creepin 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in creepin.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Jason Ayre
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# Creepin
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'creepin'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install creepin
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/creepin.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'creepin/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "creepin"
+  gem.version       = Creepin::VERSION
+  gem.authors       = ["Jason Ayre"]
+  gem.email         = ["jasonayre@gmail.com"]
+  gem.description   = %q{Creepin so logically}
+  gem.summary       = %q{Provides structured crawling, and mapping, of external sites, to your ruby classes or AR models.}
+  gem.homepage      = ""
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  
+  gem.add_dependency "activesupport"
+  gem.add_dependency 'httparty'
+  gem.add_dependency 'aasm'
+  gem.add_dependency 'nokogiri'
+end

data/lib/creepin.rb

@@ -0,0 +1,45 @@
+require "creepin/version"
+require 'aasm'
+require 'uri'
+require 'active_support'
+require 'rack'
+require 'httparty'
+require 'nokogiri'
+require 'creepin/collection_creeper'
+require 'creepin/collection'
+require 'creepin/resource_creeper'
+require 'creepin/resource'
+
+require 'creepin/on'
+
+module Creepin
+  # Your code goes here...
+  @@loaded = false
+  
+  def self.setup(&block)
+    load!
+    instance_eval(&block)
+  end
+  
+  def self.load!
+    # No work to do if we've already loaded
+    return false if loaded?
+
+    # Load files
+    files_in_load_path.each{|file| load file }
+    @@loaded = true
+  end
+  
+  def self.loaded?
+    @@loaded
+  end
+  
+  def self.load_paths
+    [File.expand_path('app/creepers', Rails.root)]
+  end
+  
+  def self.files_in_load_path
+    load_paths.flatten.compact.uniq.collect{|path| Dir["#{path}/**/*.rb"] }.flatten
+  end
+  
+end

data/lib/creepin/collection.rb

@@ -0,0 +1,103 @@
+module Creepin
+  class Collection
+    
+    def initialize(name, options={}, &block)
+      @config = {}
+      @request_params = {}
+      
+      instance_eval(&block)
+      settings = @config.dup.merge(namespace: name)
+      @request_params.merge(settings[:default_params]) if settings.has_key?(:default_params)
+      @request_params.reverse_merge(options[:params]) if options.has_key?(:params)
+      
+      class_name = "#{name.camelize}CollectionCreeper"
+      puts class_name
+      klass = Class.new CollectionCreeper
+      dsl_methods = @config.keys
+      @creeper_class = Object.const_set(class_name, klass)
+      
+      #seems to be a bug with AASM and inheritence, this should be in collection crawler file
+      @creeper_class.class_eval do
+        
+        include AASM
+
+        aasm do
+          state :waiting, :initial => true
+          state :crawling
+          state :parsing
+          state :finished
+          state :finished_and_loaded
+          
+          event :crawl, :after => :transmit do
+            transitions :from => :waiting, :to => :crawling
+          end
+          
+          event :crawl_finished, :after => :run_after_crawl_finished_callbacks do
+            transitions :from => :crawling, :to => :finished
+          end
+          
+          event :crawl_next, :after => :crawl_next_page do
+            transitions :from => :finished, :to => :waiting
+          end
+          
+          event :collection_loaded, :after => :run_after_collection_loaded_callbacks do
+            transitions :from => [:waiting, :finished], :to => :finished_and_loaded
+          end
+          
+        end
+        
+        dsl_methods.each do |sym|
+          define_method(sym.to_s) {
+            settings[sym]
+          }
+        end
+        
+      end
+      
+    end
+    
+    def base_url(string)
+      @config[:base_url] = string
+    end
+    
+    def default_params(hash)
+      @config[:default_params] = hash
+    end
+    
+    def selector(string)
+      @config[:selector] = string
+    end
+    
+    def define_element_mapping(attr_name, &block)
+      @config[:element_mappings] ||= {}
+      @config[:element_mappings][attr_name] = block
+    end
+    
+    def resource_class(string)
+      @config[:resource_class] = string
+    end
+    
+    def next_page_selector(string='', &block)
+      @config[:next_page_selector] = string unless string.nil?
+      @config[:next_page_selector] = block if block
+    end
+    
+    def skip_resource_save(bool)
+      @config[:skip_resource_save] = bool.present? ? bool : false
+    end
+    
+    def resource_load_strategy(*options, &block)
+      @config[:resource_load_strategy] = Proc.new(*options, &block)
+    end
+    
+    def resource_save_strategy(*options, &block)
+      @config[:resource_save_strategy] = Proc.new(*options, &block)
+    end
+    
+    def after(event_name, &block)
+      @config["after_#{event_name.to_s}_callbacks".to_sym] ||= []
+      @config["after_#{event_name.to_s}_callbacks".to_sym] << block
+    end
+    
+  end
+end

data/lib/creepin/collection_creeper.rb

@@ -0,0 +1,141 @@
+module Creepin
+  class CollectionCreeper
+  
+    attr_accessor :stats, :total_records, :total_pages, :loaded_collection, :started_at, :finished_at, :requested_urls
+  
+    def initialize(params = {})
+      @params ||= {}
+      @params = params if params.present?
+      @total_records ||= 0
+      @total_pages ||= 0
+      @loaded_collection ||= []
+      @requested_urls ||= []
+    end
+  
+    def run_after_crawl_callbacks
+      transmit
+    end
+  
+    def run_after_crawl_finished_callbacks
+      parse_response
+      after_crawl_finished_callbacks.each{ |callback| callback.call(self) } if after_crawl_finished_callbacks?
+      crawl_next
+    end
+  
+    def run_after_collection_loaded_callbacks
+      after_collection_loaded_callbacks.each{ |callback| callback.call(self) } if after_collection_loaded_callbacks?
+    end
+  
+    def after_crawl_finished_callbacks?
+      (respond_to?(:after_crawl_finished_callbacks) && !after_crawl_finished_callbacks.empty?) ? true : false
+    end
+  
+    def before_crawl_finished_callbacks?
+      (respond_to?(:before_crawl_finished_callbacks) && !before_crawl_finished_callbacks.empty?) ? true : false
+    end
+  
+    def after_collection_loaded_callbacks?
+      (respond_to?(:after_collection_loaded_callbacks) && !after_collection_loaded_callbacks.empty?) ? true : false
+    end
+  
+    def crawl_next_page
+      if next_page?
+        crawl
+      else
+        collection_loaded
+      end
+    end
+  
+    def transmit
+      @request_params ||=  (default_params? ? {:query => default_params.merge(@params) } : {:query => @params } )
+      @response = HTTParty.get(base_url, @request_params)
+      @requested_urls << full_request_url(base_url, @request_params)
+      @total_pages += 1
+      crawl_finished
+    end
+  
+    def build_request_params(param_string)
+      params_hash = Rack::Utils.parse_query(param_string.split('?').pop)
+      @request_params = { :query => params_hash.with_indifferent_access } if params_hash.present?
+    end
+  
+    def full_request_url(base_url, request_params)
+      base_url + request_params[:query].map{|k,v| "#{k}=#{v}"}.join("&").insert(0, '?')
+    end
+  
+    def parse_response
+      @response_html = Nokogiri::HTML::Document.parse(@response.body)
+      load_response_collection
+      map_response_collection if response_collection?
+    end
+  
+    def load_response_collection
+      @response_collection = @response_html.document.css(selector)
+    end
+  
+    def response_collection?
+      @response_collection.present?
+    end
+
+    def load_resource(collected_attributes_hash, resource_klass)
+      if resource_load_strategy?
+        resource_load_strategy.call(collected_attributes_hash, resource_klass)
+      else
+        resource_klass.new(collected_attributes_hash)
+      end
+    end
+  
+    def save_resource(collected_attributes_hash, resource)
+      if resource_save_strategy?
+        resource_save_strategy.call(collected_attributes_hash, resource)
+      else
+        resource.save unless skip_resource_save?
+      end
+    end
+  
+    def skip_resource_save?
+      respond_to?(:skip_resource_save)
+    end
+  
+    def resource_load_strategy?
+      respond_to?(:resource_load_strategy)
+    end
+
+    def resource_save_strategy?
+      respond_to?(:resource_save_strategy)
+    end
+  
+    def default_params?
+      respond_to?(:default_params)
+    end
+  
+    def map_response_collection
+      @response_collection.each do |ele|
+        collected_attributes_hash = {}
+        element_mappings.each_pair do |attribute, block|
+          value = instance_exec(ele, &block)
+          collected_attributes_hash[attribute] = value
+        end
+        resource = load_resource(collected_attributes_hash, resource_class.constantize)
+        @total_records += 1
+        resource = save_resource(collected_attributes_hash, resource)
+        loaded_collection << resource
+      end
+    end
+  
+    def next_page?
+      return false if next_page_selector.nil?
+      if next_page_selector.is_a?(Proc)
+        next_page_url = instance_exec(@response_html.document, &next_page_selector)
+        build_request_params(next_page_url) if next_page_url.present?
+        @has_next_page = next_page_url.present?
+      else
+        next_page_url = @response_html.document.at_css(next_page_selector)
+        build_request_params(next_page_url) if next_page_url.present?
+        @has_next_page = next_page_url.present?
+      end
+      @has_next_page
+    end
+  
+  end
+end

data/lib/creepin/on.rb

@@ -0,0 +1,21 @@
+module Creepin
+  
+  class On
+    
+    def initialize(name, &block)
+      @config = {}
+      @name = name
+      instance_eval(&block)
+    end
+        
+    def collection(*options, &block)
+      Creepin::Collection.new(@name, *options, &block)
+    end
+        
+    def resource(*options, &block)
+      Creepin::Resource.new(@name, *options, &block)
+    end
+      
+  end
+  
+end

data/lib/creepin/resource.rb

@@ -0,0 +1,91 @@
+module Creepin
+
+  class Resource
+    
+    def initialize(name, options={}, &block)
+      @config = {}
+      @request_params = {}
+      
+      instance_eval(&block)
+      settings = @config.dup.merge(namespace: name)
+      @request_params.merge(settings[:default_params]) if settings.has_key?(:default_params)
+      @request_params.reverse_merge(options[:params]) if options.has_key?(:params)
+      @loaded_resource = options[:loaded_resource] if options.has_key?(:loaded_resource)
+      
+      class_name = "#{name.camelize}ResourceCreeper"
+      klass = Class.new ResourceCreeper
+      dsl_methods = @config.keys
+      creeper_class = Object.const_set(class_name, klass)
+      
+      #seems to be a bug with AASM and inheritence, this should be in collection crawler file
+      creeper_class.class_eval do
+        
+        include AASM
+  
+        aasm do
+          state :waiting, :initial => true
+          state :crawling
+          state :parsing
+          state :finished
+          state :finished_and_loaded
+          
+          event :crawl, :after => :transmit do
+            transitions :from => :waiting, :to => :crawling
+          end
+          
+          event :crawl_finished, :after => :parse_response do
+            transitions :from => :crawling, :to => :finished
+          end
+          
+        end
+        
+        dsl_methods.each do |sym|
+          define_method(sym.to_s) {
+            settings[sym]
+          }
+        end
+        
+      end
+      
+    end
+    
+    def base_url(string)
+      @config[:base_url] = string
+    end
+    
+    def default_params(hash)
+      @config[:default_params] = hash
+    end
+    
+    def url_attribute(sym)
+      @config[:url_attribute] = sym.to_sym
+    end
+    
+    def selector(string)
+      @config[:selector] = string
+    end
+    
+    def define_element_mapping(attr_name, &block)
+      @config[:element_mappings] ||= {}
+      @config[:element_mappings][attr_name] = block
+    end
+    
+    def resource_class(string)
+      @config[:resource_class] = string
+    end
+    
+    def skip_resource_save(bool)
+      @config[:skip_resource_save] = bool.present? ? bool : false
+    end
+    
+    def resource_load_strategy(*options, &block)
+      @config[:resource_load_strategy] = Proc.new(*options, &block)
+    end
+    
+    def resource_save_strategy(*options, &block)
+      @config[:resource_save_strategy] = Proc.new(*options, &block)
+    end
+    
+  end
+	
+end

data/lib/creepin/resource_creeper.rb

@@ -0,0 +1,97 @@
+module Creepin
+
+  class ResourceCreeper
+  
+    attr_accessor :collected_attributes_hash, :requested_url, :loaded_resource
+  
+    def initialize(loaded_resource, params = {})
+      @params ||= {}
+      @params = params if params.present?
+      @loaded_resource = loaded_resource
+      @collected_attributes_hash = {}
+    end
+  
+    def run_after_crawl_callbacks
+      transmit
+    end
+  
+    def transmit
+      if url_attribute?
+        @response = HTTParty.get(loaded_resource.send(url_attribute))
+        @requested_url = loaded_resource.send(url_attribute)
+      else
+        @request_params ||= {:query => default_params.merge!(@params) }
+        @response = HTTParty.get(base_url, @request_params)
+        @requested_url = full_request_url(base_url, @request_params)           
+      end
+
+      crawl_finished
+    end
+  
+    def url_attribute?
+      respond_to?(:url_attribute)
+    end
+  
+    def build_request_params(param_string)
+      params_hash = Rack::Utils.parse_query(param_string.split('?').pop)
+      @request_params = { :query => params_hash.with_indifferent_access } if params_hash.present?
+    end
+  
+    def full_request_url(base_url, request_params)
+      base_url + request_params[:query].map{|k,v| "#{k}=#{v}"}.join("&").insert(0, '?')
+    end
+  
+    def parse_response
+      @response_html = Nokogiri::HTML::Document.parse(@response.body)
+      load_response_resource
+      map_response_resource if response_resource?
+    end
+  
+    def load_response_resource
+      @response_resource = @response_html.document.at_css(selector)
+    end
+  
+    def map_response_resource
+
+      element_mappings.each_pair do |attribute, block|
+        value = instance_exec(@response_resource, &block)
+        collected_attributes_hash[attribute] = value
+      end
+    
+      resource = save_resource(collected_attributes_hash, loaded_resource)
+    
+    end
+  
+    def response_resource?
+      @response_resource.present? ? true : false
+    end
+  
+    def load_resource(collected_attributes_hash, resource_klass)
+      if resource_load_strategy?
+        resource_load_strategy.call(collected_attributes_hash, resource_klass)
+      else
+        resource_klass.new(collected_attributes_hash)
+      end
+    end
+  
+    def save_resource(collected_attributes_hash, resource)
+      if resource_save_strategy?
+        resource_save_strategy.call(collected_attributes_hash, resource)
+      else
+        collected_attributes_hash.each_pair{|k,v| resource.send("#{k}=", v) }
+        resource.save unless skip_resource_save?
+        resource
+      end
+    end
+  
+    def resource_save_strategy?
+      respond_to?(:resource_save_strategy)
+    end
+  
+    def skip_resource_save?
+      respond_to?(:skip_resource_save)
+    end
+  
+  end
+
+end

data/lib/creepin/version.rb

@@ -0,0 +1,3 @@
+module Creepin
+  VERSION = "0.0.1"
+end

metadata

@@ -0,0 +1,123 @@
+--- !ruby/object:Gem::Specification
+name: creepin
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Jason Ayre
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-01-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aasm
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Creepin so logically
+email:
+- jasonayre@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- creepin.gemspec
+- lib/creepin.rb
+- lib/creepin/collection.rb
+- lib/creepin/collection_creeper.rb
+- lib/creepin/on.rb
+- lib/creepin/resource.rb
+- lib/creepin/resource_creeper.rb
+- lib/creepin/version.rb
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Provides structured crawling, and mapping, of external sites, to your ruby
+  classes or AR models.
+test_files: []