active_scraper 0.0.1 → 0.0.2

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ODhmOGY2MGY0MWNjZmExNGM5MzA5ZGQwZmFhNjg0ZDQxZTMzOGQwNw==
+    NWEwOTJjZGViNjkwYzZiMjYwY2YzMGE5YjRkMTdlNzJhMTMwZmJmMw==
   data.tar.gz: !binary |-
-    NWQ5ZDQwYzZlNTg5MjY2YjdmZjFkN2Q4MmEzMjdmOGYwNjE0OWQ2NQ==
+    OGMyNmI5MDA3MmM5YWQ5MjIxZjlhYjZlYTliMDZhZWY0YjlmYjQ5ZQ==
 SHA512:
   metadata.gz: !binary |-
-    NjY1NWFjOTg5OGI1NWFlOTgzZjNjZGNiNjk2OGJkZDIyMTRjMmQwNWU3MjY3
-    MTIyMDNlZDJiZWMwZjNhZDM4NTk1Njk5MjdkNmRiYTNlZDEzZDZmNjdhYjRl
-    ZjFjNmVlMTVjYjI5MDE0MGYyNTdlNTEyZmEzMmI3N2QwYTNhNGY=
+    YTc0Mzk5YzdhNWZlM2YyYzJlNmZiZjMxOTg4OTg0YTU0MzhkMWMxMDVlNDJl
+    Y2UzNDVmNWE1NDhkODJlZTMyYWY1OTFjNTU3ZGYyNGIwNzA4NDEzZDMxMzg5
+    YWZkZWVmMTc1MjFjNGNhZDIzYTU0NGU1MTg4ZGYyYWNhNGQ3YTI=
   data.tar.gz: !binary |-
-    YzIyNWE4M2JhODc0MmY3OWQ1ZDFkNmYwNjA1M2M5MDZiNDAyMGVlYmU2MTVl
-    NDk5Y2ZiNTg1OGFkZmZkYmJhOGQyYjUyODA4NTg2ZDliNjhhMmQyZTNhZmEx
-    YzQwYWU1NzdlZWE2MjRiMTM2YjUwNzM3NjM5YWM2NDk3YmU4NDk=
+    ZWMyYTc5MDJhZDU5NGY4ZDVlMjFkYTI0YzNkMDM2NDFkNjgyY2JhNDE3MGI2
+    M2Y5MTMzMTMzMGE3ZmFmYzk5OWNiNjQyNzRmM2E4ZjJmYjk5MWY0MzUzMjU0
+    YmMxNDQ3YWU1MTY4YTFmNDZjMjRlM2YyM2E4Nzc1NzA1MDQ3N2I=

data/app/models/active_scraper/cached_request.rb

@@ -0,0 +1,163 @@
+require 'httparty'
+require 'addressable/uri'
+require 'hashie/mash'
+
+module ActiveScraper
+  class CachedRequest < ActiveRecord::Base
+    has_many :responses, :dependent => :destroy, class_name: 'CachedResponse', foreign_key: 'cached_request_id' 
+    has_one :latest_response, ->{ order('created_at DESC') },  class_name: 'ActiveScraper::CachedResponse', foreign_key: 'cached_request_id'
+    validates_uniqueness_of :path, scope: [:host, :query, :scheme]
+
+    attr_accessor :unobfuscated_query
+
+    delegate :to_s, :to => :uri
+
+    # problematic
+    scope :with_url, ->(u){     
+      matching_request(u)
+    }
+
+    scope :matching_request, ->(req, opts={}){
+      if req.is_a?(CachedRequest)
+        req = req.to_uri
+      end
+      params = CachedRequest.build_validating_params(req, opts)
+
+      where(params)
+    }
+
+    scope :last_fetched_before, ->(some_time){
+      some_time = Time.parse(some_time) if some_time.is_a?(String)
+
+      where("last_fetched_at < ?", some_time)
+    }
+
+    def latest_response_fetched_after(time)
+      if latest_response.present?
+        return latest_response if latest_response.created_at > time
+      end
+      
+      nil      
+    end
+
+    def to_fake_party_hash
+      h = Hashie::Mash.new(self.attributes.symbolize_keys.slice(:scheme, :host, :path, :query))
+      h[:uri] = self.standard_uri
+      h[:options] ||= {}
+      h[:headers] ||= {}
+
+      return h
+    end
+
+
+
+    def obfuscated?
+      is_obfuscated == true
+    end
+
+    # to follow HTTParty conventions
+    def standard_uri
+      URI.parse(uri)
+    end
+
+    def uri
+      to_uri
+    end
+
+    # during a fresh query, we need to actually use the unobfuscated_query
+    def to_uri
+      h = self.attributes.symbolize_keys.slice(:scheme, :host, :path)
+      h[:query] = self.unobfuscated_query || self.query
+
+      return Addressable::URI.new(h)
+    end
+
+    def self.build_validating_params(uri, opts={})
+      h = build_request_params(uri, opts)
+
+      h.slice(:scheme, :host, :path, :query)
+    end
+
+#########################################################
+############ class methods
+    
+
+
+    # Returns a Hash with symbolized keys
+    def self.build_request_params(uri, opts={})
+      u = Addressable::URI.parse(uri)
+      hsh = {scheme: u.normalized_scheme, host: u.normalized_host, path: u.normalized_path, query: u.normalized_query , extname: u.extname}
+      # deal with query separately
+      unless opts[:normalize_query] == false
+        hsh[:query] = normalize_query_params(hsh[:query])        
+      end
+
+      hsh[:unobfuscated_query] = hsh[:query]
+      if ob_keys = opts[:obfuscate_query]
+        hsh[:query] = obfuscate_query_params(hsh[:query], ob_keys)
+        hsh[:is_obfuscated] = true
+      else
+        hsh[:is_obfuscated] = false
+      end
+
+      return hsh
+    end
+
+    def self.build_from_uri(uri, opts={})
+      request_params = build_request_params(uri, opts)
+      request_obj = CachedRequest.new(request_params)
+
+      return request_obj
+    end
+
+    def self.find_or_build_from_uri(uri, opts={})
+      self.matching_request(uri, opts).first || self.build_from_uri(uri, opts)
+    end
+
+1
+    def self.create_from_uri(uri, opts={})
+      req = build_from_uri(uri, opts)
+      req.save
+
+      return req
+    end
+
+
+
+    QUERY_NORMALIZER = HTTParty::Request::NON_RAILS_QUERY_STRING_NORMALIZER
+    # :q is a query String or Hash
+    # e.g.   'z=hello&b=world&a=dog'
+    #    or: {z: ['hello', 'world'], a: 'dog'}
+    #
+    # returns: (String) "a=dog&z=hello&z=world"
+    def self.normalize_query_params(q)
+      return q if q.blank?
+
+      params_hash = CGI.parse(q)
+      params_str = QUERY_NORMALIZER[params_hash]
+
+      return params_str
+    end
+
+
+    private 
+
+    def self.obfuscate_query_params(q, ob_keys)
+      string = q.dup
+      Array(ob_keys).each do |key|
+        a = Array(key)
+
+        key_to_omit = Regexp.escape(a[0].to_s)
+        char_num = a[1] || 0        
+        if val_to_omit = string.match(/(?<=#{key_to_omit}=)(.*?)(?=&|$)/)
+          val = val_to_omit[1]
+          string.sub!( val, "__OMIT__#{val[-char_num, char_num]}")
+        end
+      end
+
+      return string
+    end
+
+
+  end
+end

data/app/models/active_scraper/cached_response.rb

@@ -0,0 +1,135 @@
+require 'nokogiri'
+module ActiveScraper
+  class CachedResponse < ActiveRecord::Base
+    serialize :headers, Hash
+    belongs_to :request, touch: true, class_name: 'CachedRequest', foreign_key: 'cached_request_id'
+    before_create :encode_body_for_create
+    before_save :set_checksum
+
+    after_create :touch_request_fetched_at
+
+    def to_fake_party_hash
+      [:body, :headers, :content_type, :code].inject(Hashie::Mash.new) do |hsh, att|
+        hsh[att] = self.send(att)
+
+        hsh
+      end
+    end
+
+
+    def binary?
+      content_type =~ /pdf|image/ || !text?
+    end
+
+    def json?
+      content_type =~ /json/
+    end
+
+    def html?
+      content_type =~ /html/
+    end
+
+    def xml?
+      html? || content_type =~ /xml/
+    end
+
+    def text?
+      content_type =~ /text/ || xml? || json?
+    end
+
+    def body_changed?
+      self.changed_attributes.keys.include?('body')
+    end
+
+    def body
+      b = read_attribute(:body)
+      if b.present? && binary? && !body_changed?
+        return Base64.decode64(b)
+      else
+        return b
+      end
+    end
+ 
+    def parsed_body
+      @_parsedbody ||= if xml?
+        Nokogiri::HTML(body)
+      elsif json?
+        JSON.parse(body)
+      else
+        body
+      end
+    end
+
+    def to_s
+      body
+    end
+
+
+
+    private
+    def set_checksum
+      self.checksum = body.hash
+
+      true
+    end
+
+
+    def touch_request_fetched_at
+      if request && !request.new_record?
+        request.update_attributes(last_fetched_at: self.created_at) if self == request.latest_response
+      end
+
+      true
+    end
+
+
+    # expects @body to be populated
+    # returns string: e.g. 'utf-8', 'windows-1251'
+    def detect_encoding
+      if xml?
+        parsed_body.encoding
+      else
+        body.encoding
+      end
+    end
+
+    # converts @body to utf-8 if not already
+    def encode_body_for_create
+      if self.body.present?        
+        if binary?
+          self.body = Base64.encode64(self.body)
+        elsif
+          denc = detect_encoding
+          self.body = self.body.encode('utf-8', denc)
+        end
+      end 
+
+      true
+    end
+
+
+
+
+
+############## class methods
+    def self.find_cache_for_cached_request(cached_request, opts={})
+       time = opts[:fetched_after] || Time.at(0)
+       # smell: just goes back to CachedRequest
+       cached_request.latest_response_fetched_after(time)       
+    end
+
+    def self.find_cache_for_request(req, opts)
+      # TODO
+    end
+
+    # has one side-effect: :body is properly encoded
+    def self.build_from_response_object(resp)
+      response = self.new
+      [:body, :headers, :content_type, :code].each do |att|
+        response.send :write_attribute, att, resp.send(att)
+      end
+
+      return response
+    end
+  end
+end

data/db/migrate/20131229024155_create_active_scraper_cached_requests.rb

@@ -0,0 +1,19 @@
+class CreateActiveScraperCachedRequests < ActiveRecord::Migration
+  def change
+    create_table :active_scraper_cached_requests do |t|
+      t.string   "scheme"
+      t.string   "host"
+      t.text     "query"
+      t.string   "path"
+      t.string   "meta_tag"
+      t.string   "extname"
+      t.boolean  "is_obfuscated"
+      t.datetime "created_at"
+      t.datetime "updated_at"
+      t.datetime "last_fetched_at"
+    end
+
+    add_index "active_scraper_cached_requests", ["host", "path"], name: "index_as_requests_on_host_and_path"
+
+  end
+end

data/db/migrate/20131229033843_create_active_scraper_cached_responses.rb

@@ -0,0 +1,16 @@
+class CreateActiveScraperCachedResponses < ActiveRecord::Migration
+  def change
+    create_table :active_scraper_cached_responses do |t|
+      t.text     "body",                      limit: 4294967295
+      t.integer  "code"
+      t.text     "headers"
+      t.string   "content_type"
+      t.integer  "checksum"
+      t.integer  "cached_request_id"
+      t.timestamps      
+    end
+
+    add_index :active_scraper_cached_responses, [:cached_request_id, :created_at], name: 'index_request_id_and_created_at'
+    add_index :active_scraper_cached_responses, [:cached_request_id, :checksum], name: 'index_request_id_and_checksum'
+  end
+end

data/lib/active_scraper.rb

@@ -1,5 +1,86 @@
+# encoding: UTF-8
+
 require "active_scraper/engine"
-require "active_scraper/fetcher"
+require 'active_scraper/fake_http_party_response'
+require 'active_scraper/response_object'
 
 module ActiveScraper
+
+
+  # returns a ActiveScraper::CachedResponse
+  def self.get(uri, options={})
+    o = create_request_and_fetch_response(uri, options)
+
+    return o.response
+  end
+
+
+
+  # delegates to CachedRequest::find_or_build_from_uri
+  #   req (URI or String). If CachedRequest, is idempotent
+  #   
+  # returns a new or existing CachedRequest
+  def self.find_or_build_request(req, opts={})
+    CachedRequest.find_or_build_from_uri(req, opts)
+  end
+
+  ## cached_request (CachedRequest) => the request to find a response for
+  ##
+  ## returns a new or existing CachedResponse
+
+  def self.find_or_build_response(cached_request, opts={})
+    raise ArgumentError, "Only accepted CachedRequest, but was passed in a #{cached_request.class}" unless cached_request.is_a?(CachedRequest)
+    opts = normalize_hash(opts)
+
+    response = CachedResponse.find_cache_for_cached_request(cached_request, opts)
+
+    if response.blank?
+      fetched_obj = fetch_fresh(cached_request.uri, opts)
+      response = CachedResponse.build_from_response_object(fetched_obj)
+    end
+
+    return response
+  end
+
+
+  def self.create_request_and_fetch_response(uri, opts={})
+    opts = normalize_hash(opts)
+    # first, find or build the request
+    request = find_or_build_request(uri, opts)
+    # then find or build a matching response
+    response = find_or_build_response(request, opts)
+    # associate and save the two
+    request.responses << response
+    request.save
+     
+    obj = Hashie::Mash.new(request: request, response: response)
+    
+    return obj
+  end
+
+  # Returns an object compatible with HTTParty, i.e. an ActiveScraper::FakeHTTPartyResponse
+  # to be deprecated
+  def self.build_usable_response(request, response)
+    ActiveScraper::FakeHTTPartyResponse.new(request, response)
+  end
+
+
+
+  def self.fetch_fresh(url, opts={})
+     resp = HTTParty.get(url, opts)
+
+     return ActiveScraper::ResponseObject.factory(resp)
+  end
+
+
+
+
+  def self.normalize_hash(hsh)
+    unless hsh.is_a?(HashWithIndifferentAccess)
+      hsh = HashWithIndifferentAccess.new(hsh) 
+    end
+
+    return hsh
+  end
+
 end

data/lib/active_scraper/engine.rb

@@ -1,6 +1,17 @@
 module ActiveScraper
   class Engine < ::Rails::Engine
     isolate_namespace ActiveScraper
+ 
+    # monkey patch via: http://pivotallabs.com/leave-your-migrations-in-your-rails-engines/
+    initializer :append_migrations do |app|
+      unless app.root.to_s.match root.to_s
+        config.paths["db/migrate"].expanded.each do |expanded_path|
+          app.config.paths["db/migrate"] << expanded_path
+        end
+      end
+    end
+  
+
     config.generators do |g|
       g.test_framework      :rspec,        :fixture => false
       g.assets false

data/lib/active_scraper/fake_http_party_response.rb

@@ -0,0 +1,21 @@
+require 'httparty'
+require 'nokogiri'
+module ActiveScraper
+  class FakeHTTPartyResponse < SimpleDelegator
+    
+
+    def initialize(request, response, parsed_block=nil, options={})
+      request = request.to_fake_party_hash if request.is_a?(CachedRequest)
+      response = response.to_fake_party_hash if response.is_a?(CachedResponse)
+
+      ## making HTTParty happy...
+
+      parsed_block ||= ->(){ response.body }
+
+      super(HTTParty::Response.new request, response, parsed_block, options)      
+    end
+
+
+
+  end
+end

data/lib/active_scraper/response_object.rb

@@ -0,0 +1,13 @@
+require 'active_scraper/response_object/basic'
+
+module ActiveScraper
+  module ResponseObject
+
+
+    def self.factory(obj)
+      ActiveScraper::ResponseObject::Basic.new(obj)
+    end
+
+
+  end
+end

data/lib/active_scraper/response_object/basic.rb

@@ -0,0 +1,67 @@
+module ActiveScraper
+  module ResponseObject 
+    class Basic < SimpleDelegator
+      # I don't really know what to name this but this is passed
+      # between the various classes, including the Fetcher,
+      # and is expected to behave the same in those interactions
+
+      attr_reader :code, :headers, :body, :content_type
+
+      def initialize(obj)
+        if obj.class == (HTTParty::Response)
+          # use the Net::HTTPResponse instead
+          obj = obj.response
+        end
+
+        response_obj = if obj.is_a?(Net::HTTPResponse)
+          @body = obj.body
+          @content_type = obj.content_type
+          @headers = obj.each_header.inject({}){|h, (k, v)| h[k] = v; h }
+          @code = obj.code.to_i
+        elsif obj.is_a?(ActiveScraper::CachedResponse)
+          @body = obj.body
+          @content_type = obj.content_type
+          @headers = obj.headers
+          @code = obj.code.to_i
+        elsif obj.is_a?(StringIO) && obj.respond_to?(:meta) # OpenURI.open
+          @body = obj.read
+          @content_type = obj.content_type
+          @headers = obj.meta
+          @code = obj.status[0].to_i
+        elsif obj.nil?
+          # just do nothing
+        else
+          # other types have to raise an Error
+          raise ArgumentError, 'Improper class type'
+        end
+
+        super(ActiveSupport::HashWithIndifferentAccess.new() )
+
+        def empty?
+          @body.empty?
+        end
+
+        def nil?
+          @body.nil?
+        end
+
+
+        # now set its values
+        [:body, :headers, :content_type, :code].each do |a|
+          self[a] = self.send(a)
+        end
+      end
+
+      # def [](k)
+      #   @values[k.to_sym]
+      # end
+
+      # def [](k,v)
+      #   send(:"#{k}=", v)
+      # end
+
+    end
+  end
+end
+
+

data/lib/active_scraper/version.rb

@@ -1,3 +1,3 @@
 module ActiveScraper
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: active_scraper
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Dan Nguyen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-03 00:00:00.000000000 Z
+date: 2014-02-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -66,6 +66,20 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: hashie
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -192,6 +206,20 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: A Rails Engine using ActiveRecord to cache results of HTTP scrapes
 email:
 - dansonguyen@gmail.com
@@ -202,15 +230,16 @@ files:
 - app/assets/stylesheets/active_scraper/application.css
 - app/controllers/active_scraper/application_controller.rb
 - app/helpers/active_scraper/application_helper.rb
-- app/models/active_scraper/agnostic_response_object.rb
-- app/models/active_scraper/request.rb
-- app/models/active_scraper/response.rb
+- app/models/active_scraper/cached_request.rb
+- app/models/active_scraper/cached_response.rb
 - app/views/layouts/active_scraper/application.html.erb
 - config/routes.rb
-- db/migrate/20131229024155_create_active_scraper_requests.rb
-- db/migrate/20131229033843_create_active_scraper_responses.rb
+- db/migrate/20131229024155_create_active_scraper_cached_requests.rb
+- db/migrate/20131229033843_create_active_scraper_cached_responses.rb
 - lib/active_scraper/engine.rb
-- lib/active_scraper/fetcher.rb
+- lib/active_scraper/fake_http_party_response.rb
+- lib/active_scraper/response_object/basic.rb
+- lib/active_scraper/response_object.rb
 - lib/active_scraper/version.rb
 - lib/active_scraper.rb
 - lib/tasks/active_scraper_tasks.rake

data/app/models/active_scraper/agnostic_response_object.rb

@@ -1,47 +0,0 @@
-module ActiveScraper
-  class AgnosticResponseObject < SimpleDelegator
-
-    attr_reader :code, :headers, :body, :content_type
-
-    def initialize(obj)
-      if obj.class == (HTTParty::Response)
-        # use the Net::HTTPResponse instead
-        obj = obj.response
-      end
-
-      response_obj = if obj.is_a?(Net::HTTPResponse)
-        @body = obj.body
-        @content_type = obj.content_type
-        @headers = obj.each_header.inject({}){|h, (k, v)| h[k] = v; h }
-        @code = obj.code.to_i
-      elsif obj.is_a?(ActiveScraper::Request)
-        @body = obj.body
-        @content_type = obj.content_type
-        @headers = obj.headers
-        @code = obj.code.to_i
-      else
-        # this is probably not used
-        @body = obj.to_s
-        @headers = {}
-        @content_type = nil
-        @code = nil
-      end
-
-      super({})
-
-      # now set its values
-      [:body, :headers, :content_type, :code].each do |a|
-        self[a] = self.send(a)
-      end
-    end
-
-    # def [](k)
-    #   @values[k.to_sym]
-    # end
-
-    # def [](k,v)
-    #   send(:"#{k}=", v)
-    # end
-
-  end
-end

data/app/models/active_scraper/request.rb

@@ -1,98 +0,0 @@
-require 'addressable/uri'
-module ActiveScraper
-  class Request < ActiveRecord::Base
-    has_many :responses, :dependent => :destroy 
-    validates_uniqueness_of :path, scope: [:host, :query, :scheme]
-
-
-    scope :with_url, ->(u){     
-      params = Request.build_validating_params(u)
-      where(params)
-    }
-
-
-    def obfuscated?
-      is_obfuscated == true
-    end
-
-    def uri
-      Addressable::URI.new(
-        self.attributes.symbolize_keys.slice(:scheme, :host, :path, :query)
-      )
-    end
-
-    def self.build_validating_params(uri, opts={})
-      h = build_request_params(uri, opts)
-
-      h.slice(:scheme, :host, :path, :query)
-    end
-
-    # Returns a Hash with symbolized keys
-    def self.build_request_params(uri, opts={})
-      u = Addressable::URI.parse(uri)
-      hsh = {scheme: u.normalized_scheme, host: u.normalized_host, path: u.normalized_path, query: u.normalized_query, extname: u.extname}
-
-      if ob_keys = opts.delete(:obfuscate_query)
-        Array(ob_keys).each do |key|
-          a = Array(key)
-
-          key_to_omit = Regexp.escape(a[0].to_s)
-          char_num = a[1] || 0
-          
-          if val_to_omit = hsh[:query].match(/(?<=#{key_to_omit}=)(.*?)(?=&|$)/)
-            val = val_to_omit[1]
-            hsh[:query].sub!( val, "__OMIT__#{val[-char_num, char_num]}")
-          end
-        end
-
-        hsh[:is_obfuscated] = true
-      else
-        hsh[:is_obfuscated] = false
-      end
-
-      return hsh
-    end
-
-    def self.build_from_uri(uri, opts={})
-      request_params = build_request_params(uri, opts)
-      request_obj = Request.new(request_params)
-
-      return request_obj
-    end
-
-    def self.find_or_build_from_uri(uri, opts={})
-      self.with_url(uri).first || self.build_from_uri(uri, opts)
-    end
-
-
-    def self.create_from_uri(uri, opts={})
-      req = build_from_uri(uri, opts)
-      req.save
-
-      return req
-    end
-
-
-    def self.create_and_fetch_response(uri, opts={}, fetcher = nil)
-      request = find_or_build_from_uri(uri, opts)
-      fetcher = fetcher || Fetcher.new
-
-      if request.id.nil? 
-        # this request is new
-        # so skip to the fresh
-        resp = fetcher.fetch request, fresh: true 
-      else 
-        # will check the cache and the fresh
-        resp = fetcher.fetch request
-      end
-
-      # build the response
-      response = request.responses.build(resp)
-      # theoretically, response will be saved too
-      request.save
-
-      return request
-    end
-
-  end
-end

data/app/models/active_scraper/response.rb

@@ -1,26 +0,0 @@
-module ActiveScraper
-  class Response < ActiveRecord::Base
-    serialize :headers, Hash
-    belongs_to :request
-    before_save :set_checksum
-
-    
-
-    private
-    def set_checksum
-      self.checksum = body.hash
-
-      true
-    end
-
-############## class methods
-    def self.build_from_response_object(resp)
-      response = self.new
-      [:body, :headers, :content_type, :code].each do |att|
-        response.send :write_attribute, att, resp.send(att)
-      end
-
-      return response
-    end
-  end
-end

data/db/migrate/20131229024155_create_active_scraper_requests.rb

@@ -1,16 +0,0 @@
-class CreateActiveScraperRequests < ActiveRecord::Migration
-  def change
-    create_table :active_scraper_requests do |t|
-      t.string :host
-      t.text :query
-      t.string :path
-      t.string :meta_tag
-      t.boolean :is_obfuscated
-
-      t.timestamps
-    end
-
-    add_index :active_scraper_requests, [:host, :path]
-
-  end
-end

data/db/migrate/20131229033843_create_active_scraper_responses.rb

@@ -1,17 +0,0 @@
-class CreateActiveScraperResponses < ActiveRecord::Migration
-  def change
-    create_table :active_scraper_responses do |t|
-      t.text :body, :limit => 4294967295
-      t.integer :code
-      t.text :headers
-      t.string :content_type
-      t.integer :checksum
-      t.integer :active_scraper_request_id
-
-      t.timestamps      
-    end
-
-    add_index :active_scraper_responses, [:active_scraper_request_id, :created_at], name: 'index_request_id_and_created_at'
-    add_index :active_scraper_responses, [:active_scraper_request_id, :checksum], name: 'index_request_id_and_checksum'
-  end
-end

data/lib/active_scraper/fetcher.rb

@@ -1,65 +0,0 @@
-require 'httparty'
-
-module ActiveScraper
-  class Fetcher
-
-    def fetch(u, opts={})
-      url = convert_uri_object(u)
-      force_fresh = opts.delete :fresh
-
-      if force_fresh != true && (record = fetch_from_cache(url, opts))
-        resp_obj = record
-      else
-        resp_obj = fetch_fresh(url, opts)
-      end
-
-      build_response_object(resp_obj)
-    end
-
-
-    def fetch_fresh(url, opts={})
-      opts = opts.stringify_keys
-
-      url = url.to_s
-      verb = opts.fetch('verb'){ 'get' }
-
-      resp = HTTParty.send(verb, url)
-    end
-
-
-    # returns: 
-    #   single ScrapeCache if a valid ActiveScraper::Request exists
-    #   
-    def fetch_from_cache(uri, opts={})
-
-    end
-
-    # true or false if ActiveScraper::Request with these parameters exist
-    def has_cache?(uri, opts={})
-
-    end
-
-
-    # u can either be a Request object, a String, or Addressable::URI
-    # returns an Addressable::URI
-    def convert_uri_object(u)
-      if u.is_a?(ActiveScraper::Request)
-        x = u.uri
-      else
-        x = Addressable::URI.parse(u)
-      end
-
-      return x
-    end
-
-    def build_response_object(obj)
-      self.class.build_response_object(obj)
-    end
-
-    # returns an OpenStruct that Response can use
-    def self.build_response_object(obj)
-      return AgnosticResponseObject.new(obj)
-    end
-
-  end
-end