active_scraper 0.0.1

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ODhmOGY2MGY0MWNjZmExNGM5MzA5ZGQwZmFhNjg0ZDQxZTMzOGQwNw==
+  data.tar.gz: !binary |-
+    NWQ5ZDQwYzZlNTg5MjY2YjdmZjFkN2Q4MmEzMjdmOGYwNjE0OWQ2NQ==
+SHA512:
+  metadata.gz: !binary |-
+    NjY1NWFjOTg5OGI1NWFlOTgzZjNjZGNiNjk2OGJkZDIyMTRjMmQwNWU3MjY3
+    MTIyMDNlZDJiZWMwZjNhZDM4NTk1Njk5MjdkNmRiYTNlZDEzZDZmNjdhYjRl
+    ZjFjNmVlMTVjYjI5MDE0MGYyNTdlNTEyZmEzMmI3N2QwYTNhNGY=
+  data.tar.gz: !binary |-
+    YzIyNWE4M2JhODc0MmY3OWQ1ZDFkNmYwNjA1M2M5MDZiNDAyMGVlYmU2MTVl
+    NDk5Y2ZiNTg1OGFkZmZkYmJhOGQyYjUyODA4NTg2ZDliNjhhMmQyZTNhZmEx
+    YzQwYWU1NzdlZWE2MjRiMTM2YjUwNzM3NjM5YWM2NDk3YmU4NDk=

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2013 YOURNAME
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,23 @@
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+require 'rdoc/task'
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'ActiveScraper'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+APP_RAKEFILE = File.expand_path("../spec/dummy/Rakefile", __FILE__)
+load 'rails/tasks/engine.rake'
+
+
+
+Bundler::GemHelper.install_tasks
+

data/app/assets/stylesheets/active_scraper/application.css

@@ -0,0 +1,15 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or vendor/assets/stylesheets of plugins, if any, can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the bottom of the
+ * compiled file so the styles you add here take precedence over styles defined in any styles
+ * defined in the other CSS/SCSS files in this directory. It is generally better to create a new
+ * file per style scope.
+ *
+ *= require_tree .
+ *= require_self
+ */

data/app/controllers/active_scraper/application_controller.rb

@@ -0,0 +1,4 @@
+module ActiveScraper
+  class ApplicationController < ActionController::Base
+  end
+end

data/app/helpers/active_scraper/application_helper.rb

@@ -0,0 +1,4 @@
+module ActiveScraper
+  module ApplicationHelper
+  end
+end

data/app/models/active_scraper/agnostic_response_object.rb

@@ -0,0 +1,47 @@
+module ActiveScraper
+  class AgnosticResponseObject < SimpleDelegator
+
+    attr_reader :code, :headers, :body, :content_type
+
+    def initialize(obj)
+      if obj.class == (HTTParty::Response)
+        # use the Net::HTTPResponse instead
+        obj = obj.response
+      end
+
+      response_obj = if obj.is_a?(Net::HTTPResponse)
+        @body = obj.body
+        @content_type = obj.content_type
+        @headers = obj.each_header.inject({}){|h, (k, v)| h[k] = v; h }
+        @code = obj.code.to_i
+      elsif obj.is_a?(ActiveScraper::Request)
+        @body = obj.body
+        @content_type = obj.content_type
+        @headers = obj.headers
+        @code = obj.code.to_i
+      else
+        # this is probably not used
+        @body = obj.to_s
+        @headers = {}
+        @content_type = nil
+        @code = nil
+      end
+
+      super({})
+
+      # now set its values
+      [:body, :headers, :content_type, :code].each do |a|
+        self[a] = self.send(a)
+      end
+    end
+
+    # def [](k)
+    #   @values[k.to_sym]
+    # end
+
+    # def [](k,v)
+    #   send(:"#{k}=", v)
+    # end
+
+  end
+end

data/app/models/active_scraper/request.rb

@@ -0,0 +1,98 @@
+require 'addressable/uri'
+module ActiveScraper
+  class Request < ActiveRecord::Base
+    has_many :responses, :dependent => :destroy 
+    validates_uniqueness_of :path, scope: [:host, :query, :scheme]
+
+
+    scope :with_url, ->(u){     
+      params = Request.build_validating_params(u)
+      where(params)
+    }
+
+
+    def obfuscated?
+      is_obfuscated == true
+    end
+
+    def uri
+      Addressable::URI.new(
+        self.attributes.symbolize_keys.slice(:scheme, :host, :path, :query)
+      )
+    end
+
+    def self.build_validating_params(uri, opts={})
+      h = build_request_params(uri, opts)
+
+      h.slice(:scheme, :host, :path, :query)
+    end
+
+    # Returns a Hash with symbolized keys
+    def self.build_request_params(uri, opts={})
+      u = Addressable::URI.parse(uri)
+      hsh = {scheme: u.normalized_scheme, host: u.normalized_host, path: u.normalized_path, query: u.normalized_query, extname: u.extname}
+
+      if ob_keys = opts.delete(:obfuscate_query)
+        Array(ob_keys).each do |key|
+          a = Array(key)
+
+          key_to_omit = Regexp.escape(a[0].to_s)
+          char_num = a[1] || 0
+          
+          if val_to_omit = hsh[:query].match(/(?<=#{key_to_omit}=)(.*?)(?=&|$)/)
+            val = val_to_omit[1]
+            hsh[:query].sub!( val, "__OMIT__#{val[-char_num, char_num]}")
+          end
+        end
+
+        hsh[:is_obfuscated] = true
+      else
+        hsh[:is_obfuscated] = false
+      end
+
+      return hsh
+    end
+
+    def self.build_from_uri(uri, opts={})
+      request_params = build_request_params(uri, opts)
+      request_obj = Request.new(request_params)
+
+      return request_obj
+    end
+
+    def self.find_or_build_from_uri(uri, opts={})
+      self.with_url(uri).first || self.build_from_uri(uri, opts)
+    end
+
+
+    def self.create_from_uri(uri, opts={})
+      req = build_from_uri(uri, opts)
+      req.save
+
+      return req
+    end
+
+
+    def self.create_and_fetch_response(uri, opts={}, fetcher = nil)
+      request = find_or_build_from_uri(uri, opts)
+      fetcher = fetcher || Fetcher.new
+
+      if request.id.nil? 
+        # this request is new
+        # so skip to the fresh
+        resp = fetcher.fetch request, fresh: true 
+      else 
+        # will check the cache and the fresh
+        resp = fetcher.fetch request
+      end
+
+      # build the response
+      response = request.responses.build(resp)
+      # theoretically, response will be saved too
+      request.save
+
+      return request
+    end
+
+  end
+end

data/app/models/active_scraper/response.rb

@@ -0,0 +1,26 @@
+module ActiveScraper
+  class Response < ActiveRecord::Base
+    serialize :headers, Hash
+    belongs_to :request
+    before_save :set_checksum
+
+    
+
+    private
+    def set_checksum
+      self.checksum = body.hash
+
+      true
+    end
+
+############## class methods
+    def self.build_from_response_object(resp)
+      response = self.new
+      [:body, :headers, :content_type, :code].each do |att|
+        response.send :write_attribute, att, resp.send(att)
+      end
+
+      return response
+    end
+  end
+end

data/app/views/layouts/active_scraper/application.html.erb

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>ActiveScraper</title>
+  <%= stylesheet_link_tag    "active_scraper/application", media: "all" %>
+  <%= javascript_include_tag "active_scraper/application" %>
+  <%= csrf_meta_tags %>
+</head>
+<body>
+
+<%= yield %>
+
+</body>
+</html>

data/config/routes.rb

@@ -0,0 +1,2 @@
+ActiveScraper::Engine.routes.draw do
+end

data/db/migrate/20131229024155_create_active_scraper_requests.rb

@@ -0,0 +1,16 @@
+class CreateActiveScraperRequests < ActiveRecord::Migration
+  def change
+    create_table :active_scraper_requests do |t|
+      t.string :host
+      t.text :query
+      t.string :path
+      t.string :meta_tag
+      t.boolean :is_obfuscated
+
+      t.timestamps
+    end
+
+    add_index :active_scraper_requests, [:host, :path]
+
+  end
+end

data/db/migrate/20131229033843_create_active_scraper_responses.rb

@@ -0,0 +1,17 @@
+class CreateActiveScraperResponses < ActiveRecord::Migration
+  def change
+    create_table :active_scraper_responses do |t|
+      t.text :body, :limit => 4294967295
+      t.integer :code
+      t.text :headers
+      t.string :content_type
+      t.integer :checksum
+      t.integer :active_scraper_request_id
+
+      t.timestamps      
+    end
+
+    add_index :active_scraper_responses, [:active_scraper_request_id, :created_at], name: 'index_request_id_and_created_at'
+    add_index :active_scraper_responses, [:active_scraper_request_id, :checksum], name: 'index_request_id_and_checksum'
+  end
+end

data/lib/active_scraper/engine.rb

@@ -0,0 +1,10 @@
+module ActiveScraper
+  class Engine < ::Rails::Engine
+    isolate_namespace ActiveScraper
+    config.generators do |g|
+      g.test_framework      :rspec,        :fixture => false
+      g.assets false
+      g.helper false
+    end
+  end
+end

data/lib/active_scraper/fetcher.rb

@@ -0,0 +1,65 @@
+require 'httparty'
+
+module ActiveScraper
+  class Fetcher
+
+    def fetch(u, opts={})
+      url = convert_uri_object(u)
+      force_fresh = opts.delete :fresh
+
+      if force_fresh != true && (record = fetch_from_cache(url, opts))
+        resp_obj = record
+      else
+        resp_obj = fetch_fresh(url, opts)
+      end
+
+      build_response_object(resp_obj)
+    end
+
+
+    def fetch_fresh(url, opts={})
+      opts = opts.stringify_keys
+
+      url = url.to_s
+      verb = opts.fetch('verb'){ 'get' }
+
+      resp = HTTParty.send(verb, url)
+    end
+
+
+    # returns: 
+    #   single ScrapeCache if a valid ActiveScraper::Request exists
+    #   
+    def fetch_from_cache(uri, opts={})
+
+    end
+
+    # true or false if ActiveScraper::Request with these parameters exist
+    def has_cache?(uri, opts={})
+
+    end
+
+
+    # u can either be a Request object, a String, or Addressable::URI
+    # returns an Addressable::URI
+    def convert_uri_object(u)
+      if u.is_a?(ActiveScraper::Request)
+        x = u.uri
+      else
+        x = Addressable::URI.parse(u)
+      end
+
+      return x
+    end
+
+    def build_response_object(obj)
+      self.class.build_response_object(obj)
+    end
+
+    # returns an OpenStruct that Response can use
+    def self.build_response_object(obj)
+      return AgnosticResponseObject.new(obj)
+    end
+
+  end
+end

data/lib/active_scraper/version.rb

@@ -0,0 +1,3 @@
+module ActiveScraper
+  VERSION = "0.0.1"
+end

data/lib/active_scraper.rb

@@ -0,0 +1,5 @@
+require "active_scraper/engine"
+require "active_scraper/fetcher"
+
+module ActiveScraper
+end

data/lib/tasks/active_scraper_tasks.rake

@@ -0,0 +1,4 @@
+# desc "Explaining what the task does"
+# task :active_scraper do
+#   # Task goes here
+# end

metadata

@@ -0,0 +1,243 @@
+--- !ruby/object:Gem::Specification
+name: active_scraper
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Dan Nguyen
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-01-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 4.1.0.beta1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 4.1.0.beta1
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: database_cleaner
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A Rails Engine using ActiveRecord to cache results of HTTP scrapes
+email:
+- dansonguyen@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- app/assets/stylesheets/active_scraper/application.css
+- app/controllers/active_scraper/application_controller.rb
+- app/helpers/active_scraper/application_helper.rb
+- app/models/active_scraper/agnostic_response_object.rb
+- app/models/active_scraper/request.rb
+- app/models/active_scraper/response.rb
+- app/views/layouts/active_scraper/application.html.erb
+- config/routes.rb
+- db/migrate/20131229024155_create_active_scraper_requests.rb
+- db/migrate/20131229033843_create_active_scraper_responses.rb
+- lib/active_scraper/engine.rb
+- lib/active_scraper/fetcher.rb
+- lib/active_scraper/version.rb
+- lib/active_scraper.rb
+- lib/tasks/active_scraper_tasks.rake
+- MIT-LICENSE
+- Rakefile
+homepage: http://github.com/dannguyen/active_scraper
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.1.11
+signing_key: 
+specification_version: 4
+summary: A Rails Engine using ActiveRecord to cache results of HTTP scrapes
+test_files: []