crawl_kit 0.0.1

data/.gitignore

@@ -0,0 +1,2 @@
+*.gem
+.DS_Store

data/Rakefile

@@ -0,0 +1,13 @@
+#!/usr/bin/env rake
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end
+
+desc "Run tests"
+task :default => :test

data/crawl_kit.gemspec

@@ -0,0 +1,16 @@
+Gem::Specification.new do |s|
+  s.name        = 'crawl_kit'
+  s.version     = '0.0.1'
+  s.date        = '2011-12-08'
+  s.summary     = "A collection of serivce interfaces and models to use with the GoTime crawling infrastructure."
+  s.author      = "Austin Cargol"
+  s.email       = 'acargol@gotime.com'
+  
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test}/*`.split("\n")
+  s.require_paths = %w(lib)
+
+  s.required_ruby_version     = '>= 1.9.2'
+
+  s.add_dependency('aws-sdk', '>= 1.2.3')
+end

data/lib/crawl_kit/configuration.rb

@@ -0,0 +1,81 @@
+require 'set'
+require 'uri'
+
+module CrawlKit
+  class Configuration
+
+    # Creates a new Configuration object.
+    def initialize(options = {})
+      options.each_pair do |opt_name, value|
+        opt_name = opt_name.to_sym
+        if self.class.accepted_options.include?(opt_name)
+          supplied[opt_name] = value
+        end
+      end
+    end
+
+    # Used to create a new Configuration object with the given modifications.
+    # The current configuration object is not modified.
+    def with(options = {})
+      # symbolize option keys
+      options = options.inject({}) {|h,kv| h[kv.first.to_sym] = kv.last; h }
+
+      values = supplied.merge(options)
+
+      if supplied == values
+        self # nothing changed
+      else
+        self.class.new(values)
+      end
+    end
+
+    # @return [Hash] Returns a hash of all configuration values.
+    def to_h
+      self.class.accepted_options.inject({}) do |h,k|
+        h[k] = send(k)
+        h
+      end
+    end
+
+    # @return [Boolean] Returns true if the two configuration objects have
+    #   the same values.
+    def == other
+      other.is_a?(self.class) and self.supplied == other.supplied
+    end
+
+    alias_method :eql, :==
+
+    # @private
+    def inspect
+      "<#{self.class.name}>"
+    end
+
+    protected
+    def supplied
+      @supplied ||= {}
+    end
+
+    class << self
+
+      # @private
+      def accepted_options
+        @options ||= Set.new
+      end
+
+      # @private
+      def add_option(name, default_value = nil, options = {}, &transform)
+        accepted_options << name
+
+        define_method(name) do
+          value = supplied.has_key?(name) ? supplied[name] : default_value
+          transform ? transform.call(value) : value
+        end
+
+        alias_method("#{name}?", name) if options[:boolean]
+      end
+    end
+
+    add_option :crawl_domain, ENV['CRAWL_DOMAIN']
+    add_option :crawl_bucket, ENV['CRAWL_BUCKET']
+  end
+end

data/lib/crawl_kit/record/helpers.rb

@@ -0,0 +1,37 @@
+module CrawlKit
+  class Record < AWS::Record::Base
+    module Helpers
+      
+      def self.included(base)
+        base.extend(ClassMethods)
+      end
+
+      module ClassMethods
+        def digest
+          @digest ||= OpenSSL::Digest::Digest.new('sha1')
+        end
+        
+        def hash_uri(uri=nil)
+          digest.hexdigest(uri)
+        end
+      end
+
+      def hashed_uri
+        self.class.digest.hexdigest(uri)
+      end
+      alias :crawl_id :hashed_uri
+
+      def fetched?
+        CrawlKit::S3.file_exists?(crawl_id)
+      end
+
+      def page
+        CrawlKit::S3.get_file(crawl_id)
+      end
+      
+      def previous_version
+        CrawlKit::S3.get_previous_version(crawl_id)
+      end
+    end
+  end
+end

data/lib/crawl_kit/record.rb

@@ -0,0 +1,69 @@
+module CrawlKit
+  class Record < AWS::Record::Base
+    include CrawlKit::Record::Helpers
+    
+    # State Fields
+    string_attr :uri
+    string_attr :lifecycle, default_value: "new"
+    string_attr :page_type
+    string_attr :referer
+    integer_attr :site_id
+    integer_attr :market_id
+    integer_attr :crawl_node_id
+    integer_attr :external_region_node_id
+    integer_attr :external_category_node_id
+
+    validates_presence_of :uri
+
+    # Crawl Fields
+    string_attr :domain
+    string_attr :status
+    string_attr :status_code
+    string_attr :last_effective_url
+    string_attr :location
+    string_attr :x_city_id
+    string_attr :last_changed
+    string_attr :last_visited
+    string_attr :last_modified
+    string_attr :content_hash
+    string_attr :content_length
+    string_attr :content_type
+    string_attr :etag
+    string_attr :x_cvs_id
+    string_attr :x_category
+    
+    class << self
+      def find(id)
+        id =~ /^[^\.]+$/ ? super(id) : super(hash_uri(id))
+      end
+      
+      def find_or_create_by_uri(uri)
+        find(hash_uri(uri))
+      rescue
+        new(uri: uri)
+      end
+    end
+    
+    def seed_attributes
+      Hash.new(
+        site_id: site_id,
+        market_id: market_id,
+        crawl_node_id: crawl_node_id,
+        external_region_node_id: external_region_node_id,
+        external_category_node_id: external_category_node_id,
+        referer: uri
+      )
+    end
+    
+    def update_blank_attributes_only(attributes)
+      attributes.delete_if {|key, value| send(key) }
+      update_attributes(attributes)
+    end
+    
+    private
+    # DO NOT CHANGE. This overwrites the default behavior
+    def populate_id
+      @_id = crawl_id
+    end
+  end
+end

data/lib/crawl_kit/s3.rb

@@ -0,0 +1,58 @@
+module CrawlKit
+  class S3
+    class << self
+      def connection
+        AWS::S3.new
+      end
+      
+      def buckets
+        connection.buckets
+      end
+      
+      def crawl_bucket(bucket_name=nil)
+        bucket_name ||= CrawlKit.config.crawl_bucket
+        raise "A crawl bucket is required" unless bucket_name
+        buckets[bucket_name]
+      end
+      
+      def get_object(key)
+        crawl_bucket.objects[key]
+      end
+      
+      def file_exists?(key)
+        get_object(key).exists?
+      end
+      
+      def get_file(key)
+        if file = get_object(key)
+          decompress(file.read)
+        end
+      end
+
+      def save_page(key, body, options)
+        options.merge!(content_type: 'text/html', content_encoding: 'deflate', storage_class: :reduced_redundancy)
+        object = get_object(key)
+        object.write(Zlib::Deflate.deflate(body), options)
+      end      
+      
+      def get_versions(hashed_uri)
+        s3_object = get_object(hashed_uri)
+        versions = s3_object.versions.to_a
+      end
+
+      # A step of 0 will give the current version.
+      def get_previous_version(hashed_uri, step=1)
+        previous_version = get_versions(hashed_uri).sort_by{|version| version.head.last_modified}.reverse[step]
+        decompress(previous_version.read) if previous_version
+      end
+      
+      private
+      def decompress(str)
+        body = Zlib::Inflate.inflate(str)
+        body.force_encoding(::Encoding::UTF_8)
+        body
+      end
+      
+    end # Class Methods
+  end
+end

data/lib/crawl_kit/simple_db.rb

@@ -0,0 +1,46 @@
+module CrawlKit
+  class SimpleDB
+    class << self
+      def connection
+        AWS::SimpleDB.new
+      end
+      
+      def crawl_domain(domain_name=nil)
+        domain_name ||= CrawlKit.config.crawl_domain
+        raise "A crawl domain is required" unless domain_name
+        connection.domains[domain_name]
+      end
+      
+      def items
+        crawl_domain.items
+      end
+      
+      def get_item(hashed_uri)
+        crawl_domain.items[hashed_uri]
+      end
+      alias :find :get_item
+      
+      def delete_item(hashed_uri) #replaced delete_attributes
+        crawl_domain.items[hashed_uri].try(:delete)
+      end
+      
+      def select(params)
+        items.where(params)
+      end
+      
+      def count(params)
+        items.where(params).count
+      end
+
+      def select_each(params)
+        select(params).each do |item|
+          yield(item)
+        end
+      end
+      
+      def each_item(params)
+        select(params).each {|item| yield(item) }
+      end
+    end # Class Methods
+  end
+end

data/lib/crawl_kit/sqs.rb

@@ -0,0 +1,40 @@
+module CrawlKit
+  class SQS
+    class << self
+      def connection
+        @connection ||= AWS::SQS.new
+      end
+      
+      def get_queue(name="crawl_new")
+        queues.detect{|queue|queue.name.eql?(name)}
+      end
+      
+      def queues
+        connection.queues.to_a
+      end
+
+      def create_queue(name)
+        connection.queues.create(name)
+      end
+      alias :get_or_create_queue :create_queue
+
+      def enqueue_message(queue, message)
+        queue.send_message(ActiveSupport::JSON.encode(message))
+      end
+      
+      def consume_message(queue)
+        ActiveSupport::JSON.decode(queue.receive_message)
+      end
+
+      def clear_queue(name)
+        delete_queue(name)
+        sleep(70)
+        create_queue(name)
+      end
+
+      def delete_queue(name)
+        get_queue(name).try(:delete)
+      end
+    end # Class Methods
+  end
+end

data/lib/crawl_kit.rb

@@ -0,0 +1,24 @@
+require 'aws-sdk'
+
+require 'crawl_kit/configuration'
+
+module CrawlKit
+  
+  class << self
+    @@config = nil
+    
+    def config(options = {})
+      @@config ||= CrawlKit::Configuration.new
+      @@config = @@config.with(options) unless options.empty?
+      CrawlKit::Record.set_domain_name(@@config.crawl_domain) if @@config.crawl_domain
+      @@config
+    end
+  end
+end
+
+require 'crawl_kit/record/helpers'
+require 'crawl_kit/record'
+require 'crawl_kit/s3'
+require 'crawl_kit/simple_db'
+require 'crawl_kit/sqs'
+require 'extensions/aws/sqs/queue.rb'

data/lib/extensions/aws/sqs/queue.rb

@@ -0,0 +1,25 @@
+module AWS
+  class SQS
+    class Queue      
+      def name
+        url[/\/([^\/]+)$/, 1]
+      end
+      
+      def enqueue(message)
+        send_message(ActiveSupport::JSON.encode(message))
+      end
+      
+      def consume
+        message = receive_message
+        if message
+          message.delete
+          ActiveSupport::JSON.decode(message.body)
+        end
+      end
+      
+      def size
+        approximate_number_of_messages
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,67 @@
+--- !ruby/object:Gem::Specification
+name: crawl_kit
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Austin Cargol
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-12-08 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: &70283953721820 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.2.3
+  type: :runtime
+  prerelease: false
+  version_requirements: *70283953721820
+description: 
+email: acargol@gotime.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Rakefile
+- crawl_kit.gemspec
+- lib/crawl_kit.rb
+- lib/crawl_kit/configuration.rb
+- lib/crawl_kit/record.rb
+- lib/crawl_kit/record/helpers.rb
+- lib/crawl_kit/s3.rb
+- lib/crawl_kit/simple_db.rb
+- lib/crawl_kit/sqs.rb
+- lib/extensions/aws/sqs/queue.rb
+homepage: 
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.9.2
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: A collection of serivce interfaces and models to use with the GoTime crawling
+  infrastructure.
+test_files: []