crocoduck 0.0.5

data/lib/crocoduck/entry.rb

@@ -0,0 +1,73 @@
+# The Entry object represents a document retrieved from
+# the datastore.  By default this is a MongoDB document.
+require 'crocoduck/job'
+require 'crocoduck/redis'
+require 'crocoduck/resque'
+require 'crocoduck/store'
+
+module Crocoduck
+  class Entry    
+    attr_accessor :entry_id, :entry, :store
+
+    def initialize(entry_id)
+      @entry_id = entry_id
+    end
+
+    # A quick way to start work on an Entry is to do something
+    # like the following
+    #
+    #     >>> e = Entry.new(53029).schedule(ShortUrlJob)
+    def schedule(worker = Job)
+      Resque.enqueue worker, entry_id
+    end
+    
+    # Rather than access ``Crocoduck::Entry.entry`` directly, one can do the
+    # following:
+    #
+    #     :001 > e = Crocoduck::Entry.new(50039)
+    #      => #<Crocoduck::Entry:0x101611938 @entry_id=50039> 
+    #     :002 > e["url"]
+    #      => "/apple/news/2011/04/this-is-not-a-real-article.ars"
+    def [](key)
+      if entry.has_key? key
+        entry[key]
+      else
+        nil
+      end
+    end
+    
+    # This hasn't been field tested yet, but ``update`` should be a
+    # convienance method to manipulate a field on the entry document
+    # stored here.  If a job needed to store results or data on a
+    # different document, she could use the ``Crocoduck::Store.update`` method
+    # directly.
+    def update(field, value)
+      store.update entry_id, field, value
+    end
+    
+    # Call this method on your entries to have them close their own
+    # store object.
+    def close
+      store.close
+    end
+    
+    def setup?
+      store.setup? && !entry.nil?
+    end
+    
+    private
+    
+    # When the ``entry`` property of an Entry object is accessed
+    # we attempt to retrieve the document from the store, save it
+    # on our object, and then return it.  Further accesses get the
+    # cached copy of the document.
+    def entry
+      @entry ||= store.get entry_id
+    end
+    
+    # Accessing ``Crocoduck::Entry.store`` gets you a new store object to work with.
+    def store
+      @store ||= Store.new
+    end
+  end
+end

data/lib/crocoduck/job.rb

@@ -0,0 +1,102 @@
+# `Job` is a class that is intended to be extended to do meaningful work. A
+#  Crocoduck Job is simply a Resque style job that knows about its own
+#  datastore and an entry object (Mongo Document when using the supplied
+#  ``store`` class).
+require 'crocoduck/logging'
+require 'crocoduck/entry'
+
+module Crocoduck
+  class Job
+    # Override the value of ``@queue`` to specify which resque workers will 
+    # process this job.
+    @queue = :low
+    
+    class << self
+      attr_accessor :description
+    end
+    
+    # ``perform`` is the method called by Resque. A Crocoduck job only expects
+    # an ``entry_id`` corresponding to a record in your Mongo store.  An
+    # ``Entry`` is instantiated with said ``entry_id`` and passed to a new
+    # instance of this job and run is called on it.
+    def self.perform(entry_id)
+      init_with_id(entry_id).run
+    end
+    
+    # A convienance initializer that returns a Crocoduck::Job instance with
+    # its entry object ready to go.
+    def self.init_with_id(entry_id)
+      new(Entry.new entry_id)
+    end
+    
+    include Logging
+    
+    attr_accessor :entry
+    
+    def initialize(entry)
+      @entry = entry
+    end
+    
+    # The ``do_work`` method should be overridden to do some kind of work on
+    # the stored entry object.
+    def do_work
+      logger.info "Starting work"
+      # Do Something with entry
+      # entry.update "derp", "herp"
+      logger.info entry["url"]
+      # shorturl = shorturl.generate @entry.url
+      # store.update entry_id, 'shorturl', shorturl
+      # store.update entry_id, 'shorturl_status, job_status
+      logger.info "Ending work"
+    end
+    
+    # If you job failed, you can do something interesting here.  Generally
+    # you will want to ultimately raise the exception so Resque can track it.
+    def handle_exception(e)
+      raise e
+    end
+    
+    # This method will be called immediately before sanity checks and before
+    # ``do_work`` is called.
+    def setup
+      logger.info "Job is setup"
+    end
+    
+    # This method will be called once ``do_work`` has finished successfully.
+    # Do anything you'd need to do once the processing was finished
+    # properly (save out your entry, update stats, et cetera).
+    def finished
+      logger.info "Job finished successfully"
+    end
+    
+    # This method will always be called, regardless of the failure or
+    # success of your job.
+    def cleanup
+      entry.close
+      logger.info "Job cleaned up"
+    end
+    
+    # The ``run`` method is a thin wrapper around ``do_work`` which lets us
+    # do some setup, benchmark the work we'll do, cleanly handle exceptions if
+    # thrown by the ``do_work`` call, and clean up our store and entry on
+    # success.
+    def run
+      setup
+      # The job will not process anything unless our datastore has enough
+      # information to connect and if a valid entry object could be fetched
+      # from the store.
+      return unless entry.setup?
+      benchmark :info, "Running job" do
+        do_work
+      end
+    # Exception handling is parceled out to ``Job`` methods you can override
+    # to handle cleanup specific to your task.
+    rescue Exception => e
+      handle_exception e
+    else
+      finished
+    ensure
+      cleanup
+    end
+  end
+end

data/lib/crocoduck/logging.rb

@@ -0,0 +1,25 @@
+# Include Loggging into your class to get a logger and benchmark
+# object for logging errors or information to stdout and for profiling
+# interesting bits of code.
+require 'benchmark'
+require 'logger'
+
+module Crocoduck
+  def self.logger
+    @logger ||= Logger.new($stderr)
+  end
+
+  module Logging
+    private
+      def logger
+        Crocoduck.logger
+      end
+
+      def benchmark(level, message)
+        result = nil
+        ms = Benchmark.realtime { result = yield }
+        logger.send(level, '%s (%.5fs)' % [ message, ms ])
+        result
+      end
+  end
+end

data/lib/crocoduck/redis.rb

@@ -0,0 +1,7 @@
+require 'redis'
+require 'uri'
+
+module Crocoduck
+  uri = URI.parse "http://localhost:6379"
+  Redis = ::Redis.new({ :host => uri.host, :port => uri.port, :password => uri.password })
+end

data/lib/crocoduck/resque.rb

@@ -0,0 +1,8 @@
+require 'crocoduck/redis'
+require 'resque'
+require 'resque/server'
+
+module Crocoduck
+  Resque = ::Resque
+  ::Resque.redis = Redis
+end

data/lib/crocoduck/server.rb

@@ -0,0 +1,19 @@
+require 'sinatra/base'
+require 'crocoduck/entry'
+
+module Crocoduck
+  class Server < Sinatra::Base
+    set :root, File.dirname(__FILE__)
+
+    get "/" do
+      erb :index
+    end
+
+    post "/" do
+      entry_id = params[:entry_id]
+      entry = Entry.new entry_id
+      entry.schedule
+      redirect "/"
+    end
+  end
+end

data/lib/crocoduck/store.rb

@@ -0,0 +1,84 @@
+# The Crocoduck::Store object handles the concern of talking to your
+# data storage layer.  By default, we have implemented this on top
+# of MongoDB, so it may be that many of the choices made here highly
+# favor document-based databases.
+require 'mongo'
+require 'crocoduck/logging'
+
+module Crocoduck
+  class Store
+    include Logging
+    # We have several class properties that defined how all Store
+    # objects will connect and query for information.  As stated
+    # before, many of these will only make sense for MongoDB or
+    # other similar document-based databases.
+    @id_field = '_id'
+    @server_cluster = nil
+    @server_db = nil
+    @server_collection = nil
+
+    class << self
+      attr_accessor :id_field, :server_cluster, :server_db, :server_collection
+    end
+    
+    attr_accessor :store, :database, :collection
+    
+    # A nice method to determine if there is enough information
+    # to potentially connect to the backing database.
+    def setup?
+      Crocoduck::Store.server_cluster && 
+      Crocoduck::Store.server_db && 
+      Crocoduck::Store.server_collection
+    end
+
+    def close
+      store.close
+    end
+
+    # A simple convienance method to update a single
+    # document in your datastore.
+    def update(entry_id, field, value)
+      collection.update({
+        Crocoduck::Store.id_field => entry_id}, 
+        {'$set' => { field => value}
+      }, :safe => true)
+    end
+    
+    # Returns a single document given its ID
+    def get(id)
+      collection.find_one({
+        Crocoduck::Store.id_field => id.to_i
+      })
+    end
+    
+    # Use this method to remove documents from your datastore.  Cares
+    # has been taken to prevent accidental database destruction.  Only
+    # pass {} to this method if you are 100% sure you want to clear the
+    # database.
+    def remove(criteria=nil)
+      return if criteria.nil?
+      collection.remove criteria
+    end
+    
+    # Inserts a brand new document into the database
+    def insert(document)
+      collection.insert document
+    end
+    
+    private
+    
+    # These methods create and cache objects that maintain the state and
+    # connectivity to the backend storage.
+    def collection
+      @collection ||= database.collection Crocoduck::Store.server_collection
+    end
+    
+    def database
+      @database ||= store.db(Crocoduck::Store.server_db)
+    end
+    
+    def store
+      @store ||= Mongo::ReplSetConnection.new(*Crocoduck::Store.server_cluster)
+    end
+  end
+end

metadata

@@ -0,0 +1,142 @@
+--- !ruby/object:Gem::Specification 
+name: crocoduck
+version: !ruby/object:Gem::Version 
+  hash: 21
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 5
+  version: 0.0.5
+platform: ruby
+authors: 
+- Clint Ecker
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-06-17 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: redis
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: resque
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: sinatra
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: mongo
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: rdiscount
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id005
+description: "    Crocoduck is a Resque job system that seeks to model the pattern of mutating MongoDB documents.\n"
+email: me@clintecker.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/crocoduck/entry.rb
+- lib/crocoduck/job.rb
+- lib/crocoduck/logging.rb
+- lib/crocoduck/redis.rb
+- lib/crocoduck/resque.rb
+- lib/crocoduck/server.rb
+- lib/crocoduck/store.rb
+has_rdoc: true
+homepage: https://github.com/clintecker/crocoduck
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.4.1
+signing_key: 
+specification_version: 3
+summary: Resque Jobs working on MongoDB documents
+test_files: []
+