edamame 0.2.0

data/lib/edamame/persistent_queue.rb

@@ -0,0 +1,152 @@
+module Edamame
+  class PersistentQueue
+    DEFAULT_OPTIONS = {
+      :queue => { :type => :beanstalk_queue, :uris => ['localhost:11100'] },
+      :store => { :type => :tyrant_store,    :uri  =>           ':11101'  }
+    }
+    # Hash of options used to create this Queue.  Don't mess with this after
+    # creating the object -- it will be futile, at best.
+    attr_reader :options
+    # The default tube for the transient queue
+    # Tube name must be purely alphanumeric
+    attr_reader :tube
+    # The database backing store to use, probably a Edamame::Store
+    attr_reader :store
+    # The priority queue to use, probably a Edamame::Queue
+    attr_reader :queue
+    #
+    # Create a PersistentQueue with options
+    #
+    # @param [Hash] options the options to create a message with.
+    # @option options [String] :tube  The default tube for the transient queue
+    # @option options [String] :queue Option hash for the Edamame::Queue
+    # @option options [String] :store Option hash for the Edamame::Store
+    #
+    def initialize _options={}
+      @options = PersistentQueue::DEFAULT_OPTIONS.deep_merge(_options)
+      @tube    = options[:tube] || :default
+      @store   = Edamame::Store.create options[:store]
+      @queue   = Edamame::Queue.create options[:queue].merge(:default_tube => @tube)
+    end
+
+    #
+    # Add a new Job to the queue
+    #
+    def put job, *args
+      job.tube  = self.tube if job.tube.blank?
+      self.tube = job.tube
+      return if store.include?(job.key)
+      store.save job
+      queue.put  job, *args
+    end
+    # Alias for put(job)
+    def << job
+      put job
+    end
+
+    # Set the default tube
+    def tube= _tube
+      return if @tube == _tube
+      puts "#{self.class} setting tube to #{_tube}, was #{@tube}"
+      queue.tube = @tube = _tube
+    end
+
+    # Retrieve named record
+    def get key, klass=nil
+      klass ||= Edamame::Job
+      hsh = store.get(key) or return
+      klass.from_hash hsh
+    end
+
+    #
+    # Request a job fom the queue for processing
+    #
+    def reserve timeout=nil, klass=nil
+      qjob     = queue.reserve(timeout) or return
+      job      = get(qjob.key, klass)   or return
+      job.qjob = qjob
+      job
+    end
+
+    #
+    # Remove the job from the queue.
+    #
+    def delete job
+      store.delete job.key
+      queue.delete job.qjob
+    end
+
+    #
+    # Returns the job to the queue, to be re-run later.
+    #
+    # release'ing a job acknowledges it was completed, successfully or not
+    #
+    def release job
+      job.update!
+      store.save    job
+      queue.release job.qjob, job.priority, job.scheduling.delay
+    end
+
+    #
+    # Returns each job as it appears in the queue.
+    #
+    # all jobs -- active, inactive, running, etc -- are returned,
+    # and in some arbitrary order.
+    #
+    def each klass=nil, &block
+      klass ||= Edamame::Job
+      store.each_as(klass) do |key, job|
+        yield job
+      end
+    end
+
+    #
+    # Loads all jobs from the backing store into the queue.
+    #
+    def load &block
+      hoard do |job|
+        yield(job) if block
+        unless store.include?(job.key)
+          warn "Missing job: #{job.inspect}"
+        end
+      end
+      unhoard &block
+    end
+
+    # Returns a hash of stats about the store and queue
+    def stats
+      { :store_stats => store.stats,
+        :queue_stats => queue.stats,
+        :tube        => self.tube }
+    end
+
+  protected
+    #
+    # Destructively strips the beanstalkd queue of all of its jobs.
+    #
+    # This is the only way (I know) to enumerate all of the jobs in the queue --
+    # certainly the only way that respects concurrency.
+    #
+    # You shouldn't use this in general; the point of the backing store is to
+    # allow exactly such queries and enumeration. See #each instead.
+    #
+    def hoard &block
+      queue.empty tube, &block
+    end
+
+    #
+    # Loads all jobs from the backing store into the queue.
+    #
+    # The queue must be emptied of all jobs before running this command:
+    # otherwise jobs will be duplicated.
+    #
+    def unhoard klass=nil, &block
+      each(klass) do |job|
+        self.tube = job.tube
+        yield(job) if block
+        queue.put job, job.priority, Edamame::IMMEDIATELY
+      end
+    end
+  end
+
+end

data/lib/edamame/queue.rb

@@ -0,0 +1,6 @@
+module Edamame
+  module Queue
+    extend FactoryModule
+    autoload :BeanstalkQueue, 'edamame/queue/beanstalk'
+  end
+end

data/lib/edamame/queue/beanstalk.rb

@@ -0,0 +1,134 @@
+module Edamame
+  module Queue
+    #
+    # Persistent job queue for periodic requests.
+    #
+    # Jobs are reserved, run, and if successful put back with an updated delay parameter.
+    #
+    # This is useful for mass scraping of timelines (RSS feeds, twitter search
+    # results, etc. See http://github.com/mrflip/wuclan for )
+    #
+    class BeanstalkQueue
+      DEFAULT_OPTIONS = {
+        :priority          => 65536,    # default job queue priority
+        :time_to_run       => 60*5,     # 5 minutes to complete a job or assume dead
+        :uris              => ['localhost:11300'],
+        :default_tube      => 'default',
+      }
+      attr_accessor :options
+
+      #
+      # beanstalk_pool -- specify nil to use the default single-node ['localhost:11300'] pool
+      #
+      def initialize _options={}
+        self.options = DEFAULT_OPTIONS.deep_merge(_options.compact)
+        options[:default_tube] = options[:default_tube].to_s
+      end
+
+      #
+      # Add a new Qjob to the queue
+      #
+      def put job, priority=nil, delay=nil
+        beanstalk.put(job.key, (priority || job.priority), (delay || job.delay), job.ttr)
+      end
+
+      #
+      # Remove the qjob from the queue.
+      #
+      def delete(qjob)
+        qjob.delete
+      end
+
+      #
+      # Returns the qjob to the queue, to be re-run later.
+      #
+      # release'ing a qjob acknowledges it was completed, successfully or not
+      #
+      def release qjob, priority=nil, delay=nil
+        qjob.release( (priority || qjob.priority), (delay || qjob.delay) )
+      end
+
+      #
+      # Take the next (highest priority, delay met) qjob.
+      # Set timeout (default is 10s)
+      # Returns nil on error or timeout. Interrupt error passes through
+      #
+      def reserve timeout=10
+        begin
+          qjob = beanstalk.reserve(timeout) or return
+        rescue Beanstalk::TimedOut => e ; warn e.to_s ; sleep 0.4 ; return ;
+        rescue StandardError => e       ; warn e.to_s ; sleep 1   ; return ; end
+        qjob
+      end
+
+      #
+      # Shelves the qjob.
+      #
+      def bury
+        qjob.bury qjob.priority
+      end
+
+      # The beanstalk pool which acts as job queue
+      def beanstalk
+        return @beanstalk if @beanstalk
+        @beanstalk = Beanstalk::Pool.new(options[:uris], options[:default_tube])
+        self.tube= options[:default_tube]
+        @beanstalk
+      end
+      # Close the job queue
+      def close
+        @beanstalk.close if @beanstalk
+        @beanstalk = nil
+      end
+
+      # uses and watches the given beanstalk tube
+      def tube= _tube
+        puts "#{self.class} setting tube to #{_tube}, was #{@tube}"
+        @beanstalk.use   _tube
+        @beanstalk.watch _tube
+      end
+
+      # Stats on job count across the pool
+      def stats
+        beanstalk.stats.select{|k,v| k =~ /jobs/}
+      end
+      # Total jobs in the queue, whether reserved, ready, buried or delayed.
+      def current_jobs
+        beanstalk.
+          stats.
+          select{|k,v| (k =~ /jobs/) && (k != 'total-jobs')}.
+          inject(0){|sum,kv| sum += kv.last }
+      end
+
+      #
+      #
+      #
+      def empty tube=nil, &block
+        tube = tube.to_s if tube
+        curr_tube    = beanstalk.list_tube_used.values.first
+        curr_watches = beanstalk.list_tubes_watched.values.first
+        beanstalk.use   tube if tube
+        beanstalk.watch tube if tube
+        p ["emptying", tube, current_jobs]
+        loop do
+          kicked = beanstalk.open_connections.map{|conxn| conxn.kick(20) }
+          break if (current_jobs == 0) || (!beanstalk.peek_ready)
+          qjob = reserve(5) or break
+          yield qjob
+          qjob.delete
+        end
+        beanstalk.use curr_tube
+        beanstalk.ignore tube if (! curr_watches.include?(tube))
+      end
+
+      def empty_all &block
+        tubes = beanstalk.list_tubes.values.flatten.uniq
+        tubes.each do |tube|
+          empty tube,  &block
+        end
+      end
+
+    end # class
+  end
+end
+

data/lib/edamame/scheduling.rb

@@ -0,0 +1,79 @@
+require 'wukong/extensions/hashlike_class'
+module Edamame
+  # sugar for rescheduled jobs
+  IMMEDIATELY = 0
+
+  module Scheduling
+    extend FactoryModule
+
+    # def type
+    #   self.class.to_s
+    # end
+    # def to_hash
+    # end
+
+    class Base
+      include Wukong::HashlikeClass
+      has_members :last_run, :total_runs
+
+      def initialize *args
+        members.zip(args).each do |key, val|
+          self[key] = val if val
+        end
+      end
+
+      def last_run_time
+        last_run.is_a?(String) ? Time.parse(last_run) : last_run
+      end
+
+      def since_last
+        Time.now - last_run_time
+      end
+
+    end
+
+    class Every < Base
+      has_member :delay
+    end
+
+    class At < Base
+      attr_accessor :time
+      def initialize *args
+        super *args
+        self.time = Time.parse(time) if time.is_a?(String)
+      end
+      def delay
+        @delay ||= time - Time.now
+      end
+    end
+
+    class Once < Base
+      def delay
+        nil
+      end
+    end
+
+    #
+    # A recurring task
+    #
+    # * Run every once in a while -- often enough that you don't miss anything
+    #
+    #   want to scrape everything between now and prev_item
+    #
+    # * at the previous run, objects up to prev_time and prev_id
+    # * in the current run,  objects up to curr_time and curr_id
+    # * average rate
+    #
+    class Recurring < Base
+      has_members :delay, :prev_max, :prev_items, :prev_items_rate
+    end
+
+
+      # :total_items, :goal_items,
+      # cattr_accessor :min_resched_delay, :max_resched_delay
+      # self.min_resched_delay = 10
+      # self.max_resched_delay = 24*60*60
+
+  end
+
+end

data/lib/edamame/store.rb

@@ -0,0 +1,8 @@
+require 'monkeyshines/utils/factory_module'
+module Edamame
+  module Store
+    extend FactoryModule
+    autoload :Base,        'edamame/store/base'
+    autoload :TyrantStore, 'edamame/store/tyrant_store'
+  end
+end

data/lib/edamame/store/base.rb

@@ -0,0 +1,62 @@
+# require 'monkeyshines/utils/factory_module'
+module Edamame
+  module Store
+    class Base
+      # The actual backing store; should respond to #set and #get methods
+      attr_accessor :db
+
+      def initialize options
+      end
+
+      #
+      # Executes block once for each element in the whole DB, in whatever order
+      # the DB thinks you should see it.
+      #
+      # Your block will see |key, val|
+      #
+      # key_store.each do |key, val|
+      #   # ... stuff ...
+      # end
+      #
+      def each &block
+        db.iterinit
+        loop do
+          key = db.iternext or break
+          val = db[key]
+          yield key, val
+        end
+      end
+
+      def each_as klass, &block
+        self.each do |key, hsh|
+          yield [key, klass.from_hash(hsh)]
+        end
+      end
+
+      # Delegate to store
+      def set(key, val)
+        return unless val
+        db.put key, val.to_hash.compact
+      end
+      def save obj
+        return unless obj
+        db.put obj.key, obj.to_hash.compact
+      end
+
+      def get(key)      db[key]         end
+      def [](key)       get(key)        end
+      def put(key, val) db.put key, val end
+      def close()       db.close        end
+      def size()        db.size         end
+      def delete(key)   db.delete(key)  end
+
+      #
+      # Load from standard command-line options
+      #
+      # obvs only works when there's just one store
+      #
+      def self.create type, options
+      end
+    end
+  end
+end

data/lib/edamame/store/tyrant_store.rb

@@ -0,0 +1,49 @@
+require 'tokyotyrant'
+module Edamame
+  module Store
+
+    #
+    # Implementation of KeyStore with a Local TokyoCabinet table database (TDB)
+    #
+    class TyrantStore < Edamame::Store::Base
+      attr_accessor :db_host, :db_port
+
+      # pass in the host:port uri of the key store.
+      def initialize options
+        self.db_host, self.db_port = options[:uri].to_s.split(':')
+        super options
+      end
+
+      def db
+        return @db if @db
+        @db ||= TokyoTyrant::RDBTBL.new
+        @db.open(db_host, db_port) or raise("Can't open DB #{db_host}:#{db_port}. Pass in host:port' #{@db.ecode}: #{@db.errmsg(@db.ecode)}")
+        @db
+      end
+
+      def close
+        @db.close if @db
+        @db = nil
+      end
+
+      # Save the value into the database without waiting for a response.
+      def set_nr(key, val)
+        db.putnr key, val if val
+      end
+
+      def size()
+        db.rnum
+      end
+
+      def stats
+        {  :size => size }
+      end
+
+      def include? *args
+        db.has_key? *args
+      end
+
+    end # TyrantStore
+  end
+end
+