mrflip-edamame 0.1.1

data/lib/edamame/job.rb

@@ -0,0 +1,134 @@
+module Edamame
+
+  #
+  #
+  # id, name, body, timeouts, time-left, age, state, delay, pri, ttr
+  #
+  #
+  # * A job, pulled from the queue: it is connected to its beanstalk presence
+  #   body contains
+  #   ** obj
+  #   ** scheduling
+  #   ** stats
+  #
+  # * A DB job
+  #   body contains
+  #   ** tube, priority, ttr, state
+  #   ** obj
+  #   ** scheduling
+  #   ** stats
+  module JobCore
+
+    def key
+      [ tube, obj[:key]||obj['key'] ].join('-')
+    end
+
+    #
+    def since_last
+      scheduling.last_run - Time.now
+    end
+
+    # Beanstalk::Job stats:
+    #    { "pri"=>65536, "ttr"=>120,
+    #    {"releases"=>8, "delay"=>5, "kicks"=>0, "buries"=>0, "id"=>202,
+    #     "tube"=>"default", "time-left"=>120,
+    #     "timeouts"=>0, "age"=>1415, "state"=>"reserved"}
+    #
+    #    [ "id",
+    #      "tube", "pri", "ttr", "state",
+    #      "delay",
+    #      "releases", "kicks", "buries",
+    #      "time-left", "timeouts", "age", ]
+  end
+
+
+  Beanstalk::Job.class_eval do
+    include JobCore
+
+    def scheduling
+      @scheduling ||= Edamame::Scheduling.from_hash ybody['scheduling']
+    end
+
+    def obj
+      ybody['obj']
+    end
+
+    def priority
+      pri
+    end
+
+    def tube
+      stats['tube']
+    end
+
+    # Override this for rescheduling
+    def update!
+      scheduling.total_runs = scheduling.total_runs.to_i + stats['releases']
+      scheduling.last_run   = Time.now
+    end
+
+    def to_hash flatten=true
+      hsh =       {
+        "tube"       => tube,
+        "priority"   => priority,
+        "ttr"        => ttr,
+        "state"      => state,
+        "scheduling" => scheduling.to_hash,
+        'key'        => key,
+        "obj"        => obj.to_hash,
+      }
+      if flatten
+        hsh["scheduling"] = hsh['scheduling'].to_yaml
+        hsh["obj"]        = hsh['obj'].to_yaml
+      end
+      hsh
+    end
+  end
+
+  class Job < Struct.new(
+    :tube, :priority, :ttr, :state,
+    :scheduling, :obj
+    )
+    # Job.class_eval do
+    include JobCore
+
+    DEFAULT_OPTIONS = {
+      'priority'   => 65536,
+      'ttr'        => 120,
+      'state'      => 1,
+      'scheduling' => Edamame::Scheduling::Once.new()
+    }
+
+    # attr_accessor :runs, :failures, :prev_run_at
+    def initialize *args
+      super *args
+      DEFAULT_OPTIONS.each{|key,val| self[key] ||= val }
+      [:priority, :ttr, :state].each{|key| self[key] = self[key].to_i }
+      case self.scheduling
+      when String
+        scheduling_hash = YAML.load(self.scheduling) rescue nil
+        self.scheduling = Scheduling.from_hash(scheduling_hash) if scheduling_hash
+      when Hash
+        self.scheduling = Scheduling.from_hash(scheduling)
+      else raise "Can't build a Scheduling from #{self.scheduling}" ;  end
+      if self.obj.is_a?(String)        then self.obj        = YAML.load(self.obj)        rescue nil ; end
+    end
+
+    def delay
+      scheduling.delay
+    end
+
+    def to_hash flatten=true
+      hsh = super()
+      hsh["scheduling"] = scheduling.to_hash
+      hsh["obj"]        = obj.to_hash
+      if flatten
+        hsh["scheduling"] = hsh['scheduling'].to_yaml
+        hsh["obj"]        = hsh['obj'].to_yaml
+      end
+      hsh
+    end
+  end
+end
+
+

data/lib/edamame/queue.rb

@@ -0,0 +1,6 @@
+module Edamame
+  module Queue
+    extend FactoryModule
+    autoload :BeanstalkQueue, 'edamame/queue/beanstalk'
+  end
+end

data/lib/edamame/queue/beanstalk.rb

@@ -0,0 +1,132 @@
+module Edamame
+  module Queue
+    #
+    # Persistent job queue for periodic requests.
+    #
+    # Jobs are reserved, run, and if successful put back with an updated delay parameter.
+    #
+    # This is useful for mass scraping of timelines (RSS feeds, twitter search
+    # results, etc. See http://github.com/mrflip/wuclan for )
+    #
+    class BeanstalkQueue
+      DEFAULT_OPTIONS = {
+        :priority          => 65536,    # default job queue priority
+        :time_to_run       => 60*5,     # 5 minutes to complete a job or assume dead
+        :uris              => ['localhost:11300'],
+        :default_tube      => 'default',
+      }
+      attr_accessor :options
+
+      #
+      # beanstalk_pool -- specify nil to use the default single-node ['localhost:11300'] pool
+      #
+      def initialize _options={}
+        self.options = DEFAULT_OPTIONS.deep_merge(_options.compact)
+        options[:default_tube] = options[:default_tube].to_s
+      end
+
+      #
+      # Add a new Job to the queue
+      #
+      def put job, priority=nil, delay=nil
+        beanstalk.yput(job.to_hash(false),
+          (priority || job.priority), (delay || job.delay), job.ttr)
+      end
+
+      #
+      # Remove the job from the queue.
+      #
+      def delete(job)
+        job.delete
+      end
+
+      #
+      # Returns the job to the queue, to be re-run later.
+      #
+      # release'ing a job acknowledges it was completed, successfully or not
+      #
+      def release job, priority=nil, delay=nil
+        job.release( (priority || job.priority), (delay || job.delay) )
+      end
+
+      #
+      # Take the next (highest priority, delay met) job.
+      # Set timeout (default is 10s)
+      # Returns nil on error or timeout. Interrupt error passes through
+      #
+      def reserve timeout=10
+        begin
+          job = beanstalk.reserve(timeout) or return
+        rescue Beanstalk::TimedOut => e ; warn e.to_s ; sleep 0.4 ; return ;
+        rescue StandardError => e       ; warn e.to_s ; sleep 1   ; return ; end
+        job
+      end
+
+      #
+      # Shelves the job.
+      #
+      def bury
+        job.bury job.priority
+      end
+
+      # The beanstalk pool which acts as job queue
+      def beanstalk
+        return @beanstalk if @beanstalk
+        @beanstalk = Beanstalk::Pool.new(options[:uris], options[:default_tube])
+        self.tube= options[:default_tube]
+        @beanstalk
+      end
+      # Close the job queue
+      def close
+        @beanstalk.close if @beanstalk
+        @beanstalk = nil
+      end
+
+      # uses and watches the given beanstalk tube
+      def tube= _tube
+        puts "#{self.class} setting tube to #{_tube}, was #{@tube}"
+        @beanstalk.use   _tube
+        @beanstalk.watch _tube
+      end
+
+      # Stats on job count across the pool
+      def stats
+        beanstalk.stats.select{|k,v| k =~ /jobs/}
+      end
+      # Total jobs in the queue, whether reserved, ready, buried or delayed.
+      def total_jobs
+        [:reserved, :ready, :buried, :delayed].inject(0){|sum,type| sum += stats["current-jobs-#{type}"]}
+      end
+
+      #
+      #
+      #
+      def empty tube=nil, &block
+        tube = tube.to_s if tube
+        curr_tube    = beanstalk.list_tube_used.values.first
+        curr_watches = beanstalk.list_tubes_watched.values.first
+        beanstalk.use   tube if tube
+        beanstalk.watch tube if tube
+        p ["emptying", tube, beanstalk_total_jobs]
+        loop do
+          kicked = beanstalk.open_connections.map{|conxn| conxn.kick(20) }
+          break if (beanstalk_total_jobs == 0) || (!beanstalk.peek_ready)
+          qjob = reserve(5) or break
+          yield qjob
+          qjob.delete
+        end
+        beanstalk.use curr_tube
+        beanstalk.ignore tube if (! curr_watches.include?(tube))
+      end
+
+      def empty_all &block
+        tubes = beanstalk.list_tubes.values.flatten.uniq
+        tubes.each do |tube|
+          empty tube,  &block
+        end
+      end
+
+    end # class
+  end
+end
+

data/lib/edamame/rescheduled.rb

@@ -0,0 +1,89 @@
+module Edamame
+  module Rescheduled
+    
+    
+      # ===========================================================================
+      #
+      # Rescheduling
+
+      #
+      # Finish the qjob and re-insert it at the same priority but with the new
+      # delay setting.
+      #
+      def reschedule qjob, scrape_job
+        priority = qjob.stats['pri']
+        qjob.delete
+        self.save scrape_job, priority
+      end
+
+      #
+      # Flattens the scrape_job and enqueues it with a delay appropriate for the
+      # average item rate so far. You can explicitly supply a +priority+ to
+      # override the priority set at instantiation.
+      #
+      # This doesn't delete the job -- use reschedule if you are putting back an
+      # existing qjob.
+      #
+      def save scrape_job, priority=nil, delay=nil
+        body       = scrape_job.to_flat.join("\t")
+        delay    ||= delay_to_next_scrape(scrape_job)
+        priority ||= config[:priority]
+        log scrape_job, priority, delay
+        job_queue.put body, priority, delay, config[:time_to_run]
+      end
+      # delegates to #save() -- priority and delay are unchanged.
+      def <<(scrape_job) save scrape_job  end
+
+      #
+      # if we can't determine an actual rate, uses max_resched_delay (assumes it
+      # is rare)
+      #
+      def delay_to_next_scrape scrape_job
+        rate  = scrape_job.avg_rate or return max_resched_delay
+        delay = items_goal.to_f / rate
+        delay = delay.clamp(min_resched_delay, max_resched_delay)
+        delay.to_i
+      end
+
+      #
+      # A (very prolix) log statement
+      #
+      def log scrape_job, priority=nil, delay=nil
+        delay ||= delay_to_next_scrape(scrape_job)
+        rate_str = scrape_job.avg_rate ? "%10.5f/s" % (scrape_job.avg_rate) : " "*12
+        ll = "Rescheduling\t#{"%-23s"%scrape_job.query_term}\t"
+        ll << "%6d" % priority if priority
+        ll << "\t#{rate_str}"
+        ll << "\t#{"%7d" % (scrape_job.prev_items||0)}"
+        ll << "\t#{"%4d"%(scrape_job.new_items||0)} nu"
+        ll << "\tin #{"%8.2f" % delay} s"
+        ll << "\t#{(Time.now + delay).strftime("%Y-%m-%d %H:%M:%S")}"
+        Monkeyshines.logger.info ll
+      end
+
+      # ===========================================================================
+      #
+      # Beanstalkd interface
+      #
+
+      #
+      # De-serialize the scrape job from the queue.
+      #
+      def scrape_job_from_qjob qjob
+        args           = qjob.body.split("\t")
+        # request_klass = Wukong.class_from_resource(args.shift)
+        scrape_job     = request_klass.new(*args[1..-1])
+      end
+
+      # Take the next (highest priority, delay met) job.
+      # Set timeout (default is 10s)
+      # Returns nil on error or timeout. Interrupt error passes through
+      def reserve_job! to=10
+        begin  qjob = job_queue.reserve(to)
+        rescue Beanstalk::TimedOut => e ; Monkeyshines.logger.info e.to_s ; sleep 0.4 ; return ;
+        rescue StandardError => e       ; Monkeyshines.logger.warn e.to_s ; sleep 1   ; return ; end
+        qjob
+      end
+
+  end
+end

data/lib/edamame/scheduling.rb

@@ -0,0 +1,69 @@
+require 'wukong/extensions/hashlike_class'
+module Edamame
+
+  module Scheduling
+    extend FactoryModule
+
+    # def type
+    #   self.class.to_s
+    # end
+    # def to_hash
+    # end
+
+    class Base
+      include Wukong::HashlikeClass
+      has_members :last_run, :total_runs
+
+      def initialize *args
+        members.zip(args).each do |key, val|
+          self[key] = val if val
+        end
+      end
+
+      def last_run_time
+        last_run.is_a?(String) ? Time.parse(last_run) : last_run
+      end
+
+      def since_last
+        Time.now - last_run_time
+      end
+
+    end
+
+    class Every < Base
+      has_member :period
+      def delay
+        period
+      end
+    end
+
+    class At < Base
+      attr_accessor :time
+      def initialize *args
+        super *args
+        self.time = Time.parse(time) if time.is_a?(String)
+      end
+      def delay
+        @delay ||= time - Time.now
+      end
+    end
+
+    class Once < Base
+      def delay
+        nil
+      end
+    end
+
+    class Rescheduling < Base
+      has_members :period, :total_items, :goal_items, :prev_max
+
+      cattr_accessor :min_resched_delay, :max_resched_delay
+      self.min_resched_delay = 10
+      self.max_resched_delay = 24*60*60
+      def delay
+        period
+      end
+    end
+  end
+
+end

data/lib/edamame/store.rb

@@ -0,0 +1,8 @@
+require 'monkeyshines/utils/factory_module'
+module Edamame
+  module Store
+    extend FactoryModule
+    autoload :Base,        'edamame/store/base'
+    autoload :TyrantStore, 'edamame/store/tyrant_store'
+  end
+end

data/lib/edamame/store/base.rb

@@ -0,0 +1,62 @@
+# require 'monkeyshines/utils/factory_module'
+module Edamame
+  module Store
+    class Base
+      # The actual backing store; should respond to #set and #get methods
+      attr_accessor :db
+
+      def initialize options
+      end
+
+      #
+      # Executes block once for each element in the whole DB, in whatever order
+      # the DB thinks you should see it.
+      #
+      # Your block will see |key, val|
+      #
+      # key_store.each do |key, val|
+      #   # ... stuff ...
+      # end
+      #
+      def each &block
+        db.iterinit
+        loop do
+          key = db.iternext or break
+          val = db[key]
+          yield key, val
+        end
+      end
+
+      def each_as klass, &block
+        self.each do |key, hsh|
+          yield [key, klass.from_hash(hsh)]
+        end
+      end
+
+      # Delegate to store
+      def set(key, val)
+        return unless val
+        db.put key, val.to_hash.compact
+      end
+      def save obj
+        return unless obj
+        db.put obj.key, obj.to_hash.compact
+      end
+
+      def get(key)      db[key]         end
+      def put(key, val) db.put key, val end
+      def [](key)       db[key]         end
+      def close()       db.close        end
+      def size()        db.size         end
+      def delete(key)   db.delete(key)  end
+
+      #
+      # Load from standard command-line options
+      #
+      # obvs only works when there's just one store
+      #
+      def self.create type, config
+      end
+    end
+  end
+end