edamame 0.2.0

data/app/edamame_san/public/stylesheets/application.css

@@ -0,0 +1,32 @@
+/* ===========================================================================
+   Default styling for the upload bar
+ */
+
+body { margin: 0px ; }
+
+.bar {
+  width:         300px;
+}
+#progress {
+  border:        1px solid #222;
+  display:       block;
+  float:         left;
+  margin-right:  0.25em;
+}
+#progressbar {
+  width:         0px;
+  height:        24px;
+}
+
+#progress                      { background:    #eee; }
+#progress         #progressbar { background:    #bbf; }
+#progress.working              { background:    #eef; }
+#progress.success #progressbar { background:    #cfd; }
+#progress.error                { background:    #fcc; }
+#progress.error   #progressbar { background:    #fcd; }
+
+iframe.yuploader {
+  border: 0px none white;
+  margin: 0px;
+  padding: 0px;
+}

data/app/edamame_san/public/stylesheets/layout.css

@@ -0,0 +1,88 @@
+
+/*
+  Page layout
+*/
+
+body {
+  background-color: #4B7399;
+  font-family: Verdana, Helvetica, Arial;
+  font-size: 14px;
+  padding: 0;
+  margin:  0;
+}
+
+a img {
+  border: none;
+}
+
+a {
+  color: #0000FF;
+}
+
+.clear {
+  clear: both;
+  height: 0;
+  overflow: hidden;
+}
+
+#container {
+  width: 75%;
+  margin: 0 auto;
+  background-color: #FFF;
+  padding: 20px 40px;
+  border: solid 1px black;
+  margin-top: 20px;
+}
+
+#flash_notice, #flash_error {
+  padding: 5px 8px;
+  margin: 10px 0;
+}
+
+#flash_notice {
+  background-color: #CFC;
+  border: solid 1px #6C6;
+}
+
+#flash_error {
+  background-color: #FCC;
+  border: solid 1px #C66;
+}
+
+.fieldWithErrors {
+  display: inline;
+}
+
+#errorExplanation {
+  width: 400px;
+  border: 2px solid #CF0000;
+  padding: 0px;
+  padding-bottom: 12px;
+  margin-bottom: 20px;
+  background-color: #f0f0f0;
+}
+
+#errorExplanation h2 {
+  text-align: left;
+  font-weight: bold;
+  padding: 5px 5px 5px 15px;
+  font-size: 12px;
+  margin: 0;
+  background-color: #c00;
+  color: #fff;
+}
+
+#errorExplanation p {
+  color: #333;
+  margin-bottom: 0;
+  padding: 8px;
+}
+
+#errorExplanation ul {
+  margin: 2px 24px;
+}
+
+#errorExplanation ul li {
+  font-size: 12px;
+  list-style: disc;
+}

data/app/edamame_san/views/layout.haml

@@ -0,0 +1,13 @@
+!!! XML
+!!! Strict
+%html{ "xml:lang" => "en", :lang => "en", :xmlns => "http://www.w3.org/1999/xhtml" }
+  %head
+    %link{ :href => "/stylesheets/application.css", :rel => "stylesheet",    :type => "text/css" }
+    %link{ :href => "/favicon.ico",                 :rel => "shortcut icon", :type => "image/x-icon" }
+
+  %body
+    #container
+      =yield
+
+      -# %script{ :type => "text/javascript", :src => "http://ajax.googleapis.com/ajax/libs/jquery/1.3.2/jquery.min.js"      }
+      -# %script{ :type => "text/javascript", :src => "http://ajax.googleapis.com/ajax/libs/jqueryui/1.7.1/jquery-ui.min.js" }

data/app/edamame_san/views/load.haml

@@ -0,0 +1,37 @@
+%style{ :type => 'text/css' }
+  @import url('/stylesheets/layout.css');
+
+%p
+
+  Jobs in the edamame job store:
+
+
+%table
+  %tr
+    %th query_term
+    %th priority
+    %th prev_items
+    %th prev_rate
+    %th prev_span_min
+    %th prev_span_max
+  - @dest_store.each_as(Edamame::Job) do |key, obj|
+    %tr
+      %td=h key.inspect
+      %td=h obj.inspect
+      %td=h obj.key
+      -# %td=h obj[:query_term]
+      -# %td=h obj[:priority]
+      -# %td=h obj[:prev_items]
+      -# %td=h obj[:prev_rate]
+      -# %td=h obj[:prev_span_min]
+      -# %td=h obj[:prev_span_max]
+
+            
+            
+            
+            
+            
+            
+      
+
+  

data/app/edamame_san/views/root.haml

@@ -0,0 +1,25 @@
+%style{ :type => 'text/css' }
+  @import url('/stylesheets/layout.css');
+
+%p
+
+  Jobs in the edamame job store:
+
+
+%table
+  %tr
+    %th query_term
+    %th priority
+    %th prev_items
+    %th prev_rate
+    %th prev_span_min
+    %th prev_span_max
+  - @store.each_as(Wuclan::Domains::Twitter::Scrape::TwitterSearchJob) do |key, obj|
+    %tr
+      -# %td=h key.inspect
+      %td=h obj[:query_term]
+      %td=h obj[:priority]
+      %td=h obj[:prev_items]
+      %td=h obj[:prev_rate]
+      %td=h obj[:prev_span_min]
+      %td=h obj[:prev_span_max]

data/bin/edamame-nuke

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__)+'/../../lib'
+require 'rubygems'
+require 'edamame'
+require 'monkeyshines/monitor'
+require 'monkeyshines/utils/trollop'
+require File.dirname(__FILE__)+'/edamame_util_opts'
+
+pq = Edamame::PersistentQueue.new( :tube => CONFIG[:tube],
+  :queue => { :type => 'BeanstalkQueue', :uris => [CONFIG[:queue]] },
+  :store => { :type => 'TyrantStore',    :uri =>   CONFIG[:store]  }
+  )
+
+periodic_log = Monkeyshines::Monitor::PeriodicLogger.new(:iters => 1000, :time => 30)
+pq.queue.empty_all do |job|
+  periodic_log.periodically{ [ job.tube, job.priority, job.delay, job.scheduling, job.obj['key'] ] }
+end
+pq.store.each do |key, val|
+  pq.store.db.out key
+end

data/bin/edamame-ps

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+ps aux | egrep '(beanstalk|ttserver|god|ruby|scrape|shotgun|thin)' | egrep -v ' grep |mdns://|^_lp' | cut -c 1-200 | sort -k11

data/bin/edamame-stats

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__)+'/../../lib'
+require 'rubygems'
+require 'edamame'
+require 'monkeyshines/monitor'
+
+pq = Edamame::PersistentQueue.new(
+  :tube  => ARGV[0],
+  :queue => { :type => 'BeanstalkQueue', :uris => ['localhost:11210'] },
+  :store => { :type => 'TyrantStore',    :uri =>            ':11212'  }
+  )
+
+p pq.stats

data/bin/edamame-sync

@@ -0,0 +1,21 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__)+'/../../lib'
+require 'rubygems'
+require 'edamame'
+require 'monkeyshines/monitor'
+require 'monkeyshines/utils/trollop'
+require 'pathname'
+require File.dirname(__FILE__)+'/edamame_util_opts'
+Log = Monkeyshines.logger
+
+pq = Edamame::PersistentQueue.new( :tube => CONFIG[:tube],
+  :queue => { :type => 'BeanstalkQueue', :uris => [CONFIG[:queue]] },
+  :store => { :type => 'TyrantStore',    :uri =>   CONFIG[:store]  }
+  )
+
+periodic_log = Monkeyshines::Monitor::PeriodicLogger.new(:iters => 1, :time => 30)
+pq.load do |job|
+  obj = job.obj rescue nil
+
+  periodic_log.periodically{ [ pq.store.size, job.loggable, job.class, obj.inspect ] }
+end

data/bin/edamame_util_opts.rb

@@ -0,0 +1,10 @@
+
+CONFIG = Trollop::options do
+  opt :queue,  'host:port for the beanstalkd queue', :type => String, :required => true
+  opt :store,  'host:port for the backing store',    :type => String, :required => true
+  opt :handle, 'label for this scrape',              :type => String, :required => true
+end
+CONFIG[:store].gsub!(/^localhost:/, ':') # queue must *not* have localhost:
+CONFIG[:queue].gsub!(/^:/, 'localhost:') # queue must have localhost:
+CONFIG[:tube] = (CONFIG[:handle] || 'default').gsub(/[^A-Z0-9a-z\-]+/,'')
+

data/bin/test_run.rb

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__)+'/../../lib'
+require 'rubygems'
+require 'json'
+require 'edamame'
+
+broker = Edamame::Broker.new(
+  :queue => { :type => 'BeanstalkQueue', :uris => ['localhost:11210'] },
+  :store => { :type => 'TyrantStore',    :uri  =>           ':11212'  }
+  )
+
+broker.work do |job|
+  Log.info [job, job.scheduling, job.stats, job.obj].inspect
+end

data/lib/edamame.rb

@@ -0,0 +1,29 @@
+require 'beanstalk-client'
+require 'wukong/extensions'
+require 'monkeyshines/utils/factory_module'
+require 'monkeyshines/utils/logger'
+require 'edamame/scheduling'
+require 'edamame/job'
+require 'edamame/queue'
+require 'edamame/store'
+
+# Edamame combines the Beanstalk priority queue with a Tokyo Tyrant database to
+# produce a persistent distributed priority job queue system.
+#
+# * fast, scalable, lightweight and distributed
+# * persistent and recoverable
+# * scalable up to your memory limits
+# * queryable and enumerable jobs
+# * named jobs
+# * reasonably-good availability.
+#
+# Like beanstalk, it is a job queue, not just a message queue:
+# * priority job scheduling, not just FIFO
+# * Supports multiple queues ('tubes')
+# * reliable scheduling: jobs that time out are re-assigned
+#
+# You should start by looking at [Edamame::PersistentQueue]
+module Edamame
+  autoload :PersistentQueue, 'edamame/persistent_queue'
+  autoload :Broker,          'edamame/broker'
+end

data/lib/edamame/broker.rb

@@ -0,0 +1,38 @@
+module Edamame
+  #
+  # Repeatedly poll the queue for jobs and dispatch them to a worker loop
+  #
+  # Those jobs can be rescheduled (with updated parameters) for later
+  # re-processing.
+  #
+  class Broker < PersistentQueue
+    # Enter the work loop
+    def work timeout=nil, klass=nil, &block
+      loop do
+        job    = reserve(timeout, klass) or break
+        result = block.call(job)
+        reschedule job
+      end
+    end
+
+    # Inserts the job back into the queue at its sepcified delay --
+    # or, if delay is nil, remove the job from the queue
+    #
+    # You'll probably want to use Edamame::Scheduling with all this
+    def reschedule job
+      delay = job.scheduling.delay
+      if delay
+        release job
+      else
+        log_action 'deleting', job
+        delete job
+      end
+    end
+
+    # Log info about an action on a job
+    def log_action action, job, *stuff
+      Log.info [action, job.loggable, *stuff].flatten.join("\t")
+    end
+
+  end
+end

data/lib/edamame/job.rb

@@ -0,0 +1,114 @@
+module Edamame
+  #
+  #
+  # id, name, body, timeouts, time-left, age, state, delay, pri, ttr
+  #
+  #
+  # * A job, pulled from the queue: it is connected to its beanstalk presence
+  #   body contains
+  #   ** obj
+  #   ** scheduling
+  #   ** stats
+  #
+  # * A DB job
+  #   body contains
+  #   ** tube, priority, ttr, state
+  #   ** obj
+  #   ** scheduling
+  #   ** stats
+  class Job < Struct.new(
+    :tube, :priority, :ttr, :state,
+    :scheduling, :obj
+      )
+    # connection back to the job queue's instance of this job
+    attr_accessor :qjob
+
+    DEFAULT_OPTIONS = {
+      'priority'   => 65536,
+      'ttr'        => 120,
+      'state'      => 1,
+      'scheduling' => Edamame::Scheduling::Once.new()
+    }
+
+    # attr_accessor :runs, :failures, :prev_run_at
+    def initialize *args
+      super *args
+      DEFAULT_OPTIONS.each{|key,val| self[key] ||= val }
+      [:priority, :ttr, :state].each{|key| self[key] = self[key].to_i }
+      case self.scheduling
+      when String
+        scheduling_hash = YAML.load(self.scheduling) rescue nil
+        self.scheduling = Scheduling.from_hash(scheduling_hash) if scheduling_hash
+      when Hash
+        self.scheduling = Scheduling.from_hash(scheduling)
+      else
+        # else it should behave like a scheduling
+      end
+      if self.obj.is_a?(String) then self.obj = YAML.load(self.obj) rescue nil ; end
+    end
+
+    def key
+      key = (obj.respond_to?(:key) ? obj.key : (obj[:key]||obj['key']))
+      [ tube, key ].join('-')
+    end
+
+    #
+    def since_last
+      scheduling.last_run - Time.now
+    end
+
+    #
+    # Delegation to scheduling strategy.
+    #
+    def prev_max()            self.scheduling.prev_max              end
+    def prev_max=(val)        self.scheduling.prev_max = val        end
+    def prev_items()          self.scheduling.prev_items            end
+    def prev_items=(val)      self.scheduling.prev_items = val      end
+    def prev_items_rate()     self.scheduling.prev_items_rate       end
+    def prev_items_rate=(val) self.scheduling.prev_items_rate = val end
+    def delay()               self.scheduling.delay                 end
+    def delay=(val)           self.scheduling.delay = val           end
+    def last_run()            self.scheduling.last_run              end
+
+    # Override this for rescheduling
+    def update!
+      scheduling.total_runs = scheduling.total_runs.to_i + qjob.stats['releases']
+      scheduling.last_run   = Time.now
+    end
+
+    # Fields suitable for emission as a log line.
+    def loggable
+      "%-15s\t%7d\t%7.2f\t%-23s" % [tube, priority, delay, key]
+    end
+
+    def to_hash flatten=true
+      hsh = super()
+      hsh["scheduling"]   = scheduling.to_hash
+      hsh["obj"]          = obj.to_hash
+      if flatten
+        hsh["scheduling"] = hsh['scheduling'].to_yaml
+        hsh["obj"]        = hsh['obj'].to_yaml
+      end
+      hsh
+    end
+  end
+end
+
+Beanstalk::Job.class_eval do
+  def key
+    body
+  end
+
+  def priority
+    pri
+  end
+
+  def tube
+    stats['tube']
+  end
+
+  # Fields suitable for emission as a log line.
+  def loggable
+    "%-15s\t%7d\t%7.2f\t%-23s" % [tube, priority, delay, key]
+  end
+end