mrflip-edamame 0.1.1

data/app/edamame_san/public/stylesheets/application.css

@@ -0,0 +1,32 @@
+/* ===========================================================================
+   Default styling for the upload bar
+ */
+
+body { margin: 0px ; }
+
+.bar {
+  width:         300px;
+}
+#progress {
+  border:        1px solid #222;
+  display:       block;
+  float:         left;
+  margin-right:  0.25em;
+}
+#progressbar {
+  width:         0px;
+  height:        24px;
+}
+
+#progress                      { background:    #eee; }
+#progress         #progressbar { background:    #bbf; }
+#progress.working              { background:    #eef; }
+#progress.success #progressbar { background:    #cfd; }
+#progress.error                { background:    #fcc; }
+#progress.error   #progressbar { background:    #fcd; }
+
+iframe.yuploader {
+  border: 0px none white;
+  margin: 0px;
+  padding: 0px;
+}

data/app/edamame_san/public/stylesheets/layout.css

@@ -0,0 +1,88 @@
+
+/*
+  Page layout
+*/
+
+body {
+  background-color: #4B7399;
+  font-family: Verdana, Helvetica, Arial;
+  font-size: 14px;
+  padding: 0;
+  margin:  0;
+}
+
+a img {
+  border: none;
+}
+
+a {
+  color: #0000FF;
+}
+
+.clear {
+  clear: both;
+  height: 0;
+  overflow: hidden;
+}
+
+#container {
+  width: 75%;
+  margin: 0 auto;
+  background-color: #FFF;
+  padding: 20px 40px;
+  border: solid 1px black;
+  margin-top: 20px;
+}
+
+#flash_notice, #flash_error {
+  padding: 5px 8px;
+  margin: 10px 0;
+}
+
+#flash_notice {
+  background-color: #CFC;
+  border: solid 1px #6C6;
+}
+
+#flash_error {
+  background-color: #FCC;
+  border: solid 1px #C66;
+}
+
+.fieldWithErrors {
+  display: inline;
+}
+
+#errorExplanation {
+  width: 400px;
+  border: 2px solid #CF0000;
+  padding: 0px;
+  padding-bottom: 12px;
+  margin-bottom: 20px;
+  background-color: #f0f0f0;
+}
+
+#errorExplanation h2 {
+  text-align: left;
+  font-weight: bold;
+  padding: 5px 5px 5px 15px;
+  font-size: 12px;
+  margin: 0;
+  background-color: #c00;
+  color: #fff;
+}
+
+#errorExplanation p {
+  color: #333;
+  margin-bottom: 0;
+  padding: 8px;
+}
+
+#errorExplanation ul {
+  margin: 2px 24px;
+}
+
+#errorExplanation ul li {
+  font-size: 12px;
+  list-style: disc;
+}

data/app/edamame_san/views/layout.haml

@@ -0,0 +1,13 @@
+!!! XML
+!!! Strict
+%html{ "xml:lang" => "en", :lang => "en", :xmlns => "http://www.w3.org/1999/xhtml" }
+  %head
+    %link{ :href => "/stylesheets/application.css", :rel => "stylesheet",    :type => "text/css" }
+    %link{ :href => "/favicon.ico",                 :rel => "shortcut icon", :type => "image/x-icon" }
+
+  %body
+    #container
+      =yield
+
+      -# %script{ :type => "text/javascript", :src => "http://ajax.googleapis.com/ajax/libs/jquery/1.3.2/jquery.min.js"      }
+      -# %script{ :type => "text/javascript", :src => "http://ajax.googleapis.com/ajax/libs/jqueryui/1.7.1/jquery-ui.min.js" }

data/app/edamame_san/views/load.haml

@@ -0,0 +1,37 @@
+%style{ :type => 'text/css' }
+  @import url('/stylesheets/layout.css');
+
+%p
+
+  Jobs in the edamame job store:
+
+
+%table
+  %tr
+    %th query_term
+    %th priority
+    %th prev_items
+    %th prev_rate
+    %th prev_span_min
+    %th prev_span_max
+  - @dest_store.each_as(Edamame::Job) do |key, obj|
+    %tr
+      %td=h key.inspect
+      %td=h obj.inspect
+      %td=h obj.key
+      -# %td=h obj[:query_term]
+      -# %td=h obj[:priority]
+      -# %td=h obj[:prev_items]
+      -# %td=h obj[:prev_rate]
+      -# %td=h obj[:prev_span_min]
+      -# %td=h obj[:prev_span_max]
+
+            
+            
+            
+            
+            
+            
+      
+
+  

data/app/edamame_san/views/root.haml

@@ -0,0 +1,25 @@
+%style{ :type => 'text/css' }
+  @import url('/stylesheets/layout.css');
+
+%p
+
+  Jobs in the edamame job store:
+
+
+%table
+  %tr
+    %th query_term
+    %th priority
+    %th prev_items
+    %th prev_rate
+    %th prev_span_min
+    %th prev_span_max
+  - @store.each_as(Wuclan::Domains::Twitter::Scrape::TwitterSearchJob) do |key, obj|
+    %tr
+      -# %td=h key.inspect
+      %td=h obj[:query_term]
+      %td=h obj[:priority]
+      %td=h obj[:prev_items]
+      %td=h obj[:prev_rate]
+      %td=h obj[:prev_span_min]
+      %td=h obj[:prev_span_max]

data/bin/edamame-ps

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+ps aux | egrep '(beanstalk|ttserver|god|ruby|scrape|shotgun|thin)' | grep -v ' grep '

data/bin/empty_all.rb

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__)+'/../../lib'
+require 'rubygems'
+require 'edamame'
+require 'monkeyshines/monitor'
+
+pq = Edamame::PersistentQueue.new(
+  :queue => { :type => 'BeanstalkQueue', :uris => ['localhost:11210'] },
+  :store => { :type => 'TyrantStore',    :uri =>            ':11212'  }
+  )
+
+periodic_log = Monkeyshines::Monitor::PeriodicLogger.new(:iters => 1000, :time => 30)
+pq.queue.empty_all do |job|
+  periodic_log.periodically{ [ job.tube, job.priority, job.delay, job.scheduling, job.obj['key'] ] }
+end

data/bin/stats.rb

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__)+'/../../lib'
+require 'rubygems'
+require 'edamame'
+require 'monkeyshines/monitor'
+
+pq = Edamame::PersistentQueue.new(
+  :tube  => ARGV[0],
+  :queue => { :type => 'BeanstalkQueue', :uris => ['localhost:11210'] },
+  :store => { :type => 'TyrantStore',    :uri =>            ':11212'  }
+  )
+
+p pq.stats

data/bin/sync.rb

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__)+'/../../lib'
+require 'rubygems'
+require 'edamame'
+require 'monkeyshines/monitor'
+
+pq = Edamame::PersistentQueue.new(
+  :queue => { :type => 'BeanstalkQueue', :uris => ['localhost:11210'] },
+  :store => { :type => 'TyrantStore',    :uri =>            ':11212'  }
+  )
+
+periodic_log = Monkeyshines::Monitor::PeriodicLogger.new(:iters => 1000, :time => 30)
+pq.load do |job|
+  periodic_log.periodically{ [ pq.store.size, job, job.tube, job.priority, job.delay, job.obj['key'] ] }
+end

data/bin/test_run.rb

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+$: << File.dirname(__FILE__)+'/../../lib'
+require 'rubygems'
+require 'json'
+require 'edamame'
+
+broker = Edamame::Broker.new(
+  :queue => { :type => 'BeanstalkQueue', :uris => ['localhost:11210'] },
+  :store => { :type => 'TyrantStore',    :uri  =>           ':11212'  }
+  )
+
+broker.work do |job|
+  Monkeyshines.logger.info [job, job.scheduling, job.stats, job.obj].inspect
+end

data/edamame.gemspec

@@ -0,0 +1,110 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE
+# Instead, edit Jeweler::Tasks in Rakefile, and run `rake gemspec`
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{edamame}
+  s.version = "0.1.1"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Philip (flip) Kromer"]
+  s.date = %q{2009-08-23}
+  s.description = %q{
+
+Edamame combines the Beanstalk priority queue with a Tokyo Tyrant database and God monitoring to produce a persistent distributed priority job queue system.
+
+Like beanstalk, it is fast, lightweight, distributed, priority queuing, reliable scheduling; it adds persistence, named jobs and job querying/enumeration.
+
+}
+  s.email = %q{flip@infochimps.org}
+  s.executables = ["edamame-ps", "empty_all.rb", "stats.rb", "sync.rb", "test_run.rb"]
+  s.extra_rdoc_files = [
+    "LICENSE.textile",
+     "README.textile"
+  ]
+  s.files = [
+    ".document",
+     ".gitignore",
+     "LICENSE.textile",
+     "README.textile",
+     "Rakefile",
+     "VERSION",
+     "app/edamame_san/config.ru",
+     "app/edamame_san/config.yml",
+     "app/edamame_san/config/.gitignore",
+     "app/edamame_san/edamame_san.rb",
+     "app/edamame_san/public/favicon.ico",
+     "app/edamame_san/public/images/edamame_logo.icns",
+     "app/edamame_san/public/images/edamame_logo.ico",
+     "app/edamame_san/public/images/edamame_logo.png",
+     "app/edamame_san/public/images/edamame_logo_2.icns",
+     "app/edamame_san/public/javascripts/application.js",
+     "app/edamame_san/public/javascripts/jquery/jquery-ui.js",
+     "app/edamame_san/public/javascripts/jquery/jquery.js",
+     "app/edamame_san/public/stylesheets/application.css",
+     "app/edamame_san/public/stylesheets/layout.css",
+     "app/edamame_san/views/layout.haml",
+     "app/edamame_san/views/load.haml",
+     "app/edamame_san/views/root.haml",
+     "bin/edamame-ps",
+     "bin/empty_all.rb",
+     "bin/stats.rb",
+     "bin/sync.rb",
+     "bin/test_run.rb",
+     "edamame.gemspec",
+     "lib/edamame.rb",
+     "lib/edamame/job.rb",
+     "lib/edamame/queue.rb",
+     "lib/edamame/queue/beanstalk.rb",
+     "lib/edamame/rescheduled.rb",
+     "lib/edamame/scheduling.rb",
+     "lib/edamame/store.rb",
+     "lib/edamame/store/base.rb",
+     "lib/edamame/store/tyrant_store.rb",
+     "lib/methods.txt",
+     "spec/edamame_spec.rb",
+     "spec/spec_helper.rb",
+     "utils/god/README-god.textile",
+     "utils/god/beanstalkd_god.rb",
+     "utils/god/edamame.god",
+     "utils/god/god-etc-init-dot-d-example",
+     "utils/god/god_email.rb",
+     "utils/god/god_process.rb",
+     "utils/god/god_site_config.rb",
+     "utils/god/sinatra_god.rb",
+     "utils/god/tyrant_god.rb",
+     "utils/simulation/Add Percent Variation.vi",
+     "utils/simulation/Harmonic Average.vi",
+     "utils/simulation/Rescheduling Simulation.aliases",
+     "utils/simulation/Rescheduling Simulation.lvlps",
+     "utils/simulation/Rescheduling Simulation.lvproj",
+     "utils/simulation/Rescheduling.vi",
+     "utils/simulation/Weighted Average.vi"
+  ]
+  s.homepage = %q{http://github.com/mrflip/edamame}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.5}
+  s.summary = %q{Beanstalk + Tokyo Tyrant = Edamame, a fast persistent distributed priority job queue.}
+  s.test_files = [
+    "spec/edamame_spec.rb",
+     "spec/spec_helper.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<rspec>, [">= 0"])
+      s.add_development_dependency(%q<yard>, [">= 0"])
+    else
+      s.add_dependency(%q<rspec>, [">= 0"])
+      s.add_dependency(%q<yard>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<rspec>, [">= 0"])
+    s.add_dependency(%q<yard>, [">= 0"])
+  end
+end

data/lib/edamame.rb

@@ -0,0 +1,193 @@
+require 'wukong/extensions'
+require 'monkeyshines/utils/logger'
+require 'monkeyshines/utils/factory_module'
+require 'beanstalk-client'
+require 'edamame/scheduling'
+require 'edamame/job'
+require 'edamame/queue'
+require 'edamame/store'
+
+module Edamame
+
+  class PersistentQueue
+    DEFAULT_CONFIG = {
+      :queue => { :type => :beanstalk, :pool => ['localhost:11300'] }
+    }
+    attr_reader :tube, :store, :queue
+    def initialize options={}
+      @tube  = options[:tube] || :default
+      @store = Edamame::Store.create options[:store]
+      @queue = Edamame::Queue.create options[:queue].merge(:default_tube => @tube)
+    end
+
+    #
+    # Add a new Job to the queue
+    #
+    def put job, *args
+      job.tube  = self.tube if job.tube.blank?
+      self.tube = job.tube
+      return if store.include?(job.key)
+      store.save job
+      queue.put  job, *args
+    end
+
+    def tube= _tube
+      return if @tube == _tube
+      puts "#{self.class} setting tube to #{_tube}, was #{@tube}"
+      queue.tube = @tube = _tube
+    end
+
+    # Alias for put(job)
+    def << job
+      put job
+    end
+
+    # Retrieve named record
+    def get key
+      Edamame::Job.from_hash store.get(key)
+    end
+
+    #
+    # Request a job fom the queue for processing
+    #
+    def reserve timeout=nil
+      job = queue.reserve(timeout)
+    end
+
+    #
+    # Remove the job from the queue.
+    #
+    def delete job
+      store.delete job.key
+      queue.delete job
+    end
+
+    #
+    # Returns the job to the queue, to be re-run later.
+    #
+    # release'ing a job acknowledges it was completed, successfully or not
+    #
+    def release job
+      job.update!
+      store.save    job
+      queue.release job, job.priority, job.scheduling.delay
+    end
+
+    #
+    # Shelves the job.
+    #
+    def bury job
+      store.bury job
+      queue.bury job
+    end
+
+    #
+    # Returns each job as it appears in the queue.
+    #
+    # all jobs -- active, inactive, running, etc -- are returned,
+    # and in some arbitrary order.
+    #
+    def each *args, &block
+      store.each do |key, job_hsh|
+        yield Edamame::Job.from_hash(job_hsh)
+      end
+    end
+
+    #
+    # Loads all jobs from the backing store into the queue.
+    #
+    def load &block
+      hoard do |job|
+        yield(job) if block
+        unless store.include?(job.key)
+          store.save job
+        end
+      end
+      unhoard &block
+    end
+
+    # Returns a hash of stats about the store and queue
+    def stats
+      { :store_stats => store.stats,
+        :queue_stats => queue.stats,
+        :tube        => self.tube }
+    end
+
+  protected
+    #
+    # Destructively strips the beanstalkd queue of all of its jobs.
+    #
+    # This is the only way (I know) to enumerate all of the jobs in the queue --
+    # certainly the only way that respects concurrency.
+    #
+    # You shouldn't use this in general; the point of the backing store is to
+    # allow exactly such queries and enumeration. See #each instead.
+    #
+    def hoard &block
+      queue.empty tube, &block
+    end
+
+    #
+    # Loads all jobs from the backing store into the queue.
+    #
+    # The queue must be emptied of all jobs before running this command:
+    # otherwise jobs will be duplicated.
+    #
+    def unhoard &block
+      store.each do |key, hsh|
+        job = Edamame::Job.from_hash hsh
+        self.tube = job.tube
+        yield(job) if block
+        queue.put job
+      end
+    end
+
+    #
+    #
+    #
+    def log line
+      Monkeyshines.logger.info line
+    end
+  end
+
+  class Broker < PersistentQueue
+    def reschedule job
+      delay = job.scheduling.delay
+      if delay
+        # log_job job, 'rescheduled', delay, (Time.now + delay).to_flat, job.scheduling.to_flat.join
+        release job
+      else
+        # log_job job, 'deleted'
+        delete job
+      end
+    end
+    def log_job job, *stuff
+      log [job.tube, job.priority, job.delay, job.obj['key'], *stuff].flatten.join("\t")
+    end
+    def work timeout=10, &block
+      loop do
+        job    = reserve(timeout) or break
+        result = block.call(job)
+        job.update!
+        reschedule job
+      end
+    end
+  end
+end
+
+module Wuclan
+  module Domains
+    module Twitter
+      module Scrape
+        TwitterSearchJob = Struct.new(
+          :query_term,
+          :priority,
+          :prev_items,
+          :prev_rate,
+          :prev_span_min,
+          :prev_span_max
+          )
+      end
+    end
+  end
+end