starfish 1.0.0

data/doc/rdoc-style.css

@@ -0,0 +1,208 @@
+
+body {
+    font-family: Verdana,Arial,Helvetica,sans-serif;
+    font-size:   90%;
+    margin: 0;
+    margin-left: 40px;
+    padding: 0;
+    background: white;
+}
+
+h1,h2,h3,h4 { margin: 0; color: #efefef; background: transparent; }
+h1 { font-size: 150%; }
+h2,h3,h4 { margin-top: 1em; }
+
+a { background: #eef; color: #039; text-decoration: none; }
+a:hover { background: #039; color: #eef; }
+
+/* Override the base stylesheet's Anchor inside a table cell */
+td > a {
+  background: transparent;
+  color: #039;
+  text-decoration: none;
+}
+
+/* and inside a section title */
+.section-title > a {
+  background: transparent;
+  color: #eee;
+  text-decoration: none;
+}
+
+/* === Structural elements =================================== */
+
+div#index {
+    margin: 0;
+    margin-left: -40px;
+    padding: 0;
+    font-size: 90%;
+}
+
+
+div#index a {
+    margin-left: 0.7em;
+}
+
+div#index .section-bar {
+   margin-left: 0px;
+   padding-left: 0.7em;
+   background: #ccc;
+   font-size: small;
+}
+
+
+div#classHeader, div#fileHeader {
+    width: auto;
+    color: white;
+    padding: 0.5em 1.5em 0.5em 1.5em;
+    margin: 0;
+    margin-left: -40px;
+    border-bottom: 3px solid #006;
+}
+
+div#classHeader a, div#fileHeader a {
+    background: inherit;
+    color: white;
+}
+
+div#classHeader td, div#fileHeader td {
+    background: inherit;
+    color: white;
+}
+
+
+div#fileHeader {
+    background: #057;
+}
+
+div#classHeader {
+    background: #048;
+}
+
+
+.class-name-in-header {
+  font-size:  180%;
+  font-weight: bold;
+}
+
+
+div#bodyContent {
+    padding: 0 1.5em 0 1.5em;
+}
+
+div#description {
+    padding: 0.5em 1.5em;
+    background: #efefef;
+    border: 1px dotted #999;
+}
+
+div#description h1,h2,h3,h4,h5,h6 {
+    color: #125;;
+    background: transparent;
+}
+
+div#validator-badges {
+    text-align: center;
+}
+div#validator-badges img { border: 0; }
+
+div#copyright {
+    color: #333;
+    background: #efefef;
+    font: 0.75em sans-serif;
+    margin-top: 5em;
+    margin-bottom: 0;
+    padding: 0.5em 2em;
+}
+
+
+/* === Classes =================================== */
+
+table.header-table {
+    color: white;
+    font-size: small;
+}
+
+.type-note {
+    font-size: small;
+    color: #DEDEDE;
+}
+
+.xxsection-bar {
+    background: #eee;
+    color: #333;
+    padding: 3px;
+}
+
+.section-bar {
+   color: #333;
+   border-bottom: 1px solid #999;
+    margin-left: -20px;
+}
+
+
+.section-title {
+    background: #79a;
+    color: #eee;
+    padding: 3px;
+    margin-top: 2em;
+    margin-left: -30px;
+    border: 1px solid #999;
+}
+
+.top-aligned-row {  vertical-align: top }
+.bottom-aligned-row { vertical-align: bottom }
+
+/* --- Context section classes ----------------------- */
+
+.context-row { }
+.context-item-name { font-family: monospace; font-weight: bold; color: black; }
+.context-item-value { font-size: small; color: #448; }
+.context-item-desc { color: #333; padding-left: 2em; }
+
+/* --- Method classes -------------------------- */
+.method-detail {
+    background: #efefef;
+    padding: 0;
+    margin-top: 0.5em;
+    margin-bottom: 1em;
+    border: 1px dotted #ccc;
+}
+.method-heading {
+  color: black;
+  background: #ccc;
+  border-bottom: 1px solid #666;
+  padding: 0.2em 0.5em 0 0.5em;
+}
+.method-signature { color: black; background: inherit; }
+.method-name { font-weight: bold; }
+.method-args { font-style: italic; }
+.method-description { padding: 0 0.5em 0 0.5em; }
+
+/* --- Source code sections -------------------- */
+
+a.source-toggle { font-size: 90%; }
+div.method-source-code {
+    background: #262626;
+    color: #ffdead;
+    margin: 1em;
+    padding: 0.5em;
+    border: 1px dashed #999;
+    overflow: hidden;
+}
+
+div.method-source-code pre { color: #ffdead; overflow: hidden; }
+
+/* --- Ruby keyword styles --------------------- */
+
+.standalone-code { background: #221111; color: #ffdead; overflow: hidden; }
+
+.ruby-constant  { color: #7fffd4; background: transparent; }
+.ruby-keyword { color: #00ffff; background: transparent; }
+.ruby-ivar    { color: #eedd82; background: transparent; }
+.ruby-operator  { color: #00ffee; background: transparent; }
+.ruby-identifier { color: #ffdead; background: transparent; }
+.ruby-node    { color: #ffa07a; background: transparent; }
+.ruby-comment { color: #b22222; font-weight: bold; background: transparent; }
+.ruby-regexp  { color: #ffa07a; background: transparent; }
+.ruby-value   { color: #7fffd4; background: transparent; }

data/examples/foo.rb

@@ -0,0 +1,20 @@
+class Foo
+  attr_reader :i
+  
+  def initialize
+    @i = 0
+  end
+  
+  def inc
+    logger.info "YAY it incremented by 1 up to #{@i}"
+    @i += 1
+  end
+end
+
+server :log => "/tmp/foo.log" do |object|
+  object = Foo.new
+end
+
+client do |object|
+  object.inc
+end

data/examples/map_reduce/active_record.rb

@@ -0,0 +1,10 @@
+class Foo < ActiveRecord::Base; end
+
+server do |map_reduce|
+  map_reduce.type = Foo
+  map_reduce.conditions = ["bar = ?", 1]
+end
+
+client do |foo|
+  foo.do_some_hard_task
+end

data/examples/map_reduce/file.rb

@@ -0,0 +1,11 @@
+server :log => "/tmp/mylog" do |map_reduce|
+  map_reduce.type = File,
+  map_reduce.input = "/usr/local/big_ass_file",
+  map_reduce.per_client = 1000
+end
+
+client do |line|
+  if line =~ /some_regex/
+    logger.info(line)
+  end
+end

data/lib/map_reduce/active_record.rb

@@ -0,0 +1,98 @@
+require 'rubygems'
+require 'active_record'
+
+class MapReduce
+  module ActiveRecord
+    module Base
+      attr_accessor :queue_size, :locked_queue_wait, :empty_queue_wait, :rescan_when_complete, :vigilant
+      
+      class Client
+        include DRbUndumped
+        
+        include Enumerable
+
+        def initialize(server_object)
+          @server_object = server_object
+          @type = eval(server_object.type_to_s)
+        end
+        
+        def each
+          @server_object.limit.times do
+            yield get_value_from(@server_object.get_id)
+          end
+        end
+        
+        def logger(*args)
+          @server_object.logger(*args)
+        end
+        
+      private
+      
+        def get_value_from(object_id)
+          case object_id
+          when :locked_queue_wait
+            sleep @server_object.locked_queue_wait || 1
+            get_value_from(@server_object.get_id)
+          when :empty_queue_wait
+            sleep @server_object.empty_queue_wait || 30
+            get_value_from(@server_object.get_id)
+          else
+            @type.find(object_id)
+          end
+        end
+      end
+
+      def get_id
+        if @queue.empty? && @offset == 0
+          @total = type.count(input)
+          if not @rescan_when_complete
+            @queue_size ||= @total
+          end
+        end
+        
+        t = Time.now
+
+        object_id = if @lock
+          :locked_queue_wait
+        else
+          queue.shift || :empty_queue_wait
+        end
+
+        @time_spent_grabbing_objects += (Time.now - t)
+
+        if object_id
+          @num_objects_grabbed += 1
+        end
+
+        return object_id
+      end
+
+private
+      
+      def queue
+        if @queue.empty? && @offset == @total && !@rescan_when_complete && !@vigilant
+          exit
+        end
+        
+        if @queue.empty?
+          GC.start
+
+          @time_began = Time.now if @time_began == 0
+          @lock = true
+          t = Time.now
+
+          @queue = type.find(:all, :conditions => input, :limit => @queue_size, :offset => @offset).map{|object|object.id}
+
+          @time_spent_grabbing_queues += (Time.now - t)
+          @num_queues_grabbed += 1
+
+          @offset += @queue.size unless @queue.empty?
+          @offset = 0 if @offset == @total && @rescan_when_complete
+          @lock = false
+        end
+
+        @queue
+      end
+    end
+  end
+end

data/lib/map_reduce/array.rb

@@ -0,0 +1,4 @@
+class MapReduce
+  module Array
+  end
+end

data/lib/map_reduce/file.rb

@@ -0,0 +1,4 @@
+class MapReduce
+  module File
+  end
+end

data/lib/map_reduce.rb

@@ -0,0 +1,121 @@
+require 'drb'
+
+require 'map_reduce/active_record'
+require 'map_reduce/file'
+require 'map_reduce/array'
+
+class MapReduceError < StandardError; end
+class MapReduce
+  @@types = {
+    ::ActiveRecord::Base => [::Array, String, NilClass]
+  }
+  
+  attr_accessor :type, :input, :limit
+  alias :conditions :input
+  alias :conditions= :input=
+  
+  def initialize
+    @lock = false
+    @offset = 0
+    @limit = 1
+    
+    @time_began = 0
+
+    @num_objects_grabbed = 0
+    @time_spent_grabbing_objects = 0.0
+    @time_spent_processing_objects = 0.0
+    
+    @num_queues_grabbed = 0
+    @time_spent_grabbing_queues = 0.0
+    
+    @queue = []
+  end
+  
+  def add_time_spent_processing_objects(time)
+    @time_spent_processing_objects += time
+  end
+  
+  def stats
+    {
+      :time_began => @time_began,
+      :num_queues_grabbed => @num_queues_grabbed,
+      :time_spent_grabbing_queues => @time_spent_grabbing_objects,
+      :num_objects_grabbed => @num_objects_grabbed,
+      :time_spent_grabbing_objects => @time_spent_grabbing_objects,
+      :time_spent_processing_objects => @time_spent_processing_objects
+    }
+  end
+  
+  def type=(type)
+    @type = type
+    if valid_type?
+      self.class.instance_eval "include MapReduce::#{base_type}"
+    end
+  end
+  
+  def base_type_to_s
+    base_type.to_s if valid_type?
+  end
+
+  def type_to_s
+    type.to_s if valid_type?
+  end
+  
+  def spool=(type, input)
+    self.type = type
+    self.input = input
+  end
+  
+  def valid?
+    valid_type? && valid_input?
+  end
+  
+  def valid_type?
+    @@types.keys.each {|type| return true if @type && (@type < type || @type == type)}
+    return false
+  end
+  
+  def valid_input?
+    if not valid_type?
+      return false
+    else
+      Array(@@types[base_type]).each {|input_type| return true if input.is_a?(input_type)}
+      raise MapReduceError, "invalid input (#{@input.inspect}) for type: #{base_type}. Try one of the following: #{Array(@@types[base_type]).join(", ")}"
+    end
+  end
+  
+  def method_missing(name, *args)
+    if name.to_s =~ /(.*)=$/ && args[0].is_a?(Proc)
+      self.class.instance_eval do
+        define_method($1, args[0])
+      end
+    else
+      super
+    end
+  end
+  
+  def raise_if_invalid!
+    if not valid_type?
+      raise MapReduceError, "invalid type, please make sure you provide one of the following classes or sub-classes thereof: ActiveRecord::Base, File, or Array"
+    end
+    if not valid_input?
+      raise MapReduceError, "invalid input, please make sure you provide one of the following: #{Array(@@types[type]).join(", ")}"
+    end
+  end
+  
+  def map_reduce?
+    true
+  end
+  
+  def base_type
+    check_type = @type
+    type_found = false
+    while check_type.superclass
+      if @@types.include?(check_type)
+        return check_type
+      else
+        check_type = check_type.superclass
+      end
+    end
+  end
+end

data/lib/starfish.rb

@@ -0,0 +1,200 @@
+require 'rinda/ring'
+require 'rinda/tuplespace'
+require 'timeout'
+require 'logger'
+require 'md5'
+require 'yaml'
+
+require 'map_reduce'
+
+DRb.start_service
+
+class StarfishError < StandardError; end
+class Starfish
+  VERSION = "1.0.0"
+  
+  @@server = false
+  @@client = false
+  @@options = {
+    :log => "/tmp/#{File.basename(ARGV.first)}.log"
+  }
+  
+  include Rinda
+  attr_accessor :started, :ring_server, :server
+  
+  # set the uniq identifier
+  def initialize(uniq=ARGV.first)
+    @retry_count = 0
+    @uniq = uniq
+  end
+  
+  def uniq
+    MD5.new(@uniq).to_s
+  end
+  
+  def server
+    unless @@server
+      $stderr.puts "You must specify a server"
+      exit
+    end
+
+    map_reduce = MapReduce.new
+    object = @@server.call(map_reduce)
+    
+    if map_reduce.valid?
+      object = map_reduce
+    end
+    
+    sanitize object
+    
+    ts = Rinda::TupleSpace.new
+    begin
+      ts.write([:name, uniq.intern, object, @uniq])
+      ring_server = Rinda::RingServer.new(ts)
+    rescue Errno::EADDRINUSE
+      ts = RingFinger.primary
+      ts.write([:name, uniq.intern, object, @uniq])
+    end
+
+    File.open(@@options[:pid] || "/tmp/starfish-#{uniq}.pid","w"){|f|f<<Process.pid}
+
+    $stderr.puts "server started for #{object.inspect}"
+    
+    DRb.thread.join
+  end
+  
+  def client
+    unless @@client
+      $stderr.puts "You must specify a client"
+      exit
+    end
+
+    negotiate
+    Timeout::timeout(5) { @server_object = @ring_server.read([:name, uniq.intern, nil, nil])[2] }
+    
+    loop do
+      begin
+        @server_object.map_reduce?
+      rescue NoMethodError
+        @called = @@client.call(@server_object)
+      end
+    
+      unless @called
+        if @server_object.map_reduce? && @server_object.valid?
+          map_reduce_client = eval("MapReduce::#{@server_object.base_type_to_s}::Client").new(@server_object)
+
+          $server_object = @server_object
+          Object.instance_eval do
+            define_method(:logger) do |*args|
+              $server_object._logger(*args)
+            end
+          end
+        
+          map_reduce_client.each do |object|
+            t = Time.now
+            @@client.call(object)
+            @server_object.add_time_spent_processing_objects(Time.now-t)
+          end
+        else
+          raise MapReduceError, "invalid map reduce server (possibly missing type or input)"
+        end
+      end
+    end
+    
+  rescue Timeout::Error => m
+    spawn
+    @retry_count += 1
+    if @retry_count <= 5
+      retry
+    else
+      raise Timeout::Error, m
+    end
+  rescue DRb::DRbConnError => m
+    stop
+    negotiate
+    @retry_count += 1
+    if @retry_count <= 5
+      retry
+    else
+      raise DRb::DRbConnError, m
+    end
+  end
+  
+  def stats
+    negotiate
+    Timeout::timeout(5) { @server_object = @ring_server.read([:name, uniq.intern, nil, nil])[2] }
+    
+    puts @server_object.stats.to_yaml
+  rescue NoMethodError
+    $stderr.puts "The stats method is not defined for your server"
+  end
+  
+  def sanitize(object)
+    object.class.instance_eval { include DRbUndumped }
+    
+    @@log = case @@options[:log]
+    when String
+      Logger.new(@@options[:log])
+    when Class
+      @@options[:log].new
+    when nil, false
+      Logger.new("/dev/null")
+    else
+      @@options[:log]
+    end
+    
+    def object.logger
+      @logger ||= @@log
+    end
+    def object._logger
+      @logger ||= @@log
+    end
+  end
+  
+  def negotiate
+    begin
+      @ring_server = RingFinger.primary
+    rescue RuntimeError => m
+      # allow multiple un-cached calls to RingFinger.finger
+      def RingFinger.finger
+        @@finger = self.new
+        @@finger.lookup_ring_any
+        @@finger
+      end
+
+      spawn
+      @retry_count += 1
+      if @retry_count <= 5
+        retry
+      else
+        raise RuntimeError, m
+      end
+    end
+  end
+
+  def spawn
+    @started ||= fork { system("ruby #{File.dirname(__FILE__)}/../bin/starfish #{@uniq} server > /dev/null") }
+  end
+  
+  def stop
+    puts "stopping the server"
+    Process.kill("SIGHUP", IO.read(@@options[:pid] || "/tmp/starfish-#{uniq}.pid").to_i)
+  rescue Errno::ENOENT
+    puts "Fatal error, please kill all starfish processes manually and try again"
+    system("ps auxww|grep starfish")
+  end
+  
+  class << self
+    def server=(server)
+      @@server = server
+    end
+    
+    def client=(client)
+      @@client = client
+    end
+    
+    def options=(options={})
+      @@options.update(options)
+    end
+  end
+end