wukong 2.0.2 → 3.0.0.pre

data/lib/hanuman/resource.rb

@@ -0,0 +1,6 @@
+module Hanuman
+  class Resource < Stage
+    include Hanuman::IsOwnInputSlot
+    include Hanuman::IsOwnOutputSlot
+  end
+end

data/lib/hanuman/slot.rb

@@ -0,0 +1,87 @@
+module Hanuman
+
+  #
+  # Provides the methods required in order to accept inbound links.
+  # Including class must provide the input attribute and the owner method.
+  #
+  # @see IsOwnInputSlot
+  # @see Slottable
+  module Inlinkable
+    extend Gorillib::Concern
+
+    def set_input(stage)
+      write_attribute(:input, stage)
+      self
+    end
+
+    # wire another slot into this one
+    # @param other [Hanuman::Outlinkable] the other stage of slot
+    # @returns this object, for chaining
+    def <<(other)
+      from(other)
+      self
+    end
+
+    # wire another slot into this one
+    # @param other [Hanuman::Outlinkable] the other stage or slot
+    # @returns this object, for chaining
+    def from(other)
+      owner.connect(other, self)
+      self
+    end
+  end
+
+  #
+  # Provides the methods required in order to accept outbound links.
+  # Including class must provide the output attribute and the owner method.
+  #
+  # @see IsOwnOutputSlot
+  # @see Slottable
+  module Outlinkable
+    extend Gorillib::Concern
+
+    def set_output(stage)
+      write_attribute(:output, stage)
+      self
+    end
+
+    # wire this slot into another slot
+    # @param other [Hanuman::Slot] the other stage
+    # @returns the other slot
+    def >(other)
+      _, other = owner.connect(self, other)
+      other
+    end
+
+    # wire this stage's output into another stage's input
+    # @param other [Hanuman::Stage]the other stage
+    # @returns this stage, for chaining
+    def into(other)
+      owner.connect(self, other)
+      self
+    end
+  end
+
+  class Slot
+    include  Gorillib::Builder
+    field    :name,  Symbol
+    field    :stage, Hanuman::Stage
+    def owner
+      stage.owner
+    end
+    def to_key() name ; end
+  end
+
+  class InputSlot < Slot
+    include  Hanuman::Inlinkable
+    magic    :input,    Hanuman::Stage, :writer => false, :tester => true, :doc => 'stage/slot in graph that feeds into this one'
+    def other() input ; end
+  end
+
+  class OutputSlot < Slot
+    include  Hanuman::Outlinkable
+    magic    :output,   Hanuman::Stage, :writer => false, :tester => true, :doc => 'stage/slot in graph this one feeds into'
+    def other() ouput ; end
+  end
+
+end

data/lib/hanuman/slottable.rb

@@ -0,0 +1,220 @@
+module Hanuman
+
+  #
+  # For stages that can be linked to directly
+  # Including this means your stage has exactly one input (itself).
+  #
+  module IsOwnInputSlot
+    extend  Gorillib::Concern
+    include Inlinkable
+    included do
+      magic   :input,    Hanuman::Stage, :writer => false, :tester => true, :doc => 'stage/slot in graph that feeds into this one'
+    end
+    def inputs
+      input?  ? [input] : []
+    end
+  end
+
+  #
+  # For stages that can be linked to directly
+  # Including this means your stage has exactly one output (itself).
+  #
+  module IsOwnOutputSlot
+    extend  Gorillib::Concern
+    include Outlinkable
+    included do
+      magic  :output,   Hanuman::Stage, :writer => false, :tester => true, :doc => 'stage/slot in graph this one feeds into'
+    end
+    def outputs
+      output? ? [output] : []
+    end
+  end
+
+  #
+  # For stages with named slots
+  #
+  # A named slot is a special kind of field: saying
+  #
+  #   consumes :brain
+  #
+  # gives your class
+  #
+  # * A normal attribute `brain_slot`
+  # * methods `brain_slot`, `receive_brain_slot` to go with it
+  # * method `brain`, returning the item (if any) connected to the brain slot
+  # * method `brain=` (alias for `receive_brain`) that links the brain slot with the given item
+  #
+  # @note that at the moment you can't have an input and an output with the same name.
+  #
+  module Slottable
+    extend Gorillib::Concern
+    include Inlinkable
+    include Outlinkable
+
+    included do
+      collection :outslots, Hanuman::OutputSlot, :key_method => :name
+    end
+
+    def inputs
+      inslots.to_a.map{|slot| slot.input }.compact
+    end
+
+    def inslots
+      self.class.inslot_fields.map{|_, slot_field| read_attribute(slot_field.name) }
+    end
+
+    def handle_extra_attributes(attrs)
+      self.class.inslot_fields.each do |_, field|
+        field_name = field.basename
+        next unless attrs.has_key?(field_name)
+        self.public_send(:"receive_#{field_name}", attrs.delete(field_name))
+      end
+      super(attrs)
+    end
+
+    module ClassMethods
+      def consumes(name, options={})
+        field name, Hanuman::Stage, {:field_type => InputSlotField}.merge(options)
+      end
+      def produces(name, options={})
+        field name, Hanuman::Stage, {:field_type => OutputSlotField}.merge(options)
+      end
+
+      def define_slot_reader(field)
+        meth_name  = field.basename
+        slot_name  = field.name
+        type       = field.type
+        define_meta_module_method(meth_name, true) do ||
+          begin
+            slot = read_attribute(slot_name) or return nil
+            slot.other
+          rescue StandardError => err ; err.polish("#{self.class}.#{meth_name}") rescue nil ; raise ; end
+        end
+      end
+
+      def define_inslot_receiver(field)
+        meth_name  = field.basename
+        slot_name  = field.name
+        type       = field.type
+        define_meta_module_method("receive_#{meth_name}", true) do |stage|
+          begin
+            slot = read_attribute(slot_name) or return nil
+            slot.from(stage)
+            self
+          rescue StandardError => err ; err.polish("#{self.class} set slot #{meth_name} to #{stage}") rescue nil ; raise ; end
+        end
+        meta_module.module_eval do
+          alias_method "#{meth_name}=", "receive_#{meth_name}"
+        end
+      end
+
+      def define_outslot_receiver(field)
+        meth_name  = field.basename
+        slot_name  = field.name
+        type       = field.type
+        define_meta_module_method("receive_#{meth_name}", true) do |stage|
+          begin
+            slot = read_attribute(slot_name) or return nil
+            slot.into(stage)
+            self
+          rescue StandardError => err ; err.polish("#{self.class} set slot #{meth_name} to #{stage}") rescue nil ; raise ; end
+        end
+        meta_module.module_eval do
+          alias_method "#{meth_name}=", "receive_#{meth_name}"
+        end
+      end
+
+      def inslot_fields
+        fields.select{|_, field| field.is_a?(InputSlotField) }
+      end
+
+      def inslot_field?(field_name)
+        fields[field_name].is_a?(InputSlotField)
+      end
+    end
+
+    class SlotField < Gorillib::Model::Field
+      self.visibilities = visibilities.merge(:reader => true, :writer => false, :tester => false)
+      field :basename,   Symbol
+      field :stage_type, Whatever, :doc => 'type for stages this slot accepts'
+      class_attribute :slot_type
+
+      def initialize(model, basename, type, options={})
+        name = "#{basename}_slot"
+        options[:stage_type] = type
+        slot_type = self.slot_type
+        options[:basename] = basename
+        options[:default]  = ->{ slot_type.new(:name => basename, :stage => self) }
+        super(model, name, slot_type, options)
+      end
+    end
+
+    class InputSlotField < SlotField
+      self.slot_type = Hanuman::InputSlot
+      def inscribe_methods(model)
+        model.__send__(:define_slot_reader, self)
+        model.__send__(:define_inslot_receiver, self)
+        super
+      end
+    end
+
+    class OutputSlotField < SlotField
+      self.slot_type = Hanuman::OutputSlot
+      def inscribe_methods(model)
+        model.__send__(:define_slot_reader, self)
+        model.__send__(:define_outslot_receiver, self)
+        super
+      end
+    end
+
+  end # Slottable
+
+  module SplatInputs
+    extend  Gorillib::Concern
+    include Slottable
+
+    included do
+      collection :splat_inslots, Hanuman::InputSlot, :key_method => :name
+    end
+
+    def set_input(stage)
+      slot = Hanuman::InputSlot.new(:name => stage.name, :stage => self, :input => stage)
+      self.splat_inslots << slot
+      slot
+    end
+
+    def has_input?(slot_name)
+      self.splat_inslots.keys.include?(slot_name)
+    end
+
+    def inslots
+      super + splat_inslots.to_a
+    end
+  end
+
+  module SplatOutputs
+    extend  Gorillib::Concern
+    include Slottable
+
+    included do
+      collection :splat_outslots, Hanuman::OutputSlot, :key_method => :name
+    end
+
+    def set_output(stage)
+      slot = Hanuman::OutputSlot.new(
+        :name => stage.name, :stage => self, :output => stage)
+      self.outslots << slot
+      slot
+    end
+
+    def outputs
+      outslots.to_a.map{|slot| slot.output }
+    end
+
+    def into(*others)
+      others.each{|other| super(other)}
+      self
+    end
+  end
+
+end

data/lib/hanuman/stage.rb

@@ -0,0 +1,51 @@
+module Hanuman
+  class Stage
+    include Gorillib::Builder
+    alias_method :configure, :receive!
+
+    magic      :name,    Symbol,         :doc => 'name for this stage; should be unique among other stages on its containing graph', :tester => true
+    field      :owner,   Whatever,       :doc => 'the graph this stage sits in'
+    magic      :doc,     String,         :doc => 'freeform description of this stage type'
+
+    # @returns the stage, namespaced by the graph that owns it
+    def fullname
+      [owner.try(:fullname), name].compact.join('.')
+    end
+
+    def self.handle
+      Gorillib::Inflector.underscore(Gorillib::Inflector.demodulize(self.name))
+    end
+
+    #
+    # Methods
+    #
+
+    # Called after the graph is constructed, before the flow is run
+    def setup
+    end
+
+    # Called to signal the flow should stop. Close any open connections, flush
+    # buffers, stop supervised projects, etc.
+    def stop
+    end
+
+    def lookup(stage)
+      owner.lookup(stage)
+    end
+
+    #
+    # Graph connections
+    #
+
+    def notify(msg)
+      true
+    end
+
+    def report
+      self.attributes
+    end
+
+    def to_key()      name   ; end
+    def key_method() :name ; end
+  end
+end

data/lib/wukong.rb

@@ -1,18 +1,32 @@
-require 'wukong/extensions'
-require 'configliere'; Settings.use :define
-require 'wukong/datatypes'
-require 'wukong/periodic_monitor'
-require 'wukong/logger'
-autoload :BadRecord,   'wukong/bad_record'
-autoload :TypedStruct, 'wukong/typed_struct'
-module Wukong
-  autoload :Script,          'wukong/script'
-  autoload :Streamer,        'wukong/streamer'
-  autoload :Store,           'wukong/store'
-  autoload :FilenamePattern, 'wukong/filename_pattern'
-  autoload :Decorator,       'wukong/decorator'
+require 'multi_json'
 
-  def self.run mapper, reducer=nil, options={}
-    Wukong::Script.new(mapper, reducer, options).run
-  end
-end
+require 'configliere'
+require 'gorillib/logger/log'
+require 'gorillib/some'
+require 'gorillib/builder'
+require 'gorillib/model/serialization'
+
+require 'wukong/settings'
+
+require 'hanuman'
+
+#
+# Dataflow specific
+#
+require 'wukong/universe'
+require 'wukong/dataflow'
+require 'wukong/event'
+
+require 'wukong/processor'           # processes records in series
+require 'wukong/widget/filter'       # passes through only records that meet `accept?`
+require 'wukong/widget/source'       # generates raw records from outside
+require 'wukong/widget/sink'         # dispatch raw records to outside
+require 'wukong/widget/stringifier'  # converts raw blobs into structured records and vice/versa
+require 'wukong/mapred'              # the standard stream-sort-group-stream map/reduce flow
+require 'wukong/local_runner'
+require 'wukong/bad_record'
+
+#
+# Workflow Specific
+#
+require 'wukong/workflow/command'

data/lib/wukong/bad_record.rb

@@ -1,18 +1,15 @@
-#
-# Easily serialize bad records in-band, for later analysis or to discard if
-# neglectable.
-#
-# You can instantiate this as
-#  success = do_stuff_to record
-#  if ! success
-#    return BadRecord.new("do_stuff_to-failed", record)
-#  end
-#
-class BadRecord < Struct.new(
-    :errors,
-    :record
-    )
-  def initialize errors='', *record_fields
-    super errors, record_fields
+class BadRecord
+  include Gorillib::Model
+  field :contents, Whatever,  :doc => "The faulty contents; will be truncated at 1000 characters"
+  field :error,    Exception, :doc => "Error (optional)"
+
+  def receive_contents(contents)
+    super contents.to_s[0..1000]
+  end
+
+  def make(contents, error=nil)
+    hsh = { :contents => contents }
+    hsh[:error] = error if error
+    receive(hsh)
   end
 end