wukong 3.0.0.pre → 3.0.0.pre2

data/lib/wukong.rb

@@ -1,32 +1,31 @@
-require 'multi_json'
-
 require 'configliere'
 require 'gorillib/logger/log'
-require 'gorillib/some'
-require 'gorillib/builder'
-require 'gorillib/model/serialization'
-
-require 'wukong/settings'
+require 'vayacondios-client'
+require 'multi_json'
 
 require 'hanuman'
 
-#
-# Dataflow specific
-#
-require 'wukong/universe'
-require 'wukong/dataflow'
-require 'wukong/event'
+require 'wukong/processor'
+require 'wukong/driver'
 
-require 'wukong/processor'           # processes records in series
-require 'wukong/widget/filter'       # passes through only records that meet `accept?`
-require 'wukong/widget/source'       # generates raw records from outside
-require 'wukong/widget/sink'         # dispatch raw records to outside
-require 'wukong/widget/stringifier'  # converts raw blobs into structured records and vice/versa
-require 'wukong/mapred'              # the standard stream-sort-group-stream map/reduce flow
-require 'wukong/local_runner'
-require 'wukong/bad_record'
+require 'wukong/emitter'
+require 'wukong/dataflow'
+require 'wukong/configuration'
+require 'wukong/widgets'
 
+# The Wukong module will contain all code for Wukong's core (like
+# Processors and Dataflows) as well as all plugins.
 #
-# Workflow Specific
-#
-require 'wukong/workflow/command'
+# Plugins are expected to own their own namespace within Wukong,
+# e.g. - Wukong::Hadoop, Wukong::Storm, &c.
+module Wukong
+  extend Hanuman::Shortcuts
+
+  # A common error class intended to be raised by code within Wukong
+  # or its plugins.
+  Error = Class.new(StandardError)
+  
+  add_shortcut_method_for(:processor, ProcessorBuilder)
+  add_shortcut_method_for(:dataflow,  DataflowBuilder)
+
+end

data/lib/wukong/boot.rb

@@ -0,0 +1,87 @@
+module Wukong
+
+  # Boots Wukong, reading any and all relevant `settings` for your
+  # processors.
+  #
+  # This is most useful for any script which wants to run within a
+  # deploy.  See Wukong::Deploy for more details.
+  #
+  # @param [Configliere::Param] settings the relevant settings object that should be configured to boot within a deploy pack
+  def self.boot! settings
+    # First we load the deploy pack's environment, but only if we seem
+    # to be running within a deploy pack.
+    Boot.load_environment        if in_deploy_pack?
+
+    # Next pass the `settings` to the Deploy pack itself to add
+    # options it needs.
+    Deploy.configure(settings) if loaded_deploy_pack?
+
+    # Resolve the `settings` so we can capture all the options we need
+    # from the command line, the environment, &c.
+    settings.resolve!
+
+    # Now boot the deploy pack itself, passing in its known location
+    # and the `settings`
+    Deploy.boot!(settings, Boot.deploy_pack_dir) if loaded_deploy_pack?
+  end
+
+  # Is execution likely happening within a deploy pack?
+  #
+  # See Wukong::Deploy for more information on deploy packs.
+  #
+  # @return [true, false]
+  def self.in_deploy_pack?
+    return @in_deploy_pack unless @in_deploy_pack.nil?
+    @in_deploy_pack = (Boot.deploy_pack_dir != '/')
+  end
+
+  # Have we already loaded the environment of a deploy pack?
+  #
+  # See Wukong::Deploy for more information on deploy packs.
+  #
+  # @return [true, false]
+  def self.loaded_deploy_pack?
+    in_deploy_pack? && defined?(::Wukong::Deploy)
+  end
+
+  # Lets Wukong bootstrap by requiring an enclosing deploy pack's
+  # environment file if available.
+  #
+  # We use a simple heuristic (presence of 'Gemfile' and
+  # 'config/environment.rb' in a non-root parent directory) to
+  # determine whether or not we are in a deploy pack.
+  module Boot
+
+    # Return the directory of the enclosing deploy pack.  Will return
+    # the root ('/') if no deeper directory is identified as a deploy
+    # pack.
+    #
+    # @return [String]
+    def self.deploy_pack_dir
+      return @deploy_pack_dir if @deploy_pack_dir
+      wd     = Dir.pwd
+      parent = File.dirname(wd)
+      until wd == parent
+        return wd if File.exist?(File.join(wd, 'Gemfile')) && File.exist?(File.join(wd, 'config', 'environment.rb'))
+        wd     = parent
+        parent = File.dirname(wd)
+      end
+      @deploy_pack_dir = wd
+    end
+
+    # The default environment file that will be require'd when
+    # booting.
+    #
+    # @return [String]
+    def self.environment_file
+      File.join(deploy_pack_dir, 'config', 'environment.rb')
+    end
+
+    # Load the actual deploy pack environment.  Will not swallow any
+    # load errors.
+    def self.load_environment
+      require environment_file
+    end
+  end
+  
+end

data/lib/wukong/configuration.rb

@@ -0,0 +1,8 @@
+module Wukong
+  module Local
+
+    # All local configuration for Wukong lives within this object.
+    Configuration = Configliere::Param.new unless defined? Configuration
+    
+  end
+end

data/lib/wukong/dataflow.rb

@@ -1,102 +1,69 @@
 module Wukong
+  class DataflowBuilder < Hanuman::GraphBuilder
 
-  #
-  # Describe a dataflow of sources, processors and sinks.
-  #
-  #
-  class Dataflow < Hanuman::Graph
-    include Hanuman::SplatInputs
-    include Hanuman::SplatOutputs
+    def describe desc
+      @description = desc
+    end
 
-    # * defines the named input slot, if it doesn't exist
-    # * wires the given stage to that input slot
-    # * returns the named input slot
-    def input(slot_name, default_stage = nil)
-      if not splat_inslots.include?(slot_name)
-        slot = Hanuman::InputSlot.new(:name => slot_name, :stage => self)
-        self.splat_inslots << slot
-      else
-        slot = splat_inslots[slot_name]
-      end
-      if default_stage.present?
-        self.add_stage(default_stage) if not self.stages.include?(default_stage)
-        slot.input(default_stage)
+    def description
+      @description
+    end
+    
+    def namespace() Wukong::Dataflow ; end
+
+    def handle_dsl_arguments_for(stage, *args, &action)
+      options = args.extract_options!
+      stage.merge!(options.merge(action: action).compact)
+      stage      
+    end
+  
+    def linkable_name(direction) 
+      case direction
+      when :in  then directed_sort.first
+      when :out then directed_sort.last
       end
-      slot.input
     end
 
-    def output(slot_name, default_stage = nil)
-      if not splat_outslots.include?(slot_name)
-        slot = Hanuman::OutputSlot.new(:name => slot_name, :stage => self)
-        self.splat_outslots << slot
+    def method_missing(name, *args, &blk)
+      if stages[name]
+        handle_dsl_arguments_for(stages[name], *args, &blk)
       else
-        slot = splat_outslots[slot_name]
-      end
-      if default_stage.present?
-        self.add_stage(default_stage) if not self.stages.include?(default_stage)
-        slot.output(default_stage)
+        super
       end
-      slot.output
     end
+    
+  end
+  
+  class Dataflow < Hanuman::Graph
 
-    # FIXME: only handles one output slot
-    def process(rec)
-      input(:default).output.process(rec)
-      # stages.to_a.first.process(rec)
+    def has_input?(stage)
+      links.any?{ |link| link.into == stage }
     end
-
-    def set_output(sink)
-      stages.to_a.last.set_output sink
+    
+    def has_output?(stage)
+      links.any?{ |link| link.from == stage }
     end
 
-    #
-    # lifecycle
-    #
-
-    def setup
-      stages.each_value{|stage| stage.setup}
+    def connected?(stage)
+      input  = has_input?(stage)  || stages[stage].is_a?(Wukong::Source)
+      output = has_output?(stage) || stages[stage].is_a?(Wukong::Sink)
+      input && output
     end
 
-    # FIXME -- this is ugly, and evidence to  consider in the "Where does an input live" conundrum
-    #   ... or it means that we're thinking about message propogation wrong.
-
-    def stop
-      source_stages.each{|stage| stage.stop}
-      process_stages.each{|stage| stage.stop}
-      sink_stages.each{|stage| stage.stop}
+    def complete?
+      stages.all?{ |(name, stage)| connected? name }
     end
 
-    def source_stages()  stages.to_a.select{|st| st.is_a?(Wukong::Source) }     end
-    def process_stages() stages.to_a.select{|st| (not st.is_a?(Wukong::Source)) && (not st.is_a?(Wukong::Sink)) }  end
-    def sink_stages()    stages.to_a.select{|st| st.is_a?(Wukong::Sink) }       end
-
-    def drive(slot_name)
-      raise StandardError, "No source wired up input slot '#{slot_name.inspect}' of #{self.inspect} #{self.attributes}" unless has_input?(slot_name)
-      input(slot_name).drive
+    def setup
+      directed_sort.each{ |name| stages[name].setup }
     end
 
-    #
-    # Processor helpers
-    #
-
-    def reject(re_or_block=nil, &block)
-      raise ArgumentError, "Supply a block or regular expression, not both" if re_or_block && block
-      if re_or_block.is_a?(Regexp)
-        add_stage(Widget::RegexpRejecter.new(:pattern => re_or_block))
-      else
-        block ||= re_or_block
-        add_stage(Widget::ProcRejecter.new(block))
-      end
+    def run
+      stages[directed_sort.first].run
     end
 
-    def select(re_or_block=nil, &block)
-      raise ArgumentError, "Supply a block or regular expression, not both" if re_or_block && block
-      if re_or_block.is_a?(Regexp)
-        add_stage(Widget::RegexpFilter.new(:pattern => re_or_block))
-      else
-        block ||= re_or_block
-        add_stage(Widget::ProcFilter.new(block))
-      end
+    def stop
+      directed_sort.each{ |name| stages[name].stop }
     end
 
   end

data/lib/wukong/driver.rb

@@ -0,0 +1,99 @@
+module Wukong  
+  class Driver
+
+    class << self
+      def run(dataflow, options)
+        new(dataflow, options).run!
+      end
+    end
+
+    def lookup(label)
+      builder = Wukong.registry.retrieve(label)
+      builder.build(@settings)
+    end
+    
+    def wire(flow)
+      using = @settings[:wiring] || :emitter
+      if flow.complete?
+        flow.links.each do |link| 
+          Hanuman::LinkFactory.connect(using, flow.stages[link.from], flow.stages[link.into])
+        end
+      end
+      flow
+    end
+  end
+  
+  class LocalDriver < Driver
+    attr_accessor :dataflow, :source
+    
+    def initialize(dataflow, options = {})
+      builder    = (Wukong.registry.retrieve(dataflow) or raise Error.new("No such processor or dataflow: #{dataflow.inspect}"))
+      dataflow   = builder.build(options)
+      @dataflow  = dataflow.respond_to?(:stages) ? dataflow.directed_sort.map{ |name| dataflow.stages[name] } : [ dataflow ]
+      @dataflow << Wukong.registry.retrieve(:stdout).build
+      @source    = Wukong.registry.retrieve(:stdin).build
+    end
+
+    def emitter
+      @emitter ||= ProcEmitter.new(dataflow)
+    end
+
+    def stage_iterator(stage)
+      return dataflow.first if stage.nil?
+      position = dataflow.find_index(stage)
+      dataflow[position + 1]    
+    end
+
+    def next_record(&blk)
+      trap('SIGINT'){ break }              
+      source.process(&blk)
+    end
+
+    def run!      
+      dataflow.each(&:setup)
+      next_record do |record|
+        emitter.send_through_dataflow(record)
+      end
+      dataflow.each do |stage|
+        stage.finalize(&emitter.advance(stage)) if stage.respond_to?(:finalize)
+        stage.stop
+      end
+      nil
+    end
+  end
+end
+
+class ProcEmitter
+
+  attr_accessor :dataflow
+
+  def initialize(dataflow)
+    @dataflow = dataflow
+  end
+
+  def to_proc
+    return @wiring if @wiring
+    @wiring = Proc.new do |stage, record|
+      stage.process(record, &advance(stage)) if stage          
+    end
+  end
+
+  def send_through_dataflow(record)
+    self.call(dataflow.first, record)
+  end
+  
+  def advance(stage)
+    next_stage = stage_iterator(stage)
+    self.to_proc.curry.call(next_stage)
+  end
+
+  def stage_iterator(stage)
+    position = dataflow.find_index(stage)
+    dataflow[position + 1]    
+  end 
+
+  def call(*args)
+    to_proc.call(*args)
+  end
+  
+end

data/lib/wukong/emitter.rb

@@ -0,0 +1,22 @@
+module Wukong
+  class Emitter < Hanuman::DirectedLink
+    
+    def initialize(from, into)
+      @from = from
+      @into = into
+      @from.write_attribute(:emitter, self)
+    end
+    
+    def call(record)
+      into.process(record)
+    end
+
+    def to_s
+      "#<#{self.class}(#{from.label} -> #{into.label})>"
+    end
+
+  end
+  
+  Hanuman::LinkFactory.register(:emitter, ->(from_stage, into_stage){ Wukong::Emitter.new(from_stage, into_stage) })
+  
+end

data/lib/wukong/model/faker.rb

@@ -109,28 +109,28 @@ module Gorillib::Factory
       nil
     end
   end
-  
-  class StringFactory    ; def fake_value()        receive(Wukong::Faker::Helpers.fake_word)          ; end ; end
-  class GuidFactory      ; def fake_value()        receive(UUIDTools::UUID.random_create.to_s)        ; end ; end
-  class HostnameFactory  ; def fake_value()        receive(Wukong::Faker::Helpers.fake_hostname)      ; end ; end
-  class IpAddressFactory ; def fake_value()        receive(Wukong::Faker::Helpers.fake_ip_addresss)   ; end ; end
-
-  class SymbolFactory    ; def fake_value()        receive(Wukong::Faker::Helpers.fake_identifier)    ; end ; end
-  class PathnameFactory  ; def fake_value()        receive(Wukong::Faker::Helpers.fake_filename)      ; end ; end
-
-  class IntegerFactory   ; def fake_value(opts={}) receive(Wukong::Faker::Helpers.fake_integer(opts)) ; end ; end
-  class BignumFactory    ; def fake_value(opts={}) super({:min => 2**68, :max => 2**90}.merge(opts))  ; end ; end
-
-  class FloatFactory     ; def fake_value(opts={}) receive(Wukong::Faker::Helpers.fake_float(opts))   ; end ; end
-  class ComplexFactory   ; def fake_value()        receive(Kernel.rand, Kernel.rand)                  ; end ; end
-  class RationalFactory  ; def fake_value()        receive(Kernel.rand.to_r)                          ; end ; end
-
-  class TimeFactory      ; def fake_value()        receive(Time.now)                                  ; end ; end
-
-  class ExceptionFactory ; def fake_value()        Exception.constants.sample                         ; end ; end
-  
-  class NilFactory       ; def fake_value()        nil                                                ; end ; end
-  class TrueFactory      ; def fake_value()        true                                               ; end ; end
-  class FalseFactory     ; def fake_value()        false                                              ; end ; end
-  class BooleanFactory   ; def fake_value()        [true, false].sample                               ; end ; end
+
+  StringFactory.class_eval   { def fake_value()        receive(Wukong::Faker::Helpers.fake_word)          ; end }
+  GuidFactory.class_eval     { def fake_value()        receive(UUIDTools::UUID.random_create.to_s)        ; end }
+  HostnameFactory.class_eval { def fake_value()        receive(Wukong::Faker::Helpers.fake_hostname)      ; end }
+  IpAddressFactory.class_eval{ def fake_value()        receive(Wukong::Faker::Helpers.fake_ip_addresss)   ; end }
+
+  SymbolFactory.class_eval   { def fake_value()        receive(Wukong::Faker::Helpers.fake_identifier)    ; end }
+  PathnameFactory.class_eval { def fake_value()        receive(Wukong::Faker::Helpers.fake_filename)      ; end }
+
+  IntegerFactory.class_eval  { def fake_value(opts={}) receive(Wukong::Faker::Helpers.fake_integer(opts)) ; end }
+  BignumFactory.class_eval   { def fake_value(opts={}) super({:min => 2**68, :max => 2**90}.merge(opts))  ; end }
+
+  FloatFactory.class_eval    { def fake_value(opts={}) receive(Wukong::Faker::Helpers.fake_float(opts))   ; end }
+  ComplexFactory.class_eval  { def fake_value()        receive(Kernel.rand, Kernel.rand)                  ; end }
+  RationalFactory.class_eval { def fake_value()        receive(Kernel.rand.to_r)                          ; end }
+
+  TimeFactory.class_eval     { def fake_value()        receive(Time.now)                                  ; end }
+
+  ExceptionFactory.class_eval{ def fake_value()        Exception.constants.sample                         ; end }
+
+  NilFactory.class_eval      { def fake_value()        nil                                                ; end }
+  TrueFactory.class_eval     { def fake_value()        true                                               ; end }
+  FalseFactory.class_eval    { def fake_value()        false                                              ; end }
+  BooleanFactory.class_eval  { def fake_value()        [true, false].sample                               ; end }
 end