wukong 3.0.0.pre → 3.0.0.pre2

data/lib/hanuman/slottable.rb

@@ -1,220 +0,0 @@
-module Hanuman
-
-  #
-  # For stages that can be linked to directly
-  # Including this means your stage has exactly one input (itself).
-  #
-  module IsOwnInputSlot
-    extend  Gorillib::Concern
-    include Inlinkable
-    included do
-      magic   :input,    Hanuman::Stage, :writer => false, :tester => true, :doc => 'stage/slot in graph that feeds into this one'
-    end
-    def inputs
-      input?  ? [input] : []
-    end
-  end
-
-  #
-  # For stages that can be linked to directly
-  # Including this means your stage has exactly one output (itself).
-  #
-  module IsOwnOutputSlot
-    extend  Gorillib::Concern
-    include Outlinkable
-    included do
-      magic  :output,   Hanuman::Stage, :writer => false, :tester => true, :doc => 'stage/slot in graph this one feeds into'
-    end
-    def outputs
-      output? ? [output] : []
-    end
-  end
-
-  #
-  # For stages with named slots
-  #
-  # A named slot is a special kind of field: saying
-  #
-  #   consumes :brain
-  #
-  # gives your class
-  #
-  # * A normal attribute `brain_slot`
-  # * methods `brain_slot`, `receive_brain_slot` to go with it
-  # * method `brain`, returning the item (if any) connected to the brain slot
-  # * method `brain=` (alias for `receive_brain`) that links the brain slot with the given item
-  #
-  # @note that at the moment you can't have an input and an output with the same name.
-  #
-  module Slottable
-    extend Gorillib::Concern
-    include Inlinkable
-    include Outlinkable
-
-    included do
-      collection :outslots, Hanuman::OutputSlot, :key_method => :name
-    end
-
-    def inputs
-      inslots.to_a.map{|slot| slot.input }.compact
-    end
-
-    def inslots
-      self.class.inslot_fields.map{|_, slot_field| read_attribute(slot_field.name) }
-    end
-
-    def handle_extra_attributes(attrs)
-      self.class.inslot_fields.each do |_, field|
-        field_name = field.basename
-        next unless attrs.has_key?(field_name)
-        self.public_send(:"receive_#{field_name}", attrs.delete(field_name))
-      end
-      super(attrs)
-    end
-
-    module ClassMethods
-      def consumes(name, options={})
-        field name, Hanuman::Stage, {:field_type => InputSlotField}.merge(options)
-      end
-      def produces(name, options={})
-        field name, Hanuman::Stage, {:field_type => OutputSlotField}.merge(options)
-      end
-
-      def define_slot_reader(field)
-        meth_name  = field.basename
-        slot_name  = field.name
-        type       = field.type
-        define_meta_module_method(meth_name, true) do ||
-          begin
-            slot = read_attribute(slot_name) or return nil
-            slot.other
-          rescue StandardError => err ; err.polish("#{self.class}.#{meth_name}") rescue nil ; raise ; end
-        end
-      end
-
-      def define_inslot_receiver(field)
-        meth_name  = field.basename
-        slot_name  = field.name
-        type       = field.type
-        define_meta_module_method("receive_#{meth_name}", true) do |stage|
-          begin
-            slot = read_attribute(slot_name) or return nil
-            slot.from(stage)
-            self
-          rescue StandardError => err ; err.polish("#{self.class} set slot #{meth_name} to #{stage}") rescue nil ; raise ; end
-        end
-        meta_module.module_eval do
-          alias_method "#{meth_name}=", "receive_#{meth_name}"
-        end
-      end
-
-      def define_outslot_receiver(field)
-        meth_name  = field.basename
-        slot_name  = field.name
-        type       = field.type
-        define_meta_module_method("receive_#{meth_name}", true) do |stage|
-          begin
-            slot = read_attribute(slot_name) or return nil
-            slot.into(stage)
-            self
-          rescue StandardError => err ; err.polish("#{self.class} set slot #{meth_name} to #{stage}") rescue nil ; raise ; end
-        end
-        meta_module.module_eval do
-          alias_method "#{meth_name}=", "receive_#{meth_name}"
-        end
-      end
-
-      def inslot_fields
-        fields.select{|_, field| field.is_a?(InputSlotField) }
-      end
-
-      def inslot_field?(field_name)
-        fields[field_name].is_a?(InputSlotField)
-      end
-    end
-
-    class SlotField < Gorillib::Model::Field
-      self.visibilities = visibilities.merge(:reader => true, :writer => false, :tester => false)
-      field :basename,   Symbol
-      field :stage_type, Whatever, :doc => 'type for stages this slot accepts'
-      class_attribute :slot_type
-
-      def initialize(model, basename, type, options={})
-        name = "#{basename}_slot"
-        options[:stage_type] = type
-        slot_type = self.slot_type
-        options[:basename] = basename
-        options[:default]  = ->{ slot_type.new(:name => basename, :stage => self) }
-        super(model, name, slot_type, options)
-      end
-    end
-
-    class InputSlotField < SlotField
-      self.slot_type = Hanuman::InputSlot
-      def inscribe_methods(model)
-        model.__send__(:define_slot_reader, self)
-        model.__send__(:define_inslot_receiver, self)
-        super
-      end
-    end
-
-    class OutputSlotField < SlotField
-      self.slot_type = Hanuman::OutputSlot
-      def inscribe_methods(model)
-        model.__send__(:define_slot_reader, self)
-        model.__send__(:define_outslot_receiver, self)
-        super
-      end
-    end
-
-  end # Slottable
-
-  module SplatInputs
-    extend  Gorillib::Concern
-    include Slottable
-
-    included do
-      collection :splat_inslots, Hanuman::InputSlot, :key_method => :name
-    end
-
-    def set_input(stage)
-      slot = Hanuman::InputSlot.new(:name => stage.name, :stage => self, :input => stage)
-      self.splat_inslots << slot
-      slot
-    end
-
-    def has_input?(slot_name)
-      self.splat_inslots.keys.include?(slot_name)
-    end
-
-    def inslots
-      super + splat_inslots.to_a
-    end
-  end
-
-  module SplatOutputs
-    extend  Gorillib::Concern
-    include Slottable
-
-    included do
-      collection :splat_outslots, Hanuman::OutputSlot, :key_method => :name
-    end
-
-    def set_output(stage)
-      slot = Hanuman::OutputSlot.new(
-        :name => stage.name, :stage => self, :output => stage)
-      self.outslots << slot
-      slot
-    end
-
-    def outputs
-      outslots.to_a.map{|slot| slot.output }
-    end
-
-    def into(*others)
-      others.each{|other| super(other)}
-      self
-    end
-  end
-
-end

data/lib/wukong/bad_record.rb

@@ -1,15 +0,0 @@
-class BadRecord
-  include Gorillib::Model
-  field :contents, Whatever,  :doc => "The faulty contents; will be truncated at 1000 characters"
-  field :error,    Exception, :doc => "Error (optional)"
-
-  def receive_contents(contents)
-    super contents.to_s[0..1000]
-  end
-
-  def make(contents, error=nil)
-    hsh = { :contents => contents }
-    hsh[:error] = error if error
-    receive(hsh)
-  end
-end

data/lib/wukong/event.rb

@@ -1,44 +0,0 @@
-module Wukong
-  class EventMetadata
-    include Gorillib::Model
-
-    field :timestamp, Time,   :doc => "time the event originated, assigned by the origin (as anything they like) and unchanged afterwards. A UTC ruby time, serialized as a unix timestamp. Corresponds to Flume's `time` metadata"
-    field :origin,    String, :doc => "name for the source of this record; in flume, the dispatching `host`. This influences delivery guarantees. A downcased, dasherized, dot-separated identifier."
-    field :nano_ctr,  Bignum, :doc => "nanosecond timestamp, monotonically-increasing within each origin. The `[origin, nano_ctr]` pair may be considered globally unique. Serialized as whatever flume uses."
-
-    field :topic,     Symbol, :doc => "Topic this event belongs to"
-
-    def event_id
-      [origin, nano_ctr].join('!')
-    end
-
-  end
-
-  module Event
-    extend  Gorillib::Concern
-    include Gorillib::Model
-
-    def _metadata
-      @_metadata ||= {}
-    end
-
-    def _metadata= m
-      @_metadata = m
-    end
-
-    def to_wire options={}
-      super(options).merge(:_metadata => self._metadata)
-    end
-  end
-
-end
-
-#
-# Example Usage
-#
-# def process(blob)
-#   record   = JSON.parse(blob)
-#   metadata = blob._metadata
-#   { :_id => metadata.event_id, :time => metadata.timestamp, :type => metadata.topic, :data => record }
-#   # ... now do stuff
-# end

data/lib/wukong/local_runner.rb

@@ -1,55 +0,0 @@
-module Wukong
-  class Runner
-    include Gorillib::FancyBuilder
-    member     :flow,     Wukong::Dataflow
-
-    def run(slot_name)
-      wire_flow
-      flow.setup
-      drive_flow(slot_name)
-      flow.stop
-    end
-
-    def self.run(flow, slot_name)
-      runner = self.receive(:flow => flow)
-      runner.run(slot_name)
-    end
-
-    def validate!
-      raise StandardError, "flow is missing for #{self}" unless flow.present?
-    end
-
-  protected
-
-    # Connect sources, sinks, flows and so forth. On return, the topology of the graph should be in place.
-    # Override in your subclass
-    #
-    # @abstract
-    def wire_flow
-    end
-
-    # Launch the flow -- sources be each'ing, processors be process'n
-    # Override in your subclass
-    #
-    # @abstract
-    def drive_flow
-      puts flow
-    end
-  end
-
-  # Run dataflow in pure ruby
-  class LocalRunner < Runner
-
-  protected
-
-    def drive_flow(slot_name)
-      validate!
-      flow.drive(slot_name)
-    end
-
-    def wire_flow
-      # flow.set_output sink(:test_sink)
-      # flow.set_output sinks.to_a.last
-    end
-  end
-end

data/lib/wukong/mapred.rb

@@ -1,3 +0,0 @@
-module Wukong
-
-end

data/lib/wukong/universe.rb

@@ -1,48 +0,0 @@
-module Wukong
-
-  #
-  # Holds graphs, supplies `processor` and similar stage template methods
-  #
-  module Universe
-    def find_or_create_class(superklass, klass_name, namespace, &block)
-      klass_name = Gorillib::Inflector.camelize(klass_name.to_s).to_sym
-      if namespace.const_defined?(klass_name)
-        namespace.const_get(klass_name)
-      else
-        namespace.send(:const_set, klass_name, Class.new(superklass, &block))
-      end
-    end
-
-    def processor(processor_name, &block)
-      klass = find_or_create_class(Wukong::Processor, processor_name, Wukong::Widget) do
-        register_processor(processor_name)
-      end
-      klass.class_eval(&block) if block_given?
-      klass
-    end
-
-    def dataflow(name, attrs={}, &block)
-      attrs[:name] = name = name.to_sym
-      dataflow = @dataflows[name] ||= Dataflow.new(:name => name)
-      dataflow.receive!(attrs, &block)
-      dataflow
-    end
-
-    def workflow(name, attrs={}, &block)
-      attrs[:name] = name = name.to_sym
-      workflow = @workflows[name] ||= Workflow.new(:name => name)
-      workflow.receive!(attrs, &block)
-      workflow
-    end
-
-    def self.extended(base)
-      base.instance_eval do
-        @dataflows = Hash.new
-        @workflows = Hash.new
-      end
-    end
-  end
-
-  # Wukong can serve as a universe
-  extend Universe
-end

data/lib/wukong/widget/filter.rb

@@ -1,81 +0,0 @@
-module Wukong
-  module Widget
-
-    class Filter < Wukong::Processor
-      def process(*args) emit(*args) if select?(*args) ; end
-      def reject?(*args) not select?(*args) ; end
-    end
-
-    class Rejecter < Filter
-      def process(*args) emit(*args) if not reject?(*args) ; end
-      def select?(*args) not reject?(*args) ; end
-      def reject?(*args) true ; end
-    end
-
-    class All < Filter
-      def select?(*args) ; true ; end
-    end
-
-    class None < Rejecter
-      def reject?(*args) ; true ; end
-    end
-
-    # Selects only records matching this regexp
-    class RegexpFilter < Filter
-      field :pattern, Regexp, :doc => 'strings matching this regular expression will be selected'
-      def select?(str)
-        pattern.match(str)
-      end
-
-      def self.make(workflow, pattern, attrs={}, &block)
-        super workflow, attrs.merge(:pattern => pattern), &block
-      end
-      register_processor(:re)
-    end
-
-    class RegexpRejecter < Rejecter
-      field :pattern, Regexp, :doc => 'strings matching this regular expression will be rejected'
-      def reject?(str)
-        pattern.match(str)
-      end
-
-      def self.make(workflow, pattern, attrs={}, &block)
-        super workflow, attrs.merge(:pattern => pattern), &block
-      end
-      register_processor(:not_re)
-    end
-
-    class ProcFilter < Filter
-      # @param [Proc] proc use for body of `reject?` method
-      # @yield ...or supply a block directly
-      def initialize(prc=nil, &block)
-        prc ||= block or raise "Please supply a proc or a block to #{self.class}.new"
-        define_singleton_method(:select?, prc)
-      end
-    end
-
-    class ProcRejecter < Rejecter
-      # @param [Proc] proc use for body of `reject?` method
-      # @yield ...or supply a block directly
-      def initialize(prc=nil, &block)
-        prc ||= block or raise "Please supply a proc or a block to #{self.class}.new"
-        define_singleton_method(:reject?, prc)
-      end
-    end
-
-    class Limit < Rejecter
-      # include CountingProcessor
-      field :max_records, Integer, :doc => 'maximum records to allow', :writer => true
-
-      def reject?(*)
-        count >= max_records
-      end
-
-      def self.make(workflow, max, attrs={}, &block)
-        super workflow, attrs.merge(:max_records => max), &block
-      end
-      register_processor
-    end
-
-  end
-end