wukong 3.0.0.pre2 → 3.0.0.pre3

data/{examples/munging/wikipedia/utils/munging_utils.rb → lib/wu/munging.rb}

@@ -1,7 +1,5 @@
 # encoding:UTF-8
 
-require 'multi_json'
-
 module MungingUtils
   extend self # you can call MungingUtils.foo, or include it and call on self.
 
@@ -37,14 +35,19 @@ module MungingUtils
     text
   end
 
+  # For follow-on escaping of XML-encoded text.
+  #
   # Modifies the text in place, replacing all non-keyboard characters (newline,
   # tab, anything not between ascii 0x20 and 0x7e) with their XML entity encoding
+  #
+  # NOTE: does NOT escape the ampersand character
+  # NOTE: modifies the text in-place
+  #
   def safe_xml_encode(text)
-    text.gsub!(NON_PLAIN_ASCII_RE){|ch|  "\\u%04x" % ch.ord } unless jsonized.ascii_only?
+    text.gsub!(NON_PLAIN_ASCII_RE){|ch|  "&#%04x;" % ch.ord } unless jsonized.ascii_only?
     text
   end
 
-
   # Returns a JSON encoded string, with all non-ASCII characters escaped
   def safe_json_encode(string)
     jsonized = MultiJson.encode(string)

data/lib/wu/social/models/twitter.rb

@@ -0,0 +1,31 @@
+module Wu
+  module Social
+    module Model
+
+      class TwitterUser
+        include Gorillib::Model
+
+        field :user_id,      Integer
+        field :screen_name,  String
+        field :created_at,   Time
+        field :scraped_at,   String
+        field :protected,    Boolean
+        field :followers,    Integer
+        field :friends,      Integer
+        field :tweets_count, Integer
+        field :geo_enabled,  :boolean
+        field :time_zone,    String
+        field :utc_offset,   String
+        field :lang,         String
+        field :location,     String
+        field :url,          String
+
+        def receive_protected(val)
+          self.protected = (val.to_s == 'true' || val.to_s == '1')
+        end
+        def protected? ; !! self.protected  ; end
+      end
+
+    end
+  end
+end

data/lib/wukong.rb

@@ -1,17 +1,19 @@
 require 'configliere'
-require 'gorillib/logger/log'
 require 'vayacondios-client'
 require 'multi_json'
+require 'eventmachine'
+require 'log4r'
 
 require 'hanuman'
 
+require 'wukong/logger' 
 require 'wukong/processor'
-require 'wukong/driver'
-
-require 'wukong/emitter'
 require 'wukong/dataflow'
 require 'wukong/configuration'
 require 'wukong/widgets'
+require 'wukong/driver'
+require 'wukong/server'
+# require 'wukong/runner'
 
 # The Wukong module will contain all code for Wukong's core (like
 # Processors and Dataflows) as well as all plugins.
@@ -29,3 +31,6 @@ module Wukong
   add_shortcut_method_for(:dataflow,  DataflowBuilder)
 
 end
+
+# Alias module name for shorter namespaces
+Wu = Wukong

data/lib/wukong/boot.rb

@@ -58,11 +58,12 @@ module Wukong
     #
     # @return [String]
     def self.deploy_pack_dir
+      return File.dirname(ENV["BUNDLE_GEMFILE"]) if ENV["BUNDLE_GEMFILE"] && is_deploy_pack_dir?(File.dirname(ENV["BUNDLE_GEMFILE"]))
       return @deploy_pack_dir if @deploy_pack_dir
       wd     = Dir.pwd
       parent = File.dirname(wd)
       until wd == parent
-        return wd if File.exist?(File.join(wd, 'Gemfile')) && File.exist?(File.join(wd, 'config', 'environment.rb'))
+        return wd if is_deploy_pack_dir?(wd)
         wd     = parent
         parent = File.dirname(wd)
       end
@@ -82,6 +83,14 @@ module Wukong
     def self.load_environment
       require environment_file
     end
+
+    # Could `dir` be a deploy pack dir?
+    #
+    # @param [String] dir
+    # @return [true, false]
+    def self.is_deploy_pack_dir? dir
+      dir && !dir.empty? && File.directory?(dir) && File.exist?(File.join(dir, 'Gemfile')) && File.exist?(File.join(dir, 'config', 'environment.rb'))
+    end
   end
   
 end

data/lib/wukong/driver.rb

@@ -1,99 +1,93 @@
-module Wukong  
-  class Driver
+module Wukong
+  module DriverMethods
 
-    class << self
-      def run(dataflow, options)
-        new(dataflow, options).run!
-      end
+    def driver
+      @driver ||= Driver.new(dataflow)
     end
 
     def lookup(label)
-      builder = Wukong.registry.retrieve(label)
-      builder.build(@settings)
+      raise Wukong::Error.new("could not find definition for #{label}") unless Wukong.registry.registered?(label.to_sym)
+      Wukong.registry.retrieve(label.to_sym)
     end
     
-    def wire(flow)
-      using = @settings[:wiring] || :emitter
-      if flow.complete?
-        flow.links.each do |link| 
-          Hanuman::LinkFactory.connect(using, flow.stages[link.from], flow.stages[link.into])
-        end
-      end
-      flow
+    def lookup_and_build(label, options = {})
+      lookup(label).build(options)
     end
-  end
-  
-  class LocalDriver < Driver
-    attr_accessor :dataflow, :source
     
-    def initialize(dataflow, options = {})
-      builder    = (Wukong.registry.retrieve(dataflow) or raise Error.new("No such processor or dataflow: #{dataflow.inspect}"))
-      dataflow   = builder.build(options)
-      @dataflow  = dataflow.respond_to?(:stages) ? dataflow.directed_sort.map{ |name| dataflow.stages[name] } : [ dataflow ]
-      @dataflow << Wukong.registry.retrieve(:stdout).build
-      @source    = Wukong.registry.retrieve(:stdin).build
+    def build_serializer(direction, label, options)
+      lookup_and_build("#{direction}_#{label}", options)
     end
 
-    def emitter
-      @emitter ||= ProcEmitter.new(dataflow)
-    end
-
-    def stage_iterator(stage)
-      return dataflow.first if stage.nil?
-      position = dataflow.find_index(stage)
-      dataflow[position + 1]    
+    def add_serialization(dataflow, direction, label, options)
+      case direction
+      when :to   then dataflow.push    build_serializer(direction, label, options)
+      when :from then dataflow.unshift build_serializer(direction, label, options)
+      end
     end
 
-    def next_record(&blk)
-      trap('SIGINT'){ break }              
-      source.process(&blk)
+    def setup_dataflow
+      dataflow.each(&:setup)
     end
 
-    def run!      
-      dataflow.each(&:setup)
-      next_record do |record|
-        emitter.send_through_dataflow(record)
-      end
+    def finalize_and_stop_dataflow
       dataflow.each do |stage|
-        stage.finalize(&emitter.advance(stage)) if stage.respond_to?(:finalize)
+        stage.finalize(&driver.advance(stage)) if stage.respond_to?(:finalize)
         stage.stop
-      end
-      nil
+      end      
     end
+
+    # So pretty...
+    def construct_dataflow(label, options)
+      dataflow = lookup_and_build(label, options)
+      dataflow = dataflow.respond_to?(:stages) ? dataflow.directed_sort.map{ |name| dataflow.stages[name] } : [ dataflow ]
+      expected_input_model  = (options[:consumes].constantize rescue nil)    || dataflow.first.expected_record_type(:consumes)
+      dataflow.unshift lookup_and_build(:recordize, model: expected_input_model)  if expected_input_model
+      expected_output_model = (options[:produces].constantize rescue nil)    || dataflow.first.expected_record_type(:produces)
+      dataflow.push lookup_and_build(:recordize, model: expected_output_model)    if expected_output_model
+      expected_input_serialization  = options[:from] || dataflow.last.expected_serialization(:from)
+      add_serialization(dataflow, :from, expected_input_serialization, options)   if expected_input_serialization
+      expected_output_serialization = options[:to]   || dataflow.last.expected_serialization(:to)
+      add_serialization(dataflow, :to, expected_output_serialization, options)    if expected_output_serialization
+      dataflow.push self
+    end    
   end
-end
 
-class ProcEmitter
+  class Driver
+    attr_accessor :dataflow
 
-  attr_accessor :dataflow
+    def initialize(dataflow)
+      @dataflow = dataflow
+    end
 
-  def initialize(dataflow)
-    @dataflow = dataflow
-  end
+    def to_proc
+      return @wiring if @wiring
+      @wiring = Proc.new do |stage, record|
+        stage.process(record, &advance(stage)) if stage          
+      end
+    end
 
-  def to_proc
-    return @wiring if @wiring
-    @wiring = Proc.new do |stage, record|
-      stage.process(record, &advance(stage)) if stage          
+    def send_through_dataflow(record)
+      start_with(dataflow.first).call(record)
+    end
+    
+    def start_with(stage)
+      to_proc.curry.call(stage)
     end
-  end
 
-  def send_through_dataflow(record)
-    self.call(dataflow.first, record)
-  end
-  
-  def advance(stage)
-    next_stage = stage_iterator(stage)
-    self.to_proc.curry.call(next_stage)
-  end
+    def advance(stage)
+      next_stage = stage_iterator(stage)
+      start_with(next_stage)
+    end
 
-  def stage_iterator(stage)
-    position = dataflow.find_index(stage)
-    dataflow[position + 1]    
-  end 
+    # This should properly be defined on dataflow/builder
+    def stage_iterator(stage)
+      position = dataflow.find_index(stage)
+      dataflow[position + 1]    
+    end 
 
-  def call(*args)
-    to_proc.call(*args)
+    def call(*args)
+      to_proc.call(*args)
+    end
+    
   end
-  
 end

data/lib/wukong/logger.rb

@@ -0,0 +1,93 @@
+module Wukong  
+  class LogFactory
+
+    attr_reader :created_log
+
+    def self.defaults
+      Log4r::StderrOutputter.new('console', formatter: Log4r::PatternFormatter.new(pattern: "%l %d [%-20c] -- %m"))
+    end
+
+    def self.configure(klass, options = {})
+      factory = new(klass, options)
+      factory.created_log
+    end
+
+    def initialize(logger, config)
+      @created_log = logger.is_a?(Log4r::Logger) ? logger : Log4r::Logger.new(logger.to_s)
+      outputter(LogFactory.defaults) unless ancestry_has_outputter?(@created_log)
+      apply_options(config)
+    end
+
+    def ancestry_has_outputter? lgr
+      if lgr.respond_to?(:outputters) && !lgr.outputters.empty?
+        true
+      elsif lgr.respond_to?(:parent) && !lgr.parent.nil?
+        ancestry_has_outputter? lgr.parent
+      else
+        false        
+      end
+    end
+     
+    def apply_options config
+      config.each_pair do |option, value| 
+        begin
+          send(option, value)
+        rescue
+          raise "invalid log option"
+        end
+      end
+    end
+
+    def outputter outptr
+      created_log.outputters = outptr
+    end
+
+    def level lvl
+      created_log.level = lookup_level(lvl.to_sym)
+    end
+
+    def lookup_level lvl
+      { 
+        debug: Log4r::DEBUG,
+        info:  Log4r::INFO,
+        warn:  Log4r::WARN
+      }.fetch(lvl){ raise "invalid log level" }
+    end
+
+    def pattern ptrn
+      created_log.outputters.each do |output|
+        keep_date_format = output.formatter.date_pattern
+        output.formatter = Log4r::PatternFormatter.new(pattern: ptrn, date_format: keep_date_format)
+      end
+    end
+
+    def date_format fmt
+      created_log.outputters.each do |output|
+        keep_pattern = output.formatter.pattern
+        output.formatter = Log4r::PatternFormatter.new(pattern: keep_pattern, date_format: fmt)
+      end
+    end    
+  end
+
+  # Mixin for logging behavior
+  module Logging
+
+    def self.included klass
+      if klass.ancestors.include?(Gorillib::Model)
+        klass.class_eval do
+          field(:log, Whatever, :default => ->{ Wukong::LogFactory.configure(self.class) }) 
+
+          def receive_log params
+            @log = LogFactory.configure(self.class, params)
+          end
+        end
+      else
+        klass.class_attribute :log
+        klass.log = LogFactory.configure(klass)
+      end
+    end    
+  end
+  
+  # Default log. Parent to all logs created in Wukong namespace
+  Log = LogFactory.configure(Wukong)
+end

data/lib/wukong/processor.rb

@@ -1,5 +1,3 @@
-require 'log4r'
-
 module Wukong
   class ProcessorBuilder < Hanuman::StageBuilder
     def namespace(*args)
@@ -14,25 +12,51 @@ module Wukong
   # A Processor can be written and tested purely in Ruby and on your
   # local machine.  You can glue processors together
   class Processor < Hanuman::Stage
+    include Logging
+    include Vayacondios::Notifications
     
     field :action,   Whatever
-    field :log,      Whatever, :default => -> { log = Log4r::Logger.new(self.class.to_s) ; log.outputters = Log4r::StdoutOutputter.new('stdout', formatter: Log4r::PatternFormatter.new(pattern: "%d [%l] %c: %m")) ; log }
-    field :notifier, Vayacondios::NotifierFactory, :default => Vayacondios.default_notifier
 
-    def self.describe desc
-      @description = desc
-    end
+    class << self
 
-    def self.description
-      @description
-    end
+      def describe desc
+        @description = desc
+      end
+      
+      def description
+        @description
+      end
+      
+      def consumes(*args)
+        options   = args.extract_options!
+        @consumes = options[:as]
+        validate_and_set_serialization(:from, args.first)
+      end
 
-    def self.consumes label
-    end
+      def produces(*args)
+        options   = args.extract_options!
+        @produces = options[:as]
+        validate_and_set_serialization(:to, args.first)      
+      end
+            
+      def valid_serializer? label
+        label
+      end
 
-    def self.produces label
-    end
+      def validate_and_set_serialization(direction, label)
+        instance_variable_set("@serialization_#{direction}", label) if %w[ tsv json xml ].include?(label.to_s)
+      end
 
+    end
+        
+    def expected_record_type(type)
+      self.class.instance_variable_get("@#{type}")
+    end
+    
+    def expected_serialization(direction)
+      self.class.instance_variable_get("@serialization_#{direction.to_s}")
+    end
+    
     # This is a placeholder method intended to be overridden
     def perform_action(*args) ; end 
     
@@ -41,21 +65,6 @@ module Wukong
       self.define_singleton_method(:perform_action, &action)
     end
 
-    # Valid notifier types are currently :http or :log
-    # This processor's log is passed to vayacondios
-    def receive_notifier(type)
-      if type.is_a?(Hash)
-        @notifier = Vayacondios::NotifierFactory.receive({type: 'log'}.merge(type))
-      else
-        @notifier = Vayacondios::NotifierFactory.receive(type: type, log: log)
-      end
-    end
-
-    # Send information to Vayacondios; data goes in, the right thing happens
-    def notify(topic, cargo)
-      notifier.notify(topic, cargo)
-    end
-
     # This method is called after the processor class has been instantiated
     # but before any records are given to it to process
     def setup