wukong 3.0.0.pre → 3.0.0.pre2

data/Rakefile

@@ -1,12 +1,28 @@
-require 'bundler' ; Bundler::GemHelper.install_tasks
+require 'bundler'
+Bundler::GemHelper.install_tasks
 
 require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:specs)
+
 require 'yard'
+YARD::Rake::YardocTask.new
 
-RSpec::Core::RakeTask.new(:specs) do |spec|
-  spec.pattern = 'spec/**/*_spec.rb'
+desc 'Run RSpec with code coverage'
+task :cov do
+  ENV['WUKONG_COV'] = true
+  Rake::Task[:specs].execute
 end
 
-YARD::Rake::YardocTask.new
-
 task :default => :specs
+
+desc "Create a TAGS file for this project"
+task :tags do
+  files = [%w[Gemfile Guardfile Rakefile README.md].map { |b| File.join(File.dirname(__FILE__), b) }]
+  %w[bin examples lib spec].each do |dir|
+    files << Dir[File.join(File.dirname(__FILE__), "#{dir}/**/*.rb")]
+  end
+  files.each do |arry|
+    sh "etags", '-a', *arry unless arry.empty?
+  end
+end
+

data/TODO.md

@@ -4,21 +4,21 @@
   - hooks up source to flow,
   - if iterated source, drives it, otherwise sits in the flow
 
-* these set the contract for the inbound resources
+* these set the contract for the inbound products
 
 ### slots
 
 Typical case: one input, `:input`, one output `:output`
 
-* there are as many resources as
+* there are as many products as
   - the total number of action stage outputs
-  - the concrete input resources
+  - the concrete input products
 * The number of rsrc->action edges is at most the total number of input slots
-  - (you cannot wire multiple resources to the same input slot)
+  - (you cannot wire multiple products to the same input slot)
 
 
 1. action stage B wires up to an action stage A (which really means "the full set of A's outputs")
-2. I wire action A's output as production resource X
+2. I wire action A's output as production product X
 3. 
 
 4. How do I address other stages?
@@ -71,7 +71,7 @@ __________________________________________________________________________
         |  foo   |
         ----------
 
-create a resource with no action? action with anonymous resource, wired up later?
+create a product with no action? action with anonymous product, wired up later?
 
 
 * connections:

data/bin/wu-clean-encoding

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby
+# encoding:UTF-8
+
+if ARGV.include?('--help')
+  puts <<USAGE
+wu-clean-encoding cleans malformed characters from stdin.
+
+If a character is malformed, as defined by valid_encoding?,
+it is replaced with a '�'.
+
+wu-clean-encoding was built to work with UTF-8, and no 
+guarantees are provided for other encodings.
+USAGE
+  exit(0)
+end
+
+ARGF.each do |line|
+  if line.valid_encoding?
+    $stdout.write line
+  else 
+    repaired_line = []
+    line.each_char do |char|
+      if char.valid_encoding?
+        repaired_line << char
+      else
+        repaired_line << "�"
+      end
+    end
+    $stdout.write repaired_line.join
+  end
+end

data/bin/wu-lign

@@ -161,8 +161,8 @@ format = maxw.zip(col_types, col_minmag, col_maxmag, ARGV).map do |width, type,
   case type
   when :mixed, nil then lambda{|s| "%-#{width}s" % s }
   when :str        then lambda{|s| "%-#{width}s" % s }
-  when :int        then lambda{|s| "%#{width}d"  % s.gsub(/\D+/, "").to_i }
-  when :float      then lambda{|s| "%#{maxmag+minmag+2}.#{minmag}f" % s.to_f }
+  when :int        then lambda{|s| "%#{width}d"  % s.gsub(/[^\d\-\+]+/, "").to_i }
+  when :float      then lambda{|s| "%#{maxmag+minmag+2}.#{minmag}f" % s.gsub(/[^\d\.eE\-\+]+/, "").to_f }
   else raise "oops type #{type}"  end
 end
 

data/bin/wu-local

@@ -0,0 +1,69 @@
+#!/usr/bin/env ruby
+
+require 'wukong'
+
+settings = Wukong::Local::Configuration
+settings.use(:commandline)
+
+def settings.usage
+  "usage: #{File.basename($0)} PROCESSOR|FLOW [ --param=value | -p value | --param | -p]"
+end
+
+settings.description = <<-EOF
+wu-local is a tool for running Wukong processors and flows locally on
+the command-line.  Use wu-local by passing it a processor and feeding
+in some data:
+
+  $ echo 'UNIX is Clever and Fun...' | wu-local tokenizer.rb
+  UNIX
+  is
+  Clever
+  and
+  Fun
+
+If your processors have named fields you can pass them in as
+arguments:
+
+  $ echo 'UNIX is clever and fun...' | wu-local tokenizer.rb --min_length=4
+  UNIX
+  Clever
+
+You can chain processors and calls to wu-local together:
+
+  $ echo 'UNIX is clever and fun...' | wu-local tokenizer.rb --min_length=4 | wu-local downcaser.rb
+  unix
+  clever
+
+Which is a good way to develop a combined data flow which you can
+again test locally:
+
+  $ echo 'UNIX is clever and fun...' | wu-local tokenize_and_downcase_big_words.rb
+  unix
+  clever
+EOF
+
+settings.define :run, description: "Name of the processor or dataflow to use.  Defaults to basename of the given path.", flag: 'r'
+
+require 'wukong/boot' ; Wukong.boot!(settings)
+
+thing = settings.rest.first
+case
+when thing.nil?
+  settings.dump_help
+  exit(1)
+when Wukong.registry.registered?(thing.to_sym)
+  processor = thing.to_sym
+when File.exist?(thing)
+  load thing
+  processor = settings.run || File.basename(thing, '.rb')
+else
+  settings.dump_help
+  exit(2)
+end
+# p settings
+begin
+  Wukong::LocalDriver.run(processor.to_sym, settings)
+rescue Wukong::Error => e
+  $stderr.puts e.message
+  exit(3)
+end

data/bin/wu-server

@@ -0,0 +1,70 @@
+#!/usr/bin/env ruby
+require 'configliere'
+require 'wukong'
+require 'eventmachine'
+require 'em-synchrony'
+require 'multi_json'
+
+Settings({
+  host: "localhost",
+  port: 9500
+})
+
+Settings.use :commandline
+Settings.resolve!
+
+# Load the file on the command line
+wu_file = ARGV.shift
+load wu_file
+$processor = File.basename(wu_file, '.rb').to_sym
+
+class Wukong::Server
+  def prepare(options = {})
+    dataflow_class = Wukong.dataflow(:server){ send(options[:processor]) }
+    flow_builder = Wukong.registry.retrieve(:server)
+    flow = flow_builder.build(processor: $processor)
+    
+    @buffer = []
+    @processor = flow.stages.values.first
+    @processor.emitter = ->(value){ @buffer << value }
+  end
+  
+  def process(record)
+    @process.process(record)
+  end
+  
+  def cleanup
+  end
+end
+
+
+# EventMachine server
+
+class WukongMachine < EM::Protocols::LineAndTextProtocol
+  def post_init
+    puts "[server] Client connected"
+    @wukong = WukongInterface.new
+    @wukong.prepare(processor: $processor)
+  end
+
+  def receive_data(data)
+    @buffer = []
+    input = MultiJson.load data
+    
+    op = proc { @wukong.process(input) }
+    callback = proc { send_data MultiJson.dump(@buffer) + "\n" }
+    EM.defer(op, callback)
+      
+  rescue MultiJson::DecodeError => ex
+    STDERR.puts "[server] Dropped: Malformed request"
+  end
+
+  def unbind
+    puts "[server] Client disconnected."
+  end
+end
+
+EM::run {
+  EM::start_server(Settings[:host], Settings[:port], WukongMachine)
+  puts "Listening on #{Settings[:host]}:#{Settings[:port]}"
+}

data/examples/Gemfile

@@ -0,0 +1,38 @@
+source :rubygems
+
+gem "configliere",        '~> 0.4'
+gem "multi_json",         '>= 1.3.6'
+gem "vayacondios-client", '>= 0.0.3'
+gem "gorillib",           '>= 0.4.2'
+gem "uuidtools"
+gem "eventmachine"
+gem "log4r"
+
+group :examples do
+  gem "forgery"
+  gem "nokogiri"
+  # gem "sanitize"
+  gem "addressable"
+  gem "forgery"
+  gem "crack"
+  gem "oj"
+  gem "activesupport"
+end
+
+group :development do
+  gem "bundler",            '~> 1.1'
+  gem "rake",               '>= 0.9'
+  gem "rspec",              '>= 2.8'
+  gem "guard",              '>= 1.0'
+  gem "guard-rspec",        '>= 0.6'
+  gem "simplecov",          '>= 0.5'
+  gem "pry"
+  gem "ap"
+end
+
+group :docs do
+  gem "yard"
+  gem "redcarpet"
+  gem "addressable"
+  gem "htmlentities"
+end

data/examples/README.md

@@ -0,0 +1,9 @@
+
+
+### Geographic data 
+
+
+
+### Munging
+
+

data/examples/dataflow/apache_log_line.rb

@@ -1,23 +1,30 @@
+#
+# Parses logs in either the [Apache Common Log Format](http://en.wikipedia.org/wiki/Common_Log_Format)
+# or [Apache Combined Log Format](http://httpd.apache.org/docs/2.2/logs.html#combined)
+#
+# Common:   `%h %l %u %t "%r" %>s %b`
+# Combined: `%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"`
+#
 class ApacheLogLine
   include Gorillib::Model
 
-  field  :ip_address,    IpAddress
-  field  :junk_1,        String
-  field  :junk_2,        String
+  field  :client,        Hostname
+  field  :rfc_1413,      String
+  field  :userid,        String
   field  :log_timestamp, Time
   field  :http_method,   String
-  field  :path,          String
+  field  :rsrc,          String
   field  :protocol,      String
   field  :response_code, Integer
   field  :size,          Integer
-  field  :referer,       String
-  field  :user_agent,    String
+  # field  :referer,       String
+  # field  :user_agent,    String
 
   def page_type
     case
-    when path =~ /\.(css|js)$/                  then :asset
-    when path =~ /\.(png|gif|ico)$/             then :image
-    when path =~ /\.(pl|s?html?|asp|jsp|cgi)$/  then :page
+    when rsrc =~ /\.(css|js)$/                  then :asset
+    when rsrc =~ /\.(png|gif|ico)$/             then :image
+    when rsrc =~ /\.(pl|s?html?|asp|jsp|cgi)$/  then :page
     else                                             :other
     end
   end
@@ -25,37 +32,69 @@ class ApacheLogLine
   #
   # Regular expression to parse an apache log line.
   #
+  # local - - [24/Oct/1994:13:43:13 -0600] "GET index.html HTTP/1.0" 200 3185
   # 83.240.154.3 - - [07/Jun/2008:20:37:11 +0000] "GET /faq HTTP/1.1" 200 569 "http://infochimps.org/search?query=CAC" "Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.9.0.16) Gecko/2009120208 Firefox/3.0.16"
+  # whidbey.whidbey.com    - - [04/Sep/1995:00:30:18 -0400] "GET /pub/sshay/images/btthumb.jpg"      200 4624
+  # jgbustam-ppp.clark.net - - [04/Sep/1995:00:00:28 -0400] "GET /pub/jgbustam/famosos/alpha.html HTTP/1.0" 304 -
   #
-  LOG_RE = Regexp.compile(%r{\A
-           (\S+)                        # ip                  83.240.154.3
-         \s(\S+)                        # j1                  -
-         \s(\S+)                        # j2                  -
-       \s\[([\w\:\+\ \/]+)\]            # date part           [07/Jun/2008:20:37:11 +0000]
+  COMMON_LOG_RE = Regexp.compile(%r{\A
+           (\S+)                        # client              83.240.154.3
+         \s(\S+)                        # rfc_1413            -
+         \s(\S+)                        # userid              -
+      \s\[([\w\:\+\-\ \/]+)\]           # date part           [07/Jun/2008:20:37:11 +0000]
     \s\"(?:(\S+)                        # http_method         "GET
-         \s(\S+)                        # path                /faq
-         \s(\S+)|-)"                    # protocol            HTTP/1.1"
-         \s(\d+)                        # response_code       200
-         \s(\d+)                        # size                569
-       \s\"([^\"]*)\"                   # referer             "http://infochimps.org/search?query=CAC"
-       \s\"([^\"]*)\"                   # ua                  "Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.9.0.16) Gecko/2009120208 Firefox/3.0.16"
-      \z}x)
+         \s(.+?)                        # rsrc                /faq
+      (?:\s(HTTP/\d+\.\d+))?|-)\"       # protocol            HTTP/1.1"
+         \s(\d+|-)                      # response_code       200
+         \s(\d+|-)                      # size                569
+      \z
+      }x)
 
-  MONTHS = { 'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06', 'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12', }
+  COMBINED_LOG_RE = Regexp.compile(%r{\A
+           (\S+)                        # client              83.240.154.3
+         \s(\S+)                        # rfc_1413            -
+         \s(\S+)                        # userid              -
+      \s\[([\w\:\+\-\ \/]+)\]           # date part           [07/Jun/2008:20:37:11 +0000]
+    \s\"(?:(\S+)                        # http_method         "GET
+         \s([^\"]+?)                    # rsrc                /faq
+      (?:\s(HTTP/\d+\.\d+))?|-)\"       # protocol            HTTP/1.1"
+         \s(\d+|-)                      # response_code       200
+         \s(\d+|-)                      # size                569
+    (?:\s\"([^\"]*)\")                  # referer             "http://infochimps.org/search?query=CAC"
+    (?:\s\"([^\"]*)\")                  # ua                  "Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.9.0.16) Gecko/2009120208 Firefox/3.0.16"
+      \z
+      }x)
+
+  # LOG_RE = Regexp.compile(%r{\A(\S+)\s})
+
+  MONTHS = { 'Jan' => 1, 'Feb' => 2, 'Mar' => 3, 'Apr' => 4, 'May' => 5, 'Jun' => 6, 'Jul' => 7, 'Aug' => 8, 'Sep' => 9, 'Oct' => 10, 'Nov' => 11, 'Dec' => 12, }
 
   # Converts a time like `10/Apr/2007:10:58:27 +0300` to something parseable
   def receive_log_timestamp(raw_ts)
+    return super(nil) if raw_ts.nil?
     match = %r{(\d+)/(\w+)/(\d+):(\d+):(\d+):(\d+)\s([\+\-\w]+)}.match(raw_ts)
-    warn "Can't parse date #{raw_ts}" unless match
+    unless match then warn "Can't parse date #{raw_ts}" ; return super(nil) ; end
+    #
     day, month_name, year, hour, min, sec, tz = match.captures
     month = MONTHS[month_name]
-    super "#{year}-#{month}-#{day} #{hour}:#{min}:#{sec} #{tz}"
+    tz.insert(3, ':')  # -0600 to -06:00
+    #
+    # super "#{year}-#{month}-#{day} #{hour}:#{min}:#{sec} #{tz}"
+    super Time.new(year.to_i, month, day.to_i, hour.to_i, min.to_i, sec.to_i, tz)
+  end
+
+  # @returns the log_timestamp in the common log format
+  def unparsed_log_timestamp
+    return if log_timestamp.blank?
+    log_timestamp.strftime("%d/%b/%Y:%H:%M:%S %z")
   end
 
   # Use the regex to break line into fields
   # Emit each record as flat line
   def self.make(line)
-    m = LOG_RE.match(line.chomp) or return
+    m = COMMON_LOG_RE.match(line) or return
     from_tuple *m.captures
+  rescue ArgumentError => err
+    raise unless err.message =~ /invalid byte sequence in UTF-8/
   end
 end

data/examples/dataflow/fibonacci_series.rb

@@ -0,0 +1,101 @@
+require 'wukong/widget/many_to_many'
+require 'gorillib/enumerable/sum'
+
+#
+# An example dataflow --
+#
+
+Wukong.processor(:delay_buffer) do
+  attr_accessor :queue
+  field :delay, Integer, position: 0, doc: "number of records to hold in buffer"
+
+  def process(rec)
+    queue << rec
+    emit(next_item) if ready?
+  end
+
+  def next_item
+    queue.shift
+  end
+
+  # true if there are records at the end of the delay stage
+  def ready?
+    warn "Hmm, too many records in queue: #{queue}" if queue.size > delay+1
+    queue.size > delay
+  end
+
+  # resets to an empty state
+  def setup(*)
+    super
+    @queue = Array.new
+  end
+
+  # emits all remaining elements of the queue
+  def stop
+    queue.each{|rec| emit(rec) }
+    super
+  end
+end
+
+class Wukong::Batcher < Wukong::Processor
+  register_action
+  include Hanuman::Slottable
+  include Hanuman::OutputSlotted
+
+  attr_accessor :queues
+  consume :n_1,    Integer, doc: "n-1'th value: the one just emitted"
+  consume :tictoc, Integer, doc: "input to drive flow"
+  consume :n_2,    Integer, doc: "n-2'nd value: the one before the one just emitted"
+
+  # resets to an empty state, calls super
+  def initialize(*)
+    super
+    @queues = Hash.new{|h,k| h[k] = Array.new } # autovivifying
+  end
+
+  def process_input(channel, rec)
+    queues[channel] << rec
+    emit(next_item) if ready?
+  end
+
+  def next_item
+    queues.map{|_, queue| queue.shift }
+  end
+
+  # true if there is at least one record in each queue
+  def ready?
+    inslots.values.all?{|inslot| queues[inslot.name].length > 0 }
+  end
+end
+
+Wukong.chain(:fibonacci_series) do
+
+  delay_buffer(1, name: :my_delay)
+
+  # * I don't want to have to name everything
+  #   - are few/some/most things named?
+  # * I must be able to have the same stage type on a graph more than once
+  # * If naming things is a general case, I want it to
+  #   - be clean, and for it to
+  #   - not cause a ruckus when stage type has its own args
+  #
+
+  batcher(name: :feedback) >
+    map(name: :summer, &:sum) >
+    many_to_many(name: :fibonacci_n)
+
+  spew(6, item: 0, name: :ticker) > feedback.tictoc
+
+  fibonacci_n          > feedback.n_1
+  fibonacci_n          > output
+  fibonacci_n > :delay > feedback.n_2
+
+  # preload the feedback buffer
+  feedback.n_1.process(0)
+  feedback.n_2.process(0)
+  feedback.n_2.process(1)
+end
+
+# Wukong.dataflow(:dump) do
+#   stdout << Wukong.dataflow(:fibbonaci_series).out
+# end