wukong 1.5.4 → 2.0.0

data/lib/wukong/streamer/list_reducer.rb

@@ -8,13 +8,13 @@ module Wukong
 
       # start with an empty list
       def start! *args
-        self.values = []
+        @values = []
       end
 
       # aggregate all records.
       # note that this accumulates the full *record* -- key, value, everything.
       def accumulate *record
-        self.values << record
+        @values << record
       end
 
       # emit the key and all records, tab-separated
@@ -24,7 +24,7 @@ module Wukong
       # values)
       #
       def finalize
-        yield [key, values.to_flat.join(";")].flatten
+        yield [key, @values.to_flat.join(";")].flatten
       end
     end
   end

data/lib/wukong/streamer/reducer.rb

@@ -0,0 +1,11 @@
+ module Wukong
+   module Streamer
+     class Reducer < Wukong::Streamer::ListReducer
+
+       def finalize &block
+         reduce @values, &block
+       end
+     end
+
+   end
+end

data/lib/wukong/streamer.rb

@@ -5,14 +5,18 @@ module Wukong
     autoload :RecordStreamer,          'wukong/streamer/record_streamer'
     autoload :StructStreamer,          'wukong/streamer/struct_streamer'
     autoload :StructRecordizer,        'wukong/streamer/struct_streamer'
-    # cassandra goodies
-    autoload :CassandraStreamer,       'wukong/streamer/cassandra_streamer'
     #
     autoload :Filter,                  'wukong/streamer/filter'
     #
+    autoload :Reducer,                 'wukong/streamer/reducer'
     autoload :AccumulatingReducer,     'wukong/streamer/accumulating_reducer'
+    autoload :CountingReducer,         'wukong/streamer/counting_reducer'
     autoload :ListReducer,             'wukong/streamer/list_reducer'
+    autoload :RankAndBinReducer,       'wukong/streamer/rank_and_bin_reducer'
     autoload :UniqByLastReducer,       'wukong/streamer/uniq_by_last_reducer'
-    autoload :CountingReducer,         'wukong/streamer/counting_reducer'
+
+    class Streamer < Base
+    end
+
   end
 end

data/lib/wukong.rb

@@ -1,13 +1,17 @@
+require 'configliere'; Settings.use :define
 require 'wukong/extensions'
 require 'wukong/datatypes'
+require 'wukong/periodic_monitor'
 require 'wukong/logger'
-require 'wukong/bad_record'
+autoload :BadRecord,   'wukong/bad_record'
 autoload :TypedStruct, 'wukong/typed_struct'
-require 'configliere'; Configliere.use :define
 module Wukong
-  autoload :Dfs,             'wukong/dfs'
   autoload :Script,          'wukong/script'
   autoload :Streamer,        'wukong/streamer'
   autoload :Store,           'wukong/store'
   autoload :FilenamePattern, 'wukong/filename_pattern'
+
+  def self.run mapper, reducer=nil, options={}
+    Wukong::Script.new(mapper, reducer, options).run
+  end
 end