wukong 3.0.0.pre → 3.0.0.pre2

data/old/wukong/store/key_store.rb

@@ -1,51 +0,0 @@
-module Monkeyshines
-  module Store
-    class KeyStore < Monkeyshines::Store::Base
-      # The actual backing store; should respond to #set and #get methods
-      attr_accessor :db
-
-      #
-      # Executes block once for each element in the whole DB, in whatever order
-      # the DB thinks you should see it.
-      #
-      # Your block will see |key, val|
-      #
-      # key_store.each do |key, val|
-      #   # ... stuff ...
-      # end
-      #
-      def each &block
-        db.iterinit
-        loop do
-          key = db.iternext or break
-          val = db[key]
-          yield key, val
-        end
-      end
-
-
-      # Save the value into the database
-      def set(key, val)
-        return unless val
-        db[key] = val
-      end
-
-      alias_method :save, :set
-      def get(key)      db[key]  end
-      def [](key)       db[key]  end
-      def close()       db.close end
-      def size()        db.size  end
-
-      #
-      # Load from standard command-line options
-      #
-      # obvs only works when there's just one store
-      #
-      def self.new_from_command_line cmdline_opts, default_opts={}
-        options = default_opts.merge(cmdline_opts)
-        store = self.new(options[:store_db])
-        store
-      end
-    end
-  end
-end

data/old/wukong/store/null_store.rb

@@ -1,15 +0,0 @@
-module Monkeyshines
-  module Store
-    class NullStore < Monkeyshines::Store::Base
-
-      def each *args, &block
-      end
-
-
-      # Does nothing!
-      def set *args
-      end
-
-    end
-  end
-end

data/old/wukong/store/read_thru_store.rb

@@ -1,22 +0,0 @@
-module Monkeyshines
-  module Store
-    class ReadThruStore < Monkeyshines::Store::TyrantTdbKeyStore
-
-      #
-      # If key is absent, save the result of calling the block.
-      # If key is present, block is never called.
-      #
-      # Ex:
-      #   rt_store.set(url) do
-      #     fetcher.get url # will only be called if url isn't in rt_store
-      #   end
-      #
-      def set key, force=nil, &block
-        return if !force && db.has_key?(key)
-        result = block.call() or return
-        super(key, result)
-      end
-
-    end
-  end
-end

data/old/wukong/store/tokyo_tdb_key_store.rb

@@ -1,33 +0,0 @@
-require 'tokyocabinet'
-module Monkeyshines
-  module Store
-    #
-    # Implementation of KeyStore with a Local TokyoCabinet table database (TDB)
-    #
-    class TokyoTdbKeyStore < Monkeyshines::Store::KeyStore
-
-      # pass in the filename or URI of a tokyo cabinet table-style DB
-      # set create_db = true if you want to create a missing DB file
-      def initialize db_uri, *args
-        self.db = TokyoCabinet::TDB.new
-        db.open(db_uri, TokyoCabinet::TDB::OWRITER) or raise "#{self.class.to_s}: Can't open TokyoCabinet TDB #{db_uri}"
-        super *args
-      end
-
-
-      def each_as klass, &block
-        self.each do |key, hsh|
-          yield klass.from_hash hsh
-        end
-      end
-      # Delegate to store
-      def set(key, val)
-        return unless val
-        db.put key, val.to_hash.compact
-      end
-
-      def size()        db.rnum  end
-
-    end #class
-  end
-end

data/old/wukong/store/tyrant_rdb_key_store.rb

@@ -1,57 +0,0 @@
-require 'tokyotyrant'
-module Monkeyshines
-  module Store
-
-    #
-    # Implementation of KeyStore with a Local TokyoCabinet hash database (RDB)
-    #
-    class TyrantRdbKeyStore < Monkeyshines::Store::KeyStore
-      attr_accessor :db_host, :db_port
-
-      # pass in the host:port uri of the key store.
-      def initialize options
-        raise "URI for #{self.class} is required" if options[:uri].blank?
-        self.db_host, self.db_port = options[:uri].to_s.split(':')
-        self.db_host.gsub!(/^(localhost|127\.0\.0\.1)$/,'')
-        super options
-      end
-
-      def db
-        return @db if @db
-        @db ||= TokyoTyrant::RDB.new
-        @db.open(db_host, db_port) or raise("Can't open DB at host #{db_host} port #{db_port}. Pass in host:port' #{@db.ecode}: #{@db.errmsg(@db.ecode)}")
-        @db
-      end
-
-      def close
-        @db.close if @db
-        @db = nil
-      end
-
-      # Save the value into the database without waiting for a response.
-      def set_nr(key, val)
-        db.putnr key, val if val
-      end
-
-      def size()        db.rnum  end
-      def include? *args
-        db.has_key? *args
-      end
-
-      # require 'memcache'
-      # def initialize db_uri=nil, *args
-      #   # db_uri ||= ':1978'
-      #   # self.db_host, self.db_port = db_uri.split(':')
-      #   self.db = MemCache.new(db_uri, :no_reply => true)
-      #   if !self.db then raise("Can't open DB #{db_uri}. Pass in host:port, default is ':1978' #{db.ecode}: #{db.errmsg(db.ecode)}") end
-      #   super *args
-      # end
-      #
-      # def size
-      #   db.stats
-      # end
-
-    end #class
-  end
-end
-

data/old/wukong/store/tyrant_tdb_key_store.rb

@@ -1,20 +0,0 @@
-require 'tokyotyrant'
-require 'tyrant_rdb_key_store'
-module Monkeyshines
-  module Store
-    #
-    # Implementation of KeyStore with a Local TokyoCabinet Table database (RDBTBL)
-    #
-    class TyrantRdbKeyStore < TyrantRdbKeyStore Monkeyshines::Store::KeyStore
-
-      def db
-        return @db if @db
-        @db ||= TokyoTyrant::RDBTBL.new
-        @db.open(db_host, db_port) or raise("Can't open DB #{db_host}:#{db_port}. Pass in host:port' #{@db.ecode}: #{@db.errmsg(@db.ecode)}")
-        @db
-      end
-
-    end #class
-  end
-end
-

data/old/wukong/streamer.rb

@@ -1,30 +0,0 @@
-module Wukong
-  module Streamer
-    autoload :Base,                    'wukong/streamer/base'
-    autoload :LineStreamer,            'wukong/streamer/line_streamer'
-    autoload :RecordStreamer,          'wukong/streamer/record_streamer'
-    autoload :JsonStreamer,            'wukong/streamer/json_streamer'
-    autoload :StructStreamer,          'wukong/streamer/struct_streamer'
-    autoload :StructRecordizer,        'wukong/streamer/struct_streamer'
-    autoload :InstanceStreamer,        'wukong/streamer/instance_streamer'
-    #
-    autoload :Filter,                  'wukong/streamer/filter'
-    #
-    autoload :Reducer,                 'wukong/streamer/reducer'
-    autoload :AccumulatingReducer,     'wukong/streamer/accumulating_reducer'
-    autoload :CountingReducer,         'wukong/streamer/counting_reducer'
-    autoload :ListReducer,             'wukong/streamer/list_reducer'
-    autoload :RankAndBinReducer,       'wukong/streamer/rank_and_bin_reducer'
-    autoload :UniqByLastReducer,       'wukong/streamer/uniq_by_last_reducer'
-
-    class Streamer < Base
-    end
-
-    class IdentityMapper < Base
-    end
-
-    class IdentityReducer < Base
-    end
-
-  end
-end

data/old/wukong/streamer/accumulating_reducer.rb

@@ -1,83 +0,0 @@
- module Wukong
-   module Streamer
-
-     #
-     # AccumulatingReducer makes it easy to apply one operation across all
-     # occurrences of each key
-     #
-     # On each occurrence of a given key, AccumulatingReducer calls
-     # accumulate, and at the final occurrence calls finalize.
-     #
-     # See ListAccumulatingReducer and KeyCountingReducer for examples
-     #
-     # Make sure you don't have the bad luck, bad judgement or bad approach to
-     # accumulate more data than your box can hold before finalizing.
-     #
-     class AccumulatingReducer < Wukong::Streamer::Base
-       attr_accessor :key
-
-       #
-       # override for multiple-field keys, etc.
-       #
-       # Note that get_key is called by +process+ -- so the arguments have
-       # already been +recordize+d. In particular, if you are using
-       # StructRecordizer (or StructStreamer), you can write this as
-       #
-       #   def get_key(thing) thing.id.to_i ; end
-       #
-       # or whatever
-       def get_key *record
-         record.first
-       end
-
-       #
-       # Accumulate all records for a given key.
-       #
-       # When the last record for the key is seen, finalize processing and adopt the
-       # new key.
-       #
-       def process *args, &block
-         this_key = get_key(*args)
-         if this_key != self.key      # if this is a new key,
-           unless self.key == :__first_pass__
-             finalize(&block)         # process what we've collected so far
-           end
-           self.key = this_key        # adopt the new key
-           start! *args               # and set up for the next accumulation
-         end
-         # collect the current record
-         accumulate *args, &block
-       end
-
-       #
-       # start! is called on the the first record of the new key
-       #
-       def start! *args
-       end
-
-       #
-       # Override this to accumulate each record for the given key in turn.
-       #
-       def accumulate *args, &block
-       end
-
-       #
-       #
-       # You must override this method.
-       #
-       def finalize
-       end
-
-       # make a sentinel
-       def before_stream
-         self.key = :__first_pass__
-       end
-
-       # Finalize the last-seen group.
-       def after_stream *args
-         finalize(){|record| emit record } unless (self.key == :__first_pass__)
-         super *args
-       end
-     end
-   end
-end

data/old/wukong/streamer/base.rb

@@ -1,126 +0,0 @@
-module Wukong
-  module Streamer
-    class Base
-
-      # Options, initially set from the command-line args -- see
-      # Script#process_argv!
-      attr_reader :own_options
-
-      #
-      # Accepts option hash from script runner
-      #
-      def initialize options={}
-        @own_options = options
-      end
-
-      def options
-        Settings.deep_merge own_options
-      end
-
-      #
-      # Pass each record to +#process+
-      #
-      def stream
-        Log.info("Streaming on:\t%s" % [Script.input_file]) unless Script.input_file.blank?
-        before_stream
-        each_record do |line|
-          record = recordize(line.chomp) or next
-          process(*record) do |output_record|
-            emit output_record
-          end
-          track(record)
-        end
-        after_stream
-      end
-
-      def track record
-        monitor.periodically(record.to_s[0..1000])
-      end
-
-      def each_record &block
-        $stdin.each(&block)
-      end
-
-      # Called exactly once, before streaming begins
-      def before_stream
-      end
-
-      # Called exactly once, after streaming completes
-      def after_stream
-      end
-
-      #
-      # Default recordizer: returns array of fields by splitting at tabs
-      #
-      def recordize line
-        line.split("\t") rescue nil
-      end
-
-      #
-      # Serializes the record to output.
-      #
-      # Emits a single line of tab-separated fields created by calling #to_flat
-      # on the record and joining with "\t".
-      #
-      # Does no escaping or processing of the record -- that's to_flat's job, or
-      # yours if you override this method.
-      #
-      def emit record
-        puts record.to_flat.join("\t")
-      end
-
-      #
-      # Process each record in turn, yielding the records to emit
-      #
-      def process *args, &block
-        yield(args)
-      end
-
-      #
-      # To track processing errors inline,
-      # pass the line back to bad_record!
-      #
-      def bad_record! key, *args
-        warn "Bad record #{args.inspect[0..400]}"
-        puts ["bad_record-"+key.to_s, *args].join("\t")
-      end
-
-      # A periodic logger to track progress
-      def monitor
-        @monitor ||= PeriodicMonitor.new
-      end
-
-      # Defines a process method on the fly to execute the given mapper.
-      #
-      # This is still experimental.
-      # Among other limitations, you can't use ++yield++ -- you have to call
-      # emit() directly.
-      def mapper &mapper_block
-        @mapper_block = mapper_block.to_proc
-        self.instance_eval do
-          def process *args, &block
-            instance_exec(*args, &@mapper_block)
-          end
-        end
-        self
-      end
-
-      # Creates a new object of this class and injects the given block
-      # as the process method
-      def self.mapper *args, &block
-        self.new.mapper(*args, &block)
-      end
-
-      # Delegates back to Wukong to run this instance as a mapper
-      def run options={}
-        Wukong.run(self, nil, options)
-      end
-
-      # Creates a new object of this class and runs it
-      def self.run options={}
-        Wukong.run(self.new, nil, options)
-      end
-
-    end
-  end
-end