wukong 1.5.4 → 2.0.0

data/lib/wukong/monitor/chunked_store.rb

@@ -1,23 +0,0 @@
-require 'monkeyshines/monitor/periodic_monitor'
-module Monkeyshines
-  module Monitor
-    module ChunkedStore
-      attr_accessor :file_pattern
-      def initialize file_pattern
-        self.file_pattern = file_pattern
-        super file_pattern.make
-      end
-
-      def close_and_reopen
-        close
-        self.filename = file_pattern.make
-        dump_file
-      end
-
-      def save *args
-        chunk_monitor.periodically{ close_rename_and_open }
-        super *args
-      end
-    end
-  end
-end

data/lib/wukong/monitor/periodic_logger.rb

@@ -1,34 +0,0 @@
-module Monkeyshines
-  module Monitor
-
-    #
-    # Emits a log line but only every +iter_interval+ calls or +time_interval+
-    # lapse.
-    #
-    # Since the contents of the block aren't called until the criteria are met,
-    # you can put relatively expensive operations in the log without killing
-    # your iteration time.
-    #
-    class PeriodicLogger < PeriodicMonitor
-      #
-      # Call with a block that returns a string or array to log.
-      # If you return
-      #
-      # Ex: log if it has been at least 5 minutes since last announcement:
-      #
-      #   periodic_logger = Monkeyshines::Monitor::PeriodicLogger.new(:time => 300)
-      #   loop do
-      #     # ... stuff ...
-      #     periodic_logger.periodically{ [morbenfactor, crunkosity, exuberance] }
-      #   end
-      #
-      def periodically &block
-        super do
-          now = Time.now.utc.to_f
-          result = [ "%10d"%iter, "%7.1f"%since, "%7.1f"%inst_rate(now), (block ? block.call : nil) ].flatten.compact
-          Log.info result.join("\t")
-        end
-      end
-    end
-  end
-end

data/lib/wukong/monitor/periodic_monitor.rb

@@ -1,70 +0,0 @@
-module Wukong::Monitor
-  #
-  # Accepts a lightweight call every iteration.
-  #
-  # Once either a time or an iteration criterion is met, executes the block
-  # and resets the timer until next execution.
-  #
-  # Note that the +time_interval+ is measured *excution to execution* and not
-  # in multiples of iter_interval. Say I set a time_interval of 300s, and
-  # happen to iterate at 297s and 310s after start.  Then the monitor will
-  # execute at 310s, and the next execution will happen on or after 610s.
-  #
-  # Also note that when *either* criterion is met, *both* criteria are
-  # reset. Say I set a time interval of 300s and an +iter_interval+ of 10_000;
-  # and that at 250s I reach iteration 10_000.  Then the monitor will execute
-  # on or after 20_000 iteration or 550s, whichever happens first.
-  #
-class PeriodicMonitor
-  attr_accessor :time_interval, :iter_interval
-  attr_accessor :last_time, :current_iter, :iter, :started_at
-
-  def initialize options={}
-    self.started_at    = Time.now.utc.to_f
-    self.last_time     = started_at
-    self.iter          = 0
-    self.current_iter  = 0
-    self.time_interval = options[:time]
-    self.iter_interval = options[:iters]
-  end
-
-  # True if more than +iter_interval+ has elapsed since last execution.
-  def enough_iterations?
-    iter % iter_interval == 0 if iter_interval
-  end
-
-  # True if more than +time_interval+ has elapsed since last execution.
-  def enough_time? now
-    (now - last_time) > time_interval if time_interval
-  end
-
-  # Time since monitor was created
-  def since
-    Time.now.utc.to_f - started_at
-  end
-  # Overall iterations per second
-  def rate
-    iter.to_f / since.to_f
-  end
-  # "Instantaneous" iterations per second
-  def inst_rate now
-    current_iter.to_f / (now-last_time).to_f
-  end
-
-  #
-  # if the interval conditions are met, executes block; otherwise just does
-  # bookkeeping and returns.
-  #
-  def periodically &block
-    self.iter += 1
-    self.current_iter += 1
-    now       = Time.now.utc.to_f
-    if enough_iterations? || enough_time?(now)
-      block.call(iter, (now-last_time))
-      self.last_time = now
-      self.current_iter = 0
-    end
-  end
-end
-
-end

data/lib/wukong/monitor.rb

@@ -1,7 +0,0 @@
-module Monkeyshines
-  module Monitor
-    autoload :PeriodicMonitor, 'monkeyshines/monitor/periodic_monitor'
-    autoload :PeriodicLogger,  'monkeyshines/monitor/periodic_logger'
-  end
-end
-

data/lib/wukong/rdf.rb

@@ -1,104 +0,0 @@
-module Wukong
-  #
-  # Dump wukong object as RDF triples:
-  #
-  #   <key     attr    val     module Wukong
-  #
-  # Dump wukong object as RDF triples:
-  #
-  #   <key>     <attr>    <val> # <extra>
-  #
-  # Each element of the triple is XML encoded such that it contains no tab,
-  # newline or carriage returns, and the three are tab-separated. Any extra
-  # fields -- reification info, for instance -- are appended as a comment.
-  #
-  # This makes the result not only a valid RDF triple file but perfectly
-  # palatable to Wukong for further processing.
-  #
-  module Rdf
-
-    #
-    # RDF-formatted date
-    #
-    def self.encode_datetime dt
-      DateTime.parse_safely(dt).xmlschema
-    end
-
-    #
-    # Emit a component (subject or object) with the right semantic encoding
-    #
-    # Use :boolskip if a false property should just be left out.
-    #
-    def rdf_component val, type
-      case type
-      when :tweet         then %Q{<http://twitter.com/statuses/show/#{val}.xml>}
-      when :user          then %Q{<http://twitter.com/users/show/#{val}.xml>}
-      when :bool          then ((!val) || (val==0) || (val=="0")) ? '"false"^^<xsd:boolean>' : '"true"^^<xsd:boolean>'
-      when :boolskip      then ((!val) || (val==0) || (val=="0")) ? nil                      : '"true"^^<xsd:boolean>'
-      when :int           then %Q{"#{val.to_i}"^^<xsd:integer>}
-      when :date          then %Q{"#{TwitterRdf.encode_datetime(val)}"^^<xsd:dateTime>}
-      when :str           then %Q{"#{val}"}
-      else raise "Don't know how to encode #{type}"
-      end
-    end
-
-    #
-    # Express relationship (predicate) in RDF
-    #
-    def rdf_pred pred
-      case pred
-      when :created_at  then %Q{<http://twitter.com/##{pred}>}
-      else                   %Q{<http://twitter.com/##{pred}>}
-      end
-    end
-
-    #
-    # RDF Triple string for the given (subject, object, predicate)
-    #   http://www.w3.org/TR/rdf-testcases/#ntriples
-    #
-    def self.rdf_triple subj, pred, obj, comment=nil
-      comment = "\t# " + comment.to_s unless comment.blank?
-      %Q{%-55s\t%-39s\t%-23s\t.%s} % [subj, pred, obj, comment]
-    end
-
-    def mutable?(attr)
-      false
-    end
-
-    #
-    # Extract [subject, predicate, object, (extra)] tuples.
-    #
-    # (extra) is set to +scraped at+ for #mutable? attributes, blank otherwise.
-    #
-    def to_rdf3_tuples
-      members_with_types.map do |attr, type|
-        next if self[attr].blank?
-        subj    = rdf_resource
-        pred    = rdf_pred(attr)
-        obj     = rdf_component(self[attr], type) or next
-        comment = scraped_at if mutable?(attr)
-        [subj, pred, obj, comment]
-      end.compact
-    end
-
-    #
-    # Convert an object to an rdf triple.
-    #
-    # Appends scraped at to #mutable? attributes
-    #
-    def to_rdf3
-      to_rdf3_tuples.map do |tuple|
-        self.class.rdf_triple tuple
-      end.join("\n")
-    end
-
-  end
-end
->
-  #
-  #
-  module Rdf
-    def to_rdf
-    end
-  end
-end

data/lib/wukong/streamer/cassandra_streamer.rb

@@ -1,61 +0,0 @@
-# Defines a base class for streaming data into a cassandra db connection.
-require 'cassandra' ; include Cassandra::Constants
-module Wukong
-  module Streamer
-
-    class CassandraStreamer < Wukong::Streamer::Base
-      attr_accessor :batch_count, :batch_record_count, :batch_size, :column_space, :db_seeds, :cassandra_db
-
-      def initialize *args
-        super *args
-        self.batch_count = 0
-        self.batch_record_count = 0
-        self.column_space ||= 'Twitter'
-        self.batch_size   ||= 100
-        self.db_seeds     ||= %w[10.244.191.178 10.243.19.223 10.243.17.219 10.245.70.85 10.244.206.241].map{ |s| s.to_s+':9160'}
-        self.cassandra_db ||= Cassandra.new(self.column_space, self.db_seeds)
-      end
-
-      def stream
-        while still_lines? do
-          start_batch do
-            while still_lines? && batch_not_full? do
-              line = get_line
-              record = recordize(line.chomp) or next
-              next if record.blank?
-              process(*record) do |output_record|
-                emit output_record
-              end
-              self.batch_record_count += 1
-            end
-          end
-        end
-      end
-
-      def process *args, &blk
-        Raise "Overwrite this method to insert into cassandra db"
-      end
-
-      def start_batch &blk
-        self.batch_record_count = 0
-        self.batch_count += 1
-        self.cassandra_db.batch(&blk)
-      end
-
-      def get_line
-        $stdin.gets
-      end
-
-      def still_lines?
-        !$stdin.eof?
-      end
-
-      def batch_not_full?
-        self.batch_record_count < self.batch_size
-      end
-
-    end
-  end
-
-end
-

data/lib/wukong/streamer/count_keys.rb

@@ -1,30 +0,0 @@
-module Wukong
-  module Streamer
-    #
-    # Emit each unique key and the count of its occurrences
-    #
-    class CountKeys < Wukong::Streamer::AccumulatingReducer
-      attr_accessor :key_count
-
-      def formatted_key_count
-        "%10d"%key_count.to_i
-      end
-
-      # reset the counter to zero
-      def start! *args
-        self.key_count = 0
-      end
-
-      # record one more for this key
-      def accumulate *vals
-        self.key_count += 1
-      end
-
-      # emit each key field and the count, tab-separated.
-      def finalize
-        yield [key, formatted_key_count]
-      end
-    end
-
-  end
-end

data/lib/wukong/streamer/count_lines.rb

@@ -1,26 +0,0 @@
-module Wukong
-  module Streamer
-    #
-    # For each identical line in the map phase output, emit one representative
-    # line followed by the count of occrrences (separated by a tab).
-    #
-    # (This is the functional equivalent of +'uniq -c'+)
-    #
-    class CountLines < Wukong::Streamer::Base
-      def formatted_count item, key_count
-        "%s\t%10d" % [item, key_count.to_i]
-      end
-
-      #
-      # Delegate to +uniq -c+, but put the count last for idempotence.
-      #
-      def stream
-        %x{/usr/bin/uniq -c}.split("\n").each do |line|
-          key_count, item = line.chomp.strip.split(/\s+/, 2)
-          puts formatted_count(item, key_count)
-        end
-      end
-    end
-
-  end
-end

data/lib/wukong/streamer/em_streamer.rb

@@ -1,7 +0,0 @@
-module Wukong
-  module Streamer
-    class EmStreamer
-
-    end
-  end
-end

data/lib/wukong/streamer/preprocess_with_pipe_streamer.rb

@@ -1,22 +0,0 @@
-module Wukong
-  module Streamer
-    module PreprocessWithPipeStreamer
-      #
-      # Runs STDIN through a shell command and then begins processing.
-      #
-      # If you don't need to do anything to the output of the command, just
-      # inherit from Wukong::Script and override the #map_command.
-      #
-      # You must provide a @preprocess_pipe_command@ method that returns a shell
-      # command to run the input through.
-      #
-      def stream
-        #
-        `#{preprocess_pipe_command}`.each do |line|
-          item = itemize(line) ; next if item.blank?
-          process(*item)
-        end
-      end
-    end
-  end
-end

data/lib/wukong/wukong_class.rb

@@ -1,21 +0,0 @@
-# require 'active_support/core_ext/class/inheritable_attributes.rb'
-require 'extlib/class'
-
-module Wukong
-  #
-  # Use to instrument an actual class to behave
-  #
-  module WukongClass
-
-
-    def [](attr)
-      self.send attr
-    end
-    def []=(attr, val)
-      self.send("#{attr}=", val)
-    end
-
-  end
-
-
-end