wukong 3.0.0.pre → 3.0.0.pre2

data/old/wukong/script/local_command.rb

@@ -1,41 +0,0 @@
-module Wukong
-  #
-  # Local execution Options
-  #
-  module LocalCommand
-
-    Settings.define :sort_command, :default => 'sort'
-    Settings.define :sort_args,    :default => [], :description => 'Extra params to send to the sort function eg: Settings.sort_args = ["-t", "\t", "-S", "200M"]'
-
-    def execute_local_workflow
-      Log.info "  Reading STDIN / Writing STDOUT"
-      execute_command!(local_commandline)
-    end
-
-    # program, including arg, to sort input between mapper and reducer in local
-    # mode. You could override to for example run 'sort -n' (numeric sort).
-    def local_mode_sort_commandline
-      [ Settings.sort_command, Settings.sort_args ].flatten.join(" ")
-    end
-
-    #
-    # Commandline string to execute the job in local mode
-    #
-    # With an input path of '-', just uses $stdin
-    # With an output path of '-', just uses $stdout
-    #
-    def local_commandline
-      @input_paths = input_paths.map(&:strip).join(' ')
-      cmd_input_str  = (input_paths == '-') ? "" : "cat '#{input_paths}' | "
-      cmd_output_str = (output_path == '-') ? "" : "> '#{output_path}'"
-
-      if (reducer || options[:reduce_command])
-        %Q{ #{cmd_input_str} #{mapper_commandline} | #{local_mode_sort_commandline} | #{reducer_commandline} #{cmd_output_str} }
-      else
-        %Q{ #{cmd_input_str} #{mapper_commandline} | #{local_mode_sort_commandline} #{cmd_output_str} }
-      end
-
-    end
-
-  end
-end

data/old/wukong/store.rb

@@ -1,10 +0,0 @@
-module Wukong
-  module Store
-    autoload :Base,                    'wukong/store/base'
-    autoload :FlatFileStore,           'wukong/store/flat_file_store'
-    autoload :ChunkedFlatFileStore,    'wukong/store/chunked_flat_file_store'
-    autoload :ChhChunkedFlatFileStore, 'wukong/store/chh_chunked_flat_file_store'
-
-    autoload :CassandraModel,          'wukong/store/cassandra_model'
-  end
-end

data/old/wukong/store/base.rb

@@ -1,27 +0,0 @@
-module Wukong
-  module Store
-    class Base
-      def initialize options={}
-        Log.info "Creating #{self.class} with #{options.inspect}"
-      end
-
-      #Iterate through each object casting it as a new object of klass.
-      def each_as klass, &block
-        self.each do |*args|
-          begin
-            item = klass.new *args[1..-1]
-          rescue StandardError => e
-            Log.info [args, e.to_s, self].join("\t")
-            raise e
-          end
-          yield item
-        end
-      end
-
-      def log_line
-        nil
-      end
-
-    end
-  end
-end

data/old/wukong/store/cassandra.rb

@@ -1,10 +0,0 @@
-Settings.define :cassandra_hosts,    :default => '127.0.0.1:9160', :type => Array, :description => 'Comma-delimited list of hostname:port addresses for the Cassandra database holding Twitter API objects'
-Settings.define :cassandra_keyspace, :default => 'soc_net_tw',                     :description => 'Cassandra keyspace for Twitter objects'
-
-module Wukong
-  module Store
-    module CassandraStore
-      autoload :StructLoader, 'wukong/store/cassandra/struct_loader'
-    end
-  end
-end

data/old/wukong/store/cassandra/streaming.rb

@@ -1,75 +0,0 @@
-require 'avro'
-
-Settings.define :cassandra_avro_schema, :default => ('/usr/local/share/cassandra/interface/avro/cassandra.avpr')
-module Wukong::Store::CassandraModel
-
-  #
-  # Store model using avro writer
-  #
-  def streaming_save
-    self.class.streaming_insert id, self
-  end
-  module ClassMethods
-
-    def streaming_writer
-      @streaming_writer ||= AvroWriter.new
-    end
-
-    #
-    # Use avro and stream into cassandra
-    #
-    def streaming_insert id, hsh
-      streaming_writer.put(id.to_s, hsh.to_db_hash)
-    end
-  end
-  class AvroWriter
-    #
-    # Reads in the protocol schema
-    # creates the necessary encoder and writer.
-    #
-    def initialize
-      schema_file = Settings.cassandra_avro_schema
-      @proto  = Avro::Protocol.parse(File.read(schema_file))
-      @schema = @proto.types.detect{|schema| schema.name == 'StreamingMutation'}
-      @enc    = Avro::IO::BinaryEncoder.new($stdout)
-      # @enc    = DummyEncoder.new($stdout)
-      @writer = Avro::IO::DatumWriter.new(@schema)
-      # warn [@schema, @enc].inspect
-    end
-
-    def write key, col_name, value
-      @writer.write(smutation(key, col_name, value), @enc)
-    end
-
-    def write_directly key, col_name, value, timestamp, ttl
-      # Log.info "Insert(row_key => #{key}, col_name => #{col_name}, value => #{value}"
-      @enc.write_bytes(key)
-      @enc.write_bytes(col_name)
-      @enc.write_bytes(value)
-      @enc.write_long(timestamp)
-      @enc.write_int(ttl)
-    end
-
-    #
-    # Iterate through each key value pair in the hash to
-    # be inserted and write directly one at a time
-    #
-    def put id, hsh, timestamp=nil, ttl=0
-      timestamp ||= Time.now.to_i
-      hsh.each do |attr, val|
-        write_directly(id, attr, val, timestamp, ttl)
-      end
-    end
-
-    def smutation key, name, value
-      {
-        'key'       => key,
-        'name'      => name.to_s,
-        'value'     => value.to_s,
-        'timestamp' => Time.epoch_microseconds,
-        'ttl'       => 0
-      }
-    end
-  end
-
-end

data/old/wukong/store/cassandra/struct_loader.rb

@@ -1,21 +0,0 @@
-require 'avro'
-
-Settings.define :cassandra_avro_schema, :default => ('/usr/local/share/cassandra/interface/avro/cassandra.avpr')
-
-module Wukong::Store::Cassandra
-  class StructLoader < Wukong::Streamer::StructStreamer
-    def initialize *args
-      super(*args)
-      @log = PeriodicMonitor.new
-    end
-
-    #
-    # Blindly expects objects streaming by to have a "streaming_save" method
-    #
-    def process object, *_
-      # object.save
-      object.streaming_save
-      @log.periodically(object.to_flat)
-    end
-  end
-end

data/old/wukong/store/cassandra_model.rb

@@ -1,91 +0,0 @@
-module Wukong
-  module Store
-    #
-    # Barebones interface between a wukong class and a cassandra database
-    #
-    # Class must somehow provide a class-level cassandra_db accessor
-    # that sets the @cassandra_db instance variable.
-    #
-    module CassandraModel
-      #
-      # Store model to the DB
-      #
-      def save
-        self.class.insert key, self.to_db_hash
-      end
-
-      #
-      # Flatten attributes for storage in the DB.
-      #
-      # * omits elements whose value is nil
-      # * calls to_s on everything else
-      # * This means that blank strings are preserved;
-      # * and that false is saved as 'false'
-      #
-      # Override if you think something fancier than that should happen.
-      #
-      def to_db_hash
-        db_hsh = {}
-        each_pair{|k,v| db_hsh[k.to_s] = v.to_s unless v.nil? }
-        db_hsh
-      end
-
-
-      module ClassMethods
-        # Cassandra column family -- taken from the class name by default.
-        def table_name
-          class_basename
-        end
-
-        # Override to control how your class is instantiated from the DB hash
-        def from_db_hash *args
-          from_hash *args
-        end
-
-        # Insert into the cassandra database
-        # uses object's #to_db_hash method
-        def insert key, *args
-          hsh = args.first
-          cassandra_db.insert(table_name, key.to_s, hsh)
-        end
-
-        # Insert into the cassandra database
-        # calls out to object's #from_db_hash method
-        def load key
-          hsh = cassandra_db.get(self.class_basename, key.to_s)
-          from_db_hash(hsh) if hsh
-        end
-
-        # invalidates cassandra connection on errors where that makes sense.
-        def handle_error action, e
-          warn "#{action} failed: #{e} #{e.backtrace.join("\t")}" ;
-          @cassandra_db = nil
-          sleep 0.2
-        end
-      end
-      # The standard 'inject class methods when module is included' trick
-      def self.included base
-        base.class_eval{ extend ClassMethods}
-      end
-    end
-
-  end
-end
-
-Hash.class_eval do
-  #
-  # Flatten attributes for storage in the DB.
-  #
-  # * omits elements whose value is nil
-  # * calls to_s on everything else
-  # * This means that blank strings are preserved;
-  # * and that false is saved as 'false'
-  #
-  # Override if you think something fancier than that should happen.
-  #
-  def to_db_hash
-    db_hsh = {}
-    to_hash.each{|k,v| db_hsh[k.to_s] = v.to_s unless v.nil? }
-    db_hsh
-  end
-end

data/old/wukong/store/chh_chunked_flat_file_store.rb

@@ -1,37 +0,0 @@
-module Wukong
-  module Store
-    class ChhChunkedFlatFileStore < Wukong::Store::FlatFileStore
-      attr_accessor :filename_pattern, :handle, :rootdir
-
-      # Move to configliere
-      Settings.define :chunk_file_pattern,   :default => ":rootdir/:date/:handle:timestamp-:pid.tsv",:description => "The pattern for chunked files."
-      Settings.define :chunk_file_rootdir,   :default => nil, :description => "The root directory for the chunked files."
-
-      #Note that filemode is inherited from flat_file
-
-      def initialize options={}
-        # super wants a :filename in the options or it will fail. We need to get the initial filename
-        # set up before we call super, so we need all of the parts of the pattern set up.
-        self.rootdir          = options[:rootdir] || Settings[:chunk_file_rootdir]
-        self.handle           = options[:handle]
-        pattern               = options[:pattern] || Settings[:chunk_file_pattern]
-        self.filename_pattern = FilenamePattern.new(pattern, :handle => handle, :rootdir => self.rootdir)
-        options[:filename]    = filename_pattern.make()
-
-        super options
-
-        self.mkdir!
-      end
-
-      def new_chunk
-        new_filename = filename_pattern.make()
-        Log.info "Rotating chunked file #{filename} into #{new_filename}"
-        self.flush
-        self.close
-        @filename = new_filename
-        self.mkdir!
-      end
-
-    end
-  end
-end

data/old/wukong/store/chunked_flat_file_store.rb

@@ -1,48 +0,0 @@
-require 'wukong/monitor/periodic_monitor'
-module Wukong
-  module Store
-    class ChunkedFlatFileStore < Wukong::Store::FlatFileStore
-      attr_accessor :filename_pattern, :chunk_monitor, :handle, :chunktime, :rootdir
-
-      # Move to configliere
-      Settings.define :chunk_file_pattern,   :default => ":rootdir/:date/:handle-:timestamp-:pid.tsv",:description => "The pattern for chunked files."
-      Settings.define :chunk_file_interval,  :default => 4*60*60,  :description => "The time interval to keep a chunk file open."
-      Settings.define :chunk_file_rootdir,   :default => '/tmp',   :description => "The root directory for the chunked files."
-
-      #Note that filemode is inherited from flat_file
-
-      def initialize options={}
-        # super wants a :filename in the options or it will fail. We need to get the initial filename
-        # set up before we call super, so we need all of the parts of the pattern set up.
-        self.chunktime        = options[:interval] || Settings[:chunk_file_interval]
-        self.rootdir          = options[:rootdir]  || Settings[:chunk_file_rootdir]
-        self.handle           = options[:handle]
-        pattern               = options[:pattern]  || Settings[:chunk_file_pattern]
-        self.filename_pattern = FilenamePattern.new(pattern, :handle => handle, :rootdir => self.rootdir)
-        options[:filename]    = filename_pattern.make()
-        options[:filemode]  ||= 'a'
-        Log.warn "You don't really want a chunk time this small: #{self.chunktime}" unless self.chunktime > 600
-        self.chunk_monitor    = Wukong::Monitor::PeriodicMonitor.new( :time => self.chunktime )
-
-        super options
-        self.mkdir!
-      end
-
-      def new_chunk!
-        new_filename = filename_pattern.make()
-        Log.info "Rotating chunked file #{filename} into #{new_filename}"
-        self.flush
-        self.close
-        @filename = new_filename
-        self.mkdir!
-      end
-
-      def save *args
-        result = super *args
-        chunk_monitor.periodically{ new_chunk! }
-        result
-      end
-
-    end
-  end
-end

data/old/wukong/store/conditional_store.rb

@@ -1,57 +0,0 @@
-module Monkeyshines
-  module Store
-    class ConditionalStore < Monkeyshines::Store::Base
-      attr_accessor :options, :cache, :store, :misses
-
-      DEFAULT_OPTIONS = {
-        :cache => { :type => :tyrant_rdb_key_store    },
-        :store => { :type => :chunked_flat_file_store },
-      }
-
-      #
-      #
-      # +cache+ must behave like a hash (Hash and
-      #  Monkeyshines::Store::TyrantRdbKeyStore are both cromulent
-      #  choices).
-      #
-      #
-      #
-      def initialize _options
-        self.options = DEFAULT_OPTIONS.deep_merge(_options)
-        self.cache  = Monkeyshines::Store.create(options[:cache])
-        self.store  = Monkeyshines::Store.create(options[:store])
-        self.misses = 0
-      end
-
-      #
-      # If key is absent, save the result of calling the block.
-      # If key is present, block is never called.
-      #
-      # Ex:
-      #   rt_store.set(url) do
-      #     fetcher.get url # will only be called if url isn't in rt_store
-      #   end
-      #
-      def set key, force=nil, &block
-        return if (!force) && cache.include?(key)
-        cache_val, store_val = block.call()
-        return unless cache_val
-        cache.set_nr key, cache_val # update cache
-        store << store_val          # save value
-        self.misses += 1            # track the cache miss
-        store_val
-      end
-
-      def size() cache.size  end
-
-      def log_line
-        [size, "%8d misses"%misses]
-      end
-
-      def close()
-        cache.close
-        store.close
-      end
-    end
-  end
-end

data/old/wukong/store/factory.rb

@@ -1,8 +0,0 @@
-module Monkeyshines
-  module Store
-    class Factory
-      def self.generate type, opts
-      end
-    end
-  end
-end

data/old/wukong/store/flat_file_store.rb

@@ -1,89 +0,0 @@
-require 'fileutils'; include FileUtils
-
-module Wukong
-  module Store
-    #
-    class FlatFileStore < Store::Base
-      attr_accessor :filename, :filemode
-
-      #
-      # +filename_root+  : first part of name for files
-      #
-      def initialize options={}
-        super options
-        self.filename = options[:filename] or raise "Missing filename in #{self.class}"
-        self.filemode = options[:filemode] || 'r'
-        skip!(options[:skip]) if options[:skip]
-      end
-
-      #
-      #
-      #
-      def each &block
-        file.each do |line|
-          attrs = line.chomp.split("\t")
-          next if attrs.blank?
-          yield *attrs
-        end
-      end
-
-      #
-      # Read ahead n_lines lines in the file
-      #
-      def skip! n_lines
-        Log.info "Skipping #{n_lines} in #{self.class}:#{filename}"
-        n_lines.times do
-          file.readline
-        end
-      end
-
-      #
-      # Open the timestamped file,
-      # ensuring its directory exists
-      #
-      def file
-        return @file if @file
-        Log.info "Opening file #{filename} with mode #{filemode}"
-        @file = File.open(filename, filemode)
-      end
-
-      # Close the dump file
-      def close
-        @file.close if @file
-        @file = nil
-      end
-
-      def flush
-        @file.flush if @file
-      end
-
-      # Ensure the file's directory exists
-      def mkdir!
-        dir = File.dirname(filename)
-        return if File.directory?(dir)
-        Log.info "Making directory #{dir}"
-        FileUtils.mkdir_p dir
-      end
-
-      # write to the file
-      def save obj
-        file.puts obj
-        obj
-      end
-      
-      # returns the size of the current file
-      def size
-        return 0 if !@file
-        File.size(filename)
-      end
-
-      # delegates to +#save+ -- writes the object to the file. Returns self for chaining on the stream.
-      def <<(obj)
-        save obj
-	self
-      end
-
-    end
-  end
-end
-