wukong 1.5.3 → 1.5.4

data/lib/wukong/store.rb

@@ -1,15 +1,10 @@
 module Wukong
   module Store
-#    extend FactoryModule
     autoload :Base,                    'wukong/store/base'
     autoload :FlatFileStore,           'wukong/store/flat_file_store'
-#    autoload :ConditionalStore,       'monkeyshines/store/conditional_store'
     autoload :ChunkedFlatFileStore,    'wukong/store/chunked_flat_file_store'
     autoload :ChhChunkedFlatFileStore, 'wukong/store/chh_chunked_flat_file_store'
-#    autoload :KeyStore,               'monkeyshines/store/key_store'
-#    autoload :TokyoTdbKeyStore,       'monkeyshines/store/tokyo_tdb_key_store'
-#    autoload :TyrantTdbKeyStore,      'monkeyshines/store/tyrant_tdb_key_store'
-#    autoload :TyrantRdbKeyStore,      'monkeyshines/store/tyrant_rdb_key_store'
-#    autoload :ReadThruStore,          'monkeyshines/store/read_thru_store'
+
+    autoload :CassandraModel,          'wukong/store/cassandra_model'
   end
 end

data/lib/wukong/store/cassandra.rb

@@ -0,0 +1,10 @@
+Settings.define :cassandra_hosts,    :default => '127.0.0.1:9160', :type => Array, :description => 'Comma-delimited list of hostname:port addresses for the Cassandra database holding Twitter API objects'
+Settings.define :cassandra_keyspace, :default => 'soc_net_tw',                     :description => 'Cassandra keyspace for Twitter objects'
+
+module Wukong
+  module Store
+    module CassandraStore
+      autoload :StructLoader, 'wukong/store/cassandra/struct_loader'
+    end
+  end
+end

data/lib/wukong/store/cassandra/streaming.rb

@@ -0,0 +1,75 @@
+require 'avro'
+
+Settings.define :cassandra_avro_schema, :default => ('/usr/local/share/cassandra/interface/avro/cassandra.avpr')
+module Wukong::Store::CassandraModel
+
+  #
+  # Store model using avro writer
+  #
+  def streaming_save
+    self.class.streaming_insert id, self
+  end
+  module ClassMethods
+
+    def streaming_writer
+      @streaming_writer ||= AvroWriter.new
+    end
+
+    #
+    # Use avro and stream into cassandra
+    #
+    def streaming_insert id, hsh
+      streaming_writer.put(id.to_s, hsh.to_db_hash)
+    end
+  end
+  class AvroWriter
+    #
+    # Reads in the protocol schema
+    # creates the necessary encoder and writer.
+    #
+    def initialize
+      schema_file = Settings.cassandra_avro_schema
+      @proto  = Avro::Protocol.parse(File.read(schema_file))
+      @schema = @proto.types.detect{|schema| schema.name == 'StreamingMutation'}
+      @enc    = Avro::IO::BinaryEncoder.new($stdout)
+      # @enc    = DummyEncoder.new($stdout)
+      @writer = Avro::IO::DatumWriter.new(@schema)
+      # warn [@schema, @enc].inspect
+    end
+
+    def write key, col_name, value
+      @writer.write(smutation(key, col_name, value), @enc)
+    end
+
+    def write_directly key, col_name, value, timestamp, ttl
+      # Log.info "Insert(row_key => #{key}, col_name => #{col_name}, value => #{value}"
+      @enc.write_bytes(key)
+      @enc.write_bytes(col_name)
+      @enc.write_bytes(value)
+      @enc.write_long(timestamp)
+      @enc.write_int(ttl)
+    end
+
+    #
+    # Iterate through each key value pair in the hash to
+    # be inserted and write directly one at a time
+    #
+    def put id, hsh, timestamp=nil, ttl=0
+      timestamp ||= Time.now.to_i
+      hsh.each do |attr, val|
+        write_directly(id, attr, val, timestamp, ttl)
+      end
+    end
+
+    def smutation key, name, value
+      {
+        'key'       => key,
+        'name'      => name.to_s,
+        'value'     => value.to_s,
+        'timestamp' => Time.epoch_microseconds,
+        'ttl'       => 0
+      }
+    end
+  end
+
+end

data/lib/wukong/store/cassandra/struct_loader.rb

@@ -0,0 +1,21 @@
+require 'avro'
+
+Settings.define :cassandra_avro_schema, :default => ('/usr/local/share/cassandra/interface/avro/cassandra.avpr')
+
+module Wukong::Store::Cassandra
+  class StructLoader < Wukong::Streamer::StructStreamer
+    def initialize *args
+      super(*args)
+      @log = PeriodicMonitor.new
+    end
+
+    #
+    # Blindly expects objects streaming by to have a "streaming_save" method
+    #
+    def process object, *_
+      # object.save
+      object.streaming_save
+      @log.periodically(object.to_flat)
+    end
+  end
+end

data/lib/wukong/store/cassandra_model.rb

@@ -0,0 +1,90 @@
+module Wukong
+  module Store
+    #
+    # Barebones interface between a wukong class and a cassandra database
+    #
+    # Class must somehow provide a class-level cassandra_db accessor
+    # that sets the @cassandra_db instance variable.
+    #
+    module CassandraModel
+      #
+      # Store model to the DB
+      #
+      def save
+        self.class.insert key, self.to_db_hash
+      end
+
+      #
+      # Flatten attributes for storage in the DB.
+      #
+      # * omits elements whose value is nil
+      # * calls to_s on everything else
+      # * This means that blank strings are preserved;
+      # * and that false is saved as 'false'
+      #
+      # Override if you think something fancier than that should happen.
+      #
+      def to_db_hash
+        db_hsh = {}
+        to_hash.each{|k,v| db_hsh[k.to_s] = v.to_s unless v.nil? }
+        db_hsh
+      end
+
+      module ClassMethods
+        # Cassandra column family -- taken from the class name by default.
+        def table_name
+          class_basename
+        end
+
+        # Override to control how your class is instantiated from the DB hash
+        def from_db_hash *args
+          from_hash *args
+        end
+
+        # Insert into the cassandra database
+        # uses object's #to_db_hash method
+        def insert key, *args
+          hsh = args.first
+          cassandra_db.insert(table_name, key.to_s, hsh)
+        end
+
+        # Insert into the cassandra database
+        # calls out to object's #from_db_hash method
+        def load key
+          hsh = cassandra_db.get(self.class_basename, key.to_s)
+          from_db_hash(hsh) if hsh
+        end
+
+        # invalidates cassandra connection on errors where that makes sense.
+        def handle_error action, e
+          warn "#{action} failed: #{e} #{e.backtrace.join("\t")}" ;
+          @cassandra_db = nil
+          sleep 0.2
+        end
+      end
+      # The standard 'inject class methods when module is included' trick
+      def self.included base
+        base.class_eval{ extend ClassMethods}
+      end
+    end
+
+  end
+end
+
+Hash.class_eval do
+  #
+  # Flatten attributes for storage in the DB.
+  #
+  # * omits elements whose value is nil
+  # * calls to_s on everything else
+  # * This means that blank strings are preserved;
+  # * and that false is saved as 'false'
+  #
+  # Override if you think something fancier than that should happen.
+  #
+  def to_db_hash
+    db_hsh = {}
+    to_hash.each{|k,v| db_hsh[k.to_s] = v.to_s unless v.nil? }
+    db_hsh
+  end
+end

data/lib/wukong/store/chh_chunked_flat_file_store.rb

@@ -12,7 +12,7 @@ module Wukong
       def initialize options={}
         # super wants a :filename in the options or it will fail. We need to get the initial filename
         # set up before we call super, so we need all of the parts of the pattern set up.
-        self.rootdir          = options[:rootdir]   || Settings[:chunk_file_rootdir]
+        self.rootdir          = options[:rootdir] || Settings[:chunk_file_rootdir]
         self.handle           = options[:handle]
         pattern               = options[:pattern] || Settings[:chunk_file_pattern]
         self.filename_pattern = FilenamePattern.new(pattern, :handle => handle, :rootdir => self.rootdir)

data/lib/wukong/store/chunked_flat_file_store.rb

@@ -1,41 +1,45 @@
+require 'wukong/monitor/periodic_monitor'
 module Wukong
   module Store
     class ChunkedFlatFileStore < Wukong::Store::FlatFileStore
       attr_accessor :filename_pattern, :chunk_monitor, :handle, :chunktime, :rootdir
 
       # Move to configliere
-      Settings.define :chunk_file_pattern,   :default => ":rootdir/:date/:handle:timestamp-:pid.tsv",:description => "The pattern for chunked files."
-      Settings.define :chunk_file_chunktime, :default => 4*60*60,:description => "The time interval to keep a chunk file open."
-      Settings.define :chunk_file_rootdir,   :default => nil, :description => "The root directory for the chunked files."
-      
+      Settings.define :chunk_file_pattern,   :default => ":rootdir/:date/:handle-:timestamp-:pid.tsv",:description => "The pattern for chunked files."
+      Settings.define :chunk_file_interval,  :default => 4*60*60,  :description => "The time interval to keep a chunk file open."
+      Settings.define :chunk_file_rootdir,   :default => '/tmp',   :description => "The root directory for the chunked files."
+
       #Note that filemode is inherited from flat_file
 
       def initialize options={}
         # super wants a :filename in the options or it will fail. We need to get the initial filename
-        # set up before we call super, so we need all of the parts of the pattern set up. 
-        self.chunktime        = options[:chunktime] || Settings[:chunk_file_chunktime]
-        self.rootdir          = options[:rootdir]   || Settings[:chunk_file_rootdir]
-        self.handle           = options[:handle] 
-        pattern               = options[:pattern] || Settings[:chunk_file_pattern]
+        # set up before we call super, so we need all of the parts of the pattern set up.
+        self.chunktime        = options[:interval] || Settings[:chunk_file_interval]
+        self.rootdir          = options[:rootdir]  || Settings[:chunk_file_rootdir]
+        self.handle           = options[:handle]
+        pattern               = options[:pattern]  || Settings[:chunk_file_pattern]
         self.filename_pattern = FilenamePattern.new(pattern, :handle => handle, :rootdir => self.rootdir)
-        options[:filename]    = filename_pattern.make() 
-      
+        options[:filename]    = filename_pattern.make()
+        options[:filemode]  ||= 'a'
+        Log.warn "You don't really want a chunk time this small: #{self.chunktime}" unless self.chunktime > 600
+        self.chunk_monitor    = Wukong::Monitor::PeriodicMonitor.new( :time => self.chunktime )
+
         super options
+        self.mkdir!
+      end
 
-        Log.warn "You don't really want a chunk time this small: #{self.chunktime}" unless self.chunktime > 600
-        self.chunk_monitor    = Wukong::PeriodicMonitor.new( :time => self.chunktime )
+      def new_chunk!
+        new_filename = filename_pattern.make()
+        Log.info "Rotating chunked file #{filename} into #{new_filename}"
+        self.flush
+        self.close
+        @filename = new_filename
         self.mkdir!
       end
 
       def save *args
         result = super *args
-        chunk_monitor.periodically do
-          new_filename = filename_pattern.make()
-          Log.info "Rotating chunked file #{filename} into #{new_filename}"
-          self.close
-          @filename = new_filename
-          self.mkdir!
-        end
+        chunk_monitor.periodically{ new_chunk! }
         result
       end
 

data/wukong.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{wukong}
-  s.version = "1.5.3"
+  s.version = "1.5.4"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Philip (flip) Kromer"]
-  s.date = %q{2010-08-19}
+  s.date = %q{2010-11-02}
   s.description = %q{  Treat your dataset like a:
 
       * stream of lines when it's efficient to process by lines
@@ -30,10 +30,10 @@ Gem::Specification.new do |s|
      "LICENSE.textile",
      "README.textile",
      "TODO.textile",
-     "bin/bootstrap.sh",
      "bin/cutc",
      "bin/cuttab",
      "bin/greptrue",
+     "bin/hdp-bin",
      "bin/hdp-bzip",
      "bin/hdp-cat",
      "bin/hdp-catd",
@@ -75,6 +75,7 @@ Gem::Specification.new do |s|
      "docpages/UsingWukong-part3-parsing.textile",
      "docpages/_config.yml",
      "docpages/avro/avro_notes.textile",
+     "docpages/avro/performance.textile",
      "docpages/avro/tethering.textile",
      "docpages/bigdata-tips.textile",
      "docpages/code/api_response_example.txt",
@@ -129,6 +130,21 @@ Gem::Specification.new do |s|
      "docpages/wutils.textile",
      "examples/README.txt",
      "examples/binning_percentile_estimator.rb",
+     "examples/cassandra_streaming/avromapper.rb",
+     "examples/cassandra_streaming/berlitz_for_cassandra.textile",
+     "examples/cassandra_streaming/cassandra.avpr",
+     "examples/cassandra_streaming/cassandra_random_partitioner.rb",
+     "examples/cassandra_streaming/catter.sh",
+     "examples/cassandra_streaming/client_interface_notes.textile",
+     "examples/cassandra_streaming/client_schema.avpr",
+     "examples/cassandra_streaming/client_schema.textile",
+     "examples/cassandra_streaming/foofile.avr",
+     "examples/cassandra_streaming/pymap.sh",
+     "examples/cassandra_streaming/pyreduce.sh",
+     "examples/cassandra_streaming/smutation.avpr",
+     "examples/cassandra_streaming/streamer.sh",
+     "examples/cassandra_streaming/struct_loader.rb",
+     "examples/cassandra_streaming/tuning.textile",
      "examples/contrib/jeans/README.markdown",
      "examples/contrib/jeans/data/normalized_sizes",
      "examples/contrib/jeans/data/orders.tsv",
@@ -138,8 +154,11 @@ Gem::Specification.new do |s|
      "examples/corpus/words_to_bigrams.rb",
      "examples/count_keys.rb",
      "examples/count_keys_at_mapper.rb",
+     "examples/emr/README-elastic_map_reduce.textile",
+     "examples/emr/dot_wukong_dir/credentials.json",
+     "examples/emr/dot_wukong_dir/emr.yaml",
+     "examples/emr/dot_wukong_dir/emr_bootstrap.sh",
      "examples/emr/elastic_mapreduce_example.rb",
-     "examples/emr/emr.yaml",
      "examples/keystore/cassandra_batch_test.rb",
      "examples/keystore/conditional_outputter_example.rb",
      "examples/network_graph/adjacency_list.rb",
@@ -171,6 +190,7 @@ Gem::Specification.new do |s|
      "lib/wukong/datatypes/fake_types.rb",
      "lib/wukong/dfs.rb",
      "lib/wukong/encoding.rb",
+     "lib/wukong/encoding/asciize.rb",
      "lib/wukong/extensions.rb",
      "lib/wukong/extensions/array.rb",
      "lib/wukong/extensions/blank.rb",
@@ -203,11 +223,16 @@ Gem::Specification.new do |s|
      "lib/wukong/schema.rb",
      "lib/wukong/script.rb",
      "lib/wukong/script/avro_command.rb",
+     "lib/wukong/script/cassandra_loader_script.rb",
      "lib/wukong/script/emr_command.rb",
      "lib/wukong/script/hadoop_command.rb",
      "lib/wukong/script/local_command.rb",
      "lib/wukong/store.rb",
      "lib/wukong/store/base.rb",
+     "lib/wukong/store/cassandra.rb",
+     "lib/wukong/store/cassandra/streaming.rb",
+     "lib/wukong/store/cassandra/struct_loader.rb",
+     "lib/wukong/store/cassandra_model.rb",
      "lib/wukong/store/chh_chunked_flat_file_store.rb",
      "lib/wukong/store/chunked_flat_file_store.rb",
      "lib/wukong/store/conditional_store.rb",
@@ -259,6 +284,9 @@ Gem::Specification.new do |s|
      "spec/wukong/encoding_spec.rb",
      "spec/wukong/script_spec.rb",
      "examples/binning_percentile_estimator.rb",
+     "examples/cassandra_streaming/avromapper.rb",
+     "examples/cassandra_streaming/cassandra_random_partitioner.rb",
+     "examples/cassandra_streaming/struct_loader.rb",
      "examples/contrib/jeans/normalize.rb",
      "examples/contrib/jeans/sizes.rb",
      "examples/corpus/words_to_bigrams.rb",

metadata

@@ -1,13 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: wukong
 version: !ruby/object:Gem::Version 
-  hash: 5
   prerelease: false
   segments: 
   - 1
   - 5
-  - 3
-  version: 1.5.3
+  - 4
+  version: 1.5.4
 platform: ruby
 authors: 
 - Philip (flip) Kromer
@@ -15,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-08-19 00:00:00 -05:00
+date: 2010-11-02 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -26,7 +25,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 13
         segments: 
         - 1
         - 2
@@ -42,7 +40,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
         segments: 
         - 0
         version: "0"
@@ -56,7 +53,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
         segments: 
         - 0
         version: "0"
@@ -70,7 +66,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
         segments: 
         - 0
         version: "0"
@@ -84,7 +79,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
         segments: 
         - 0
         version: "0"
@@ -98,7 +92,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
         segments: 
         - 0
         version: "0"
@@ -124,10 +117,10 @@ files:
 - LICENSE.textile
 - README.textile
 - TODO.textile
-- bin/bootstrap.sh
 - bin/cutc
 - bin/cuttab
 - bin/greptrue
+- bin/hdp-bin
 - bin/hdp-bzip
 - bin/hdp-cat
 - bin/hdp-catd
@@ -169,6 +162,7 @@ files:
 - docpages/UsingWukong-part3-parsing.textile
 - docpages/_config.yml
 - docpages/avro/avro_notes.textile
+- docpages/avro/performance.textile
 - docpages/avro/tethering.textile
 - docpages/bigdata-tips.textile
 - docpages/code/api_response_example.txt
@@ -223,6 +217,21 @@ files:
 - docpages/wutils.textile
 - examples/README.txt
 - examples/binning_percentile_estimator.rb
+- examples/cassandra_streaming/avromapper.rb
+- examples/cassandra_streaming/berlitz_for_cassandra.textile
+- examples/cassandra_streaming/cassandra.avpr
+- examples/cassandra_streaming/cassandra_random_partitioner.rb
+- examples/cassandra_streaming/catter.sh
+- examples/cassandra_streaming/client_interface_notes.textile
+- examples/cassandra_streaming/client_schema.avpr
+- examples/cassandra_streaming/client_schema.textile
+- examples/cassandra_streaming/foofile.avr
+- examples/cassandra_streaming/pymap.sh
+- examples/cassandra_streaming/pyreduce.sh
+- examples/cassandra_streaming/smutation.avpr
+- examples/cassandra_streaming/streamer.sh
+- examples/cassandra_streaming/struct_loader.rb
+- examples/cassandra_streaming/tuning.textile
 - examples/contrib/jeans/README.markdown
 - examples/contrib/jeans/data/normalized_sizes
 - examples/contrib/jeans/data/orders.tsv
@@ -232,8 +241,11 @@ files:
 - examples/corpus/words_to_bigrams.rb
 - examples/count_keys.rb
 - examples/count_keys_at_mapper.rb
+- examples/emr/README-elastic_map_reduce.textile
+- examples/emr/dot_wukong_dir/credentials.json
+- examples/emr/dot_wukong_dir/emr.yaml
+- examples/emr/dot_wukong_dir/emr_bootstrap.sh
 - examples/emr/elastic_mapreduce_example.rb
-- examples/emr/emr.yaml
 - examples/keystore/cassandra_batch_test.rb
 - examples/keystore/conditional_outputter_example.rb
 - examples/network_graph/adjacency_list.rb
@@ -265,6 +277,7 @@ files:
 - lib/wukong/datatypes/fake_types.rb
 - lib/wukong/dfs.rb
 - lib/wukong/encoding.rb
+- lib/wukong/encoding/asciize.rb
 - lib/wukong/extensions.rb
 - lib/wukong/extensions/array.rb
 - lib/wukong/extensions/blank.rb
@@ -297,11 +310,16 @@ files:
 - lib/wukong/schema.rb
 - lib/wukong/script.rb
 - lib/wukong/script/avro_command.rb
+- lib/wukong/script/cassandra_loader_script.rb
 - lib/wukong/script/emr_command.rb
 - lib/wukong/script/hadoop_command.rb
 - lib/wukong/script/local_command.rb
 - lib/wukong/store.rb
 - lib/wukong/store/base.rb
+- lib/wukong/store/cassandra.rb
+- lib/wukong/store/cassandra/streaming.rb
+- lib/wukong/store/cassandra/struct_loader.rb
+- lib/wukong/store/cassandra_model.rb
 - lib/wukong/store/chh_chunked_flat_file_store.rb
 - lib/wukong/store/chunked_flat_file_store.rb
 - lib/wukong/store/conditional_store.rb
@@ -356,7 +374,6 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
       segments: 
       - 0
       version: "0"
@@ -365,7 +382,6 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
       segments: 
       - 0
       version: "0"
@@ -381,6 +397,9 @@ test_files:
 - spec/wukong/encoding_spec.rb
 - spec/wukong/script_spec.rb
 - examples/binning_percentile_estimator.rb
+- examples/cassandra_streaming/avromapper.rb
+- examples/cassandra_streaming/cassandra_random_partitioner.rb
+- examples/cassandra_streaming/struct_loader.rb
 - examples/contrib/jeans/normalize.rb
 - examples/contrib/jeans/sizes.rb
 - examples/corpus/words_to_bigrams.rb