wukong 1.5.4 → 2.0.0

metadata

@@ -1,12 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: wukong
 version: !ruby/object:Gem::Version 
-  prerelease: false
+  hash: 15
+  prerelease: 
   segments: 
-  - 1
-  - 5
-  - 4
-  version: 1.5.4
+  - 2
+  - 0
+  - 0
+  version: 2.0.0
 platform: ruby
 authors: 
 - Philip (flip) Kromer
@@ -14,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-11-02 00:00:00 -05:00
+date: 2011-01-29 00:00:00 -06:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -25,6 +26,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 13
         segments: 
         - 1
         - 2
@@ -40,6 +42,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 0
         version: "0"
@@ -53,6 +56,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 0
         version: "0"
@@ -66,6 +70,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 0
         version: "0"
@@ -79,6 +84,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 0
         version: "0"
@@ -92,6 +98,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 0
         version: "0"
@@ -144,7 +151,9 @@ files:
 - bin/hdp-sync
 - bin/hdp-wc
 - bin/md5sort
+- bin/setcat
 - bin/tabchar
+- bin/uniq-ord
 - bin/uniqc
 - bin/wu-date
 - bin/wu-datetime
@@ -216,38 +225,23 @@ files:
 - docpages/usage.textile
 - docpages/wutils.textile
 - examples/README.txt
-- examples/binning_percentile_estimator.rb
-- examples/cassandra_streaming/avromapper.rb
-- examples/cassandra_streaming/berlitz_for_cassandra.textile
-- examples/cassandra_streaming/cassandra.avpr
-- examples/cassandra_streaming/cassandra_random_partitioner.rb
-- examples/cassandra_streaming/catter.sh
-- examples/cassandra_streaming/client_interface_notes.textile
-- examples/cassandra_streaming/client_schema.avpr
-- examples/cassandra_streaming/client_schema.textile
-- examples/cassandra_streaming/foofile.avr
-- examples/cassandra_streaming/pymap.sh
-- examples/cassandra_streaming/pyreduce.sh
-- examples/cassandra_streaming/smutation.avpr
-- examples/cassandra_streaming/streamer.sh
-- examples/cassandra_streaming/struct_loader.rb
-- examples/cassandra_streaming/tuning.textile
 - examples/contrib/jeans/README.markdown
 - examples/contrib/jeans/data/normalized_sizes
 - examples/contrib/jeans/data/orders.tsv
 - examples/contrib/jeans/data/sizes
 - examples/contrib/jeans/normalize.rb
 - examples/contrib/jeans/sizes.rb
+- examples/corpus/bucket_counter.rb
+- examples/corpus/dbpedia_abstract_to_sentences.rb
+- examples/corpus/sentence_coocurrence.rb
 - examples/corpus/words_to_bigrams.rb
-- examples/count_keys.rb
-- examples/count_keys_at_mapper.rb
-- examples/emr/README-elastic_map_reduce.textile
+- examples/emr/README.textile
 - examples/emr/dot_wukong_dir/credentials.json
 - examples/emr/dot_wukong_dir/emr.yaml
 - examples/emr/dot_wukong_dir/emr_bootstrap.sh
 - examples/emr/elastic_mapreduce_example.rb
-- examples/keystore/cassandra_batch_test.rb
-- examples/keystore/conditional_outputter_example.rb
+- examples/ignore_me/counting.rb
+- examples/ignore_me/grouper.rb
 - examples/network_graph/adjacency_list.rb
 - examples/network_graph/breadth_first_search.rb
 - examples/network_graph/gen_2paths.rb
@@ -258,15 +252,16 @@ files:
 - examples/pagerank/pagerank.rb
 - examples/pagerank/pagerank_initialize.rb
 - examples/pagerank/run_pagerank.sh
-- examples/rank_and_bin.rb
 - examples/sample_records.rb
 - examples/server_logs/apache_log_parser.rb
 - examples/server_logs/breadcrumbs.rb
+- examples/server_logs/logline.rb
 - examples/server_logs/user_agent.rb
 - examples/size.rb
 - examples/stats/avg_value_frequency.rb
+- examples/stats/binning_percentile_estimator.rb
 - examples/stats/data/avg_value_frequency.tsv
-- examples/store/chunked_store_example.rb
+- examples/stats/rank_and_bin.rb
 - examples/stupidly_simple_filter.rb
 - examples/word_count.rb
 - lib/wukong.rb
@@ -275,7 +270,6 @@ files:
 - lib/wukong/datatypes.rb
 - lib/wukong/datatypes/enum.rb
 - lib/wukong/datatypes/fake_types.rb
-- lib/wukong/dfs.rb
 - lib/wukong/encoding.rb
 - lib/wukong/encoding/asciize.rb
 - lib/wukong/extensions.rb
@@ -295,18 +289,8 @@ files:
 - lib/wukong/extensions/struct.rb
 - lib/wukong/extensions/symbol.rb
 - lib/wukong/filename_pattern.rb
-- lib/wukong/keystore/cassandra_conditional_outputter.rb
-- lib/wukong/keystore/redis_db.rb
-- lib/wukong/keystore/tyrant_db.rb
-- lib/wukong/keystore/tyrant_notes.textile
 - lib/wukong/logger.rb
-- lib/wukong/models/graph.rb
-- lib/wukong/monitor.rb
-- lib/wukong/monitor/chunked_store.rb
-- lib/wukong/monitor/periodic_logger.rb
-- lib/wukong/monitor/periodic_monitor.rb
 - lib/wukong/periodic_monitor.rb
-- lib/wukong/rdf.rb
 - lib/wukong/schema.rb
 - lib/wukong/script.rb
 - lib/wukong/script/avro_command.rb
@@ -334,23 +318,22 @@ files:
 - lib/wukong/streamer.rb
 - lib/wukong/streamer/accumulating_reducer.rb
 - lib/wukong/streamer/base.rb
-- lib/wukong/streamer/cassandra_streamer.rb
-- lib/wukong/streamer/count_keys.rb
-- lib/wukong/streamer/count_lines.rb
 - lib/wukong/streamer/counting_reducer.rb
-- lib/wukong/streamer/em_streamer.rb
 - lib/wukong/streamer/filter.rb
 - lib/wukong/streamer/line_streamer.rb
 - lib/wukong/streamer/list_reducer.rb
-- lib/wukong/streamer/preprocess_with_pipe_streamer.rb
 - lib/wukong/streamer/rank_and_bin_reducer.rb
 - lib/wukong/streamer/record_streamer.rb
+- lib/wukong/streamer/reducer.rb
 - lib/wukong/streamer/set_reducer.rb
 - lib/wukong/streamer/struct_streamer.rb
 - lib/wukong/streamer/summing_reducer.rb
 - lib/wukong/streamer/uniq_by_last_reducer.rb
 - lib/wukong/typed_struct.rb
-- lib/wukong/wukong_class.rb
+- old/cassandra_streaming/berlitz_for_cassandra.textile
+- old/cassandra_streaming/client_interface_notes.textile
+- old/cassandra_streaming/client_schema.textile
+- old/cassandra_streaming/tuning.textile
 - spec/data/a_atsigns_b.tsv
 - spec/data/a_follows_b.tsv
 - spec/data/tweet.tsv
@@ -365,8 +348,8 @@ homepage: http://mrflip.github.com/wukong
 licenses: []
 
 post_install_message: 
-rdoc_options: 
-- --charset=UTF-8
+rdoc_options: []
+
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
@@ -374,6 +357,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
@@ -382,32 +366,27 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.7
+rubygems_version: 1.4.2
 signing_key: 
 specification_version: 3
 summary: Hadoop Streaming for Ruby. Wukong makes Hadoop so easy a chimpanzee can use it, yet handles terabyte-scale computation with ease.
 test_files: 
-- spec/spec_helper.rb
-- spec/wukong/encoding_spec.rb
-- spec/wukong/script_spec.rb
-- examples/binning_percentile_estimator.rb
-- examples/cassandra_streaming/avromapper.rb
-- examples/cassandra_streaming/cassandra_random_partitioner.rb
-- examples/cassandra_streaming/struct_loader.rb
 - examples/contrib/jeans/normalize.rb
 - examples/contrib/jeans/sizes.rb
+- examples/corpus/bucket_counter.rb
+- examples/corpus/dbpedia_abstract_to_sentences.rb
+- examples/corpus/sentence_coocurrence.rb
 - examples/corpus/words_to_bigrams.rb
-- examples/count_keys.rb
-- examples/count_keys_at_mapper.rb
 - examples/emr/elastic_mapreduce_example.rb
-- examples/keystore/cassandra_batch_test.rb
-- examples/keystore/conditional_outputter_example.rb
+- examples/ignore_me/counting.rb
+- examples/ignore_me/grouper.rb
 - examples/network_graph/adjacency_list.rb
 - examples/network_graph/breadth_first_search.rb
 - examples/network_graph/gen_2paths.rb
@@ -415,13 +394,17 @@ test_files:
 - examples/network_graph/gen_symmetric_links.rb
 - examples/pagerank/pagerank.rb
 - examples/pagerank/pagerank_initialize.rb
-- examples/rank_and_bin.rb
 - examples/sample_records.rb
 - examples/server_logs/apache_log_parser.rb
 - examples/server_logs/breadcrumbs.rb
+- examples/server_logs/logline.rb
 - examples/server_logs/user_agent.rb
 - examples/size.rb
 - examples/stats/avg_value_frequency.rb
-- examples/store/chunked_store_example.rb
+- examples/stats/binning_percentile_estimator.rb
+- examples/stats/rank_and_bin.rb
 - examples/stupidly_simple_filter.rb
 - examples/word_count.rb
+- spec/spec_helper.rb
+- spec/wukong/encoding_spec.rb
+- spec/wukong/script_spec.rb

data/examples/cassandra_streaming/avromapper.rb

@@ -1,85 +0,0 @@
-#!/usr/bin/env ruby
-
-# To install avro gem
-#   cd avro/lang/ruby ; gem package ; sudo gem install pkg/avro-1.4.0.pre1.gem
-
-require 'rubygems'
-require 'avro'
-require 'wukong'
-require 'wukong/periodic_monitor'
-
-Settings.define :cassandra_avro_schema, :default => ('/usr/local/share/cassandra/interface/avro/cassandra.avpr')
-Settings.define :cassandra_thrift_uri,  :default => `hostname`.chomp.strip+':9160'
-Settings.define :log_interval, :default => 10_000
-
-class AvroStreamer < Wukong::Streamer::RecordStreamer
-  def initialize *args
-    super(*args)
-    @writer = SmutWriter.new
-    @log = PeriodicMonitor.new
-  end
-
-  def process word, count, *_
-    @writer.write_directly(word, 'count', count)
-    @log.periodically( word, count )
-  end
-end
-
-class SmutWriter
-  # Reads in the protocol schema
-  # creates the necessary encoder and writer.
-  def initialize
-    schema_file = Settings.cassandra_avro_schema
-    @proto  = Avro::Protocol.parse(File.read(schema_file))
-    @schema = @proto.types.detect{|schema| schema.name == 'StreamingMutation'}
-    @enc    = Avro::IO::BinaryEncoder.new($stdout)
-    @writer = Avro::IO::DatumWriter.new(@schema)
-  end
-
-  # Directly write the simplified StreamingMutation schema; uses patch from @stuhood
-  def write_directly key, col_name, value
-    @enc.write_bytes(key)
-    @enc.write_bytes(col_name)
-    @enc.write_bytes(value)
-    @enc.write_long(Time.epoch_microseconds)
-    @enc.write_int(0)
-  end
-
-  # Write using the datumwriter
-  def write key, col_name, value
-    @writer.write(smutation(key, col_name, value), @enc)
-  end
-
-  # Simplified StreamingMutation schema uses patch from @stuhood
-  def smutation key, name, value
-    {
-      'key'       => key,
-      'name'      => name.to_s,
-      'value'     => value.to_s,
-      'timestamp' => Time.epoch_microseconds,
-      'ttl'       => 0
-    }
-  end
-
-  # The StreamingMutation schema defined in trunk.
-  # Becomes monstrously inefficient due to implementation of unions.
-  def smutation_from_trunk key, name, value
-    {
-      'key'      => key,
-      'mutation' => { 'column_or_supercolumn' => { 'column' => {
-            'name'  => name.to_s,
-            'value' => value.to_s,
-            'clock' => { 'timestamp' => Time.epoch_microseconds },
-            'ttl'   => 0
-          }}}
-    }
-  end
-end
-
-Time.class_eval do
-  def self.epoch_microseconds
-    (Time.now.utc.to_i * 1_000_000)
-  end
-end
-
-Wukong::Script.new(AvroStreamer, nil, :map_speculative => false).run