wukong 1.4.11 → 1.4.12

data/bin/hdp-rm

@@ -14,9 +14,9 @@ if [ "$1" == "-r" ] ; then
     shift
     hadoop fs -test -e "$@"
     if [ "$?" == "0" ] ; then
-           # echo "File exists, skipping trash, removing it..."
-      echo hadoop dfs -rmr "$@"
-      exec hadoop dfs -rmr "$@"
+      # echo "File exists, skipping trash, removing it..."
+      echo hadoop dfs -rmr -skipTrash "$@"
+      exec hadoop dfs -rmr -skipTrash "$@"
     fi
   else
     hadoop fs -test -e "$@"

data/lib/wukong/and_pig.rb

@@ -2,10 +2,16 @@ module Enumerable
   #
   # Convert an array of values to a string representing it as a pig tuple
   #
+  # def to_pig_tuple
+  #   map{|*vals| '(' + vals.join(',') + ')' }
+  # end
+
+  #
+  # Convert an array to a pig tuple
+  #
   def to_pig_tuple
-    map{|*vals| '(' + vals.join(',') + ')' }
+    '(' + self.join(',') + ')'
   end
-
   #
   # Convert an array of values to a string pig format
   # Delegates to to_pig_tuple -- see also to_pig_bag
@@ -17,7 +23,29 @@ module Enumerable
   #
   # Convert an array of values to a string representing it as a pig bag
   #
+  # def to_pig_bag
+  #   '{' + self.join(',') + '}'
+  # end
+
+  #
+  # Convert and array of values to a string representing it as a pig bag
+  #
   def to_pig_bag
-    '{' + self.join(',') + '}'
+    '{' + self.map{|*vals| vals.to_pig_tuple}.join(",") + '}'
+  end
+
+  #
+  # Convert a string representing a pig bag into a nested array
+  #
+  def from_pig_bag
+    self.split("),(").map{|t| t.gsub(/[\{\}]/, '').from_pig_tuple} rescue []
+  end
+
+  #
+  # Convert a string representing a pig tuple into an array
+  #
+  def from_pig_tuple
+    self.gsub(/[\(\)]/, '').split(',')
   end
+
 end

data/lib/wukong/extensions/date_time.rb

@@ -1,21 +1,23 @@
 require 'time'
+require 'date'
 DateTime.class_eval do
   #
   # Parses the time but never fails.
   # Return value is always in the UTC time zone.
   #
-  # A flattened time -- a 12-digit YYYYmmddHHMMMSS -- is treated as a UTC
-  # datetime.
+  # A flattened datetime -- a 12-digit YYYYmmddHHMMMSS -- is fixed to the UTC
+  # time zone by parsing it as YYYYmmddHHMMMSSZ <- 'Z' at end
   #
   def self.parse_safely dt
+    return nil if dt.blank?
     begin
       if dt.to_s =~ /\A\d{12}Z?\z/
-        parse(dt+'Z', true).utc
+        parse(dt+'Z', true)
       else
         parse(dt, true).utc
       end
-    rescue StandardError
-      nil
+    rescue StandardError => e
+      Log.info e
     end
   end
 

data/lib/wukong/keystore/tyrant_db.rb

@@ -53,7 +53,6 @@ class TokyoTyrant::Balancer::Base
   def close
     @servers.all?{ |server| server.close rescue nil}
   end
-
 end
 
 module TokyoDbConnection
@@ -67,11 +66,27 @@ module TokyoDbConnection
     ].freeze unless defined?(TokyoDbConnection::TyrantDb::DB_SERVERS)
 
     DB_PORTS = {
-      :user_ids      => 12001,
-      :screen_names  => 12002,
-      :search_ids    => 12003,
-      :tweets_parsed => 12004,
-      :users_parsed  => 12005,
+      :screen_names    => 12002,
+      :search_ids      => 12003,
+      #
+      :tw_user_info    => 14000,
+      :tw_wordbag      => 14101,
+      :tw_influence    => 14102,
+      :tw_trstrank     => 14103,
+      :tw_conversation => 14104,
+      #
+      :screen_names2   => 12004,
+      :search_ids2     => 12005,
+      #
+      :tw_user_info2    => 14200,
+      :tw_wordbag2      => 14201,
+      :tw_influence2    => 14202,
+      :tw_trstrank2     => 14203,
+      :tw_conversation2 => 14204,
+      :tw_strong_links2 => 14205,
+      :tw_word_stats2   => 14206,
+      #
+      :ip_geo_census    => 14400,
     } unless defined?(TokyoDbConnection::TyrantDb::DB_PORTS)
 
     def initialize dataset
@@ -82,8 +97,6 @@ module TokyoDbConnection
       return @db if @db
       port = DB_PORTS[dataset] or raise "Don't know how to reach dataset #{dataset}"
       @db = TokyoTyrant::Balancer::DB.new(DB_SERVERS.map{|s| s+':'+port.to_s})
-      # @db = TokyoTyrant::DB.new(DB_SERVERS.first, port.to_i)
-      @db
     end
 
     def [](*args)      ; db[*args]        ; end

data/wukong.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{wukong}
-  s.version = "1.4.11"
+  s.version = "1.4.12"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Philip (flip) Kromer"]
-  s.date = %q{2010-07-30}
+  s.date = %q{2010-08-03}
   s.description = %q{  Treat your dataset like a:
 
       * stream of lines when it's efficient to process by lines
@@ -245,31 +245,31 @@ Gem::Specification.new do |s|
     "spec/spec_helper.rb",
      "spec/wukong/encoding_spec.rb",
      "spec/wukong/script_spec.rb",
-     "examples/binning_percentile_estimator.rb",
-     "examples/contrib/jeans/normalize.rb",
-     "examples/contrib/jeans/sizes.rb",
-     "examples/corpus/words_to_bigrams.rb",
-     "examples/count_keys.rb",
-     "examples/count_keys_at_mapper.rb",
-     "examples/keystore/cassandra_batch_test.rb",
-     "examples/keystore/conditional_outputter_example.rb",
-     "examples/network_graph/adjacency_list.rb",
-     "examples/network_graph/breadth_first_search.rb",
-     "examples/network_graph/gen_2paths.rb",
-     "examples/network_graph/gen_multi_edge.rb",
-     "examples/network_graph/gen_symmetric_links.rb",
      "examples/pagerank/pagerank.rb",
      "examples/pagerank/pagerank_initialize.rb",
-     "examples/rank_and_bin.rb",
      "examples/sample_records.rb",
      "examples/server_logs/apache_log_parser.rb",
      "examples/server_logs/breadcrumbs.rb",
      "examples/server_logs/user_agent.rb",
+     "examples/corpus/words_to_bigrams.rb",
+     "examples/count_keys.rb",
+     "examples/rank_and_bin.rb",
+     "examples/binning_percentile_estimator.rb",
      "examples/size.rb",
-     "examples/stats/avg_value_frequency.rb",
      "examples/store/chunked_store_example.rb",
+     "examples/network_graph/breadth_first_search.rb",
+     "examples/network_graph/gen_symmetric_links.rb",
+     "examples/network_graph/gen_multi_edge.rb",
+     "examples/network_graph/adjacency_list.rb",
+     "examples/network_graph/gen_2paths.rb",
+     "examples/keystore/cassandra_batch_test.rb",
+     "examples/keystore/conditional_outputter_example.rb",
+     "examples/stats/avg_value_frequency.rb",
+     "examples/contrib/jeans/sizes.rb",
+     "examples/contrib/jeans/normalize.rb",
+     "examples/word_count.rb",
      "examples/stupidly_simple_filter.rb",
-     "examples/word_count.rb"
+     "examples/count_keys_at_mapper.rb"
   ]
 
   if s.respond_to? :specification_version then

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: wukong
 version: !ruby/object:Gem::Version 
-  hash: 17
+  hash: 31
   prerelease: false
   segments: 
   - 1
   - 4
-  - 11
-  version: 1.4.11
+  - 12
+  version: 1.4.12
 platform: ruby
 authors: 
 - Philip (flip) Kromer
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-07-30 00:00:00 -05:00
+date: 2010-08-03 00:00:00 +00:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -367,28 +367,28 @@ test_files:
 - spec/spec_helper.rb
 - spec/wukong/encoding_spec.rb
 - spec/wukong/script_spec.rb
-- examples/binning_percentile_estimator.rb
-- examples/contrib/jeans/normalize.rb
-- examples/contrib/jeans/sizes.rb
-- examples/corpus/words_to_bigrams.rb
-- examples/count_keys.rb
-- examples/count_keys_at_mapper.rb
-- examples/keystore/cassandra_batch_test.rb
-- examples/keystore/conditional_outputter_example.rb
-- examples/network_graph/adjacency_list.rb
-- examples/network_graph/breadth_first_search.rb
-- examples/network_graph/gen_2paths.rb
-- examples/network_graph/gen_multi_edge.rb
-- examples/network_graph/gen_symmetric_links.rb
 - examples/pagerank/pagerank.rb
 - examples/pagerank/pagerank_initialize.rb
-- examples/rank_and_bin.rb
 - examples/sample_records.rb
 - examples/server_logs/apache_log_parser.rb
 - examples/server_logs/breadcrumbs.rb
 - examples/server_logs/user_agent.rb
+- examples/corpus/words_to_bigrams.rb
+- examples/count_keys.rb
+- examples/rank_and_bin.rb
+- examples/binning_percentile_estimator.rb
 - examples/size.rb
-- examples/stats/avg_value_frequency.rb
 - examples/store/chunked_store_example.rb
-- examples/stupidly_simple_filter.rb
+- examples/network_graph/breadth_first_search.rb
+- examples/network_graph/gen_symmetric_links.rb
+- examples/network_graph/gen_multi_edge.rb
+- examples/network_graph/adjacency_list.rb
+- examples/network_graph/gen_2paths.rb
+- examples/keystore/cassandra_batch_test.rb
+- examples/keystore/conditional_outputter_example.rb
+- examples/stats/avg_value_frequency.rb
+- examples/contrib/jeans/sizes.rb
+- examples/contrib/jeans/normalize.rb
 - examples/word_count.rb
+- examples/stupidly_simple_filter.rb
+- examples/count_keys_at_mapper.rb