wukong 1.5.3 → 1.5.4

data/examples/cassandra_streaming/streamer.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+input_file="$1" 		 ; shift
+output_file="$1" 		 ; shift
+map_script=${1-/bin/cat}	 ; shift
+reduce_script=${1-/usr/bin/uniq} ; shift
+
+dest_keyspace=${dest_keyspace-soc_net_tw}
+dest_col_family=${dest_col_family-Wordbag}
+
+hostname=`hostname`
+
+# Path to cassandra and hadoop dirs
+script_dir=$(readlink -f `dirname $0`)
+CASSANDRA_HOME=${CASSANDRA_HOME-/usr/local/share/cassandra}
+HADOOP_HOME=${HADOOP_HOME-/usr/lib/hadoop}
+avro_file=${avro_file-$CASSANDRA_HOME/interface/avro/cassandra.avpr}
+
+ARCHIVES=`/bin/ls -1 $CASSANDRA_HOME/build/apache-cassandra*.jar`
+for jar in `/bin/ls -1 $CASSANDRA_HOME/build/lib/jars/*.jar $CASSANDRA_HOME/lib/*.jar`; do
+    ARCHIVES=$ARCHIVES,$jar
+done
+
+${HADOOP_HOME}/bin/hadoop                                                                        \
+     jar ${HADOOP_HOME}/contrib/streaming/hadoop-*streaming*.jar                                 \
+    -D stream.map.output=cassandra_avro_output                                                   \
+    -D stream.io.identifier.resolver.class=org.apache.cassandra.hadoop.streaming.AvroResolver    \
+    -D cassandra.output.keyspace="$dest_keyspace"                                                \
+    -D cassandra.output.columnfamily="$dest_col_family"                                          \
+    -D cassandra.thrift.address=10.204.41.193,10.204.30.11,10.204.58.238,10.204.239.133,10.196.191.31,10.204.103.21,10.202.74.223,10.202.143.95 \
+    -D cassandra.partitioner.class=org.apache.cassandra.dht.RandomPartitioner                    \
+    -D cassandra.thrift.port=9160                                                                \
+    -D mapreduce.output.columnfamilyoutputformat.batch.threshold=1024                            \
+    -D mapred.reduce.tasks=0                                                                     \
+    -D mapred.map.tasks.speculative.execution=false                                              \
+    -libjars $ARCHIVES                                                                           \
+    -file $avro_file                                                                             \
+    -outputformat org.apache.cassandra.hadoop.ColumnFamilyOutputFormat                           \
+    -mapper  	 "ruby $script_dir/avromapper.rb --map "                                         \
+    -input       "$input_file"                                                                   \
+    -output  	 "$output_file"                                                                  \
+    "$@"
+
+    # -D cassandra.thrift.address=10.204.54.190,10.244.42.31,10.244.42.176,10.244.42.112,10.244.42.143,10.244.42.79,10.244.42.4,10.204.53.166 \
+    # -D cassandra.thrift.address=10.204.221.230,10.243.79.223,10.245.19.159,10.242.154.159,10.242.153.155,10.242.153.203 \
+
+
+# cat /tmp/mj-flip/chimchim-info.log | cut -f5 | ruby -e 'puts $stdin.readlines.map{|l| l.chomp.gsub(/ip-([0-9\-]+)\..*/,"\\1").gsub(/-/,".") }.join(",")'
+
+
+

data/examples/cassandra_streaming/struct_loader.rb

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'wukong'
+require 'wukong/periodic_monitor'
+require 'wukong/store/cassandra'
+require 'wukong/script/cassandra_loader_script'
+
+Settings.use :commandline
+Settings.define :log_interval,    :default => 1
+Settings.cassandra_keyspace   = 'soc_net_tw'
+Settings.cassandra_col_family = 'TwitterUser'
+Settings.cassandra_hosts      = "ip-10-204-41-193.ec2.internal:9160,ip-10-204-30-11.ec2.internal:9160,ip-10-204-58-238.ec2.internal:9160,ip-10-204-239-133.ec2.internal:9160,ip-10-196-191-31.ec2.internal:9160,ip-10-204-103-21.ec2.internal:9160,ip-10-202-74-223.ec2.internal:9160,ip-10-202-143-95.ec2.internal:9160"
+Settings.resolve!
+
+require 'cassandra/0.7'
+require 'wuclan/twitter' ; include Wuclan::Twitter
+require 'wuclan/twitter/cassandra_db'
+require 'wukong/store/cassandra/streaming'
+
+# hdp-catd s3://s3hdfs.infinitemonkeys.info/data/sn/tw/fixd/objects/twitter_user | head
+
+# CassandraScript.new(Wukong::Store::Cassandra::StructLoader, nil).run
+Wukong::CassandraScript.new(Wukong::Store::Cassandra::StructLoader, nil).run
+

data/examples/cassandra_streaming/tuning.textile

@@ -0,0 +1,73 @@
+
+
+Start
+
+5 c1.xlarge
+  2000 writes/sec
+  40   clients
+
+4 m2.xlarge
+
+      :java_max_heap                => "12500M",          #
+      # :flush_data_buffer_size     => 32,                # 32,
+      # :flush_index_buffer_size    => 8,                 # 8,
+      # :binary_memtable_throughput => 256,               # 256,
+      # :memtable_flush_after       => 60,                # 60,
+      # :memtable_throughput        => 64,                # 64,
+      # :memtable_ops               => 0.3,               # 0.3,
+      # :column_index_size          => 64,                # 64,
+      # :in_memory_compaction_limit => 64                 # 64
+      :concurrent_reads             => 8,                 # 8
+      :concurrent_writes            => 250,               # 32
+
+/usr/bin/java -ea                                                       \
+    -Xms128M                                                            \
+    -Xmx12500M                                                          \
+    -XX:TargetSurvivorRatio=90                                          \
+    -XX:+AggressiveOpts                                                 \
+    -XX:+UseParNewGC                                                    \
+    -XX:+UseConcMarkSweepGC                                             \
+    -XX:+CMSParallelRemarkEnabled                                       \
+    -XX:+HeapDumpOnOutOfMemoryError                                     \
+    -XX:SurvivorRatio=128                                               \
+    -XX:MaxTenuringThreshold=0                                          \
+    -Djava.rmi.server.hostname=ec2-184-73-20-37.compute-1.amazonaws.com \
+    -Dcom.sun.management.jmxremote.port=12345                           \
+    -Dcom.sun.management.jmxremote.ssl=false                            \
+    -Dcom.sun.management.jmxremote.authenticate=false                   \
+    -Dcassandra                                                         \
+    -Dstorage-config=/etc/cassandra                                     \
+    -Dcassandra-foreground=yes                                          \
+    -cp /etc/cassandra:/usr/local/share/cassandra/build/classes:/usr/local/share/cassandra/lib/antlr-3.1.3.jar:/usr/local/share/cassandra/lib/avro-1.3.3-sources~cust1.jar:/usr/local/share/cassandra/lib/avro-1.3.3~cust2.jar:/usr/local/share/cassandra/lib/clhm-production.jar:/usr/local/share/cassandra/lib/commons-cli-1.1.jar:/usr/local/share/cassandra/lib/commons-codec-1.2.jar:/usr/local/share/cassandra/lib/commons-collections-3.2.1.jar:/usr/local/share/cassandra/lib/commons-lang-2.4.jar:/usr/local/share/cassandra/lib/guava-r05.jar:/usr/local/share/cassandra/lib/hadoop-core-0.20.1.jar:/usr/local/share/cassandra/lib/high-scale-lib.jar:/usr/local/share/cassandra/lib/jackson-core-asl-1.4.0.jar:/usr/local/share/cassandra/lib/jackson-mapper-asl-1.4.0.jar:/usr/local/share/cassandra/lib/jetty-6.1.21.jar:/usr/local/share/cassandra/lib/jetty-util-6.1.21.jar:/usr/local/share/cassandra/lib/jline-0.9.94.jar:/usr/local/share/cassandra/lib/json-simple-1.1.jar:/usr/local/share/cassandra/lib/jug-2.0.0.jar:/usr/local/share/cassandra/lib/libthrift-r959516.jar:/usr/local/share/cassandra/lib/log4j-1.2.16.jar:/usr/local/share/cassandra/lib/servlet-api-2.5-20081211.jar:/usr/local/share/cassandra/lib/slf4j-api-1.5.8.jar:/usr/local/share/cassandra/lib/slf4j-log4j12-1.5.8.jar:/usr/local/share/cassandra/lib/snakeyaml-1.6.jar\
+    org.apache.cassandra.thrift.CassandraDaemon
+
+
+avg-cpu:  %user   %nice %system %iowait  %steal   %idle
+          81.83    0.00    1.96    0.00    0.00   16.21
+
+Device:            tps   Blk_read/s   Blk_wrtn/s   Blk_read   Blk_wrtn
+sda1            155.12         9.45     11450.39         48      58168
+sdb               2.76         0.00        22.05          0        112
+
+avg-cpu:  %user   %nice %system %iowait  %steal   %idle
+          83.72    0.00    3.80    0.20    0.00   12.29
+
+Device:            tps   Blk_read/s   Blk_wrtn/s   Blk_read   Blk_wrtn
+sda1             66.53         1.59      3921.91          8      19688
+sdb             100.20         0.00      6686.85          0      33568
+
+avg-cpu:  %user   %nice %system %iowait  %steal   %idle
+          66.40    0.00    5.00    0.80    0.40   27.40
+
+Device:            tps   Blk_read/s   Blk_wrtn/s   Blk_read   Blk_wrtn
+sda1              2.40         0.00        19.20          0         96
+sdb             186.80         0.00     15318.40          0      76592
+
+avg-cpu:  %user   %nice %system %iowait  %steal   %idle
+          80.98    0.00    6.08    1.99    0.00   10.96
+
+Device:            tps   Blk_read/s   Blk_wrtn/s   Blk_read   Blk_wrtn
+sda1            113.97         0.00      7426.75          0      37208
+sdb             360.28         1.60     29232.73          8     146456
+
+    

data/examples/emr/README-elastic_map_reduce.textile

@@ -0,0 +1,26 @@
+
+# Download the Amazon elastic-mapreduce runner from http://elasticmapreduce.s3.amazonaws.com/elastic-mapreduce-ruby.zip
+
+# Create a bucket and path to hold your EMR logs, scripts and other ephemera. For instance you might choose 'emr.yourdomain.com' as the bucket and '/wukong' as a scoping path within that bucket. In that case you will refer to it with a path like s3n://emr.yourdomain.com/wukong (see notes below about s3n:// vs. s3:// URLs).
+
+# Copy the contents of wukong/examples/emr/dot_wukong_dir to ~/.wukong
+# Edit emr.yaml -- it has instructions for the 
+
+
+
+
+
+h3. s3n:// vs. s3:// URLs
+
+Many external tools use a URI convention to address files in S3; they typically use the 's3://' scheme, which makes a lot of sense:
+  s3://emr.yourcompany.com/wukong/happy_job_1/logs/whatever-20100808.log
+
+Hadoop can maintain an HDFS on the Amazon S3: it uses a block structure and has optimizations for streaming, no file size limitation, and other goodness. However, only hadoop tools can interpret the contents of those blocks -- to everything else it just looks like a soup of blocks labelled block_-8675309 and so forth.  Hadoop unfortunately chose the 's3://' scheme for URIs in this filesystem:
+  s3://s3hdfs.yourcompany.com/path/to/data
+
+Hadoop is happy to read s3 native files -- 'native' as in, you can look at them with a browser and upload them an download them with any S3 tool out there. There's a 5GB limit on file size, and in some cases a performance hit (but not in our experience enough to worry about).  You refer to these files with the 's3n://' scheme ('n' as in 'native'):
+  s3n://emr.yourcompany.com/wukong/happy_job_1/code/happy_job_1-mapper.rb
+  s3n://emr.yourcompany.com/wukong/happy_job_1/code/happy_job_1-reducer.rb
+  s3n://emr.yourcompany.com/wukong/happy_job_1/logs/whatever-20100808.log
+
+Wukong will coerce things to the right scheme when it knows what that scheme should be (eg. code should be s3n://). It will otherwise leave the path alone. Specifically, if you use a URI scheme for input and output paths you must use 's3n://' for normal s3 files.

data/examples/emr/dot_wukong_dir/credentials.json

@@ -0,0 +1,7 @@
+{
+  "key-pair":      "gibbon",
+  "key-pair-file": "/home/your/.wukong/keypairs/gibbon.pem",
+  "access-id":     "YOURACCESSID",
+  "private-key":   "YOURPRIVATEKEY",
+  "region":        "us-east-1",
+}

data/examples/emr/{emr.yaml → dot_wukong_dir/emr.yaml}

@@ -2,51 +2,68 @@
 # Elastic MapReduce config in wukong
 #
 
+# ===========================================================================
 #
 # Infrastructure options
 #
 
-# == Fill all your information into yet another file with your amazon key Sorry
-#    that it needs to be in so many stupid places, nobody can agree on a
+# == Fill all your information into yet another file with your amazon key
+#    It needs to be in so many stupid places because nobody can agree on a
 #    filename or format.
+#
 :emr_credentials_file:          ~/.wukong/credentials.json
+
 #
-# == Set the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY env vars, or enter them here:
+# == Set the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY env vars, or enter them here
+#  
 # :access_key:                  ASDFAHKHASDF
 # :secret_access_key:           ADSGHASDFJASDFASDF
+
+# == Path to your keypair file.
 #
-# == Path to your keypair file. 
 :key_pair_file:                 ~/.wukong/keypairs/gibbon.pem
-# == Keypair will be named after your file, or force the name:
+
+# == Keypair will be named after your file, or force the name
+#
 # :key_pair:                    ~
 
 # == Path to the Amazon elastic-mapreduce runner. Get a copy from
 #    http://elasticmapreduce.s3.amazonaws.com/elastic-mapreduce-ruby.zip
+#
 :emr_runner:                    ~/ics/hadoop/elastic-mapreduce/elastic-mapreduce
 
+# ===========================================================================
+#
+# Remote Paths
+#
+
+# == Wukong is opinionated about the paths and locations of scripts and
+#    everything. It will organize files by job name within the following path:
+#
+:emr_root:                      s3://s3n.infinitemonkeys.info/emr
+
+# == If you specify the :emr_data_root path, then relative pathnames -- ones that
+#    do not look like a URI (s3://yadda/yada) and do not start with a '/' -- will
+#    be prefixed with this path prefix.
+:emr_data_root:                 s3n://s3n.infinitemonkeys.info/data
+
+
+# ===========================================================================
 #
 # Cluster Config
 #
 :num_instances:                 1
-:instance_type:                 m2.xlarge
+:instance_type:                 m1.small
 :master_instance_type:          ~
 :hadoop_version:                '0.20'
 :availability_zone:             us-east-1b
 
+# ===========================================================================
 #
 # Running and reporting options
 #
-:alive:                         false
+:alive:                         true
 :enable_debugging:              true
 :emr_runner_verbose:            true
 :emr_runner_debug:              ~
 :step_action:                   CANCEL_AND_WAIT         # CANCEL_AND_WAIT, TERMINATE_JOB_FLOW or CONTINUE
-
-#
-# Remote Paths
-#
-# Wukong is opinionated about the paths and locations of scripts and
-# everything. Make an S3 bucket and let the wookiee win -- or hack
-# lib/wukong/script/emr_command.rb to be more flexible and send us back a patch.
-#
-:emr_root:                      s3n://emr.infinitemonkeys.info

data/{bin/bootstrap.sh → examples/emr/dot_wukong_dir/emr_bootstrap.sh}

@@ -24,7 +24,7 @@ sudo apt-get install -y unzip build-essential git-core ruby ruby1.8-dev rubygems
 echo "`date` Unchaining rubygems from the tyrrany of ubuntu" 
 sudo gem install --no-rdoc --no-ri rubygems-update --version=1.3.7 ; sudo /var/lib/gems/1.8/bin/update_rubygems; sudo gem update --no-rdoc --no-ri --system ; gem --version ;
 
-echo "`date` Installing wukong gems" 
+echo "`date` Installing wukong and related gems" 
 sudo gem install --no-rdoc --no-ri addressable extlib htmlentities configliere yard wukong right_aws uuidtools cheat
 sudo gem list 
 

data/examples/emr/elastic_mapreduce_example.rb

@@ -24,4 +24,5 @@ class FooStreamer < Wukong::Streamer::LineStreamer
   end
 end
 
+Settings.resolve!
 Wukong::Script.new(FooStreamer, FooStreamer).run

data/lib/wukong/encoding/asciize.rb

@@ -0,0 +1,108 @@
+# -*- coding: utf-8 -*-
+#
+# http://www.jroller.com/obie/tags/unicode
+# http://www.unicode.org/faq/casemap_charprop.html
+# http://unicode.org/reports/tr10/#Conformance
+# http://intertwingly.net/stories/2009/11/30/asciize.rb
+# http://blog.stevenlevithan.com/archives/javascript-regex-and-unicode
+#
+# http://xregexp.com/tests/unicode.html
+
+class String
+  #
+  # Taken from http://intertwingly.net/stories/2009/11/30/asciize.rb
+  #
+  def asciize(name)
+    if name =~ /[^\x00-\x7F]/
+      # digraphs.  May be culturally sensitive
+      name.gsub! /\xc3\x9f/, 'ss'
+      name.gsub! /\xc3\xa4|a\xcc\x88/, 'ae'
+      name.gsub! /\xc3\xa5|a\xcc\x8a/, 'aa'
+      name.gsub! /\xc3\xa6/, 'ae'
+      name.gsub! /\xc3\xb1|n\xcc\x83/, 'ny'
+      name.gsub! /\xc3\xb6|o\xcc\x88/, 'oe'
+      name.gsub! /\xc3\xbc|u\xcc\x88/, 'ue'
+
+      # latin 1
+      name.gsub! /\xc3[\xa0-\xa5]/, 'a'
+      name.gsub! /\xc3\xa7/, 'c'
+      name.gsub! /\xc3[\xa8-\xab]/, 'e'
+      name.gsub! /\xc3[\xac-\xaf]/, 'i'
+      name.gsub! /\xc3[\xb2-\xb6]|\xc3\xb8/, 'o'
+      name.gsub! /\xc3[\xb9-\xbc]/, 'u'
+      name.gsub! /\xc3[\xbd\xbf]/, 'y'
+
+      # Latin Extended-A
+      name.gsub! /\xc4[\x80-\x85]/, 'a'
+      name.gsub! /\xc4[\x86-\x8d]/, 'c'
+      name.gsub! /\xc4[\x8e-\x91]/, 'd'
+      name.gsub! /\xc4[\x92-\x9b]/, 'e'
+      name.gsub! /\xc4[\x9c-\xa3]/, 'g'
+      name.gsub! /\xc4[\xa4-\xa7]/, 'h'
+      name.gsub! /\xc4[\xa8-\xb1]/, 'i'
+      name.gsub! /\xc4[\xb2-\xb3]/, 'ij'
+      name.gsub! /\xc4[\xb4-\xb5]/, 'j'
+      name.gsub! /\xc4[\xb6-\xb8]/, 'k'
+      name.gsub! /\xc4[\xb9-\xff]|\xc5[\x80-\x82]/, 'l'
+      name.gsub! /\xc5[\x83-\x8b]/, 'n'
+      name.gsub! /\xc5[\x8c-\x91]/, 'o'
+      name.gsub! /\xc5[\x92-\x93]/, 'oe'
+      name.gsub! /\xc5[\x94-\x99]/, 'r'
+      name.gsub! /\xc5[\x9a-\xa2]/, 's'
+      name.gsub! /\xc5[\xa2-\xa7]/, 't'
+      name.gsub! /\xc5[\xa8-\xb3]/, 'u'
+      name.gsub! /\xc5[\xb4-\xb5]/, 'w'
+      name.gsub! /\xc5[\xb6-\xb8]/, 'y'
+      name.gsub! /\xc5[\xb9-\xbe]/, 'z'
+
+      # denormalized diacritics
+      name.gsub! /\xcc[\x80-\xff]|\xcd[\x80-\xaf]/, ''
+    end
+
+    name.gsub /[^\w]+/, '-'
+  end
+
+end
+
+if __FILE__ == $PROGRAM_NAME
+  i18n = "I\xc3\xb1t\xc3\xabrn\xc3\xa2ti\xc3\xb4n\xc3\xa0liz\xc3\xa6ti\xc3\xb8n"
+  puts "#{i18n} => #{i18n.asciize}"
+end
+
+# http://www.jroller.com/obie/tags/unicode
+#
+# require 'iconv'
+# require 'unicode'
+# 
+# class String
+#   
+#   def to_ascii
+#     # split in muti-byte aware fashion and translate characters over 127
+#     # and dropping characters not in the translation hash
+#     self.chars.split('').collect { |c| (c[0] <= 127) ? c : translation_hash[c[0]] }.join
+#   end
+#     
+#   def to_url_format
+#     url_format = self.to_ascii
+#     url_format = url_format.gsub(/[^A-Za-z0-9]/, '') # all non-word
+#     url_format.downcase!
+#     url_format
+#   end
+#   
+#   protected
+#   
+#     def translation_hash
+#       @@translation_hash ||= setup_translation_hash      
+#     end
+#     
+#     def setup_translation_hash
+#       accented_chars   = "ÀÁÂÃÄÅÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝàáâãäåçèéêëìíîïñòóôõöøùúûüý"
+#       unaccented_chars = "AAAAAACEEEEIIIIDNOOOOOxOUUUUYaaaaaaceeeeiiiinoooooouuuuy"
+#   
+#       translation_hash = Hash.zip(accented_chars.chars, unaccented_chars.chars)
+#       translation_hash["Æ".chars[0]] = 'AE'
+#       translation_hash["æ".chars[0]] = 'ae'
+#       translation_hash
+#     end
+#     
+# end

data/lib/wukong/extensions/date_time.rb

@@ -1,23 +1,31 @@
 require 'time'
 require 'date'
-DateTime.class_eval do
+
+class Time
+  # strftime() format to flatten a date
+  FLAT_FORMAT = "%Y%m%d%H%M%S"
+  # Flatten
+  def to_flat
+    utc.strftime(FLAT_FORMAT)
+  end
+
   #
   # Parses the time but never fails.
   # Return value is always in the UTC time zone.
   #
-  # A flattened datetime -- a 12-digit YYYYmmddHHMMMSS -- is fixed to the UTC
+  # A flattened datetime -- a 14-digit YYYYmmddHHMMMSS -- is fixed to the UTC
   # time zone by parsing it as YYYYmmddHHMMMSSZ <- 'Z' at end
   #
   def self.parse_safely dt
     return nil if dt.blank?
     begin
-      if dt.to_s =~ /\A\d{12}Z?\z/
-        parse(dt+'Z', true)
-      else
-        parse(dt, true).utc
+      case
+      when dt.is_a?(Time)               then dt.utc
+      when (dt.to_s =~ /\A\d{14}\z/)    then parse(dt.to_s+'Z', true)
+      else                                   parse(dt.to_s,     true).utc
       end
     rescue StandardError => e
-      Log.info e
+      Log.debug e
     end
   end
 
@@ -25,3 +33,21 @@ DateTime.class_eval do
     parse_safely(str).to_flat
   end
 end
+
+class DateTime < Date
+  # strftime() format to flatten a date
+  FLAT_FORMAT = "%Y%m%d%H%M%S"
+  # Flatten
+  def to_flat
+    strftime(FLAT_FORMAT)
+  end
+end
+
+class Date
+  # strftime() format to flatten a date
+  FLAT_FORMAT = "%Y%m%d"
+  # Flatten
+  def to_flat
+    strftime(FLAT_FORMAT)
+  end
+end