wukong 1.5.3 → 1.5.4

data/CHANGELOG.textile

@@ -1,3 +1,7 @@
+h2. Wukong v1.5.4
+
+* EMR support now works very well
+
 h2. Wukong v1.5.3
 
 * A couple of bugfixes. Sorry about that.

data/bin/hdp-bin

@@ -0,0 +1,44 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'wukong'
+require 'wukong/streamer/count_keys'
+
+#
+# Run locally for testing:
+#
+# hdp-cat /hdfs/sometable.tsv | head -n100 | ./hdp-bin --column=4 --bin_width=0.1 --map | sort | ./hdp-bin --reduce
+#
+# Run on a giant dataset:
+#
+# hdp-bin --run --column=4 --bin_width=0.1 /hdfs/sometable.tsv /hdfs/sometable_col4_binned
+#
+
+Settings.define :column,    :default => 1,   :type => Integer, :description => "The column to bin"
+Settings.define :bin_width, :default => 0.5, :type => Float,   :description => "What should the bin width be?"
+
+module HadoopBinning
+  
+  class Mapper < Wukong::Streamer::RecordStreamer
+
+    def initialize *args
+      super(*args)
+      @bin_width = options.bin_width
+      @column    = options.column
+    end
+    
+    def process *args
+      yield bin_field(args[@column])
+    end
+
+    def bin_field field
+      (field.to_f/@bin_width).round*@bin_width
+    end
+    
+  end
+
+  class Reducer < Wukong::Streamer::CountKeys; end
+  
+end
+
+Wukong::Script.new(HadoopBinning::Mapper, HadoopBinning::Reducer).run

data/bin/hdp-ls

@@ -7,4 +7,5 @@ else
     action=ls
 fi
 
-exec hadoop dfs -$action "$@"
+HADOOP_HOME=${HADOOP_HOME-/usr/lib/hadoop}
+exec $HADOOP_HOME/bin/hadoop dfs -$action "$@"

data/docpages/avro/performance.textile

@@ -0,0 +1,36 @@
+
+
+h2. Bulk Streaming use cases
+
+* Take a bunch of nightly calculations and need to flood it into the DB -- http://sna-projects.com/blog/2009/06/building-a-1-tb-data-cycle-at-linkedin-with-hadoop-and-project-voldemort/ In this case, it's important that the bulk load happen efficiently but with low stress on the DB.  I'm willing to make it so that data streams to each cassandra node in the sort order and pre-partitioned just like the node wants to use it. Should run at the full streaming speed of the disk.
+
+* Building a new table or moving a legacy database over to cassandra.  I want to write from one or several nodes, probably not in the cluster, and data will be completely unpartitioned.  I might be able to make some guarantees about uniqueness of keys and rows (that is, you'll generally only see a key once, and/or when you see a key it will contain the entire row).  20k inserts/s / receiving node.
+
+* Using cassandra to replace HDFS. Replication is for compute, not for availability -- so efficient writing at consistency level ANY is important. Would like to get 100k inserts/s per receiving node. 
+
+* A brand new user wants to just stuff his goddamn data into the goddamn database and start playing with it. It had better be not-terribly-slow, and it had better be really easy to take whatever insane format it shows up in and cram that into the data hole.  It should also be conceptually straighforward: it should look like I'm writing hashes or hashes of hashes.
+
+
+===========================================================================
+From http://sna-projects.com/blog/2009/06/building-a-1-tb-data-cycle-at-linkedin-with-hadoop-and-project-voldemort/
+
+Here are the times taken:
+
+* 100GB: 28mins (400 mappers, 90 reducers)
+* 512GB: 2hrs, 16mins (2313 mappers, 350 reducers)
+* 1TB: 5hrs, 39mins (4608 mappers, 700 reducers)
+
+Data transfer between the clusters happens at a steady rate bound by the disk or network. For our Amazon instances this is around 40MB/second.
+
+Online Performance
+
+Lookup time for a single Voldemort node compares well to a single MySQL instance as well. To test this we ran local tests against the 100GB per-node data from the 1 TB test. This test as well was run on an Amazon Extra Large instance with 15GB of RAM and the 4 ephemeral disks in a RAID 10 configuration. To run the tests we simulated we simulated 1 million requests from a real request stream recorded on our production system against each of storage systems. We see the following performance for 1 million requests against a single node:
+	                 MySQL   	Voldemort
+Reqs per sec. 	         727    	1291
+Median req. time 	0.23 ms 	0.05 ms
+Avg. req. time   	13.7 ms 	7.7 ms
+
+99th percentile req. time
+	127.2 ms 	100.7 ms
+
+These numbers are both for local requests with no network involved as the only intention is to benchmark the storage layer of these systems.

data/examples/cassandra_streaming/avromapper.rb

@@ -0,0 +1,85 @@
+#!/usr/bin/env ruby
+
+# To install avro gem
+#   cd avro/lang/ruby ; gem package ; sudo gem install pkg/avro-1.4.0.pre1.gem
+
+require 'rubygems'
+require 'avro'
+require 'wukong'
+require 'wukong/periodic_monitor'
+
+Settings.define :cassandra_avro_schema, :default => ('/usr/local/share/cassandra/interface/avro/cassandra.avpr')
+Settings.define :cassandra_thrift_uri,  :default => `hostname`.chomp.strip+':9160'
+Settings.define :log_interval, :default => 10_000
+
+class AvroStreamer < Wukong::Streamer::RecordStreamer
+  def initialize *args
+    super(*args)
+    @writer = SmutWriter.new
+    @log = PeriodicMonitor.new
+  end
+
+  def process word, count, *_
+    @writer.write_directly(word, 'count', count)
+    @log.periodically( word, count )
+  end
+end
+
+class SmutWriter
+  # Reads in the protocol schema
+  # creates the necessary encoder and writer.
+  def initialize
+    schema_file = Settings.cassandra_avro_schema
+    @proto  = Avro::Protocol.parse(File.read(schema_file))
+    @schema = @proto.types.detect{|schema| schema.name == 'StreamingMutation'}
+    @enc    = Avro::IO::BinaryEncoder.new($stdout)
+    @writer = Avro::IO::DatumWriter.new(@schema)
+  end
+
+  # Directly write the simplified StreamingMutation schema; uses patch from @stuhood
+  def write_directly key, col_name, value
+    @enc.write_bytes(key)
+    @enc.write_bytes(col_name)
+    @enc.write_bytes(value)
+    @enc.write_long(Time.epoch_microseconds)
+    @enc.write_int(0)
+  end
+
+  # Write using the datumwriter
+  def write key, col_name, value
+    @writer.write(smutation(key, col_name, value), @enc)
+  end
+
+  # Simplified StreamingMutation schema uses patch from @stuhood
+  def smutation key, name, value
+    {
+      'key'       => key,
+      'name'      => name.to_s,
+      'value'     => value.to_s,
+      'timestamp' => Time.epoch_microseconds,
+      'ttl'       => 0
+    }
+  end
+
+  # The StreamingMutation schema defined in trunk.
+  # Becomes monstrously inefficient due to implementation of unions.
+  def smutation_from_trunk key, name, value
+    {
+      'key'      => key,
+      'mutation' => { 'column_or_supercolumn' => { 'column' => {
+            'name'  => name.to_s,
+            'value' => value.to_s,
+            'clock' => { 'timestamp' => Time.epoch_microseconds },
+            'ttl'   => 0
+          }}}
+    }
+  end
+end
+
+Time.class_eval do
+  def self.epoch_microseconds
+    (Time.now.utc.to_i * 1_000_000)
+  end
+end
+
+Wukong::Script.new(AvroStreamer, nil, :map_speculative => false).run

data/examples/cassandra_streaming/berlitz_for_cassandra.textile

@@ -0,0 +1,22 @@
+
+
+
+* concurrent_reads, concurrent_writes
+
+* ColumnFamily, SuperColumnFamily
+
+* Keyspace
+
+create temp table uncommonplaces (
+  id integer primary key asc,
+  nvisits integer)
+  ;
+
+
+mmap means index data files and index files
+  mmap in cassandra consumes up to 2GB
+
+  the data we store in the ring is immutable (most of it) or live but independently replay-able (some of it). If I lost say an hour of writes, that's not such a big deal; I could re-play or loa. Is it safe 
+  
+
+* Don't use the cli.  Use Ruby (irb) or Python

data/examples/cassandra_streaming/cassandra.avpr

@@ -0,0 +1,468 @@
+{
+  "protocol" : "Cassandra",
+  "namespace" : "org.apache.cassandra.avro",
+  "types" : [ {
+    "type" : "enum",
+    "name" : "AccessLevel",
+    "symbols" : [ "NONE", "READONLY", "READWRITE", "FALL" ]
+  }, {
+    "type" : "record",
+    "name" : "ColumnPath",
+    "fields" : [ {
+      "name" : "column_family",
+      "type" : "string"
+    }, {
+      "name" : "super_column",
+      "type" : [ "bytes", "null" ]
+    }, {
+      "name" : "column",
+      "type" : [ "bytes", "null" ]
+    } ]
+  }, {
+    "type" : "record",
+    "name" : "ColumnParent",
+    "fields" : [ {
+      "name" : "column_family",
+      "type" : "string"
+    }, {
+      "name" : "super_column",
+      "type" : [ "bytes", "null" ]
+    } ]
+  }, {
+    "type" : "record",
+    "name" : "Clock",
+    "fields" : [ {
+      "name" : "timestamp",
+      "type" : "long"
+    } ]
+  }, {
+    "type" : "record",
+    "name" : "Column",
+    "fields" : [ {
+      "name" : "name",
+      "type" : "bytes"
+    }, {
+      "name" : "value",
+      "type" : "bytes"
+    }, {
+      "name" : "clock",
+      "type" : "Clock"
+    }, {
+      "name" : "ttl",
+      "type" : [ "int", "null" ]
+    } ]
+  }, {
+    "type" : "record",
+    "name" : "SuperColumn",
+    "fields" : [ {
+      "name" : "name",
+      "type" : "bytes"
+    }, {
+      "name" : "columns",
+      "type" : {
+        "type" : "array",
+        "items" : "Column"
+      }
+    } ]
+  }, {
+    "type" : "record",
+    "name" : "ColumnOrSuperColumn",
+    "fields" : [ {
+      "name" : "column",
+      "type" : [ "Column", "null" ]
+    }, {
+      "name" : "super_column",
+      "type" : [ "SuperColumn", "null" ]
+    } ]
+  }, {
+    "type" : "record",
+    "name" : "SliceRange",
+    "fields" : [ {
+      "name" : "start",
+      "type" : "bytes"
+    }, {
+      "name" : "finish",
+      "type" : "bytes"
+    }, {
+      "name" : "reversed",
+      "type" : "boolean"
+    }, {
+      "name" : "count",
+      "type" : "int"
+    }, {
+      "name" : "bitmasks",
+      "type" : [ {
+        "type" : "array",
+        "items" : "bytes"
+      }, "null" ]
+    } ]
+  }, {
+    "type" : "record",
+    "name" : "SlicePredicate",
+    "fields" : [ {
+      "name" : "column_names",
+      "type" : [ {
+        "type" : "array",
+        "items" : "bytes"
+      }, "null" ]
+    }, {
+      "name" : "slice_range",
+      "type" : [ "SliceRange", "null" ]
+    } ]
+  }, {
+    "type" : "record",
+    "name" : "Deletion",
+    "fields" : [ {
+      "name" : "clock",
+      "type" : "Clock"
+    }, {
+      "name" : "super_column",
+      "type" : [ "bytes", "null" ]
+    }, {
+      "name" : "predicate",
+      "type" : [ "SlicePredicate", "null" ]
+    } ]
+  }, {
+    "type" : "record",
+    "name" : "Mutation",
+    "fields" : [ {
+      "name" : "column_or_supercolumn",
+      "type" : [ "ColumnOrSuperColumn", "null" ]
+    }, {
+      "name" : "deletion",
+      "type" : [ "Deletion", "null" ]
+    } ]
+  }, {
+    "type" : "enum",
+    "name" : "IndexType",
+    "symbols" : [ "KEYS" ]
+  }, {
+    "type" : "record",
+    "name" : "ColumnDef",
+    "fields" : [ {
+      "name" : "name",
+      "type" : "bytes"
+    }, {
+      "name" : "validation_class",
+      "type" : "string"
+    }, {
+      "name" : "index_type",
+      "type" : [ "IndexType", "null" ]
+    }, {
+      "name" : "index_name",
+      "type" : [ "string", "null" ]
+    } ]
+  }, {
+    "type" : "record",
+    "name" : "CfDef",
+    "fields" : [ {
+      "name" : "keyspace",
+      "type" : "string"
+    }, {
+      "name" : "name",
+      "type" : "string"
+    }, {
+      "name" : "column_type",
+      "type" : [ "string", "null" ]
+    }, {
+      "name" : "clock_type",
+      "type" : [ "string", "null" ]
+    }, {
+      "name" : "comparator_type",
+      "type" : [ "string", "null" ]
+    }, {
+      "name" : "subcomparator_type",
+      "type" : [ "string", "null" ]
+    }, {
+      "name" : "reconciler",
+      "type" : [ "string", "null" ]
+    }, {
+      "name" : "comment",
+      "type" : [ "string", "null" ]
+    }, {
+      "name" : "row_cache_size",
+      "type" : [ "double", "null" ]
+    }, {
+      "name" : "preload_row_cache",
+      "type" : [ "boolean", "null" ]
+    }, {
+      "name" : "key_cache_size",
+      "type" : [ "double", "null" ]
+    }, {
+      "name" : "read_repair_chance",
+      "type" : [ "double", "null" ]
+    }, {
+      "name" : "gc_grace_seconds",
+      "type" : [ "int", "null" ]
+    }, {
+      "name" : "column_metadata",
+      "type" : [ {
+        "type" : "array",
+        "items" : "ColumnDef"
+      }, "null" ]
+    }, {
+      "name" : "id",
+      "type" : [ "int", "null" ]
+    } ]
+  }, {
+    "type" : "record",
+    "name" : "KsDef",
+    "fields" : [ {
+      "name" : "name",
+      "type" : "string"
+    }, {
+      "name" : "strategy_class",
+      "type" : "string"
+    }, {
+      "name" : "strategy_options",
+      "type" : [ {
+        "type" : "map",
+        "values" : "string"
+      }, "null" ]
+    }, {
+      "name" : "replication_factor",
+      "type" : "int"
+    }, {
+      "name" : "cf_defs",
+      "type" : {
+        "type" : "array",
+        "items" : "CfDef"
+      }
+    } ]
+  }, {
+    "type" : "record",
+    "name" : "StreamingMutation",
+    "fields" : [ {
+      "name" : "key",
+      "type" : "bytes"
+    }, {
+      "name" : "name",
+      "type" : "bytes"
+    }, {
+      "name" : "value",
+      "type" : "bytes"
+    }, {
+      "name" : "timestamp",
+      "type" : "long"
+    }, {
+      "name" : "ttl",
+      "type" : "int"
+    } ]
+  }, {
+    "type" : "record",
+    "name" : "MutationsMapEntry",
+    "fields" : [ {
+      "name" : "key",
+      "type" : "bytes"
+    }, {
+      "name" : "mutations",
+      "type" : {
+        "type" : "map",
+        "values" : {
+          "type" : "array",
+          "items" : "Mutation"
+        }
+      }
+    } ]
+  }, {
+    "type" : "record",
+    "name" : "CoscsMapEntry",
+    "fields" : [ {
+      "name" : "key",
+      "type" : "bytes"
+    }, {
+      "name" : "columns",
+      "type" : {
+        "type" : "array",
+        "items" : "ColumnOrSuperColumn"
+      }
+    } ]
+  }, {
+    "type" : "enum",
+    "name" : "ConsistencyLevel",
+    "symbols" : [ "ZERO", "ONE", "QUORUM", "DCQUORUM", "DCQUORUMSYNC", "ALL" ]
+  }, {
+    "type" : "error",
+    "name" : "InvalidRequestException",
+    "fields" : [ {
+      "name" : "why",
+      "type" : [ "string", "null" ]
+    } ]
+  }, {
+    "type" : "error",
+    "name" : "NotFoundException",
+    "fields" : [ {
+      "name" : "why",
+      "type" : [ "string", "null" ]
+    } ]
+  }, {
+    "type" : "error",
+    "name" : "UnavailableException",
+    "fields" : [ {
+      "name" : "why",
+      "type" : [ "string", "null" ]
+    } ]
+  }, {
+    "type" : "error",
+    "name" : "TimedOutException",
+    "fields" : [ {
+      "name" : "why",
+      "type" : [ "string", "null" ]
+    } ]
+  } ],
+  "messages" : {
+    "get" : {
+      "request" : [ {
+        "name" : "key",
+        "type" : "bytes"
+      }, {
+        "name" : "column_path",
+        "type" : "ColumnPath"
+      }, {
+        "name" : "consistency_level",
+        "type" : "ConsistencyLevel"
+      } ],
+      "response" : "ColumnOrSuperColumn",
+      "errors" : [ "InvalidRequestException", "NotFoundException", "UnavailableException", "TimedOutException" ]
+    },
+    "get_slice" : {
+      "request" : [ {
+        "name" : "key",
+        "type" : "bytes"
+      }, {
+        "name" : "column_parent",
+        "type" : "ColumnParent"
+      }, {
+        "name" : "predicate",
+        "type" : "SlicePredicate"
+      }, {
+        "name" : "consistency_level",
+        "type" : "ConsistencyLevel"
+      } ],
+      "response" : {
+        "type" : "array",
+        "items" : "ColumnOrSuperColumn"
+      },
+      "errors" : [ "InvalidRequestException", "UnavailableException", "TimedOutException" ]
+    },
+    "multiget_slice" : {
+      "request" : [ {
+        "name" : "keys",
+        "type" : {
+          "type" : "array",
+          "items" : "bytes"
+        }
+      }, {
+        "name" : "column_parent",
+        "type" : "ColumnParent"
+      }, {
+        "name" : "predicate",
+        "type" : "SlicePredicate"
+      }, {
+        "name" : "consistency_level",
+        "type" : "ConsistencyLevel"
+      } ],
+      "response" : {
+        "type" : "array",
+        "items" : "CoscsMapEntry"
+      },
+      "errors" : [ "InvalidRequestException", "UnavailableException", "TimedOutException" ]
+    },
+    "get_count" : {
+      "request" : [ {
+        "name" : "key",
+        "type" : "bytes"
+      }, {
+        "name" : "column_parent",
+        "type" : "ColumnParent"
+      }, {
+        "name" : "predicate",
+        "type" : "SlicePredicate"
+      }, {
+        "name" : "consistency_level",
+        "type" : "ConsistencyLevel"
+      } ],
+      "response" : "int",
+      "errors" : [ "InvalidRequestException", "UnavailableException", "TimedOutException" ]
+    },
+    "insert" : {
+      "request" : [ {
+        "name" : "key",
+        "type" : "bytes"
+      }, {
+        "name" : "column_parent",
+        "type" : "ColumnParent"
+      }, {
+        "name" : "column",
+        "type" : "Column"
+      }, {
+        "name" : "consistency_level",
+        "type" : "ConsistencyLevel"
+      } ],
+      "response" : "null",
+      "errors" : [ "InvalidRequestException", "UnavailableException", "TimedOutException" ]
+    },
+    "remove" : {
+      "request" : [ {
+        "name" : "key",
+        "type" : "bytes"
+      }, {
+        "name" : "column_path",
+        "type" : "ColumnPath"
+      }, {
+        "name" : "clock",
+        "type" : "Clock"
+      }, {
+        "name" : "consistency_level",
+        "type" : "ConsistencyLevel"
+      } ],
+      "response" : "null",
+      "errors" : [ "InvalidRequestException", "UnavailableException", "TimedOutException" ]
+    },
+    "batch_mutate" : {
+      "request" : [ {
+        "name" : "mutation_map",
+        "type" : {
+          "type" : "array",
+          "items" : "MutationsMapEntry"
+        }
+      }, {
+        "name" : "consistency_level",
+        "type" : "ConsistencyLevel"
+      } ],
+      "response" : "null",
+      "errors" : [ "InvalidRequestException", "UnavailableException", "TimedOutException" ]
+    },
+    "system_add_keyspace" : {
+      "request" : [ {
+        "name" : "ks_def",
+        "type" : "KsDef"
+      } ],
+      "response" : "null",
+      "errors" : [ "InvalidRequestException" ]
+    },
+    "set_keyspace" : {
+      "request" : [ {
+        "name" : "keyspace",
+        "type" : "string"
+      } ],
+      "response" : "null",
+      "errors" : [ "InvalidRequestException" ]
+    },
+    "describe_keyspaces" : {
+      "request" : [ ],
+      "response" : {
+        "type" : "array",
+        "items" : "string"
+      }
+    },
+    "describe_cluster_name" : {
+      "request" : [ ],
+      "response" : "string"
+    },
+    "describe_version" : {
+      "request" : [ ],
+      "response" : "string"
+    }
+  }
+}