wukong 1.5.4 → 2.0.0

data/examples/cassandra_streaming/cassandra.avpr

@@ -1,468 +0,0 @@
-{
-  "protocol" : "Cassandra",
-  "namespace" : "org.apache.cassandra.avro",
-  "types" : [ {
-    "type" : "enum",
-    "name" : "AccessLevel",
-    "symbols" : [ "NONE", "READONLY", "READWRITE", "FALL" ]
-  }, {
-    "type" : "record",
-    "name" : "ColumnPath",
-    "fields" : [ {
-      "name" : "column_family",
-      "type" : "string"
-    }, {
-      "name" : "super_column",
-      "type" : [ "bytes", "null" ]
-    }, {
-      "name" : "column",
-      "type" : [ "bytes", "null" ]
-    } ]
-  }, {
-    "type" : "record",
-    "name" : "ColumnParent",
-    "fields" : [ {
-      "name" : "column_family",
-      "type" : "string"
-    }, {
-      "name" : "super_column",
-      "type" : [ "bytes", "null" ]
-    } ]
-  }, {
-    "type" : "record",
-    "name" : "Clock",
-    "fields" : [ {
-      "name" : "timestamp",
-      "type" : "long"
-    } ]
-  }, {
-    "type" : "record",
-    "name" : "Column",
-    "fields" : [ {
-      "name" : "name",
-      "type" : "bytes"
-    }, {
-      "name" : "value",
-      "type" : "bytes"
-    }, {
-      "name" : "clock",
-      "type" : "Clock"
-    }, {
-      "name" : "ttl",
-      "type" : [ "int", "null" ]
-    } ]
-  }, {
-    "type" : "record",
-    "name" : "SuperColumn",
-    "fields" : [ {
-      "name" : "name",
-      "type" : "bytes"
-    }, {
-      "name" : "columns",
-      "type" : {
-        "type" : "array",
-        "items" : "Column"
-      }
-    } ]
-  }, {
-    "type" : "record",
-    "name" : "ColumnOrSuperColumn",
-    "fields" : [ {
-      "name" : "column",
-      "type" : [ "Column", "null" ]
-    }, {
-      "name" : "super_column",
-      "type" : [ "SuperColumn", "null" ]
-    } ]
-  }, {
-    "type" : "record",
-    "name" : "SliceRange",
-    "fields" : [ {
-      "name" : "start",
-      "type" : "bytes"
-    }, {
-      "name" : "finish",
-      "type" : "bytes"
-    }, {
-      "name" : "reversed",
-      "type" : "boolean"
-    }, {
-      "name" : "count",
-      "type" : "int"
-    }, {
-      "name" : "bitmasks",
-      "type" : [ {
-        "type" : "array",
-        "items" : "bytes"
-      }, "null" ]
-    } ]
-  }, {
-    "type" : "record",
-    "name" : "SlicePredicate",
-    "fields" : [ {
-      "name" : "column_names",
-      "type" : [ {
-        "type" : "array",
-        "items" : "bytes"
-      }, "null" ]
-    }, {
-      "name" : "slice_range",
-      "type" : [ "SliceRange", "null" ]
-    } ]
-  }, {
-    "type" : "record",
-    "name" : "Deletion",
-    "fields" : [ {
-      "name" : "clock",
-      "type" : "Clock"
-    }, {
-      "name" : "super_column",
-      "type" : [ "bytes", "null" ]
-    }, {
-      "name" : "predicate",
-      "type" : [ "SlicePredicate", "null" ]
-    } ]
-  }, {
-    "type" : "record",
-    "name" : "Mutation",
-    "fields" : [ {
-      "name" : "column_or_supercolumn",
-      "type" : [ "ColumnOrSuperColumn", "null" ]
-    }, {
-      "name" : "deletion",
-      "type" : [ "Deletion", "null" ]
-    } ]
-  }, {
-    "type" : "enum",
-    "name" : "IndexType",
-    "symbols" : [ "KEYS" ]
-  }, {
-    "type" : "record",
-    "name" : "ColumnDef",
-    "fields" : [ {
-      "name" : "name",
-      "type" : "bytes"
-    }, {
-      "name" : "validation_class",
-      "type" : "string"
-    }, {
-      "name" : "index_type",
-      "type" : [ "IndexType", "null" ]
-    }, {
-      "name" : "index_name",
-      "type" : [ "string", "null" ]
-    } ]
-  }, {
-    "type" : "record",
-    "name" : "CfDef",
-    "fields" : [ {
-      "name" : "keyspace",
-      "type" : "string"
-    }, {
-      "name" : "name",
-      "type" : "string"
-    }, {
-      "name" : "column_type",
-      "type" : [ "string", "null" ]
-    }, {
-      "name" : "clock_type",
-      "type" : [ "string", "null" ]
-    }, {
-      "name" : "comparator_type",
-      "type" : [ "string", "null" ]
-    }, {
-      "name" : "subcomparator_type",
-      "type" : [ "string", "null" ]
-    }, {
-      "name" : "reconciler",
-      "type" : [ "string", "null" ]
-    }, {
-      "name" : "comment",
-      "type" : [ "string", "null" ]
-    }, {
-      "name" : "row_cache_size",
-      "type" : [ "double", "null" ]
-    }, {
-      "name" : "preload_row_cache",
-      "type" : [ "boolean", "null" ]
-    }, {
-      "name" : "key_cache_size",
-      "type" : [ "double", "null" ]
-    }, {
-      "name" : "read_repair_chance",
-      "type" : [ "double", "null" ]
-    }, {
-      "name" : "gc_grace_seconds",
-      "type" : [ "int", "null" ]
-    }, {
-      "name" : "column_metadata",
-      "type" : [ {
-        "type" : "array",
-        "items" : "ColumnDef"
-      }, "null" ]
-    }, {
-      "name" : "id",
-      "type" : [ "int", "null" ]
-    } ]
-  }, {
-    "type" : "record",
-    "name" : "KsDef",
-    "fields" : [ {
-      "name" : "name",
-      "type" : "string"
-    }, {
-      "name" : "strategy_class",
-      "type" : "string"
-    }, {
-      "name" : "strategy_options",
-      "type" : [ {
-        "type" : "map",
-        "values" : "string"
-      }, "null" ]
-    }, {
-      "name" : "replication_factor",
-      "type" : "int"
-    }, {
-      "name" : "cf_defs",
-      "type" : {
-        "type" : "array",
-        "items" : "CfDef"
-      }
-    } ]
-  }, {
-    "type" : "record",
-    "name" : "StreamingMutation",
-    "fields" : [ {
-      "name" : "key",
-      "type" : "bytes"
-    }, {
-      "name" : "name",
-      "type" : "bytes"
-    }, {
-      "name" : "value",
-      "type" : "bytes"
-    }, {
-      "name" : "timestamp",
-      "type" : "long"
-    }, {
-      "name" : "ttl",
-      "type" : "int"
-    } ]
-  }, {
-    "type" : "record",
-    "name" : "MutationsMapEntry",
-    "fields" : [ {
-      "name" : "key",
-      "type" : "bytes"
-    }, {
-      "name" : "mutations",
-      "type" : {
-        "type" : "map",
-        "values" : {
-          "type" : "array",
-          "items" : "Mutation"
-        }
-      }
-    } ]
-  }, {
-    "type" : "record",
-    "name" : "CoscsMapEntry",
-    "fields" : [ {
-      "name" : "key",
-      "type" : "bytes"
-    }, {
-      "name" : "columns",
-      "type" : {
-        "type" : "array",
-        "items" : "ColumnOrSuperColumn"
-      }
-    } ]
-  }, {
-    "type" : "enum",
-    "name" : "ConsistencyLevel",
-    "symbols" : [ "ZERO", "ONE", "QUORUM", "DCQUORUM", "DCQUORUMSYNC", "ALL" ]
-  }, {
-    "type" : "error",
-    "name" : "InvalidRequestException",
-    "fields" : [ {
-      "name" : "why",
-      "type" : [ "string", "null" ]
-    } ]
-  }, {
-    "type" : "error",
-    "name" : "NotFoundException",
-    "fields" : [ {
-      "name" : "why",
-      "type" : [ "string", "null" ]
-    } ]
-  }, {
-    "type" : "error",
-    "name" : "UnavailableException",
-    "fields" : [ {
-      "name" : "why",
-      "type" : [ "string", "null" ]
-    } ]
-  }, {
-    "type" : "error",
-    "name" : "TimedOutException",
-    "fields" : [ {
-      "name" : "why",
-      "type" : [ "string", "null" ]
-    } ]
-  } ],
-  "messages" : {
-    "get" : {
-      "request" : [ {
-        "name" : "key",
-        "type" : "bytes"
-      }, {
-        "name" : "column_path",
-        "type" : "ColumnPath"
-      }, {
-        "name" : "consistency_level",
-        "type" : "ConsistencyLevel"
-      } ],
-      "response" : "ColumnOrSuperColumn",
-      "errors" : [ "InvalidRequestException", "NotFoundException", "UnavailableException", "TimedOutException" ]
-    },
-    "get_slice" : {
-      "request" : [ {
-        "name" : "key",
-        "type" : "bytes"
-      }, {
-        "name" : "column_parent",
-        "type" : "ColumnParent"
-      }, {
-        "name" : "predicate",
-        "type" : "SlicePredicate"
-      }, {
-        "name" : "consistency_level",
-        "type" : "ConsistencyLevel"
-      } ],
-      "response" : {
-        "type" : "array",
-        "items" : "ColumnOrSuperColumn"
-      },
-      "errors" : [ "InvalidRequestException", "UnavailableException", "TimedOutException" ]
-    },
-    "multiget_slice" : {
-      "request" : [ {
-        "name" : "keys",
-        "type" : {
-          "type" : "array",
-          "items" : "bytes"
-        }
-      }, {
-        "name" : "column_parent",
-        "type" : "ColumnParent"
-      }, {
-        "name" : "predicate",
-        "type" : "SlicePredicate"
-      }, {
-        "name" : "consistency_level",
-        "type" : "ConsistencyLevel"
-      } ],
-      "response" : {
-        "type" : "array",
-        "items" : "CoscsMapEntry"
-      },
-      "errors" : [ "InvalidRequestException", "UnavailableException", "TimedOutException" ]
-    },
-    "get_count" : {
-      "request" : [ {
-        "name" : "key",
-        "type" : "bytes"
-      }, {
-        "name" : "column_parent",
-        "type" : "ColumnParent"
-      }, {
-        "name" : "predicate",
-        "type" : "SlicePredicate"
-      }, {
-        "name" : "consistency_level",
-        "type" : "ConsistencyLevel"
-      } ],
-      "response" : "int",
-      "errors" : [ "InvalidRequestException", "UnavailableException", "TimedOutException" ]
-    },
-    "insert" : {
-      "request" : [ {
-        "name" : "key",
-        "type" : "bytes"
-      }, {
-        "name" : "column_parent",
-        "type" : "ColumnParent"
-      }, {
-        "name" : "column",
-        "type" : "Column"
-      }, {
-        "name" : "consistency_level",
-        "type" : "ConsistencyLevel"
-      } ],
-      "response" : "null",
-      "errors" : [ "InvalidRequestException", "UnavailableException", "TimedOutException" ]
-    },
-    "remove" : {
-      "request" : [ {
-        "name" : "key",
-        "type" : "bytes"
-      }, {
-        "name" : "column_path",
-        "type" : "ColumnPath"
-      }, {
-        "name" : "clock",
-        "type" : "Clock"
-      }, {
-        "name" : "consistency_level",
-        "type" : "ConsistencyLevel"
-      } ],
-      "response" : "null",
-      "errors" : [ "InvalidRequestException", "UnavailableException", "TimedOutException" ]
-    },
-    "batch_mutate" : {
-      "request" : [ {
-        "name" : "mutation_map",
-        "type" : {
-          "type" : "array",
-          "items" : "MutationsMapEntry"
-        }
-      }, {
-        "name" : "consistency_level",
-        "type" : "ConsistencyLevel"
-      } ],
-      "response" : "null",
-      "errors" : [ "InvalidRequestException", "UnavailableException", "TimedOutException" ]
-    },
-    "system_add_keyspace" : {
-      "request" : [ {
-        "name" : "ks_def",
-        "type" : "KsDef"
-      } ],
-      "response" : "null",
-      "errors" : [ "InvalidRequestException" ]
-    },
-    "set_keyspace" : {
-      "request" : [ {
-        "name" : "keyspace",
-        "type" : "string"
-      } ],
-      "response" : "null",
-      "errors" : [ "InvalidRequestException" ]
-    },
-    "describe_keyspaces" : {
-      "request" : [ ],
-      "response" : {
-        "type" : "array",
-        "items" : "string"
-      }
-    },
-    "describe_cluster_name" : {
-      "request" : [ ],
-      "response" : "string"
-    },
-    "describe_version" : {
-      "request" : [ ],
-      "response" : "string"
-    }
-  }
-}

data/examples/cassandra_streaming/cassandra_random_partitioner.rb

@@ -1,62 +0,0 @@
-#!/usr/bin/env ruby
-require 'rubygems'
-require 'avro'
-require 'wukong'
-require 'wukong/periodic_monitor'
-Settings.define :log_interval, :default => 10_000
-
-require 'digest/md5'
-Settings.define :ring_nodes
-
-MAX_HASH = 2**127
-RING_NODES = 72
-RING_WIDTH = MAX_HASH / RING_NODES
-OUT_DIR    = '/mnt/tmp/partitioned_words'
-
-# for foo in pw0/part-000* ; do echo $foo ; time cat $foo | ~/ics/wukong/examples/cassandra_streaming/cassandra_random_partitioner.rb --map 2>/tmp/split-`basename $foo`.log & done 
-
-module CassandraRandomPartitioner
-  def partition_hash key
-    uval = Digest::MD5.hexdigest(key).to_i(16)
-    (uval > 2**127) ? (2**128 - uval) : uval
-  end
-
-  def partition key
-    partition_hash(key) / RING_WIDTH
-  end
-  
-  def files
-    @files ||= Hash.new{|h,part| h[part] = File.open(OUT_DIR+"/chunk-#{"%03d" % part}", 'w') }
-  end
-
-end
-
-module PeriodicLog
-  def log
-    @log ||= PeriodicMonitor.new
-  end  
-end
-
-class HashingStreamer < Wukong::Streamer::RecordStreamer
-  include CassandraRandomPartitioner
-  include PeriodicLog
-
-  def process word, count, *_
-    log.periodically( word, count )
-    part = partition(word)
-    # yield [part, word, count]
-    files[part] << [word, count].join("\t") << "\n"
-  end
-end
-
-class HashingReducer <  Wukong::Streamer::RecordStreamer
-  include CassandraRandomPartitioner
-  include PeriodicLog
-
-  def process part, word, count, *_
-    log.periodically( word, count )
-    yield [word, count]
-  end
-end
-
-Wukong::Script.new(HashingStreamer, HashingReducer, :map_speculative => false).run

data/examples/cassandra_streaming/catter.sh

@@ -1,45 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Cat a binary-encoded avro file into the bulk loader
-#
-
-input_file="$1" 		 ; shift
-output_file="$1" 		 ; shift
-map_script=${1-/bin/cat}	 ; shift
-reduce_script=${1-/usr/bin/uniq} ; shift
-
-dest_keyspace=${dest_keyspace-soc_net_tw}
-dest_col_family=${dest_col_family-Wordbag}
-
-hostname=`hostname`
-
-# Path to cassandra and hadoop dirs
-script_dir=$(readlink -f `dirname $0`)
-CASSANDRA_HOME=${CASSANDRA_HOME-/usr/local/share/cassandra}
-HADOOP_HOME=${HADOOP_HOME-/usr/lib/hadoop}
-avro_file=${avro_file-$CASSANDRA_HOME/interface/avro/cassandra.avpr}
-
-ARCHIVES=`/bin/ls -1 $CASSANDRA_HOME/build/apache-cassandra*.jar`
-for jar in `/bin/ls -1 $CASSANDRA_HOME/build/lib/jars/*.jar $CASSANDRA_HOME/lib/*.jar`; do
-    ARCHIVES=$ARCHIVES,$jar
-done
-
-${HADOOP_HOME}/bin/hadoop                                                                        \
-     jar ${HADOOP_HOME}/contrib/streaming/hadoop-*streaming*.jar                                 \
-    -D stream.map.output=cassandra_avro_output                                                   \
-    -D stream.io.identifier.resolver.class=org.apache.cassandra.hadoop.streaming.AvroResolver    \
-    -D cassandra.output.keyspace="$dest_keyspace"                                                \
-    -D cassandra.output.columnfamily="$dest_col_family"                                          \
-    -D cassandra.partitioner.class=org.apache.cassandra.dht.RandomPartitioner                    \
-    -D cassandra.thrift.address="10.104.9.68"                                                    \
-    -D cassandra.thrift.port=9160                                                                \
-    -D mapred.reduce.tasks=0                                                                     \
-    -libjars $ARCHIVES                                                                           \
-    -file $avro_file                                                                             \
-    -outputformat org.apache.cassandra.hadoop.ColumnFamilyOutputFormat                           \
-    -mapper  	 `which cat`                                                                     \
-    -input       "$input_file"                                                                   \
-    -output  	 "$output_file"                                                                  \
-    "$@"
-