wukong 1.5.4 → 2.0.0

data/examples/cassandra_streaming/client_schema.avpr

@@ -1,211 +0,0 @@
-{
-    "protocol" : "Cassandra",
-    "namespace" : "org.apache.cassandra.avro",  "types" : [
-    
-Add/insert one value
-    
-  Mutate        ks, [col_ref], 'val', ts, ttl }
-
-Add/insert multiple cols to same row
-  
-  MutateRow     ks, supercol_or_nil, { [col, val, ts, ttl], [col,val,ts,ttl],...}}
-  MutateCRow    ks,                  { [col, val, ts, ttl], [col,val,ts,ttl],...}}
-  MutateSCRow   ks, supercol,        { [col, val, ts, ttl], [col,val,ts,ttl],...}}
-
-Get one, many or all columns from given row
-
-  get
-  Multiget      ks, supercol_or_nil, [col1, col2, ...] or nil
-
-Get one, many or all columns from a slice of sequential rows
-
-  get_range
-
-Remove one column from a row
-  
-  remove
-  
-Remove many columns from a row
-  
-Remove all columns in a row
-
-
-
-h3. Mo
-    
-        
-        { "name" : "AccessLevel",               "type" : "enum",       "symbols" : [ "NONE", "READONLY", "READWRITE", "FALL" ]  },
-        { "name" : "ColumnPath",                "type" : "record",    "fields" : [
-            { "name" : "column_family",         "type" : "string"},
-            { "name" : "super_column",          "type" : [ "bytes", "null" ]},
-            { "name" : "column",                "type" : [ "bytes", "null" ] } ]},
-        { "name" : "ColumnParent",              "type" : "record",	"fields" : [
-            { "name" : "column_family",         "type" : "string"},
-            { "name" : "super_column",          "type" : [ "bytes", "null" ] } ]},
-        { "name" : "SliceRange",                "type" : "record",	"fields" : [
-            { "name" : "start",                 "type" : "bytes"},
-            { "name" : "finish",                "type" : "bytes"},
-            { "name" : "reversed",              "type" : "boolean"},
-            { "name" : "count",                 "type" : "int"},
-            { "name" : "bitmasks",              "type" : [ { "type" : "array",        "items" : "bytes"},  "null" ]    } ]},
-        { "name" : "SlicePredicate",            "type" : "record",	"fields" : [
-            { "name" : "column_names",          "type" : [ { "type" : "array",        "items" : "bytes"},  "null" ]},
-            { "name" : "slice_range",           "type" : [ "SliceRange", "null" ]    } ]},
-
-        { "name" : "Clock",                     "type" : "record",	"fields" : [
-            { "name" : "timestamp",	"type" : "long"   } ]},
-        { "name" : "Column",                    "type" : "record",	"fields" : [
-            { "name" : "name",                  "type" : "bytes"},
-            { "name" : "value",                 "type" : "bytes"},
-            { "name" : "clock",                 "type" : "Clock"},
-            { "name" : "ttl",                   "type" : "int"   } ]},
-        { "name" : "SuperColumn",               "type" : "record",	"fields" : [
-            { "name" : "name",                  "type" : "bytes"},
-            { "name" : "columns",               "type" : { "type" : "array",        "items" : "Column" } } ]},
-        { "name" : "ColumnOrSuperColumn",       "type" : "record",	"fields" : [
-            { "name" : "column",                "type" : "Column" },
-            { "name" : "super_column",          "type" : "null"     } ]},
-        { "name" : "Deletion",                  "type" : "record",	"fields" : [
-            { "name" : "clock",                 "type" : "Clock"},
-            { "name" : "super_column",          "type" : [ "bytes", "null" ]},
-            { "name" : "predicate",             "type" : [ "SlicePredicate", "null" ]    } ]},
-        { "name" : "Mutation",                  "type" : "record",	"fields" : [
-            { "name" : "column_or_supercolumn", "type" : "ColumnOrSuperColumn" },
-            { "name" : "deletion",              "type" : "null" }
-        ]},
-        { "name" : "StreamingMutation",         "type" : "record",	"fields" : [
-            { "name" : "key",	                "type" : "bytes"       },
-            { "name" : "mutation",	        "type" : "Mutation"    } ]},
-
-        { "name" : "IndexType",                 "type" : "enum",	"symbols" : [ "KEYS" ]},
-        { "name" : "ColumnDef",                 "type" : "record",	"fields" : [
-            { "name" : "name",                  "type" : "bytes"},
-            { "name" : "validation_class",      "type" : "string"},
-            { "name" : "index_type",            "type" : [ "IndexType", "null" ]},
-            { "name" : "index_name",            "type" : [ "string", "null" ]    } ]},
-        { "name" : "CfDef",                     "type" : "record",	"fields" : [
-            { "name" : "keyspace",              "type" : "string"},
-            { "name" : "name",                  "type" : "string"},
-            { "name" : "column_type",           "type" : [ "string", "null" ]},
-            { "name" : "clock_type",            "type" : [ "string", "null" ]},
-            { "name" : "comparator_type",       "type" : [ "string", "null" ]},
-            { "name" : "subcomparator_type",    "type" : [ "string", "null" ]},
-            { "name" : "reconciler",            "type" : [ "string", "null" ]},
-            { "name" : "comment",               "type" : [ "string", "null" ]},
-            { "name" : "row_cache_size",        "type" : [ "double", "null" ]},
-            { "name" : "preload_row_cache",     "type" : [ "boolean", "null" ]},
-            { "name" : "key_cache_size",        "type" : [ "double", "null" ]},
-            { "name" : "read_repair_chance",    "type" : [ "double", "null" ]},
-            { "name" : "gc_grace_seconds",      "type" : [ "int", "null" ]},
-            { "name" : "column_metadata",       "type" : [ { "type" : "array",        "items" : "ColumnDef"},  "null" ]},
-            { "name" : "id",                    "type" : [ "int", "null" ]    } ]},
-        { "name" : "KsDef",                     "type" : "record",	"fields" : [
-            { "name" : "name",                  "type" : "string"},  { "name" : "strategy_class",	"type" : "string"},
-            { "name" : "strategy_options",      "type" : [ { "type" : "map",        "values" : "string"},  "null" ]},
-            { "name" : "replication_factor",    "type" : "int"},  { "name" : "cf_defs",	"type" : { "type" : "array",        "items" : "CfDef"      }    } ]},
-        { "name" : "MutationsMapEntry",         "type" : "record",	"fields" : [ { "name" : "key",	"type" : "bytes"},  { "name" : "mutations",	"type" : { "type" : "map",        "values" : { "type" : "array",          "items" : "Mutation"        }      }    } ]},
-        { "name" : "CoscsMapEntry",             "type" : "record",	"fields" : [ { "name" : "key",	"type" : "bytes"},  { "name" : "columns",	"type" : { "type" : "array",        "items" : "ColumnOrSuperColumn"      }    } ]},
-        { "name" : "ConsistencyLevel",          "type" : "enum",	"symbols" : [ "ZERO", "ONE", "QUORUM", "DCQUORUM", "DCQUORUMSYNC", "ALL" ]},
-        { "name" : "InvalidRequestException",   "type" : "error",	"fields" : [ { "name" : "why",	"type" : [ "string", "null" ]    } ]},
-        { "name" : "NotFoundException",         "type" : "error",	"fields" : [ { "name" : "why",	"type" : [ "string", "null" ]    } ]},
-        { "name" : "UnavailableException",      "type" : "error",	"fields" : [ { "name" : "why",	"type" : [ "string", "null" ]    } ]},
-        { "name" : "TimedOutException",         "type" : "error",	"fields" : [ { "name" : "why",	"type" : [ "string", "null" ]    } ]  }
-    ],
-
-    
-  "messages" : { "get" : {
-      "request" : [ { "name" : "key",	"type" : "bytes"},
-  { "name" : "column_path",	"type" : "ColumnPath"},
-  { "name" : "consistency_level",	"type" : "ConsistencyLevel"
-      } ],
-      "response" : "ColumnOrSuperColumn",
-      "errors" : [ "InvalidRequestException", "NotFoundException", "UnavailableException", "TimedOutException" ]
-    },
-    "get_slice" : {
-      "request" : [ { "name" : "key",	"type" : "bytes"},
-  { "name" : "column_parent",	"type" : "ColumnParent"},
-  { "name" : "predicate",	"type" : "SlicePredicate"},
-  { "name" : "consistency_level",	"type" : "ConsistencyLevel"
-      } ],
-      "response" : { "type" : "array",
-        "items" : "ColumnOrSuperColumn"
-      },
-      "errors" : [ "InvalidRequestException", "UnavailableException", "TimedOutException" ]
-    },
-    "multiget_slice" : {
-      "request" : [ { "name" : "keys",	"type" : { "type" : "array",
-          "items" : "bytes"
-        }},
-  { "name" : "column_parent",	"type" : "ColumnParent"},
-  { "name" : "predicate",	"type" : "SlicePredicate"},
-  { "name" : "consistency_level",	"type" : "ConsistencyLevel"
-      } ],
-      "response" : { "type" : "array",
-        "items" : "CoscsMapEntry"
-      },
-      "errors" : [ "InvalidRequestException", "UnavailableException", "TimedOutException" ]
-    },
-    "get_count" : {
-      "request" : [ { "name" : "key",	"type" : "bytes"},
-  { "name" : "column_parent",	"type" : "ColumnParent"},
-  { "name" : "predicate",	"type" : "SlicePredicate"},
-  { "name" : "consistency_level",	"type" : "ConsistencyLevel"
-      } ],
-      "response" : "int",
-      "errors" : [ "InvalidRequestException", "UnavailableException", "TimedOutException" ]
-    },
-    "insert" : {
-      "request" : [ { "name" : "key",	"type" : "bytes"},
-  { "name" : "column_parent",	"type" : "ColumnParent"},
-  { "name" : "column",	"type" : "Column"},
-  { "name" : "consistency_level",	"type" : "ConsistencyLevel"
-      } ],
-      "response" : "null",
-      "errors" : [ "InvalidRequestException", "UnavailableException", "TimedOutException" ]
-    },
-    "remove" : {
-      "request" : [ { "name" : "key",	"type" : "bytes"},
-  { "name" : "column_path",	"type" : "ColumnPath"},
-  { "name" : "clock",	"type" : "Clock"},
-  { "name" : "consistency_level",	"type" : "ConsistencyLevel"
-      } ],
-      "response" : "null",
-      "errors" : [ "InvalidRequestException", "UnavailableException", "TimedOutException" ]
-    },
-    "batch_mutate" : {
-      "request" : [ { "name" : "mutation_map",	"type" : { "type" : "array",
-          "items" : "MutationsMapEntry"
-        }},
-  { "name" : "consistency_level",	"type" : "ConsistencyLevel"
-      } ],
-      "response" : "null",
-      "errors" : [ "InvalidRequestException", "UnavailableException", "TimedOutException" ]
-    },
-    "system_add_keyspace" : {
-      "request" : [ { "name" : "ks_def",	"type" : "KsDef"
-      } ],
-      "response" : "null",
-      "errors" : [ "InvalidRequestException" ]
-    },
-    "set_keyspace" : {
-      "request" : [ { "name" : "keyspace",	"type" : "string"
-      } ],
-      "response" : "null",
-      "errors" : [ "InvalidRequestException" ]
-    },
-    "describe_keyspaces" : {
-      "request" : [ ],
-      "response" : { "type" : "array",
-        "items" : "string"
-      }
-    },
-    "describe_cluster_name" : {
-      "request" : [ ],
-      "response" : "string"
-    },
-    "describe_version" : {
-      "request" : [ ],
-      "response" : "string"
-    }
-  }
-}

data/examples/cassandra_streaming/pymap.sh

@@ -1 +0,0 @@
-python /usr/local/share/cassandra/contrib/hadoop_streaming_output/bin/mapper.py

data/examples/cassandra_streaming/pyreduce.sh

@@ -1 +0,0 @@
-python /usr/local/share/cassandra/contrib/hadoop_streaming_output/bin/reducer.py

data/examples/cassandra_streaming/smutation.avpr

@@ -1,188 +0,0 @@
-{
-    "protocol" : "Cassandra",
-    "namespace" : "org.apache.cassandra.avro",  "types" : [
-        
-        { "name" : "AccessLevel",               "type" : "enum",       "symbols" : [ "NONE", "READONLY", "READWRITE", "FALL" ]  },
-        { "name" : "ColumnPath",                "type" : "record",    "fields" : [
-            { "name" : "column_family",         "type" : "string"},
-            { "name" : "super_column",          "type" : [ "bytes", "null" ]},
-            { "name" : "column",                "type" : [ "bytes", "null" ] } ]},
-        { "name" : "ColumnParent",              "type" : "record",	"fields" : [
-            { "name" : "column_family",         "type" : "string"},
-            { "name" : "super_column",          "type" : [ "bytes", "null" ] } ]},
-        { "name" : "SliceRange",                "type" : "record",	"fields" : [
-            { "name" : "start",                 "type" : "bytes"},
-            { "name" : "finish",                "type" : "bytes"},
-            { "name" : "reversed",              "type" : "boolean"},
-            { "name" : "count",                 "type" : "int"},
-            { "name" : "bitmasks",              "type" : [ { "type" : "array",        "items" : "bytes"},  "null" ]    } ]},
-        { "name" : "SlicePredicate",            "type" : "record",	"fields" : [
-            { "name" : "column_names",          "type" : [ { "type" : "array",        "items" : "bytes"},  "null" ]},
-            { "name" : "slice_range",           "type" : [ "SliceRange", "null" ]    } ]},
-
-        { "name": "StupidColumnMutation", "type": "record",
-          "fields": [
-              { "name" : "key",	                "type" : "bytes"  },
-              { "name" : "name",                "type" : "bytes"  },
-              { "name" : "value",               "type" : "bytes"  },
-              { "name" : "timestamp",	        "type" : "long"   },
-              { "name" : "ttl",                 "type" : "int"    }
-          ]},
-        
-        { "name" : "Clock",                     "type" : "record",	"fields" : [
-            { "name" : "timestamp",	"type" : "long"   } ]},
-        { "name" : "Column",                    "type" : "record",	"fields" : [
-            { "name" : "name",                  "type" : "bytes"},
-            { "name" : "value",                 "type" : "bytes"},
-            { "name" : "clock",                 "type" : "Clock"},
-            { "name" : "ttl",                   "type" : "int"   } ]},
-        { "name" : "SuperColumn",               "type" : "record",	"fields" : [
-            { "name" : "name",                  "type" : "bytes"},
-            { "name" : "columns",               "type" : { "type" : "array",        "items" : "Column" } } ]},
-        { "name" : "ColumnOrSuperColumn",       "type" : "record",	"fields" : [
-            { "name" : "column",                "type" : "Column" },
-            { "name" : "super_column",          "type" : "null"     } ]},
-        { "name" : "Deletion",                  "type" : "record",	"fields" : [
-            { "name" : "clock",                 "type" : "Clock"},
-            { "name" : "super_column",          "type" : [ "bytes", "null" ]},
-            { "name" : "predicate",             "type" : [ "SlicePredicate", "null" ]    } ]},
-        { "name" : "Mutation",                  "type" : "record",	"fields" : [
-            { "name" : "column_or_supercolumn", "type" : "ColumnOrSuperColumn" },
-            { "name" : "deletion",              "type" : "null" }
-        ]},
-        { "name" : "StreamingMutation",         "type" : "record",	"fields" : [
-            { "name" : "key",	                "type" : "bytes"       },
-            { "name" : "mutation",	        "type" : "Mutation"    } ]},
-
-        { "name" : "IndexType",                 "type" : "enum",	"symbols" : [ "KEYS" ]},
-        { "name" : "ColumnDef",                 "type" : "record",	"fields" : [
-            { "name" : "name",                  "type" : "bytes"},
-            { "name" : "validation_class",      "type" : "string"},
-            { "name" : "index_type",            "type" : [ "IndexType", "null" ]},
-            { "name" : "index_name",            "type" : [ "string", "null" ]    } ]},
-        { "name" : "CfDef",                     "type" : "record",	"fields" : [
-            { "name" : "keyspace",              "type" : "string"},
-            { "name" : "name",                  "type" : "string"},
-            { "name" : "column_type",           "type" : [ "string", "null" ]},
-            { "name" : "clock_type",            "type" : [ "string", "null" ]},
-            { "name" : "comparator_type",       "type" : [ "string", "null" ]},
-            { "name" : "subcomparator_type",    "type" : [ "string", "null" ]},
-            { "name" : "reconciler",            "type" : [ "string", "null" ]},
-            { "name" : "comment",               "type" : [ "string", "null" ]},
-            { "name" : "row_cache_size",        "type" : [ "double", "null" ]},
-            { "name" : "preload_row_cache",     "type" : [ "boolean", "null" ]},
-            { "name" : "key_cache_size",        "type" : [ "double", "null" ]},
-            { "name" : "read_repair_chance",    "type" : [ "double", "null" ]},
-            { "name" : "gc_grace_seconds",      "type" : [ "int", "null" ]},
-            { "name" : "column_metadata",       "type" : [ { "type" : "array",        "items" : "ColumnDef"},  "null" ]},
-            { "name" : "id",                    "type" : [ "int", "null" ]    } ]},
-        { "name" : "KsDef",                     "type" : "record",	"fields" : [
-            { "name" : "name",                  "type" : "string"},  { "name" : "strategy_class",	"type" : "string"},
-            { "name" : "strategy_options",      "type" : [ { "type" : "map",        "values" : "string"},  "null" ]},
-            { "name" : "replication_factor",    "type" : "int"},  { "name" : "cf_defs",	"type" : { "type" : "array",        "items" : "CfDef"      }    } ]},
-        { "name" : "MutationsMapEntry",         "type" : "record",	"fields" : [ { "name" : "key",	"type" : "bytes"},  { "name" : "mutations",	"type" : { "type" : "map",        "values" : { "type" : "array",          "items" : "Mutation"        }      }    } ]},
-        { "name" : "CoscsMapEntry",             "type" : "record",	"fields" : [ { "name" : "key",	"type" : "bytes"},  { "name" : "columns",	"type" : { "type" : "array",        "items" : "ColumnOrSuperColumn"      }    } ]},
-        { "name" : "ConsistencyLevel",          "type" : "enum",	"symbols" : [ "ZERO", "ONE", "QUORUM", "DCQUORUM", "DCQUORUMSYNC", "ALL" ]},
-        { "name" : "InvalidRequestException",   "type" : "error",	"fields" : [ { "name" : "why",	"type" : [ "string", "null" ]    } ]},
-        { "name" : "NotFoundException",         "type" : "error",	"fields" : [ { "name" : "why",	"type" : [ "string", "null" ]    } ]},
-        { "name" : "UnavailableException",      "type" : "error",	"fields" : [ { "name" : "why",	"type" : [ "string", "null" ]    } ]},
-        { "name" : "TimedOutException",         "type" : "error",	"fields" : [ { "name" : "why",	"type" : [ "string", "null" ]    } ]  }
-    ],
-
-    
-  "messages" : { "get" : {
-      "request" : [ { "name" : "key",	"type" : "bytes"},
-  { "name" : "column_path",	"type" : "ColumnPath"},
-  { "name" : "consistency_level",	"type" : "ConsistencyLevel"
-      } ],
-      "response" : "ColumnOrSuperColumn",
-      "errors" : [ "InvalidRequestException", "NotFoundException", "UnavailableException", "TimedOutException" ]
-    },
-    "get_slice" : {
-      "request" : [ { "name" : "key",	"type" : "bytes"},
-  { "name" : "column_parent",	"type" : "ColumnParent"},
-  { "name" : "predicate",	"type" : "SlicePredicate"},
-  { "name" : "consistency_level",	"type" : "ConsistencyLevel"
-      } ],
-      "response" : { "type" : "array",
-        "items" : "ColumnOrSuperColumn"
-      },
-      "errors" : [ "InvalidRequestException", "UnavailableException", "TimedOutException" ]
-    },
-    "multiget_slice" : {
-      "request" : [ { "name" : "keys",	"type" : { "type" : "array",
-          "items" : "bytes"
-        }},
-  { "name" : "column_parent",	"type" : "ColumnParent"},
-  { "name" : "predicate",	"type" : "SlicePredicate"},
-  { "name" : "consistency_level",	"type" : "ConsistencyLevel"
-      } ],
-      "response" : { "type" : "array",
-        "items" : "CoscsMapEntry"
-      },
-      "errors" : [ "InvalidRequestException", "UnavailableException", "TimedOutException" ]
-    },
-    "get_count" : {
-      "request" : [ { "name" : "key",	"type" : "bytes"},
-  { "name" : "column_parent",	"type" : "ColumnParent"},
-  { "name" : "predicate",	"type" : "SlicePredicate"},
-  { "name" : "consistency_level",	"type" : "ConsistencyLevel"
-      } ],
-      "response" : "int",
-      "errors" : [ "InvalidRequestException", "UnavailableException", "TimedOutException" ]
-    },
-    "insert" : {
-      "request" : [ { "name" : "key",	"type" : "bytes"},
-  { "name" : "column_parent",	"type" : "ColumnParent"},
-  { "name" : "column",	"type" : "Column"},
-  { "name" : "consistency_level",	"type" : "ConsistencyLevel"
-      } ],
-      "response" : "null",
-      "errors" : [ "InvalidRequestException", "UnavailableException", "TimedOutException" ]
-    },
-    "remove" : {
-      "request" : [ { "name" : "key",	"type" : "bytes"},
-  { "name" : "column_path",	"type" : "ColumnPath"},
-  { "name" : "clock",	"type" : "Clock"},
-  { "name" : "consistency_level",	"type" : "ConsistencyLevel"
-      } ],
-      "response" : "null",
-      "errors" : [ "InvalidRequestException", "UnavailableException", "TimedOutException" ]
-    },
-    "batch_mutate" : {
-      "request" : [ { "name" : "mutation_map",	"type" : { "type" : "array",
-          "items" : "MutationsMapEntry"
-        }},
-  { "name" : "consistency_level",	"type" : "ConsistencyLevel"
-      } ],
-      "response" : "null",
-      "errors" : [ "InvalidRequestException", "UnavailableException", "TimedOutException" ]
-    },
-    "system_add_keyspace" : {
-      "request" : [ { "name" : "ks_def",	"type" : "KsDef"
-      } ],
-      "response" : "null",
-      "errors" : [ "InvalidRequestException" ]
-    },
-    "set_keyspace" : {
-      "request" : [ { "name" : "keyspace",	"type" : "string"
-      } ],
-      "response" : "null",
-      "errors" : [ "InvalidRequestException" ]
-    },
-    "describe_keyspaces" : {
-      "request" : [ ],
-      "response" : { "type" : "array",
-        "items" : "string"
-      }
-    },
-    "describe_cluster_name" : {
-      "request" : [ ],
-      "response" : "string"
-    },
-    "describe_version" : {
-      "request" : [ ],
-      "response" : "string"
-    }
-  }
-}

data/examples/cassandra_streaming/streamer.sh

@@ -1,51 +0,0 @@
-#!/usr/bin/env bash
-
-input_file="$1" 		 ; shift
-output_file="$1" 		 ; shift
-map_script=${1-/bin/cat}	 ; shift
-reduce_script=${1-/usr/bin/uniq} ; shift
-
-dest_keyspace=${dest_keyspace-soc_net_tw}
-dest_col_family=${dest_col_family-Wordbag}
-
-hostname=`hostname`
-
-# Path to cassandra and hadoop dirs
-script_dir=$(readlink -f `dirname $0`)
-CASSANDRA_HOME=${CASSANDRA_HOME-/usr/local/share/cassandra}
-HADOOP_HOME=${HADOOP_HOME-/usr/lib/hadoop}
-avro_file=${avro_file-$CASSANDRA_HOME/interface/avro/cassandra.avpr}
-
-ARCHIVES=`/bin/ls -1 $CASSANDRA_HOME/build/apache-cassandra*.jar`
-for jar in `/bin/ls -1 $CASSANDRA_HOME/build/lib/jars/*.jar $CASSANDRA_HOME/lib/*.jar`; do
-    ARCHIVES=$ARCHIVES,$jar
-done
-
-${HADOOP_HOME}/bin/hadoop                                                                        \
-     jar ${HADOOP_HOME}/contrib/streaming/hadoop-*streaming*.jar                                 \
-    -D stream.map.output=cassandra_avro_output                                                   \
-    -D stream.io.identifier.resolver.class=org.apache.cassandra.hadoop.streaming.AvroResolver    \
-    -D cassandra.output.keyspace="$dest_keyspace"                                                \
-    -D cassandra.output.columnfamily="$dest_col_family"                                          \
-    -D cassandra.thrift.address=10.204.41.193,10.204.30.11,10.204.58.238,10.204.239.133,10.196.191.31,10.204.103.21,10.202.74.223,10.202.143.95 \
-    -D cassandra.partitioner.class=org.apache.cassandra.dht.RandomPartitioner                    \
-    -D cassandra.thrift.port=9160                                                                \
-    -D mapreduce.output.columnfamilyoutputformat.batch.threshold=1024                            \
-    -D mapred.reduce.tasks=0                                                                     \
-    -D mapred.map.tasks.speculative.execution=false                                              \
-    -libjars $ARCHIVES                                                                           \
-    -file $avro_file                                                                             \
-    -outputformat org.apache.cassandra.hadoop.ColumnFamilyOutputFormat                           \
-    -mapper  	 "ruby $script_dir/avromapper.rb --map "                                         \
-    -input       "$input_file"                                                                   \
-    -output  	 "$output_file"                                                                  \
-    "$@"
-
-    # -D cassandra.thrift.address=10.204.54.190,10.244.42.31,10.244.42.176,10.244.42.112,10.244.42.143,10.244.42.79,10.244.42.4,10.204.53.166 \
-    # -D cassandra.thrift.address=10.204.221.230,10.243.79.223,10.245.19.159,10.242.154.159,10.242.153.155,10.242.153.203 \
-
-
-# cat /tmp/mj-flip/chimchim-info.log | cut -f5 | ruby -e 'puts $stdin.readlines.map{|l| l.chomp.gsub(/ip-([0-9\-]+)\..*/,"\\1").gsub(/-/,".") }.join(",")'
-
-
-

data/examples/cassandra_streaming/struct_loader.rb

@@ -1,24 +0,0 @@
-#!/usr/bin/env ruby
-require 'rubygems'
-require 'wukong'
-require 'wukong/periodic_monitor'
-require 'wukong/store/cassandra'
-require 'wukong/script/cassandra_loader_script'
-
-Settings.use :commandline
-Settings.define :log_interval,    :default => 1
-Settings.cassandra_keyspace   = 'soc_net_tw'
-Settings.cassandra_col_family = 'TwitterUser'
-Settings.cassandra_hosts      = "ip-10-204-41-193.ec2.internal:9160,ip-10-204-30-11.ec2.internal:9160,ip-10-204-58-238.ec2.internal:9160,ip-10-204-239-133.ec2.internal:9160,ip-10-196-191-31.ec2.internal:9160,ip-10-204-103-21.ec2.internal:9160,ip-10-202-74-223.ec2.internal:9160,ip-10-202-143-95.ec2.internal:9160"
-Settings.resolve!
-
-require 'cassandra/0.7'
-require 'wuclan/twitter' ; include Wuclan::Twitter
-require 'wuclan/twitter/cassandra_db'
-require 'wukong/store/cassandra/streaming'
-
-# hdp-catd s3://s3hdfs.infinitemonkeys.info/data/sn/tw/fixd/objects/twitter_user | head
-
-# CassandraScript.new(Wukong::Store::Cassandra::StructLoader, nil).run
-Wukong::CassandraScript.new(Wukong::Store::Cassandra::StructLoader, nil).run
-

data/examples/count_keys.rb

@@ -1,56 +0,0 @@
-#!/usr/bin/env ruby
-$: << File.dirname(__FILE__)+'/../lib'
-require 'wukong'
-require 'wukong/streamer/count_keys'
-require 'wukong/streamer/count_lines'
-
-#
-#
-class CountKeysReducer < Wukong::Streamer::CountLines
-  #
-  # Taken from the actionpack Rails component ('action_view/helpers/number_helper')
-  #
-  # Formats a +number+ with grouped thousands using +delimiter+. You
-  # can customize the format using optional <em>delimiter</em> and <em>separator</em> parameters.
-  # * <tt>delimiter</tt>  - Sets the thousands delimiter, defaults to ","
-  # * <tt>separator</tt>  - Sets the separator between the units, defaults to "."
-  #
-  #  number_with_delimiter(12345678)      => 12,345,678
-  #  number_with_delimiter(12345678.05)   => 12,345,678.05
-  #  number_with_delimiter(12345678, ".")   => 12.345.678
-  def number_with_delimiter(number, delimiter=",", separator=".")
-    begin
-      parts = number.to_s.split('.')
-      parts[0].gsub!(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1#{delimiter}")
-      parts.join separator
-    rescue
-      number
-    end
-  end
-
-  # Override to look nice
-  def formatted_count item, key_count
-    key_count_str = number_with_delimiter(key_count.to_i)
-    "%-25s\t%12s" % [item, key_count_str]
-  end
-end
-
-#
-class CountKeysScript < Wukong::Script
-  def map_command
-    # Use `cut` to extract the first field
-    %Q{ cut -d"\t" -f1 }
-  end
-
-  #
-  # There's just the one field
-  #
-  def default_options
-    super.merge :sort_fields => 1
-  end
-end
-
-# Executes the script when run from command line
-if __FILE__ == $0
-  CountKeysScript.new(nil, CountKeysReducer).run
-end

data/examples/count_keys_at_mapper.rb

@@ -1,57 +0,0 @@
-#!/usr/bin/env ruby
-$: << File.dirname(__FILE__)+'/../lib'
-require 'wukong'
-
-#
-#
-module CountKeys
-  #
-  class Mapper < Wukong::Streamer::Base
-    attr_accessor :keys_count
-    def initialize *args
-      self.keys_count = {}
-    end
-    def process key, *args
-      key.gsub!(/-.*/, '')  # kill off the slug
-      self.keys_count[key] ||= 0
-      self.keys_count[key]  += 1
-    end
-    def stream *args
-      super *args
-      self.keys_count.each do |key, count|
-        emit [key, count].to_flat
-      end
-    end
-  end
-  # Identity Mapper
-  class Reducer < Wukong::Streamer::AccumulatingReducer
-    attr_accessor :key_count
-    require 'active_support'
-    require 'action_view/helpers/number_helper'; include ActionView::Helpers::NumberHelper
-
-    # Override to look nice
-    def formatted_count item, key_count
-      key_count_str = number_with_delimiter(key_count.to_i, :delimiter => ',')
-      "%-25s\t%12s" % [item, key_count_str]
-    end
-    def start!  *args
-      self.key_count = 0
-    end
-    def accumulate key, count
-      self.key_count += count.to_i
-    end
-    def finalize
-      yield formatted_count(key, key_count)
-    end
-  end
-
-  #
-  class Script < Wukong::Script
-    # There's just the one field
-    def default_options
-      super.merge :sort_fields => 1, :reduce_tasks => 1
-    end
-  end
-end
-
-CountKeys::Script.new(CountKeys::Mapper, CountKeys::Reducer).run

data/examples/emr/README-elastic_map_reduce.textile

@@ -1,26 +0,0 @@
-
-# Download the Amazon elastic-mapreduce runner from http://elasticmapreduce.s3.amazonaws.com/elastic-mapreduce-ruby.zip
-
-# Create a bucket and path to hold your EMR logs, scripts and other ephemera. For instance you might choose 'emr.yourdomain.com' as the bucket and '/wukong' as a scoping path within that bucket. In that case you will refer to it with a path like s3n://emr.yourdomain.com/wukong (see notes below about s3n:// vs. s3:// URLs).
-
-# Copy the contents of wukong/examples/emr/dot_wukong_dir to ~/.wukong
-# Edit emr.yaml -- it has instructions for the 
-
-
-
-
-
-h3. s3n:// vs. s3:// URLs
-
-Many external tools use a URI convention to address files in S3; they typically use the 's3://' scheme, which makes a lot of sense:
-  s3://emr.yourcompany.com/wukong/happy_job_1/logs/whatever-20100808.log
-
-Hadoop can maintain an HDFS on the Amazon S3: it uses a block structure and has optimizations for streaming, no file size limitation, and other goodness. However, only hadoop tools can interpret the contents of those blocks -- to everything else it just looks like a soup of blocks labelled block_-8675309 and so forth.  Hadoop unfortunately chose the 's3://' scheme for URIs in this filesystem:
-  s3://s3hdfs.yourcompany.com/path/to/data
-
-Hadoop is happy to read s3 native files -- 'native' as in, you can look at them with a browser and upload them an download them with any S3 tool out there. There's a 5GB limit on file size, and in some cases a performance hit (but not in our experience enough to worry about).  You refer to these files with the 's3n://' scheme ('n' as in 'native'):
-  s3n://emr.yourcompany.com/wukong/happy_job_1/code/happy_job_1-mapper.rb
-  s3n://emr.yourcompany.com/wukong/happy_job_1/code/happy_job_1-reducer.rb
-  s3n://emr.yourcompany.com/wukong/happy_job_1/logs/whatever-20100808.log
-
-Wukong will coerce things to the right scheme when it knows what that scheme should be (eg. code should be s3n://). It will otherwise leave the path alone. Specifically, if you use a URI scheme for input and output paths you must use 's3n://' for normal s3 files.