tros 1.7.6.1

data/lib/tros/version.rb

@@ -0,0 +1,3 @@
+module Tros
+  VERSION = "1.7.6.1"
+end

data/test/datafile_test.rb

@@ -0,0 +1,193 @@
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'test_helper'
+
+class DataFileTest < Minitest::Test
+  TEST_FILE = File.join(TMP_DIR, 'data_file_test.avro')
+  
+  def setup
+    File.unlink(TEST_FILE) if File.exists?(TEST_FILE)
+  end
+
+  def teardown
+    File.unlink(TEST_FILE) if File.exists?(TEST_FILE)
+  end
+
+  def test_differing_schemas_with_primitives
+    writer_schema = <<-JSON
+      { 
+        "type": "record",
+        "name": "User",
+        "fields" : [
+          {"name": "username", "type": "string"},
+          {"name": "age", "type": "int"},
+          {"name": "verified", "type": "boolean", "default": "false"}
+        ]
+      }
+    JSON
+
+    data = [{"username" => "john", "age" => 25, "verified" => true},
+            {"username" => "ryan", "age" => 23, "verified" => false}]
+
+    Tros::DataFile.open(TEST_FILE, 'w', writer_schema) do |dw|
+      data.each{|h| dw << h }
+    end
+
+    # extract the username only from the tros serialized file
+    reader_schema = <<-JSON
+      { 
+        "type": "record",
+        "name": "User",
+        "fields" : [
+          {"name": "username", "type": "string"}
+        ]
+      }
+    JSON
+
+    Tros::DataFile.open(TEST_FILE, 'r', reader_schema) do |dr|
+      dr.each_with_index do |record, i|
+        assert_equal data[i]['username'], record['username']
+      end
+    end
+  end
+
+  def test_differing_schemas_with_complex_objects
+    writer_schema = <<-JSON
+      { 
+        "type": "record",
+        "name": "something",
+        "fields": [
+          {"name": "something_fixed", "type": {"name": "inner_fixed",
+                                               "type": "fixed", "size": 3}},
+          {"name": "something_enum", "type": {"name": "inner_enum",
+                                              "type": "enum",
+                                              "symbols": ["hello", "goodbye"]}},
+          {"name": "something_array", "type": {"type": "array", "items": "int"}},
+          {"name": "something_map", "type": {"type": "map", "values": "int"}},
+          {"name": "something_record", "type": {"name": "inner_record",
+                                                "type": "record",
+                                                "fields": [
+                                                  {"name": "inner", "type": "int"}
+                                                ]}},
+          {"name": "username", "type": "string"}
+        ]
+      }
+    JSON
+
+    data = [{"username" => "john",
+              "something_fixed" => "foo",
+              "something_enum" => "hello",
+              "something_array" => [1,2,3],
+              "something_map" => {"a" => 1, "b" => 2},
+              "something_record" => {"inner" => 2},
+              "something_error" => {"code" => 403}
+            },
+            {"username" => "ryan",
+              "something_fixed" => "bar",
+              "something_enum" => "goodbye",
+              "something_array" => [1,2,3],
+              "something_map" => {"a" => 2, "b" => 6},
+              "something_record" => {"inner" => 1},
+              "something_error" => {"code" => 401}
+            }]
+
+    Tros::DataFile.open(TEST_FILE, 'w', writer_schema) do |dw|
+      data.each{|d| dw << d }
+    end
+
+    %w[fixed enum record error array map union].each do |s|
+      reader = JSON.load(writer_schema)
+      reader['fields'] = reader['fields'].reject{|f| f['type']['type'] == s}
+      Tros::DataFile.open(TEST_FILE, 'r', JSON.dump(reader)) do |dr|
+        dr.each_with_index do |obj, i|
+          reader['fields'].each do |field|
+            assert_equal data[i][field['name']], obj[field['name']]
+          end
+        end
+      end
+    end
+  end
+
+  def test_data_writer_handles_sync_interval
+    writer_schema = <<-JSON
+      { 
+        "type": "record",
+        "name": "something",
+        "fields": [
+          {"name": "something_boolean", "type": "boolean"}
+        ]
+      }
+    JSON
+
+    data = {"something_boolean" => true }
+
+    Tros::DataFile.open(TEST_FILE, 'w', writer_schema) do |dw|
+      while dw.writer.tell < Tros::DataFile::SYNC_INTERVAL
+        dw << data
+      end
+      block_count = dw.block_count
+      dw << data
+      # ensure we didn't just write another block
+      assert_equal(block_count+1, dw.block_count)
+    end
+  end
+
+  def test_utf8
+    datafile = Tros::DataFile.open(TEST_FILE, 'w', '"string"')
+    datafile << "家"
+    datafile.close
+
+    datafile = Tros::DataFile.open(TEST_FILE)
+    datafile.each do |s|
+      assert_equal "家", s
+    end
+    datafile.close
+  end
+
+  def test_deflate
+    Tros::DataFile.open(TEST_FILE, 'w', '"string"', :deflate) do |writer|
+      writer << 'a' * 10_000
+    end
+    assert(File.size(TEST_FILE) < 500)
+
+    records = []
+    Tros::DataFile.open(TEST_FILE) do |reader|
+      reader.each { |record| records << record }
+    end
+    assert_equal records, ['a' * 10_000]
+  end
+
+  def test_append_to_deflated_file
+    schema = Tros::Schema.parse('"string"')
+    writer = Tros::IO::DatumWriter.new(schema)
+    file = Tros::DataFile::Writer.new(File.open(TEST_FILE, 'wb'), writer, schema, :deflate)
+    file << 'a' * 10_000
+    file.close
+
+    file = Tros::DataFile::Writer.new(File.open(TEST_FILE, 'a+b'), writer)
+    file << 'b' * 10_000
+    file.close
+    assert(File.size(TEST_FILE) < 1_000)
+
+    records = []
+    Tros::DataFile.open(TEST_FILE) do |reader|
+      reader.each {|record| records << record }
+    end
+    assert_equal records, ['a' * 10_000, 'b' * 10_000]
+  end
+end

data/test/fixtures/schemas/org/apache/avro/data/Json.avsc

@@ -0,0 +1,15 @@
+{"type": "record", "name": "Json", "namespace":"org.apache.avro.data",
+ "fields": [
+     {"name": "value",
+      "type": [
+          "long",
+          "double",
+          "string",
+          "boolean",
+          "null",
+          {"type": "array", "items": "Json"},
+          {"type": "map", "values": "Json"}
+      ]
+     }
+ ]
+}

data/test/fixtures/schemas/org/apache/avro/ipc/HandshakeRequest.avsc

@@ -0,0 +1,11 @@
+{
+    "type": "record",
+    "name": "HandshakeRequest", "namespace":"org.apache.avro.ipc",
+    "fields": [
+        {"name": "clientHash",
+	 "type": {"type": "fixed", "name": "MD5", "size": 16}},
+        {"name": "clientProtocol", "type": ["null", "string"]},
+        {"name": "serverHash", "type": "MD5"},
+ 	{"name": "meta", "type": ["null", {"type": "map", "values": "bytes"}]}
+ ]
+}

data/test/fixtures/schemas/org/apache/avro/ipc/HandshakeResponse.avsc

@@ -0,0 +1,15 @@
+{
+    "type": "record",
+    "name": "HandshakeResponse", "namespace": "org.apache.avro.ipc",
+    "fields": [
+        {"name": "match",
+         "type": {"type": "enum", "name": "HandshakeMatch",
+                  "symbols": ["BOTH", "CLIENT", "NONE"]}},
+        {"name": "serverProtocol",
+         "type": ["null", "string"]},
+        {"name": "serverHash",
+         "type": ["null", {"type": "fixed", "name": "MD5", "size": 16}]},
+ 	{"name": "meta",
+         "type": ["null", {"type": "map", "values": "bytes"}]}
+    ]
+}

data/test/fixtures/schemas/org/apache/avro/ipc/trace/avroTrace.avdl

@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * A Span is our basic unit of tracing. It tracks the critical points
+ * of a single RPC call and records other call meta-data. It also
+ * allows arbitrary string annotations. Both the client and server create
+ * Span objects, each of which is populated with half of the relevant event
+ * data. They share a span ID, which allows us to merge them into one complete
+ * span later on.
+ */
+@namespace("org.apache.avro.ipc.trace")
+
+protocol AvroTrace {
+  enum SpanEvent { SERVER_RECV, SERVER_SEND, CLIENT_RECV, CLIENT_SEND }
+
+  fixed ID(8);
+
+  record TimestampedEvent {
+    long timeStamp; // Unix time, in nanoseconds
+    union { SpanEvent, string} event;
+  }
+
+  /**
+   * An individual span is the basic unit of testing.
+   * The record is used by both \"client\" and \"server\".
+   */
+  record Span {
+    ID  traceID;  // ID shared by all Spans in a given trace
+    ID spanID;    // Random ID for this Span
+    union { ID, null } parentSpanID; // Parent Span ID (null if root Span)
+    string messageName;       // Function call represented
+    long requestPayloadSize;  // Size (bytes) of the request
+    long responsePayloadSize; // Size (byts) of the response
+    union { string, null} requestorHostname; // Hostname of requestor
+//    int requestorPort;     // Port of the requestor (currently unused)
+    union { string, null } responderHostname; // Hostname of the responder
+//    int responderPort;     // Port of the responder (currently unused)
+    array<TimestampedEvent> events;  // List of critical events
+    boolean complete; // Whether includes data from both sides
+  }
+
+  /**
+   * Get all spans stored on this host.
+   */
+  array<Span> getAllSpans();
+
+  /**
+   * Get spans occuring between start and end. Each is a unix timestamp
+   * in nanosecond units (for consistency with TimestampedEvent).
+   */
+  array<Span> getSpansInRange(long start, long end);
+}

data/test/fixtures/schemas/org/apache/avro/ipc/trace/avroTrace.avpr

@@ -0,0 +1,82 @@
+{
+  "protocol" : "AvroTrace",
+  "namespace" : "org.apache.avro.ipc.trace",
+  "types" : [ {
+    "type" : "enum",
+    "name" : "SpanEvent",
+    "symbols" : [ "SERVER_RECV", "SERVER_SEND", "CLIENT_RECV", "CLIENT_SEND" ]
+  }, {
+    "type" : "fixed",
+    "name" : "ID",
+    "size" : 8
+  }, {
+    "type" : "record",
+    "name" : "TimestampedEvent",
+    "fields" : [ {
+      "name" : "timeStamp",
+      "type" : "long"
+    }, {
+      "name" : "event",
+      "type" : [ "SpanEvent", "string" ]
+    } ]
+  }, {
+    "type" : "record",
+    "name" : "Span",
+    "fields" : [ {
+      "name" : "traceID",
+      "type" : "ID"
+    }, {
+      "name" : "spanID",
+      "type" : "ID"
+    }, {
+      "name" : "parentSpanID",
+      "type" : [ "ID", "null" ]
+    }, {
+      "name" : "messageName",
+      "type" : "string"
+    }, {
+      "name" : "requestPayloadSize",
+      "type" : "long"
+    }, {
+      "name" : "responsePayloadSize",
+      "type" : "long"
+    }, {
+      "name" : "requestorHostname",
+      "type" : [ "string", "null" ]
+    }, {
+      "name" : "responderHostname",
+      "type" : [ "string", "null" ]
+    }, {
+      "name" : "events",
+      "type" : {
+        "type" : "array",
+        "items" : "TimestampedEvent"
+      }
+    }, {
+      "name" : "complete",
+      "type" : "boolean"
+    } ]
+  } ],
+  "messages" : {
+    "getAllSpans" : {
+      "request" : [ ],
+      "response" : {
+        "type" : "array",
+        "items" : "Span"
+      }
+    },
+    "getSpansInRange" : {
+      "request" : [ {
+        "name" : "start",
+        "type" : "long"
+      }, {
+        "name" : "end",
+        "type" : "long"
+      } ],
+      "response" : {
+        "type" : "array",
+        "items" : "Span"
+      }
+    }
+  }
+}

data/test/fixtures/schemas/org/apache/avro/mapred/tether/InputProtocol.avpr

@@ -0,0 +1,64 @@
+{"namespace":"org.apache.avro.mapred.tether",
+ "protocol": "InputProtocol",
+ "doc": "Transmit inputs to a map or reduce task sub-process.",
+
+ "types": [
+     {"name": "TaskType", "type": "enum", "symbols": ["MAP","REDUCE"]}
+ ],
+
+ "messages": {
+
+     "configure": {
+         "doc": "Configure the task.  Sent before any other message.",
+         "request": [
+             {"name": "taskType", "type": "TaskType",
+              "doc": "Whether this is a map or reduce task."},
+             {"name": "inSchema", "type": "string",
+              "doc": "The Avro schema for task input data."},
+             {"name": "outSchema", "type": "string",
+              "doc": "The Avro schema for task output data."}
+         ],
+         "response": "null",
+         "one-way": true
+     },
+
+     "partitions": {
+         "doc": "Set the number of map output partitions.",
+         "request": [
+             {"name": "partitions", "type": "int",
+              "doc": "The number of map output partitions."}
+         ],
+         "response": "null",
+         "one-way": true
+     },
+
+     "input": {
+         "doc": "Send a block of input data to a task.",
+         "request": [
+             {"name": "data", "type": "bytes",
+              "doc": "A sequence of instances of the declared schema."},
+             {"name": "count", "type": "long",
+              "default": 1,
+              "doc": "The number of instances in this block."}
+         ],
+         "response": "null",
+         "one-way": true
+     },
+
+     "abort": {
+         "doc": "Called to abort the task.",
+         "request": [],
+         "response": "null",
+         "one-way": true
+     },
+
+     "complete": {
+         "doc": "Called when a task's input is complete.",
+         "request": [],
+         "response": "null",
+         "one-way": true
+     }
+
+ }
+
+}