avro-salsify-fork 1.9.0.0

data/test/sample_ipc_server.rb

@@ -0,0 +1,92 @@
+#!/usr/bin/env ruby
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'socket'
+require 'avro'
+
+MAIL_PROTOCOL_JSON = <<-EOS
+{"namespace": "example.proto",
+ "protocol": "Mail",
+
+ "types": [
+     {"name": "Message", "type": "record",
+      "fields": [
+          {"name": "to",   "type": "string"},
+          {"name": "from", "type": "string"},
+          {"name": "body", "type": "string"}
+      ]
+     }
+ ],
+
+ "messages": {
+     "send": {
+         "request": [{"name": "message", "type": "Message"}],
+         "response": "string"
+     },
+     "replay": {
+         "request": [],
+         "response": "string"
+     }
+ }
+}
+EOS
+
+MAIL_PROTOCOL = Avro::Protocol.parse(MAIL_PROTOCOL_JSON)
+
+class MailResponder < Avro::IPC::Responder
+  def initialize
+    super(MAIL_PROTOCOL)
+  end
+
+  def call(message, request)
+    if message.name == 'send'
+      request_content = request['message']
+      "Sent message to #{request_content['to']} from #{request_content['from']} with body #{request_content['body']}"
+    elsif message.name == 'replay'
+      'replay'
+    end
+  end
+end
+
+class RequestHandler
+  def initialize(address, port)
+    @ip_address = address
+    @port = port
+  end
+
+  def run
+    server = TCPServer.new(@ip_address, @port)
+    while (session = server.accept)
+      handle(session)
+      session.close
+    end
+  end
+end
+
+class MailHandler < RequestHandler
+  def handle(request)
+    responder = MailResponder.new()
+    transport = Avro::IPC::SocketTransport.new(request)
+    str = transport.read_framed_message
+    transport.write_framed_message(responder.respond(str))
+  end
+end
+
+if $0 == __FILE__
+  handler = MailHandler.new('localhost', 9090)
+  handler.run
+end

data/test/test_datafile.rb

@@ -0,0 +1,214 @@
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'test_help'
+
+class TestDataFile < Test::Unit::TestCase
+  HERE = File.expand_path File.dirname(__FILE__)
+  def setup
+    if File.exists?(HERE + '/data.avr')
+      File.unlink(HERE + '/data.avr')
+    end
+  end
+
+  def teardown
+    if File.exists?(HERE + '/data.avr')
+      File.unlink(HERE + '/data.avr')
+    end
+  end
+
+  def test_differing_schemas_with_primitives
+    writer_schema = <<-JSON
+{ "type": "record",
+  "name": "User",
+  "fields" : [
+    {"name": "username", "type": "string"},
+    {"name": "age", "type": "int"},
+    {"name": "verified", "type": "boolean", "default": "false"}
+  ]}
+JSON
+
+    data = [{"username" => "john", "age" => 25, "verified" => true},
+            {"username" => "ryan", "age" => 23, "verified" => false}]
+
+    Avro::DataFile.open('data.avr', 'w', writer_schema) do |dw|
+      data.each{|h| dw << h }
+    end
+
+    # extract the username only from the avro serialized file
+    reader_schema = <<-JSON
+{ "type": "record",
+  "name": "User",
+  "fields" : [
+    {"name": "username", "type": "string"}
+ ]}
+JSON
+
+    Avro::DataFile.open('data.avr', 'r', reader_schema) do |dr|
+      dr.each_with_index do |record, i|
+        assert_equal data[i]['username'], record['username']
+      end
+    end
+  end
+
+  def test_differing_schemas_with_complex_objects
+    writer_schema = <<-JSON
+{ "type": "record",
+  "name": "something",
+  "fields": [
+    {"name": "something_fixed", "type": {"name": "inner_fixed",
+                                         "type": "fixed", "size": 3}},
+    {"name": "something_enum", "type": {"name": "inner_enum",
+                                        "type": "enum",
+                                        "symbols": ["hello", "goodbye"]}},
+    {"name": "something_array", "type": {"type": "array", "items": "int"}},
+    {"name": "something_map", "type": {"type": "map", "values": "int"}},
+    {"name": "something_record", "type": {"name": "inner_record",
+                                          "type": "record",
+                                          "fields": [
+                                            {"name": "inner", "type": "int"}
+                                          ]}},
+    {"name": "username", "type": "string"}
+]}
+JSON
+
+    data = [{"username" => "john",
+              "something_fixed" => "foo",
+              "something_enum" => "hello",
+              "something_array" => [1,2,3],
+              "something_map" => {"a" => 1, "b" => 2},
+              "something_record" => {"inner" => 2},
+              "something_error" => {"code" => 403}
+            },
+            {"username" => "ryan",
+              "something_fixed" => "bar",
+              "something_enum" => "goodbye",
+              "something_array" => [1,2,3],
+              "something_map" => {"a" => 2, "b" => 6},
+              "something_record" => {"inner" => 1},
+              "something_error" => {"code" => 401}
+            }]
+
+    Avro::DataFile.open('data.avr', 'w', writer_schema) do |dw|
+      data.each{|d| dw << d }
+    end
+
+    %w[fixed enum record error array map union].each do |s|
+      reader = MultiJson.load(writer_schema)
+      reader['fields'] = reader['fields'].reject{|f| f['type']['type'] == s}
+      Avro::DataFile.open('data.avr', 'r', MultiJson.dump(reader)) do |dr|
+        dr.each_with_index do |obj, i|
+          reader['fields'].each do |field|
+            assert_equal data[i][field['name']], obj[field['name']]
+          end
+        end
+      end
+    end
+  end
+
+  def test_data_writer_handles_sync_interval
+    writer_schema = <<-JSON
+{ "type": "record",
+  "name": "something",
+  "fields": [
+    {"name": "something_boolean", "type": "boolean"}
+]}
+JSON
+
+    data = {"something_boolean" => true }
+
+    Avro::DataFile.open('data.avr', 'w', writer_schema) do |dw|
+      while dw.writer.tell < Avro::DataFile::SYNC_INTERVAL
+        dw << data
+      end
+      block_count = dw.block_count
+      dw << data
+      # ensure we didn't just write another block
+      assert_equal(block_count+1, dw.block_count)
+    end
+  end
+
+  def test_utf8
+    datafile = Avro::DataFile::open('data.avr', 'w', '"string"')
+    datafile << "家"
+    datafile.close
+
+    datafile = Avro::DataFile.open('data.avr')
+    datafile.each do |s|
+      assert_equal "家", s
+    end
+    datafile.close
+  end
+
+  def test_deflate
+    Avro::DataFile.open('data.avr', 'w', '"string"', :deflate) do |writer|
+      writer << 'a' * 10_000
+    end
+    assert(File.size('data.avr') < 500)
+
+    records = []
+    Avro::DataFile.open('data.avr') do |reader|
+      reader.each {|record| records << record }
+    end
+    assert_equal records, ['a' * 10_000]
+  end
+
+  def test_snappy
+    Avro::DataFile.open('data.avr', 'w', '"string"', :snappy) do |writer|
+      writer << 'a' * 10_000
+    end
+    assert(File.size('data.avr') < 600)
+
+    records = []
+    Avro::DataFile.open('data.avr') do |reader|
+      reader.each {|record| records << record }
+    end
+    assert_equal records, ['a' * 10_000]
+  end
+
+  def test_append_to_deflated_file
+    schema = Avro::Schema.parse('"string"')
+    writer = Avro::IO::DatumWriter.new(schema)
+    file = Avro::DataFile::Writer.new(File.open('data.avr', 'wb'), writer, schema, :deflate)
+    file << 'a' * 10_000
+    file.close
+
+    file = Avro::DataFile::Writer.new(File.open('data.avr', 'a+b'), writer)
+    file << 'b' * 10_000
+    file.close
+    assert(File.size('data.avr') < 1_000)
+
+    records = []
+    Avro::DataFile.open('data.avr') do |reader|
+      reader.each {|record| records << record }
+    end
+    assert_equal records, ['a' * 10_000, 'b' * 10_000]
+  end
+
+  def test_custom_meta
+    meta = { 'x.greeting' => 'yo' }
+
+    schema = Avro::Schema.parse('"string"')
+    writer = Avro::IO::DatumWriter.new(schema)
+    file = Avro::DataFile::Writer.new(File.open('data.avr', 'wb'), writer, schema, nil, meta)
+    file.close
+
+    Avro::DataFile.open('data.avr') do |reader|
+      assert_equal 'yo', reader.meta['x.greeting']
+    end
+  end
+end

data/test/test_fingerprints.rb

@@ -0,0 +1,37 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'test_help'
+
+class TestFingerprints < Test::Unit::TestCase
+  def test_md5_fingerprint
+    schema = Avro::Schema.parse <<-SCHEMA
+      { "type": "int" }
+    SCHEMA
+
+    assert_equal 318112854175969537208795771590915775282,
+      schema.md5_fingerprint
+  end
+
+  def test_sha256_fingerprint
+    schema = Avro::Schema.parse <<-SCHEMA
+      { "type": "int" }
+    SCHEMA
+
+    assert_equal 28572620203319713300323544804233350633246234624932075150020181448463213378117,
+      schema.sha256_fingerprint
+  end
+end

data/test/test_help.rb

@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'rubygems'
+require 'test/unit'
+require 'stringio'
+require 'fileutils'
+FileUtils.mkdir_p('tmp')
+require 'avro'
+require 'random_data'

data/test/test_io.rb

@@ -0,0 +1,451 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'test_help'
+
+class TestIO < Test::Unit::TestCase
+  DATAFILE = 'tmp/test.rb.avro'
+  Schema = Avro::Schema
+
+  def test_null
+    check('"null"')
+    check_default('"null"', "null", nil)
+  end
+
+  def test_boolean
+    check('"boolean"')
+    check_default('"boolean"', "true", true)
+    check_default('"boolean"', "false", false)
+  end
+
+  def test_string
+    check('"string"')
+    check_default('"string"', '"foo"', "foo")
+  end
+
+  def test_bytes
+    check('"bytes"')
+    check_default('"bytes"', '"foo"', "foo")
+  end
+
+  def test_int
+    check('"int"')
+    check_default('"int"', "5", 5)
+  end
+
+  def test_long
+    check('"long"')
+    check_default('"long"', "9", 9)
+  end
+
+  def test_float
+    check('"float"')
+    check_default('"float"', "1.2", 1.2)
+  end
+
+  def test_double
+    check('"double"')
+    check_default('"double"', "1.2", 1.2)
+  end
+
+  def test_array
+    array_schema = '{"type": "array", "items": "long"}'
+    check(array_schema)
+    check_default(array_schema, "[1]", [1])
+  end
+
+  def test_map
+    map_schema = '{"type": "map", "values": "long"}'
+    check(map_schema)
+    check_default(map_schema, '{"a": 1}', {"a" => 1})
+  end
+
+  def test_record
+    record_schema = <<EOS
+      {"type": "record",
+       "name": "Test",
+       "fields": [{"name": "f",
+                   "type": "long"}]}
+EOS
+    check(record_schema)
+    check_default(record_schema, '{"f": 11}', {"f" => 11})
+  end
+
+  def test_error
+    error_schema = <<EOS
+      {"type": "error",
+       "name": "TestError",
+       "fields": [{"name": "message",
+                   "type": "string"}]}
+EOS
+    check(error_schema)
+    check_default(error_schema, '{"message": "boom"}', {"message" => "boom"})
+  end
+
+  def test_enum
+    enum_schema = '{"type": "enum", "name": "Test","symbols": ["A", "B"]}'
+    check(enum_schema)
+    check_default(enum_schema, '"B"', "B")
+  end
+
+  def test_recursive
+    recursive_schema = <<EOS
+      {"type": "record",
+       "name": "Node",
+       "fields": [{"name": "label", "type": "string"},
+                  {"name": "children",
+                   "type": {"type": "array", "items": "Node"}}]}
+EOS
+    check(recursive_schema)
+  end
+
+  def test_union
+    union_schema = <<EOS
+      ["string",
+       "null",
+       "long",
+       {"type": "record",
+        "name": "Cons",
+        "fields": [{"name": "car", "type": "string"},
+                   {"name": "cdr", "type": "string"}]}]
+EOS
+    check(union_schema)
+    check_default('["double", "long"]', "1.1", 1.1)
+  end
+
+  def test_lisp
+    lisp_schema = <<EOS
+      {"type": "record",
+       "name": "Lisp",
+       "fields": [{"name": "value",
+                   "type": ["null", "string",
+                            {"type": "record",
+                             "name": "Cons",
+                             "fields": [{"name": "car", "type": "Lisp"},
+                                        {"name": "cdr", "type": "Lisp"}]}]}]}
+EOS
+    check(lisp_schema)
+  end
+
+  def test_fixed
+    fixed_schema = '{"type": "fixed", "name": "Test", "size": 1}'
+    check(fixed_schema)
+    check_default(fixed_schema, '"a"', "a")
+  end
+
+  def test_enum_with_duplicate
+    str = '{"type": "enum", "name": "Test","symbols" : ["AA", "AA"]}'
+    assert_raises(Avro::SchemaParseError) do
+      schema = Avro::Schema.parse str
+    end
+  end
+
+  BINARY_INT_ENCODINGS = [
+    [0, '00'],
+    [-1, '01'],
+    [1, '02'],
+    [-2, '03'],
+    [2, '04'],
+    [-64, '7f'],
+    [64, '80 01'],
+    [8192, '80 80 01'],
+    [-8193, '81 80 01'],
+  ]
+
+  def avro_hexlify(reader)
+    bytes = []
+    current_byte = reader.read(1)
+    bytes << hexlify(current_byte)
+    while (current_byte.unpack('C').first & 0x80) != 0
+      current_byte = reader.read(1)
+      bytes << hexlify(current_byte)
+    end
+    bytes.join ' '
+  end
+
+  def hexlify(msg)
+    msg.unpack("H*")
+  end
+
+  def test_binary_int_encoding
+    for value, hex_encoding in BINARY_INT_ENCODINGS
+      # write datum in binary to string buffer
+      buffer = StringIO.new
+      encoder = Avro::IO::BinaryEncoder.new(buffer)
+      datum_writer = Avro::IO::DatumWriter.new(Avro::Schema.parse('"int"'))
+      datum_writer.write(value, encoder)
+
+      buffer.seek(0)
+      hex_val = avro_hexlify(buffer)
+
+      assert_equal hex_encoding, hex_val
+    end
+  end
+
+  def test_binary_long_encoding
+    for value, hex_encoding in BINARY_INT_ENCODINGS
+      buffer = StringIO.new
+      encoder = Avro::IO::BinaryEncoder.new(buffer)
+      datum_writer = Avro::IO::DatumWriter.new(Avro::Schema.parse('"long"'))
+      datum_writer.write(value, encoder)
+
+      # read it out of the buffer and hexlify it
+      buffer.seek(0)
+      hex_val = avro_hexlify(buffer)
+
+      assert_equal hex_encoding, hex_val
+    end
+  end
+
+  def test_utf8_string_encoding
+    [
+      "\xC3".force_encoding('ISO-8859-1'),
+      "\xC3\x83".force_encoding('UTF-8')
+    ].each do |value|
+      output = ''.force_encoding('BINARY')
+      encoder = Avro::IO::BinaryEncoder.new(StringIO.new(output))
+      datum_writer = Avro::IO::DatumWriter.new(Avro::Schema.parse('"string"'))
+      datum_writer.write(value, encoder)
+
+      assert_equal "\x04\xc3\x83".force_encoding('BINARY'), output
+    end
+  end
+
+  def test_bytes_encoding
+    [
+      "\xC3\x83".force_encoding('BINARY'),
+      "\xC3\x83".force_encoding('ISO-8859-1'),
+      "\xC3\x83".force_encoding('UTF-8')
+    ].each do |value|
+      output = ''.force_encoding('BINARY')
+      encoder = Avro::IO::BinaryEncoder.new(StringIO.new(output))
+      datum_writer = Avro::IO::DatumWriter.new(Avro::Schema.parse('"bytes"'))
+      datum_writer.write(value, encoder)
+
+      assert_equal "\x04\xc3\x83".force_encoding('BINARY'), output
+    end
+  end
+
+  def test_fixed_encoding
+    [
+      "\xC3\x83".force_encoding('BINARY'),
+      "\xC3\x83".force_encoding('ISO-8859-1'),
+      "\xC3\x83".force_encoding('UTF-8')
+    ].each do |value|
+      output = ''.force_encoding('BINARY')
+      encoder = Avro::IO::BinaryEncoder.new(StringIO.new(output))
+      schema = '{"type": "fixed", "name": "TwoBytes", "size": 2}'
+      datum_writer = Avro::IO::DatumWriter.new(Avro::Schema.parse(schema))
+      datum_writer.write(value, encoder)
+
+      assert_equal "\xc3\x83".force_encoding('BINARY'), output
+    end
+  end
+
+  def test_skip_long
+    for value_to_skip, hex_encoding in BINARY_INT_ENCODINGS
+      value_to_read = 6253
+
+      # write some data in binary to string buffer
+      writer = StringIO.new
+      encoder = Avro::IO::BinaryEncoder.new(writer)
+      datum_writer = Avro::IO::DatumWriter.new(Avro::Schema.parse('"long"'))
+      datum_writer.write(value_to_skip, encoder)
+      datum_writer.write(value_to_read, encoder)
+
+      # skip the value
+      reader = StringIO.new(writer.string())
+      decoder = Avro::IO::BinaryDecoder.new(reader)
+      decoder.skip_long()
+
+      # read data from string buffer
+      datum_reader = Avro::IO::DatumReader.new(Avro::Schema.parse('"long"'))
+      read_value = datum_reader.read(decoder)
+
+      # check it
+      assert_equal value_to_read, read_value
+    end
+  end
+
+  def test_skip_int
+    for value_to_skip, hex_encoding in BINARY_INT_ENCODINGS
+      value_to_read = 6253
+
+      writer = StringIO.new
+      encoder = Avro::IO::BinaryEncoder.new(writer)
+      datum_writer = Avro::IO::DatumWriter.new(Avro::Schema.parse('"int"'))
+      datum_writer.write(value_to_skip, encoder)
+      datum_writer.write(value_to_read, encoder)
+
+      reader = StringIO.new(writer.string)
+      decoder = Avro::IO::BinaryDecoder.new(reader)
+      decoder.skip_int
+
+      datum_reader = Avro::IO::DatumReader.new(Avro::Schema.parse('"int"'))
+      read_value = datum_reader.read(decoder)
+
+      assert_equal value_to_read, read_value
+    end
+  end
+
+  def test_skip_union
+    ["hello", -1, 32, nil].each do |value_to_skip|
+      value_to_read = 6253
+
+      schema = Avro::Schema.parse('["int", "string", "null"]')
+      writer = StringIO.new
+      encoder = Avro::IO::BinaryEncoder.new(writer)
+      datum_writer = Avro::IO::DatumWriter.new(schema)
+      datum_writer.write(value_to_skip, encoder)
+      datum_writer.write(value_to_read, encoder)
+
+      reader = StringIO.new(writer.string)
+      decoder = Avro::IO::BinaryDecoder.new(reader)
+      datum_reader = Avro::IO::DatumReader.new(schema)
+      datum_reader.skip_data(schema, decoder)
+      read_value = datum_reader.read(decoder)
+
+      assert_equal value_to_read, read_value
+    end
+  end
+
+
+  def test_schema_promotion
+    promotable_schemas = ['"int"', '"long"', '"float"', '"double"']
+    incorrect = 0
+    promotable_schemas.each_with_index do |ws, i|
+      writers_schema = Avro::Schema.parse(ws)
+      datum_to_write = 219
+      for rs in promotable_schemas[(i + 1)..-1]
+        readers_schema = Avro::Schema.parse(rs)
+        writer, enc, dw = write_datum(datum_to_write, writers_schema)
+        datum_read = read_datum(writer, writers_schema, readers_schema)
+        if datum_read != datum_to_write
+          incorrect += 1
+        end
+      end
+      assert_equal(incorrect, 0)
+    end
+  end
+  private
+
+  def check_default(schema_json, default_json, default_value)
+    actual_schema = '{"type": "record", "name": "Foo", "fields": []}'
+    actual = Avro::Schema.parse(actual_schema)
+
+    expected_schema = <<EOS
+      {"type": "record",
+       "name": "Foo",
+       "fields": [{"name": "f", "type": #{schema_json}, "default": #{default_json}}]}
+EOS
+    expected = Avro::Schema.parse(expected_schema)
+
+    reader = Avro::IO::DatumReader.new(actual, expected)
+    record = reader.read(Avro::IO::BinaryDecoder.new(StringIO.new))
+    assert_equal default_value, record["f"]
+  end
+
+  def check(str)
+    # parse schema, then convert back to string
+    schema = Avro::Schema.parse str
+
+    parsed_string = schema.to_s
+
+     # test that the round-trip didn't mess up anything
+    # NB: I don't think we should do this. Why enforce ordering?
+    assert_equal(MultiJson.load(str),
+                  MultiJson.load(parsed_string))
+
+    # test __eq__
+    assert_equal(schema, Avro::Schema.parse(str))
+
+    # test hashcode doesn't generate infinite recursion
+    schema.hash
+
+    # test serialization of random data
+    randomdata = RandomData.new(schema)
+    9.times { checkser(schema, randomdata) }
+
+    # test writing of data to file
+    check_datafile(schema)
+  end
+
+  def checkser(schm, randomdata)
+    datum = randomdata.next
+    assert validate(schm, datum)
+    w = Avro::IO::DatumWriter.new(schm)
+    writer = StringIO.new "", "w"
+    w.write(datum, Avro::IO::BinaryEncoder.new(writer))
+    r = datum_reader(schm)
+    reader = StringIO.new(writer.string)
+    ob = r.read(Avro::IO::BinaryDecoder.new(reader))
+    assert_equal(datum, ob) # FIXME check on assertdata conditional
+  end
+
+  def check_datafile(schm)
+    seed = 0
+    count = 10
+    random_data = RandomData.new(schm, seed)
+
+
+    f = File.open(DATAFILE, 'wb')
+    dw = Avro::DataFile::Writer.new(f, datum_writer(schm), schm)
+    count.times{ dw << random_data.next }
+    dw.close
+
+    random_data = RandomData.new(schm, seed)
+
+
+    f = File.open(DATAFILE, 'r+')
+    dr = Avro::DataFile::Reader.new(f, datum_reader(schm))
+
+    last_index = nil
+    dr.each_with_index do |data, c|
+      last_index = c
+      # FIXME assertdata conditional
+      assert_equal(random_data.next, data)
+    end
+    dr.close
+    assert_equal count, last_index+1
+  end
+
+  def validate(schm, datum)
+    Avro::Schema.validate(schm, datum)
+  end
+
+  def datum_writer(schm)
+    Avro::IO::DatumWriter.new(schm)
+  end
+
+  def datum_reader(schm)
+    Avro::IO::DatumReader.new(schm)
+  end
+
+  def write_datum(datum, writers_schema)
+    writer = StringIO.new
+    encoder = Avro::IO::BinaryEncoder.new(writer)
+    datum_writer = Avro::IO::DatumWriter.new(writers_schema)
+    datum_writer.write(datum, encoder)
+    [writer, encoder, datum_writer]
+  end
+
+  def read_datum(buffer, writers_schema, readers_schema=nil)
+    reader = StringIO.new(buffer.string)
+    decoder = Avro::IO::BinaryDecoder.new(reader)
+    datum_reader = Avro::IO::DatumReader.new(writers_schema, readers_schema)
+    datum_reader.read(decoder)
+  end
+end