avro-salsify-fork 1.9.0.0

data/test/test_schema_normalization.rb

@@ -0,0 +1,171 @@
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'test_help'
+require 'case_finder'
+
+class TestSchemaNormalization < Test::Unit::TestCase
+  def test_primitives
+    %w[null boolean string bytes int long float double].each do |type|
+      schema = Avro::Schema.parse(<<-JSON)
+        { "type": "#{type}" }
+      JSON
+
+      canonical_form = Avro::SchemaNormalization.to_parsing_form(schema)
+
+      assert_equal %("#{type}"), canonical_form
+    end
+  end
+
+  def test_records
+    schema = Avro::Schema.parse(<<-JSON)
+      {
+        "type": "record",
+        "name": "test",
+        "namespace": "random",
+        "doc": "some record",
+        "fields": [
+          { "name": "height", "type": "int", "doc": "the height" }
+        ]
+      }
+    JSON
+
+    expected_type = <<-JSON.strip
+      {"name":"random.test","type":"record","fields":[{"name":"height","type":"int"}]}
+    JSON
+
+    canonical_form = Avro::SchemaNormalization.to_parsing_form(schema)
+
+    assert_equal expected_type, canonical_form
+  end
+
+  def test_recursive_records
+    schema = Avro::Schema.parse(<<-JSON)
+      {
+        "type": "record",
+        "name": "item",
+        "fields": [
+          { "name": "next", "type": "item" }
+        ]
+      }
+    JSON
+
+    expected_type = <<-JSON.strip
+      {"name":"item","type":"record","fields":[{"name":"next","type":"item"}]}
+    JSON
+
+    canonical_form = Avro::SchemaNormalization.to_parsing_form(schema)
+
+    assert_equal expected_type, canonical_form
+  end
+
+  def test_enums
+    schema = Avro::Schema.parse(<<-JSON)
+      {
+        "type": "enum",
+        "name": "suit",
+        "namespace": "cards",
+        "doc": "the different suits of cards",
+        "symbols": ["club", "hearts", "diamond", "spades"]
+      }
+    JSON
+
+    expected_type = <<-JSON.strip
+      {"name":"cards.suit","type":"enum","symbols":["club","hearts","diamond","spades"]}
+    JSON
+
+    canonical_form = Avro::SchemaNormalization.to_parsing_form(schema)
+
+    assert_equal expected_type, canonical_form
+  end
+
+  def test_fixed
+    schema = Avro::Schema.parse(<<-JSON)
+      {
+        "type": "fixed",
+        "name": "id",
+        "namespace": "db",
+        "size": 64
+      }
+    JSON
+
+    expected_type = <<-JSON.strip
+      {"name":"db.id","type":"fixed","size":64}
+    JSON
+
+    canonical_form = Avro::SchemaNormalization.to_parsing_form(schema)
+
+    assert_equal expected_type, canonical_form
+  end
+
+  def test_arrays
+    schema = Avro::Schema.parse(<<-JSON)
+      {
+        "type": "array",
+        "doc": "the items",
+        "items": "int"
+      }
+    JSON
+
+    expected_type = <<-JSON.strip
+      {"type":"array","items":"int"}
+    JSON
+
+    canonical_form = Avro::SchemaNormalization.to_parsing_form(schema)
+
+    assert_equal expected_type, canonical_form
+  end
+
+  def test_maps
+    schema = Avro::Schema.parse(<<-JSON)
+      {
+        "type": "map",
+        "doc": "the items",
+        "values": "int"
+      }
+    JSON
+
+    expected_type = <<-JSON.strip
+      {"type":"map","values":"int"}
+    JSON
+
+    canonical_form = Avro::SchemaNormalization.to_parsing_form(schema)
+
+    assert_equal expected_type, canonical_form
+  end
+
+  def test_unions
+    schema = Avro::Schema.parse(<<-JSON)
+      ["int", "string"]
+    JSON
+
+    expected_type = <<-JSON.strip
+      ["int","string"]
+    JSON
+
+    canonical_form = Avro::SchemaNormalization.to_parsing_form(schema)
+
+    assert_equal expected_type, canonical_form
+  end
+
+  def test_shared_dataset
+    CaseFinder.cases.each do |test_case|
+      schema = Avro::Schema.parse(test_case.input)
+      assert_equal test_case.canonical, Avro::SchemaNormalization.to_parsing_form(schema)
+    end
+  end
+end

data/test/test_socket_transport.rb

@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'test_help'
+
+class TestSocketTransport < Test::Unit::TestCase
+  def test_buffer_writing
+    io = StringIO.new
+    st = Avro::IPC::SocketTransport.new(io)
+    buffer_length = "\000\000\000\006"  # 6 in big-endian
+    message = 'abcdef'
+    null_ending = "\000\000\000\000" # 0 in big-endian
+    full = buffer_length + message + null_ending
+    st.write_framed_message('abcdef')
+    assert_equal full, io.string
+  end
+
+  def test_buffer_reading
+    buffer_length = "\000\000\000\005" # 5 in big-endian
+    message = "hello"
+    null_ending = "\000\000\000\000" # 0 in big-endian
+    full = buffer_length + message + null_ending
+    io = StringIO.new(full)
+    st = Avro::IPC::SocketTransport.new(io)
+    assert_equal 'hello', st.read_framed_message
+  end
+end

data/test/tool.rb

@@ -0,0 +1,144 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'avro'
+require 'webrick'
+require 'uri'
+require 'logger'
+
+class GenericResponder < Avro::IPC::Responder
+  def initialize(proto, msg, datum)
+    proto_json = open(proto).read
+    super(Avro::Protocol.parse(proto_json))
+    @msg = msg
+    @datum = datum
+  end
+
+  def call(message, request)
+    if message.name == @msg
+      STDERR.puts "Message: #{message.name} Datum: #{@datum.inspect}"
+      @datum
+    end
+  end
+end
+
+class GenericHandler < WEBrick::HTTPServlet::AbstractServlet
+  def do_POST(req, resp)
+    call_request = Avro::IPC::FramedReader.new(StringIO.new(req.body)).read_framed_message
+    unframed_resp = $responder.respond(call_request)
+    writer = Avro::IPC::FramedWriter.new(StringIO.new)
+    writer.write_framed_message(unframed_resp)
+    resp.body = writer.to_s
+    @server.stop
+  end
+end
+
+def run_server(uri, proto, msg, datum)
+  uri = URI.parse(uri)
+  $responder = GenericResponder.new(proto, msg, datum)
+  server = WEBrick::HTTPServer.new(:BindAddress => uri.host,
+                                   :Port => uri.port,
+                                   :Logger => Logger.new(StringIO.new))
+  server.mount '/', GenericHandler
+  puts "Port: #{server.config[:Port]}"
+  STDOUT.flush
+  trap("INT") { server.stop }
+  trap("TERM") { server.stop }
+  server.start
+end
+
+def send_message(uri, proto, msg, datum)
+  uri = URI.parse(uri)
+  trans = Avro::IPC::HTTPTransceiver.new(uri.host, uri.port)
+  proto_json = open(proto).read
+  requestor = Avro::IPC::Requestor.new(Avro::Protocol.parse(proto_json),
+                                       trans)
+  p requestor.request(msg, datum)
+end
+
+def file_or_stdin(f)
+  f == "-" ? STDIN : open(f)
+end
+
+def main
+  if ARGV.size == 0
+    puts "Usage: #{$0} [dump|rpcreceive|rpcsend]"
+    return 1
+  end
+
+  case ARGV[0]
+  when "dump"
+    if ARGV.size != 3
+      puts "Usage: #{$0} dump input_file"
+      return 1
+    end
+    d = Avro::DataFile.new(file_or_stdin(ARGV[1]), Avro::IO::DatumReader.new)
+    d.each{|o| puts o.inspect }
+    d.close
+  when "rpcreceive"
+    usage_str = "Usage: #{$0} rpcreceive uri protocol_file "
+    usage_str += "message_name (-data d | -file f)"
+
+    unless [4, 6].include?(ARGV.size)
+      puts usage_str
+      return 1
+    end
+    uri, proto, msg = ARGV[1,3]
+    datum = nil
+    if ARGV.size > 4
+      case ARGV[4]
+      when "-file"
+        Avro::DataFile.open(ARGV[5]) {|f|
+          f.each{|d| datum = d; break }
+        }
+      when "-data"
+        puts "JSON Decoder not yet implemented."
+        return 1
+      else
+        puts usage_str
+        return 1
+      end
+    end
+    run_server(uri, proto, msg, datum)
+  when "rpcsend"
+    usage_str = "Usage: #{$0} rpcsend uri protocol_file "
+    usage_str += "message_name (-data d | -file f)"
+    unless [4,6].include?(ARGV.size)
+      puts usage_str
+      return 1
+    end
+    uri, proto, msg = ARGV[1,3]
+    datum = nil
+    if ARGV.size > 4
+      case ARGV[4]
+      when "-file"
+        Avro::DataFile.open(ARGV[5]){|f| f.each{|d| datum = d; break } }
+      when "-data"
+        puts "JSON Decoder not yet implemented"
+        return 1
+      else
+        puts usage_str
+        return 1
+      end
+    end
+    send_message(uri, proto, msg, datum)
+  end
+  return 0
+end
+
+if __FILE__ == $0
+  exit(main)
+end

metadata

@@ -0,0 +1,114 @@
+--- !ruby/object:Gem::Specification
+name: avro-salsify-fork
+version: !ruby/object:Gem::Version
+  version: 1.9.0.0
+platform: ruby
+authors:
+- Apache Software Foundation / Salsify Engineering
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-08-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: |-
+  Avro is a data serialization and RPC format.
+  This release contains the changes submitted in https://github.com/apache/avro/pull/116
+  to support logical types in the Ruby gem.
+email: engineering@salsify.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- CHANGELOG
+- LICENSE
+- lib/avro.rb
+- lib/avro/data_file.rb
+- lib/avro/io.rb
+- lib/avro/ipc.rb
+- lib/avro/logical_types.rb
+- lib/avro/protocol.rb
+- lib/avro/schema.rb
+- lib/avro/schema_normalization.rb
+files:
+- CHANGELOG
+- LICENSE
+- Manifest
+- NOTICE
+- Rakefile
+- avro-salsify-fork.gemspec
+- avro.gemspec
+- interop/test_interop.rb
+- lib/avro.rb
+- lib/avro/data_file.rb
+- lib/avro/io.rb
+- lib/avro/ipc.rb
+- lib/avro/logical_types.rb
+- lib/avro/protocol.rb
+- lib/avro/schema.rb
+- lib/avro/schema_normalization.rb
+- test/case_finder.rb
+- test/random_data.rb
+- test/sample_ipc_client.rb
+- test/sample_ipc_http_client.rb
+- test/sample_ipc_http_server.rb
+- test/sample_ipc_server.rb
+- test/test_datafile.rb
+- test/test_fingerprints.rb
+- test/test_help.rb
+- test/test_io.rb
+- test/test_logical_types.rb
+- test/test_protocol.rb
+- test/test_schema.rb
+- test/test_schema_normalization.rb
+- test/test_socket_transport.rb
+- test/tool.rb
+homepage: https://github.com/salsify/avro
+licenses:
+- Apache License 2.0 (Apache-2.0)
+metadata: {}
+post_install_message: 
+rdoc_options:
+- "--line-numbers"
+- "--title"
+- Avro-salsify-fork
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '1.2'
+requirements: []
+rubyforge_project: avro-salsify-fork
+rubygems_version: 2.6.6
+signing_key: 
+specification_version: 4
+summary: Apache Avro for Ruby with logical types patch
+test_files:
+- test/test_datafile.rb
+- test/test_fingerprints.rb
+- test/test_help.rb
+- test/test_io.rb
+- test/test_logical_types.rb
+- test/test_protocol.rb
+- test/test_schema.rb
+- test/test_schema_normalization.rb
+- test/test_socket_transport.rb