avro-salsify-fork 1.9.0.0

data/lib/avro/schema_normalization.rb

@@ -0,0 +1,83 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Avro
+  class SchemaNormalization
+    def self.to_parsing_form(schema)
+      new.to_parsing_form(schema)
+    end
+
+    def initialize
+      @processed_names = []
+    end
+
+    def to_parsing_form(schema)
+      MultiJson.dump(normalize_schema(schema))
+    end
+
+    private
+
+    def normalize_schema(schema)
+      type = schema.type_sym.to_s
+
+      if Schema::NAMED_TYPES.include?(type)
+        if @processed_names.include?(schema.name)
+          return schema.name
+        else
+          @processed_names << schema.name
+        end
+      end
+
+      case type
+      when *Schema::PRIMITIVE_TYPES
+        type
+      when "record"
+        fields = schema.fields.map {|field| normalize_field(field) }
+
+        normalize_named_type(schema, fields: fields)
+      when "enum"
+        normalize_named_type(schema, symbols: schema.symbols)
+      when "fixed"
+        normalize_named_type(schema, size: schema.size)
+      when "array"
+        { type: type, items: normalize_schema(schema.items) }
+      when "map"
+        { type: type, values: normalize_schema(schema.values) }
+      when "union"
+        if schema.schemas.nil?
+          []
+        else
+          schema.schemas.map {|s| normalize_schema(s) }
+        end
+      else
+        raise "unknown type #{type}"
+      end
+    end
+
+    def normalize_field(field)
+      {
+        name: field.name,
+        type: normalize_schema(field.type)
+      }
+    end
+
+    def normalize_named_type(schema, attributes = {})
+      name = Name.make_fullname(schema.name, schema.namespace)
+
+      { name: name, type: schema.type_sym.to_s }.merge(attributes)
+    end
+  end
+end

data/test/case_finder.rb

@@ -0,0 +1,87 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+require 'strscan'
+
+class CaseFinder
+  PATH = File.expand_path("../../../../share/test/data/schema-tests.txt", __FILE__)
+
+  Case = Struct.new(:id, :input, :canonical, :fingerprint)
+
+  def self.cases
+    new.cases
+  end
+
+  def initialize
+    @scanner = StringScanner.new(File.read(PATH))
+    @cases = []
+  end
+
+  def cases
+    until @scanner.eos?
+      test_case = scan_case
+      @cases << test_case if test_case
+    end
+
+    @cases
+  end
+
+  private
+
+  def scan_case
+    if id = @scanner.scan(/\/\/ \d+\n/)
+      while @scanner.skip(/\/\/ .*\n/); end
+
+      input = scan_input
+      canonical = scan_canonical
+      fingerprint = scan_fingerprint
+
+      Case.new(id, input, canonical, fingerprint)
+    else
+      @scanner.skip(/.*\n/)
+      nil
+    end
+  end
+
+  def scan_item(name)
+    if @scanner.scan(/<<#{name}\n/)
+      lines = []
+      while line = @scanner.scan(/.+\n/)
+        break if line.chomp == name
+        lines << line
+      end
+      lines.join
+    elsif @scanner.scan(/<<#{name} /)
+      input = @scanner.scan(/.+$/)
+      @scanner.skip(/\n/)
+      input
+    end
+  end
+
+  def scan_input
+    scan_item("INPUT")
+  end
+
+  def scan_canonical
+    scan_item("canonical")
+  end
+
+  def scan_fingerprint
+    scan_item("fingerprint")
+  end
+end

data/test/random_data.rb

@@ -0,0 +1,90 @@
+#!/usr/bin/env ruby
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class RandomData
+  def initialize(schm, seed=nil)
+    srand(seed) if seed
+    @seed = seed
+    @schm = schm
+  end
+
+  def next
+    nextdata(@schm)
+  end
+
+  def nextdata(schm, d=0)
+    case schm.type_sym
+    when :boolean
+      rand > 0.5
+    when :string
+      randstr()
+    when :int
+      rand(Avro::Schema::INT_MAX_VALUE - Avro::Schema::INT_MIN_VALUE) + Avro::Schema::INT_MIN_VALUE
+    when :long
+      rand(Avro::Schema::LONG_MAX_VALUE - Avro::Schema::LONG_MIN_VALUE) + Avro::Schema::LONG_MIN_VALUE
+    when :float
+      (-1024 + 2048 * rand).round.to_f
+    when :double
+      Avro::Schema::LONG_MIN_VALUE + (Avro::Schema::LONG_MAX_VALUE - Avro::Schema::LONG_MIN_VALUE) * rand
+    when :bytes
+      randstr(BYTEPOOL)
+    when :null
+      nil
+    when :array
+      arr = []
+      len = rand(5) + 2 - d
+      len = 0 if len < 0
+      len.times{ arr << nextdata(schm.items, d+1) }
+      arr
+    when :map
+      map = {}
+      len = rand(5) + 2 - d
+      len = 0 if len < 0
+      len.times do
+        map[nextdata(Avro::Schema::PrimitiveSchema.new(:string))] = nextdata(schm.values, d+1)
+      end
+      map
+    when :record, :error
+      m = {}
+      schm.fields.each do |field|
+        m[field.name] = nextdata(field.type, d+1)
+      end
+      m
+    when :union
+      types = schm.schemas
+      nextdata(types[rand(types.size)], d)
+    when :enum
+      symbols = schm.symbols
+      len = symbols.size
+      return nil if len == 0
+      symbols[rand(len)]
+    when :fixed
+      f = ""
+      schm.size.times { f << BYTEPOOL[rand(BYTEPOOL.size), 1] }
+      f
+    end
+  end
+
+  CHARPOOL = 'abcdefghjkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789'
+  BYTEPOOL = '12345abcd'
+
+  def randstr(chars=CHARPOOL, length=20)
+    str = ''
+    rand(length+1).times { str << chars[rand(chars.size)] }
+    str
+  end
+end

data/test/sample_ipc_client.rb

@@ -0,0 +1,85 @@
+#!/usr/bin/env ruby
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'socket'
+require 'avro'
+
+MAIL_PROTOCOL_JSON = <<-JSON
+{"namespace": "example.proto",
+ "protocol": "Mail",
+
+ "types": [
+     {"name": "Message", "type": "record",
+      "fields": [
+          {"name": "to",   "type": "string"},
+          {"name": "from", "type": "string"},
+          {"name": "body", "type": "string"}
+      ]
+     }
+ ],
+
+ "messages": {
+     "send": {
+         "request": [{"name": "message", "type": "Message"}],
+         "response": "string"
+     },
+     "replay": {
+         "request": [],
+         "response": "string"
+     }
+ }
+}
+JSON
+
+MAIL_PROTOCOL = Avro::Protocol.parse(MAIL_PROTOCOL_JSON)
+
+def make_requestor(server_address, port, protocol)
+  sock = TCPSocket.new(server_address, port)
+  client = Avro::IPC::SocketTransport.new(sock)
+  Avro::IPC::Requestor.new(protocol, client)
+end
+
+if $0 == __FILE__
+  if ![3, 4].include?(ARGV.length)
+    raise "Usage: <to> <from> <body> [<count>]"
+  end
+
+  # client code - attach to the server and send a message
+  # fill in the Message record
+  message = {
+    'to'   => ARGV[0],
+    'from' => ARGV[1],
+    'body' => ARGV[2]
+  }
+
+  num_messages = (ARGV[3] || 1).to_i
+
+  # build the parameters for the request
+  params = {'message' => message}
+
+  # send the requests and print the result
+  num_messages.times do
+    requestor = make_requestor('localhost', 9090, MAIL_PROTOCOL)
+    result = requestor.request('send', params)
+    puts("Result: " + result)
+  end
+
+  # try out a replay message
+  requestor = make_requestor('localhost', 9090, MAIL_PROTOCOL)
+  result = requestor.request('replay', {})
+  puts("Replay Result: " + result)
+end

data/test/sample_ipc_http_client.rb

@@ -0,0 +1,84 @@
+#!/usr/bin/env ruby
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'socket'
+require 'avro'
+
+MAIL_PROTOCOL_JSON = <<-JSON
+{"namespace": "example.proto",
+ "protocol": "Mail",
+
+ "types": [
+     {"name": "Message", "type": "record",
+      "fields": [
+          {"name": "to",   "type": "string"},
+          {"name": "from", "type": "string"},
+          {"name": "body", "type": "string"}
+      ]
+     }
+ ],
+
+ "messages": {
+     "send": {
+         "request": [{"name": "message", "type": "Message"}],
+         "response": "string"
+     },
+     "replay": {
+         "request": [],
+         "response": "string"
+     }
+ }
+}
+JSON
+
+MAIL_PROTOCOL = Avro::Protocol.parse(MAIL_PROTOCOL_JSON)
+
+def make_requestor(server_address, port, protocol)
+  transport = Avro::IPC::HTTPTransceiver.new(server_address, port)
+  Avro::IPC::Requestor.new(protocol, transport)
+end
+
+if $0 == __FILE__
+  if ![3, 4].include?(ARGV.length)
+    raise "Usage: <to> <from> <body> [<count>]"
+  end
+
+  # client code - attach to the server and send a message
+  # fill in the Message record
+  message = {
+    'to'   => ARGV[0],
+    'from' => ARGV[1],
+    'body' => ARGV[2]
+  }
+
+  num_messages = (ARGV[3] || 1).to_i
+
+  # build the parameters for the request
+  params = {'message' => message}
+  # send the requests and print the result
+
+  num_messages.times do
+    requestor = make_requestor('localhost', 9090, MAIL_PROTOCOL)
+    result = requestor.request('send', params)
+    puts("Result: " + result)
+  end
+
+  # try out a replay message
+  requestor = make_requestor('localhost', 9090, MAIL_PROTOCOL)
+  result = requestor.request('replay', {})
+  puts("Replay Result: " + result)
+end

data/test/sample_ipc_http_server.rb

@@ -0,0 +1,79 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+require 'avro'
+require 'webrick'
+
+MAIL_PROTOCOL_JSON = <<-JSON
+{"namespace": "example.proto",
+ "protocol": "Mail",
+
+ "types": [
+     {"name": "Message", "type": "record",
+      "fields": [
+          {"name": "to",   "type": "string"},
+          {"name": "from", "type": "string"},
+          {"name": "body", "type": "string"}
+      ]
+     }
+ ],
+
+ "messages": {
+     "send": {
+         "request": [{"name": "message", "type": "Message"}],
+         "response": "string"
+     },
+     "replay": {
+         "request": [],
+         "response": "string"
+     }
+ }
+}
+JSON
+
+MAIL_PROTOCOL = Avro::Protocol.parse(MAIL_PROTOCOL_JSON)
+
+class MailResponder < Avro::IPC::Responder
+  def initialize
+    super(MAIL_PROTOCOL)
+  end
+
+  def call(message, request)
+    if message.name == 'send'
+      request_content = request['message']
+      "Sent message to #{request_content['to']} from #{request_content['from']} with body #{request_content['body']}"
+    elsif message.name == 'replay'
+      'replay'
+    end
+  end
+end
+
+class MailHandler < WEBrick::HTTPServlet::AbstractServlet
+  def do_POST(req, resp)
+    responder = MailResponder.new
+    call_request = Avro::IPC::FramedReader.new(StringIO.new(req.body)).read_framed_message
+    unframed_resp = responder.respond(call_request)
+    writer = Avro::IPC::FramedWriter.new(StringIO.new)
+    writer.write_framed_message(unframed_resp)
+    resp.body = writer.to_s
+  end
+end
+
+if $0 == __FILE__
+  server = WEBrick::HTTPServer.new(:Host => 'localhost', :Port => 9090)
+  server.mount '/', MailHandler
+  trap("INT") { server.shutdown }
+  server.start
+end