dataMetaAvro 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 499130a7de4920319df3b2432a8ec85a6cf46a42
+  data.tar.gz: c01912259141df8d2aba49eb97e8da67b73e8b58
+SHA512:
+  metadata.gz: f103e5ef77050cc11c5cd7ae6037197cfa412589a80b4bd1ef84c688c61367624906cdc89e79750e5c71ab74362bebf829fd68d335466f575c6e72f0ded69794
+  data.tar.gz: adc11fd55bbba2761ac8b6f860f89da4726d6273e468252fdd2572373c583606cea1710ff8532bee08c0de630a39326c2435feb77f4ccfb9e9e2f4215d1ac93d

data/.yardopts

@@ -0,0 +1 @@
+--title "DataMeta support for Avro" -r README.md --charset UTF-8 lib/**/* - README.md

data/History.md

@@ -0,0 +1,5 @@
+# `legaDomAvro` Release history:
+
+## `1.0.0` released `2017-02-14 Tue`
+* 1 major enhancement:
+  * Initial release

data/PostInstall.txt

@@ -0,0 +1,2 @@
+No special steps
+

data/README.md

@@ -0,0 +1,46 @@
+# DataMetaAvro
+
+DataMeta [Avro](http://avro.apache.org/docs/1.7.4) utilities, such as DataMetaDOM source to
+[Avro Schema](http://avro.apache.org/docs/1.7.4/spec.html) converter.
+
+References to this gem's:
+
+* [Source](https://github.com/eBayDataMeta/DataMeta-gems)
+
+
+## DESCRIPTION:
+
+See the [DataMeta Project](https://github.com/eBayDataMeta/DataMeta).
+
+## FEATURES/PROBLEMS:
+
+Avro support:
+
+* ver `1.8.1` or newer
+
+Since Avro supports limited subset of the DataMetaDOM features, DataMeta's features that are not supported by Avro
+cause an error during export.
+
+## SYNOPSIS:
+
+### Avro schema generator
+
+* Runnables:
+  * <tt>dataMetaAvroSchemaGen.rb</tt> - generate [Avro](http://avro.apache.org/docs/current) [Schemas](http://avro.apache.org/docs/current/spec.html),
+    one file per class
+
+Usage:
+
+    dataMetaAvroSchemaGen.rb <DataMetaDOM source> <Avro Schemas target dir>
+
+## REQUIREMENTS:
+
+* No special requirements
+
+## INSTALL:
+
+    gem install dataMetaAvro
+
+## LICENSE:
+
+[Apache v 2.0](https://github.com/eBayDataMeta/DataMeta/blob/master/LICENSE.md)

data/Rakefile

@@ -0,0 +1,12 @@
+%w(yard rake/testtask).each{ |r| require r}
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+end
+
+desc 'Regen RDocs'
+task :default => :docs
+
+YARD::Rake::YardocTask.new('docs') {|r|
+    r.stats_options = ['--list-undoc']
+}

data/bin/dataMetaAvroSchemaGen.rb

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+
+%w( dataMetaDom dataMetaAvro ).each(&method(:require))
+
+# sample arguments from the gem's root
+# dataMetaAvroSchemaGen.rb ../../../test/dmDom/showCase.dmDom ../../../../src/test/avsc
+
+@source, @target = ARGV
+DataMetaAvro::helpAvroSchemaGen __FILE__ unless @source && @target
+DataMetaAvro::helpAvroSchemaGen(__FILE__, "DataMetaDom source #{@source} is not a file") unless File.file?(@source)
+DataMetaAvro::helpAvroSchemaGen(__FILE__, "Schema destination directory #{@target} is not a dir") unless File.directory?(@target)
+
+puts "Generating #{@source} into #{@target}"
+
+@parser = DataMetaDom::Model.new
+begin
+    @parser.parse(@source)
+    DataMetaAvro::genSchema(@parser, @target)
+rescue Exception => e
+   puts "ERROR #{e.message}; #{@parser.diagn}"
+   puts e.backtrace.inspect
+end

data/lib/dataMetaAvro.rb

@@ -0,0 +1,147 @@
+$:.unshift(File.dirname(__FILE__)) unless $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+require 'erb'
+require 'fileutils'
+require 'dataMetaDom'
+require 'dataMetaDom/help'
+require 'dataMetaDom/pojo'
+require 'dataMetaDom/record'
+require 'dataMetaDom/util'
+require 'ostruct'
+
+=begin rdoc
+DataMetaDOM and {Avro}[http://avro.apache.org/docs/current] {Schemas}[http://avro.apache.org/docs/current/spec.html].
+
+For command line details either check the new method's source or the README, the usage section.
+=end
+
+module DataMetaAvro
+    # Current version
+    VERSION = '1.0.0'
+
+    # The root of the gem.
+    GEM_ROOT = File.realpath(File.dirname(__FILE__) + '/../')
+
+    # Location of templates.
+    TMPL_ROOT = File.join(GEM_ROOT, 'tmpl')
+
+=begin rdoc
+Mapping from a DataMeta DOM type to a matching renderer of Avro schema JSON.
+The lambda expect whole DataMetaDom::Field instance, must return the whole
+specification that you would put under the <tt>"type":</tt> JSON tag, such as:
+    "int"
+or, for a type with a size:
+    { "type": "fixed", "name": "theFieldName", "size": 16}
+
+Note that wrapping this type into optional specification, i.e. unioned with <tt>"null"</tt> is done by calling
+the avroType method.
+=end
+    AVRO_TYPES = {
+            DataMetaDom::BOOL => lambda{|dt| %q<"boolean">},
+            DataMetaDom::INT => lambda{ |dt|
+              len = dt.length
+              case
+                when len <= 4; %q<"int">
+                when len <= 8; %q<"long">
+                else; raise "Invalid integer length #{len}"
+              end
+            },
+            DataMetaDom::FLOAT => lambda{|dt|
+              len = dt.length
+              case
+                when len <= 4; %q<"float">
+                when len <= 8; %q<"double">
+                else; raise "Invalid float length #{len}"
+              end
+            },
+            DataMetaDom::RAW => lambda{|dt| %q<"bytes">},
+            DataMetaDom::STRING => lambda{|dt| %q<"string">},
+=begin
+Unlike DataMeta DOM, Avro does not support temporal types such as date, time and datetime,
+they have a ticket filed for it but no idea when it is going to be implemented.
+They use {integral types}[http://avro.apache.org/docs/current/spec.html#Time+%28millisecond+precision%29] for
+everything temporal.
+=end
+        DataMetaDom::DATETIME => lambda{|dt| %q<"long">},
+# No support for these in this release:
+          #NUMERIC => lambda{|t| "BigDecimal"}
+    }
+
+=begin rdoc
+Converts DataMeta DOM type to Avro schema type.
+=end
+    def avroType(dataMetaType)
+        renderer = AVRO_TYPES[dataMetaType.type]
+        raise "Unsupported type #{dataMetaType}" unless renderer
+        renderer.call(dataMetaType)
+    end
+
+# Wraps required/optional in proper enclosure
+    def wrapReqOptional(field, baseType)
+        field.isRequired ? baseType : %Q^[#{baseType}, "null"]^
+    end
+
+=begin rdoc
+Generates an {Avro Schema}[http://avro.apache.org/docs/current/spec.html] for the given model's record.
+
+It makes impression that some parameters are not used, but it is not so: they are used by the ERB template
+as the part of the method's binding.
+
+The parameters nameSpace and the base can be derived from rec, but since they are evaluated previously by calling
+assertNamespace, can just as well reuse them.
+
+* Params:
+  * model - DataMetaDom::Model
+  * outFile - output file name
+  * rec - DataMetaDom::Record
+  * nameSpace - the namespace for the record
+  * base - base name of the record
+=end
+    def genRecordJson(model, outFile, rec, nameSpace, base)
+        vars =  OpenStruct.new # for template's local variables. ERB does not make them visible to the binding
+        IO.write(outFile, "#{ERB.new(IO.read("#{TMPL_ROOT}/dataClass.avsc.erb"), 0, '-').result(binding)}", {:mode => 'wb'})
+    end
+
+=begin rdoc
+Splits the full name of a class into the namespace and the base, returns an array of
+the namespace (empty string if there is no namespace on the name) and the base name.
+
+Examples:
+* <tt>'BaseNameAlone'</tt> -> <tt>['', 'BaseNameAlone']</tt>
+* <tt>'one.package.another.pack.FinallyTheName'</tt> -> <tt>['one.package.another.pack', 'FinallyTheName']</tt>
+=end
+    def assertNamespace(fullName)
+      ns, base = DataMetaDom::splitNameSpace(fullName)
+      [DataMetaDom.validNs?(ns, base) ? ns : '', base]
+    end
+
+=begin rdoc
+Generates the {Avro Schema}[http://avro.apache.org/docs/current/spec.html], one +avsc+ file per a record.
+=end
+    def genSchema(model, outRoot)
+      model.records.values.each { |rec| # loop through all the records in the model
+        nameSpace, base = assertNamespace(rec.name)
+        FileUtils.mkdir_p outRoot # write json files named as one.package.another.package.ClassName.json in one dir
+        outFile = File.join(outRoot, "#{rec.name}.avsc")
+          case
+            when rec.kind_of?(DataMetaDom::Record)
+                genRecordJson model, outFile, rec, nameSpace, base
+            else # since we are cycling through records, should never get here
+              raise "Unsupported Entity: #{rec.inspect}"
+          end
+      }
+    end
+
+    # Shortcut to help for the Hadoop Writables generator.
+    def helpAvroSchemaGen(file, errorText=nil)
+        DataMetaDom::help(file, "DataMeta DOM Avro Schema Generation ver #{VERSION}",
+                          '<DataMeta DOM source> <Avro Schemas target dir>', errorText)
+    end
+
+    def assertMapKeyType(fld, type)
+        raise ArgumentError, %<Field "#{fld.name}": Avro supports only strings as map keys, "#{
+            type}" is not supported as a map key by Avro> unless type == DataMetaDom::STRING
+    end
+module_function :helpAvroSchemaGen, :genSchema, :assertNamespace, :genRecordJson, :avroType, :assertMapKeyType,
+            :wrapReqOptional
+end

data/test/sample.dmDom

@@ -0,0 +1,61 @@
+# DataMetaDOM with all field types accounted for testing.
+namespace org.ebay.datameta.examples.conv.avro
+
+ver 1.0.0
+
+enum BaseColor
+   Red, Green, Blue
+end
+
+# Avro supports only stings as map keys
+mapping Timings string[3] datetime
+   "now" => DateTime.now,
+   'then' => DateTime.parse("2012-09-29T23:45:59Z")
+end
+
+mapping Depths string[16] float[3]
+   "shallow" => 0.1,
+   "medium" => 353.232,
+   "deep" => 787.0
+end
+
+record AllTypes
+    +int[4] id
+    +int[8] count
+    +bool isIt
+# this is a very special case in Avro, "fixed", see http://avro.apache.org/docs/current/spec.html#Fixed
+    +char[5] code
+    -float[4] width
+    +float[8] height
+    +string anyLength
+    +string[16] name
+    +set{string} aliases
+    -deque{datetime} accesses
+    +list{int[4]} quants
+# Avro supports only stings as map keys
+    +map{string[32], int[4]} strToInt
+    +Depths depths
+    +Timings lengths
+    +BaseColor color
+    identity id
+end
+
+# Check how optional fields are rendered
+record Optionals
+    +int[4] id
+    -int[8] count
+    -bool isIt
+    -char[5] code
+    -float[4] width
+    +float[8] height
+    -string anyLength
+    +string[16] name
+    +set{string} aliases
+    -deque{datetime} accesses
+    -list{int[4]} quants
+    -map{string[32], int[4]} strToInt
+    -Depths depths
+    -Timings lengths
+    -BaseColor color
+    identity id
+end

data/test/test_dataMetaAvro.rb

@@ -0,0 +1,34 @@
+# keep this underscore naming in the test subdir, it's easier to append files names to test
+require './test/test_helper.rb'
+require 'avro'
+
+# Unit test cases for the DataMetaAvro
+#  See for instance:
+#  - test_full
+class TestNewGem < Test::Unit::TestCase
+
+    L = Logger.new('dataMetaAvroTests.log', 0, 10_000_000)
+    L.level = Logger::DEBUG
+    L.datetime_format = '%Y-%m-%d %H:%M:%S'
+
+    GEN_TARGET = '.tmp'
+
+    # an empty stub for now
+    def setup; end
+
+    # Smell-check the parsing
+    def test_parsing
+        model = DataMetaDom::Model.new
+        model.parse(File.join(File.dirname(__FILE__), 'sample.dmDom'), options={autoNsVer: true})
+        L.info(%<Model: #{model}>)
+        FileUtils.rmtree(GEN_TARGET) if File.exist?(GEN_TARGET)
+        FileUtils.mkpath GEN_TARGET
+        DataMetaAvro.genSchema(model, GEN_TARGET)
+        Dir.entries(GEN_TARGET).select{|e| e.end_with?('.avsc')}.each{ |e|
+            L.info("Verifying schema #{e}")
+            schema = IO.read(File.join(GEN_TARGET, e))
+            projection = Avro::Schema.parse(schema) # if schema is invalid, this will cause an error
+            L.info(projection.inspect)
+        }
+    end
+end

data/test/test_helper.rb

@@ -0,0 +1,8 @@
+## keep this underscore naming in the test subdir, it's easier to append files names to test
+
+require 'test/unit'
+require 'dataMetaDom'
+require 'fileutils'
+
+# this is expected to run from the project root, normally by the rake file
+require './lib/dataMetaAvro'

data/tmpl/dataClass.avsc.erb

@@ -0,0 +1,43 @@
+<%#
+Template for Avro data only protocol for one class.
+
+-%>
+{
+    "type": "record", "namespace":"<%= nameSpace %>", "name" : "<%= base %>",
+    "fields": [
+<%
+
+ vars.flds = rec.fields
+ delim = nil; rec.fields.each_key { |fldId| vars.fld=rec.fields[fldId]
+        vars.dataType = vars.fld.dataType
+        vars.dt = vars.dataType.type
+        vars.enum = model.enums[vars.dt]
+%><%=delim ? delim : ' '%><%
+        if vars.enum && vars.enum.is_a?(DataMetaDom::Enum)
+          vars.ns, vars.base = DataMetaDom.splitNameSpace(vars.dt)
+          values = vars.enum.keys.map{|k| vars.enum[k]} # sort by ordinals to preserve the order
+%>
+      {"name": "<%=vars.fld.name%>", "type": <%=wrapReqOptional(vars.fld, %^{ "name": "#{vars.base}", "type": "enum", "symbols" : [#{ values.map{|v| %<"#{v}">}.join(', ')}]}^)%>}
+<%
+        elsif vars.enum && vars.enum.is_a?(DataMetaDom::Mappings)
+          assertMapKeyType(vars.fld, vars.enum.fromT.type)
+          vars.ns, vars.base = DataMetaDom.splitNameSpace(vars.dt)
+%>
+      {"name": "<%=vars.fld.name%>", "type": <%=wrapReqOptional(vars.fld, %^{"type": "map", "values": #{avroType(vars.enum.toT)}}^)%>}<%
+        elsif vars.dt == DataMetaDom::CHAR
+%>
+      {"name": "<%=vars.fld.name%>", "type":<%=wrapReqOptional(vars.fld, %^{"name": "#{vars.fld.name}", "type": "fixed",  "size": #{vars.dataType.length}}^)%>}<%
+        elsif vars.fld.map?
+          assertMapKeyType(vars.fld, vars.dt)
+        %>
+      {"name": "<%=vars.fld.name%>", "type": <%=wrapReqOptional(vars.fld, %^{"type": "map", "values": #{avroType(vars.fld.trgType)}}^)%>} <%
+        elsif vars.fld.aggr%>
+      {"name": "<%=vars.fld.name%>", "type": <%=wrapReqOptional(vars.fld, %^{"type": "array", "items": #{avroType(vars.dataType)}}^)%>} <%
+        else %>
+      {"name" : "<%= vars.fld.name %>", "type": <%= wrapReqOptional(vars.fld, avroType(vars.dataType)) %>} <%
+        end
+   delim =","
+        }
+%>
+        ]
+}

metadata

@@ -0,0 +1,77 @@
+--- !ruby/object:Gem::Specification
+name: dataMetaAvro
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Michael Bergens
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-02-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: dataMetaDom
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+description: DataMeta DOM to Avro schema generator
+email: michael.bergens@gmail.com
+executables:
+- dataMetaAvroSchemaGen.rb
+extensions: []
+extra_rdoc_files: []
+files:
+- ".yardopts"
+- History.md
+- PostInstall.txt
+- README.md
+- Rakefile
+- bin/dataMetaAvroSchemaGen.rb
+- lib/dataMetaAvro.rb
+- test/sample.dmDom
+- test/test_dataMetaAvro.rb
+- test/test_helper.rb
+- tmpl/dataClass.avsc.erb
+homepage: https://github.com/eBayDataMeta
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.1.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements:
+- No special requirements
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: DataMeta Avro
+test_files:
+- test/test_dataMetaAvro.rb