dataMetaProtobuf 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 091be0378d3394410e5d9bd5a4747083a63b8083
+  data.tar.gz: c34ba9da421151b2c3fda17d654db24045a311f7
+SHA512:
+  metadata.gz: 732ba6ee73997abb8db2092912b659fddcddd013d7ff842a6fb89afe689cbd47f8466be69dece2e9945b1e1913fb7578751ddd7845479b1965cea032503f454e
+  data.tar.gz: f93506ea7441ad5742803559588ba6f46122bcd052704e0bb10a25dc9f64d28b8824e245a379efb7e0e2bde69650dc916694585c129bce8af5ad831ad002c9ee

data/.yardopts

@@ -0,0 +1 @@
+--title "DataMeta support for Protobuf" -r README.md --charset UTF-8 lib/**/* - README.md

data/History.md

@@ -0,0 +1,5 @@
+# `dataMetaProtobuf` Release history:
+
+## `1.0.0` released `2017-02-19 Sun`
+* 1 major enhancement:
+  * Initial release

data/PostInstall.txt

@@ -0,0 +1,2 @@
+No special steps
+

data/README.md

@@ -0,0 +1,63 @@
+# DataMetaProtobuf
+
+DataMeta [Protobuf](https://github.com/google/protobuf/wiki) utilities, such as DataMetaDOM source to
+[Protobuf IDL](https://developers.google.com/protocol-buffers/docs/proto) converter.
+
+References to this gem's:
+
+* [Source](https://github.com/eBayDataMeta/DataMeta-gems)
+
+
+## DESCRIPTION:
+
+See the [DataMeta Project](https://github.com/eBayDataMeta/DataMeta).
+
+## FEATURES/PROBLEMS:
+
+Protobuf support:
+
+* ver `3.2.0` or newer
+
+Since Protobuf supports limited subset of the DataMetaDOM features, DataMeta's features that are not supported by
+Protobuf cause an error during export.
+
+## SYNOPSIS:
+
+### Protobuf IDL generator
+
+Since DataMeta DOM is superset of Protobuf data types, tradeoffs are made:
+
+
+* Runnables:
+  * `dataMetaProtobufGen.rb` - generate [Protobuf IDL](https://developers.google.com/protocol-buffers/docs/proto)
+
+Usage:
+
+Pipe your DataMeta DOM source to:
+
+    dataMetaProtobufGen.rb
+
+It will output your Protobuf IDL to STDOUT and write the log file named `dataMetaProtobuf.log` in which you may find
+some useful information about what just happened. This log is written into the current directory.
+
+DataMeta type conversions:
+
+* `datetime` - Protobuf 3 does have a type named `Timestamp`, but it's new and flakey, therefore we export `datetime` as
+    the Protobuf IDL `string` type.
+* Any DataMeta aggregate type except `mapping`: all of `list`, `deque`, `set` are exported as Protobuf's `repeated`.    
+* Since Protobuf has dropped support for the "required" vs "optional", we use `repeatable` for the optional fields.
+* Protobuf makes maps non-repeatable, therefore maps must be required in DataMetaDOM.
+* DataMeta DOM's `mapping` translates to Protobuf's `map`. Since Protobuf does not support mapping values, 
+    therefore DataMeta DOM's `mapping` values are dropped.
+
+## REQUIREMENTS:
+
+* No special requirements
+
+## INSTALL:
+
+    gem install dataMetaProtobuf
+
+## LICENSE:
+
+[Apache v 2.0](https://github.com/eBayDataMeta/DataMeta/blob/master/LICENSE.md)

data/Rakefile

@@ -0,0 +1,12 @@
+%w(yard rake/testtask).each{ |r| require r}
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+end
+
+desc 'Regen RDocs'
+task :default => :docs
+
+YARD::Rake::YardocTask.new('docs') {|r|
+    r.stats_options = ['--list-undoc']
+}

data/bin/dataMetaProtobufGen.rb

@@ -0,0 +1,21 @@
+#!/usr/bin/env ruby
+
+%w( dataMetaDom dataMetaProtobuf ).each(&method(:require))
+
+require 'dataMetaDom'
+require 'dataMetaProtobuf'
+
+@source = $*[0]
+
+DataMetaProtobuf.helpProtobufGen __FILE__, %q<Argument missing: pass the name of the file with DataMeta DOM source to convert> unless @source
+DataMetaProtobuf.helpProtobufGen __FILE__, %<"#{@source}" is not a valid file> unless File.file?(@source)
+
+@model = DataMetaDom::Model.new
+begin
+    @model.parse(@source)
+    puts "Current source: #{@model.sources.inspect}"
+    puts DataMetaProtobuf.genSchema(@model)
+rescue Exception => e
+   $stderr.puts "ERROR #{e.message}; #{@model.diagn}"
+   $stderr.puts e.backtrace.join("\n\t")
+end

data/lib/dataMetaProtobuf.rb

@@ -0,0 +1,182 @@
+$:.unshift(File.dirname(__FILE__)) unless $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+require 'erb'
+require 'fileutils'
+require 'dataMetaDom'
+require 'dataMetaDom/help'
+require 'dataMetaDom/pojo'
+require 'dataMetaDom/record'
+require 'dataMetaDom/util'
+require 'ostruct'
+
+=begin rdoc
+DataMetaDOM and {Protobuf}[https://github.com/google/protobuf/wiki] {IDL}[https://developers.google.com/protocol-buffers/docs/proto].
+
+For command line details either check the new method's source or the README, the usage section.
+=end
+
+module DataMetaProtobuf
+    # Current version
+    VERSION = '1.0.0'
+
+# First level indent
+    INDENT = ' ' * 4
+
+    # Since we are piping source in and spilling the result into the STDOUT, need logger for any other output.
+    L = Logger.new('dataMetaProtobuf.log', 0, 10_000_000)
+    L.level = Logger::INFO
+    L.datetime_format = '%Y-%m-%d %H:%M:%S'
+
+    # The root of the gem.
+    GEM_ROOT = File.realpath(File.dirname(__FILE__) + '/../')
+
+    # Location of templates.
+    TMPL_ROOT = File.join(GEM_ROOT, 'tmpl')
+
+=begin rdoc
+Mapping from a DataMeta DOM type to a matching renderer of Protobuf IDL.
+=end
+    PROTO_TYPES = {
+            DataMetaDom::BOOL => lambda{|dt| %q<bool>},
+            DataMetaDom::CHAR => lambda{|dt| %q<string>},
+            DataMetaDom::INT => lambda{ |dt|
+              len = dt.length
+              case
+                when len <= 4; %q<int32>
+                when len <= 8; %q<int64>
+                else; raise "Invalid integer length #{len}"
+              end
+            },
+            DataMetaDom::FLOAT => lambda{|dt|
+              len = dt.length
+              case
+                when len <= 4; %q<float>
+                when len <= 8; %q<double>
+                else; raise "Invalid float length #{len}"
+              end
+            },
+            DataMetaDom::RAW => lambda{|dt| %q<bytes>},
+            DataMetaDom::STRING => lambda{|dt| %q<string>},
+=begin
+Unlike DataMeta DOM, Protobuf does not support temporal types such as date, time and datetime, therefore we export
+DataMeta +datetime+ as +string+.
+=end
+        DataMetaDom::DATETIME => lambda{|dt| %q<string>},
+# No support for these in this release:
+          #NUMERIC => lambda{|t| "BigDecimal"}
+    }
+
+=begin rdoc
+Converts DataMeta DOM type to Protobuf IDL type.
+=end
+    def protoType(dataMetaType, model, rec)
+        ty = dataMetaType.type
+        if model.records[ty]
+            nameSpace, base = assertNamespace(ty)
+            base
+        elsif model.enums[ty]
+            nameSpace, base = assertNamespace(ty)
+            base
+        else
+            renderer = PROTO_TYPES[dataMetaType.type]
+            raise "Unsupported type #{dataMetaType}" unless renderer
+            renderer.call(dataMetaType)
+        end
+    end
+
+=begin rdoc
+Splits the full name of a class into the namespace and the base, returns an array of
+the namespace (empty string if there is no namespace on the name) and the base name.
+
+Examples:
+* <tt>'BaseNameAlone'</tt> -> <tt>['', 'BaseNameAlone']</tt>
+* <tt>'one.package.another.pack.FinallyTheName'</tt> -> <tt>['one.package.another.pack', 'FinallyTheName']</tt>
+=end
+    def assertNamespace(fullName)
+      ns, base = DataMetaDom::splitNameSpace(fullName)
+      [DataMetaDom.validNs?(ns, base) ? ns : '', base]
+    end
+
+=begin rdoc
+Generates the {Protobuf IDL}[https://developers.google.com/protocol-buffers/docs/proto], returns the source.
+
+We decided against using the template and use simple string concatenation instead; because the way Protobuf IDL is
+designed, it's easier to do the string concat. For example, rendering a message in a message in a message...
+=end
+    def genSchema(model)
+        result = %<
+// This Protobuf schema is generated by DataMeta exporter.
+
+syntax = "proto3";
+package #{model.sources[model.sources.doneKeys[0]].namespace};
+
+>
+        model.enums.values.each { |e| renderEnum model, e, result }
+        model.records.values.each { |r| renderRec model, r, result }
+        result << "\n"
+        result
+    end
+
+# Render a single record
+    def renderRec(model, rec, result)
+        nameSpace, base = assertNamespace(rec.name)
+        L.info("Rendering record #{rec.name}")
+        result << %<
+message #{base} {
+>
+        pbInt = 1
+        rec.fields.each_key { | fldId|
+            fld = rec.fields[fldId]
+            ty = fld.dataType.type
+            if fld.aggr && !fld.map?
+                result << "#{INDENT}repeated #{protoType(fld.dataType, model, rec)} #{fldId} = #{pbInt};\n"
+            elsif fld.map?
+                raise ArgumentError, %<Field "#{fldId}" of the rec "#{base}": for Protobuf, map can not be optional, it must be required> unless fld.isRequired
+                result << "#{INDENT}map<#{protoType(fld.dataType, model, rec)}, #{protoType(fld.trgType, model, rec)}> #{fldId} = #{pbInt};\n"
+            elsif model.enums[ty] && model.enums[ty].is_a?(DataMetaDom::Mapping)
+                raise ArgumentError, %<Field "#{fldId}" of the rec "#{base}": for Protobuf, map can not be optional, it must be required> unless fld.isRequired
+                result << "#{INDENT}map<#{protoType(model.enums[ty].fromT, model, rec)}, #{protoType(model.enums[ty].toT, model, rec)}> #{fldId} = #{pbInt};\n"
+            elsif fld.isRequired
+                result << "#{INDENT}#{protoType(fld.dataType, model, rec)} #{fldId} = #{pbInt};\n"
+            else
+                result << "#{INDENT}repeated #{protoType(fld.dataType, model, rec)} #{fldId} = #{pbInt};\n"
+            end
+            pbInt += 1
+        }
+        result << %q<}
+>
+    end
+
+# Render one single enum
+    def renderEnum(model, enm, result)
+        nameSpace, base = assertNamespace(enm.name)
+        L.info("Rendering enum #{enm.name} of the type #{enm.class}")
+        case enm
+            when DataMetaDom::Enum
+                values = enm.keys.map{|k| enm[k]} # sorted by ordinals to preserve the original order
+                pbInt = 0
+                result << %<
+enum #{base} {
+>
+                values.each{|v|
+                    result << "#{INDENT}#{v} = #{pbInt};\n"
+                    pbInt += 1
+                }
+                result << %q<}
+>
+            when DataMetaDom::Mapping
+                # taken care of in renderDer
+            else
+                raise ArgumentError, %<Unsupported enum type "#{enm.class}" for the name "#{enm.name}">
+
+        end
+
+    end
+
+    # Shortcut to help for the Protobuf IDL generator
+    def helpProtobufGen(file, errorText=nil)
+        DataMetaDom::help(file, %<DataMeta DOM Protobuf IDL Generation ver #{VERSION}>, '<DataMeta DOM source file>', errorText)
+    end
+
+module_function :helpProtobufGen, :genSchema, :assertNamespace, :protoType, :renderRec, :renderEnum
+end

data/test/sample.dmDom

@@ -0,0 +1,68 @@
+# DataMetaDOM with all field types accounted for testing.
+namespace org.ebay.datameta.examples.conv.protobuf
+
+ver 1.0.0
+
+enum BaseColor
+   Red, Green, Blue
+end
+
+mapping Timings string[3] datetime
+   "now" => DateTime.now,
+   'then' => DateTime.parse("2012-09-29T23:45:59Z")
+end
+
+mapping Depths string[16] float[3]
+   "shallow" => 0.1,
+   "medium" => 353.232,
+   "deep" => 787.0
+end
+
+record AllTypes
+    +int[4] id
+    +int[8] count
+    +bool isIt
+    +char[5] code
+    -float[4] width
+    +float[8] height
+    +string anyLength
+    +string[16] name
+    +set{string} aliases
+    +deque{datetime} accesses
+    +list{int[4]} quants
+    +map{string[32], int[4]} strToInt
+    +map{int[8], string[40]} longToString
+    +Depths depths
+    +Timings lengths
+    +BaseColor color
+    +Includable inclo
+    identity id
+end
+
+# Check how optional fields are rendered
+record Optionals
+    +int[4] id
+    -int[8] count
+    -bool isIt
+    -char[5] code
+    -float[4] width
+    +float[8] height
+    -string anyLength
+    +string[16] name
+    -set{string} aliases
+    -deque{datetime} accesses
+    -list{int[4]} quants
+    +map{string[32], int[4]} strToInt
+    +Depths depths
+    +Timings lengths
+    -BaseColor color
+    -Includable inclo
+    identity id
+end
+
+record Includable
+   +int[8] incId
+   +string[32] name
+   +float[4] amount
+end
+

data/test/test_dataMetaProtobuf.rb

@@ -0,0 +1,53 @@
+# keep this underscore naming in the test subdir, it's easier to append files names to test
+require './test/test_helper.rb'
+require 'google/protobuf'
+require 'open3'
+
+# Unit test cases for the DataMetaProtobuf
+#  See for instance:
+#  - test_full
+class TestNewGem < Test::Unit::TestCase
+    H1 ||= '*' * 15
+
+    L = Logger.new('dataMetaProtobufTests.log', 0, 10_000_000)
+    L.level = Logger::DEBUG
+    L.datetime_format = '%Y-%m-%d %H:%M:%S'
+
+    MODEL_BASE = 'sample'
+    GEN_TARGET = '.tmp'
+    PROTO_SRC = File.join(GEN_TARGET, "#{MODEL_BASE}.proto")
+
+    # an empty stub for now
+    def setup; end
+
+    # Smell-check the parsing
+    def test_parsing
+        model = DataMetaDom::Model.new
+        model.parse(File.join(File.dirname(__FILE__), 'sample.dmDom'), options={autoNsVer: true})
+        L.info(%<Model: #{model}>)
+        FileUtils.rmtree(GEN_TARGET) if File.exist?(GEN_TARGET)
+        FileUtils.mkpath GEN_TARGET
+        IO.write(PROTO_SRC, DataMetaProtobuf.genSchema(model), mode: 'wb')
+        cmd = "protoc --ruby_out=. #{PROTO_SRC}" # as counterintuitive it is, but . means actually the dir where the source is found
+        L.info(%<Running "#{cmd}">)
+        o,e,s= Open3.capture3(cmd, :binmode => true)
+        unless s.to_i == 0
+            $stderr.puts %|#{H1} OUT #{H1}
+#{o}
+#{H1} ERR #{H1}
+#{e}
+#{H1} state=#{s.inspect}
+|
+            raise RuntimeError, %|ERRORS running "#{cmd}"|
+        end
+
+        L.info("Verifying schema #{PROTO_SRC}")
+        require "./#{GEN_TARGET}/#{MODEL_BASE}_pb"
+        # see if I can use those messages and the enum
+        optionals = Org::Ebay::Datameta::Examples::Conv::Protobuf::Optionals.new
+        allTypes = Org::Ebay::Datameta::Examples::Conv::Protobuf::AllTypes.new
+        red = Org::Ebay::Datameta::Examples::Conv::Protobuf::BaseColor::Red
+        blue = Org::Ebay::Datameta::Examples::Conv::Protobuf::BaseColor::Blue
+        L.info("Red = #{red}; Blue = #{blue}")
+    end
+end

data/test/test_helper.rb

@@ -0,0 +1,8 @@
+## keep this underscore naming in the test subdir, it's easier to append files names to test
+
+require 'test/unit'
+require 'dataMetaDom'
+require 'fileutils'
+
+# this is expected to run from the project root, normally by the rake file
+require './lib/dataMetaProtobuf'

metadata

@@ -0,0 +1,96 @@
+--- !ruby/object:Gem::Specification
+name: dataMetaProtobuf
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Michael Bergens
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-02-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: dataMetaDom
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+- !ruby/object:Gem::Dependency
+  name: google-protobuf
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.2.0
+description: DataMeta DOM to Protobuf IDL generator
+email: michael.bergens@gmail.com
+executables:
+- dataMetaProtobufGen.rb
+extensions: []
+extra_rdoc_files: []
+files:
+- ".yardopts"
+- History.md
+- PostInstall.txt
+- README.md
+- Rakefile
+- bin/dataMetaProtobufGen.rb
+- lib/dataMetaProtobuf.rb
+- test/sample.dmDom
+- test/test_dataMetaProtobuf.rb
+- test/test_helper.rb
+homepage: https://github.com/eBayDataMeta
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.1.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements:
+- No special requirements
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: DataMeta Protobuf
+test_files:
+- test/test_dataMetaProtobuf.rb