dataMetaDom 1.0.0

data/lib/dataMetaDom/python.rb

@@ -0,0 +1,168 @@
+$:.unshift(File.dirname(__FILE__)) unless $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+require 'set'
+require 'fileutils'
+require 'dataMetaDom/help'
+require 'dataMetaDom/field'
+require 'dataMetaDom/pojo' # we borrow some useful methods from the POJO Lexer
+require 'erb'
+require 'ostruct'
+
+module DataMetaDom
+=begin rdoc
+Definition for generating Python basic classes that do not relate to serialization layer.
+
+For command line details either check the new method's source or the README.rdoc file, the usage section.
+=end
+module PythonLexer
+    include DataMetaDom
+
+    # Standard header for every __init__.py file
+    INIT_PY_HEADER = %q<
+# see https://docs.python.org/3/library/pkgutil.html
+# without this, Python will have trouble finding packages that share some common tree off the root
+from pkgutil import extend_path
+__path__ = extend_path(__path__, __name__)
+
+>
+
+# The top of each model file.
+    MODEL_HEADER = %q|# This file is generated by DataMeta DOM. Do not edit manually!
+# package %s
+
+import re
+
+from ebay_datameta_core.base import Verifiable, DateTime, Migrator, SemVer
+from ebay_datameta_core.canned_re import CannedRe
+
+# noinspection PyCompatibility
+from enum import Enum
+
+|
+
+# Augment the RegExRoster class with Python specifics
+    class PyRegExRoster < RegExRoster
+# converts the registry to the Python variable -- compiled Re
+        def to_patterns
+            i_to_r.keys.map { |ix|
+                r = i_to_r[ix]
+                rx = r.r.to_s
+                %<#{RegExRoster.ixToVarName(ix)} = re.compile(#{rx.inspect}) # #{r.vars.to_a.sort.join(', ')}>
+            }.join("\n#{INDENT}")
+        end
+
+# converts the registry to the verification code for the verify() method
+        def to_verifications(baseName)
+            result = (canned.keys.map { |r|
+                r = canned[r]
+                vs = r.vars.to_a.sort
+                vs.map { |v|
+                    rx = r.r.to_s
+                    %<\n#{INDENT*2}if(#{r.req? ? '' : "self.__#{v} is not None and "}CannedRe.CANNED_RES[#{rx.inspect}].match(self.__#{v}) is None):
+#{INDENT*3}raise AttributeError("Property \\"#{v}\\" == {{%s}} didn't match canned expression \\"#{rx}\\"" % self.__#{v} )>
+                }
+            }).flatten
+            (result << i_to_r.keys.map { |ix|
+                r = i_to_r[ix]
+                vs = r.vars.to_a.sort
+                rv = "#{baseName}.#{RegExRoster.ixToVarName(ix)}"
+                vs.map { |v|
+                    %<\n#{INDENT*2}if(#{r.req? ? '' : "self.__#{v} is not None and "}#{rv}.match(self.__#{v}) is None):
+#{INDENT*3}raise AttributeError("Property \\"#{v}\\" == {{%s}} didn't match custom expression {{%s}}" %(self.__#{v}, #{rv}))>
+                }
+            }).flatten
+            result.join("\n")
+        end
+    end
+
+=begin rdoc
+Generates Python sources for the model, the "plain" Python part, without fancy dependencies.
+* Parameters
+  * +parser+ - instance of Model
+  * +outRoot+ - output directory
+=end
+    def genPy(model, outRoot)
+        firstRecord = model.records.values.first
+        @pyPackage, baseName, packagePath = DataMetaDom::PojoLexer::assertNamespace(firstRecord.name)
+
+        # Next: replace dots with underscores.
+        # The path also adjusted accordingly.
+        #
+        # Rationale for this, quoting PEP 8:
+        #
+        #    Package and Module Names
+        #
+        #    Modules should have short, all-lowercase names. Underscores can be used in the module name if it improves
+        #    readability. Python packages should also have short, all-lowercase names, although the use of underscores
+        #    is discouraged.
+        #
+        # Short and all-lowercase names, and improving readability if you have complex system and need long package names,
+        # is "discouraged". Can't do this here, our system is more complicated for strictly religous, "pythonic" Python.
+        # A tool must be enabling, and in this case, this irrational ruling gets in the way.
+        # And dots are a no-no, Python can't find packages with complicated package structures and imports.
+        #
+        # Hence, we opt for long package names with underscores for distinctiveness and readability:
+        @pyPackage = @pyPackage.gsub('.', '_')
+        packagePath = packagePath.gsub('/', '_')
+        destDir = File.join(outRoot, packagePath)
+        FileUtils.mkdir_p destDir
+        # build the package path and create __init__.py files in each package's dir if it's not there.
+        @destDir = packagePath.split(File::SEPARATOR).reduce(outRoot){ |s,v|
+            r = s.empty? ? v : File.join(s, v) # next package dir
+            FileUtils.mkdir_p r # create if not there
+            # create the __init__.py with proper content if it's not there
+            df = File.join(r, '__init__.py'); IO.write(df, INIT_PY_HEADER, mode: 'wb') unless r == outRoot || File.file?(df)
+            r # pass r for the next reduce iteration
+        }
+        vars =  OpenStruct.new # for template's local variables. ERB does not make them visible to the binding
+        modelFile = File.join(@destDir, 'model.py')
+        [modelFile].each{|f| FileUtils.rm f if File.file?(f)}
+        IO.write(modelFile, MODEL_HEADER % @pyPackage, mode: 'wb')
+        (model.records.values + model.enums.values).each { |srcE| # it is important that the records render first
+            _, baseName, _ = DataMetaDom::PojoLexer::assertNamespace(srcE.name)
+
+            pyClassName = baseName
+
+            case
+                when srcE.kind_of?(DataMetaDom::Record)
+                    fields = srcE.fields
+                    rxRoster = PyRegExRoster.new
+                    eqHashFields = srcE.identity ? srcE.identity.args : fields.keys.sort
+                    reqFields = fields.values.select{|f| f.isRequired }.map{|f| f.name}
+                    verCalls = reqFields.map{|r| %<if(self.__#{r} is None): missingFields.append("#{r}");>}.join("\n#{INDENT * 2}")
+                    fieldVerifications = ''
+                    fields.each_key { |k|
+                        f = fields[k]
+                        dt = f.dataType
+                        rxRoster.register(f) if f.regex
+                        if f.trgType # Maps: if either the key or the value is verifiable, do it
+                            mainVf = model.records[dt.type] # main data type is verifiable
+                            trgVf = model.records[f.trgType.type]  # target type is verifiable
+                            if mainVf || trgVf
+                                fieldVerifications << "\n#{INDENT*2}#{!f.isRequired ? "if(self.__#{f.name} is not None):\n#{INDENT*3}" : '' }for k, v in self.__#{f.name}.iteritems():#{mainVf ? 'k.verify();' : ''} #{trgVf ? 'v.verify()' :''}\n"
+                            end
+                        end
+
+                        if model.records[dt.type] && !f.trgType # maps handled separately
+                            fieldVerifications << "\n#{INDENT*2}#{!f.isRequired ? "if(self.__#{f.name} is not None): " : '' }#{f.aggr ? "[v___#{f.name}.verify() for v___#{f.name} in self.__#{f.name} ]" : "self.__#{f.name}.verify()"}"
+                            # the Verifiable::verify method reference works just fine, tested it: Java correctly calls the method on the object
+                        end
+                    }
+                    IO::write(File.join(@destDir, 'model.py'),
+                              ERB.new(IO.read(File.join(File.dirname(__FILE__), '../../tmpl/python/entity.erb')),
+                                      $SAFE, '%<>').result(binding), mode: 'ab')
+                when srcE.kind_of?(DataMetaDom::Mappings)
+#                        FIXME -- implement!!!
+                when srcE.kind_of?(DataMetaDom::Enum)
+                    IO.write(modelFile, %|#{baseName} = Enum("#{baseName}", "#{srcE.rawVals.join(' ')}")\n\n|, mode: 'ab')
+#                        # handled below, bundled in one huge file
+                when srcE.kind_of?(DataMetaDom::BitSet)
+#                        FIXME -- implement!!!
+                else
+                    raise "Unsupported Entity: #{srcE.inspect}"
+            end
+        }
+    end
+
+end
+end

data/lib/dataMetaDom/recAttr.rb

@@ -0,0 +1,271 @@
+$:.unshift(File.dirname(__FILE__)) unless $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+%w(fileutils set).each { |r| require r }
+
+module DataMetaDom
+
+=begin rdoc
+Record Attribute such as unique fields set, identity information, indexes, references etc
+the common structure is like this:
+keyword (hint1, hint2, hint3...) arg1, arg2
+
+For command line details either check the new method's source or the README.rdoc file, the usage section.
+=end
+class RecAttr
+
+=begin rdoc
+The keyword for the attribute.
+=end
+    attr_reader :keyword
+
+=begin rdoc
+A Set of hints, empty set if there are no hints on this attribute.
+=end
+    attr_reader :hints
+
+=begin rdoc
+Arguments on this attribute if any, an array in the order as listed in the DataMeta DOM source. Order is important,
+for example, for an identity.
+=end
+    attr_reader :args
+
+=begin rdoc
+Unique key for the given attribute to distinguish between those and use in a map. Rebuilt by getKey method
+defined on the subclasses.
+=end
+    attr_reader :key
+
+=begin rdoc
+Determines if this attribute has the given hint.
+=end
+    def hasHint?(hint)
+        @hints.member?(hint)
+    end
+
+=begin rdoc
+Creates an instance for the given keyword.
+=end
+    def initialize(keyword)
+        @keyword = keyword.to_sym
+        raise "Unsupported keyword #@keyword" unless REC_ATTR_KEYWORDS.member?(@keyword)
+        @args = []; @hints = Set.new
+    end
+
+=begin rdoc
+Adds the given argument, updates the key
+=end
+    def addArg(val)
+        @args << val.to_sym
+        updateKey
+        self
+    end
+
+=begin rdoc
+Adds the given hint.
+=end
+    def addHint(val); @hints << val end
+
+=begin rdoc
+Adds an array of arguments.
+=end
+    def addArgs(vals); vals.each { |v| addArg v }; self end
+
+=begin rdoc
+Adds a collection of hints.
+=end
+    def addHints(vals); vals.each { |h| addHint h }; self end
+
+=begin rdoc
+Updates the key, returns self for call chaining
+=end
+    def updateKey; @key = getKey; self end
+
+=begin rdoc
+Returns the count of arguments.
+=end
+    def length; @args.length end
+
+=begin rdoc
+Returns the arguments in the given position, zero-based.
+=end
+    def [](index); @args[index] end
+
+
+=begin rdoc
+Joins the arguments with the given delimiter.
+=end
+    def join(delimiter); @args.join(delimiter) end
+
+=begin rdoc
+Parses this instance from the given source.
+* Parameter:
+  * +source+ - an instance of SourceFile
+=end
+    def parse(source)
+        @sourceRef = source.snapshot
+        line = source.line
+        recAttrMatch = line.scan(/^\s*(\w*)\s*(\([^\)]+\))?\s+(.+)$/)
+        raise "Invalid record attribute spec '#{line}'" unless recAttrMatch
+        keyw, hintList, argList = recAttrMatch[0]
+        raise "Wrong keyword '#{keyw}', '#@keyword' expected instead" unless keyw && keyw.to_sym == @keyword
+        @args = argList.split(/[\(\)\s\,]+/).map { |a| a.to_sym }
+        if hintList
+            @hints = Set.new hintList.split(/[\(\)\s\,]+/).select { |h| !h.strip.empty? }.map { |h| h.strip.to_sym }
+        else
+            @hints = Set.new
+        end
+    end
+
+# textual representation of this instance
+    def to_s; "#@keyword:#@key; #@sourceRef" end
+
+    private :initialize
+end
+
+=begin rdoc
+The record attribute with the unordered set of arguments.
+See the RecAttrList for the ordered list implementation.
+=end
+class RecAttrSet < RecAttr
+
+# Unordered unique set of the arguments
+    attr_reader :argSet
+
+# Creates an instance with the given keyword
+    def initialize(keyword); super(keyword); @argSet = Set.new end
+
+# Engages the super's parse method via the alias
+    alias :recAttrParse :parse
+
+# Determines if the instance has the given argument
+    def hasArg?(arg); argSet.member?(arg) end
+
+# Builds textual for the set of the arguments, for diagnostics.
+    def argSetTextual; @argSet.map { |a| a.to_s }.sort.join(':') end
+
+# Builds the unique key for the set of arguments on the instance
+    def getKey; argSetTextual.to_sym end
+
+# Adds the given argument to the instance
+    def addArg(val)
+        k = val.to_sym
+        raise "Duplicate arg #{k} in the set of #{argSetTextual}" if @argSet.member?(k)
+        @argSet << k
+        #RecAttr.instance_method(:addArg).bind(self).call k - fortunately, overkill in this case, can do with just:
+        super k
+    end
+
+=begin rdoc
+Parses the instance from the given source.
+* Parameters
+  * +src+ - an instance of SourceFile
+=end
+    def parse(src)
+        recAttrParse(src)
+        # look if there are any duplicates, if there are it's an error:
+        counterHash = Hash.new(0)
+        args.each { |a| k=a.to_sym; counterHash[k] += 1 }
+        dupes = []; counterHash.each { |k, v| dupes << k if v > 1 }
+        raise "Duplicate arguments for #{self} - [#{dupes.join(',')}]" unless dupes.empty?
+        @argSet = Set.new(args)
+        updateKey
+        self
+    end
+end
+
+=begin rdoc
+The record attribute with the ordered list of arguments.
+See RecAttrSet for the unordered set implementation.
+=end
+class RecAttrList < RecAttr
+
+# Engages the super's parse method via the alias
+    alias :recAttrParse :parse
+
+# Creates an instance with the given keyword
+    def initialize(keyword); super(keyword) end
+
+# Builds the unique key for the list of arguments on the instance
+    def getKey; @args.map { |a| a.to_s }.join(':').to_sym end
+
+=begin rdoc
+Parses the instance from the given source preserving the order of arguments, returns self for call chaining.
+* Parameters
+  * +src+ - an instance of SourceFile
+=end
+    def parse(src)
+        recAttrParse(src)
+        updateKey
+        self
+    end
+end
+
+=begin rdoc
+Record attrubute "<tt>unique</tt>"
+=end
+class RecUnique < RecAttrSet
+
+# Creates an instance with the keyword "<tt>unique</tt>"
+    def initialize
+        #noinspection RubyArgCount
+        super(UNIQUE)
+    end
+
+=begin rdoc
+Attempts to consume the "<tt>unique</tt>" attribute for the given Record from the given source.
+* Parameters
+  * +source+ - an instance of SourceFile
+=end
+    def self.consumed?(source, record)
+        source.line =~ /^#{UNIQUE}\W.+$/ ? record.addUnique(RecUnique.new.parse(source)) : nil
+    end
+end
+
+=begin rdoc
+Record attrubute "<tt>identity</tt>"
+=end
+class RecIdentity < RecAttrSet
+
+# Creates an instance with the keyword "<tt>identity</tt>"
+    def initialize
+        #noinspection RubyArgCount
+        super(IDENTITY)
+    end
+
+=begin rdoc
+Attempts to consume the "<tt>identity</tt>" attribute for the given Record from the given source.
+* Parameters
+  * +source+ - an instance of SourceFile
+=end
+    def self.consumed?(source, record)
+        source.line =~ /^#{IDENTITY}\W+.+$/ ? record.identity = RecIdentity.new.parse(source) : nil
+    end
+end
+
+=begin rdoc
+Record attrubute "<tt>index</tt>"
+=end
+class RecIndex < RecAttrList
+
+# Creates an instance with the keyword "<tt>index</tt>"
+    def initialize
+    #noinspection RubyArgCount
+        super(INDEX)
+    end
+
+=begin rdoc
+Attempts to consume the "<tt>index</tt>" attribute for the given Record from the given source.
+* Parameters
+  * +source+ - an instance of SourceFile
+=end
+    def self.consumed?(source, record)
+        source.line =~ /^#{INDEX}\W+.+$/ ? record.addIndex(RecIndex.new.parse(source)) : nil
+    end
+end
+
+=begin rdoc
+An array of record level parse token classes, namely RecIdentity, RecIndex, RecUnique
+=end
+RECORD_LEVEL_TOKENS=[RecIdentity, RecIndex, RecUnique]
+
+end