dataMetaDom 1.0.0

data/lib/dataMetaDom/model.rb

@@ -0,0 +1,274 @@
+$:.unshift(File.dirname(__FILE__)) unless $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+require 'set'
+require 'dataMetaDom/converter'
+require 'dataMetaDom/docs'
+require 'dataMetaDom/field'
+require 'dataMetaDom/ver'
+require 'date'
+
+module DataMetaDom
+=begin rdoc
+Metadata Model, including parsing.
+
+For command line details either check the new method's source or the README.rdoc file, the usage section.
+=end
+class Model
+    include DataMetaDom
+
+=begin rdoc
+The instance of SourceFile currently being parsed.
+=end
+    attr_reader :currentSource
+
+=begin rdoc
+All sources, including the includes.
+=end
+    attr_reader :sources
+=begin rdoc
+Instances of Enum, Map and BitSet on this model, keyed by the full name including the namespace if any.
+=end
+    attr_reader :enums
+
+=begin rdoc
+Instances of Record on this model, keyed by the full name including the namespace if any.
+=end
+    attr_reader :records
+
+=begin rdoc
+Reverse references keyed by reference target names
+=end
+    attr_reader :reRefs
+=begin
+Documentation for all options is in this section.
+
+* +autoNsVer+ - if set to True, advises a generator to append a +vN+ where +N+ is a version number to the namespace.
+=end
+    attr_accessor :options
+# Version on the model level
+    attr_accessor :ver
+
+=begin rdoc
+Creates a blank model.
+=end
+    def initialize() # no file name if want to build model manually
+        @reRefs = Hash.new(*[]) # references to the entity, hash keyed by entity
+        @enums={}; @records = {}
+        @ver = nil
+    end
+
+=begin rdoc
+Resolves references after parsing all the sources to the types that were used before than they
+were defined. Verifies integrity.
+=end
+    def resolveVerify
+        duplicateGuard = {}
+        @records.each_key { |recKey|
+            rec = @records[recKey]
+            rec.refs.each { |ref|
+                ref.resolve self
+                preExisting = duplicateGuard[ref.key]
+                raise "Duplicate reference spec: #{r}(#{r.sourceRef}), pre-existing: #{preExisting}(#{preExisting.sourceRef})" if preExisting
+                duplicateGuard[ref.key] = ref
+                reKey = ref.toEntity.name
+                @reRefs[reKey] = [] unless @reRefs[reKey]
+                @reRefs[reKey] << ref
+            }
+        }
+        raise RuntimeError, "No version defined on #{self}" unless ver
+        self
+    end
+
+=begin rdoc
+Builds diagnostics string, including the source info.
+=end
+    def diagn; "; Src: #{@currentSource ? @currentSource : '<no source>'}" end
+
+    # master parse, initializes process queue and seeds it with the master file
+    def parse(fileName, options={autoVerNs: false})
+        @options = options
+        @sources = Sources.new(fileName)
+        while (@currentSource=@sources.next)
+            @currentSource.parse self
+        end
+        resolveVerify
+        self
+    end
+
+=begin rdoc
+Adds the given record to the model
+* Parameter
+  * +rec+ - instance of a Record
+=end
+    def addRecord(rec); @records[rec.key] = rec end
+
+=begin rdoc
+Adds the given records to the model
+* Parameter
+  * +recs+ - an array of instances of a Record
+=end
+    def addRecords(recs); recs.each { |r| addRecord r } end
+
+=begin rdoc
+Adds the given enum to the model
+* Parameter
+  * +rec+ - instance of a Enum or a BitSet or a Map
+=end
+    def addEnum(newEnum); @enums[newEnum.name] = newEnum end
+
+=begin rdoc
+Adds the given enums to the model
+* Parameter
+  * +rec+ - an array of instances of a Enum or a BitSet or a Map
+=end
+    def addEnums(enums); enums.each { |e| addEnum e } end
+
+=begin rdoc
+Generates DataMeta DOM source for the given Enum, yielding the lines to the caller's block.
+
+* Parameters
+  * +e+ - instance of a Enum or a BitSet or a Map to generate the DataMeta DOM source for
+  * +baseName+ - the base name excluding the namespace if any, usually available on the caller's side.
+=end
+    def genSourceEnum(e, baseName)
+        yield '' # yield empty line before a type
+
+        case
+            when e.kind_of?(Enum)
+                if e.docs
+                    genDocs(e.docs){|line| yield line}
+                end
+                yield "#{e.sourceKeyWord} #{baseName}"
+                #genVer(e) { |line| yield line }
+                e.values.each { |v|
+                    yield "#{SOURCE_INDENT}#{v}"
+                }
+            when e.kind_of?(BitSet), e.kind_of?(Mappings)
+                if e.docs
+                    genDocs(e.docs){|line| yield line}
+                end
+                yield "#{e.sourceKeyWord} #{baseName} #{e.kind_of?(BitSet) ? '' : e.fromT.to_s + ' '}#{e.toT}"
+                #genVer(e) { |line| yield line }
+                e.keys.each { |k|
+                    fromConv = CONVS[e.fromT.type]
+                    toConv = CONVS[e.toT.type]
+                    #DataMetaDom::L.debug "k=#{k.inspect}, e=#{e[k].inspect}"
+                    raise "Invalid convertor for #{e}: (#{fromConv.inspect} => #{toConv.inspect})" unless fromConv && toConv
+                    yield "#{SOURCE_INDENT}#{fromConv.ser.call(k)} => #{toConv.ser.call(e[k])},"
+                }
+            else
+                raise "Enum #{e} - unsupported format"
+        end
+        yield END_KW
+    end
+
+=begin rdoc
+Renders the source for the docs property of Documentable.
+=end
+    def genDocs(docs)
+        docs.each_key{ |t|
+            yield "#{DOC} #{t}"
+            d = docs[t]
+            yield d.text
+            yield END_KW
+        }
+    end
+
+=begin rdoc
+Renders the source for the docs property of Ver.
+=end
+    def genVer(e)
+        raise "No version on #{e}" unless e.ver
+        v = e.ver
+        raise "Version on #{e} is wrong type: #{v.inspect}" unless v.kind_of?(Ver)
+        yield "#{VER_KW} #{v.full}"
+    end
+
+=begin rdoc
+Generates DataMeta DOM source for the given Record, yielding the lines to the caller's block.
+
+* Parameters
+  * +r+ - instance of a Record to generate the DataMeta DOM source for
+  * +namespace+ - the namespace of the record, usually available on the caller's side.
+  * +baseName+ - the base name excluding the namespace if any, usually available on the caller's side.
+=end
+    def genSourceRec(r, namespace, baseName)
+        yield '' # yield empty line before a type
+        if r.docs
+            genDocs(r.docs){|line| yield line}
+        end
+
+        yield "#{RECORD} #{baseName}"
+        #genVer(r) { |line| yield line }
+        r.fields.values.each { |f|
+            if f.docs
+                genDocs(f.docs) { |line| yield line}
+            end
+            t = f.dataType
+            #puts ">>F: #{f}, ns=#{ns}, base=#{base}, bn=#{baseName}"
+            # render names from other namespaces than the current in full
+            renderType = qualName(namespace, t.type)
+            srcLine = if f.map?
+                          trgRender = qualName(namespace, f.trgType.type)
+                          "#{SOURCE_INDENT}#{f.req_spec}#{Field::MAP}{#{renderType}#{t.length_spec}, #{trgRender}#{
+                                f.trgType.length_spec}} #{f.name}#{f.default_spec}"
+                      elsif f.aggr?
+                          "#{SOURCE_INDENT}#{f.req_spec}#{f.aggr}{#{renderType}#{t.length_spec}} #{f.name}#{f.default_spec}"
+                      else
+                          "#{SOURCE_INDENT}#{f.req_spec}#{renderType}#{t.length_spec} #{f.name}#{f.default_spec}#{f.matches_spec}"
+                      end
+            yield  srcLine
+        }
+
+        yield "#{SOURCE_INDENT}#{IDENTITY}#{r.identity.hints.empty? ? '' : "(#{r.identity.hints.to_a.join(', ')})"} "\
+          "#{r.identity.args.join(', ')}" if r.identity
+        if r.uniques
+            r.uniques.each_value { |uq|
+                yield "#{SOURCE_INDENT}#{UNIQUE}#{uq.hints.empty? ? '' : "(#{uq.hints.to_a.join(', ')})"} #{uq.args.join(', ')}"
+            }
+        end
+        if r.indexes
+            r.indexes.each_value { |ix|
+                yield "#{SOURCE_INDENT}#{INDEX}#{ix.hints.empty? ? '' : "(#{ix.hints.to_a.join(', ')})"} #{ix.args.join(', ')}"
+            }
+        end
+        if r.refs
+            r.refs.each { |ref|
+                yield "# #{ref}"
+            }
+        end
+        yield END_KW
+    end
+
+=begin rdoc
+Generates the source lines for the given model,
+yields the lines to the caller's block, use as:
+
+    genSource{|line| ... }
+=end
+    def genSource
+        yield '# model definition exported into the source code by DataMeta DOM'
+        namespace = ''
+        (@enums.keys + @records.keys).sort { |a, b| a.to_s <=> b.to_s }.each { |k|
+            ns, base = DataMetaDom.splitNameSpace(k.to_s)
+            if DataMetaDom.validNs?(ns, base) && ns != namespace
+                namespace = ns
+                yield "#{NAMESPACE} #{namespace}"
+            end
+
+            raise 'No version on the model' unless @ver
+            raise "Version on the model is wrong type: #{@ver.inspect}" unless @ver.kind_of?(Ver)
+            yield "#{VER_KW} #{@ver.full}"
+            case
+                when @records[k]
+                    genSourceRec(@records[k], namespace, base) { |line| yield line }
+                when @enums[k]
+                    genSourceEnum(@enums[k], base) { |line| yield line }
+                else
+                    raise "Unsupported entity: #{e.inspect}"
+            end
+        }
+    end
+end
+
+end

data/lib/dataMetaDom/mySql.rb

@@ -0,0 +1,256 @@
+$:.unshift(File.dirname(__FILE__)) unless $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+require 'set'
+require 'fileutils'
+
+module DataMetaDom
+
+=begin rdoc
+Definition for generating MySQL 5 artifacts such as schemas, select statements,
+ORM input files etc etc
+
+TODO this isn't a bad way, but beter use templating next time such as {ERB}[http://ruby-doc.org/stdlib-1.9.3/libdoc/erb/rdoc/ERB.html].
+
+For command line details either check the new method's source or the README.rdoc file, the usage section.
+=end
+module MySqlLexer
+
+=begin rdoc
+Integer types
+=end
+    INT_TYPES = {2 => 'smallint', 4 => 'int', 8 => 'bigint'}
+
+=begin rdoc
+Float types
+=end
+    FLOAT_TYPES = {4 => 'float', 8 => 'double'}
+=begin rdoc
+Not null (required) wording per MySQL DDL syntax
+=end
+    NOT_NULL=' not null'
+
+=begin rdoc
+\Mapping from DataMeta DOM standard types to correspondent MySQL types renderer lambdas.
+=end
+    SQL_TYPES={
+            INT => lambda { |len, isReq|
+                concreteType = INT_TYPES[len]
+                raise "Invalid integer type length #{len} " unless concreteType
+                "#{concreteType}#{isReq ? NOT_NULL : ''}"
+            },
+            DataMetaDom::FLOAT => lambda { |len, isReq|
+                concreteType = FLOAT_TYPES[len]
+                raise "Invalid integer type length #{len} " unless concreteType
+                "#{concreteType}#{isReq ? NOT_NULL : ''}"
+            },
+            STRING => lambda { |len, isReq| "varchar(#{len})#{isReq ? NOT_NULL : ''}" },
+            DATETIME => lambda { |len, isReq| "datetime#{isReq ? NOT_NULL : ''}" },
+            BOOL => lambda { |len, isReq| "bool#{isReq ? NOT_NULL : ''}" }
+    }
+
+=begin rdoc
+Encapsulates 4 parts of DDL related SQL output:
+* Creates
+* Drops
+* Linking aka Coupling aka creating Foreign Keys
+* Unlinking aka Uncoupling aka dropping Foreign Keys
+=end
+    class SqlOutput
+
+=begin rdoc
+Open output file into create SQL DDL statements (CREATE TABLE)
+=end
+        attr_reader :create
+
+=begin rdoc
+Open output file into drop SQL DDL statements (DROP TABLE)
+=end
+        attr_reader :drop
+
+=begin rdoc
+Open output file into the \couple SQL DDL statements, creating foreign keys
+=end
+        attr_reader :couple
+=begin rdoc
+Open output file into the \uncouple SQL DDL statements, dropping foreign keys
+=end
+        attr_reader :uncouple
+
+=begin rdoc
+Creates an instance into the given target directory in which all 4 parts of the SQL DDL
+process will be created.
+=end
+        def initialize(sqlTargetDir)
+            @selTargetDir = sqlTargetDir
+            @create = File.new("#{sqlTargetDir}/DDL-create.sql", 'wb')
+            @drop = File.new("#{sqlTargetDir}/DDL-drop.sql", 'wb')
+            @couple = File.new("#{sqlTargetDir}/DDL-couple.sql", 'wb')
+            @uncouple = File.new("#{sqlTargetDir}/DDL-uncouple.sql", 'wb')
+            @allScriptFiles = [@create, @drop, @couple, @uncouple]
+            @dropScripts = [@uncouple, @drop]
+            @allScriptFiles.each { |f|
+                f.puts %q</* Generated by DataMeta DOM MySQL utility
+DO NOT EDIT MANUALLY, update the DataMeta DOM source and regen.
+*/
+>
+            }
+            @dropScripts.each { |ds|
+                ds.puts %q<
+/* Disable all checks for safe dropping without any errors */
+SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0;
+SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0;
+SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='TRADITIONAL,ALLOW_INVALID_DATES';
+
+>
+            }
+        end
+
+=begin rdoc
+Safely closes all the output files.
+=end
+        def close
+            @dropScripts.each { |ds|
+                ds.puts %q<
+
+/* Re-enable all checks disabled earlier */
+SET SQL_MODE=@OLD_SQL_MODE;
+SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS;
+SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS;
+>
+            }
+            @allScriptFiles.each { |f|
+                begin
+                    f.close
+                rescue Exception => x;
+                    $stderr.puts x.message
+                end
+            }
+        end
+    end
+
+=begin rdoc
+Builds and returns an autoincrement clause if applicable, for the given record and the field.
+
+If the field is the one and only identity on the record *and* if it is an integral type, returns
+the auto increment clause, otherwise returns and empty string.
+=end
+    def autoGenClauseIfAny(record, field)
+        record.identity && record.identity.length == 1 && field.name == record.identity[0] &&
+                field.dataType.type == DataMetaDom::INT ? ' AUTO_INCREMENT' : ''
+    end
+
+=begin rdoc
+Renders the given field into create statement.
+* Parameters:
+  * +createStatement+ - the create statement to append the field definition to.
+  * +parser+ - the instance of the Model
+  * +record+ - the instance of the Record to which the field belongs
+  * +fieldKey+ - the full name of the field to render turned into a symbol.
+  * +isFirstField+ - the boolean, true if the field is first in the create statement.
+=end
+    def renderField(createStatement, parser, record, fieldKey, isFirstField)
+        field = record[fieldKey]
+        ty = field.dataType
+        stdRenderer = SQL_TYPES[ty.type]
+        typeEnum = parser.enums[ty.type]
+        typeRec = parser.records[ty.type]
+
+        typeDef = if stdRenderer
+                      stdRenderer.call ty.length, field.isRequired
+                  elsif typeEnum
+                      "enum('#{typeEnum.values.join("','")}')"
+                  elsif typeRec
+                      raise "Invalid ref to #{typeRec} - it has no singular ID" unless typeRec.identity.length == 1
+                      idField = typeRec[typeRec.identity[0]]
+                      idRenderer = SQL_TYPES[idField.dataType.type]
+                      raise 'Only one-level prim type references only allowed in this version' unless idRenderer
+                      idRenderer.call idField.dataType.length, field.isRequired
+                  else
+                      raise ArgumentError, "Unsupported datatype #{ty}"
+                  end
+        createStatement << ",\n" unless isFirstField
+        createStatement << "\t#{field.name} #{typeDef}#{autoGenClauseIfAny(record, field)}"
+    end
+
+=begin rdoc
+Builds and returns the foreign key name for the given entity (Record) name and the counting number of these.
+* Parameters:
+  * +bareEntityName+ - the entity name without the namespace
+  * +index+ - an integer, an enumerated counting number, starting from one. For each subsequent FK this number is
+    incremented.
+=end
+    def fkName(bareEntityName, index)
+        "fk_#{bareEntityName}_#{index}"
+    end
+
+=begin rdoc
+Render SQL record with for the given model into the given output.
+* Parameters
+  * +out+ - an instance of SqlOutput
+  * +parser+ - an instance of Model
+  * +recordKey+ - full name of the record datatype including namespeace if any turned into a symbol.
+=end
+    def renderRecord(out, parser, recordKey)
+        record = parser.records[recordKey]
+        ns, entityName = DataMetaDom.splitNameSpace record.name
+        isFirstField = true
+        out.drop.puts "\ndrop table if exists #{entityName};"
+        fkNumber = 1 # to generate unique names that fit in 64 characters of identifier max length for MySQL
+        record.refs.select { |r| r.type == Reference::RECORD }.each { |ref|
+            ns, fromEntityBareName = DataMetaDom.splitNameSpace ref.fromEntity.name
+            ns, toEntityBareName = DataMetaDom.splitNameSpace ref.toEntity.name
+            out.couple.puts "alter table #{fromEntityBareName} add constraint #{fkName(fromEntityBareName, fkNumber)} "\
+  " foreign key (#{ref.fromField.name}) references #{toEntityBareName}(#{ref.toFields.name});"
+            out.uncouple.puts "alter table #{fromEntityBareName} drop foreign key #{fkName(fromEntityBareName, fkNumber)};"
+            fkNumber += 1
+        }
+        ids = record.identity ? record.identity.args : []
+        createStatement = "create table #{entityName} (\n"
+        fieldKeys = [] << ids.map { |i| i.to_s }.sort.map { |i| i.to_sym } \
+   << record.fields.keys.select { |k| !ids.include?(k) }.map { |k| k.to_s }.sort.map { |k| k.to_sym }
+
+        fieldKeys.flatten.each { |f|
+            renderField(createStatement, parser, record, f, isFirstField)
+            isFirstField = false
+        }
+        if record.identity && record.identity.length > 0
+            createStatement << ",\n\tprimary key(#{ids.sort.join(', ')})"
+        end
+        unless record.uniques.empty?
+            uqNumber = 1
+            record.uniques.each_value { |uq|
+                createStatement << ",\n\tunique uq_#{entityName}_#{uqNumber}(#{uq.args.join(', ')})"
+                uqNumber += 1 # to generate unique names that fit in 64 characters of identifier max length for MySQL
+            }
+        end
+        unless record.indexes.empty?
+            ixNumber = 1
+            record.indexes.each_value { |ix|
+                createStatement << ",\n\tindex ix_#{entityName}_#{ixNumber}(#{ix.args.join(', ')})"
+                ixNumber += 1 # to generate unique names that fit in 64 characters of identifier max length for MySQL
+            }
+        end
+        createStatement << "\n) Engine=InnoDB;\n\n" # MyISAM, the default engine does not support FKs
+
+        out.create.puts createStatement
+    end
+
+=begin rdoc
+Generate the MySQL DDL from the given Model into the given output directory.
+* Parameters
+  * +parser+ - an instance of a Model
+  * +outDir+ - a String, the directory to generate the DDL into.
+=end
+    def genDdl(parser, outDir)
+        out = SqlOutput.new(outDir)
+        begin
+            parser.records.each_key { |r|
+                renderRecord(out, parser, r)
+            }
+        ensure
+            out.close
+        end
+    end
+
+end
+end