dataMetaPii 1.0.1

data/test/appLinkFull.dmPii

@@ -0,0 +1,71 @@
+# DSL describing Connection between the PII abstract definition and application concretics
+# This is full definition that can be parsed and structured.
+
+ver "12.489.2.alpha"
+
+# Reusable Attributes Division
+attributes {
+   universalConstants {
+       piPositive: 3.14, # numers and decimal point: Float
+       piNegative: -3.14, # check the negative case
+       universe: "Forty-two",
+       @sam# reference to the reusable, see further down
+   }
+   ints {
+       maxInstanceCount: 0012, # numbers only, no decimal point: perceived/rendered as an Integer
+       foo: -019 # check the negative case; note that octal conversion does not apply like with literals in Ruby code
+   }
+   sam { # Can the parser handle strings with C-like backslash escapes?
+       says: "Sam \t says, \"You kidding me, right?\"",
+       @donald # should be able to handle cycles in the inclusion tree
+   }
+   donald {
+       quacks: "Quack!",
+       @sam # should be able to handle cycles in the inclusion tree
+   }
+   # see if the parser can handle a one-liner
+   noise { @donald, spectrum: "white", @sam, @ints }
+}
+
+# Application Division
+apps {
+    pollingServices { # Unique application ID; should be unique throughout the config, each app responsible for its own def
+        # or make it centralized, standard??
+        Name_Full { # the PII key from the abstract definition
+            # Application-specific attributes for this field (need to standardize them?):
+
+            # DataMeta DOM record's name, without the version let alone the package: the application
+            # should be aware of those details and be flexible. From this, can infer the InOutable and migration calls
+            voClass: Name,
+            # list of database fields that store this value
+            bigTableFields: "users:name, clients:fullName",
+            @ints # reference to the reusables
+        }
+
+        TwoFA_Token_Serial_Number {
+            voClass: TwoFaSerialNumber,
+            bigTableFields: "security:twoFa",
+            @universalConstants,@ints
+        }
+    }
+
+    bcastingServices {
+        Name_First {
+            voClass: Name,
+            oracleFields: "USERS:FIRST_NAME",
+            lapseCount: 0123,
+            median: -123.4567,
+            weight: 987.65432
+        }
+        Name_Last {
+            voClass: Name,
+            oracleFields: "USERS:LAST_NAME"
+        }
+        Date_Of_Birth {
+            voClass: BirthDate, # plain symbol, no enclosure: perceived/rendered as a string
+            logFormat: "yyyy-MM-dd HH:mm:SS", # enclosured in a string: perceived/rendered as a string
+            @universalConstants,
+            bigTableFields: "users:dob,clients:birthDate" # enclosured in a string: perceived/rendered as a string
+        }
+    }
+}

data/test/appLinkNoReusables.dmPii

@@ -0,0 +1,35 @@
+# DSL describing Connection between the PII abstract definition and application concretics
+# This is full definition that can be parsed and structured.
+# For testing/structuring purposes, the attributes division is missing and
+# no reusables are referred to.
+
+ver "3.423.11.19"
+
+# Application Division
+apps {
+    pollingServices { # Unique application ID; should be unique throughout the config, each app responsible for its own def
+        # or make it centralized, standard??
+        Name_Full { # the PII key from the abstract definition
+            # Application-specific attributes for this field (need to standardize them?):
+
+            # DataMeta DOM record's name, without the version let alone the package: the application
+            # should be aware of those details and be flexible. From this, can infer the InOutable and migration calls
+            voClass: Name,
+            # list of database fields that store this value
+            oracleFields: "users:name, clients:fullName"
+        }
+
+        TwoFA_Token_Serial_Number {
+            voClass: TwoFaSerialNumber,
+            oracleFields: "security:twoFa"
+        }
+    }
+
+    bcastingServices {
+        Date_Of_Birth {
+            voClass: BirthDate, # plain symbol, no enclosure: perceived/rendered as a string
+            logFormat: "yyyy-MM-dd HH:mm:SS", # enclosured in a string: perceived/rendered as a string
+            oracleFields: "users:dob,clients:birthDate" # enclosured in a string: perceived/rendered as a string
+        }
+    }
+}

data/test/appLinkShowcase.dmPii

@@ -0,0 +1,73 @@
+# This definition is for grammar testing/debugging only.
+# An attempt to pass it through structuring should fail because there are intentional violations
+# of a exogrammatical structure: for example, multiple VO definitions are allowed by the grammar
+# but the CST builder rejects those with an error.
+
+ver "1.2.3.4.beta" # Full version presented here
+
+# Reusable Attributes Division
+attributes {
+  universalConstants {
+       piPositive: 3.14, # numers and decimal point: Float
+       piNegative: -3.14, # check the negative case
+       universe: "Forty-two",
+       @sam, # reference to the reusable, see further down
+@symbolSoloOnLine,
+boo: 1.23, foo:-35.36,
+    @symbolLeadinWsAndTrailingComment, # endline comment for this
+@symbolWithTrailingComment, # endline comment
+    @symbolWithLeadingWhitespace, str:"Some \n string"
+  }
+   ints {
+       maxInstanceCount: 4, # numbers only, no decimal point: perceived/rendered as an Integer
+       foo: -4 # check the negative case
+   }
+   sam { # Can the parser handle strings with C-like backslash escapes?
+       says: "Sam \t says, \"You kidding me, right?\""
+   }
+   donald {
+       quacks: "Quack!"
+   }
+   # see if the parser can handle a one-liner
+   noise { @donald, spectrum: "white", @sam }
+}
+
+# Application Division
+apps {
+    pollingServices { # Unique application ID; should be unique throughout the config, each app responsible for its own def
+        # or make it centralized, standard??
+        Name_Full { # the PII key from the abstract definition
+            # Application-specific attributes for this field (need to standardize them?):
+
+            # DataMeta DOM record's name, without the version let alone the package: the application
+            # should be aware of those details and be flexible. From this, can infer the InOutable and migration calls
+            voClass: Name,
+            # list of database fields that store this value
+            mySqlFields: "users:name, clients:fullName",
+            @ints # reference to the reusables
+        }
+
+        TwoFA_Token_Serial_Number {
+            voClass: TwoFaSerialNumber,
+            db2Fields: "security:twoFa",
+            @universalConstants,@ints
+        }
+    } # try this
+    bcastingServices {
+    Date_Of_Birth {
+voClass:FirstClass,
+voClass: SecondClass,
+voClass : ThirdClass,
+voClass :FourthClass,
+# Leading space
+    voClass: FifthClass,
+    voClass: SixthClass, # tail comment
+voClass :SeventhClass # tail with no leading whitespace
+    }
+    TwoFA_Token_Serial_Number {
+        voClass: TwoFaSerialNumber,
+        bigTableFields: "security:twoFa",
+        @universalConstants,@ints
+    }
+    }
+}

data/test/registryShowcase.dmPii

@@ -0,0 +1,28 @@
+# Master Definition Of the PII fields
+# the "taboo" level means that under no circumstances the value should be stored.
+
+# PII field definition format:
+#             PII-Field-UniqueKey {PROFILE-KEY {level=IMPACT_LEVEL}}
+
+# These impact levels are picked at random for testing, check with your company's policies
+# for real impact levels. 
+
+ver "1.0.0"
+
+Corp_User_Name { # PII field unique key
+        level=taboo # "taboo" : in this profile, under no circumstances this field can be stored or rendered in the logs
+}
+
+Date_Of_Birth { level=restricted }
+
+TwoFA_Token {level=confidential, dataType=access}
+
+Home_City {level=confidential, dataType=location}
+Home_Country {level=confidential}
+Home_State {level=confidential} # State or Province
+Home_Zip {level=confidential} # U.S. ZIP code
+Name_First {level=confidential}
+Name_Last {level=internal}
+Name_Middle {level=public}
+Name_Full {level=confidential,enabled=true, owner=nameKeeper}
+

data/test/test_dataMetaPii.rb

@@ -0,0 +1,42 @@
+# keep this underscore naming in the test subdir, it's easier to append files names to test
+require './test/test_helper.rb'
+
+=begin rdoc
+Unit test cases for the DataMetaPii
+=end
+class TestNewGem < Test::Unit::TestCase
+    include DataMetaPiiTests
+
+    # an empty stub for now
+    def setup;
+        @impactConstVals = DataMetaPii::Impact.constants.map{|c| ":#{DataMetaPii::Impact.const_get(c)}"}
+    end
+
+=begin rdoc
+Test obtaining a constant list from a module
+=end
+    def test_constantList
+        v = DataMetaPii::RegKeyVo.new('A_PII_field',
+                                  {DataMetaPii::RegKeyVo::LEVEL => DataMetaPii::Impact::CONFIDENTIAL, "foo" => "bar", "val" => 123.456})
+
+        L.info(%<Impact constants: #{DataMetaPii::Impact.constants.map{|c| "#{c}=#{DataMetaPii::Impact.const_get(c)}"}.join('; ')}
+first PII VO created: #{v.to_tree_image(DataMetaPii::INDENT)}
+>)
+        xcp = assert_raise(ArgumentError) { DataMetaPii::RegKeyVo.new('A_PII_field', {DataMetaPii::RegKeyVo::LEVEL => ''}) }
+        assert(xcp.message.start_with?('Impact level missing or empty in'))
+
+        xcp = assert_raise(ArgumentError) { DataMetaPii::RegKeyVo.new('A_PII_field', {'foo' => 'bar'}) }
+        assert(xcp.message.start_with?('Impact level missing or empty in'))
+
+        xcp = assert_raise(ArgumentError) { DataMetaPii::RegKeyVo.new('A PII field',
+                                                                  {DataMetaPii::RegKeyVo::LEVEL => DataMetaPii::Impact::CONFIDENTIAL}) }
+
+        assert(xcp.message.start_with?('Invalid PII key: '))
+
+        xcp = assert_raise(ArgumentError) { DataMetaPii::RegKeyVo.new('A_PII_field',
+                                                                  {DataMetaPii::RegKeyVo::LEVEL => :bad_impact_level}) }
+
+        L.info(%<Testing #{:bad_impact_level}: #{xcp.message}\n#{xcp.backtrace.join("\n\t")}\n>)
+        assert(xcp.message.start_with?('Unsupported Impact Level '))
+    end
+end

data/test/test_grammars.rb

@@ -0,0 +1,152 @@
+require './test/test_helper.rb'
+
+=begin rdoc
+Test for the DataMetaPii grammars and parsing.
+
+Assertions: http://ruby-doc.org/stdlib-1.9.3/libdoc/test/unit/rdoc/Test/Unit/Assertions.html
+=end
+class TestPiiFmts < Test::Unit::TestCase
+
+    include DataMetaPiiTests
+
+    # loads the fmt grammar and creates the parser instance
+    def setup
+
+        @registryParser = PiiRegistryParser.new
+        @piiAppLinkParser = PiiAppLinkParser.new
+        L.info %<DataMeta PII Registry parser: #{@registryParser.inspect}
+DataMeta PII App Link parser: #{@piiAppLinkParser.inspect}
+>
+    end
+
+=begin rdoc
+Test Pii Registry DML parsing
+=end
+    def test_registry_parse
+        ast = DataMetaParse.parse(@registryParser, IO.read('./test/registryShowcase.dmPii'))
+        L.info 'Registry parse start'
+        raise 'Registry File Format parse unsuccessful' unless ast
+        raise ast if ast.is_a?(DataMetaParse::Err)
+        #L.info "AST:\n#{ast.fields.inspect}"
+        #L.info "----------AST:\n#{ast.inspect}"
+        #elems = [ast.items.elements]
+        #elems = [ast.fields]
+        #L.info "===========AST Fields:\n#{elems.inspect}\n==================="
+        ast.fields.elements.each { |f|
+            L.info("PII Key: #{f.pk}: #{f.attrbLst.attrbs.map{|e| "#{e.k}::#{e.v}"}.join('; ')}")
+        }
+        L.info(%<PII Registry:
+#{DataMetaPii.buildRegCst(ast).to_tree_image(DataMetaPii::INDENT)}>)
+    end
+
+=begin rdoc
+Test Pii Registry DML parsing and structuring - full version
+=end
+    def test_app_link_parse_full
+        L.info 'Full applink parse start'
+        ast = DataMetaParse.parse(@piiAppLinkParser, IO.read('./test/appLinkFull.dmPii'))
+        raise 'Full AppLink parse unsuccessful' unless ast
+        if ast.is_a?(DataMetaParse::Err)
+            raise %<#{ast.parser.failure_line}
+#{ast.parser.failure_reason}>
+        end
+        #L.info "AST:\n#{ast.fields.inspect}"
+        #L.info "----------AST:\n#{ast.inspect}"
+        #elems = [ast.items.elements]
+        #elems = [ast.fields]
+        #L.info "===========AST Fields:\n#{elems.inspect}\n==================="
+        ast.elements[1].elements.each { |f|
+            if f.respond_to?(:type) && f.type == 'refVal'
+                L.info(%<refVal: #{f.sym}>)
+            else
+                L.info(%<AppLink Elem: #{f.inspect}>)
+            end
+        }
+        ast.elements[3].elements.each { |f|
+            if f.respond_to?(:type) && f.type == 'refVal'
+                L.info(%<refVal: #{f.sym}>)
+            else
+                L.info(%<AppLink Elem: #{f.inspect}>)
+            end
+        }
+        log = ''
+        appLinkObj = DataMetaPii.buildAlCst(ast, log)
+
+        L.info(%<AppLink Full:
+#{appLinkObj}
+Building log:
+#{log}
+>)#.to_tree_image(DataMetaPii::INDENT_STEP)}>)
+
+    end
+
+    def test_app_link_parse_method
+        appLinkModel = DataMetaPii.parseAppLink(IO.read('./test/appLinkFull.dmPii'))
+        L.info(%<AppLink Model - full:
+#{appLinkModel.inspect}>)
+    end
+
+=begin rdoc
+Test Pii Registry DML parsing - showcase version, do not run this tree through CST builder!
+=end
+    def test_app_link_parse_showcase
+        L.info 'Showcase applink parse start'
+        ast = DataMetaParse.parse(@piiAppLinkParser, IO.read('./test/appLinkShowcase.dmPii'))
+        raise 'Showcase File Format parse unsuccessful' unless ast
+        if ast.is_a?(DataMetaParse::Err)
+            raise %<#{ast.parser.failure_line}
+                  #{ast.parser.failure_reason}>
+        end
+        #L.info "AST:\n#{ast.fields.inspect}"
+        #L.info "----------AST:\n#{ast.inspect}"
+        #elems = [ast.items.elements]
+        #elems = [ast.fields]
+        #L.info "===========AST Fields:\n#{elems.inspect}\n==================="
+        ast.elements[1].elements.each { |f|
+            if f.respond_to?(:type) && f.type == 'refVal'
+                L.info(%<refVal: #{f.sym}>)
+            else
+                L.info(%<AppLink Elem: #{f.inspect}>)
+            end
+        }
+        ast.elements[3].elements.each { |f|
+            if f.respond_to?(:type) && f.type == 'refVal'
+                L.info(%<refVal: #{f.sym}>)
+            else
+                L.info(%<AppLink Elem: #{f.inspect}>)
+            end
+        }
+    end
+
+=begin rdoc
+Test Pii Registry DML parsing - version with no reusables, must be successfully processed by the CST builder
+=end
+    def test_app_link_parse_no_reusables
+        L.info 'No-reusables applink parse start'
+        ast = DataMetaParse.parse(@piiAppLinkParser, IO.read('./test/appLinkNoReusables.dmPii'))
+        raise 'No-reusables File Format parse unsuccessful' unless ast
+        if ast.is_a?(DataMetaParse::Err)
+            raise %<#{ast.parser.failure_line}
+                  #{ast.parser.failure_reason}>
+        end
+        #L.info "AST:\n#{ast.fields.inspect}"
+        #L.info "----------AST:\n#{ast.inspect}"
+        #elems = [ast.items.elements]
+        #elems = [ast.fields]
+        #L.info "===========AST Fields:\n#{elems.inspect}\n==================="
+        ast.elements[0].elements.each { |f|
+            if f.respond_to?(:type) && f.type == 'refVal'
+                L.info(%<refVal: #{f.sym}>)
+            else
+                L.info(%<AppLink Elem: #{f.inspect}>)
+            end
+        }
+        log = ''
+        applinkObj = DataMetaPii.buildAlCst(ast, log)
+        L.info(%<AppLink No Reusables:
+#{applinkObj.inspect}
+Building log:
+#{log}
+>)#.to_tree_image(DataMetaPii::INDENT_STEP)}>)
+    end
+end

data/test/test_helper.rb

@@ -0,0 +1,24 @@
+## keep this underscore naming in the test subdir, it's easier to append files names to test
+%w(stringio test/unit).each { |r| require r }
+# this is expected to run from the project root, normally by the rake file
+
+$VERBOSE = false # turn off noisy for the 1st release
+
+require './lib/dataMetaPii'
+require 'logger'
+require 'dataMetaParse'
+
+module DataMetaPiiTests
+
+    L = Logger.new('piiTests.log', 0, 10_000_000)
+    L.level = Logger::DEBUG
+    L.datetime_format = '%Y-%m-%d %H:%M:%S'
+
+# By the way, inspecting a parser does not make any difference compared to just to_s:
+    L.info(%<Loaded base rules: #{DataMetaPii::BASE_RULES}
+
+Loaded PII Commons Rules: #{DataMetaPii::PII_COMMONS.inspect}
+Loaded Regstry Rules: #{DataMetaPii::REGISTRY}
+Loaded AppLink Rules: #{DataMetaPii::APP_LINK}
+>)
+end

data/tpl/java/master.erb

@@ -0,0 +1,64 @@
+package <%= namespace %>;
+
+/*
+ ****************  This file is generated by DataMeta; do not edit manually!!
+*/
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static java.util.Collections.unmodifiableMap;
+
+/**
+ * The encapsulation of the version <%= reg.ver %> of the Abstract PII definition.
+ */
+public class <%= className %> {
+<% reg.keyVos.keys.sort.each { |k| fldName = k.to_s.inspect; %>
+  public final static String <%= k.to_s %> = <%=fldName%>;
+<%}%>
+
+  /**
+   * Impact Levels:
+  */
+  public static enum Level {
+<%= (codeIndent * 3)[0..-2] %><%= ALL_IMPACTS.map{|i| i.to_s.upcase}.join(",\n#{(codeIndent * 3)[0..-2]}") %>
+
+  }
+
+  /**
+   * One abstract key definition encapsulation.
+   */
+  public static class Def {
+    private final String key;
+    private final Level level;
+    private final Map<String, String> props;
+
+    public Def(String key, final Level level, final Map<String, String> props) {
+      this.key = key;
+      this.level = level;
+      this.props = props;
+    }
+
+    public Level getLevel() { return level; }
+
+    public Map<String, String> getProps() { return props; }
+
+    public String getKey() { return key; }
+  }
+
+  /**
+   * The map keyed by field name pointing to the matching instance of the {@link Def} class.
+   */
+  public static final Map<String, Def> DEFS = unmodifiableMap(new HashMap<String, Def>() {
+    {
+<% reg.keyVos.keys.sort.each { |k| fldName = k.to_s; fldDef = reg.keyVos[k] %>
+      put(<%= fldName %>, new Def(<%= fldName %>, Level.<%= fldDef.level.to_s.upcase %>, unmodifiableMap(new HashMap<String, String>() {
+        {<% fldDef.attrs.keys.sort.each {|k| keyStr = k.to_s.inspect; valStr = fldDef.attrs[k].inspect %>
+          put(<%=keyStr%>, <%=valStr%>); <% } %>
+        }
+      })));
+<%   } %>
+      }
+    });
+}
+