dataMetaParse 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7a0e72fa6c8ea304157a65156fb66f908b74bef4
+  data.tar.gz: 0cb917be27d2ca056bc345e277e3377dcbdd1334
+SHA512:
+  metadata.gz: 4aa00db56ca8992e43119ac430759bc0429390b8c3dc6f1f4ecfc0000eebd87e711ac5ea4852472f10d15e50868ec0a9d97c320aa6f8c50325967e0f4b4dfeff
+  data.tar.gz: d94b08fa48f84f458d8bc18c5e5ccc9feebba35c6e62caaf58070c9188157fe204285b906271d5ea6b1e9db874c2ccdc3e04b1948db0c5075bc6d833332d369a

data/.yardopts

@@ -0,0 +1 @@
+--title "DataMeta Parsing Utils" -r README.md --charset UTF-8 lib/**/*rb - README.md

data/History.md

@@ -0,0 +1,5 @@
+# `dataMetaParse` Release history:
+
+## `1.0.0` released `2017-01-15`
+* 1 major enhancement:
+    * Initial release

data/PostInstall.txt

@@ -0,0 +1,2 @@
+No special steps
+

data/README.md

@@ -0,0 +1,53 @@
+# dataMetaParse
+
+DataMeta Parser commons: common rules and some reusable grammars
+
+References to this gem's:
+
+* [Source](https://github.com/eBayDataMeta/DataMeta-gems)
+
+## DESCRIPTION:
+
+See the [DataMeta Project](https://github.com/eBayDataMeta/DataMeta)
+
+## FEATURES/PROBLEMS:
+
+* This gem uses [treetop](http://treetop.rubyforge.org) for grammar processing which only works with
+[PEGs](http://en.wikipedia.org/wiki/Parsing_expression_grammar), same as [Antlr](http://www.antlr.org) and many other
+popular grammar processors. Hence, be careful with features that
+[PEGs](http://en.wikipedia.org/wiki/Parsing_expression_grammar) do not support,
+like [left recursion](http://en.wikipedia.org/wiki/Left_recursion).
+
+### DataMeta URI parsing
+
+This gem provides convenient class for URI parsing with DataMeta Specifics.
+
+The URI format is [typical](http://support.microsoft.com/kb/135975 "URL Format - MS Knowledge Base"):
+
+    protocol://user:password@server:port/path?query
+
+Out of which,
+
+* `protocol`: required, corresponds with DataMeta "platform", can be:
+    * `oracle` - for Oracle connections
+    * `mysql` - for MySQL connections
+* `user`: optional, the user name for authentication
+* `password`: password for the user, can be only used in conjunction with the `user`. Depending on a protocol,
+    can be either required or optional.
+* `server`: required, host name or IP address
+* `port`: optional, port number to connect to
+* `path`: optional, protocol specific, may refer either to a full path on the server's filesystem or a name of the database
+* `?query`: optional, regular format for the URL query, in `key=value` format separated by <tt>&</tt>, any special
+    characters encoded in the <tt>%xx</tt> format.
+
+## SYNOPSIS:
+
+* No command line runnables in this gem, it is a library only.
+
+## INSTALL:
+
+    gem install dataMetaParse
+
+## LICENSE:
+
+[Apache v 2.0](https://github.com/eBayDataMeta/DataMeta/blob/master/LICENSE.md)

data/Rakefile

@@ -0,0 +1,13 @@
+%w(yard rdoc/task rake/testtask fileutils ./lib/dataMetaParse).each{ |r| require r}
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+end
+
+desc 'Regen RDocs'
+task :default => :docs
+
+YARD::Rake::YardocTask.new('docs') {|r|
+    r.stats_options = ['--list-undoc']
+}
+

data/lib/dataMetaParse.rb

@@ -0,0 +1,58 @@
+$:.unshift(File.dirname(__FILE__)) unless $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+require 'treetop'
+=begin rdoc
+Grammar parsing commons for the dataMeta Project.
+
+For command line details either check the new method's source or the README.rdoc file, the usage section.
+=end
+module DataMetaParse
+    # Current version
+    VERSION = '1.0.0'
+
+=begin rdoc
+Parsing error, RuntimeError augmented with report feature
+
+=end
+    class Err < RuntimeError
+        attr_reader :source, :parser
+=begin rdoc
+Constructor, constructs also the error message passed to the super.
+
+@param [String] source the next have been parsed to get this error
+@param [Object] parser Treetop compiled parser whichever class it is. It may be +Treetop::Runtime::CompiledParser+
+=end
+        def initialize(source, parser)
+            @source, @parser = source, parser
+            parser.failure_reason =~ /^(Expected .+) after/m
+            reason = $1 || 'REASONLESS'
+            # replace newlines with <EOL> to make them stand out
+            super %Q<ERROR at index #{parser.index}
+#{reason.gsub("\n", '<EOL>')}:
+#{source.lines.to_a[parser.failure_line - 1]}
+#{'~' * (parser.failure_column - 1)}^
+>
+        end
+    end
+=begin rdoc
+Loads the base rules from +dataMetaParse/basic.treetop+
+=end
+    def loadBaseRulz
+        Treetop.load("#{File.dirname(__FILE__)}/dataMetaParse/basic")
+    end
+
+=begin rdoc
+Parse with error handling, convenience shortcut to the content of this method.
+
+@param [Object] parser Treetop compiled parser whichever class it is. It may be +Treetop::Runtime::CompiledParser+
+@param [String] source the data to parse with the given parser
+@return [Object] either the AST, likely as +Treetop::Runtime::SyntaxNode+ if the parsing was successful or {Err} if it was not
+    or +nil+ if there is no match. It's not very consistent of when you get an Err or when you get a +nil+, it's
+    not exact science. One way to get a +nil+ is to cause mismatch in the very first token.
+=end
+    def parse(parser, source)
+        parser.parse(source) || ( parser.failure_reason ? Err.new(source, parser) : nil)
+    end
+
+    module_function :loadBaseRulz, :parse
+end

data/lib/dataMetaParse/basic.treetop

@@ -0,0 +1,236 @@
+grammar DataMetaCommonsRoot
+
+# Some staple rules
+# w in the name means “whitespace”, e means End of Line, and capitalization means that the
+# capitalized part is required (contrary to being optional).
+
+
+    # Required whitespace
+    rule W
+       [\s]+
+    #  [ \t]+
+    end
+
+    rule notBlank
+      (!W .)
+    end
+
+    rule notBlanks
+      notBlank+
+    end
+
+    rule E
+       "\n"
+    end
+
+    # to the End Of Line inclusively
+    rule toE
+      tillE? E
+    end
+
+    # to the End Of Line exclusively
+    rule tillE
+      notE+
+    end
+
+    rule notE
+       (!E .)
+    end
+
+    # Optional whitespace
+    rule we
+      wE?
+    end
+
+    # Optional whitespace with required EOL
+    rule wE
+      ( ( W "\n" / w meshLineComment / w "\n" ) w meshLineComment? )+
+    end
+
+    # Optional whitespace
+    rule w
+      W?
+    end
+
+    # Comment used in scripting languages like Bash, Ruby etc.
+    rule meshLineComment
+      '#' (!"\n" .)* "\n"
+    end
+
+    # /* */ comment used in C, Java etc, multiline
+    rule slashStarComment
+      '/*'
+      (
+        !'*/'
+        (. / "\n")
+      )*
+      '*/'
+    end
+
+    # Uppercase A to Z
+    rule AZ
+      [A-Z]+
+    end
+
+    # Lowercase A to Z
+    rule az
+      [a-z]+
+    end
+
+    # Alphabetical
+    rule alpha
+      [a-zA-Z]+
+    end
+
+    # Decimal digit
+    rule digit
+      [0-9]
+    end
+
+    rule dot
+       '.'
+    end
+
+    rule plus
+       '+'
+    end
+
+    rule minus
+       '-'
+    end
+
+    rule sign
+      plus / minus
+    end
+
+    rule decIntNoSign
+        digit+
+    end
+
+    rule decIntSignable
+       sign? decIntNoSign
+    end
+
+    rule decFraction
+       dot decIntNoSign
+    end
+
+    rule signDotDecFrac
+       sign? dot decIntNoSign
+    end
+
+    rule decIntDotFrac
+      decIntSignable dot
+    end
+
+    rule fullDecFrac
+      decIntSignable decFraction
+    end
+
+    rule fixedDecimal
+       fullDecFrac /signDotDecFrac / decIntDotFrac
+    end
+
+    # Hexacedimal digit
+    rule hexDigit
+      [0-9A-Fa-f]
+    end
+
+    # Alphanumeric
+    rule alphaNum
+      [0-9A-Za-z]
+    end
+
+    # dataMeta "Word" character, can be a part of a indentifier name.
+    rule wordChar
+      [0-9A-Za-z_]
+    end
+
+    rule dmWord
+      wordChar+
+    end
+
+    # Class name, first uppercase, then any of the word components
+    rule className
+      AZ wordChar?
+    end
+
+    # Variable name, first
+    rule varName
+      [_a-z] wordChar? # allows single underscore as a var name.
+    end
+
+    # C-style comment to reuse everywhere
+    rule c_comment
+      '/*'
+      (
+        !'*/'
+        (. / "\n")
+      )*
+      '*/'
+    end
+
+    # C-Style whitespace
+    rule c_whitespace
+        c_comment / W
+    end
+
+    # C-Style End of Line comment
+    rule cEolComment
+      '//' (!"\n" .)* "\n"
+    end
+
+    rule string
+      '"' letters:( !'"' stringLetter )* '"' {
+        def fetch
+          letters.elements.map { |el| el.elements.last.fetch }.join
+        end
+      }
+    end
+
+    rule stringLetter
+      '\\' char:["ntr] {
+        def fetch
+          case char.text_value
+            when '"'; '"'
+            when 'n'; "\n"
+            when 'r'; 13.chr
+            when 't'; 9.chr
+            when '\\'; "\\"
+            else
+               raise ArgumentException, "Invalid string escape '#{char.text_value}'"
+          end
+        end
+      }
+      /
+      . {
+        def fetch
+          text_value
+        end
+      }
+    end
+
+    rule sn
+      sN?
+    end
+
+    # borrowed from: http://whitequark.org/blog/2011/09/08/treetop-typical-errors
+    # C End of line
+    rule cEol
+      ( ( S "\n" / s cEolComment / s "\n" ) s cEolComment? )+
+    end
+
+    # Scripting EOL, with mesh comment
+    rule sEol
+      ( ( S "\n" / s meshLineComment / s "\n" ) s meshLineComment? )+
+    end
+
+    rule s
+      S?
+    end
+
+    rule S
+      [ \t]+
+    end
+
+end

data/lib/dataMetaParse/uri.treetop

@@ -0,0 +1,229 @@
+# Borrowed from: https://github.com/juretta/uri-templates/blob/master/grammar/uri_template.treetop
+
+grammar UriTemplate
+
+  include DataMetaCommonsRoot
+
+  rule uri_template
+    uri_element more_elements:(uri_element)* {
+      def value(env={})
+        uri_element.value(env) << more_elements.elements.map{|el| el.value(env)}.join
+      end
+    }
+  end
+
+  rule uri_element
+    expansion / uri_part
+  end
+
+  rule expansion
+    '{'
+   c:(
+      var
+      /
+      operator
+   )
+   '}'
+    {
+      def value(env = {})
+        c.value(env)
+      end
+      }
+  end
+
+  rule uri_part
+    (unreserved / reserved / pct_encoded) {
+      def value(env = {})
+        text_value
+      end
+    }
+  end
+
+  rule arg
+    (reserved / unreserved / pct_encoded)*
+  end
+
+  rule op
+    (
+    'opt' {
+		# If each variable is undefined or an empty list then substitute the
+		# empty string, otherwise substitute the value of 'arg'.
+         def exec
+           lambda do |env, arg, vars|
+             ret = ''
+             vars.split(',').each do |var|
+               if env[var] && (env[var].respond_to?(:length) ? env[var].length > 0 : true)
+                 ret = "#{arg}"
+                 break
+               end
+             end
+             ret
+           end
+         end
+       }
+    /
+    'neg' {
+     # If all of the variables are un-defined or empty then substitute the
+     # value of arg, otherwise substitute the empty string.
+      def exec
+        lambda do |env, arg, vars|
+          ret = "#{arg}"
+          vars.split(',').each do |var|
+            if !env[var].to_s.blank?
+              ret = ""
+              break
+            end
+          end
+          ret
+        end
+      end
+    }
+    /
+    'prefix' {
+     # The prefix operator MUST only have one variable in its expansion.  If
+     # the variable is defined and non-empty then substitute the value of
+     # arg followed by the value of the variable, otherwise substitute the
+     # empty string.
+      def exec
+        lambda do |env, prefix, vars|
+          v = env[vars]
+          if vars =~ /([^=]+)=([^=]+)/
+            var, default = $1.dup, $2.dup
+            v = env[var]
+            v = default if v.to_s.blank?
+          end
+          !v.blank? ? "#{prefix}#{UriTemplate::Encoder.encode(v)}" : ""
+        end
+      end
+    }
+    /
+    'suffix' {
+     # The suffix operator MUST only have one variable in its expansion.  If
+     # the variable is defined and non-empty then substitute the value of
+     # the variable followed by the value of arg, otherwise substitute the
+     # empty string.
+      def exec
+        lambda do |env, append, vars|
+          v = env[vars]
+          if vars =~ /([^=]+)=([^=]+)/
+            var, default = $1.dup, $2.dup
+            v = env[var]
+            v = default if v.to_s.blank?
+          end
+          if v
+            val = UriTemplate::Encoder.encode(v)
+            !val.blank? ? "#{val}#{append}" : ""
+          else
+            ''
+          end
+        end
+      end
+    }
+    /
+    'join' {
+     # For each variable that is defined and non-empty create a keyvalue
+     # string that is the concatenation of the variable name, "=", and the
+     # variable value.  Concatenate more than one keyvalue string with
+     # intervening values of arg to create the substitution value.
+     def exec
+        lambda do |env, joinop, vars|
+          vars.split(',').map do |var|
+          v = env[var]
+          if var =~ /([^=]+)=([^=]+)/
+            var, default = $1.dup, $2.dup
+            v = env[var]
+            v = default if v.to_s.blank?
+          end
+          "#{var}=#{UriTemplate::Encoder.encode(v)}" if v
+        end.compact.join(joinop)
+      end
+     end
+   }
+    /
+    'list' {
+    # 	The listjoin operator MUST have only one variable in its expansion
+    # and that variable must be a list.  More than one variable is an
+    # error.  If the list is non-empty then substitute the concatenation of
+    # all the list members with intervening values of arg.  If the list is
+    # empty or the variable is undefined them substitute the empty string.
+     def exec
+       lambda do |env, joinop, vars|
+         return "" unless env[vars].respond_to? :each
+         env[vars].map do |v|
+           "#{UriTemplate::Encoder.encode(v)}" if v
+         end.compact.join(joinop)
+       end
+     end
+   }
+    )
+  end
+
+  rule vars
+    var ("," var)*
+  end
+
+  rule vardefault
+    (unreserved / pct_encoded)*
+  end
+
+  rule var
+    varname defaults:('=' vardefault)* {
+      def value(env={} )
+        return UriTemplate::Encoder.encode(env[name]) unless env[name].nil?
+        defaults.text_value.gsub(/=/, '')
+      end
+
+      def name
+        varname.text_value
+      end
+    }
+  end
+
+  rule operator
+    "-" op "|" arg "|" vars {
+      def value(env={})
+        op.exec.call(env, arg.text_value, vars.text_value) # if op.respond_to?(:exec)
+      end
+    }
+  end
+
+  rule varname
+    [a-zA-Z0-9] [a-zA-Z0-9_.-]*
+  end
+
+  rule alpha
+    [A-Za-z_]
+  end
+
+  rule alphanumeric
+    alpha / [0-9]
+  end
+
+  # see http://www.ietf.org/rfc/rfc3986.txt
+  rule unreserved
+    alphanumeric / "-" / "." / "_" / "~"
+  end
+
+  # see http://www.ietf.org/rfc/rfc3986.txt
+  rule pct_encoded
+    '%' hexdig hexdig
+  end
+
+  rule hexdig
+    [a-fA-F0-9]
+  end
+
+  # see http://www.ietf.org/rfc/rfc3986.txt
+  rule reserved
+    gen_delims / sub_delims
+  end
+
+  rule gen_delims
+    ":" / "/" / "?" / "#" / "[" / "]" / "@"
+  end
+
+  rule sub_delims
+    "!" / "$" / "&" / "'" / "(" / ")" / "*" / "+" / "," / ";" / "="
+  end
+
+end

data/lib/dataMetaParse/uriDataMeta.rb

@@ -0,0 +1,139 @@
+$:.unshift(File.dirname(__FILE__)) unless $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+require 'uri'
+require 'treetop'
+
+module DataMetaParse
+=begin rdoc
+DataMeta URI with all the parts.
+
+The user story:
+
+* DataMeta URIs are used in DataMeta Scripts to specify all aspects of a data set identity and location.
+* For physical access, a URI may be disassembled using this grammar and parser, the parts obtained so may be used
+  to access concrete physical resources.
+
+@!attribute [r] proto
+    @return [String] the protocol part, such as +http+, +ftp+, +socparc+ etc
+
+@!attribute [r] user
+    @return [String] the user id part of the URI, can be +nil+ and for some URIs may be in properties
+
+@!attribute [r] pwd
+    @return [String] the password part of the URI, can be +nil+ and for some URIs may be in properties
+
+@!attribute [r] host
+    @return [String] the host part of the URI
+
+@!attribute [r] port
+    @return [Fixnum] the port number specified in the URI, can be +nil+
+
+@!attribute [r] path
+    @return [String] for the +file+ protocol, path as specified, full or relative. For any other URI, the part between
+        the closing '/' after the +host:port+ part and the query part starting with '?'.
+        This means, for all other protocols except +file+, the path part will never have an initial slash.
+
+@!attribute [r] props
+    @return [Hash] hash of properties keyed by the property name and pointing to a value if any
+
+=end
+    class Uri
+        attr_reader :proto, :user, :pwd, :host, :port, :path, :props
+
+=begin rdoc
+Creates an instance of the object.
+
+@param [String] proto see the property {#proto}
+@param [String] user see the property {#user}
+@param [String] pwd see the property {#pwd}
+@param [String] host see the property {#host}
+@param [String] port see the property {#port}
+@param [String] path see the property {#path}
+@param [String] props see the property {#props}
+=end
+        def initialize(proto, user, pwd, host, port, path, props)
+            raise ArgumentError, 'Password specified but user not' if !user && pwd
+            raise ArgumentError, 'For file protocol, only path can be specified' if proto == 'file' && (
+                user || pwd || host || port || !props.empty?)
+
+            @proto, @user, @pwd, @host, @port, @path, @props = proto, user, pwd, host, port, path, props
+        end
+
+=begin rdoc
+Equality to the other
+=end
+        def ==(other)
+            @proto == other.proto && @user == other.user && @pwd == other.pwd && @host == other.host &&
+                    @port == other.port && @path == other.path && @props.eql?(other.props)
+        end
+
+=begin rdoc
+Same as the {#==}
+=end
+        def eql?(other); self == other end
+
+=begin rdoc
+Loads the grammar - has to be done only once per RVM start. Relies on loading the basics.
+=end
+        def self.loadRulz
+            Treetop.load(File.join(File.dirname(__FILE__), 'urlDataMeta'))
+        end
+
+=begin rdoc
+Instance to textual.
+=end
+        def to_s
+            if @proto == 'file'
+                "file://#{@path}"
+            else
+                result = ''
+                result << @proto << '://'
+                result << URI.encode_www_form_component(@user) if @user
+                result << ':' << URI.encode_www_form_component(@pwd) if @user && @pwd
+                result << '@' if @user
+                result << @host
+                result << ':' << @port.to_s if @port
+                result << '/' if @path || !@props.empty?
+                result << @path if @path
+
+                result << '?' << @props.keys.map { |k|
+                    v=@props[k]; v ? "#{k}=#{URI.encode_www_form_component(v)}" : "#{k}"
+                }.join('&') unless @props.empty?
+
+                result
+            end
+        end
+
+=begin rdoc
+Parses the source into the instance of the object.
+@param [String] source the source, the URI specification to parse into the instance of this class
+=end
+        def self.parse(source)
+            fileSignature = 'file://'
+            if source.start_with?(fileSignature)
+                Uri.new('file', nil, nil, nil, nil, source[fileSignature.length..-1], {})
+            else
+                parser = DataMetaUrlParser.new
+                ast = parser.parse(source)
+                return nil unless ast
+                proto = ast.proto.text_value
+                user = ast.user? ? ast.userPwd.user.text_value : nil
+                pwd = ast.pwd? ? URI.decode_www_form_component(ast.userPwd.password) : nil
+                host = ast.host.text_value
+                port = ast.port? ? ast.port.number : nil
+                path = ast.path? ? ast.uTail.path : nil
+                query = ast.query? ? ast.uTail.query : nil
+                props = {}
+                if query
+                    pairs = query.split('&')
+                    pairs.each { |pairString|
+                        key, val = pairString.split('=')
+                        props[key] = val ? URI.decode_www_form_component(val) : nil # this is caused by &paramA&paramB=b, in which case paramA will be nil
+                    }
+                end
+                Uri.new(proto, user, pwd, host, port, path, props)
+            end
+        end
+
+    end
+end

data/lib/dataMetaParse/urlDataMeta.treetop

@@ -0,0 +1,110 @@
+grammar DataMetaUrl
+
+# DataMeta URL definition, a part of a URL definition
+
+    include DataMetaCommonsRoot
+
+    rule dataMetaUri
+       proto:urlProtocol '://' userPwd:userSpec? host:hostName port:portSpec? uTail:urlTail? {
+           def user?
+              !userPwd.text_value.empty? && userPwd.name && !userPwd.name.empty?
+           end
+           def pwd?
+              !userPwd.text_value.empty? && userPwd.password && !userPwd.password.empty?
+           end
+           def port?
+              !port.text_value.empty?
+           end
+
+           def tail?
+              !uTail.text_value.empty?
+           end
+
+           def path?
+               !uTail.text_value.empty? && uTail.path
+           end
+           def path
+               uTail.path
+           end
+           def query?
+               !uTail.text_value.empty? && uTail.query
+           end
+           def query
+               uTail.query
+           end
+       }
+    end
+
+# Order is important, if you put "http" in front of the "https", https match will fail
+    rule urlProtocol
+       'https' / 'http' / 'ftp' / 'hdfs' / 'mysql' / 'oracle'
+    end
+
+    rule urlTail
+      '/' uPath:urlPath? uQuery:urlQuery? {
+         def path
+           uPath.text_value.empty? ? nil : uPath.text_value
+         end
+         def query
+            uQuery.text_value.empty? ? nil : uQuery.query
+         end
+      }
+    end
+
+    rule urlPath
+      (urlPathChar+)
+    end
+
+    rule urlPathChar
+      wordChar / '-' / '/' / '.'
+      #!'?' # - this causes infinite loop
+    end
+
+  rule urlQuery
+    '?' queryText:(notBlanks+) {
+       def query
+          queryText.text_value.empty? ? nil : queryText.text_value
+       end
+    }
+  end
+
+  rule hostChar
+     wordChar / '-' / '.'
+  end
+
+  rule hostName
+     hostChar+
+  end
+
+  rule portSpec
+    ':' portNumber:(digit+) {
+       def number # port number or nil if none, can not default it here because it depends on the protocol
+          text_value.empty? ? nil : portNumber.text_value.to_i
+       end
+    }
+  end
+
+  rule notAtSymbol
+    !'@' .
+  end
+
+  rule pwdSpec
+    ':' pwd:(notAtSymbol+) {
+       def empty?
+          text_value.empty?
+       end
+    }
+  end
+
+  rule userSpec
+    user:(wordChar+) pwdOpt:pwdSpec? '@' {
+       def name
+           user.text_value.empty? ? nil : user.text_value
+       end
+       def password
+          pwdOpt.text_value.empty? ? nil : pwdOpt.pwd.text_value
+       end
+    }
+  end
+
+end

data/test/numbers.treetop

@@ -0,0 +1,23 @@
+grammar TestNumbers
+
+    include DataMetaCommonsRoot
+    rule testNumbers
+        singleDecDigitNoSign:decIntNoSign W mulDecDigitNoSign:decIntNoSign
+        W signableIntNoSign:decIntSignable W signableIntPlus:decIntSignable W signableIntMinus:decIntSignable
+        W singleDigSingleDigFrac:fixedDecimal
+        W singDigDoubleDigFrac:fixedDecimal
+        W doubleDigSingleDigFrac:fixedDecimal
+        W doubleDigDotFrac:fixedDecimal
+        W dotDigitsFrac:fixedDecimal
+        W singleDigSingleDigFracPlus:fixedDecimal
+        W singDigDoubleDigFracPlus:fixedDecimal
+        W doubleDigSingleDigFracPlus:fixedDecimal
+        W doubleDigDotFracPlus:fixedDecimal
+        W dotDigitsFracPlus:fixedDecimal
+        W singleDigSingleDigFracMinus:fixedDecimal
+        W singDigDoubleDigFracMinus:fixedDecimal
+        W doubleDigSingleDigFracMinus:fixedDecimal
+        W doubleDigDotFracMinus:fixedDecimal
+        W dotDigitsFracMinus:fixedDecimal
+    end
+end

data/test/test_helper.rb

@@ -0,0 +1,6 @@
+## keep this underscore naming in the test subdir, it's easier to append files names to test
+%w(stringio test/unit logger).each { |r| require r }
+# this is expected to run from the project root, normally by the rake file
+require './lib/dataMetaParse'
+require './lib/dataMetaParse/uriDataMeta'
+require './test/utils'

data/test/test_numbers.rb

@@ -0,0 +1,52 @@
+# keep this underscore naming in the test subdir, it's easier to append files names to test
+require './test/utils'
+
+# Unit test cases for the DataMetaParse
+#  See for instance:
+#  - test_full
+# Assertions: http://ruby-doc.org/stdlib-1.9.3/libdoc/test/unit/rdoc/Test/Unit/Assertions.html
+class TestNumberParse < Test::Unit::TestCase
+
+    include DataMetaParseTestUtil
+    L.info "Loaded Base Rules: #{BASE_RULS}"
+    # Loads the grammars, creates a parser
+    def setup
+        numbers = Treetop.load('./test/numbers')
+        L.info "Loaded numbers: #{numbers.inspect}"
+
+        @parser = TestNumbersParser.new
+        L.info "#{@parser.inspect}"
+    end
+
+=begin rdoc
+Numbers parsing test
+=end
+    def test_numbers
+        ast = DataMetaParse.parse(@parser,
+                %q<1 123 4321 +4321 -4321 1.1 1.23 12.3 12. .12 +1.1 +1.23 +12.3 +12.  +.12 -1.1 -1.23 -12.3 -12. -.12>)
+        raise 'Numbers parse unsuccessful' unless ast
+        raise ast if ast.is_a?(DataMetaParse::Err)
+        L.info "AST:\n#{ast.inspect}"
+        assert_equal(1, ast.singleDecDigitNoSign.text_value.to_i)
+        assert_equal(123, ast.mulDecDigitNoSign.text_value.to_i)
+        assert_equal(4321, ast.signableIntNoSign.text_value.to_i)
+        assert_equal(4321, ast.signableIntPlus.text_value.to_i)
+        assert_equal(-4321, ast.signableIntMinus.text_value.to_i)
+        assert_equal(1.1, ast.singleDigSingleDigFrac.text_value.to_f)
+        assert_equal(1.23, ast.singDigDoubleDigFrac.text_value.to_f)
+        assert_equal(12.3, ast.doubleDigSingleDigFrac.text_value.to_f)
+        assert_equal(12.0, ast.doubleDigDotFrac.text_value.to_f)
+        assert_equal(0.12, ast.dotDigitsFrac.text_value.to_f)
+        assert_equal(1.1, ast.singleDigSingleDigFracPlus.text_value.to_f)
+        assert_equal(1.23, ast.singDigDoubleDigFracPlus.text_value.to_f)
+        assert_equal(12.3, ast.doubleDigSingleDigFracPlus.text_value.to_f)
+        assert_equal(12.0, ast.doubleDigDotFracPlus.text_value.to_f)
+        assert_equal(0.12, ast.dotDigitsFracPlus.text_value.to_f)
+        assert_equal(-1.1, ast.singleDigSingleDigFracMinus.text_value.to_f)
+        assert_equal(-1.23, ast.singDigDoubleDigFracMinus.text_value.to_f)
+        assert_equal(-12.3, ast.doubleDigSingleDigFracMinus.text_value.to_f)
+        assert_equal(-12.0, ast.doubleDigDotFracMinus.text_value.to_f)
+        assert_equal(-0.12, ast.dotDigitsFracMinus.text_value.to_f)
+    end
+
+end

data/test/test_uriParser.rb

@@ -0,0 +1,162 @@
+# keep this underscore naming in the test subdir, it's easier to append files names to test
+require './test/utils'
+
+# Unit test cases for the DataMetaParse
+#  See for instance:
+#  - test_full
+# Assertions: https://ruby-doc.org/stdlib-2.1.4/libdoc/test/unit/rdoc/Test/Unit/Assertions.html
+#noinspection RubyStringKeysInHashInspection
+class TestDataMetaParse < Test::Unit::TestCase
+
+    include DataMetaParseTestUtil
+
+    # Stub
+    def setup
+        DataMetaParse::Uri.loadRulz
+    end
+
+=begin rdoc
+Checks one URI specification, reports results
+=end
+    def assertUri(uriSpec, expected)
+        uri = DataMetaParse::Uri.parse(uriSpec)
+        if uri
+            L.info %Q<parsed "#{uriSpec}"; protocol: #{uri.proto}, user: #{uri.user}, pwd=#{uri.pwd}, host:#{uri.host}> +
+                   ", port=#{uri.port}, path:#{uri.path}, props: #{uri.props.inspect}\nre:#{uri}"
+        else
+            L.info %Q<parsed "#{uriSpec}", no match>
+        end
+
+        assert_equal(expected, uri) # assert_equal goes by object.eql?
+    end
+
+    # Tests General URI grammar
+    def test_GeneralUriGrammar
+#proto, user, pwd, host, port, path, props
+        assertUri(%q<http://www.domain.tld>, DataMetaParse::Uri.new('http', nil, nil, 'www.domain.tld', nil, nil, {}))
+        assertUri(%q<http://www.domain.tld/>, DataMetaParse::Uri.new('http', nil, nil, 'www.domain.tld', nil, nil, {}))
+
+        assertUri(%q<http://www.domain.tld:9090>,
+                  DataMetaParse::Uri.new('http', nil, nil, 'www.domain.tld', 9090, nil, {}))
+
+        assertUri(%q<http://www.domain.tld:9090/>,
+                  DataMetaParse::Uri.new('http', nil, nil, 'www.domain.tld', 9090, nil, {}))
+
+        assertUri(%q<http://joe_1@www.domain.tld>,
+                  DataMetaParse::Uri.new('http', 'joe_1', nil, 'www.domain.tld', nil, nil, {}))
+
+        assertUri(%q<https://joe_1@www.domain.tld>,
+                  DataMetaParse::Uri.new('https', 'joe_1', nil, 'www.domain.tld', nil, nil, {}))
+
+        assertUri(%q<http://joe_1:secret@www.domain.tld>,
+                  DataMetaParse::Uri.new('http', 'joe_1', 'secret', 'www.domain.tld', nil, nil, {}))
+
+        assertUri(%q<https://joe_1:secr%26et@www.domain.tld>,
+                  DataMetaParse::Uri.new('https', 'joe_1', 'secr&et', 'www.domain.tld', nil, nil, {}))
+
+        assertUri(%q<ftp://www.domain.tld/path/dir>,
+                  DataMetaParse::Uri.new('ftp', nil, nil, 'www.domain.tld', nil, 'path/dir', {}))
+
+        assertUri(%q<http://www.domain.tld/path/dir>,
+                  DataMetaParse::Uri.new('http', nil, nil, 'www.domain.tld', nil, 'path/dir', {}))
+
+        assertUri(%q<http://www.domain.tld/path/dir?qa=aVal&qb=bVal>,
+                  DataMetaParse::Uri.new('http', nil, nil, 'www.domain.tld', nil, 'path/dir',
+                                       {'qa' => 'aVal', 'qb' => 'bVal'}))
+
+        assertUri(%q<http://joe:secret@www.domain.tld/path/dir?qa=aVal&qb&qc=cVal>,
+                  DataMetaParse::Uri.new('http', 'joe', 'secret', 'www.domain.tld', nil, 'path/dir',
+                                       {'qa' => 'aVal', 'qb' => nil, 'qc' => 'cVal'}))
+
+        assertUri(%q<http://joe:secret@www.domain.tld/path/dir?qa=a%2FVal&qb=b%26Val&qc>,
+                  DataMetaParse::Uri.new('http', 'joe', 'secret', 'www.domain.tld', nil, 'path/dir',
+                                       {'qa' => 'a/Val', 'qb' => 'b&Val', 'qc' => nil}))
+
+        assertUri(%q<http://joe:secret@www.domain.tld:8443/path/dir?qa=a%2FVal&qb=b%26Val&qc>,
+                  DataMetaParse::Uri.new('http', 'joe', 'secret', 'www.domain.tld', 8443, 'path/dir',
+                                       {'qa' => 'a/Val', 'qb' => 'b&Val', 'qc' => nil}))
+
+        assertUri(%q<file:///dir/otherDir/file.ext>,
+                  DataMetaParse::Uri.new('file', nil, nil, nil, nil, '/dir/otherDir/file.ext', {}))
+
+        assertUri(%q<file://dir/otherDir/file.ext>,
+                  DataMetaParse::Uri.new('file', nil, nil, nil, nil, 'dir/otherDir/file.ext', {}))
+
+        assertUri(%q<hdfs://node-geo-ss.vip.acme.com/dir/otherDir/file.ext?cluster=hadoopCluster&format=seqFile&blkSize=128M>,
+                  DataMetaParse::Uri.new('hdfs', nil, nil, 'node-geo-ss.vip.acme.com', nil, 'dir/otherDir/file.ext',
+                                       {'cluster' => 'hadoopCluster', 'format' => 'seqFile', 'blkSize' => '128M'}))
+
+        assertUri(%q<hdfs://node-geo-ss.vip.acme.com:8020/dir/otherDir/file.ext?cluster=hadoopCluster&format=seqFile&blkSize=128M>,
+                  DataMetaParse::Uri.new('hdfs', nil, nil, 'node-geo-ss.vip.acme.com', 8020, 'dir/otherDir/file.ext',
+                                       {'cluster' => 'hadoopCluster', 'format' => 'seqFile', 'blkSize' => '128M'}))
+
+        assertUri(%q<mysql://DM_USER:DataMeta_PWD@db-host:3306/database?sql=select%20*%20from%20entity%20where%20id%20%3D%201>,
+                  DataMetaParse::Uri.new('mysql', 'DM_USER', 'DataMeta_PWD', 'db-host', 3306, 'database',
+                                       {'sql' => 'select * from entity where id = 1'}))
+
+        assertUri(%q<mysql://DM_USER:DataMeta_PWD@db-host/database?sql=select%20*%20from%20entity%20where%20id%20%3D%201>,
+                  DataMetaParse::Uri.new('mysql', 'DM_USER', 'DataMeta_PWD', 'db-host', nil, 'database',
+                                       {'sql' => 'select * from entity where id = 1'}))
+
+        assertUri(%q<mysql://DM_USER:DataMeta_PWD@db-host/database?sql=select%20*%20from%20entity%20where%20id%20%3D%201%20and%20c2%20%3D%20%27abc%27>,
+                  DataMetaParse::Uri.new('mysql', 'DM_USER', 'DataMeta_PWD', 'db-host', nil, 'database',
+                                       {'sql' => %q<select * from entity where id = 1 and c2 = 'abc'>}))
+
+        assertUri(%q<mysql://DM_USER:DataMeta_PWD@db-host/database?sql=select%20*%20from%20entity%20where%20id%20!%3D%201>,
+                  DataMetaParse::Uri.new('mysql', 'DM_USER', 'DataMeta_PWD', 'db-host', nil, 'database',
+                                       {'sql' => %q|select * from entity where id != 1|}))
+
+        assertUri(%q<mysql://DM_USER:DataMeta_PWD@db-host/database?sql=select%20*%20from%20entity%20where%20id%20%3C%3E%201>,
+                  DataMetaParse::Uri.new('mysql', 'DM_USER', 'DataMeta_PWD', 'db-host', nil, 'database',
+                                       {'sql' => %q|select * from entity where id <> 1|}))
+
+        assertUri(%q<mysql://DM_USER:DataMeta_PWD@db-host/database?sql=select%20*%20from%20entity%20where%20id%20%3E%3D%201>,
+                  DataMetaParse::Uri.new('mysql', 'DM_USER', 'DataMeta_PWD', 'db-host', nil, 'database',
+                                       {'sql' => %q|select * from entity where id >= 1|}))
+
+        assertUri(%q<mysql://DM_USER:DataMeta_PWD@db-host/database?sql=select%20*%20from%20database.entity%20where%20id%20%3E%3D%201>,
+                  DataMetaParse::Uri.new('mysql', 'DM_USER', 'DataMeta_PWD', 'db-host', nil, 'database',
+                                       {'sql' => %q|select * from database.entity where id >= 1|}))
+
+        assertUri(%q<mysql://DM_USER:DataMeta_PWD@db-host/database?sql=select%20*%20from%20entity%20where%20id%20in%20(1%2C2%2C3)>,
+                  DataMetaParse::Uri.new('mysql', 'DM_USER', 'DataMeta_PWD', 'db-host', nil, 'database',
+                                       {'sql' => %q|select * from entity where id in (1,2,3)|}))
+
+        assertUri(%q<oracle://DM_USER:DataMeta_PWD@db-host:3306/database?sql=select%20*%20from%20entity%20where%20id%20%3D%201>,
+                  DataMetaParse::Uri.new('oracle', 'DM_USER', 'DataMeta_PWD', 'db-host', 3306, 'database',
+                                       {'sql' => 'select * from entity where id = 1'}))
+
+        assertUri(%q<mysql://DM_USER:DataMeta_PWD@db-host/database?sql=select%20*%20from%20database.entity%20where%20id%20like%20%27%25a%25%27>,
+                  DataMetaParse::Uri.new('mysql', 'DM_USER', 'DataMeta_PWD', 'db-host', nil, 'database',
+                                       {'sql' => %q|select * from database.entity where id like '%a%'|}))
+    end
+
+=begin rdoc
+Check bad URLs, must raise errors:
+=end
+    def test_badUris
+        assert_raise(ArgumentError) {
+            DataMetaParse::Uri.new('file', 'blah', nil, nil, nil, 'dir/otherDir/file.ext', {})
+        }
+
+        assert_raise(ArgumentError) {
+            DataMetaParse::Uri.new('file', nil, 'blah', nil, nil, 'dir/otherDir/file.ext', {})
+        }
+
+        assert_raise(ArgumentError) {
+            DataMetaParse::Uri.new('file', nil, nil, 'blah', nil, 'dir/otherDir/file.ext', {})
+        }
+
+        assert_raise(ArgumentError) {
+            DataMetaParse::Uri.new('file', nil, nil, nil, 8080, 'dir/otherDir/file.ext', {})
+        }
+
+        assert_raise(ArgumentError) { # password but no user, no good:
+            DataMetaParse::Uri.new('http', nil, 'secret', 'www.domain.tld', nil, 'path/dir',
+                                 {'qa' => 'aVal', 'qb' => 'bVal'})
+        }
+        # Hadoop Good Practice: disallow hdfs specifications without namenode
+        assert_equal(nil, DataMetaParse::Uri.parse('hdfs:///dir/otherDir/file.ext'))
+
+    end
+end

data/test/utils.rb

@@ -0,0 +1,12 @@
+
+=begin rdoc
+Utilities for testing
+=end
+module DataMetaParseTestUtil
+    L = Logger.new('parseTests.log', 0, 10000)
+    L.level = Logger::DEBUG
+    L.datetime_format = '%Y-%m-%d %H:%M:%S'
+    # same as: DataMetaParse.loadBaseRulz
+    BASE_RULS = Treetop.load('./lib/dataMetaParse/basic')
+
+end

metadata

@@ -0,0 +1,83 @@
+--- !ruby/object:Gem::Specification
+name: dataMetaParse
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Michael Bergens
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-01-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: treetop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.8
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.8
+description: DataMeta Parser commons; common rules and some reusable grammars
+email: michael.bergens@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".yardopts"
+- History.md
+- PostInstall.txt
+- README.md
+- Rakefile
+- lib/dataMetaParse.rb
+- lib/dataMetaParse/basic.treetop
+- lib/dataMetaParse/uri.treetop
+- lib/dataMetaParse/uriDataMeta.rb
+- lib/dataMetaParse/urlDataMeta.treetop
+- test/numbers.treetop
+- test/test_helper.rb
+- test/test_numbers.rb
+- test/test_uriParser.rb
+- test/utils.rb
+homepage: https://github.com/eBayDataMeta
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements:
+- No specific requirements
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: DataMeta Parser commons
+test_files:
+- test/test_numbers.rb
+- test/test_uriParser.rb
+- test/numbers.treetop
+- test/test_helper.rb