cddl 0.10.3 → 0.12.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9e60b48882767d7e76926473b44c79369891e4203326d4e31f5693a3819f391d
-  data.tar.gz: 6ed2f45e7fa8468557e73c96a4dbfdd142d65fdebeeb362efeebec8007b7948f
+  metadata.gz: a4b5bfb31fd9b7fcf4cf66a1d0f2a76550c9682215c486d99a03997bab1ae30b
+  data.tar.gz: 97f11378fb18576f115ffb8cbd668fe9e14e33e91e4aac01d874c69e206f2fbd
 SHA512:
-  metadata.gz: 2124a4fc046c3dc03d007ca949e4f9191497b359fe19fafe307ed2227c4b5654d909815575504e452add310dfc8487bfc13db94794d4b6c06a7af0bb2701e55f
-  data.tar.gz: 8a223332d81282eb9ad651dad887579ceb7b44a01d05ccff6458e271ea999a3c5fad5f81c87a6a2ccf927307b3bb9c4e06075361b0f8fc8b02005926a3d9a5b7
+  metadata.gz: 623f3945c3fff7c04dd2a4e1c77879d585d347092c87b54d5ead6b8fb3ed3c5ab68a58ae67b7f3f936437c8bc9f92dea6f8597509ba8913fbc5e2e47f43d398a
+  data.tar.gz: a04d3f282b0d42182f63dd66988fece8fb0ab7ddb4e159b2f8213705503cff0a97b622a172096b881dc4fe70f16bdcc48a51fa035413bc55421bb5e15d1bcda3

data/bin/cddl

@@ -7,10 +7,13 @@ require 'json'
 
 Encoding.default_external = "UTF-8" # wake up, smell the coffee
 
+CDDL_VERSION = Gem.loaded_specs["cddl"].version rescue "unknown-version"
+
 EX_USAGE = 64
 EX_DATAERR = 65
 
 def usage
+  warn "cddl tool version #{CDDL_VERSION}"
   warn "Usage:"
   warn "#$0 spec.cddl generate [n]"
   warn "#$0 spec.cddl json-generate [n]"
@@ -19,6 +22,13 @@ def usage
   exit EX_USAGE
 end
 
+def utfify(s)
+  s.force_encoding(Encoding::UTF_8).scrub { |c|
+      warn "*** replaced invalid UTF-8 byte sequence #{c.inspect} by U+FFFD REPLACEMENT CHARACTER"
+      0xFFFD.chr(Encoding::UTF_8)
+  }
+end
+
 def read_arg(arg, remember_fn = true)
   if arg == "-"
     $fn = "(stdin)" if remember_fn
@@ -31,7 +41,7 @@ def read_arg(arg, remember_fn = true)
 end
 
 def parser
-  @parser ||= CDDL::Parser.new(read_arg(ARGV[0], false))
+  @parser ||= CDDL::Parser.new(utfify(read_arg(ARGV[0], false)))
 end
 
 def my_pp(v)
@@ -96,7 +106,12 @@ begin
       instance = if $sequence
                    CBOR.decode("\x9f".b << instance.b << "\xff".b)
                  else
-                   CBOR.decode(instance.b) rescue JSON.load(instance)
+                   begin
+                     CBOR.decode(instance.b)
+                   rescue => e
+                     warn e.inspect if verbose
+                     JSON.load(utfify(instance))
+                   end
                  end
       instance = instance.cbor_clone if $annotate && ENV["EXPERIMENTAL_ANNOTATE"]
       warn "#{fn}:" if verbose
@@ -104,6 +119,7 @@ begin
       # my_pp ann
       instance.cbor_add_annotations_from(ann) rescue nil
       my_diag(instance) if $annotate
+      warn "ann: #{!!ann}" if verbose
       unless ann
         retcode = 1
       end

data/cddl.gemspec

@@ -1,24 +1,25 @@
 spec = Gem::Specification.new do |s|
   s.name = 'cddl'
-  s.version = '0.10.3'
+  s.version = '0.12.14'
   s.summary = "CDDL generator and validator."
   s.description = %{A parser, generator, and validator for CDDL}
   s.add_dependency('cbor-diag')
-  s.add_dependency('abnc')
+  s.add_dependency('abnc', '~> 0.1.1')
   s.add_dependency('json_pure')
   s.add_dependency('abnftt')
-  s.add_dependency('regexp-examples') # , '1.1.0')
+  s.add_dependency('regexp-examples', '~> 1.5.1') # , '1.1.0')
   s.add_dependency('colorize')
   s.add_dependency('base32', '~> 0.3')
+  s.add_dependency('base45_lite', '~> 1.0')
+  s.add_dependency('scanf', '~> 1.0')
   s.files = `git ls-files`.split("\n").grep(/^[a-z]/)
   s.files = Dir['lib/**/*.rb'] + %w(cddl.gemspec) + Dir['data/**/*.abnf'] + Dir['data/**/*.cddl'] + Dir['test-data/**/*.cddl'] + Dir['test/**/*.rb']
   s.require_path = 'lib'
   s.executables = ['cddl']
-  s.default_executable = 'cddl'
-  s.required_ruby_version = '>= 2.0'
+  s.required_ruby_version = '>= 2.3'
   s.author = "Carsten Bormann"
   s.email = "cabo@tzi.org"
-  s.homepage = "http://github.com/cabo/cddl"
+  s.homepage = "https://www.rfc-editor.org/rfc/rfc8610#appendix-F"
   s.license = 'MIT'
 end
 

data/data/cddl.abnf

@@ -13,7 +13,8 @@ genericarg = "<" S type1 S *("," S type1 S ) ">"
 type = type1 S *("/" S type1 S)
 
 type1 = type2 [S (rangeop / annotator) S type2]
-     / "#" "6" ["." uint] "(" S type S ")" ; note no space!
+     / "#" "6" ["." headnumber] "(" S type S ")" ; note no space!
+     / "#" "7" ["." headnumber]             ; note no space!
      / "#" DIGIT ["." uint]                ; major/ai
      / "#"                                 ; any
      / "~" S typename [genericarg]
@@ -22,6 +23,8 @@ type1 = type2 [S (rangeop / annotator) S type2]
      / "&" S "(" S group S ")"
      / "&" S groupname [genericarg]
 
+headnumber = uint / ("<" type ">")
+
 type2 = value
       / typename [genericarg]
       / "(" type ")"
@@ -38,7 +41,7 @@ grpent = [occur S] [memberkey S] type optcom
        / [occur S] groupname [genericarg] optcom ; preempted by above
        / [occur S] "(" S group S ")" optcom
 
-memberkey = type1 S "=>"
+memberkey = type1 S ["^" S] "=>"
           / bareword S ":"
           / value S ":"
 
@@ -62,32 +65,45 @@ value = number
 int = ["-"] uint
 
 ; This is a float if it has fraction or exponent; int otherwise
-number = int ["." fraction] ["e" exponent ]
+number = hexfloat / (int ["." fraction] ["e" exponent ])
+hexfloat = ["-"] "0x" 1*HEXDIG ["." 1*HEXDIG] "p" exponent
 fraction = 1*DIGIT
-exponent = int
+exponent = ["+"/"-"] 1*DIGIT
 
 text = %x22 *SCHAR %x22
-SCHAR = %x20-21 / %x23-7E / SESC
-SESC = "\" %x20-7E
+SCHAR = %x20-21 / %x23-5B / %x5D-7E / NONASCII / SESC
+
+SESC = "\" ( %x22 / "/" / "\" /                 ; \" \/ \\
+             %x62 / %x66 / %x6E / %x72 / %x74 / ; \b \f \n \r \t
+             (%x75 hexchar) )                   ; \uXXXX
+
+hexchar = "{" (1*"0" [ hexscalar ] / hexscalar) "}" /
+          non-surrogate / (high-surrogate "\" %x75 low-surrogate)
+non-surrogate = ((DIGIT / "A"/"B"/"C" / "E"/"F") HEXDIG HEXDIG HEXDIG) /
+                ("D" %x30-37 HEXDIG HEXDIG)
+high-surrogate = "D" ("8"/"9"/"A"/"B") HEXDIG HEXDIG
+low-surrogate = "D" ("C"/"D"/"E"/"F") HEXDIG HEXDIG
+hexscalar = "10" HEXDIG HEXDIG HEXDIG HEXDIG / HEXDIG1 HEXDIG HEXDIG HEXDIG HEXDIG
+          / non-surrogate / HEXDIG [HEXDIG [HEXDIG]]
 
 bytes = [bsqual] %x27 *BCHAR %x27
-BCHAR = %x20-26 / %x28-5B / %x5D-7E / SESC / CRLF
-bsqual = %x68 ; "h"
-       / %x62.36.34 ; "b64"
+BCHAR = %x20-26 / %x28-5B / %x5D-7E / NONASCII / SESC / "\'" / CRLF
+bsqual = "h" / "b64"
 
 id = EALPHA *(*("-" / ".") (EALPHA / DIGIT))
 ALPHA = %x41-5A / %x61-7A
-EALPHA = %x41-5A / %x61-7A / "@" / "_" / "$"
+EALPHA = ALPHA / "@" / "_" / "$"
 DIGIT = %x30-39
 DIGIT1 = %x31-39
 HEXDIG = DIGIT / "A" / "B" / "C" / "D" / "E" / "F"
+HEXDIG1 = DIGIT1 / "A" / "B" / "C" / "D" / "E" / "F"
 BINDIG = %x30-31
 
 S = *WS
 WS = SP / NL
 SP = %x20
 NL = COMMENT / CRLF
-COMMENT = ";" *(SP / VCHAR) CRLF
-VCHAR = %x21-7E
+COMMENT = ";" *PCHAR CRLF
+PCHAR = %x20-7E / NONASCII
+NONASCII = %xA0-D7FF / %xE000-10FFFD
 CRLF = %x0A / %x0D.0A
-

data/lib/cddl.rb

@@ -2,6 +2,7 @@ require 'abnc'
 require 'pp'
 require 'pathname'
 require 'cbor-pure' unless defined?(CBOR::Tagged)
+require 'half'
 require 'cbor-deterministic'
 require 'regexp-examples'
 require 'abnftt'
@@ -9,6 +10,7 @@ require 'colorize'
 require 'base64'
 require 'base32'
 require 'base45_lite'
+require 'scanf'
 
 module CDDL
 
@@ -38,6 +40,7 @@ module CDDL
                    [{}, {}]
                  end
 
+    CDDL_UNUSED_OK = ENV["CDDL_UNUSED_OK"]
 
     class BackTrack < Exception; end
 
@@ -59,7 +62,7 @@ module CDDL
         if upto - presult < 100
           part2 = source_text[presult...upto]
         else
-          part2 = source_text[presult, 50] + "......." + source_text[upto-50, 50]
+          part2 = source_text[presult, 50] + "......." + (source_text[upto-50, 50] || "")
         end
         warn "*** Look for syntax problems around the #{
                "%%%".colorize(background: :light_yellow)} markers:\n#{
@@ -116,6 +119,16 @@ module CDDL
       r.compact
     end
 
+    def longest_initial_match(old, now)
+      md = /\A(.*)(.*)\n\1(.*)\z/.match([old,now].join("\n"))
+      md[1..3]
+    end
+
+    def diff_message(old, now)
+      c, l, r = longest_initial_match(old, now)
+      "-- common: >#{c}< -- old: >#{l}< -- new: >#{r}<"
+    end
+
     def cname(s)
       s.to_s.gsub(/-/, "_")
     end
@@ -202,7 +215,8 @@ module CDDL
           ids = g.children(:id).map(&:to_s)
           # puts ["ids", ids].inspect
           if b = @generics[n]
-            fail "Duplicate generics definition #{n} as #{rule_ast} (was #{b})"
+            fail "Duplicate generics definition #{n} #{
+              diff_message(strip_nodes(b[0]).inspect, strip_nodes(rule_ast).inspect)}"
           end
           @generics[n] = [rule_ast, ids]
         else
@@ -214,7 +228,7 @@ module CDDL
               if a == b
                 warn "*** Warning: Identical redefinition of #{n} as #{a}"
               else
-                fail "Duplicate rule definition #{n} as #{b} (was #{a})"
+                fail "Duplicate rule definition #{n} #{diff_message(b, a)}"
               end
             end
             @rules[n] = rule_ast
@@ -246,7 +260,7 @@ module CDDL
       r_process("used_in_cddl_prelude", @rules["used_in_cddl_prelude"])
       @rules.each do |n, r|
       #   r_process(n, r)     # debug only loop
-        warn "*** Unused rule #{n}" unless @stage1[n]
+        warn "*** Unused rule #{n}" unless @stage1[n] || CDDL_UNUSED_OK
       end
       if result[0] == :grpent
         warn "Group at top -- first rule must be a type!"
@@ -300,6 +314,12 @@ module CDDL
       abnfb: Encoding::BINARY
     }
 
+    def myfrand(exp)
+      exprange = 1 << exp
+      expoffset= exprange >> 1
+      (rand(2)*2-1) * rand() * 2.0**(rand(exprange)-expoffset)
+    end
+
     def generate
       @recursion = 0
       generate1(rules)
@@ -422,9 +442,17 @@ module CDDL
         when 3
           gen_word
         when 6
-          CBOR::Tagged.new(where[2], generate1(where[3]))
+          tn = Integer === where[2] ? where[2] : generate1(where[2])
+          unless Integer === tn && tn >= 0
+            fail "Can't generate a tag number out of #{where[2]}"
+          end
+          CBOR::Tagged.new(tn, generate1(where[3]))
         when 7
-          case where[2]
+          w2 = where[2]
+          if Array === w2
+            w2 = generate1(w2)
+          end
+          case w2
           when nil
             Math::PI
           when 20
@@ -433,10 +461,22 @@ module CDDL
             true
           when 22
             nil
-          when 23
-            :undefined
-          when 25, 26, 27
-            rand()
+          when 0..19, 23, 32..255
+            CBOR::Simple.new(w2)
+          when 25
+            while !(hs = Half.encode_from_single_bytes([myfrand(5)].pack("g")))
+            end
+            Half.decode(hs)
+          when 26
+            while (a = [myfrand(8)].pack("g").unpack("g").first).to_cbor.size != 5
+            end
+            a
+          when 27
+            while (a = myfrand(11)).to_cbor.size != 9
+            end
+            a
+          else
+            fail "Can't generate prim #7.#{where[2].inspect}" # XXX retry array generate
           end
         else
           fail "Can't generate prim #{where[1]}"
@@ -551,8 +591,7 @@ module CDDL
               warn "HUH gen #{where.inspect} #{try.inspect}" unless try.nil?
             end
           end
-          32.times do
-            content = generate1(target)
+          try_generate(target) do |content|
             if validate1(content, where)
               return content
             end
@@ -598,29 +637,35 @@ module CDDL
           end
           content = JSON::dump(generate1(control))
           content
-        when :decimal
+        when :decimal, :base10
+          if conop == :decimal
+            unless @decimal_warned
+              warn "** .decimal control operator is now called .base10"
+              @decimal_warned = true
+            end
+          end
           unless target == [:prim, 3]
             fail "Don't know yet how to generate #{where}"
           end
           content = Integer(generate1(control)).to_s
           content
         when :join
-          content = generate1(control)
-          unless Array === content && 
-                 content.all? {|x| String === x &&
-                               ((target == [:prim, 2] && x.encoding == Encoding::BINARY) ||
-                                (target == [:prim, 3] && x.encoding != Encoding::BINARY))}
-            fail "Don't know yet how to generate #{where}"
+          try_generate(control) do |content|
+            if Array === content &&
+               content.all? {|x| String === x} &&
+               Set[content.map {|x| x.encoding}].size == 1
+              content = content.join
+              if validate1(content, target)
+                return content
+              end
+            end
           end
-          content = content.join
-          content
+          fail "Don't know yet how to generate #{where}"
         when :b64u, :"b64u-sloppy", :b64c, :"b64c-sloppy",
              :b45, :b32, :h32, :hex, :hexlc, :hexuc
-          content = generate1(control)
-          unless String === content
-            fail "Don't know yet how to generate #{where}"
-          end
-          content = case conop
+          try_generate(control) do |content|
+            if String === content
+              content = case conop
                     when :b64u, :"b64u-sloppy"
                       Base64.urlsafe_encode64(content, padding: false)
                     when :b64c, :"b64c-sloppy"
@@ -639,10 +684,33 @@ module CDDL
                       content.unpack("H*")[0]
                     else fail "Cannot happen"
                     end
-          content
+              if validate1(content, target)
+                return content
+              end
+            end
+          end
+          fail "Not smart enough to generate #{where}"
+        when :printf
+          try_generate(control) do |content|
+            if Array === content && content.size >= 1
+              fmt, *data = content
+              if String === fmt
+                begin
+                  content = fmt % data
+                  if validate1(content, target)
+                    return content
+                  end
+                rescue ArgumentError => e
+                  # be verbose about mismatches here
+                  @last_message << "\n** #{fmt.inspect} ArgumentError #{e}"
+                end
+              end
+            end
+          end
+          fail "Not smart enough to generate #{where}#{@last_message}"
         when :within, :and
-          32.times do
-            content = generate1(target)
+          try_generate(target) do |content|
+            $features ||= Hash.new {|h, k| h[k] = {}}
             if validate1(content, control)
               return content
             elsif conop == :within
@@ -670,22 +738,41 @@ module CDDL
       end
     end
 
+    def try_generate(target)
+      32.times do
+        content = generate1(target)
+        yield content           # should return if success
+      end
+    end
+
     VALUE_TYPE = {text: String, bytes: String, int: Integer, float: Float}
     SIMPLE_VALUE = {
       [:prim, 7, 20] => [true, false, :bool],
       [:prim, 7, 21] => [true, true, :bool],
       [:prim, 7, 22] => [true, nil, :nil],
-      [:prim, 7, 23] => [true, :undefined, :undefined],
     }
+    SIMPLE_VALUE_SIMPLE = Set[23] + (0..19) + (32..255)
 
     def extract_value(t)        # []
       if vt = VALUE_TYPE[t[0]]
         [true, t[1], vt]
-      elsif v = SIMPLE_VALUE[t]
-        v
+      elsif t[0] == :prim && t[1] == 7
+        case t2 = t[2]
+        when Integer
+        when Array
+          a, b, c = extract_value(t2)
+          if a && c == Integer
+            t2 = b
+          end
+        end
+        if v = SIMPLE_VALUE[[:prim, 7, t2]]
+          v
+        elsif SIMPLE_VALUE_SIMPLE === t2
+          [true, CBOR::Simple.new(t2), :simple]
+        end
       elsif t[0] == :anno
         _, conop, target, control = t
-        # warn ["EXV0", conop, target, control].inspect
+        warn ["EXV0", conop, target, control].inspect   if ENV["CDDL_TRACE"]
         if conop == :cat || conop == :plus || conop == :det
           ok1, v1, vt1 = extract_value(target)
           ok2, v2, vt2 = extract_value(control)
@@ -717,6 +804,19 @@ module CDDL
        }]
     end
 
+
+    def extract_arg0(t)
+      return [false] unless t[0] == :array
+      [true,
+       (el = t[1]
+        return [false] unless el[0..3] == [:member, 1, 1, nil]
+        ok, v, vt = extract_value(el[4])
+        return [false] unless ok
+        [v, vt]
+       ),
+       *t[2..-1]]
+    end
+
     def extract_feature(control, d)
       ok, v, vt = extract_value(control)
       if ok
@@ -905,6 +1005,19 @@ module CDDL
       }
     end
 
+    def validate1a_ignore_encoding(d, where)
+      if String === d
+        validate1a(d.force_encoding(Encoding::BINARY), where) or (
+          text = d.force_encoding(Encoding::UTF_8).scrub {
+            fail "can't use bytes as text"
+          } rescue :NOT_A_TEXT_STRING
+          validate1a(text, where)
+        )
+      else
+        validate1a(d, where)
+      end
+    end
+
     def validate1a(d, where)
       if ann = validate1(d, where)
         here = [d, where]
@@ -917,6 +1030,8 @@ module CDDL
     end
 
     OPERATORS = {lt: :<, le: :<=, gt: :>, ge: :>=, eq: :==, ne: :!=}
+    OPERATORS_MATCH = {eq: true, ne: false}
+    FLOAT_AI_FROM_SIZE = {3 => 25, 5 => 26, 9 => 27}
 
     def validate1(d, where)
       if ENV["CDDL_TRACE"]
@@ -927,6 +1042,9 @@ module CDDL
       @last_data = d
       @last_rule = where
       ann = nil
+      if CBOR::Tagged === d && d.tag === 888 && ENV["CDDL_888"]
+        return []
+      end
       case where[0]
       when :type1
         if where[1..-1].any? {|r| ann = validate1a(d, r)}
@@ -963,7 +1081,7 @@ module CDDL
         if conop == :cat || conop == :plus || conop == :det
           ok1, v1, vt1 = extract_value(target)
           ok2, v2, vt2 = extract_value(control)
-          # warn ["ANNO0", ok1, v1, vt1, ok2, v2, vt2, d].inspect
+          warn ["ANNO0", ok1, v1, vt1, ok2, v2, vt2, d].inspect  if ENV["CDDL_TRACE"]
           if ok1 && ok2
             v2 = Integer(v2) if vt1 == Integer
             if conop == :det
@@ -972,6 +1090,30 @@ module CDDL
             end
             # warn ["ANNO", ok1, v1, vt1, ok2, v2, vt2, d].inspect
             [] if d == v1 + v2  # XXX Focus ArgumentError
+          elsif conop == :cat && String === d
+            warn ["CAT-L", ok1, v1, vt1, ok2, v2, vt2, d].inspect  if ENV["CDDL_TRACE"]
+            if ok1 && vt1 == String
+              # d and lhs (v1) need to agree in encoding
+              if d.encoding == v1.encoding
+                if d[0...v1.length] == v1
+                  d2 = d[v1.length..-1]
+                  warn ["CAT-L1", d2, d2.encoding, control].inspect  if ENV["CDDL_TRACE"]
+                  validate1a_ignore_encoding(d2, control)
+                end
+              end
+            elsif ok2 && vt2 == String
+            warn ["CAT-R", ok1, v1, vt1, ok2, v2, vt2, d].inspect  if ENV["CDDL_TRACE"]
+              if d[-v2.length..-1] == v2
+                d1 = d[0...-v2.length]
+                validate1a(d1, control)
+              end
+            end
+          elsif conop == :plus && Integer === d
+            if ok1 && vt1 == Integer
+              validate1a(d - Integer(v1), control)
+            elsif ok2 && vt2 == Integer
+              validate1a(d - Integer(v2), target)
+            end
           end
         elsif ann = validate1a(d, target)
           case conop
@@ -1022,6 +1164,11 @@ module CDDL
             ok, v, _vt = extract_value(control)
             if ok
               ann if d.send(op, v) rescue nil # XXX Focus ArgumentError
+            else
+              needs_to_match = OPERATORS_MATCH[conop]
+              unless needs_to_match.nil?
+                ann if !!validate1(d, control) == needs_to_match
+              end
             end
           when :regexp
             ann if (
@@ -1074,21 +1221,89 @@ module CDDL
             end
           when :json
             ann if validate1((JSON::load(d) rescue :BAD_JSON), control)
-          when :decimal
+          when :decimal, :base10
+            if conop == :decimal
+              unless @decimal_warned
+                warn "** .decimal control operator is now called .base10"
+                @decimal_warned = true
+              end
+            end
+
             ann if (
                 if /\A0|-?[1-9][0-9]*\z/ === d
                   validate1((d.to_i rescue :BAD_JSON), control)
                 end
               )
           when :join
-            ok, *v = extract_array(control)
-            # warn "@@@JJJ@@@ #{ok.inspect} #{v.inspect}"
-            if ok
-              expected = v.map {|ve|
-                fail "Not a string in #{where}" unless String === ve[0]
-                ve[0]
-              }.join
-              ann if d == expected
+            t = control
+            v = if t[0] == :array
+                  t[1..-1].map { |el|
+                    if el[0..2] == [:member, 1, 1]
+                      ok, v, vt = extract_value(el[4])
+                      if ok
+                        [true, v, vt]
+                      else
+                        [false, el[4]]
+                      end
+                    end
+                  }
+                end
+            warn "@@@JOIN@@@ #{v.inspect}" if ENV["CDDL_TRACE"]
+            ok = true
+            if v
+              ix = 0
+              rest = d.dup
+              while ix < v.length
+                if left = v[ix]
+                  if left[0]    # match constant value first
+                    fail "Not a string for #{left.inspect} in #{where}" unless String == left[2]
+                    want = left[1]
+                    have = rest[0...left[1].length]
+                    if want == have
+                      rest[0...left[1].length] = ''
+                      ix += 1
+                      next
+                    else
+                      fail ".join: want #{want.inspect}, have #{have.inspect}"
+                    end
+                  else
+                    ix += 1
+                    if ix == v.length # match remaining
+                      warn "@@@JOIN ok in #{ok.inspect} rest #{rest.inspect}"  if ENV["CDDL_TRACE"]
+                      ok &&= validate1(rest, left[1])
+                      warn "@@@JOIN ok out #{ok.inspect}"  if ENV["CDDL_TRACE"]
+                      # more diag
+                      rest = ''
+                      next
+                    else
+                      if mid = v[ix]
+                        if mid[0] # have constant value to split over
+                          fail "Not a string for #{mid} in #{where}" unless String == mid[2]
+                          rest1, rest2 = rest.split(mid[1], 2)
+                          if rest2
+                            warn "@@@JOIN ok in #{ok.inspect} rest1 #{rest1.inspect}"   if ENV["CDDL_TRACE"]
+                            ok &&= validate1(rest1, left[1])
+                            warn "@@@JOIN ok out #{ok.inspect}"   if ENV["CDDL_TRACE"]
+                            rest = rest2
+                            ix += 1
+                            next
+                          else
+                            fail "Can't find #{mid[1].inspect} in #{rest.inspect}"
+                          end
+                        else
+                          fail "Cannot validate consecutive non-constant members of .join in #{where}"
+                        end
+                      else
+                        fail "Cannot handle .join over #{t[ix+1]} in #{where}"
+                      end
+                    end
+                  end
+                else
+                  fail "Cannot handle .join over #{t[ix+1]} in #{where}"
+                end
+              end
+              fail "Can't match #{rest.inspect} for .join in #{where}" if rest != ''
+              ann if ok
             else
               fail "Don't know yet how to validate against #{where}"
             end
@@ -1097,7 +1312,7 @@ module CDDL
                 String === d && (
                 decoded = case conop
                     when :b64u
-                      /=/ !~ d &&
+                      /[+\/=]/ !~ d &&
                       Base64.urlsafe_decode64(d)
                     when :"b64u-sloppy"
                       /[-_=]/ !~ d &&
@@ -1126,6 +1341,21 @@ module CDDL
                     end
                 ) && validate1(decoded.b, control)
               )
+          when :printf
+            ann if String === d && (
+                     ok, fmt, *v = extract_arg0(control)
+                     if ok && String == fmt[1]
+                       fmt = fmt[0]
+                       # warn "** ok #{ok.inspect} fmt #{fmt.inspect} v #{v.inspect}"
+                       decoded = d.scanf(fmt) # this is a bit too lenient, so let's do:
+                       encode_again = fmt % decoded
+                       if encode_again != d
+                         warn "** fmt #{fmt.inspect} d #{d.inspect} decoded #{decoded.inspect} encode_again #{encode_again.inspect}"
+                       else
+                         validate1(decoded, [:array, *v])
+                       end
+                     end
+                   )
           when :within
             if validate1(d, control)
               ann
@@ -1161,20 +1391,35 @@ module CDDL
             end
             d = CBOR::Tagged.new(t, d == 0 ? "".b : d.digits(256).reverse!.pack("C*"))
           end
-          CBOR::Tagged === d && d.tag == where[2] && validate1a(d.data, where[3])
+          CBOR::Tagged === d && (
+            Integer === where[2] ? d.tag == where[2] : validate1a(d.tag, where[2])
+          ) && validate1a(d.data, where[3])
         when 7
           t, v = extract_value(where)
           if t
             v.eql? d
           else
-            case where[2]
+            case w2 = where[2]
             when nil
-              # XXX
-              fail
+              Float === d || CBOR::Simple === d || [false, true, nil].include?(d)
+            when Array
+              headnum = case d
+                        when Float
+                          FLOAT_AI_FROM_SIZE[d.to_cbor.size]
+                        when false
+                          20
+                        when true
+                          21
+                        when nil
+                          22
+                        when CBOR::Simple
+                          d.value
+                        end
+              validate1a(headnum, w2)
             when 25, 26, 27
-              Float === d
+              Float === d && FLOAT_AI_FROM_SIZE[d.to_cbor.size] == w2
             else
-              fail
+              fail [:val7, d, where].inspect
             end
           end
         else
@@ -1224,6 +1469,10 @@ module CDDL
                                else
                                  [:type1]
                                end
+          elsif s = ENV["CDDL_INVENT"]
+            s = "_" if s == ""
+            r = [:type1, [:text, "#{s}-#{name}"]]
+            return r
           end
         end
         if r
@@ -1269,6 +1518,18 @@ module CDDL
                      end
     end
 
+    STRING_ESCAPES = {
+      "\"" => "\"",
+      "/" => "/",
+      "\\" => "\\",
+      "'" => "'",
+      "b" => "\b",
+      "f" => "\f",
+      "n" => "\n",
+      "r" => "\r",
+      "t" => "\t",
+    }
+
     def value(n)
       # cheat:
       # warn n
@@ -1278,7 +1539,20 @@ module CDDL
         if parts[-1] != ""
           warn "*** Problem decoding byte string #{s.inspect}"
         end
-        bsval = parts[0...-1].join("'").gsub(/\\(.)/){$1}
+        bsval = parts[0...-1].join("'").gsub(/\\u([Dd][89AaBb][0-9a-zA-Z]{2})\\u([Dd][CcDdEeFf][0-9a-zA-Z]{2})|\\u([0-9a-zA-Z]{4})|\\u\{([0-9a-zA-Z]+)\}|\r?(\n)|\\([^u])/) {
+          if hex = $3 || $4
+            hex.to_i(16).chr(Encoding::UTF_8)
+          elsif lf = $5
+            lf
+          elsif escaped = $6
+            STRING_ESCAPES[$6] or (
+              fail "Invalid String Escape #{escaped.inspect} in #{s.inspect}"
+            )
+          else
+            ((($1.to_i(16) & 0x3ff) << 10) +
+             ($2.to_i(16) & 0x3ff) + 0x10000).chr(Encoding::UTF_8)
+          end
+        }
         [:bytes,
          case bsqual
          when ""
@@ -1292,7 +1566,21 @@ module CDDL
          end
         ]
       else
-        val = eval(n.to_s)
+        if s[-1] == '"'
+          s.gsub!(/\\u([Dd][89AaBb][0-9a-zA-Z]{2})\\u([Dd][CcDdEeFf][0-9a-zA-Z]{2})|\\([^u]|u[0-9a-zA-Z]{4})/) {
+            if $3
+              "\\#$3" # skip this, use eval parser
+            else
+              ((($1.to_i(16) & 0x3ff) << 10) +
+               ($2.to_i(16) & 0x3ff) + 0x10000).chr(Encoding::UTF_8)
+            end
+          }
+        end
+        if s =~ /p.*\d\z/       # hexfloat
+          val = s.scanf("%a").first
+        else
+          val = eval(s)
+        end
         # warn val
         case val
          when Integer; [:int, val]
@@ -1485,9 +1773,10 @@ module CDDL
                              :within, :and,
                              :default, :lt, :le, :gt, :ge, :eq, :ne,
                              :feature, :abnf, :abnfb, :det, :cat, :plus,
-                             :json, :decimal, :join,
+                             :json, :decimal, :base10, :join,
                              :b64u, :"b64u-sloppy", :b64c, :"b64c-sloppy",
                              :b45, :b32, :h32, :hex, :hexlc, :hexuc,
+                             :printf,
                             ]
 
     def type1(n, canbegroup = false)
@@ -1520,6 +1809,13 @@ module CDDL
         when /\A#(\d+)/
           maj = $1.to_i
           s = [:prim, maj, *n.children(:uint).map(&:to_s).map(&:to_i)]
+          if tn = n.headnumber
+            if ui = tn.uint
+              s << ui.to_s.to_i
+            elsif tt = tn.type
+              s << type(tt)
+            end
+          end
           if tagged_type = n.type
               s << type(tagged_type)
           end

data/test-data/b64u-strict.cddl

@@ -0,0 +1,12 @@
+a = text .b64u bytes
+
+;;gp 10
+
+;;vp "AAAA"
+;;vp "AAA-"
+;;vp "AAA_"
+; fail with 0.11.5:
+;;vp "AAA+"
+;;vp "AAA/"
+;;vp "AAA="
+;;vp "+kd8"

data/test-data/cat-re.cddl

@@ -1,2 +1,4 @@
 foo = text .regexp myregexp
 myregexp = "A" .cat "B"
+
+;;gp 10

data/test-data/catb64.cddl

@@ -0,0 +1,8 @@
+a1 = '' .cat a2
+a2 = text .b64u ain
+; a = "e30" .b64u ain
+ain = '{}'
+
+;;gp 3
+;;vp 'e30'
+;;vp "e30"

data/test-data/decimal-warn.cddl

@@ -0,0 +1 @@
+s = [text .decimal 4711, text .base10 4712]

data/test-data/dotplus1.cddl

@@ -0,0 +1,3 @@
+a = b<13..15>
+b<T> = #7.<32 .plus T>
+; cddl dotplus1.cddl gp 10

data/test-data/dotplus2.cddl

@@ -0,0 +1,3 @@
+a = d<17..19>
+d<T> = #7.<T .plus 32>
+; cddl dotplus.cddl gp 10

data/test-data/dotplus3.cddl

@@ -0,0 +1,3 @@
+a = d<17..19>
+d<T> = 32.5 .plus T
+; cddl dotplus4.cddl gp 10

data/test-data/dotplus4.cddl

@@ -0,0 +1,3 @@
+a = d<17..19>
+d<T> = T .plus 32.5
+; cddl dotplus4.cddl gp 10

data/test-data/enum.cddl

@@ -0,0 +1,14 @@
+              terminal-color = &basecolors
+              basecolors = (
+                black: 0,  red: 1,  green: 2,  yellow: 3,
+                blue: 4,  magenta: 5,  cyan: 6,  white: 7,
+                reserved: (8..255) .feature "extension"
+              )
+              ; extended-color = &(
+              ;   basecolors,
+              ;   orange: 8,  pink: 9,  purple: 10,  brown: 11,
+              ; )
+
+;;gp 10
+;;vp 42
+;;vp 433

data/test-data/enum1.cddl

@@ -0,0 +1,12 @@
+terminal-color = $terminal-color / text .feature "extension"
+$terminal-color /= &basecolors
+basecolors = (
+               black: "black",  red: "red",  green: "green", 
+             )
+
+pastelcolors = ( pink: "pink", mauve: "mauve", peach: "peach", lavender: "lavender" )
+$terminal-color /= &pastelcolors
+
+;;gp 10
+;;vp "florp"
+;;vp "lavender"

data/test-data/json1.cddl

@@ -0,0 +1,15 @@
+; JWT-JWS = text .join ([
+;              b64u<jwt-headers>, ".",
+;              b64u<jwt-payload>, ".",
+;              b64u<jwt-signature>])
+; ; a = jwt-headers
+; ; a = b64u<"abc"> ; fails
+; ; a = b64u<'abc'> ; succeeds
+; b64u<B> = text .b64u B
+jwt-headers = '' .cat jwt-headers1
+jwt-headers1 = text .json jwt-headermap
+jwt-headermap = { * text => any } ; simplified
+jwt-payload = bytes
+jwt-signature = bytes
+
+;;gp 10

data/test-data/jwt.cddl

@@ -0,0 +1,15 @@
+JWT-JWS = text .join ([
+             b64u<jwt-headers>, ".",
+             b64u<jwt-payload>, ".",
+             b64u<jwt-signature>])
+; a = jwt-headers
+; a = b64u<"abc"> ; fails
+; a = b64u<'abc'> ; succeeds
+b64u<B> = text .b64u B
+jwt-headers = '' .cat jwt-headers1
+jwt-headers1 = text .json jwt-headermap
+jwt-headermap = { * text => any } ; simplified
+jwt-payload = bytes
+jwt-signature = bytes
+
+;;gp 10

data/test-data/prim.cddl

@@ -0,0 +1,7 @@
+start = [a, e, b, c, d]
+a = false
+e = undefined
+b = float32
+; c = #7.32
+c = #6.<12>(3)
+d = #6.13(4)

data/test-data/printf.cddl

@@ -0,0 +1,2 @@
+a = text .printf (["%8d %06d", ~arr])
+arr = [uint, uint]

data/test-data/printf0.cddl

@@ -0,0 +1 @@
+my_label = text .printf (["0x%x: %a", 4711, 81.5])

data/test-data/printf1.cddl

@@ -0,0 +1,2 @@
+my_alg_sha = hexlabel<4>
+hexlabel<K> = text .printf (["0x%04x", K])

data/test-data/printf2.cddl

@@ -0,0 +1,2 @@
+my_alg_sha = hexlabel<int>
+hexlabel<K> = text .printf (["0x%04x", K])

data/test-data/printf3.cddl

@@ -0,0 +1,2 @@
+any_alg = hexlabel<1..20>
+hexlabel<K> = text .printf (["0x%04x", K])

data/test-data/valemb.cddl

@@ -0,0 +1,16 @@
+cwt = [
+  bstr .cbor protected-map,
+  unprotected-map,
+  bstr .cbor payload-map,
+  bstr ; signature
+]
+
+protected-map = {
+   &(alg: 1) => int, 
+   ? &(typ: 16) => text,
+   * key => any   
+}
+
+unprotected-map = ()
+key = "key"
+payload-map = { payload: "map"}

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: cddl
 version: !ruby/object:Gem::Version
-  version: 0.10.3
+  version: 0.12.14
 platform: ruby
 authors:
 - Carsten Bormann
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-03-09 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: cbor-diag
@@ -28,16 +27,16 @@ dependencies:
   name: abnc
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.1.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.1.1
 - !ruby/object:Gem::Dependency
   name: json_pure
   requirement: !ruby/object:Gem::Requirement
@@ -70,16 +69,16 @@ dependencies:
   name: regexp-examples
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.5.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.5.1
 - !ruby/object:Gem::Dependency
   name: colorize
   requirement: !ruby/object:Gem::Requirement
@@ -108,6 +107,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: base45_lite
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: scanf
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 description: A parser, generator, and validator for CDDL
 email: cabo@tzi.org
 executables:
@@ -119,7 +146,6 @@ files:
 - cddl.gemspec
 - data/cddl.abnf
 - data/prelude.cddl
-- lib/base45_lite.rb
 - lib/cbor-pp-play.rb
 - lib/cbor-pp.rb
 - lib/cddl.rb
@@ -136,6 +162,7 @@ files:
 - test-data/b.cddl
 - test-data/b64c-sloppy.cddl
 - test-data/b64c.cddl
+- test-data/b64u-strict.cddl
 - test-data/b64u.cddl
 - test-data/badaddr.cddl
 - test-data/base32.cddl
@@ -146,6 +173,7 @@ files:
 - test-data/bpv7a.cddl
 - test-data/bpv7b.cddl
 - test-data/cat-re.cddl
+- test-data/catb64.cddl
 - test-data/cdni-ct.cddl
 - test-data/complex-occ.cddl
 - test-data/coral.cddl
@@ -155,10 +183,17 @@ files:
 - test-data/cose.cddl
 - test-data/dcaf.cddl
 - test-data/dcaf1.cddl
+- test-data/decimal-warn.cddl
 - test-data/decimal.cddl
 - test-data/decimal2.cddl
 - test-data/det1.cddl
+- test-data/dotplus1.cddl
+- test-data/dotplus2.cddl
+- test-data/dotplus3.cddl
+- test-data/dotplus4.cddl
 - test-data/dotsize.cddl
+- test-data/enum.cddl
+- test-data/enum1.cddl
 - test-data/extractor-demo.cddl
 - test-data/feat1.cddl
 - test-data/feature-controller.cddl
@@ -186,8 +221,10 @@ files:
 - test-data/joini.cddl
 - test-data/joinx.cddl
 - test-data/json.cddl
+- test-data/json1.cddl
 - test-data/json2.cddl
 - test-data/jsoniodef.cddl
+- test-data/jwt.cddl
 - test-data/kevin5.cddl
 - test-data/lint1.cddl
 - test-data/map-group.cddl
@@ -208,6 +245,12 @@ files:
 - test-data/oidbat.cddl
 - test-data/patch1.cddl
 - test-data/plus.cddl
+- test-data/prim.cddl
+- test-data/printf.cddl
+- test-data/printf0.cddl
+- test-data/printf1.cddl
+- test-data/printf2.cddl
+- test-data/printf3.cddl
 - test-data/reused_named_group.cddl
 - test-data/sasl.cddl
 - test-data/sequence.cddl
@@ -220,6 +263,7 @@ files:
 - test-data/toerless0.cddl
 - test-data/toerless1.cddl
 - test-data/two_anonymous_groups.cddl
+- test-data/valemb.cddl
 - test-data/wrong1.cddl
 - test-data/wrong1a.cddl
 - test-data/wrong2.cddl
@@ -228,11 +272,10 @@ files:
 - test-data/yangid.cddl
 - test-data/yaron1.cddl
 - test/test-cddl.rb
-homepage: http://github.com/cabo/cddl
+homepage: https://www.rfc-editor.org/rfc/rfc8610#appendix-F
 licenses:
 - MIT
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -240,15 +283,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.0'
+      version: '2.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.6
-signing_key:
+rubygems_version: 4.0.8
 specification_version: 4
 summary: CDDL generator and validator.
 test_files: []

data/lib/base45_lite.rb

@@ -1,71 +0,0 @@
-# frozen_string_literal: true
-
-# Copyright (c) 2022 Weihang Jian <https://tonytonyjan.net>
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in all
-# copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
-
-# An implementation of draft-faltstrom-base45-10, see
-# https://datatracker.ietf.org/doc/draft-faltstrom-base45/
-module Base45Lite
-  class Error < ::StandardError; end
-  class OverflowError < Error; end
-  class InvalidCharacterError < Error; end
-  class ForbiddenLengthError < Error; end
-
-  MAX_UINT18 = 2**16 - 1
-  SQUARED_45 = 45**2
-  MAPPING = [
-    *'0'..'9',
-    *'A'..'Z',
-    ' ', '$', '%', '*', '+', '-', '.', '/', ':'
-  ].map!.with_index { |x, i| [i, x] }.to_h.freeze
-  REVERSE_MAPPING = MAPPING.invert.freeze
-
-  def self.encode(input)
-    sequence = []
-    input.unpack('n*').map! do |uint16|
-      i, c = uint16.divmod(45)
-      i, d = i.divmod(45)
-      _, e = i.divmod(45)
-      sequence.push(c, d, e)
-    end
-    if input.bytesize.odd?
-      i, c = input.getbyte(-1).divmod(45)
-      _, d = i.divmod(45)
-      sequence.push(c, d)
-    end
-    sequence.map!{ |n| MAPPING[n] }.join
-  end
-
-  def self.decode(input)
-    input
-      .chars.map! { |c| REVERSE_MAPPING[c] || raise(InvalidCharacterError) }
-      .each_slice(3).map do |slice|
-        c, d, e = slice
-        raise ForbiddenLengthError if d.nil?
-
-        sum = c + d * 45
-        sum += e * SQUARED_45 if e
-        raise OverflowError if sum > MAX_UINT18
-
-        sum
-      end
-      .pack((input.length % 3).zero? ? 'n*' : "n#{input.length / 3}C")
-  end
-end