cddl 0.8.26 → 0.12.14

data/lib/cddl.rb

@@ -2,10 +2,15 @@ require 'abnc'
 require 'pp'
 require 'pathname'
 require 'cbor-pure' unless defined?(CBOR::Tagged)
+require 'half'
+require 'cbor-deterministic'
 require 'regexp-examples'
 require 'abnftt'
 require 'colorize'
 require 'base64'
+require 'base32'
+require 'base45_lite'
+require 'scanf'
 
 module CDDL
 
@@ -22,6 +27,21 @@ module CDDL
 
   class Parser
 
+    FEATURE_REJECT_RE = /\A\^/
+    # CDDL_FEATURE_OK=cbor,^json
+    CDDL_FEATURE_OK, CDDL_FEATURE_REJECT =
+                 if ok = ENV["CDDL_FEATURE_OK"]
+                   ok.split(/,\s*/)
+                     .partition{|s| s[0] !~ FEATURE_REJECT_RE}
+                     .map {|l| Hash[l.map {|feature|
+                                      [feature.sub(FEATURE_REJECT_RE, ''),
+                                       true]}]}
+                 else
+                   [{}, {}]
+                 end
+
+    CDDL_UNUSED_OK = ENV["CDDL_UNUSED_OK"]
+
     class BackTrack < Exception; end
 
     def initialize(source_text)
@@ -42,7 +62,7 @@ module CDDL
         if upto - presult < 100
           part2 = source_text[presult...upto]
         else
-          part2 = source_text[presult, 50] + "......." + source_text[upto-50, 50]
+          part2 = source_text[presult, 50] + "......." + (source_text[upto-50, 50] || "")
         end
         warn "*** Look for syntax problems around the #{
                "%%%".colorize(background: :light_yellow)} markers:\n#{
@@ -99,6 +119,16 @@ module CDDL
       r.compact
     end
 
+    def longest_initial_match(old, now)
+      md = /\A(.*)(.*)\n\1(.*)\z/.match([old,now].join("\n"))
+      md[1..3]
+    end
+
+    def diff_message(old, now)
+      c, l, r = longest_initial_match(old, now)
+      "-- common: >#{c}< -- old: >#{l}< -- new: >#{r}<"
+    end
+
     def cname(s)
       s.to_s.gsub(/-/, "_")
     end
@@ -130,7 +160,7 @@ module CDDL
         if subtree[0] == :type1 && subtree[1..-1].all? {|x| x[0] == :int}
           if enumname = subtree.cbor_annotations rescue nil
             enumname = cname(enumname.first)
-            subtree[1..-1].each do |x| 
+            subtree[1..-1].each do |x|
               if memname = x.cbor_annotations
                 memname = "#{enumname}_#{cname(memname.first)}"
                 add.(memname, x[1].to_s)
@@ -185,7 +215,8 @@ module CDDL
           ids = g.children(:id).map(&:to_s)
           # puts ["ids", ids].inspect
           if b = @generics[n]
-            fail "Duplicate generics definition #{n} as #{rule_ast} (was #{b})"
+            fail "Duplicate generics definition #{n} #{
+              diff_message(strip_nodes(b[0]).inspect, strip_nodes(rule_ast).inspect)}"
           end
           @generics[n] = [rule_ast, ids]
         else
@@ -197,7 +228,7 @@ module CDDL
               if a == b
                 warn "*** Warning: Identical redefinition of #{n} as #{a}"
               else
-                fail "Duplicate rule definition #{n} as #{b} (was #{a})"
+                fail "Duplicate rule definition #{n} #{diff_message(b, a)}"
               end
             end
             @rules[n] = rule_ast
@@ -229,7 +260,7 @@ module CDDL
       r_process("used_in_cddl_prelude", @rules["used_in_cddl_prelude"])
       @rules.each do |n, r|
       #   r_process(n, r)     # debug only loop
-        warn "*** Unused rule #{n}" unless @stage1[n]
+        warn "*** Unused rule #{n}" unless @stage1[n] || CDDL_UNUSED_OK
       end
       if result[0] == :grpent
         warn "Group at top -- first rule must be a type!"
@@ -252,10 +283,25 @@ module CDDL
       l.map {|l| l.sub(/^ {0,#{indent}}/, "")}.join
     end
 
+    def unpack_array_to_sequence(content, where)
+      # remove the first head
+      n = case content.getbyte(0) - (4 << 5)
+          when 0..23; 1
+          when 24; 2
+          when 25; 3
+          when 26; 5
+          when 27; 9      # unlikely :-)
+          else fail ".cborseq sequence for #{where} not an array"
+          end
+      content[0...n] = ''
+      content
+    end
+
+
     # Memoize a bit here
 
     REGEXP_FOR_STRING = Hash.new {|h, k|
-      h[k] = Regexp.new("\\A#{k}\\z")
+      h[k] = Regexp.new("\\A(?:#{k})\\z")
     }
 
     ABNF_PARSER_FOR_STRING = Hash.new {|h, k|
@@ -268,6 +314,12 @@ module CDDL
       abnfb: Encoding::BINARY
     }
 
+    def myfrand(exp)
+      exprange = 1 << exp
+      expoffset= exprange >> 1
+      (rand(2)*2-1) * rand() * 2.0**(rand(exprange)-expoffset)
+    end
+
     def generate
       @recursion = 0
       generate1(rules)
@@ -390,9 +442,17 @@ module CDDL
         when 3
           gen_word
         when 6
-          CBOR::Tagged.new(where[2], generate1(where[3]))
+          tn = Integer === where[2] ? where[2] : generate1(where[2])
+          unless Integer === tn && tn >= 0
+            fail "Can't generate a tag number out of #{where[2]}"
+          end
+          CBOR::Tagged.new(tn, generate1(where[3]))
         when 7
-          case where[2]
+          w2 = where[2]
+          if Array === w2
+            w2 = generate1(w2)
+          end
+          case w2
           when nil
             Math::PI
           when 20
@@ -401,10 +461,22 @@ module CDDL
             true
           when 22
             nil
-          when 23
-            :undefined
-          when 25, 26, 27
-            rand()
+          when 0..19, 23, 32..255
+            CBOR::Simple.new(w2)
+          when 25
+            while !(hs = Half.encode_from_single_bytes([myfrand(5)].pack("g")))
+            end
+            Half.decode(hs)
+          when 26
+            while (a = [myfrand(8)].pack("g").unpack("g").first).to_cbor.size != 5
+            end
+            a
+          when 27
+            while (a = myfrand(11)).to_cbor.size != 9
+            end
+            a
+          else
+            fail "Can't generate prim #7.#{where[2].inspect}" # XXX retry array generate
           end
         else
           fail "Can't generate prim #{where[1]}"
@@ -459,6 +531,11 @@ module CDDL
           end
           generate1(target, inmap)
         when :feature
+          feature, = extract_feature(control, nil)
+          # warn "@@@ GENFEAT #{feature.inspect} #{CDDL_FEATURE_REJECT[feature].inspect}"
+          if CDDL_FEATURE_REJECT[feature]
+            fail BackTrack.new("Feature '#{feature}' rejected")
+          end
           generate1(target, inmap)
         when :cat, :det
           lhs = generate1(target, inmap)
@@ -514,8 +591,7 @@ module CDDL
               warn "HUH gen #{where.inspect} #{try.inspect}" unless try.nil?
             end
           end
-          32.times do
-            content = generate1(target)
+          try_generate(target) do |content|
             if validate1(content, where)
               return content
             end
@@ -534,33 +610,107 @@ module CDDL
           unless !bytes.nil? && String === grammar
             fail "Don't know yet how to generate #{where}"
           end
+          grammar << "\n" unless grammar =~ /\n\z/
           out = ABNF_PARSER_FOR_STRING[grammar].generate
           if conop == :abnfb
             out = out.codepoints.pack("C*")
           end
           enc = bytes ? Encoding::BINARY : Encoding::UTF_8
           out.force_encoding(enc)
-        when :cbor, :cborseq
+        when :cbor, :cborseq, :cbordet, :cborseqdet
           unless target == [:prim, 2]
             fail "Don't know yet how to generate #{where}"
           end
-          content = CBOR::encode(generate1(control))
-          if conop == :cborseq
+          input = generate1(control)
+          if conop == :cbordet || conop == :cborseqdet
+            input = input.cbor_prepare_deterministic
+          end
+          content = CBOR::encode(input)
+          if conop == :cborseq || conop == :cborseqdet
             # remove the first head
-            n = case content.getbyte(0) - (4 << 5)
-                when 0..23; 1
-                when 24; 2
-                when 25; 3
-                when 26; 5
-                when 27; 9      # unlikely :-)
-                else fail "Generated .cborseq sequence for #{where} not an array"
-                end
-            content[0...n] = ''
+            content = unpack_array_to_sequence(content, where)
+          end
+          content
+        when :json
+          unless target == [:prim, 3]
+            fail "Don't know yet how to generate #{where}"
+          end
+          content = JSON::dump(generate1(control))
+          content
+        when :decimal, :base10
+          if conop == :decimal
+            unless @decimal_warned
+              warn "** .decimal control operator is now called .base10"
+              @decimal_warned = true
+            end
+          end
+          unless target == [:prim, 3]
+            fail "Don't know yet how to generate #{where}"
           end
+          content = Integer(generate1(control)).to_s
           content
+        when :join
+          try_generate(control) do |content|
+            if Array === content &&
+               content.all? {|x| String === x} &&
+               Set[content.map {|x| x.encoding}].size == 1
+              content = content.join
+              if validate1(content, target)
+                return content
+              end
+            end
+          end
+          fail "Don't know yet how to generate #{where}"
+        when :b64u, :"b64u-sloppy", :b64c, :"b64c-sloppy",
+             :b45, :b32, :h32, :hex, :hexlc, :hexuc
+          try_generate(control) do |content|
+            if String === content
+              content = case conop
+                    when :b64u, :"b64u-sloppy"
+                      Base64.urlsafe_encode64(content, padding: false)
+                    when :b64c, :"b64c-sloppy"
+                      Base64.strict_encode64(content)
+                    when :b45
+                      Base45Lite.encode(content)
+                    when :b32
+                      Base32.table = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"
+                      Base32.encode(content).gsub("=", "")
+                    when :h32
+                      Base32.table = "0123456789ABCDEFGHIJKLMNOPQRSTUV"
+                      Base32.encode(content).gsub("=", "")
+                    when :hex, :hexuc
+                      content.unpack("H*")[0].upcase
+                    when :hexlc
+                      content.unpack("H*")[0]
+                    else fail "Cannot happen"
+                    end
+              if validate1(content, target)
+                return content
+              end
+            end
+          end
+          fail "Not smart enough to generate #{where}"
+        when :printf
+          try_generate(control) do |content|
+            if Array === content && content.size >= 1
+              fmt, *data = content
+              if String === fmt
+                begin
+                  content = fmt % data
+                  if validate1(content, target)
+                    return content
+                  end
+                rescue ArgumentError => e
+                  # be verbose about mismatches here
+                  @last_message << "\n** #{fmt.inspect} ArgumentError #{e}"
+                end
+              end
+            end
+          end
+          fail "Not smart enough to generate #{where}#{@last_message}"
         when :within, :and
-          32.times do
-            content = generate1(target)
+          try_generate(target) do |content|
+            $features ||= Hash.new {|h, k| h[k] = {}}
             if validate1(content, control)
               return content
             elsif conop == :within
@@ -588,22 +738,41 @@ module CDDL
       end
     end
 
+    def try_generate(target)
+      32.times do
+        content = generate1(target)
+        yield content           # should return if success
+      end
+    end
+
     VALUE_TYPE = {text: String, bytes: String, int: Integer, float: Float}
     SIMPLE_VALUE = {
       [:prim, 7, 20] => [true, false, :bool],
       [:prim, 7, 21] => [true, true, :bool],
       [:prim, 7, 22] => [true, nil, :nil],
-      [:prim, 7, 23] => [true, :undefined, :undefined],
     }
+    SIMPLE_VALUE_SIMPLE = Set[23] + (0..19) + (32..255)
 
     def extract_value(t)        # []
       if vt = VALUE_TYPE[t[0]]
         [true, t[1], vt]
-      elsif v = SIMPLE_VALUE[t]
-        v
+      elsif t[0] == :prim && t[1] == 7
+        case t2 = t[2]
+        when Integer
+        when Array
+          a, b, c = extract_value(t2)
+          if a && c == Integer
+            t2 = b
+          end
+        end
+        if v = SIMPLE_VALUE[[:prim, 7, t2]]
+          v
+        elsif SIMPLE_VALUE_SIMPLE === t2
+          [true, CBOR::Simple.new(t2), :simple]
+        end
       elsif t[0] == :anno
         _, conop, target, control = t
-        # warn ["EXV0", conop, target, control].inspect
+        warn ["EXV0", conop, target, control].inspect   if ENV["CDDL_TRACE"]
         if conop == :cat || conop == :plus || conop == :det
           ok1, v1, vt1 = extract_value(target)
           ok2, v2, vt2 = extract_value(control)
@@ -635,6 +804,38 @@ module CDDL
        }]
     end
 
+
+    def extract_arg0(t)
+      return [false] unless t[0] == :array
+      [true,
+       (el = t[1]
+        return [false] unless el[0..3] == [:member, 1, 1, nil]
+        ok, v, vt = extract_value(el[4])
+        return [false] unless ok
+        [v, vt]
+       ),
+       *t[2..-1]]
+    end
+
+    def extract_feature(control, d)
+      ok, v, vt = extract_value(control)
+      if ok
+        nm = v
+        det = d
+        warn "*** feature controller should be a string: #{control.inspect}" unless String == vt
+      else
+        ok, *v = extract_array(control)
+        if ok && v.size == 2
+          nm = v[0][0]
+          det = v[1][0]
+          warn "*** first element of feature controller should be a string: #{control.inspect}" unless String === nm
+        else
+          warn "*** feature controller not implemented: #{control.inspect}"
+        end
+      end
+      [nm, det]
+    end
+
     def validate_diag
       [@last_data, @last_rule, @last_message]
     end
@@ -646,9 +847,8 @@ module CDDL
       if warn
         if result
           if $features != {}
-            ok = ENV["CDDL_FEATURE_OK"] and ok = ok.split(/,\s*/) or ok = []
-            # warn([:OK, ok, $features].inspect)
-            features = $features.reject {|k, v| ok.include? k.to_s }
+            features = $features.reject {|k, v| CDDL_FEATURE_OK[k.to_s] }
+            # warn "@@@ FEAT #{CDDL_FEATURE_OK.inspect} #{CDDL_FEATURE_REJECT.inspect}"
             warn "** Features potentially used (#$fn): #{features.map {|k, v| "#{k}: #{v.keys}"}.join(", ")}" if features != {}
           end
         else
@@ -805,6 +1005,19 @@ module CDDL
       }
     end
 
+    def validate1a_ignore_encoding(d, where)
+      if String === d
+        validate1a(d.force_encoding(Encoding::BINARY), where) or (
+          text = d.force_encoding(Encoding::UTF_8).scrub {
+            fail "can't use bytes as text"
+          } rescue :NOT_A_TEXT_STRING
+          validate1a(text, where)
+        )
+      else
+        validate1a(d, where)
+      end
+    end
+
     def validate1a(d, where)
       if ann = validate1(d, where)
         here = [d, where]
@@ -817,6 +1030,8 @@ module CDDL
     end
 
     OPERATORS = {lt: :<, le: :<=, gt: :>, ge: :>=, eq: :==, ne: :!=}
+    OPERATORS_MATCH = {eq: true, ne: false}
+    FLOAT_AI_FROM_SIZE = {3 => 25, 5 => 26, 9 => 27}
 
     def validate1(d, where)
       if ENV["CDDL_TRACE"]
@@ -827,6 +1042,9 @@ module CDDL
       @last_data = d
       @last_rule = where
       ann = nil
+      if CBOR::Tagged === d && d.tag === 888 && ENV["CDDL_888"]
+        return []
+      end
       case where[0]
       when :type1
         if where[1..-1].any? {|r| ann = validate1a(d, r)}
@@ -863,7 +1081,7 @@ module CDDL
         if conop == :cat || conop == :plus || conop == :det
           ok1, v1, vt1 = extract_value(target)
           ok2, v2, vt2 = extract_value(control)
-          # warn ["ANNO0", ok1, v1, vt1, ok2, v2, vt2, d].inspect
+          warn ["ANNO0", ok1, v1, vt1, ok2, v2, vt2, d].inspect  if ENV["CDDL_TRACE"]
           if ok1 && ok2
             v2 = Integer(v2) if vt1 == Integer
             if conop == :det
@@ -872,6 +1090,30 @@ module CDDL
             end
             # warn ["ANNO", ok1, v1, vt1, ok2, v2, vt2, d].inspect
             [] if d == v1 + v2  # XXX Focus ArgumentError
+          elsif conop == :cat && String === d
+            warn ["CAT-L", ok1, v1, vt1, ok2, v2, vt2, d].inspect  if ENV["CDDL_TRACE"]
+            if ok1 && vt1 == String
+              # d and lhs (v1) need to agree in encoding
+              if d.encoding == v1.encoding
+                if d[0...v1.length] == v1
+                  d2 = d[v1.length..-1]
+                  warn ["CAT-L1", d2, d2.encoding, control].inspect  if ENV["CDDL_TRACE"]
+                  validate1a_ignore_encoding(d2, control)
+                end
+              end
+            elsif ok2 && vt2 == String
+            warn ["CAT-R", ok1, v1, vt1, ok2, v2, vt2, d].inspect  if ENV["CDDL_TRACE"]
+              if d[-v2.length..-1] == v2
+                d1 = d[0...-v2.length]
+                validate1a(d1, control)
+              end
+            end
+          elsif conop == :plus && Integer === d
+            if ok1 && vt1 == Integer
+              validate1a(d - Integer(v1), control)
+            elsif ok2 && vt2 == Integer
+              validate1a(d - Integer(v2), target)
+            end
           end
         elsif ann = validate1a(d, target)
           case conop
@@ -914,28 +1156,19 @@ module CDDL
             end
             ann
           when :feature
-            ok, v, vt = extract_value(control)
-            if ok
-              nm = v
-              det = d
-              warn "*** feature controller should be a string: #{control.inspect}" unless String == vt
-            else
-              ok, *v = extract_array(control)
-              if ok && v.size == 2
-                nm = v[0][0]
-                det = v[1][0]
-                warn "*** first element of feature controller should be a string: #{control.inspect}" unless String === nm
-              else
-                warn "*** feature controller not implemented: #{control.inspect}"
-              end
-            end
+            nm, det = extract_feature(control, d)
             $features[nm][det] = true
-            ann
+            ann if !CDDL_FEATURE_REJECT[nm]
           when :lt, :le, :gt, :ge, :eq, :ne
             op = OPERATORS[conop]
             ok, v, _vt = extract_value(control)
             if ok
               ann if d.send(op, v) rescue nil # XXX Focus ArgumentError
+            else
+              needs_to_match = OPERATORS_MATCH[conop]
+              unless needs_to_match.nil?
+                ann if !!validate1(d, control) == needs_to_match
+              end
             end
           when :regexp
             ann if (
@@ -970,6 +1203,159 @@ module CDDL
             ann if validate1((CBOR::decode(d) rescue :BAD_CBOR), control)
           when :cborseq
             ann if validate1((CBOR::decode("\x9f".b << d << "\xff".b) rescue :BAD_CBOR), control)
+          when :cbordet
+            decoded = CBOR::decode(d) rescue :BAD_CBOR
+            if d != decoded.to_deterministic_cbor
+              @last_message = "CBOR #{d.inspect} not deterministically encoded"
+              nil
+            else
+              ann if validate1(decoded, control)
+            end
+          when :cborseqdet
+            decoded = CBOR::decode("\x9f".b << d << "\xff".b) rescue :BAD_CBOR
+            if d != unpack_array_to_sequence(decoded.to_deterministic_cbor, d.inspect)
+              @last_message = "CBOR Sequence #{d.inspect} not deterministically encoded"
+              nil
+            else
+              ann if validate1(decoded, control)
+            end
+          when :json
+            ann if validate1((JSON::load(d) rescue :BAD_JSON), control)
+          when :decimal, :base10
+            if conop == :decimal
+              unless @decimal_warned
+                warn "** .decimal control operator is now called .base10"
+                @decimal_warned = true
+              end
+            end
+
+            ann if (
+                if /\A0|-?[1-9][0-9]*\z/ === d
+                  validate1((d.to_i rescue :BAD_JSON), control)
+                end
+              )
+          when :join
+            t = control
+            v = if t[0] == :array
+                  t[1..-1].map { |el|
+                    if el[0..2] == [:member, 1, 1]
+                      ok, v, vt = extract_value(el[4])
+                      if ok
+                        [true, v, vt]
+                      else
+                        [false, el[4]]
+                      end
+                    end
+                  }
+                end
+            warn "@@@JOIN@@@ #{v.inspect}" if ENV["CDDL_TRACE"]
+            ok = true
+            if v
+              ix = 0
+              rest = d.dup
+              while ix < v.length
+                if left = v[ix]
+                  if left[0]    # match constant value first
+                    fail "Not a string for #{left.inspect} in #{where}" unless String == left[2]
+                    want = left[1]
+                    have = rest[0...left[1].length]
+                    if want == have
+                      rest[0...left[1].length] = ''
+                      ix += 1
+                      next
+                    else
+                      fail ".join: want #{want.inspect}, have #{have.inspect}"
+                    end
+                  else
+                    ix += 1
+                    if ix == v.length # match remaining
+                      warn "@@@JOIN ok in #{ok.inspect} rest #{rest.inspect}"  if ENV["CDDL_TRACE"]
+                      ok &&= validate1(rest, left[1])
+                      warn "@@@JOIN ok out #{ok.inspect}"  if ENV["CDDL_TRACE"]
+                      # more diag
+                      rest = ''
+                      next
+                    else
+                      if mid = v[ix]
+                        if mid[0] # have constant value to split over
+                          fail "Not a string for #{mid} in #{where}" unless String == mid[2]
+                          rest1, rest2 = rest.split(mid[1], 2)
+                          if rest2
+                            warn "@@@JOIN ok in #{ok.inspect} rest1 #{rest1.inspect}"   if ENV["CDDL_TRACE"]
+                            ok &&= validate1(rest1, left[1])
+                            warn "@@@JOIN ok out #{ok.inspect}"   if ENV["CDDL_TRACE"]
+                            rest = rest2
+                            ix += 1
+                            next
+                          else
+                            fail "Can't find #{mid[1].inspect} in #{rest.inspect}"
+                          end
+                        else
+                          fail "Cannot validate consecutive non-constant members of .join in #{where}"
+                        end
+                      else
+                        fail "Cannot handle .join over #{t[ix+1]} in #{where}"
+                      end
+                    end
+                  end
+                else
+                  fail "Cannot handle .join over #{t[ix+1]} in #{where}"
+                end
+              end
+              fail "Can't match #{rest.inspect} for .join in #{where}" if rest != ''
+              ann if ok
+            else
+              fail "Don't know yet how to validate against #{where}"
+            end
+          when :b64u, :"b64u-sloppy", :b64c, :"b64c-sloppy", :b45, :b32, :h32, :hex, :hexlc, :hexuc
+            ann if (
+                String === d && (
+                decoded = case conop
+                    when :b64u
+                      /[+\/=]/ !~ d &&
+                      Base64.urlsafe_decode64(d)
+                    when :"b64u-sloppy"
+                      /[-_=]/ !~ d &&
+                      Base64.decode64(d.tr("+/", "-_"))
+                    when :b64c
+                      Base64.strict_decode64(d)
+                    when :"b64c-sloppy"
+                      Base64.decode64(d)
+                    when :b32
+                      /=/ !~ d &&
+                        (Base32.table = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567") &&
+                        Base32.decode(d)
+                    when :b45
+                      Base45Lite.decode(d)
+                    when :h32
+                      /=/ !~ d &&
+                        (Base32.table = "0123456789ABCDEFGHIJKLMNOPQRSTUV") &&
+                        Base32.decode(d)
+                    when :hex
+                      /\A[0-9a-fA-F]*\z/ =~ d && [d].pack("H*")
+                    when :hexuc
+                      /\A[0-9A-F]*\z/ =~ d && [d].pack("H*")
+                    when :hexlc
+                      /\A[0-9a-f]*\z/ =~ d && [d].pack("H*")
+                    else fail "Cannot happen"
+                    end
+                ) && validate1(decoded.b, control)
+              )
+          when :printf
+            ann if String === d && (
+                     ok, fmt, *v = extract_arg0(control)
+                     if ok && String == fmt[1]
+                       fmt = fmt[0]
+                       # warn "** ok #{ok.inspect} fmt #{fmt.inspect} v #{v.inspect}"
+                       decoded = d.scanf(fmt) # this is a bit too lenient, so let's do:
+                       encode_again = fmt % decoded
+                       if encode_again != d
+                         warn "** fmt #{fmt.inspect} d #{d.inspect} decoded #{decoded.inspect} encode_again #{encode_again.inspect}"
+                       else
+                         validate1(decoded, [:array, *v])
+                       end
+                     end
+                   )
           when :within
             if validate1(d, control)
               ann
@@ -1005,20 +1391,35 @@ module CDDL
             end
             d = CBOR::Tagged.new(t, d == 0 ? "".b : d.digits(256).reverse!.pack("C*"))
           end
-          CBOR::Tagged === d && d.tag == where[2] && validate1a(d.data, where[3])
+          CBOR::Tagged === d && (
+            Integer === where[2] ? d.tag == where[2] : validate1a(d.tag, where[2])
+          ) && validate1a(d.data, where[3])
         when 7
           t, v = extract_value(where)
           if t
             v.eql? d
           else
-            case where[2]
+            case w2 = where[2]
             when nil
-              # XXX
-              fail
+              Float === d || CBOR::Simple === d || [false, true, nil].include?(d)
+            when Array
+              headnum = case d
+                        when Float
+                          FLOAT_AI_FROM_SIZE[d.to_cbor.size]
+                        when false
+                          20
+                        when true
+                          21
+                        when nil
+                          22
+                        when CBOR::Simple
+                          d.value
+                        end
+              validate1a(headnum, w2)
             when 25, 26, 27
-              Float === d
+              Float === d && FLOAT_AI_FROM_SIZE[d.to_cbor.size] == w2
             else
-              fail
+              fail [:val7, d, where].inspect
             end
           end
         else
@@ -1068,6 +1469,10 @@ module CDDL
                                else
                                  [:type1]
                                end
+          elsif s = ENV["CDDL_INVENT"]
+            s = "_" if s == ""
+            r = [:type1, [:text, "#{s}-#{name}"]]
+            return r
           end
         end
         if r
@@ -1113,29 +1518,69 @@ module CDDL
                      end
     end
 
+    STRING_ESCAPES = {
+      "\"" => "\"",
+      "/" => "/",
+      "\\" => "\\",
+      "'" => "'",
+      "b" => "\b",
+      "f" => "\f",
+      "n" => "\n",
+      "r" => "\r",
+      "t" => "\t",
+    }
+
     def value(n)
       # cheat:
       # warn n
       s = n.to_s
       if s[-1] == "'"
-        parts = s.split("'", 3)
-        if parts[2] != "" || parts[1] =~ /\\/
-          warn "*** Can't handle backslashes in #{s.inspect} yet"
+        bsqual, *parts = s.split("'", -1)
+        if parts[-1] != ""
+          warn "*** Problem decoding byte string #{s.inspect}"
         end
+        bsval = parts[0...-1].join("'").gsub(/\\u([Dd][89AaBb][0-9a-zA-Z]{2})\\u([Dd][CcDdEeFf][0-9a-zA-Z]{2})|\\u([0-9a-zA-Z]{4})|\\u\{([0-9a-zA-Z]+)\}|\r?(\n)|\\([^u])/) {
+          if hex = $3 || $4
+            hex.to_i(16).chr(Encoding::UTF_8)
+          elsif lf = $5
+            lf
+          elsif escaped = $6
+            STRING_ESCAPES[$6] or (
+              fail "Invalid String Escape #{escaped.inspect} in #{s.inspect}"
+            )
+          else
+            ((($1.to_i(16) & 0x3ff) << 10) +
+             ($2.to_i(16) & 0x3ff) + 0x10000).chr(Encoding::UTF_8)
+          end
+        }
         [:bytes,
-         case parts[0]
+         case bsqual
          when ""
-           parts[1].b
+           bsval.b
          when "h"
-           parts[1].gsub(/\s/, "").chars.each_slice(2).map{ |x| Integer(x.join, 16).chr("BINARY") }.join.b
+           bsval.gsub(/\s/, "").chars.each_slice(2).map{ |x| Integer(x.join, 16).chr("BINARY") }.join.b
          when "b64"
-            Base64.urlsafe_decode64(parts[1])
+            Base64.urlsafe_decode64(bsval)
          else
-           warn "*** Can't handle byte string type #{parts[0].inspect} yet"
+           warn "*** Can't handle byte string type #{bsqual.inspect} yet"
          end
         ]
       else
-        val = eval(n.to_s)
+        if s[-1] == '"'
+          s.gsub!(/\\u([Dd][89AaBb][0-9a-zA-Z]{2})\\u([Dd][CcDdEeFf][0-9a-zA-Z]{2})|\\([^u]|u[0-9a-zA-Z]{4})/) {
+            if $3
+              "\\#$3" # skip this, use eval parser
+            else
+              ((($1.to_i(16) & 0x3ff) << 10) +
+               ($2.to_i(16) & 0x3ff) + 0x10000).chr(Encoding::UTF_8)
+            end
+          }
+        end
+        if s =~ /p.*\d\z/       # hexfloat
+          val = s.scanf("%a").first
+        else
+          val = eval(s)
+        end
         # warn val
         case val
          when Integer; [:int, val]
@@ -1241,9 +1686,13 @@ module CDDL
     def g_process(name, g, genericargs)
       r, ids = g
       args = genericargs.children(:type1).map {|x| type1(x, true)}
+      # warn "** args #{args.inspect}"
       fail "number of args #{name} #{ids.size} #{args.size}" if ids.size != args.size
       bindings = Hash[ids.zip(args)]
-      r_process("#{name}<#{genericargs._strip}>", r, bindings)
+      # warn "** bindings: #{bindings.inspect}"
+      genname = "#{name}<#{args.inspect}>" # XXX this works up to a RECURSE...
+      # warn "** genname #{genname}"
+      r_process(genname, r, bindings)
     end
 
     def type_recall(name, canbegroup, genericargs)
@@ -1319,9 +1768,16 @@ module CDDL
 
     BRACE = {"{" => :map, "[" => :array}
     RANGE_EXCLUDE_END = {".." => false, "..." => true}
-    SUPPORTED_ANNOTATIONS = [:bits, :size, :regexp, :cbor, :cborseq, :within, :and,
+    SUPPORTED_ANNOTATIONS = [:bits, :size, :regexp,
+                             :cbor, :cborseq, :cbordet, :cborseqdet,
+                             :within, :and,
                              :default, :lt, :le, :gt, :ge, :eq, :ne,
-                             :feature, :abnf, :abnfb, :det, :cat, :plus]
+                             :feature, :abnf, :abnfb, :det, :cat, :plus,
+                             :json, :decimal, :base10, :join,
+                             :b64u, :"b64u-sloppy", :b64c, :"b64c-sloppy",
+                             :b45, :b32, :h32, :hex, :hexlc, :hexuc,
+                             :printf,
+                            ]
 
     def type1(n, canbegroup = false)
 #      puts "NVALUE #{n.value.inspect}"
@@ -1353,6 +1809,13 @@ module CDDL
         when /\A#(\d+)/
           maj = $1.to_i
           s = [:prim, maj, *n.children(:uint).map(&:to_s).map(&:to_i)]
+          if tn = n.headnumber
+            if ui = tn.uint
+              s << ui.to_s.to_i
+            elsif tt = tn.type
+              s << type(tt)
+            end
+          end
           if tagged_type = n.type
               s << type(tagged_type)
           end