cddl 0.8.9 → 0.12.14

data/lib/cddl.rb

@@ -2,9 +2,15 @@ require 'abnc'
 require 'pp'
 require 'pathname'
 require 'cbor-pure' unless defined?(CBOR::Tagged)
+require 'half'
+require 'cbor-deterministic'
 require 'regexp-examples'
+require 'abnftt'
 require 'colorize'
 require 'base64'
+require 'base32'
+require 'base45_lite'
+require 'scanf'
 
 module CDDL
 
@@ -21,6 +27,21 @@ module CDDL
 
   class Parser
 
+    FEATURE_REJECT_RE = /\A\^/
+    # CDDL_FEATURE_OK=cbor,^json
+    CDDL_FEATURE_OK, CDDL_FEATURE_REJECT =
+                 if ok = ENV["CDDL_FEATURE_OK"]
+                   ok.split(/,\s*/)
+                     .partition{|s| s[0] !~ FEATURE_REJECT_RE}
+                     .map {|l| Hash[l.map {|feature|
+                                      [feature.sub(FEATURE_REJECT_RE, ''),
+                                       true]}]}
+                 else
+                   [{}, {}]
+                 end
+
+    CDDL_UNUSED_OK = ENV["CDDL_UNUSED_OK"]
+
     class BackTrack < Exception; end
 
     def initialize(source_text)
@@ -41,7 +62,7 @@ module CDDL
         if upto - presult < 100
           part2 = source_text[presult...upto]
         else
-          part2 = source_text[presult, 50] + "......." + source_text[upto-50, 50]
+          part2 = source_text[presult, 50] + "......." + (source_text[upto-50, 50] || "")
         end
         warn "*** Look for syntax problems around the #{
                "%%%".colorize(background: :light_yellow)} markers:\n#{
@@ -55,6 +76,8 @@ module CDDL
       @ast = @abnf.ast?
       # our little argument stack for rule processing
       @insides = []
+      # collect error information
+      @last_message = ""
     end
 
     def apr                     # for debugging
@@ -96,6 +119,16 @@ module CDDL
       r.compact
     end
 
+    def longest_initial_match(old, now)
+      md = /\A(.*)(.*)\n\1(.*)\z/.match([old,now].join("\n"))
+      md[1..3]
+    end
+
+    def diff_message(old, now)
+      c, l, r = longest_initial_match(old, now)
+      "-- common: >#{c}< -- old: >#{l}< -- new: >#{r}<"
+    end
+
     def cname(s)
       s.to_s.gsub(/-/, "_")
     end
@@ -127,7 +160,7 @@ module CDDL
         if subtree[0] == :type1 && subtree[1..-1].all? {|x| x[0] == :int}
           if enumname = subtree.cbor_annotations rescue nil
             enumname = cname(enumname.first)
-            subtree[1..-1].each do |x| 
+            subtree[1..-1].each do |x|
               if memname = x.cbor_annotations
                 memname = "#{enumname}_#{cname(memname.first)}"
                 add.(memname, x[1].to_s)
@@ -182,7 +215,8 @@ module CDDL
           ids = g.children(:id).map(&:to_s)
           # puts ["ids", ids].inspect
           if b = @generics[n]
-            fail "Duplicate generics definition #{n} as #{rule_ast} (was #{b})"
+            fail "Duplicate generics definition #{n} #{
+              diff_message(strip_nodes(b[0]).inspect, strip_nodes(rule_ast).inspect)}"
           end
           @generics[n] = [rule_ast, ids]
         else
@@ -194,7 +228,7 @@ module CDDL
               if a == b
                 warn "*** Warning: Identical redefinition of #{n} as #{a}"
               else
-                fail "Duplicate rule definition #{n} as #{b} (was #{a})"
+                fail "Duplicate rule definition #{n} #{diff_message(b, a)}"
               end
             end
             @rules[n] = rule_ast
@@ -226,7 +260,7 @@ module CDDL
       r_process("used_in_cddl_prelude", @rules["used_in_cddl_prelude"])
       @rules.each do |n, r|
       #   r_process(n, r)     # debug only loop
-        warn "*** Unused rule #{n}" unless @stage1[n]
+        warn "*** Unused rule #{n}" unless @stage1[n] || CDDL_UNUSED_OK
       end
       if result[0] == :grpent
         warn "Group at top -- first rule must be a type!"
@@ -243,12 +277,49 @@ module CDDL
       [rule[0], *rule[1]]
     end
 
+    def remove_indentation(s)
+      l = s.lines
+      indent = l.grep(/\S/).map {|l| l[/^\s*/].size}.min
+      l.map {|l| l.sub(/^ {0,#{indent}}/, "")}.join
+    end
+
+    def unpack_array_to_sequence(content, where)
+      # remove the first head
+      n = case content.getbyte(0) - (4 << 5)
+          when 0..23; 1
+          when 24; 2
+          when 25; 3
+          when 26; 5
+          when 27; 9      # unlikely :-)
+          else fail ".cborseq sequence for #{where} not an array"
+          end
+      content[0...n] = ''
+      content
+    end
+
+
     # Memoize a bit here
 
     REGEXP_FOR_STRING = Hash.new {|h, k|
-      h[k] = Regexp.new("\\A#{k}\\z")
+      h[k] = Regexp.new("\\A(?:#{k})\\z")
+    }
+
+    ABNF_PARSER_FOR_STRING = Hash.new {|h, k|
+      grammar = "cddl-t0p--1eve1-f0r--abnf = " << k # XXX
+      h[k] = ABNF.from_abnf(grammar)
+    }
+
+    ABNF_ENCODING_FOR_CONOP = {
+      abnf: Encoding::UTF_8,
+      abnfb: Encoding::BINARY
     }
 
+    def myfrand(exp)
+      exprange = 1 << exp
+      expoffset= exprange >> 1
+      (rand(2)*2-1) * rand() * 2.0**(rand(exprange)-expoffset)
+    end
+
     def generate
       @recursion = 0
       generate1(rules)
@@ -371,9 +442,17 @@ module CDDL
         when 3
           gen_word
         when 6
-          CBOR::Tagged.new(where[2], generate1(where[3]))
+          tn = Integer === where[2] ? where[2] : generate1(where[2])
+          unless Integer === tn && tn >= 0
+            fail "Can't generate a tag number out of #{where[2]}"
+          end
+          CBOR::Tagged.new(tn, generate1(where[3]))
         when 7
-          case where[2]
+          w2 = where[2]
+          if Array === w2
+            w2 = generate1(w2)
+          end
+          case w2
           when nil
             Math::PI
           when 20
@@ -382,10 +461,22 @@ module CDDL
             true
           when 22
             nil
-          when 23
-            :undefined
-          when 25, 26, 27
-            rand()
+          when 0..19, 23, 32..255
+            CBOR::Simple.new(w2)
+          when 25
+            while !(hs = Half.encode_from_single_bytes([myfrand(5)].pack("g")))
+            end
+            Half.decode(hs)
+          when 26
+            while (a = [myfrand(8)].pack("g").unpack("g").first).to_cbor.size != 5
+            end
+            a
+          when 27
+            while (a = myfrand(11)).to_cbor.size != 9
+            end
+            a
+          else
+            fail "Can't generate prim #7.#{where[2].inspect}" # XXX retry array generate
           end
         else
           fail "Can't generate prim #{where[1]}"
@@ -393,7 +484,7 @@ module CDDL
       when :anno
         target = where[2]
         control = where[3]
-        case where[1]
+        case conop = where[1]
         when :size
           should_be_int = generate1(control)
           unless (Array === target && target[0] == :prim && [0, 2, 3].include?(target[1])) && Integer === should_be_int && should_be_int >= 0
@@ -439,6 +530,40 @@ module CDDL
             $default_warned = true
           end
           generate1(target, inmap)
+        when :feature
+          feature, = extract_feature(control, nil)
+          # warn "@@@ GENFEAT #{feature.inspect} #{CDDL_FEATURE_REJECT[feature].inspect}"
+          if CDDL_FEATURE_REJECT[feature]
+            fail BackTrack.new("Feature '#{feature}' rejected")
+          end
+          generate1(target, inmap)
+        when :cat, :det
+          lhs = generate1(target, inmap)
+          rhs = generate1(control)
+          if conop == :det
+            lhs = remove_indentation(lhs)
+            rhs = remove_indentation(rhs)
+          end
+          begin
+            lhs + rhs
+          rescue Exception => e
+            fail "Can't #{lhs.inspect} .cat #{rhs.inspect}: #{e.message}"
+          end
+        when :plus
+          lhs = generate1(target, inmap)
+          rhs = generate1(control)
+          begin
+            case lhs
+            when Integer
+              lhs + Integer(rhs)
+            when Numeric
+              lhs + rhs
+            else
+              fail "#{lhs.inspect}: Not a number"
+            end
+          rescue Exception => e
+            fail "Can't #{lhs.inspect} .plus #{rhs.inspect}: #{e.message}"
+          end
         when :eq
           content = generate1(control)
           if validate1(content, where)
@@ -452,7 +577,7 @@ module CDDL
                     content = Integer(content)
                     case target[1]
                     when 0
-                      case where[1]
+                      case conop
                       when :lt
                         rand(0...content)
                       when :le
@@ -466,8 +591,7 @@ module CDDL
               warn "HUH gen #{where.inspect} #{try.inspect}" unless try.nil?
             end
           end
-          32.times do
-            content = generate1(target)
+          try_generate(target) do |content|
             if validate1(content, where)
               return content
             end
@@ -479,30 +603,117 @@ module CDDL
             fail "Don't know yet how to generate #{where}"
           end
           REGEXP_FOR_STRING[regexp].random_example(max_repeater_variance: 5)
-        when :cbor, :cborseq
+        when :abnf, :abnfb
+          grammar = generate1(control)
+          bytes = true if target == [:prim, 2]
+          bytes = false if target == [:prim, 3]
+          unless !bytes.nil? && String === grammar
+            fail "Don't know yet how to generate #{where}"
+          end
+          grammar << "\n" unless grammar =~ /\n\z/
+          out = ABNF_PARSER_FOR_STRING[grammar].generate
+          if conop == :abnfb
+            out = out.codepoints.pack("C*")
+          end
+          enc = bytes ? Encoding::BINARY : Encoding::UTF_8
+          out.force_encoding(enc)
+        when :cbor, :cborseq, :cbordet, :cborseqdet
           unless target == [:prim, 2]
             fail "Don't know yet how to generate #{where}"
           end
-          content = CBOR::encode(generate1(control))
-          if where[1] == :cborseq
+          input = generate1(control)
+          if conop == :cbordet || conop == :cborseqdet
+            input = input.cbor_prepare_deterministic
+          end
+          content = CBOR::encode(input)
+          if conop == :cborseq || conop == :cborseqdet
             # remove the first head
-            n = case content.getbyte(0) - (4 << 5)
-                when 0..23; 1
-                when 24; 2
-                when 25; 3
-                when 26; 5
-                when 27; 9      # unlikely :-)
-                else fail "Generated .cborseq sequence for #{where} not an array"
-                end
-            content[0...n] = ''
+            content = unpack_array_to_sequence(content, where)
           end
           content
+        when :json
+          unless target == [:prim, 3]
+            fail "Don't know yet how to generate #{where}"
+          end
+          content = JSON::dump(generate1(control))
+          content
+        when :decimal, :base10
+          if conop == :decimal
+            unless @decimal_warned
+              warn "** .decimal control operator is now called .base10"
+              @decimal_warned = true
+            end
+          end
+          unless target == [:prim, 3]
+            fail "Don't know yet how to generate #{where}"
+          end
+          content = Integer(generate1(control)).to_s
+          content
+        when :join
+          try_generate(control) do |content|
+            if Array === content &&
+               content.all? {|x| String === x} &&
+               Set[content.map {|x| x.encoding}].size == 1
+              content = content.join
+              if validate1(content, target)
+                return content
+              end
+            end
+          end
+          fail "Don't know yet how to generate #{where}"
+        when :b64u, :"b64u-sloppy", :b64c, :"b64c-sloppy",
+             :b45, :b32, :h32, :hex, :hexlc, :hexuc
+          try_generate(control) do |content|
+            if String === content
+              content = case conop
+                    when :b64u, :"b64u-sloppy"
+                      Base64.urlsafe_encode64(content, padding: false)
+                    when :b64c, :"b64c-sloppy"
+                      Base64.strict_encode64(content)
+                    when :b45
+                      Base45Lite.encode(content)
+                    when :b32
+                      Base32.table = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"
+                      Base32.encode(content).gsub("=", "")
+                    when :h32
+                      Base32.table = "0123456789ABCDEFGHIJKLMNOPQRSTUV"
+                      Base32.encode(content).gsub("=", "")
+                    when :hex, :hexuc
+                      content.unpack("H*")[0].upcase
+                    when :hexlc
+                      content.unpack("H*")[0]
+                    else fail "Cannot happen"
+                    end
+              if validate1(content, target)
+                return content
+              end
+            end
+          end
+          fail "Not smart enough to generate #{where}"
+        when :printf
+          try_generate(control) do |content|
+            if Array === content && content.size >= 1
+              fmt, *data = content
+              if String === fmt
+                begin
+                  content = fmt % data
+                  if validate1(content, target)
+                    return content
+                  end
+                rescue ArgumentError => e
+                  # be verbose about mismatches here
+                  @last_message << "\n** #{fmt.inspect} ArgumentError #{e}"
+                end
+              end
+            end
+          end
+          fail "Not smart enough to generate #{where}#{@last_message}"
         when :within, :and
-          32.times do
-            content = generate1(target)
+          try_generate(target) do |content|
+            $features ||= Hash.new {|h, k| h[k] = {}}
             if validate1(content, control)
               return content
-            elsif where[1] == :within
+            elsif conop == :within
               warn "*** #{content.inspect} meets #{target.inspect} but not #{control.inspect}"
             end
           end
@@ -527,22 +738,102 @@ module CDDL
       end
     end
 
-    VALUE_TYPE = {text: String, int: Integer, float: Float}
+    def try_generate(target)
+      32.times do
+        content = generate1(target)
+        yield content           # should return if success
+      end
+    end
+
+    VALUE_TYPE = {text: String, bytes: String, int: Integer, float: Float}
     SIMPLE_VALUE = {
       [:prim, 7, 20] => [true, false, :bool],
       [:prim, 7, 21] => [true, true, :bool],
       [:prim, 7, 22] => [true, nil, :nil],
-      [:prim, 7, 23] => [true, :undefined, :undefined],
     }
+    SIMPLE_VALUE_SIMPLE = Set[23] + (0..19) + (32..255)
 
     def extract_value(t)        # []
       if vt = VALUE_TYPE[t[0]]
         [true, t[1], vt]
-      elsif v = SIMPLE_VALUE[t]
-        v
+      elsif t[0] == :prim && t[1] == 7
+        case t2 = t[2]
+        when Integer
+        when Array
+          a, b, c = extract_value(t2)
+          if a && c == Integer
+            t2 = b
+          end
+        end
+        if v = SIMPLE_VALUE[[:prim, 7, t2]]
+          v
+        elsif SIMPLE_VALUE_SIMPLE === t2
+          [true, CBOR::Simple.new(t2), :simple]
+        end
+      elsif t[0] == :anno
+        _, conop, target, control = t
+        warn ["EXV0", conop, target, control].inspect   if ENV["CDDL_TRACE"]
+        if conop == :cat || conop == :plus || conop == :det
+          ok1, v1, vt1 = extract_value(target)
+          ok2, v2, vt2 = extract_value(control)
+          # warn ["EXV", ok1, v1, vt1, ok2, v2, vt2].inspect
+          if ok1 && ok2
+            if vt1 == Integer
+              [true, v1 + Integer(v2), Integer] if vt2 == Integer || vt2 == Float
+            elsif vt1 == Float
+              [true, v1 + v2, vt1] if vt2 == Integer || vt2 == Float
+            else
+              if conop == :det
+                v1 = remove_indentation(v1)
+                v2 = remove_indentation(v2)
+              end
+              [true, v1 + v2, vt1] if vt1 == vt2
+            end
+          end rescue nil
+        end
+      end || [false]
+    end
+
+    def extract_array(t)
+      return [false] unless t[0] == :array
+      [true, *t[1..-1].map { |el|
+         return [false] unless el[0..3] == [:member, 1, 1, nil]
+         ok, v, vt = extract_value(el[4])
+         return [false] unless ok
+         [v, vt]
+       }]
+    end
+
+
+    def extract_arg0(t)
+      return [false] unless t[0] == :array
+      [true,
+       (el = t[1]
+        return [false] unless el[0..3] == [:member, 1, 1, nil]
+        ok, v, vt = extract_value(el[4])
+        return [false] unless ok
+        [v, vt]
+       ),
+       *t[2..-1]]
+    end
+
+    def extract_feature(control, d)
+      ok, v, vt = extract_value(control)
+      if ok
+        nm = v
+        det = d
+        warn "*** feature controller should be a string: #{control.inspect}" unless String == vt
       else
-        [false]
+        ok, *v = extract_array(control)
+        if ok && v.size == 2
+          nm = v[0][0]
+          det = v[1][0]
+          warn "*** first element of feature controller should be a string: #{control.inspect}" unless String === nm
+        else
+          warn "*** feature controller not implemented: #{control.inspect}"
+        end
       end
+      [nm, det]
     end
 
     def validate_diag
@@ -551,11 +842,19 @@ module CDDL
 
     def validate(d, warn=true)
       @recursion = 0
+      $features = Hash.new {|h, k| h[k] = {}}
       result = validate1a(d, rules)
-      unless result
-        if warn
+      if warn
+        if result
+          if $features != {}
+            features = $features.reject {|k, v| CDDL_FEATURE_OK[k.to_s] }
+            # warn "@@@ FEAT #{CDDL_FEATURE_OK.inspect} #{CDDL_FEATURE_REJECT.inspect}"
+            warn "** Features potentially used (#$fn): #{features.map {|k, v| "#{k}: #{v.keys}"}.join(", ")}" if features != {}
+          end
+        else
           warn "CDDL validation failure (#{result.inspect} for #{d.inspect}):"
           PP::pp(validate_diag, STDERR)
+          puts(validate_diag.cbor_diagnostic)
         end
       end
       result
@@ -563,7 +862,7 @@ module CDDL
 
     def validate_result(check)
       check || (
-        @last_message = yield
+        @last_message << yield
         false
       )
     end
@@ -596,7 +895,9 @@ module CDDL
             ann.concat(ann2)
           end
           if occ < s
-            @last_message = "occur not reached in array #{d} for #{where}"
+            # warn "*** lme #{@last_message.encoding} #{@last_message}"
+            # warn "*** #{"\noccur #{occ} < #{s}, not reached at #{i} in array #{d} for #{where}".encoding}"
+            @last_message << "\noccur #{occ} < #{s}, not reached at #{i} in array #{d} for #{where}"
             return [false, ann]
           end
         end
@@ -685,13 +986,17 @@ module CDDL
                           puts "COMPLEX: #{k.inspect} #{simple.inspect} #{simpleval.inspect}"         if ENV["CDDL_TRACE"]
             keys = d_check.keys
             ta, keys = keys.partition{ |key| validate1(key, k)}
-            # XXX check ta.size against s/e
-            ta.all? { |val|
-              if (ann2 = validate1a(d[val], v)) && 
-                 d_check.delete(val) {:not_found} != :not_found
-                anno.concat(ann2)
-              end
-            }
+            count = 0
+            catch :enough do
+              ta.all? { |val|
+                if (ann2 = validate1a(d[val], v)) && # XXX check cut or not!
+                   d_check.delete(val) {:not_found} != :not_found
+                  anno.concat(ann2)
+                  throw :enough, true if (count += 1) == e
+                  true
+                end
+              }
+            end and validate_result(count >= s) { "not enough #{ta.inspect} for #{r.inspect}" }
           end
           end
         else
@@ -700,6 +1005,19 @@ module CDDL
       }
     end
 
+    def validate1a_ignore_encoding(d, where)
+      if String === d
+        validate1a(d.force_encoding(Encoding::BINARY), where) or (
+          text = d.force_encoding(Encoding::UTF_8).scrub {
+            fail "can't use bytes as text"
+          } rescue :NOT_A_TEXT_STRING
+          validate1a(text, where)
+        )
+      else
+        validate1a(d, where)
+      end
+    end
+
     def validate1a(d, where)
       if ann = validate1(d, where)
         here = [d, where]
@@ -712,6 +1030,8 @@ module CDDL
     end
 
     OPERATORS = {lt: :<, le: :<=, gt: :>, ge: :>=, eq: :==, ne: :!=}
+    OPERATORS_MATCH = {eq: true, ne: false}
+    FLOAT_AI_FROM_SIZE = {3 => 25, 5 => 26, 9 => 27}
 
     def validate1(d, where)
       if ENV["CDDL_TRACE"]
@@ -722,6 +1042,9 @@ module CDDL
       @last_data = d
       @last_rule = where
       ann = nil
+      if CBOR::Tagged === d && d.tag === 888 && ENV["CDDL_888"]
+        return []
+      end
       case where[0]
       when :type1
         if where[1..-1].any? {|r| ann = validate1a(d, r)}
@@ -745,16 +1068,55 @@ module CDDL
           idx, ann = validate_forward(d, 0, where)
           ann if validate_result(idx == d.size) { "#{validate_diag.inspect} -- cannot complete (#{idx}, #{d.size}) array #{d} for #{where}" }
         end
-      when :text, :int, :float, :bytes
+      when :int, :float
         _, v = extract_value(where)
         [] if d == v
+      when :text, :bytes
+        _, v = extract_value(where)
+        [] if d == v && ((d.encoding == Encoding::BINARY) == (v.encoding == Encoding::BINARY))
       when :range
         [] if where[2] === d && where[1].include?(d)
       when :anno
-        target = where[2]
-        if ann = validate1a(d, target)
-          control = where[3]
-          case where[1]
+        _, conop, target, control = where
+        if conop == :cat || conop == :plus || conop == :det
+          ok1, v1, vt1 = extract_value(target)
+          ok2, v2, vt2 = extract_value(control)
+          warn ["ANNO0", ok1, v1, vt1, ok2, v2, vt2, d].inspect  if ENV["CDDL_TRACE"]
+          if ok1 && ok2
+            v2 = Integer(v2) if vt1 == Integer
+            if conop == :det
+              v1 = remove_indentation(v1)
+              v2 = remove_indentation(v2)
+            end
+            # warn ["ANNO", ok1, v1, vt1, ok2, v2, vt2, d].inspect
+            [] if d == v1 + v2  # XXX Focus ArgumentError
+          elsif conop == :cat && String === d
+            warn ["CAT-L", ok1, v1, vt1, ok2, v2, vt2, d].inspect  if ENV["CDDL_TRACE"]
+            if ok1 && vt1 == String
+              # d and lhs (v1) need to agree in encoding
+              if d.encoding == v1.encoding
+                if d[0...v1.length] == v1
+                  d2 = d[v1.length..-1]
+                  warn ["CAT-L1", d2, d2.encoding, control].inspect  if ENV["CDDL_TRACE"]
+                  validate1a_ignore_encoding(d2, control)
+                end
+              end
+            elsif ok2 && vt2 == String
+            warn ["CAT-R", ok1, v1, vt1, ok2, v2, vt2, d].inspect  if ENV["CDDL_TRACE"]
+              if d[-v2.length..-1] == v2
+                d1 = d[0...-v2.length]
+                validate1a(d1, control)
+              end
+            end
+          elsif conop == :plus && Integer === d
+            if ok1 && vt1 == Integer
+              validate1a(d - Integer(v1), control)
+            elsif ok2 && vt2 == Integer
+              validate1a(d - Integer(v2), target)
+            end
+          end
+        elsif ann = validate1a(d, target)
+          case conop
           when :size
             case d
             when Integer
@@ -793,11 +1155,20 @@ module CDDL
               $default_warned = true
             end
             ann
+          when :feature
+            nm, det = extract_feature(control, d)
+            $features[nm][det] = true
+            ann if !CDDL_FEATURE_REJECT[nm]
           when :lt, :le, :gt, :ge, :eq, :ne
-            op = OPERATORS[where[1]]
+            op = OPERATORS[conop]
             ok, v, _vt = extract_value(control)
             if ok
               ann if d.send(op, v) rescue nil # XXX Focus ArgumentError
+            else
+              needs_to_match = OPERATORS_MATCH[conop]
+              unless needs_to_match.nil?
+                ann if !!validate1(d, control) == needs_to_match
+              end
             end
           when :regexp
             ann if (
@@ -810,10 +1181,181 @@ module CDDL
               end
             end
             )
+          when :abnf, :abnfb
+            ann if (
+            if String === d
+              ok, v, vt = extract_value(control)
+              if ok && vt == String
+                begin
+                  ABNF_PARSER_FOR_STRING[v].validate(
+                    d.dup.force_encoding(ABNF_ENCODING_FOR_CONOP[conop]).codepoints.pack("U*")
+                  )
+                  true
+                rescue => e
+                  # warn "*** #{e}" # XXX
+                  @last_message = e.to_s.force_encoding(Encoding::UTF_8)
+                  nil
+                end
+              end
+            end
+            )
           when :cbor
             ann if validate1((CBOR::decode(d) rescue :BAD_CBOR), control)
           when :cborseq
             ann if validate1((CBOR::decode("\x9f".b << d << "\xff".b) rescue :BAD_CBOR), control)
+          when :cbordet
+            decoded = CBOR::decode(d) rescue :BAD_CBOR
+            if d != decoded.to_deterministic_cbor
+              @last_message = "CBOR #{d.inspect} not deterministically encoded"
+              nil
+            else
+              ann if validate1(decoded, control)
+            end
+          when :cborseqdet
+            decoded = CBOR::decode("\x9f".b << d << "\xff".b) rescue :BAD_CBOR
+            if d != unpack_array_to_sequence(decoded.to_deterministic_cbor, d.inspect)
+              @last_message = "CBOR Sequence #{d.inspect} not deterministically encoded"
+              nil
+            else
+              ann if validate1(decoded, control)
+            end
+          when :json
+            ann if validate1((JSON::load(d) rescue :BAD_JSON), control)
+          when :decimal, :base10
+            if conop == :decimal
+              unless @decimal_warned
+                warn "** .decimal control operator is now called .base10"
+                @decimal_warned = true
+              end
+            end
+
+            ann if (
+                if /\A0|-?[1-9][0-9]*\z/ === d
+                  validate1((d.to_i rescue :BAD_JSON), control)
+                end
+              )
+          when :join
+            t = control
+            v = if t[0] == :array
+                  t[1..-1].map { |el|
+                    if el[0..2] == [:member, 1, 1]
+                      ok, v, vt = extract_value(el[4])
+                      if ok
+                        [true, v, vt]
+                      else
+                        [false, el[4]]
+                      end
+                    end
+                  }
+                end
+            warn "@@@JOIN@@@ #{v.inspect}" if ENV["CDDL_TRACE"]
+            ok = true
+            if v
+              ix = 0
+              rest = d.dup
+              while ix < v.length
+                if left = v[ix]
+                  if left[0]    # match constant value first
+                    fail "Not a string for #{left.inspect} in #{where}" unless String == left[2]
+                    want = left[1]
+                    have = rest[0...left[1].length]
+                    if want == have
+                      rest[0...left[1].length] = ''
+                      ix += 1
+                      next
+                    else
+                      fail ".join: want #{want.inspect}, have #{have.inspect}"
+                    end
+                  else
+                    ix += 1
+                    if ix == v.length # match remaining
+                      warn "@@@JOIN ok in #{ok.inspect} rest #{rest.inspect}"  if ENV["CDDL_TRACE"]
+                      ok &&= validate1(rest, left[1])
+                      warn "@@@JOIN ok out #{ok.inspect}"  if ENV["CDDL_TRACE"]
+                      # more diag
+                      rest = ''
+                      next
+                    else
+                      if mid = v[ix]
+                        if mid[0] # have constant value to split over
+                          fail "Not a string for #{mid} in #{where}" unless String == mid[2]
+                          rest1, rest2 = rest.split(mid[1], 2)
+                          if rest2
+                            warn "@@@JOIN ok in #{ok.inspect} rest1 #{rest1.inspect}"   if ENV["CDDL_TRACE"]
+                            ok &&= validate1(rest1, left[1])
+                            warn "@@@JOIN ok out #{ok.inspect}"   if ENV["CDDL_TRACE"]
+                            rest = rest2
+                            ix += 1
+                            next
+                          else
+                            fail "Can't find #{mid[1].inspect} in #{rest.inspect}"
+                          end
+                        else
+                          fail "Cannot validate consecutive non-constant members of .join in #{where}"
+                        end
+                      else
+                        fail "Cannot handle .join over #{t[ix+1]} in #{where}"
+                      end
+                    end
+                  end
+                else
+                  fail "Cannot handle .join over #{t[ix+1]} in #{where}"
+                end
+              end
+              fail "Can't match #{rest.inspect} for .join in #{where}" if rest != ''
+              ann if ok
+            else
+              fail "Don't know yet how to validate against #{where}"
+            end
+          when :b64u, :"b64u-sloppy", :b64c, :"b64c-sloppy", :b45, :b32, :h32, :hex, :hexlc, :hexuc
+            ann if (
+                String === d && (
+                decoded = case conop
+                    when :b64u
+                      /[+\/=]/ !~ d &&
+                      Base64.urlsafe_decode64(d)
+                    when :"b64u-sloppy"
+                      /[-_=]/ !~ d &&
+                      Base64.decode64(d.tr("+/", "-_"))
+                    when :b64c
+                      Base64.strict_decode64(d)
+                    when :"b64c-sloppy"
+                      Base64.decode64(d)
+                    when :b32
+                      /=/ !~ d &&
+                        (Base32.table = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567") &&
+                        Base32.decode(d)
+                    when :b45
+                      Base45Lite.decode(d)
+                    when :h32
+                      /=/ !~ d &&
+                        (Base32.table = "0123456789ABCDEFGHIJKLMNOPQRSTUV") &&
+                        Base32.decode(d)
+                    when :hex
+                      /\A[0-9a-fA-F]*\z/ =~ d && [d].pack("H*")
+                    when :hexuc
+                      /\A[0-9A-F]*\z/ =~ d && [d].pack("H*")
+                    when :hexlc
+                      /\A[0-9a-f]*\z/ =~ d && [d].pack("H*")
+                    else fail "Cannot happen"
+                    end
+                ) && validate1(decoded.b, control)
+              )
+          when :printf
+            ann if String === d && (
+                     ok, fmt, *v = extract_arg0(control)
+                     if ok && String == fmt[1]
+                       fmt = fmt[0]
+                       # warn "** ok #{ok.inspect} fmt #{fmt.inspect} v #{v.inspect}"
+                       decoded = d.scanf(fmt) # this is a bit too lenient, so let's do:
+                       encode_again = fmt % decoded
+                       if encode_again != d
+                         warn "** fmt #{fmt.inspect} d #{d.inspect} decoded #{decoded.inspect} encode_again #{encode_again.inspect}"
+                       else
+                         validate1(decoded, [:array, *v])
+                       end
+                     end
+                   )
           when :within
             if validate1(d, control)
               ann
@@ -841,20 +1383,43 @@ module CDDL
         when 3
           String === d && d.encoding != Encoding::BINARY # cheat
         when 6
-          CBOR::Tagged === d && d.tag == where[2] && validate1a(d.data, where[3])
+          if Integer === d
+            t = 2               # assuming only 2/3 match an Integer
+            if d < 0
+              d = ~d
+              t = 3
+            end
+            d = CBOR::Tagged.new(t, d == 0 ? "".b : d.digits(256).reverse!.pack("C*"))
+          end
+          CBOR::Tagged === d && (
+            Integer === where[2] ? d.tag == where[2] : validate1a(d.tag, where[2])
+          ) && validate1a(d.data, where[3])
         when 7
           t, v = extract_value(where)
           if t
             v.eql? d
           else
-            case where[2]
+            case w2 = where[2]
             when nil
-              # XXX
-              fail
+              Float === d || CBOR::Simple === d || [false, true, nil].include?(d)
+            when Array
+              headnum = case d
+                        when Float
+                          FLOAT_AI_FROM_SIZE[d.to_cbor.size]
+                        when false
+                          20
+                        when true
+                          21
+                        when nil
+                          22
+                        when CBOR::Simple
+                          d.value
+                        end
+              validate1a(headnum, w2)
             when 25, 26, 27
-              Float === d
+              Float === d && FLOAT_AI_FROM_SIZE[d.to_cbor.size] == w2
             else
-              fail
+              fail [:val7, d, where].inspect
             end
           end
         else
@@ -904,6 +1469,10 @@ module CDDL
                                else
                                  [:type1]
                                end
+          elsif s = ENV["CDDL_INVENT"]
+            s = "_" if s == ""
+            r = [:type1, [:text, "#{s}-#{name}"]]
+            return r
           end
         end
         if r
@@ -949,29 +1518,69 @@ module CDDL
                      end
     end
 
+    STRING_ESCAPES = {
+      "\"" => "\"",
+      "/" => "/",
+      "\\" => "\\",
+      "'" => "'",
+      "b" => "\b",
+      "f" => "\f",
+      "n" => "\n",
+      "r" => "\r",
+      "t" => "\t",
+    }
+
     def value(n)
       # cheat:
       # warn n
       s = n.to_s
       if s[-1] == "'"
-        parts = s.split("'", 3)
-        if parts[2] != "" || parts[1] =~ /\\/
-          warn "*** Can't handle backslashes in #{s.inspect} yet"
+        bsqual, *parts = s.split("'", -1)
+        if parts[-1] != ""
+          warn "*** Problem decoding byte string #{s.inspect}"
         end
+        bsval = parts[0...-1].join("'").gsub(/\\u([Dd][89AaBb][0-9a-zA-Z]{2})\\u([Dd][CcDdEeFf][0-9a-zA-Z]{2})|\\u([0-9a-zA-Z]{4})|\\u\{([0-9a-zA-Z]+)\}|\r?(\n)|\\([^u])/) {
+          if hex = $3 || $4
+            hex.to_i(16).chr(Encoding::UTF_8)
+          elsif lf = $5
+            lf
+          elsif escaped = $6
+            STRING_ESCAPES[$6] or (
+              fail "Invalid String Escape #{escaped.inspect} in #{s.inspect}"
+            )
+          else
+            ((($1.to_i(16) & 0x3ff) << 10) +
+             ($2.to_i(16) & 0x3ff) + 0x10000).chr(Encoding::UTF_8)
+          end
+        }
         [:bytes,
-         case parts[0]
+         case bsqual
          when ""
-           parts[1].b
+           bsval.b
          when "h"
-           parts[1].gsub(/\s/, "").chars.each_slice(2).map{ |x| Integer(x.join, 16).chr("BINARY") }.join
+           bsval.gsub(/\s/, "").chars.each_slice(2).map{ |x| Integer(x.join, 16).chr("BINARY") }.join.b
          when "b64"
-            Base64.urlsafe_decode64(parts[1])
+            Base64.urlsafe_decode64(bsval)
          else
-           warn "*** Can't handle byte string type #{parts[0].inspect} yet"
+           warn "*** Can't handle byte string type #{bsqual.inspect} yet"
          end
         ]
       else
-        val = eval(n.to_s)
+        if s[-1] == '"'
+          s.gsub!(/\\u([Dd][89AaBb][0-9a-zA-Z]{2})\\u([Dd][CcDdEeFf][0-9a-zA-Z]{2})|\\([^u]|u[0-9a-zA-Z]{4})/) {
+            if $3
+              "\\#$3" # skip this, use eval parser
+            else
+              ((($1.to_i(16) & 0x3ff) << 10) +
+               ($2.to_i(16) & 0x3ff) + 0x10000).chr(Encoding::UTF_8)
+            end
+          }
+        end
+        if s =~ /p.*\d\z/       # hexfloat
+          val = s.scanf("%a").first
+        else
+          val = eval(s)
+        end
         # warn val
         case val
          when Integer; [:int, val]
@@ -1077,9 +1686,13 @@ module CDDL
     def g_process(name, g, genericargs)
       r, ids = g
       args = genericargs.children(:type1).map {|x| type1(x, true)}
+      # warn "** args #{args.inspect}"
       fail "number of args #{name} #{ids.size} #{args.size}" if ids.size != args.size
       bindings = Hash[ids.zip(args)]
-      r_process("#{name}<#{genericargs._strip}>", r, bindings)
+      # warn "** bindings: #{bindings.inspect}"
+      genname = "#{name}<#{args.inspect}>" # XXX this works up to a RECURSE...
+      # warn "** genname #{genname}"
+      r_process(genname, r, bindings)
     end
 
     def type_recall(name, canbegroup, genericargs)
@@ -1155,8 +1768,16 @@ module CDDL
 
     BRACE = {"{" => :map, "[" => :array}
     RANGE_EXCLUDE_END = {".." => false, "..." => true}
-    SUPPORTED_ANNOTATIONS = [:bits, :size, :regexp, :cbor, :cborseq, :within, :and,
-                             :default, :lt, :le, :gt, :ge, :eq, :ne]
+    SUPPORTED_ANNOTATIONS = [:bits, :size, :regexp,
+                             :cbor, :cborseq, :cbordet, :cborseqdet,
+                             :within, :and,
+                             :default, :lt, :le, :gt, :ge, :eq, :ne,
+                             :feature, :abnf, :abnfb, :det, :cat, :plus,
+                             :json, :decimal, :base10, :join,
+                             :b64u, :"b64u-sloppy", :b64c, :"b64c-sloppy",
+                             :b45, :b32, :h32, :hex, :hexlc, :hexuc,
+                             :printf,
+                            ]
 
     def type1(n, canbegroup = false)
 #      puts "NVALUE #{n.value.inspect}"
@@ -1188,6 +1809,13 @@ module CDDL
         when /\A#(\d+)/
           maj = $1.to_i
           s = [:prim, maj, *n.children(:uint).map(&:to_s).map(&:to_i)]
+          if tn = n.headnumber
+            if ui = tn.uint
+              s << ui.to_s.to_i
+            elsif tt = tn.type
+              s << type(tt)
+            end
+          end
           if tagged_type = n.type
               s << type(tagged_type)
           end
@@ -1218,6 +1846,26 @@ module CDDL
              v
            }
           ]
+        when /\A~/
+          if tn = n.typename
+            s = type_recall(tn.to_s, false, n.genericarg).flatten(1)
+          else
+            fail "No typename in unwrap #{n}"
+          end
+          if s[0] != :type1
+            fail [:UNWRAPs0, s].inspect
+          end
+          case s[1]
+          when :map, :array
+            [:grpent, s[2..-1]]
+          when :prim
+            if s[2] != 6
+              fail [:UNWRAPs2, s].inspect
+            end
+            s[4]                # check :type1?
+          else
+            fail [:UNWRAPs1, s].inspect
+          end
         else
           "unimplemented #{n}"
         end