cddl 0.8.8 → 0.10.3

data/lib/cddl.rb

@@ -2,9 +2,13 @@ require 'abnc'
 require 'pp'
 require 'pathname'
 require 'cbor-pure' unless defined?(CBOR::Tagged)
+require 'cbor-deterministic'
 require 'regexp-examples'
+require 'abnftt'
 require 'colorize'
 require 'base64'
+require 'base32'
+require 'base45_lite'
 
 module CDDL
 
@@ -21,6 +25,20 @@ module CDDL
 
   class Parser
 
+    FEATURE_REJECT_RE = /\A\^/
+    # CDDL_FEATURE_OK=cbor,^json
+    CDDL_FEATURE_OK, CDDL_FEATURE_REJECT =
+                 if ok = ENV["CDDL_FEATURE_OK"]
+                   ok.split(/,\s*/)
+                     .partition{|s| s[0] !~ FEATURE_REJECT_RE}
+                     .map {|l| Hash[l.map {|feature|
+                                      [feature.sub(FEATURE_REJECT_RE, ''),
+                                       true]}]}
+                 else
+                   [{}, {}]
+                 end
+
+
     class BackTrack < Exception; end
 
     def initialize(source_text)
@@ -55,6 +73,8 @@ module CDDL
       @ast = @abnf.ast?
       # our little argument stack for rule processing
       @insides = []
+      # collect error information
+      @last_message = ""
     end
 
     def apr                     # for debugging
@@ -127,7 +147,7 @@ module CDDL
         if subtree[0] == :type1 && subtree[1..-1].all? {|x| x[0] == :int}
           if enumname = subtree.cbor_annotations rescue nil
             enumname = cname(enumname.first)
-            subtree[1..-1].each do |x| 
+            subtree[1..-1].each do |x|
               if memname = x.cbor_annotations
                 memname = "#{enumname}_#{cname(memname.first)}"
                 add.(memname, x[1].to_s)
@@ -243,10 +263,41 @@ module CDDL
       [rule[0], *rule[1]]
     end
 
+    def remove_indentation(s)
+      l = s.lines
+      indent = l.grep(/\S/).map {|l| l[/^\s*/].size}.min
+      l.map {|l| l.sub(/^ {0,#{indent}}/, "")}.join
+    end
+
+    def unpack_array_to_sequence(content, where)
+      # remove the first head
+      n = case content.getbyte(0) - (4 << 5)
+          when 0..23; 1
+          when 24; 2
+          when 25; 3
+          when 26; 5
+          when 27; 9      # unlikely :-)
+          else fail ".cborseq sequence for #{where} not an array"
+          end
+      content[0...n] = ''
+      content
+    end
+
+
     # Memoize a bit here
 
     REGEXP_FOR_STRING = Hash.new {|h, k|
-      h[k] = Regexp.new("\\A#{k}\\z")
+      h[k] = Regexp.new("\\A(?:#{k})\\z")
+    }
+
+    ABNF_PARSER_FOR_STRING = Hash.new {|h, k|
+      grammar = "cddl-t0p--1eve1-f0r--abnf = " << k # XXX
+      h[k] = ABNF.from_abnf(grammar)
+    }
+
+    ABNF_ENCODING_FOR_CONOP = {
+      abnf: Encoding::UTF_8,
+      abnfb: Encoding::BINARY
     }
 
     def generate
@@ -329,8 +380,12 @@ module CDDL
           unless kr
             case vr[0]
             when :grpent
-              fail "grpent in map #{vr.inspect}" unless vr.size == 2
-              g = Array.new(st) { generate1(vr[1], true) }.flatten(1)
+              # fail "grpent in map #{vr.inspect}" unless vr.size == 2
+              g = Array.new(st) {
+                vr[1..-1].map{ |vrx|
+                  generate1(vrx, true)
+                }.flatten(1)
+              }.flatten(1)
               # warn "GGG #{g.inspect}"
               return g
             when :grpchoice
@@ -389,7 +444,7 @@ module CDDL
       when :anno
         target = where[2]
         control = where[3]
-        case where[1]
+        case conop = where[1]
         when :size
           should_be_int = generate1(control)
           unless (Array === target && target[0] == :prim && [0, 2, 3].include?(target[1])) && Integer === should_be_int && should_be_int >= 0
@@ -435,6 +490,40 @@ module CDDL
             $default_warned = true
           end
           generate1(target, inmap)
+        when :feature
+          feature, = extract_feature(control, nil)
+          # warn "@@@ GENFEAT #{feature.inspect} #{CDDL_FEATURE_REJECT[feature].inspect}"
+          if CDDL_FEATURE_REJECT[feature]
+            fail BackTrack.new("Feature '#{feature}' rejected")
+          end
+          generate1(target, inmap)
+        when :cat, :det
+          lhs = generate1(target, inmap)
+          rhs = generate1(control)
+          if conop == :det
+            lhs = remove_indentation(lhs)
+            rhs = remove_indentation(rhs)
+          end
+          begin
+            lhs + rhs
+          rescue Exception => e
+            fail "Can't #{lhs.inspect} .cat #{rhs.inspect}: #{e.message}"
+          end
+        when :plus
+          lhs = generate1(target, inmap)
+          rhs = generate1(control)
+          begin
+            case lhs
+            when Integer
+              lhs + Integer(rhs)
+            when Numeric
+              lhs + rhs
+            else
+              fail "#{lhs.inspect}: Not a number"
+            end
+          rescue Exception => e
+            fail "Can't #{lhs.inspect} .plus #{rhs.inspect}: #{e.message}"
+          end
         when :eq
           content = generate1(control)
           if validate1(content, where)
@@ -448,7 +537,7 @@ module CDDL
                     content = Integer(content)
                     case target[1]
                     when 0
-                      case where[1]
+                      case conop
                       when :lt
                         rand(0...content)
                       when :le
@@ -475,30 +564,88 @@ module CDDL
             fail "Don't know yet how to generate #{where}"
           end
           REGEXP_FOR_STRING[regexp].random_example(max_repeater_variance: 5)
-        when :cbor, :cborseq
+        when :abnf, :abnfb
+          grammar = generate1(control)
+          bytes = true if target == [:prim, 2]
+          bytes = false if target == [:prim, 3]
+          unless !bytes.nil? && String === grammar
+            fail "Don't know yet how to generate #{where}"
+          end
+          grammar << "\n" unless grammar =~ /\n\z/
+          out = ABNF_PARSER_FOR_STRING[grammar].generate
+          if conop == :abnfb
+            out = out.codepoints.pack("C*")
+          end
+          enc = bytes ? Encoding::BINARY : Encoding::UTF_8
+          out.force_encoding(enc)
+        when :cbor, :cborseq, :cbordet, :cborseqdet
           unless target == [:prim, 2]
             fail "Don't know yet how to generate #{where}"
           end
-          content = CBOR::encode(generate1(control))
-          if where[1] == :cborseq
+          input = generate1(control)
+          if conop == :cbordet || conop == :cborseqdet
+            input = input.cbor_prepare_deterministic
+          end
+          content = CBOR::encode(input)
+          if conop == :cborseq || conop == :cborseqdet
             # remove the first head
-            n = case content.getbyte(0) - (4 << 5)
-                when 0..23; 1
-                when 24; 2
-                when 25; 3
-                when 26; 5
-                when 27; 9      # unlikely :-)
-                else fail "Generated .cborseq sequence for #{where} not an array"
-                end
-            content[0...n] = ''
+            content = unpack_array_to_sequence(content, where)
+          end
+          content
+        when :json
+          unless target == [:prim, 3]
+            fail "Don't know yet how to generate #{where}"
           end
+          content = JSON::dump(generate1(control))
+          content
+        when :decimal
+          unless target == [:prim, 3]
+            fail "Don't know yet how to generate #{where}"
+          end
+          content = Integer(generate1(control)).to_s
+          content
+        when :join
+          content = generate1(control)
+          unless Array === content && 
+                 content.all? {|x| String === x &&
+                               ((target == [:prim, 2] && x.encoding == Encoding::BINARY) ||
+                                (target == [:prim, 3] && x.encoding != Encoding::BINARY))}
+            fail "Don't know yet how to generate #{where}"
+          end
+          content = content.join
+          content
+        when :b64u, :"b64u-sloppy", :b64c, :"b64c-sloppy",
+             :b45, :b32, :h32, :hex, :hexlc, :hexuc
+          content = generate1(control)
+          unless String === content
+            fail "Don't know yet how to generate #{where}"
+          end
+          content = case conop
+                    when :b64u, :"b64u-sloppy"
+                      Base64.urlsafe_encode64(content, padding: false)
+                    when :b64c, :"b64c-sloppy"
+                      Base64.strict_encode64(content)
+                    when :b45
+                      Base45Lite.encode(content)
+                    when :b32
+                      Base32.table = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"
+                      Base32.encode(content).gsub("=", "")
+                    when :h32
+                      Base32.table = "0123456789ABCDEFGHIJKLMNOPQRSTUV"
+                      Base32.encode(content).gsub("=", "")
+                    when :hex, :hexuc
+                      content.unpack("H*")[0].upcase
+                    when :hexlc
+                      content.unpack("H*")[0]
+                    else fail "Cannot happen"
+                    end
           content
         when :within, :and
           32.times do
             content = generate1(target)
             if validate1(content, control)
               return content
-            elsif where[1] == :within
+            elsif conop == :within
               warn "*** #{content.inspect} meets #{target.inspect} but not #{control.inspect}"
             end
           end
@@ -523,7 +670,7 @@ module CDDL
       end
     end
 
-    VALUE_TYPE = {text: String, int: Integer, float: Float}
+    VALUE_TYPE = {text: String, bytes: String, int: Integer, float: Float}
     SIMPLE_VALUE = {
       [:prim, 7, 20] => [true, false, :bool],
       [:prim, 7, 21] => [true, true, :bool],
@@ -536,9 +683,57 @@ module CDDL
         [true, t[1], vt]
       elsif v = SIMPLE_VALUE[t]
         v
+      elsif t[0] == :anno
+        _, conop, target, control = t
+        # warn ["EXV0", conop, target, control].inspect
+        if conop == :cat || conop == :plus || conop == :det
+          ok1, v1, vt1 = extract_value(target)
+          ok2, v2, vt2 = extract_value(control)
+          # warn ["EXV", ok1, v1, vt1, ok2, v2, vt2].inspect
+          if ok1 && ok2
+            if vt1 == Integer
+              [true, v1 + Integer(v2), Integer] if vt2 == Integer || vt2 == Float
+            elsif vt1 == Float
+              [true, v1 + v2, vt1] if vt2 == Integer || vt2 == Float
+            else
+              if conop == :det
+                v1 = remove_indentation(v1)
+                v2 = remove_indentation(v2)
+              end
+              [true, v1 + v2, vt1] if vt1 == vt2
+            end
+          end rescue nil
+        end
+      end || [false]
+    end
+
+    def extract_array(t)
+      return [false] unless t[0] == :array
+      [true, *t[1..-1].map { |el|
+         return [false] unless el[0..3] == [:member, 1, 1, nil]
+         ok, v, vt = extract_value(el[4])
+         return [false] unless ok
+         [v, vt]
+       }]
+    end
+
+    def extract_feature(control, d)
+      ok, v, vt = extract_value(control)
+      if ok
+        nm = v
+        det = d
+        warn "*** feature controller should be a string: #{control.inspect}" unless String == vt
       else
-        [false]
+        ok, *v = extract_array(control)
+        if ok && v.size == 2
+          nm = v[0][0]
+          det = v[1][0]
+          warn "*** first element of feature controller should be a string: #{control.inspect}" unless String === nm
+        else
+          warn "*** feature controller not implemented: #{control.inspect}"
+        end
       end
+      [nm, det]
     end
 
     def validate_diag
@@ -547,11 +742,19 @@ module CDDL
 
     def validate(d, warn=true)
       @recursion = 0
+      $features = Hash.new {|h, k| h[k] = {}}
       result = validate1a(d, rules)
-      unless result
-        if warn
+      if warn
+        if result
+          if $features != {}
+            features = $features.reject {|k, v| CDDL_FEATURE_OK[k.to_s] }
+            # warn "@@@ FEAT #{CDDL_FEATURE_OK.inspect} #{CDDL_FEATURE_REJECT.inspect}"
+            warn "** Features potentially used (#$fn): #{features.map {|k, v| "#{k}: #{v.keys}"}.join(", ")}" if features != {}
+          end
+        else
           warn "CDDL validation failure (#{result.inspect} for #{d.inspect}):"
           PP::pp(validate_diag, STDERR)
+          puts(validate_diag.cbor_diagnostic)
         end
       end
       result
@@ -559,7 +762,7 @@ module CDDL
 
     def validate_result(check)
       check || (
-        @last_message = yield
+        @last_message << yield
         false
       )
     end
@@ -592,7 +795,9 @@ module CDDL
             ann.concat(ann2)
           end
           if occ < s
-            @last_message = "occur not reached in array #{d} for #{where}"
+            # warn "*** lme #{@last_message.encoding} #{@last_message}"
+            # warn "*** #{"\noccur #{occ} < #{s}, not reached at #{i} in array #{d} for #{where}".encoding}"
+            @last_message << "\noccur #{occ} < #{s}, not reached at #{i} in array #{d} for #{where}"
             return [false, ann]
           end
         end
@@ -681,13 +886,17 @@ module CDDL
                           puts "COMPLEX: #{k.inspect} #{simple.inspect} #{simpleval.inspect}"         if ENV["CDDL_TRACE"]
             keys = d_check.keys
             ta, keys = keys.partition{ |key| validate1(key, k)}
-            # XXX check ta.size against s/e
-            ta.all? { |val|
-              if (ann2 = validate1a(d[val], v)) && 
-                 d_check.delete(val) {:not_found} != :not_found
-                anno.concat(ann2)
-              end
-            }
+            count = 0
+            catch :enough do
+              ta.all? { |val|
+                if (ann2 = validate1a(d[val], v)) && # XXX check cut or not!
+                   d_check.delete(val) {:not_found} != :not_found
+                  anno.concat(ann2)
+                  throw :enough, true if (count += 1) == e
+                  true
+                end
+              }
+            end and validate_result(count >= s) { "not enough #{ta.inspect} for #{r.inspect}" }
           end
           end
         else
@@ -741,16 +950,31 @@ module CDDL
           idx, ann = validate_forward(d, 0, where)
           ann if validate_result(idx == d.size) { "#{validate_diag.inspect} -- cannot complete (#{idx}, #{d.size}) array #{d} for #{where}" }
         end
-      when :text, :int, :float, :bytes
+      when :int, :float
         _, v = extract_value(where)
         [] if d == v
+      when :text, :bytes
+        _, v = extract_value(where)
+        [] if d == v && ((d.encoding == Encoding::BINARY) == (v.encoding == Encoding::BINARY))
       when :range
         [] if where[2] === d && where[1].include?(d)
       when :anno
-        target = where[2]
-        if ann = validate1a(d, target)
-          control = where[3]
-          case where[1]
+        _, conop, target, control = where
+        if conop == :cat || conop == :plus || conop == :det
+          ok1, v1, vt1 = extract_value(target)
+          ok2, v2, vt2 = extract_value(control)
+          # warn ["ANNO0", ok1, v1, vt1, ok2, v2, vt2, d].inspect
+          if ok1 && ok2
+            v2 = Integer(v2) if vt1 == Integer
+            if conop == :det
+              v1 = remove_indentation(v1)
+              v2 = remove_indentation(v2)
+            end
+            # warn ["ANNO", ok1, v1, vt1, ok2, v2, vt2, d].inspect
+            [] if d == v1 + v2  # XXX Focus ArgumentError
+          end
+        elsif ann = validate1a(d, target)
+          case conop
           when :size
             case d
             when Integer
@@ -789,8 +1013,12 @@ module CDDL
               $default_warned = true
             end
             ann
+          when :feature
+            nm, det = extract_feature(control, d)
+            $features[nm][det] = true
+            ann if !CDDL_FEATURE_REJECT[nm]
           when :lt, :le, :gt, :ge, :eq, :ne
-            op = OPERATORS[where[1]]
+            op = OPERATORS[conop]
             ok, v, _vt = extract_value(control)
             if ok
               ann if d.send(op, v) rescue nil # XXX Focus ArgumentError
@@ -806,10 +1034,98 @@ module CDDL
               end
             end
             )
+          when :abnf, :abnfb
+            ann if (
+            if String === d
+              ok, v, vt = extract_value(control)
+              if ok && vt == String
+                begin
+                  ABNF_PARSER_FOR_STRING[v].validate(
+                    d.dup.force_encoding(ABNF_ENCODING_FOR_CONOP[conop]).codepoints.pack("U*")
+                  )
+                  true
+                rescue => e
+                  # warn "*** #{e}" # XXX
+                  @last_message = e.to_s.force_encoding(Encoding::UTF_8)
+                  nil
+                end
+              end
+            end
+            )
           when :cbor
             ann if validate1((CBOR::decode(d) rescue :BAD_CBOR), control)
           when :cborseq
             ann if validate1((CBOR::decode("\x9f".b << d << "\xff".b) rescue :BAD_CBOR), control)
+          when :cbordet
+            decoded = CBOR::decode(d) rescue :BAD_CBOR
+            if d != decoded.to_deterministic_cbor
+              @last_message = "CBOR #{d.inspect} not deterministically encoded"
+              nil
+            else
+              ann if validate1(decoded, control)
+            end
+          when :cborseqdet
+            decoded = CBOR::decode("\x9f".b << d << "\xff".b) rescue :BAD_CBOR
+            if d != unpack_array_to_sequence(decoded.to_deterministic_cbor, d.inspect)
+              @last_message = "CBOR Sequence #{d.inspect} not deterministically encoded"
+              nil
+            else
+              ann if validate1(decoded, control)
+            end
+          when :json
+            ann if validate1((JSON::load(d) rescue :BAD_JSON), control)
+          when :decimal
+            ann if (
+                if /\A0|-?[1-9][0-9]*\z/ === d
+                  validate1((d.to_i rescue :BAD_JSON), control)
+                end
+              )
+          when :join
+            ok, *v = extract_array(control)
+            # warn "@@@JJJ@@@ #{ok.inspect} #{v.inspect}"
+            if ok
+              expected = v.map {|ve|
+                fail "Not a string in #{where}" unless String === ve[0]
+                ve[0]
+              }.join
+              ann if d == expected
+            else
+              fail "Don't know yet how to validate against #{where}"
+            end
+          when :b64u, :"b64u-sloppy", :b64c, :"b64c-sloppy", :b45, :b32, :h32, :hex, :hexlc, :hexuc
+            ann if (
+                String === d && (
+                decoded = case conop
+                    when :b64u
+                      /=/ !~ d &&
+                      Base64.urlsafe_decode64(d)
+                    when :"b64u-sloppy"
+                      /[-_=]/ !~ d &&
+                      Base64.decode64(d.tr("+/", "-_"))
+                    when :b64c
+                      Base64.strict_decode64(d)
+                    when :"b64c-sloppy"
+                      Base64.decode64(d)
+                    when :b32
+                      /=/ !~ d &&
+                        (Base32.table = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567") &&
+                        Base32.decode(d)
+                    when :b45
+                      Base45Lite.decode(d)
+                    when :h32
+                      /=/ !~ d &&
+                        (Base32.table = "0123456789ABCDEFGHIJKLMNOPQRSTUV") &&
+                        Base32.decode(d)
+                    when :hex
+                      /\A[0-9a-fA-F]*\z/ =~ d && [d].pack("H*")
+                    when :hexuc
+                      /\A[0-9A-F]*\z/ =~ d && [d].pack("H*")
+                    when :hexlc
+                      /\A[0-9a-f]*\z/ =~ d && [d].pack("H*")
+                    else fail "Cannot happen"
+                    end
+                ) && validate1(decoded.b, control)
+              )
           when :within
             if validate1(d, control)
               ann
@@ -837,6 +1153,14 @@ module CDDL
         when 3
           String === d && d.encoding != Encoding::BINARY # cheat
         when 6
+          if Integer === d
+            t = 2               # assuming only 2/3 match an Integer
+            if d < 0
+              d = ~d
+              t = 3
+            end
+            d = CBOR::Tagged.new(t, d == 0 ? "".b : d.digits(256).reverse!.pack("C*"))
+          end
           CBOR::Tagged === d && d.tag == where[2] && validate1a(d.data, where[3])
         when 7
           t, v = extract_value(where)
@@ -950,20 +1274,21 @@ module CDDL
       # warn n
       s = n.to_s
       if s[-1] == "'"
-        parts = s.split("'", 3)
-        if parts[2] != "" || parts[1] =~ /\\/
-          warn "*** Can't handle backslashes in #{s.inspect} yet"
+        bsqual, *parts = s.split("'", -1)
+        if parts[-1] != ""
+          warn "*** Problem decoding byte string #{s.inspect}"
         end
+        bsval = parts[0...-1].join("'").gsub(/\\(.)/){$1}
         [:bytes,
-         case parts[0]
+         case bsqual
          when ""
-           parts[1].b
+           bsval.b
          when "h"
-           parts[1].gsub(/\s/, "").chars.each_slice(2).map{ |x| Integer(x.join, 16).chr("BINARY") }.join
+           bsval.gsub(/\s/, "").chars.each_slice(2).map{ |x| Integer(x.join, 16).chr("BINARY") }.join.b
          when "b64"
-            Base64.urlsafe_decode64(parts[1])
+            Base64.urlsafe_decode64(bsval)
          else
-           warn "*** Can't handle byte string type #{parts[0].inspect} yet"
+           warn "*** Can't handle byte string type #{bsqual.inspect} yet"
          end
         ]
       else
@@ -1073,9 +1398,13 @@ module CDDL
     def g_process(name, g, genericargs)
       r, ids = g
       args = genericargs.children(:type1).map {|x| type1(x, true)}
+      # warn "** args #{args.inspect}"
       fail "number of args #{name} #{ids.size} #{args.size}" if ids.size != args.size
       bindings = Hash[ids.zip(args)]
-      r_process("#{name}<#{genericargs._strip}>", r, bindings)
+      # warn "** bindings: #{bindings.inspect}"
+      genname = "#{name}<#{args.inspect}>" # XXX this works up to a RECURSE...
+      # warn "** genname #{genname}"
+      r_process(genname, r, bindings)
     end
 
     def type_recall(name, canbegroup, genericargs)
@@ -1142,7 +1471,7 @@ module CDDL
           end
         when :recurse_grpent # XXX we don't have the entry yet
         when :member
-          fail "map entry without member key given: #{member}" unless member[3] || member[4][0] == :grpent || member[4][0] == :grpchoice
+          fail "map entry #{member.inspect} without member key given: #{member}" unless member[3] || member[4][0] == :grpent || member[4][0] == :grpchoice
         else
           fail ["type1 member", member].inspect unless member[0] == :member
         end
@@ -1151,8 +1480,15 @@ module CDDL
 
     BRACE = {"{" => :map, "[" => :array}
     RANGE_EXCLUDE_END = {".." => false, "..." => true}
-    SUPPORTED_ANNOTATIONS = [:bits, :size, :regexp, :cbor, :cborseq, :within, :and,
-                             :default, :lt, :le, :gt, :ge, :eq, :ne]
+    SUPPORTED_ANNOTATIONS = [:bits, :size, :regexp,
+                             :cbor, :cborseq, :cbordet, :cborseqdet,
+                             :within, :and,
+                             :default, :lt, :le, :gt, :ge, :eq, :ne,
+                             :feature, :abnf, :abnfb, :det, :cat, :plus,
+                             :json, :decimal, :join,
+                             :b64u, :"b64u-sloppy", :b64c, :"b64c-sloppy",
+                             :b45, :b32, :h32, :hex, :hexlc, :hexuc,
+                            ]
 
     def type1(n, canbegroup = false)
 #      puts "NVALUE #{n.value.inspect}"
@@ -1214,6 +1550,26 @@ module CDDL
              v
            }
           ]
+        when /\A~/
+          if tn = n.typename
+            s = type_recall(tn.to_s, false, n.genericarg).flatten(1)
+          else
+            fail "No typename in unwrap #{n}"
+          end
+          if s[0] != :type1
+            fail [:UNWRAPs0, s].inspect
+          end
+          case s[1]
+          when :map, :array
+            [:grpent, s[2..-1]]
+          when :prim
+            if s[2] != 6
+              fail [:UNWRAPs2, s].inspect
+            end
+            s[4]                # check :type1?
+          else
+            fail [:UNWRAPs1, s].inspect
+          end
         else
           "unimplemented #{n}"
         end