cddl 0.8.8 → 0.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6c877883cb125426e061f527ba4375c0f021d70dfef060336fe0bb0f5e2ca3d4
-  data.tar.gz: 7a3dc4065f3616bf57c229b410782b35140018ecada4418c0ef45056c62aef0b
+  metadata.gz: 1c5fe473658f795abcc5e1599dd18ea3e62525c2f595a8e6c36d4fc9fa7654e8
+  data.tar.gz: 1d3789efc876946703170afb3aca08e458ad19c18ef8c8a5477852ab02fcc78d
 SHA512:
-  metadata.gz: b3987fbe82e80dddba2f650b1a12941ed6092708297302cf1a78dfe45b20b79d08e85901cb8739b5fc79eab8cde70ca8efb88698f4340da72c722a7a461565c6
-  data.tar.gz: 68c35688bac9b30411ee419e171860c463df28c3452bce1741399274e51295b609e0acc8df54e9876037f9d96ba02e369fd9465c2fe7c38bb9682d5ec19018cd
+  metadata.gz: f108776ab78dd95aa075e3450bcf121c8374508cbe9df27a539090af5faa4f9e70d378b39318bdd0b47ef97b29228ec4796e32b0229084942ee7106cdcb567c4
+  data.tar.gz: b642fe30b4c4aef6ee12fc42a7f1caab6b94ef3797789c5d7b8f190b447fdc4133669ddfa9ba923b91ce31760cfb41da6859b1e1c2d0d5b976f8133c2e742dc9

data/bin/cddl

@@ -1,4 +1,4 @@
-#!/usr/bin/env ruby
+#!/usr/bin/env RUBY_THREAD_VM_STACK_SIZE=5000000 ruby
 # -*- coding: utf-8 -*-
 # require_relative '../lib/cddl'
 require 'cddl'
@@ -14,22 +14,24 @@ def usage
   warn "Usage:"
   warn "#$0 spec.cddl generate [n]"
   warn "#$0 spec.cddl json-generate [n]"
-  warn "#$0 spec.cddl validate instance.cbor"
-  warn "#$0 spec.cddl validate instance.json"
+  warn "#$0 spec.cddl validate instance.cbor..."
+  warn "#$0 spec.cddl validate instance.json..."
   exit EX_USAGE
 end
 
-def read_arg(arg)
+def read_arg(arg, remember_fn = true)
   if arg == "-"
+    $fn = "(stdin)" if remember_fn
     STDIN.read
   else
     usage unless arg
+    $fn = arg if remember_fn
     File.read(arg, mode: "rb")
   end
 end
 
 def parser
-  @parser ||= CDDL::Parser.new(read_arg(ARGV[0]))
+  @parser ||= CDDL::Parser.new(read_arg(ARGV[0], false))
 end
 
 def my_pp(v)
@@ -48,13 +50,19 @@ def my_diag(v)
   end
 end
 
+retcode = 0
+
 begin
   case ARGV[1]
-  when /\A.p/
+  when /\A..*p/
     $annotate = true
     require_relative '../lib/cbor-pp'
   end
   case ARGV[1]
+  when /\A..*s/
+    $sequence = true
+  end
+  case ARGV[1]
   when /\Ar/                      # secret
     $advanced = true
     my_pp parser.rules
@@ -81,15 +89,25 @@ begin
       g = parser.generate
       puts JSON.pretty_generate(g)
     end
-  when /\Av/
-    instance = read_arg(ARGV[2])
-    instance = CBOR.decode(instance.b) rescue JSON.load(instance)
-    instance = instance.cbor_clone if $annotate && ENV["EXPERIMENTAL_ANNOTATE"]
-    ann = parser.validate(instance)
-    # my_pp ann
-    instance.cbor_add_annotations_from(ann) rescue nil
-    my_diag(instance) if $annotate
-    exit 1 unless ann
+  when /\Av(v)?/
+    verbose = $1
+    ARGV[2..-1].each do |fn|
+      instance = read_arg(fn)
+      instance = if $sequence
+                   CBOR.decode("\x9f".b << instance.b << "\xff".b)
+                 else
+                   CBOR.decode(instance.b) rescue JSON.load(instance)
+                 end
+      instance = instance.cbor_clone if $annotate && ENV["EXPERIMENTAL_ANNOTATE"]
+      warn "#{fn}:" if verbose
+      ann = parser.validate(instance)
+      # my_pp ann
+      instance.cbor_add_annotations_from(ann) rescue nil
+      my_diag(instance) if $annotate
+      unless ann
+        retcode = 1
+      end
+    end
   else
     usage
   end
@@ -97,3 +115,5 @@ rescue CDDL::ParseError => e
   warn e.message
   exit EX_DATAERR
 end
+
+exit retcode

data/cddl.gemspec

@@ -1,11 +1,12 @@
 spec = Gem::Specification.new do |s|
   s.name = 'cddl'
-  s.version = '0.8.8'
+  s.version = '0.9.1'
   s.summary = "CDDL generator and validator."
   s.description = %{A parser, generator, and validator for CDDL}
   s.add_dependency('cbor-diag')
   s.add_dependency('abnc')
-  s.add_dependency('json')
+  s.add_dependency('json_pure')
+  s.add_dependency('abnftt')
   s.add_dependency('regexp-examples') # , '1.1.0')
   s.add_dependency('colorize')
   s.files = `git ls-files`.split("\n").grep(/^[a-z]/)
@@ -19,3 +20,4 @@ spec = Gem::Specification.new do |s|
   s.homepage = "http://github.com/cabo/cddl"
   s.license = 'MIT'
 end
+

data/data/cddl.abnf

@@ -16,6 +16,7 @@ type1 = type2 [S (rangeop / annotator) S type2]
      / "#" "6" ["." uint] "(" S type S ")" ; note no space!
      / "#" DIGIT ["." uint]                ; major/ai
      / "#"                                 ; any
+     / "~" S typename [genericarg]
      / "{" S group S "}"
      / "[" S group S "]"
      / "&" S "(" S group S ")"

data/lib/cbor-pp-play.rb

@@ -2,6 +2,7 @@ require_relative "cbor-pp"
 
 a = [1, {foo: "bar", busel: "basel", baz: "bass", ant: "cat", bat: "dog", eel: "fox"},
              2, 3, 4, [3]*55, "foo".b, 0.00006103515625, 0.0000099, 1e-7, nil]
+a[0].cbor_annotation_add "flurp"
 a[1].cbor_annotation_add "fasel"
 CBOR::PP.pp a
 

data/lib/cddl.rb

@@ -3,6 +3,7 @@ require 'pp'
 require 'pathname'
 require 'cbor-pure' unless defined?(CBOR::Tagged)
 require 'regexp-examples'
+require 'abnftt'
 require 'colorize'
 require 'base64'
 
@@ -55,6 +56,8 @@ module CDDL
       @ast = @abnf.ast?
       # our little argument stack for rule processing
       @insides = []
+      # collect error information
+      @last_message = ""
     end
 
     def apr                     # for debugging
@@ -243,12 +246,28 @@ module CDDL
       [rule[0], *rule[1]]
     end
 
+    def remove_indentation(s)
+      l = s.lines
+      indent = l.grep(/\S/).map {|l| l[/^\s*/].size}.min
+      l.map {|l| l.sub(/^ {0,#{indent}}/, "")}.join
+    end
+
     # Memoize a bit here
 
     REGEXP_FOR_STRING = Hash.new {|h, k|
       h[k] = Regexp.new("\\A#{k}\\z")
     }
 
+    ABNF_PARSER_FOR_STRING = Hash.new {|h, k|
+      grammar = "cddl-t0p--1eve1-f0r--abnf = " << k # XXX
+      h[k] = ABNF.from_abnf(grammar)
+    }
+
+    ABNF_ENCODING_FOR_CONOP = {
+      abnf: Encoding::UTF_8,
+      abnfb: Encoding::BINARY
+    }
+
     def generate
       @recursion = 0
       generate1(rules)
@@ -329,8 +348,12 @@ module CDDL
           unless kr
             case vr[0]
             when :grpent
-              fail "grpent in map #{vr.inspect}" unless vr.size == 2
-              g = Array.new(st) { generate1(vr[1], true) }.flatten(1)
+              # fail "grpent in map #{vr.inspect}" unless vr.size == 2
+              g = Array.new(st) {
+                vr[1..-1].map{ |vrx|
+                  generate1(vrx, true)
+                }.flatten(1)
+              }.flatten(1)
               # warn "GGG #{g.inspect}"
               return g
             when :grpchoice
@@ -389,7 +412,7 @@ module CDDL
       when :anno
         target = where[2]
         control = where[3]
-        case where[1]
+        case conop = where[1]
         when :size
           should_be_int = generate1(control)
           unless (Array === target && target[0] == :prim && [0, 2, 3].include?(target[1])) && Integer === should_be_int && should_be_int >= 0
@@ -435,6 +458,35 @@ module CDDL
             $default_warned = true
           end
           generate1(target, inmap)
+        when :feature
+          generate1(target, inmap)
+        when :cat, :det
+          lhs = generate1(target, inmap)
+          rhs = generate1(control)
+          if conop == :det
+            lhs = remove_indentation(lhs)
+            rhs = remove_indentation(rhs)
+          end
+          begin
+            lhs + rhs
+          rescue Exception => e
+            fail "Can't #{lhs.inspect} .cat #{rhs.inspect}: #{e.message}"
+          end
+        when :plus
+          lhs = generate1(target, inmap)
+          rhs = generate1(control)
+          begin
+            case lhs
+            when Integer
+              lhs + Integer(rhs)
+            when Numeric
+              lhs + rhs
+            else
+              fail "#{lhs.inspect}: Not a number"
+            end
+          rescue Exception => e
+            fail "Can't #{lhs.inspect} .plus #{rhs.inspect}: #{e.message}"
+          end
         when :eq
           content = generate1(control)
           if validate1(content, where)
@@ -448,7 +500,7 @@ module CDDL
                     content = Integer(content)
                     case target[1]
                     when 0
-                      case where[1]
+                      case conop
                       when :lt
                         rand(0...content)
                       when :le
@@ -475,12 +527,26 @@ module CDDL
             fail "Don't know yet how to generate #{where}"
           end
           REGEXP_FOR_STRING[regexp].random_example(max_repeater_variance: 5)
+        when :abnf, :abnfb
+          grammar = generate1(control)
+          bytes = true if target == [:prim, 2]
+          bytes = false if target == [:prim, 3]
+          unless !bytes.nil? && String === grammar
+            fail "Don't know yet how to generate #{where}"
+          end
+          grammar << "\n" unless grammar =~ /\n\z/
+          out = ABNF_PARSER_FOR_STRING[grammar].generate
+          if conop == :abnfb
+            out = out.codepoints.pack("C*")
+          end
+          enc = bytes ? Encoding::BINARY : Encoding::UTF_8
+          out.force_encoding(enc)
         when :cbor, :cborseq
           unless target == [:prim, 2]
             fail "Don't know yet how to generate #{where}"
           end
           content = CBOR::encode(generate1(control))
-          if where[1] == :cborseq
+          if conop == :cborseq
             # remove the first head
             n = case content.getbyte(0) - (4 << 5)
                 when 0..23; 1
@@ -498,7 +564,7 @@ module CDDL
             content = generate1(target)
             if validate1(content, control)
               return content
-            elsif where[1] == :within
+            elsif conop == :within
               warn "*** #{content.inspect} meets #{target.inspect} but not #{control.inspect}"
             end
           end
@@ -523,7 +589,7 @@ module CDDL
       end
     end
 
-    VALUE_TYPE = {text: String, int: Integer, float: Float}
+    VALUE_TYPE = {text: String, bytes: String, int: Integer, float: Float}
     SIMPLE_VALUE = {
       [:prim, 7, 20] => [true, false, :bool],
       [:prim, 7, 21] => [true, true, :bool],
@@ -536,9 +602,38 @@ module CDDL
         [true, t[1], vt]
       elsif v = SIMPLE_VALUE[t]
         v
-      else
-        [false]
-      end
+      elsif t[0] == :anno
+        _, conop, target, control = t
+        # warn ["EXV0", conop, target, control].inspect
+        if conop == :cat || conop == :plus || conop == :det
+          ok1, v1, vt1 = extract_value(target)
+          ok2, v2, vt2 = extract_value(control)
+          # warn ["EXV", ok1, v1, vt1, ok2, v2, vt2].inspect
+          if ok1 && ok2
+            if vt1 == Integer
+              [true, v1 + Integer(v2), Integer] if vt2 == Integer || vt2 == Float
+            elsif vt1 == Float
+              [true, v1 + v2, vt1] if vt2 == Integer || vt2 == Float
+            else
+              if conop == :det
+                v1 = remove_indentation(v1)
+                v2 = remove_indentation(v2)
+              end
+              [true, v1 + v2, vt1] if vt1 == vt2
+            end
+          end rescue nil
+        end
+      end || [false]
+    end
+
+    def extract_array(t)
+      return [false] unless t[0] == :array
+      [true, *t[1..-1].map { |el|
+         return [false] unless el[0..3] == [:member, 1, 1, nil]
+         ok, v, vt = extract_value(el[4])
+         return [false] unless ok
+         [v, vt]
+       }]
     end
 
     def validate_diag
@@ -547,11 +642,20 @@ module CDDL
 
     def validate(d, warn=true)
       @recursion = 0
+      $features = Hash.new {|h, k| h[k] = {}}
       result = validate1a(d, rules)
-      unless result
-        if warn
+      if warn
+        if result
+          if $features != {}
+            ok = ENV["CDDL_FEATURE_OK"] and ok = ok.split(/,\s*/) or ok = []
+            # warn([:OK, ok, $features].inspect)
+            features = $features.reject {|k, v| ok.include? k.to_s }
+            warn "** Features potentially used (#$fn): #{features.map {|k, v| "#{k}: #{v.keys}"}.join(", ")}" if features != {}
+          end
+        else
           warn "CDDL validation failure (#{result.inspect} for #{d.inspect}):"
           PP::pp(validate_diag, STDERR)
+          puts(validate_diag.cbor_diagnostic)
         end
       end
       result
@@ -559,7 +663,7 @@ module CDDL
 
     def validate_result(check)
       check || (
-        @last_message = yield
+        @last_message << yield
         false
       )
     end
@@ -592,7 +696,9 @@ module CDDL
             ann.concat(ann2)
           end
           if occ < s
-            @last_message = "occur not reached in array #{d} for #{where}"
+            # warn "*** lme #{@last_message.encoding} #{@last_message}"
+            # warn "*** #{"\noccur #{occ} < #{s}, not reached at #{i} in array #{d} for #{where}".encoding}"
+            @last_message << "\noccur #{occ} < #{s}, not reached at #{i} in array #{d} for #{where}"
             return [false, ann]
           end
         end
@@ -681,13 +787,17 @@ module CDDL
                           puts "COMPLEX: #{k.inspect} #{simple.inspect} #{simpleval.inspect}"         if ENV["CDDL_TRACE"]
             keys = d_check.keys
             ta, keys = keys.partition{ |key| validate1(key, k)}
-            # XXX check ta.size against s/e
-            ta.all? { |val|
-              if (ann2 = validate1a(d[val], v)) && 
-                 d_check.delete(val) {:not_found} != :not_found
-                anno.concat(ann2)
-              end
-            }
+            count = 0
+            catch :enough do
+              ta.all? { |val|
+                if (ann2 = validate1a(d[val], v)) && # XXX check cut or not!
+                   d_check.delete(val) {:not_found} != :not_found
+                  anno.concat(ann2)
+                  throw :enough, true if (count += 1) == e
+                  true
+                end
+              }
+            end and validate_result(count >= s) { "not enough #{ta.inspect} for #{r.inspect}" }
           end
           end
         else
@@ -741,16 +851,31 @@ module CDDL
           idx, ann = validate_forward(d, 0, where)
           ann if validate_result(idx == d.size) { "#{validate_diag.inspect} -- cannot complete (#{idx}, #{d.size}) array #{d} for #{where}" }
         end
-      when :text, :int, :float, :bytes
+      when :int, :float
         _, v = extract_value(where)
         [] if d == v
+      when :text, :bytes
+        _, v = extract_value(where)
+        [] if d == v && ((d.encoding == Encoding::BINARY) == (v.encoding == Encoding::BINARY))
       when :range
         [] if where[2] === d && where[1].include?(d)
       when :anno
-        target = where[2]
-        if ann = validate1a(d, target)
-          control = where[3]
-          case where[1]
+        _, conop, target, control = where
+        if conop == :cat || conop == :plus || conop == :det
+          ok1, v1, vt1 = extract_value(target)
+          ok2, v2, vt2 = extract_value(control)
+          # warn ["ANNO0", ok1, v1, vt1, ok2, v2, vt2, d].inspect
+          if ok1 && ok2
+            v2 = Integer(v2) if vt1 == Integer
+            if conop == :det
+              v1 = remove_indentation(v1)
+              v2 = remove_indentation(v2)
+            end
+            # warn ["ANNO", ok1, v1, vt1, ok2, v2, vt2, d].inspect
+            [] if d == v1 + v2  # XXX Focus ArgumentError
+          end
+        elsif ann = validate1a(d, target)
+          case conop
           when :size
             case d
             when Integer
@@ -789,8 +914,26 @@ module CDDL
               $default_warned = true
             end
             ann
+          when :feature
+            ok, v, vt = extract_value(control)
+            if ok
+              nm = v
+              det = d
+              warn "*** feature controller should be a string: #{control.inspect}" unless String == vt
+            else
+              ok, *v = extract_array(control)
+              if ok && v.size == 2
+                nm = v[0][0]
+                det = v[1][0]
+                warn "*** first element of feature controller should be a string: #{control.inspect}" unless String === nm
+              else
+                warn "*** feature controller not implemented: #{control.inspect}"
+              end
+            end
+            $features[nm][det] = true
+            ann
           when :lt, :le, :gt, :ge, :eq, :ne
-            op = OPERATORS[where[1]]
+            op = OPERATORS[conop]
             ok, v, _vt = extract_value(control)
             if ok
               ann if d.send(op, v) rescue nil # XXX Focus ArgumentError
@@ -806,6 +949,24 @@ module CDDL
               end
             end
             )
+          when :abnf, :abnfb
+            ann if (
+            if String === d
+              ok, v, vt = extract_value(control)
+              if ok && vt == String
+                begin
+                  ABNF_PARSER_FOR_STRING[v].validate(
+                    d.dup.force_encoding(ABNF_ENCODING_FOR_CONOP[conop]).codepoints.pack("U*")
+                  )
+                  true
+                rescue => e
+                  # warn "*** #{e}" # XXX
+                  @last_message = e.to_s.force_encoding(Encoding::UTF_8)
+                  nil
+                end
+              end
+            end
+            )
           when :cbor
             ann if validate1((CBOR::decode(d) rescue :BAD_CBOR), control)
           when :cborseq
@@ -837,6 +998,14 @@ module CDDL
         when 3
           String === d && d.encoding != Encoding::BINARY # cheat
         when 6
+          if Integer === d
+            t = 2               # assuming only 2/3 match an Integer
+            if d < 0
+              d = ~d
+              t = 3
+            end
+            d = CBOR::Tagged.new(t, d == 0 ? "".b : d.digits(256).reverse!.pack("C*"))
+          end
           CBOR::Tagged === d && d.tag == where[2] && validate1a(d.data, where[3])
         when 7
           t, v = extract_value(where)
@@ -959,7 +1128,7 @@ module CDDL
          when ""
            parts[1].b
          when "h"
-           parts[1].gsub(/\s/, "").chars.each_slice(2).map{ |x| Integer(x.join, 16).chr("BINARY") }.join
+           parts[1].gsub(/\s/, "").chars.each_slice(2).map{ |x| Integer(x.join, 16).chr("BINARY") }.join.b
          when "b64"
             Base64.urlsafe_decode64(parts[1])
          else
@@ -1073,9 +1242,13 @@ module CDDL
     def g_process(name, g, genericargs)
       r, ids = g
       args = genericargs.children(:type1).map {|x| type1(x, true)}
+      # warn "** args #{args.inspect}"
       fail "number of args #{name} #{ids.size} #{args.size}" if ids.size != args.size
       bindings = Hash[ids.zip(args)]
-      r_process("#{name}<#{genericargs._strip}>", r, bindings)
+      # warn "** bindings: #{bindings.inspect}"
+      genname = "#{name}<#{args.inspect}>" # XXX this works up to a RECURSE...
+      # warn "** genname #{genname}"
+      r_process(genname, r, bindings)
     end
 
     def type_recall(name, canbegroup, genericargs)
@@ -1142,7 +1315,7 @@ module CDDL
           end
         when :recurse_grpent # XXX we don't have the entry yet
         when :member
-          fail "map entry without member key given: #{member}" unless member[3] || member[4][0] == :grpent || member[4][0] == :grpchoice
+          fail "map entry #{member.inspect} without member key given: #{member}" unless member[3] || member[4][0] == :grpent || member[4][0] == :grpchoice
         else
           fail ["type1 member", member].inspect unless member[0] == :member
         end
@@ -1152,7 +1325,8 @@ module CDDL
     BRACE = {"{" => :map, "[" => :array}
     RANGE_EXCLUDE_END = {".." => false, "..." => true}
     SUPPORTED_ANNOTATIONS = [:bits, :size, :regexp, :cbor, :cborseq, :within, :and,
-                             :default, :lt, :le, :gt, :ge, :eq, :ne]
+                             :default, :lt, :le, :gt, :ge, :eq, :ne,
+                             :feature, :abnf, :abnfb, :det, :cat, :plus]
 
     def type1(n, canbegroup = false)
 #      puts "NVALUE #{n.value.inspect}"
@@ -1214,6 +1388,26 @@ module CDDL
              v
            }
           ]
+        when /\A~/
+          if tn = n.typename
+            s = type_recall(tn.to_s, false, n.genericarg).flatten(1)
+          else
+            fail "No typename in unwrap #{n}"
+          end
+          if s[0] != :type1
+            fail [:UNWRAPs0, s].inspect
+          end
+          case s[1]
+          when :map, :array
+            [:grpent, s[2..-1]]
+          when :prim
+            if s[2] != 6
+              fail [:UNWRAPs2, s].inspect
+            end
+            s[4]                # check :type1?
+          else
+            fail [:UNWRAPs1, s].inspect
+          end
         else
           "unimplemented #{n}"
         end

data/test/test-cddl.rb

@@ -171,6 +171,57 @@ HERE
     assert parser.validate_for_test(true)
   end
 
+  def test_validate_bignum
+    parser = CDDL::Parser.new <<HERE
+test = bigint
+HERE
+    assert parser.validate_for_test(2**4711)
+    assert parser.validate_for_test(-2**815)
+    assert parser.validate_for_test(1)
+    assert parser.validate_for_test(0)
+    assert parser.validate_for_test(-11)
+  end
+
+  def test_aaaaaaaa_invalidate_bignum
+    parser = CDDL::Parser.new <<HERE
+test = bignint
+HERE
+    refute parser.validate_for_test(2**4711, false)
+    assert parser.validate_for_test(-2**815)
+    refute parser.validate_for_test(1, false)
+    refute parser.validate_for_test(0, false)
+    assert parser.validate_for_test(-11)
+  end
+
+  
+  def test_aaaaaaaa_validate_bignum0b
+    parser = CDDL::Parser.new <<HERE
+test = #6.2("")
+HERE
+    refute parser.validate_for_test(1, false)
+    refute parser.validate_for_test(0, false)
+  end
+
+  
+  def test_aaaaaaaa_validate_bignum0
+    parser = CDDL::Parser.new <<HERE
+test = #6.2(h'')
+HERE
+    assert parser.validate_for_test(0)
+    refute parser.validate_for_test(1, false)
+  end
+
+  
+  def test_aaaaaaaa_validate_bignum1
+    parser = CDDL::Parser.new <<HERE
+test = #6.2(h'01')
+HERE
+    assert parser.validate_for_test(1)
+    refute parser.validate_for_test(0, false)
+    refute parser.validate_for_test(2, false)
+  end
+
+
   def test_validate_a
     parser = CDDL::Parser.new <<HERE
 test = [* one: 1]

data/test-data/7807.cddl

@@ -0,0 +1,14 @@
+coap-problem-details = {
+ type => uint,
+ ? title => text,
+ ? response-code => bstr .size 1,
+ ? detail => text,
+ ? instance => uri,
+ * $$coap-problem-details-extension,
+}
+
+type = 0
+title = 1
+response-code = 2
+detail = 3
+instance = 4

data/test-data/abignum.cddl

@@ -0,0 +1 @@
+abignum = unsigned

data/test-data/abnf1.cddl

@@ -0,0 +1,3 @@
+foo = text .abnf '4DIGIT
+DIGIT          =  %x30-39 ; 0-9
+'