cddl 0.8.1 → 0.8.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: bc5efc3a37bc0a936d6e03eb4f7d689196758914
-  data.tar.gz: 2ec4a61c192d6ec61ce94bbbcfa805693ef127fb
+SHA256:
+  metadata.gz: 2ede21b4292e6f546745029c7a071eb1af1259366e676d90e19f400dfe089d7a
+  data.tar.gz: 57ed90f75d58941bd28c5afef7d5eb29eea5d5483a6a3b2c4fc65d49df0370aa
 SHA512:
-  metadata.gz: e09a9a2e079f8eafa1c879ed3a2d5c30f5c507624dff1222f68bf54ee1e05b8d56e30060ee9e27da2124834158ec7f9d222fe61ebb92a251da1070fd0d966509
-  data.tar.gz: 3f045307ce12737a799c238fb3eef169c9481b405d3f009af18aa6819ba12b86317e2473272685c244c278ea48f471b93c972411da7ba7610d8be3da1b36aab3
+  metadata.gz: 65de704e842bb6d48265e7024922edf0406d6cb521d2977b07db2926d396b8b2d6bdc103619d579b861cb53b8914b4383e25c2e610e86f7b4d88d89983a64dff
+  data.tar.gz: c0919845b3ee5bba3b2a5d77f9e182a4a9f1904f187843501abf29954d37033d62f1cfd50545865ba878c92583f26151f9024dc1093c0d4607442038b36008b0

data/bin/cddl

@@ -7,13 +7,16 @@ require 'json'
 
 Encoding.default_external = "UTF-8" # wake up, smell the coffee
 
+EX_USAGE = 64
+EX_DATAERR = 65
+
 def usage
   warn "Usage:"
   warn "#$0 spec.cddl generate [n]"
   warn "#$0 spec.cddl json-generate [n]"
   warn "#$0 spec.cddl validate instance.cbor"
   warn "#$0 spec.cddl validate instance.json"
-  exit 1
+  exit EX_USAGE
 end
 
 def read_arg(arg)
@@ -21,7 +24,7 @@ def read_arg(arg)
     STDIN.read
   else
     usage unless arg
-    File.read(arg)
+    File.read(arg, mode: "rb")
   end
 end
 
@@ -86,10 +89,11 @@ begin
     # my_pp ann
     instance.cbor_add_annotations_from(ann) rescue nil
     my_diag(instance) if $annotate
+    exit 1 unless ann
   else
     usage
   end
 rescue CDDL::ParseError => e
   warn e.message
-  exit 1
+  exit EX_DATAERR
 end

data/cddl.gemspec

@@ -1,6 +1,6 @@
 spec = Gem::Specification.new do |s|
   s.name = 'cddl'
-  s.version = '0.8.1'
+  s.version = '0.8.12'
   s.summary = "CDDL generator and validator."
   s.description = %{A parser, generator, and validator for CDDL}
   s.add_dependency('cbor-diag')
@@ -19,3 +19,4 @@ spec = Gem::Specification.new do |s|
   s.homepage = "http://github.com/cabo/cddl"
   s.license = 'MIT'
 end
+

data/data/cddl.abnf

@@ -16,6 +16,7 @@ type1 = type2 [S (rangeop / annotator) S type2]
      / "#" "6" ["." uint] "(" S type S ")" ; note no space!
      / "#" DIGIT ["." uint]                ; major/ai
      / "#"                                 ; any
+     / "~" S typename [genericarg]
      / "{" S group S "}"
      / "[" S group S "]"
      / "&" S "(" S group S ")"
@@ -49,8 +50,10 @@ occur = [uint] "*" [uint]
       / "+"
       / "?"
 
-uint = ["0x" / "0b"] "0"
-     / ["0x" / "0b"] DIGIT1 *DIGIT
+uint = DIGIT1 *DIGIT
+     / "0x" 1*HEXDIG
+     / "0b" 1*BINDIG
+     / "0"
 
 value = number
       / text
@@ -77,6 +80,9 @@ ALPHA = %x41-5A / %x61-7A
 EALPHA = %x41-5A / %x61-7A / "@" / "_" / "$"
 DIGIT = %x30-39
 DIGIT1 = %x31-39
+HEXDIG = DIGIT / "A" / "B" / "C" / "D" / "E" / "F"
+BINDIG = %x30-31
+
 S = *WS
 WS = SP / NL
 SP = %x20

data/lib/cddl.rb

@@ -4,6 +4,7 @@ require 'pathname'
 require 'cbor-pure' unless defined?(CBOR::Tagged)
 require 'regexp-examples'
 require 'colorize'
+require 'base64'
 
 module CDDL
 
@@ -14,7 +15,7 @@ module CDDL
 
   MANY = Float::INFINITY
 
-  MAX_RECURSE = 32              # XXX
+  MAX_RECURSE = 128              # XXX
 
   class ParseError < ArgumentError; end
 
@@ -319,7 +320,8 @@ module CDDL
         end
       when :member
         st = where[1]
-        en = [where[2], [st, 4].max].min # truncate to 4 unless must be more
+        fudge = 4 * (1 - (@recursion / MAX_RECURSE.to_f))**3 # get less generate-happy with more recursion
+        en = [where[2], [st, fudge].max].min # truncate to fudge unless must be more
         st += rand(en + 1 - st) if en != st
         kr = where[3]
         vr = where[4]
@@ -327,8 +329,12 @@ module CDDL
           unless kr
             case vr[0]
             when :grpent
-              fail "grpent in map #{vr.inspect}" unless vr.size == 2
-              g = Array.new(st) { generate1(vr[1], true) }.flatten(1)
+              # fail "grpent in map #{vr.inspect}" unless vr.size == 2
+              g = Array.new(st) {
+                vr[1..-1].map{ |vrx|
+                  generate1(vrx, true)
+                }.flatten(1)
+              }.flatten(1)
               # warn "GGG #{g.inspect}"
               return g
             when :grpchoice
@@ -348,7 +354,7 @@ module CDDL
           fail BackTrack.new("Need #{where[1]}..#{where[2]} of these: #{[kr, vr].inspect}") unless where[1] == 0
           []
         end
-      when :text, :int, :float
+      when :text, :int, :float, :bytes
         where[1]
       when :range
         rand(where[1])
@@ -390,15 +396,19 @@ module CDDL
         case where[1]
         when :size
           should_be_int = generate1(control)
-          unless (target == [:prim, 2] || target == [:prim, 0]) && Integer === should_be_int && should_be_int >= 0
+          unless (Array === target && target[0] == :prim && [0, 2, 3].include?(target[1])) && Integer === should_be_int && should_be_int >= 0
             fail "Don't know yet how to generate #{where}"
           end
           s = Random.new.bytes(should_be_int)
-          if target == [:prim, 0]
+          case target[1]
+          when 0
             # silently restrict to what we can put into a uint
             s[0...8].bytes.inject(0) {|a, b| a << 8 | b }
-          else
+          when 2
             s
+          when 3
+            Base64.urlsafe_encode64(s)[0...should_be_int]
+            # XXX generate word a la w = gen_word
           end
         when :bits
           set_of_bits = Array.new(10) { generate1(control) } # XXX: ten?
@@ -429,6 +439,8 @@ module CDDL
             $default_warned = true
           end
           generate1(target, inmap)
+        when :feature
+          generate1(target, inmap)
         when :eq
           content = generate1(control)
           if validate1(content, where)
@@ -517,7 +529,7 @@ module CDDL
       end
     end
 
-    VALUE_TYPE = {text: String, int: Integer, float: Float}
+    VALUE_TYPE = {text: String, bytes: String, int: Integer, float: Float}
     SIMPLE_VALUE = {
       [:prim, 7, 20] => [true, false, :bool],
       [:prim, 7, 21] => [true, true, :bool],
@@ -525,7 +537,7 @@ module CDDL
       [:prim, 7, 23] => [true, :undefined, :undefined],
     }
 
-    def extract_value(t)
+    def extract_value(t)        # []
       if vt = VALUE_TYPE[t[0]]
         [true, t[1], vt]
       elsif v = SIMPLE_VALUE[t]
@@ -541,11 +553,17 @@ module CDDL
 
     def validate(d, warn=true)
       @recursion = 0
+      $features = Hash.new {|h, k| h[k] = {}}
       result = validate1a(d, rules)
-      unless result
-        if warn
+      if warn
+        if result
+          if $features != {}
+            warn "** Features potentially used: #{$features.map {|k, v| "#{k}: #{v.keys}"}.join(", ")}"
+          end
+        else
           warn "CDDL validation failure (#{result.inspect} for #{d.inspect}):"
           PP::pp(validate_diag, STDERR)
+          puts(validate_diag.cbor_diagnostic)
         end
       end
       result
@@ -660,7 +678,7 @@ module CDDL
           # this is mostly quadratic; let's do the linear thing if possible
           simple, simpleval = extract_value(k)
           if simple
-            #              puts "SIMPLE: #{d_check.inspect} #{simpleval}"
+                          puts "SIMPLE: #{d_check.inspect} #{simpleval}"         if ENV["CDDL_TRACE"]
             # add occurrence check; check that val is present in the first place
             actual = d.fetch(simpleval, :not_found)
             if actual == :not_found
@@ -672,7 +690,7 @@ module CDDL
               end
             end
           else
-            #              puts "COMPLEX: #{k.inspect} #{simple.inspect} #{simpleval.inspect}"
+                          puts "COMPLEX: #{k.inspect} #{simple.inspect} #{simpleval.inspect}"         if ENV["CDDL_TRACE"]
             keys = d_check.keys
             ta, keys = keys.partition{ |key| validate1(key, k)}
             # XXX check ta.size against s/e
@@ -735,9 +753,12 @@ module CDDL
           idx, ann = validate_forward(d, 0, where)
           ann if validate_result(idx == d.size) { "#{validate_diag.inspect} -- cannot complete (#{idx}, #{d.size}) array #{d} for #{where}" }
         end
-      when :text, :int, :float
+      when :int, :float
         _, v = extract_value(where)
         [] if d == v
+      when :text, :bytes
+        _, v = extract_value(where)
+        [] if d == v && ((d.encoding == Encoding::BINARY) == (v.encoding == Encoding::BINARY))
       when :range
         [] if where[2] === d && where[1].include?(d)
       when :anno
@@ -783,6 +804,10 @@ module CDDL
               $default_warned = true
             end
             ann
+          when :feature
+            ok, v, vt = extract_value(control)
+            $features[v][d] = true
+            ann
           when :lt, :le, :gt, :ge, :eq, :ne
             op = OPERATORS[where[1]]
             ok, v, _vt = extract_value(control)
@@ -831,6 +856,14 @@ module CDDL
         when 3
           String === d && d.encoding != Encoding::BINARY # cheat
         when 6
+          if Integer === d
+            t = 2               # assuming only 2/3 match an Integer
+            if d < 0
+              d = ~d
+              t = 3
+            end
+            d = CBOR::Tagged.new(t, d == 0 ? "".b : d.digits(256).reverse!.pack("C*"))
+          end
           CBOR::Tagged === d && d.tag == where[2] && validate1a(d.data, where[3])
         when 7
           t, v = extract_value(where)
@@ -942,14 +975,34 @@ module CDDL
     def value(n)
       # cheat:
       # warn n
-      val = eval(n.to_s)
-      # warn val
-      [case val
-       when Integer; :int
-       when Numeric; :float
-       when String; :text
-       else fail "huh? value #{val.inspect}"
-       end, val]
+      s = n.to_s
+      if s[-1] == "'"
+        parts = s.split("'", 3)
+        if parts[2] != "" || parts[1] =~ /\\/
+          warn "*** Can't handle backslashes in #{s.inspect} yet"
+        end
+        [:bytes,
+         case parts[0]
+         when ""
+           parts[1].b
+         when "h"
+           parts[1].gsub(/\s/, "").chars.each_slice(2).map{ |x| Integer(x.join, 16).chr("BINARY") }.join.b
+         when "b64"
+            Base64.urlsafe_decode64(parts[1])
+         else
+           warn "*** Can't handle byte string type #{parts[0].inspect} yet"
+         end
+        ]
+      else
+        val = eval(n.to_s)
+        # warn val
+        case val
+         when Integer; [:int, val]
+         when Numeric; [:float, val]
+         when String; [:text, val.force_encoding(Encoding::UTF_8)]
+         else fail "huh? value #{val.inspect}"
+        end
+      end
     end
 
     def workaround1(n)
@@ -988,7 +1041,7 @@ module CDDL
             fail [n, n.children, mk, mk.children].inspect
           end
           name = bw.to_s
-          [[:member, *occ, [:text, name], type(nt)]]
+          [[:member, *occ, [:text, name.force_encoding(Encoding::UTF_8)], type(nt)]]
         end
       else
         t = if nbw = nt.bareword
@@ -1000,8 +1053,8 @@ module CDDL
             else
               type(nt, true)              # type can actually be a group here!
             end
-        # pp ["t is", t]
-        if t[0] == :grpent && (@insides.last != :array || occ == [1, 1])
+        pp ["t is", t]               if ENV["CDDL_TRACE"]
+        if t[0] == :grpent && (occ == [1, 1])
           # go through the members here and multiply the occs
           t1 = t[1..-1].flatten(1)
           if t1[0] == :recurse_grpent
@@ -1116,7 +1169,7 @@ module CDDL
           end
         when :recurse_grpent # XXX we don't have the entry yet
         when :member
-          fail "map entry without member key given: #{member}" unless member[3] || member[4][0] == :grpent || member[4][0] == :grpchoice
+          fail "map entry #{member.inspect} without member key given: #{member}" unless member[3] || member[4][0] == :grpent || member[4][0] == :grpchoice
         else
           fail ["type1 member", member].inspect unless member[0] == :member
         end
@@ -1126,7 +1179,7 @@ module CDDL
     BRACE = {"{" => :map, "[" => :array}
     RANGE_EXCLUDE_END = {".." => false, "..." => true}
     SUPPORTED_ANNOTATIONS = [:bits, :size, :regexp, :cbor, :cborseq, :within, :and,
-                             :default, :lt, :le, :gt, :ge, :eq, :ne]
+                             :default, :lt, :le, :gt, :ge, :eq, :ne, :feature]
 
     def type1(n, canbegroup = false)
 #      puts "NVALUE #{n.value.inspect}"
@@ -1188,6 +1241,26 @@ module CDDL
              v
            }
           ]
+        when /\A~/
+          if tn = n.typename
+            s = type_recall(tn.to_s, false, n.genericarg).flatten(1)
+          else
+            fail "No typename in unwrap #{n}"
+          end
+          if s[0] != :type1
+            fail [:UNWRAPs0, s].inspect
+          end
+          case s[1]
+          when :map, :array
+            [:grpent, s[2..-1]]
+          when :prim
+            if s[2] != 6
+              fail [:UNWRAPs2, s].inspect
+            end
+            s[4]                # check :type1?
+          else
+            fail [:UNWRAPs1, s].inspect
+          end
         else
           "unimplemented #{n}"
         end

data/test/test-cddl.rb

@@ -59,9 +59,9 @@ class TestABNF < Test::Unit::TestCase
 
   def test_generate_hex_bin
     parser = CDDL::Parser.new <<HERE
-test = [0x4711, 0b1011001110001111000]
+test = [0x4711, 0xabcDEF, 0b1011001110001111000]
 HERE
-    assert_equal [0x4711, 0b1011001110001111000], parser.generate
+    assert_equal [0x4711, 0xabcDEF, 0b1011001110001111000], parser.generate
   end
 
   def test_generate_le
@@ -171,6 +171,57 @@ HERE
     assert parser.validate_for_test(true)
   end
 
+  def test_validate_bignum
+    parser = CDDL::Parser.new <<HERE
+test = bigint
+HERE
+    assert parser.validate_for_test(2**4711)
+    assert parser.validate_for_test(-2**815)
+    assert parser.validate_for_test(1)
+    assert parser.validate_for_test(0)
+    assert parser.validate_for_test(-11)
+  end
+
+  def test_aaaaaaaa_invalidate_bignum
+    parser = CDDL::Parser.new <<HERE
+test = bignint
+HERE
+    refute parser.validate_for_test(2**4711, false)
+    assert parser.validate_for_test(-2**815)
+    refute parser.validate_for_test(1, false)
+    refute parser.validate_for_test(0, false)
+    assert parser.validate_for_test(-11)
+  end
+
+  
+  def test_aaaaaaaa_validate_bignum0b
+    parser = CDDL::Parser.new <<HERE
+test = #6.2("")
+HERE
+    refute parser.validate_for_test(1, false)
+    refute parser.validate_for_test(0, false)
+  end
+
+  
+  def test_aaaaaaaa_validate_bignum0
+    parser = CDDL::Parser.new <<HERE
+test = #6.2(h'')
+HERE
+    assert parser.validate_for_test(0)
+    refute parser.validate_for_test(1, false)
+  end
+
+  
+  def test_aaaaaaaa_validate_bignum1
+    parser = CDDL::Parser.new <<HERE
+test = #6.2(h'01')
+HERE
+    assert parser.validate_for_test(1)
+    refute parser.validate_for_test(0, false)
+    refute parser.validate_for_test(2, false)
+  end
+
+
   def test_validate_a
     parser = CDDL::Parser.new <<HERE
 test = [* one: 1]
@@ -202,6 +253,54 @@ HERE
     assert parser.validate_for_test({"one" => 1, "two" => 1}, false)
   end
 
+  def test_validate_a_nontrivial_group_map
+    parser = CDDL::Parser.new <<HERE
+test = {* (text => 1, text => 2)}
+HERE
+    assert parser.validate_for_test({})
+    assert parser.validate_for_test({"one" => 2, "two" => 1})
+    assert parser.validate_for_test({"one" => 1, "two" => 2})
+    refute parser.validate_for_test({"one" => 1}, false)
+    assert parser.validate_for_test({"one" => 1, "two" => 2, "uno" => 1, "due" => 2})
+  end
+
+
+  def test_murray
+    parser = CDDL::Parser.new <<HERE
+a = {
+  b
+}
+
+b = (
+  (x: int // y: int), z: int
+)
+HERE
+    assert parser.validate_for_test({})
+    assert parser.validate_for_test({"x" => 2, "z" => 1})
+    assert parser.validate_for_test({"y" => 1, "z" => 2})
+    refute parser.validate_for_test({"z" => 1}, false)
+    assert parser.validate_for_test({"one" => 1, "two" => 2, "uno" => 1, "due" => 2})
+  end
+
+
+  def test_murray_workaround
+    parser = CDDL::Parser.new <<HERE
+a = {
+  b
+}
+
+b = ( z: int
+  (x: int // y: int)
+)
+HERE
+    assert parser.validate_for_test({})
+    assert parser.validate_for_test({"x" => 2, "z" => 1})
+    assert parser.validate_for_test({"y" => 1, "z" => 2})
+    refute parser.validate_for_test({"z" => 1}, false)
+    assert parser.validate_for_test({"one" => 1, "two" => 2, "uno" => 1, "due" => 2})
+  end
+
+
   def test_validate_a_string
     parser = CDDL::Parser.new <<HERE
 test = [* one: "one"]

data/test-data/7807.cddl

@@ -0,0 +1,14 @@
+coap-problem-details = {
+ type => uint,
+ ? title => text,
+ ? response-code => bstr .size 1,
+ ? detail => text,
+ ? instance => uri,
+ * $$coap-problem-details-extension,
+}
+
+type = 0
+title = 1
+response-code = 2
+detail = 3
+instance = 4

data/test-data/abignum.cddl

@@ -0,0 +1 @@
+abignum = unsigned

data/test-data/ambig.cddl

@@ -0,0 +1,2 @@
+b = [ a ]
+a = uint

data/test-data/badaddr.cddl

@@ -0,0 +1 @@
+badaddr = text .regexp "[0-9A-Fa-f][0-9A-Fa-f]:[0-9A-Fa-f][0-9A-Fa-f]:[0-9A-Fa-f][0-9A-Fa-f]:[0-9A-Fa-f][0-9A-Fa-f]:[0-9A-Fa-f][0-9A-Fa-f]:[0-9A-Fa-f][0-9A-Fa-f]"

data/test-data/bpv7.cddl

@@ -0,0 +1,136 @@
+   start = bundle
+
+   dtn-time = uint
+
+   creation-timestamp = [dtn-time, sequence: uint]
+
+   eid-generic = [uri-code, SSP: any]
+
+   uri-code = uint
+
+   eid = eid-choice .within eid-generic
+
+   eid-choice /= [dtn-code, SSP: (text / 0)]
+
+   dtn-code = 1 ; TBD
+
+   eid-choice /= [ipn-code, SSP: [nodenum: uint, servicenum: uint]]
+
+   ipn-code = 2 ; TBD
+
+   bundle-control-flags = uint .bits bundleflagbits
+
+   bundleflagbits = &(
+
+     reserved: 15
+
+     reserved: 14
+
+     reserved: 13
+
+     bundle-deletion-status-reports-are-requested: 12
+
+     bundle-delivery-status-reports-are-requested: 11
+
+     bundle-forwarding-status-reports-are-requested: 10
+
+     reserved: 9
+
+     bundle-reception-status-reports-are-requested: 8
+
+     bundle-contains-a-Manifest-block: 7
+
+     status-time-is-requested-in-all-status-reports: 6
+
+     user-application-acknowledgement-is-requested: 5
+
+     reserved: 4
+
+     reserved: 3
+
+     bundle-must-not-be-fragmented: 2
+
+     payload-is-an-administrative-record: 1
+
+     bundle-is-a-fragment: 0
+
+   )
+
+   crc = bytes
+
+   block-control-flags = uint .bits blockflagbits
+
+   blockflagbits = &(
+
+     reserved: 7
+
+     reserved: 6
+
+     reserved: 5
+
+     reserved: 4
+
+     bundle-must-be-deleted-if-block-cannot-be-processed: 3
+
+     status-report-must-be-transmitted-if-block-cannot-be-processed: 2
+
+     block-must-be-removed-from-bundle-if-it-cannot-be-processed: 1
+
+     block-must-be-replicated-in-every-fragment: 0
+
+   )
+
+   bundle = [primary-block, *extension-block, payload-block]
+
+   primary-block = [
+
+                 version: 7,
+
+                 bundle-control-flags,
+
+                 crc-type: uint,
+
+                 destination: eid,
+
+                 source-node: eid,
+
+                 report-to: eid,
+
+                 creation-timestamp,
+
+                 lifetime: uint,
+
+                 ? fragment-offset: uint,
+
+                 ? total-application-data-length: uint,
+
+                 ? crc,
+
+   ]
+
+   canonical-block = [
+
+                           block-type-code: uint,
+
+                           canonical-block-common,
+
+                           ? crc
+
+   ]
+
+   canonical-block-common = (
+
+                          block-number: uint,
+
+                          block-control-flags,
+
+                          crc-type: uint,
+
+                          block-type-specific-data: bytes
+
+   )
+
+
+payload-block = ([block-type-code: 1, canonical-block-common, ?crc]) .within canonical-block
+
+extension-block = ([block-type-code: (uint .ne 1), canonical-block-common, ?crc]) .within canonical-block