cddl 0.9.3 → 0.10.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 53440c040b2f038a19f13930772e0c274d40c2074818c333883f571048b094f1
-  data.tar.gz: 606acb287909e0abbf9fe6c038bd1a29649a966a4feab28fa3ffaa20090e3531
+  metadata.gz: 9e60b48882767d7e76926473b44c79369891e4203326d4e31f5693a3819f391d
+  data.tar.gz: 6ed2f45e7fa8468557e73c96a4dbfdd142d65fdebeeb362efeebec8007b7948f
 SHA512:
-  metadata.gz: 7ede3ed88ccb5dba90731fa47da28ed2bd68a05b207fdd4c10cbf495c3fea7da31a65b25aa9d24000276ba1a7518d3d11d6c5a156239e7901b105646bfec6b1f
-  data.tar.gz: 5003a424b4e7edaca09bff529acc1349006dcfba8d3b645b1e3b084aacaf3a0b146f0e371a77427dfbedeb978ff12a10896cc87e1ef44854f6bae94f0b614031
+  metadata.gz: 2124a4fc046c3dc03d007ca949e4f9191497b359fe19fafe307ed2227c4b5654d909815575504e452add310dfc8487bfc13db94794d4b6c06a7af0bb2701e55f
+  data.tar.gz: 8a223332d81282eb9ad651dad887579ceb7b44a01d05ccff6458e271ea999a3c5fad5f81c87a6a2ccf927307b3bb9c4e06075361b0f8fc8b02005926a3d9a5b7

data/cddl.gemspec

@@ -1,6 +1,6 @@
 spec = Gem::Specification.new do |s|
   s.name = 'cddl'
-  s.version = '0.9.3'
+  s.version = '0.10.3'
   s.summary = "CDDL generator and validator."
   s.description = %{A parser, generator, and validator for CDDL}
   s.add_dependency('cbor-diag')
@@ -9,6 +9,7 @@ spec = Gem::Specification.new do |s|
   s.add_dependency('abnftt')
   s.add_dependency('regexp-examples') # , '1.1.0')
   s.add_dependency('colorize')
+  s.add_dependency('base32', '~> 0.3')
   s.files = `git ls-files`.split("\n").grep(/^[a-z]/)
   s.files = Dir['lib/**/*.rb'] + %w(cddl.gemspec) + Dir['data/**/*.abnf'] + Dir['data/**/*.cddl'] + Dir['test-data/**/*.cddl'] + Dir['test/**/*.rb']
   s.require_path = 'lib'

data/lib/base45_lite.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2022 Weihang Jian <https://tonytonyjan.net>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# An implementation of draft-faltstrom-base45-10, see
+# https://datatracker.ietf.org/doc/draft-faltstrom-base45/
+module Base45Lite
+  class Error < ::StandardError; end
+  class OverflowError < Error; end
+  class InvalidCharacterError < Error; end
+  class ForbiddenLengthError < Error; end
+
+  MAX_UINT18 = 2**16 - 1
+  SQUARED_45 = 45**2
+  MAPPING = [
+    *'0'..'9',
+    *'A'..'Z',
+    ' ', '$', '%', '*', '+', '-', '.', '/', ':'
+  ].map!.with_index { |x, i| [i, x] }.to_h.freeze
+  REVERSE_MAPPING = MAPPING.invert.freeze
+
+  def self.encode(input)
+    sequence = []
+    input.unpack('n*').map! do |uint16|
+      i, c = uint16.divmod(45)
+      i, d = i.divmod(45)
+      _, e = i.divmod(45)
+      sequence.push(c, d, e)
+    end
+    if input.bytesize.odd?
+      i, c = input.getbyte(-1).divmod(45)
+      _, d = i.divmod(45)
+      sequence.push(c, d)
+    end
+    sequence.map!{ |n| MAPPING[n] }.join
+  end
+
+  def self.decode(input)
+    input
+      .chars.map! { |c| REVERSE_MAPPING[c] || raise(InvalidCharacterError) }
+      .each_slice(3).map do |slice|
+        c, d, e = slice
+        raise ForbiddenLengthError if d.nil?
+
+        sum = c + d * 45
+        sum += e * SQUARED_45 if e
+        raise OverflowError if sum > MAX_UINT18
+
+        sum
+      end
+      .pack((input.length % 3).zero? ? 'n*' : "n#{input.length / 3}C")
+  end
+end

data/lib/cddl.rb

@@ -2,10 +2,13 @@ require 'abnc'
 require 'pp'
 require 'pathname'
 require 'cbor-pure' unless defined?(CBOR::Tagged)
+require 'cbor-deterministic'
 require 'regexp-examples'
 require 'abnftt'
 require 'colorize'
 require 'base64'
+require 'base32'
+require 'base45_lite'
 
 module CDDL
 
@@ -22,6 +25,20 @@ module CDDL
 
   class Parser
 
+    FEATURE_REJECT_RE = /\A\^/
+    # CDDL_FEATURE_OK=cbor,^json
+    CDDL_FEATURE_OK, CDDL_FEATURE_REJECT =
+                 if ok = ENV["CDDL_FEATURE_OK"]
+                   ok.split(/,\s*/)
+                     .partition{|s| s[0] !~ FEATURE_REJECT_RE}
+                     .map {|l| Hash[l.map {|feature|
+                                      [feature.sub(FEATURE_REJECT_RE, ''),
+                                       true]}]}
+                 else
+                   [{}, {}]
+                 end
+
+
     class BackTrack < Exception; end
 
     def initialize(source_text)
@@ -130,7 +147,7 @@ module CDDL
         if subtree[0] == :type1 && subtree[1..-1].all? {|x| x[0] == :int}
           if enumname = subtree.cbor_annotations rescue nil
             enumname = cname(enumname.first)
-            subtree[1..-1].each do |x| 
+            subtree[1..-1].each do |x|
               if memname = x.cbor_annotations
                 memname = "#{enumname}_#{cname(memname.first)}"
                 add.(memname, x[1].to_s)
@@ -252,6 +269,21 @@ module CDDL
       l.map {|l| l.sub(/^ {0,#{indent}}/, "")}.join
     end
 
+    def unpack_array_to_sequence(content, where)
+      # remove the first head
+      n = case content.getbyte(0) - (4 << 5)
+          when 0..23; 1
+          when 24; 2
+          when 25; 3
+          when 26; 5
+          when 27; 9      # unlikely :-)
+          else fail ".cborseq sequence for #{where} not an array"
+          end
+      content[0...n] = ''
+      content
+    end
+
+
     # Memoize a bit here
 
     REGEXP_FOR_STRING = Hash.new {|h, k|
@@ -459,6 +491,11 @@ module CDDL
           end
           generate1(target, inmap)
         when :feature
+          feature, = extract_feature(control, nil)
+          # warn "@@@ GENFEAT #{feature.inspect} #{CDDL_FEATURE_REJECT[feature].inspect}"
+          if CDDL_FEATURE_REJECT[feature]
+            fail BackTrack.new("Feature '#{feature}' rejected")
+          end
           generate1(target, inmap)
         when :cat, :det
           lhs = generate1(target, inmap)
@@ -541,23 +578,67 @@ module CDDL
           end
           enc = bytes ? Encoding::BINARY : Encoding::UTF_8
           out.force_encoding(enc)
-        when :cbor, :cborseq
+        when :cbor, :cborseq, :cbordet, :cborseqdet
           unless target == [:prim, 2]
             fail "Don't know yet how to generate #{where}"
           end
-          content = CBOR::encode(generate1(control))
-          if conop == :cborseq
+          input = generate1(control)
+          if conop == :cbordet || conop == :cborseqdet
+            input = input.cbor_prepare_deterministic
+          end
+          content = CBOR::encode(input)
+          if conop == :cborseq || conop == :cborseqdet
             # remove the first head
-            n = case content.getbyte(0) - (4 << 5)
-                when 0..23; 1
-                when 24; 2
-                when 25; 3
-                when 26; 5
-                when 27; 9      # unlikely :-)
-                else fail "Generated .cborseq sequence for #{where} not an array"
-                end
-            content[0...n] = ''
+            content = unpack_array_to_sequence(content, where)
+          end
+          content
+        when :json
+          unless target == [:prim, 3]
+            fail "Don't know yet how to generate #{where}"
           end
+          content = JSON::dump(generate1(control))
+          content
+        when :decimal
+          unless target == [:prim, 3]
+            fail "Don't know yet how to generate #{where}"
+          end
+          content = Integer(generate1(control)).to_s
+          content
+        when :join
+          content = generate1(control)
+          unless Array === content && 
+                 content.all? {|x| String === x &&
+                               ((target == [:prim, 2] && x.encoding == Encoding::BINARY) ||
+                                (target == [:prim, 3] && x.encoding != Encoding::BINARY))}
+            fail "Don't know yet how to generate #{where}"
+          end
+          content = content.join
+          content
+        when :b64u, :"b64u-sloppy", :b64c, :"b64c-sloppy",
+             :b45, :b32, :h32, :hex, :hexlc, :hexuc
+          content = generate1(control)
+          unless String === content
+            fail "Don't know yet how to generate #{where}"
+          end
+          content = case conop
+                    when :b64u, :"b64u-sloppy"
+                      Base64.urlsafe_encode64(content, padding: false)
+                    when :b64c, :"b64c-sloppy"
+                      Base64.strict_encode64(content)
+                    when :b45
+                      Base45Lite.encode(content)
+                    when :b32
+                      Base32.table = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"
+                      Base32.encode(content).gsub("=", "")
+                    when :h32
+                      Base32.table = "0123456789ABCDEFGHIJKLMNOPQRSTUV"
+                      Base32.encode(content).gsub("=", "")
+                    when :hex, :hexuc
+                      content.unpack("H*")[0].upcase
+                    when :hexlc
+                      content.unpack("H*")[0]
+                    else fail "Cannot happen"
+                    end
           content
         when :within, :and
           32.times do
@@ -636,6 +717,25 @@ module CDDL
        }]
     end
 
+    def extract_feature(control, d)
+      ok, v, vt = extract_value(control)
+      if ok
+        nm = v
+        det = d
+        warn "*** feature controller should be a string: #{control.inspect}" unless String == vt
+      else
+        ok, *v = extract_array(control)
+        if ok && v.size == 2
+          nm = v[0][0]
+          det = v[1][0]
+          warn "*** first element of feature controller should be a string: #{control.inspect}" unless String === nm
+        else
+          warn "*** feature controller not implemented: #{control.inspect}"
+        end
+      end
+      [nm, det]
+    end
+
     def validate_diag
       [@last_data, @last_rule, @last_message]
     end
@@ -647,9 +747,8 @@ module CDDL
       if warn
         if result
           if $features != {}
-            ok = ENV["CDDL_FEATURE_OK"] and ok = ok.split(/,\s*/) or ok = []
-            # warn([:OK, ok, $features].inspect)
-            features = $features.reject {|k, v| ok.include? k.to_s }
+            features = $features.reject {|k, v| CDDL_FEATURE_OK[k.to_s] }
+            # warn "@@@ FEAT #{CDDL_FEATURE_OK.inspect} #{CDDL_FEATURE_REJECT.inspect}"
             warn "** Features potentially used (#$fn): #{features.map {|k, v| "#{k}: #{v.keys}"}.join(", ")}" if features != {}
           end
         else
@@ -915,23 +1014,9 @@ module CDDL
             end
             ann
           when :feature
-            ok, v, vt = extract_value(control)
-            if ok
-              nm = v
-              det = d
-              warn "*** feature controller should be a string: #{control.inspect}" unless String == vt
-            else
-              ok, *v = extract_array(control)
-              if ok && v.size == 2
-                nm = v[0][0]
-                det = v[1][0]
-                warn "*** first element of feature controller should be a string: #{control.inspect}" unless String === nm
-              else
-                warn "*** feature controller not implemented: #{control.inspect}"
-              end
-            end
+            nm, det = extract_feature(control, d)
             $features[nm][det] = true
-            ann
+            ann if !CDDL_FEATURE_REJECT[nm]
           when :lt, :le, :gt, :ge, :eq, :ne
             op = OPERATORS[conop]
             ok, v, _vt = extract_value(control)
@@ -971,6 +1056,76 @@ module CDDL
             ann if validate1((CBOR::decode(d) rescue :BAD_CBOR), control)
           when :cborseq
             ann if validate1((CBOR::decode("\x9f".b << d << "\xff".b) rescue :BAD_CBOR), control)
+          when :cbordet
+            decoded = CBOR::decode(d) rescue :BAD_CBOR
+            if d != decoded.to_deterministic_cbor
+              @last_message = "CBOR #{d.inspect} not deterministically encoded"
+              nil
+            else
+              ann if validate1(decoded, control)
+            end
+          when :cborseqdet
+            decoded = CBOR::decode("\x9f".b << d << "\xff".b) rescue :BAD_CBOR
+            if d != unpack_array_to_sequence(decoded.to_deterministic_cbor, d.inspect)
+              @last_message = "CBOR Sequence #{d.inspect} not deterministically encoded"
+              nil
+            else
+              ann if validate1(decoded, control)
+            end
+          when :json
+            ann if validate1((JSON::load(d) rescue :BAD_JSON), control)
+          when :decimal
+            ann if (
+                if /\A0|-?[1-9][0-9]*\z/ === d
+                  validate1((d.to_i rescue :BAD_JSON), control)
+                end
+              )
+          when :join
+            ok, *v = extract_array(control)
+            # warn "@@@JJJ@@@ #{ok.inspect} #{v.inspect}"
+            if ok
+              expected = v.map {|ve|
+                fail "Not a string in #{where}" unless String === ve[0]
+                ve[0]
+              }.join
+              ann if d == expected
+            else
+              fail "Don't know yet how to validate against #{where}"
+            end
+          when :b64u, :"b64u-sloppy", :b64c, :"b64c-sloppy", :b45, :b32, :h32, :hex, :hexlc, :hexuc
+            ann if (
+                String === d && (
+                decoded = case conop
+                    when :b64u
+                      /=/ !~ d &&
+                      Base64.urlsafe_decode64(d)
+                    when :"b64u-sloppy"
+                      /[-_=]/ !~ d &&
+                      Base64.decode64(d.tr("+/", "-_"))
+                    when :b64c
+                      Base64.strict_decode64(d)
+                    when :"b64c-sloppy"
+                      Base64.decode64(d)
+                    when :b32
+                      /=/ !~ d &&
+                        (Base32.table = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567") &&
+                        Base32.decode(d)
+                    when :b45
+                      Base45Lite.decode(d)
+                    when :h32
+                      /=/ !~ d &&
+                        (Base32.table = "0123456789ABCDEFGHIJKLMNOPQRSTUV") &&
+                        Base32.decode(d)
+                    when :hex
+                      /\A[0-9a-fA-F]*\z/ =~ d && [d].pack("H*")
+                    when :hexuc
+                      /\A[0-9A-F]*\z/ =~ d && [d].pack("H*")
+                    when :hexlc
+                      /\A[0-9a-f]*\z/ =~ d && [d].pack("H*")
+                    else fail "Cannot happen"
+                    end
+                ) && validate1(decoded.b, control)
+              )
           when :within
             if validate1(d, control)
               ann
@@ -1120,7 +1275,7 @@ module CDDL
       s = n.to_s
       if s[-1] == "'"
         bsqual, *parts = s.split("'", -1)
-        if parts[-1] != "" 
+        if parts[-1] != ""
           warn "*** Problem decoding byte string #{s.inspect}"
         end
         bsval = parts[0...-1].join("'").gsub(/\\(.)/){$1}
@@ -1325,9 +1480,15 @@ module CDDL
 
     BRACE = {"{" => :map, "[" => :array}
     RANGE_EXCLUDE_END = {".." => false, "..." => true}
-    SUPPORTED_ANNOTATIONS = [:bits, :size, :regexp, :cbor, :cborseq, :within, :and,
+    SUPPORTED_ANNOTATIONS = [:bits, :size, :regexp,
+                             :cbor, :cborseq, :cbordet, :cborseqdet,
+                             :within, :and,
                              :default, :lt, :le, :gt, :ge, :eq, :ne,
-                             :feature, :abnf, :abnfb, :det, :cat, :plus]
+                             :feature, :abnf, :abnfb, :det, :cat, :plus,
+                             :json, :decimal, :join,
+                             :b64u, :"b64u-sloppy", :b64c, :"b64c-sloppy",
+                             :b45, :b32, :h32, :hex, :hexlc, :hexuc,
+                            ]
 
     def type1(n, canbegroup = false)
 #      puts "NVALUE #{n.value.inspect}"

data/test/test-cddl.rb

@@ -275,7 +275,7 @@ b = (
   (x: int // y: int), z: int
 )
 HERE
-    assert parser.validate_for_test({})
+    refute parser.validate_for_test({}, false)
     assert parser.validate_for_test({"x" => 2, "z" => 1})
     assert parser.validate_for_test({"y" => 1, "z" => 2})
     refute parser.validate_for_test({"z" => 1}, false)
@@ -293,7 +293,7 @@ b = ( z: int
   (x: int // y: int)
 )
 HERE
-    assert parser.validate_for_test({})
+    refute parser.validate_for_test({}, false)
     assert parser.validate_for_test({"x" => 2, "z" => 1})
     assert parser.validate_for_test({"y" => 1, "z" => 2})
     refute parser.validate_for_test({"z" => 1}, false)

data/test-data/b64c-sloppy.cddl

@@ -0,0 +1 @@
+a = text .b64c-sloppy 'mnoabcdue=='

data/test-data/b64c.cddl

@@ -0,0 +1 @@
+a = text .b64c 'mnoabcd'

data/test-data/b64u.cddl

@@ -0,0 +1 @@
+a = text .b64u 'mnoabcd'

data/test-data/base32.cddl

@@ -0,0 +1 @@
+a = text .b32 'mnoabcd'

data/test-data/base32hex.cddl

@@ -0,0 +1 @@
+a = text .h32 'mnoabcd'

data/test-data/decimal.cddl

@@ -0,0 +1 @@
+a = text .decimal uint

data/test-data/decimal2.cddl

@@ -0,0 +1 @@
+a = text .decimal 4711

data/test-data/hex.cddl

@@ -0,0 +1 @@
+a = text .hex 'mnoabcd'

data/test-data/hexlc.cddl

@@ -0,0 +1 @@
+a = text .hexlc 'mnoabcd'

data/test-data/hexuc.cddl

@@ -0,0 +1 @@
+a = text .hexuc 'mnoabcd'

data/test-data/join.cddl

@@ -0,0 +1 @@
+a = text .join (["a", "b", "c"])

data/test-data/joini.cddl

@@ -0,0 +1 @@
+a = text .join (["a", "b", 3])

data/test-data/joinx.cddl

@@ -0,0 +1 @@
+a = text .join (["a", "b", text])

data/test-data/json.cddl

@@ -0,0 +1 @@
+a = text .json ([4, uint .size 1, {"a": "b"}])

data/test-data/json2.cddl

@@ -0,0 +1 @@
+a = text .json ([4, uint .size 2, {"a": "b"}])

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cddl
 version: !ruby/object:Gem::Version
-  version: 0.9.3
+  version: 0.10.3
 platform: ruby
 authors:
 - Carsten Bormann
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-10-28 00:00:00.000000000 Z
+date: 2023-03-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: cbor-diag
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: base32
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
 description: A parser, generator, and validator for CDDL
 email: cabo@tzi.org
 executables:
@@ -105,6 +119,7 @@ files:
 - cddl.gemspec
 - data/cddl.abnf
 - data/prelude.cddl
+- lib/base45_lite.rb
 - lib/cbor-pp-play.rb
 - lib/cbor-pp.rb
 - lib/cddl.rb
@@ -119,7 +134,12 @@ files:
 - test-data/abnf3.cddl
 - test-data/ambig.cddl
 - test-data/b.cddl
+- test-data/b64c-sloppy.cddl
+- test-data/b64c.cddl
+- test-data/b64u.cddl
 - test-data/badaddr.cddl
+- test-data/base32.cddl
+- test-data/base32hex.cddl
 - test-data/basic_syntax_example.cddl
 - test-data/bat.cddl
 - test-data/bpv7.cddl
@@ -135,6 +155,8 @@ files:
 - test-data/cose.cddl
 - test-data/dcaf.cddl
 - test-data/dcaf1.cddl
+- test-data/decimal.cddl
+- test-data/decimal2.cddl
 - test-data/det1.cddl
 - test-data/dotsize.cddl
 - test-data/extractor-demo.cddl
@@ -148,6 +170,9 @@ files:
 - test-data/grasp-09.cddl
 - test-data/grasp-v1.cddl
 - test-data/grasp-v2X.cddl
+- test-data/hex.cddl
+- test-data/hexlc.cddl
+- test-data/hexuc.cddl
 - test-data/homenet-de.cddl
 - test-data/homenet-fe.cddl
 - test-data/ifmap-base-2.0v17.cddl
@@ -157,6 +182,11 @@ files:
 - test-data/jcr-ex.cddl
 - test-data/jim-cut-2.cddl
 - test-data/jim-cut.cddl
+- test-data/join.cddl
+- test-data/joini.cddl
+- test-data/joinx.cddl
+- test-data/json.cddl
+- test-data/json2.cddl
 - test-data/jsoniodef.cddl
 - test-data/kevin5.cddl
 - test-data/lint1.cddl
@@ -217,7 +247,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.11
+rubygems_version: 3.4.6
 signing_key:
 specification_version: 4
 summary: CDDL generator and validator.