cddl 0.8.16 → 0.8.21

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4939bdcbf4646dc3e2606b7590db6967cc5ccdb0fcb3ac278376b2393baefef2
-  data.tar.gz: ee4e8dc9a948843a3ad4ee3adbc9a128c3de921a5c2056c8d91c82ecbdf85372
+  metadata.gz: 207ea6c166a60cfc705009af0a15b9633e301429cd8958d279f9d710800765bb
+  data.tar.gz: 6b0e05ce4c0059d2a40a39892784da5e838451772bbad0d301c056a2c83418cb
 SHA512:
-  metadata.gz: 07360a1f513c23f9cabb0f64a870bbd961cc336f722b7d857f6d9e92dd47d8fe328899772dcd27afa24af47085e579c8a029ed4d02b8cbecb95b7af7ea2d50cf
-  data.tar.gz: 9dbf6d31bb5656b958274ec32d39bb76ebaf57e015cd819487f48c752b2d786667166f78f9d822322cfba6c7750a5559dc2baf8e938c42e4ce35b7bf3d509dd0
+  metadata.gz: 8fd0733ddb7c869b89578d92aee41d818ae3a4c85fbb9db0d4377136498195c054c17670c00ae89322578d5f6dfa64e1f10c5ed6c9c1cfcee27e9d135d9ac843
+  data.tar.gz: 7a64af74fe96ae30fe834c36ca7c59326257f27e2d9a16ac3edb97bd1772c46e6f77c4a8dd715c27996a1b59369b8efe7be8332f82215afd7a0014cb72097f59

data/bin/cddl

@@ -14,8 +14,8 @@ def usage
   warn "Usage:"
   warn "#$0 spec.cddl generate [n]"
   warn "#$0 spec.cddl json-generate [n]"
-  warn "#$0 spec.cddl validate instance.cbor"
-  warn "#$0 spec.cddl validate instance.json"
+  warn "#$0 spec.cddl validate instance.cbor..."
+  warn "#$0 spec.cddl validate instance.json..."
   exit EX_USAGE
 end
 
@@ -48,6 +48,8 @@ def my_diag(v)
   end
 end
 
+retcode = 0
+
 begin
   case ARGV[1]
   when /\A..*p/
@@ -85,19 +87,25 @@ begin
       g = parser.generate
       puts JSON.pretty_generate(g)
     end
-  when /\Av/
-    instance = read_arg(ARGV[2])
-    instance = if $sequence
-      CBOR.decode("\x9f".b << instance.b << "\xff".b)
-    else
-      CBOR.decode(instance.b) rescue JSON.load(instance)
+  when /\Av(v)?/
+    verbose = $1
+    ARGV[2..-1].each do |fn|
+      instance = read_arg(fn)
+      instance = if $sequence
+                   CBOR.decode("\x9f".b << instance.b << "\xff".b)
+                 else
+                   CBOR.decode(instance.b) rescue JSON.load(instance)
+                 end
+      instance = instance.cbor_clone if $annotate && ENV["EXPERIMENTAL_ANNOTATE"]
+      warn "#{fn}:" if verbose
+      ann = parser.validate(instance)
+      # my_pp ann
+      instance.cbor_add_annotations_from(ann) rescue nil
+      my_diag(instance) if $annotate
+      unless ann
+        retcode = 1
+      end
     end
-    instance = instance.cbor_clone if $annotate && ENV["EXPERIMENTAL_ANNOTATE"]
-    ann = parser.validate(instance)
-    # my_pp ann
-    instance.cbor_add_annotations_from(ann) rescue nil
-    my_diag(instance) if $annotate
-    exit 1 unless ann
   else
     usage
   end
@@ -105,3 +113,5 @@ rescue CDDL::ParseError => e
   warn e.message
   exit EX_DATAERR
 end
+
+exit retcode

data/cddl.gemspec

@@ -1,11 +1,12 @@
 spec = Gem::Specification.new do |s|
   s.name = 'cddl'
-  s.version = '0.8.16'
+  s.version = '0.8.21'
   s.summary = "CDDL generator and validator."
   s.description = %{A parser, generator, and validator for CDDL}
   s.add_dependency('cbor-diag')
   s.add_dependency('abnc')
   s.add_dependency('json')
+  s.add_dependency('abnftt')
   s.add_dependency('regexp-examples') # , '1.1.0')
   s.add_dependency('colorize')
   s.files = `git ls-files`.split("\n").grep(/^[a-z]/)

data/lib/cddl.rb

@@ -3,6 +3,7 @@ require 'pp'
 require 'pathname'
 require 'cbor-pure' unless defined?(CBOR::Tagged)
 require 'regexp-examples'
+require 'abnftt'
 require 'colorize'
 require 'base64'
 
@@ -243,12 +244,28 @@ module CDDL
       [rule[0], *rule[1]]
     end
 
+    def remove_indentation(s)
+      l = s.lines
+      indent = l.grep(/\S/).map {|l| l[/^\s*/].size}.min
+      l.map {|l| l.sub(/^ {0,#{indent}}/, "")}.join
+    end
+
     # Memoize a bit here
 
     REGEXP_FOR_STRING = Hash.new {|h, k|
       h[k] = Regexp.new("\\A#{k}\\z")
     }
 
+    ABNF_PARSER_FOR_STRING = Hash.new {|h, k|
+      grammar = "cddl-t0p--1eve1-f0r--abnf = " << k # XXX
+      h[k] = ABNF.from_abnf(grammar)
+    }
+
+    ABNF_ENCODING_FOR_CONOP = {
+      abnf: Encoding::UTF_8,
+      abnfb: Encoding::BINARY
+    }
+
     def generate
       @recursion = 0
       generate1(rules)
@@ -393,7 +410,7 @@ module CDDL
       when :anno
         target = where[2]
         control = where[3]
-        case where[1]
+        case conop = where[1]
         when :size
           should_be_int = generate1(control)
           unless (Array === target && target[0] == :prim && [0, 2, 3].include?(target[1])) && Integer === should_be_int && should_be_int >= 0
@@ -441,9 +458,10 @@ module CDDL
           generate1(target, inmap)
         when :feature
           generate1(target, inmap)
-        when :cat
+        when :cat, :det
           lhs = generate1(target, inmap)
           rhs = generate1(control)
+          rhs = remove_indentation(rhs) if conop == :det
           begin
             lhs + rhs
           rescue Exception => e
@@ -477,7 +495,7 @@ module CDDL
                     content = Integer(content)
                     case target[1]
                     when 0
-                      case where[1]
+                      case conop
                       when :lt
                         rand(0...content)
                       when :le
@@ -504,12 +522,25 @@ module CDDL
             fail "Don't know yet how to generate #{where}"
           end
           REGEXP_FOR_STRING[regexp].random_example(max_repeater_variance: 5)
+        when :abnf, :abnfb
+          grammar = generate1(control)
+          bytes = true if target == [:prim, 2]
+          bytes = false if target == [:prim, 3]
+          unless !bytes.nil? && String === grammar
+            fail "Don't know yet how to generate #{where}"
+          end
+          out = ABNF_PARSER_FOR_STRING[grammar].generate
+          if conop == :abnfb
+            out = out.codepoints.pack("C*")
+          end
+          enc = bytes ? Encoding::BINARY : Encoding::UTF_8
+          out.force_encoding(enc)
         when :cbor, :cborseq
           unless target == [:prim, 2]
             fail "Don't know yet how to generate #{where}"
           end
           content = CBOR::encode(generate1(control))
-          if where[1] == :cborseq
+          if conop == :cborseq
             # remove the first head
             n = case content.getbyte(0) - (4 << 5)
                 when 0..23; 1
@@ -527,7 +558,7 @@ module CDDL
             content = generate1(target)
             if validate1(content, control)
               return content
-            elsif where[1] == :within
+            elsif conop == :within
               warn "*** #{content.inspect} meets #{target.inspect} but not #{control.inspect}"
             end
           end
@@ -568,7 +599,7 @@ module CDDL
       elsif t[0] == :anno
         _, conop, target, control = t
         # warn ["EXV0", conop, target, control].inspect
-        if conop == :cat || conop == :plus
+        if conop == :cat || conop == :plus || conop == :det
           ok1, v1, vt1 = extract_value(target)
           ok2, v2, vt2 = extract_value(control)
           # warn ["EXV", ok1, v1, vt1, ok2, v2, vt2].inspect
@@ -578,6 +609,7 @@ module CDDL
             elsif vt1 == Float
               [true, v1 + v2, vt1] if vt2 == Integer || vt2 == Float
             else
+              v2 = remove_indentation(v2) if conop == :det
               [true, v1 + v2, vt1] if vt1 == vt2
             end
           end rescue nil
@@ -622,7 +654,7 @@ module CDDL
 
     def validate_result(check)
       check || (
-        @last_message = yield
+        @last_message << yield
         false
       )
     end
@@ -655,7 +687,9 @@ module CDDL
             ann.concat(ann2)
           end
           if occ < s
-            @last_message = "occur not reached in array #{d} for #{where}"
+            # warn "*** lme #{@last_message.encoding} #{@last_message}"
+            # warn "*** #{"\noccur #{occ} < #{s}, not reached at #{i} in array #{d} for #{where}".encoding}"
+            @last_message << "\noccur #{occ} < #{s}, not reached at #{i} in array #{d} for #{where}"
             return [false, ann]
           end
         end
@@ -820,6 +854,7 @@ module CDDL
           # warn ["ANNO0", ok1, v1, vt1, ok2, v2, vt2, d].inspect
           if ok1 && ok2
             v2 = Integer(v2) if vt1 == Integer
+            v2 = remove_indentation(v2) if conop == :det
             # warn ["ANNO", ok1, v1, vt1, ok2, v2, vt2, d].inspect
             [] if d == v1 + v2  # XXX Focus ArgumentError
           end
@@ -896,6 +931,24 @@ module CDDL
               end
             end
             )
+          when :abnf, :abnfb
+            ann if (
+            if String === d
+              ok, v, vt = extract_value(control)
+              if ok && vt == String
+                begin
+                  ABNF_PARSER_FOR_STRING[v].validate(
+                    d.dup.force_encoding(ABNF_ENCODING_FOR_CONOP[conop]).codepoints.pack("U*")
+                  )
+                  true
+                rescue => e
+                  # warn "*** #{e}" # XXX
+                  @last_message = e.to_s.force_encoding(Encoding::UTF_8)
+                  nil
+                end
+              end
+            end
+            )
           when :cbor
             ann if validate1((CBOR::decode(d) rescue :BAD_CBOR), control)
           when :cborseq
@@ -1250,7 +1303,8 @@ module CDDL
     BRACE = {"{" => :map, "[" => :array}
     RANGE_EXCLUDE_END = {".." => false, "..." => true}
     SUPPORTED_ANNOTATIONS = [:bits, :size, :regexp, :cbor, :cborseq, :within, :and,
-                             :default, :lt, :le, :gt, :ge, :eq, :ne, :feature, :cat, :plus]
+                             :default, :lt, :le, :gt, :ge, :eq, :ne,
+                             :feature, :abnf, :abnfb, :det, :cat, :plus]
 
     def type1(n, canbegroup = false)
 #      puts "NVALUE #{n.value.inspect}"

data/test-data/abnf1.cddl

@@ -0,0 +1,3 @@
+foo = text .abnf '4DIGIT
+DIGIT          =  %x30-39 ; 0-9
+'

data/test-data/abnf2.cddl

@@ -0,0 +1,37 @@
+start = [Tag0, Tag1004]
+
+; for draft-ietf-cbor-date-tag
+Tag1004 = #6.1004(text .abnf full-date)
+; for RFC 7049
+Tag0 = #6.0(text .abnf date-time)
+
+full-date = "full-date" .cat rfc3339
+date-time = "date-time" .cat rfc3339
+
+; Note the trick of idiomatically starting with a newline, separating
+;   off the element in the .cat from the rule-list
+rfc3339 = '
+date-fullyear   = 4DIGIT
+date-month      = 2DIGIT  ; 01-12
+date-mday       = 2DIGIT  ; 01-28, 01-29, 01-30, 01-31 based on
+                        ; month/year
+time-hour       = 2DIGIT  ; 00-23
+time-minute     = 2DIGIT  ; 00-59
+time-second     = 2DIGIT  ; 00-58, 00-59, 00-60 based on leap sec
+                        ; rules
+time-secfrac    = "." 1*DIGIT
+time-numoffset  = ("+" / "-") time-hour ":" time-minute
+time-offset     = "Z" / time-numoffset
+
+partial-time    = time-hour ":" time-minute ":" time-second
+                [time-secfrac]
+full-date       = date-fullyear "-" date-month "-" date-mday
+full-time       = partial-time time-offset
+
+date-time       = full-date "T" full-time
+' .cat rfc5234-core
+
+rfc5234-core = '
+DIGIT          =  %x30-39 ; 0-9
+; abbreviated here
+'

data/test-data/abnf3.cddl

@@ -0,0 +1,32 @@
+start = [tt, tb, bt, bb]
+
+
+tt = text .abnf '4DIGIT 1FOO
+DIGIT          =  %x30-39 ; 0-9
+FOO = %xc0-cf
+'
+tb = text .abnfb '4DIGIT 1FOO
+DIGIT          =  %x30-39 ; 0-9
+FOO = %xc0-cf
+'
+bt = bytes .abnf '4DIGIT 1FOO
+DIGIT          =  %x30-39 ; 0-9
+FOO = %xc0-cf
+'
+bb = bytes .abnfb '4DIGIT 1FOO
+DIGIT          =  %x30-39 ; 0-9
+FOO = %xc0-cf
+'
+
+; ["7408\xC6", "7073\xC3", h'30383131CD', h'32363738CD']
+; ["5892\u00ca", "2145\u00c6", h'33333338C38B', h'31343033C388']
+
+
+; ** ENCOED UTF-8
+; ** ENCOED UTF-8
+; ** ENCOED UTF-8
+; ** ENCOED UTF-8
+; ["5845\u00c6", "9329\u00cb", h'38393538C389', h'33343230C386']
+
+
+; ["1066\u00cc", "9253\xC9", h'38333533C38A', h'32373337C4']

data/test-data/bat.cddl

@@ -0,0 +1,16 @@
+start = [v1, v2, v3, v4]
+
+v1 = "a" .bat "b"
+
+v2 = "a" .bat " b"
+
+v3 = "a" .bat '
+  oid = 1*arc
+  roid = *arc
+  arc = [nlsb] %x00-7f
+  nlsb = %x81-ff *%x80-ff
+'
+
+bbat<v1, v2> = ("" .bat v1) .bat v2
+
+v4 = bbat<" a", "   b">

data/test-data/cat-re.cddl

@@ -0,0 +1,2 @@
+foo = text .regexp myregexp
+myregexp = "A" .cat "B"

data/test-data/oid.cddl

@@ -0,0 +1,8 @@
+oid = bytes .abnfb ("oid" .cat cbor-tags-oid)
+
+cbor-tags-oid = '
+oid = 1*arc
+roid = *arc
+arc = [nlsb] %x00-7f
+nlsb = %x81-ff *%x80-ff
+'

data/test-data/oidbat.cddl

@@ -0,0 +1,8 @@
+oid = bytes .abnfb ("oid" .bat cbor-tags-oid)
+
+cbor-tags-oid = '
+  oid = 1*arc
+  roid = *arc
+  arc = [nlsb] %x00-7f
+  nlsb = %x81-ff *%x80-ff
+'

data/test-data/sasl.cddl

@@ -0,0 +1,24 @@
+message = text .abnfb ("message" .cat rfc4505)
+
+rfc4505 = '
+message     = [ email / token ]
+              ;; to be prepared in accordance with Section 3
+
+UTF1        = %x00-3F / %x41-7F ;; less "@" (U+0040)
+UTF2        = %xC2-DF UTF0
+UTF3        = %xE0 %xA0-BF UTF0 / %xE1-EC 2(UTF0) /
+              %xED %x80-9F UTF0 / %xEE-EF 2(UTF0)
+UTF4        = %xF0 %x90-BF 2(UTF0) / %xF1-F3 3(UTF0) /
+              %xF4 %x80-8F 2(UTF0)
+UTF0        = %x80-BF
+
+TCHAR       = UTF1 / UTF2 / UTF3 / UTF4
+              ;; any UTF-8 encoded Unicode character
+              ;; except "@" (U+0040)
+
+email        = "too@much.work"; for this example
+;email       = addr-spec
+              ;; as defined in [IMAIL]
+
+token       = 1*255TCHAR
+'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cddl
 version: !ruby/object:Gem::Version
-  version: 0.8.16
+  version: 0.8.21
 platform: ruby
 authors:
 - Carsten Bormann
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-02-15 00:00:00.000000000 Z
+date: 2021-03-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: cbor-diag
@@ -52,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: abnftt
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: regexp-examples
   requirement: !ruby/object:Gem::Requirement
@@ -100,13 +114,18 @@ files:
 - test-data/7807.cddl
 - test-data/a.cddl
 - test-data/abignum.cddl
+- test-data/abnf1.cddl
+- test-data/abnf2.cddl
+- test-data/abnf3.cddl
 - test-data/ambig.cddl
 - test-data/b.cddl
 - test-data/badaddr.cddl
 - test-data/basic_syntax_example.cddl
+- test-data/bat.cddl
 - test-data/bpv7.cddl
 - test-data/bpv7a.cddl
 - test-data/bpv7b.cddl
+- test-data/cat-re.cddl
 - test-data/cdni-ct.cddl
 - test-data/coral.cddl
 - test-data/coral1.cddl
@@ -148,8 +167,11 @@ files:
 - test-data/mon-val.cddl
 - test-data/multipart-ct.cddl
 - test-data/named-group.cddl
+- test-data/oid.cddl
+- test-data/oidbat.cddl
 - test-data/patch1.cddl
 - test-data/reused_named_group.cddl
+- test-data/sasl.cddl
 - test-data/sequence.cddl
 - test-data/structure.cddl
 - test-data/test-gen.cddl