abnftt 0.2.4 → 0.2.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 152ef34be3428f0a7cda4f157ec08a1986f5cf295436be8186ee53d61eddb386
-  data.tar.gz: fdc3504304d4e9c5244dcfce7a2e103e2801d3f6087de307bcd976a099fe0b5d
+  metadata.gz: 436afd21fea00a7369bf5dfdfdcc0a61b33ee0b14f482a6d1c00488a520c93c1
+  data.tar.gz: 93cb2255e048e4482c4245e9beb6bf635a79b253124359a8c0debbd2abb204b2
 SHA512:
-  metadata.gz: e5b786e95c96464bf516e2d6ba00f1363a6a720d0dfd01b83401fef3ade2dc51116fda365881fb770fa9756f0ec71a14e7c6e85e7e3a7bc3589a930e8b9e5381
-  data.tar.gz: '08eb748abf720cfdfe4a742e675e7b5b0144a60cdd2fce90f85783a2b7eac2d856e138f6536394dcccc123e72eb6fa8439b472b1e8ff26fff7630938445b2316'
+  metadata.gz: '09621b3970222f8de758894c6fa6c58821fca89980612234ca31cdedae11fcc1c2c9b2e1fd9edb873865169b640926e049388888a82da7baa4b1dfa972f37dde'
+  data.tar.gz: 4f77b32132f12ae7a2f6c91b250df5213fdb386c21ecd6fe02c6470115be7996448eefbd94f0e9cd833cc446a7a451c7ff65567069cd15695e6093c11505bd43

data/abnftt.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = "abnftt"
-  s.version = "0.2.4"
+  s.version = "0.2.7"
   s.summary = "RFC 5234+7405 ABNF to Treetop"
   s.description = %q{Less shifty support for tools based on IETF's ABNF}
   s.author = "Carsten Bormann"

data/bin/abnfrob

@@ -0,0 +1,149 @@
+#!/usr/bin/env ruby -Ku
+require 'pp'
+require 'yaml'
+
+Encoding.default_external = Encoding::UTF_8
+require 'optparse'
+require 'ostruct'
+
+$options = OpenStruct.new
+begin
+  op = OptionParser.new do |opts|
+    opts.version = "(from abnftt #{Gem.loaded_specs['abnftt'].version})" rescue "unknown-version"
+
+    opts.banner = "Usage: abnfrob [options] file.abnf"
+    opts.on("-b", "--bap=[OPTIONS]", "Pretty-print using bap") do |v|
+      $options.bap = true
+      $options.bap_options = v ? " #{v}" : ""
+      # warn "** bap_options #{$options.bap_options.inspect}"
+    end
+    opts.on("-tFMT", "--to=FMT", [:abnf, :json, :pp, :yaml], "Target format") do |v|
+      $options.target = v
+    end
+    opts.on("-y", "--yaml", "Output separate YAML copy of rules") do |v|
+      $options.yaml = v
+    end
+    opts.on("-a", "--asr33", "Line-break to fit on teletype") do |v|
+      $options.asr33 = v
+    end
+    opts.on("--ascii", "No escaping of ASCII printables %x20-7e") do |v|
+      $options.ascii = v
+    end
+    opts.on("--squash=PREFIX", String, "Squash to app-prefix") do |v|
+      $options.squash = v
+    end
+  end
+  op.parse!
+rescue StandardError => e
+  warn e
+  exit 1
+end
+
+require 'abnftt'
+
+unless fn = ARGV[0]
+  warn op
+  exit 1
+end
+unless File.extname(fn) =~ /\A.abnf/
+  warn op
+  exit 1
+end
+outfn = (Pathname.new(File.dirname(fn)) + File.basename(fn, ".*")).to_s
+
+parser = ABNFGrammarParser.new
+abnf_file = File.read(fn)
+ast = parser.parse abnf_file
+unless ast
+  puts parser.failure_reason
+  parser.failure_reason =~ /^(Expected .+) after/m
+  puts "#{$1.gsub("\n", '<<<NEWLINE>>>')}:"
+  puts abnf_file.lines.to_a[parser.failure_line - 1]
+  puts "#{'~' * (parser.failure_column - 1)}^"
+  exit 1
+end
+
+abnf = ABNF.new(ast)
+
+if $options.yaml
+  File.open("#{outfn}.yaml", "w") do |f|
+    f.puts abnf.rules.to_yaml
+  end
+end
+
+if $options.squash
+  require 'abnftt/abnf-flattener.rb'
+  require 'abnftt/abnf-util.rb'
+  require 'abnftt/abnf-squasher.rb'
+
+  prefix = $options.squash + "-"
+
+  abnf.flatten_ops
+  abnf.flatten_strings
+
+  abnf.squash_edn_levels(ascii: $options.ascii)
+  abnf.char_range_to_string
+
+  abnf.share_hex("sq")
+
+  abnf.share_alt("sq")
+
+  abnf.rules.replace(Hash[abnf.rules.map do |k, v|
+                       [k.sub(/^(?:#{prefix})?/, prefix), abnf.visit(v) do |prod|
+                          if String === prod 
+                            [true, prod.sub(/^(?:#{prefix})?/, prefix)]
+                          end
+                        end]
+                     end])
+
+  rule1 = abnf.rules.first
+  outer_name = "sq-#{rule1[0]}"
+  outer_elements = ["seq",
+                    ["cs", $options.squash+"'"],
+                    rule1[0],
+                    ["cs", "'"]
+                   ]
+  abnf.rules.replace(a = Hash[[[outer_name, outer_elements],
+                               *abnf.rules.to_a]])
+  File.open("#{outfn}-sq.yaml", "w") do |f|
+    f.puts abnf.rules.to_yaml
+  end
+end
+
+## Work on abnf.rules
+
+case $options.target
+when :pp
+  pp abnf.rules
+when :json
+  require 'neatjson'
+  puts JSON.neat_generate(abnf.rules, after_comma: 1, after_colon: 1)
+when :yaml
+  puts abnf.rules.to_yaml
+when :abnf, nil
+  require_relative '../lib/abnftt/abnf-writer.rb'
+  result = abnf.to_s
+
+  if $options.bap
+    require 'open3'
+    result, err, status =
+                 Open3.capture3("bap -o RFC7405#{$options.bap_options}",
+                                stdin_data: result)
+    warn err.gsub(/^/, "** ") unless err == ""
+    unless status.success?
+      warn "*** Giving up"
+      exit 1
+    end
+  end
+
+  if $options.asr33
+    result = abnf.breaker(result)
+  end
+
+  puts result
+  File.open("#{outfn}-sq.abnf", "w") do |f|
+    f.puts result
+  end
+else
+  warn ["Unknown target format: ", $options.target].inspect
+end

data/bin/abnfrob~

@@ -0,0 +1,40 @@
+#!/usr/bin/env ruby -Ku
+
+require 'pp'
+require 'yaml'
+
+Encoding.default_external = Encoding::UTF_8
+
+require 'abnftt'
+
+unless fn = ARGV[0]
+  warn "Usage: abnftt grammar.abnftt"
+  exit 1
+end
+outfn = fn.sub(/\.abnftt\z/, "")
+
+parser = ABNFGrammarParser.new
+abnf_file = File.read(fn)
+ast = parser.parse abnf_file
+if ast
+#  p ast
+  abnf = ABNF.new(ast)
+  File.open("#{outfn}.yaml", "w") do |f|
+    f.puts abnf.tree.to_yaml
+  end
+  # pp ast.ast
+  File.open("#{outfn}.treetop", "w") do |f|
+    modname = File.basename(outfn).gsub("-", "_").gsub(/[^_a-zA-Z0-9]/, "").upcase
+    f.puts abnf.to_treetop(modname)
+  end
+  File.open("#{outfn}.abnf", "w") do |f|
+    f.puts ast.clean_abnf.lines.map(&:rstrip).join("\n")
+  end
+else
+
+  puts parser.failure_reason
+  parser.failure_reason =~ /^(Expected .+) after/m
+  puts "#{$1.gsub("\n", '<<<NEWLINE>>>')}:"
+  puts abnf_file.lines.to_a[parser.failure_line - 1]
+  puts "#{'~' * (parser.failure_column - 1)}^"
+end

data/lib/abnftt/abnf-flattener.rb

@@ -0,0 +1,32 @@
+require "abnftt/abnf-visitor.rb"
+
+class ABNF
+  def expand_op_into(s, op, out = [op])
+    s.each do |el|
+      case el
+      in [^op, *inner]
+        expand_op_into(inner, op, out)
+      else
+        out << flatten_ops_1(el)
+      end
+    end
+    out
+  end
+  def flatten_ops_1(prod)
+    visit(prod) do |here|
+        case here
+        in ["seq", *rest]
+          [true, expand_op_into(rest, "seq")]
+        in ["alt", *rest]
+          [true, expand_op_into(rest, "alt")]
+        else
+          false
+        end
+    end
+  end
+  def flatten_ops
+    rules.each do |name, prod|
+      rules[name] = flatten_ops_1(prod)
+    end
+  end
+end

data/lib/abnftt/abnf-squasher.rb

@@ -0,0 +1,65 @@
+require "abnftt"
+require "abnftt/abnf-flattener"
+require "abnftt/abnf-util"
+require "abnftt/abnf-visitor"
+
+class ABNF
+
+  UNESCAPED_SQSTR_RANGES =
+    # [[0xA, 0xA], [0x20, 0x21], [0x23, 0x26], -- but DQUOTE is allowed
+    [[0xA, 0xA], [0x20, 0x26],  # "'"
+     # [0x28, 0x5b], [0x5d, 0x7e], [0xa0, 0xd7ff], -- but JSON allows 7F-9F
+     [0x28, 0x5b], [0x5d, 0xd7ff], # \
+     [0xe000, 0x10ffff]].map {|l, r|
+    [l.chr(Encoding::UTF_8), r.chr(Encoding::UTF_8)]
+  }
+
+  ESCAPED_SQSTR_MAPPINGS = [
+    ["\x08", "b"],
+    ["\x09", "t"],
+    ["\x0A", "n"],
+    ["\x0C", "f"],
+    ["\x0D", "r"],
+    ["\x27", "'"],
+    ["\x2F", "/"],
+    ["\x5C", "\\"]]
+
+  def squash_edn_levels_1(prod, **options)
+    f1 = visit(prod) do |here|
+      case here
+      in ["char-range", c1, c2]
+        lit = UNESCAPED_SQSTR_RANGES.map { |u1, u2|
+          overlap(here, u1, u2) }.compact
+        esc = ESCAPED_SQSTR_MAPPINGS.map {|cv, ev|
+          if cv >= c1 && cv <= c2
+            ["seq", ["char-range", "\\", "\\"], ["char-range", ev, ev]]
+          end
+        }.compact
+        u_escapes = []
+        if options[:ascii]
+          do_ranges_outside(here, " ", "~") do |l, r|
+            u_escapes << alt_ranges_legacy(l, r) # old
+            u_escapes << alt_ranges_modern(l, r) # new
+          end
+        else
+          u_escapes << alt_ranges_legacy(c1.ord, c2.ord) # old
+          u_escapes << alt_ranges_modern(c1.ord, c2.ord) # new
+        end
+        oldnew = ["seq",
+                  ["cs", "\\u"],
+                  wrap_flat("alt", u_escapes) ] if u_escapes != []
+        [true, wrap_flat("alt", [*lit, *esc, oldnew].compact.sort)]
+      else
+        false
+      end
+    end
+    flatten_ops_1(f1)
+  end
+
+  def squash_edn_levels(**options)
+    rules.each do |name, prod|
+      rules[name] = squash_edn_levels_1(prod, **options)
+    end
+  end
+
+end

data/lib/abnftt/abnf-util.rb

@@ -0,0 +1,388 @@
+require "abnftt/abnf-visitor"
+require "abnftt/abnf-flattener"
+
+class ABNF
+
+  def wrap(head, all)
+    if all.size == 1
+      all.first
+    else
+      [head, *all]
+    end
+  end
+
+  def wrap_flat(head, all)
+    if all.size == 1
+      all.first
+    else
+      [head, *all.collect_concat {|el|
+         case el
+         in [^head, *rest]
+           rest
+         else
+           [el]
+         end
+       }]
+    end
+  end
+
+  def overlap(cr, l, r)
+    if cr[2] >= l && cr[1] <= r
+      ["char-range", [cr[1], l].max, [cr[2], r].min]
+    end
+  end
+
+  def do_ranges_outside(cr, l, r)
+    if cr[2] < l || cr[1] > r # outside
+      yield cr[1].ord, cr[2].ord
+    else
+      if cr[1] < l
+        yield cr[1].ord, l.ord - 1
+      end
+      if cr[2] > r
+        yield r.ord + 1, cr[2].ord
+      end
+    end
+  end
+
+  # Utilities for creating hexadecimal rules from unsigned integers
+
+  def hexdig_range(l, r)
+    alt = []
+    if l < 10
+      alt << ["char-range",
+              (l+0x30).chr(Encoding::UTF_8),
+              ([r, 9].min+0x30).chr(Encoding::UTF_8)]
+    end
+    if r >= 10
+      alt << ["char-range", ([l, 10].max+0x41-0xA).chr(Encoding::UTF_8),
+              (r+0x41-0xA).chr(Encoding::UTF_8)]
+      alt << ["char-range", ([l, 10].max+0x61-0xA).chr(Encoding::UTF_8),
+              (r+0x61-0xA).chr(Encoding::UTF_8)]
+    end
+    wrap("alt", alt)
+  end
+
+  # This assumes l and r are preprocessed to have single or full ranges except in one place
+  def hex_ranges(l, r, ndig = false)
+    ld = l.digits(16)
+    rd = r.digits(16)
+    ndig ||= rd.size
+    seq = []
+    (0...ndig).each do |dig|
+      seq << hexdig_range(ld[dig] || 0, rd[dig] || 0)
+    end
+    wrap("seq", seq.reverse)
+  end
+
+  # split range into passages that have the property needed for hex_ranges
+  def do_range(l, r, step = 4)
+    column = 0
+    while l <= r
+      mask = (1 << step * (column + 1)) - 1
+      new_r = l | mask
+      if new_r > r # right hand side: come down from mountain
+        while column >= 0
+          mask >>= step
+          new_r = (r + 1) & ~mask
+          yield l, new_r - 1, column + 1 if l != new_r
+          l = new_r
+          column -= 1
+        end
+        return
+      else
+        column += 1
+        if (l & mask) != 0
+          yield l, new_r, column
+          l = new_r + 1
+        end
+      end
+    end
+  end
+
+  # Support legacy JSON \u/\u\u and \u{...} hex unicode
+
+  def alt_ranges(l, r, step = 4, ndig = false)
+    alt = []
+    do_range(l.ord, r.ord, step) do |l, r, column|
+      alt << hex_ranges(l, r, ndig)
+    end
+    wrap("alt", alt.reverse)    # work around prioritized choice
+  end
+
+  def alt_ranges_legacy(l, r)
+    alt = []
+    if l < 0x10000
+      alt << ["alt", alt_ranges(l, [r, 0xFFFF].min, 4, 4)]
+    end
+    if r >= 0x10000
+      l1 = [l, 0x10000].max - 0x10000
+      r1 = r - 0x10000
+      do_range(l1, r1, 10) do |l2, r2, column|
+        alt << ["seq",
+                alt_ranges((l2 >> 10) + 0xD800, (r2 >> 10) + 0xD800, 4, 4),
+                expand_string("\\u"),
+                alt_ranges((l2 & 0x3FF) + 0xDC00, (r2 & 0x3FF) + 0xDC00, 4, 4)]
+      end
+    end
+    wrap_flat("alt", alt)
+  end
+
+  def alt_ranges_modern(l, r, step = 4)
+    ["seq",
+     expand_string("{"),
+     ["rep", 0, true, ["cs","0"]],
+     alt_ranges(l, r, 4, false),
+     expand_string("}")]
+  end
+
+  # flatten_strings: reduce all strings to char-range/seq/alt
+
+  def expand_string(s, case_fold = false)
+    wrap("seq",
+         s.chars.map do |ch|
+           if case_fold &&
+              (u = ch.upcase; d = ch.downcase; u != d)
+             ["alt", expand_string(u), expand_string(d)]
+           else
+             ["char-range", ch, ch]
+           end
+         end)
+  end
+
+  def flatten_strings_1(prod)
+    f1 = visit(prod) do |here|
+        case here
+        in ["cs", string]
+          [true, expand_string(string, false)]
+        in ["ci", string]
+          [true, expand_string(string, true)]
+        else
+          false
+        end
+    end
+    merge_strings_1(flatten_ops_1(f1))
+  end
+
+
+  def merge_strings_1(prod)
+    visit(prod) do |here|
+      case here
+      in ["alt", *rest]
+        ranges = []
+        i = 0
+        while i < rest.size
+          case rest[i]
+          in ["char-range", _ic1, _ic2]
+            j = i
+            while j+1 < rest.size && (rest[j+1] in ["char-range", _jc1, _jc2])
+              j += 1
+            end
+            ranges << [i, j] if i != j # inclusive right
+            i = j
+          else
+            here[i+1] = merge_strings_1(rest[i]) # XXX could be part of a range
+          end
+          i += 1
+        end
+        ranges.reverse.each do |i, j|
+          sorted = here[i+1..j+1].sort
+          l = sorted.length
+          while l > 1
+            l -= 1              # index to last item
+            if sorted[l][1].ord == sorted[l-1][2].ord+1 # merge:
+              sorted[l-1..l] = [["char-range", sorted[l-1][1], sorted[l][2]]]
+            end
+          end
+          here[i+1..j+1] = sorted
+        end
+        [true, here]
+      else
+        false
+      end
+    end
+  end
+
+  def flatten_strings
+    rules.each do |name, prod|
+      rules[name] = flatten_strings_1(prod)
+    end
+  end
+
+  # Cleanup operations
+
+  def expand_range_into(s, op, out = [op])
+    s.each do |el|
+      case el
+      in [^op, *inner]
+        expand_range_into(inner, op, out)
+      else
+        out << char_range_to_string1(el)
+      end
+    end
+    out
+  end
+  def char_range_to_string1(prod)
+    visit(prod) do |here|
+        case here
+        in ["seq", *rest]
+          rest = expand_range_into(rest, "seq")
+          i = rest.size
+          while i > 1
+            if (rest[i-1] in ["cs", s2]) && (rest[i-2] in ["cs", s1])
+              rest[i-2..i-1] = [["cs", s1 + s2]]
+            end
+            i -= 1
+          end
+          [true, rest]
+        in ["char-range", chr, ^chr] if chr.between?(" ", "!") || chr.between?("#", "~")
+          [true, ["cs", chr]]
+        else
+          false
+        end
+    end
+  end
+  def char_range_to_string
+    rules.each do |name, prod|
+      rules[name] = ci_cs_merge(detect_ci(char_range_to_string1(prod)))
+    end
+  end
+
+  def detect_ci(prod)
+    visit(prod) do |here|
+        case here
+        in ["alt", ["cs", c1], ["cs", c2]] if c1.downcase == c2 && c2.upcase == c1
+          [true, ["ci", c1]]
+        else
+          false
+        end
+    end
+  end
+  def ci_compat(prod)
+    case prod
+    in ["ci", s]
+      s
+    in ["cs", s] if s =~ /\A[^A-Za-z]*\z/
+      s
+    else
+      nil
+    end
+  end
+  def ci_cs_merge(prod)
+    visit(prod) do |here|
+        case here
+        in ["seq", *rest]
+          rest = rest.map{|x| ci_cs_merge(x)}
+          i = rest.size
+          while i > 1
+            if (s2 = ci_compat(rest[i-1])) && (s1 = ci_compat(rest[i-2]))
+              rest[i-2..i-1] = [["ci", s1 + s2]]
+            end
+            i -= 1
+          end
+          [true, wrap_flat("seq", rest)]
+        else
+          false
+        end
+    end
+  end
+
+  def seq_rep(prod)
+    visit(prod) do |here|
+        case here
+        in ["seq", *rest]
+          rest = rest.map{|x| seq_rep(x)}
+          i = rest.size         # behind last element
+          while i > 1
+            j = i - 1           # end of range
+            s_end = rest[j]
+            k = j               # start of range
+            while k > 0 && rest[k-1] == s_end
+              k -= 1
+            end
+            if k != j
+              n = j - k + 1
+              rest[k..j] = [["rep", n, n, s_end]]
+            end
+            i = k               # skip element k
+          end
+          [true, wrap_flat("seq", rest)]
+        else
+          false
+        end
+    end
+  end
+
+  # sharing
+  def count_alt(counter, prod)
+    visit(prod) do |here|
+      case here
+      in ["alt", *rest]
+        rest.each {|pr| count_alt(counter, pr)}
+        counter[here] += 1
+      else
+        false
+      end
+    end
+  end
+
+  def share_alt(prefix)
+    counter = Hash.new(0)
+    rules.each do |name, prod|
+      count_alt(counter, prod)
+    end
+    subs = {}
+    counter.to_a.select{|k, v| v > 2}.sort_by{|k, v| -v}.each_with_index do |(el, _count), i|
+      name = "#{prefix}-a#{i}"
+      rules[name] = el
+      subs[el] = name
+    end
+    rules.each do |name, prod|
+      count_alt(counter, prod)
+    end
+    rules.replace(Hash[rules.map do |k, v|
+                    [k, seq_rep(visit(v) do |prod|
+                       if (s = subs[prod]) && k != s
+                         [true, s]
+                       end
+                     end)]
+                  end])
+  end
+
+  def share_hex_1(prod, rules)
+    visit(prod) do |here|
+      case here
+      in ["alt",
+          ["char-range", c3l, "9"],
+          ["char-range", "A", c4r],
+          ["char-range", "a", c6r]] if c4r == c6r.upcase && c3l >= "0" && c6r <= "f"
+        name = "x#{c3l}#{c6r}"
+        rules[name] ||= here
+        [true, name]
+      in ["alt",
+          ["char-range", c4l, c4r],
+          ["char-range", c6l, c6r]] if c4r == c6r.upcase &&
+                                       c4l == c6l.upcase &&
+                                       c6l.between?("a", "f") &&
+                                       c6r.between?("a", "f")
+        name = "x#{c6l}#{c6r}"
+        rules[name] ||= here
+        [true, name]
+      in ["char-range", l, r] if l >= "0" && r <= "9"
+        name = "x#{l}#{r}"
+        rules[name] ||= here
+        [true, name]
+      else
+        false
+      end
+    end
+  end
+
+  def share_hex(_prefix)
+    newrules = {}
+    rules.each do |name, prod|
+      rules[name] = share_hex_1(prod, newrules)
+    end
+    rules.merge!(Hash[newrules.sort])
+  end
+end

data/lib/abnftt/abnf-visitor.rb

@@ -0,0 +1,26 @@
+require_relative "../abnftt.rb"
+
+class ABNF
+  def visit_all(prod_array, &block)
+    prod_array.map {|prod| visit(prod, &block)}
+  end
+  def visit(prod, &block)
+    done, ret = block.call(prod, &block)
+    if done
+      return ret
+    end
+
+    case prod
+    in ["alt", *prods]
+      ["alt", *visit_all(prods, &block)]
+    in ["tadd", *prods]
+      ["tadd", *visit_all(prods, &block)]
+    in ["seq", *prods]
+      ["seq", *visit_all(prods, &block)]
+    in ["rep", s, e, prod]
+      ["rep", s, e, visit(prod, &block)]
+    else
+      prod
+    end
+  end
+end

data/lib/abnftt/abnf-writer.rb

@@ -0,0 +1,106 @@
+class ABNF
+
+  # return [precedence ((2 if seq needed)), string]
+
+  def stringify(s)
+    fail "Can't stringify #{s.inspect} yet" unless s =~ /\A[ !#-~]*\z/
+    %{"#{s}"}
+  end
+
+  def write_lhs(k)
+      k
+  end
+
+  # precedence:
+  # 1: / alt -> (type1)
+  # 2: »« seq
+  # 4: atomic
+
+  def prec_check(inner, targetprec, prec)
+    if targetprec >= prec
+      "(#{inner})"
+    else
+      inner
+    end
+  end
+
+  def write_rhs(v, targetprec = 0)
+    prec, ret =
+    case v
+    in String                   # this should really be ["name", id]
+      [4, v]
+    in ["name", id]
+      [4, id]
+    in ["alt" | "tadd", *types]
+      [1, types.map{write_rhs(_1, 1)}.join(" / ")]
+    in ["seq", *groups]
+      case groups.size
+      when 0; [4, ""]           # XXX
+      else
+        [2, "#{groups.map{write_rhs(_1, 2)}.join(" ")}"]
+      end
+    in ["ci", s]
+      [4, stringify(s)]
+    in ["cs", s]
+      if s =~ /\A[^A-Za-z]*\z/
+        [4, stringify(s)]
+      else
+        [4, "%s" << stringify(s)]  # reduce noise if no alphabetics
+      end
+    in ["char-range", c1, c2]
+      nc1 = "%02x" % c1.ord
+
+      nc2 = "%02x" % c2.ord
+      nc2add = "-#{nc2}" if nc2 != nc1
+      [4, "%x#{nc1}#{nc2add}"]
+    in ["rep", s, e, group]
+      if s == 0 && e == 1
+        [4, "[#{write_rhs(group)}]"]
+      else
+        occur = case [s, e]
+                in [1, 1];    ""
+                in [0, true]; "*"
+                in [n, ^n]; n.to_s
+                else
+                  "#{s}*#{e != true ? e : ""}"
+                end
+        [4, "#{occur}#{write_rhs(group, 4)}"]
+      end
+    else
+      fail [:WRITE_NOMATCH, v].inspect
+    end
+    prec_check(ret, targetprec, prec)
+  end
+
+  def write_rule(k, v)
+      case v
+      in ["tadd", *_rest]
+        assign = "=/"
+      else
+        assign = "="
+      end
+      "#{write_lhs(k)} #{assign} #{write_rhs(v, 0)}"
+  end
+
+  def to_s
+    rules.map {|k, v| write_rule(k, v) }.join("\n").sub(/.\z/) {$& << "\n"}
+  end
+
+  # primitively break down lines so they fit on a teletype
+  def breaker(s, col = 69)
+    ret = ""
+    s.each_line do |*l|
+      while l[-1].size > col
+        breakpoint = l[-1][0...col].rindex(' ')
+        break unless breakpoint && breakpoint > 4
+        l[-1..-1] = [
+          l[-1][0...breakpoint],
+          "    " << l[-1][breakpoint+1..-1]
+        ]
+      end
+      ret << l.join("\n")
+    end
+    ret
+  end
+
+end

data/lib/abnftt.rb

@@ -53,11 +53,17 @@ class ABNF
     ABNF.new(ast)
   end
 
+  def self.from_rules(r)
+    ABNF.new(nil, r)
+  end
+
   attr_accessor :ast, :rules, :tree
-  def initialize(ast_)
-    @ast = ast_
-    @tree = ast.ast
-    @rules = {}
+  def initialize(ast_ = nil, rules_ = {})
+    if ast_
+      @ast = ast_
+      @tree = ast.ast
+    end
+    @rules = rules_
     @tree.each do |x|
       op, name, val, rest = x
       fail rest if rest
@@ -72,7 +78,7 @@ class ABNF
         else
           val
         end
-    end
+    end if @tree
     # warn "** rules #{rules.inspect}"
   end
 

metadata

@@ -1,28 +1,35 @@
 --- !ruby/object:Gem::Specification
 name: abnftt
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.2.7
 platform: ruby
 authors:
 - Carsten Bormann
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-08-26 00:00:00.000000000 Z
+date: 2025-01-11 00:00:00.000000000 Z
 dependencies: []
 description: Less shifty support for tools based on IETF's ABNF
 email: cabo@tzi.org
 executables:
+- abnfrob
+- abnfrob~
 - abnftt
-- abnftt~
 extensions: []
 extra_rdoc_files: []
 files:
 - abnftt.gemspec
+- bin/abnfrob
+- bin/abnfrob~
 - bin/abnftt
-- bin/abnftt~
 - lib/abnfgrammar.rb
 - lib/abnftt.rb
+- lib/abnftt/abnf-flattener.rb
+- lib/abnftt/abnf-squasher.rb
+- lib/abnftt/abnf-util.rb
+- lib/abnftt/abnf-visitor.rb
+- lib/abnftt/abnf-writer.rb
 homepage: http://github.com/cabo/abnftt
 licenses:
 - MIT
@@ -42,7 +49,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.5.14
 signing_key:
 specification_version: 4
 summary: RFC 5234+7405 ABNF to Treetop

data/bin/abnftt~

@@ -1,135 +0,0 @@
-#!/usr/bin/env ruby -Ku
-
-require 'pp'
-require 'yaml'
-require 'treetop'
-
-Encoding.default_external = Encoding::UTF_8
-
-require 'abnf'
-
-class Treetop::Runtime::SyntaxNode
-  def clean_abnf
-    if elements
-      elements.map {|el| el.clean_abnf}.join
-    else
-      text_value
-    end
-  end
-  def ast
-    fail "undefined_ast #{inspect}"
-  end
-  def ast_from_percent(base, first, second)
-    c1 = first.to_i(base).chr(Encoding::UTF_8)
-    case second[0]
-    when nil
-      ["cs", c1]
-    when "-"
-      c2 = second[1..-1].to_i(base).chr(Encoding::UTF_8)
-      ["char-range", c1, c2]
-    when "."
-      el = second.split(".")
-      el[0] = first
-      ["cs", el.map {|c| c.to_i(base).chr(Encoding::UTF_8)}.join]
-    else
-      fail "ast_from_percent"
-    end
-  end
-end
-
-def to_treetop(ast)
-  <<~EOS
-    # Encoding: UTF-8
-    grammar TESTME
-    #{ast.map {|x| to_treetop0(x)}.join}
-    end
-  EOS
-end
-def to_treetop0(ast)
-  fail ast.inspect unless ast[0] == "="
-  <<~EOS
-    rule #{to_treetop1(ast[1])}
-    #{to_treetop1(ast[2])}
-    end
-  EOS
-end
-FIXUP_NAMES = Hash.new {|h, k| k}
-FIXUP_NAMES.merge!({
-                     "rule" => "r__rule",
-                   })
-def to_treetop1(ast)
-  case ast
-  when String
-    FIXUP_NAMES[ast].gsub("-", "_")
-  when Array
-    case ast[0]
-    when "alt" # ["alt", *a]
-      "(#{ast[1..-1].map {|x| to_treetop1(x)}.join(" / ")})"
-    when "seq" # ["seq", *a]
-      "(#{ast[1..-1].map {|x| to_treetop1(x)}.join(" ")})"
-    when "rep" # ["rep", s, e, a]
-      t = to_treetop1(ast[3]) || "@@@"
-      case [ast[1], ast[2]]
-      when [0, 1]
-        t + "?"
-      when [0, true]
-        t + "*"
-      when [1, true]
-        t + "+"
-      else
-        t + " #{ast[1]}..#{ast[2] == true ? '' : ast[2]}"
-      end
-    when "prose" # ["prose", text]
-      fail "prose not implemented #{ast.inspect}"
-    when "ci" # ["ci", text]
-      s = ast[1]
-      if s =~ /\A[^A-Za-z]*\z/
-        s.inspect
-      else
-        s.inspect << "i"        # could do this always, but reduce noise
-      end
-    when "cs" # ["cs", text]
-      ast[1].inspect
-    when "char-range" # ["char-range", c1, c2]
-      c1 = Regexp.quote(ast[1])
-      c2 = Regexp.quote(ast[2])
-      "[#{c1}-#{c2}]"           # XXX does that always work
-    when "im" # ["im", a, text]
-      to_treetop1(ast[1]) + " " + ast[2]
-    else
-      fail "to_treetop(#{ast.inspect})"
-    end
-  else
-    fail "to_treetop(#{ast.inspect})"
-  end
-end
-
-unless fn = ARGV[0]
-  warn "Usage: abnftt grammar.abnftt"
-  exit 1
-end
-outfn = fn.sub(/\.abnftt\z/, "")
-
-parser = ABNFParser.new
-abnf_file = File.read(fn)
-ast = parser.parse abnf_file
-if ast
-#  p ast
-  File.open("#{outfn}.yaml", "w") do |f|
-    f.puts ast.ast.to_yaml
-  end
-  # pp ast.ast
-  File.open("#{outfn}.treetop", "w") do |f|
-    f.puts to_treetop(ast.ast)
-  end
-  File.open("#{outfn}.abnf", "w") do |f|
-    f.puts ast.clean_abnf
-  end
-else
-  
-  puts parser.failure_reason
-  parser.failure_reason =~ /^(Expected .+) after/m
-  puts "#{$1.gsub("\n", '<<<NEWLINE>>>')}:"
-  puts abnf_file.lines.to_a[parser.failure_line - 1]
-  puts "#{'~' * (parser.failure_column - 1)}^"
-end