parsanol 1.3.107-arm-linux → 1.3.109-arm-linux

data/lib/parsanol/parg/export.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module Parsanol
+  module PARG
+    # Lowers a compiled artifact to a decoder-facing grammar form
+    # (ruby#321): XGrammar's EBNF dialect first. One artifact feeding
+    # both worlds is the point — human-facing PARG with provenance,
+    # tests, and five-engine replay on one side; the constrained
+    # decoder on the other.
+    #
+    # The supported subset lowers 1:1. Everything the target cannot
+    # express — lookaround, behind-guards, custom atoms, fold groups —
+    # is rejected NAMING the construct: the importer philosophy in
+    # reverse, never silently dropped.
+    module Export
+      class Error < StandardError
+      end
+
+      TARGETS = %i[xgrammar].freeze
+
+      class << self
+        def call(artifact_json, target: :xgrammar)
+          case target
+          when :xgrammar then Xgrammar.new(artifact_json).call
+          else raise Error, "unknown export target #{target.inspect} (supported: #{TARGETS.join(', ')})"
+          end
+        end
+      end
+
+      # XGrammar's EBNF: one rule per entry; literals "…"; classes
+      # […]; suffixes * + ? {m,n}. Ordered choice is lost — the target
+      # resolves ambiguity its own way; the export lowers alternatives
+      # in the same order and the round-trip corpus pins acceptance
+      # equality. The artifact wire has no rule boundaries (all atoms
+      # inline), so each entry lowers as one expression.
+      class Xgrammar
+        def initialize(artifact_json)
+          @envelope = JSON.parse(artifact_json)
+        end
+
+        def call
+          rules = @envelope["entries"].map do |entry, meta|
+            grammar = meta.fetch("grammar")
+            "#{entry}_entry ::= #{lower(grammar.fetch('atoms'), grammar.fetch('root'))}"
+          end
+          "#{rules.join("\n")}\n"
+        end
+
+        private
+
+        def lower(atoms, index)
+          tag, fields = atoms.fetch(index).first
+          case tag
+          when "Str" then "\"#{fields['pattern'].gsub('\\') { '\\\\\\\\' }.gsub('"') { '\\\\\"' }}\""
+          when "Re" then re_class(fields["pattern"])
+          when "Sequence" then fields["atoms"].map { |child| lower(atoms, child) }.join(" ")
+          when "Alternative" then "( #{fields['atoms'].map { |child| lower(atoms, child) }.join(' | ')} )"
+          when "Repetition" then suffix(lower(atoms, fields["atom"]), fields["min"], fields["max"])
+          when "Maybe" then suffix(lower(atoms, fields["atom"]), 0, 1)
+          when "Named", "Entity" then "( #{lower(atoms, fields['atom'])} )"
+          when "Lookahead" then unsupported("lookahead (!x / &x)")
+          when "BehindGuard" then unsupported("behind-guard (!<x / &<x)")
+          when "Custom" then unsupported("custom atom")
+          when "Constant" then unsupported("constant atom")
+          else unsupported("atom #{tag}")
+          end
+        end
+
+        def suffix(body, min, max)
+          min ||= 0
+          if min.zero? && max.nil?
+            "#{wrap(body)}*"
+          elsif min == 1 && max.nil?
+            "#{wrap(body)}+"
+          elsif min.zero? && max == 1
+            "#{wrap(body)}?"
+          else
+            "#{wrap(body)}{#{min},#{max || ''}}"
+          end
+        end
+
+        def wrap(body)
+          return body if body =~ /\A"[^"]*"\z/ || body =~ /\A\[[^\]]*\]\z/ || body =~ /\A[A-Za-z_][A-Za-z0-9_]*\z/
+
+          "( #{body} )"
+        end
+
+        # A Re pattern the class form cannot carry is unsupported
+        # rather than approximated — EXCEPT the ASCII fold literal:
+        # (?i:...) expands mechanically to per-character [Xx] classes,
+        # which IS the simple-case-fold semantics for ASCII. A non-ASCII
+        # fold group needs a fold table the target does not have.
+        def re_class(pattern)
+          return pattern if pattern.start_with?("[") && pattern.end_with?("]")
+
+          if (folded = pattern[/\A\(\?i:(.*)\)\z/m, 1])
+            return expand_fold(folded, pattern)
+          end
+
+          unsupported("regex pattern #{pattern.inspect}")
+        end
+
+        def expand_fold(text, pattern)
+          unsupported("non-ASCII fold group #{pattern.inspect} -- the target has no fold table") unless text.ascii_only?
+
+          text.chars.map do |char|
+            if char =~ /[A-Za-z]/ && char.bytesize == 1
+              "[#{char.upcase}#{char.downcase}]"
+            elsif char == '"'
+              '\\"'
+            else
+              char
+            end
+          end.join
+        end
+
+        def unsupported(what)
+          raise Error, "xgrammar export: #{what} has no XGrammar equivalent"
+        end
+      end
+    end
+  end
+end

data/lib/parsanol/parg/import.rb

@@ -12,8 +12,9 @@ module Parsanol
       autoload :Abnf, "parsanol/parg/importers/abnf"
       autoload :Ebnf, "parsanol/parg/importers/ebnf"
       autoload :Pest, "parsanol/parg/importers/pest"
+      autoload :Antlr, "parsanol/parg/importers/antlr"
 
-      KINDS = %i[abnf ebnf pest].freeze
+      KINDS = %i[abnf ebnf pest antlr].freeze
 
       module_function
 
@@ -22,6 +23,7 @@ module Parsanol
                  when :abnf then Abnf.call(text)
                  when :ebnf then Ebnf.call(text)
                  when :pest then Pest.call(text)
+                 when :antlr then Antlr.call(text)
                  else
                    raise Error,
                          "unknown import kind #{kind.inspect} (supported: #{KINDS.map(&:inspect).join(', ')})"

data/lib/parsanol/parg/importers/abnf.rb

@@ -12,13 +12,17 @@ module Parsanol
       #   first-set lint flags order-dependent branches after import.
       # - rule names are case-insensitive in ABNF -> normalized to snake_case
       # - prose-vals (<...>) are rejected: they are not machine-parseable
+      # - quoted strings have NO escape mechanism (RFC 5234 3.4): a bare
+      #   backslash is an ordinary character; a literal DQUOTE needs the
+      #   DQUOTE core rule, not a backslash
       class Abnf
         TOKEN = /
           (?<ws>[ \t]+)
         | (?<comment>;[^\n]*)
         | (?<crlf>\r?\n)
-        | (?<sstr>%s"(?:[^"\\]|\\.)*")
-        | (?<cistr>"(?:[^"\\]|\\.)*")
+        | (?<sstr>%s"[^"]*")
+        | (?<istr>%i"[^"]*")
+        | (?<cistr>"[^"]*")
         | (?<numval>%[xbdo][0-9A-Za-z]+(?:-[0-9A-Za-z]+|(?:\.[0-9A-Za-z]+)+)?)
         | (?<prose><[^>\n]*>)
         | (?<definedas>=\/|=)
@@ -50,6 +54,12 @@ module Parsanol
           "WSP" => "%x20 / %x09",
         }.freeze
 
+        # RFC 3986's `0<pchar>` empty idiom lowers to a captured empty
+        # literal — the compiler's documented marker form (bare ""
+        # zero-width matches are rejected). The sentinel is replaced
+        # with the rule's own capture name in parse_rule.
+        EMPTY_IDIOM = ":empty0:"
+
         class << self
           def call(text)
             new(text).call
@@ -66,6 +76,7 @@ module Parsanol
 
         def call
           parse_rules
+          @nullable = nullable_rules
           emit
         end
 
@@ -100,6 +111,19 @@ module Parsanol
           advance while peek && INLINE_WS.include?(peek[0])
         end
 
+        # A CRLF followed by an indented line is a continuation: body
+        # whitespace, not the end of the rule (RFC 5234 section 3.4 —
+        # RFC 3986's IPv6address styles its alternations this way).
+        def skip_body_ws
+          loop do
+            skip_inline_ws
+            break unless peek&.[](0) == :crlf && peek(1)&.[](0) == :ws
+
+            advance
+            advance
+          end
+        end
+
         def parse_rules
           loop do
             skip_leading
@@ -132,8 +156,13 @@ module Parsanol
           end
 
           body = parse_alternation
+          body = body.gsub(EMPTY_IDIOM) { %(("" as #{name})) }
           if defined[1] == "=/"
-            @rules[name] = @rules[name] ? "#{@rules[name]} / #{body}" : body
+            merged = @rules[name] ? "#{@rules[name]} / #{body}" : body
+            # Incremental definitions may restate a branch; identical
+            # branches are one branch in ABNF's unordered semantics and
+            # a compile error in PARG's.
+            @rules[name] = split_top_level(merged, " / ").uniq.join(" / ")
           elsif @rules.key?(name)
             raise Error, "ABNF: duplicate rule #{name.inspect}"
           else
@@ -143,7 +172,10 @@ module Parsanol
 
         def parse_alternation
           parts = [parse_concatenation]
-          while peek&.[](0) == :punct && peek[1] == "/"
+          loop do
+            skip_body_ws
+            break unless peek&.[](0) == :punct && peek[1] == "/"
+
             advance
             parts << parse_concatenation
           end
@@ -152,27 +184,19 @@ module Parsanol
 
         def parse_concatenation
           parts = [parse_repetition]
-          while continuation?
-            parts << parse_repetition
-          end
-          parts.join(" ")
-        end
-
-        def continuation?
-          skip_inline_ws
-          token = peek
-          return false if token.nil?
-
-          if token[0] == :crlf
-            unless peek(1)&.[](0) == :ws
-              return false
+          loop do
+            save = @pos
+            skip_body_ws
+            if peek && element_start?(peek)
+              parts << parse_repetition
+            else
+              # A close paren, an alternation slash, or end of rule:
+              # not ours to consume.
+              @pos = save
+              break
             end
-
-            advance
-            advance
-            return true
           end
-          element_start?(token)
+          parts.join(" ")
         end
 
         def element_start?(token)
@@ -181,7 +205,14 @@ module Parsanol
         end
 
         def parse_repetition
-          skip_inline_ws
+          skip_body_ws
+          # RFC 3986's empty idiom: `0<pchar>` — zero of an unparseable
+          # prose-val is exactly the empty match.
+          if peek&.[](0) == :num && peek[1] == "0" && peek(1)&.[](0) == :prose
+            advance
+            advance
+            return EMPTY_IDIOM
+          end
           if peek&.[](0) == :num
             prefix = advance[1]
             if peek&.[](0) == :punct && peek[1] == "*"
@@ -200,16 +231,19 @@ module Parsanol
         end
 
         def parse_element
-          skip_inline_ws
+          skip_body_ws
           token = advance
           raise Error, "ABNF: unexpected end of rule body" if token.nil?
 
           case token[0]
           when :cistr then emit_string(token[1][1..-2], fold: true)
+          when :istr then emit_string(token[1][3..-2], fold: true)
           when :sstr then emit_string(token[1][3..-2], fold: false)
           when :numval then emit_numval(token[1])
           when :name then normalize_name(token[1])
           when :prose
+            # RFC 3986's `0<pchar>` idiom: zero occurrences of anything
+            # is the empty match. A real prose-val stays unimportable.
             raise Error,
                   "ABNF: prose-val #{token[1].inspect} cannot be imported — " \
                   "replace it with a machine-parseable rule"
@@ -232,7 +266,7 @@ module Parsanol
         end
 
         def expect_punct(char)
-          skip_inline_ws
+          skip_body_ws
           token = advance
           unless token && token[0] == :punct && token[1] == char
             raise Error, "ABNF: expected #{char.inspect}"
@@ -240,7 +274,10 @@ module Parsanol
         end
 
         def emit_string(body, fold:)
-          escaped = body.gsub("\\", "\\\\").gsub('"', '\\"')
+          # Block form: a String replacement treats backslash escapes
+          # specially, so "\\\\" would emit ONE backslash (the silent
+          # no-op this bug was).
+          escaped = body.gsub("\\") { "\\\\" }.gsub('"') { '\\"' }
           fold ? "%i\"#{escaped}\"" : "\"#{escaped}\""
         end
 
@@ -267,6 +304,89 @@ module Parsanol
           end
         end
 
+        # ABNF alternation is unordered; PARG's is ordered. A branch
+        # that provably matches empty (repetitions of zero, optionals,
+        # the empty idiom) would shadow everything after it in ordered
+        # choice, so definitely-nullable branches move last — the same
+        # acceptance, no shadowing. Branches with rule references are
+        # never moved: their nullability is not provable textually.
+        def reorder_nullable_branches_last(body)
+          return body unless body.include?(" / ")
+
+          branches = split_top_level(body, " / ")
+          return body unless branches.size > 1
+
+          nullable, ordered = branches.partition { |branch| branch_nullable?(branch, @nullable) }
+          return body if nullable.empty?
+
+          (ordered + nullable).join(" / ")
+        end
+
+        # Fixpoint over the rule graph: a rule is nullable when any
+        # alternative's every item is nullable — repetitions of zero,
+        # optionals, the empty idiom, or references to nullable rules.
+        def nullable_rules
+          nullable = {}
+          changed = true
+          while changed
+            changed = false
+            @rules.each do |name, body|
+              next if nullable[name]
+
+              if split_top_level(body, " / ").any? { |branch| branch_nullable?(branch, nullable) }
+                nullable[name] = true
+                changed = true
+              end
+            end
+          end
+          nullable
+        end
+
+        def branch_nullable?(branch, nullable)
+          items = split_top_level(branch, " ").reject(&:empty?)
+          !items.empty? && items.all? { |item| item_nullable?(item, nullable) }
+        end
+
+        def item_nullable?(item, nullable)
+          return true if item.start_with?("*")
+          return true if item.start_with?("[") && item.end_with?("]")
+          return true if item.start_with?('("" as')
+
+          name = item[/\A[a-z0-9_]+\z/]
+          name ? nullable[name] == true : false
+        end
+
+        def split_top_level(text, separator)
+          out = []
+          depth = 0
+          current = +""
+          index = 0
+          while index < text.length
+            piece = text[index..]
+            if piece.start_with?(separator) && depth.zero?
+              out << current
+              current = +""
+              index += separator.length
+              next
+            end
+            case piece[0]
+            when "(" then depth += 1
+            when ")" then depth -= 1
+            end
+            current << piece[0]
+            index += 1
+          end
+          out << current
+          out
+        end
+
+        def provably_empty?(branch)
+          items = split_top_level(branch, " ").reject(&:empty?)
+          !items.empty? && items.all? do |item|
+            item.start_with?("*", "[", '("" as')
+          end
+        end
+
         def normalize_name(name)
           name.downcase.tr("-", "_")
         end
@@ -278,7 +398,7 @@ module Parsanol
 
             "  #{normalized} = #{body}"
           end
-          rules = @rules.map { |name, body| "  #{name} = #{body}" }
+          rules = @rules.map { |name, body| "  #{name} = #{reorder_nullable_branches_last(body)}" }
           lines = [
             "# Imported from ABNF (RFC 5234/7405).",
             "# Notes:",