parsanol 1.3.56-arm-linux → 1.3.57-arm-linux

data/lib/parsanol/parg/frontend.rb

@@ -0,0 +1,173 @@
+# frozen_string_literal: true
+
+module Parsanol
+  module PARG
+    # F10 phase 2: the artifact-driven front end.
+    #
+    # The parg artifact is the parser of record: SelfHost.validate accepts
+    # or rejects the source with the native engine before anything else.
+    # Document construction then routes each top-level item —
+    # line-classified, since PARG's document grammar is line-oriented
+    # (PN 1) — through the reference semantics. Acceptance gate: the
+    # compiled envelope checksum must equal the reference compiler's for
+    # every grammar in the corpus.
+    module Frontend
+      NAMED_SECTION = /\A(render|bindings)\s+(\w+)\s*\{/
+      PREPROCESS_SECTION = /\Apreprocess\s+(\w+)\s*\{/
+      DERIVE_LINE = /\Aderive\s+(\w+)\s+"((?:[^"\\]|\\.)*)"/
+      TEST_SECTION = /\Atest\s*\{/
+      RULE_LINE = /\A([a-z_][a-z0-9_]*)\s*=\s*/
+      ENTRY_LINE = /\Aentry\s+(\w+)\s*:\s*(\w+)/
+      USE_LINE = /\Ause\s+(\w+)/
+      GRAMMAR_LINE = /\Agrammar\s+(\w+)\s+version\s+"([^"]+)"/
+
+      module_function
+
+      def parse(source)
+        SelfHost.validate(source) unless SelfHost.available? && !SelfHost.valid?(source)
+
+        document = Document.new
+        document.source = source
+        state = { section: nil, section_name: nil, section_lines: [],
+                  depth: 0, doc_comments: [], rule_lines: {},
+                  entry_lines: [] }
+        deferred = []
+        preprocess_texts = []
+        document.own_entries ||= []
+
+        source.each_line do |line|
+          stripped = line.strip
+          if stripped.start_with?("##")
+            state[:doc_comments] << stripped.sub(/\A##\s?/, "")
+            next
+          end
+          next if stripped.empty? || stripped.start_with?("#")
+
+          if state[:section]
+            advance_section(document, state, line, deferred, preprocess_texts)
+          else
+            route_top_level(document, stripped, line, state,
+                            preprocess_texts)
+          end
+        end
+
+        replay_deferred(document, deferred, state[:rule_lines],
+                        state[:entry_lines], preprocess_texts)
+        document
+      end
+
+      # Consumes one line inside a deferred/closed-by-brace section.
+      def advance_section(document, state, line, deferred, preprocess_texts)
+        section = state[:section]
+        state[:depth] += line.scan("{").count - line.scan("}").count
+        if state[:depth].positive?
+          preprocess_texts.last << line if section == "preprocess"
+          state[:section_lines] << line
+          return
+        end
+
+        case section
+        when "test"
+          deferred << ["test {", state[:section_lines]]
+        when "bindings"
+          deferred << ["bindings #{state[:section_name]} {", state[:section_lines]]
+        when "preprocess"
+          preprocess_texts.last << "}\n"
+          close_section(document, section, state[:section_name],
+                        state[:section_lines])
+        else
+          close_section(document, section, state[:section_name],
+                        state[:section_lines])
+        end
+        state[:section] = nil
+        state[:section_name] = nil
+        state[:section_lines] = []
+      end
+
+      def route_top_level(document, stripped, line, state, preprocess_texts)
+        doc_comments = state[:doc_comments]
+
+        case stripped
+        when USE_LINE
+          document.uses << Regexp.last_match(1)
+          doc_comments = []
+        when GRAMMAR_LINE
+          document.grammar_name = Regexp.last_match(1)
+          document.version = Regexp.last_match(2)
+          doc_comments = []
+        when NAMED_SECTION
+          state[:section] = Regexp.last_match(1)
+          state[:section_name] = Regexp.last_match(2)
+          state[:section_lines] = []
+          state[:depth] = 1
+          doc_comments = []
+        when PREPROCESS_SECTION
+          state[:section] = "preprocess"
+          state[:section_name] = Regexp.last_match(1)
+          state[:section_lines] = []
+          preprocess_texts << "preprocess #{state[:section_name]} {\n"
+          state[:depth] = 1
+          doc_comments = []
+        when TEST_SECTION
+          state[:section] = "test"
+          state[:section_lines] = []
+          state[:depth] = 1
+          doc_comments = []
+        when ENTRY_LINE
+          entry_name = Regexp.last_match(1)
+          document.entries[entry_name] = Regexp.last_match(2)
+          document.own_entries << entry_name
+          state[:entry_lines] << line
+          doc_comments = []
+        when DERIVE_LINE
+          document.derive[Regexp.last_match(1)] =
+            Regexp.last_match(2).gsub(/\\(.)/, '\1')
+          doc_comments = []
+        when RULE_LINE
+          rule_name = Regexp.last_match(1)
+          document.docs[rule_name] = doc_comments.join("\n") unless doc_comments.empty?
+          doc_comments = []
+          state[:rule_lines][rule_name] = line
+          merge_rule(document, line)
+        else
+          doc_comments = []
+        end
+        state[:doc_comments] = doc_comments
+      end
+
+      # Deferred test/bindings sections re-parse against a mini preamble
+      # built from the collected rule/entry/preprocess text.
+      def replay_deferred(document, deferred, rule_lines, entry_lines,
+                          preprocess_texts)
+        return if deferred.empty?
+
+        preamble = "grammar Mini version \"1\" {\n" \
+                   "#{rule_lines.values.join}#{entry_lines.join}" \
+                   "#{preprocess_texts.join}}\n"
+        deferred.each do |(opener, lines)|
+          text = "#{preamble[0..-2]}#{opener}\n#{lines.join}}\n"
+          mini = Parser.new(text).parse
+          if opener.start_with?("test")
+            document.tests.concat(mini.tests)
+          else
+            document.bindings.merge!(mini.bindings)
+          end
+        end
+      end
+
+      def merge_rule(document, line)
+        mini = Parser.new("grammar Mini version \"1\" {\n#{line}}\n").parse
+        document.rules.merge!(mini.rules)
+        mini.docs.each { |rule, text| document.docs[rule] = text }
+      end
+
+      def close_section(document, kind, name, lines)
+        mini = Parser.new("grammar Mini version \"1\" {\nb = \"q\"\n}#{kind} #{name} {\n#{lines.join}}\n").parse
+        document.bindings.merge!(mini.bindings)
+        document.preprocess.merge!(mini.preprocess)
+        document.render.merge!(mini.render)
+        document.derive.merge!(mini.derive)
+      end
+    end
+  end
+end

data/lib/parsanol/parg/import.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Parsanol
+  module PARG
+    # Foreign-grammar importers. Each converts a foreign grammar notation to
+    # PARG source text — the .parg file is committed as the single source of
+    # truth, then compiled like any hand-written grammar. Import results are
+    # self-checked: the emitted source must round-trip through PARG::Parser.
+    module Import
+      class Error < PARG::Error; end
+
+      autoload :Abnf, "parsanol/parg/importers/abnf"
+      autoload :Ebnf, "parsanol/parg/importers/ebnf"
+      autoload :Pest, "parsanol/parg/importers/pest"
+
+      KINDS = %i[abnf ebnf pest].freeze
+
+      module_function
+
+      def import(kind, text)
+        source = case kind
+                 when :abnf then Abnf.call(text)
+                 when :ebnf then Ebnf.call(text)
+                 when :pest then Pest.call(text)
+                 else
+                   raise Error,
+                         "unknown import kind #{kind.inspect} (supported: #{KINDS.map(&:inspect).join(', ')})"
+                 end
+        Parser.new(source).parse
+        source
+      end
+    end
+  end
+end

data/lib/parsanol/parg/importers/abnf.rb

@@ -0,0 +1,299 @@
+# frozen_string_literal: true
+
+module Parsanol
+  module PARG
+    module Import
+      # RFC 5234 (+ RFC 7405 case prefixes) ABNF importer.
+      #
+      # Semantic conversions (recorded in the emitted header):
+      # - bare ABNF strings are CASE-INSENSITIVE -> emitted as %i"..."
+      # - %s"..." (case-sensitive) -> emitted as plain "..."
+      # - ABNF alternation is unordered; PARG's is ordered. The PARG compiler's
+      #   first-set lint flags order-dependent branches after import.
+      # - rule names are case-insensitive in ABNF -> normalized to snake_case
+      # - prose-vals (<...>) are rejected: they are not machine-parseable
+      class Abnf
+        TOKEN = /
+          (?<ws>[ \t]+)
+        | (?<comment>;[^\n]*)
+        | (?<crlf>\r?\n)
+        | (?<sstr>%s"(?:[^"\\]|\\.)*")
+        | (?<cistr>"(?:[^"\\]|\\.)*")
+        | (?<numval>%[xbdo][0-9A-Za-z]+(?:-[0-9A-Za-z]+|(?:\.[0-9A-Za-z]+)+)?)
+        | (?<prose><[^>\n]*>)
+        | (?<definedas>=\/|=)
+        | (?<name>[A-Za-z][A-Za-z0-9-]*)
+        | (?<num>\d+)
+        | (?<punct>[*\/()\[\]])
+        /x
+
+        INLINE_WS = %i[ws comment].freeze
+        LEADING = %i[ws comment crlf].freeze
+        ELEMENT_START_TYPES = %i[name cistr sstr numval num].freeze
+        ELEMENT_START_PUNCT = ["*", "(", "["].freeze
+
+        CORE_RULES = {
+          "ALPHA" => "%x41-5A / %x61-7A",
+          "BIT" => '"0" / "1"',
+          "CHAR" => "%x01-7F",
+          "CR" => "%x0D",
+          "CRLF" => "%x0D %x0A",
+          "CTL" => "%x00-1F / %x7F",
+          "DIGIT" => "%x30-39",
+          "DQUOTE" => "%x22",
+          "HEXDIG" => '%x30-39 / %i"a" / %i"b" / %i"c" / %i"d" / %i"e" / %i"f"',
+          "HTAB" => "%x09",
+          "LF" => "%x0A",
+          "OCTET" => "%x00-FF",
+          "SP" => "%x20",
+          "VCHAR" => "%x21-7E",
+          "WSP" => "%x20 / %x09",
+        }.freeze
+
+        class << self
+          def call(text)
+            new(text).call
+          end
+        end
+
+        def initialize(text)
+          @text = text
+          @tokens = []
+          @pos = 0
+          @rules = {}
+          scan
+        end
+
+        def call
+          parse_rules
+          emit
+        end
+
+        private
+
+        def scan
+          pos = 0
+          until pos >= @text.length
+            match = TOKEN.match(@text, pos)
+            if match.nil? || match.begin(0) != pos
+              raise Error,
+                    "ABNF: unexpected #{@text[pos].inspect} at offset #{pos}"
+            end
+
+            name = TOKEN.names.find { |n| match[n] }
+            @tokens << [name.to_sym, match[0], pos]
+            pos = match.end(0)
+          end
+        end
+
+        def peek(offset = 0)
+          @tokens[@pos + offset]
+        end
+
+        def advance
+          token = @tokens[@pos]
+          @pos += 1
+          token
+        end
+
+        def skip_inline_ws
+          advance while peek && INLINE_WS.include?(peek[0])
+        end
+
+        def parse_rules
+          loop do
+            skip_leading
+            break if peek.nil?
+
+            parse_rule
+          end
+        end
+
+        def skip_leading
+          loop do
+            token = peek
+            break unless token && LEADING.include?(token[0])
+
+            advance
+          end
+        end
+
+        def parse_rule
+          token = advance
+          unless token[0] == :name
+            raise Error, "ABNF: expected rule name, got #{token[1].inspect}"
+          end
+
+          name = normalize_name(token[1])
+          skip_inline_ws
+          defined = advance
+          unless defined && defined[0] == :definedas
+            raise Error, "ABNF: expected '=' after #{token[1].inspect}"
+          end
+
+          body = parse_alternation
+          if defined[1] == "=/"
+            @rules[name] = @rules[name] ? "#{@rules[name]} / #{body}" : body
+          elsif @rules.key?(name)
+            raise Error, "ABNF: duplicate rule #{name.inspect}"
+          else
+            @rules[name] = body
+          end
+        end
+
+        def parse_alternation
+          parts = [parse_concatenation]
+          while peek&.[](0) == :punct && peek[1] == "/"
+            advance
+            parts << parse_concatenation
+          end
+          parts.join(" / ")
+        end
+
+        def parse_concatenation
+          parts = [parse_repetition]
+          while continuation?
+            parts << parse_repetition
+          end
+          parts.join(" ")
+        end
+
+        def continuation?
+          skip_inline_ws
+          token = peek
+          return false if token.nil?
+
+          if token[0] == :crlf
+            unless peek(1)&.[](0) == :ws
+              return false
+            end
+
+            advance
+            advance
+            return true
+          end
+          element_start?(token)
+        end
+
+        def element_start?(token)
+          ELEMENT_START_TYPES.include?(token[0]) ||
+            (token[0] == :punct && ELEMENT_START_PUNCT.include?(token[1]))
+        end
+
+        def parse_repetition
+          skip_inline_ws
+          if peek&.[](0) == :num
+            prefix = advance[1]
+            if peek&.[](0) == :punct && peek[1] == "*"
+              advance
+              prefix += "*"
+              prefix += advance[1] if peek&.[](0) == :num
+            end
+            return "#{prefix}#{parse_element}"
+          elsif peek&.[](0) == :punct && peek[1] == "*"
+            advance
+            prefix = "*"
+            prefix += advance[1] if peek&.[](0) == :num
+            return "#{prefix}#{parse_element}"
+          end
+          parse_element
+        end
+
+        def parse_element
+          skip_inline_ws
+          token = advance
+          raise Error, "ABNF: unexpected end of rule body" if token.nil?
+
+          case token[0]
+          when :cistr then emit_string(token[1][1..-2], fold: true)
+          when :sstr then emit_string(token[1][3..-2], fold: false)
+          when :numval then emit_numval(token[1])
+          when :name then normalize_name(token[1])
+          when :prose
+            raise Error,
+                  "ABNF: prose-val #{token[1].inspect} cannot be imported — " \
+                  "replace it with a machine-parseable rule"
+          when :punct
+            case token[1]
+            when "("
+              body = parse_alternation
+              expect_punct(")")
+              "(#{body})"
+            when "["
+              body = parse_alternation
+              expect_punct("]")
+              "[#{body}]"
+            else
+              raise Error, "ABNF: unexpected #{token[1].inspect}"
+            end
+          else
+            raise Error, "ABNF: unexpected token #{token[0]}"
+          end
+        end
+
+        def expect_punct(char)
+          skip_inline_ws
+          token = advance
+          unless token && token[0] == :punct && token[1] == char
+            raise Error, "ABNF: expected #{char.inspect}"
+          end
+        end
+
+        def emit_string(body, fold:)
+          escaped = body.gsub("\\", "\\\\").gsub('"', '\\"')
+          fold ? "%i\"#{escaped}\"" : "\"#{escaped}\""
+        end
+
+        def emit_numval(spec)
+          kind = spec[1]
+          body = spec[2..]
+          if body.include?("-")
+            lo, hi = body.split("-")
+            "%x#{convert_value(kind, lo)}-#{convert_value(kind, hi)}"
+          elsif body.include?(".")
+            body.split(".").map { |value| "%x#{convert_value(kind, value)}" }.join(" ")
+          else
+            "%x#{convert_value(kind, body)}"
+          end
+        end
+
+        def convert_value(kind, value)
+          case kind
+          when "x" then format("%02x", value.to_i(16))
+          when "d" then format("%02x", value.to_i)
+          when "b" then format("%02x", value.to_i(2))
+          when "o" then format("%02x", value.to_i(8))
+          else raise Error, "ABNF: unknown numeric value kind #{kind.inspect}"
+          end
+        end
+
+        def normalize_name(name)
+          name.downcase.tr("-", "_")
+        end
+
+        def emit
+          preamble = CORE_RULES.filter_map do |name, body|
+            normalized = normalize_name(name)
+            next if @rules.key?(normalized)
+
+            "  #{normalized} = #{body}"
+          end
+          rules = @rules.map { |name, body| "  #{name} = #{body}" }
+          lines = [
+            "# Imported from ABNF (RFC 5234/7405).",
+            "# Notes:",
+            "# - bare ABNF strings are case-insensitive; imported as %i\"...\"",
+            "# - ABNF alternation is unordered; PARG's is ordered — the compiler",
+            "#   lints order-dependent branches",
+            "grammar imported_abnf version \"0.0.0\" {",
+          ]
+          lines.concat(preamble)
+          lines << "" unless preamble.empty?
+          lines.concat(rules)
+          lines << "}"
+          "#{lines.join("\n")}\n"
+        end
+      end
+    end
+  end
+end

data/lib/parsanol/parg/importers/ebnf.rb

@@ -0,0 +1,201 @@
+# frozen_string_literal: true
+
+module Parsanol
+  module PARG
+    module Import
+      # ISO 14977 EBNF importer.
+      #
+      # Semantic conversions (recorded in the emitted header):
+      # - meta-identifiers are case-insensitive in ISO EBNF -> downcased
+      # - terminals are exact -> emitted as plain PARG strings
+      # - sequence "," -> juxtaposition; "|" -> ordered "/"
+      # - { x } (zero-or-more) -> *( x )
+      # - syntactic exceptions "term - exception" are approximated as
+      #   !( exception ) term — exact whenever the exception matches a
+      #   prefix of the term's match; each occurrence is commented
+      # - special sequences ? ... ? are rejected
+      class Ebnf
+        COMMENT = /\(\*(?:[^*]|\*(?!\)))*\*\)/m
+
+        TOKEN = /
+          (?<ws>\s+)
+        | (?<squote>'[^']*')
+        | (?<dquote>"[^"]*")
+        | (?<special>\?[^?\n]*\?)
+        | (?<name>[A-Za-z][A-Za-z0-9]*)
+        | (?<punct>[=;,|()\[\]{}-])
+        /x
+
+        class << self
+          def call(text)
+            new(text).call
+          end
+        end
+
+        def initialize(text)
+          @text = text.gsub(COMMENT, "")
+          @tokens = []
+          @pos = 0
+          @rules = {}
+          @notes = []
+          @current_name = nil
+          scan
+        end
+
+        def call
+          parse_rules
+          emit
+        end
+
+        private
+
+        def scan
+          pos = 0
+          until pos >= @text.length
+            match = TOKEN.match(@text, pos)
+            if match.nil? || match.begin(0) != pos
+              raise Error,
+                    "EBNF: unexpected #{@text[pos].inspect} at offset #{pos}"
+            end
+
+            name = TOKEN.names.find { |n| match[n] }
+            unless name == "ws"
+              if name == "special"
+                raise Error,
+                      "EBNF: special sequence #{match[0].inspect} cannot be " \
+                      "imported — express it with PARG syntax"
+              end
+              @tokens << [name.to_sym, match[0], pos]
+            end
+            pos = match.end(0)
+          end
+        end
+
+        def peek(offset = 0)
+          @tokens[@pos + offset]
+        end
+
+        def advance
+          token = @tokens[@pos]
+          @pos += 1
+          token
+        end
+
+        def parse_rules
+          until peek.nil?
+            token = advance
+            unless token[0] == :name
+              raise Error, "EBNF: expected rule name, got #{token[1].inspect}"
+            end
+
+            name = normalize_name(token[1])
+            equals = advance
+            unless equals && equals[0] == :punct && equals[1] == "="
+              raise Error, "EBNF: expected '=' after #{token[1].inspect}"
+            end
+
+            if @rules.key?(name)
+              raise Error, "EBNF: duplicate rule #{name.inspect}"
+            end
+
+            @current_name = name
+            @rules[name] = parse_alternation
+            terminator = advance
+            unless terminator && terminator[0] == :punct && terminator[1] == ";"
+              raise Error, "EBNF: expected ';' after rule #{name.inspect}"
+            end
+          end
+        end
+
+        def parse_alternation
+          parts = [parse_sequence]
+          while peek&.[](0) == :punct && peek[1] == "|"
+            advance
+            parts << parse_sequence
+          end
+          parts.join(" / ")
+        end
+
+        def parse_sequence
+          parts = [parse_term]
+          while peek&.[](0) == :punct && peek[1] == ","
+            advance
+            parts << parse_term
+          end
+          parts.join(" ")
+        end
+
+        def parse_term
+          factor = parse_factor
+          if peek&.[](0) == :punct && peek[1] == "-"
+            advance
+            exception = parse_factor
+            @notes << "rule '#{@current_name}': syntactic exception " \
+                      "'#{factor} - #{exception}' approximated as " \
+                      "'!(#{exception}) #{factor}'"
+            "!(#{exception}) #{factor}"
+          else
+            factor
+          end
+        end
+
+        def parse_factor
+          token = advance
+          raise Error, "EBNF: unexpected end of rule body" if token.nil?
+
+          case token[0]
+          when :squote, :dquote then emit_string(token[1][1..-2])
+          when :name then normalize_name(token[1])
+          when :punct
+            case token[1]
+            when "("
+              body = parse_alternation
+              expect_punct(")")
+              "(#{body})"
+            when "["
+              body = parse_alternation
+              expect_punct("]")
+              "[#{body}]"
+            when "{"
+              body = parse_alternation
+              expect_punct("}")
+              "*(#{body})"
+            else
+              raise Error, "EBNF: unexpected #{token[1].inspect}"
+            end
+          else
+            raise Error, "EBNF: unexpected token #{token[0]}"
+          end
+        end
+
+        def expect_punct(char)
+          token = advance
+          unless token && token[0] == :punct && token[1] == char
+            raise Error, "EBNF: expected #{char.inspect}"
+          end
+        end
+
+        def emit_string(body)
+          body.gsub("\\", "\\\\").gsub('"', '\\"').then { |escaped| "\"#{escaped}\"" }
+        end
+
+        def normalize_name(name)
+          name.downcase
+        end
+
+        def emit
+          lines = [
+            "# Imported from ISO 14977 EBNF.",
+            "# Notes:",
+            "# - meta-identifiers were case-insensitive; downcased",
+          ]
+          @notes.each { |note| lines << "# - #{note}" }
+          lines << "grammar imported_ebnf version \"0.0.0\" {"
+          lines.concat(@rules.map { |name, body| "  #{name} = #{body}" })
+          lines << "}"
+          "#{lines.join("\n")}\n"
+        end
+      end
+    end
+  end
+end