parsanol 1.3.56-arm-linux → 1.3.57-arm-linux

data/lib/parsanol/parg/importers/pest.rb

@@ -0,0 +1,314 @@
+# frozen_string_literal: true
+
+module Parsanol
+  module PARG
+    module Import
+      # pest (Rust PEG) importer — the common subset, mapped to PARG.
+      #
+      # Conversions:
+      # - ordered choice "|", predicates "!"/"&", postfix "*"/"+"/"?" map
+      #   directly (PARG: *x / 1*x / [ x ])
+      # - "lit" exact; ^"lit" case-insensitive -> %i"..."
+      # - 'a'..'z' ranges -> %x61-7A
+      # - builtin character classes (ASCII_DIGIT, ...) -> %x ranges
+      # - "a" ~ "b": pest inserts implicit WHITESPACE at ~; PARG emits plain
+      #   sequence — whitespace must be explicit (recorded in the header)
+      # - rule modifiers _/@/$ are accepted and noted (PARG captures stay
+      #   enabled inside them); silent name_ rules keep their name
+      # - PUSH/POP/PEEK/EOI/SOI and the implicit WHITESPACE/Comment rules
+      #   are rejected with an explanatory error
+      class Pest
+        TOKEN = /
+          (?<ws>\s+)
+        | (?<comment>\/\/[^\n]*)
+        | (?<insens>\^(?:"(?:[^"\\]|\\.)*"|i"(?:[^"\\]|\\.)*"))
+        | (?<str>"(?:[^"\\]|\\.)*")
+        | (?<range>'(?:[^'\\]|\\.)'\.\.'(?:[^'\\]|\\.)')
+        | (?<ident>[A-Za-z_][A-Za-z0-9_]*)
+        | (?<punct>[|~!&*+?(){}=@$])
+        /x
+
+        BUILTINS = {
+          "ANY" => "%x00-FF",
+          "ASCII_DIGIT" => "%x30-39",
+          "ASCII_NONZERO_DIGIT" => "%x31-39",
+          "ASCII_BIN_DIGIT" => "%x30-31",
+          "ASCII_OCT_DIGIT" => "%x30-37",
+          "ASCII_HEX_DIGIT" => "%x30-39 / %x41-46 / %x61-66",
+          "ASCII_ALPHA" => "%x41-5A / %x61-7A",
+          "ASCII_ALPHA_LOWER" => "%x61-7A",
+          "ASCII_ALPHA_UPPER" => "%x41-5A",
+          "ASCII_ALPHANUMERIC" => "%x30-39 / %x41-5A / %x61-7A",
+          "NEWLINE" => '"\n" / "\r\n"',
+        }.freeze
+
+        SKIPPED_NAMES = %w[ws comment].freeze
+        MODIFIER_CHARS = %w[_ @ $].freeze
+
+        REJECTED = %w[SOI EOI WHITESPACE COMMENT PUSH POP PEEK PEEK_ALL DROP
+                      RESET].freeze
+
+        class << self
+          def call(text)
+            new(text).call
+          end
+        end
+
+        def initialize(text)
+          @tokens = []
+          @pos = 0
+          @rules = {}
+          @notes = []
+          scan(text)
+        end
+
+        def call
+          parse_rules
+          emit
+        end
+
+        private
+
+        def scan(text)
+          pos = 0
+          until pos >= text.length
+            match = TOKEN.match(text, pos)
+            if match.nil? || match.begin(0) != pos
+              raise Error,
+                    "pest: unexpected #{text[pos].inspect} at offset #{pos}"
+            end
+
+            name = TOKEN.names.find { |n| match[n] }
+            @tokens << [name.to_sym, match[0], pos] unless SKIPPED_NAMES.include?(name)
+            pos = match.end(0)
+          end
+        end
+
+        def peek(offset = 0)
+          @tokens[@pos + offset]
+        end
+
+        def advance
+          token = @tokens[@pos]
+          @pos += 1
+          token
+        end
+
+        def parse_rules
+          until peek.nil?
+            token = advance
+            unless token[0] == :ident
+              raise Error, "pest: expected rule name, got #{token[1].inspect}"
+            end
+
+            name = token[1]
+            if REJECTED.include?(name)
+              raise Error,
+                    "pest: #{name} is not importable — " \
+                    "#{name == 'WHITESPACE' ? 'pest applies it implicitly; define whitespace explicitly in PARG' : 'express it in PARG syntax'}"
+            end
+            @notes << "silent rule #{name.inspect} imported as a normal rule" if name.end_with?("_")
+            equals = advance
+            unless equals && equals[0] == :punct && equals[1] == "="
+              raise Error, "pest: expected '=' after #{name.inspect}"
+            end
+
+            nil
+            if peek&.[](0) == :punct && MODIFIER_CHARS.include?(peek[1])
+              modifier = advance[1]
+              @notes << "rule #{name.inspect}: modifier #{modifier.inspect} " \
+                        "accepted; PARG captures stay enabled inside it"
+            end
+            expect_punct("{")
+            if @rules.key?(name)
+              raise Error, "pest: duplicate rule #{name.inspect}"
+            end
+
+            @rules[name] = parse_alt
+            expect_punct("}")
+          end
+        end
+
+        def parse_alt
+          branches = [parse_seq]
+          while peek&.[](0) == :punct && peek[1] == "|"
+            advance
+            branches << parse_seq
+          end
+          branches.length == 1 ? branches.first : Node.new(:alt, branches)
+        end
+
+        def parse_seq
+          items = [parse_prefixed]
+          until seq_stops?
+            items << parse_prefixed
+          end
+          items.length == 1 ? items.first : Node.new(:seq, items)
+        end
+
+        def seq_stops?
+          token = peek
+          return true if token.nil?
+
+          token[0] == :punct && %w[| ) }].include?(token[1])
+        end
+
+        def parse_prefixed
+          sign = nil
+          if peek&.[](0) == :punct && %w[! &].include?(peek[1])
+            sign = advance[1] == "&"
+          end
+          node = parse_suffixed
+          sign.nil? ? node : Node.new(:pred, sign, node)
+        end
+
+        def parse_suffixed
+          node = parse_primary
+          op = peek
+          return node unless op&.[](0) == :punct && %w[* + ?].include?(op[1])
+
+          advance
+          case op[1]
+          when "*" then Node.new(:rep, node, 0, nil)
+          when "+" then Node.new(:rep, node, 1, nil)
+          else Node.new(:opt, node)
+          end
+        end
+
+        def parse_primary
+          token = advance
+          raise Error, "pest: unexpected end of rule body" if token.nil?
+
+          case token[0]
+          when :str then Node.new(:lit, unescape(token[1][1..-2]), false)
+          when :insens
+            body = token[1][1..]
+            body = body[1..] if body.start_with?("i")
+            Node.new(:lit, unescape(body[1..-2]), true)
+          when :range
+            lo = token[1][1]
+            hi = token[1][-2]
+            Node.new(:class, [[lo.ord, hi.ord]])
+          when :ident
+            if BUILTINS.key?(token[1])
+              parse_builtin(token[1])
+            elsif REJECTED.include?(token[1])
+              raise Error,
+                    "pest: #{token[1]} is not importable — " \
+                    "#{token[1] == 'WHITESPACE' ? 'pest applies it implicitly; define whitespace explicitly in PARG' : 'express it in PARG syntax'}"
+            else
+              Node.new(:ref, token[1])
+            end
+          when :punct
+            if token[1] == "("
+              node = parse_alt
+              expect_punct(")")
+              node
+            elsif token[1] == "~"
+              unless @noted_tilde
+                @notes << "~ treated as plain sequence (pest inserts implicit " \
+                          "WHITESPACE; PARG does not)"
+              end
+              @noted_tilde = true
+              Node.new(:lit, " ", false)
+            else
+              raise Error, "pest: unexpected #{token[1].inspect}"
+            end
+          else
+            raise Error, "pest: unexpected token #{token[0]}"
+          end
+        end
+
+        def parse_builtin(name)
+          spec = BUILTINS.fetch(name)
+          branches = spec.split(" / ").map do |part|
+            if part.start_with?("%x")
+              parse_hex_node(part)
+            else
+              Node.new(:lit, unescape(part[1..-2]), false)
+            end
+          end
+          branches.length == 1 ? branches.first : Node.new(:alt, branches)
+        end
+
+        def parse_hex_node(spec)
+          body = spec[2..]
+          if body.include?("-")
+            lo, hi = body.split("-")
+            Node.new(:class, [[lo.to_i(16), hi.to_i(16)]])
+          else
+            byte = body.to_i(16)
+            Node.new(:class, [[byte, byte]])
+          end
+        end
+
+        def expect_punct(char)
+          token = advance
+          unless token && token[0] == :punct && token[1] == char
+            raise Error, "pest: expected #{char.inspect}"
+          end
+        end
+
+        def unescape(body)
+          body.gsub(/\\(.)/) do
+            case Regexp.last_match(1)
+            when "n" then "\n"
+            when "t" then "\t"
+            when "r" then "\r"
+            when "0" then "\0"
+            else Regexp.last_match(1)
+            end
+          end
+        end
+
+        # ---- emission ------------------------------------------------------
+
+        def emit
+          lines = [
+            "# Imported from pest (Rust PEG).",
+            "# Notes:",
+            "# - ordered choice and predicates map directly",
+          ]
+          @notes.each { |note| lines << "# - #{note}" }
+          lines << "grammar imported_pest version \"0.0.0\" {"
+          lines.concat(@rules.map { |name, node| "  #{name} = #{emit_node(node, :top)}" })
+          lines << "}"
+          "#{lines.join("\n")}\n"
+        end
+
+        def emit_node(node, context)
+          case node.kind
+          when :lit then emit_string(node.a, fold: node.b)
+          when :class
+            node.a.map { |lo, hi| "%x#{lo.to_s(16)}-#{hi.to_s(16)}" }.join(" ")
+          when :ref then node.a
+          when :alt
+            text = node.a.map { |branch| emit_node(branch, :branch) }.join(" / ")
+            context == :branch ? "(#{text})" : text
+          when :seq
+            text = node.a.map { |item| emit_node(item, :seq_item) }.join(" ")
+            context == :branch ? "(#{text})" : text
+          when :rep
+            prefix = node.b.zero? ? "*" : "#{node.b}*"
+            "#{prefix}#{emit_operand(node.a)}"
+          when :opt then "[ #{emit_node(node.a, :top)} ]"
+          when :pred
+            "#{node.a ? '&' : '!'}#{emit_operand(node.b)}"
+          end
+        end
+
+        def emit_operand(node)
+          case node.kind
+          when :lit, :class, :ref then emit_node(node, :top)
+          else "(#{emit_node(node, :top)})"
+          end
+        end
+
+        def emit_string(body, fold:)
+          escaped = body.gsub("\\", "\\\\").gsub('"', '\\"')
+          fold ? "%i\"#{escaped}\"" : "\"#{escaped}\""
+        end
+      end
+    end
+  end
+end

data/lib/parsanol/parg/imports.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Parsanol
+  module PARG
+    # Cross-file imports (PN 1): `use <name>` at the top of a .parg file
+    # loads <name>.parg from an import directory, prefixes every rule with
+    # "<name>." (capture names are preserved — embedded grammars keep
+    # their capture contract), and merges entries, bindings, preprocess
+    # steps, tests, and docs. Cycles and missing files are compile
+    # errors. The artifact bakes the expansion: consumers never see
+    # imports.
+    module Imports
+      module_function
+
+      def merge!(document, import_dirs, merging = [])
+        return document if document.uses.empty?
+
+        document.uses.each do |name|
+          raise CompileError, "import cycle: #{(merging + [name]).join(' -> ')}" if merging.include?(name)
+
+          source = find_file(name, import_dirs)
+          used = Parser.new(File.read(source)).parse
+          merge!(used, import_dirs, merging + [name])
+          merge_used(document, used, "#{name}.")
+        end
+        document.uses = []
+        document
+      end
+
+      IMPORT_EXTENSIONS = %w[.yaml .parg .json].freeze
+
+      def find_file(name, import_dirs)
+        dirs = Array(import_dirs)
+        raise CompileError, "no import directories configured for `use #{name}`" if dirs.empty?
+
+        path = dirs.flat_map { |dir| IMPORT_EXTENSIONS.map { |ext| File.join(dir, "#{name}#{ext}") } }
+          .select { |candidate| candidate.end_with?(".parg") }
+          .find { |candidate| File.file?(candidate) }
+        path || raise(CompileError, "import `use #{name}` not found in #{dirs.inspect}")
+      end
+
+      def merge_used(document, used, prefix)
+        used.rules.each do |name, node|
+          qualified = "#{prefix}#{name}"
+          raise CompileError, "imported rule #{qualified.inspect} collides" if document.rules.key?(qualified)
+
+          document.rules[qualified] = copy(node, prefix)
+        end
+        used.entries.each do |entry, rule|
+          document.entries["#{prefix}#{entry}"] = "#{prefix}#{rule}"
+        end
+        used.bindings.each do |rule, list|
+          document.bindings["#{prefix}#{rule}"] = list
+        end
+        used.preprocess.each do |name, steps|
+          document.preprocess["#{prefix}#{name}"] = steps
+        end
+        used.docs.each do |rule, text|
+          document.docs["#{prefix}#{rule}"] = text
+        end
+        default_used_entry = used.entries.size == 1 ? used.entries.keys.first : nil
+        used.tests.each do |test|
+          entry = test.entry || default_used_entry
+          document.tests << Document::Test.new(
+            entry ? "#{prefix}#{entry}" : test.entry,
+            test.kind, test.input, test.expect
+          )
+        end
+      end
+
+      def copy(node, prefix)
+        return node unless node.is_a?(Node)
+
+        case node.kind
+        when :seq then Node.new(:seq, node.a.map { |child| copy(child, prefix) })
+        when :alt then Node.new(:alt, node.a.map { |child| copy(child, prefix) })
+        when :rep then Node.new(:rep, copy(node.a, prefix), node.b, node.c)
+        when :opt then Node.new(:opt, copy(node.a, prefix))
+        when :pred then Node.new(:pred, node.a, copy(node.b, prefix))
+        when :cap then Node.new(:cap, node.a, copy(node.b, prefix))
+        when :ref then Node.new(:ref, "#{prefix}#{node.a}")
+        else node
+        end
+      end
+    end
+  end
+end

data/lib/parsanol/parg/lexer.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Parsanol
+  module PARG
+    # Tokenizer for PARG source text.
+    #
+    # Emits Token structs. Comments (# to end of line) and whitespace are
+    # dropped. Punctuation arrives as :punct tokens carrying the character.
+    class Lexer
+      Token = Struct.new(:type, :value, :offset)
+
+      DROPPED = %i[ws comment].freeze
+
+      TOKEN = /
+        (?<newline>\r?\n)
+      | (?<ws>[ \t]+)
+      | (?<doc>\#\#[^\n]*)
+      | (?<comment>\#[^\n]*)
+      | (?<hex>%x[0-9A-Fa-f]{2,6}(?:-[0-9A-Fa-f]{2,6}|(?:\.[0-9A-Fa-f]{2,6})+)?)
+      | (?<istr>%i"(?:[^"\\]|\\.)*")
+      | (?<sstr>%s"(?:[^"\\]|\\.)*")
+      | (?<str>"(?:[^"\\]|\\.)*")
+      | (?<arrow>->)
+      | (?<num>\d+)
+      | (?<ident>[A-Za-z_][A-Za-z0-9_]*)
+      | (?<punct>[=\/()\[\]{}*!&:,.])
+      /x
+
+      def initialize(text)
+        @text = text
+        @tokens = []
+        scan
+      end
+
+      attr_reader :tokens
+
+      private
+
+      def scan
+        pos = 0
+        until pos >= @text.length
+          match = TOKEN.match(@text, pos)
+          if match.nil? || match.begin(0) != pos
+            raise ParseError,
+                  "unexpected character #{@text[pos].inspect} at offset #{pos}"
+          end
+
+          type = matched_type(match)
+          if type && !DROPPED.include?(type)
+            value = token_value(type, match)
+            @tokens << Token.new(type, value, pos)
+          end
+          pos = match.end(0)
+        end
+      end
+
+      def matched_type(match)
+        TOKEN.names.each do |name|
+          return name.to_sym if match[name]
+        end
+        nil
+      end
+
+      def token_value(type, match)
+        case type
+        when :hex, :istr, :sstr then match[0][2..]
+        else match[0]
+        end
+      end
+    end
+  end
+end

data/lib/parsanol/parg/lints.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+module Parsanol
+  module PARG
+    # Registry of compile-time lints (OCP): each lint is a class with
+    # `errors(document, compiler)` and `warnings(document, compiler)` —
+    # both pure (data in, data out; no side effects). The compiler runs
+    # every registered lint; adding a lint never touches the pipeline.
+    # Lints use the compiler only as an analysis facade (find_left_cycle,
+    # first_set, nullable?).
+    module Lints
+      class << self
+        def register(name, strategy)
+          strategies[name.to_sym] = strategy
+        end
+
+        def errors(document, compiler)
+          strategies.values.flat_map { |strategy| strategy.errors(document, compiler) }
+        end
+
+        def warnings(document, compiler)
+          strategies.values.flat_map { |strategy| strategy.warnings(document, compiler) }
+        end
+
+        def strategies
+          @strategies ||= {}
+        end
+      end
+
+      # Left recursion (direct + indirect, leftmost positions) rejects
+      # the compile: every PEG engine would loop forever.
+      class LeftRecursion
+        def errors(document, compiler)
+          document.rules.filter_map do |name, _node|
+            cycle = compiler.find_left_cycle(name)
+            "left recursion detected: #{cycle.join(' -> ')}" if cycle
+          end
+        end
+
+        def warnings(_document, _compiler) = []
+      end
+
+      # Ordered-choice hazards: the class of silent failures PEGs are
+      # known for, turned into build failures plus recorded warnings.
+      class Alternatives
+        def errors(document, compiler)
+          checked(document, compiler).flat_map { |result| result[:errors] }
+        end
+
+        def warnings(document, compiler)
+          checked(document, compiler).flat_map { |result| result[:warnings] }
+        end
+
+        private
+
+        def checked(document, compiler)
+          @checked ||= {}
+          @checked[[document.object_id, compiler.object_id]] ||= begin
+            results = []
+            document.rules.each do |name, node|
+              next unless node.kind == :alt
+
+              results.concat(check_alternatives(name, node, compiler))
+            end
+            results
+          end
+        end
+
+        def check_alternatives(name, node, compiler)
+          results = []
+          branches = node.a
+          branches.each_with_index do |branch, index|
+            if compiler.nullable?(branch) && index < branches.length - 1
+              results << { errors: [empty_shadow_error(name, index + 1)], warnings: [] }
+            end
+            branches[(index + 1)..].each_with_index do |later, j|
+              results << compare_branches(name, branch, index + 1, later, index + j + 2, compiler)
+            end
+          end
+          results
+        end
+
+        def empty_shadow_error(name, branch_number)
+          "rule #{name}: branch #{branch_number} can match empty input and " \
+            "shadows all later branches"
+        end
+
+        def compare_branches(name, earlier, earlier_n, later, later_n, compiler)
+          earlier_lit = literal_string(earlier)
+          later_lit = literal_string(later)
+          if earlier_lit && later_lit
+            if earlier_lit == later_lit
+              return { errors: ["rule #{name}: branches #{earlier_n} and #{later_n} are identical"], warnings: [] }
+            elsif later_lit.start_with?(earlier_lit)
+              return { errors: [shadow_error(name, earlier_n, earlier_lit, later_n, later_lit)], warnings: [] }
+            end
+          end
+          first_earlier = compiler.first_set(earlier) - [Compiler::EPS]
+          first_later = compiler.first_set(later) - [Compiler::EPS]
+          if first_earlier.include?(Compiler::ANY) || first_later.include?(Compiler::ANY)
+            return { errors: [], warnings: [order_warning(name, earlier_n, later_n, "first set not statically known")] }
+          end
+          return { errors: [], warnings: [] } if !first_earlier.intersect?(first_later)
+
+          { errors: [],
+            warnings: [order_warning(name, earlier_n, later_n, "shared first bytes; ordered choice is decisive")] }
+        end
+
+        def shadow_error(name, earlier_n, earlier_lit, later_n, later_lit)
+          "rule #{name}: branch #{earlier_n} (#{earlier_lit.inspect}) shadows " \
+            "branch #{later_n} (#{later_lit.inspect}) — reorder longest-first or " \
+            "the shorter always wins"
+        end
+
+        def order_warning(name, earlier_n, later_n, reason)
+          "rule #{name}: branches #{earlier_n} and #{later_n} are order-dependent (#{reason})"
+        end
+
+        def literal_string(node)
+          case node.kind
+          when :lit then node.a
+          when :seq
+            node.a.map { |child| literal_string(child) }.join if node.a.all? { |child| child.kind == :lit }
+          end
+        end
+      end
+
+      register :left_recursion, LeftRecursion.new
+      register :alternatives, Alternatives.new
+
+      # An unbounded repetition whose body can match empty input is an
+      # INVALID GRAMMAR: the parse can never terminate, and any VM that
+      # tolerates it allocates without bound. Rejected at compile time —
+      # unboundedness is a grammar-validity property, not a runtime
+      # hazard to be hardened against.
+      class Repetitions
+        def errors(document, compiler)
+          document.rules.flat_map do |name, node|
+            walk(name, node, compiler)
+          end
+        end
+
+        def warnings(_document, _compiler) = []
+
+        private
+
+        def walk(name, node, compiler)
+          return [] if node.nil?
+
+          kind = node.kind
+          errors =
+            if kind == :rep && node.c.nil? && compiler.nullable?(node.a)
+              ["rule #{name}: repetition body can match empty input — " \
+               "unbounded repetition is an invalid grammar"]
+            else
+              []
+            end
+          children =
+            case kind
+            when :rep, :opt then [node.a]
+            when :pred, :cap then [node.b]
+            when :seq, :alt then node.a
+            else []
+            end
+          errors + children.flat_map { |child| walk(name, child, compiler) }
+        end
+      end
+      register :repetitions, Repetitions.new
+    end
+  end
+end