parsanol 1.3.107-arm-linux → 1.3.109-arm-linux

data/lib/parsanol/parg/lints.rb

@@ -9,6 +9,15 @@ module Parsanol
     # Lints use the compiler only as an analysis facade (find_left_cycle,
     # first_set, nullable?).
     module Lints
+      # Advisory warnings are author guidance — they never ride the
+      # envelope. The envelope's lint.order_warnings is a wire contract:
+      # recompiling an unchanged grammar must reproduce it byte-for-byte,
+      # so style/authoring findings (which fire on perfectly valid
+      # grammars) stay out of it.
+      Advisory = Struct.new(:message) do
+        def to_s = message
+      end
+
       class << self
         def register(name, strategy)
           strategies[name.to_sym] = strategy
@@ -22,6 +31,14 @@ module Parsanol
           strategies.values.flat_map { |strategy| strategy.warnings(document, compiler) }
         end
 
+        # The envelope-carried subset: strategies that are not advisory.
+        def envelope_warnings(document, compiler)
+          strategies.values
+            .reject { |strategy| strategy.respond_to?(:advisory?) && strategy.advisory? }
+            .flat_map { |strategy| strategy.warnings(document, compiler) }
+            .grep_v(Advisory)
+        end
+
         def strategies
           @strategies ||= {}
         end
@@ -71,8 +88,19 @@ module Parsanol
           branches = node.a
           branches.each_with_index do |branch, index|
             if compiler.nullable?(branch) && index < branches.length - 1
-              results << { errors: [empty_shadow_error(name, index + 1)], warnings: [] }
+              later = branches[(index + 1)..]
+              results << if later.all? { |candidate| compiler.nullable?(candidate) }
+                           # Both sides always succeed: the later branches are dead
+                           # code, but acceptance is identical. Imported grammars
+                           # legitimately produce this (unordered ABNF choice), so
+                           # it is a warning, not the shadowing hazard.
+                           { errors: [],
+                             warnings: [Advisory.new(dead_nullable_warning(name, index + 1))] }
+                         else
+                           { errors: [empty_shadow_error(name, index + 1)], warnings: [] }
+                         end
             end
+
             branches[(index + 1)..].each_with_index do |later, j|
               results << compare_branches(name, branch, index + 1, later, index + j + 2, compiler)
             end
@@ -85,14 +113,25 @@ module Parsanol
             "shadows all later branches"
         end
 
+        def dead_nullable_warning(name, branch_number)
+          "rule #{name}: branches after #{branch_number} are unreachable " \
+            "(that branch always succeeds) — dead code, acceptance unchanged"
+        end
+
         def compare_branches(name, earlier, earlier_n, later, later_n, compiler)
           earlier_lit = literal_string(earlier)
           later_lit = literal_string(later)
+          earlier_fold = literal_fold(earlier)
           if earlier_lit && later_lit
             if earlier_lit == later_lit
               return { errors: ["rule #{name}: branches #{earlier_n} and #{later_n} are identical"], warnings: [] }
             elsif later_lit.start_with?(earlier_lit)
               return { errors: [shadow_error(name, earlier_n, earlier_lit, later_n, later_lit)], warnings: [] }
+            elsif earlier_fold && earlier_lit.casecmp(later_lit).zero?
+              # A case-insensitive earlier literal matches every casing
+              # the later literal could: the later branch is dead (same
+              # class as the prefix shadow above).
+              return { errors: [case_shadow_error(name, earlier_n, earlier_lit, later_n, later_lit)], warnings: [] }
             end
           end
           first_earlier = compiler.first_set(earlier) - [Compiler::EPS]
@@ -116,6 +155,16 @@ module Parsanol
           "rule #{name}: branches #{earlier_n} and #{later_n} are order-dependent (#{reason})"
         end
 
+        def literal_fold(node)
+          node.kind == :lit ? node.b : nil
+        end
+
+        def case_shadow_error(name, earlier_n, earlier_lit, later_n, later_lit)
+          "rule #{name}: branch #{later_n} (#{later_lit.inspect}) can never match -- " \
+            "branch #{earlier_n} is case-insensitive and already matches " \
+            "#{earlier_lit.inspect} in every casing"
+        end
+
         def literal_string(node)
           case node.kind
           when :lit then node.a
@@ -125,8 +174,276 @@ module Parsanol
         end
       end
 
+      # TODO.sota/9 T1/T2 lints (ruby#307): the core-rule prelude and
+      # the brace form both carry a migration path — the lint carries
+      # the suggestion. Core-rule shadows are deliberate-compatibility
+      # (a local rule wins), so they are warnings, never errors.
+      class CoreRuleStyle
+        def advisory? = true
+
+        CORE_NAMES = (
+          Compiler::BUILTIN_CLASSES.keys + Compiler::STRUCTURED_CORE_RULE_NAMES
+        ).freeze
+
+        def warnings(document, _compiler)
+          authored = document.authored_rules || document.rules.keys
+          shadows = authored.select { |name| CORE_NAMES.include?(name) }
+          shadow_warnings = shadows.map do |name|
+            "rule #{name}: shadows the RFC 5234 core rule — drop the local " \
+              "definition unless the narrowing is deliberate"
+          end
+          shadow_warnings + style_mixing_warning(document)
+        end
+
+        def errors(_document, _compiler) = []
+
+        private
+
+        def style_mixing_warning(document)
+          brace = document.brace_rules.size
+          line = document.rules.size - brace
+          return [] if brace.zero? || line.zero?
+          return [] if document.rules.size < 4
+
+          ["grammar mixes brace (#{brace}) and line (#{line}) rule bodies — " \
+           "pick one style per grammar"]
+        end
+      end
+
       register :left_recursion, LeftRecursion.new
+      register :core_rule_style, CoreRuleStyle.new
+
+      # ruby#307: authoring-contract lints. Each flags a grammar whose
+      # artifact ships without something its consumers will want: pins,
+      # docs, or a canonical entry placement. All warnings — the
+      # constructs are legal; the omissions are the hazard.
+      class AuthoringContracts
+        def advisory? = true
+
+        def warnings(document, compiler)
+          unpinned(document) + undocumented(document, compiler)
+        end
+
+        def errors(_document, _compiler) = []
+
+        private
+
+        def unpinned(document)
+          return [] unless document.tests.empty?
+
+          ["grammar ships without a test section — an unpinned grammar is " \
+           "unpinned behavior (accept/reject/capture tests ride the artifact " \
+           "to every engine)"]
+        end
+
+        def undocumented(document, _compiler)
+          # Injected core rules and left-recursion tier rules are
+          # compiler-generated: they are reachable and necessarily
+          # undocumented, and flagging them would be noise.
+          injected = Compiler::BUILTIN_CLASSES.keys +
+            Compiler::STRUCTURED_CORE_RULE_NAMES
+          authored = document.authored_rules
+          prefix = Compiler::LEFT_RECURSION_PREFIX
+          entry_reachable(document).filter_map do |name|
+            next if authored && !authored.include?(name)
+            next if document.docs.key?(name)
+            next if injected.include?(name)
+            next if name.to_s.start_with?(prefix)
+
+            "rule #{name}: entry-reachable without a ## doc comment — the " \
+              "artifact ships docs for LSP hovers; a public rule without one " \
+              "is a missing contract"
+          end
+        end
+
+        # Rules reachable from any entry root through rule references.
+        def entry_reachable(document)
+          reachable = []
+          seen = {}
+          walk = lambda do |node|
+            case node.kind
+            when :ref
+              name = node.a
+              next if seen[name]
+
+              seen[name] = true
+              reachable << name
+              walk.call(document.rules[name]) if document.rules.key?(name)
+            when :alt, :seq then node.a.each { |child| walk.call(child) }
+            when :cap, :pred then walk.call(node.b)
+            when :rep, :opt then walk.call(node.a)
+            end
+          end
+          document.entries.each_value do |root|
+            walk.call(document.rules[root]) if document.rules.key?(root)
+          end
+          reachable
+        end
+      end
+
+      register :authoring_contracts, AuthoringContracts.new
+      # ruby#308 first slice: the two halves of binding coverage.
+      # A binding that names a capture the grammar never produces binds
+      # nothing; a capture no binding consumes is dead weight. Both are
+      # advisory: the grammar compiles either way.
+      class BindingCoverage
+        def advisory? = true
+
+        def warnings(document, _compiler)
+          all_captures = produced_captures(document)
+          bound = []
+          document.bindings.each_value do |list|
+            Array(list).each { |binding| bound << binding.capture }
+          end
+
+          # No bindings section = the author opted out of binding (a
+          # parse-only grammar): uncovered captures are not oversights.
+          # The orphan half moved to BindingPaths, whose resolution
+          # failures fail the build (ruby#308 v2) -- a warning could
+          # never fire past that error.
+          dead = document.bindings.empty? ? [] : all_captures - bound.uniq
+          dead.map do |capture|
+            "capture #{capture.inspect} is bound by no bindings section -- " \
+              "ship the binding beside the capture, or drop the capture"
+          end
+        end
+
+        def errors(_document, _compiler) = []
+
+        private
+
+        def produced_captures(document)
+          captures = []
+          seen = {}
+          document.rules.each_value do |node|
+            collect_captures(node, document, captures, seen)
+          end
+          captures.uniq
+        end
+
+        def collect_captures(node, document, captures, seen)
+          case node.kind
+          when :cap
+            captures << node.a
+            collect_captures(node.b, document, captures, seen)
+          when :ref
+            rule = node.a
+            return if seen[rule]
+
+            seen[rule] = true
+            target = document.rules[rule]
+            collect_captures(target, document, captures, seen) if target
+          when :alt, :seq
+            node.a.each { |child| collect_captures(child, document, captures, seen) }
+          when :pred
+            collect_captures(node.b, document, captures, seen)
+          when :rep, :opt
+            collect_captures(node.a, document, captures, seen)
+          end
+        end
+      end
+
       register :alternatives, Alternatives.new
+      register :binding_coverage, BindingCoverage.new
+
+      # ruby#308 v2: compile-time binding resolution. The CAPTURE a
+      # binding reads must exist in its rule's capture subtree (the
+      # orphan check, now with teeth); a `x[]...` path needs that
+      # capture to sit under a repetition (the flattening presumes
+      # iteration). The path's names are output naming -- free -- but
+      # its SHAPE is a contract: one `[]` level, one dotted-free leaf,
+      # and a nested path always carries the marker. Anything the
+      # runtime would silently bind as nothing fails the build instead,
+      # naming the rule, the path, and the offending segment.
+      class BindingPaths
+        def errors(document, _compiler)
+          document.bindings.flat_map do |rule, bindings|
+            tree = CaptureTree.new(document, rule)
+            Array(bindings).flat_map { |binding| resolve(rule, binding, tree) }
+          end
+        end
+
+        def warnings(_document, _compiler) = []
+        def advisory? = false
+
+        private
+
+        # The validated language is what the runtime honors today:
+        # `capture` and `capture[].leaf` -- one repetition level, one
+        # leaf. Anything else fails the build instead of silently
+        # binding nothing (the runtime's leaf lookup would miss).
+        def resolve(rule, binding, tree)
+          capture = binding.capture.to_s
+          path = binding.path.to_s
+          return [unresolvable(rule, path, path)] if path.include?(".") && !path.include?("[]")
+
+          scope = tree.scope
+          return [unresolvable(rule, capture, capture)] unless scope.key?(capture)
+          return [] unless path.include?("[]")
+
+          levels = path.split("[]", -1)
+          return [unresolvable(rule, path, "[]")] if levels.size > 2
+
+          leaf = levels[1].to_s.delete_prefix(".")
+          return [unresolvable(rule, path, "[]")] unless scope[capture][0]
+          return [unresolvable(rule, path, leaf)] if leaf.empty? || leaf.include?(".")
+
+          []
+        end
+
+        def unresolvable(rule, path, segment)
+          "binding on #{rule.inspect}: path #{path.inspect} does not resolve -- " \
+            "segment #{segment.inspect} names no capture at that position " \
+            "(supported: capture, capture[].leaf)"
+        end
+      end
+
+      # The capture TREE a binding path resolves against: per capture,
+      # whether it sits under a repetition and which captures its
+      # subtree contains (transitively -- the runtime's capture lookup
+      # is subtree-wide too).
+      class CaptureTree
+        def initialize(document, rule)
+          @document = document
+          @scope = Hash.new { |hash, name| hash[name] = [false, []] }
+          walk(document.rules[rule], { rule => true }) if document.rules.key?(rule)
+        end
+
+        def scope
+          @scope
+        end
+
+        def walk(node, seen, under_rep: false, inside: [])
+          case node.kind
+          when :cap
+            entry = @scope[node.a]
+            # `1*x as n` compiles to cap(n, rep): the capture WRAPS its
+            # repetition, so a repetition in either position makes the
+            # capture repeated.
+            repeated = under_rep || node.b&.kind == :rep
+            entry[0] ||= repeated
+            inside.each { |parent| entry[1] |= [parent] unless entry[1].include?(parent) }
+            walk(node.b, seen, under_rep: repeated, inside: inside + [node.a])
+          when :ref
+            rule = node.a
+            return if seen[rule]
+
+            seen[rule] = true
+            target = @document.rules[rule]
+            walk(target, seen, under_rep: under_rep, inside: inside) if target
+          when :alt, :seq
+            node.a.each { |child| walk(child, seen, under_rep: under_rep, inside: inside) }
+          when :pred
+            walk(node.b, seen, under_rep: under_rep, inside: inside)
+          when :rep
+            walk(node.a, seen, under_rep: true, inside: inside)
+          when :opt
+            walk(node.a, seen, under_rep: under_rep, inside: inside)
+          end
+        end
+      end
+
+      register :binding_paths, BindingPaths.new
 
       # An unbounded repetition whose body can match empty input is an
       # INVALID GRAMMAR: the parse can never terminate, and any VM that
@@ -166,6 +483,136 @@ module Parsanol
         end
       end
       register :repetitions, Repetitions.new
+
+      # ruby#307 lint completion: a rule no other rule references is
+      # dead weight (unless it carries an entry); a rule referenced
+      # exactly once is an inline candidate. Both advisory -- the
+      # grammar is correct either way. Entry roots, the injected core
+      # rules, and the left-recursion tier rules are exempt.
+      class RuleUsage
+        def advisory? = true
+
+        def warnings(document, _compiler)
+          counts = Hash.new(0)
+          document.rules.each do |name, node|
+            collect_refs(node, document, { name => true }, counts)
+          end
+
+          injected = Compiler::BUILTIN_CLASSES.keys +
+            Compiler::STRUCTURED_CORE_RULE_NAMES
+          resync_prefix = Compiler::LEFT_RECURSION_PREFIX
+          document.rules.filter_map do |name, _node|
+            next if injected.include?(name)
+            next if name.to_s.start_with?(resync_prefix)
+            next if document.entries.value?(name)
+
+            case counts[name]
+            when 0
+              "rule #{name.inspect} is referenced by no rule -- dead " \
+              "unless it is a future entry; drop it or wire it in"
+            when 1
+              "rule #{name.inspect} has exactly one reference -- inline " \
+              "candidate (keep it if the name carries the domain)"
+            end
+          end
+        end
+
+        def errors(_document, _compiler) = []
+
+        private
+
+        def collect_refs(node, document, seen, counts)
+          case node.kind
+          when :ref
+            rule = node.a
+            counts[rule] += 1
+            return if seen[rule]
+
+            seen[rule] = true
+            target = document.rules[rule]
+            collect_refs(target, document, seen, counts) if target
+          when :alt, :seq
+            node.a.each { |child| collect_refs(child, document, seen, counts) }
+          when :cap, :pred
+            collect_refs(node.b, document, seen, counts)
+          when :rep, :opt
+            collect_refs(node.a, document, seen, counts)
+          end
+        end
+      end
+      register :rule_usage, RuleUsage.new
+
+      # ruby#307: canonical entry placement. Every corpus grammar and
+      # the guide's canonical demo declare entries at the FOOT of the
+      # grammar block, after the rules they root; an entry above its
+      # rule reads as a forward table-of-contents and breaks the
+      # reading order. Advisory — placement is style, resolution is
+      # order-independent.
+      class EntryPlacement
+        def advisory? = true
+
+        def warnings(document, _compiler)
+          index = {}
+          document.declaration_order.each_with_index do |(kind, name), i|
+            index[[kind, name]] = i
+          end
+          document.entries.filter_map do |name, rule|
+            entry_at = index[[:entry, name]]
+            rule_at = index[[:rule, rule]]
+            next if entry_at.nil? || rule_at.nil? || entry_at > rule_at
+
+            "entry #{name.inspect} is declared before the rule #{rule.inspect} it " \
+              "roots -- canonical placement: entries at the foot of the grammar block"
+          end
+        end
+
+        def errors(_document, _compiler) = []
+      end
+      register :entry_placement, EntryPlacement.new
+
+      # ruby#307 B5 --suggest-upgrades: machine-applicable authoring
+      # upgrades. NOT registered -- its output is a suggestion surface
+      # (parg lint --suggest-upgrades), never a compile diagnostic and
+      # never part of the envelope.
+      class CoreRuleDuplicates
+        def warnings(document, _compiler)
+          document.rules.filter_map do |name, node|
+            next if Compiler::BUILTIN_CLASSES.key?(name)
+
+            core = core_rule_for(node)
+            next if core.nil?
+
+            "rule #{name.inspect} duplicates the core rule #{core} -- " \
+              "reference #{core} instead (or keep the local spelling deliberate)"
+          end
+        end
+
+        def errors(_document, _compiler) = []
+        def advisory? = true
+
+        private
+
+        def core_rule_for(node)
+          case node.kind
+          when :class
+            Compiler::BUILTIN_CLASSES.find { |_, ranges| normalized(ranges) == normalized(node.a) }&.first
+          when :seq
+            crlf?(node) ? "CRLF" : nil
+          end
+        end
+
+        def crlf?(node)
+          parts = node.a
+          parts.length == 2 &&
+            parts.all? { |part| part.kind == :class } &&
+            normalized(parts[0].a) == [[0x0D, 0x0D]] &&
+            normalized(parts[1].a) == [[0x0A, 0x0A]]
+        end
+
+        def normalized(ranges)
+          ranges.map(&:sort).sort
+        end
+      end
     end
   end
 end

data/lib/parsanol/parg/parser.rb

@@ -27,6 +27,7 @@ module Parsanol
         @tokens = Lexer.new(text).tokens
         @pos = 0
         @source_text = text
+        @brace_depth = 0
       end
 
       def parse
@@ -65,6 +66,20 @@ module Parsanol
         advance while peek&.type == :newline
       end
 
+      # Whitespace inside a brace body: newlines and comments (the `##`
+      # doc token included — inside a body it is a comment, not a doc
+      # attachment). No-op outside braces, so line-mode parsing is
+      # unchanged.
+      def skip_body_ws
+        return unless @brace_depth.positive?
+
+        skip_newlines
+        while peek&.type == :doc
+          advance
+          skip_newlines
+        end
+      end
+
       # Doc comments (##) accumulate and attach to the next rule.
       def take_doc_comments
         docs = []
@@ -294,12 +309,41 @@ module Parsanol
           name = rule_name
           document.docs[name] = docs.join("\n") unless docs.empty?
           punct("=")
+          # Brace bodies (TODO.sota/9 T2, ruby#302): `name = { ... }`
+          # is self-delimiting — newlines and comments are whitespace
+          # inside, the body extends to the matching `}`. Additive: `{`
+          # never appeared at the rule head before.
+          brace = peek&.type == :punct && peek.value == "{"
+          if !brace
+            # The brace may also sit on the next line: `name =` NL `{`.
+            # Peek past the newline; restore when it is not a brace so
+            # line-mode error behavior is untouched.
+            saved = @pos
+            skip_newlines
+            if peek&.type == :punct && peek.value == "{"
+              brace = true
+            else
+              @pos = saved
+            end
+          end
+          if brace
+            advance
+            @brace_depth += 1
+            skip_body_ws
+          end
           node = parse_choice
+          if brace
+            skip_body_ws
+            punct("}")
+            @brace_depth -= 1
+          end
           if document.rules.key?(name)
             raise ParseError, "duplicate rule #{name.inspect}"
           end
 
           document.rules[name] = node
+          document.declaration_order << [:rule, name]
+          document.brace_rules << name if brace
           document.atomic_rules << name if atomic
           skip_newlines
         end
@@ -365,7 +409,10 @@ module Parsanol
 
       def parse_choice
         branches = [parse_seq]
-        while peek&.type == :punct && peek.value == "/"
+        loop do
+          skip_body_ws
+          break unless peek&.type == :punct && peek.value == "/"
+
           advance
           branches << parse_seq
         end
@@ -373,15 +420,22 @@ module Parsanol
       end
 
       def parse_seq
+        skip_body_ws
         items = [parse_element]
         until stop_element?
+          skip_body_ws
+          # After skipped body whitespace the terminator (`}`, `)`, `/`)
+          # may only now be visible: re-check before parsing an element.
+          break if stop_element?
+          break if @brace_depth.positive? && peek&.type == :punct && peek.value == "/"
+
           items << parse_element
         end
         items.length == 1 ? items.first : Node.new(:seq, items)
       end
 
       def stop_element?
-        peek.nil? || peek.type == :newline ||
+        peek.nil? || (peek.type == :newline && @brace_depth.zero?) ||
           (peek.type == :punct && ["/", ")", "]", "}"].include?(peek.value)) ||
           # `capture:` suffix terminates the skip declaration's RHS
           # (parsanol-ruby#152); everywhere else parse_element's guard
@@ -452,6 +506,7 @@ module Parsanol
       end
 
       def capture_postfix(node)
+        skip_body_ws
         if peek&.type == :ident && peek.value == "as"
           advance
           node = Node.new(:cap, ident.value, node)
@@ -477,6 +532,7 @@ module Parsanol
         when :str, :sstr then Node.new(:lit, unquote(token.value), false)
         when :istr then Node.new(:lit, unquote(token.value), true)
         when :hex then parse_hex(token.value)
+        when :category then Node.new(:category, token.value[3..-2])
         when :punct then parse_punctuated(token)
         when :ident then parse_ident(token)
         else raise ParseError, "unexpected token #{token.type}:#{token.value}"
@@ -580,6 +636,7 @@ module Parsanol
         name = ident.value
         punct(":")
         document.entries[name] = rule_name
+        document.declaration_order << [:entry, name]
       end
 
       def parse_bindings(document)

data/lib/parsanol/parg.rb

@@ -18,6 +18,7 @@ module Parsanol
     autoload :CompileError, "parsanol/parg/error"
     autoload :ArtifactError, "parsanol/parg/error"
     autoload :Node, "parsanol/parg/node"
+    autoload :Export, "parsanol/parg/export"
     autoload :Lexer, "parsanol/parg/lexer"
     autoload :Parser, "parsanol/parg/parser"
     autoload :Document, "parsanol/parg/document"
@@ -33,6 +34,7 @@ module Parsanol
     autoload :CLI, "parsanol/parg/cli"
     autoload :Lsp, "parsanol/parg/lsp"
     autoload :Lints, "parsanol/parg/lints"
+    autoload :Diff, "parsanol/parg/diff"
     autoload :Preprocess, "parsanol/parg/preprocess"
     autoload :Visitor, "parsanol/parg/visitor"
     autoload :Imports, "parsanol/parg/imports"

data/lib/parsanol/slice.rb

@@ -143,12 +143,21 @@ module Parsanol
     # Unified line/column computation:
     # - String input: compute from input string
     # - Source / LineCache: delegate
+    def prefix_bytes(pos)
+      (@input.byteslice(0, pos) || "").b
+    end
+
     def line_and_column_at(pos)
       if @input.is_a?(String)
-        prefix = @input.byteslice(0, pos) || ""
-        line = 1 + prefix.count("\n")
-        last_nl = prefix.rindex("\n")
-        column = last_nl ? pos - last_nl : pos + 1
+        # All index math in BINARY (rindex returns a character index on
+        # a text string — mixing it with byteslice was the off-by-one);
+        # the column is a CHARACTER count (the guide's wire contract;
+        # rs walks chars()). Offsets stay bytes.
+        raw = prefix_bytes(pos)
+        line = 1 + raw.count("\n".b)
+        last_nl = raw.rindex("\n".b)
+        tail = last_nl ? raw.byteslice(last_nl + 1, pos - last_nl - 1) : raw
+        column = tail.force_encoding(Encoding::UTF_8).length + 1
         [line, column]
       else
         @input.line_and_column(pos)

data/lib/parsanol/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Parsanol
-  VERSION = "1.3.107"
+  VERSION = "1.3.109"
 end