parsanol 1.3.66-aarch64-linux → 1.3.67-aarch64-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a8535329ca9b9c8836c7cbfd2644ca86f3bf79b2d42809ce1b4f949c27f71050
-  data.tar.gz: be575af69b7a661f0b1168791af56ea55ba16e504b10436aea1cc72c456ded8a
+  metadata.gz: ccfa3bf06978a6ae539be84940ca4bbff440b324644bf539960f979ca0237f5b
+  data.tar.gz: 79ab5492c25ac5b12f69c36a0761cd5894ba641fd6cae5322044d3965ad161e5
 SHA512:
-  metadata.gz: df07c688d90a26aece7c8f628350a7b8ac5bfdf86da38b43ea3141add51e918374c0dbbe20544deac0a95dab28e41d9bec65116455de47cdaced7a3011e53c08
-  data.tar.gz: 6fac38097606659f867bab918c2972f181c0a765987a08537325455e7cf12030f2259af8272b1c3a4558d48c15773c9e05c13c51a12946a5f7a6f6c3a2ddc50f
+  metadata.gz: 33723ecbdfc86c933c24024e7b317c4d8d099a0759381f28587093847ad6bbe85249d7e8ec333f10914734735cc510bd77f0f1a2ad35bcbbae50b10d86737468
+  data.tar.gz: c17bf0042c56e63aeffb1a8efdcdc56f97888b00c0c4d22a8aeef9f18227315fa62cfc2b12c204e900f5d0e00ea88dd63711a550a6e78fe40599979eda0092a2

data/lib/parsanol/parg/compiler.rb

@@ -55,6 +55,17 @@ module Parsanol
         @warnings = []
       end
 
+      # QoL shorthands (ABNF core rules): materialized on reference when
+      # the grammar does not define its own rule of that name.
+      BUILTIN_CLASSES = {
+        "DIGIT" => [[0x30, 0x39]],
+        "ALPHA" => [[0x41, 0x5A], [0x61, 0x7A]],
+        "ALNUM" => [[0x30, 0x39], [0x41, 0x5A], [0x61, 0x7A]],
+        "WORD" => [[0x30, 0x39], [0x41, 0x5A], [0x5F, 0x5F], [0x61, 0x7A]],
+        "HEXDIG" => [[0x30, 0x39], [0x41, 0x46], [0x61, 0x66]],
+        "ANY" => [[0x00, 0x10FFFF]],
+      }.freeze
+
       def compile
         @document.rules.each_key { |name| atom_for(name) }
         errors = Lints.errors(@document, self)
@@ -79,7 +90,124 @@ module Parsanol
       end
 
       def atom_for(name)
-        @atom_cache[name] ||= build_atom(fetch_rule(name))
+        @atom_cache[name] ||= begin
+          atom = build_atom(fetch_rule(name))
+          if @document.skip_declaration && !skip_exempt_names.include?(name)
+            atom = inject_skip_trivia(atom)
+            # Entry points take leading and trailing skips: tail trivia
+            # (the tail-comment case) has no following terminal to
+            # trigger an injection, and full-consumption would reject
+            # it; leading trivia has no preceding interleave either.
+            if entry_rule?(name)
+              atom = Atoms::Sequence.new(skip_maybe_atom, atom, skip_maybe_atom)
+            end
+          end
+          atom
+        end
+      end
+
+      def entry_rule?(name)
+        @document.entries.value?(name) || @document.own_entries&.include?(name)
+      end
+
+      # The skip declaration's rule references (and their transitive
+      # rule references) build without injection: the skip atom must
+      # not contain itself.
+      def skip_exempt_names
+        @skip_exempt_names ||= begin
+          names = Set.new
+          walk_refs = lambda do |node|
+            case node.kind
+            when :ref then names << node.a
+            when :alt, :seq then node.a.each { |b| walk_refs.call(b) }
+            when :cap then walk_refs.call(node.b)
+            end
+          end
+          walk_refs.call(@document.skip_declaration) if @document.skip_declaration
+          closed = Set.new
+          until names.subset?(closed)
+            (names - closed).each do |ref|
+              closed << ref
+              walk_refs.call(@document.rules[ref]) if @document.rules.key?(ref)
+            end
+            names += closed
+          end
+          names
+        end
+      end
+
+      def skip_wrapper?(atom)
+        @skip_wrapper_ids&.key?(atom)
+      end
+
+      def skip_maybe_atom
+        @skip_maybe_atom ||= begin
+          atom = build_atom(@document.skip_declaration)
+          while atom.is_a?(Atoms::Entity)
+            atom = atom.parslet
+          end
+          # the VM's internal compiler owns the ATOM-level nullability
+          # analysis (the node-level nullable? here works on rule nodes)
+          if Parsanol::VM::Compiler.new.nullable?(atom)
+            raise CompileError,
+                  "skip rule must be non-nullable: a skip that can match " \
+                  "empty loops forever at injection points"
+          end
+          # Ignored: consumed but contributes nothing to the enclosing
+          # sequence/repetition value — trivia never leaks into
+          # span-joined captures.
+          wrapper = Atoms::Ignored.new(Atoms::Repetition.new(atom, 0, 1))
+          (@skip_wrapper_ids ||= {}.compare_by_identity)[wrapper] = true
+          wrapper
+        end
+      end
+
+      # parsanol-ruby#134: inject an optional match of the declared skip
+      # rule before every terminal of every rule. Injected skips are
+      # capture-free (a bare Repetition contributes no Named results),
+      # optional (ordered-choice/backtracking semantics preserved), and
+      # leave first sets unchanged (an optional prefix is epsilon-able).
+      def inject_skip_trivia(atom)
+        return atom if @document.skip_declaration.nil?
+
+        case atom
+        # rubocop:disable Lint/DuplicateBranch -- pass-through is the
+        # deliberate default; identical bodies, different decisions
+        when Atoms::Sequence
+          kids = atom.parslets.map { |child| inject_skip_trivia(child) }
+          out = []
+          kids.each do |child|
+            out << skip_maybe_atom unless skip_wrapper?(child)
+            out << child
+          end
+          out.length == 1 ? out.first : Atoms::Sequence.new(*out)
+        when Atoms::Alternative
+          Atoms::Alternative.new(*atom.alternatives.map { |a| inject_skip_trivia(a) })
+        when Atoms::Repetition
+          if skip_wrapper?(atom)
+            atom
+          else
+            Atoms::Repetition.new(inject_skip_trivia(atom.parslet), atom.min, atom.max,
+                                  atom.result_tag)
+          end
+        when Atoms::Named
+          # Pass-through: trivia must stay OUTSIDE captures. Injecting
+          # inside a Named makes the captured span include the skipped
+          # bytes (span-join swallows them into the captured value).
+          # Inter-token trivia around a capture is injected by the
+          # enclosing sequence's interleave.
+          atom
+        when Atoms::Lookahead
+          Atoms::Lookahead.new(inject_skip_trivia(atom.bound_parslet), atom.positive)
+        when Atoms::Str, Atoms::Re
+          # bare terminal as a whole rule body: leading trivia position
+          Atoms::Sequence.new(skip_maybe_atom, atom)
+        # pass-through is the deliberate default; identical bodies,
+        # different decisions
+        else
+          atom
+        end
+        # rubocop:enable Lint/DuplicateBranch
       end
 
       def record_warning(message)
@@ -174,6 +302,10 @@ module Parsanol
         if !@document.rules.key?(name) && @document.customs.key?(name)
           return Node.new(:lit, "\u0001", false)
         end
+        if !@document.rules.key?(name) && BUILTIN_CLASSES.key?(name)
+          # QoL shorthand stand-in: the analysis passes see the class
+          return Node.new(:class, BUILTIN_CLASSES[name])
+        end
 
         @document.rules[name] || raise(CompileError, "unknown rule #{name.inspect}")
       end
@@ -193,6 +325,15 @@ module Parsanol
             Atoms::Str.new(node.a)
           end
         when :class then Atoms::Re.new(class_pattern(node.a))
+        when :until
+          # QoL: (*( !terminal any_char )) — consume everything up to
+          # (and excluding) the delimiter.
+          any_char = Atoms::Re.new(class_pattern([[0x00, 0x10FFFF]]))
+          Atoms::Repetition.new(
+            Atoms::Sequence.new(Atoms::Lookahead.new(Atoms::Str.new(node.a), false),
+                                any_char),
+            0, nil
+          )
         when :seq
           atoms = node.a.map { |child| build_atom(child) }
           atoms.length == 1 ? atoms.first : Atoms::Sequence.new(*atoms)
@@ -234,6 +375,10 @@ module Parsanol
           if @document.customs.key?(node.a)
             @dynamic = true
             resolve_custom(node.a)
+          elsif !@document.rules.key?(node.a) && BUILTIN_CLASSES.key?(node.a)
+            # QoL: character-class shorthands (ABNF core rules) — usable
+            # unless the grammar defines its own rule of the same name.
+            Atoms::Re.new(class_pattern(BUILTIN_CLASSES[node.a]))
           else
             Atoms::Entity.new(node.a) { atom_for(node.a) }
           end
@@ -472,6 +617,13 @@ module Parsanol
           "docs" => @document.docs,
           "source" => @document.source,
         }
+        if @document.skip_declaration
+          # parsanol-ruby#134: declared trivia rule. Injection is baked
+          # into the compiled trees (plain atom kinds), so consumers
+          # without skip support still parse these artifacts correctly;
+          # the field records the grammar-level declaration.
+          envelope["skip"] = @document.skip_source
+        end
         envelope["customs"] = @document.customs unless @document.customs.empty?
         envelope["states"] = @document.states unless @document.states.empty?
         envelope["dynamic"] = true if @dynamic

data/lib/parsanol/parg/document.rb

@@ -9,7 +9,7 @@ module Parsanol
 
       Test = Struct.new(:entry, :kind, :input, :expect)
 
-      attr_accessor :grammar_name, :version, :source, :uses
+      attr_accessor :grammar_name, :version, :source, :uses, :skip_declaration, :skip_source
       attr_reader :rules, :bindings, :preprocess, :entries, :docs, :tests, :render, :derive,
                   :customs, :states
       # Entries authored in this file (pre-import); nil-entry tests resolve

data/lib/parsanol/parg/parser.rb

@@ -20,7 +20,8 @@ module Parsanol
 
       STATE_KEYWORDS = %w[custom state].freeze
       KEYWORDS = %w[grammar as alt from_table column bindings render derive
-                    preprocess entry table_lookup custom state set switch].freeze
+                    preprocess entry table_lookup custom state set switch
+                    skip until].freeze
 
       def initialize(text)
         @tokens = Lexer.new(text).tokens
@@ -161,6 +162,19 @@ module Parsanol
             skip_newlines
             next
           end
+          if peek&.type == :ident && peek.value == "skip"
+            advance
+            punct("=")
+            if document.skip_declaration
+              raise ParseError, "duplicate skip declaration"
+            end
+
+            node = parse_choice
+            document.skip_declaration = validate_skip_node(node)
+            document.skip_source = skip_source_string(node)
+            skip_newlines
+            next
+          end
           if peek&.type == :ident && STATE_KEYWORDS.include?(peek.value)
             keyword = advance.value
             slot = ident.value
@@ -188,6 +202,30 @@ module Parsanol
         punct("}")
       end
 
+      # The skip declaration may only name rules (a rule reference or an
+      # alternation of rule references) — arbitrary expressions would
+      # make the injected trivia unanalyzable.
+      def skip_source_string(node)
+        case node.kind
+        when :ref then node.a
+        when :alt then node.a.map { |b| skip_source_string(b) }.join(" / ")
+        end
+      end
+
+      def validate_skip_node(node)
+        case node.kind
+        when :ref
+          node
+        when :alt
+          node.a.each { |branch| validate_skip_node(branch) }
+          node
+        else
+          raise ParseError,
+                "skip must be a rule reference or an alternation of rule " \
+                "references, got #{node.kind.inspect}"
+        end
+      end
+
       def rule_name
         token = ident
         if KEYWORDS.include?(token.value)
@@ -325,6 +363,10 @@ module Parsanol
           end
           punct("}")
           Node.new(:switch, slot, arms)
+        when "until"
+          # QoL: any-until terminal — (*( !terminal any_char )), the
+          # idiomatic "consume everything up to the delimiter" run.
+          Node.new(:until, unquote(expect(:str).value))
         when "alt"
           ident("from_table")
           table = unquote(expect(:str).value)

data/lib/parsanol/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Parsanol
-  VERSION = "1.3.66"
+  VERSION = "1.3.67"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: parsanol
 version: !ruby/object:Gem::Version
-  version: 1.3.66
+  version: 1.3.67
 platform: aarch64-linux
 authors:
 - Ribose Inc.