parsanol 1.3.65-x86_64-linux → 1.3.67-x86_64-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6c86ab4a722d2773cb11b28cc325d8ae79ec53a09b250b69b8f1ebc3aca45535
-  data.tar.gz: 5c106ddaa883cd3d5ce93644350be5b1f05dac998430ee2898d23edbb75f75db
+  metadata.gz: 5a813b154280a5449a18ff5ef6d34e8832753535c5ef796e205879c43227e746
+  data.tar.gz: 8653688985a9a44ca4508991b87c3e5bd1f64ba1192b6cae6ee74f5fba6c236a
 SHA512:
-  metadata.gz: 5e0f672b0e632599033401bf3404832a6dcaae9308039c8b3f10a52b853f172f38af80e57198617a317b64bd9ed5644657f35a55fe6d42988090786d47b62792
-  data.tar.gz: 3daacb2565d78eac43ab5783e64f82887f022ebab7687674cff6f65f9d0ffe69b9a375829f08956f43e28bd738fa58885fad9ee5d0e5944471045fa31278dae2
+  metadata.gz: 804f2d7afab07c5ae7144fd22de2e69135df734673db91e76873754d8552f7c4086835b2209edf283011817aee709026524c1e0ddd6bd6ca0dd57c03dddab96d
+  data.tar.gz: db333c204ea7f3d4936b3672239ea12cf0d10fd146223ef7abbc81993dcb76689d44af314f88fb6188bea43fa29383d9607bb7d0f0648078f56f9b9fc8a7cb8d

data/lib/parsanol/parg/compiler.rb

@@ -55,6 +55,17 @@ module Parsanol
         @warnings = []
       end
 
+      # QoL shorthands (ABNF core rules): materialized on reference when
+      # the grammar does not define its own rule of that name.
+      BUILTIN_CLASSES = {
+        "DIGIT" => [[0x30, 0x39]],
+        "ALPHA" => [[0x41, 0x5A], [0x61, 0x7A]],
+        "ALNUM" => [[0x30, 0x39], [0x41, 0x5A], [0x61, 0x7A]],
+        "WORD" => [[0x30, 0x39], [0x41, 0x5A], [0x5F, 0x5F], [0x61, 0x7A]],
+        "HEXDIG" => [[0x30, 0x39], [0x41, 0x46], [0x61, 0x66]],
+        "ANY" => [[0x00, 0x10FFFF]],
+      }.freeze
+
       def compile
         @document.rules.each_key { |name| atom_for(name) }
         errors = Lints.errors(@document, self)
@@ -79,7 +90,124 @@ module Parsanol
       end
 
       def atom_for(name)
-        @atom_cache[name] ||= build_atom(fetch_rule(name))
+        @atom_cache[name] ||= begin
+          atom = build_atom(fetch_rule(name))
+          if @document.skip_declaration && !skip_exempt_names.include?(name)
+            atom = inject_skip_trivia(atom)
+            # Entry points take leading and trailing skips: tail trivia
+            # (the tail-comment case) has no following terminal to
+            # trigger an injection, and full-consumption would reject
+            # it; leading trivia has no preceding interleave either.
+            if entry_rule?(name)
+              atom = Atoms::Sequence.new(skip_maybe_atom, atom, skip_maybe_atom)
+            end
+          end
+          atom
+        end
+      end
+
+      def entry_rule?(name)
+        @document.entries.value?(name) || @document.own_entries&.include?(name)
+      end
+
+      # The skip declaration's rule references (and their transitive
+      # rule references) build without injection: the skip atom must
+      # not contain itself.
+      def skip_exempt_names
+        @skip_exempt_names ||= begin
+          names = Set.new
+          walk_refs = lambda do |node|
+            case node.kind
+            when :ref then names << node.a
+            when :alt, :seq then node.a.each { |b| walk_refs.call(b) }
+            when :cap then walk_refs.call(node.b)
+            end
+          end
+          walk_refs.call(@document.skip_declaration) if @document.skip_declaration
+          closed = Set.new
+          until names.subset?(closed)
+            (names - closed).each do |ref|
+              closed << ref
+              walk_refs.call(@document.rules[ref]) if @document.rules.key?(ref)
+            end
+            names += closed
+          end
+          names
+        end
+      end
+
+      def skip_wrapper?(atom)
+        @skip_wrapper_ids&.key?(atom)
+      end
+
+      def skip_maybe_atom
+        @skip_maybe_atom ||= begin
+          atom = build_atom(@document.skip_declaration)
+          while atom.is_a?(Atoms::Entity)
+            atom = atom.parslet
+          end
+          # the VM's internal compiler owns the ATOM-level nullability
+          # analysis (the node-level nullable? here works on rule nodes)
+          if Parsanol::VM::Compiler.new.nullable?(atom)
+            raise CompileError,
+                  "skip rule must be non-nullable: a skip that can match " \
+                  "empty loops forever at injection points"
+          end
+          # Ignored: consumed but contributes nothing to the enclosing
+          # sequence/repetition value — trivia never leaks into
+          # span-joined captures.
+          wrapper = Atoms::Ignored.new(Atoms::Repetition.new(atom, 0, 1))
+          (@skip_wrapper_ids ||= {}.compare_by_identity)[wrapper] = true
+          wrapper
+        end
+      end
+
+      # parsanol-ruby#134: inject an optional match of the declared skip
+      # rule before every terminal of every rule. Injected skips are
+      # capture-free (a bare Repetition contributes no Named results),
+      # optional (ordered-choice/backtracking semantics preserved), and
+      # leave first sets unchanged (an optional prefix is epsilon-able).
+      def inject_skip_trivia(atom)
+        return atom if @document.skip_declaration.nil?
+
+        case atom
+        # rubocop:disable Lint/DuplicateBranch -- pass-through is the
+        # deliberate default; identical bodies, different decisions
+        when Atoms::Sequence
+          kids = atom.parslets.map { |child| inject_skip_trivia(child) }
+          out = []
+          kids.each do |child|
+            out << skip_maybe_atom unless skip_wrapper?(child)
+            out << child
+          end
+          out.length == 1 ? out.first : Atoms::Sequence.new(*out)
+        when Atoms::Alternative
+          Atoms::Alternative.new(*atom.alternatives.map { |a| inject_skip_trivia(a) })
+        when Atoms::Repetition
+          if skip_wrapper?(atom)
+            atom
+          else
+            Atoms::Repetition.new(inject_skip_trivia(atom.parslet), atom.min, atom.max,
+                                  atom.result_tag)
+          end
+        when Atoms::Named
+          # Pass-through: trivia must stay OUTSIDE captures. Injecting
+          # inside a Named makes the captured span include the skipped
+          # bytes (span-join swallows them into the captured value).
+          # Inter-token trivia around a capture is injected by the
+          # enclosing sequence's interleave.
+          atom
+        when Atoms::Lookahead
+          Atoms::Lookahead.new(inject_skip_trivia(atom.bound_parslet), atom.positive)
+        when Atoms::Str, Atoms::Re
+          # bare terminal as a whole rule body: leading trivia position
+          Atoms::Sequence.new(skip_maybe_atom, atom)
+        # pass-through is the deliberate default; identical bodies,
+        # different decisions
+        else
+          atom
+        end
+        # rubocop:enable Lint/DuplicateBranch
       end
 
       def record_warning(message)
@@ -174,6 +302,10 @@ module Parsanol
         if !@document.rules.key?(name) && @document.customs.key?(name)
           return Node.new(:lit, "\u0001", false)
         end
+        if !@document.rules.key?(name) && BUILTIN_CLASSES.key?(name)
+          # QoL shorthand stand-in: the analysis passes see the class
+          return Node.new(:class, BUILTIN_CLASSES[name])
+        end
 
         @document.rules[name] || raise(CompileError, "unknown rule #{name.inspect}")
       end
@@ -193,6 +325,15 @@ module Parsanol
             Atoms::Str.new(node.a)
           end
         when :class then Atoms::Re.new(class_pattern(node.a))
+        when :until
+          # QoL: (*( !terminal any_char )) — consume everything up to
+          # (and excluding) the delimiter.
+          any_char = Atoms::Re.new(class_pattern([[0x00, 0x10FFFF]]))
+          Atoms::Repetition.new(
+            Atoms::Sequence.new(Atoms::Lookahead.new(Atoms::Str.new(node.a), false),
+                                any_char),
+            0, nil
+          )
         when :seq
           atoms = node.a.map { |child| build_atom(child) }
           atoms.length == 1 ? atoms.first : Atoms::Sequence.new(*atoms)
@@ -234,6 +375,10 @@ module Parsanol
           if @document.customs.key?(node.a)
             @dynamic = true
             resolve_custom(node.a)
+          elsif !@document.rules.key?(node.a) && BUILTIN_CLASSES.key?(node.a)
+            # QoL: character-class shorthands (ABNF core rules) — usable
+            # unless the grammar defines its own rule of the same name.
+            Atoms::Re.new(class_pattern(BUILTIN_CLASSES[node.a]))
           else
             Atoms::Entity.new(node.a) { atom_for(node.a) }
           end
@@ -472,6 +617,13 @@ module Parsanol
           "docs" => @document.docs,
           "source" => @document.source,
         }
+        if @document.skip_declaration
+          # parsanol-ruby#134: declared trivia rule. Injection is baked
+          # into the compiled trees (plain atom kinds), so consumers
+          # without skip support still parse these artifacts correctly;
+          # the field records the grammar-level declaration.
+          envelope["skip"] = @document.skip_source
+        end
         envelope["customs"] = @document.customs unless @document.customs.empty?
         envelope["states"] = @document.states unless @document.states.empty?
         envelope["dynamic"] = true if @dynamic

data/lib/parsanol/parg/document.rb

@@ -9,7 +9,7 @@ module Parsanol
 
       Test = Struct.new(:entry, :kind, :input, :expect)
 
-      attr_accessor :grammar_name, :version, :source, :uses
+      attr_accessor :grammar_name, :version, :source, :uses, :skip_declaration, :skip_source
       attr_reader :rules, :bindings, :preprocess, :entries, :docs, :tests, :render, :derive,
                   :customs, :states
       # Entries authored in this file (pre-import); nil-entry tests resolve

data/lib/parsanol/parg/lutaml.rb

@@ -1,5 +1,9 @@
 # frozen_string_literal: true
 
+# Standalone requires of this file bypass the parg autoload registry —
+# declare the namespace dependency so Artifact resolves.
+require "parsanol/parg"
+
 module Parsanol
   module PARG
     # lutaml-model integration: register a PARG artifact as a lutaml-model

data/lib/parsanol/parg/parser.rb

@@ -20,7 +20,8 @@ module Parsanol
 
       STATE_KEYWORDS = %w[custom state].freeze
       KEYWORDS = %w[grammar as alt from_table column bindings render derive
-                    preprocess entry table_lookup custom state set switch].freeze
+                    preprocess entry table_lookup custom state set switch
+                    skip until].freeze
 
       def initialize(text)
         @tokens = Lexer.new(text).tokens
@@ -161,6 +162,19 @@ module Parsanol
             skip_newlines
             next
           end
+          if peek&.type == :ident && peek.value == "skip"
+            advance
+            punct("=")
+            if document.skip_declaration
+              raise ParseError, "duplicate skip declaration"
+            end
+
+            node = parse_choice
+            document.skip_declaration = validate_skip_node(node)
+            document.skip_source = skip_source_string(node)
+            skip_newlines
+            next
+          end
           if peek&.type == :ident && STATE_KEYWORDS.include?(peek.value)
             keyword = advance.value
             slot = ident.value
@@ -188,6 +202,30 @@ module Parsanol
         punct("}")
       end
 
+      # The skip declaration may only name rules (a rule reference or an
+      # alternation of rule references) — arbitrary expressions would
+      # make the injected trivia unanalyzable.
+      def skip_source_string(node)
+        case node.kind
+        when :ref then node.a
+        when :alt then node.a.map { |b| skip_source_string(b) }.join(" / ")
+        end
+      end
+
+      def validate_skip_node(node)
+        case node.kind
+        when :ref
+          node
+        when :alt
+          node.a.each { |branch| validate_skip_node(branch) }
+          node
+        else
+          raise ParseError,
+                "skip must be a rule reference or an alternation of rule " \
+                "references, got #{node.kind.inspect}"
+        end
+      end
+
       def rule_name
         token = ident
         if KEYWORDS.include?(token.value)
@@ -325,6 +363,10 @@ module Parsanol
           end
           punct("}")
           Node.new(:switch, slot, arms)
+        when "until"
+          # QoL: any-until terminal — (*( !terminal any_char )), the
+          # idiomatic "consume everything up to the delimiter" run.
+          Node.new(:until, unquote(expect(:str).value))
         when "alt"
           ident("from_table")
           table = unquote(expect(:str).value)

data/lib/parsanol/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Parsanol
-  VERSION = "1.3.65"
+  VERSION = "1.3.67"
 end

data/lib/parsanol/vm.rb

@@ -175,8 +175,10 @@ module Parsanol
         compiler.to_program
       end
 
+      # Returns violation descriptions for this atom's ordered-choice
+      # branches (empty-matchable non-last branches shadow later ones).
       def validate_choice_branches(atom)
-        atom.alternatives.each_with_index do |branch, idx|
+        atom.alternatives.each_with_index.filter_map do |branch, idx|
           walk_branch = branch
           next if idx == atom.alternatives.size - 1
 
@@ -194,14 +196,13 @@ module Parsanol
           next if walk_branch.nil?
           next unless Compiler.new.nullable?(walk_branch)
 
-          raise Parsanol::GrammarError,
-                "wrong grammar: alternative branch #{idx} " \
-                "(#{branch.inspect}) can match empty input and is not " \
-                "the last branch — ordered choice commits to its empty " \
-                "match, so #{atom.alternatives.size - idx - 1} later " \
-                "branch(es) can never match at positions where it " \
-                "matches empty. Reorder branches so empty-matchable " \
-                "ones come last, or make the branch non-empty."
+          "alternative branch #{idx} (#{branch.inspect}) can match " \
+            "empty input and is not the last branch — ordered choice " \
+            "commits to its empty match, so " \
+            "#{atom.alternatives.size - idx - 1} later branch(es) can " \
+            "never match at positions where it matches empty. Reorder " \
+            "branches so empty-matchable ones come last, or make the " \
+            "branch non-empty."
         end
       end
 
@@ -211,6 +212,7 @@ module Parsanol
       # the grammar is wrong, and inputs it would mis-parse fail with a
       # baffling cause. Raise before any input is seen.
       def validate_alternatives(root)
+        violations = []
         visited = {}.compare_by_identity
         walk = lambda do |atom, depth|
           next if atom.nil? || depth > 20
@@ -219,26 +221,26 @@ module Parsanol
           visited[atom] = true
           case atom
           when Parsanol::Atoms::Alternative
-            validate_choice_branches(atom)
+            violations.concat(validate_choice_branches(atom))
             atom.alternatives.each { |c| walk.call(c, depth + 1) }
           when Parsanol::Atoms::Sequence
             atom.parslets.each { |c| walk.call(c, depth + 1) }
           when Parsanol::Atoms::Repetition, Parsanol::Atoms::Named
             walk.call(atom.parslet, depth + 1)
-          when Parsanol::Atoms::Lookahead
-            walk.call(atom.bound_parslet, depth + 1)
-          when Parsanol::Atoms::Ignored
-            walk.call(atom.wrapped_atom, depth + 1)
           when Parsanol::Atoms::Entity
+            # Rule bodies hide behind the entity wrapper; without this
+            # descent the lint passes DSL grammars that fail the same
+            # grammar compiled from PARG (where bodies are inlined).
+            # Entities that cannot resolve outside a parse are skipped.
             begin
               walk.call(atom.parslet, depth + 1)
-            rescue Parsanol::GrammarError
-              raise
             rescue StandardError
-              # Lazily resolved / recursive entity that cannot resolve
-              # outside a parse: skip this branch of the walk.
               nil
             end
+          when Parsanol::Atoms::Lookahead
+            walk.call(atom.bound_parslet, depth + 1)
+          when Parsanol::Atoms::Ignored
+            walk.call(atom.wrapped_atom, depth + 1)
           when Parsanol::Atoms::Scope
             begin
               walk.call(atom.block.call, depth + 1)
@@ -248,6 +250,11 @@ module Parsanol
           end
         end
         walk.call(root, 0)
+        return if violations.empty?
+
+        raise Parsanol::GrammarError,
+              "wrong grammar: #{violations.size} shadowed-alternative " \
+              "violation(s):\n  - #{violations.join('\n  - ')}"
       end
 
       # Executes a compiled program. Returns [true, value] on success;

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: parsanol
 version: !ruby/object:Gem::Version
-  version: 1.3.65
+  version: 1.3.67
 platform: x86_64-linux
 authors:
 - Ribose Inc.