parsanol 1.3.108 → 1.3.112

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0c753a3a41a099b3306b3b18861cd2236b4b7a122c032cbf2a3815bc2777dd9e
-  data.tar.gz: 0b7fdf682dfa6150ea51cc30d36892bb5657e0bb6c7a833a09a810a146aa5845
+  metadata.gz: 683289b7d0453a88c2dfb691d54479ebacf6be93cc30f527c4199557bb99b48d
+  data.tar.gz: '0182a60c9bca2c3c1fcf1f450fed4623aeb915fb3b3e80f3a14ee716c09f3ffd'
 SHA512:
-  metadata.gz: a94d7bcebe6d9780adc68d3005beccb194b78a6e48b71c3a5479c6dc37f9fa38a9efcf45ec979bb69c6fa521618b5f733909cb28e4eefc7f6baf95a457c1c52e
-  data.tar.gz: 677c550497c09949077db2c70083e09b72e0b3f9adf57dab4ba68bb69e05ad3de81bc8914e40509326bed4ca4f8fc22dab00c85a0555d52072e86a04f7588683
+  metadata.gz: 15607095c23776f36bf34dcf4e53849a739cd373e9a31a766c484331fd998a01b4d66faebd528fb61077d648e58e7e35173e6ec7e3280640e1b2eaee17a884cc
+  data.tar.gz: 8fc517dc729088dcac71596911ed5a57a8b72c580d246b315a1b8313cb0157ef7ee63b04b20d723583f03ac4699af4810ca1cb61e5c91d9aed1514fe68806a27

data/Cargo.lock

@@ -307,9 +307,9 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
 
 [[package]]
 name = "parsanol"
-version = "0.18.19"
+version = "0.19.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30daf4ef27d78639cb883ea9d71ec990e7fcfc7ffd295c3651098fe2bcdf70cd"
+checksum = "8ffe7e92d626c65e83d2c53c148a18db5897879858bcd37ebd0f406f5a251036"
 dependencies = [
  "ahash",
  "getrandom 0.3.4",
@@ -326,9 +326,9 @@ dependencies = [
 
 [[package]]
 name = "parsanol-derive"
-version = "0.18.19"
+version = "0.19.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f2c09d80a3a6c4efe1f756642877f73cc830e25f2011f101fc9b8e8ec1f33cd"
+checksum = "3b94bb50df3fe1a9f66b6d8ee784672c27262e9e609568d5e9d14e1f7d6bec91"
 dependencies = [
  "proc-macro2",
  "quote",

data/ext/parsanol_cdylib/Cargo.toml

@@ -13,4 +13,4 @@ doc = false
 
 [dependencies]
 # 0.17.0 carries LookSpec + the widened opaque writeback
-parsanol = { version = "0.18.0" }
+parsanol = { version = "0.19.1" }

data/ext/parsanol_native/Cargo.toml

@@ -32,7 +32,7 @@ magnus = { version = "0.9" }
 # parsanol parser library (crates.io; 0.17.0 carries the LookSpec regex
 # lookbehind (rs#163 gap 1) and the widened opaque writeback: the
 # lazy-continuation fix, parsanol-ruby#160)
-parsanol = { version = "0.18.0", features = ["ruby"] }
+parsanol = { version = "0.19.1", features = ["ruby"] }
 
 # Logging
 log = "0.4"

data/lib/parsanol/atoms/repetition.rb

@@ -195,8 +195,19 @@ module Parsanol
 
           unless success
             last_failure = value
+            if ENV["PARSANOL_REP_TRACE"]
+              warn "REP body FAIL at iter=#{occurrence} start=#{iteration_start} pos_after=#{source.bytepos} restoring"
+            end
+            # A failed iteration may have consumed input before its own
+            # inner failure (an ordered choice's partial alternative);
+            # the repetition owns the rollback — the rep ends at the
+            # last SUCCESSFUL iteration's end (ruby#357).
+            source.bytepos = iteration_start
             break
           end
+          if ENV["PARSANOL_REP_TRACE"]
+            warn "REP body OK at iter=#{occurrence} start=#{iteration_start} end=#{source.bytepos}"
+          end
 
           occurrence += 1
           result[occurrence] = value
@@ -276,8 +287,19 @@ module Parsanol
 
           unless success
             last_failure = value
+            if ENV["PARSANOL_REP_TRACE"]
+              warn "REP body FAIL at iter=#{occurrence} start=#{iteration_start} pos_after=#{source.bytepos} restoring"
+            end
+            # A failed iteration may have consumed input before its own
+            # inner failure (an ordered choice's partial alternative);
+            # the repetition owns the rollback — the rep ends at the
+            # last SUCCESSFUL iteration's end (ruby#357).
+            source.bytepos = iteration_start
             break
           end
+          if ENV["PARSANOL_REP_TRACE"]
+            warn "REP body OK at iter=#{occurrence} start=#{iteration_start} end=#{source.bytepos}"
+          end
 
           occurrence += 1
           result[occurrence] = value

data/lib/parsanol/parg/artifact.rb

@@ -26,6 +26,10 @@ module Parsanol
 
       SUPPORTED_SHAPE = "parsanol-tree/v2"
 
+      # The editions this runtime parses (ruby#357's consolidation):
+      # v1 = the legacy shapes; v2 = named trees + explicit spans.
+      SUPPORTED_EDITIONS = %w[v1 v2].freeze
+
       def initialize(envelope, path, tables_dir, imports_dir: nil)
         unless envelope["shape"] == SUPPORTED_SHAPE
           raise ArtifactError,
@@ -38,6 +42,18 @@ module Parsanol
         bindings = envelope["bindings"]
         validate_bindings!(bindings) unless bindings.nil?
         @envelope = envelope
+        # ruby#353: absent field = the pre-1.3.109 semantics. A level
+        # above this runtime's is a hard error (the trees would carry
+        # nodes this engine never emits); a lower one warns — parsing
+        # works, but frozen tree comparisons will differ.
+        # The declared edition must be one this runtime understands
+        # (ruby#357's consolidation: one axis, `edition`, values v1/v2).
+        edition = envelope["edition"] || "v1"
+        unless SUPPORTED_EDITIONS.include?(edition)
+          raise ArtifactError,
+                "artifact declares edition #{edition.inspect}; this " \
+                "runtime supports #{SUPPORTED_EDITIONS.map(&:inspect).join(', ')}"
+        end
         @path = path
         @tables_dir = tables_dir
         # parsanol-ruby#208: `use`-importing grammars need the imported
@@ -233,13 +249,27 @@ module Parsanol
         # initial parse's (the native FFI's coarse failure position
         # loses the deepest offset).
         captures, reparse_error = recovery_journal(entry_name, input, mode)
-        partial = recovery_partial(entry_name)
+        # The spine boundary, not the failure offset, is where the
+        # resync scan starts (rs#304 convergence): junk frequently lies
+        # BEFORE the deepest failure (the failed iteration consumed its
+        # own prefix), and the scan from the failure offset can never
+        # match it. When no iteration matched, the boundary is 0 —
+        # recovery still runs (rs#304 Gap B: a declared pattern is
+        # usable from the start).
+        stash = Parsanol::Recovery.take_partial
+        partial = recovery_partial(entry_name, stash)
+        boundary = stash&.first || 0
         resyncs = []
-        if partial && reparse_error
-          items = partial.values.first
-          extended, resyncs = resync_extension(entry_name, input,
-                                               reparse_error["offset"])
-          items.concat(extended) if extended
+        if reparse_error
+          root_name = entry(entry_name)["root"].to_sym
+          items = partial&.values&.first
+          created_partial = items.nil?
+          items ||= []
+          extended, resyncs = resync_extension(entry_name, input, boundary)
+          items.concat(extended)
+          # Recovered iterations with no prior partial form the partial
+          # (the rs engine's stash exists from offset 0).
+          partial = { root_name => items } if created_partial && !items.empty?
         end
         { "ok" => false,
           "error" => reparse_error || recovery_error(e),
@@ -367,8 +397,7 @@ module Parsanol
       # R1.5 (TODO.sota/1): the deepest repetition prefix the failed
       # journaling pass matched, shaped like the root (the item array
       # under the root rule's name). Nil when nothing matched.
-      def recovery_partial(entry_name)
-        stash = Parsanol::Recovery.take_partial
+      def recovery_partial(entry_name, stash = Parsanol::Recovery.take_partial)
         return nil if stash.nil?
 
         _end_pos, items, _tag = stash

data/lib/parsanol/parg/authoring.rb

@@ -64,22 +64,213 @@ module Parsanol
         end
       end
 
+      # The bound model's shape: a nested type tree derived from the
+      # binding paths (ruby#308 v2). Dot segments nest objects, a
+      # `[]` segment is an array of objects (or of leaf values when
+      # the array is itself the leaf), and the leaf carries the
+      # binding's type and card (`0..1` = nullable, `*` cards arrive
+      # through the array node itself).
+      #
+      #   shape({ "items[].n" => { "type" => "integer", "card" => "1" } })
+      #   # => { "items" => { "kind" => "array",
+      #   #                    "of" => { "n" => { "kind" => "leaf", "type" => "integer", "card" => "1" } } } }
+      def shape(fields)
+        root = {}
+        fields.each do |path, requirements|
+          segments = path.split(".")
+          node = root
+          segments.each_with_index do |segment, index|
+            array = segment.end_with?("[]")
+            name = array ? segment.delete_suffix("[]") : segment
+            leaf = index == segments.length - 1
+            if array && leaf
+              node[name] = {
+                "kind" => "array",
+                "of" => { "kind" => "leaf", "type" => requirements["type"], "card" => requirements["card"] },
+              }
+              break
+            end
+            slot = (node[name] ||= array ? { "kind" => "array", "of" => {} } : {})
+            node = array ? slot["of"] : slot
+            next unless leaf
+
+            slot.update("kind" => "leaf", "type" => requirements["type"], "card" => requirements["card"])
+          end
+        end
+        root
+      end
+
       # TypeScript materialization types: one interface per entry, one
-      # property per bound field. Cards ending in * become arrays;
+      # interface per nested object node (v2: the interfaces mirror
+      # the bound model's shape). Cards ending in * become arrays;
       # 0..1 becomes nullable.
       def to_typescript(schema)
         lines = ["// Generated by parsanol parg schema --ts — binding-requirements types."]
         schema.each do |entry, data|
           lines << ""
-          lines << "export interface #{ts_name(entry)} {"
-          data["fields"].each do |path, requirements|
-            lines << "  #{ts_prop(path)}: #{ts_type(requirements)};"
+          emit_ts_interface(lines, ts_name(entry), shape(data["fields"]))
+        end
+        "#{lines.join("\n")}\n"
+      end
+
+      def emit_ts_interface(lines, name, node)
+        nested = []
+        lines << "export interface #{name} {"
+        node.each do |prop, child|
+          case child["kind"]
+          when "array"
+            element = child["of"]
+            inner = element["kind"] == "leaf" ? ts_leaf(element) : ts_ref_name(name, prop)
+            lines << "  #{ts_prop(prop)}: #{inner}[];"
+            nested << [ts_ref_name(name, prop), element] unless element["kind"] == "leaf"
+          when "leaf"
+            lines << "  #{ts_prop(prop)}: #{ts_leaf(child)};"
           end
-          lines << "}"
+        end
+        lines << "}"
+        lines << ""
+        nested.each { |inner, element| emit_ts_interface(lines, inner, element) }
+      end
+
+      def ts_leaf(leaf)
+        base = ts_base(leaf["type"])
+        if leaf["card"].to_s.end_with?("*")
+          "#{base}[]"
+        elsif leaf["card"] == "0..1"
+          "#{base} | null"
+        else
+          base
+        end
+      end
+
+      def ts_base(type)
+        case type
+        when "integer", "float" then "number"
+        when "boolean" then "boolean"
+        else "string"
+        end
+      end
+
+      def ts_ref_name(parent, prop)
+        "#{parent}#{ts_prop(prop).split('.').map(&:capitalize).join}"
+      end
+
+      # Ruby materialization models: lutaml-model classes (the
+      # established serialization framework — the generated file
+      # requires it; the consumer's bundle carries the gem).
+      def to_ruby(schema)
+        lines = [
+          "# frozen_string_literal: true",
+          "",
+          "# Generated by parsanol parg schema --ruby — lutaml-model binding types.",
+          "require \"lutaml/model\"",
+          "",
+        ]
+        schema.each do |entry, data|
+          emit_ruby_class(lines, ts_name(entry), shape(data["fields"]))
         end
         "#{lines.join("\n")}\n"
       end
 
+      def emit_ruby_class(lines, name, node)
+        nested = []
+        node.each do |prop, child|
+          next unless child["kind"] == "array" && child["of"]["kind"] != "leaf"
+
+          inner = lutaml_ref_name(name, prop)
+          nested << [inner, child["of"]]
+        end
+        # Ruby resolves the constant at class-body time: the nested
+        # models precede their parents.
+        nested.each { |inner, element| emit_ruby_class(lines, inner, element) }
+        lines << "class #{name} < Lutaml::Model::Serializable"
+        lines <<
+          node.each do |prop, child|
+            attr_name = ts_prop(prop)
+            case child["kind"]
+            when "array"
+              element = child["of"]
+              if element["kind"] == "leaf"
+                lines << "  attribute :#{attr_name}, #{lutaml_type(element)}, collection: true"
+              else
+                inner = lutaml_ref_name(name, prop)
+                lines << "  attribute :#{attr_name}, #{inner}, collection: true"
+                nested << [inner, element]
+              end
+            when "leaf"
+              # lutaml-model attributes are nilable unless validated:
+              # a 0..1 card is the plain attribute; presence-validated
+              # cards are the consumer's policy, not the schema's.
+              lines << "  attribute :#{attr_name}, #{lutaml_type(child)}"
+            end
+          end
+        lines << "end"
+        lines << ""
+      end
+
+      def lutaml_type(leaf)
+        case leaf["type"]
+        when "integer" then ":integer"
+        when "float" then ":float"
+        when "boolean" then ":boolean"
+        else ":string"
+        end
+      end
+
+      def lutaml_ref_name(parent, prop)
+        "#{parent}#{ts_prop(prop).split('.').map(&:capitalize).join}"
+      end
+
+      # Rust materialization structs (serde): one struct per nested
+      # object node, i64/f64/bool/String leaves, Vec for arrays,
+      # Option for 0..1.
+      def to_rust(schema)
+        lines = [
+          "// Generated by parsanol parg schema --rust — serde binding types.",
+          "",
+        ]
+        schema.each do |entry, data|
+          emit_rust_struct(lines, ts_name(entry), shape(data["fields"]))
+        end
+        "#{lines.join("\n")}\n"
+      end
+
+      def emit_rust_struct(lines, name, node)
+        nested = []
+        lines << "#[derive(Debug, serde::Serialize, serde::Deserialize)]"
+        lines << "pub struct #{name} {"
+        node.each do |prop, child|
+          field = ts_prop(prop)
+          snake = field.gsub(/([A-Z])/) { "_#{Regexp.last_match(1).downcase}" }
+          case child["kind"]
+          when "array"
+            element = child["of"]
+            inner = element["kind"] == "leaf" ? rust_leaf(element) : rust_ref_name(name, prop)
+            lines << "    pub #{snake}: Vec<#{inner}>,"
+            nested << [rust_ref_name(name, prop), element] unless element["kind"] == "leaf"
+          when "leaf"
+            base = rust_leaf(child)
+            lines << "    pub #{snake}: #{child['card'] == '0..1' ? "Option<#{base}>" : base},"
+          end
+        end
+        lines << "}"
+        lines << ""
+        nested.each { |inner, element| emit_rust_struct(lines, inner, element) }
+      end
+
+      def rust_leaf(leaf)
+        case leaf["type"]
+        when "integer" then "i64"
+        when "float" then "f64"
+        when "boolean" then "bool"
+        else "String"
+        end
+      end
+
+      def rust_ref_name(parent, prop)
+        ts_ref_name(parent, prop)
+      end
+
       def ts_name(entry)
         entry.split(/[_.]/).map(&:capitalize).join
       end

data/lib/parsanol/parg/cli.rb

@@ -29,6 +29,13 @@ module Parsanol
           repl    <file> [--entry E]        interactive parse loop (blank
                                             line quits, :quit exits)
           doc     <file> [rule]             print rule documentation
+          export  <file|artifact.json> --target xgrammar
+                                            lower the compiled artifact to a
+                                            decoder-facing grammar (ruby#321)
+          lint    <file> [--suggest-upgrades]
+                                            print lint errors (exit 1) and
+                                            warnings; the flag adds
+                                            machine-applicable upgrades
 
         options:
           --tables DIR   directory for alt-from-table data (default:
@@ -51,6 +58,7 @@ module Parsanol
           @argv.delete_at(index)
         end
         @json = !!@argv.delete("--json")
+        @suggest_upgrades = !!@argv.delete("--suggest-upgrades")
         command = @argv.shift
         case command
         when "compile" then compile
@@ -60,6 +68,9 @@ module Parsanol
         when "check" then check
         when "repl" then repl
         when "doc" then doc
+        when "lint" then lint
+        when "export" then export_grammar
+        when "diff" then diff_artifacts
         when "lsp" then Lsp.new.run && 0
         else
           warn USAGE
@@ -84,6 +95,53 @@ module Parsanol
         end
       end
 
+      # `parg export FILE --target xgrammar`: the decoder-facing
+      # lowering (ruby#321). Reads a .parg (compiling it) or an
+      # artifact json, prints the target grammar on stdout.
+      def export_grammar
+        source = @argv.shift
+        target = (flag_value("--target") || "xgrammar").to_sym
+        json = if source.end_with?(".json")
+                 File.read(source)
+               else
+                 JSON.generate(compile_source(source).envelope)
+               end
+        print Export.call(json, target: target)
+        0
+      end
+
+      # `parg lint FILE`: the lint suite on one grammar — errors
+      # first (exit 1), then every warning including the advisory
+      # strategies. --suggest-upgrades adds the machine-applicable
+      # upgrade suggestions on top.
+      def lint
+        source = @argv.shift
+        @captures_radar = @argv.delete("--captures") ? true : false
+        tables = @tables_dir || default_tables_dir(source)
+        document = Parsanol::PARG::Parser.new(PARG.read_utf8(source)).parse
+        Parsanol::PARG::Imports.merge!(document, [File.dirname(source)])
+        compiler = Parsanol::PARG::Compiler.new(document, tables)
+        document.rules.each_key { |rule| compiler.atom_for(rule) }
+
+        errors = Lints.errors(document, compiler)
+        errors.each { |error| puts "error: #{error}" }
+        return 1 unless errors.empty?
+
+        Lints.warnings(document, compiler).each { |warning| puts "warning: #{warning}" }
+        if @suggest_upgrades
+          Lints::CoreRuleDuplicates.new.warnings(document, compiler)
+            .each { |upgrade| puts "upgrade: #{upgrade}" }
+        end
+        if @captures_radar
+          # The migration radar (ruby#304): what modern mode would name.
+          compiler.modern_radar.each { |line| puts "captures: #{line}" }
+        end
+        0
+      rescue Parsanol::PARG::ParseError, Parsanol::PARG::Error => e
+        warn "#{e.class}: #{e.message}"
+        1
+      end
+
       def compile
         source = @argv.shift
         out = flag_value("-o") || default_artifact_name(source)
@@ -152,16 +210,66 @@ module Parsanol
 
       def schema
         ts = @argv.delete("--ts")
+        ruby = @argv.delete("--ruby")
+        rust = @argv.delete("--rust")
         artifact = artifact_for(@argv.shift)
         schema = Parsanol::PARG::Schema.from_artifact(artifact)
         if ts
           puts Parsanol::PARG::Schema.to_typescript(schema)
+        elsif ruby
+          puts Parsanol::PARG::Schema.to_ruby(schema)
+        elsif rust
+          puts Parsanol::PARG::Schema.to_rust(schema)
         else
           puts JSON.pretty_generate(schema)
         end
         0
       end
 
+      # parg diff OLD.artifact.json NEW.artifact.json [--json] [--suggest-bump] [--check LEVEL]
+      # Semantic classification of a grammar change (ruby#306): majors
+      # break consumers, minors extend them, patches preserve behavior.
+      def diff_artifacts
+        old_path = @argv.shift
+        new_path = @argv.shift
+        unless old_path && new_path
+          warn "usage: parsanol parg diff OLD.artifact.json NEW.artifact.json [--json] [--suggest-bump] [--check minor]"
+          return 2
+        end
+        suggest = !!@argv.delete("--suggest-bump")
+        check = @argv.delete("--check")
+        check_level = check && check.split("=", 2).last&.to_sym
+
+        old_env = JSON.parse(File.read(old_path))
+        new_env = JSON.parse(File.read(new_path))
+        findings = Diff.diff(old_env, new_env)
+        level = Diff.level(findings)
+
+        if @json
+          payload = {
+            level: level.to_s,
+            findings: findings.map { |f| { level: f.level.to_s, kind: f.kind.to_s, detail: f.detail } },
+          }
+          payload[:suggested] = Diff.suggest_bump(findings) if suggest
+          puts JSON.generate(payload)
+        else
+          if findings.empty?
+            puts "no semantic changes (#{level})"
+          else
+            findings.each { |f| puts "#{f.level.upcase} #{f.kind}: #{f.detail}" }
+            puts "level: #{level}"
+          end
+          puts Diff.suggest_bump(findings) if suggest
+        end
+
+        if check_level
+          ok = Diff::SEVERITY[level] >= Diff::SEVERITY[check_level]
+          warn "check: #{level} is #{ok ? 'at least' : 'BELOW'} #{check_level}" unless @json
+          return ok ? 0 : 3
+        end
+        0
+      end
+
       def parse
         file = @argv.shift
         entry = flag_value("--entry")