parsanol 1.3.54 → 1.3.57

data/lib/parsanol/parg/artifact.rb

@@ -0,0 +1,213 @@
+# frozen_string_literal: true
+
+require "json"
+require "yaml"
+
+module Parsanol
+  module PARG
+    # A compiled artifact envelope: load, verify checksum, parse inputs.
+    #
+    # The envelope's grammar section is the portable JSON the native engines
+    # consume; the embedded PARG source lets a Ruby runtime recompile the atom
+    # tree locally without any other dependency.
+    class Artifact
+      attr_reader :envelope, :path, :tables_dir
+
+      def self.load(path, tables_dir: nil)
+        envelope = JSON.parse(File.read(path))
+        new(envelope, path, tables_dir || File.dirname(path))
+      end
+
+      def self.from_json(text, tables_dir: nil)
+        new(JSON.parse(text), nil, tables_dir)
+      end
+
+      SUPPORTED_SHAPE = "parsanol-tree/v2"
+
+      def initialize(envelope, path, tables_dir)
+        unless envelope["shape"] == SUPPORTED_SHAPE
+          raise ArtifactError,
+                "unsupported artifact shape #{envelope['shape'].inspect} " \
+                "(this runtime implements #{SUPPORTED_SHAPE.inspect})"
+        end
+        @envelope = envelope
+        @path = path
+        @tables_dir = tables_dir
+        verify_checksum!
+        @compiler = nil
+      end
+
+      def version = envelope["version"]
+
+      def entries = envelope["entries"].keys
+
+      def entry(name)
+        envelope["entries"].fetch(name) do
+          raise ArtifactError, "unknown entry #{name.inspect}"
+        end
+      end
+
+      # Parse via the native engine: the envelope's grammar section is the
+      # exact portable JSON the Rust side registers, so the artifact parse
+      # path is a straight register-and-run — no Ruby recompilation. The
+      # Ruby atom runtime (mode: :ruby) remains available explicitly, and
+      # serves as the fallback on platforms without the extension.
+      def parse(entry_name, input, mode: :native)
+        case mode
+        when :native
+          if Native.available?
+            return Native.parse(JSON.generate(entry(entry_name).fetch("grammar")), input)
+          end
+
+          root_atom(entry_name).parse(input)
+        when :ruby
+          root_atom(entry_name).parse(input)
+        else
+          raise ArgumentError, "unknown mode #{mode.inspect} (use :native or :ruby)"
+        end
+      end
+
+      def apply_bindings(entry_name, shape)
+        Bindings.apply(self, entry(entry_name), shape)
+      end
+
+      # Render the identifier string from a bound attribute map (F6).
+      def render(_entry_name, bound, variant: "default")
+        Render.apply(envelope["render"] || {}, variant, bound)
+      end
+
+      def render_string(entry_name, input, variant: "default")
+        render(entry_name, parse_and_bind(entry_name, input), variant: variant)
+      end
+
+      # Evaluate a named derive spec against a bound map (F6).
+      def derive(name, bound)
+        Derive.apply(envelope["derive"] || {}, name, bound)
+      end
+
+      def derive_string(entry_name, input, name)
+        derive(name, parse_and_bind(entry_name, input))
+      end
+
+      def parse_and_bind(entry_name, input, mode: :native)
+        apply_bindings(entry_name, parse(entry_name, input, mode: mode))
+      end
+
+      # Re-run the grammar's in-file tests against this artifact. Returns
+      # the failure descriptions; empty means every test passes.
+      def run_tests(mode: :native)
+        run_test_list(envelope.fetch("tests", []), mode: mode)
+      end
+
+      # Run an external test list (suite files yield Document::Test
+      # structs; embedded tests are artifact-shaped hashes).
+      def run_test_list(tests, mode: :native)
+        tests.filter_map do |test|
+          hash = test.is_a?(Document::Test) ? test_to_hash(test) : test
+          entry_name = hash["entry"] || entries.first
+          begin
+            bound = parse_and_bind(entry_name, hash["input"], mode: mode)
+            if hash["kind"] == "reject"
+              "test #{hash['input'].inspect}: expected the input to be rejected"
+            elsif hash["kind"] == "example"
+              expect = hash["expect"].to_h { |key, value| [key.to_sym, value] }
+              mismatched = expect.reject { |key, value| bound.key?(key) && bound[key] == value }
+              next if mismatched.empty?
+
+              "test #{hash['input'].inspect}: expected captures " \
+                "#{mismatched.transform_values(&:inspect).inspect}, got #{bound.inspect}"
+            end
+          rescue Parsanol::ParseFailed
+            unless hash["kind"] == "reject"
+              "test #{hash['input'].inspect}: expected the input to parse"
+            end
+          end
+        end
+      end
+
+      def test_to_hash(test)
+        {
+          "entry" => test.entry,
+          "kind" => test.kind.to_s,
+          "input" => test.input,
+          "expect" => test.expect,
+        }
+      end
+
+      # Rule documentation embedded from ## doc comments.
+      def rule_docs = envelope.fetch("docs", {})
+
+      # Structured parse diagnostics (PN 2): {offset, message} for the
+      # deepest failure of the most recent parse attempt on this entry.
+      def parse_with_diagnostics(entry_name, input, mode: :native)
+        shape = parse(entry_name, input, mode: mode)
+        { "ok" => true, "offset" => nil, "message" => nil, "shape" => shape }
+      rescue Parsanol::ParseFailed => e
+        cause = deepest_cause(e.parse_failure_cause)
+        { "ok" => false, "offset" => cause&.position,
+          "message" => cause&.message || e.message, "shape" => nil }
+      end
+
+      def deepest_cause(cause)
+        return cause if cause.nil? || cause.children.empty?
+
+        deepest = cause.children.filter_map { |child| deepest_cause(child) }
+          .max_by { |node| node.position.to_i }
+        (deepest&.position.to_i >= cause.position.to_i ? deepest : cause)
+      end
+
+      def table_rows(name)
+        @table_cache ||= {}
+        return @table_cache[name] if @table_cache.key?(name)
+
+        declared = envelope["tables"].fetch(name) do
+          raise ArtifactError, "artifact does not declare table #{name.inspect}"
+        end
+        if declared.is_a?(Hash) && declared.key?("rows")
+          return @table_cache[name] = declared["rows"]
+        end
+
+        file = declared
+        path = File.join(@tables_dir.to_s, file)
+        raw = path.end_with?(".json") ? JSON.parse(File.read(path)) : YAML.safe_load_file(path, aliases: true)
+        rows = case raw
+               when Hash
+                 raw.map { |key, value| { "name" => key.to_s }.merge(value.to_h.transform_keys(&:to_s)) }
+               when Array then raw.map { |row| row.to_h.transform_keys(&:to_s) }
+               else raise ArtifactError, "table #{name.inspect} must be a map or array"
+               end
+        @table_cache[name] = rows
+      end
+
+      def compiler
+        @compiler ||= begin
+          document = Parser.new(envelope.fetch("source")).parse
+          # `use`-importing grammars carry dotted cross-grammar rule
+          # references that only resolve once the imported sources merge;
+          # without this the pure-Ruby path fails on artifacts whose
+          # native atom graphs inline everything. The imported sources
+          # sit beside the artifact (vendors ship them together).
+          imports_dir = path && File.directory?(File.dirname(path)) ? File.dirname(path) : nil
+          Imports.merge!(document, [imports_dir].compact) if imports_dir
+          Compiler.new(document, @tables_dir)
+        end
+      end
+
+      def root_atom(entry_name)
+        rule = entry(entry_name).fetch("root")
+        @compiled ||= {}
+        @compiled[rule] ||= compiler.atom_for(rule)
+      end
+
+      def verify_checksum!
+        stored = envelope["checksum"]
+        computed = Compiler.checksum(envelope)
+        return if stored == computed
+
+        raise ArtifactError,
+              "artifact checksum mismatch: stored #{stored.inspect}, " \
+              "computed #{computed.inspect}"
+      end
+    end
+  end
+end

data/lib/parsanol/parg/authoring.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Parsanol
+  module PARG
+    # External test suites: *.pgtest files carrying the same test syntax
+    # as the inline test section (accept / reject / example). Suites are
+    # development-time corpora — they iterate a grammar against its
+    # binding requirements without being embedded in the artifact.
+    #
+    #   suite smoke for entry identifier {
+    #     accept "ISO 8601-1:2019"
+    #     example "ISO/CD 12345" { stage: "draft20" }
+    #   }
+    module Suite
+      module_function
+
+      # Load every *.pgtest file in a directory.
+      # Returns { suite_name => [Document::Test, ...] }.
+      def load(dir)
+        Dir.glob(File.join(dir, "*.pgtest")).to_h do |file|
+          [File.basename(file, ".pgtest"), read_file(file)]
+        end
+      end
+
+      def read_file(file)
+        content = File.read(file).gsub(/^#[^\n]*\n/, "")
+        header = content.match(/suite\s+\w+(?:\s+for\s+entry\s+(?<entry>\w+))?\s*\{/)
+        unless header
+          raise ParseError,
+                "#{file}: expected 'suite <name> [for <entry>] { ... }'"
+        end
+
+        body = content[header.end(0)..].sub(/\}\s*\z/, "")
+        document = Parser.new("grammar Suite version \"0.0.0\" {\n}\ntest {\n#{body}\n}\n").parse
+        document.tests.map do |test|
+          Document::Test.new(header[:entry] || test.entry, test.kind, test.input, test.expect)
+        end
+      end
+    end
+
+    # The binding-requirements schema: what a language-specific data model
+    # must implement for an entry. Derived from the declared bindings plus
+    # the observed example values.
+    module Schema
+      module_function
+
+      def from_artifact(artifact)
+        artifact.entries.to_h do |entry|
+          entry_data = artifact.entry(entry)
+          fields = entry_data["bindings"].to_h do |binding|
+            requirements = {
+              "type" => binding["type"],
+              "card" => binding["card"] || "1",
+            }
+            requirements["preprocess"] = binding["preprocess"] if binding["preprocess"]
+            [binding["path"], requirements]
+          end
+          examples = artifact.envelope.fetch("tests", [])
+            .select { |test| (test["kind"] == "example" && test["entry"].nil?) || test["entry"] == entry }
+            .map { |test| { "input" => test["input"], "captures" => test["expect"] } }
+          [entry, { "root" => entry_data["root"], "fields" => fields, "examples" => examples }]
+        end
+      end
+
+      # TypeScript materialization types: one interface per entry, one
+      # property per bound field. Cards ending in * become arrays;
+      # 0..1 becomes nullable.
+      def to_typescript(schema)
+        lines = ["// Generated by parsanol parg schema --ts — binding-requirements types."]
+        schema.each do |entry, data|
+          lines << ""
+          lines << "export interface #{ts_name(entry)} {"
+          data["fields"].each do |path, requirements|
+            lines << "  #{ts_prop(path)}: #{ts_type(requirements)};"
+          end
+          lines << "}"
+        end
+        "#{lines.join("\n")}\n"
+      end
+
+      def ts_name(entry)
+        entry.split(/[_.]/).map(&:capitalize).join
+      end
+
+      def ts_prop(path)
+        leaf = path.include?("[]") ? path.split("].").last : path
+        leaf.gsub(/([A-Za-z0-9])_([a-z])/) do
+          "#{Regexp.last_match(1)}#{Regexp.last_match(2).upcase}"
+        end
+      end
+
+      def ts_type(requirements)
+        base = case requirements["type"]
+               when "integer", "float" then "number"
+               when "boolean" then "boolean"
+               else "string"
+               end
+        if requirements["card"].to_s.end_with?("*")
+          "#{base}[]"
+        elsif requirements["card"] == "0..1"
+          "#{base} | null"
+        else
+          base
+        end
+      end
+    end
+  end
+end

data/lib/parsanol/parg/bindings.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+module Parsanol
+  module PARG
+    # Applies an entry's artifact bindings to a parsanol-shape parse tree,
+    # producing the attribute hash that fills a lutaml-model class:
+    # capture collection -> preprocessing -> type cast -> path assignment.
+    module Bindings
+      module_function
+
+      def apply(artifact, entry, shape)
+        bindings = entry["bindings"] || []
+        captures = collect_captures(shape)
+        out = {}
+        scalar, arrays = bindings.partition { |binding| !binding["path"].to_s.include?("[]") }
+        scalar.each do |binding|
+          validate_path!(binding)
+          assign_scalar(out, artifact, binding, captures)
+        end
+        arrays.group_by { |binding| binding["path"].split("[]").first }.each do |prefix, group|
+          assign_array(out, artifact, prefix, group, captures)
+        end
+        out
+      end
+
+      def collect_captures(shape)
+        found = Hash.new { |hash, key| hash[key] = [] }
+        queue = [shape]
+        until queue.empty?
+          node = queue.shift
+          case node
+          when Hash
+            node.each do |key, value|
+              found[key.to_sym] << value
+              queue << value
+            end
+          when Array then queue.concat(node)
+          end
+        end
+        found
+      end
+
+      def assign_scalar(out, artifact, binding, captures)
+        value = captures[binding["capture"].to_sym].first
+        return if value.nil?
+
+        out[leaf_key(binding)] = finalize(artifact, binding, value)
+      end
+
+      def assign_array(out, artifact, prefix, group, captures)
+        count = group.map { |binding| captures[binding["capture"].to_sym].length }.max || 0
+        return if count.zero?
+
+        list = (out[prefix] ||= [])
+        count.times do |index|
+          element = {}
+          group.each do |binding|
+            value = captures[binding["capture"].to_sym][index]
+            next if value.nil?
+
+            element[leaf_key(binding)] = finalize(artifact, binding, value)
+          end
+          list << element
+        end
+      end
+
+      def validate_path!(binding)
+        path = binding["path"].to_s
+        return unless path.include?(".") || path.include?("[")
+
+        raise ArtifactError,
+              "binding #{binding['capture'].inspect}: nested path " \
+              "#{path.inspect} is not supported; bind the components instead"
+      end
+
+      def leaf_key(binding)
+        path = binding["path"].to_s
+        leaf = path.include?("[]") ? path.split("].").last : path
+        leaf.to_sym
+      end
+
+      def finalize(artifact, binding, value)
+        cast(preprocess(artifact, binding, value), binding["type"])
+      end
+
+      def preprocess(artifact, binding, value)
+        return value if binding["preprocess"].nil?
+
+        steps = artifact.envelope["preprocess"].fetch(binding["preprocess"]) do
+          raise ArtifactError,
+                "preprocess step #{binding['preprocess'].inspect} not declared"
+        end
+        steps.each do |step|
+          case step["op"]
+          when "table_lookup"
+            map = artifact.table_rows(step["table"]).to_h do |row|
+              [row[step["from"]].to_s, row[step["to"]]]
+            end
+            value = map[value.to_s] || value
+          else
+            raise ArtifactError, "unknown preprocess op #{step['op'].inspect}"
+          end
+        end
+        value
+      end
+
+      def cast(value, type)
+        case type
+        when "integer" then Integer(value)
+        when "float" then Float(value)
+        when "string"
+          # Parsanol::Slice (Ruby atom runtime) and native leaf objects are
+          # the shape contract's capture leaves; their to_s is the text.
+          scalar = value.nil? || value.is_a?(String) || value.is_a?(Numeric) ||
+            value == true || value == false || value.is_a?(Parsanol::Slice)
+          unless scalar
+            raise ArtifactError,
+                  "cannot bind a #{value.class} capture to (string); " \
+                  "captures must be scalar"
+          end
+
+          value.to_s
+        when "boolean" then value == true || value.to_s.casecmp("true").zero?
+        else value
+        end
+      end
+    end
+  end
+end

data/lib/parsanol/parg/cli.rb

@@ -0,0 +1,251 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Parsanol
+  module PARG
+    # Command-line tool for authoring and testing PARG grammars without any
+    # language binding: compile, run the in-file tests, parse inputs
+    # interactively, and read the rule documentation.
+    #
+    #   parsanol parg compile grammars/iso.parg -o iso.json
+    #   parsanol parg test grammars/iso.parg
+    #   parsanol parg parse grammars/iso.parg "ISO 8601-1:2019"
+    #   parsanol parg repl iso.json
+    #   parsanol parg doc grammars/iso.parg publisher
+    class CLI
+      USAGE = <<~TEXT
+        usage: parsanol parg <command> <grammar> [arguments]
+
+        commands:
+          compile <file.parg> [-o out.json]   compile to an artifact envelope
+          test    <file.parg|artifact.json>   run the grammar's test section
+          parse   <file> [--entry E] INPUT  parse one input; prints the tree
+                                            and the bound captures
+          repl    <file> [--entry E]        interactive parse loop (blank
+                                            line quits, :quit exits)
+          doc     <file> [rule]             print rule documentation
+
+        options:
+          --tables DIR   directory for alt-from-table data (default:
+                         the grammar file's directory)
+      TEXT
+
+      def self.run(argv)
+        new(argv).run
+      end
+
+      def initialize(argv)
+        @argv = argv.dup
+        @tables_dir = nil
+      end
+
+      def run
+        if (index = @argv.index("--tables"))
+          @tables_dir = @argv.delete_at(index + 1)
+          @argv.delete_at(index)
+        end
+        @json = !!@argv.delete("--json")
+        command = @argv.shift
+        case command
+        when "compile" then compile
+        when "test" then test
+        when "schema" then schema
+        when "parse" then parse
+        when "repl" then repl
+        when "doc" then doc
+        when "lsp" then Lsp.new.run && 0
+        else
+          warn USAGE
+          1
+        end
+      rescue Parsanol::PARG::Error, Parsanol::ParseFailed => e
+        warn "#{e.class}: #{e.message}"
+        1
+      end
+
+      private
+
+      # parsanol-tree/v2 JSON: leaves become {value, line, column, offset,
+      # length}; Ruby atom-runtime leaves are Parsanol::Slice (to_s is the
+      # captured text, positions unknown at this layer).
+      def json_shape(node)
+        case node
+        when Hash then node.to_h { |k, v| [k.to_s, json_shape(v)] }
+        when Array then node.map { |item| json_shape(item) }
+        when Parsanol::Slice then { "value" => node.to_s }
+        else node
+        end
+      end
+
+      def compile
+        source = @argv.shift
+        out = flag_value("-o") || default_artifact_name(source)
+        envelope = compile_source(source).envelope
+        File.write(out, "#{JSON.generate(envelope)}\n")
+        puts "#{out}  #{envelope['checksum']}  warnings=#{envelope['lint']['order_warnings'].size}"
+        0
+      end
+
+      def test
+        target = @argv.shift
+        if File.directory?(target)
+          return batch(target) { |file| artifact_for(file).run_tests }
+        end
+
+        artifact = artifact_for(target)
+        failures = artifact.run_tests
+        suite_dir = flag_value("--suite")
+        suites = suite_dir ? Parsanol::PARG::Suite.load(suite_dir) : {}
+        suites.each_value do |tests|
+          failures.concat(artifact.run_test_list(tests))
+        end
+        if failures.empty?
+          if @json
+            puts JSON.generate({ "ok" => true, "failures" => [] })
+          else
+            puts suites.empty? ? "all tests pass" : "all tests pass (#{suites.keys.join(', ')})"
+          end
+          0
+        else
+          if @json
+            puts JSON.generate({ "ok" => false, "failures" => failures })
+          else
+            failures.each { |failure| warn failure }
+          end
+          1
+        end
+      end
+
+      def schema
+        ts = @argv.delete("--ts")
+        artifact = artifact_for(@argv.shift)
+        schema = Parsanol::PARG::Schema.from_artifact(artifact)
+        if ts
+          puts Parsanol::PARG::Schema.to_typescript(schema)
+        else
+          puts JSON.pretty_generate(schema)
+        end
+        0
+      end
+
+      def parse
+        file = @argv.shift
+        entry = flag_value("--entry")
+        input = @argv.join(" ")
+        if input.empty?
+          warn "usage: parsanol parg parse <file> [--entry E] INPUT"
+          return 1
+        end
+        artifact = artifact_for(file)
+        entry_name = entry || artifact.entries.first
+        shape = artifact.parse(entry_name, input)
+        if @json
+          puts JSON.generate({
+                               "entry" => entry_name,
+                               "shape" => json_shape(shape),
+                               "bound" => artifact.apply_bindings(entry_name, shape),
+                             })
+        else
+          puts "tree: #{shape.inspect}"
+          puts "captures: #{artifact.apply_bindings(entry_name, shape).inspect}"
+        end
+        0
+      end
+
+      def repl
+        file = @argv.shift
+        entry = flag_value("--entry")
+        artifact = artifact_for(file)
+        entry_name = entry || artifact.entries.first
+        puts "parsanol parg repl — #{file} entry #{entry_name.inspect}; blank line quits"
+        loop do
+          print "> "
+          input = $stdin.gets
+          break if input.nil? || input.strip.empty?
+
+          begin
+            shape = artifact.parse(entry_name, input.rstrip)
+            puts "tree: #{shape.inspect}"
+            puts "captures: #{artifact.apply_bindings(entry_name, shape).inspect}"
+          rescue Parsanol::ParseFailed => e
+            puts "parse failed: #{e.message[0, 160]}"
+          end
+        end
+        0
+      end
+
+      def doc
+        file = @argv.shift
+        rule = @argv.shift
+        docs = artifact_for(file).rule_docs
+        selected = rule ? { rule => docs[rule] } : docs
+        if selected.values.all?(&:nil?)
+          warn "no documentation#{" for #{rule.inspect}" if rule}"
+          return 1
+        end
+        selected.each do |name, text|
+          puts "#{name}:"
+          text.to_s.split("\n").each { |line| puts "  #{line}" }
+        end
+        0
+      end
+
+      def for_source_or_artifact
+        file = @argv.shift
+        artifact = artifact_for(file)
+        yield artifact
+      end
+
+      def artifact_for(file)
+        if file.end_with?(".json")
+          Parsanol::PARG::Artifact.load(file, tables_dir: @tables_dir || default_tables_dir(file))
+        else
+          artifact = compile_source(file)
+          Parsanol::PARG::Artifact.from_json(JSON.generate(artifact.envelope),
+                                             tables_dir: @tables_dir || default_tables_dir(file))
+        end
+      end
+
+      def compile_source(file)
+        tables = @tables_dir || default_tables_dir(file)
+        document = Parsanol::PARG::Parser.new(File.read(file)).parse
+        Parsanol::PARG::Imports.merge!(document, [File.dirname(file)])
+        Parsanol::PARG::Compiler.compile(document, tables_dir: tables)
+      end
+
+      def default_artifact_name(source)
+        "#{File.basename(source, '.parg')}.artifact.json"
+      end
+
+      # Batch mode: run a command over every *.parg in a directory.
+      def batch(target)
+        failures = {}
+        Dir.glob(File.join(target, "*.parg")).each do |file|
+          result = yield artifact_for(file)
+          failures[file] = result unless result.empty?
+        end
+        if failures.empty?
+          puts "all flavors pass"
+          0
+        else
+          failures.each { |file, list| list.each { |f| warn "#{file}: #{f}" } }
+          1
+        end
+      end
+
+      # Tables conventionally live in a "tables" directory beside the
+      # grammar (as a sibling of the grammars directory).
+      def default_tables_dir(file)
+        dir = File.dirname(file)
+        candidate = File.join(dir, "tables")
+        candidate = File.join(File.dirname(dir), "tables") unless File.directory?(candidate)
+        File.directory?(candidate) ? candidate : dir
+      end
+
+      def flag_value(flag)
+        @argv.delete_at(@argv.index(flag) + 1).tap { @argv.delete(flag) } if @argv.include?(flag)
+      end
+    end
+  end
+end