parsanol 1.3.96 → 1.3.98

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3361d31dc03ae64b82fbc5ac5429f28984b387f47b6de6f4dc782a192d75a396
-  data.tar.gz: '08c24046572b0bff4f377dfd959503d2fd86aa31c442af01e5c354f0a08e6af6'
+  metadata.gz: 141dfea6902c1018accd9962fc346017dbb8d070303323df0d98e5f16bf12516
+  data.tar.gz: ec3ae0172d80d7c6c84743376958fb2ceb7dffd9072afbfe641fcf4e0267e079
 SHA512:
-  metadata.gz: 87807af6107e2df8acd840da4a815a7e432fb84cdd7f5dce3b6c737122179fe2d808d9cd259829f40b41945e9f20e85d76a99f92b04d4d31e6dc9e497518af44
-  data.tar.gz: 0a0250d855f6330a8bd0f559d8a02419f448e0f402801aa9eba97272d0e683f02a8f3557f35ad4b4b6b10ca59f11035542fb55f545048eb7af0d4030c4dd81bc
+  metadata.gz: 40e41398c910b644fe3be0a7042d0a2ed763e2e622d61e98d3efa01d5bf74301c2adc61dbfdd689f99987c6677a7737402d97f013050b4cd6080c3706ee656b9
+  data.tar.gz: 46e233a13a96163779eb6eca5d07687cc6fe9c7fbd3e88c81e20d2af9aa43939ddca682e1210b3d2e5cc5b1555c2660aad8cecc01d9ff45f0adc25b6d0fc9ff7

data/Cargo.lock

@@ -307,9 +307,9 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
 
 [[package]]
 name = "parsanol"
-version = "0.18.9"
+version = "0.18.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "456a6ee7850bd50d2254b3232d537af2de447ea354a40a06a33904da899cd57b"
+checksum = "54bbd6459639572bc085c504112bc9a4acbca47719736aebad19d9113549bdad"
 dependencies = [
  "ahash",
  "getrandom 0.3.4",
@@ -326,9 +326,9 @@ dependencies = [
 
 [[package]]
 name = "parsanol-derive"
-version = "0.18.9"
+version = "0.18.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "280663d376aefe51ad6850535618f67fbb281d48f6ac024fad560ee39029ace1"
+checksum = "40adc63c6d2afcd1b095518d55f0095c5c04445f46994592161f0fa60cf6331b"
 dependencies = [
  "proc-macro2",
  "quote",

data/lib/parsanol/parg/artifact.rb

@@ -178,8 +178,9 @@ module Parsanol
             when "capture"
               captures = Bindings.collect_captures(shape)
               mismatched = hash["expect"].reject do |key, value|
-                actual = captures[key.to_sym]&.first
-                !actual.nil? && Bindings.canonical(actual) == Bindings.canonical(value)
+                values = captures[key.to_sym]
+                values && !values.empty? &&
+                  Bindings.canonical(values.first) == Bindings.canonical(value)
               end
               next if mismatched.empty?
 

data/lib/parsanol/parg/bindings.rb

@@ -64,7 +64,9 @@ module Parsanol
         count = group.map { |binding| captures[binding["capture"].to_sym].length }.max || 0
         return if count.zero?
 
-        list = (out[prefix] ||= [])
+        # Field names are symbols everywhere else (scalar bindings, element
+        # fields); the array prefix must match or `example` lookups miss it.
+        list = (out[prefix.to_sym] ||= [])
         count.times do |index|
           element = {}
           group.each do |binding|

data/lib/parsanol/parg/compiler.rb

@@ -337,6 +337,12 @@ module Parsanol
         find_left_cycle_from(name, [], {})
       end
 
+      # In-file test failures without raising: the LSP surfaces them as
+      # diagnostics instead of failing a build.
+      def test_failures
+        run_tests
+      end
+
       private
 
       # In-file grammar tests run at compile time: the build fails when an
@@ -372,8 +378,9 @@ module Parsanol
             when :capture
               captures = Bindings.collect_captures(shape)
               mismatched = test.expect.reject do |key, value|
-                actual = captures[key.to_sym]&.first
-                !actual.nil? && Bindings.canonical(actual) == Bindings.canonical(value)
+                values = captures[key.to_sym]
+                values && !values.empty? &&
+                  Bindings.canonical(values.first) == Bindings.canonical(value)
               end
               next if mismatched.empty?
 

data/lib/parsanol/parg/lexer.rb

@@ -23,7 +23,7 @@ module Parsanol
       | (?<arrow>->)
       | (?<num>\d+)
       | (?<ident>[A-Za-z_][A-Za-z0-9_]*)
-      | (?<punct>[=\/()\[\]{}*!&:,.<])
+      | (?<punct>[=\/()\[\]{}*!&:,.<-])
       /x
 
       def initialize(text)

data/lib/parsanol/parg/lsp.rb

@@ -1,26 +1,63 @@
 # frozen_string_literal: true
 
 require "json"
-require "rbconfig"
+require "strscan"
+require "uri"
 
 module Parsanol
   module PARG
-    # F11: minimal PARG LSP over stdio (JSON-RPC 2.0, no framework).
+    # PARG language server over stdio (JSON-RPC 2.0, no framework).
     #
     #   initialize / shutdown / exit
-    #   textDocument/didOpen|didChange -> publishDiagnostics
-    #     (parse errors, lint errors, failing inline tests)
-    #   textDocument/hover -> the rule's ## doc comment
+    #   textDocument/didOpen|didChange|didClose -> publishDiagnostics
+    #     (parse errors, lint errors, failing inline tests; imports merged)
+    #   textDocument/hover              -> rule docs (Imports-aware)
+    #   textDocument/definition         -> rule definition / imported file
+    #   textDocument/references         -> rule occurrences
+    #   textDocument/completion         -> keywords, rule names, snippets
+    #   textDocument/semanticTokens/full
+    #   textDocument/codeAction         -> longest-first reorder
     #
-    # Diagnostics and hover run entirely on the compiler API — one
-    # language implementation, every editor.
+    # Everything runs on the compiler API and the parsed Document model — one
+    # language implementation, every editor. Imported grammars are resolved
+    # from the document's directory and PG_IMPORT_DIR (path-separator list).
     class Lsp
       CONTENT_LENGTH = "Content-Length: "
 
+      SEMANTIC_TYPES = %w[keyword rule capture string number comment].freeze
+      SEMANTIC_MODIFIERS = %w[declaration].freeze
+      DECLARATION_BIT = 1
+
+      KEYWORDS = (%w[version test accept reject example use] + Parser::KEYWORDS).uniq.freeze
+
+      COMPLETION_KEYWORDS = KEYWORDS.map do |keyword|
+        { "label" => keyword, "kind" => 14 }
+      end.freeze
+
+      SNIPPETS = [
+        ["%i\"...\"", "%i\"${1:text}\"", "case-insensitive literal"],
+        ["%x41", "%x41", "one code point"],
+        ["%x41-5A", "%x41-5A", "code point range"],
+        ["%x0D.0A", "%x0D.0A", "code point sequence"],
+        ["[ ... ]", "[ ${1:content} ]", "optional"],
+        ["*( ... )", "*( ${1:content} )", "zero or more"],
+        ["1*( ... )", "1*( ${1:content} )", "one or more"],
+        ["!x", "!${1:x}", "negative lookahead"],
+        ["&x", "&${1:x}", "positive lookahead"],
+        ["!<x", "!< ${1:x}", "negative behind-guard"],
+        ["&<x", "&< ${1:x}", "positive behind-guard"],
+        ["as name", "as ${1:name}", "capture"],
+      ].map do |label, insert, detail|
+        { "label" => label, "kind" => 15, "detail" => detail,
+          "insertText" => insert, "insertTextFormat" => 2 }
+      end.freeze
+
       def initialize(input = $stdin, output = $stdout)
         @input = input
         @output = output
         @documents = {}
+        @parsed = {}
+        @import_paths = {}
       end
 
       def run
@@ -45,6 +82,19 @@ module Parsanol
                     "capabilities" => {
                       "textDocumentSync" => 1,
                       "hoverProvider" => true,
+                      "definitionProvider" => true,
+                      "referencesProvider" => true,
+                      "codeActionProvider" => true,
+                      "completionProvider" => {
+                        "triggerCharacters" => [".", "%", "\""],
+                      },
+                      "semanticTokensProvider" => {
+                        "legend" => {
+                          "tokenTypes" => SEMANTIC_TYPES,
+                          "tokenModifiers" => SEMANTIC_MODIFIERS,
+                        },
+                        "full" => true,
+                      },
                     },
                     "serverInfo" => { "name" => "parsanol-parg-lsp" },
                   })
@@ -56,17 +106,32 @@ module Parsanol
           doc = message["params"]["textDocument"]
           @documents[doc["uri"]] =
             doc["text"] || message.dig("params", "contentChanges", -1, "text")
+          @parsed.delete(doc["uri"])
           publish_diagnostics(doc["uri"])
+        when "textDocument/didClose"
+          uri = message.dig("params", "textDocument", "uri")
+          @documents.delete(uri)
+          @parsed.delete(uri)
         when "textDocument/codeAction"
           respond(message, code_action(message["params"]))
         when "textDocument/hover"
-          respond(message, hover(message.dig("params", "textDocument", "uri"),
-                                 message.dig("params", "position")))
+          respond(message, hover(*position_params(message["params"])))
+        when "textDocument/definition"
+          respond(message, definition(*position_params(message["params"])))
+        when "textDocument/references"
+          params = message["params"]
+          respond(message, references(params.dig("textDocument", "uri"),
+                                      params["position"],
+                                      params.dig("context", "includeDeclaration")))
+        when "textDocument/completion"
+          respond(message, completion(message.dig("params", "textDocument", "uri")))
+        when "textDocument/semanticTokens/full"
+          respond(message, semantic_tokens(message.dig("params", "textDocument", "uri")))
         end
       end
 
       def respond(message, result)
-        send_message("id" => message["id"], "result" => result)
+        send_message("jsonrpc" => "2.0", "id" => message["id"], "result" => result)
       end
 
       def notify(method, params)
@@ -79,12 +144,379 @@ module Parsanol
         @output.flush
       end
 
+      # ---- documents and imports -------------------------------------------
+
+      def position_params(params)
+        [params.dig("textDocument", "uri"), params["position"]]
+      end
+
+      def document_source(uri)
+        return @documents[uri] if @documents.key?(uri)
+
+        path = @import_paths[uri] || file_path(uri)
+        return nil unless path && File.file?(path)
+
+        File.read(path)
+      end
+
+      def parsed(uri)
+        source = @documents[uri]
+        return nil unless source
+
+        cached = @parsed[uri]
+        return cached[1] if cached && cached[0] == source
+
+        document = begin
+          Parser.new(source).parse
+        rescue ParseError
+          nil
+        end
+        @parsed[uri] = [source, document]
+        document
+      end
+
+      def file_path(uri)
+        text = uri.to_s
+        return nil unless text.start_with?("file://")
+
+        URI::DEFAULT_PARSER.unescape(text.delete_prefix("file://"))
+      end
+
+      def import_dirs(uri)
+        dirs = []
+        path = file_path(uri)
+        dirs << File.dirname(path) if path
+        ENV.fetch("PG_IMPORT_DIR", "").split(File::PATH_SEPARATOR).each do |dir|
+          dirs << dir unless dir.to_s.empty?
+        end
+        dirs.uniq
+      end
+
+      def import_uri(uri, name)
+        import_dirs(uri).each do |dir|
+          path = File.join(dir, "#{name}.parg")
+          next unless File.file?(path)
+
+          expanded = File.expand_path(path)
+          target = "file://#{URI::DEFAULT_PARSER.escape(expanded)}"
+          @import_paths[target] = expanded
+          return target
+        end
+        nil
+      end
+
+      # ---- rule model ------------------------------------------------------
+
+      Definition = Struct.new(:line, :character, :name_length, :docs)
+
+      # Rule definitions with their preceding ## doc comments. Works on a
+      # source that does not parse: hover/definition stay useful mid-edit.
+      def rule_definitions(source)
+        defs = {}
+        lines = source.to_s.lines
+        lines.each_with_index do |line, index|
+          match = line.match(/\A(\s*)(?:atomic\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=/)
+          next unless match
+
+          name = match[2]
+          docs = []
+          cursor = index - 1
+          while cursor.negative? == false && lines[cursor].to_s.strip.start_with?("##")
+            docs.unshift(lines[cursor].strip.sub(/\A##\s?/, ""))
+            cursor -= 1
+          end
+          defs[name] = Definition.new(index, match.begin(2), name.length, docs.join("\n"))
+        end
+        defs
+      end
+
+      def rule_names(uri)
+        document = parsed(uri)
+        return document.rules.keys if document
+
+        rule_definitions(document_source(uri)).keys
+      end
+
+      # Resolve a (possibly dotted) rule reference to its defining file.
+      def resolve(uri, word)
+        segments = word.to_s.split(".").reject(&:empty?)
+        return nil if segments.empty?
+
+        resolve_segments(uri, segments)
+      end
+
+      def resolve_segments(uri, segments)
+        source = document_source(uri)
+        return nil unless source
+
+        if segments.size == 1
+          name = segments.first
+          defs = rule_definitions(source)
+          definition = defs[name]
+          return nil unless definition
+
+          return { uri: uri, name: name, definition: definition }
+        end
+
+        document = if @documents.key?(uri)
+                     parsed(uri)
+                   else
+                     begin
+                       Parser.new(source).parse
+                     rescue ParseError
+                       nil
+                     end
+                   end
+        return nil unless document&.uses&.include?(segments.first)
+
+        target = import_uri(uri, segments.first)
+        return nil unless target
+
+        resolve_segments(target, segments[1..])
+      end
+
+      def word_at(source, position)
+        return nil unless source && position
+
+        line = source.lines[position["line"].to_i].to_s
+        character = position["character"].to_i
+        left = line[0...character].to_s
+        right = line[character..].to_s
+        prefix = left[/[A-Za-z0-9_.]*\z/] || ""
+        suffix = right[/\A[A-Za-z0-9_.]*/] || ""
+        word = (prefix + suffix).sub(/\A\.+/, "").sub(/\.+\z/, "")
+        return nil if word.empty?
+
+        word
+      end
+
+      def import_use_line?(source, position)
+        line = source.lines[position["line"].to_i].to_s
+        line.match?(/\A\s*use\s+/)
+      end
+
+      def location(uri, line, character, length)
+        {
+          "uri" => uri,
+          "range" => {
+            "start" => { "line" => line, "character" => character },
+            "end" => { "line" => line, "character" => character + length },
+          },
+        }
+      end
+
+      # ---- hover / definition / references ---------------------------------
+
+      def hover(uri, position)
+        source = document_source(uri)
+        return nil unless source && position
+
+        word = word_at(source, position)
+        return nil unless word
+
+        if import_use_line?(source, position)
+          target = import_uri(uri, word)
+          return nil unless target
+
+          return { "contents" => { "kind" => "markdown",
+                                   "value" => "Import `#{word}`\n\n`#{File.basename(target)}`" } }
+        end
+
+        resolved = resolve(uri, word)
+        return nil unless resolved
+
+        definition = resolved[:definition]
+        return nil unless definition
+
+        value = if definition.docs.empty?
+                  "`#{resolved[:name]}` — rule"
+                else
+                  "`#{resolved[:name]}`\n\n#{definition.docs}"
+                end
+        { "contents" => { "kind" => "markdown", "value" => value } }
+      end
+
+      def definition(uri, position)
+        source = document_source(uri)
+        return nil unless source && position
+
+        word = word_at(source, position)
+        return nil unless word
+
+        if import_use_line?(source, position)
+          target = import_uri(uri, word)
+          return target ? location(target, 0, 0, 0) : nil
+        end
+
+        resolved = resolve(uri, word)
+        return nil unless resolved
+
+        definition = resolved[:definition]
+        location(resolved[:uri], definition.line, definition.character,
+                 definition.name_length)
+      end
+
+      def references(uri, position, include_declaration)
+        source = document_source(uri)
+        return nil unless source && position
+
+        word = word_at(source, position)
+        resolved = word && resolve(uri, word)
+        return nil unless resolved
+
+        name = resolved[:name]
+        uris = [resolved[:uri], uri].uniq
+        results = []
+        uris.each do |target_uri|
+          target_source = document_source(target_uri)
+          next unless target_source
+
+          qualified = target_uri == uri && resolved[:uri] != uri && word.include?(".")
+          needle = qualified ? word : name
+          declaration = rule_definitions(target_source)[name]
+          target_source.lines.each_with_index do |line, index|
+            identifier_occurrences(line, needle).each do |character, length|
+              declaration_here = declaration && declaration.line == index &&
+                character == declaration.character && needle == name
+              results << location(target_uri, index, character, length) if
+                include_declaration || !declaration_here
+            end
+          end
+        end
+        results
+      end
+
+      # Identifier tokens outside strings and comments: a rule name inside a
+      # literal or a doc comment is not a reference.
+      def identifier_occurrences(line, name)
+        found = []
+        scanner = StringScanner.new(line)
+        until scanner.eos?
+          start = scanner.pos
+          start_char = line.byteslice(0, start).length
+          if scanner.scan(/#.*/)
+            break
+          elsif scanner.scan(/%(?:i|s)?"(?:\\.|[^"\\])*"/) ||
+              scanner.scan(/"(?:\\.|[^"\\])*"/) ||
+              scanner.scan(/%x[0-9A-Fa-f]{2}(?:[.-][0-9A-Fa-f]{2})*|[0-9]+/) ||
+              scanner.scan(/\s+/)
+            next
+          elsif (text = scanner.scan(/[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*/))
+            found << [start_char, text.length] if text == name
+          else
+            scanner.getch
+          end
+        end
+        found
+      end
+
+      # ---- completion ------------------------------------------------------
+
+      def completion(uri)
+        source = document_source(uri)
+        return nil unless source
+
+        items = COMPLETION_KEYWORDS.dup + SNIPPETS.dup
+        rule_names(uri).each do |name|
+          items << { "label" => name, "kind" => 18, "detail" => "rule" }
+        end
+        document = parsed(uri)
+        (document&.uses || []).each do |import|
+          target = import_uri(uri, import)
+          next unless target
+
+          rule_definitions(document_source(target)).each_key do |name|
+            items << { "label" => "#{import}.#{name}", "kind" => 18,
+                       "detail" => "imported rule" }
+          end
+        end
+        { "isIncomplete" => false, "items" => items }
+      end
+
+      # ---- semantic tokens -------------------------------------------------
+
+      def semantic_tokens(uri)
+        source = document_source(uri)
+        return nil unless source
+
+        names = rule_names(uri)
+        tokens = []
+        source.lines.each_with_index do |line, line_no|
+          scan_line(line, line_no, names, tokens)
+        end
+        { "data" => encode_tokens(tokens) }
+      end
+
+      def scan_line(line, line_no, names, tokens)
+        scanner = StringScanner.new(line)
+        previous_word = nil
+        until scanner.eos?
+          start = scanner.pos
+          start_char = line.byteslice(0, start).length
+
+          if (text = scanner.scan(/#.*/))
+            tokens << [line_no, start_char, text.length, "comment", 0]
+            break
+          elsif (text = scanner.scan(/%(?:i|s)?"(?:\\.|[^"\\])*"|"(?:\\.|[^"\\])*"/))
+            tokens << [line_no, start_char, text.length, "string", 0]
+            previous_word = nil
+          elsif (text = scanner.scan(/%x[0-9A-Fa-f]{2}(?:[.-][0-9A-Fa-f]{2})*|[0-9]+/))
+            tokens << [line_no, start_char, text.length, "number", 0]
+            previous_word = nil
+          elsif (text = scanner.scan(/[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*/))
+            type, modifiers = classify_word(line, start_char, text, names, previous_word)
+            tokens << [line_no, start_char, text.length, type, modifiers] if type
+            previous_word = text
+          elsif (text = scanner.scan(/\s+/))
+            _ = text
+          else
+            scanner.getch
+            previous_word = nil
+          end
+        end
+      end
+
+      def classify_word(line, start_char, text, names, previous_word)
+        base = text.split(".").last
+        if KEYWORDS.include?(text)
+          ["keyword", 0]
+        elsif previous_word == "as"
+          ["capture", 0]
+        elsif names.include?(base) || text.include?(".")
+          modifier = declaration_here?(line, start_char, text) ? DECLARATION_BIT : 0
+          ["rule", modifier]
+        elsif %w[entry bindings test].include?(previous_word)
+          ["rule", DECLARATION_BIT]
+        end
+      end
+
+      def declaration_here?(line, start_char, text)
+        before = line[0...start_char].to_s
+        after = line[(start_char + text.length)..].to_s
+        before.match?(/\A\s*(?:atomic\s+)?\z/) && after.match?(/\A\s*=/)
+      end
+
+      def encode_tokens(tokens)
+        data = []
+        last_line = 0
+        last_char = 0
+        tokens.sort_by { |token| [token[0], token[1]] }.each do |line, char, length, type, modifiers|
+          delta_line = line - last_line
+          delta_char = delta_line.zero? ? char - last_char : char
+          data.push(delta_line, delta_char, length, SEMANTIC_TYPES.index(type), modifiers)
+          last_line = line
+          last_char = char
+        end
+        data
+      end
+
+      # ---- diagnostics -----------------------------------------------------
+
       def publish_diagnostics(uri)
         source = @documents[uri]
         diagnostics = []
-        document = nil
-        begin
-          document = Parser.new(source).parse
+        document = begin
+          Parser.new(source).parse
         rescue ParseError => e
           line = 0
           # The self-hosting artifact yields a precise offset (F7 wire);
@@ -97,16 +529,23 @@ module Parsanol
             end
           end
           diagnostics << diagnostic(line, e.message)
+          nil
         end
         if document
+          dirs = import_dirs(uri)
           begin
-            tables_dir = ENV.fetch("PG_TABLES_DIR", nil)
+            Imports.merge!(document, dirs) if document.uses.any?
+          rescue PARG::Error, ParseError => e
+            diagnostics << diagnostic(use_line(source, document), e.message)
+          end
+          begin
+            tables_dir = ENV.fetch("PG_TABLES_DIR", nil) || dirs.first
             compiler = Compiler.new(document, tables_dir)
             document.rules.each_key { |rule| compiler.atom_for(rule) }
             Lints.errors(document, compiler).each do |error|
               diagnostics << diagnostic(rule_line(source, error[/\Arule (\w+):/, 1]), error)
             end
-            compiler.run_tests.each do |failure|
+            compiler.test_failures.each do |failure|
               diagnostics << diagnostic(0, failure)
             end
           rescue ParseError, PARG::Error => e
@@ -117,11 +556,18 @@ module Parsanol
                "uri" => uri, "diagnostics" => diagnostics)
       end
 
+      def use_line(source, document)
+        name = document.uses.first
+        return 0 unless name
+
+        source.lines.index { |line| line.match?(/^\s*use\s+#{Regexp.escape(name)}\b/) } || 0
+      end
+
       # Rule-granular position: the line the named rule is defined on.
       def rule_line(source, rule_name)
         return 0 unless rule_name
 
-        source.lines.index { |l| l.match?(/^\s*#{rule_name}\s*=/) } || 0
+        source.lines.index { |line| line.match?(/^\s*#{rule_name}\s*=/) } || 0
       end
 
       def diagnostic(line, message)
@@ -134,6 +580,8 @@ module Parsanol
         }
       end
 
+      # ---- code action -----------------------------------------------------
+
       # Reorder code action: only when EVERY top-level alternative of the
       # rule is a quoted literal is the reorder mechanically safe.
       def code_action(params)
@@ -176,33 +624,6 @@ module Parsanol
         { "start" => { "line" => line_no, "character" => 0 },
           "end" => { "line" => line_no, "character" => 1000 } }
       end
-
-      def hover(uri, position)
-        source = @documents[uri]
-        return nil unless source && position
-
-        line_no = position.dig("position", "line") || position["line"]
-        text = source.lines[line_no.to_i].to_s
-        rule = text[/^\s*([a-z_]+)\s*=/, 1]
-        return nil unless rule
-
-        lines = source.lines
-        rule_line = lines.index { |l| l.match?(/^\s*#{rule}\s*=/) }
-        return nil unless rule_line
-
-        doc_lines = []
-        cursor = rule_line - 1
-        while cursor >= 0 && lines[cursor].strip.start_with?("##")
-          doc_lines.unshift(lines[cursor])
-          cursor -= 1
-        end
-        return nil if doc_lines.empty?
-
-        doc = doc_lines.map(&:strip).join("\n")
-
-        { "contents" => { "kind" => "markdown",
-                          "value" => doc.gsub(/^##\s?/, "").strip } }
-      end
     end
   end
 end

data/lib/parsanol/parg/parser.rb

@@ -91,6 +91,8 @@ module Parsanol
               punct(":")
               expect_pairs[key.to_sym] = parse_test_value
               skip_newlines
+              advance if peek&.type == :punct && peek.value == ","
+              skip_newlines
             end
             punct("}")
           end
@@ -101,18 +103,84 @@ module Parsanol
         skip_newlines
       end
 
+      # Expectation values: strings, numbers, true/false/null, arrays, and
+      # nested objects. Nested structure is what pins array bindings,
+      # absent-optional nil, and flatten/merge results in the corpus;
+      # before this the test grammar only accepted scalars.
       def parse_test_value
-        token = advance
+        token = peek
         case token&.type
-        when :str then unquote(token.value)
-        when :num then token.value.to_i
-        when :ident
-          raise ParseError, "expected true or false" unless %w[true false].include?(token.value)
+        when :str then unquote(advance.value)
+        when :num then parse_test_number
+        when :punct
+          case token.value
+          when "[" then parse_test_array
+          when "{" then parse_test_object
+          when "-"
+            advance
+            unless peek&.type == :num
+              raise ParseError, "expected a number after '-' in a test value"
+            end
 
-          token.value == "true"
+            -parse_test_number
+          else
+            raise ParseError,
+                  "expected a test value (string, number, true, false, null, array, object)"
+          end
+        when :ident
+          advance
+          case token.value
+          when "true" then true
+          when "false" then false
+          when "null" then nil
+          else
+            raise ParseError,
+                  "expected true, false, or null, got #{token.value.inspect}"
+          end
         else
-          raise ParseError, "expected a string, number, true or false"
+          raise ParseError,
+                "expected a test value (string, number, true, false, null, array, object)"
+        end
+      end
+
+      # The lexer emits integers only; a decimal is num "." num.
+      def parse_test_number
+        text = advance.value
+        if peek&.type == :punct && peek.value == "." && @tokens[@pos + 1]&.type == :num
+          advance
+          text = "#{text}.#{advance.value}"
         end
+        text.include?(".") ? Float(text) : Integer(text, 10)
+      end
+
+      def parse_test_array
+        punct("[")
+        values = []
+        skip_newlines
+        until peek&.type == :punct && peek.value == "]"
+          values << parse_test_value
+          skip_newlines
+          advance if peek&.type == :punct && peek.value == ","
+          skip_newlines
+        end
+        punct("]")
+        values
+      end
+
+      def parse_test_object
+        punct("{")
+        value = {}
+        skip_newlines
+        until peek&.type == :punct && peek.value == "}"
+          key = ident.value
+          punct(":")
+          value[key.to_sym] = parse_test_value
+          skip_newlines
+          advance if peek&.type == :punct && peek.value == ","
+          skip_newlines
+        end
+        punct("}")
+        value
       end
 
       def eof? = @pos >= @tokens.length

data/lib/parsanol/parg/selfhost.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "json"
+
 module Parsanol
   module PARG
     # F10 phase 2 (gate slice): every .parg source must parse under the parg

data/lib/parsanol/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Parsanol
-  VERSION = "1.3.96"
+  VERSION = "1.3.98"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: parsanol
 version: !ruby/object:Gem::Version
-  version: 1.3.96
+  version: 1.3.98
 platform: ruby
 authors:
 - Ribose Inc.