parsanol 1.3.95-aarch64-linux → 1.3.97-aarch64-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 294aa3c79aee85b7dc03fb8579568c1946757227561f12ce1fbe3414219e85fd
-  data.tar.gz: 8733d41a4a3e5520a185f07d091ef411d949cff5d8116be126f35d251e15edab
+  metadata.gz: f259ea55092d5673e2d9b3604df6fb9adba3807dba48500b765562de3b1b6aee
+  data.tar.gz: d6c1ba491efa08d1ccc551d79899558577c72138909a17d46a39560ca818b80a
 SHA512:
-  metadata.gz: 2cf7ed489972829333d106fc3d4ab2385ac3a4a6394dbbcb5765693ba48c5a110c11d20f27428a5833a0b5c406c1c89751783c9ab5ae6554751ff56a352f1531
-  data.tar.gz: 24282d91f4148bcddda12871d91111e58bc47deffdf457e022a05e020ff94e6a73f53d38bc6ea3b6488d82dcda733019194a62eec5d14e18b45470db212d40d4
+  metadata.gz: 066d798f2dc0b057a46db9f077aee2e0d4a424b076c836cce9152ef1d969b2f3d587b0071ed9f1b3524a0534c223aa9cd58b2d43d8511ec58eb8d867438a8fd9
+  data.tar.gz: 8bce87d1b44af2a4ec860b5a1072837bda1ac171a6aee8d1fd1dd505cac84a3ae84f7529f413430014de1dd902ce69197d2c8f9ae6cb16c465591d520a36af37

data/lib/parsanol/3.2/parsanol_native.so

@@ -1,4 +1,4 @@
 [diffend] Oversized file quarantined before diffing.
 name: data/lib/parsanol/3.2/parsanol_native.so
-size: 40550656 bytes
-sha256: cff224d02b36baf6455b961827d069556a7fed497e0a8e778009065f12613f00
+size: 40739336 bytes
+sha256: 4cad7bf4894a1cea08ee2b319696f166f2e7cc38ed6eef38dee79057f9fd684a

data/lib/parsanol/3.3/parsanol_native.so

@@ -1,4 +1,4 @@
 [diffend] Oversized file quarantined before diffing.
 name: data/lib/parsanol/3.3/parsanol_native.so
-size: 40785744 bytes
-sha256: ae67525c0f460813aed0b9fccb58809b517da84d72dae383bc232bd3c0c5d4f0
+size: 40974392 bytes
+sha256: 25df57a50ccd92bd51baaf86008559487f4259b81a899c30c61e8e023ff7ec23

data/lib/parsanol/3.4/parsanol_native.so

@@ -1,4 +1,4 @@
 [diffend] Oversized file quarantined before diffing.
 name: data/lib/parsanol/3.4/parsanol_native.so
-size: 40781360 bytes
-sha256: e30f6696ef7f343f3edf49240039e9287277f6973d4e0bfee918939d82684056
+size: 40961728 bytes
+sha256: e6e8cae20b2ffc4754570fd4e4e19081e24e4b5c37ccc0e122619ada30ca7d97

data/lib/parsanol/4.0/parsanol_native.so

@@ -1,4 +1,4 @@
 [diffend] Oversized file quarantined before diffing.
 name: data/lib/parsanol/4.0/parsanol_native.so
-size: 40791392 bytes
-sha256: 2f8c4e6eb46bc01f1271aeb19e5b3beb3b1c81ce1deb99b500094fabf3b11517
+size: 40971784 bytes
+sha256: 4c426cf8bfd77fb270f32e171664208d9b3fd3414bcf22b2d677fddc2ec14e9f

data/lib/parsanol/atoms/context.rb

@@ -72,6 +72,10 @@ module Parsanol
         # and the NEXT successful Named capture attaches (and drains)
         # them under `comments:`.
         @pending_trivia = []
+
+        # Recovery journal (TODO.sota/1): non-nil only in recovery
+        # re-parses; Named captures append to the shared array.
+        @recovery_journal = Parsanol::Recovery.journal if Parsanol::Recovery.journaling?
         @evict_threshold = 200
         @evict_counter = 0
         @evict_interval = 100
@@ -327,6 +331,10 @@ module Parsanol
       # @return [Parsanol::Scope] capture variable bindings
       attr_reader :captures
 
+      # Recovery journal (TODO.sota/1): a shared array when this
+      # context was created inside a recovery re-parse; nil otherwise.
+      attr_accessor :recovery_journal
+
       # @return [Parsanol::Pools::ArrayPool] array object pool
       attr_reader :array_pool
 

data/lib/parsanol/atoms/named.rb

@@ -42,6 +42,8 @@ module Parsanol
         comments = context.take_pending_trivia
         result = wrap_result(value)
         result = attach_comments(result, comments, source) unless comments.empty?
+        # Recovery journal (TODO.sota/1): nil on every normal parse.
+        context.recovery_journal << [@name, result] if context.recovery_journal
         ok(result)
       end
 

data/lib/parsanol/native/libparsanol.so

@@ -1,4 +1,4 @@
 [diffend] Oversized file quarantined before diffing.
 name: data/lib/parsanol/native/libparsanol.so
-size: 36352992 bytes
-sha256: fc4e4581f2ff40db6d747bd6b6377e34da0d7389eccc7b1ecb30f485f2e59e25
+size: 36519512 bytes
+sha256: e6eab7ae66da64b0abf1e86985786423cce6eba79524e64771a52fd53d541530

data/lib/parsanol/native.rb

@@ -382,6 +382,20 @@ module Parsanol
             :unavailable
           end
         end
+        name_cache_version if @native_state == :loaded
+        @native_state
+      end
+
+      private
+
+      # Compiled VM programs are cached on disk across processes; naming
+      # this gem release in the cache key means a new release never runs
+      # a program an older one compiled. Extensions built before the
+      # engine grew the hook simply skip it.
+      def name_cache_version
+        return unless respond_to?(:_set_cache_version)
+
+        _set_cache_version("parsanol-ruby #{Parsanol::VERSION}")
       end
     end
   end

data/lib/parsanol/parg/artifact.rb

@@ -197,6 +197,53 @@ module Parsanol
         end
       end
 
+      # Best-effort recovery (TODO.sota/1, R1): on success returns
+      # {"ok": true, "shape": tree}; on ParseFailed returns
+      # {"ok": false, "error": {"offset", "message"}, "captures":
+      # {name => [values...]}} — the captures observed before the
+      # failure, from a journaling re-parse on the Ruby engine (tree
+      # parity with the native engine is enforced by the differential
+      # specs). The journal is best-effort: it may include captures
+      # from branches that later failed.
+      def parse_recovering(entry_name, input, mode: :native)
+        shape = parse(entry_name, input, mode: mode)
+        { "ok" => true, "shape" => shape }
+      rescue Parsanol::ParseFailed => e
+        { "ok" => false,
+          "error" => recovery_error(e),
+          "captures" => recovery_journal(entry_name, input, mode) }
+      end
+
+      def recovery_error(err)
+        pos = err.parse_failure_cause&.position
+        { "offset" => pos, "message" => err.message }
+      end
+
+      # Journaling re-parse — always on the Ruby engine (journaling
+      # rides the interpreter's diagnostic reparse; the native engine
+      # has no journal). Tree parity with the native engine is enforced
+      # by the differential specs, so the observed captures agree.
+      def recovery_journal(entry_name, input, _mode)
+        Parsanol::Recovery.start_journal
+        begin
+          parse(entry_name, input, mode: :ruby)
+        rescue Parsanol::ParseFailed
+          # expected: the journaling pass fails at the same point
+        end
+        Parsanol::Recovery.stop_journal.each_with_object({}) do |(name, value), out|
+          # Both the initial attempt and the diagnostic re-parse journal
+          # into the shared thread-local; identical observations dedupe.
+          wire = value.is_a?(Slice) ? slice_wire(value) : value
+          (out[name.to_s] ||= []) << wire
+          out[name.to_s].uniq!
+        end
+      end
+
+      def slice_wire(slice)
+        { "value" => slice.to_s, "offset" => slice.offset,
+          "length" => slice.content.length }
+      end
+
       def test_to_hash(test)
         {
           "entry" => test.entry,

data/lib/parsanol/parg/compiler.rb

@@ -337,6 +337,12 @@ module Parsanol
         find_left_cycle_from(name, [], {})
       end
 
+      # In-file test failures without raising: the LSP surfaces them as
+      # diagnostics instead of failing a build.
+      def test_failures
+        run_tests
+      end
+
       private
 
       # In-file grammar tests run at compile time: the build fails when an

data/lib/parsanol/parg/lsp.rb

@@ -1,26 +1,63 @@
 # frozen_string_literal: true
 
 require "json"
-require "rbconfig"
+require "strscan"
+require "uri"
 
 module Parsanol
   module PARG
-    # F11: minimal PARG LSP over stdio (JSON-RPC 2.0, no framework).
+    # PARG language server over stdio (JSON-RPC 2.0, no framework).
     #
     #   initialize / shutdown / exit
-    #   textDocument/didOpen|didChange -> publishDiagnostics
-    #     (parse errors, lint errors, failing inline tests)
-    #   textDocument/hover -> the rule's ## doc comment
+    #   textDocument/didOpen|didChange|didClose -> publishDiagnostics
+    #     (parse errors, lint errors, failing inline tests; imports merged)
+    #   textDocument/hover              -> rule docs (Imports-aware)
+    #   textDocument/definition         -> rule definition / imported file
+    #   textDocument/references         -> rule occurrences
+    #   textDocument/completion         -> keywords, rule names, snippets
+    #   textDocument/semanticTokens/full
+    #   textDocument/codeAction         -> longest-first reorder
     #
-    # Diagnostics and hover run entirely on the compiler API — one
-    # language implementation, every editor.
+    # Everything runs on the compiler API and the parsed Document model — one
+    # language implementation, every editor. Imported grammars are resolved
+    # from the document's directory and PG_IMPORT_DIR (path-separator list).
     class Lsp
       CONTENT_LENGTH = "Content-Length: "
 
+      SEMANTIC_TYPES = %w[keyword rule capture string number comment].freeze
+      SEMANTIC_MODIFIERS = %w[declaration].freeze
+      DECLARATION_BIT = 1
+
+      KEYWORDS = (%w[version test accept reject example use] + Parser::KEYWORDS).uniq.freeze
+
+      COMPLETION_KEYWORDS = KEYWORDS.map do |keyword|
+        { "label" => keyword, "kind" => 14 }
+      end.freeze
+
+      SNIPPETS = [
+        ["%i\"...\"", "%i\"${1:text}\"", "case-insensitive literal"],
+        ["%x41", "%x41", "one code point"],
+        ["%x41-5A", "%x41-5A", "code point range"],
+        ["%x0D.0A", "%x0D.0A", "code point sequence"],
+        ["[ ... ]", "[ ${1:content} ]", "optional"],
+        ["*( ... )", "*( ${1:content} )", "zero or more"],
+        ["1*( ... )", "1*( ${1:content} )", "one or more"],
+        ["!x", "!${1:x}", "negative lookahead"],
+        ["&x", "&${1:x}", "positive lookahead"],
+        ["!<x", "!< ${1:x}", "negative behind-guard"],
+        ["&<x", "&< ${1:x}", "positive behind-guard"],
+        ["as name", "as ${1:name}", "capture"],
+      ].map do |label, insert, detail|
+        { "label" => label, "kind" => 15, "detail" => detail,
+          "insertText" => insert, "insertTextFormat" => 2 }
+      end.freeze
+
       def initialize(input = $stdin, output = $stdout)
         @input = input
         @output = output
         @documents = {}
+        @parsed = {}
+        @import_paths = {}
       end
 
       def run
@@ -45,6 +82,19 @@ module Parsanol
                     "capabilities" => {
                       "textDocumentSync" => 1,
                       "hoverProvider" => true,
+                      "definitionProvider" => true,
+                      "referencesProvider" => true,
+                      "codeActionProvider" => true,
+                      "completionProvider" => {
+                        "triggerCharacters" => [".", "%", "\""],
+                      },
+                      "semanticTokensProvider" => {
+                        "legend" => {
+                          "tokenTypes" => SEMANTIC_TYPES,
+                          "tokenModifiers" => SEMANTIC_MODIFIERS,
+                        },
+                        "full" => true,
+                      },
                     },
                     "serverInfo" => { "name" => "parsanol-parg-lsp" },
                   })
@@ -56,17 +106,32 @@ module Parsanol
           doc = message["params"]["textDocument"]
           @documents[doc["uri"]] =
             doc["text"] || message.dig("params", "contentChanges", -1, "text")
+          @parsed.delete(doc["uri"])
           publish_diagnostics(doc["uri"])
+        when "textDocument/didClose"
+          uri = message.dig("params", "textDocument", "uri")
+          @documents.delete(uri)
+          @parsed.delete(uri)
         when "textDocument/codeAction"
           respond(message, code_action(message["params"]))
         when "textDocument/hover"
-          respond(message, hover(message.dig("params", "textDocument", "uri"),
-                                 message.dig("params", "position")))
+          respond(message, hover(*position_params(message["params"])))
+        when "textDocument/definition"
+          respond(message, definition(*position_params(message["params"])))
+        when "textDocument/references"
+          params = message["params"]
+          respond(message, references(params.dig("textDocument", "uri"),
+                                      params["position"],
+                                      params.dig("context", "includeDeclaration")))
+        when "textDocument/completion"
+          respond(message, completion(message.dig("params", "textDocument", "uri")))
+        when "textDocument/semanticTokens/full"
+          respond(message, semantic_tokens(message.dig("params", "textDocument", "uri")))
         end
       end
 
       def respond(message, result)
-        send_message("id" => message["id"], "result" => result)
+        send_message("jsonrpc" => "2.0", "id" => message["id"], "result" => result)
       end
 
       def notify(method, params)
@@ -79,12 +144,379 @@ module Parsanol
         @output.flush
       end
 
+      # ---- documents and imports -------------------------------------------
+
+      def position_params(params)
+        [params.dig("textDocument", "uri"), params["position"]]
+      end
+
+      def document_source(uri)
+        return @documents[uri] if @documents.key?(uri)
+
+        path = @import_paths[uri] || file_path(uri)
+        return nil unless path && File.file?(path)
+
+        File.read(path)
+      end
+
+      def parsed(uri)
+        source = @documents[uri]
+        return nil unless source
+
+        cached = @parsed[uri]
+        return cached[1] if cached && cached[0] == source
+
+        document = begin
+          Parser.new(source).parse
+        rescue ParseError
+          nil
+        end
+        @parsed[uri] = [source, document]
+        document
+      end
+
+      def file_path(uri)
+        text = uri.to_s
+        return nil unless text.start_with?("file://")
+
+        URI::DEFAULT_PARSER.unescape(text.delete_prefix("file://"))
+      end
+
+      def import_dirs(uri)
+        dirs = []
+        path = file_path(uri)
+        dirs << File.dirname(path) if path
+        ENV.fetch("PG_IMPORT_DIR", "").split(File::PATH_SEPARATOR).each do |dir|
+          dirs << dir unless dir.to_s.empty?
+        end
+        dirs.uniq
+      end
+
+      def import_uri(uri, name)
+        import_dirs(uri).each do |dir|
+          path = File.join(dir, "#{name}.parg")
+          next unless File.file?(path)
+
+          expanded = File.expand_path(path)
+          target = "file://#{URI::DEFAULT_PARSER.escape(expanded)}"
+          @import_paths[target] = expanded
+          return target
+        end
+        nil
+      end
+
+      # ---- rule model ------------------------------------------------------
+
+      Definition = Struct.new(:line, :character, :name_length, :docs)
+
+      # Rule definitions with their preceding ## doc comments. Works on a
+      # source that does not parse: hover/definition stay useful mid-edit.
+      def rule_definitions(source)
+        defs = {}
+        lines = source.to_s.lines
+        lines.each_with_index do |line, index|
+          match = line.match(/\A(\s*)(?:atomic\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=/)
+          next unless match
+
+          name = match[2]
+          docs = []
+          cursor = index - 1
+          while cursor.negative? == false && lines[cursor].to_s.strip.start_with?("##")
+            docs.unshift(lines[cursor].strip.sub(/\A##\s?/, ""))
+            cursor -= 1
+          end
+          defs[name] = Definition.new(index, match.begin(2), name.length, docs.join("\n"))
+        end
+        defs
+      end
+
+      def rule_names(uri)
+        document = parsed(uri)
+        return document.rules.keys if document
+
+        rule_definitions(document_source(uri)).keys
+      end
+
+      # Resolve a (possibly dotted) rule reference to its defining file.
+      def resolve(uri, word)
+        segments = word.to_s.split(".").reject(&:empty?)
+        return nil if segments.empty?
+
+        resolve_segments(uri, segments)
+      end
+
+      def resolve_segments(uri, segments)
+        source = document_source(uri)
+        return nil unless source
+
+        if segments.size == 1
+          name = segments.first
+          defs = rule_definitions(source)
+          definition = defs[name]
+          return nil unless definition
+
+          return { uri: uri, name: name, definition: definition }
+        end
+
+        document = if @documents.key?(uri)
+                     parsed(uri)
+                   else
+                     begin
+                       Parser.new(source).parse
+                     rescue ParseError
+                       nil
+                     end
+                   end
+        return nil unless document&.uses&.include?(segments.first)
+
+        target = import_uri(uri, segments.first)
+        return nil unless target
+
+        resolve_segments(target, segments[1..])
+      end
+
+      def word_at(source, position)
+        return nil unless source && position
+
+        line = source.lines[position["line"].to_i].to_s
+        character = position["character"].to_i
+        left = line[0...character].to_s
+        right = line[character..].to_s
+        prefix = left[/[A-Za-z0-9_.]*\z/] || ""
+        suffix = right[/\A[A-Za-z0-9_.]*/] || ""
+        word = (prefix + suffix).sub(/\A\.+/, "").sub(/\.+\z/, "")
+        return nil if word.empty?
+
+        word
+      end
+
+      def import_use_line?(source, position)
+        line = source.lines[position["line"].to_i].to_s
+        line.match?(/\A\s*use\s+/)
+      end
+
+      def location(uri, line, character, length)
+        {
+          "uri" => uri,
+          "range" => {
+            "start" => { "line" => line, "character" => character },
+            "end" => { "line" => line, "character" => character + length },
+          },
+        }
+      end
+
+      # ---- hover / definition / references ---------------------------------
+
+      def hover(uri, position)
+        source = document_source(uri)
+        return nil unless source && position
+
+        word = word_at(source, position)
+        return nil unless word
+
+        if import_use_line?(source, position)
+          target = import_uri(uri, word)
+          return nil unless target
+
+          return { "contents" => { "kind" => "markdown",
+                                   "value" => "Import `#{word}`\n\n`#{File.basename(target)}`" } }
+        end
+
+        resolved = resolve(uri, word)
+        return nil unless resolved
+
+        definition = resolved[:definition]
+        return nil unless definition
+
+        value = if definition.docs.empty?
+                  "`#{resolved[:name]}` — rule"
+                else
+                  "`#{resolved[:name]}`\n\n#{definition.docs}"
+                end
+        { "contents" => { "kind" => "markdown", "value" => value } }
+      end
+
+      def definition(uri, position)
+        source = document_source(uri)
+        return nil unless source && position
+
+        word = word_at(source, position)
+        return nil unless word
+
+        if import_use_line?(source, position)
+          target = import_uri(uri, word)
+          return target ? location(target, 0, 0, 0) : nil
+        end
+
+        resolved = resolve(uri, word)
+        return nil unless resolved
+
+        definition = resolved[:definition]
+        location(resolved[:uri], definition.line, definition.character,
+                 definition.name_length)
+      end
+
+      def references(uri, position, include_declaration)
+        source = document_source(uri)
+        return nil unless source && position
+
+        word = word_at(source, position)
+        resolved = word && resolve(uri, word)
+        return nil unless resolved
+
+        name = resolved[:name]
+        uris = [resolved[:uri], uri].uniq
+        results = []
+        uris.each do |target_uri|
+          target_source = document_source(target_uri)
+          next unless target_source
+
+          qualified = target_uri == uri && resolved[:uri] != uri && word.include?(".")
+          needle = qualified ? word : name
+          declaration = rule_definitions(target_source)[name]
+          target_source.lines.each_with_index do |line, index|
+            identifier_occurrences(line, needle).each do |character, length|
+              declaration_here = declaration && declaration.line == index &&
+                character == declaration.character && needle == name
+              results << location(target_uri, index, character, length) if
+                include_declaration || !declaration_here
+            end
+          end
+        end
+        results
+      end
+
+      # Identifier tokens outside strings and comments: a rule name inside a
+      # literal or a doc comment is not a reference.
+      def identifier_occurrences(line, name)
+        found = []
+        scanner = StringScanner.new(line)
+        until scanner.eos?
+          start = scanner.pos
+          start_char = line.byteslice(0, start).length
+          if scanner.scan(/#.*/)
+            break
+          elsif scanner.scan(/%(?:i|s)?"(?:\\.|[^"\\])*"/) ||
+              scanner.scan(/"(?:\\.|[^"\\])*"/) ||
+              scanner.scan(/%x[0-9A-Fa-f]{2}(?:[.-][0-9A-Fa-f]{2})*|[0-9]+/) ||
+              scanner.scan(/\s+/)
+            next
+          elsif (text = scanner.scan(/[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*/))
+            found << [start_char, text.length] if text == name
+          else
+            scanner.getch
+          end
+        end
+        found
+      end
+
+      # ---- completion ------------------------------------------------------
+
+      def completion(uri)
+        source = document_source(uri)
+        return nil unless source
+
+        items = COMPLETION_KEYWORDS.dup + SNIPPETS.dup
+        rule_names(uri).each do |name|
+          items << { "label" => name, "kind" => 18, "detail" => "rule" }
+        end
+        document = parsed(uri)
+        (document&.uses || []).each do |import|
+          target = import_uri(uri, import)
+          next unless target
+
+          rule_definitions(document_source(target)).each_key do |name|
+            items << { "label" => "#{import}.#{name}", "kind" => 18,
+                       "detail" => "imported rule" }
+          end
+        end
+        { "isIncomplete" => false, "items" => items }
+      end
+
+      # ---- semantic tokens -------------------------------------------------
+
+      def semantic_tokens(uri)
+        source = document_source(uri)
+        return nil unless source
+
+        names = rule_names(uri)
+        tokens = []
+        source.lines.each_with_index do |line, line_no|
+          scan_line(line, line_no, names, tokens)
+        end
+        { "data" => encode_tokens(tokens) }
+      end
+
+      def scan_line(line, line_no, names, tokens)
+        scanner = StringScanner.new(line)
+        previous_word = nil
+        until scanner.eos?
+          start = scanner.pos
+          start_char = line.byteslice(0, start).length
+
+          if (text = scanner.scan(/#.*/))
+            tokens << [line_no, start_char, text.length, "comment", 0]
+            break
+          elsif (text = scanner.scan(/%(?:i|s)?"(?:\\.|[^"\\])*"|"(?:\\.|[^"\\])*"/))
+            tokens << [line_no, start_char, text.length, "string", 0]
+            previous_word = nil
+          elsif (text = scanner.scan(/%x[0-9A-Fa-f]{2}(?:[.-][0-9A-Fa-f]{2})*|[0-9]+/))
+            tokens << [line_no, start_char, text.length, "number", 0]
+            previous_word = nil
+          elsif (text = scanner.scan(/[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*/))
+            type, modifiers = classify_word(line, start_char, text, names, previous_word)
+            tokens << [line_no, start_char, text.length, type, modifiers] if type
+            previous_word = text
+          elsif (text = scanner.scan(/\s+/))
+            _ = text
+          else
+            scanner.getch
+            previous_word = nil
+          end
+        end
+      end
+
+      def classify_word(line, start_char, text, names, previous_word)
+        base = text.split(".").last
+        if KEYWORDS.include?(text)
+          ["keyword", 0]
+        elsif previous_word == "as"
+          ["capture", 0]
+        elsif names.include?(base) || text.include?(".")
+          modifier = declaration_here?(line, start_char, text) ? DECLARATION_BIT : 0
+          ["rule", modifier]
+        elsif %w[entry bindings test].include?(previous_word)
+          ["rule", DECLARATION_BIT]
+        end
+      end
+
+      def declaration_here?(line, start_char, text)
+        before = line[0...start_char].to_s
+        after = line[(start_char + text.length)..].to_s
+        before.match?(/\A\s*(?:atomic\s+)?\z/) && after.match?(/\A\s*=/)
+      end
+
+      def encode_tokens(tokens)
+        data = []
+        last_line = 0
+        last_char = 0
+        tokens.sort_by { |token| [token[0], token[1]] }.each do |line, char, length, type, modifiers|
+          delta_line = line - last_line
+          delta_char = delta_line.zero? ? char - last_char : char
+          data.push(delta_line, delta_char, length, SEMANTIC_TYPES.index(type), modifiers)
+          last_line = line
+          last_char = char
+        end
+        data
+      end
+
+      # ---- diagnostics -----------------------------------------------------
+
       def publish_diagnostics(uri)
         source = @documents[uri]
         diagnostics = []
-        document = nil
-        begin
-          document = Parser.new(source).parse
+        document = begin
+          Parser.new(source).parse
         rescue ParseError => e
           line = 0
           # The self-hosting artifact yields a precise offset (F7 wire);
@@ -97,16 +529,23 @@ module Parsanol
             end
           end
           diagnostics << diagnostic(line, e.message)
+          nil
         end
         if document
+          dirs = import_dirs(uri)
           begin
-            tables_dir = ENV.fetch("PG_TABLES_DIR", nil)
+            Imports.merge!(document, dirs) if document.uses.any?
+          rescue PARG::Error, ParseError => e
+            diagnostics << diagnostic(use_line(source, document), e.message)
+          end
+          begin
+            tables_dir = ENV.fetch("PG_TABLES_DIR", nil) || dirs.first
             compiler = Compiler.new(document, tables_dir)
             document.rules.each_key { |rule| compiler.atom_for(rule) }
             Lints.errors(document, compiler).each do |error|
               diagnostics << diagnostic(rule_line(source, error[/\Arule (\w+):/, 1]), error)
             end
-            compiler.run_tests.each do |failure|
+            compiler.test_failures.each do |failure|
               diagnostics << diagnostic(0, failure)
             end
           rescue ParseError, PARG::Error => e
@@ -117,11 +556,18 @@ module Parsanol
                "uri" => uri, "diagnostics" => diagnostics)
       end
 
+      def use_line(source, document)
+        name = document.uses.first
+        return 0 unless name
+
+        source.lines.index { |line| line.match?(/^\s*use\s+#{Regexp.escape(name)}\b/) } || 0
+      end
+
       # Rule-granular position: the line the named rule is defined on.
       def rule_line(source, rule_name)
         return 0 unless rule_name
 
-        source.lines.index { |l| l.match?(/^\s*#{rule_name}\s*=/) } || 0
+        source.lines.index { |line| line.match?(/^\s*#{rule_name}\s*=/) } || 0
       end
 
       def diagnostic(line, message)
@@ -134,6 +580,8 @@ module Parsanol
         }
       end
 
+      # ---- code action -----------------------------------------------------
+
       # Reorder code action: only when EVERY top-level alternative of the
       # rule is a quoted literal is the reorder mechanically safe.
       def code_action(params)
@@ -176,33 +624,6 @@ module Parsanol
         { "start" => { "line" => line_no, "character" => 0 },
           "end" => { "line" => line_no, "character" => 1000 } }
       end
-
-      def hover(uri, position)
-        source = @documents[uri]
-        return nil unless source && position
-
-        line_no = position.dig("position", "line") || position["line"]
-        text = source.lines[line_no.to_i].to_s
-        rule = text[/^\s*([a-z_]+)\s*=/, 1]
-        return nil unless rule
-
-        lines = source.lines
-        rule_line = lines.index { |l| l.match?(/^\s*#{rule}\s*=/) }
-        return nil unless rule_line
-
-        doc_lines = []
-        cursor = rule_line - 1
-        while cursor >= 0 && lines[cursor].strip.start_with?("##")
-          doc_lines.unshift(lines[cursor])
-          cursor -= 1
-        end
-        return nil if doc_lines.empty?
-
-        doc = doc_lines.map(&:strip).join("\n")
-
-        { "contents" => { "kind" => "markdown",
-                          "value" => doc.gsub(/^##\s?/, "").strip } }
-      end
     end
   end
 end

data/lib/parsanol/recovery.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Parsanol
+  # Recovery journaling (TODO.sota/1): when enabled for the current
+  # thread, every Named capture appends to a shared journal for the
+  # duration of the recovery re-parse. Success paths never enable it —
+  # normal parses see only a nil-context check.
+  module Recovery
+    module_function
+
+    def start_journal
+      Thread.current[:parsanol_recovery_journal] = []
+      Thread.current[:parsanol_recovery_journaling] = true
+    end
+
+    # Stops journaling and returns the journal: [name, value] pairs in
+    # capture order, including captures from branches that later failed
+    # (best-effort superset — same semantics as the rs recovery
+    # journal).
+    def stop_journal
+      Thread.current[:parsanol_recovery_journaling] = false
+      journal
+    end
+
+    def journal
+      Thread.current[:parsanol_recovery_journal] || []
+    end
+
+    def journaling?
+      Thread.current[:parsanol_recovery_journaling] == true
+    end
+  end
+end

data/lib/parsanol/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Parsanol
-  VERSION = "1.3.95"
+  VERSION = "1.3.97"
 end

data/lib/parsanol.rb

@@ -273,6 +273,7 @@ module Parsanol
   end
   module_function :exp
 
+  autoload :Recovery, "parsanol/recovery"
   autoload :Resettable, "parsanol/resettable"
   autoload :Result, "parsanol/result"
   autoload :Slice, "parsanol/slice"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parsanol
 version: !ruby/object:Gem::Version
-  version: 1.3.95
+  version: 1.3.97
 platform: aarch64-linux
 authors:
 - Ribose Inc.
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2026-10-19 00:00:00.000000000 Z
+date: 2026-10-20 00:00:00.000000000 Z
 dependencies: []
 description: A small Ruby library for constructing parsers in the PEG (Parsing Expression
   Grammar) fashion. Parsanol provides Parslet-compatible API with additional features
@@ -133,6 +133,7 @@ files:
 - lib/parsanol/pools/position_pool.rb
 - lib/parsanol/pools/slice_pool.rb
 - lib/parsanol/position.rb
+- lib/parsanol/recovery.rb
 - lib/parsanol/resettable.rb
 - lib/parsanol/result.rb
 - lib/parsanol/result_builder.rb