ibex 0.1.0 → 0.3.0

data/lib/ibex/lsp/document_store.rb

@@ -0,0 +1,264 @@
+# frozen_string_literal: true
+
+module Ibex
+  module LSP
+    # Owns open buffers, parsed snapshots, include closures, and reverse dependencies.
+    class DocumentStore
+      include DocumentStoreValidation
+      include DocumentStoreDiagnostics
+
+      # @rbs!
+      #   type snapshot = {
+      #     uri: String,
+      #     path: String,
+      #     version: Integer?,
+      #     source: String,
+      #     open: bool,
+      #     document: Frontend::SourceDocument?
+      #   }
+      #   type publication = {
+      #     uri: String,
+      #     ?version: Integer,
+      #     diagnostics: Array[WorkspaceAnalyzer::lsp_diagnostic]
+      #   }
+
+      attr_reader :loader #: Frontend::SourceLoader
+      attr_reader :workspace #: Workspace
+
+      # @rbs (Workspace workspace, Frontend::SourceLoader loader) -> void
+      def initialize(workspace, loader)
+        @workspace = workspace
+        @loader = loader
+        @analyzer = WorkspaceAnalyzer.new(workspace, loader)
+        @snapshots = {} #: Hash[String, snapshot]
+        @roots = {} #: Hash[String, bool]
+        @closures = {} #: Hash[String, Array[String]]
+        @reverse_dependencies = {} #: Hash[String, Array[String]]
+        @resolutions = {} #: Hash[String, Frontend::Resolution]
+        @document_diagnostics = {} #: Hash[String, Array[WorkspaceAnalyzer::lsp_diagnostic]]
+        @root_diagnostics = {} #: Hash[String, Hash[String, Array[WorkspaceAnalyzer::lsp_diagnostic]]]
+      end
+
+      # @rbs (String uri, Integer version, String source) -> Array[publication]
+      def open(uri, version, source)
+        path = workspace.path(uri)
+        raise ProtocolError.new("document is already open: #{uri}", code: -32_602) if @snapshots.dig(path, :open)
+
+        validate_version(version)
+        validate_source(source)
+        loader.set_overlay(path, source)
+        @snapshots[path] = snapshot(uri, path, version, source, true, nil)
+        refresh(path)
+      end
+
+      # @rbs (String uri, Integer version, String source) -> Array[publication]
+      def change(uri, version, source)
+        path = workspace.path(uri)
+        current = open_snapshot(path, uri)
+        validate_version(version)
+        unless current.fetch(:version) && version > current.fetch(:version)
+          raise ProtocolError.new("document version must increase monotonically", code: -32_602)
+        end
+
+        validate_source(source)
+        loader.set_overlay(path, source)
+        @snapshots[path] = snapshot(uri, path, version, source, true, nil)
+        refresh(path)
+      end
+
+      # @rbs (String uri, ?source: String?) -> Array[publication]
+      def save(uri, source: nil)
+        path = workspace.path(uri)
+        current = open_snapshot(path, uri)
+        if source
+          validate_source(source)
+          loader.set_overlay(path, source)
+          current = snapshot(uri, path, current.fetch(:version), source, true, nil)
+          @snapshots[path] = current
+        end
+        refresh(path)
+      end
+
+      # Clear the open version first, then replace it with a disk snapshot if one exists.
+      # @rbs (String uri) -> Array[publication]
+      def close(uri)
+        path = workspace.path(uri)
+        open_snapshot(path, uri)
+        clear = { uri: uri, diagnostics: [] } #: publication
+        old_files = files_for(path)
+        was_root = @roots[path]
+        disk_path = disk_path_for_close(path, uri)
+        disk_source = loader.disk_source(disk_path) if disk_path && loader.disk_file?(disk_path)
+        loader.delete_overlay(path)
+        if disk_source
+          @snapshots[path] = snapshot(uri, path, nil, disk_source, false, nil)
+          if was_root
+            analyzed = @analyzer.analyze(path)
+            update_snapshot(path, analyzed)
+            remove_root_state(path)
+            [clear] + publish_paths((old_files + [path]).uniq)
+          else
+            [clear] + refresh(path)
+          end
+        else
+          remove_path(path)
+          [clear] + refresh_dependents(path)
+        end
+      end
+
+      # @rbs (String path) -> snapshot?
+      def snapshot_for(path)
+        @snapshots[path]
+      end
+
+      # Preserve the URI spelling used to open a document; closed files use their canonical URI.
+      # @rbs (String path) -> String
+      def uri_for(path)
+        entry = @snapshots[path]
+        return entry.fetch(:uri) if entry&.fetch(:open)
+
+        workspace.uri(path)
+      end
+
+      # @rbs (String path) -> Array[String]
+      def files_for(path)
+        roots = affected_roots(path)
+        files = roots.flat_map { |root| @closures.fetch(root, [root]) }
+        (files + [path]).uniq
+      end
+
+      # @rbs (String path) -> Array[Frontend::Resolution]
+      def resolutions_for(path)
+        affected_roots(path).filter_map { |root| @resolutions[root] }
+      end
+
+      # @rbs (String path) -> Array[String]
+      def roots_for(path)
+        affected_roots(path)
+      end
+
+      private
+
+      # @rbs (String path) -> Array[publication]
+      def refresh(path)
+        old_files = files_for(path)
+        previous_roots = affected_roots(path)
+        was_root = @roots[path]
+        analyzed = @analyzer.analyze(path)
+        update_snapshot(path, analyzed)
+        update_root_membership(path, analyzed.fetch(:document))
+        remove_root_state(path) if was_root && !@roots[path]
+        roots = (previous_roots + affected_roots(path)).uniq
+        roots.delete(path) unless @roots[path]
+        roots.each { |root| refresh_root(root) }
+        publish_paths((old_files + files_for(path) + [path]).uniq)
+      end
+
+      # @rbs (String path) -> Array[publication]
+      def refresh_dependents(path)
+        roots = affected_roots(path)
+        roots.each { |root| refresh_root(root) }
+        publish_paths(roots.flat_map { |root| @closures.fetch(root, [root]) }.uniq)
+      end
+
+      # @rbs (String root) -> void
+      def refresh_root(root)
+        files, analyzed = @analyzer.closure(root)
+        @closures[root] = files
+        rebuild_reverse_dependencies
+        analyzed.each { |path, result| update_snapshot(path, result, own_diagnostics: false) }
+        parse_failed = analyzed.any? do |path, result|
+          !result.fetch(:diagnostics).empty? && loader.file?(path)
+        end
+        resolution, errors = parse_failed ? [nil, []] : @analyzer.resolve(root)
+        resolution ? @resolutions[root] = resolution : @resolutions.delete(root)
+        replace_root_diagnostics(root, files, analyzed, errors)
+      end
+
+      # @rbs () -> void
+      def rebuild_reverse_dependencies
+        reverse = Hash.new { |hash, key| hash[key] = [] } #: Hash[String, Array[String]]
+        @closures.each { |root, files| files.each { |file| reverse[file] << root } }
+        @reverse_dependencies = reverse
+      end
+
+      # @rbs (String path) -> Array[String]
+      def affected_roots(path)
+        roots = @reverse_dependencies.fetch(path, empty_paths).dup
+        roots << path if @roots[path]
+        roots.uniq
+      end
+
+      # @rbs (String path, WorkspaceAnalyzer::analyzed_document analyzed, ?own_diagnostics: bool) -> void
+      def update_snapshot(path, analyzed, own_diagnostics: true)
+        current = @snapshots[path]
+        uri = current&.fetch(:uri) || workspace.uri(path)
+        version = current&.fetch(:version)
+        open = current&.fetch(:open) || false
+        source = open ? current.fetch(:source) : analyzed.fetch(:source)
+        document = analyzed.fetch(:document)
+        @snapshots[path] = snapshot(uri, path, version, source, open, document)
+        @document_diagnostics[path] = analyzed.fetch(:diagnostics) if own_diagnostics
+      end
+
+      # @rbs (String path, Frontend::SourceDocument? document) -> void
+      def update_root_membership(path, document)
+        if document&.ast.is_a?(Frontend::AST::Root)
+          @roots[path] = true
+        else
+          @roots.delete(path)
+        end
+      end
+
+      # @rbs (Array[String] paths) -> Array[publication]
+      def publish_paths(paths)
+        paths.filter_map do |path|
+          entry = @snapshots[path]
+          next unless entry
+
+          publication = {
+            uri: entry.fetch(:uri),
+            diagnostics: aggregated_diagnostics(path)
+          } #: publication
+          publication[:version] = entry.fetch(:version) if entry.fetch(:open) && entry.fetch(:version)
+          publication
+        end
+      end
+
+      # @rbs (String uri, String path, Integer? version, String source, bool open,
+      #   Frontend::SourceDocument? document) -> snapshot
+      def snapshot(uri, path, version, source, open, document)
+        { uri: uri, path: path, version: version, source: source, open: open, document: document }
+      end
+
+      # @rbs (String path, String uri) -> snapshot
+      def open_snapshot(path, uri)
+        entry = @snapshots[path]
+        return entry if entry&.fetch(:open) && entry.fetch(:uri) == uri
+
+        raise ProtocolError.new("document is not open: #{uri}", code: -32_602)
+      end
+
+      # @rbs () -> Array[String]
+      def empty_paths
+        [] #: Array[String]
+      end
+
+      # @rbs (String path) -> void
+      def remove_path(path)
+        @snapshots.delete(path)
+        @document_diagnostics.delete(path)
+        remove_root_state(path)
+      end
+
+      # @rbs (String path) -> void
+      def remove_root_state(path)
+        @roots.delete(path)
+        @closures.delete(path)
+        @resolutions.delete(path)
+        @root_diagnostics.delete(path)
+        rebuild_reverse_dependencies
+      end
+    end
+  end
+end

data/lib/ibex/lsp/document_store_diagnostics.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Ibex
+  module LSP
+    # Owns per-document and per-root diagnostic aggregation for DocumentStore.
+    module DocumentStoreDiagnostics
+      private
+
+      # @rbs (String root, Array[String] files, Hash[String, WorkspaceAnalyzer::analyzed_document] analyzed,
+      #   Array[WorkspaceAnalyzer::lsp_diagnostic] errors) -> void
+      def replace_root_diagnostics(root, files, analyzed, errors)
+        # @type self: DocumentStore
+        owned = {} #: Hash[String, Array[WorkspaceAnalyzer::lsp_diagnostic]]
+        files.each { |path| owned[path] = analyzed.dig(path, :diagnostics) || [] }
+        errors.each do |diagnostic|
+          file = diagnostic.dig("data", "file")
+          next unless file
+
+          existing = owned[file] || []
+          duplicate = existing.any? do |entry|
+            entry["range"] == diagnostic["range"] && entry["message"] == diagnostic["message"]
+          end
+          owned[file] = existing + [diagnostic] unless duplicate
+        end
+        @root_diagnostics[root] = owned
+      end
+
+      # @rbs (String path) -> Array[WorkspaceAnalyzer::lsp_diagnostic]
+      def aggregated_diagnostics(path)
+        # @type self: DocumentStore
+        diagnostics = @document_diagnostics.fetch(path, empty_diagnostics).dup
+        @root_diagnostics.each_value do |owned|
+          diagnostics.concat(owned.fetch(path, empty_diagnostics))
+        end
+        diagnostics.uniq
+      end
+
+      # @rbs () -> Array[WorkspaceAnalyzer::lsp_diagnostic]
+      def empty_diagnostics
+        [] #: Array[WorkspaceAnalyzer::lsp_diagnostic]
+      end
+    end
+  end
+end

data/lib/ibex/lsp/document_store_validation.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Ibex
+  module LSP
+    # Validates versions, source bounds, and prospective multi-file workspace edits.
+    module DocumentStoreValidation
+      # @rbs (Hash[String, String] replacements) -> bool
+      def valid_replacements?(replacements)
+        # @type self: DocumentStore
+        candidate = Frontend::SourceLoader.new
+        @snapshots.each_value do |entry|
+          candidate.set_overlay(entry.fetch(:path), entry.fetch(:source)) if entry.fetch(:open)
+        end
+        replacements.each do |path, source|
+          candidate.set_overlay(path, source)
+          parse_candidate(path, source)
+        end
+        roots = replacements.keys.flat_map { |path| affected_roots(path) }.uniq
+        roots.each { |root| Frontend::Resolver.new(root, mode: :extended, loader: candidate).resolve }
+        true
+      rescue Ibex::Error, SystemCallError
+        false
+      end
+
+      private
+
+      # @rbs (Integer version) -> void
+      def validate_version(version)
+        raise ProtocolError.new("document version must be a non-negative integer", code: -32_602) unless
+          version.is_a?(Integer) && !version.negative?
+      end
+
+      # @rbs (String source) -> void
+      def validate_source(source)
+        raise ProtocolError.new("document text must be a string", code: -32_602) unless source.is_a?(String)
+        return if source.bytesize <= Limits::MAX_DOCUMENT_BYTES
+
+        raise ProtocolError.new("document exceeds #{Limits::MAX_DOCUMENT_BYTES} bytes", code: -32_602)
+      end
+
+      # @rbs (String path, String source) -> void
+      def parse_candidate(path, source)
+        Frontend::Parser.new(source, file: path, mode: :extended).parse_source_document
+      end
+
+      # Resolve disk independently from overlay aliases before removing an open buffer.
+      # @rbs (String path, String uri) -> String?
+      def disk_path_for_close(path, uri)
+        # @type self: DocumentStore
+        canonical = loader.disk_path(path)
+        return canonical if workspace.root_for(canonical)
+
+        raise ProtocolError.new("document disk target is outside workspace roots: #{uri}", code: -32_602)
+      rescue Errno::ENOENT, Errno::ENOTDIR
+        nil
+      rescue SystemCallError => e
+        raise ProtocolError.new("cannot resolve document disk target #{uri.inspect}: #{e.message}", code: -32_602)
+      end
+    end
+  end
+end

data/lib/ibex/lsp/initialization_handlers.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Ibex
+  module LSP
+    # Handles initialize/initialized/shutdown/exit lifecycle transitions.
+    module InitializationHandlers
+      include RequestSupport
+
+      private
+
+      # @rbs (lsp_value? raw_params) -> lsp_object
+      def initialize_workspace(raw_params)
+        raise ProtocolError.new("server is already initialized", code: -32_600) unless @state == :uninitialized
+
+        params = params_hash(raw_params)
+        loader = Frontend::SourceLoader.new
+        @workspace = Workspace.new(initialization_roots(params), loader)
+        @store = DocumentStore.new(@workspace, loader)
+        @state = :initialized
+        {
+          "capabilities" => capabilities,
+          "serverInfo" => { "name" => "ibex", "version" => Ibex::VERSION }
+        }
+      end
+
+      # @rbs (lsp_value? params) -> nil
+      def initialized_notification(_params)
+        require_state!(:initialized)
+        @state = :running
+        nil
+      end
+
+      # @rbs (lsp_value? params) -> nil
+      def shutdown_request(_params)
+        require_state!(:running)
+        @state = :shutdown
+        nil
+      end
+
+      # @rbs (lsp_value? params) -> nil
+      def exit_notification(_params)
+        @exit_status = @state == :shutdown ? 0 : 1
+        @exit_requested = true
+        nil
+      end
+
+      # @rbs () -> lsp_object
+      def capabilities
+        {
+          "positionEncoding" => "utf-16",
+          "textDocumentSync" => { "openClose" => true, "change" => 1, "save" => { "includeText" => true } },
+          "definitionProvider" => true,
+          "referencesProvider" => true,
+          "renameProvider" => { "prepareProvider" => true },
+          "hoverProvider" => true,
+          "completionProvider" => { "resolveProvider" => false },
+          "workspace" => { "workspaceFolders" => { "supported" => true, "changeNotifications" => false } }
+        }
+      end
+
+      # @rbs (lsp_object params) -> Array[String]
+      def initialization_roots(params)
+        folders = params["workspaceFolders"]
+        if folders.is_a?(Array) && !folders.empty?
+          return folders.map do |folder|
+            raise ProtocolError.new("workspaceFolders entries must be objects", code: -32_602) unless folder.is_a?(Hash)
+
+            folder_object = folder #: lsp_object
+            string_member(folder_object, "uri")
+          end
+        end
+
+        root = params["rootUri"]
+        return [root] if root.is_a?(String)
+
+        raise ProtocolError.new("initialize requires rootUri or workspaceFolders", code: -32_602)
+      end
+    end
+  end
+end

data/lib/ibex/lsp/navigation_handlers.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Ibex
+  module LSP
+    # Serves symbol navigation, rename, and hover requests.
+    module NavigationHandlers
+      include RequestSupport
+
+      private
+
+      # @rbs (lsp_object? raw_params) -> Array[lsp_object]
+      def definition(raw_params)
+        with_index(raw_params) { |index, path, position| index.definition(path, position) } #: Array[lsp_object]
+      end
+
+      # @rbs (lsp_object? raw_params) -> Array[lsp_object]
+      def references(raw_params)
+        params = params_hash(raw_params)
+        context = hash_member(params, "context")
+        include_declaration = context["includeDeclaration"] == true
+        with_index(params) do |index, path, position|
+          index.references(path, position, include_declaration: include_declaration)
+        end #: Array[lsp_object]
+      end
+
+      # @rbs (lsp_object? raw_params) -> lsp_object?
+      def prepare_rename(raw_params)
+        with_index(raw_params) { |index, path, position| index.prepare_rename(path, position) } #: lsp_object?
+      end
+
+      # @rbs (lsp_object? raw_params) -> lsp_object
+      def rename(raw_params)
+        params = params_hash(raw_params)
+        new_name = string_member(params, "newName")
+        with_index(params) { |index, path, position| index.rename(path, position, new_name) } #: lsp_object
+      end
+
+      # @rbs (lsp_object? raw_params) -> lsp_object?
+      def hover(raw_params)
+        with_index(raw_params) do |index, path, position|
+          index.hover(path, position) || ParserConfigurationAssistance.new(store, path).hover(position)
+        end #: lsp_object?
+      end
+
+      # @rbs (lsp_object? raw_params) -> lsp_object
+      def completion(raw_params)
+        require_running!
+        params = params_hash(raw_params)
+        document = hash_member(params, "textDocument")
+        path = workspace.path(string_member(document, "uri"))
+        position = hash_member(params, "position")
+        ParserConfigurationAssistance.new(store, path).completion(position)
+      rescue ArgumentError => e
+        raise ProtocolError.new(e.message, code: -32_602)
+      end
+
+      # @rbs (lsp_object? raw_params) { (SymbolIndex, String, lsp_object) -> Object? } -> Object?
+      def with_index(raw_params)
+        require_running!
+        params = params_hash(raw_params)
+        document = hash_member(params, "textDocument")
+        path = workspace.path(string_member(document, "uri"))
+        position = hash_member(params, "position")
+        yield SymbolIndex.new(store, path), path, position
+      rescue ArgumentError => e
+        raise ProtocolError.new(e.message, code: -32_602)
+      end
+    end
+  end
+end

data/lib/ibex/lsp/parser_configuration_assistance.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+module Ibex
+  module LSP
+    # Provides static editor assistance for the root parser declaration without evaluating grammar code.
+    class ParserConfigurationAssistance
+      SETTING_VALUES = Frontend::ParserConfigurationSupport::PARSER_SETTING_VALUES #: Hash[String, Array[String]]
+      SETTING_DOCUMENTATION = {
+        "algorithm" => "`parser.algorithm` selects parser-table construction. Values: `slr`, `lalr`, `ielr`, `lr1`.",
+        "entries" => "`parser.entries` selects shared or isolated construction for multiple start symbols. " \
+                     "Values: `shared`, `isolated`.",
+        "cst_trivia" => "`cst.trivia` selects CST trivia ownership. Values: `leading`, `balanced`, `drop`; " \
+                        "requires `pragma cst`."
+      }.freeze #: Hash[String, String]
+      VALUE_DOCUMENTATION = {
+        "slr" => "`slr` — SLR parser-table construction.",
+        "lalr" => "`lalr` — LALR parser-table construction.",
+        "ielr" => "`ielr` — IELR parser-table construction with canonical LR(1) verification support.",
+        "lr1" => "`lr1` — canonical LR(1) parser-table construction.",
+        "shared" => "`shared` — construct multiple entries in one shared automaton.",
+        "isolated" => "`isolated` — construct each of multiple entries independently.",
+        "leading" => "`leading` — attach leading trivia to the following CST node.",
+        "balanced" => "`balanced` — attach trivia between the surrounding CST nodes.",
+        "drop" => "`drop` — omit CST trivia; location and incremental APIs are unavailable."
+      }.freeze #: Hash[String, String]
+
+      # @rbs (DocumentStore store, String path) -> void
+      def initialize(store, path)
+        @store = store
+        @path = path
+      end
+
+      # @rbs (lsp_object position) -> lsp_object
+      def completion(position)
+        source = source!
+        codec = PositionCodec.new(source)
+        offset = codec.byte_offset(position)
+        line_start = line_start(source, offset)
+        return completion_list([]) unless parser_block_open?(source.byteslice(0, line_start) || "")
+
+        prefix = source.byteslice(line_start, offset - line_start) || ""
+        value_match = prefix.match(/\A\s*(algorithm|entries|cst_trivia)\s+([A-Za-z_]*)\z/)
+        if value_match
+          setting = value_match[1] || raise("parser setting capture is missing")
+          return completion_list(value_items(setting))
+        end
+        return completion_list(setting_items) if prefix.match?(/\A\s*[A-Za-z_]*\z/)
+
+        completion_list([])
+      end
+
+      # @rbs (lsp_object position) -> lsp_object?
+      def hover(position)
+        source = source!
+        codec = PositionCodec.new(source)
+        offset = codec.byte_offset(position)
+        start = line_start(source, offset)
+        return unless parser_block_open?(source.byteslice(0, start) || "")
+
+        finish = source.index("\n", start) || source.bytesize
+        line = source.byteslice(start, finish - start) || ""
+        match = line.match(
+          /\A\s*(algorithm|entries|cst_trivia)\s+(slr|lalr|ielr|lr1|shared|isolated|leading|balanced|drop)\s*(?:#.*)?\z/
+        )
+        return unless match
+
+        token, token_start, token_end = hovered_token(match, line, start, offset)
+        return unless token
+
+        documentation = SETTING_DOCUMENTATION[token] || VALUE_DOCUMENTATION[token]
+        {
+          "contents" => { "kind" => "markdown", "value" => documentation },
+          "range" => { "start" => codec.position(token_start), "end" => codec.position(token_end) }
+        }
+      end
+
+      private
+
+      # @rbs () -> String
+      def source!
+        snapshot = @store.snapshot_for(@path)
+        raise ArgumentError, "document is unavailable" unless snapshot
+
+        snapshot.fetch(:source)
+      end
+
+      # @rbs (String source, Integer offset) -> Integer
+      def line_start(source, offset)
+        newline = offset.positive? ? source.rindex("\n", offset - 1) : nil
+        newline ? newline + 1 : 0
+      end
+
+      # @rbs (String source) -> bool
+      def parser_block_open?(source)
+        root = false
+        open = false
+        source.each_line do |line|
+          content = line.sub(/#.*/, "").strip
+          next if content.empty?
+
+          root = true if content.match?(/\Aclass\b/)
+          root = false if content.match?(/\Afragment\b/)
+          open = true if root && content == "parser"
+          open = false if open && content == "end"
+          return false if content.match?(/\Arule\b/)
+        end
+        root && open
+      end
+
+      # @rbs () -> Array[lsp_object]
+      def setting_items
+        SETTING_VALUES.keys.map do |name|
+          completion_item(name, 10, SETTING_DOCUMENTATION.fetch(name))
+        end
+      end
+
+      # @rbs (String setting) -> Array[lsp_object]
+      def value_items(setting)
+        SETTING_VALUES.fetch(setting).map do |name|
+          completion_item(name, 12, VALUE_DOCUMENTATION.fetch(name))
+        end
+      end
+
+      # @rbs (String label, Integer kind, String documentation) -> lsp_object
+      def completion_item(label, kind, documentation)
+        { "label" => label, "kind" => kind, "documentation" => { "kind" => "markdown", "value" => documentation } }
+      end
+
+      # @rbs (Array[lsp_object] items) -> lsp_object
+      def completion_list(items)
+        { "isIncomplete" => false, "items" => items }
+      end
+
+      # @rbs (MatchData match, String line, Integer line_start, Integer offset) -> [String?, Integer, Integer]
+      def hovered_token(match, line, line_start, offset)
+        [1, 2].each do |capture|
+          token = match[capture]
+          next unless token
+
+          before = line[0...match.begin(capture)] || ""
+          start = line_start + before.bytesize
+          finish = start + token.bytesize
+          return [token, start, finish] if offset.between?(start, finish - 1)
+        end
+        [nil, offset, offset]
+      end
+    end
+  end
+end