canopus 0.1.0

data/lib/canopus/language/background_analysis.rb

@@ -0,0 +1,280 @@
+# frozen_string_literal: true
+
+module Canopus
+  module Language
+    class BackgroundAnalysis
+      EMPTY = [].freeze
+      SOURCE_LIMIT = 2 << 20
+      LINE_LIMIT = 16 << 10
+      ROW_LIMIT = 256
+      CACHE_ROWS = 512
+      Snapshot = Data.define(:rope, :version, :id, :name, :lexer, :first, :last, :syntax)
+      attr_reader :analysis_error
+
+      def initialize(document)
+        @document, @buffer = document, document.buffer
+        @id = "#{Process.pid}:#{object_id}"
+        @tokens, @stale, @provisional, @generation = {}, {}, {}, 0
+        @syntax_rows = {}
+        @disabled = @buffer.rope.respond_to?(:lazy?) && @buffer.rope.lazy?
+      end
+
+      def tokens_for(row)
+        validate_row(row)
+        return @tokens[row] if @tokens.key?(row)
+        unless @disabled || @disposed || @explicit_rows
+          first, last = @range || [row, row]
+          if row < first
+            first, last = row, [last, row + ROW_LIMIT - 1].min
+          elsif row > last
+            first, last = [first, row - ROW_LIMIT + 1].max, row
+          end
+          request(first_line: first, last_line: last)
+        end
+        return @stale[row] if @stale.key?(row)
+        return @provisional[row] if @provisional.key?(row)
+        @provisional.shift if @provisional.length >= CACHE_ROWS
+        @provisional[row] = [["Text", self.class.line(@buffer.rope, row)].freeze].freeze
+      end
+
+      def request(first_line: nil, last_line: nil, rows: nil, syntax: false)
+        return if @disabled || @disposed
+        if rows
+          unless rows.is_a?(Array) && rows.length.between?(1, ROW_LIMIT) && first_line.nil? && last_line.nil?
+            raise ArgumentError, "language rows must contain 1..#{ROW_LIMIT} entries without a range"
+          end
+          rows.each { |row| validate_row(row) }
+          rows = rows.uniq.sort.freeze
+          @explicit_rows = true
+        elsif first_line || last_line || !@rows
+          first = first_line || @range&.first || 0
+          last = last_line || @range&.last || first
+          validate_row(first)
+          validate_row(last)
+          raise ArgumentError, "inverted language range" if last < first
+          rows = (first..[last, first + ROW_LIMIT - 1].min).to_a.freeze
+          @explicit_rows = false
+        else
+          rows = @rows
+        end
+        changed = rows != @rows || (syntax && !@want_syntax)
+        if rows != @rows && !syntax_complete?
+          @syntax = nil
+          @syntax_rows.clear
+        end
+        @want_syntax ||= syntax
+        if changed
+          @job&.cancel
+          @job = nil
+          @generation += 1
+          @failed = false
+          @analysis_error = nil
+        end
+        @rows, @range = rows, [rows.first, rows.last].freeze
+        @dirty = !@failed && missing_work?
+        self
+      end
+
+      def pending? = !@disposed && !@disabled && (!!@job || !!@dirty)
+      def tokens_current?(row) = @tokens.key?(row)
+      def syntax_ready? = !!@syntax
+      def syntax_complete? = !!(@syntax && @syntax[:complete])
+      def outline = (request(syntax: true); @syntax ? @syntax[:outline] : EMPTY)
+      def diagnostics = (request(syntax: true); @syntax ? @syntax[:diagnostics] : EMPTY)
+      def fold_ranges = (request(syntax: true); @syntax ? @syntax[:folds] : EMPTY)
+      def bracket_at(offset) = (request(syntax: true); @syntax && @syntax[:brackets][offset])
+
+      def invalidate(patch)
+        @job&.cancel
+        @job = @syntax = nil
+        @syntax_rows.clear
+        if patch.is_a?(Patch::Reload)
+          @tokens.clear
+          @stale.clear
+        else
+          (patch.is_a?(Patch::Composite) ? patch.patches : [patch]).each { |item| retain_unchanged_rows(item) }
+        end
+        @provisional.clear
+        @generation += 1
+        @failed = false
+        @analysis_error = nil
+        @disabled = @buffer.rope.respond_to?(:lazy?) && @buffer.rope.lazy?
+        @rows = @rows&.map { |row| [row, @buffer.line_count - 1].min }&.uniq&.freeze
+        @range = [@rows.first, @rows.last].freeze if @rows
+        @dirty = !!@rows && !@disabled
+      end
+
+      def poll
+        return false unless pending?
+        changed = false
+        if @job&.future&.done?
+          job, @job = @job, nil
+          result = job.future.await(timeout: 0)
+          if @submitted_version == @buffer.version && @submitted_generation == @generation
+            result[:tokens].each { |row, tokens| @tokens[row] = tokens; @stale.delete(row) }
+            while @tokens.length > CACHE_ROWS
+              @tokens.delete(@tokens.each_key.find { |row| !@rows.include?(row) })
+            end
+            @stale.shift while @stale.length + @tokens.length > CACHE_ROWS
+            if result[:syntax]
+              @syntax = result[:syntax]
+              result[:tokens].each_key { |row| @syntax_rows[row] = true }
+            end
+            @dirty = missing_work?
+            changed = true
+          end
+        end
+        if @dirty && !@job
+          @scheduler ||= Scheduler.acquire
+          # A folded viewport can contain distant source rows. Fill one missing
+          # contiguous run at a time without paint changing/cancelling its job.
+          missing = @rows.select { |row| row_needed?(row) }
+          missing = @rows if missing.empty?
+          first = last = missing.first
+          missing.drop(1).each do |row|
+            break unless row == last + 1
+            last = row
+          end
+          snapshot = Snapshot.new(@buffer.rope, @buffer.version, @id,
+            @document.definition.name, @document.definition.lexer, first, last, !!(@want_syntax && !syntax_complete?))
+          if (@job = @scheduler.submit(snapshot, prior_syntax: @syntax))
+            @submitted_version, @submitted_generation = @buffer.version, @generation
+          end
+        end
+        changed
+      rescue StandardError => error
+        @job&.cancel
+        @job = nil
+        @failed, @dirty = true, false
+        @analysis_error = error
+        raise
+      end
+
+      def dispose
+        return if @disposed
+        @disposed = true
+        @job&.cancel
+        @job = nil
+        Scheduler.release(@scheduler) if @scheduler
+        @scheduler = nil
+        @tokens.clear
+        @stale.clear
+        @provisional.clear
+        @syntax_rows.clear
+        @syntax = nil
+        @analysis_error = nil
+      end
+
+      def self.line(rope, row)
+        first = rope.line_start(row)
+        last = row + 1 < rope.line_count ? rope.line_start(row + 1) : rope.bytesize
+        ending = boundary(rope, [last, first + LINE_LIMIT].min)
+        rope.byteslice(first, ending - first).to_s.freeze
+      end
+
+      def self.boundary(rope, offset)
+        rope.point_at(offset)
+        offset
+      rescue RangeError
+        offset -= 1
+        retry
+      end
+
+      def self.prepare(snapshot, job)
+        rope, first, last = snapshot.rope, snapshot.first, snapshot.last
+        complete = rope.bytesize <= 1 << 20
+        from, to, context = 0, rope.bytesize, 0
+        unless complete
+          context = rope.bytesize <= 10 << 20 ? 500 : 0
+          start_row = [first - context, 0].max
+          finish_row = [last + context + 1, rope.line_count].min
+          requested_end = last + 1 < rope.line_count ? rope.line_start(last + 1) : rope.bytesize
+          while start_row < first && requested_end - rope.line_start(start_row) > SOURCE_LIMIT
+            job.check!
+            start_row += 1
+          end
+          from = rope.line_start(start_row)
+          to = finish_row < rope.line_count ? rope.line_start(finish_row) : rope.bytesize
+          to = boundary(rope, [to, from + SOURCE_LIMIT].min)
+        end
+        # A capped snapshot may end before later requested rows, or exactly at
+        # the start of a row whose content was not copied. Never cache that row
+        # as an empty, supposedly current token result; the next batch reads it.
+        end_row = rope.point_at(to).row
+        end_row -= 1 if to < rope.bytesize && rope.line_start(end_row) == to
+        last = [last, end_row].min
+        job.check!
+        source = rope.byteslice(from, to - from).to_s.freeze
+        {"id" => snapshot.id, "version" => snapshot.version, "name" => snapshot.name,
+          "lexer" => snapshot.lexer, "source" => source, "base" => from,
+          "base_line" => rope.point_at(from).row, "first" => first, "last" => last,
+          "syntax" => snapshot.syntax, "complete" => complete, "context" => context}.freeze
+      end
+
+      def self.decode(response, prior_syntax: nil)
+        tokens = response.fetch("tokens").to_h do |row, pairs|
+          [row, pairs.map { |name, value| [name.freeze, value.freeze].freeze }.freeze]
+        end.freeze
+        raw = response["syntax"]
+        syntax = if raw
+          outline = raw.fetch("outline").map do |name, kind, first, last, start, finish, depth|
+            Language::Symbol.new(name.freeze, kind.to_sym, (first...last).freeze, (start...finish).freeze, depth)
+          end.freeze
+          brackets = {}
+          raw.fetch("brackets").each do |first, last|
+            range = (first...last).freeze
+            brackets[first] = brackets[last - 1] = range
+          end
+          {outline: outline, diagnostics: raw.fetch("diagnostics").map { |first, last, message| {range: (first...last).freeze, message: message.freeze, severity: 1}.freeze }.freeze,
+            folds: raw.fetch("folds").map { |first, last| (first...last).freeze }.freeze,
+            brackets: brackets.freeze, complete: response.fetch("complete")}.freeze
+        end
+        syntax = merge_syntax(prior_syntax, syntax) if prior_syntax && syntax && !syntax[:complete]
+        {tokens: tokens, syntax: syntax}.freeze
+      end
+
+      # Decoding and bounded merging happen on a preparation thread, never in
+      # paint/poll. Overlapping context windows must not duplicate symbols.
+      def self.merge_syntax(previous, current)
+        return previous if previous[:complete]
+        outlines = (previous[:outline] + current[:outline]).uniq { |item| [item.name, item.kind, item.range, item.selection] }
+        brackets = (previous[:brackets].values + current[:brackets].values).uniq.first(8192)
+        {outline: outlines.sort_by { |item| item.selection.begin }.first(10_000).freeze,
+          diagnostics: (previous[:diagnostics] + current[:diagnostics]).uniq.first(1000).freeze,
+          folds: (previous[:folds] + current[:folds]).uniq.sort_by(&:begin).first(10_000).freeze,
+          brackets: brackets.each_with_object({}) { |range, result| result[range.begin] = result[range.end - 1] = range }.freeze,
+          complete: false}.freeze
+      end
+
+      private
+      def row_needed?(row) = !@tokens.key?(row) || (@want_syntax && !syntax_complete? && !@syntax_rows.key?(row))
+      def missing_work? = @rows.any? { |row| row_needed?(row) }
+
+      def retain_unchanged_rows(patch)
+        return if patch.edits.empty?
+        changes = patch.edits.map do |edit|
+          first, last = patch.before.point_at(edit.old_range.begin).row, patch.before.point_at(edit.old_range.end).row
+          inserted = patch.after.point_at(edit.new_range.end).row - patch.after.point_at(edit.new_range.begin).row
+          [first, last, inserted - (last - first)]
+        end
+        earliest = changes.map(&:first).min
+        fresh, stale = {}, {}
+        [@stale, @tokens].each do |cache|
+          cache.each do |row, tokens|
+            next if changes.any? { |first, last, _| row.between?(first, last) }
+            shift = changes.sum { |_, last, delta| row > last ? delta : 0 }
+            destination = cache.equal?(@tokens) && row < earliest ? fresh : stale
+            destination[row + shift] = tokens
+          end
+        end
+        @tokens, @stale = fresh, stale
+      end
+      def validate_row(row)
+        raise RangeError, "language row outside buffer" unless row.is_a?(Integer) && row >= 0 && row < @buffer.line_count
+      end
+    end
+  end
+end
+
+require_relative "background_analysis/job"
+require_relative "background_analysis/scheduler"

data/lib/canopus/language/definition.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Canopus
+  module Language
+    Definition = Data.define(:name, :lexer, :extensions, :comment, :indent_open, :indent_close, :servers)
+  end
+end

data/lib/canopus/language/document.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+module Canopus
+  module Language
+    class Document
+      attr_reader :buffer, :definition
+      def initialize(buffer, definition: Language.for_path(buffer.path), background: true)
+        @buffer, @definition, @syntax_version = buffer, definition, -1
+        @background = BackgroundAnalysis.new(self) if background
+        @subscription = buffer.on_edit { |patch| invalidate(patch) }
+      end
+      def dispose
+        @subscription.detach
+        @background&.dispose
+      end
+      def background? = !!@background
+      def pending? = @background ? @background.pending? : false
+      def analysis_error = @background&.analysis_error
+      def poll = @background ? @background.poll : false
+      def request(first_line: nil, last_line: nil, rows: nil, syntax: false)
+        @background&.request(first_line: first_line, last_line: last_line, rows: rows, syntax: syntax)
+        self
+      end
+      def syntax_complete? = @background ? @background.syntax_complete? : true
+      def syntax_ready? = @background ? @background.syntax_ready? : true
+      def tokens_current?(row) = @background ? @background.tokens_current?(row) : true
+      def tokens_for(row)
+        return @background.tokens_for(row) if @background
+        highlighter.tokens_for(row)
+      end
+      # Explicit synchronous escape hatch. Asynchronous UI paths never call this.
+      def highlighter(strategy: nil, window_context: nil)
+        return @highlighter if @highlighter
+        require "antares"
+        large = @buffer.rope.bytesize > 1 << 20
+        @highlighter = Antares::Highlighter.new(lexer: Rouge::Lexer.find(@definition.lexer).new,
+          lines: ->(row) { @buffer.rope.byteslice(@buffer.rope.line_start(row), (row + 1 < @buffer.line_count ? @buffer.rope.line_start(row + 1) : @buffer.rope.bytesize) - @buffer.rope.line_start(row)).to_s },
+          line_count: -> { @buffer.line_count }, strategy: strategy || (large ? :window : :auto),
+          window_context: window_context || (@buffer.rope.bytesize > 10 << 20 ? 0 : 500))
+      end
+      def invalidate(patch)
+        @background&.invalidate(patch)
+        return unless @highlighter
+        if patch.is_a?(Patch::Composite) || patch.is_a?(Patch::Reload)
+          @highlighter = nil
+        else
+          first = patch.edits.map { |edit| patch.before.point_at(edit.old_range.begin).row }.min
+          old_last = patch.edits.map { |edit| patch.before.point_at(edit.old_range.end).row }.max
+          new_last = patch.edits.map { |edit| patch.after.point_at(edit.new_range.end).row }.max
+          @highlighter.edit(from_line: first, removed: old_last - first + 1, inserted: new_last - first + 1)
+        end
+      end
+      # Explicit local Prism API; use outline/diagnostics for nonblocking UI work.
+      def ruby_syntax
+        return nil unless @definition.name == "ruby"
+        if @syntax_version != @buffer.version
+          require "prism"
+          @syntax = Prism.parse(@buffer.text)
+          @syntax_version = @buffer.version
+        end
+        @syntax
+      end
+      def outline
+        return @background.outline if @background
+        return [] unless ruby_syntax
+        symbols, stack = [], [[ruby_syntax.value, 0]]
+        until stack.empty?
+          node, depth = stack.pop
+          kind = case node
+          when Prism::DefNode then :method
+          when Prism::ClassNode then :class
+          when Prism::ModuleNode then :module
+          end
+          if kind
+            name = kind == :method ? node.name.to_s : node.constant_path.slice
+            location = kind == :method ? node.name_loc : node.constant_path.location
+            symbols << Symbol.new(name, kind, node.location.start_offset...node.location.end_offset,
+              location.start_offset...location.end_offset, depth)
+          end
+          node.compact_child_nodes.reverse_each { |child| stack << [child, depth + (kind ? 1 : 0)] }
+        end
+        symbols
+      end
+      def diagnostics
+        return @background.diagnostics if @background
+        return [] unless ruby_syntax
+        ruby_syntax.errors.map { |error| {range: error.location.start_offset...error.location.end_offset, message: error.message, severity: 1} }
+      end
+      def fold_ranges
+        return @background.fold_ranges if @background
+        outline.filter_map { |symbol| symbol.range if @buffer.rope.point_at(symbol.range.end).row > @buffer.rope.point_at(symbol.range.begin).row }
+      end
+      def indent_for(offset, tab_size: 4, use_tabs: false)
+        row = @buffer.rope.point_at(offset).row
+        before = @buffer.rope.byteslice(@buffer.rope.line_start(row), offset - @buffer.rope.line_start(row)).to_s
+        indentation = before[/\A[ \t]*/]
+        indentation += use_tabs ? "\t" : " " * tab_size if @definition.indent_open.match?(before)
+        indentation
+      end
+      def bracket_at(offset)
+        return @background.bracket_at(offset) if @background
+        bracket_ranges[offset]
+      end
+
+      private
+      def bracket_ranges
+        return @brackets if @brackets_version == @buffer.version
+        pairs = {"(" => ")", "[" => "]", "{" => "}"}
+        tokens = if @definition.name == "ruby"
+          require "prism"
+          Prism.lex(@buffer.text).value.filter_map do |token, _state|
+            next unless %w[PARENTHESIS_LEFT PARENTHESIS_RIGHT BRACKET_LEFT BRACKET_RIGHT BRACE_LEFT BRACE_RIGHT].include?(token.type.to_s)
+            [token.location.start_offset, token.value]
+          end
+        else
+          require "rouge"
+          collected, byte = [], 0
+          Rouge::Lexer.find(@definition.lexer).new.lex(@buffer.text).each do |token, value|
+            unless token.qualname.start_with?("Literal.String", "Comment")
+              value.b.scan(/[()\[\]{}]/n) { |character| collected << [byte + Regexp.last_match.begin(0), character] }
+            end
+            byte += value.bytesize
+          end
+          collected
+        end
+        stack, @brackets = [], {}
+        tokens.each do |position, character|
+          if pairs.key?(character)
+            stack << [position, character]
+          elsif stack.last && pairs[stack.last[1]] == character
+            opening = stack.pop[0]
+            range = (opening...position + character.bytesize).freeze
+            @brackets[opening] = @brackets[position] = range
+          else
+            stack.clear
+          end
+        end
+        @brackets_version = @buffer.version
+        @brackets
+      end
+    end
+  end
+end

data/lib/canopus/language/symbol.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Canopus
+  module Language
+    Symbol = Data.define(:name, :kind, :range, :selection, :depth)
+  end
+end

data/lib/canopus/language/syntax_worker.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require_relative "../../canopus"
+
+module Canopus
+  module Language
+    # Fixed ProcessPool handler. This state never lives on the UI thread.
+    module SyntaxWorker
+      MAX_DOCUMENTS = 4
+      MAX_CACHE_BYTES = 4 << 20
+      MAX_SYMBOLS = 10_000
+      MAX_BRACKETS = 8192
+
+      def self.call(request)
+        source = request.fetch("source")
+        unless source.is_a?(String) && source.valid_encoding? && source.encoding == Encoding::UTF_8 && source.bytesize <= BackgroundAnalysis::SOURCE_LIMIT
+          raise ArgumentError, "language snapshot must be bounded UTF-8"
+        end
+        first_row, last_row, base_line, base = request.values_at("first", "last", "base_line", "base")
+        unless [first_row, last_row, base_line, base].all? { |value| value.is_a?(Integer) && value >= 0 } &&
+            base_line <= first_row && last_row >= first_row && last_row - first_row < BackgroundAnalysis::ROW_LIMIT
+          raise ArgumentError, "language token range exceeds limit"
+        end
+        @documents ||= {}
+        key = [request.fetch("id"), base, request.fetch("name"), request.fetch("lexer"), request.fetch("complete")]
+        cached = @documents.delete(key)
+        if cached
+          buffer, document, previous = cached
+          update(buffer, previous, source) unless previous == source
+        else
+          buffer = Buffer.new(source)
+          definition = Language::Definition.new(request.fetch("name"), request.fetch("lexer"), [], "", /\A\z/, /\A\z/, [])
+          document = Document.new(buffer, definition: definition, background: false)
+        end
+        @documents[key] = [buffer, document, source]
+        while @documents.length > MAX_DOCUMENTS || @documents.values.sum { |entry| entry[2].bytesize } > MAX_CACHE_BYTES
+          _, discarded = @documents.shift
+          discarded[1].dispose
+        end
+        first = request.fetch("first") - request.fetch("base_line")
+        last = [request.fetch("last") - request.fetch("base_line"), buffer.line_count - 1].min
+        highlighter = document.highlighter(strategy: request.fetch("complete") ? nil : :window,
+          window_context: request.fetch("complete") ? nil : request.fetch("context"))
+        rows = first <= last ? highlighter.tokens_in(first..last) : []
+        budget = 65_536
+        tokens = (request.fetch("first")..request.fetch("last")).map.with_index do |row, index|
+          pairs = rows[index] || []
+          if pairs.length > 4096 || pairs.length > budget || pairs.sum { |_, text| text.bytesize } > BackgroundAnalysis::LINE_LIMIT
+            local = row - request.fetch("base_line")
+            plain = local < buffer.line_count ? BackgroundAnalysis.line(buffer.rope, local) : ""
+            values = [["Text", plain]]
+          else
+            values = pairs.map { |token, text| [token.qualname, text] }
+          end
+          budget -= values.length
+          [row, values]
+        end
+        syntax = analyze(document, request.fetch("base"), request.fetch("complete")) if request.fetch("syntax")
+        complete = request.fetch("complete") && (!syntax || !syntax.delete("truncated"))
+        {"tokens" => tokens, "syntax" => syntax, "complete" => complete}
+      end
+
+      def self.update(buffer, previous, source)
+        first, shared = 0, [previous.bytesize, source.bytesize].min
+        first += 4096 while first + 4096 <= shared && previous.byteslice(first, 4096) == source.byteslice(first, 4096)
+        first += 1 while first < shared && previous.getbyte(first) == source.getbyte(first)
+        first -= 1 while first.positive? && previous.getbyte(first)&.&(0xc0) == 0x80
+        tail = 0
+        tail += 4096 while tail + 4096 <= shared - first && previous.byteslice(previous.bytesize - tail - 4096, 4096) == source.byteslice(source.bytesize - tail - 4096, 4096)
+        tail += 1 while tail < shared - first && previous.getbyte(previous.bytesize - tail - 1) == source.getbyte(source.bytesize - tail - 1)
+        tail -= 1 while tail.positive? && previous.getbyte(previous.bytesize - tail)&.&(0xc0) == 0x80
+        buffer.edit([[first...(previous.bytesize - tail), source.byteslice(first, source.bytesize - first - tail)]])
+        buffer.history.clear
+      end
+
+      def self.analyze(document, base, complete)
+        all_symbols = document.outline
+        outline = all_symbols.first(MAX_SYMBOLS).map do |symbol|
+          [symbol.name, symbol.kind.to_s, base + symbol.range.begin, base + symbol.range.end,
+            base + symbol.selection.begin, base + symbol.selection.end, symbol.depth]
+        end
+        diagnostics = complete ? document.diagnostics.first(1000).map { |item| [base + item[:range].begin, base + item[:range].end, item[:message]] } : []
+        all_brackets = document.__send__(:bracket_ranges).values.uniq
+        brackets = all_brackets.first(MAX_BRACKETS).map { |range| [base + range.begin, base + range.end] }
+        folds = document.fold_ranges.first(MAX_SYMBOLS).map { |range| [base + range.begin, base + range.end] }
+        {"outline" => outline, "diagnostics" => diagnostics, "brackets" => brackets, "folds" => folds,
+          "truncated" => all_symbols.length > MAX_SYMBOLS || all_brackets.length > MAX_BRACKETS || (complete && document.diagnostics.length > 1000)}
+      end
+    end
+  end
+end

data/lib/canopus/language.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative "language/definition"
+require_relative "language/symbol"
+
+module Canopus
+  module Language
+    DEFINITIONS = [
+      ["ruby", "ruby", %w[.rb .rake .gemspec Gemfile Rakefile], "#", /(?:^\s*(?:class|module|def|if|unless|case|while|until|for|begin)\b.*|(?:\bdo|[\[{(])(?:\s*\|[^|]*\|)?\s*)$/, /^\s*(?:end\b|else\b|elsif\b|rescue\b|ensure\b|[\]})])/, [["ruby-lsp"]]],
+      ["javascript", "javascript", %w[.js .jsx .mjs .cjs], "//", /[\[{(]\s*$/, /^\s*[\]})]/, [["typescript-language-server", "--stdio"]]],
+      ["typescript", "typescript", %w[.ts .tsx], "//", /[\[{(]\s*$/, /^\s*[\]})]/, [["typescript-language-server", "--stdio"]]],
+      ["python", "python", %w[.py .pyi], "#", /:\s*(?:#.*)?$/, /^\s*(?:else|elif|except|finally)\b/, [["pyright-langserver", "--stdio"]]],
+      ["rust", "rust", %w[.rs], "//", /[\[{(]\s*$/, /^\s*[\]})]/, [["rust-analyzer"]]],
+      ["go", "go", %w[.go], "//", /[\[{(]\s*$/, /^\s*[\]})]/, [["gopls"]]],
+      ["c", "c", %w[.c .h], "//", /[\[{(]\s*$/, /^\s*[\]})]/, [["clangd"]]],
+      ["cpp", "cpp", %w[.cpp .hpp .cc .cxx], "//", /[\[{(]\s*$/, /^\s*[\]})]/, [["clangd"]]],
+      ["json", "json", %w[.json .jsonc], "//", /[\[{]\s*$/, /^\s*[\]}]/, []],
+      ["yaml", "yaml", %w[.yml .yaml], "#", /:\s*$/, /^\s*$/, []],
+      ["toml", "toml", %w[.toml], "#", /[\[{]\s*$/, /^\s*[\]}]/, []],
+      ["markdown", "markdown", %w[.md .markdown], "<!--", /\A\z/, /\A\z/, []],
+      ["html", "html", %w[.html .erb], "<!--", /<[^\/>]+>\s*$/, /^\s*<\//, []],
+      ["css", "css", %w[.css .scss], "/*", /\{\s*$/, /^\s*}/, []],
+      ["shellscript", "shell", %w[.sh .bash .zsh], "#", /\b(?:then|do|case)\s*$/, /^\s*(?:fi|done|esac)\b/, []]
+    ].map { |values| Definition.new(*values) }.freeze
+    PLAIN = Definition.new("text", "plaintext", [], "#", /\A\z/, /\A\z/, [])
+    def self.for_path(path)
+      return PLAIN unless path
+      extension, base = File.extname(path), File.basename(path)
+      DEFINITIONS.find { |language| language.extensions.include?(extension) || language.extensions.include?(base) } || PLAIN
+    end
+    def self.executable?(name)
+      paths = name.include?(File::SEPARATOR) ? [name] : ENV.fetch("PATH", "").split(File::PATH_SEPARATOR).map { |path| File.join(path, name) }
+      paths.any? { |path| File.file?(path) && File.executable?(path) }
+    end
+
+  end
+end
+
+require_relative "language/background_analysis"
+require_relative "language/document"
+Canopus::Language.send(:private_constant, :BackgroundAnalysis)
+require_relative "snippet"