antares 0.1.0

data/lib/antares/highlighter.rb

@@ -0,0 +1,284 @@
+# frozen_string_literal: true
+
+require "timeout"
+
+module Antares
+  class Highlighter
+    STRATEGIES = %i[auto incremental window full].freeze
+    attr_reader :strategy, :requested_strategy, :frontier, :checkpoints, :last_scanned_lines, :fallback_reason
+
+    def initialize(lexer:, lines:, line_count:, strategy: :auto, checkpoint_interval: 64,
+      window_context: 200, max_bytes: 8 * 1024 * 1024, max_lines: 100_000,
+      max_line_bytes: 16_384, max_checkpoint_bytes: 256 * 1024, max_seconds: 0.25)
+      raise ArgumentError, "unknown strategy #{strategy.inspect}" unless STRATEGIES.include?(strategy)
+      raise ArgumentError, "lines and line_count must be callable" unless lines.respond_to?(:call) && line_count.respond_to?(:call)
+      {checkpoint_interval: checkpoint_interval, max_bytes: max_bytes, max_lines: max_lines,
+        max_line_bytes: max_line_bytes, max_checkpoint_bytes: max_checkpoint_bytes}.each do |name, value|
+        raise ArgumentError, "#{name} must be a positive integer" unless value.is_a?(Integer) && value.positive?
+      end
+      raise ArgumentError, "window_context must be nonnegative" unless window_context.is_a?(Integer) && window_context >= 0
+      raise ArgumentError, "max_seconds must be positive" unless max_seconds.is_a?(Numeric) && max_seconds.positive?
+      @template = lexer
+      @lines, @line_count = lines, line_count
+      @requested_strategy = strategy
+      @strategy = strategy == :auto ? Antares.compatible?(lexer) : strategy
+      @checkpoint_interval, @window_context = checkpoint_interval, window_context
+      @max_bytes, @max_lines, @max_line_bytes = max_bytes, max_lines, max_line_bytes
+      @max_checkpoint_bytes, @max_seconds = max_checkpoint_bytes, max_seconds
+      @tokens, @fingerprints, @checkpoints = {}, {}, {}
+      @frontier = @last_scanned_lines = 0
+      @count = count
+      @dirty_until = 0
+      initialize_lexer
+    rescue UnsupportedLexerError => error
+      raise unless strategy == :auto || strategy == :window || strategy == :full
+      @template = lexer
+      @strategy = :window if strategy == :auto
+      @fallback_reason = error.message
+    end
+
+    def tokens_for(index)
+      validate_index(index)
+      advance(until_line: index) unless @tokens.key?(index) && index < frontier
+      @tokens.fetch(index) { plain(index) }
+    end
+
+    def tokens_in(range)
+      indexes = range.to_a
+      return [] if indexes.empty?
+      indexes.each { |index| validate_index(index) }
+      advance(until_line: indexes.max, from_line: indexes.min)
+      indexes.map { |index| @tokens.fetch(index) { plain(index) } }
+    end
+
+    # Provider contents must already reflect this line edit. Indices are zero based.
+    def edit(from_line:, removed:, inserted:)
+      values = [from_line, removed, inserted]
+      raise ArgumentError, "edit fields must be nonnegative integers" unless values.all? { |value| value.is_a?(Integer) && value >= 0 }
+      raise RangeError, "edit outside previous document" unless from_line <= @count && from_line + removed <= @count
+      updated_count = count
+      raise ArgumentError, "line_count disagrees with edit" unless updated_count == @count - removed + inserted
+      @last_scanned_lines = 0
+      if strategy == :incremental
+        @tokens.delete_if { |line, _| line >= frontier } if @old_fingerprints
+        start = @checkpoints.keys.select { |line| line <= from_line && line <= frontier }.max || 0
+        restart = @checkpoints.fetch(start)
+        shift_cache(@tokens, from_line, removed, inserted)
+        shift_cache(@fingerprints, from_line, removed, inserted)
+        shift_cache(@checkpoints, from_line, removed, inserted)
+        @old_fingerprints = @fingerprints.dup
+        @old_checkpoints = @checkpoints.dup
+        @fingerprints.delete_if { |line, _| line > start }
+        @checkpoints.delete_if { |line, _| line > start }
+        @checkpoints[start] = restart
+        @frontier = start
+        @dirty_until = from_line + inserted
+        @lexer = @checkpoints.fetch(start).restore
+      else
+        @tokens.clear
+        @frontier = 0
+      end
+      @count = updated_count
+      @source = @offsets = @driver = nil
+      self
+    end
+
+    # frontier is the first line not yet proven current (an exclusive boundary).
+    def advance(until_line:, from_line: nil)
+      return self if @count.zero?
+      validate_index(until_line)
+      @last_scanned_lines = 0
+      return self if until_line < frontier && @tokens.key?(until_line) &&
+        (from_line.nil? || (from_line..until_line).all? { |line| @tokens.key?(line) })
+      Timeout.timeout(@max_seconds) do
+        if @count > @max_lines
+          fallback!(:window, "document exceeds #{@max_lines} lines")
+        end
+        case strategy
+        when :incremental then advance_incremental(until_line)
+        when :full then advance_full
+        when :window then advance_window(from_line || until_line, until_line)
+        end
+      end
+      self
+    rescue ResourceLimitError, UnsupportedLexerError => error
+      raise if requested_strategy == :incremental && error.is_a?(UnsupportedLexerError)
+      fallback!(:window, error.message)
+      advance_window(from_line || until_line, until_line)
+      self
+    rescue Timeout::Error
+      fallback!(:window, "lexing exceeded #{@max_seconds} seconds")
+      ((from_line || until_line)..until_line).each { |index| @tokens[index] = plain(index) }
+      @frontier = until_line + 1
+      self
+    end
+
+    def checkpoint_bytes = checkpoints.values.sum(&:bytesize)
+
+    private
+
+    def initialize_lexer
+      @lexer = fresh_lexer
+      if strategy == :incremental
+        raise UnsupportedLexerError, "#{@lexer.class.tag} has a custom stream driver" unless LexerDriver.supported?(@lexer)
+        @checkpoints[0] = LexerStateSnapshot.new(@lexer, max_bytes: @max_checkpoint_bytes)
+      end
+    end
+
+    def fresh_lexer
+      lexer = LexerStateSnapshot.copy(@template)
+      lexer.reset!
+      lexer
+    rescue UnsupportedLexerError
+      @template.class.new(@template.options).tap(&:reset!)
+    end
+
+    def count
+      value = @line_count.call
+      raise ArgumentError, "line_count must return a nonnegative integer" unless value.is_a?(Integer) && value >= 0
+      value
+    end
+
+    def validate_index(index)
+      raise RangeError, "line outside document" unless index.is_a?(Integer) && index >= 0 && index < @count
+    end
+
+    def source_line(index)
+      value = @lines.call(index)
+      raise TypeError, "line provider must return a String" unless value.is_a?(String)
+      raise EncodingError, "line provider must return valid UTF-8" unless value.valid_encoding? && [Encoding::UTF_8, Encoding::US_ASCII].include?(value.encoding)
+      raise ArgumentError, "line provider returned multiple logical lines" if value.count("\n") > 1 || (value.include?("\n") && !value.end_with?("\n"))
+      # Providers may omit separators, but an empty final line remains empty.
+      index < @count - 1 && !value.end_with?("\n") ? value + "\n" : value
+    end
+
+    def build_source
+      return if @source
+      source = +""
+      offsets = [0]
+      @count.times do |index|
+        line = source_line(index)
+        raise ResourceLimitError, "line exceeds #{@max_line_bytes} bytes" if line.bytesize > @max_line_bytes
+        raise ResourceLimitError, "document exceeds #{@max_bytes} bytes" if source.bytesize + line.bytesize > @max_bytes
+        source << line
+        offsets << source.bytesize
+      end
+      @source, @offsets = source.freeze, offsets.freeze
+    end
+
+    def advance_incremental(until_line)
+      build_source
+      @driver ||= LexerDriver.new(@lexer, @source, @offsets, start_line: frontier) do |line, tokens|
+        @tokens[line] = tokens
+      end
+      before = @driver.scanned_lines
+      @driver.advance(until_line: until_line) do |line, lexer|
+        fingerprint = LexerStateSnapshot.fingerprint(lexer)
+        previous = @old_fingerprints&.[](line)
+        @fingerprints[line] = fingerprint
+        @frontier = line
+        if line >= @dirty_until && previous == fingerprint && @tokens.key?(line)
+          # Only the declared edited interval changed. Equal state at an aligned
+          # old boundary makes the untouched cached suffix reusable.
+          @frontier += 1 while @tokens.key?(@frontier)
+          @fingerprints.merge!(@old_fingerprints.select { |position, _| position >= line && position <= frontier })
+          @checkpoints.merge!(@old_checkpoints.select { |position, _| position >= line && position <= frontier })
+          @old_fingerprints = @old_checkpoints = nil
+          frontier > until_line
+        else
+          checkpoint = @checkpoints.keys.select { |position| position < line }.max || 0
+          if line - checkpoint >= @checkpoint_interval || line == @count || @checkpoints.key?(line)
+            @checkpoints[line] = LexerStateSnapshot.new(lexer, max_bytes: @max_checkpoint_bytes)
+          end
+          false
+        end
+      end
+      @last_scanned_lines = @driver.scanned_lines - before
+      @frontier = [@frontier, @driver.line].max
+      @old_fingerprints = @old_checkpoints = nil if frontier >= @count
+    end
+
+    def advance_full
+      build_source
+      @tokens = split_tokens(fresh_lexer.lex(@source), 0)
+      @frontier = @count
+      @last_scanned_lines = @count
+    end
+
+    def advance_window(first, last)
+      Timeout.timeout(@max_seconds) { window_tokens(first, last) }
+    rescue Timeout::Error
+      @fallback_reason = "window lexing exceeded #{@max_seconds} seconds"
+      (first..last).each { |index| @tokens[index] = plain(index) }
+      @frontier = last + 1
+    end
+
+    def window_tokens(first, last)
+      first = [first - @window_context, 0].max
+      source = +""
+      selected = []
+      (first..last).each do |index|
+        line = source_line(index)
+        if line.bytesize > @max_line_bytes || source.bytesize + line.bytesize > @max_bytes
+          @tokens[index] = plain(index)
+        else
+          selected << index
+          source << line
+        end
+      end
+      unless selected.empty?
+        rows = split_tokens(fresh_lexer.lex(source), 0)
+        selected.each_with_index { |index, offset| @tokens[index] = rows.fetch(offset, [].freeze) }
+      end
+      @frontier = last + 1
+      @last_scanned_lines = selected.length
+    end
+
+    def split_tokens(tokens, first)
+      result = {first => []}
+      line = first
+      tokens.each do |type, value|
+        next if value.empty?
+        unless value.include?("\n")
+          row = result[line] ||= []
+          row.last&.first == type ? row.last[1] << value : row << [type, value.dup]
+          next
+        end
+        value.each_line do |part|
+          row = result[line] ||= []
+          row.last&.first == type ? row.last[1] << part : row << [type, part.dup]
+          line += 1 if part.end_with?("\n")
+        end
+      end
+      result.transform_values { |row| row.each { |pair| pair.last.freeze; pair.freeze }.freeze }
+    end
+
+    def plain(index)
+      value = source_line(index)
+      value.empty? ? [].freeze : [[Rouge::Token::Tokens::Text, value.dup.freeze].freeze].freeze
+    end
+
+    def fallback!(strategy, reason)
+      @strategy, @fallback_reason = strategy, reason
+      @driver = @source = @offsets = nil
+      @tokens.clear
+      @checkpoints.clear
+      @fingerprints.clear
+      @old_fingerprints = @old_checkpoints = nil
+      @frontier = 0
+    end
+
+    def shift_cache(cache, first, removed, inserted)
+      delta = inserted - removed
+      shifted = {}
+      cache.each do |line, value|
+        if line < first
+          shifted[line] = value
+        elsif line >= first + removed
+          shifted[line + delta] = value
+        end
+      end
+      cache.replace(shifted)
+    end
+  end
+end

data/lib/antares/lexer_driver.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require "strscan"
+
+module Antares
+  # Reuses Rouge's own rule interpreter. Supplying the complete bounded source
+  # preserves cross-line matches, fixed anchors and lookbehind at checkpoints.
+  class LexerDriver
+    attr_reader :lexer, :line, :scanner, :scanned_lines
+
+    def self.supported?(lexer)
+      lexer.is_a?(Rouge::RegexLexer) && lexer.method(:stream_tokens).owner == Rouge::RegexLexer
+    end
+
+    def initialize(lexer, source, offsets, start_line: 0, &emit)
+      raise UnsupportedLexerError, "#{lexer.class.tag} has a custom stream driver" unless self.class.supported?(lexer)
+      @lexer = lexer
+      @source = source
+      @offsets = offsets
+      @line = start_line
+      @scanned_lines = 0
+      @emit = emit
+      @tokens = []
+      @scanner = StringScanner.new(source, fixed_anchor: true)
+      @scanner.pos = offsets.fetch(start_line)
+      lexer.instance_variable_set(:@current_stream, scanner)
+      lexer.instance_variable_set(:@output_stream, method(:token).to_proc)
+      lexer.instance_variable_set(:@states, lexer.class.states)
+      lexer.instance_variable_set(:@null_steps, 0)
+    end
+
+    # Checkpoint only after an entire Rouge rule callback, never inside a yield:
+    # callbacks frequently emit a token before updating their persistent state.
+    def advance(until_line:)
+      until scanner.eos? || line > until_line
+        before = scanner.pos
+        success = lexer.step(lexer.state, scanner)
+        token(Rouge::Token::Tokens::Error, scanner.getch) unless success
+        if scanner.pos > before && @tokens.empty? && scanner.pos == @offsets[line]
+          return if block_given? && yield(line, lexer)
+        end
+      end
+      if scanner.eos?
+        finish_line unless @tokens.empty?
+        finish_line while line < @offsets.length - 1
+        yield(line, lexer) if block_given?
+      end
+      self
+    end
+
+    private
+
+    def token(type, value)
+      return if value.nil? || value.empty?
+      value.each_line do |part|
+        if @tokens.last&.first == type
+          @tokens.last[1] << part
+        else
+          @tokens << [type, part.dup]
+        end
+        finish_line if part.end_with?("\n")
+      end
+    end
+
+    def finish_line
+      @emit.call(line, @tokens.map { |pair| pair[1].freeze; pair.freeze }.freeze)
+      @tokens = []
+      @line += 1
+      @scanned_lines += 1
+    end
+  end
+end

data/lib/antares/lexer_state_snapshot.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require "digest"
+require "set"
+
+module Antares
+  # The RegexLexer engine replaces these on every scan; retaining them would
+  # also retain a whole source String and an output callback in every checkpoint.
+  class LexerStateSnapshot
+    VOLATILE = %i[@current_stream @output_stream @states @null_steps].freeze
+    attr_reader :fingerprint, :bytesize
+
+    def initialize(lexer, max_bytes: 256 * 1024)
+      @value = self.class.copy(lexer)
+      normalized = Marshal.dump(self.class.normalize(@value))
+      raise ResourceLimitError, "lexer checkpoint exceeds #{max_bytes} bytes" if normalized.bytesize > max_bytes
+      @bytesize = normalized.bytesize
+      @fingerprint = Digest::SHA256.digest(normalized).freeze
+      freeze
+    end
+
+    def restore = self.class.copy(@value)
+
+    def self.fingerprint(value)
+      Digest::SHA256.digest(Marshal.dump(normalize(value)))
+    end
+
+    def self.variables(value)
+      names = value.instance_variables.sort
+      value.is_a?(Rouge::Lexer) ? names - VOLATILE : names
+    end
+
+    def self.immutable?(value)
+      value.nil? || value == true || value == false || value.is_a?(Numeric) || value.is_a?(Symbol) ||
+        value.is_a?(Module) || value.is_a?(Regexp) || value.is_a?(Encoding) || value.is_a?(Rouge::RegexLexer::State)
+    end
+
+    def self.copy(value, memo = {})
+      return value if immutable?(value)
+      return memo[value.object_id] if memo.key?(value.object_id)
+      raise UnsupportedLexerError, "lexer state graph exceeds 10000 objects" if memo.length >= 10_000
+      case value
+      when String
+        memo[value.object_id] = value.dup
+      when Array
+        result = memo[value.object_id] = value.dup.clear
+        value.each { |item| result << copy(item, memo) }
+        variables(value).each { |name| result.instance_variable_set(name, copy(value.instance_variable_get(name), memo)) }
+        result
+      when Hash
+        raise UnsupportedLexerError, "state Hash has a closure default" if value.default_proc
+        result = memo[value.object_id] = value.dup.clear
+        result.default = copy(value.default, memo)
+        value.each { |key, item| result[copy(key, memo)] = copy(item, memo) }
+        variables(value).each { |name| result.instance_variable_set(name, copy(value.instance_variable_get(name), memo)) }
+        result
+      when Set
+        result = memo[value.object_id] = value.dup.clear
+        value.each { |item| result.add(copy(item, memo)) }
+        variables(value).each { |name| result.instance_variable_set(name, copy(value.instance_variable_get(name), memo)) }
+        result
+      when Struct
+        result = memo[value.object_id] = value.dup
+        value.each_pair { |name, item| result[name] = copy(item, memo) }
+        result
+      when Proc, Method, IO, StringScanner, MatchData
+        raise UnsupportedLexerError, "cannot safely snapshot #{value.class}"
+      else
+        names = variables(value)
+        raise UnsupportedLexerError, "opaque state object #{value.class}" if names.empty? && !value.is_a?(Rouge::Lexer)
+        result = memo[value.object_id] = value.class.allocate
+        names.each { |name| result.instance_variable_set(name, copy(value.instance_variable_get(name), memo)) }
+        result
+      end
+    end
+
+    def self.normalize(value, memo = {})
+      case value
+      when nil, true, false, Numeric, Symbol then value
+      when String then [:string, value.encoding.name, value]
+      when Regexp then [:regexp, value.source, value.options]
+      when Module then [:module, value.name]
+      when Encoding then [:encoding, value.name]
+      when Rouge::RegexLexer::State
+        # Static states are immutable rule templates. Dynamic states contain
+        # closures, so identity is conservative: never converge distinct ones.
+        [:state, value.name.is_a?(Symbol) ? value.name : value.object_id]
+      else
+        return [:reference, memo[value.object_id]] if memo.key?(value.object_id)
+        raise UnsupportedLexerError, "lexer state graph exceeds 10000 objects" if memo.length >= 10_000
+        memo[value.object_id] = memo.length
+        case value
+        when Array then [value.class.name, value.map { |item| normalize(item, memo) }, variables(value).map { |name| [name, normalize(value.instance_variable_get(name), memo)] }]
+        when Hash
+          raise UnsupportedLexerError, "state Hash has a closure default" if value.default_proc
+          [value.class.name, normalize(value.default, memo), value.map { |key, item| [normalize(key, memo), normalize(item, memo)] }, variables(value).map { |name| [name, normalize(value.instance_variable_get(name), memo)] }]
+        when Set then [value.class.name, value.map { |item| normalize(item, memo) }, variables(value).map { |name| [name, normalize(value.instance_variable_get(name), memo)] }]
+        when Struct then [value.class.name, value.each_pair.map { |name, item| [name, normalize(item, memo)] }]
+        else
+          raise UnsupportedLexerError, "cannot fingerprint #{value.class}" if [Proc, Method, IO, StringScanner, MatchData].any? { |klass| value.is_a?(klass) }
+          [value.class.name, variables(value).map { |name| [name, normalize(value.instance_variable_get(name), memo)] }]
+        end
+      end
+    end
+  end
+end

data/lib/antares/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Antares
+  VERSION = "0.1.0"
+end

data/lib/antares.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require_relative "antares/version"
+require "rouge"
+
+module Antares
+  class Error < StandardError; end
+  class UnsupportedLexerError < Error; end
+  class ResourceLimitError < Error; end
+
+  def self.compatible?(lexer)
+    klass = lexer.is_a?(Class) ? lexer : lexer.class
+    return :window unless Rouge.version == COMPATIBILITY_VERSION
+    COMPATIBILITY.fetch(klass.tag, :window)
+  end
+end
+
+require_relative "antares/compatibility"
+require_relative "antares/lexer_state_snapshot"
+require_relative "antares/lexer_driver"
+require_relative "antares/highlighter"

data/sig/antares.rbs

@@ -0,0 +1,47 @@
+module Antares
+  VERSION: String
+  COMPATIBILITY_VERSION: String
+  COMPATIBILITY: Hash[String, :incremental | :window]
+  COMPATIBILITY_DETAILS: Hash[String, Hash[Symbol, untyped]]
+
+  class Error < StandardError
+  end
+  class UnsupportedLexerError < Error
+  end
+  class ResourceLimitError < Error
+  end
+
+  type strategy = :auto | :incremental | :window | :full
+  type token = [Class, String]
+  type row = Array[token]
+
+  def self.compatible?: (untyped lexer) -> (:incremental | :window)
+
+  class LexerStateSnapshot
+    attr_reader fingerprint: String
+    attr_reader bytesize: Integer
+    def initialize: (untyped lexer, ?max_bytes: Integer) -> void
+    def restore: () -> untyped
+    def self.copy: (untyped value, ?Hash[Integer, untyped] memo) -> untyped
+    def self.fingerprint: (untyped value) -> String
+  end
+
+  class Highlighter
+    attr_reader strategy: strategy
+    attr_reader requested_strategy: strategy
+    attr_reader frontier: Integer
+    attr_reader last_scanned_lines: Integer
+    attr_reader fallback_reason: String?
+    attr_reader checkpoints: Hash[Integer, LexerStateSnapshot]
+
+    def initialize: (lexer: untyped, lines: ^(Integer) -> String, line_count: ^() -> Integer,
+      ?strategy: strategy, ?checkpoint_interval: Integer, ?window_context: Integer,
+      ?max_bytes: Integer, ?max_lines: Integer, ?max_line_bytes: Integer,
+      ?max_checkpoint_bytes: Integer, ?max_seconds: Numeric) -> void
+    def tokens_for: (Integer index) -> row
+    def tokens_in: (Range[Integer] range) -> Array[row]
+    def edit: (from_line: Integer, removed: Integer, inserted: Integer) -> self
+    def advance: (until_line: Integer, ?from_line: Integer?) -> self
+    def checkpoint_bytes: () -> Integer
+  end
+end

metadata

@@ -0,0 +1,67 @@
+--- !ruby/object:Gem::Specification
+name: antares
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Yudai Takada
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rouge
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+email:
+- t.yudai92@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- lib/antares.rb
+- lib/antares/compatibility.rb
+- lib/antares/highlighter.rb
+- lib/antares/lexer_driver.rb
+- lib/antares/lexer_state_snapshot.rb
+- lib/antares/version.rb
+- sig/antares.rbs
+homepage: https://github.com/noxdea/antares
+licenses:
+- MIT
+metadata:
+  source_code_uri: https://github.com/noxdea/antares
+  changelog_uri: https://github.com/noxdea/antares/blob/main/CHANGELOG.md
+  allowed_push_host: https://rubygems.org
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.1'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.19
+specification_version: 4
+summary: Incremental highlighting for Rouge lexers
+test_files: []