antares 0.1.0 → 0.2.0

data/lib/antares/grammar.rb

@@ -0,0 +1,274 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "tm_language_plist"
+require_relative "tm_language_lexer"
+
+module Antares
+  module Grammar
+    MAX_BYTES = 2 * 1024 * 1024
+    MAX_DEPTH = 64
+    MAX_RULES = 10_000
+    MAX_REGEX_BYTES = 16_384
+    TOP_KEYS = %w[$schema name scopeName fileTypes firstLineMatch foldingStartMarker
+      foldingStopMarker patterns repository uuid version comment
+      information_for_contributors hideFromUser].freeze
+    RULE_KEYS = %w[include match captures name begin end beginCaptures endCaptures
+      contentName patterns applyEndPatternLast comment].freeze
+    UNSUPPORTED_RULE_KEYS = %w[while whileCaptures repository disabled].freeze
+
+    MatchRule = Struct.new(:regexp, :token, :captures, keyword_init: true)
+    BeginRule = Struct.new(:regexp, :end_source, :ending, :dynamic_end, :token,
+      :content_token, :begin_captures, :end_captures, :patterns, :apply_end_last,
+      keyword_init: true)
+    IncludeRule = Struct.new(:target, keyword_init: true)
+    GroupRule = Struct.new(:patterns, keyword_init: true)
+    Definition = Struct.new(:scope_name, :tag, :patterns, :repository, keyword_init: true)
+
+    class UniqueHash < Hash
+      def []=(key, value)
+        raise GrammarError, "duplicate JSON object key #{key.inspect}" if key?(key)
+
+        super
+      end
+    end
+    private_constant :UniqueHash
+
+    module_function
+
+    def load_tmlanguage(path)
+      raise ArgumentError, "path must be a String" unless path.is_a?(String) && !path.include?("\0")
+
+      source = File.open(path, "rb") do |file|
+        raise GrammarError, "tmLanguage must be a regular file" unless file.stat.file?
+        raise ResourceLimitError, "tmLanguage exceeds #{MAX_BYTES} bytes" if file.stat.size > MAX_BYTES
+
+        file.read(MAX_BYTES + 1) || "".b
+      end
+      raise ResourceLimitError, "tmLanguage exceeds #{MAX_BYTES} bytes" if source.bytesize > MAX_BYTES
+      source = source.delete_prefix("\xEF\xBB\xBF".b).force_encoding(Encoding::UTF_8)
+      raise GrammarError, "tmLanguage must be valid UTF-8" unless source.valid_encoding?
+
+      value = parse(source)
+      definition = Compiler.new.compile(value)
+      TMLanguageLexer.build(definition)
+    rescue SystemCallError => error
+      raise GrammarError, "cannot read tmLanguage: #{error.message}"
+    end
+
+    def parse(source)
+      case source.lstrip.getbyte(0)
+      when 123
+        JSON.parse(source, max_nesting: MAX_DEPTH, object_class: UniqueHash,
+          allow_duplicate_key: false)
+      when 60
+        TMLanguagePlist.parse(source, max_depth: MAX_DEPTH)
+      else
+        raise GrammarError, "tmLanguage must be JSON or XML plist"
+      end
+    rescue JSON::ParserError => error
+      raise GrammarError, "invalid tmLanguage JSON: #{error.message.byteslice(0, 256)}"
+    end
+
+    def token_for_scope(scope)
+      return Rouge::Token::Tokens::Text unless scope.is_a?(String)
+
+      name = scope.split.first.to_s.downcase
+      case name
+      when /\Ainvalid(?:\.|\z)/ then Rouge::Token::Tokens::Error
+      when /\Acomment(?:\.|\z)/ then Rouge::Token::Tokens::Comment
+      when /\A(?:string\.)?(?:regexp|regex)(?:\.|\z)/ then Rouge::Token::Tokens::Literal::String::Regex
+      when /\Aconstant\.character\.escape(?:\.|\z)/ then Rouge::Token::Tokens::Literal::String::Escape
+      when /\Astring(?:\.|\z)/ then Rouge::Token::Tokens::Literal::String
+      when /\Aconstant\.numeric(?:\.|\z)/ then Rouge::Token::Tokens::Literal::Number
+      when /\Aconstant\.language(?:\.|\z)/ then Rouge::Token::Tokens::Keyword::Constant
+      when /\Akeyword(?:\.|\z)/ then Rouge::Token::Tokens::Keyword
+      when /\Astorage\.type(?:\.|\z)/ then Rouge::Token::Tokens::Keyword::Type
+      when /\Astorage(?:\.|\z)/ then Rouge::Token::Tokens::Keyword::Declaration
+      when /\Aentity\.name\.function(?:\.|\z)/ then Rouge::Token::Tokens::Name::Function
+      when /\A(?:entity\.name\.(?:class|type)|support\.class)(?:\.|\z)/ then Rouge::Token::Tokens::Name::Class
+      when /\Aentity\.other\.attribute-name(?:\.|\z)/ then Rouge::Token::Tokens::Name::Attribute
+      when /\Asupport\.function(?:\.|\z)/ then Rouge::Token::Tokens::Name::Builtin
+      when /\Avariable(?:\.|\z)/ then Rouge::Token::Tokens::Name::Variable
+      when /\Apunctuation(?:\.|\z)/ then Rouge::Token::Tokens::Punctuation
+      when /\Amarkup\.heading(?:\.|\z)/ then Rouge::Token::Tokens::Generic::Heading
+      when /\Amarkup\.bold(?:\.|\z)/ then Rouge::Token::Tokens::Generic::Strong
+      when /\Amarkup\.italic(?:\.|\z)/ then Rouge::Token::Tokens::Generic::Emph
+      when /\A(?:constant|support\.constant)(?:\.|\z)/ then Rouge::Token::Tokens::Name::Constant
+      when /\Aentity\.name(?:\.|\z)/ then Rouge::Token::Tokens::Name
+      else Rouge::Token::Tokens::Text
+      end
+    end
+
+    class Compiler
+      def initialize
+        @rule_count = 0
+      end
+
+      def compile(value)
+        object(value, "tmLanguage root")
+        unknown = value.keys - TOP_KEYS
+        raise GrammarError, "unsupported tmLanguage key #{unknown.first.inspect}" unless unknown.empty?
+
+        scope = string(value["scopeName"], "scopeName")
+        patterns = compile_patterns(value.fetch("patterns", []), 0)
+        source_repository = value.fetch("repository", {})
+        object(source_repository, "repository")
+        repository = source_repository.to_h do |name, rule|
+          raise GrammarError, "invalid repository name" unless name.is_a?(String) && !name.empty?
+
+          [name.freeze, compile_rule(rule, 1)]
+        end.freeze
+        validate_includes(patterns, repository)
+        repository.each_value { |rule| validate_includes([rule], repository) }
+        tag = scope.split(".").last.to_s.gsub(/[^a-zA-Z0-9_+.-]/, "_")
+        raise GrammarError, "scopeName does not contain a usable lexer tag" if tag.empty?
+
+        Definition.new(scope_name: scope.freeze, tag: tag.downcase.freeze,
+          patterns: patterns, repository: repository).freeze
+      end
+
+      private
+
+      def compile_patterns(value, depth)
+        raise GrammarError, "patterns must be an Array" unless value.is_a?(Array)
+        raise ResourceLimitError, "tmLanguage nesting exceeds #{MAX_DEPTH}" if depth >= MAX_DEPTH
+
+        value.map { |rule| compile_rule(rule, depth + 1) }.freeze
+      end
+
+      def compile_rule(value, depth)
+        object(value, "grammar rule")
+        unsupported = value.keys & UNSUPPORTED_RULE_KEYS
+        raise GrammarError, "unsupported grammar rule key #{unsupported.first.inspect}" unless unsupported.empty?
+        unknown = value.keys - RULE_KEYS
+        raise GrammarError, "unsupported grammar rule key #{unknown.first.inspect}" unless unknown.empty?
+        @rule_count += 1
+        raise ResourceLimitError, "tmLanguage exceeds #{MAX_RULES} rules" if @rule_count > MAX_RULES
+
+        kinds = %w[include match begin].select { |key| value.key?(key) }
+        raise GrammarError, "grammar rule has incompatible operations" if kinds.length > 1
+
+        case kinds.first
+        when "include" then compile_include(value)
+        when "match" then compile_match(value)
+        when "begin" then compile_begin(value, depth)
+        else
+          raise GrammarError, "grammar rule must have one operation" unless value.key?("patterns")
+
+          compile_group(value, depth)
+        end
+      end
+
+      def compile_include(value)
+        target = string(value["include"], "include")
+        valid = %w[$self $base].include?(target) || target.match?(/\A#[^#\s]+\z/)
+        raise GrammarError, "external grammar include #{target.inspect} is unsupported" unless valid
+        raise GrammarError, "include rule has incompatible fields" unless (value.keys - %w[include comment]).empty?
+
+        IncludeRule.new(target: target.freeze).freeze
+      end
+
+      def compile_match(value)
+        allowed = %w[match captures name comment]
+        raise GrammarError, "match rule has incompatible fields" unless (value.keys - allowed).empty?
+
+        MatchRule.new(regexp: compile_regexp(value["match"]), token: optional_token(value["name"]),
+          captures: compile_captures(value["captures"])).freeze
+      end
+
+      def compile_begin(value, depth)
+        raise GrammarError, "begin rule requires end" unless value.key?("end")
+
+        regexp = compile_regexp(value["begin"])
+        ending = string(value["end"], "end")
+        compiled_ending = validate_regexp(dynamic_source(ending), "end")
+        shared = value["captures"]
+        BeginRule.new(regexp: regexp, end_source: ending.freeze, ending: compiled_ending,
+          dynamic_end: ending.match?(/(?<!\\)\\[1-9]/), token: optional_token(value["name"]),
+          content_token: optional_token(value["contentName"]),
+          begin_captures: compile_captures(value["beginCaptures"] || shared),
+          end_captures: compile_captures(value["endCaptures"] || shared),
+          patterns: compile_patterns(value.fetch("patterns", []), depth),
+          apply_end_last: boolean(value.fetch("applyEndPatternLast", false), "applyEndPatternLast")).freeze
+      end
+
+      def compile_group(value, depth)
+        raise GrammarError, "pattern group has incompatible fields" unless (value.keys - %w[patterns comment]).empty?
+
+        GroupRule.new(patterns: compile_patterns(value["patterns"], depth)).freeze
+      end
+
+      def compile_captures(value)
+        return {}.freeze if value.nil?
+        object(value, "captures")
+
+        value.to_h do |index, capture|
+          raise GrammarError, "capture keys must be decimal indexes" unless index.is_a?(String) && index.match?(/\A\d{1,3}\z/)
+          object(capture, "capture")
+          unknown = capture.keys - %w[name comment]
+          raise GrammarError, "unsupported capture key #{unknown.first.inspect}" unless unknown.empty?
+
+          [Integer(index, 10), scope_token(string(capture["name"], "capture name"))]
+        end.freeze
+      end
+
+      def compile_regexp(value)
+        source = string(value, "match")
+        validate_regexp(source, "match")
+      end
+
+      def validate_regexp(source, field)
+        raise ResourceLimitError, "#{field} regex exceeds #{MAX_REGEX_BYTES} bytes" if source.bytesize > MAX_REGEX_BYTES
+
+        Regexp.new(source)
+      rescue RegexpError => error
+        raise GrammarError, "invalid #{field} regex: #{error.message.byteslice(0, 256)}"
+      end
+
+      def dynamic_source(source)
+        source.gsub(/(?<!\\)\\[1-9]/, "x")
+      end
+
+      def validate_includes(patterns, repository, seen = {})
+        patterns.each do |rule|
+          if rule.is_a?(IncludeRule) && rule.target.start_with?("#")
+            name = rule.target.delete_prefix("#")
+            raise GrammarError, "unknown repository include #{rule.target.inspect}" unless repository.key?(name)
+          elsif rule.is_a?(BeginRule) || rule.is_a?(GroupRule)
+            next if seen[rule.object_id]
+
+            seen[rule.object_id] = true
+            validate_includes(rule.patterns, repository, seen)
+          end
+        end
+      end
+
+      def optional_token(value)
+        value.nil? ? nil : scope_token(string(value, "scope name"))
+      end
+
+      def scope_token(value)
+        Grammar.__send__(:token_for_scope, value)
+      end
+
+      def object(value, name)
+        raise GrammarError, "#{name} must be an object" unless value.is_a?(Hash) && value.keys.all? { |key| key.is_a?(String) }
+      end
+
+      def string(value, name)
+        raise GrammarError, "#{name} must be a nonempty String" unless value.is_a?(String) && !value.empty?
+
+        value
+      end
+
+      def boolean(value, name)
+        raise GrammarError, "#{name} must be true or false" unless value == true || value == false
+
+        value
+      end
+    end
+    private_constant :Compiler
+    private_class_method :parse, :token_for_scope
+  end
+end

data/lib/antares/highlighter.rb

@@ -51,6 +51,17 @@ module Antares
       indexes.map { |index| @tokens.fetch(index) { plain(index) } }
     end
 
+    def structure
+      @structure ||= Structure.build(
+        language: @template.class.tag,
+        lines: @lines,
+        line_count: @line_count,
+        tokens_for: method(:tokens_for),
+        tokens_in: method(:tokens_in),
+        stabilize: method(:stabilize_structure)
+      )
+    end
+
     # Provider contents must already reflect this line edit. Indices are zero based.
     def edit(from_line:, removed:, inserted:)
       values = [from_line, removed, inserted]
@@ -58,10 +69,16 @@ module Antares
       raise RangeError, "edit outside previous document" unless from_line <= @count && from_line + removed <= @count
       updated_count = count
       raise ArgumentError, "line_count disagrees with edit" unless updated_count == @count - removed + inserted
+      separator_line = if from_line.positive? &&
+        ((from_line == @count && inserted.positive?) || (from_line + removed == @count && inserted.zero? && removed.positive?))
+        from_line - 1
+      end
+      restart_from = separator_line || from_line
+      source_edit = @source_edit ? nil : (@source && [@source, @offsets, @count, from_line, removed, inserted])
       @last_scanned_lines = 0
       if strategy == :incremental
         @tokens.delete_if { |line, _| line >= frontier } if @old_fingerprints
-        start = @checkpoints.keys.select { |line| line <= from_line && line <= frontier }.max || 0
+        start = @checkpoints.keys.select { |line| line <= restart_from && line <= frontier }.max || 0
         restart = @checkpoints.fetch(start)
         shift_cache(@tokens, from_line, removed, inserted)
         shift_cache(@fingerprints, from_line, removed, inserted)
@@ -79,7 +96,10 @@ module Antares
         @frontier = 0
       end
       @count = updated_count
+      @source_edit = source_edit
       @source = @offsets = @driver = nil
+      @structure&.edit(from_line: from_line, removed: removed, inserted: inserted)
+      @structure&.edit(from_line: separator_line, removed: 1, inserted: 1) if separator_line
       self
     end
 
@@ -146,22 +166,70 @@ module Antares
     def source_line(index)
       value = @lines.call(index)
       raise TypeError, "line provider must return a String" unless value.is_a?(String)
-      raise EncodingError, "line provider must return valid UTF-8" unless value.valid_encoding? && [Encoding::UTF_8, Encoding::US_ASCII].include?(value.encoding)
-      raise ArgumentError, "line provider returned multiple logical lines" if value.count("\n") > 1 || (value.include?("\n") && !value.end_with?("\n"))
+      encoding = value.encoding
+      unless value.valid_encoding? && (encoding == Encoding::UTF_8 || encoding == Encoding::US_ASCII)
+        raise EncodingError, "line provider must return valid UTF-8"
+      end
+      newline = value.index("\n")
+      raise ArgumentError, "line provider returned multiple logical lines" if newline && newline != value.length - 1
       # Providers may omit separators, but an empty final line remains empty.
-      index < @count - 1 && !value.end_with?("\n") ? value + "\n" : value
+      index < @count - 1 && newline.nil? ? value + "\n" : value
     end
 
     def build_source
       return if @source
-      source = +""
-      offsets = [0]
-      @count.times do |index|
+      if @source_edit
+        build_edited_source
+        @source_edit = nil
+        return
+      end
+      parts = Array.new(@count)
+      offsets = Array.new(@count + 1, 0)
+      bytes = 0
+      index = 0
+      while index < @count
+        line = source_line(index)
+        line_bytes = line.bytesize
+        raise ResourceLimitError, "line exceeds #{@max_line_bytes} bytes" if line_bytes > @max_line_bytes
+        bytes += line_bytes
+        raise ResourceLimitError, "document exceeds #{@max_bytes} bytes" if bytes > @max_bytes
+        parts[index] = line
+        offsets[index + 1] = bytes
+        index += 1
+      end
+      @source, @offsets = parts.join.freeze, offsets.freeze
+    end
+
+    def build_edited_source
+      old_source, old_offsets, old_count, from_line, removed, inserted = @source_edit
+      first = [from_line - 1, 0].max
+      old_suffix = from_line + removed
+      new_suffix = from_line + inserted
+      prefix_bytes = old_offsets.fetch(first)
+      suffix_bytes = old_offsets.fetch(old_suffix)
+      offsets = old_offsets[0..first]
+      middle = []
+      bytes = prefix_bytes
+      index = first
+      while index < new_suffix
         line = source_line(index)
-        raise ResourceLimitError, "line exceeds #{@max_line_bytes} bytes" if line.bytesize > @max_line_bytes
-        raise ResourceLimitError, "document exceeds #{@max_bytes} bytes" if source.bytesize + line.bytesize > @max_bytes
-        source << line
-        offsets << source.bytesize
+        line_bytes = line.bytesize
+        raise ResourceLimitError, "line exceeds #{@max_line_bytes} bytes" if line_bytes > @max_line_bytes
+        bytes += line_bytes
+        middle << line
+        offsets << bytes
+        index += 1
+      end
+      total = bytes + old_source.bytesize - suffix_bytes
+      raise ResourceLimitError, "document exceeds #{@max_bytes} bytes" if total > @max_bytes
+
+      source = String.new(capacity: total)
+      source << old_source.byteslice(0, prefix_bytes) << middle.join << old_source.byteslice(suffix_bytes..)
+      delta = bytes - suffix_bytes
+      boundary = old_suffix + 1
+      while boundary <= old_count
+        offsets << old_offsets.fetch(boundary) + delta
+        boundary += 1
       end
       @source, @offsets = source.freeze, offsets.freeze
     end
@@ -207,8 +275,9 @@ module Antares
 
     def advance_window(first, last)
       Timeout.timeout(@max_seconds) { window_tokens(first, last) }
-    rescue Timeout::Error
-      @fallback_reason = "window lexing exceeded #{@max_seconds} seconds"
+    rescue Timeout::Error, ResourceLimitError => error
+      @fallback_reason = error.is_a?(Timeout::Error) ?
+        "window lexing exceeded #{@max_seconds} seconds" : error.message
       (first..last).each { |index| @tokens[index] = plain(index) }
       @frontier = last + 1
     end
@@ -260,6 +329,7 @@ module Antares
 
     def fallback!(strategy, reason)
       @strategy, @fallback_reason = strategy, reason
+      @source_edit = nil
       @driver = @source = @offsets = nil
       @tokens.clear
       @checkpoints.clear
@@ -270,6 +340,10 @@ module Antares
 
     def shift_cache(cache, first, removed, inserted)
       delta = inserted - removed
+      if delta.zero?
+        removed.times { |offset| cache.delete(first + offset) }
+        return
+      end
       shifted = {}
       cache.each do |line, value|
         if line < first
@@ -280,5 +354,11 @@ module Antares
       end
       cache.replace(shifted)
     end
+
+    def stabilize_structure(from_line)
+      return 0 if @count.zero?
+      advance(until_line: @count - 1, from_line: from_line)
+      strategy == :incremental && @driver ? [@driver.line, @count].min : @count
+    end
   end
 end