mt-lang 0.3.38 → 0.3.39

data/lib/milk_tea/lsp/server/semantic_tokens.rb

@@ -12,8 +12,10 @@ module MilkTea
 
           content = @workspace.get_content(uri)
           cache_key = content.hash
+          facts = @workspace.peek_facts(uri)
+          facts_id = facts&.object_id
           cached = @semantic_tokens_cache[uri]
-          if cached && cached[:content_hash] == cache_key
+          if cached && cached[:content_hash] == cache_key && cached[:facts_id] == facts_id
             elapsed = elapsed_ms(total_start)
             short_uri = shorten_uri(uri) || uri
             log_perf_breakdown('textDocument/semanticTokens/full', elapsed,
@@ -25,10 +27,6 @@ module MilkTea
           tokens = @workspace.get_tokens(uri) || []
           tokens_ms = elapsed_ms(tokens_start)
 
-          facts_start = monotonic_time
-          facts = @workspace.get_facts(uri, allow_last_good_fallback: semantic_tokens_allow_last_good_fallback?(uri))
-          facts_ms = elapsed_ms(facts_start)
-
           build_start = monotonic_time
           semantic_entries = build_semantic_token_entries(tokens, facts)
           build_ms = elapsed_ms(build_start)
@@ -38,17 +36,18 @@ module MilkTea
           encode_ms = elapsed_ms(encode_start)
 
           result_id = next_semantic_token_result_id(uri)
-          @semantic_tokens_cache[uri] = { content_hash: cache_key, data: data, result_id: result_id }
+          @semantic_tokens_cache[uri] = { content_hash: cache_key, data: data, result_id: result_id, facts_id: facts_id }
           @semantic_tokens_delta_cache[uri] = {
             result_id: result_id,
             content_hash: cache_key,
+            facts_id: facts_id,
             entries: semantic_entries,
           }
 
           elapsed = elapsed_ms(total_start)
           short_uri = shorten_uri(uri) || uri
           log_perf_breakdown('textDocument/semanticTokens/full', elapsed,
-                            "uri=#{short_uri} bytes=#{content.bytesize} lines=#{content.count("\n") + 1} cache=miss tokens=#{tokens.length} entries=#{semantic_entries.length} data_len=#{data.length} facts=on stages_ms=tokens:#{tokens_ms},facts:#{facts_ms},build:#{build_ms},encode:#{encode_ms}")
+                            "uri=#{short_uri} bytes=#{content.bytesize} lines=#{content.count("\n") + 1} cache=miss tokens=#{tokens.length} entries=#{semantic_entries.length} data_len=#{data.length} facts=#{facts_id ? 'on' : 'off'} stages_ms=tokens:#{tokens_ms},build:#{build_ms},encode:#{encode_ms}")
 
           { resultId: result_id, data: data }
         rescue StandardError => e
@@ -68,18 +67,20 @@ module MilkTea
 
           content = @workspace.get_content(uri)
           cache_key = content.hash
-          if cached[:content_hash] == cache_key
+          facts = @workspace.peek_facts(uri)
+          facts_id = facts&.object_id
+          if cached[:content_hash] == cache_key && cached[:facts_id] == facts_id
             return { resultId: cached[:result_id], edits: [] }
           end
 
           tokens = @workspace.get_tokens(uri) || []
-          facts = @workspace.get_facts(uri, allow_last_good_fallback: semantic_tokens_allow_last_good_fallback?(uri))
           new_entries = build_semantic_token_entries(tokens, facts)
 
           new_result_id = next_semantic_token_result_id(uri)
           @semantic_tokens_delta_cache[uri] = {
             result_id: new_result_id,
             content_hash: cache_key,
+            facts_id: facts_id,
             entries: new_entries,
           }
 
@@ -109,7 +110,7 @@ module MilkTea
               ![:newline, :indent, :dedent, :eof].include?(t.type)
           end
 
-          facts = @workspace.get_facts(uri, allow_last_good_fallback: semantic_tokens_allow_last_good_fallback?(uri))
+          facts = @workspace.peek_facts(uri)
           semantic_entries = build_semantic_token_entries(range_tokens, facts)
 
           entries_in_range = semantic_entries.select do |entry|
@@ -128,6 +129,7 @@ module MilkTea
           # is O(1) per call instead of O(n), turning the overall build from O(n²) to O(n).
           trivia_types = Set[:newline, :indent, :dedent, :eof]
           @tokens_by_line_cache = Hash.new { |h, k| h[k] = [] }
+          @semantic_token_build_cache = {}
           tokens.each { |t| @tokens_by_line_cache[t.line] << t unless trivia_types.include?(t.type) }
           @attribute_name_semantic_overrides = build_attribute_name_semantic_overrides(tokens, facts)
 
@@ -152,6 +154,24 @@ module MilkTea
         ensure
           @tokens_by_line_cache = nil
           @attribute_name_semantic_overrides = nil
+          @semantic_token_build_cache = nil
+        end
+
+        # Memoize facts-derived lookups for the duration of one semantic-token
+        # build. The build cache is scoped per build (set up and torn down in
+        # build_semantic_token_entries) so repeated per-token classifications do
+        # not re-scan frames/type scopes/types for every identifier token.
+        def semantic_build_memo?(key)
+          !@semantic_token_build_cache.nil? && @semantic_token_build_cache.key?(key)
+        end
+
+        def semantic_build_memo(key)
+          @semantic_token_build_cache[key][1] if @semantic_token_build_cache&.key?(key)
+        end
+
+        def store_semantic_build_memo(key, value)
+          @semantic_token_build_cache[key] = [:ok, value] if @semantic_token_build_cache
+          value
         end
 
         def fstring_interpolation_entries(fstring_tok, facts)
@@ -1052,11 +1072,23 @@ module MilkTea
 
         def known_type_name?(facts, name)
           return false unless facts
-          return true if facts.types.key?(name)
-          facts.types.each_value do |type|
-            return true if type.respond_to?(:nested_types) && type.nested_types.key?(name)
+          memo_key = [:known_type, facts.object_id, name]
+          return semantic_build_memo(memo_key) if semantic_build_memo?(memo_key)
+
+          result = if facts.types.key?(name)
+            true
+          else
+            found = false
+            facts.types.each_value do |type|
+              if type.respond_to?(:nested_types) && type.nested_types.key?(name)
+                found = true
+                break
+              end
+            end
+            found
           end
-          false
+
+          store_semantic_build_memo(memo_key, result)
         end
 
         def dot_nested_type_member?(tokens, index, facts)
@@ -1317,6 +1349,9 @@ module MilkTea
         end
 
         def type_parameter_names_in_scope(facts, line)
+          memo_key = [:type_param_names, facts.object_id, line]
+          return semantic_build_memo(memo_key) if semantic_build_memo?(memo_key)
+
           scopes = @type_parameter_scope_cache ||= {}
           cached = scopes[facts.object_id]
           unless cached
@@ -1330,11 +1365,15 @@ module MilkTea
             scopes[facts.object_id] = cached
           end
 
+          result = []
           cached.reverse_each do |scope|
-            return scope[:names] if line >= scope[:start_line] && line <= scope[:end_line]
+            if line >= scope[:start_line] && line <= scope[:end_line]
+              result = scope[:names]
+              break
+            end
           end
 
-          []
+          store_semantic_build_memo(memo_key, result)
         end
 
         def type_parameter_scopes_for_declaration(decl, end_line: Float::INFINITY)

data/lib/milk_tea/lsp/server/text_documents.rb

@@ -10,7 +10,7 @@ module MilkTea
           content = params['textDocument']['text']
           source = @workspace.document_source(uri) || 'unknown'
           open_start = monotonic_time
-          open_stats = @workspace.open_document(uri, content)
+          open_stats = @workspace.open_document(uri, content, warm_facts: false)
           open_ms = elapsed_ms(open_start)
           @semantic_tokens_cache.delete(uri)
           @semantic_tokens_delta_cache.delete(uri)
@@ -38,7 +38,7 @@ module MilkTea
           changes = params['contentChanges'] || []
           previous_content = @workspace.get_content(uri)
 
-          @workspace.apply_incremental_changes(uri, changes)
+          @workspace.apply_incremental_changes(uri, changes, warm_facts: false)
 
           invalidate_document_caches(uri)
           current_content = @workspace.get_content(uri)

data/lib/milk_tea/lsp/server/type_hierarchy.rb

@@ -115,12 +115,12 @@ module MilkTea
         def find_type_info(facts, name)
           if (t = facts.types[name])
             ast = t.respond_to?(:ast_declaration) ? t.ast_declaration : nil
-            return { kind: type_kind_from_object(t), line: ast&.line, column: ast.column || 0 }
+            return { kind: type_kind_from_object(t), line: ast&.line, column: ast&.column || 0 }
           end
 
           if (t = facts.interfaces[name])
             ast = t.respond_to?(:ast_declaration) ? t.ast_declaration : nil
-            return { kind: :interface, line: ast&.line, column: ast.column || 0 }
+            return { kind: :interface, line: ast&.line, column: ast&.column || 0 }
           end
 
           nil

data/lib/milk_tea/lsp/server/utilities.rb

@@ -177,6 +177,7 @@ module MilkTea
 
         def handle_did_change_watched_files(params)
           changes = params['changes'] || []
+          @workspace.apply_module_index_events(changes)
           affected_uris = Set.new
           changes.each do |change|
             uri = change['uri']
@@ -198,6 +199,9 @@ module MilkTea
           @semantic_tokens_cache.delete(uri)
           @semantic_tokens_delta_cache.delete(uri)
           @fixall_cache.delete(uri)
+          @document_symbol_cache.delete(uri)
+          @completion_docs_cache.clear
+          @completion_resolve_cache.clear
           path = uri_to_path(uri)
           if path
             prefix = "#{path}:"

data/lib/milk_tea/lsp/server.rb

@@ -209,6 +209,15 @@ module MilkTea
         @fixall_cache = {}
         @definition_file_token_cache = {}
         @definition_file_ast_cache = {}
+        # Completion session re-filtering pool keyed by [uri, line].
+        @completion_session_cache = {}
+        @completion_session_order = []
+        @completion_session_order_set = Set.new
+        # Server-scoped documentation / resolve memoization keyed by [uri, name].
+        @completion_docs_cache = {}
+        @completion_resolve_cache = {}
+        # Enriched documentSymbol outline keyed by uri -> content_hash.
+        @document_symbol_cache = {}
         @diagnostics_perf = {
           scheduled: 0,
           skipped_unchanged: 0,

data/lib/milk_tea/lsp/workspace/analysis.rb

@@ -62,8 +62,11 @@ module MilkTea
           nil
         end
 
-        def warm_document_facts(uri, content)
-          ast = ast_for_content(uri, content)
+        # Eagerly compute and cache facts for +uri+. The keystroke path
+        # (didChange) passes warm: false so the sema/import-resolution cost runs
+        # on the debounced diagnostics worker instead of the request thread;
+        # requests then serve last-good facts until the background pass lands.
+        def warm_document_facts(uri, content, warm: true)
           stats = {
             bytes: content.bytesize,
             lines: content.count("\n") + 1,
@@ -71,7 +74,7 @@ module MilkTea
             facts_mode: nil,
             facts_ms: nil,
             skip_reason: nil,
-            import_count: ast.respond_to?(:imports) ? ast.imports.length : 0,
+            import_count: 0,
             shared_module_cache_size: @shared_module_cache.length,
           }
 
@@ -80,6 +83,14 @@ module MilkTea
             return stats
           end
 
+          unless warm
+            stats[:skip_reason] = :deferred
+            return stats
+          end
+
+          ast = ast_for_content(uri, content)
+          stats[:import_count] = ast.respond_to?(:imports) ? ast.imports.length : 0
+
           facts_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
           facts_path = uri_to_path(uri)
           get_facts(uri)

data/lib/milk_tea/lsp/workspace/caches.rb

@@ -23,7 +23,12 @@ module MilkTea
         end
 
         def get_ast(uri)
-          @ast_cache[uri] ||= parse_document(uri)
+          @ast_cache[uri] ||= begin
+            # Prefer the facts' own AST so consumers that iterate the AST and
+            # resolve bindings via facts.binding_resolution operate on the same
+            # node instances (binding IDs are keyed by AST node object_id).
+            peek_facts(uri)&.ast || parse_document(uri)
+          end
         end
 
         def get_facts(uri, allow_last_good_fallback: true)
@@ -31,6 +36,17 @@ module MilkTea
           snapshot&.facts
         end
 
+        # Return the most recent facts already computed for +uri+ without ever
+        # running analysis or blocking on the analysis mutex. Used by hot request
+        # paths (semantic tokens) that must not stall the request thread on sema;
+        # they serve the latest cached/last-good facts and are told to re-fetch
+        # once fresh facts land in the background.
+        def peek_facts(uri)
+          @facts_cache_mutex.synchronize do
+            @facts_cache[uri] || @last_good_facts_cache[uri]
+          end
+        end
+
         def get_tooling_snapshot(uri, allow_last_good_fallback: true)
           total_start = perf_logging? ? monotonic_time : nil
           cache_state = 'miss'

data/lib/milk_tea/lsp/workspace/collection.rb

@@ -67,6 +67,10 @@ module MilkTea
                   @facts_cache[uri] = facts if facts
                   @last_good_facts_cache[uri] = facts if facts
                   @document_module_names[uri] = facts.module_name if facts&.module_name
+                  # Consumers resolve binding_resolution against get_ast(uri) node
+                  # object_ids; keep the AST cache aligned with the facts that
+                  # reference it so binding lookups line up across the worker path.
+                  @ast_cache[uri] = snapshot.facts.ast if snapshot&.facts&.ast
                   update_dependency_index(uri, facts)
                   @diagnostics_cache[uri] = {
                     content_hash: hash,

data/lib/milk_tea/lsp/workspace/module_index.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+module MilkTea
+  module LSP
+    class Workspace
+      # Persistent module-name index for import-line completion.
+      #
+      # `import_completions` previously walked the entire module-root tree on
+      # every keystroke (stat'ing every directory entry). This index records, per
+      # module root, the importable module names reachable from each directory:
+      # `.mt` file basenames directly in a directory, plus subdirectory names that
+      # transitively contain a `.mt` file. It is built once per root and kept in
+      # sync with workspace/didChangeWatchedFiles events, so per-keystroke import
+      # completion becomes an in-memory hash lookup.
+      module WorkspaceModuleIndex
+        def reset_module_index
+          @module_index_mutex.synchronize { @module_index.clear }
+        end
+
+        # Rebuild the index for every module root reachable from the workspace
+        # root. Called after initialization and after workspace-folder changes.
+        def refresh_module_index_for_workspace
+          root_path = @workspace_root_path
+          return unless root_path
+
+          module_roots_for_path(root_path).each { |root| ensure_module_index(root) }
+        end
+
+        # Ensure +root+ has an index entry, building it on first use. A race
+        # between two threads simply builds twice and the later write wins.
+        def ensure_module_index(root)
+          present = @module_index_mutex.synchronize { @module_index.key?(root) }
+          return if present
+
+          build_module_index(root)
+        end
+
+        # Return importable module names for +fs_dir+ under +root+, excluding the
+        # currently-open +current_path+ file. Returns nil when +root+ is unknown.
+        def module_importable_names(root, fs_dir, current_path: nil)
+          ensure_module_index(root)
+
+          entry = @module_index_mutex.synchronize { @module_index[root] }
+          return nil unless entry
+
+          result = {}
+          files = entry[:mt_files][fs_dir]
+          if files
+            files.each do |mod_name, file_path|
+              next if current_path && file_path == current_path
+
+              result[mod_name] = true
+            end
+          end
+          entry[:mt_dirs][fs_dir]&.each { |subdir| result[subdir] = true }
+          result
+        end
+
+        # Apply workspace/didChangeWatchedFiles events to the index. Content
+        # changes (type 2) do not affect the name index; creates (1) and deletes
+        # (3) of `.mt` files rebuild their root once per event batch so a build
+        # producing many events triggers a single rescan. Open documents are
+        # source-of-truth and are ignored (skip_open: true), matching
+        # apply_watched_file_change; rename handling passes skip_open: false so
+        # the index never keeps a stale name for a renamed module.
+        def apply_module_index_events(changes, skip_open: true)
+          open_uris = @document_state_mutex.synchronize { @open_documents.keys }
+          roots_to_rebuild = Set.new
+          changes.each do |change|
+            uri = change.is_a?(Hash) ? change['uri'] : nil
+            change_type = change.is_a?(Hash) ? change['type'] : nil
+            next unless uri && [1, 3].include?(change_type&.to_i)
+            next if skip_open && open_uris.include?(uri)
+
+            path = uri_to_path(uri)
+            next unless path && path.end_with?('.mt')
+
+            root = module_index_root_for_path(path)
+            roots_to_rebuild << root if root
+          end
+
+          roots_to_rebuild.each { |root| build_module_index(root) }
+          roots_to_rebuild.length
+        end
+
+        def module_index_roots
+          @module_index_mutex.synchronize { @module_index.keys }
+        end
+
+        private
+
+        def module_index_root_for_path(path)
+          expanded = File.expand_path(path)
+          @module_index_mutex.synchronize do
+            @module_index.each_key do |root|
+              return root if expanded == root || expanded.start_with?(root + File::SEPARATOR)
+            end
+          end
+          nil
+        end
+
+        # Scan +root+ once and record, for every directory, the importable module
+        # names: `.mt` file basenames in that directory and subdirectory names
+        # that transitively contain a `.mt` file.
+        def build_module_index(root)
+          mt_files = Hash.new { |hash, key| hash[key] = {} }
+          mt_dirs = Hash.new { |hash, key| hash[key] = Set.new }
+
+          files = Dir.glob(File.join(root, '**', '*.mt')).sort
+          prefix = root + File::SEPARATOR
+          files.each do |file_path|
+            rel = file_path.delete_prefix(prefix)
+            dir = File.dirname(rel)
+            dir = '' if dir == '.'
+            mod_name = File.basename(rel, '.mt')
+
+            mt_files[dir][mod_name] = File.expand_path(file_path)
+
+            acc = ''
+            dir.split(File::SEPARATOR).each do |segment|
+              mt_dirs[acc] << segment
+              acc = acc.empty? ? segment : "#{acc}#{File::SEPARATOR}#{segment}"
+            end
+          end
+
+          @module_index_mutex.synchronize do
+            @module_index[root] = { mt_files: mt_files, mt_dirs: mt_dirs }
+          end
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/milk_tea/lsp/workspace/store.rb

@@ -29,13 +29,13 @@ module MilkTea
 
         # ── Document lifecycle ──────────────────────────────────────────────────
 
-        def open_document(uri, content)
+        def open_document(uri, content, warm_facts: true)
           @document_state_mutex.synchronize do
             @open_documents[uri] = content
           end
           invalidate_cache(uri)
           enqueue_definition_warmup(uri) unless background_document?(uri)
-          warm_document_facts(uri, content)
+          warm_document_facts(uri, content, warm: warm_facts)
         end
 
         def close_document(uri)
@@ -56,7 +56,7 @@ module MilkTea
           enqueue_definition_warmup(uri) unless background_document?(uri)
         end
 
-        def apply_incremental_changes(uri, changes)
+        def apply_incremental_changes(uri, changes, warm_facts: true)
           content = get_content(uri)
           edits = Array(changes)
 
@@ -79,7 +79,7 @@ module MilkTea
           end
           dependent_uris.each { |dep_uri| invalidate_cache(dep_uri) }
 
-          warm_document_facts(uri, new_content)
+          warm_document_facts(uri, new_content, warm: warm_facts)
         end
 
         def apply_incremental_change_to_content(content, change)
@@ -107,11 +107,16 @@ module MilkTea
         end
 
         # Index all .mt files under root_uri so they are available for workspace-wide queries.
+        # Skips the per-file sweep when the on-disk file set is unchanged since the last
+        # index, so repeated workspace/symbol calls do not re-walk and re-read the tree.
         def index_workspace(root_uri, &progress)
           root_path = uri_to_path(root_uri)
           return unless root_path && File.directory?(root_path)
 
           paths = Dir.glob(File.join(root_path, '**', '*.mt')).sort
+          return if @indexed_workspace_paths == paths
+
+          @indexed_workspace_paths = paths
           total = paths.length
           paths.each_with_index do |path, idx|
             file_uri = path_to_uri(path)

data/lib/milk_tea/lsp/workspace.rb

@@ -11,6 +11,7 @@ require_relative 'workspace/analysis'
 require_relative 'workspace/dependency_graph'
 require_relative 'workspace/definition_index'
 require_relative 'workspace/collection'
+require_relative 'workspace/module_index'
 require_relative 'workspace/utilities'
 
 module MilkTea
@@ -81,6 +82,10 @@ module MilkTea
         @identifier_index = {}
         @identifier_index_mutex = Mutex.new
         @indexed_uris = Set.new
+        # Module index: root path -> { mt_files: {dir => {name => path}}, mt_dirs: {dir => Set<subdir>} }
+        @module_index = {}
+        @module_index_mutex = Mutex.new
+        @indexed_workspace_paths = nil
       end
 
       def reset
@@ -113,6 +118,8 @@ module MilkTea
         @reverse_import_dependents.clear
         @diagnostics_cache.clear
         @facts_generation.clear
+        @module_index_mutex.synchronize { @module_index.clear }
+        @indexed_workspace_paths = nil
       end
 
       include WorkspaceStore
@@ -121,6 +128,7 @@ module MilkTea
       include WorkspaceDependencyGraph
       include WorkspaceDefinitionIndex
       include WorkspaceCollection
+      include WorkspaceModuleIndex
       include WorkspaceUtilities
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mt-lang
 version: !ruby/object:Gem::Version
-  version: 0.3.38
+  version: 0.3.39
 platform: ruby
 authors:
 - Long (Teefan) Tran
@@ -149,6 +149,7 @@ files:
 - docs/index.html
 - docs/language-design.md
 - docs/language-manual.md
+- docs/lsp-performance.md
 - lib/milk_tea.rb
 - lib/milk_tea/base.rb
 - lib/milk_tea/bindings.rb
@@ -338,6 +339,7 @@ files:
 - lib/milk_tea/lsp/workspace/collection.rb
 - lib/milk_tea/lsp/workspace/definition_index.rb
 - lib/milk_tea/lsp/workspace/dependency_graph.rb
+- lib/milk_tea/lsp/workspace/module_index.rb
 - lib/milk_tea/lsp/workspace/store.rb
 - lib/milk_tea/lsp/workspace/utilities.rb
 - lib/milk_tea/packages.rb
@@ -624,7 +626,7 @@ metadata:
   homepage_uri: https://teefan.github.io/mt-lang/
   source_code_uri: https://github.com/teefan/mt-lang
 post_install_message: |
-  Milk Tea 0.3.38 installed!
+  Milk Tea 0.3.39 installed!
 
   System requirements:
     - A C compiler (gcc or clang) must be available on PATH