klenod-build 0.0.14 → 0.0.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: af646580bbf44262cf5f2262b5fd81eb7a77eeb94498d7ae00e44383cc26ce42
-  data.tar.gz: 6c84bd8dcc4d42be762dabd59bcb918d3aa42418772bd54f68b55e5cd0f3d17d
+  metadata.gz: abf589452fadde26fe16d0bbc326e0161e8ed8a719ea516e12e286ef8e1609aa
+  data.tar.gz: 9f54e971b5dc88cd07348be0d96e9885928db10c64b4228bfb64e350b202f669
 SHA512:
-  metadata.gz: ce727e4c814158c472b04f3fed395faa53c812d9b6b397ea7bdf57f7c30305459fb057631021cdaf83979860f252d91f7c3d86ac2fde9518d92f983c284cb900
-  data.tar.gz: 72bc4a860585664b9b9c210d89676c50feb0a594d10120d788516ef7cea0c735e983e25145aa89f022b42d6fee0df8064f6f69ac15b45e8c86bd78b55464f389
+  metadata.gz: af84591cca45a914ffbf087f308eea54d264cb14a0e03d6e60732a806d5e4cf44b00d291679b1f3dfe1f6d39f58bda043ff6b1de625b9d904667d2db5f8a6c07
+  data.tar.gz: 8f5fbdc8411f73fee7b7f6b86f803d7e04e588db97dc3d4d5019b7bf08cc63a1d73b05d82a97e74135fc2370be072b00524b7df3ac301c0ed2f4eb025c4352d7

data/lib/klenod/build/context.rb

@@ -113,7 +113,8 @@ module Klenod
         asset_generation_concurrency: AssetGenerationQueue::DEFAULT_CONCURRENCY,
         asset_download_concurrency: AssetGenerationQueue::DEFAULT_DOWNLOAD_CONCURRENCY,
         profiler: nil,
-        namespace: Module.new
+        namespace: Module.new,
+        analysis: false
       )
         @source_dir = source_dir
         plugins = plugins.to_a if plugins.equal?(DEFAULT_PLUGINS)
@@ -131,12 +132,17 @@ module Klenod
             asset_generation_concurrency: asset_generation_concurrency,
             asset_download_concurrency: asset_download_concurrency,
             profiler: profiler,
-            namespace: namespace
+            namespace: namespace,
+            analysis: analysis
           )
       end
 
       attr_reader :graph, :mode, :base, :asset_origin
 
+      def analysis?
+        @graph.analysis?
+      end
+
       def evaluate(specifier)
         @graph.load(specifier)
       end

data/lib/klenod/build/filesystem_resolver.rb

@@ -15,6 +15,14 @@ module Klenod
         @root = Pathname.new(root).expand_path
         @extensions = extensions.freeze
         @path_prefix = path_prefix.to_s
+        @spelling_index = nil
+      end
+
+      # Forget the cached file listing behind did-you-mean suggestions. The
+      # owning resolver clears it whenever files change, so a suggestion never
+      # walks the source tree more than once between file events.
+      def clear_cache
+        @spelling_index = nil
       end
 
       def resolve(relative_path)
@@ -101,14 +109,21 @@ module Klenod
         []
       end
 
+      SpellingIndex = Data.define(:aliases, :checker)
+
       def spelling_corrections(relative_path)
-        aliases = spelling_aliases
-        matches =
-          DidYouMean::TreeSpellChecker
-            .new(dictionary: aliases.keys.sort, separator: "/")
-            .correct(relative_path)
+        index = spelling_index
+        matches = index.checker.correct(relative_path)
 
-        matches.flat_map { aliases.fetch(it) }.uniq.first(MAX_CORRECTIONS)
+        matches.flat_map { index.aliases.fetch(it) }.uniq.first(MAX_CORRECTIONS)
+      end
+
+      def spelling_index
+        @spelling_index ||=
+          begin
+            aliases = spelling_aliases
+            SpellingIndex.new(aliases, DidYouMean::TreeSpellChecker.new(dictionary: aliases.keys.sort, separator: "/"))
+          end
       end
 
       def spelling_aliases

data/lib/klenod/build/graph/invalidator.rb

@@ -24,7 +24,7 @@ module Klenod
           errors = []
 
           removed_module_ids.each do |module_id|
-            records.delete(module_id)
+            graph.remove_record(module_id)
             mods.delete(module_id)
           end
 
@@ -189,9 +189,7 @@ module Klenod
         end
 
         def direct_dependents(module_id)
-          records.filter_map do |candidate_id, record|
-            candidate_id if record.resolved_dependencies.any? { |dependency| dependency.module_id == module_id }
-          end
+          graph.dependents(module_id)
         end
 
         def mark_module_failed(module_id, error)
@@ -206,7 +204,8 @@ module Klenod
           source_hash = loaded_source.source_hash || Hashing.hexdigest(source)
           version = cached ? cached.version + 1 : 0
 
-          records[module_id] =
+          graph.store_record(
+            module_id,
             ModuleRecord.new(
               module_id,
               source_hash,
@@ -222,6 +221,7 @@ module Klenod
               version,
               :failed
             )
+          )
           mods[module_id] = FailedModule.new(error)
         end
 

data/lib/klenod/build/graph.rb

@@ -71,9 +71,11 @@ module Klenod
         asset_generation_concurrency: AssetGenerationQueue::DEFAULT_CONCURRENCY,
         asset_download_concurrency: AssetGenerationQueue::DEFAULT_DOWNLOAD_CONCURRENCY,
         profiler: nil,
-        namespace: Module.new
+        namespace: Module.new,
+        analysis: false
       )
         @profiler = profiler || Profiler.new
+        @analysis = analysis
         @resolver = Resolver.new(source_dir: source_dir, profiler: @profiler)
         @plugins = plugins
         @mode = mode
@@ -90,6 +92,16 @@ module Klenod
         @virtual_metadata = {}
         @virtual_owners = {}
         @loading_tasks = {}
+        @source_overrides = {}
+        @dependents = {}
+      end
+
+      # Analysis mode collects the graph for tooling such as the language
+      # server. Plugins must keep the record shape but skip work whose result
+      # is never used there: network fetches, hashing whole files, compiling
+      # JavaScript, and building asset bytes.
+      def analysis?
+        @analysis
       end
 
       def load(specifier)
@@ -211,10 +223,89 @@ module Klenod
         end
       end
 
+      # Editor buffers take precedence over files on disk for the given
+      # module until cleared. The next collection re-reads and, when the text
+      # changed, re-transforms the module. A caller that already transformed
+      # the same text can pass that `transform` so collection reuses it.
+      def override_source(module_id, source, transform: nil)
+        @source_overrides[module_id] = LoadResult.new(source, nil, transform)
+      end
+
+      def clear_source_override(module_id)
+        @source_overrides.delete(module_id)
+      end
+
+      def source_override?(module_id)
+        @source_overrides.key?(module_id)
+      end
+
+      # Module ids whose collected record imports the given module, eagerly
+      # or lazily.
+      def dependents(module_id)
+        @dependents.fetch(module_id) { [] }.to_a
+      end
+
+      def store_record(module_id, record)
+        previous = @records[module_id]
+        previous&.resolved_dependencies&.each { |dependency| @dependents[dependency.module_id]&.delete(module_id) }
+        record.resolved_dependencies.each { |dependency| (@dependents[dependency.module_id] ||= Set.new) << module_id }
+        @records[module_id] = record
+      end
+
+      def remove_record(module_id)
+        record = @records.delete(module_id)
+        record&.resolved_dependencies&.each { |dependency| @dependents[dependency.module_id]&.delete(module_id) }
+        record
+      end
+
+      # Collect everything reachable from a collected module through eager
+      # and lazy dependencies. Each module the walk collects itself is
+      # yielded with `nil`; a module that fails to collect is yielded with
+      # its error and the walk continues past it.
+      def collect_reachable(module_id)
+        queue = @records.fetch(module_id).resolved_dependencies.dup
+        seen = Set.new([module_id])
+        index = 0
+
+        while index < queue.length
+          dependency_id = queue[index].module_id
+          index += 1
+          next unless seen.add?(dependency_id)
+
+          record = @records[dependency_id]
+          unless record
+            begin
+              record = collect_module(dependency_id)
+            rescue StandardError, ScriptError => error
+              yield dependency_id, error if block_given?
+              next
+            end
+            yield dependency_id, nil if block_given?
+          end
+          queue.concat(record.resolved_dependencies)
+        end
+      end
+
+      # Transform source for a module without collecting it. Editor tooling
+      # such as the language server uses this to analyze unsaved text with the
+      # same plugins as a build, without adding a record or evaluating anything.
+      def transform_source(module_id, source)
+        transform = transform_module_source(module_id, source)
+        assert_supported_transform!(module_id, source, transform)
+        transform
+      end
+
       def absolute_path(module_id)
         @resolver.absolute_path(module_id)
       end
 
+      # Forget resolved paths so files created, renamed, or deleted outside
+      # a build invalidation are looked up again. Invalidation does this
+      # itself; tooling that only transforms must ask for it.
+      def clear_resolver_cache
+        @resolver.clear_cache
+      end
+
       def source_dir
         @resolver.source_dir
       end
@@ -235,7 +326,7 @@ module Klenod
           @virtual_sources.delete(module_id)
           @virtual_metadata.delete(module_id)
           @virtual_owners.delete(module_id)
-          @records.delete(module_id)
+          remove_record(module_id)
           @mods.delete(module_id)
         end
       end
@@ -290,7 +381,7 @@ module Klenod
           mod = instantiate_module(module_id, transform, resolved_dependencies, dependency_records, cached, source)
           record = build_module_record(module_id, source, source_hash, transformed_hash, transform, resolved_dependencies, mod)
 
-          @records[module_id] = record
+          store_record(module_id, record)
           @mods[module_id] = mod
           record
         end
@@ -345,7 +436,7 @@ module Klenod
           transformed_hash = Hashing.hexdigest(transform.code)
           record = build_module_record(module_id, source, source_hash, transformed_hash, transform, resolved_dependencies, cached)
 
-          @records[module_id] = record
+          store_record(module_id, record)
           @mods.delete(module_id)
           @profiler.progress(:collect_module, module_id: record.id.to_s, total_records: @records.length)
           record
@@ -851,6 +942,11 @@ module Klenod
       end
 
       def load_source(module_id)
+        if (override = @source_overrides[module_id])
+          @profiler.count(:source_override_hit)
+          return override
+        end
+
         @plugins.each do |plugin|
           @profiler.count(:plugin_load_check)
           loaded =

data/lib/klenod/build/hashing.rb

@@ -13,6 +13,13 @@ module Klenod
         Digest::SHA256.file(path.to_s).hexdigest
       end
 
+      # A cheap change-detection key for a file when its bytes are not
+      # needed, e.g. graph analysis that skips asset work.
+      def self.file_key(path)
+        stat = File.stat(path.to_s)
+        short("#{path}:#{stat.mtime.to_f}:#{stat.size}")
+      end
+
       def self.short(value, length: 16)
         hexdigest(value)[0, length]
       end

data/lib/klenod/build/plugins/gem_import_plugin.rb

@@ -39,11 +39,11 @@ module Klenod
             raise error.with_resolution_context(dependency: dependency, importer_id: dependency.importer_id)
           end
 
-          def load(module_id, _context)
+          def load(module_id, context)
             return nil unless module_id.scheme == :gem
 
             path = path_for(module_id)
-            LoadResult.new(path.binread, Hashing.file_hexdigest(path), nil)
+            LoadResult.new(path.binread, context.analysis? ? Hashing.file_key(path) : Hashing.file_hexdigest(path), nil)
           end
 
           private

data/lib/klenod/build/plugins/google_fonts_plugin.rb

@@ -163,7 +163,9 @@ module Klenod
             return nil unless google_fonts_module?(module_id)
 
             url = google_fonts_url_for(module_id)
-            css = fetch_css(url)
+            css = context.analysis? ? cached_css(url) : fetch_css(url)
+            return analysis_load(module_id, url, context) unless css
+
             font_faces = FontFaceParser.new(css).font_faces_by_url
             font_assets = {}
             rewritten_css =
@@ -220,6 +222,22 @@ module Klenod
             raise Error, "Could not download Google Fonts asset #{url.inspect}: #{error.message}"
           end
 
+          # Analysis never touches the network: a warm cache is used as is,
+          # and a cold one yields an empty stylesheet that keeps importers
+          # transforming.
+          def cached_css(url)
+            return nil unless @css_cache && !@refresh_cache
+
+            @css_cache.read(url)
+          end
+
+          def analysis_load(module_id, url, context)
+            css_asset = css_asset(module_id, url, "", [])
+            css_asset.url = context.asset_url(css_asset.output_path)
+            @assets_by_module_id[module_id] = [css_asset]
+            ruby_module_source(css_asset.url)
+          end
+
           def fetch_css(url)
             if @css_cache && !@refresh_cache
               css = @css_cache.read(url)

data/lib/klenod/build/plugins/haml_plugin/transformer.rb

@@ -136,6 +136,8 @@ module Klenod
             end
           rescue RubyParseError => error
             raise ParseError.new(error, source: source, module_id: module_id)
+          rescue RubyImportRewriter::ParseError => error
+            raise ParseError.new(RubyParseError.new(error.message, line: error.line), source: source, module_id: module_id)
           ensure
             @profiler = previous_profiler
             @module_id = previous_module_id

data/lib/klenod/build/plugins/haml_plugin.rb

@@ -78,6 +78,10 @@ module Klenod
             @transformer = Transformer.new
           end
 
+          # The validated `variables` mapping, so tooling such as the language
+          # server can tell whether `$name` reads a framework prop.
+          attr_reader :variables
+
           def resolve(dependency, _context)
             styles_dependency = resolve_class_names_runtime(dependency)
             return styles_dependency if styles_dependency

data/lib/klenod/build/plugins/image_plugin.rb

@@ -91,6 +91,8 @@ module Klenod
             return nil unless EXTENSIONS.include?(module_id.extname)
 
             source_path = context.absolute_path(module_id)
+            return analysis_load(module_id, source_path, context) if context.analysis?
+
             source_hash = Hashing.file_hexdigest(source_path)
             dimensions = image_dimensions(source_path, module_id)
             image_options = image_options_for(module_id)
@@ -158,6 +160,38 @@ module Klenod
             raise DecodeError.new(error, module_id: module_id)
           end
 
+          # Analysis keeps the module shape (dimensions, content type, a
+          # single asset reference) without hashing the file, decoding it, or
+          # describing variants nobody will generate.
+          def analysis_load(module_id, source_path, context)
+            source_hash = Hashing.file_key(source_path)
+            dimensions = image_dimensions(source_path, module_id)
+            asset = static_image_asset(module_id, source_path, source_hash, dimensions)
+            asset.url = context.asset_url(asset.output_path)
+            image_runtime_dependency = image_runtime_dependency_for(module_id)
+            transform =
+              TransformResult.new(
+                image_module_source(module_id, [asset], image_runtime_dependency, context, placeholder: nil),
+                [image_runtime_dependency],
+                nil,
+                [asset],
+                [],
+                {}
+              )
+
+            LoadResult.new(image_source(module_id), source_hash, transform)
+          end
+
+          def image_runtime_dependency_for(module_id)
+            Dependency
+              .create(
+                specifier: IMAGE_RUNTIME_SPECIFIER,
+                importer_id: module_id,
+                kind: :image_runtime
+              )
+              .with(id: "#{module_id}:image_runtime")
+          end
+
           def default_image_asset(module_id, source_path, source_hash, dimensions, image_options, queue)
             return static_image_asset(module_id, source_path, source_hash, dimensions) unless generated_default_image?(image_options)
 

data/lib/klenod/build/plugins/intl_plugin.rb

@@ -26,6 +26,12 @@ module Klenod
         class Plugin < Klenod::Build::Plugin
           INTL_FILE_RE = /\.intl\.(?<locale>[^\/]+)\.toml\z/
 
+          CachedTranslations = Data.define(:mtime, :size, :translations)
+
+          def initialize
+            @cache = {}
+          end
+
           def translations_for(context, module_id)
             base = module_id.path.delete_suffix(module_id.extname)
             pattern = context.absolute_path(ModuleId.new("#{base}.intl.*.toml", nil)).to_s
@@ -39,18 +45,36 @@ module Klenod
               end
           end
 
+          # Changed or removed companions leave the cache immediately, so it
+          # only ever holds live files. Nothing else needs invalidating: the
+          # owning Haml module is re-transformed by HamlPlugin's own hook.
+          def invalidate_module_ids(paths, _context)
+            paths.each { |path| @cache.delete(File.expand_path(path)) }
+            []
+          end
+
           private
 
-          # Read the file here rather than using TomlRB.load_file, so a parse
-          # failure can carry the source for the excerpt.
+          # Parsed translations are kept per file and validated by stat, so a
+          # module that transforms often, as in the language server, does not
+          # re-read and re-parse companions that did not change. Read the file
+          # here rather than using TomlRB.load_file, so a parse failure can
+          # carry the source for the excerpt.
           def translations_from(path, module_id)
+            stat = File.stat(path)
+            cached = @cache[path]
+            return cached.translations if cached && cached.mtime == stat.mtime && cached.size == stat.size
+
             source = File.read(path)
+            translations =
+              begin
+                TomlRB.parse(source)
+              rescue TomlRB::Error => error
+                raise ParseError.new(error, source: source, module_id: companion_id(path, module_id))
+              end
 
-            begin
-              TomlRB.parse(source)
-            rescue TomlRB::Error => error
-              raise ParseError.new(error, source: source, module_id: companion_id(path, module_id))
-            end
+            @cache[path] = CachedTranslations.new(stat.mtime, stat.size, translations)
+            translations
           end
 
           # The companion is not a module in the graph, but its id resolves to a

data/lib/klenod/build/plugins/static_asset_plugin.rb

@@ -2,6 +2,7 @@
 
 require_relative "../asset"
 require_relative "../hashing"
+require_relative "../load_result"
 require_relative "../plugin"
 require_relative "../transform_result"
 
@@ -21,6 +22,13 @@ module Klenod
             ".woff2" => "font/woff2"
           }.freeze
 
+          # Analysis never needs the bytes of a font, so the file is not read.
+          def load(module_id, context)
+            return nil unless context.analysis? && CONTENT_TYPES.key?(module_id.extname)
+
+            LoadResult.new("", Hashing.short(module_id.to_s), TransformResult.new("Default = nil\n", [], nil, [], [], {}))
+          end
+
           def transform(module_id, code, context)
             return super unless CONTENT_TYPES.key?(module_id.extname)
 

data/lib/klenod/build/resolver.rb

@@ -64,6 +64,7 @@ module Klenod
       def clear_cache
         @resolved_module_ids.clear
         @absolute_paths.clear
+        @filesystem_resolver.clear_cache
       end
 
       private

data/lib/klenod/build/ruby_import_rewriter.rb

@@ -39,6 +39,18 @@ module Klenod
       IMPORT_SOURCE_PATTERN = /(?<![A-Za-z0-9_])(?:import|lazy_import|import_glob)\s*(?:\(|["'])/
       FAST_LITERAL_IMPORT_PATTERN = /(?<![A-Za-z0-9_])(import|lazy_import)\s*\(\s*("(?:\\.|[^"\\#])*")\s*\)/
 
+      # Ruby that does not parse, reported at its line in the enclosing
+      # source. Haml wraps it into its own parse error with a source excerpt.
+      class ParseError < Klenod::Build::Error
+        attr_reader :line, :column
+
+        def initialize(message, line:, column:)
+          @line = line
+          @column = column
+          super(message)
+        end
+      end
+
       def initialize(module_id:, kind:, source_dir: nil, profiler: nil, dependency_id_offset: 0, source_line_offset: 0, source_column_offset: 0)
         @module_id = module_id
         @kind = kind
@@ -54,7 +66,12 @@ module Klenod
         fast_result = rewrite_literal_import_calls(code)
         return fast_result if fast_result
 
-        ast = measure(:ruby_import_parse) { SyntaxTree.parse(code) }
+        ast =
+          begin
+            measure(:ruby_import_parse) { SyntaxTree.parse(code) }
+          rescue SyntaxTree::Parser::ParseError => error
+            raise ParseError.new(error.message, line: error.lineno + @source_line_offset, column: error.column + 1 + @source_column_offset)
+          end
         calls = measure(:ruby_import_scan) { import_calls(ast) }
         expanded = expand_import_calls(calls)
 

data/lib/klenod/build/version.rb

@@ -2,6 +2,6 @@
 
 module Klenod
   module Build
-    VERSION = "0.0.14"
+    VERSION = "0.0.15"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: klenod-build
 version: !ruby/object:Gem::Version
-  version: 0.0.14
+  version: 0.0.15
 platform: ruby
 authors:
 - Andrés Alin
@@ -15,14 +15,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.0.14
+        version: 0.0.15
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.0.14
+        version: 0.0.15
 - !ruby/object:Gem::Dependency
   name: async
   requirement: !ruby/object:Gem::Requirement