klenod-build 0.0.13 → 0.0.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4b99708644a67ef46ca03814e2ea175347cfdea450ac8ac33d53e8a7b0c9a9f4
-  data.tar.gz: f7b5935d6d9ddcbd442dfdb95873c5e23fe3ed5c28ab891c4c5996b26ce8b9f0
+  metadata.gz: abf589452fadde26fe16d0bbc326e0161e8ed8a719ea516e12e286ef8e1609aa
+  data.tar.gz: 9f54e971b5dc88cd07348be0d96e9885928db10c64b4228bfb64e350b202f669
 SHA512:
-  metadata.gz: 1cf296164f848ae53f748d9dff80dffcccaae0abcbad3eee0b0071366042ae5ad3c66069526baadb02ec34154b90453c0cf464998613fd286293f7b896b88108
-  data.tar.gz: 726527a490d7dc39cd1eb13515f80351dcdbfcf1735519131a52c74ac046b041580dcf131090e3484a17d98c385dc750ab721a3688b5f54ae8c98f9b10610bd8
+  metadata.gz: af84591cca45a914ffbf087f308eea54d264cb14a0e03d6e60732a806d5e4cf44b00d291679b1f3dfe1f6d39f58bda043ff6b1de625b9d904667d2db5f8a6c07
+  data.tar.gz: 8f5fbdc8411f73fee7b7f6b86f803d7e04e588db97dc3d4d5019b7bf08cc63a1d73b05d82a97e74135fc2370be072b00524b7df3ac301c0ed2f4eb025c4352d7

data/lib/klenod/build/context.rb

@@ -113,7 +113,8 @@ module Klenod
         asset_generation_concurrency: AssetGenerationQueue::DEFAULT_CONCURRENCY,
         asset_download_concurrency: AssetGenerationQueue::DEFAULT_DOWNLOAD_CONCURRENCY,
         profiler: nil,
-        namespace: Module.new
+        namespace: Module.new,
+        analysis: false
       )
         @source_dir = source_dir
         plugins = plugins.to_a if plugins.equal?(DEFAULT_PLUGINS)
@@ -131,12 +132,17 @@ module Klenod
             asset_generation_concurrency: asset_generation_concurrency,
             asset_download_concurrency: asset_download_concurrency,
             profiler: profiler,
-            namespace: namespace
+            namespace: namespace,
+            analysis: analysis
           )
       end
 
       attr_reader :graph, :mode, :base, :asset_origin
 
+      def analysis?
+        @graph.analysis?
+      end
+
       def evaluate(specifier)
         @graph.load(specifier)
       end

data/lib/klenod/build/filesystem_resolver.rb

@@ -15,6 +15,14 @@ module Klenod
         @root = Pathname.new(root).expand_path
         @extensions = extensions.freeze
         @path_prefix = path_prefix.to_s
+        @spelling_index = nil
+      end
+
+      # Forget the cached file listing behind did-you-mean suggestions. The
+      # owning resolver clears it whenever files change, so a suggestion never
+      # walks the source tree more than once between file events.
+      def clear_cache
+        @spelling_index = nil
       end
 
       def resolve(relative_path)
@@ -101,14 +109,21 @@ module Klenod
         []
       end
 
+      SpellingIndex = Data.define(:aliases, :checker)
+
       def spelling_corrections(relative_path)
-        aliases = spelling_aliases
-        matches =
-          DidYouMean::TreeSpellChecker
-            .new(dictionary: aliases.keys.sort, separator: "/")
-            .correct(relative_path)
+        index = spelling_index
+        matches = index.checker.correct(relative_path)
 
-        matches.flat_map { aliases.fetch(it) }.uniq.first(MAX_CORRECTIONS)
+        matches.flat_map { index.aliases.fetch(it) }.uniq.first(MAX_CORRECTIONS)
+      end
+
+      def spelling_index
+        @spelling_index ||=
+          begin
+            aliases = spelling_aliases
+            SpellingIndex.new(aliases, DidYouMean::TreeSpellChecker.new(dictionary: aliases.keys.sort, separator: "/"))
+          end
       end
 
       def spelling_aliases

data/lib/klenod/build/graph/invalidator.rb

@@ -24,10 +24,11 @@ module Klenod
           errors = []
 
           removed_module_ids.each do |module_id|
-            records.delete(module_id)
+            graph.remove_record(module_id)
             mods.delete(module_id)
           end
 
+          failed_reload_ids = []
           reloaded_module_ids =
             reload_module_ids.filter_map do |module_id|
               if evaluated_module_ids.include?(module_id)
@@ -36,12 +37,15 @@ module Klenod
                 graph.collect_module(module_id, force: true)
               end
               module_id
-            rescue => e
+            rescue StandardError, ScriptError => e
+              # ScriptError too: a syntax error in a module is not a
+              # StandardError, and letting it escape here kills the watcher
+              # thread rather than reporting the module that failed.
               mark_module_failed(module_id, e)
-              errors << [module_id, e]
+              failed_reload_ids << module_id
+              record_error(errors, module_id, e)
               nil
             end
-          failed_reload_ids = errors.map(&:first) & reload_module_ids
           blocked_dependent_ids = dependent_closure(failed_reload_ids)
           blocked_dependent_ids.each { |module_id| mods.delete(module_id) }
 
@@ -58,8 +62,8 @@ module Klenod
                 graph.collect_module(module_id, force: true)
                 nil
               end
-            rescue => e
-              errors << [module_id, e]
+            rescue StandardError, ScriptError => e
+              record_error(errors, module_id, e)
               nil
             end
           asset_updates = diff_assets(previous_assets, graph.assets)
@@ -80,6 +84,16 @@ module Klenod
 
         private
 
+        # One failure, reported once. A module whose dependency already failed
+        # in this invalidation re-raises that same error when it reloads, so a
+        # broken companion would otherwise be reported once for itself and
+        # again for every module that imports it.
+        def record_error(errors, module_id, error)
+          return if errors.any? { |(_id, recorded)| recorded.equal?(error) }
+
+          errors << [module_id, error]
+        end
+
         attr_reader :graph, :resolver, :source_loader
 
         def records
@@ -175,9 +189,7 @@ module Klenod
         end
 
         def direct_dependents(module_id)
-          records.filter_map do |candidate_id, record|
-            candidate_id if record.resolved_dependencies.any? { |dependency| dependency.module_id == module_id }
-          end
+          graph.dependents(module_id)
         end
 
         def mark_module_failed(module_id, error)
@@ -192,7 +204,8 @@ module Klenod
           source_hash = loaded_source.source_hash || Hashing.hexdigest(source)
           version = cached ? cached.version + 1 : 0
 
-          records[module_id] =
+          graph.store_record(
+            module_id,
             ModuleRecord.new(
               module_id,
               source_hash,
@@ -208,6 +221,7 @@ module Klenod
               version,
               :failed
             )
+          )
           mods[module_id] = FailedModule.new(error)
         end
 

data/lib/klenod/build/graph.rb

@@ -7,6 +7,7 @@ require "klenod/runtime/mod"
 require "klenod/runtime/bundle"
 require_relative "asset_generation_queue"
 require_relative "errors"
+require_relative "source_error"
 require_relative "graph/invalidator"
 require_relative "hashing"
 require_relative "invalidation_result"
@@ -24,9 +25,12 @@ module Klenod
       include TSort
 
       AsyncResult = Data.define(:value, :error) do
+        # ScriptError too, so a syntax error in a module comes back to the
+        # parent graph path instead of surfacing as an unhandled Async::Task
+        # exception.
         def self.capture
           new(yield, nil)
-        rescue => e
+        rescue StandardError, ScriptError => e
           new(nil, e)
         end
 
@@ -67,9 +71,11 @@ module Klenod
         asset_generation_concurrency: AssetGenerationQueue::DEFAULT_CONCURRENCY,
         asset_download_concurrency: AssetGenerationQueue::DEFAULT_DOWNLOAD_CONCURRENCY,
         profiler: nil,
-        namespace: Module.new
+        namespace: Module.new,
+        analysis: false
       )
         @profiler = profiler || Profiler.new
+        @analysis = analysis
         @resolver = Resolver.new(source_dir: source_dir, profiler: @profiler)
         @plugins = plugins
         @mode = mode
@@ -86,6 +92,16 @@ module Klenod
         @virtual_metadata = {}
         @virtual_owners = {}
         @loading_tasks = {}
+        @source_overrides = {}
+        @dependents = {}
+      end
+
+      # Analysis mode collects the graph for tooling such as the language
+      # server. Plugins must keep the record shape but skip work whose result
+      # is never used there: network fetches, hashing whole files, compiling
+      # JavaScript, and building asset bytes.
+      def analysis?
+        @analysis
       end
 
       def load(specifier)
@@ -207,10 +223,89 @@ module Klenod
         end
       end
 
+      # Editor buffers take precedence over files on disk for the given
+      # module until cleared. The next collection re-reads and, when the text
+      # changed, re-transforms the module. A caller that already transformed
+      # the same text can pass that `transform` so collection reuses it.
+      def override_source(module_id, source, transform: nil)
+        @source_overrides[module_id] = LoadResult.new(source, nil, transform)
+      end
+
+      def clear_source_override(module_id)
+        @source_overrides.delete(module_id)
+      end
+
+      def source_override?(module_id)
+        @source_overrides.key?(module_id)
+      end
+
+      # Module ids whose collected record imports the given module, eagerly
+      # or lazily.
+      def dependents(module_id)
+        @dependents.fetch(module_id) { [] }.to_a
+      end
+
+      def store_record(module_id, record)
+        previous = @records[module_id]
+        previous&.resolved_dependencies&.each { |dependency| @dependents[dependency.module_id]&.delete(module_id) }
+        record.resolved_dependencies.each { |dependency| (@dependents[dependency.module_id] ||= Set.new) << module_id }
+        @records[module_id] = record
+      end
+
+      def remove_record(module_id)
+        record = @records.delete(module_id)
+        record&.resolved_dependencies&.each { |dependency| @dependents[dependency.module_id]&.delete(module_id) }
+        record
+      end
+
+      # Collect everything reachable from a collected module through eager
+      # and lazy dependencies. Each module the walk collects itself is
+      # yielded with `nil`; a module that fails to collect is yielded with
+      # its error and the walk continues past it.
+      def collect_reachable(module_id)
+        queue = @records.fetch(module_id).resolved_dependencies.dup
+        seen = Set.new([module_id])
+        index = 0
+
+        while index < queue.length
+          dependency_id = queue[index].module_id
+          index += 1
+          next unless seen.add?(dependency_id)
+
+          record = @records[dependency_id]
+          unless record
+            begin
+              record = collect_module(dependency_id)
+            rescue StandardError, ScriptError => error
+              yield dependency_id, error if block_given?
+              next
+            end
+            yield dependency_id, nil if block_given?
+          end
+          queue.concat(record.resolved_dependencies)
+        end
+      end
+
+      # Transform source for a module without collecting it. Editor tooling
+      # such as the language server uses this to analyze unsaved text with the
+      # same plugins as a build, without adding a record or evaluating anything.
+      def transform_source(module_id, source)
+        transform = transform_module_source(module_id, source)
+        assert_supported_transform!(module_id, source, transform)
+        transform
+      end
+
       def absolute_path(module_id)
         @resolver.absolute_path(module_id)
       end
 
+      # Forget resolved paths so files created, renamed, or deleted outside
+      # a build invalidation are looked up again. Invalidation does this
+      # itself; tooling that only transforms must ask for it.
+      def clear_resolver_cache
+        @resolver.clear_cache
+      end
+
       def source_dir
         @resolver.source_dir
       end
@@ -231,7 +326,7 @@ module Klenod
           @virtual_sources.delete(module_id)
           @virtual_metadata.delete(module_id)
           @virtual_owners.delete(module_id)
-          @records.delete(module_id)
+          remove_record(module_id)
           @mods.delete(module_id)
         end
       end
@@ -281,11 +376,12 @@ module Klenod
           dependency_records = load_eager_dependency_records(resolved_dependencies)
           transform = finalize_transform_result(module_id, transform, resolved_dependencies, dependency_records)
           assert_supported_transform!(module_id, source, transform)
+          assert_generated_ruby_parses!(module_id, transform)
           transformed_hash = Hashing.hexdigest(transform.code)
-          mod = instantiate_module(module_id, transform, resolved_dependencies, dependency_records, cached)
+          mod = instantiate_module(module_id, transform, resolved_dependencies, dependency_records, cached, source)
           record = build_module_record(module_id, source, source_hash, transformed_hash, transform, resolved_dependencies, mod)
 
-          @records[module_id] = record
+          store_record(module_id, record)
           @mods[module_id] = mod
           record
         end
@@ -336,10 +432,11 @@ module Klenod
           dependency_records = collect_eager_dependency_records(resolved_dependencies)
           transform = finalize_transform_result(module_id, transform, resolved_dependencies, dependency_records)
           assert_supported_transform!(module_id, source, transform)
+          assert_generated_ruby_parses!(module_id, transform)
           transformed_hash = Hashing.hexdigest(transform.code)
           record = build_module_record(module_id, source, source_hash, transformed_hash, transform, resolved_dependencies, cached)
 
-          @records[module_id] = record
+          store_record(module_id, record)
           @mods.delete(module_id)
           @profiler.progress(:collect_module, module_id: record.id.to_s, total_records: @records.length)
           record
@@ -347,22 +444,31 @@ module Klenod
       end
 
       def evaluate_module(module_id)
-        return @mods.fetch(module_id) if @mods.key?(module_id)
+        if @mods.key?(module_id)
+          mod = @mods.fetch(module_id)
+          # A failed reload is remembered so later demand raises the stored
+          # error rather than serving stale exports.
+          raise mod.error if mod.is_a?(FailedModule)
+
+          return mod
+        end
 
         record = @records.fetch(module_id) { collect_module(module_id) }
         raise_failed_module!(record)
         evaluate_eager_dependencies(record.resolved_dependencies)
         dependency_records = dependency_records_for(eager_dependencies(record.resolved_dependencies))
         mod =
-          Runtime::Mod.new(
-            module_id.to_s,
-            record.transformed_source,
-            imports: imports_for(record.resolved_dependencies, dependency_records),
-            source_map: record.source_map,
-            version: record.version,
-            eval_path: eval_path_for(module_id),
-            namespace: namespace
-          )
+          evaluating(module_id, record.transformed_source, record.source) do
+            Runtime::Mod.new(
+              module_id.to_s,
+              record.transformed_source,
+              imports: imports_for(record.resolved_dependencies, dependency_records),
+              source_map: record.source_map,
+              version: record.version,
+              eval_path: eval_path_for(module_id),
+              namespace: namespace
+            )
+          end
 
         @mods[module_id] = mod
       end
@@ -532,20 +638,50 @@ module Klenod
         raise UnsupportedFileError, "No plugin transformed #{module_id.path.inspect}. Add a plugin for #{module_id.extname.inspect} files."
       end
 
+      # A plugin that generates Ruby can generate Ruby that does not parse.
+      # Without this the failure waits until the module is evaluated, and a
+      # production build never evaluates application modules, so the bundle
+      # would ship broken. Ruby modules are already checked by RubyPlugin
+      # against their original source, which reports better locations.
+      def assert_generated_ruby_parses!(module_id, transform)
+        return if ruby_module_extension?(module_id.extname)
+        return if Prism.parse_success?(transform.code)
+
+        raise GeneratedRubyError.new(Prism.parse(transform.code), source: transform.code, module_id: module_id)
+      end
+
       def ruby_module_extension?(extname)
         extname.empty? || extname == ".rb"
       end
 
-      def instantiate_module(module_id, transform, resolved_dependencies, dependency_records, cached)
-        Runtime::Mod.new(
-          module_id.to_s,
-          transform.code,
-          imports: imports_for(resolved_dependencies, dependency_records),
-          source_map: transform.source_map,
-          version: cached ? cached.version + 1 : 0,
-          eval_path: eval_path_for(module_id),
-          namespace: namespace
-        )
+      def instantiate_module(module_id, transform, resolved_dependencies, dependency_records, cached, source)
+        evaluating(module_id, transform.code, source) do
+          Runtime::Mod.new(
+            module_id.to_s,
+            transform.code,
+            imports: imports_for(resolved_dependencies, dependency_records),
+            source_map: transform.source_map,
+            version: cached ? cached.version + 1 : 0,
+            eval_path: eval_path_for(module_id),
+            namespace: namespace
+          )
+        end
+      end
+
+      # Evaluating a module runs its Ruby, and a syntax error there is a
+      # ScriptError that escapes every `rescue => e` in the build. Only a syntax
+      # error is wrapped: a LoadError from the module's own `require` is also a
+      # ScriptError, and reporting that as a syntax error would be wrong.
+      #
+      # The reported line refers to the evaluated source, so the excerpt has to
+      # come from that -- except for a Ruby module, whose transform only
+      # rewrites import calls in place and leaves every line where it was. There
+      # the original source reads better.
+      def evaluating(module_id, evaluated_source, original_source)
+        yield
+      rescue ::SyntaxError => error
+        excerpt_source = ruby_module_extension?(module_id.extname) ? original_source : evaluated_source
+        raise GeneratedRubyError.new(error, source: excerpt_source || evaluated_source, module_id: module_id)
       end
 
       def eval_path_for(module_id)
@@ -670,7 +806,7 @@ module Klenod
 
         tasks.each_with_object({}) do |(dependency_id, child_task), records|
           records[dependency_id] = AsyncResult.unwrap(child_task.wait)
-        rescue => e
+        rescue StandardError, ScriptError => e
           first_error ||= e
         end.tap do
           raise first_error if first_error
@@ -806,6 +942,11 @@ module Klenod
       end
 
       def load_source(module_id)
+        if (override = @source_overrides[module_id])
+          @profiler.count(:source_override_hit)
+          return override
+        end
+
         @plugins.each do |plugin|
           @profiler.count(:plugin_load_check)
           loaded =

data/lib/klenod/build/hashing.rb

@@ -13,6 +13,13 @@ module Klenod
         Digest::SHA256.file(path.to_s).hexdigest
       end
 
+      # A cheap change-detection key for a file when its bytes are not
+      # needed, e.g. graph analysis that skips asset work.
+      def self.file_key(path)
+        stat = File.stat(path.to_s)
+        short("#{path}:#{stat.mtime.to_f}:#{stat.size}")
+      end
+
       def self.short(value, length: 16)
         hexdigest(value)[0, length]
       end

data/lib/klenod/build/invalidation_result.rb

@@ -38,6 +38,23 @@ module Klenod
         :asset_updates,
         :errors
       ) do
+        # An invalidation that blew up before it could inspect any module, so
+        # subscribers still get told about the failure instead of the build
+        # going quiet.
+        def self.failed(error, module_id: nil)
+          new(
+            changed_module_ids: [].freeze,
+            removed_module_ids: [].freeze,
+            reloaded_module_ids: [].freeze,
+            reevaluated_module_ids: [].freeze,
+            added_assets: [].freeze,
+            changed_assets: [].freeze,
+            removed_assets: [].freeze,
+            asset_updates: [].freeze,
+            errors: [[module_id, error]].freeze
+          )
+        end
+
         def asset_changes
           AssetChanges.new(added_assets, changed_assets, removed_assets)
         end

data/lib/klenod/build/plugins/data_plugin.rb

@@ -5,6 +5,7 @@ require "toml-rb"
 require "yaml"
 
 require_relative "../plugin"
+require_relative "../source_error"
 require_relative "../transform_result"
 
 module Klenod
@@ -16,14 +17,24 @@ module Klenod
         end
 
         class Plugin < Klenod::Build::Plugin
+          # A format that cannot fail to parse declares no wrapped errors, and
+          # `rescue *[]` then catches nothing.
+          WRAPPED_ERRORS = [].freeze
+
           def self.extensions(*values)
             const_set(:EXTENSIONS, values.freeze)
           end
 
+          # Names the SourceError to raise and the library exceptions it wraps.
+          def self.parse_error(error_class, *wrapped)
+            const_set(:PARSE_ERROR, error_class)
+            const_set(:WRAPPED_ERRORS, wrapped.freeze)
+          end
+
           def transform(module_id, code, _context)
             return super unless self.class::EXTENSIONS.include?(module_id.extname)
 
-            data = parse(code)
+            data = parse_source(code, module_id)
             TransformResult.new(module_source(data), [], nil, [], [], {data: data})
           end
 
@@ -41,6 +52,15 @@ module Klenod
 
           private
 
+          # Each format's library raises its own exception type. Wrapping them
+          # here keeps every `parse` hook a one-liner and gives all four formats
+          # the same report.
+          def parse_source(code, module_id)
+            parse(code)
+          rescue *self.class::WRAPPED_ERRORS => error
+            raise self.class::PARSE_ERROR.new(error, source: code, module_id: module_id)
+          end
+
           def module_source(data)
             <<~RUBY
               Default = #{data.inspect}
@@ -54,8 +74,32 @@ module Klenod
           Plugin.new(...)
         end
 
+        class ParseError < Klenod::Build::SourceError
+          # "expected ':' after object key, got: '2' at line 3 column 7"
+          LOCATION = /\s+at line \d+ column \d+\z/
+
+          def kind
+            "JSON parse error"
+          end
+
+          private
+
+          def location(error)
+            return nil unless error.line
+
+            # The location is rendered as the title and the caret, so drop the
+            # copy the parser appends to its message.
+            Location.new(
+              line: error.line,
+              column: error.column,
+              detail: error.message.sub(LOCATION, "")
+            )
+          end
+        end
+
         class Plugin < DataPlugin::Plugin
           extensions ".json"
+          parse_error ParseError, JSON::ParserError
 
           private
 
@@ -70,8 +114,32 @@ module Klenod
           Plugin.new(...)
         end
 
+        class ParseError < Klenod::Build::SourceError
+          def kind
+            "YAML parse error"
+          end
+
+          private
+
+          def location(error)
+            # Psych::DisallowedClass and friends carry no location.
+            return nil unless error.is_a?(Psych::SyntaxError)
+
+            # One-based. libyaml points at the construct it was parsing when it
+            # gave up, which is not always the offending line, so `context` is
+            # worth surfacing.
+            Location.new(
+              line: error.line,
+              column: error.column,
+              detail: error.problem,
+              hints: [error.context&.capitalize].compact
+            )
+          end
+        end
+
         class Plugin < DataPlugin::Plugin
           extensions ".yaml", ".yml"
+          parse_error ParseError, Psych::Exception
 
           private
 
@@ -86,8 +154,34 @@ module Klenod
           Plugin.new(...)
         end
 
+        class ParseError < Klenod::Build::SourceError
+          # "Failed to parse input on line 2 at offset 9\ninvalid =\n\n         ^"
+          LOCATION = /\AFailed to parse input on line (?<line>\d+) at offset (?<column>\d+)/
+
+          def kind
+            "TOML parse error"
+          end
+
+          private
+
+          def location(error)
+            found = LOCATION.match(error.message)
+            return nil unless found
+
+            # toml-rb embeds its own caret diagram in the message; we render our
+            # own from the line and column instead.
+            Location.new(
+              line: found[:line].to_i,
+              # citrus reports a zero-based offset into the line.
+              column: found[:column].to_i + 1,
+              detail: "Could not parse TOML"
+            )
+          end
+        end
+
         class Plugin < DataPlugin::Plugin
           extensions ".toml"
+          parse_error ParseError, TomlRB::Error
 
           private
 

data/lib/klenod/build/plugins/gem_import_plugin.rb

@@ -39,11 +39,11 @@ module Klenod
             raise error.with_resolution_context(dependency: dependency, importer_id: dependency.importer_id)
           end
 
-          def load(module_id, _context)
+          def load(module_id, context)
             return nil unless module_id.scheme == :gem
 
             path = path_for(module_id)
-            LoadResult.new(path.binread, Hashing.file_hexdigest(path), nil)
+            LoadResult.new(path.binread, context.analysis? ? Hashing.file_key(path) : Hashing.file_hexdigest(path), nil)
           end
 
           private