importmap-plus 1.1.1 → 2.0.0

data/lib/importmap/commands.rb

@@ -1,6 +1,12 @@
 require "thor"
 require "importmap/packager"
+require "importmap/package_graph"
+require "importmap/vendored_graph"
 require "importmap/npm"
+require "importmap/provider_chain"
+require "importmap/batch_resolver"
+require "importmap/integrity"
+require "importmap/doctor"
 
 class Importmap::Commands < Thor
   include Thor::Actions
@@ -17,19 +23,24 @@ class Importmap::Commands < Thor
   option :minify, type: :boolean, desc: "Minify the vendored download with bun, esbuild or terser"
   option :lock, type: :boolean, desc: "Lock the pin at this version; update, pin and pristine leave it there until unlocked"
   option :force, type: :boolean, default: false, desc: "Re-pin locked packages, keeping each lock at the new version"
+  option :vendor, type: :boolean, default: false, desc: "Vendor the download even when it looks like it needs sibling files; converts a pin kept remote back"
+  option :integrity, type: :boolean, default: true, desc: "Write a subresource-integrity hash on a pin kept remote; --no-integrity skips the fetch that computes it"
   def pin(*packages)
-    packages = without_locked(packages, lock: options[:lock], force: options[:force])
+    packages = resolve_latest_versions(without_locked(packages, lock: options[:lock], force: options[:force]))
     # jspm resolves a package together with its dependencies; --lock and
     # --no-lock are about the packages that were asked for, not those.
     requested = packages.map { |spec| packager.package_key_for(spec) }
 
-    for_each_import_grouped_by_provider(packages, env: options[:env], from: options[:from]) do |package, url|
+    for_each_import_grouped_by_provider(packages, env: options[:env], from: options[:from], fallback: true) do |package, url|
       next if keep_locked_dependency(package, requested)
 
       pin_package(package, url, preload: options[:preload], remote: options[:remote], env: options[:env],
                                 minify: options[:minify], from: options[:from],
-                                lock: requested.include?(package) ? options[:lock] : nil)
+                                lock: requested.include?(package) ? options[:lock] : nil,
+                                vendor: requested.include?(package) && options[:vendor])
     end
+
+    exit 1 if skipped.any? || resolver.unresolved.any?
   end
 
   desc "lock [*PACKAGES]", "Lock packages at their pinned version"
@@ -60,6 +71,7 @@ class Importmap::Commands < Thor
   option :minify, type: :boolean, desc: "Minify every download; defaults to what each vendored file already is"
   def pristine
     packages = prepare_packages_with_versions
+    unrestored = []
 
     for_each_import_grouped_by_provider(packages, env: options[:env], from: options[:from]) do |package, url|
       if packager.remote_pin?(package)
@@ -67,14 +79,13 @@ class Importmap::Commands < Thor
       elsif (resolved = version_drift_of_locked(package, url))
         puts %(Skipping "#{package}" (locked at #{packager.pin_provenance(package)[:version]}, CDN resolved #{resolved}))
       else
-        minify = options[:minify].nil? ? vendored_minified?(package) : options[:minify]
-
-        puts %(Downloading "#{package}" to #{packager.vendor_path}/#{package}.js from #{url}#{" (minified)" if minify})
-
-        pin_esm_run_dependencies packager.download(package, url, minify: minify), minify: minify
-        record_provenance(package, url, minify) if provenance_changed?(package, url, minify)
+        unrestored << package unless restore_package(package, url)
       end
     end
+
+    # skipped: an esm.run bundle's dependency is pinned on the way, and one the
+    # CDN failed on is skipped the same way pin skips it.
+    exit 1 if unrestored.any? || skipped.any? || resolver.unresolved.any?
   end
 
   desc "json", "Show the full importmap in json"
@@ -149,14 +160,14 @@ class Importmap::Commands < Thor
     else
       keys = packages.any? ? requested_keys_for(packages, names) : outdated_keys_for(names)
 
-      for_each_import_grouped_by_provider(keys, env: "production") do |package, url|
+      for_each_import_grouped_by_provider(keys, env: "production", fallback: true) do |package, url|
         next if keep_locked_dependency(package, keys)
 
         pin_package(package, url)
       end
     end
 
-    exit 1 if unchecked_packages.any?
+    exit 1 if unchecked_packages.any? || skipped.any? || resolver.unresolved.any?
   end
 
   desc "packages", "Print out packages with version numbers"
@@ -164,7 +175,32 @@ class Importmap::Commands < Thor
     puts npm.packages_with_versions.map { |x| x.join(' ') }
   end
 
+  desc "doctor", "Check the importmap and the vendored files for problems"
+  option :online, type: :boolean, default: false, desc: "Also fetch every remote pin and check its integrity hash"
+  def doctor
+    require Rails.root.join("config/environment")
+
+    doctor = Importmap::Doctor.new(
+      importmap: Rails.application.importmap,
+      resolver: ActionController::Base.helpers,
+      root: Rails.root,
+      asset_paths: asset_paths,
+      online: options[:online]
+    )
+
+    doctor.diagnose.each { |finding| puts finding }
+    puts doctor.summary
+
+    exit 1 if doctor.errors?
+  end
+
   private
+    # Where a logical path is looked for on disk. Every engine adds its own, so
+    # a file a gem serves resolves here exactly as it does in a request.
+    def asset_paths
+      Rails.application.config.respond_to?(:assets) ? Rails.application.config.assets.paths : []
+    end
+
     def packager
       @packager ||= Importmap::Packager.new
     end
@@ -175,35 +211,121 @@ class Importmap::Commands < Thor
 
     # A lock outlives a rewrite unless the caller says otherwise, so update
     # and --force re-lock at the version they move to.
-    def pin_package(package, url, preload: nil, remote: false, env: "production", minify: nil, from: nil, lock: nil)
+    def pin_package(package, url, preload: nil, remote: false, env: "production", minify: nil, from: nil, lock: nil, vendor: false)
       existing_options = packager.extract_existing_pin_options(package)[package] || {}
       preload = existing_options[:preload] if preload.nil?
       integrity = existing_options[:integrity]
       locked = lock.nil? ? packager.locked?(package) : lock
+      # A pin vendored on purpose stays vendored: without this an update would
+      # inspect the download again, refuse it again, and quietly undo --vendor.
+      vendor ||= packager.vendored?(package)
+      # A pin kept remote only because its file imports siblings is vendored
+      # again now that the whole graph can be. The reason its comment records
+      # is one this gem has since learned to answer, and without this an app
+      # that hit the check once would never see the fix arrive.
+      regraph = packager.remote_reason(package) == Importmap::PackageGraph::REASON
       existing_url = existing_options[:to] if existing_options[:to].to_s.match?(Importmap::Packager::REMOTE_URL_REGEXP)
 
-      if existing_url
+      report_graph_shadowing(package)
+
+      if existing_url && !vendor && !regraph
         repin_remote_package(package, url, existing_url, preload, env: env, from: from, integrity: integrity, locked: locked)
+      elsif existing_url && from.nil? && packager.provider_for_url(existing_url).nil?
+        keep_custom_url(package, existing_url)
       elsif remote
         pin_remote_package(package, url, preload, integrity: integrity, locked: locked)
       else
-        pin_vendored_package(package, url, preload, minify: minify, integrity: integrity, locked: locked)
+        pin_vendored_package(package, url, preload, minify: minify, integrity: integrity, locked: locked, vendor: vendor)
       end
     end
 
-    def pin_vendored_package(package, url, preload, minify: nil, integrity: nil, locked: false)
+    # Nothing is said about a download until it has arrived and been found fit
+    # to serve on its own, so a package kept remote reports that and only that.
+    def pin_vendored_package(package, url, preload, minify: nil, integrity: nil, locked: false, vendor: false)
       minify = vendored_minified?(package) if minify.nil?
 
-      puts %(Pinning "#{package}" to #{packager.vendor_path}/#{package}.js via download from #{url}#{" (minified)" if minify})
+      begin
+        dependencies = packager.download(package, url, minify: minify, force: vendor, graph: !vendor)
+      rescue Importmap::Packager::Unvendorable, Importmap::Packager::NotAnEsModule => refusal
+        return pin_remote_package(package, url, preload, integrity: integrity, locked: locked, kept_remote: refusal)
+      rescue Importmap::VendoredGraph::Occupied => occupied
+        # Nothing is written, including the pin: the app is asked to move its
+        # own directory rather than told afterwards that this gem took it.
+        return skip(package, occupied.message)
+      rescue Importmap::Packager::Error => error
+        # The CDN couldn't be read once its retries were spent. The pin is left
+        # exactly as it was: a package that vendors today must not turn into a
+        # remote pin — losing the files that make it work — because of a 503.
+        return skip(package, error.message)
+      end
+
+      graph = packager.last_graph
+
+      puts %(Pinning "#{package}" to #{packager.vendor_path}/#{package}.js via download from #{url}#{" (minified)" if minify}#{graph_note(graph)})
 
-      dependencies = packager.download(package, url, minify: minify)
+      update_importmap_with_pin(package, packager.vendored_pin_for(package, url, preload, minify: minify, integrity: integrity, locked: locked, vendored: vendor))
+
+      # The line that maps the graph directory goes with the download: a
+      # package that stopped needing one takes its line and its files with it,
+      # or pin_all_from keeps mapping files nothing imports.
+      if graph
+        update_importmap_with_graph_pin(package, packager.graph_pin_for(package, url, preload))
+      else
+        packager.remove_graph(package)
+      end
 
-      update_importmap_with_pin(package, packager.vendored_pin_for(package, url, preload, minify: minify, integrity: integrity, locked: locked))
       report_lock(package) if locked
 
       pin_esm_run_dependencies(dependencies, preload: preload, minify: minify)
     end
 
+    # A package that couldn't be pinned has said why on stdout; this is what
+    # makes the exit code say so too, or `pin x && git commit` commits a tree
+    # with no pin in it. Kept on the command rather than returned up through
+    # pin_package, because an esm.run bundle's dependencies are pinned two
+    # calls down and are skipped the same way.
+    def skip(package, reason)
+      puts %(Skipping "#{package}": #{reason})
+      skipped << package
+    end
+
+    def skipped
+      @skipped ||= []
+    end
+
+    # pristine restores what each pin already says, so a package vendored
+    # before the single-file check existed — or on purpose with --vendor — is
+    # downloaded again rather than converted, and only a pin that already maps
+    # a graph directory has one crawled again. pin is where both are decided.
+    #
+    # A package the CDN can no longer serve the way its pin describes stops
+    # that one package being restored, not the run: pristine is the repair
+    # command, and the packages after it in the batch still need repairing.
+    def restore_package(package, url)
+      minify = options[:minify].nil? ? vendored_minified?(package) : options[:minify]
+
+      puts %(Downloading "#{package}" to #{packager.vendor_path}/#{package}.js from #{url}#{" (minified)" if minify})
+
+      dependencies = packager.download(package, url, minify: minify, force: true,
+                                                     graph: packager.vendored_graph(package).mapped?)
+
+      # A download that came back as one file — a --from that moved the package
+      # to a bundling CDN — takes its graph line with it, or pin_all_from is
+      # left mapping the directory that download just removed.
+      packager.remove_graph(package) unless packager.last_graph
+
+      pin_esm_run_dependencies dependencies, minify: minify
+      record_provenance(package, url, minify) if provenance_changed?(package, url, minify)
+
+      true
+    rescue Importmap::Packager::Unvendorable, Importmap::Packager::NotAnEsModule => refusal
+      puts %(Couldn't restore "#{package}": it #{refusal.message})
+      false
+    rescue Importmap::Packager::Error => error
+      puts %(Couldn't restore "#{package}": #{error.message})
+      false
+    end
+
     def lock_package(spec)
       package = packager.package_key_for(spec)
 
@@ -301,12 +423,20 @@ class Importmap::Commands < Thor
 
     # The registry knows a package by name and the import map by key: a pin of
     # "photoswipe/lightbox" is outdated when "photoswipe" is. A named update
-    # re-pins the keys that were asked for, not the name the registry answered with.
+    # re-pins the keys that were asked for: a package name means every pin
+    # that carries it, the way outdated reports it and a bare update moves it,
+    # while "photoswipe/lightbox" means that one key.
     def requested_keys_for(specs, outdated_names)
-      specs.map { |spec| packager.package_key_for(spec) }
+      specs.flat_map { |spec| keys_for(packager.package_key_for(spec)) }
            .select { |key| outdated_names.include?(packager.package_name_for(key)) }
     end
 
+    def keys_for(key)
+      return [ key ] unless key == packager.package_name_for(key)
+
+      versioned_keys_by_package[key] || [ key ]
+    end
+
     # An outdated package is outdated in every pin that carries it, so a bare
     # update re-pins those keys: "photoswipe" moving updates the app's
     # "photoswipe/lightbox" pin rather than appending a bare one beside it.
@@ -320,10 +450,12 @@ class Importmap::Commands < Thor
     # doesn't match its key (pin "buffer", to: ".../npm:jspm-core@..."); it is
     # still the only handle there is, so it goes through as itself.
     def outdated_keys_for(names)
-      keys = packager.pinned_packages.select { |key| packager.pin_version(key) }
-                                     .group_by { |key| packager.package_name_for(key) }
+      names.flat_map { |name| versioned_keys_by_package[name] || [ name ] }
+    end
 
-      names.flat_map { |name| keys[name] || [ name ] }
+    def versioned_keys_by_package
+      packager.pinned_packages.select { |key| packager.pin_version(key) }
+                              .group_by { |key| packager.package_name_for(key) }
     end
 
     def locked_pin_covering(name)
@@ -370,6 +502,7 @@ class Importmap::Commands < Thor
 
       update_importmap_with_pin(package, packager.vendored_pin_for(package, url, existing_options[:preload],
                                                                    minify: minify, integrity: existing_options[:integrity],
+                                                                   vendored: packager.vendored?(package),
                                                                    locked: packager.locked?(package)))
     end
 
@@ -387,25 +520,119 @@ class Importmap::Commands < Thor
     # A vendored package keeps coming from the CDN its pin comment names, the
     # way a remote pin keeps its provider, unless --from says otherwise.
     # Packages that aren't pinned yet resolve from jspm.
-    def for_each_import_grouped_by_provider(packages, env:, from: nil, &block)
-      packages.group_by { |spec| from || vendored_provider_for(spec) || "jspm" }.each do |provider, group|
-        for_each_import(group, env: env, from: provider, &block)
+    #
+    # +fallback+ lets the group that named no CDN try the rest of the chain
+    # when jspm can't build it, which is what pin and update want. pristine
+    # doesn't: it restores what each pin already says, and a pin that moved to
+    # another CDN would be a rewrite, not a restore.
+    def for_each_import_grouped_by_provider(packages, env:, from: nil, fallback: false, &block)
+      packages.group_by { |spec| from || vendored_provider_for(spec) || Importmap::ProviderChain::DEFAULT }.each do |provider, group|
+        resolver.each_import(group, env: env, from: provider,
+          fallback: fallback && from.nil? && Importmap::ProviderChain.default?(provider), &block)
       end
     end
 
+    # Shared by every group of a single run, so the exit code can ask it once
+    # whether anything was left unresolved.
+    def resolver
+      @resolver ||= Importmap::BatchResolver.new(packager, on_miss: method(:handle_package_not_found))
+    end
+
+    # A spec with no version means the latest, and the registry is what knows
+    # which that is: a CDN answers with the latest it has got round to indexing,
+    # which lags npm by anything from hours to a major release. Resolving it up
+    # front also pins the version for the whole chain, so falling back to
+    # another CDN can't quietly land the app on a different one. A registry
+    # that can't be reached leaves the spec alone, and the CDN chooses as before.
+    def resolve_latest_versions(specs)
+      specs.map do |spec|
+        name, version, _subpath = spec.to_s.match(Importmap::Packager::PACKAGE_SPEC_REGEXP)&.captures
+        next spec if version || name.nil?
+
+        if (latest = npm.latest_version(name))
+          puts %(Resolved "#{name}" to #{latest} from the npm registry)
+          packager.package_spec_for(spec, "@#{latest}")
+        else
+          spec
+        end
+      end
+    end
+
+    # A subpath pin that names no CDN of its own comes from wherever its
+    # package's pin came from: pdfjs-dist and pdfjs-dist/build/pdf.worker.min.mjs
+    # were pinned together, and jspm, the default, can't resolve either.
+    # A pin answers for itself first and only then for its subpaths: the URL on
+    # pin "photoswipe/lightbox" says more about where that file comes from than
+    # its package's pin does. Asking the package first grouped the two together,
+    # and one spec the CDN can't answer for fails the whole batch.
     def vendored_provider_for(spec)
-      packager.pin_provenance(packager.package_key_for(spec))&.dig(:provider)
+      key  = packager.package_key_for(spec)
+      name = packager.package_name_for(key)
+
+      return packager.pin_provenance(key)&.dig(:provider) || kept_remote_provider(key) if key == name
+
+      provider_of_pin(key) || provider_of_pin(name)
+    end
+
+    # A pin the single-file check kept remote records its CDN only in its URL:
+    # the comment's provider slot is taken by the reason it was kept. Asking the
+    # URL keeps such a pin on its own CDN when it is re-pinned — and, now that a
+    # chunked package can be vendored whole, when it converts to a download.
+    # A pin the app made remote itself carries no reason and is left to
+    # repin_remote_package, which resolves it from its URL as it always did.
+    def kept_remote_provider(key)
+      return unless packager.remote_reason(key)
+
+      packager.provider_for_url(packager.extract_existing_pin_options(key).dig(key, :to))
+    end
+
+    # A vendored pin records its CDN in the version comment; a remote one
+    # carries it in the URL and, unlocked, has no comment at all. Only a subpath
+    # asks this, and a bare package pin above stops at its comment: reading the
+    # URL of every pin would change where a plain remote pin resolves its
+    # dependencies from, which is not this lookup's business.
+    def provider_of_pin(package)
+      packager.pin_provenance(package)&.dig(:provider) ||
+        packager.provider_for_url(packager.extract_existing_pin_options(package).dig(package, :to))
     end
 
-    def pin_remote_package(package, url, preload, integrity: nil, locked: false)
-      puts %(Pinning "#{package}" to #{url})
+    # +kept_remote+ is the Packager's refusal to vendor a download it just
+    # fetched — its reasons, and the hash of the bytes it already has; without
+    # it the reason the pin already records is carried over, so update and a
+    # plain pin don't drop it.
+    def pin_remote_package(package, url, preload, integrity: nil, locked: false, kept_remote: nil)
+      if compute_integrity?
+        integrity = remote_integrity_for(url, integrity, kept_remote&.integrity)
+      elsif packager.integrity_hash?(package)
+        puts %(Dropping the integrity hash on "#{package}" (--no-integrity))
+      end
+
+      notes = +""
+      notes << %( (kept remote: #{kept_remote.reasons.to_sentence})) if kept_remote
+      notes << %( (integrity #{integrity})) if Importmap::Integrity.hash?(integrity)
+
+      puts %(Pinning "#{package}" to #{url}#{notes})
+
+      remote = kept_remote&.reasons&.first || packager.remote_reason(package)
 
       packager.remove_existing_package_file(package)
+      packager.remove_graph(package)
 
-      update_importmap_with_pin(package, packager.pin_for(package, url, preloads: preload, integrity: integrity, locked: locked))
+      update_importmap_with_pin(package, packager.pin_for(package, url, preloads: preload, integrity: integrity,
+                                                           locked: locked, remote: remote))
       report_lock(package) if locked
     end
 
+    # A URL on a host no provider answers for is the app's own choice, and the
+    # branch above skips it — but only while `vendor` is false. --vendor is
+    # about overriding the check that refuses a download, not about moving a
+    # pin to another source, so it lands here instead of vendoring whatever the
+    # spec happened to resolve to. Naming a CDN with --from is how a pin is
+    # moved on purpose, and that still goes through repin_remote_package.
+    def keep_custom_url(package, existing_url)
+      puts %(Skipping "#{package}" pinned to custom URL #{existing_url} (--vendor doesn't move a pin to another source; pass --from to choose one))
+    end
+
     def repin_remote_package(package, url, existing_url, preload, env:, from: nil, integrity: nil, locked: false)
       # `url` was already resolved from the requested CDN, so an explicit
       # --from moves the pin instead of being overruled by its current one.
@@ -424,6 +651,31 @@ class Importmap::Commands < Thor
       end
     end
 
+    # Read from the command's own options rather than threaded through every
+    # pin path: update, pristine and an esm.run bundle's dependency pins all
+    # reach pin_remote_package too, and a keyword one of them forgets to pass
+    # silently hashes a pin the app asked not to hash. Only pin declares the
+    # flag, so anything else sees nil and hashes.
+    def compute_integrity?
+      options["integrity"] != false
+    end
+
+    # A remote pin is the one place an app trusts a CDN at runtime, so the pin
+    # records the hash of what was resolved and the browser checks the bytes it
+    # gets against it. integrity: false is an app's decision and stays; a hash
+    # the line already carried was computed for the old URL and was dropped
+    # when the line was read, so nil here means "hash this". +computed+ is the
+    # hash a download that was just refused already carries — the file isn't
+    # fetched again for it.
+    def remote_integrity_for(url, integrity, computed = nil)
+      return integrity if integrity == false
+
+      computed || Importmap::Integrity.for(packager.fetch_remote(url))
+    rescue Importmap::Packager::Error => error
+      puts %(Couldn't hash #{url} (#{error.message}); pinning it without an integrity hash)
+      nil
+    end
+
     def resolve_url_from_provider(package, reference_url, provider, env:)
       version  = packager.extract_package_version_from(reference_url)
       response = packager.import(packager.package_spec_for(package, version), env: env, from: provider)
@@ -434,8 +686,52 @@ class Importmap::Commands < Thor
       nil
     end
 
+    # Importmap::Map expands a directory over the packages, so a key some other
+    # package's graph already maps resolves to that file whatever the pin about
+    # to be written says. Say so rather than write a line that does nothing.
+    def report_graph_shadowing(package)
+      mapped = Importmap::VendoredGraph.mapping_for(importmap_source, package)
+
+      puts %(Note: the graph of "#{mapped}" already maps "#{package}", and a mapped directory wins over a pin) if mapped
+    end
+
+    # Two directories mapping one package write their files under the same
+    # prefix, so at two versions the app gets one of them for every file the
+    # two share, and which one depends on the order of the lines.
+    def report_graph_conflict(pin)
+      directory, under, version = Importmap::VendoredGraph.mappings_in(pin).first
+      conflict = Importmap::VendoredGraph.conflict_in(importmap_source, under: under, version: version, except: directory)
+
+      puts %(Note: #{conflict.first} already maps "#{under}" at #{conflict.last}; a file both carry resolves to one of them) if conflict
+    end
+
+    def importmap_source
+      File.exist?("config/importmap.rb") ? File.read("config/importmap.rb") : ""
+    end
+
+    # How many files came down with the entry, so an app can see that a chunked
+    # package was vendored whole rather than as the one file that would 404.
+    def graph_note(graph)
+      %( (with #{graph.size} sibling #{"file".pluralize(graph.size)})) if graph && graph.size.positive?
+    end
+
+    def update_importmap_with_graph_pin(package, pin)
+      report_graph_conflict(pin)
+
+      if packager.vendored_graph(package).mapped?
+        gsub_file("config/importmap.rb", packager.vendored_graph(package).line_regexp, pin, verbose: false)
+      else
+        append_to_file("config/importmap.rb", "#{pin}\n", verbose: false)
+      end
+
+      packager.reload!
+    end
+
     def update_importmap_with_pin(package, pin)
       new_pin = "#{pin}\n"
+      # Two esm.run bundles can need one dependency: skipped on the first
+      # bundle's turn and pinned on the second's, it is in the tree after all.
+      skipped.delete(package)
 
       if packager.packaged?(package)
         gsub_file("config/importmap.rb", Importmap::Map.pin_line_regexp_for(package), pin, verbose: false)
@@ -446,8 +742,13 @@ class Importmap::Commands < Thor
       packager.reload!
     end
 
-    def handle_package_not_found(packages, from)
-      puts "Couldn't find any packages in #{packages.inspect} on #{from}"
+    # jspm answers 401 with its generator's reason in the body — the subpath a
+    # dependency doesn't export, the module it couldn't find — and that is the
+    # one thing that tells an app developer whether to try another CDN.
+    def handle_package_not_found(packages, from, reason: packager.last_import_error)
+      detail = Importmap::ProviderChain.tidy_reason(reason)
+
+      puts "Couldn't find any packages in #{packages.inspect} on #{from}#{" (#{detail})" if detail}"
     end
 
     def remove_line_from_file(path, pattern)