importmap-plus 1.0.0 → 2.0.0

data/lib/importmap/import_scanner.rb

@@ -0,0 +1,53 @@
+require "importmap/module_inspector"
+
+# Every module specifier a JavaScript file names, in source order, with whether
+# the browser resolves it while linking the module or later at runtime. What an
+# import map has to define, read back out of the files the map serves.
+#
+# Like Importmap::ModuleInspector and Importmap::PackageGraph::IMPORT_REGEXP
+# this reads the source with regexes rather than parsing JavaScript, so an
+# import statement spelled out inside a string literal is reported too, and a
+# form the regexp can't read — a magic comment between the keyword and the
+# specifier, a specifier built at runtime — isn't reported at all. Both
+# mistakes are the cautious direction for the one caller: a specifier reported
+# that the browser never asks for costs a pin, and a specifier missed leaves
+# the app exactly as broken as it was before anyone ran the check.
+#
+# Block comments are discounted first, through ModuleInspector#code, because a
+# published bundle is full of `/** @typedef {import('./slide.js').Slide} */` —
+# type annotations naming files the package never loads.
+class Importmap::ImportScanner
+  Import = Struct.new(:specifier, :kind, keyword_init: true) # :nodoc:
+
+  # `import("x")` of a string literal, then every static spelling: the bare
+  # `import "x"` and the `from "x"` that ends `import a from "x"`,
+  # `import {a} from "x"`, `import * as a from "x"`, `export {a} from "x"` and
+  # `export * from "x"`. The lookbehind keeps `loader.import(` and identifiers
+  # ending in `import` or `from` out, and the dynamic branch comes first so an
+  # `import(` is never read as the bare form.
+  #
+  # The closing `[),]` on the dynamic branch is what makes a computed
+  # specifier — `import(name)`, `import(`./${lang}.js`)` — match nothing
+  # rather than match half of something; it is the same test
+  # Importmap::ModuleInspector::COMPUTED_IMPORT_REGEXP makes from the other
+  # side, and the two must agree about which files hold one.
+  IMPORT_REGEXP = /
+    (?<![\w.$])
+    (?:
+      import\s*\(\s*(["'])([^"'\n]*)\1\s*[),] |
+      (?:from|import)\s*(["'])([^"'\n]*)\3
+    )
+  /x.freeze # :nodoc:
+
+  attr_reader :source
+
+  def initialize(source)
+    @source = source.to_s
+  end
+
+  def imports
+    @imports ||= Importmap::ModuleInspector.new(source).code.scan(IMPORT_REGEXP).map do |_, dynamic, _, static|
+      dynamic ? Import.new(specifier: dynamic, kind: :dynamic) : Import.new(specifier: static, kind: :static)
+    end
+  end
+end

data/lib/importmap/integrity.rb

@@ -0,0 +1,25 @@
+require "digest"
+
+# The subresource-integrity hash of a file, in the form the import map's
+# integrity section and a modulepreload link take: "sha384-" and the digest in
+# base64. Computed from the bytes a CDN hands back, which are the bytes the
+# browser will hash when it loads the module.
+#
+# pack("m0") rather than Base64.strict_encode64: base64 stopped being a default
+# gem in Ruby 3.4, and this gem depends on railties, activesupport and actionpack
+# and nothing else.
+module Importmap::Integrity
+  ALGORITHM = "sha384".freeze
+
+  class << self
+    def for(body)
+      "#{ALGORITHM}-#{[ Digest::SHA384.digest(body) ].pack("m0")}"
+    end
+
+    # A pin's integrity option is true, false, nil or a hash string; only the
+    # last is a value this computed, and only it is quoted and printed.
+    def hash?(integrity)
+      integrity.is_a?(String)
+    end
+  end
+end

data/lib/importmap/map.rb

@@ -1,4 +1,5 @@
 require "pathname"
+require "importmap/graph"
 
 class Importmap::Map
   attr_reader :packages, :directories
@@ -193,6 +194,14 @@ class Importmap::Map
     end
   end
 
+  # Yields every key the map defines and the +MappedFile+ it maps to, with the
+  # `pin_all_from` directories expanded into the keys they contribute — what
+  # #to_json is about to resolve, before a resolver has touched it. Returns an
+  # Enumerator without a block.
+  def each_expanded_package(&block)
+    expanded_packages_and_directories.each(&block)
+  end
+
   private
     MappedDir  = Struct.new(:dir, :path, :under, :preload, :integrity, keyword_init: true)
     MappedFile = Struct.new(:name, :path, :preload, :integrity, keyword_init: true)
@@ -207,6 +216,7 @@ class Importmap::Map
 
     def clear_cache
       @cache.clear
+      @graph = nil
     end
 
     def rescuable_asset_error?(error)
@@ -265,7 +275,34 @@ class Importmap::Map
     end
 
     def expanded_preloading_packages_and_directories(entry_point:)
-      expanded_packages_and_directories.select { |name, mapping| mapping.preload.in?([true, false]) ? mapping.preload : (Array(mapping.preload) & Array(entry_point)).any? }
+      preloading = expanded_packages_and_directories.select { |name, mapping| mapping.preload.in?([true, false]) ? mapping.preload : (Array(mapping.preload) & Array(entry_point)).any? }
+      reachable_only(preloading, entry_point: entry_point)
+    end
+
+    # With config.importmap.preload_strategy == :reachable, a pin that says
+    # `preload: true` is preloaded only when the entry point's own imports
+    # reach it, so a package behind an `import()` stops being fetched on every
+    # page without anyone maintaining a `preload: false` for it and for
+    # everything it depends on. A pin naming the entry point is the app
+    # overruling the graph, and `preload: false` is off either way.
+    def reachable_only(packages, entry_point:)
+      return packages unless Rails.application.config.importmap.preload_strategy == :reachable
+
+      reachable = graph.reachable_from(Array(entry_point))
+      packages.select { |name, mapping| mapping.preload != true || reachable.include?(name) }
+    end
+
+    # Memoised beside the rendered map rather than inside it: #cache_as shares
+    # one namespace with the cache_key the preload helper passes, which is the
+    # entry point's own name, so an app with an entry point named "graph" would
+    # read this back as its preload set. Dropped by the same clear_cache the
+    # sweeper calls when a .js file under a watched directory changes.
+    def graph
+      @graph ||= Importmap::Graph.new(self, roots: asset_paths)
+    end
+
+    def asset_paths
+      (config = Rails.application.config).respond_to?(:assets) ? config.assets.paths : []
     end
 
     def expanded_packages_and_directories

data/lib/importmap/module_inspector.rb

@@ -0,0 +1,200 @@
+require "strscan"
+
+# Decides whether a downloaded ESM file can stand alone as the single file an
+# import map entry points at. A vendored package is exactly one file served
+# under a digested asset path, so anything the file expects to find beside
+# itself — a sibling module, a worker script, a wasm binary, its own directory
+# via import.meta.url — resolves to a 404 in the browser.
+#
+# Like Importmap::EsmRun::IMPORT_REGEXP this reads the source with regexes
+# rather than parsing JavaScript, so the same text inside a string still counts.
+# It is deliberately the cautious direction: a false positive keeps a working
+# remote pin, and `pin --vendor` is the escape hatch. Every judgement call here
+# leans that way, because the two mistakes are not equal — a package wrongly
+# kept remote still works, a package wrongly vendored 404s in production.
+class Importmap::ModuleInspector
+  # `from "./x"`, `from '../x'`, a bare `import "./x"` and a dynamic
+  # `import("./x")`. Anchored on the keyword, and the lookbehind keeps
+  # `obj.import(` and identifiers ending in `import` out.
+  RELATIVE_IMPORT_REGEXP = /(?<![\w.$])(?:from|import)\s*\(?\s*["']\.{1,2}\//.freeze # :nodoc:
+  # import() of anything but a string literal: the specifier is computed at
+  # runtime, so what it resolves to can't be known here, let alone vendored.
+  # The whitespace lives inside the lookahead on purpose: as `import\s*\(\s*`
+  # followed by a negative lookahead, `\s*` backtracks to zero and the lookahead
+  # then reads the space rather than the quote, so `import( "crypt" )` reads as
+  # computed.
+  COMPUTED_IMPORT_REGEXP = /(?<![\w.$])import\s*\((?!\s*["'][^"']*["']\s*[),])/.freeze # :nodoc:
+  # A worker is fetched as its own top-level script and never goes through the
+  # import map, so its URL has to exist on its own.
+  # The qualifier group catches `new window.Worker(…)` and `new self.Worker(…)`;
+  # it needs the dot, so `new WorkerPool(…)` is still left alone.
+  WORKER_REGEXP = /(?<![\w.$])new\s+(?:[\w$]+\s*\.\s*)*(?:Shared)?Worker\s*\(/.freeze # :nodoc:
+  # import.meta.url is the file's own digested asset path, which is not the
+  # directory the package's other files were published to.
+  IMPORT_META_URL_REGEXP = /(?<![\w.$])import\s*\.\s*meta\s*\.\s*url\b/.freeze # :nodoc:
+  # A .wasm binary is fetched at runtime by a path the package computes; it is
+  # never part of the JavaScript file that names it. Template literals count —
+  # the path is usually built from a base — and so does a cache-busting query
+  # or a fragment after the extension.
+  WASM_REGEXP = /(["'`])[^"'`\n]*\.wasm(?:[?#][^"'`\n]*)?\1/.freeze # :nodoc:
+
+  # In precedence order: the first one that matches is the reason reported, and
+  # relative imports come first because they are both the commonest cause and
+  # the one an app developer can act on.
+  PATTERNS = {
+    "relative imports" => RELATIVE_IMPORT_REGEXP,
+    "dynamic imports"  => COMPUTED_IMPORT_REGEXP,
+    "workers"          => WORKER_REGEXP,
+    "import.meta.url"  => IMPORT_META_URL_REGEXP,
+    "wasm"             => WASM_REGEXP
+  }.freeze # :nodoc:
+
+  # A string literal, consumed whole so nothing inside it is ever read as
+  # code. Unterminated, it simply doesn't match and the quote is stepped over.
+  STRING_REGEXP = /"(?:[^"\\\n]|\\.)*"|'(?:[^'\\\n]|\\.)*'|`(?:[^`\\]|\\.)*`/m.freeze # :nodoc:
+  # A regex literal, consumed whole for the same reason: `/[/*]/` otherwise
+  # hands the block-comment matcher an opener and loses the file to the next
+  # `*/`. The body allows an escape or a character class, since both can hold
+  # the delimiter.
+  REGEXP_LITERAL_REGEXP = %r{/(?![*/])(?:[^/\\\n\[]|\\.|\[(?:[^\]\\\n]|\\.)*\])+/[dgimsuvy]*}.freeze # :nodoc:
+  # Whether a `/` opens a regex literal or divides depends on the token before
+  # it. Getting it wrong can only keep text that should have been dropped —
+  # never drop text that should have been kept — so the cautious reading is
+  # the safe one here too. Only the tail of what has been kept is looked at:
+  # matching the whole buffer at every slash is quadratic, and pdf.js is a
+  # megabyte of minified source with a slash in every other line.
+  # `)` and `}` are in the set even though they also end an expression that a
+  # `/` would divide: reading `(a + b) / 2` as a regex keeps the text either
+  # way, while leaving them out lets `if (x) /[/*]/` open a false comment.
+  # Verified against 31 published packages — no verdict changes.
+  BEFORE_REGEXP_LITERAL_REGEXP =
+    /(?:[(,=:\[!&|?{};+\-*%~^<>)\}]|\b(?:return|throw|typeof|case|in|of|do|else|yield|await|delete|void|instanceof|new))\s*\z/.freeze # :nodoc:
+  BLOCK_COMMENT_REGEXP = %r{/\*.*?\*/}m.freeze # :nodoc:
+
+  # An ESM statement, in every spelling a published bundle uses: `import "x"`,
+  # `import a from "b"`, `import a, {b} from "c"`, `import{a}from"b"`,
+  # `export{a}`, `export * from "b"`, `export default`, and `export` in front
+  # of a declaration. The lookbehind keeps `obj.import(` and `reimport` out;
+  # the space the identifier form insists on keeps lodash's `importsKeys,` out,
+  # which the `exports` of a UMD wrapper needs no help with.
+  ESM_STATEMENT_REGEXP = /
+    (?<![\w.$])
+    (?:
+      import\s*["'{*] | import\s+[\w$]+\s*(?:,|\bfrom\b) |
+      export\s*(?:[{*]|\b(?:default|var|let|const|function|class|async)\b)
+    )
+  /x.freeze # :nodoc:
+  # What a CommonJS, AMD or UMD bundle says instead: it assigns to an
+  # `exports`, it requires or defines, or it sniffs for the loader it is
+  # running under. None of these is
+  # proof on its own — an ESM file may well mention `require(` — so they only
+  # decide a file that declares no exports of its own, where the only mistake
+  # they can make is sending a package on to the next CDN.
+  #
+  # The loader sniff has to be here because the assignment often isn't:
+  # lodash reaches its `exports` through `freeModule.exports`, and spells
+  # `module.exports` out only in a comment, which is stripped before any of
+  # this is read.
+  COMMONJS_REGEXP = /
+    (?<![\w.$])(?:module\s*\.\s*exports|exports\s*(?:\.\s*[\w$]+|\[[^\]]+\])\s*=|require\s*\(|define\s*\(|typeof\s+(?:exports|module|define)\s*[!=]=) |
+    \.\s*exports\s*=
+  /x.freeze # :nodoc:
+  LINE_COMMENT_REGEXP = %r{//[^\n]*}.freeze # :nodoc:
+
+  attr_reader :source
+
+  # The source as it would be written to vendor/javascript: after an esm.run
+  # bundle's imports have been rewritten to bare specifiers, before minifying.
+  def initialize(source)
+    @source = source.to_s
+  end
+
+  # Every pattern the source matches, in precedence order.
+  def reasons
+    @reasons ||= PATTERNS.filter_map { |reason, regexp| reason if code.match?(regexp) }
+  end
+
+  def reason
+    reasons.first
+  end
+
+  def vendorable?
+    reasons.empty?
+  end
+
+  # Whether the file is something an import map entry can resolve to: it says
+  # what it exports, or at least never says it is CommonJS. A UMD bundle loaded
+  # as a module runs and exports nothing, so `import x from "pkg"` fails to
+  # link — "The requested module does not provide an export named 'default'" —
+  # and takes every module that imported it down too, in the browser only.
+  #
+  # The two halves read different text, each in the direction that keeps a
+  # non-module out. An import statement counts only outside a string literal
+  # and outside a line comment, because a bundle that ships a usage example in
+  # either is still CommonJS; a `module.exports` counts wherever it appears,
+  # because a UMD wrapper hidden in a string is a UMD wrapper.
+  def es_module?
+    statements.match?(ESM_STATEMENT_REGEXP) || !code.match?(COMMONJS_REGEXP)
+  end
+
+  # The source with its block comments discounted: what every pattern above is
+  # matched against, and what Importmap::PackageGraph reads its specifiers out
+  # of, so the crawl follows exactly the imports this class counted.
+  def code
+    @code ||= without_block_comments
+  end
+
+  private
+    def statements
+      @statements ||= without_block_comments(statements_only: true)
+    end
+
+    # Block comments are discounted before anything is matched. A published
+    # bundle is full of `/** @typedef {import('./slide.js').Slide} Slide */` —
+    # type annotations naming files the package never loads, which would
+    # otherwise keep a self-contained package remote; photoswipe alone carries
+    # 76 of them.
+    #
+    # The scan walks the source instead of running a `/\*.*?\*/` over it,
+    # because that regex reads a `"/*"` inside a string as a comment opener and
+    # swallows the code up to the next `*/` — and an `import "./sibling.js"`
+    # swallowed there is precisely the 404 this class exists to catch. Strings
+    # are matched first and kept whole, so a comment opener inside one is never
+    # reached.
+    #
+    # Line comments are left alone. Stripping them would mean reading `//` as
+    # an opener inside a regex literal such as `[//]`, which is the same trap
+    # in the same dangerous direction, and nothing is known to hide behind one.
+    # With +statements_only+ the line comments go too, and every literal is
+    # emptied rather than kept — its delimiters stay, so `import "x"` still
+    # reads as an import statement while the text inside it stops being read
+    # as code at all. That mode feeds only the ES-module check, where the
+    # `[//]` trap above can at worst send a package on to the next CDN.
+    def without_block_comments(statements_only: false)
+      scanner = StringScanner.new(source)
+      kept    = +""
+
+      until scanner.eos?
+        if scanner.skip(BLOCK_COMMENT_REGEXP) || (statements_only && scanner.skip(LINE_COMMENT_REGEXP))
+          next
+        elsif (literal = scanner.scan(STRING_REGEXP)) ||
+              (regexp_literal_next?(kept, scanner) && (literal = scanner.scan(REGEXP_LITERAL_REGEXP)))
+          kept << (statements_only ? literal[0, 1] * 2 : literal)
+        else
+          kept << scanner.getch
+        end
+      end
+
+      kept
+    end
+
+    # Wide enough for the longest keyword above plus the indentation a
+    # pretty-printed file can put between it and the slash; a keyword the
+    # window cuts in half simply reads as division, which keeps less.
+    REGEXP_LOOKBEHIND_LIMIT = 32 # :nodoc:
+
+    def regexp_literal_next?(kept, scanner)
+      scanner.match?(%r{/}) &&
+        (kept[-REGEXP_LOOKBEHIND_LIMIT..] || kept).match?(BEFORE_REGEXP_LITERAL_REGEXP)
+    end
+end

data/lib/importmap/npm.rb

@@ -19,8 +19,13 @@ class Importmap::Npm
     @vendor_path    = Pathname.new(vendor_path)
   end
 
-  def outdated_packages
-    packages_with_versions.each_with_object([]) do |(package, current_version), outdated_packages|
+  # With +only:+, just those packages are looked up; names may carry a
+  # subpath (apexcharts/core), which the registry doesn't know about.
+  def outdated_packages(only: nil)
+    wanted = only&.map { |name| extract_base_package_name(name) }
+    candidates = wanted ? packages_with_versions.select { |package, _| wanted.include?(package) } : packages_with_versions
+
+    candidates.each_with_object([]) do |(package, current_version), outdated_packages|
       outdated_package = OutdatedPackage.new(name: package, current_version: current_version)
 
       if !(response = get_package(package))
@@ -38,6 +43,15 @@ class Importmap::Npm
     end.sort_by(&:name)
   end
 
+  # The version the registry calls latest, or nil when it couldn't be asked.
+  # The registry is authoritative about what a package's latest version is,
+  # where a CDN answers with the latest it happens to have built.
+  def latest_version(package)
+    response = get_package(package)
+
+    find_latest_version(response)&.to_s unless response.nil? || response["error"]
+  end
+
   def vulnerable_packages
     get_audit.flat_map do |package, vulnerabilities|
       vulnerabilities.map do |vulnerability|
@@ -51,21 +65,25 @@ class Importmap::Npm
     end.sort_by { |p| [p.name, p.severity] }
   end
 
+  # Memoized: a command that asks twice would otherwise report the
+  # unversioned packages twice.
   def packages_with_versions
-    # We cannot use the name after "pin" because some dependencies are loaded from inside packages
-    # Eg. pin "buffer", to: "https://ga.jspm.io/npm:@jspm/core@2.0.0-beta.19/nodelibs/browser/buffer.js"
-    with_versions = importmap.scan(/^pin .*(?<=npm:|npm\/|skypack\.dev\/|unpkg\.com\/|esm\.sh\/|esm\.sh\/\*)([^@\/]+)@(\d+\.\d+\.\d+(?:[^\/\s"']*))/) |
-      importmap.scan(/#{PIN_REGEX} #.*@(\d+\.\d+\.\d+(?:[^\s]*)).*$/)
-
-    with_versions.map! do |package, version|
-      [extract_base_package_name(package), version]
-    end.uniq!
+    @packages_with_versions ||= begin
+      # We cannot use the name after "pin" because some dependencies are loaded from inside packages
+      # Eg. pin "buffer", to: "https://ga.jspm.io/npm:@jspm/core@2.0.0-beta.19/nodelibs/browser/buffer.js"
+      with_versions = importmap.scan(/^pin .*(?<=npm:|npm\/|skypack\.dev\/|unpkg\.com\/|esm\.sh\/|esm\.sh\/\*)([^@\/]+)@(\d+\.\d+\.\d+(?:[^\/\s"']*))/) |
+        importmap.scan(/#{PIN_REGEX} #.*@(\d+\.\d+\.\d+(?:[^\s]*)).*$/)
+
+      with_versions.map! do |package, version|
+        [extract_base_package_name(package), version]
+      end.uniq!
+
+      vendored_packages_without_version(with_versions).each do |package, path|
+        $stdout.puts "Ignoring #{package} (#{path}) since no version is specified in the importmap"
+      end
 
-    vendored_packages_without_version(with_versions).each do |package, path|
-      $stdout.puts "Ignoring #{package} (#{path}) since no version is specified in the importmap"
+      with_versions
     end
-
-    with_versions
   end
 
   private
@@ -84,6 +102,11 @@ class Importmap::Npm
       JSON.parse(response)
     rescue JSON::ParserError
       nil
+    rescue HTTPError => error
+      # One package the registry won't answer for shouldn't end the run: the
+      # caller records it as unchecked, so the rest are still reported on.
+      # with_retries has already spent its attempts by the time we get here.
+      { "error" => error.message }
     end
 
     def get_json(uri)