importmap-plus 1.0.0

data/lib/importmap/packager.rb

@@ -0,0 +1,428 @@
+require "net/http"
+require "uri"
+require "json"
+require "importmap/minifier"
+require "importmap/http_retries"
+
+class Importmap::Packager
+  include Importmap::HttpRetries
+
+  PIN_REGEX = /#{Importmap::Map::PIN_REGEX}(.*)/.freeze # :nodoc:
+  PRELOAD_OPTION_REGEXP = /preload:\s*(\[[^\]]+\]|true|false|["'][^"']*["'])/.freeze # :nodoc:
+  TO_OPTION_REGEXP = /to:\s*["']([^"']*)["']/.freeze # :nodoc:
+  REMOTE_URL_REGEXP = %r{\Ahttps?://}.freeze # :nodoc:
+
+  PROVIDER_HOSTS = {
+    "ga.jspm.io"       => "jspm.io",
+    "unpkg.com"        => "unpkg",
+    "cdn.jsdelivr.net" => "jsdelivr",
+    "cdn.skypack.dev"  => "skypack",
+    "esm.sh"           => "esm.sh"
+  }.freeze # :nodoc:
+
+  # jsDelivr's bundling endpoint (https://www.jsdelivr.com/esm): one minified
+  # ESM file per package, with its dependencies referenced as /npm/dep@ver/+esm.
+  ESM_RUN_PROVIDER    = "esm.run".freeze # :nodoc:
+  ESM_RUN_CDN         = "https://cdn.jsdelivr.net/npm/".freeze # :nodoc:
+  ESM_RUN_URL_REGEXP  = %r{\Ahttps://cdn\.jsdelivr\.net/npm/.+/\+esm\z}.freeze # :nodoc:
+  # An esm.run bundle's own imports: `from"/npm/dep@1.2.3/+esm"`, `import"…"`,
+  # `import("…")`, `export … from"…"`. Anchored on the keyword so an ordinary
+  # string that happens to look like a bundle URL is left alone. What it does
+  # not do is parse JavaScript, so the same text inside a string or a comment
+  # would still be rewritten — a jsDelivr bundle is esbuild output whose only
+  # surviving comment is the banner, and a root-relative /npm/ URL is
+  # meaningless anywhere but in one of its own imports.
+  ESM_RUN_IMPORT_REGEXP =
+    %r{((?:\bfrom|\bimport)\s*\(?\s*)(["'])/npm/((?:@[^/"'@]+/)?[^/"'@]+)@([^/"']+)((?:/[^"']*?)?)/\+esm\2}.freeze # :nodoc:
+  # name[@version][/subpath] — a leading "@" distinguishes a scoped name
+  # (@scope/pkg) from an unscoped name with a subpath (apexcharts/core).
+  PACKAGE_SPEC_REGEXP = %r{\A(@[^@/]+/[^@/]+|[^@/]+)(?:@([^/]+))?(/.+)?\z}.freeze # :nodoc:
+  # The version comment on a vendored pin, plus what it was built with:
+  #   pin "luxon" # @3.7.2
+  #   pin "luxon" # @3.7.2 (esm.run, minified)
+  PIN_PROVENANCE_REGEXP = /#\s*@([^\s(]+)(?:\s+\(([^)]*)\))?/.freeze # :nodoc:
+  DEFAULT_PROVIDER = "jspm.io".freeze # :nodoc:
+
+  Error        = Class.new(StandardError)
+  HTTPError    = Class.new(Error)
+  ServiceError = Error.new(Error)
+
+  singleton_class.attr_accessor :endpoint
+  self.endpoint = URI("https://api.jspm.io/generate")
+
+  singleton_class.attr_accessor :esm_run_resolver
+  self.esm_run_resolver = URI("https://data.jsdelivr.com/v1/packages/npm/")
+
+  # CDNs reset connections and rate-limit bursts. Each request is tried this
+  # many times, pausing retry_wait × attempt between tries, before it fails.
+  # Shared with Importmap::Npm, which talks to the registry the same way.
+  class << self
+    def retry_attempts = Importmap::HttpRetries.attempts
+    def retry_attempts=(value)
+      Importmap::HttpRetries.attempts = value
+    end
+
+    def retry_wait = Importmap::HttpRetries.wait
+    def retry_wait=(value)
+      Importmap::HttpRetries.wait = value
+    end
+  end
+
+  # Anything responding to #call(source) => String. Defaults to the first of
+  # bun, esbuild or terser found on the machine.
+  singleton_class.attr_writer :minifier
+
+  def self.minifier
+    @minifier ||= Importmap::Minifier.new
+  end
+
+  attr_reader :vendor_path
+
+  def initialize(importmap_path = "config/importmap.rb", vendor_path: "vendor/javascript")
+    @importmap_path = Pathname.new(importmap_path)
+    @vendor_path    = Pathname.new(vendor_path)
+  end
+
+  def import(*packages, env: "production", from: "jspm")
+    return import_from_esm_run(packages) if esm_run?(from)
+
+    response = post_json({
+      "install"      => Array(packages),
+      "flattenScope" => true,
+      "env"          => [ "browser", "module", env ],
+      "provider"     => normalize_provider(from),
+    })
+
+    case response.code
+    when "200"
+      extract_parsed_response(response)
+    when "404", "401"
+      nil
+    else
+      handle_failure_response(response)
+    end
+  end
+
+  def pin_for(package, url = nil, preloads: nil)
+    to = url ? %(, to: "#{url}") : ""
+    preload_param = preload(preloads)
+
+     %(pin "#{package}") + to + preload_param
+  end
+
+  # The pin line for a vendored download. The version comment also records
+  # the CDN when it isn't jspm and whether the file was minified, so a later
+  # update or pristine can do the same again:
+  #
+  #   pin "luxon" # @3.7.2
+  #   pin "luxon" # @3.7.2 (esm.run, minified)
+  #
+  def vendored_pin_for(package, url, preloads = nil, minify: false)
+    filename = package_filename(package)
+    version  = extract_package_version_from(url)
+    to = "#{package}.js" != filename ? filename : nil
+
+    provenance = []
+    provenance << provider_for_url(url) if provider_for_url(url) && provider_for_url(url) != DEFAULT_PROVIDER
+    provenance << "minified" if minify
+
+    pin_for(package, to, preloads: preloads) + %( # #{version}) + (provenance.any? ? %( (#{provenance.join(", ")})) : "")
+  end
+
+  # What the pin's version comment says a vendored package was built with:
+  # { version:, provider:, minified: }, or nil for a pin without one.
+  def pin_provenance(package)
+    return unless @importmap_path.exist?
+
+    line = importmap.lines.find { |candidate| candidate.match?(Importmap::Map.pin_line_regexp_for(package)) }
+    match = line&.match(PIN_PROVENANCE_REGEXP)
+    return unless match
+
+    details = match[2].to_s.split(",").map(&:strip)
+    minified = details.delete("minified") ? true : false
+
+    { version: match[1], provider: details.first, minified: minified }
+  end
+
+  def packaged?(package)
+    importmap.match(Importmap::Map.pin_line_regexp_for(package))
+  end
+
+  # Downloads +url+ into vendor/javascript. With +minify: true+ the source is
+  # run through .minifier first and the file header records it, so later
+  # updates keep minifying. Returns the dependencies an esm.run bundle imports
+  # as [package, url] pairs (empty for every other provider), with the bundle's
+  # absolute /npm/... imports rewritten to bare specifiers on the way in.
+  def download(package, url, minify: false)
+    ensure_vendor_directory_exists
+    remove_existing_package_file(package)
+    download_package_file(package, url, minify: minify)
+  end
+
+  def remove(package)
+    remove_existing_package_file(package)
+    remove_package_from_importmap(package)
+  end
+
+  def extract_existing_pin_options(packages)
+    return {} unless @importmap_path.exist?
+
+    packages = Array(packages)
+
+    all_package_options = build_package_options_lookup(importmap.lines)
+
+    packages.to_h do |package|
+      [package, all_package_options[package] || {}]
+    end
+  end
+
+  # Drops the cached import map so a read after a write sees the new file:
+  # pinning an esm.run bundle appends pins and then asks the map what its
+  # dependencies still need.
+  def reload!
+    @importmap = nil
+    self
+  end
+
+  # The provenance a pin would record for +url+, shaped like #pin_provenance
+  # returns, so a caller can tell whether the existing pin already says this.
+  def provenance_for(url, minify: false)
+    provider = provider_for_url(url)
+
+    { provider: provider == DEFAULT_PROVIDER ? nil : provider, minified: minify ? true : false }
+  end
+
+  def remote_pin?(package)
+    options = extract_existing_pin_options(package)[package] || {}
+    options[:to].to_s.match?(REMOTE_URL_REGEXP)
+  end
+
+  def provider_for_url(url)
+    return ESM_RUN_PROVIDER if url.to_s.match?(ESM_RUN_URL_REGEXP)
+
+    PROVIDER_HOSTS[URI(url.to_s).host]
+  rescue URI::InvalidURIError
+    nil
+  end
+
+  def esm_run?(provider)
+    provider.to_s == ESM_RUN_PROVIDER
+  end
+
+  # The import-map key a package spec pins: "apexcharts@7.1.0/core" pins
+  # "apexcharts/core", "@hotwired/stimulus@3" pins "@hotwired/stimulus".
+  def package_key_for(spec)
+    name, _version, subpath = spec.to_s.match(PACKAGE_SPEC_REGEXP)&.captures
+    name ? "#{name}#{subpath}" : spec.to_s
+  end
+
+  def remove_existing_package_file(package)
+    FileUtils.rm_rf vendored_package_path(package)
+  end
+
+  def extract_package_version_from(url)
+    url.match(/@\d+\.\d+\.\d+[^\/\s"']*/)&.to_a&.first
+  end
+
+  private
+    def build_package_options_lookup(lines)
+      lines.each_with_object({}) do |line, package_options|
+        match = line.strip.match(PIN_REGEX)
+
+        if match
+          package_name = match[1]
+          options_part = match[2]
+
+          options = {}
+
+          if (preload_match = options_part.match(PRELOAD_OPTION_REGEXP))
+            options[:preload] = preload_from_string(preload_match[1])
+          end
+
+          if (to_match = options_part.match(TO_OPTION_REGEXP))
+            options[:to] = to_match[1]
+          end
+
+          package_options[package_name] = options if options.any?
+        end
+      end
+    end
+
+    def preload_from_string(value)
+      case value
+      when "true"
+        true
+      when "false"
+        false
+      when /^\[.*\]$/
+        JSON.parse(value)
+      else
+        value.gsub(/["']/, "")
+      end
+    end
+
+    def preload(preloads)
+      case Array(preloads)
+      in []
+        ""
+      in ["true"] | [true]
+        %(, preload: true)
+      in ["false"] | [false]
+        %(, preload: false)
+      in [string]
+        %(, preload: "#{string}")
+      else
+        %(, preload: #{preloads})
+      end
+    end
+
+    def post_json(body)
+      with_retries("posting to #{self.class.endpoint}") do
+        Net::HTTP.post(self.class.endpoint, body.to_json, "Content-Type" => "application/json")
+      end
+    rescue HTTPError
+      raise
+    rescue => error
+      raise HTTPError, "Unexpected transport error (#{error.class}: #{error.message})"
+    end
+
+    def normalize_provider(name)
+      name.to_s == "jspm" ? "jspm.io" : name.to_s
+    end
+
+    def extract_parsed_response(response)
+      parsed = JSON.parse(response.body)
+      imports = parsed.dig("map", "imports")
+
+      {
+        imports: imports,
+      }
+    end
+
+    def handle_failure_response(response)
+      if error_message = parse_service_error(response)
+        raise ServiceError, error_message
+      else
+        raise HTTPError, "Unexpected response code (#{response.code})"
+      end
+    end
+
+    def parse_service_error(response)
+      JSON.parse(response.body.to_s)["error"]
+    rescue JSON::ParserError
+      nil
+    end
+
+    def importmap
+      @importmap ||= File.read(@importmap_path)
+    end
+
+
+    def ensure_vendor_directory_exists
+      FileUtils.mkdir_p @vendor_path
+    end
+
+    def remove_package_from_importmap(package)
+      all_lines = File.readlines(@importmap_path)
+      with_lines_removed = all_lines.grep_v(Importmap::Map.pin_line_regexp_for(package))
+
+      File.open(@importmap_path, "w") do |file|
+        with_lines_removed.each { |line| file.write(line) }
+      end
+    end
+
+    def download_package_file(package, url, minify: false)
+      response = with_retries("downloading #{url}") { Net::HTTP.get_response(URI(url)) }
+
+      if response.code == "200"
+        source = response.body.dup.force_encoding("UTF-8")
+        source, dependencies = rewrite_esm_run_imports(source) if url.match?(ESM_RUN_URL_REGEXP)
+        source = self.class.minifier.call(source) if minify
+
+        save_vendored_package(package, url, source, minified: minify)
+
+        dependencies || []
+      else
+        handle_failure_response(response)
+      end
+    end
+
+    def save_vendored_package(package, url, source, minified: false)
+      File.open(vendored_package_path(package), "w+") do |vendored_package|
+        vendored_package.write "// #{package}#{extract_package_version_from(url)} downloaded from #{url}#{" (minified)" if minified}\n\n"
+
+        vendored_package.write remove_sourcemap_comment_from(source).force_encoding("UTF-8")
+      end
+    end
+
+    # Turns import "/npm/dep@1.2.3/+esm" into import "dep" so the bundle
+    # resolves through the import map, and lists what it needs pinned.
+    def rewrite_esm_run_imports(source)
+      dependencies = {}
+      versions = Hash.new { |hash, key| hash[key] = [] }
+
+      rewritten = source.gsub(ESM_RUN_IMPORT_REGEXP) do
+        keyword, quote, name, version, subpath = $1, $2, $3, $4, $5.to_s
+        key = "#{name}#{subpath}"
+        dependencies[key] ||= "#{ESM_RUN_CDN}#{name}@#{version}#{subpath}/+esm"
+        versions[key] << version unless versions[key].include?(version)
+        "#{keyword}#{quote}#{name}#{subpath}#{quote}"
+      end
+
+      # An import map maps a bare specifier to one file, so a bundle that
+      # imports the same package at two versions can only get the first one
+      # it asked for. Say so rather than pick silently.
+      versions.each do |key, seen|
+        next if seen.one?
+
+        warn %(#{key} is imported at #{seen.join(", ")} by this bundle; pinning @#{seen.first}, an import map holds one version)
+      end
+
+      [rewritten, dependencies.to_a]
+    end
+
+    def import_from_esm_run(packages)
+      imports = packages.to_h do |spec|
+        name, requested, subpath = spec.match(PACKAGE_SPEC_REGEXP)&.captures
+        raise Error, "Can't parse package spec #{spec.inspect}" unless name
+
+        version = resolve_esm_run_version(name, requested)
+        return nil unless version
+
+        ["#{name}#{subpath}", "#{ESM_RUN_CDN}#{name}@#{version}#{subpath}/+esm"]
+      end
+
+      { imports: imports }
+    end
+
+    def resolve_esm_run_version(name, requested)
+      uri = self.class.esm_run_resolver.dup
+      uri.path += "#{name}/resolved"
+      uri.query = "specifier=#{URI.encode_www_form_component(requested)}" if requested
+
+      response = with_retries("resolving #{uri}") { Net::HTTP.get_response(uri) }
+
+      case response.code
+      when "200"
+        JSON.parse(response.body)["version"]
+      when "404"
+        nil
+      else
+        handle_failure_response(response)
+      end
+    rescue JSON::ParserError
+      raise HTTPError, "Unexpected response from #{uri}"
+    end
+
+    def remove_sourcemap_comment_from(source)
+      source.gsub(/^\/\/# sourceMappingURL=.*/, "")
+    end
+
+    def vendored_package_path(package)
+      @vendor_path.join(package_filename(package))
+    end
+
+    def package_filename(package)
+      package.gsub("/", "--") + ".js"
+    end
+end

data/lib/importmap/reloader.rb

@@ -0,0 +1,23 @@
+require "active_support"
+require "active_support/core_ext/module/delegation"
+
+class Importmap::Reloader
+  delegate :execute_if_updated, :execute, :updated?, to: :updater
+
+  def reload!
+    import_map_paths.each { |path| Rails.application.importmap.draw(path) }
+  end
+
+  private
+    def updater
+      @updater ||= config.file_watcher.new(import_map_paths) { reload! }
+    end
+
+    def import_map_paths
+      config.importmap.paths
+    end
+
+    def config
+      Rails.application.config
+    end
+end

data/lib/importmap/version.rb

@@ -0,0 +1,6 @@
+module Importmap
+  # importmap-plus versions independently of the importmap-rails it forks.
+  # UPSTREAM_VERSION is the importmap-rails release this tracks.
+  VERSION = "1.0.0"
+  UPSTREAM_VERSION = "2.2.3"
+end

data/lib/importmap-plus.rb

@@ -0,0 +1,4 @@
+# The entry point Bundler requires for `gem "importmap-plus"`. The constants
+# stay under Importmap:: so this gem drops into any app that used
+# importmap-rails without touching a single call site.
+require "importmap-rails"

data/lib/importmap-rails.rb

@@ -0,0 +1,6 @@
+module Importmap
+end
+
+require "importmap/version"
+require "importmap/reloader"
+require "importmap/engine" if defined?(Rails::Railtie)

data/lib/install/bin/importmap

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require_relative "../config/application"
+require "importmap/commands"

data/lib/install/config/importmap.rb

@@ -0,0 +1,3 @@
+# Pin npm packages by running ./bin/importmap
+
+pin "application"

data/lib/install/install.rb

@@ -0,0 +1,32 @@
+APPLICATION_LAYOUT_PATH = Rails.root.join("app/views/layouts/application.html.erb")
+
+if APPLICATION_LAYOUT_PATH.exist?
+  say "Add Importmap include tags in application layout"
+  insert_into_file APPLICATION_LAYOUT_PATH.to_s, "\n    <%= javascript_importmap_tags %>", before: /\s*<\/head>/
+else
+  say "Default application.html.erb is missing!", :red
+  say "        Add <%= javascript_importmap_tags %> within the <head> tag in your custom layout."
+end
+
+say "Create application.js module as entrypoint"
+create_file Rails.root.join("app/javascript/application.js") do <<-JS
+// Configure your import map in config/importmap.rb. Read more: https://github.com/zoolutions/importmap-plus
+JS
+end
+
+say "Use vendor/javascript for downloaded pins"
+empty_directory "vendor/javascript"
+keep_file "vendor/javascript"
+
+if (sprockets_manifest_path = Rails.root.join("app/assets/config/manifest.js")).exist?
+  say "Ensure JavaScript files are in the Sprocket manifest"
+  append_to_file sprockets_manifest_path,
+    %(//= link_tree ../../javascript .js\n//= link_tree ../../../vendor/javascript .js\n)
+end
+
+say "Configure importmap paths in config/importmap.rb"
+copy_file "#{__dir__}/config/importmap.rb", "config/importmap.rb"
+
+say "Copying binstub"
+copy_file "#{__dir__}/bin/importmap", "bin/importmap"
+chmod "bin", 0755 & ~File.umask, verbose: false

data/lib/tasks/importmap_tasks.rake

@@ -0,0 +1,9 @@
+namespace :importmap do
+  desc "Setup Importmap for the app"
+  task :install do
+    previous_location = ENV["LOCATION"]
+    ENV["LOCATION"] = File.expand_path("../install/install.rb", __dir__)
+    Rake::Task["app:template"].invoke
+    ENV["LOCATION"] = previous_location
+  end
+end

metadata

@@ -0,0 +1,110 @@
+--- !ruby/object:Gem::Specification
+name: importmap-plus
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- David Heinemeier Hansson
+- Mikael Henriksson
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: railties
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 6.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 6.0.0
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 6.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 6.0.0
+- !ruby/object:Gem::Dependency
+  name: actionpack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 6.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 6.0.0
+description: 'A drop-in replacement for importmap-rails. Same Importmap:: API, same
+  pins, same import map, plus `bin/importmap pin --minify`, `--from esm.run` for jsDelivr''s
+  bundled builds, and a pin comment that records the CDN and minification so later
+  updates keep them. Use this gem or importmap-rails, never both.'
+email: mikael@mhenrixon.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- MIT-LICENSE
+- README.md
+- Rakefile
+- app/controllers/importmap/freshness.rb
+- app/helpers/importmap/importmap_tags_helper.rb
+- lib/importmap-plus.rb
+- lib/importmap-rails.rb
+- lib/importmap/commands.rb
+- lib/importmap/engine.rb
+- lib/importmap/http_retries.rb
+- lib/importmap/map.rb
+- lib/importmap/minifier.rb
+- lib/importmap/npm.rb
+- lib/importmap/packager.rb
+- lib/importmap/reloader.rb
+- lib/importmap/version.rb
+- lib/install/bin/importmap
+- lib/install/config/importmap.rb
+- lib/install/install.rb
+- lib/tasks/importmap_tasks.rake
+homepage: https://github.com/zoolutions/importmap-plus
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/zoolutions/importmap-plus
+  source_code_uri: https://github.com/zoolutions/importmap-plus
+  changelog_uri: https://github.com/zoolutions/importmap-plus/blob/main/CHANGELOG.md
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.1.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.19
+specification_version: 4
+summary: importmap-rails with vendoring that minifies, bundles from esm.run, and remembers
+  where each package came from.
+test_files: []