@wardby/cli 0.6.0 → 0.8.0

package/dist/coding/registry/npm.js

@@ -2,12 +2,13 @@
  * npm registry adapter: parses npm allowlist syntax and semver ranges,
  * routes proxy requests against the npm HTTP protocol (packuments,
  * `-/tarball/<name>/<version>` downloads, and the standard
- * `<name>/-/<unscoped>-<version>.tgz` path lockfiles record), and configures npm/npmrc for
- * the sandboxed worker to use the proxy with install-time scripts disabled.
+ * `<name>/-/<unscoped>-<version>.tgz` path lockfiles record), and configures npm, pnpm,
+ * yarn (classic and Berry) and corepack in the sandboxed worker to use the proxy with
+ * install-time scripts disabled.
  */
 import semver from "semver";
-import { npmLockfiles } from "./npm-lockfile.js";
-import { npmLockfilePlan, registryDependencyEdges } from "./npm-plan.js";
+import { npmClientLockfilePlan, npmClientLockfiles } from "./npm-lockfile-formats.js";
+import { registryDependencyEdges } from "./npm-plan.js";
 import { AllowlistEntryError, RegistryError, } from "./types.js";
 /** Scoped names are lower case (npm never allowed capitals in them); an
  *  unscoped legacy name may contain capitals, and npm treats it as a
@@ -81,6 +82,19 @@ function standardTarball(name, filename) {
         return null;
     return { kind: "download", name, version, filename: `${version}.tgz` };
 }
+/** The download URL the proxy hands out: the registry's usual tarball path,
+ *  `<name>/-/<unscoped>-<version>.tgz`, under the proxy's base (routed by
+ *  `standardTarball`). pnpm and yarn 2+ record a download URL in their
+ *  lockfile (and yarn in .pnp.cjs and its cache names) only when it isn't
+ *  that usual one, so they record nothing of the proxy. */
+function proxyTarballUrl(proxyBase, name, version) {
+    // The usual path routes only a canonically spelled version (standardTarball);
+    // any other kept version (build metadata, `1.0.0+build`) keeps the proxy's
+    // own route, which the collection rewrite turns public all the same.
+    if (semver.valid(version) !== version)
+        return `${proxyBase}-/tarball/${encodeURIComponent(name)}/${encodeURIComponent(version)}`;
+    return `${proxyBase}${name}/-/${name.slice(name.indexOf("/") + 1)}-${version}.tgz`;
+}
 function validName(name) {
     if (!NAME.test(name))
         throw new AllowlistEntryError(`"${name}" is not a valid npm package name`);
@@ -138,8 +152,8 @@ export const npmAdapter = {
     upstreamHosts: ["registry.npmjs.org"],
     collectExclude: ["node_modules"],
     dependenciesInMetadata: true,
-    lockfiles: npmLockfiles,
-    lockfilePlan: npmLockfilePlan,
+    lockfiles: npmClientLockfiles,
+    lockfilePlan: npmClientLockfilePlan,
     parseAllowlistEntry(raw) {
         const value = raw.trim();
         // Scopes are always lower case on npm, so folding one cannot select a
@@ -173,6 +187,9 @@ export const npmAdapter = {
         // one segment (`react`, `@scope%2fpkg`) or a literal scope plus name
         // (`@scope/pkg`); the filename is one raw segment.
         const standard = subpath.match(/^((?:@[^/]+\/)?[^/]+)\/-\/([^/]+)$/);
+        // One version's document, `<name>/<version>` (corepack reads these to
+        // download a pinned pnpm or yarn), for an exact version only.
+        const versionDocument = subpath.match(/^((?:@[^/]+\/)?[^/]+)\/([^/]+)$/);
         try {
             if (tarball) {
                 const name = decodeURIComponent(tarball[1]);
@@ -183,6 +200,14 @@ export const npmAdapter = {
             }
             if (standard)
                 return standardTarball(decodeURIComponent(standard[1]), decodeURIComponent(standard[2]));
+            if (versionDocument) {
+                // `@scope/pkg` matches too (as name `@scope`, version `pkg`), and
+                // falls through: no package name is a scope alone.
+                const name = decodeURIComponent(versionDocument[1]);
+                const version = decodeURIComponent(versionDocument[2]);
+                if (NAME.test(name) && semver.valid(version) === version)
+                    return { kind: "metadata", name, version };
+            }
             const name = decodeURIComponent(subpath);
             return NAME.test(name) ? { kind: "metadata", name } : null;
         }
@@ -214,7 +239,7 @@ export const npmAdapter = {
             const info = JSON.parse(manifest);
             versions[version] = {
                 ...info,
-                dist: { ...info.dist, tarball: `${proxyBase}-/tarball/${encodeURIComponent(meta.name)}/${version}` },
+                dist: { ...info.dist, tarball: proxyTarballUrl(proxyBase, meta.name, version) },
             };
             const published = raw.times.get(version);
             if (published !== undefined)
@@ -229,13 +254,37 @@ export const npmAdapter = {
             body: JSON.stringify({ name: raw.name, "dist-tags": tags, time, versions }),
         };
     },
+    renderVersion(meta, version, keptFiles, proxyBase) {
+        const raw = meta.raw;
+        if (!raw)
+            throw new Error(`npm metadata for "${meta.name}" was fetched without render data`);
+        const manifest = raw.manifests.get(version);
+        if (manifest === undefined || !keptFiles.has(`${version}.tgz`))
+            return null;
+        const info = JSON.parse(manifest);
+        return {
+            contentType: "application/json",
+            body: JSON.stringify({
+                ...info,
+                dist: { ...info.dist, tarball: proxyTarballUrl(proxyBase, meta.name, version) },
+            }),
+        };
+    },
     resolveDownload(route, meta) {
         return meta.versions.get(route.version)?.files.find((file) => file.filename === route.filename) ?? null;
     },
     workerConfig({ registryUrl, token, cacheDir }) {
         const npmrc = `${cacheDir}/npmrc`;
+        // yarn Berry and corepack append paths to a registry without a trailing slash.
+        const registryServer = registryUrl.replace(/\/+$/, "");
+        const proxy = new URL(registryUrl);
         return {
             env: {
+                // npm, and pnpm (which reads the same npm_config_* settings and
+                // npmrc). Only settings npm knows: npm warns about any other on
+                // every command, so pnpm's store and cache and yarn classic's cache
+                // folder are set by the shim for that tool alone (npm-shim.mjs),
+                // under npm_config_cache.
                 npm_config_registry: registryUrl,
                 npm_config_userconfig: npmrc,
                 npm_config_ignore_scripts: "true",
@@ -243,7 +292,28 @@ export const npmAdapter = {
                 npm_config_audit: "false",
                 npm_config_fund: "false",
                 npm_config_update_notifier: "false",
-                // The only registry the npm shim (npm-shim.mjs) sends a lockfile to.
+                // yarn classic (1.x): the registry and token from the npm settings
+                // above, and its own switch for install scripts. Nothing here is a
+                // YARN_ setting yarn Berry would reject as unknown.
+                YARN_IGNORE_SCRIPTS: "true",
+                // yarn Berry (2+) ignores npm's settings. Its global folder holds its
+                // shared cache; the per-project cacheFolder is left alone, so a
+                // repository that commits its cache (zero-install) keeps it.
+                YARN_NPM_REGISTRY_SERVER: registryServer,
+                YARN_NPM_AUTH_TOKEN: token,
+                YARN_NPM_ALWAYS_AUTH: "true",
+                ...(proxy.protocol === "http:" ? { YARN_UNSAFE_HTTP_WHITELIST: proxy.hostname } : {}),
+                YARN_ENABLE_TELEMETRY: "0",
+                YARN_ENABLE_SCRIPTS: "false",
+                YARN_GLOBAL_FOLDER: `${cacheDir}/yarn-berry`,
+                // corepack, which downloads the pnpm or yarn release a project pins.
+                COREPACK_NPM_REGISTRY: registryServer,
+                COREPACK_NPM_TOKEN: token,
+                COREPACK_HOME: `${cacheDir}/corepack`,
+                COREPACK_ENABLE_DOWNLOAD_PROMPT: "0",
+                COREPACK_ENABLE_AUTO_PIN: "0",
+                COREPACK_DEFAULT_TO_LATEST: "0",
+                // The only registry the package-manager shim (npm-shim.mjs) sends a lockfile to.
                 WARDBY_NPM_PLAN_REGISTRY: registryUrl,
             },
             files: [{ path: npmrc, content: `${registryUrl.replace(/^https?:/, "")}:_authToken=${token}\n`, mode: 0o600 }],

package/dist/coding/registry/pnpm-lockfile.d.ts

@@ -0,0 +1,17 @@
+import { type ParsedLockfile } from "./types.js";
+export declare function parsePnpmLockfile(body: string, options: {
+    maxEntries: number;
+    proxyRegistryUrl: string;
+}): ParsedLockfile;
+/** Whether `body` is a pnpm lockfile: a top-level `lockfileVersion:` key
+ *  (a Berry yarn.lock has `__metadata:` instead, and npm's is JSON). */
+export declare function isPnpmLockfile(body: string): boolean;
+/**
+ * Rewrites what pnpm records of the proxy in a `pnpm-lock.yaml`. pnpm
+ * records a package's `tarball` only when the registry's download URL is
+ * not the conventional one, which the proxy's own route is not: a proxy
+ * `tarball` next to an `integrity` is dropped (exactly what pnpm writes for
+ * a package from the public registry), and one without an integrity is
+ * rewritten to the public registry URL. Only matched values change.
+ */
+export declare function normalizePnpmLockfile(content: string, registryUrl: string): string;

package/dist/coding/registry/pnpm-lockfile.js

@@ -0,0 +1,189 @@
+/**
+ * `pnpm-lock.yaml` for npm lockfile plans (`POST /registry/npm/-/plan`) and
+ * for the rewrite of proxy URLs before collection.
+ *
+ * Supported: lockfileVersion 9 (pnpm 9+: `packages` holds each
+ * name@version's resolution and integrity, `snapshots` each installed
+ * instance with its resolved dependencies) and 6 (pnpm 8: `packages` keyed
+ * `/name@version`, holding both). An entry is one installed instance (a
+ * snapshot, or a v6 package key), so the same name@version with different
+ * peers is several entries; plans approve per name@version anyway.
+ *
+ * pnpm resolves a dependency by the reference the parent's entry records
+ * (`ms: 2.1.3`, an alias's `string-width@4.2.3`, both possibly with a
+ * `(peer@x)` suffix): `resolve` follows exactly that. `link:` and `file:`
+ * references and `directory` resolutions are local and skipped; git and
+ * non-registry tarballs are refused. Nothing here decides anything: plan.ts
+ * verifies every claim against the registry.
+ *
+ * Kept free of imports from npm.ts, which imports this module.
+ */
+import { parseProxyPath, publicNpmTarballUrl } from "./npm-lockfile.js";
+import { registryDependencyEdges } from "./npm-plan.js";
+import { buildResolve, entryOf, fromRegistry, mapOf, parseLockfileYaml, splitDescriptor, stringOf, tooLarge, yamlEntryLines, } from "./lockfile-shared.js";
+import { RegistryError } from "./types.js";
+/** Two-space-indented keys a pnpm lockfile may have per entry (a package
+ *  and its snapshot, plus slack for importers), for the cheap pre-check. */
+const KEY_LINES_PER_ENTRY = 4;
+/** `name@1.0.0(peer@2.0.0)(other@3)` without its peer suffix. */
+function withoutPeers(reference) {
+    const open = reference.indexOf("(");
+    return open < 0 ? reference : reference.slice(0, open);
+}
+/** The string-valued dependencies of a snapshot or package entry. */
+function referencesOf(entry) {
+    const references = new Map();
+    for (const field of ["dependencies", "optionalDependencies"]) {
+        for (const [key, value] of Object.entries(mapOf(entry[field]))) {
+            const reference = stringOf(value);
+            if (reference !== undefined && !references.has(key))
+                references.set(key, reference);
+        }
+    }
+    return references;
+}
+const IMPORTER_FIELDS = ["dependencies", "devDependencies", "optionalDependencies"];
+/** An importer's declared dependencies (`{ specifier, version }` per name). */
+function importerDeclarations(importer) {
+    const specs = {};
+    for (const field of IMPORTER_FIELDS) {
+        for (const [key, value] of Object.entries(mapOf(importer[field]))) {
+            const specifier = stringOf(mapOf(value).specifier) ?? stringOf(value);
+            if (specifier !== undefined && !(key in specs))
+                specs[key] = specifier;
+        }
+    }
+    return registryDependencyEdges(specs);
+}
+/** The reference (`version`) an importer records for each dependency it
+ *  declares, which `resolve` follows like a snapshot's. */
+function importerReferences(importer) {
+    const references = new Map();
+    for (const field of IMPORTER_FIELDS) {
+        for (const [key, value] of Object.entries(mapOf(importer[field]))) {
+            const reference = stringOf(mapOf(value).version);
+            if (reference !== undefined && !references.has(key))
+                references.set(key, reference);
+        }
+    }
+    return references;
+}
+/** The entry for one installed instance: local (null), or registry or
+ *  refused by its resolution. */
+function classify(path, name, version, resolution, proxyRegistryUrl) {
+    if (version.startsWith("link:") || version.startsWith("file:"))
+        return null;
+    const fields = mapOf(resolution);
+    const type = stringOf(fields.type);
+    if (type === "directory")
+        return null;
+    const tarball = stringOf(fields.tarball);
+    if (tarball?.startsWith("file:"))
+        return null;
+    const integrity = stringOf(fields.integrity) ?? null;
+    const elsewhere = type !== undefined || (tarball !== undefined && !fromRegistry(tarball, proxyRegistryUrl));
+    return entryOf(path, name, version, integrity, elsewhere ? "elsewhere" : "registry");
+}
+export function parsePnpmLockfile(body, options) {
+    const lines = yamlEntryLines(body);
+    if (lines > (options.maxEntries + 1) * KEY_LINES_PER_ENTRY)
+        throw tooLarge(lines, options.maxEntries);
+    const document = parseLockfileYaml(body);
+    const lockfileVersion = stringOf(document.lockfileVersion) ?? "";
+    const major = Number.parseInt(lockfileVersion, 10);
+    if (major !== 6 && major !== 9)
+        throw new RegistryError(400, "wardby_lockfile_unsupported", `pnpm lockfileVersion ${JSON.stringify(lockfileVersion)} is not supported; regenerate the lockfile with pnpm 8 or later (lockfileVersion 6 or 9)`);
+    const packages = mapOf(document.packages);
+    const snapshots = mapOf(document.snapshots);
+    // v9 lists every installed instance under `snapshots`; v6 has only `packages`.
+    const instances = major === 9 && Object.keys(snapshots).length > 0 ? snapshots : packages;
+    const count = Object.keys(instances).length;
+    if (count > options.maxEntries)
+        throw tooLarge(count, options.maxEntries);
+    const entries = [];
+    const byPath = new Map();
+    const byKey = new Map();
+    for (const [path, raw] of Object.entries(instances)) {
+        if (!raw || typeof raw !== "object" || Array.isArray(raw))
+            throw new RegistryError(400, "wardby_bad_request", `pnpm-lock.yaml entry "${path}" is not a mapping`);
+        const instance = raw;
+        let target;
+        if (major === 9) {
+            const base = withoutPeers(path);
+            const { name, range: version } = splitDescriptor(base);
+            target = classify(path, name, version, mapOf(packages[base]).resolution, options.proxyRegistryUrl);
+        }
+        else if (path.startsWith("/")) {
+            const { name, range: version } = splitDescriptor(withoutPeers(path.slice(1)));
+            target = classify(path, name, version, instance.resolution, options.proxyRegistryUrl);
+        }
+        else {
+            // A v6 key without a leading slash is a package from outside the
+            // registry (a git host, a URL, a local path); pnpm records its
+            // name and version as fields.
+            target = classify(path, stringOf(instance.name) ?? path, stringOf(instance.version) ?? "", { type: "elsewhere", ...mapOf(instance.resolution) }, options.proxyRegistryUrl);
+        }
+        if (target)
+            entries.push(target);
+        byPath.set(path, { dependencies: referencesOf(instance) });
+        byKey.set(path, target);
+    }
+    const importers = mapOf(document.importers);
+    const importerOf = Object.keys(importers).length > 0
+        ? Object.entries(importers).map(([path, importer]) => [path, mapOf(importer)])
+        : [[".", document]];
+    const projects = importerOf.map(([path, importer]) => ({ path, dependencies: importerDeclarations(importer) }));
+    // A project resolves what it declares like an instance does (an importer
+    // path that is also an instance key keeps the instance's references).
+    for (const [path, importer] of importerOf)
+        if (!byPath.has(path))
+            byPath.set(path, { dependencies: importerReferences(importer) });
+    const candidates = (key, reference) => {
+        if (reference.startsWith("link:"))
+            return [];
+        if (major === 9) {
+            // An alias records `name@version`; anything else is this key's version.
+            return withoutPeers(reference).indexOf("@", 1) > 0 ? [reference] : [`${key}@${reference}`];
+        }
+        if (reference.startsWith("/"))
+            return [reference];
+        return /^\d/.test(reference) ? [`/${key}@${reference}`] : [reference];
+    };
+    return { entries, projects, ...buildResolve(byPath, byKey, candidates) };
+}
+/** Whether `body` is a pnpm lockfile: a top-level `lockfileVersion:` key
+ *  (a Berry yarn.lock has `__metadata:` instead, and npm's is JSON). */
+export function isPnpmLockfile(body) {
+    return /^lockfileVersion:/m.test(body);
+}
+/** A `resolution: {…}` flow mapping on one line of a pnpm lockfile. */
+const RESOLUTION = /^(\s+resolution:\s*\{)([^}\n]*)(\}\s*)$/gm;
+/**
+ * Rewrites what pnpm records of the proxy in a `pnpm-lock.yaml`. pnpm
+ * records a package's `tarball` only when the registry's download URL is
+ * not the conventional one, which the proxy's own route is not: a proxy
+ * `tarball` next to an `integrity` is dropped (exactly what pnpm writes for
+ * a package from the public registry), and one without an integrity is
+ * rewritten to the public registry URL. Only matched values change.
+ */
+export function normalizePnpmLockfile(content, registryUrl) {
+    const base = registryUrl.endsWith("/") ? registryUrl : `${registryUrl}/`;
+    return content.replace(RESOLUTION, (match, open, inner, close) => {
+        const parts = inner.split(",").map((part) => part.trim());
+        const tarballAt = parts.findIndex((part) => part.startsWith("tarball:"));
+        if (tarballAt < 0)
+            return match;
+        const url = parts[tarballAt]
+            .slice("tarball:".length)
+            .trim()
+            .replace(/^(['"])(.*)\1$/, "$2");
+        const target = url.startsWith(base) ? parseProxyPath(url.slice(base.length)) : null;
+        if (!target)
+            return match;
+        if (parts.some((part) => part.startsWith("integrity:")))
+            parts.splice(tarballAt, 1);
+        else
+            parts[tarballAt] = `tarball: ${publicNpmTarballUrl(target.name, target.version)}`;
+        return `${open}${parts.join(", ")}${close}`;
+    });
+}

package/dist/coding/registry/pypi-plan.d.ts

@@ -0,0 +1,50 @@
+import { type DeclaredDependency, type LockfilePlanSupport, type ParsedLockfile, type UpstreamFetch, type VersionFact } from "./types.js";
+/**
+ * A pinned requirements file: every `name==version` (or `===`) line is an
+ * entry; a direct reference (`name @ url`) is an unsupported one. Options
+ * (`-r`, `-c`, `--index-url`, …), editable or local projects and unpinned
+ * lines are not entries: pip resolves those the usual way. The file records
+ * no dependency graph, so a parent's dependency resolves to every entry of
+ * that name (marker-split pins).
+ */
+export declare function parseRequirements(body: string, maxEntries: number): ParsedLockfile;
+/**
+ * A uv.lock (`version = 1`). A package from PyPI (or the proxy) is an
+ * entry; the lock's own projects (editable, virtual and directory sources)
+ * are its projects; one from git, a URL, a path or another index is
+ * unsupported. A dependency resolves to the version the lock names for it
+ * when the lock pins the name at several versions.
+ */
+export declare function parseUvLock(body: string, maxEntries: number, proxyRegistryUrl: string): ParsedLockfile;
+/** Which file a plan body is: a uv.lock, a requirements file, or one this registry cannot verify. */
+export declare function parsePypiLockfile(body: string, options: {
+    maxEntries: number;
+    proxyRegistryUrl: string;
+}): ParsedLockfile;
+/** One release's facts: its dependency lines (with their extras and extra
+ *  gates), and its earliest wheel's upload time, URL and sha256. A release
+ *  with no wheel is refused (`wardby_file_not_allowed`): nothing in it may
+ *  be installed. */
+export declare function fetchPypiVersionFact(name: string, version: string, upstream: UpstreamFetch, signal?: AbortSignal): Promise<VersionFact | null>;
+/** Publish times for releases whose fact carried none (no wheel recorded an upload time). */
+export declare function fetchPypiPublishTimes(name: string, versions: readonly string[], upstream: UpstreamFetch, options: {
+    signal: AbortSignal;
+}): Promise<Map<string, Date>>;
+/** Whether a locked version is inside a dependency line's specifier, a
+ *  pinned pre-release included. */
+export declare function pypiEdgeSatisfies(version: string, range: string): boolean;
+/** The dependencies a release installed with `extras` follows, under the
+ *  same extras policy as the graph walk (pypi-requirements.ts). */
+export declare function pypiDependenciesOf(fact: VersionFact, extras: readonly string[]): DeclaredDependency[];
+export declare const pypiLockfilePlan: LockfilePlanSupport;
+/** PyPI's own path for a file by project and filename, which redirects to the
+ *  file (a wheel's python tag, or `source` for an sdist). */
+export declare function publicFileUrl(name: string, filename: string): string | undefined;
+/**
+ * A uv.lock with the proxy's URLs (which uv records when it locks through
+ * the proxy) put back to PyPI's: its index as `https://pypi.org/simple`, each
+ * file as PyPI's redirecting path (publicFileUrl). Credentials are never in
+ * them (uv records none), but any userinfo before the proxy host is dropped
+ * with the URL anyway. Everything else is left exactly as it was.
+ */
+export declare function normalizeUvLock(content: string, registryUrl: string): string;