@mjasnikovs/pi-task 0.39.4 → 0.40.0

package/dist/workers/docs-core.js

@@ -4,7 +4,8 @@ import * as os from 'node:os';
 import * as path from 'node:path';
 import { openCache as defaultOpenCache } from './docs-cache.js';
 import { ensureIndexed as defaultEnsureIndexed } from './docs-index.js';
-import { resolvePackage as defaultResolvePackage, ResolveError, isDtsFile, resolveTypeSource, typesPackageName, splitRuntimeNamespace } from './docs-resolve.js';
+import { npmProfile, chooseEcosystem, defaultEcosystemIo, ECOSYSTEMS } from './docs-ecosystems.js';
+import { resolvePackage as defaultResolvePackage, ResolveError, resolveTypeSource, typesPackageName, splitRuntimeNamespace } from './docs-resolve.js';
 import { retrieveChunks as defaultRetrieveChunks, PACKAGE_RETRIEVE_LIMIT, RETRIEVE_CONTENT_BUDGET } from './docs-retrieve.js';
 import { npmVersionLookup as defaultNpmVersionLookup } from './npm-version.js';
 import { runChild } from '../shared/child-process.js';
@@ -126,16 +127,29 @@ export function findDeclaredRange(parentPkg, cwd) {
  * `bun`, and cannot declare `bun-types`, so a sentence about what package.json
  * does or does not say has to be a sentence about `bun`.
  */
-export function buildVersionBanner(pin, resolved, version, cwd) {
+export function buildVersionBanner(pin, resolved, version, cwd, profile = ECOSYSTEMS.npm) {
     if (!pin)
         return '';
     const asked = pin.asked ?? resolved;
     const grounded = resolved !== asked ? ` The types this answer reads come from ${resolved}.` : '';
+    const manifest = profile.manifestLabel;
+    const registry = profile.registryLabel;
     if (pin.source === 'declared-range') {
         return (`[VERSION] "${asked}" resolved to this project's declared range `
             + `${pin.range} (installed v${version}); the answer below is pinned to that `
             + `version.${grounded}\n\n`);
     }
+    // The @types redirect chain and the four dependency maps are npm ideas, so the
+    // "declared, but only as X" wordings below can only be reached for npm. Every
+    // other registry gets the plain not-declared sentence, which is the true one:
+    // its own `declaredRange` already returned null.
+    if (profile.id !== 'npm') {
+        return (`[VERSION — verify] "${asked}" is not pinned by this project's ${manifest}, so `
+            + `this answer is based on the latest ${registry} release (v${version}). Your `
+            + `project may target a different MAJOR — confirm the version you intend and `
+            + `treat any API that differs across majors as unverified until you check `
+            + `it.${grounded}\n\n`);
+    }
     // The install fell back to npm latest. A usable declaration can still exist
     // further along the chain (`@types/<name>`) — it did not pin THIS install, so
     // the banner reports it as provenance, not as a pin.
@@ -227,8 +241,11 @@ export async function runAutoInstall(spawn, packageName, opts = {}) {
  */
 export async function acquirePackage(input) {
     const { name, cwd, spawn, resolvePackage, signal } = input;
+    const profile = input.profile ?? ECOSYSTEMS.npm;
+    const io = input.io ?? defaultEcosystemIo({ spawn, signal });
+    const resolve = (from) => resolvePackage ? resolvePackage(name, from) : profile.resolve(name, from, io);
     try {
-        return { ok: true, pkg: resolvePackage(name, cwd), autoInstalled: false };
+        return { ok: true, pkg: resolve(cwd), autoInstalled: false };
     }
     catch (firstErr) {
         if (!(firstErr instanceof ResolveError) || firstErr.kind !== 'not_installed') {
@@ -238,22 +255,19 @@ export async function acquirePackage(input) {
         // dep is declared in the project's package.json, install that range so a
         // scaffolding answer is grounded in the project's major, not whatever npm
         // currently tags `latest`.
-        const asked = extractParentPackage(name);
-        const declaredRange = findDeclaredRange(asked, cwd);
+        const asked = profile.parentPackage(name);
+        const declaredRange = profile.declaredRange(asked, cwd);
         const pin = declaredRange ?
             { source: 'declared-range', range: declaredRange, asked }
             : { source: 'npm-latest', asked };
-        const install = await runAutoInstall(spawn, asked, {
-            signal,
-            versionRange: declaredRange ?? undefined
-        });
+        const install = await profile.acquire(asked, declaredRange, io);
         if (!install.success) {
             return { ok: false, stage: 'install', stderr: install.stderr, pin, asked };
         }
         try {
             return {
                 ok: true,
-                pkg: resolvePackage(name, install.installDir),
+                pkg: resolve(install.installDir),
                 autoInstalled: true,
                 pin
             };
@@ -276,7 +290,7 @@ async function tryResolveOrInstall(name, cwd, spawn, resolvePackage, signal, onA
 /** The docs pipeline's adapter over the shared redirect walk (docs-resolve.ts):
  *  hops resolve through the auto-installing lookup, so a declaration package that is
  *  declared but not yet on disk is fetched rather than abandoned. */
-async function resolveTypeSourceForDocs(pkg, requested, cwd, spawn, resolvePackage, signal) {
+export async function resolveTypeSourceForDocs(pkg, requested, cwd, spawn, resolvePackage, signal) {
     // A package acquired ONLY through a hop reports neither `autoInstalled` nor a
     // pin unless this runs, so pi-worker-docs would emit no version banner for it.
     // Report the last hop that actually installed.
@@ -295,17 +309,53 @@ export async function docsRaw(input) {
     const openCache = input.openCache ?? defaultOpenCache;
     const spawn = input.spawn ?? defaultSpawn;
     const npmVersionLookup = input.npmVersionLookup ?? defaultNpmVersionLookup;
+    const io = defaultEcosystemIo({ spawn, signal: input.signal, ...input.io });
     // A runtime builtin specifier (`bun:sql`, `node:fs`) is typed by the runtime's
     // own types package, not a literal package of that colon-name — so resolve the
     // runtime instead. This is what turns a `bun:sql` lookup into Bun's real SQL
     // surface (`declare module "bun"` → `const sql: SQL`) rather than an
     // `invalid_name` error, and it lets the docs tool disprove a phantom submodule.
     const requested = splitRuntimeNamespace(input.pkg)?.runtime ?? input.pkg;
+    // Which registry, decided by the MANIFEST before anything else runs. A refusal
+    // must reach the caller having spawned nothing and asked no registry, so this
+    // sits above the version lookup and the resolve ladder both.
+    const choice = chooseEcosystem({
+        cwd: input.cwd,
+        requested: input.ecosystem,
+        // Read-only, and deliberately so: an ambiguous name is exactly the one that
+        // would otherwise be fetched from the wrong registry.
+        resolvesLocally: candidate => {
+            try {
+                candidate.resolve(requested, input.cwd, io);
+                return true;
+            }
+            catch {
+                return false;
+            }
+        },
+        declaresPackage: candidate => candidate.declaredRange(candidate.parentPackage(requested), input.cwd) !== null
+    });
+    if (!choice.ok) {
+        return {
+            kind: 'error',
+            message: `${choice.message} Use pi-worker-search or pi-worker-fetch for `
+                + `"${input.pkg}" instead.`,
+            resolveError: choice.reason === 'ambiguous' ? 'ambiguous_ecosystem' : 'unsupported_ecosystem'
+        };
+    }
+    // A per-call row for npm, not the static one: `resolvePackage` and
+    // `npmVersionLookup` are injection hooks, and a row built from the
+    // module-level exports would route straight past whatever a caller injected.
+    // They are npm's hooks, so no other row takes them.
+    const profile = choice.profile.id === 'npm' ?
+        npmProfile({ resolvePackage, npmVersionLookup })
+        : choice.profile;
+    const registryLabel = profile.registryLabel;
     // Fire the npm registry lookup in parallel with resolve/index/retrieve.
     // It returns null on any failure, so it never blocks the local pipeline.
-    const npmVersionPromise = npmVersionLookup(extractParentPackage(requested), {
-        signal: input.signal
-    }).catch(() => null);
+    const npmVersionPromise = profile
+        .latest(profile.parentPackage(requested), io)
+        .catch(() => null);
     // Step 1: acquire the package — resolve, or install-at-the-declared-range and
     // resolve again. The ladder is `acquirePackage`; this maps its stages onto the
     // rich error results the docs tool reports.
@@ -313,8 +363,9 @@ export async function docsRaw(input) {
         name: requested,
         cwd: input.cwd,
         spawn,
-        resolvePackage,
-        signal: input.signal
+        signal: input.signal,
+        profile,
+        io
     });
     if (!got.ok) {
         if (got.stage === 'install') {
@@ -324,7 +375,8 @@ export async function docsRaw(input) {
                 resolveError: 'not_installed',
                 installError: got.stderr,
                 autoInstallPin: got.pin,
-                npmVersion: await npmVersionPromise
+                npmVersion: await npmVersionPromise,
+                registryLabel
             };
         }
         if (got.stage === 'reresolve') {
@@ -335,7 +387,8 @@ export async function docsRaw(input) {
                     resolveError: got.err.kind,
                     autoInstalled: true,
                     autoInstallPin: got.pin,
-                    npmVersion: await npmVersionPromise
+                    npmVersion: await npmVersionPromise,
+                    registryLabel
                 };
             }
             return {
@@ -343,7 +396,8 @@ export async function docsRaw(input) {
                 message: `Could not resolve "${input.pkg}" after install: ${got.err instanceof Error ? got.err.message : String(got.err)}`,
                 autoInstalled: true,
                 autoInstallPin: got.pin,
-                npmVersion: await npmVersionPromise
+                npmVersion: await npmVersionPromise,
+                registryLabel
             };
         }
         if (got.err instanceof ResolveError) {
@@ -351,13 +405,15 @@ export async function docsRaw(input) {
                 kind: 'error',
                 message: got.err.message,
                 resolveError: got.err.kind,
-                npmVersion: await npmVersionPromise
+                npmVersion: await npmVersionPromise,
+                registryLabel
             };
         }
         return {
             kind: 'error',
             message: `Could not resolve "${input.pkg}": ${got.err instanceof Error ? got.err.message : String(got.err)}`,
-            npmVersion: await npmVersionPromise
+            npmVersion: await npmVersionPromise,
+            registryLabel
         };
     }
     let pkg = got.pkg;
@@ -369,11 +425,13 @@ export async function docsRaw(input) {
     // @types/<name> + triple-slash `<reference types>` chain to the package that
     // actually holds the declarations (e.g. bun -> @types/bun -> bun-types).
     // Best-effort: any failure leaves the original resolution untouched.
-    const viaTypes = await resolveTypeSourceForDocs(pkg, requested, input.cwd, spawn, resolvePackage, input.signal);
-    pkg = viaTypes.pkg;
-    if (viaTypes.installed) {
-        autoInstalled = true;
-        autoInstallPin ??= viaTypes.pin;
+    const viaTypes = await profile.afterResolve?.(pkg, requested, input.cwd, io);
+    if (viaTypes) {
+        pkg = viaTypes.pkg;
+        if (viaTypes.installed) {
+            autoInstalled = true;
+            autoInstallPin ??= viaTypes.pin;
+        }
     }
     // Step 2: open cache
     let cache = null;
@@ -385,18 +443,19 @@ export async function docsRaw(input) {
         cacheError = err instanceof Error ? err.message : String(err);
     }
     const result = cache ?
-        docsRawCached(cache, pkg, input.query, ensureIndexed, retrieveChunks, autoInstalled)
-        : docsRawUncached(pkg, cacheError ?? 'unknown cache error', autoInstalled);
+        docsRawCached(cache, pkg, profile, input.query, ensureIndexed, retrieveChunks, autoInstalled)
+        : docsRawUncached(pkg, profile, cacheError ?? 'unknown cache error', autoInstalled);
     result.npmVersion = await npmVersionPromise;
+    result.registryLabel = registryLabel;
     if (autoInstallPin)
         result.autoInstallPin = autoInstallPin;
     return result;
 }
-function docsRawCached(cache, pkg, query, ensureIndexed, retrieveChunks, autoInstalled) {
+function docsRawCached(cache, pkg, profile, query, ensureIndexed, retrieveChunks, autoInstalled) {
     let indexResult;
     const t0 = Date.now();
     try {
-        indexResult = ensureIndexed(cache, pkg);
+        indexResult = ensureIndexed(cache, pkg, profile);
     }
     catch (err) {
         return {
@@ -407,8 +466,8 @@ function docsRawCached(cache, pkg, query, ensureIndexed, retrieveChunks, autoIns
     }
     const indexingMs = indexResult.hitCache ? undefined : Date.now() - t0;
     const chunkCount = cache.db
-        .prepare('SELECT count(*) AS c FROM chunks WHERE name = ? AND version = ?')
-        .get(pkg.name, pkg.version)?.c ?? 0;
+        .prepare('SELECT count(*) AS c FROM chunks WHERE ecosystem = ? AND name = ? AND version = ?')
+        .get(profile.id, pkg.name, pkg.version)?.c ?? 0;
     if (chunkCount === 0) {
         return {
             kind: 'no_chunks',
@@ -421,6 +480,7 @@ function docsRawCached(cache, pkg, query, ensureIndexed, retrieveChunks, autoIns
     let chunks;
     try {
         chunks = retrieveChunks(cache, {
+            ecosystem: profile.id,
             name: pkg.name,
             version: pkg.version,
             query,
@@ -453,10 +513,10 @@ function docsRawCached(cache, pkg, query, ensureIndexed, retrieveChunks, autoIns
         autoInstalled: autoInstalled ? true : undefined
     };
 }
-function docsRawUncached(pkg, cacheError, autoInstalled) {
+function docsRawUncached(pkg, profile, cacheError, autoInstalled) {
     const parts = [];
-    const dtsFiles = walkDtsAlpha(pkg.root);
-    const entryFirst = pkg.entryDts ? [pkg.entryDts, ...dtsFiles.filter(f => f !== pkg.entryDts)] : dtsFiles;
+    const surfaceFiles = walkSurfaceAlpha(pkg.root, profile);
+    const entryFirst = pkg.entry ? [pkg.entry, ...surfaceFiles.filter(f => f !== pkg.entry)] : surfaceFiles;
     for (const abs of entryFirst) {
         let raw;
         try {
@@ -466,7 +526,7 @@ function docsRawUncached(pkg, cacheError, autoInstalled) {
             continue;
         }
         const rel = path.relative(pkg.root, abs);
-        parts.push(`// ${rel}\n${raw}`);
+        parts.push(`${profile.commentPrefix} ${rel}\n${profile.surface(raw)}`);
     }
     if (pkg.readme) {
         const rel = path.relative(pkg.root, pkg.readme);
@@ -502,7 +562,7 @@ function docsRawUncached(pkg, cacheError, autoInstalled) {
         autoInstalled: autoInstalled ? true : undefined
     };
 }
-function walkDtsAlpha(root) {
+function walkSurfaceAlpha(root, profile) {
     const out = [];
     const stack = [root];
     while (stack.length) {
@@ -515,12 +575,12 @@ function walkDtsAlpha(root) {
             continue;
         }
         for (const entry of entries) {
-            if (entry.name === 'node_modules')
+            if (profile.skipDirs.includes(entry.name))
                 continue;
             const full = path.join(dir, entry.name);
             if (entry.isDirectory())
                 stack.push(full);
-            else if (entry.isFile() && isDtsFile(entry.name))
+            else if (entry.isFile() && profile.isSurfaceFile(entry.name))
                 out.push(full);
         }
     }
@@ -538,7 +598,8 @@ export async function docsFocused(input) {
         throw new Error(rawResult.message);
     }
     if (rawResult.kind === 'no_chunks') {
-        throw new Error(`Package ${rawResult.pkg.name}@${rawResult.pkg.version} has no .d.ts files or README.`);
+        throw new Error(`Package ${rawResult.pkg.name}@${rawResult.pkg.version} has no `
+            + `${ECOSYSTEMS[rawResult.pkg.ecosystem].surfaceLabel}.`);
     }
     const { pkg, chunks, hitCache, indexingMs } = rawResult;
     const r = await docsLookup({
@@ -581,7 +642,9 @@ export async function docsFocused(input) {
 export function buildPrompt(pkg, query, content) {
     return buildExtractionPrompt({
         kind: 'package',
-        subject: 'an npm package',
+        // The extracting child is reading Rust or Haskell whenever the row is not
+        // npm's, and this is the one sentence it is told about what it has.
+        subject: ECOSYSTEMS[pkg.ecosystem].packageSubject,
         tag: 'package',
         identity: `${pkg.name}@${pkg.version}`,
         query,

package/dist/workers/docs-ecosystems.d.ts

@@ -0,0 +1,173 @@
+/**
+ * docs-ecosystems — one row per package registry the docs Worker tool can read.
+ *
+ * The docs pipeline used to be npm all the way down with nothing saying so: it
+ * resolved through `node_modules`, chunked TypeScript, and auto-installed any
+ * unknown name from npm. Common Rust and Haskell package names also exist on
+ * npm, so a question about `aeson` or `tokio` returned a confident answer about
+ * an unrelated JavaScript package — a wrong answer, not a miss.
+ *
+ * A row states the whole of what one registry needs: how to spot its manifest,
+ * how to find a package on disk, how to fetch one that is absent, and how to cut
+ * its source into retrievable chunks. Rows live in code and arrive as pull
+ * requests, so a new ecosystem is reviewable and testable rather than a user
+ * string that either works or silently does not.
+ */
+import { type AutoInstallPin } from './docs-core.js';
+import { resolvePackage, type ResolvedPackage } from './docs-resolve.js';
+import { npmVersionLookup, type NpmVersionInfo } from './npm-version.js';
+import { type SpawnFn } from '../shared/child-process.js';
+export type EcosystemId = 'npm' | 'cargo' | 'hackage';
+/**
+ * Every filesystem, process and network reach a row is allowed. Rows read no
+ * environment and call no global directly, so a test injects a fake registry and
+ * a fake extractor instead of needing a real toolchain on the machine.
+ */
+export interface EcosystemIo {
+    spawn: SpawnFn;
+    fetch: typeof fetch;
+    /** Where a package the project does not have is put once fetched. */
+    modulesDir: string;
+    /** Root of the cargo checkout cache — `CARGO_HOME`, or its default. */
+    cargoHome: string;
+    /** Every directory cabal may have filed a downloaded tarball under. */
+    cabalPackageDirs: readonly string[];
+    signal?: AbortSignal | undefined;
+}
+/**
+ * Production defaults for {@link EcosystemIo}. The environment is read HERE and
+ * nowhere in a row, so a test injects a directory instead of setting a variable
+ * that outlives it.
+ */
+export declare function defaultEcosystemIo(overrides?: Partial<EcosystemIo>): EcosystemIo;
+export interface AcquireResult {
+    success: boolean;
+    installDir: string;
+    stderr: string;
+}
+export interface EcosystemProfile {
+    id: EcosystemId;
+    /** What this row covers, and what it deliberately does not. */
+    why: string;
+    /** The registry's name, as it appears in text the model reads. */
+    registryLabel: string;
+    /** The manifest file this row is detected by, named for a refusal message. */
+    manifestLabel: string;
+    /** True when `cwd` looks like a project of this ecosystem. */
+    detect: (cwd: string) => boolean;
+    isValidName: (name: string) => boolean;
+    /** The installable package a possibly-subpath specifier belongs to. */
+    parentPackage: (name: string) => string;
+    /** Find the package on disk. Throws `ResolveError` when it is absent. */
+    resolve: (name: string, cwd: string, io: EcosystemIo) => ResolvedPackage;
+    /** The version range the project pins this package to, if it pins one. */
+    declaredRange: (name: string, cwd: string) => string | null;
+    /** Fetch a package that is not on disk into `io.modulesDir`. */
+    acquire: (name: string, range: string | null, io: EcosystemIo) => Promise<AcquireResult>;
+    /**
+     * A second resolution hop, for ecosystems where the package that ships the
+     * documented surface is not the one that was asked for.
+     */
+    afterResolve?: (pkg: ResolvedPackage, requested: string, cwd: string, io: EcosystemIo) => Promise<{
+        pkg: ResolvedPackage;
+        installed: boolean;
+        pin?: AutoInstallPin;
+    }>;
+    /** The registry's own newest version, for grounding an answer in the present. */
+    latest: (name: string, io: EcosystemIo) => Promise<NpmVersionInfo | null>;
+    /** True for a file that carries the package's public API surface. */
+    isSurfaceFile: (name: string) => boolean;
+    /**
+     * Reduce a source file to its public surface. Identity where the ecosystem
+     * already ships a declarations-only file, as npm does with `.d.ts`.
+     */
+    surface: (content: string) => string;
+    /** Where a declaration begins, so a chunk never splits a signature. */
+    declSplitRe: RegExp;
+    /** Line-comment marker, used to label a chunk with the file it came from. */
+    commentPrefix: string;
+    /** Directories the surface walk never descends into: tests, build output. */
+    skipDirs: readonly string[];
+    /** What this ecosystem's packages ship, for a "there is nothing to read" answer. */
+    surfaceLabel: string;
+    /**
+     * How the extraction child is told what it is reading. A written phrase, not
+     * a label plus an article: "a npm package" is what deriving one gives you.
+     */
+    packageSubject: string;
+    /** Which of the PROJECT's own files this ecosystem contributes to its index. */
+    projectGlobs: readonly string[];
+    /** The project's own name from its manifest, or null when it declares none. */
+    projectName: (cwd: string) => string | null;
+    /**
+     * What the project's manifest declares, name to version. Undefined when the
+     * manifest is missing or unreadable — the caller then cannot prove any
+     * package's version, which is a different fact from "declares nothing".
+     */
+    declaredDeps: (cwd: string) => Record<string, string> | undefined;
+}
+/** Overrides a caller has already been given its own copies of. */
+export interface NpmProfileHooks {
+    resolvePackage?: typeof resolvePackage;
+    npmVersionLookup?: typeof npmVersionLookup;
+}
+/**
+ * The npm row, with the pieces a caller may have replaced left as parameters.
+ *
+ * `docsRaw` already takes `resolvePackage` and `npmVersionLookup` as injection
+ * hooks, and those hooks must keep reaching the resolution they are injected
+ * for. A per-call row carries them; {@link ECOSYSTEMS} holds the plain one.
+ */
+export declare function npmProfile(hooks?: NpmProfileHooks): EcosystemProfile;
+/**
+ * The package.json manifest, not the lockfile: a lockfile is rewritten by
+ * installs that change no resolved version, and pruning digests on that would
+ * cost reuse for no correctness gain.
+ */
+export declare function npmDeclaredDeps(cwd: string): Record<string, string> | undefined;
+/** A project's own name from its package.json, or null when it declares none. */
+export declare function npmProjectName(cwd: string): string | null;
+export declare const ECOSYSTEMS: {
+    readonly npm: EcosystemProfile;
+    readonly cargo: EcosystemProfile;
+    readonly hackage: EcosystemProfile;
+};
+/** Which ecosystems `cwd` looks like a project of, in roster order. */
+export declare function detectEcosystems(cwd: string, roster?: readonly EcosystemProfile[]): EcosystemId[];
+export type EcosystemChoice = {
+    ok: true;
+    profile: EcosystemProfile;
+    detected: EcosystemId[];
+} | {
+    ok: false;
+    reason: 'none' | 'ambiguous' | 'not_detected';
+    detected: EcosystemId[];
+    message: string;
+};
+export interface ChooseEcosystemInput {
+    cwd: string;
+    /** An explicit ecosystem, for a repo that holds more than one manifest. */
+    requested?: EcosystemId;
+    /**
+     * Does this row already have the package on disk? The tie-break for a
+     * polyglot repo, and it must not install: an ambiguous name is exactly the
+     * one that would be installed from the wrong registry.
+     */
+    resolvesLocally?: (profile: EcosystemProfile) => boolean;
+    /**
+     * Does this row's MANIFEST name the package? Stronger evidence than a copy
+     * on disk, and it outranks it below.
+     */
+    declaresPackage?: (profile: EcosystemProfile) => boolean;
+    roster?: readonly EcosystemProfile[];
+}
+/**
+ * Which ecosystem a lookup belongs to. The MANIFEST decides, never the model:
+ * `text`, `base`, `aeson`, `tokio` and `clap` are all real npm packages as well
+ * as Haskell or Rust ones, so a name alone cannot say which registry was meant,
+ * and guessing npm returns a confident answer about the wrong package.
+ *
+ * A refusal is a result, not a failure: the caller reports it and installs
+ * nothing.
+ */
+export declare function chooseEcosystem(input: ChooseEcosystemInput): EcosystemChoice;