@mjasnikovs/pi-task 0.39.4 → 0.40.0

package/dist/workers/docs-ecosystems.js

@@ -0,0 +1,449 @@
+/**
+ * docs-ecosystems — one row per package registry the docs Worker tool can read.
+ *
+ * The docs pipeline used to be npm all the way down with nothing saying so: it
+ * resolved through `node_modules`, chunked TypeScript, and auto-installed any
+ * unknown name from npm. Common Rust and Haskell package names also exist on
+ * npm, so a question about `aeson` or `tokio` returned a confident answer about
+ * an unrelated JavaScript package — a wrong answer, not a miss.
+ *
+ * A row states the whole of what one registry needs: how to spot its manifest,
+ * how to find a package on disk, how to fetch one that is absent, and how to cut
+ * its source into retrievable chunks. Rows live in code and arrive as pull
+ * requests, so a new ecosystem is reviewable and testable rather than a user
+ * string that either works or silently does not.
+ */
+import { spawn as nodeSpawn } from 'node:child_process';
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import { runAutoInstall, findDeclaredRange, extractParentPackage, resolveTypeSourceForDocs, getDocsModulesDir } from './docs-core.js';
+import { resolvePackage, isDtsFile, isValidModuleName } from './docs-resolve.js';
+import { DECL_SPLIT_RE } from './docs-chunk.js';
+import { npmVersionLookup } from './npm-version.js';
+import { resolveCrate, cratesLatest, crateTarballUrl, crateOf, isValidCrateName, isRustFile, lockedVersion, rustSurface, cargoProjectName, childDirs, lockedDeps, CARGO_DECL_SPLIT_RE } from './eco-cargo.js';
+import { resolveHackage, hackageLatest, hackageVersion, hackageTarballUrl, hackageExtractDir, hackageProjectName, findCabalTarball, cachedVersions, resolvedVersions, isValidHackageName, isHaskellFile, haskellSurface, HACKAGE_DECL_SPLIT_RE, HACKAGE_SKIP_DIRS } from './eco-hackage.js';
+import { runChild } from '../shared/child-process.js';
+/**
+ * Is any of `names` present at `cwd` or above it?
+ *
+ * Detection has to reach as far as resolution does. Node resolves a package by
+ * walking `node_modules` UPWARD, and cargo and cabal read the nearest manifest
+ * above them, so a session started in `~/project/src` is in an npm project by
+ * every rule that matters — and a cwd-only check would refuse every lookup
+ * there.
+ */
+function existsAtOrAbove(cwd, names) {
+    let dir = cwd;
+    while (true) {
+        if (names.some(n => fs.existsSync(path.join(dir, n))))
+            return true;
+        const up = path.dirname(dir);
+        if (up === dir)
+            return false;
+        dir = up;
+    }
+}
+/**
+ * The same walk, checking each ancestor's immediate children too.
+ *
+ * npm does not need this — node resolves straight up — but cargo and cabal keep
+ * their manifest in a SIBLING of where a session usually starts. From a Tauri
+ * repo's `src/`, the crate is in `../src-tauri`, and without the sideways step
+ * the project reads as npm-only and `tokio` resolves to npm's web scraper: the
+ * original bug, from one directory over.
+ */
+function foundAtOrAbove(cwd, atDir) {
+    let dir = cwd;
+    while (true) {
+        if (atDir(dir) || childDirs(dir).some(atDir))
+            return true;
+        const up = path.dirname(dir);
+        if (up === dir)
+            return false;
+        dir = up;
+    }
+}
+/**
+ * Production defaults for {@link EcosystemIo}. The environment is read HERE and
+ * nowhere in a row, so a test injects a directory instead of setting a variable
+ * that outlives it.
+ */
+export function defaultEcosystemIo(overrides = {}) {
+    return {
+        spawn: nodeSpawn,
+        fetch,
+        modulesDir: getDocsModulesDir(),
+        cargoHome: process.env.CARGO_HOME?.trim() || path.join(os.homedir(), '.cargo'),
+        cabalPackageDirs: defaultCabalPackageDirs(),
+        ...overrides
+    };
+}
+/** What acquiring a package produced, whatever the registry. */
+/**
+ * Both cabal layouts, because a machine may have either: `CABAL_DIR` when set,
+ * then the classic `~/.cabal`, then the XDG path newer cabal versions use.
+ */
+function defaultCabalPackageDirs() {
+    const home = os.homedir();
+    const configured = process.env.CABAL_DIR?.trim();
+    const cacheHome = process.env.XDG_CACHE_HOME?.trim() || path.join(home, '.cache');
+    return [
+        ...(configured ? [path.join(configured, 'packages')] : []),
+        path.join(home, '.cabal', 'packages'),
+        path.join(cacheHome, 'cabal', 'packages')
+    ];
+}
+/**
+ * The npm row, with the pieces a caller may have replaced left as parameters.
+ *
+ * `docsRaw` already takes `resolvePackage` and `npmVersionLookup` as injection
+ * hooks, and those hooks must keep reaching the resolution they are injected
+ * for. A per-call row carries them; {@link ECOSYSTEMS} holds the plain one.
+ */
+export function npmProfile(hooks = {}) {
+    const resolve = hooks.resolvePackage ?? resolvePackage;
+    const versionLookup = hooks.npmVersionLookup ?? npmVersionLookup;
+    return {
+        id: 'npm',
+        why: "The original and only ecosystem this tool read. Resolution is Node's own "
+            + 'node_modules walk, the documented surface is the .d.ts files a package ships, '
+            + 'and a missing package is installed with --ignore-scripts because the name is '
+            + 'model-chosen.',
+        registryLabel: 'npm',
+        manifestLabel: 'package.json',
+        // `node_modules` without a package.json counts: a directory that has one
+        // is an npm project whether or not it declares itself.
+        detect: cwd => existsAtOrAbove(cwd, ['package.json', 'node_modules']),
+        isValidName: isValidModuleName,
+        parentPackage: extractParentPackage,
+        resolve: (name, cwd) => resolve(name, cwd),
+        declaredRange: findDeclaredRange,
+        acquire: (name, range, io) => runAutoInstall(io.spawn, name, {
+            signal: io.signal,
+            versionRange: range ?? undefined
+        }),
+        afterResolve: (pkg, requested, cwd, io) => resolveTypeSourceForDocs(pkg, requested, cwd, io.spawn, resolve, io.signal),
+        latest: (name, io) => versionLookup(name, io.signal === undefined ? {} : { signal: io.signal }),
+        isSurfaceFile: isDtsFile,
+        surface: content => content,
+        declSplitRe: DECL_SPLIT_RE,
+        commentPrefix: '//',
+        // A nested node_modules is another package's surface, never this one's.
+        skipDirs: ['node_modules'],
+        surfaceLabel: '.d.ts files or README',
+        packageSubject: 'an npm package',
+        projectGlobs: ['*.ts', '*.tsx'],
+        projectName: npmProjectName,
+        declaredDeps: npmDeclaredDeps
+    };
+}
+const NPM_DEP_BLOCKS = [
+    'dependencies',
+    'devDependencies',
+    'peerDependencies',
+    'optionalDependencies'
+];
+/**
+ * The package.json manifest, not the lockfile: a lockfile is rewritten by
+ * installs that change no resolved version, and pruning digests on that would
+ * cost reuse for no correctness gain.
+ */
+export function npmDeclaredDeps(cwd) {
+    let json;
+    try {
+        json = JSON.parse(fs.readFileSync(path.join(cwd, 'package.json'), 'utf8'));
+    }
+    catch {
+        return undefined;
+    }
+    const out = {};
+    for (const block of NPM_DEP_BLOCKS) {
+        const deps = json[block];
+        if (!deps || typeof deps !== 'object')
+            continue;
+        for (const [name, range] of Object.entries(deps)) {
+            if (typeof range === 'string' && !(name in out))
+                out[name] = range;
+        }
+    }
+    // No dependency block at all is a real, stable state (a dependency-free repo).
+    return out;
+}
+/** A project's own name from its package.json, or null when it declares none. */
+export function npmProjectName(cwd) {
+    try {
+        const pkg = JSON.parse(fs.readFileSync(path.join(cwd, 'package.json'), 'utf8'));
+        return pkg.name ?? null;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * A crate's `.crate` tarball is fetched to a FILE and handed to `tar`, not piped:
+ * `runChild` writes only strings to a child's stdin, so gzip bytes cannot go
+ * through it. Windows ships bsdtar as `tar`, which reads `-xzf` the same way.
+ */
+async function acquireCrate(name, version, io) {
+    const dir = path.join(io.modulesDir, 'cargo');
+    const crate = crateOf(name);
+    // A unique name per download. Research children run concurrently, and on a
+    // fixed path one child's post-extract delete lands between another's write
+    // and its `tar`, which then fails on a crate that was in fact fetched.
+    const archive = path.join(dir, `.${crate}-${version}.${process.pid}.${Date.now()}.crate`);
+    try {
+        fs.mkdirSync(dir, { recursive: true });
+        const response = await io.fetch(crateTarballUrl(crate, version), {
+            headers: { 'user-agent': 'pi-task (github.com/mjasnikovs/pi-task)' },
+            ...(io.signal ? { signal: io.signal } : {})
+        });
+        if (!response.ok) {
+            return {
+                success: false,
+                installDir: dir,
+                stderr: `crates.io returned ${response.status} for ${crate} ${version}`
+            };
+        }
+        fs.writeFileSync(archive, Buffer.from(await response.arrayBuffer()));
+    }
+    catch (err) {
+        return {
+            success: false,
+            installDir: dir,
+            stderr: err instanceof Error ? err.message : String(err)
+        };
+    }
+    return extractArchive(archive, dir, io);
+}
+/**
+ * Unpack an archive and then delete it. The extracted tree is what every later
+ * call reads, so keeping the tarball beside it only spends disk — and a `.tar.gz`
+ * sitting among the package directories trips anything that treats the extract
+ * directory as a list of packages.
+ */
+async function extractArchive(archive, dir, io) {
+    const result = await runChild(io.spawn, { command: 'tar', args: ['-xzf', archive, '-C', dir] }, dir, io.signal, { mode: 'text', discardStdout: true });
+    const success = result.exitCode === 0 && !result.aborted;
+    if (success)
+        fs.rmSync(archive, { force: true });
+    return { success, installDir: dir, stderr: result.stderr };
+}
+const cargoProfile = {
+    id: 'cargo',
+    why: 'Rust ships no declarations file, so the surface is cut out of .rs source. '
+        + 'Versions come from Cargo.lock, not from a range: cargo has already resolved '
+        + 'them, and the newest wins when a workspace holds two majors — the answer '
+        + 'header states which was read. Note the asymmetry this leaves: in a repo with '
+        + 'both manifests the docs tool answers from cargo while the final gate still '
+        + 'runs the npm test command. Widening the gate is a separate change.',
+    registryLabel: 'crates.io',
+    manifestLabel: 'Cargo.toml',
+    // One level down as well: a Tauri repo declares package.json at the root and
+    // keeps its crate in `src-tauri/`.
+    detect: cwd => foundAtOrAbove(cwd, d => fs.existsSync(path.join(d, 'Cargo.toml'))),
+    isValidName: isValidCrateName,
+    parentPackage: crateOf,
+    resolve: (name, cwd, io) => resolveCrate(name, cwd, { cargoHome: io.cargoHome, modulesDir: io.modulesDir }),
+    // Cargo has already resolved every version; the lock IS the pin.
+    declaredRange: (name, cwd) => lockedVersion(name, cwd),
+    acquire: async (name, range, io) => {
+        // Asked even when the range is known: the download host wants the name as
+        // PUBLISHED, and only the API knows whether that is `tokio-util` or
+        // `tokio_util`. A null answer falls back to the caller's spelling.
+        const info = await cratesLatest(name, io.fetch, io.signal);
+        const version = range ?? info?.latest;
+        if (!version) {
+            return {
+                success: false,
+                installDir: path.join(io.modulesDir, 'cargo'),
+                stderr: `No published version found for crate "${crateOf(name)}".`
+            };
+        }
+        return acquireCrate(info?.pkg ?? name, version, io);
+    },
+    latest: (name, io) => cratesLatest(name, io.fetch, io.signal),
+    isSurfaceFile: isRustFile,
+    surface: content => rustSurface(content),
+    declSplitRe: CARGO_DECL_SPLIT_RE,
+    commentPrefix: '//',
+    skipDirs: ['tests', 'benches', 'examples', 'target'],
+    surfaceLabel: '.rs source or README',
+    packageSubject: 'a Rust crate from crates.io',
+    projectGlobs: ['*.rs'],
+    projectName: cargoProjectName,
+    declaredDeps: lockedDeps
+};
+/**
+ * Unpack a Hackage tarball into the tool's own directory. Whether it came from
+ * cabal's cache or from Hackage, a `.tar.gz` is not readable in place, and
+ * `runChild` writes only strings to stdin — so the archive is always a file on
+ * disk and `tar` always reads it from there. Windows ships bsdtar as `tar`.
+ */
+async function extractTarball(archive, dir, io) {
+    // Cabal's own cached tarball must survive; only a copy this tool downloaded
+    // into its own directory is deleted after unpacking.
+    if (path.dirname(archive) !== dir) {
+        const result = await runChild(io.spawn, { command: 'tar', args: ['-xzf', archive, '-C', dir] }, dir, io.signal, { mode: 'text', discardStdout: true });
+        return {
+            success: result.exitCode === 0 && !result.aborted,
+            installDir: dir,
+            stderr: result.stderr
+        };
+    }
+    return extractArchive(archive, dir, io);
+}
+const hackageProfile = {
+    id: 'hackage',
+    why: 'A Hackage package is a tarball, not a checked-out tree, so it is unpacked '
+        + 'before it can be read — which is why acquire runs even for a package cabal '
+        + 'already holds. Versions come from what the build actually resolved '
+        + '(dist-newstyle/cache/plan.json), then a freeze file, then a stack lock. The '
+        + 'row refuses a dotted MODULE name outright: Data.Aeson is not a package, and '
+        + 'answering it from the wrong registry is the bug this whole table exists for.',
+    registryLabel: 'hackage',
+    manifestLabel: '*.cabal',
+    detect: cwd => foundAtOrAbove(cwd, hasCabalManifest),
+    isValidName: isValidHackageName,
+    parentPackage: name => name,
+    resolve: (name, cwd, io) => resolveHackage(name, cwd, { modulesDir: io.modulesDir }),
+    declaredRange: (name, cwd) => hackageVersion(name, cwd),
+    acquire: async (name, range, io) => {
+        const dir = hackageExtractDir(io.modulesDir);
+        const version = range
+            ?? cachedVersions(name, io.cabalPackageDirs).pop()
+            ?? (await hackageLatest(name, io.fetch, io.signal))?.latest;
+        if (!version) {
+            return {
+                success: false,
+                installDir: dir,
+                stderr: `No published version found for Hackage package "${name}".`
+            };
+        }
+        try {
+            fs.mkdirSync(dir, { recursive: true });
+            const cached = findCabalTarball(name, version, io.cabalPackageDirs);
+            if (cached)
+                return await extractTarball(cached, dir, io);
+            const response = await io.fetch(hackageTarballUrl(name, version), {
+                ...(io.signal ? { signal: io.signal } : {})
+            });
+            if (!response.ok) {
+                return {
+                    success: false,
+                    installDir: dir,
+                    stderr: `hackage returned ${response.status} for ${name} ${version}`
+                };
+            }
+            const archive = path.join(dir, `.${name}-${version}.${process.pid}.${Date.now()}.tar.gz`);
+            fs.writeFileSync(archive, Buffer.from(await response.arrayBuffer()));
+            return await extractTarball(archive, dir, io);
+        }
+        catch (err) {
+            return {
+                success: false,
+                installDir: dir,
+                stderr: err instanceof Error ? err.message : String(err)
+            };
+        }
+    },
+    latest: (name, io) => hackageLatest(name, io.fetch, io.signal),
+    isSurfaceFile: isHaskellFile,
+    surface: haskellSurface,
+    declSplitRe: HACKAGE_DECL_SPLIT_RE,
+    commentPrefix: '--',
+    skipDirs: HACKAGE_SKIP_DIRS,
+    surfaceLabel: '.hs source or README',
+    packageSubject: 'a Haskell package from Hackage',
+    projectGlobs: ['*.hs'],
+    projectName: hackageProjectName,
+    declaredDeps: resolvedVersions
+};
+/** A cabal, stack or hpack project declares itself with one of these. */
+function hasCabalManifest(cwd) {
+    if (['cabal.project', 'stack.yaml', 'package.yaml'].some(f => fs.existsSync(path.join(cwd, f)))) {
+        return true;
+    }
+    try {
+        // A FILE named `<pkg>.cabal`. `~/.cabal` is cabal's own config DIRECTORY
+        // and ends in the same six characters, so a bare suffix test makes every
+        // directory under a Haskell developer's home look like a cabal project.
+        return fs
+            .readdirSync(cwd, { withFileTypes: true })
+            .some(e => e.isFile() && e.name.length > '.cabal'.length && e.name.endsWith('.cabal'));
+    }
+    catch {
+        return false;
+    }
+}
+export const ECOSYSTEMS = {
+    npm: npmProfile(),
+    cargo: cargoProfile,
+    hackage: hackageProfile
+};
+/** Which ecosystems `cwd` looks like a project of, in roster order. */
+export function detectEcosystems(cwd, roster = Object.values(ECOSYSTEMS)) {
+    return roster.filter(p => p.detect(cwd)).map(p => p.id);
+}
+/**
+ * Which ecosystem a lookup belongs to. The MANIFEST decides, never the model:
+ * `text`, `base`, `aeson`, `tokio` and `clap` are all real npm packages as well
+ * as Haskell or Rust ones, so a name alone cannot say which registry was meant,
+ * and guessing npm returns a confident answer about the wrong package.
+ *
+ * A refusal is a result, not a failure: the caller reports it and installs
+ * nothing.
+ */
+export function chooseEcosystem(input) {
+    const roster = input.roster ?? Object.values(ECOSYSTEMS);
+    const detected = detectEcosystems(input.cwd, roster);
+    const rowOf = (id) => roster.find(p => p.id === id);
+    const manifests = roster.map(p => p.manifestLabel).join(', ');
+    if (input.requested) {
+        if (detected.includes(input.requested)) {
+            return { ok: true, profile: rowOf(input.requested), detected };
+        }
+        const wanted = roster.find(p => p.id === input.requested);
+        return {
+            ok: false,
+            reason: 'not_detected',
+            detected,
+            message: `No ${input.requested} project here: ${input.cwd} has no `
+                + `${wanted?.manifestLabel ?? input.requested} manifest.`
+        };
+    }
+    if (detected.length === 0) {
+        return {
+            ok: false,
+            reason: 'none',
+            detected,
+            message: `${input.cwd} holds no package manifest this tool reads (${manifests}), `
+                + 'so there is no registry to look the name up in and nothing was installed.'
+        };
+    }
+    if (detected.length === 1)
+        return { ok: true, profile: rowOf(detected[0]), detected };
+    // Several manifests. A row whose MANIFEST names the package wins: in a Tauri
+    // repo `semver` is pinned by Cargo.lock at 1.0.28 and merely sits in
+    // node_modules as a transitive copy nothing declared, and taking the npm one
+    // because npm leads the roster is the issue-#18 mistake with both packages
+    // installed instead of neither.
+    const declaring = input.declaresPackage ? detected.filter(id => input.declaresPackage(rowOf(id))) : [];
+    if (declaring.length === 1)
+        return { ok: true, profile: rowOf(declaring[0]), detected };
+    if (declaring.length === 0 && input.resolvesLocally) {
+        // Nobody declares it. A copy on disk is the only evidence left.
+        for (const id of detected) {
+            if (input.resolvesLocally(rowOf(id)))
+                return { ok: true, profile: rowOf(id), detected };
+        }
+    }
+    return {
+        ok: false,
+        reason: 'ambiguous',
+        detected,
+        message: `${input.cwd} holds manifests for ${detected.join(' and ')}, and this package `
+            + 'is installed in none of them, so which registry to read is not decidable. '
+            + `Pass ecosystem: "${detected[0]}" (or one of: ${detected.join(', ')}) to say which.`
+    };
+}

package/dist/workers/docs-index.d.ts

@@ -1,9 +1,10 @@
 import type { CacheHandle } from './docs-cache.js';
 import { type ResolvedPackage } from './docs-resolve.js';
+import { type EcosystemProfile } from './docs-ecosystems.js';
 export interface IndexResult {
     hitCache: boolean;
     filesIngested: number;
     chunksWritten: number;
     contentHash: string;
 }
-export declare function ensureIndexed(cache: CacheHandle, pkg: ResolvedPackage): IndexResult;
+export declare function ensureIndexed(cache: CacheHandle, pkg: ResolvedPackage, profile?: EcosystemProfile): IndexResult;

package/dist/workers/docs-index.js

@@ -1,15 +1,39 @@
 import { createHash } from 'node:crypto';
 import * as fs from 'node:fs';
 import * as path from 'node:path';
-import { isDtsFile } from './docs-resolve.js';
+import {} from './docs-resolve.js';
 import { chunkDeclarations, chunkReadme } from './docs-chunk.js';
+import { ECOSYSTEMS } from './docs-ecosystems.js';
 const ZERO_SEP = Buffer.from([0]);
-function computeContentHash(pkg) {
+/**
+ * The gate that decides whether a package needs re-indexing.
+ *
+ * The entry file goes in SURFACED, not raw. What is cached is the extractor's
+ * OUTPUT, so a build whose extractor changed has stale chunks even though every
+ * byte on disk is identical — a crate indexed before the braced-`use` fix keeps
+ * `pub use crate::runtime::;` forever, because name, version and file bytes all
+ * still match. Surfacing here costs one file and makes the hash answer the
+ * question actually being asked: would re-reading produce the same chunks?
+ *
+ * The CHUNKER counts too, for the same reason: the rows are chunks, not surface,
+ * so a fix to where a declaration is cut leaves stale rows behind on its own.
+ *
+ * It is not total. An extractor change that alters only files BELOW the entry
+ * goes unnoticed; deleting the cache is still the escape hatch for that.
+ */
+function computeContentHash(pkg, profile) {
     const hash = createHash('sha256');
     hash.update(Buffer.from(`${pkg.name}@${pkg.version}`, 'utf8'));
     hash.update(ZERO_SEP);
-    if (pkg.entryDts && fs.existsSync(pkg.entryDts)) {
-        hash.update(fs.readFileSync(pkg.entryDts));
+    hash.update(Buffer.from(`${profile.declSplitRe.source}\u0000${profile.commentPrefix}`, 'utf8'));
+    hash.update(ZERO_SEP);
+    if (pkg.entry && fs.existsSync(pkg.entry)) {
+        try {
+            hash.update(Buffer.from(profile.surface(fs.readFileSync(pkg.entry, 'utf8')), 'utf8'));
+        }
+        catch {
+            hash.update(fs.readFileSync(pkg.entry));
+        }
     }
     hash.update(ZERO_SEP);
     if (pkg.readme && fs.existsSync(pkg.readme)) {
@@ -17,7 +41,7 @@ function computeContentHash(pkg) {
     }
     return hash.digest('hex');
 }
-function walkDts(root) {
+function walkSurface(root, profile) {
     const out = [];
     const stack = [root];
     while (stack.length) {
@@ -30,7 +54,7 @@ function walkDts(root) {
             continue;
         }
         for (const entry of entries) {
-            if (entry.name === 'node_modules')
+            if (profile.skipDirs.includes(entry.name))
                 continue;
             const full = path.join(dir, entry.name);
             if (entry.isSymbolicLink()) {
@@ -47,37 +71,40 @@ function walkDts(root) {
                 const stat = fs.statSync(realPath);
                 if (stat.isDirectory())
                     stack.push(realPath);
-                else if (stat.isFile() && isDtsFile(realPath))
+                else if (stat.isFile() && profile.isSurfaceFile(realPath))
                     out.push(realPath);
                 continue;
             }
             if (entry.isDirectory())
                 stack.push(full);
-            else if (entry.isFile() && isDtsFile(entry.name))
+            else if (entry.isFile() && profile.isSurfaceFile(entry.name))
                 out.push(full);
         }
     }
     return out.sort();
 }
-function collectFiles(pkg) {
+function collectFiles(pkg, profile) {
     return {
-        dts: walkDts(pkg.root),
+        surface: walkSurface(pkg.root, profile),
         readme: pkg.readme
     };
 }
-function ingestBody(cache, pkg, contentHash) {
+function ingestBody(cache, pkg, profile, contentHash) {
+    const ecosystem = profile.id;
     const inside = cache.db
-        .prepare('SELECT content_hash FROM packages WHERE name = ? AND version = ?')
-        .get(pkg.name, pkg.version);
+        .prepare('SELECT content_hash FROM packages WHERE ecosystem = ? AND name = ? AND version = ?')
+        .get(ecosystem, pkg.name, pkg.version);
     if (inside && inside.content_hash === contentHash) {
         return { hitCache: true, filesIngested: 0, chunksWritten: 0 };
     }
-    cache.db.prepare('DELETE FROM chunks WHERE name = ? AND version = ?').run(pkg.name, pkg.version);
-    const files = collectFiles(pkg);
+    cache.db
+        .prepare('DELETE FROM chunks WHERE ecosystem = ? AND name = ? AND version = ?')
+        .run(ecosystem, pkg.name, pkg.version);
+    const files = collectFiles(pkg, profile);
     let chunksWritten = 0;
     let filesIngested = 0;
-    const insertChunk = cache.db.prepare('INSERT INTO chunks (name, version, file_path, kind, content) VALUES (?, ?, ?, ?, ?)');
-    for (const abs of files.dts) {
+    const insertChunk = cache.db.prepare('INSERT INTO chunks (ecosystem, name, version, file_path, kind, content) VALUES (?, ?, ?, ?, ?, ?)');
+    for (const abs of files.surface) {
         // Normalise the separator before storing, so the same package indexes to
         // the same rows whatever built the path. The value is a MODEL-FACING
         // label: chunkDeclarations turns it into the chunk's `// <path>` header,
@@ -90,12 +117,12 @@ function ingestBody(cache, pkg, contentHash) {
         catch {
             continue;
         }
-        const chunks = chunkDeclarations(raw, rel);
+        const chunks = chunkDeclarations(profile.surface(raw), rel, profile.declSplitRe, profile.commentPrefix);
         if (!chunks.length)
             continue;
         filesIngested++;
         for (const c of chunks) {
-            insertChunk.run(pkg.name, pkg.version, rel, 'dts', c);
+            insertChunk.run(ecosystem, pkg.name, pkg.version, rel, 'dts', c);
             chunksWritten++;
         }
     }
@@ -106,28 +133,29 @@ function ingestBody(cache, pkg, contentHash) {
         if (chunks.length) {
             filesIngested++;
             for (const c of chunks) {
-                insertChunk.run(pkg.name, pkg.version, rel, 'readme', c);
+                insertChunk.run(ecosystem, pkg.name, pkg.version, rel, 'readme', c);
                 chunksWritten++;
             }
         }
     }
     cache.db
-        .prepare('INSERT OR REPLACE INTO packages (name, version, content_hash, indexed_at) VALUES (?, ?, ?, ?)')
-        .run(pkg.name, pkg.version, contentHash, Date.now());
+        .prepare('INSERT OR REPLACE INTO packages (ecosystem, name, version, content_hash, indexed_at) VALUES (?, ?, ?, ?, ?)')
+        .run(ecosystem, pkg.name, pkg.version, contentHash, Date.now());
     return { hitCache: false, filesIngested, chunksWritten };
 }
-export function ensureIndexed(cache, pkg) {
-    const contentHash = computeContentHash(pkg);
+export function ensureIndexed(cache, pkg, profile = ECOSYSTEMS[pkg.ecosystem]) {
+    const ecosystem = profile.id;
+    const contentHash = computeContentHash(pkg, profile);
     const existing = cache.db
-        .prepare('SELECT content_hash FROM packages WHERE name = ? AND version = ?')
-        .get(pkg.name, pkg.version);
+        .prepare('SELECT content_hash FROM packages WHERE ecosystem = ? AND name = ? AND version = ?')
+        .get(ecosystem, pkg.name, pkg.version);
     if (existing && existing.content_hash === contentHash) {
         return { hitCache: true, filesIngested: 0, chunksWritten: 0, contentHash };
     }
     cache.db.exec('BEGIN IMMEDIATE');
     let result;
     try {
-        result = ingestBody(cache, pkg, contentHash);
+        result = ingestBody(cache, pkg, profile, contentHash);
         cache.db.exec('COMMIT');
     }
     catch (err) {

package/dist/workers/docs-project.d.ts

@@ -2,7 +2,15 @@ import type { CacheHandle } from './docs-cache.js';
 import { retrieveChunks as defaultRetrieveChunks } from './docs-retrieve.js';
 import type { RetrievedChunk } from './docs-retrieve.js';
 import type { DocsCorpus } from './docs-lookup.js';
+/**
+ * Scope value for project-source rows. It sits in the same column as a registry
+ * id because these rows share the tables, but it is NOT one: a project is keyed
+ * by a hash of its cwd, so no registry could name it.
+ */
+export declare const PROJECT_SCOPE = "project";
 export declare function getProjectName(cwd: string): string;
+/** The extensions a given project is indexed for — `.ts/.tsx`, `.rs`, `.hs`. */
+export declare function projectSourceLabel(cwd: string): string;
 export declare function cwdKey(cwd: string): string;
 /**
  * Which source files make up the project.
@@ -48,6 +56,8 @@ export type ProjectDocsRawResult = {
     version: string;
     hitCache: boolean;
     filesIngested: number;
+    /** The extensions this project was actually searched for, e.g. `.rs`. */
+    sourceLabel: string;
 } | {
     kind: 'error';
     projectName: string;