@mjasnikovs/pi-task 0.39.4 → 0.40.0

package/dist/workers/docs-project.js

@@ -5,17 +5,49 @@ import * as path from 'node:path';
 import { retrieveChunks as defaultRetrieveChunks, PROJECT_RETRIEVE_LIMIT, RETRIEVE_CONTENT_BUDGET } from './docs-retrieve.js';
 import { buildExtractionPrompt } from './abstention.js';
 import { chunkDeclarations } from './docs-chunk.js';
+import { ECOSYSTEMS, detectEcosystems } from './docs-ecosystems.js';
 const DEFAULT_LIMIT = PROJECT_RETRIEVE_LIMIT;
 const DEFAULT_BUDGET = RETRIEVE_CONTENT_BUDGET;
+/**
+ * Scope value for project-source rows. It sits in the same column as a registry
+ * id because these rows share the tables, but it is NOT one: a project is keyed
+ * by a hash of its cwd, so no registry could name it.
+ */
+export const PROJECT_SCOPE = 'project';
+/**
+ * The rows whose manifest is present, or npm alone when none is. The npm
+ * fallback is what keeps a bare directory indexing its `.ts` exactly as before.
+ */
+function projectProfiles(cwd) {
+    const detected = detectEcosystems(cwd);
+    return detected.length ? detected.map(id => ECOSYSTEMS[id]) : [ECOSYSTEMS.npm];
+}
 export function getProjectName(cwd) {
-    try {
-        const pkg = JSON.parse(fs.readFileSync(path.join(cwd, 'package.json'), 'utf8'));
-        if (pkg.name)
-            return pkg.name;
+    for (const profile of projectProfiles(cwd)) {
+        const name = profile.projectName(cwd);
+        if (name)
+            return name;
     }
-    catch { }
     return path.basename(cwd);
 }
+/** The file extension a glob like `*.rs` selects. */
+function extensionOf(glob) {
+    return glob.replace(/^\*/, '');
+}
+/** The extensions a given project is indexed for — `.ts/.tsx`, `.rs`, `.hs`. */
+export function projectSourceLabel(cwd) {
+    return [...new Set(projectProfiles(cwd).flatMap(p => p.projectGlobs))]
+        .map(extensionOf)
+        .join('/');
+}
+/** Which row chunks a given project file, by its extension. */
+function profileForFile(file, profiles) {
+    for (const profile of profiles) {
+        if (profile.projectGlobs.some(g => file.endsWith(extensionOf(g))))
+            return profile;
+    }
+    return profiles[0];
+}
 export function cwdKey(cwd) {
     return createHash('sha256').update(cwd).digest('hex').slice(0, 8);
 }
@@ -38,8 +70,9 @@ export function cwdKey(cwd) {
  * and a temp dir that is not itself inside a repo.
  */
 export function getProjectFiles(cwd) {
+    const globs = [...new Set(projectProfiles(cwd).flatMap(p => p.projectGlobs))];
     try {
-        const result = spawnSync('git', ['ls-files', '--cached', '--others', '--exclude-standard', '*.ts', '*.tsx'], { cwd, encoding: 'utf8', timeout: 5000 });
+        const result = spawnSync('git', ['ls-files', '--cached', '--others', '--exclude-standard', ...globs], { cwd, encoding: 'utf8', timeout: 5000 });
         if (result.status === 0 && result.stdout?.trim()) {
             return result.stdout
                 .trim()
@@ -49,10 +82,25 @@ export function getProjectFiles(cwd) {
         }
     }
     catch { }
-    return walkTsFiles(cwd);
+    return walkSourceFiles(cwd, globs.map(extensionOf));
 }
-function walkTsFiles(root) {
-    const SKIP = new Set(['node_modules', '.git', 'dist', 'build', 'coverage']);
+/**
+ * Build output, per ecosystem. NOT `profile.skipDirs` — that list says which
+ * directories of a DOWNLOADED package are not its published API, and `tests/`
+ * and `examples/` of the project's OWN repo are exactly what a question about
+ * the project may be asking. `git ls-files` filters none of them either.
+ */
+const BUILD_OUTPUT_DIRS = [
+    'node_modules',
+    '.git',
+    'dist',
+    'build',
+    'coverage',
+    'target',
+    'dist-newstyle'
+];
+function walkSourceFiles(root, extensions) {
+    const SKIP = new Set(BUILD_OUTPUT_DIRS);
     const out = [];
     const stack = [root];
     while (stack.length) {
@@ -70,8 +118,7 @@ function walkTsFiles(root) {
             const full = path.join(dir, entry.name);
             if (entry.isDirectory())
                 stack.push(full);
-            else if (entry.isFile()
-                && (entry.name.endsWith('.ts') || entry.name.endsWith('.tsx'))) {
+            else if (entry.isFile() && extensions.some(ext => entry.name.endsWith(ext))) {
                 out.push(full);
             }
         }
@@ -91,9 +138,10 @@ export function getMaxMtime(files) {
     return String(Math.floor(max));
 }
 export function ensureProjectIndexed(cache, name, version, files, cwd) {
+    const profiles = projectProfiles(cwd);
     const existing = cache.db
-        .prepare('SELECT content_hash FROM packages WHERE name = ? AND version = ?')
-        .get(name, version);
+        .prepare('SELECT content_hash FROM packages WHERE ecosystem = ? AND name = ? AND version = ?')
+        .get(PROJECT_SCOPE, name, version);
     if (existing)
         return { hitCache: true, filesIngested: 0, chunksWritten: 0 };
     const t0 = Date.now();
@@ -101,9 +149,13 @@ export function ensureProjectIndexed(cache, name, version, files, cwd) {
     try {
         // Delete by NAME with no version: unlike the package index, a project
         // keeps only its newest max-mtime version, so every older one goes.
-        cache.db.prepare('DELETE FROM chunks WHERE name = ?').run(name);
-        cache.db.prepare('DELETE FROM packages WHERE name = ?').run(name);
-        const insertChunk = cache.db.prepare('INSERT INTO chunks (name, version, file_path, kind, content) VALUES (?, ?, ?, ?, ?)');
+        cache.db
+            .prepare('DELETE FROM chunks WHERE ecosystem = ? AND name = ?')
+            .run(PROJECT_SCOPE, name);
+        cache.db
+            .prepare('DELETE FROM packages WHERE ecosystem = ? AND name = ?')
+            .run(PROJECT_SCOPE, name);
+        const insertChunk = cache.db.prepare('INSERT INTO chunks (ecosystem, name, version, file_path, kind, content) VALUES (?, ?, ?, ?, ?, ?)');
         let filesIngested = 0;
         let chunksWritten = 0;
         for (const abs of files) {
@@ -115,18 +167,25 @@ export function ensureProjectIndexed(cache, name, version, files, cwd) {
                 continue;
             }
             const rel = path.relative(cwd, abs);
-            const chunks = chunkDeclarations(raw, rel);
+            // The project's own source goes in WHOLE. `profile.surface` strips a
+            // dependency's implementation down to its published API, which is the
+            // opposite of what is wanted here: a binary crate's `main.rs` has no
+            // `pub` item in it at all, so surfacing it indexes to nothing and
+            // `pi-worker-docs(".", …)` answers "no chunks" for the whole project.
+            // Only the CHUNK BOUNDARY is language-specific.
+            const profile = profileForFile(abs, profiles);
+            const chunks = chunkDeclarations(raw, rel, profile.declSplitRe, profile.commentPrefix);
             if (!chunks.length)
                 continue;
             filesIngested++;
             for (const c of chunks) {
-                insertChunk.run(name, version, rel, 'dts', c);
+                insertChunk.run(PROJECT_SCOPE, name, version, rel, 'dts', c);
                 chunksWritten++;
             }
         }
         cache.db
-            .prepare('INSERT OR REPLACE INTO packages (name, version, content_hash, indexed_at) VALUES (?, ?, ?, ?)')
-            .run(name, version, version, Date.now());
+            .prepare('INSERT OR REPLACE INTO packages (ecosystem, name, version, content_hash, indexed_at) VALUES (?, ?, ?, ?, ?)')
+            .run(PROJECT_SCOPE, name, version, version, Date.now());
         cache.db.exec('COMMIT');
         return { hitCache: false, filesIngested, chunksWritten, indexingMs: Date.now() - t0 };
     }
@@ -154,8 +213,8 @@ listFiles = getProjectFiles) {
         };
     }
     const chunkCount = cache.db
-        .prepare('SELECT count(*) AS c FROM chunks WHERE name = ? AND version = ?')
-        .get(cacheKey, version)?.c ?? 0;
+        .prepare('SELECT count(*) AS c FROM chunks WHERE ecosystem = ? AND name = ? AND version = ?')
+        .get(PROJECT_SCOPE, cacheKey, version)?.c ?? 0;
     if (chunkCount === 0) {
         return {
             kind: 'no_chunks',
@@ -163,12 +222,14 @@ listFiles = getProjectFiles) {
             cacheKey,
             version,
             hitCache: indexResult.hitCache,
-            filesIngested: indexResult.filesIngested
+            filesIngested: indexResult.filesIngested,
+            sourceLabel: projectSourceLabel(cwd)
         };
     }
     let chunks;
     try {
         chunks = retrieveChunksFn(cache, {
+            ecosystem: PROJECT_SCOPE,
             name: cacheKey,
             version,
             query,
@@ -190,7 +251,8 @@ listFiles = getProjectFiles) {
             cacheKey,
             version,
             hitCache: indexResult.hitCache,
-            filesIngested: indexResult.filesIngested
+            filesIngested: indexResult.filesIngested,
+            sourceLabel: projectSourceLabel(cwd)
         };
     }
     return {

package/dist/workers/docs-resolve.d.ts

@@ -1,16 +1,21 @@
+import type { EcosystemId } from './docs-ecosystems.js';
 /** True for TypeScript declaration files: .d.ts, .d.mts, .d.cts. */
 export declare function isDtsFile(name: string): boolean;
 export interface ResolvedPackage {
+    /** Which registry this package came from. Scopes it in the docs cache. */
+    ecosystem: EcosystemId;
     name: string;
     version: string;
     root: string;
-    entryDts: string | null;
+    /** The file holding the package's public API surface, if it has one. */
+    entry: string | null;
     readme: string | null;
 }
 export declare class ResolveError extends Error {
     readonly kind: 'not_installed' | 'invalid_name';
     constructor(kind: 'not_installed' | 'invalid_name', message: string);
 }
+export declare function isValidModuleName(name: string): boolean;
 /**
  * Split a runtime builtin specifier (`bun:sql`, `node:fs/promises`) into its
  * runtime and submodule. Returns null for ordinary specifiers (including scoped

package/dist/workers/docs-resolve.js

@@ -15,7 +15,7 @@ export class ResolveError extends Error {
     }
 }
 const MODULE_NAME_RE = /^(?:@[a-z0-9-_.]+\/)?[a-z0-9-_.]+(?:\/[a-z0-9-_./]+)?$/i;
-function isValidModuleName(name) {
+export function isValidModuleName(name) {
     if (!name || name.includes('..') || name.startsWith('/'))
         return false;
     return MODULE_NAME_RE.test(name);
@@ -147,10 +147,11 @@ export function resolvePackage(moduleName, cwd) {
     const root = path.dirname(pkgJsonPath);
     const pkg = readPackageJson(pkgJsonPath);
     return {
+        ecosystem: 'npm',
         name: pkg.name ?? parent,
         version: pkg.version ?? '0.0.0',
         root,
-        entryDts: resolveEntryDts(moduleName, parent, root, pkg),
+        entry: resolveEntryDts(moduleName, parent, root, pkg),
         readme: findReadme(root)
     };
 }
@@ -282,7 +283,7 @@ export function detectTypesRedirect(pkg) {
     // redirect stub — use its own types.
     if (countTypeFiles(pkg.root, 2) > 1)
         return null;
-    const entry = pkg.entryDts ?? findIndexDts(pkg.root);
+    const entry = pkg.entry ?? findIndexDts(pkg.root);
     if (!entry)
         return null;
     let content;

package/dist/workers/docs-retrieve.d.ts

@@ -6,6 +6,8 @@ export interface RetrievedChunk {
     rank: number;
 }
 export interface RetrieveOptions {
+    /** Registry scope of the rows to search, or the project-source scope. */
+    ecosystem: string;
     name: string;
     version: string;
     query: string;

package/dist/workers/docs-retrieve.js

@@ -37,13 +37,13 @@ function tokenize(query) {
 function buildFtsQuery(tokens) {
     return tokens.map(t => `"${t}"`).join(' OR ');
 }
-function fallbackChunks(cache, name, version) {
+function fallbackChunks(cache, ecosystem, name, version) {
     const dts = cache.db
-        .prepare("SELECT file_path, kind, content, 0 AS rank FROM chunks WHERE name = ? AND version = ? AND kind = 'dts' ORDER BY file_path, id LIMIT 1")
-        .all(name, version);
+        .prepare("SELECT file_path, kind, content, 0 AS rank FROM chunks WHERE ecosystem = ? AND name = ? AND version = ? AND kind = 'dts' ORDER BY file_path, id LIMIT 1")
+        .all(ecosystem, name, version);
     const readme = cache.db
-        .prepare("SELECT file_path, kind, content, 0 AS rank FROM chunks WHERE name = ? AND version = ? AND kind = 'readme' ORDER BY id LIMIT 1")
-        .all(name, version);
+        .prepare("SELECT file_path, kind, content, 0 AS rank FROM chunks WHERE ecosystem = ? AND name = ? AND version = ? AND kind = 'readme' ORDER BY id LIMIT 1")
+        .all(ecosystem, name, version);
     const out = [];
     for (const r of dts) {
         out.push({
@@ -91,7 +91,7 @@ export function retrieveChunks(cache, opts) {
     const budget = opts.contentBudget ?? DEFAULT_BUDGET;
     const tokens = tokenize(opts.query);
     if (tokens.length === 0) {
-        return enforceBudget(fallbackChunks(cache, opts.name, opts.version), budget);
+        return enforceBudget(fallbackChunks(cache, opts.ecosystem, opts.name, opts.version), budget);
     }
     const ftsQuery = buildFtsQuery(tokens);
     let rows;
@@ -100,16 +100,16 @@ export function retrieveChunks(cache, opts) {
             .prepare(`SELECT c.file_path, c.kind, c.content, bm25(chunks_fts) AS rank
                  FROM chunks_fts
                  JOIN chunks c ON c.id = chunks_fts.rowid
-                 WHERE c.name = ?1 AND c.version = ?2 AND chunks_fts MATCH ?3
+                 WHERE c.ecosystem = ?1 AND c.name = ?2 AND c.version = ?3 AND chunks_fts MATCH ?4
                  ORDER BY rank
-                 LIMIT ?4`)
-            .all(opts.name, opts.version, ftsQuery, limit);
+                 LIMIT ?5`)
+            .all(opts.ecosystem, opts.name, opts.version, ftsQuery, limit);
     }
     catch {
-        return enforceBudget(fallbackChunks(cache, opts.name, opts.version), budget);
+        return enforceBudget(fallbackChunks(cache, opts.ecosystem, opts.name, opts.version), budget);
     }
     if (rows.length === 0) {
-        return enforceBudget(fallbackChunks(cache, opts.name, opts.version), budget);
+        return enforceBudget(fallbackChunks(cache, opts.ecosystem, opts.name, opts.version), budget);
     }
     const mapped = rows.map(r => ({
         filePath: r.file_path,

package/dist/workers/eco-cargo.d.ts

@@ -0,0 +1,115 @@
+/**
+ * eco-cargo — reading Rust crates for the docs Worker tool.
+ *
+ * Rust ships no declarations file, so the documented surface has to be cut out
+ * of `.rs` source: doc comments, attributes and public item heads kept, function
+ * bodies and private items dropped. That is what `surface` below does, and it is
+ * why this row needs code where the npm row needed none.
+ *
+ * Nothing here parses TOML. `Cargo.lock` is a generated file with a fixed
+ * `[[package]]` shape, and a line reader over it costs one small function where
+ * a TOML dependency would cost a dependency.
+ */
+import { type ResolvedPackage } from './docs-resolve.js';
+import type { NpmVersionInfo } from './npm-version.js';
+export declare function isValidCrateName(name: string): boolean;
+/** `serde_json::Value` is a path into `serde_json`; the crate is what installs. */
+export declare function crateOf(name: string): string;
+/** Every version of `name` recorded in a `Cargo.lock`, oldest first. */
+export declare function lockVersions(lockText: string, name: string): string[];
+/** Directories that never hold a project's own manifest. */
+export declare const SKIP_DIRS: Set<string>;
+/**
+ * Immediate child directories of `cwd`, for the one-level-down manifest scan —
+ * and EMPTY unless `cwd` is itself a project root.
+ *
+ * Without that guard the scan reaches into any directory that merely happens to
+ * contain projects. `/tmp` is the case that bites: a single unrelated checkout
+ * under it would make every lookup run from `/tmp` believe it was in that
+ * ecosystem.
+ */
+export declare function childDirs(cwd: string): string[];
+/**
+ * The `Cargo.lock` governing `cwd`: cargo's own upward walk first, then ONE level
+ * down. The step down is the Tauri shape — `package.json` at the repo root and
+ * the crate under `src-tauri/` — where the upward walk from the root finds
+ * nothing at all.
+ */
+export declare function findLock(cwd: string): string | null;
+/**
+ * The first `<dir>/<name>` at or above `cwd`, checking each ancestor's immediate
+ * children too.
+ *
+ * The sideways step is the Tauri shape: `package.json` at the repo root and the
+ * crate under `src-tauri/`. Checking children of `cwd` alone is not enough —
+ * from the frontend directory `src/`, the crate is in a SIBLING, so the scan has
+ * to happen at every level on the way up or the whole project reads as npm-only.
+ */
+export declare function findAtOrAbove(cwd: string, ...rel: string[]): string | null;
+/**
+ * The version the project pins `name` to. Several is not an error — a workspace
+ * legitimately holds two majors of one crate — and the NEWEST is taken, which
+ * the answer then states in its `Per <name>@<version>:` header.
+ */
+export declare function lockedVersion(name: string, cwd: string): string | null;
+/**
+ * Every crate the lock pins, name to version. Cargo has already resolved these,
+ * so unlike an npm range these are exact — and a `cargo update` that moves one
+ * is what a cached answer about it has to be dropped on.
+ */
+export declare function lockedDeps(cwd: string): Record<string, string> | undefined;
+export interface CargoResolveDirs {
+    cargoHome: string;
+    /** Where a crate fetched by this tool was extracted. */
+    modulesDir: string;
+}
+/**
+ * Find a crate's source on disk.
+ *
+ * The lock is consulted first, so a project reading `tokio` gets the `tokio` it
+ * builds against rather than the newest copy the machine happens to hold. With
+ * no lock in sight — which is where the post-fetch re-resolve arrives, since it
+ * is handed the download directory — the newest extracted copy wins.
+ */
+export declare function resolveCrate(name: string, cwd: string, dirs: CargoResolveDirs): ResolvedPackage;
+/** The newest published version of a crate, or null on any failure. */
+export declare function cratesLatest(name: string, fetchFn: typeof fetch, signal?: AbortSignal): Promise<NpmVersionInfo | null>;
+export declare function crateTarballUrl(name: string, version: string): string;
+/**
+ * Where a Rust declaration begins, so a chunk never splits a signature. Column 0
+ * only: `rustSurface` INDENTS the members of an impl or a trait, so an `^\s*`
+ * anchor cut every method into its own chunk — an orphan signature with no
+ * receiver type, carrying the next method's doc comment.
+ */
+export declare const CARGO_DECL_SPLIT_RE: RegExp;
+interface Item {
+    /** Doc comments and attributes immediately above the item. */
+    pending: string;
+    /** The item text up to its `{` or `;`. */
+    head: string;
+    /** The brace body, without the braces. Null for a `;`-terminated item. */
+    body: string | null;
+}
+/**
+ * Split one nesting level of Rust source into items.
+ *
+ * Braces are counted with strings, chars, comments and lifetimes skipped: `"{"`
+ * and `'{'` both appear in real source, and a scanner that counts them never
+ * finds the end of the item it is in.
+ */
+export declare function splitRustItems(src: string): Item[];
+/**
+ * Reduce Rust source to its public API surface.
+ *
+ * Function bodies go — they are the bulk of the file and answer no question the
+ * docs tool is asked. Everything a caller can name stays: the item head, its doc
+ * comment, its attributes, and for a struct or enum its fields and variants.
+ *
+ * Inside a `trait` every member is public by definition, so the `pub` test is
+ * suspended there; anywhere else a bare or `pub(crate)` item is dropped.
+ */
+export declare function rustSurface(src: string, insideTrait?: boolean, topLevel?: boolean): string;
+export declare function isRustFile(name: string): boolean;
+/** The `[package] name` of a cargo project, for labelling its own source. */
+export declare function cargoProjectName(cwd: string): string | null;
+export {};