@mjasnikovs/pi-task 0.39.5 → 0.40.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/README.md +18 -3
  2. package/dist/task/context-attribution.js +18 -6
  3. package/dist/task/external-context.d.ts +8 -1
  4. package/dist/task/external-context.js +52 -4
  5. package/dist/task/phases.js +2 -1
  6. package/dist/task/prompts.d.ts +7 -1
  7. package/dist/task/prompts.js +14 -4
  8. package/dist/workers/docs-cache.js +50 -3
  9. package/dist/workers/docs-chunk.d.ts +6 -3
  10. package/dist/workers/docs-chunk.js +8 -5
  11. package/dist/workers/docs-core.d.ts +27 -3
  12. package/dist/workers/docs-core.js +104 -41
  13. package/dist/workers/docs-ecosystems.d.ts +173 -0
  14. package/dist/workers/docs-ecosystems.js +449 -0
  15. package/dist/workers/docs-index.d.ts +2 -1
  16. package/dist/workers/docs-index.js +55 -27
  17. package/dist/workers/docs-project.d.ts +10 -0
  18. package/dist/workers/docs-project.js +86 -24
  19. package/dist/workers/docs-resolve.d.ts +6 -1
  20. package/dist/workers/docs-resolve.js +4 -3
  21. package/dist/workers/docs-retrieve.d.ts +2 -0
  22. package/dist/workers/docs-retrieve.js +11 -11
  23. package/dist/workers/eco-cargo.d.ts +115 -0
  24. package/dist/workers/eco-cargo.js +793 -0
  25. package/dist/workers/eco-hackage.d.ts +93 -0
  26. package/dist/workers/eco-hackage.js +508 -0
  27. package/dist/workers/npm-version.d.ts +5 -3
  28. package/dist/workers/npm-version.js +6 -4
  29. package/dist/workers/pi-worker-docs.d.ts +18 -4
  30. package/dist/workers/pi-worker-docs.js +57 -19
  31. package/dist/workers/research-cache.d.ts +2 -13
  32. package/dist/workers/research-cache.js +22 -46
  33. package/dist/workers/shared.d.ts +16 -5
  34. package/dist/workers/shared.js +0 -0
  35. package/package.json +1 -1
package/README.md CHANGED
@@ -191,12 +191,25 @@ Fetches a URL, cleans HTML to markdown ([Readability](https://github.com/mozilla
191
191
  - The extraction child runs with `--no-tools` to mitigate visible-text prompt injection.
192
192
 
193
193
  ### `pi-worker-docs`
194
- Resolves an installed npm package, indexes its `.d.ts` files and README into a local SQLite cache, retrieves the most relevant chunks for your `query`, and passes them to an isolated child that extracts the focused answer. Version-pinned to whatever is in your `node_modules`.
194
+ Resolves an installed package, indexes its API surface and README into a local SQLite cache, retrieves the most relevant chunks for your `query`, and passes them to an isolated child that extracts the focused answer. Version-pinned to whatever the project actually resolved.
195
195
 
196
- - The package must be installed in the project's `node_modules`; otherwise a one-time auto-install into a dedicated cache dir is attempted.
197
- - The first call for a `(package, version)` pair pays a one-time ingestion cost; later calls are FTS-only.
196
+ **The manifest decides which registry, not the model.** `text`, `base`, `aeson`, `tokio` and `clap` are all real npm packages *and* real Rust/Haskell ones, so a name alone cannot say which was meant — and guessing npm returns a confident answer about an unrelated package. That is a wrong answer, not a miss.
197
+
198
+ | Ecosystem | Detected by | Surface read | Version comes from |
199
+ | --- | --- | --- | --- |
200
+ | `npm` | `package.json`, or a `node_modules/` directory | the `.d.ts` files the package ships, plus README | the installed `package.json` |
201
+ | `cargo` | `Cargo.toml`, at the directory or one level below it | `.rs` source reduced to public item heads, doc comments and attributes | `Cargo.lock` |
202
+ | `hackage` | `*.cabal`, `cabal.project`, `stack.yaml` or `package.yaml` | `.hs` source reduced to the export list, signatures and type declarations | `dist-newstyle/cache/plan.json`, then `cabal.project.freeze`, then `stack.yaml.lock` |
203
+
204
+ - **No manifest, no lookup.** In a directory with none of the above the tool refuses, spawns nothing and installs nothing, and points you at `pi-worker-search` / `pi-worker-fetch` instead.
205
+ - **Two manifests** (a Tauri app, say) are resolved by whichever registry already has the package on disk. If neither does, the call is refused as ambiguous and you pass `ecosystem: "cargo"` to say which.
206
+ - A package the project does not have is fetched once into a dedicated cache dir: `npm install --ignore-scripts` for npm, the `.crate` tarball for cargo, the Hackage tarball (or cabal's own cached copy) for hackage.
207
+ - A Haskell **module** name is refused by name — `Data.Aeson` is not a package, `aeson` is.
208
+ - The first call for a `(ecosystem, package, version)` triple pays a one-time ingestion cost; later calls are FTS-only.
198
209
  - Cache lives at `${XDG_CACHE_HOME:-~/.cache}/pi-worker/docs.sqlite` — delete it to reset.
199
210
 
211
+ **Known gaps.** The rest of pi-task is still npm-shaped: the final gate, repo-health and orientation read the working directory first-wins, so a Tauri repo gets cargo docs answers while the gate runs `bun run test`, not `cargo test`. Phantom-import checking, dependency-name extraction for research enrichment, and the `@types/…` redirect chain are npm concepts and are no-ops elsewhere. Adding an ecosystem is one row in `src/workers/docs-ecosystems.ts` plus its parsers — profiles live in code and arrive as pull requests with tests, never as user configuration.
212
+
200
213
  ## Settings — `/task-config`
201
214
 
202
215
  Run `/task-config` to toggle pi-task's behavior in an editor dialog. Settings persist to `~/.config/pi-task/config.json`.
@@ -226,6 +239,8 @@ Run `/task-config` to toggle pi-task's behavior in an editor dialog. Settings pe
226
239
  | --- | --- | --- |
227
240
  | `BRAVE_SEARCH_API_KEY` / `BRAVE_API_KEY` | `pi-worker-search`, research enrichment | Required only when the **Brave** search engine is selected in `/task-config`. |
228
241
  | `XDG_CACHE_HOME` | `pi-worker-docs` | Overrides the docs cache location (defaults to `~/.cache`). |
242
+ | `CARGO_HOME` | `pi-worker-docs` | Where crate source checkouts are read from (defaults to `~/.cargo`). |
243
+ | `CABAL_DIR` | `pi-worker-docs` | Where cabal's downloaded package tarballs are read from (also checks `~/.cabal/packages` and `${XDG_CACHE_HOME:-~/.cache}/cabal/packages`). |
229
244
  | `XDG_DATA_HOME` | remote push | Where the VAPID keypair is stored (defaults to `~/.local/share`). |
230
245
  | `PI_REMOTE_PUSH_SUBJECT` | remote push | VAPID JWT `sub` contact. Defaults to the project URL; set your own `mailto:you@domain.com` or `https://…`. |
231
246
  | `PI_REMOTE_PUSH_DEBUG` | remote push | When set (e.g. `1`), logs push delivery and push-service HTTP status. Off by default. |
@@ -154,17 +154,29 @@ export function splitBulletSpans(context) {
154
154
  export function splitBullets(context) {
155
155
  return splitBulletSpans(context).map(s => s.text);
156
156
  }
157
+ /**
158
+ * A version block is headed by its REGISTRY, so the heading is open-ended
159
+ * (`crates.io`, `hackage`). Anything else is one: the widened pattern also reads
160
+ * headings out of a retrieved README or page BODY, and calling those `docs` would
161
+ * let a package's own text forge the block that sources a semantics claim.
162
+ */
163
+ function blockKind(heading) {
164
+ if (heading === 'freshness-check')
165
+ return 'freshness-skipped';
166
+ if (heading === 'docs' || heading === 'url' || heading === 'service')
167
+ return heading;
168
+ return 'npm';
169
+ }
157
170
  /** Parse the `### npm:` / `### docs:` / `### url:` / `### service:` blocks out of an EXTERNAL CONTEXT header. */
158
171
  export function parseContextBlocks(externalContext) {
159
172
  const out = [];
160
- const re = /^###\s+(npm|docs|url|service|freshness-check)\s*:?\s*(.*)$/gim;
173
+ // A version block is headed by its REGISTRY, so `npm` is one of several
174
+ // (`crates.io`, `hackage`). Matching only the fixed words made every non-npm
175
+ // block invisible to the parser.
176
+ const re = /^###\s+([A-Za-z][\w.-]*)\s*:?\s*(.*)$/gim;
161
177
  let m;
162
178
  while ((m = re.exec(externalContext)) !== null) {
163
- const kind = m[1].toLowerCase();
164
- out.push({
165
- kind: kind === 'freshness-check' ? 'freshness-skipped' : kind,
166
- subject: m[2].trim()
167
- });
179
+ out.push({ kind: blockKind(m[1].toLowerCase()), subject: m[2].trim() });
168
180
  }
169
181
  return out;
170
182
  }
@@ -36,6 +36,8 @@ type GatherDeps = Pick<PhaseDeps, 'cwd' | 'signal' | 'recordSubStep' | 'docsRaw'
36
36
  export interface ExternalTargetResult {
37
37
  /** Emitted as an `### npm:` block ahead of every body. Absent for url targets. */
38
38
  npmVersion?: NpmVersionInfo | null;
39
+ /** Which registry the version block came from; npm when absent. */
40
+ registryLabel?: string;
39
41
  /**
40
42
  * The `### docs:`/`### url:` body. `undefined` means "this target contributes no
41
43
  * body block" — and the two call paths draw that line differently ON PURPOSE:
@@ -68,6 +70,11 @@ export interface ExternalContextLookups {
68
70
  * question, in front of a waiting user, while research is uncapped and trails its
69
71
  * sub-step.
70
72
  */
73
+ /** A live version answer and the registry it came from, for the block heading. */
74
+ export interface VersionBlock {
75
+ info: NpmVersionInfo;
76
+ label: string;
77
+ }
71
78
  export interface ExternalContextPolicy {
72
79
  /**
73
80
  * Max combined docs+url targets fanned out, packages first. Omit for uncapped
@@ -83,7 +90,7 @@ export interface ExternalContextPolicy {
83
90
  * version at all, and a "which version?" question falls back to whatever the
84
91
  * model remembers — which is how a dependency gets pinned to a stale major.
85
92
  */
86
- versionLookup?: (pkg: string) => Promise<NpmVersionInfo | null>;
93
+ versionLookup?: (pkg: string) => Promise<VersionBlock | null>;
87
94
  /** Sub-step label recorded via `deps.recordSubStep`. Omit to record nothing. */
88
95
  subStepLabel?: string;
89
96
  /**
@@ -22,6 +22,7 @@
22
22
  * Run against a mixed source, the emitted headings come out in exactly that order:
23
23
  * `### npm:` then `### docs:` then `### url:` then `### service:`.
24
24
  */
25
+ import { chooseEcosystem, defaultEcosystemIo } from '../workers/docs-ecosystems.js';
25
26
  import { docsRaw } from '../workers/docs-core.js';
26
27
  import { fetchRaw } from '../workers/fetch-core.js';
27
28
  import { formatNpmVersionSection, npmVersionLookup } from '../workers/npm-version.js';
@@ -61,6 +62,14 @@ export async function buildExternalContext(source, deps, lookups, policy = {}) {
61
62
  }).catch(() => null))),
62
63
  Promise.all(versionLookup ? extraVersionPkgs.map(pkg => versionLookup(pkg).catch(() => null)) : [])
63
64
  ]);
65
+ // A docs target that came back WITHOUT a version — a refused ecosystem, a dead
66
+ // registry — was dropped from `extraVersionPkgs` on the assumption docs would
67
+ // supply it. Ask for it now, or its block is silently lost.
68
+ const missedVersionPkgs = versionLookup ?
69
+ enrichTargets.versionPackages.filter(p => docsTargets.has(p)
70
+ && !targets.some((t, i) => t.name === p && targetResults[i]?.npmVersion))
71
+ : [];
72
+ const missedVersionResults = await Promise.all(missedVersionPkgs.map(pkg => versionLookup(pkg).catch(() => null)));
64
73
  const sections = [];
65
74
  // npm version blocks lead the section so the model anchors on live version
66
75
  // data before reading any docs body. The docs-fetched packages carry their
@@ -69,11 +78,11 @@ export async function buildExternalContext(source, deps, lookups, policy = {}) {
69
78
  // named dep.
70
79
  for (const r of targetResults) {
71
80
  if (r?.npmVersion)
72
- sections.push(formatNpmVersionSection(r.npmVersion));
81
+ sections.push(formatNpmVersionSection(r.npmVersion, r.registryLabel));
73
82
  }
74
- for (const v of extraVersionResults) {
83
+ for (const v of [...extraVersionResults, ...missedVersionResults]) {
75
84
  if (v)
76
- sections.push(formatNpmVersionSection(v));
85
+ sections.push(formatNpmVersionSection(v.info, v.label));
77
86
  }
78
87
  for (let i = 0; i < targets.length; i++) {
79
88
  const body = targetResults[i]?.body;
@@ -118,6 +127,44 @@ export async function gatherExternalContext(refined, deps) {
118
127
  const fetchRawFn = deps.fetchRaw ?? fetchRaw;
119
128
  const npmVersionFn = deps.npmVersionLookup ?? npmVersionLookup;
120
129
  const docsQuery = refined.split('\n').find(l => l.trim()) ?? refined;
130
+ // Which registry to ask, decided PER PACKAGE.
131
+ //
132
+ // A cargo-only project's dependency names are crate names, and asking npm
133
+ // about them returns a real but unrelated package's versions — the exact
134
+ // confusion the docs tool refuses. But deciding once for the whole repo makes
135
+ // every polyglot project ambiguous, and then NO version block is emitted at
136
+ // all: worse than the old always-npm, because the prompts tell the model to
137
+ // quote a block that will never be there. Each name carries its own evidence,
138
+ // so each name gets its own answer.
139
+ const io = defaultEcosystemIo(deps.signal ? { signal: deps.signal } : {});
140
+ const askNpm = async (pkg) => {
141
+ const info = await npmVersionFn(pkg, { signal: deps.signal });
142
+ return info ? { info, label: 'npm' } : null;
143
+ };
144
+ const versionLookup = async (pkg) => {
145
+ const choice = chooseEcosystem({
146
+ cwd: deps.cwd,
147
+ declaresPackage: p => p.declaredRange(p.parentPackage(pkg), deps.cwd) !== null,
148
+ resolvesLocally: p => {
149
+ try {
150
+ p.resolve(pkg, deps.cwd, io);
151
+ return true;
152
+ }
153
+ catch {
154
+ return false;
155
+ }
156
+ }
157
+ });
158
+ // No manifest at all keeps npm, which is what a bare directory has always
159
+ // done. A name genuinely ambiguous between two registries gets no block —
160
+ // a version from the wrong registry is worse than none.
161
+ if (!choice.ok)
162
+ return choice.reason === 'none' ? askNpm(pkg) : null;
163
+ if (choice.profile.id === 'npm')
164
+ return askNpm(pkg);
165
+ const info = await choice.profile.latest(pkg, io);
166
+ return info ? { info, label: choice.profile.registryLabel } : null;
167
+ };
121
168
  return buildExternalContext(refined, deps, {
122
169
  docs: async (pkg) => {
123
170
  const r = await docsRawFn({
@@ -128,6 +175,7 @@ export async function gatherExternalContext(refined, deps) {
128
175
  });
129
176
  return {
130
177
  npmVersion: r.npmVersion,
178
+ ...(r.registryLabel ? { registryLabel: r.registryLabel } : {}),
131
179
  body: r.kind === 'ok' && r.chunks.length > 0 ?
132
180
  r.chunks
133
181
  .map(c => c.content)
@@ -142,7 +190,7 @@ export async function gatherExternalContext(refined, deps) {
142
190
  },
143
191
  search: deps.searchFn
144
192
  }, {
145
- versionLookup: pkg => npmVersionFn(pkg, { signal: deps.signal }),
193
+ versionLookup,
146
194
  subStepLabel: 'enrichment',
147
195
  earlyReturnOnNoTargets: true
148
196
  });
@@ -4,6 +4,7 @@
4
4
  */
5
5
  import { fileURLToPath } from 'node:url';
6
6
  import { docsFocused } from '../workers/docs-core.js';
7
+ import { detectEcosystems } from '../workers/docs-ecosystems.js';
7
8
  import { fetchFocused } from '../workers/fetch-core.js';
8
9
  import { runWorker } from '../workers/pi-worker-core.js';
9
10
  import { findPhantomImports, formatApiCorrections, rewritePhantomSpecifiers } from '../workers/phantom-imports.js';
@@ -550,7 +551,7 @@ export async function phaseResearch(deps, refined) {
550
551
  // queries the FILES worker just answered.
551
552
  prompt: prior => orientation.block
552
553
  + promptHeader
553
- + RESEARCH_APIS_PROMPT(refined, prior.find(s => s.name === 'FILES')?.text || undefined)
554
+ + RESEARCH_APIS_PROMPT(refined, prior.find(s => s.name === 'FILES')?.text || undefined, detectEcosystems(deps.cwd))
554
555
  + (searchConfigured() ? RESEARCH_SEARCH_HINT : '')
555
556
  // Empty unless PI_TASK_PROJECT_DOCS_BUDGET is set. The tool-side
556
557
  // half lives in pi-worker-docs.ts; a budget enforced without being
@@ -26,7 +26,13 @@ export declare const COMPRESS_LABEL_PROMPT: (title: string, maxChars: number) =>
26
26
  declare const REFINE_PROMPT: (raw: string, planContext?: string, existingFiles?: string, contracts?: string, directives?: string) => string;
27
27
  declare const RESEARCH_READ_ONLY_CONSTRAINT = "IMPORTANT: You are ONLY allowed to READ. Do NOT create, modify, or delete any files. Use the read, grep, find, and ls tools to inspect the repo.";
28
28
  declare const RESEARCH_FILES_PROMPT: (refined: string) => string;
29
- declare const RESEARCH_APIS_PROMPT: (refined: string, filesMap?: string) => string;
29
+ /**
30
+ * `ecosystems` is what `detectEcosystems` found in the worker's cwd. It is REQUIRED,
31
+ * and an empty list is a real answer: a directory with no package manifest has no
32
+ * registry to look a library up in, and the paragraph below has to say so rather
33
+ * than inviting a lookup that will be refused.
34
+ */
35
+ declare const RESEARCH_APIS_PROMPT: (refined: string, filesMap: string | undefined, ecosystems: readonly string[]) => string;
30
36
  declare const RESEARCH_CONTEXT_PROMPT: (refined: string) => string;
31
37
  declare const RESEARCH_TOOLING_PROMPT: (refined: string) => string;
32
38
  declare const GRILL_GEN_PROMPT: (refined: string, research: string, priorQA: string) => string;
@@ -97,13 +97,23 @@ No section header. No other sections. No preamble.
97
97
 
98
98
  Task:
99
99
  ${refined}`;
100
- const RESEARCH_APIS_PROMPT = (refined, filesMap) => `You are doing targeted research for an AI coding agent. Use the read, grep, find, and ls tools — and \`pi-worker-docs\` for installed npm packages — to identify the commands, functions, types, and interfaces the agent will use for the following task.
100
+ /**
101
+ * `ecosystems` is what `detectEcosystems` found in the worker's cwd. It is REQUIRED,
102
+ * and an empty list is a real answer: a directory with no package manifest has no
103
+ * registry to look a library up in, and the paragraph below has to say so rather
104
+ * than inviting a lookup that will be refused.
105
+ */
106
+ const RESEARCH_APIS_PROMPT = (refined, filesMap, ecosystems) => `You are doing targeted research for an AI coding agent. Use the read, grep, find, and ls tools${ecosystems.length ? ' — and `pi-worker-docs` for installed packages —' : ''} to identify the commands, functions, types, and interfaces the agent will use for the following task.
101
107
 
102
- NPM PACKAGES — use pi-worker-docs, NOT file reads: for any third-party npm package (e.g. "zod", "hono", "drizzle-orm"), call \`pi-worker-docs(module, query)\` to get its type signatures and API surface. Do NOT open node_modules source files directly — those reads are expensive and produce far more noise than the tool. The tool returns a compact, focused excerpt in a fraction of the token cost.
108
+ ${ecosystems.length ?
109
+ `LIBRARY PACKAGES — use pi-worker-docs, NOT file reads: for any third-party package, call \`pi-worker-docs(module, query)\` to get its type signatures and API surface. Do NOT open installed-package source files directly — those reads are expensive and produce far more noise than the tool. The tool returns a compact, focused excerpt in a fraction of the token cost. This project's package ecosystems are: ${ecosystems.join(', ')}.`
110
+ : 'LIBRARY PACKAGES — do NOT look them up: this directory holds no package manifest, so `pi-worker-docs` has no registry to read and will refuse. Do not list an external library API you cannot check here.'}
103
111
 
104
- PROJECT SOURCE — use pi-worker-docs with module ".", NOT file reads: for any function, class, type, or interface defined in THIS project's own .ts/.tsx source (e.g. "what does requireAuth check?", "what does CreateListingSchema look like?", "what does the listings query module export?"), call \`pi-worker-docs(".", query)\` instead of reading the file. The tool indexes all git-tracked source files and returns only the relevant chunks — far cheaper than reading whole files.
112
+ PROJECT SOURCE — use pi-worker-docs with module ".", NOT file reads: for any function, class, type, or interface defined in THIS project's own source (e.g. "what does requireAuth check?", "what does CreateListingSchema look like?", "what does the listings query module export?"), call \`pi-worker-docs(".", query)\` instead of reading the file. The tool indexes all git-tracked source files and returns only the relevant chunks — far cheaper than reading whole files.
105
113
 
106
- RUNTIME BUILTINS — verify, do NOT echo: a task (or the spec doc it references) may name a runtime/builtin import like \`bun:sql\`, \`bun:sqlite\`, \`node:fs\`, or \`Bun.password\`. A runtime exposes only a small FIXED set of \`<runtime>:<submodule>\` modules, and a spec doc can confidently name one that does not exist. Before you list ANY \`<pkg>:<sub>\` specifier, confirm it with \`pi-worker-docs\` (e.g. \`pi-worker-docs("bun:sql", "sql tagged template and SQL class — the import")\` — the tool resolves the runtime's real types) and emit the CANONICAL import the types actually prove, NOT the string copied from the task. Concretely: Bun's SQL client is \`import { sql } from "bun"\` (or \`Bun.sql\` / \`new SQL()\`) — there is NO \`bun:sql\` module. Never pass an unverified colon-specifier through to the APIS list; a phantom import laundered here becomes fabricated \`declare module\` shims in the implementation.
114
+ ${ecosystems.includes('npm') ?
115
+ `RUNTIME BUILTINS — verify, do NOT echo: a task (or the spec doc it references) may name a runtime/builtin import like \`bun:sql\`, \`bun:sqlite\`, \`node:fs\`, or \`Bun.password\`. A runtime exposes only a small FIXED set of \`<runtime>:<submodule>\` modules, and a spec doc can confidently name one that does not exist. Before you list ANY \`<pkg>:<sub>\` specifier, confirm it with \`pi-worker-docs\` (e.g. \`pi-worker-docs("bun:sql", "sql tagged template and SQL class — the import")\` — the tool resolves the runtime's real types) and emit the CANONICAL import the types actually prove, NOT the string copied from the task. Concretely: Bun's SQL client is \`import { sql } from "bun"\` (or \`Bun.sql\` / \`new SQL()\`) — there is NO \`bun:sql\` module. Never pass an unverified colon-specifier through to the APIS list; a phantom import laundered here becomes fabricated \`declare module\` shims in the implementation.`
116
+ : ''}
107
117
 
108
118
  APIS owns symbols and commands BY NAME ONLY. Do NOT include any file path or path fragment — no \`package.json\`, no \`./src/foo.ts\`, no \`package.json#scripts.lint\`. If the symbol is a script defined in package.json, write the invocation (\`npm run lint\`), not its location. If the symbol is a config file, it does not belong in APIS at all — it belongs in FILES.
109
119
 
@@ -2,24 +2,37 @@ import { createRequire } from 'node:module';
2
2
  import * as fs from 'node:fs';
3
3
  import * as path from 'node:path';
4
4
  import * as os from 'node:os';
5
+ /**
6
+ * Bump whenever the shape below changes in a way old rows cannot satisfy. The
7
+ * migration is a DROP and rebuild: this is a derived cache of package sources,
8
+ * so re-indexing costs a walk, and hand-written ALTERs cost a defect class.
9
+ */
10
+ const SCHEMA_VERSION = 1;
11
+ /**
12
+ * `ecosystem` scopes every row to the registry it came from, so `text` on npm and
13
+ * `text` on Hackage are different packages. Project-source rows use their own
14
+ * scope value rather than a registry id.
15
+ */
5
16
  const SCHEMA_SQL = `
6
17
  CREATE TABLE IF NOT EXISTS packages (
18
+ ecosystem TEXT NOT NULL DEFAULT 'npm',
7
19
  name TEXT NOT NULL,
8
20
  version TEXT NOT NULL,
9
21
  content_hash TEXT NOT NULL,
10
22
  indexed_at INTEGER NOT NULL,
11
- PRIMARY KEY (name, version)
23
+ PRIMARY KEY (ecosystem, name, version)
12
24
  );
13
25
 
14
26
  CREATE TABLE IF NOT EXISTS chunks (
15
27
  id INTEGER PRIMARY KEY AUTOINCREMENT,
28
+ ecosystem TEXT NOT NULL DEFAULT 'npm',
16
29
  name TEXT NOT NULL,
17
30
  version TEXT NOT NULL,
18
31
  file_path TEXT NOT NULL,
19
32
  kind TEXT NOT NULL CHECK (kind IN ('dts','readme')),
20
33
  content TEXT NOT NULL
21
34
  );
22
- CREATE INDEX IF NOT EXISTS chunks_pkg ON chunks(name, version);
35
+ CREATE INDEX IF NOT EXISTS chunks_pkg ON chunks(ecosystem, name, version);
23
36
 
24
37
  CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
25
38
  content,
@@ -34,6 +47,14 @@ CREATE TRIGGER IF NOT EXISTS chunks_ad AFTER DELETE ON chunks BEGIN
34
47
  INSERT INTO chunks_fts(chunks_fts, rowid, content) VALUES('delete', old.id, old.content);
35
48
  END;
36
49
  `;
50
+ // Order matters: the triggers and the FTS index reference `chunks`.
51
+ const DROP_SQL = `
52
+ DROP TRIGGER IF EXISTS chunks_ai;
53
+ DROP TRIGGER IF EXISTS chunks_ad;
54
+ DROP TABLE IF EXISTS chunks_fts;
55
+ DROP TABLE IF EXISTS chunks;
56
+ DROP TABLE IF EXISTS packages;
57
+ `;
37
58
  const req = createRequire(import.meta.url);
38
59
  // Branch on the RUNTIME, not on a try/catch: each builtin exists in exactly one
39
60
  // of them. Requiring `bun:sqlite` under node throws MODULE_NOT_FOUND, and
@@ -61,9 +82,35 @@ export function openCache(dbPath) {
61
82
  db.exec('PRAGMA journal_mode = WAL;');
62
83
  db.exec('PRAGMA synchronous = NORMAL;');
63
84
  db.exec('PRAGMA foreign_keys = ON;');
64
- db.exec(SCHEMA_SQL);
85
+ // A blocking busy handler, not a tuned delay: research children open this
86
+ // cache concurrently, and the loser of the migration lock has to WAIT for the
87
+ // winner's COMMIT rather than throw SQLITE_BUSY on the spot. The bound only
88
+ // stops a wedged handle hanging the worker forever.
89
+ db.exec('PRAGMA busy_timeout = 30000;');
90
+ migrate(db);
65
91
  return {
66
92
  db,
67
93
  close: () => db.close()
68
94
  };
69
95
  }
96
+ /**
97
+ * `CREATE TABLE IF NOT EXISTS` is a no-op against an older shape, so the version
98
+ * check has to run BEFORE it or a stale table survives untouched.
99
+ */
100
+ function migrate(db) {
101
+ db.exec('BEGIN IMMEDIATE');
102
+ try {
103
+ const row = db.prepare('PRAGMA user_version').get();
104
+ // A brand-new database also reads 0, hence DROP ... IF EXISTS.
105
+ if ((row?.user_version ?? 0) < SCHEMA_VERSION) {
106
+ db.exec(DROP_SQL);
107
+ db.exec(`PRAGMA user_version = ${SCHEMA_VERSION};`);
108
+ }
109
+ db.exec(SCHEMA_SQL);
110
+ db.exec('COMMIT');
111
+ }
112
+ catch (err) {
113
+ db.exec('ROLLBACK');
114
+ throw err;
115
+ }
116
+ }
@@ -53,15 +53,18 @@ export declare function splitAtMatches(text: string, re: RegExp): string[];
53
53
  */
54
54
  export declare function sliceBytes(s: string, maxBytes: number): string[];
55
55
  /**
56
- * Chunk a declaration file (`.d.ts`, `.ts`, `.tsx`), one chunk per declaration,
57
- * each labelled with the file it came from.
56
+ * Chunk a declaration file, one chunk per declaration, each labelled with the
57
+ * file it came from.
58
58
  *
59
59
  * `relPath` is a MODEL-FACING label and is used exactly as given. docs-index.ts
60
60
  * normalises it to POSIX (`.replace(/\\/g, '/')`) so a package index is identical
61
61
  * across platforms; docs-project.ts passes `path.relative` through with the native
62
62
  * separator. It is never re-joined to the filesystem, so neither is wrong — this
63
63
  * leaves the choice with the caller that has a reason for it.
64
+ *
65
+ * `splitRe` and `commentPrefix` default to the TypeScript pair, which is what
66
+ * both the project corpus and npm packages are written in.
64
67
  */
65
- export declare function chunkDeclarations(content: string, relPath: string): string[];
68
+ export declare function chunkDeclarations(content: string, relPath: string, splitRe?: RegExp, commentPrefix?: string): string[];
66
69
  /** Chunk a README, one chunk per top-level section, each labelled by heading. */
67
70
  export declare function chunkReadme(content: string): string[];
@@ -90,22 +90,25 @@ export function sliceBytes(s, maxBytes) {
90
90
  return out;
91
91
  }
92
92
  /**
93
- * Chunk a declaration file (`.d.ts`, `.ts`, `.tsx`), one chunk per declaration,
94
- * each labelled with the file it came from.
93
+ * Chunk a declaration file, one chunk per declaration, each labelled with the
94
+ * file it came from.
95
95
  *
96
96
  * `relPath` is a MODEL-FACING label and is used exactly as given. docs-index.ts
97
97
  * normalises it to POSIX (`.replace(/\\/g, '/')`) so a package index is identical
98
98
  * across platforms; docs-project.ts passes `path.relative` through with the native
99
99
  * separator. It is never re-joined to the filesystem, so neither is wrong — this
100
100
  * leaves the choice with the caller that has a reason for it.
101
+ *
102
+ * `splitRe` and `commentPrefix` default to the TypeScript pair, which is what
103
+ * both the project corpus and npm packages are written in.
101
104
  */
102
- export function chunkDeclarations(content, relPath) {
105
+ export function chunkDeclarations(content, relPath, splitRe = DECL_SPLIT_RE, commentPrefix = '//') {
103
106
  const chunks = [];
104
- for (const part of splitAtMatches(content, new RegExp(DECL_SPLIT_RE.source, 'gm'))) {
107
+ for (const part of splitAtMatches(content, new RegExp(splitRe.source, 'gm'))) {
105
108
  const trimmed = part.trim();
106
109
  if (!trimmed)
107
110
  continue;
108
- const prefixed = `// ${relPath}\n${trimmed}`;
111
+ const prefixed = `${commentPrefix} ${relPath}\n${trimmed}`;
109
112
  if (Buffer.byteLength(prefixed, 'utf8') > MAX_CHUNK_BYTES) {
110
113
  for (const slice of sliceBytes(prefixed, MAX_CHUNK_BYTES))
111
114
  chunks.push(slice);
@@ -1,5 +1,6 @@
1
1
  import { openCache as defaultOpenCache } from './docs-cache.js';
2
2
  import { ensureIndexed as defaultEnsureIndexed } from './docs-index.js';
3
+ import { type EcosystemId, type EcosystemIo, type EcosystemProfile } from './docs-ecosystems.js';
3
4
  import { resolvePackage as defaultResolvePackage, type ResolvedPackage } from './docs-resolve.js';
4
5
  import { retrieveChunks as defaultRetrieveChunks, type RetrievedChunk } from './docs-retrieve.js';
5
6
  import { npmVersionLookup as defaultNpmVersionLookup, type NpmVersionInfo } from './npm-version.js';
@@ -37,6 +38,8 @@ export type DocsRawResult = {
37
38
  autoInstalled?: boolean;
38
39
  autoInstallPin?: AutoInstallPin;
39
40
  npmVersion?: NpmVersionInfo | null;
41
+ /** The registry the answer came from, for the block that leads it. */
42
+ registryLabel?: string;
40
43
  } | {
41
44
  kind: 'no_chunks';
42
45
  pkg: ResolvedPackage;
@@ -46,10 +49,12 @@ export type DocsRawResult = {
46
49
  autoInstalled?: boolean;
47
50
  autoInstallPin?: AutoInstallPin;
48
51
  npmVersion?: NpmVersionInfo | null;
52
+ /** The registry the answer came from, for the block that leads it. */
53
+ registryLabel?: string;
49
54
  } | {
50
55
  kind: 'error';
51
56
  message: string;
52
- resolveError?: 'not_installed' | 'invalid_name';
57
+ resolveError?: 'not_installed' | 'invalid_name' | 'unsupported_ecosystem' | 'ambiguous_ecosystem';
53
58
  installError?: string;
54
59
  version?: string;
55
60
  hitCache?: boolean;
@@ -57,11 +62,15 @@ export type DocsRawResult = {
57
62
  autoInstalled?: boolean;
58
63
  autoInstallPin?: AutoInstallPin;
59
64
  npmVersion?: NpmVersionInfo | null;
65
+ /** The registry the answer came from, for the block that leads it. */
66
+ registryLabel?: string;
60
67
  };
61
68
  export interface DocsRawInput {
62
69
  pkg: string;
63
70
  query: string;
64
71
  cwd: string;
72
+ /** Which registry to read, for a repo holding more than one manifest. */
73
+ ecosystem?: EcosystemId;
65
74
  autoInstall?: boolean;
66
75
  resolvePackage?: typeof defaultResolvePackage;
67
76
  ensureIndexed?: typeof defaultEnsureIndexed;
@@ -69,6 +78,8 @@ export interface DocsRawInput {
69
78
  openCache?: typeof defaultOpenCache;
70
79
  spawn?: SpawnFn;
71
80
  npmVersionLookup?: typeof defaultNpmVersionLookup;
81
+ /** Overrides for the filesystem and network a non-npm row reaches through. */
82
+ io?: Partial<EcosystemIo>;
72
83
  signal?: AbortSignal;
73
84
  }
74
85
  export interface DocsFocusedResult {
@@ -162,7 +173,7 @@ export declare function findDeclaredRange(parentPkg: string, cwd: string): strin
162
173
  * `bun`, and cannot declare `bun-types`, so a sentence about what package.json
163
174
  * does or does not say has to be a sentence about `bun`.
164
175
  */
165
- export declare function buildVersionBanner(pin: AutoInstallPin | undefined, resolved: string, version: string, cwd: string): string;
176
+ export declare function buildVersionBanner(pin: AutoInstallPin | undefined, resolved: string, version: string, cwd: string, profile?: EcosystemProfile): string;
166
177
  export declare function getDocsModulesDir(): string;
167
178
  export declare function ensureDocsModulesDir(dir: string): void;
168
179
  /**
@@ -207,8 +218,13 @@ export interface AcquireInput {
207
218
  name: string;
208
219
  cwd: string;
209
220
  spawn: SpawnFn;
210
- resolvePackage: typeof defaultResolvePackage;
221
+ /** Overrides `profile.resolve` when given, for callers that inject a resolver. */
222
+ resolvePackage?: typeof defaultResolvePackage;
211
223
  signal: AbortSignal | undefined;
224
+ /** Which registry to acquire from. */
225
+ profile?: EcosystemProfile;
226
+ /** The row's filesystem and network reach. Built from `spawn` when absent. */
227
+ io?: EcosystemIo;
212
228
  }
213
229
  /**
214
230
  * Get a package onto disk and resolved: resolve from `cwd`, and on
@@ -231,6 +247,14 @@ export interface AcquireInput {
231
247
  * sites should differ only about WHETHER to install, never about HOW.
232
248
  */
233
249
  export declare function acquirePackage(input: AcquireInput): Promise<AcquireOutcome>;
250
+ /** The docs pipeline's adapter over the shared redirect walk (docs-resolve.ts):
251
+ * hops resolve through the auto-installing lookup, so a declaration package that is
252
+ * declared but not yet on disk is fetched rather than abandoned. */
253
+ export declare function resolveTypeSourceForDocs(pkg: ResolvedPackage, requested: string, cwd: string, spawn: SpawnFn, resolvePackage: typeof defaultResolvePackage, signal: AbortSignal | undefined): Promise<{
254
+ pkg: ResolvedPackage;
255
+ installed: boolean;
256
+ pin?: AutoInstallPin;
257
+ }>;
234
258
  export declare function docsRaw(input: DocsRawInput): Promise<DocsRawResult>;
235
259
  export declare function docsFocused(input: DocsFocusedInput): Promise<DocsFocusedResult>;
236
260
  export declare function buildPrompt(pkg: ResolvedPackage, query: string, content: string): string;