@mjasnikovs/pi-task 0.39.5 → 0.40.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -3
- package/dist/task/context-attribution.js +18 -6
- package/dist/task/external-context.d.ts +8 -1
- package/dist/task/external-context.js +52 -4
- package/dist/task/phases.js +2 -1
- package/dist/task/prompts.d.ts +7 -1
- package/dist/task/prompts.js +14 -4
- package/dist/workers/docs-cache.js +50 -3
- package/dist/workers/docs-chunk.d.ts +6 -3
- package/dist/workers/docs-chunk.js +8 -5
- package/dist/workers/docs-core.d.ts +27 -3
- package/dist/workers/docs-core.js +104 -41
- package/dist/workers/docs-ecosystems.d.ts +173 -0
- package/dist/workers/docs-ecosystems.js +449 -0
- package/dist/workers/docs-index.d.ts +2 -1
- package/dist/workers/docs-index.js +55 -27
- package/dist/workers/docs-project.d.ts +10 -0
- package/dist/workers/docs-project.js +86 -24
- package/dist/workers/docs-resolve.d.ts +6 -1
- package/dist/workers/docs-resolve.js +4 -3
- package/dist/workers/docs-retrieve.d.ts +2 -0
- package/dist/workers/docs-retrieve.js +11 -11
- package/dist/workers/eco-cargo.d.ts +115 -0
- package/dist/workers/eco-cargo.js +793 -0
- package/dist/workers/eco-hackage.d.ts +93 -0
- package/dist/workers/eco-hackage.js +508 -0
- package/dist/workers/npm-version.d.ts +5 -3
- package/dist/workers/npm-version.js +6 -4
- package/dist/workers/pi-worker-docs.d.ts +18 -4
- package/dist/workers/pi-worker-docs.js +57 -19
- package/dist/workers/research-cache.d.ts +2 -13
- package/dist/workers/research-cache.js +22 -46
- package/dist/workers/shared.d.ts +16 -5
- package/dist/workers/shared.js +0 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -191,12 +191,25 @@ Fetches a URL, cleans HTML to markdown ([Readability](https://github.com/mozilla
|
|
|
191
191
|
- The extraction child runs with `--no-tools` to mitigate visible-text prompt injection.
|
|
192
192
|
|
|
193
193
|
### `pi-worker-docs`
|
|
194
|
-
Resolves an installed
|
|
194
|
+
Resolves an installed package, indexes its API surface and README into a local SQLite cache, retrieves the most relevant chunks for your `query`, and passes them to an isolated child that extracts the focused answer. Version-pinned to whatever the project actually resolved.
|
|
195
195
|
|
|
196
|
-
|
|
197
|
-
|
|
196
|
+
**The manifest decides which registry, not the model.** `text`, `base`, `aeson`, `tokio` and `clap` are all real npm packages *and* real Rust/Haskell ones, so a name alone cannot say which was meant — and guessing npm returns a confident answer about an unrelated package. That is a wrong answer, not a miss.
|
|
197
|
+
|
|
198
|
+
| Ecosystem | Detected by | Surface read | Version comes from |
|
|
199
|
+
| --- | --- | --- | --- |
|
|
200
|
+
| `npm` | `package.json`, or a `node_modules/` directory | the `.d.ts` files the package ships, plus README | the installed `package.json` |
|
|
201
|
+
| `cargo` | `Cargo.toml`, at the directory or one level below it | `.rs` source reduced to public item heads, doc comments and attributes | `Cargo.lock` |
|
|
202
|
+
| `hackage` | `*.cabal`, `cabal.project`, `stack.yaml` or `package.yaml` | `.hs` source reduced to the export list, signatures and type declarations | `dist-newstyle/cache/plan.json`, then `cabal.project.freeze`, then `stack.yaml.lock` |
|
|
203
|
+
|
|
204
|
+
- **No manifest, no lookup.** In a directory with none of the above the tool refuses, spawns nothing and installs nothing, and points you at `pi-worker-search` / `pi-worker-fetch` instead.
|
|
205
|
+
- **Two manifests** (a Tauri app, say) are resolved by whichever registry already has the package on disk. If neither does, the call is refused as ambiguous and you pass `ecosystem: "cargo"` to say which.
|
|
206
|
+
- A package the project does not have is fetched once into a dedicated cache dir: `npm install --ignore-scripts` for npm, the `.crate` tarball for cargo, the Hackage tarball (or cabal's own cached copy) for hackage.
|
|
207
|
+
- A Haskell **module** name is refused by name — `Data.Aeson` is not a package, `aeson` is.
|
|
208
|
+
- The first call for a `(ecosystem, package, version)` triple pays a one-time ingestion cost; later calls are FTS-only.
|
|
198
209
|
- Cache lives at `${XDG_CACHE_HOME:-~/.cache}/pi-worker/docs.sqlite` — delete it to reset.
|
|
199
210
|
|
|
211
|
+
**Known gaps.** The rest of pi-task is still npm-shaped: the final gate, repo-health and orientation read the working directory first-wins, so a Tauri repo gets cargo docs answers while the gate runs `bun run test`, not `cargo test`. Phantom-import checking, dependency-name extraction for research enrichment, and the `@types/…` redirect chain are npm concepts and are no-ops elsewhere. Adding an ecosystem is one row in `src/workers/docs-ecosystems.ts` plus its parsers — profiles live in code and arrive as pull requests with tests, never as user configuration.
|
|
212
|
+
|
|
200
213
|
## Settings — `/task-config`
|
|
201
214
|
|
|
202
215
|
Run `/task-config` to toggle pi-task's behavior in an editor dialog. Settings persist to `~/.config/pi-task/config.json`.
|
|
@@ -226,6 +239,8 @@ Run `/task-config` to toggle pi-task's behavior in an editor dialog. Settings pe
|
|
|
226
239
|
| --- | --- | --- |
|
|
227
240
|
| `BRAVE_SEARCH_API_KEY` / `BRAVE_API_KEY` | `pi-worker-search`, research enrichment | Required only when the **Brave** search engine is selected in `/task-config`. |
|
|
228
241
|
| `XDG_CACHE_HOME` | `pi-worker-docs` | Overrides the docs cache location (defaults to `~/.cache`). |
|
|
242
|
+
| `CARGO_HOME` | `pi-worker-docs` | Where crate source checkouts are read from (defaults to `~/.cargo`). |
|
|
243
|
+
| `CABAL_DIR` | `pi-worker-docs` | Where cabal's downloaded package tarballs are read from (also checks `~/.cabal/packages` and `${XDG_CACHE_HOME:-~/.cache}/cabal/packages`). |
|
|
229
244
|
| `XDG_DATA_HOME` | remote push | Where the VAPID keypair is stored (defaults to `~/.local/share`). |
|
|
230
245
|
| `PI_REMOTE_PUSH_SUBJECT` | remote push | VAPID JWT `sub` contact. Defaults to the project URL; set your own `mailto:you@domain.com` or `https://…`. |
|
|
231
246
|
| `PI_REMOTE_PUSH_DEBUG` | remote push | When set (e.g. `1`), logs push delivery and push-service HTTP status. Off by default. |
|
|
@@ -154,17 +154,29 @@ export function splitBulletSpans(context) {
|
|
|
154
154
|
export function splitBullets(context) {
|
|
155
155
|
return splitBulletSpans(context).map(s => s.text);
|
|
156
156
|
}
|
|
157
|
+
/**
|
|
158
|
+
* A version block is headed by its REGISTRY, so the heading is open-ended
|
|
159
|
+
* (`crates.io`, `hackage`). Anything else is one: the widened pattern also reads
|
|
160
|
+
* headings out of a retrieved README or page BODY, and calling those `docs` would
|
|
161
|
+
* let a package's own text forge the block that sources a semantics claim.
|
|
162
|
+
*/
|
|
163
|
+
function blockKind(heading) {
|
|
164
|
+
if (heading === 'freshness-check')
|
|
165
|
+
return 'freshness-skipped';
|
|
166
|
+
if (heading === 'docs' || heading === 'url' || heading === 'service')
|
|
167
|
+
return heading;
|
|
168
|
+
return 'npm';
|
|
169
|
+
}
|
|
157
170
|
/** Parse the `### npm:` / `### docs:` / `### url:` / `### service:` blocks out of an EXTERNAL CONTEXT header. */
|
|
158
171
|
export function parseContextBlocks(externalContext) {
|
|
159
172
|
const out = [];
|
|
160
|
-
|
|
173
|
+
// A version block is headed by its REGISTRY, so `npm` is one of several
|
|
174
|
+
// (`crates.io`, `hackage`). Matching only the fixed words made every non-npm
|
|
175
|
+
// block invisible to the parser.
|
|
176
|
+
const re = /^###\s+([A-Za-z][\w.-]*)\s*:?\s*(.*)$/gim;
|
|
161
177
|
let m;
|
|
162
178
|
while ((m = re.exec(externalContext)) !== null) {
|
|
163
|
-
|
|
164
|
-
out.push({
|
|
165
|
-
kind: kind === 'freshness-check' ? 'freshness-skipped' : kind,
|
|
166
|
-
subject: m[2].trim()
|
|
167
|
-
});
|
|
179
|
+
out.push({ kind: blockKind(m[1].toLowerCase()), subject: m[2].trim() });
|
|
168
180
|
}
|
|
169
181
|
return out;
|
|
170
182
|
}
|
|
@@ -36,6 +36,8 @@ type GatherDeps = Pick<PhaseDeps, 'cwd' | 'signal' | 'recordSubStep' | 'docsRaw'
|
|
|
36
36
|
export interface ExternalTargetResult {
|
|
37
37
|
/** Emitted as an `### npm:` block ahead of every body. Absent for url targets. */
|
|
38
38
|
npmVersion?: NpmVersionInfo | null;
|
|
39
|
+
/** Which registry the version block came from; npm when absent. */
|
|
40
|
+
registryLabel?: string;
|
|
39
41
|
/**
|
|
40
42
|
* The `### docs:`/`### url:` body. `undefined` means "this target contributes no
|
|
41
43
|
* body block" — and the two call paths draw that line differently ON PURPOSE:
|
|
@@ -68,6 +70,11 @@ export interface ExternalContextLookups {
|
|
|
68
70
|
* question, in front of a waiting user, while research is uncapped and trails its
|
|
69
71
|
* sub-step.
|
|
70
72
|
*/
|
|
73
|
+
/** A live version answer and the registry it came from, for the block heading. */
|
|
74
|
+
export interface VersionBlock {
|
|
75
|
+
info: NpmVersionInfo;
|
|
76
|
+
label: string;
|
|
77
|
+
}
|
|
71
78
|
export interface ExternalContextPolicy {
|
|
72
79
|
/**
|
|
73
80
|
* Max combined docs+url targets fanned out, packages first. Omit for uncapped
|
|
@@ -83,7 +90,7 @@ export interface ExternalContextPolicy {
|
|
|
83
90
|
* version at all, and a "which version?" question falls back to whatever the
|
|
84
91
|
* model remembers — which is how a dependency gets pinned to a stale major.
|
|
85
92
|
*/
|
|
86
|
-
versionLookup?: (pkg: string) => Promise<
|
|
93
|
+
versionLookup?: (pkg: string) => Promise<VersionBlock | null>;
|
|
87
94
|
/** Sub-step label recorded via `deps.recordSubStep`. Omit to record nothing. */
|
|
88
95
|
subStepLabel?: string;
|
|
89
96
|
/**
|
|
@@ -22,6 +22,7 @@
|
|
|
22
22
|
* Run against a mixed source, the emitted headings come out in exactly that order:
|
|
23
23
|
* `### npm:` then `### docs:` then `### url:` then `### service:`.
|
|
24
24
|
*/
|
|
25
|
+
import { chooseEcosystem, defaultEcosystemIo } from '../workers/docs-ecosystems.js';
|
|
25
26
|
import { docsRaw } from '../workers/docs-core.js';
|
|
26
27
|
import { fetchRaw } from '../workers/fetch-core.js';
|
|
27
28
|
import { formatNpmVersionSection, npmVersionLookup } from '../workers/npm-version.js';
|
|
@@ -61,6 +62,14 @@ export async function buildExternalContext(source, deps, lookups, policy = {}) {
|
|
|
61
62
|
}).catch(() => null))),
|
|
62
63
|
Promise.all(versionLookup ? extraVersionPkgs.map(pkg => versionLookup(pkg).catch(() => null)) : [])
|
|
63
64
|
]);
|
|
65
|
+
// A docs target that came back WITHOUT a version — a refused ecosystem, a dead
|
|
66
|
+
// registry — was dropped from `extraVersionPkgs` on the assumption docs would
|
|
67
|
+
// supply it. Ask for it now, or its block is silently lost.
|
|
68
|
+
const missedVersionPkgs = versionLookup ?
|
|
69
|
+
enrichTargets.versionPackages.filter(p => docsTargets.has(p)
|
|
70
|
+
&& !targets.some((t, i) => t.name === p && targetResults[i]?.npmVersion))
|
|
71
|
+
: [];
|
|
72
|
+
const missedVersionResults = await Promise.all(missedVersionPkgs.map(pkg => versionLookup(pkg).catch(() => null)));
|
|
64
73
|
const sections = [];
|
|
65
74
|
// npm version blocks lead the section so the model anchors on live version
|
|
66
75
|
// data before reading any docs body. The docs-fetched packages carry their
|
|
@@ -69,11 +78,11 @@ export async function buildExternalContext(source, deps, lookups, policy = {}) {
|
|
|
69
78
|
// named dep.
|
|
70
79
|
for (const r of targetResults) {
|
|
71
80
|
if (r?.npmVersion)
|
|
72
|
-
sections.push(formatNpmVersionSection(r.npmVersion));
|
|
81
|
+
sections.push(formatNpmVersionSection(r.npmVersion, r.registryLabel));
|
|
73
82
|
}
|
|
74
|
-
for (const v of extraVersionResults) {
|
|
83
|
+
for (const v of [...extraVersionResults, ...missedVersionResults]) {
|
|
75
84
|
if (v)
|
|
76
|
-
sections.push(formatNpmVersionSection(v));
|
|
85
|
+
sections.push(formatNpmVersionSection(v.info, v.label));
|
|
77
86
|
}
|
|
78
87
|
for (let i = 0; i < targets.length; i++) {
|
|
79
88
|
const body = targetResults[i]?.body;
|
|
@@ -118,6 +127,44 @@ export async function gatherExternalContext(refined, deps) {
|
|
|
118
127
|
const fetchRawFn = deps.fetchRaw ?? fetchRaw;
|
|
119
128
|
const npmVersionFn = deps.npmVersionLookup ?? npmVersionLookup;
|
|
120
129
|
const docsQuery = refined.split('\n').find(l => l.trim()) ?? refined;
|
|
130
|
+
// Which registry to ask, decided PER PACKAGE.
|
|
131
|
+
//
|
|
132
|
+
// A cargo-only project's dependency names are crate names, and asking npm
|
|
133
|
+
// about them returns a real but unrelated package's versions — the exact
|
|
134
|
+
// confusion the docs tool refuses. But deciding once for the whole repo makes
|
|
135
|
+
// every polyglot project ambiguous, and then NO version block is emitted at
|
|
136
|
+
// all: worse than the old always-npm, because the prompts tell the model to
|
|
137
|
+
// quote a block that will never be there. Each name carries its own evidence,
|
|
138
|
+
// so each name gets its own answer.
|
|
139
|
+
const io = defaultEcosystemIo(deps.signal ? { signal: deps.signal } : {});
|
|
140
|
+
const askNpm = async (pkg) => {
|
|
141
|
+
const info = await npmVersionFn(pkg, { signal: deps.signal });
|
|
142
|
+
return info ? { info, label: 'npm' } : null;
|
|
143
|
+
};
|
|
144
|
+
const versionLookup = async (pkg) => {
|
|
145
|
+
const choice = chooseEcosystem({
|
|
146
|
+
cwd: deps.cwd,
|
|
147
|
+
declaresPackage: p => p.declaredRange(p.parentPackage(pkg), deps.cwd) !== null,
|
|
148
|
+
resolvesLocally: p => {
|
|
149
|
+
try {
|
|
150
|
+
p.resolve(pkg, deps.cwd, io);
|
|
151
|
+
return true;
|
|
152
|
+
}
|
|
153
|
+
catch {
|
|
154
|
+
return false;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
});
|
|
158
|
+
// No manifest at all keeps npm, which is what a bare directory has always
|
|
159
|
+
// done. A name genuinely ambiguous between two registries gets no block —
|
|
160
|
+
// a version from the wrong registry is worse than none.
|
|
161
|
+
if (!choice.ok)
|
|
162
|
+
return choice.reason === 'none' ? askNpm(pkg) : null;
|
|
163
|
+
if (choice.profile.id === 'npm')
|
|
164
|
+
return askNpm(pkg);
|
|
165
|
+
const info = await choice.profile.latest(pkg, io);
|
|
166
|
+
return info ? { info, label: choice.profile.registryLabel } : null;
|
|
167
|
+
};
|
|
121
168
|
return buildExternalContext(refined, deps, {
|
|
122
169
|
docs: async (pkg) => {
|
|
123
170
|
const r = await docsRawFn({
|
|
@@ -128,6 +175,7 @@ export async function gatherExternalContext(refined, deps) {
|
|
|
128
175
|
});
|
|
129
176
|
return {
|
|
130
177
|
npmVersion: r.npmVersion,
|
|
178
|
+
...(r.registryLabel ? { registryLabel: r.registryLabel } : {}),
|
|
131
179
|
body: r.kind === 'ok' && r.chunks.length > 0 ?
|
|
132
180
|
r.chunks
|
|
133
181
|
.map(c => c.content)
|
|
@@ -142,7 +190,7 @@ export async function gatherExternalContext(refined, deps) {
|
|
|
142
190
|
},
|
|
143
191
|
search: deps.searchFn
|
|
144
192
|
}, {
|
|
145
|
-
versionLookup
|
|
193
|
+
versionLookup,
|
|
146
194
|
subStepLabel: 'enrichment',
|
|
147
195
|
earlyReturnOnNoTargets: true
|
|
148
196
|
});
|
package/dist/task/phases.js
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
*/
|
|
5
5
|
import { fileURLToPath } from 'node:url';
|
|
6
6
|
import { docsFocused } from '../workers/docs-core.js';
|
|
7
|
+
import { detectEcosystems } from '../workers/docs-ecosystems.js';
|
|
7
8
|
import { fetchFocused } from '../workers/fetch-core.js';
|
|
8
9
|
import { runWorker } from '../workers/pi-worker-core.js';
|
|
9
10
|
import { findPhantomImports, formatApiCorrections, rewritePhantomSpecifiers } from '../workers/phantom-imports.js';
|
|
@@ -550,7 +551,7 @@ export async function phaseResearch(deps, refined) {
|
|
|
550
551
|
// queries the FILES worker just answered.
|
|
551
552
|
prompt: prior => orientation.block
|
|
552
553
|
+ promptHeader
|
|
553
|
-
+ RESEARCH_APIS_PROMPT(refined, prior.find(s => s.name === 'FILES')?.text || undefined)
|
|
554
|
+
+ RESEARCH_APIS_PROMPT(refined, prior.find(s => s.name === 'FILES')?.text || undefined, detectEcosystems(deps.cwd))
|
|
554
555
|
+ (searchConfigured() ? RESEARCH_SEARCH_HINT : '')
|
|
555
556
|
// Empty unless PI_TASK_PROJECT_DOCS_BUDGET is set. The tool-side
|
|
556
557
|
// half lives in pi-worker-docs.ts; a budget enforced without being
|
package/dist/task/prompts.d.ts
CHANGED
|
@@ -26,7 +26,13 @@ export declare const COMPRESS_LABEL_PROMPT: (title: string, maxChars: number) =>
|
|
|
26
26
|
declare const REFINE_PROMPT: (raw: string, planContext?: string, existingFiles?: string, contracts?: string, directives?: string) => string;
|
|
27
27
|
declare const RESEARCH_READ_ONLY_CONSTRAINT = "IMPORTANT: You are ONLY allowed to READ. Do NOT create, modify, or delete any files. Use the read, grep, find, and ls tools to inspect the repo.";
|
|
28
28
|
declare const RESEARCH_FILES_PROMPT: (refined: string) => string;
|
|
29
|
-
|
|
29
|
+
/**
|
|
30
|
+
* `ecosystems` is what `detectEcosystems` found in the worker's cwd. It is REQUIRED,
|
|
31
|
+
* and an empty list is a real answer: a directory with no package manifest has no
|
|
32
|
+
* registry to look a library up in, and the paragraph below has to say so rather
|
|
33
|
+
* than inviting a lookup that will be refused.
|
|
34
|
+
*/
|
|
35
|
+
declare const RESEARCH_APIS_PROMPT: (refined: string, filesMap: string | undefined, ecosystems: readonly string[]) => string;
|
|
30
36
|
declare const RESEARCH_CONTEXT_PROMPT: (refined: string) => string;
|
|
31
37
|
declare const RESEARCH_TOOLING_PROMPT: (refined: string) => string;
|
|
32
38
|
declare const GRILL_GEN_PROMPT: (refined: string, research: string, priorQA: string) => string;
|
package/dist/task/prompts.js
CHANGED
|
@@ -97,13 +97,23 @@ No section header. No other sections. No preamble.
|
|
|
97
97
|
|
|
98
98
|
Task:
|
|
99
99
|
${refined}`;
|
|
100
|
-
|
|
100
|
+
/**
|
|
101
|
+
* `ecosystems` is what `detectEcosystems` found in the worker's cwd. It is REQUIRED,
|
|
102
|
+
* and an empty list is a real answer: a directory with no package manifest has no
|
|
103
|
+
* registry to look a library up in, and the paragraph below has to say so rather
|
|
104
|
+
* than inviting a lookup that will be refused.
|
|
105
|
+
*/
|
|
106
|
+
const RESEARCH_APIS_PROMPT = (refined, filesMap, ecosystems) => `You are doing targeted research for an AI coding agent. Use the read, grep, find, and ls tools${ecosystems.length ? ' — and `pi-worker-docs` for installed packages —' : ''} to identify the commands, functions, types, and interfaces the agent will use for the following task.
|
|
101
107
|
|
|
102
|
-
|
|
108
|
+
${ecosystems.length ?
|
|
109
|
+
`LIBRARY PACKAGES — use pi-worker-docs, NOT file reads: for any third-party package, call \`pi-worker-docs(module, query)\` to get its type signatures and API surface. Do NOT open installed-package source files directly — those reads are expensive and produce far more noise than the tool. The tool returns a compact, focused excerpt in a fraction of the token cost. This project's package ecosystems are: ${ecosystems.join(', ')}.`
|
|
110
|
+
: 'LIBRARY PACKAGES — do NOT look them up: this directory holds no package manifest, so `pi-worker-docs` has no registry to read and will refuse. Do not list an external library API you cannot check here.'}
|
|
103
111
|
|
|
104
|
-
PROJECT SOURCE — use pi-worker-docs with module ".", NOT file reads: for any function, class, type, or interface defined in THIS project's own
|
|
112
|
+
PROJECT SOURCE — use pi-worker-docs with module ".", NOT file reads: for any function, class, type, or interface defined in THIS project's own source (e.g. "what does requireAuth check?", "what does CreateListingSchema look like?", "what does the listings query module export?"), call \`pi-worker-docs(".", query)\` instead of reading the file. The tool indexes all git-tracked source files and returns only the relevant chunks — far cheaper than reading whole files.
|
|
105
113
|
|
|
106
|
-
|
|
114
|
+
${ecosystems.includes('npm') ?
|
|
115
|
+
`RUNTIME BUILTINS — verify, do NOT echo: a task (or the spec doc it references) may name a runtime/builtin import like \`bun:sql\`, \`bun:sqlite\`, \`node:fs\`, or \`Bun.password\`. A runtime exposes only a small FIXED set of \`<runtime>:<submodule>\` modules, and a spec doc can confidently name one that does not exist. Before you list ANY \`<pkg>:<sub>\` specifier, confirm it with \`pi-worker-docs\` (e.g. \`pi-worker-docs("bun:sql", "sql tagged template and SQL class — the import")\` — the tool resolves the runtime's real types) and emit the CANONICAL import the types actually prove, NOT the string copied from the task. Concretely: Bun's SQL client is \`import { sql } from "bun"\` (or \`Bun.sql\` / \`new SQL()\`) — there is NO \`bun:sql\` module. Never pass an unverified colon-specifier through to the APIS list; a phantom import laundered here becomes fabricated \`declare module\` shims in the implementation.`
|
|
116
|
+
: ''}
|
|
107
117
|
|
|
108
118
|
APIS owns symbols and commands BY NAME ONLY. Do NOT include any file path or path fragment — no \`package.json\`, no \`./src/foo.ts\`, no \`package.json#scripts.lint\`. If the symbol is a script defined in package.json, write the invocation (\`npm run lint\`), not its location. If the symbol is a config file, it does not belong in APIS at all — it belongs in FILES.
|
|
109
119
|
|
|
@@ -2,24 +2,37 @@ import { createRequire } from 'node:module';
|
|
|
2
2
|
import * as fs from 'node:fs';
|
|
3
3
|
import * as path from 'node:path';
|
|
4
4
|
import * as os from 'node:os';
|
|
5
|
+
/**
|
|
6
|
+
* Bump whenever the shape below changes in a way old rows cannot satisfy. The
|
|
7
|
+
* migration is a DROP and rebuild: this is a derived cache of package sources,
|
|
8
|
+
* so re-indexing costs a walk, and hand-written ALTERs cost a defect class.
|
|
9
|
+
*/
|
|
10
|
+
const SCHEMA_VERSION = 1;
|
|
11
|
+
/**
|
|
12
|
+
* `ecosystem` scopes every row to the registry it came from, so `text` on npm and
|
|
13
|
+
* `text` on Hackage are different packages. Project-source rows use their own
|
|
14
|
+
* scope value rather than a registry id.
|
|
15
|
+
*/
|
|
5
16
|
const SCHEMA_SQL = `
|
|
6
17
|
CREATE TABLE IF NOT EXISTS packages (
|
|
18
|
+
ecosystem TEXT NOT NULL DEFAULT 'npm',
|
|
7
19
|
name TEXT NOT NULL,
|
|
8
20
|
version TEXT NOT NULL,
|
|
9
21
|
content_hash TEXT NOT NULL,
|
|
10
22
|
indexed_at INTEGER NOT NULL,
|
|
11
|
-
PRIMARY KEY (name, version)
|
|
23
|
+
PRIMARY KEY (ecosystem, name, version)
|
|
12
24
|
);
|
|
13
25
|
|
|
14
26
|
CREATE TABLE IF NOT EXISTS chunks (
|
|
15
27
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
28
|
+
ecosystem TEXT NOT NULL DEFAULT 'npm',
|
|
16
29
|
name TEXT NOT NULL,
|
|
17
30
|
version TEXT NOT NULL,
|
|
18
31
|
file_path TEXT NOT NULL,
|
|
19
32
|
kind TEXT NOT NULL CHECK (kind IN ('dts','readme')),
|
|
20
33
|
content TEXT NOT NULL
|
|
21
34
|
);
|
|
22
|
-
CREATE INDEX IF NOT EXISTS chunks_pkg ON chunks(name, version);
|
|
35
|
+
CREATE INDEX IF NOT EXISTS chunks_pkg ON chunks(ecosystem, name, version);
|
|
23
36
|
|
|
24
37
|
CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
|
|
25
38
|
content,
|
|
@@ -34,6 +47,14 @@ CREATE TRIGGER IF NOT EXISTS chunks_ad AFTER DELETE ON chunks BEGIN
|
|
|
34
47
|
INSERT INTO chunks_fts(chunks_fts, rowid, content) VALUES('delete', old.id, old.content);
|
|
35
48
|
END;
|
|
36
49
|
`;
|
|
50
|
+
// Order matters: the triggers and the FTS index reference `chunks`.
|
|
51
|
+
const DROP_SQL = `
|
|
52
|
+
DROP TRIGGER IF EXISTS chunks_ai;
|
|
53
|
+
DROP TRIGGER IF EXISTS chunks_ad;
|
|
54
|
+
DROP TABLE IF EXISTS chunks_fts;
|
|
55
|
+
DROP TABLE IF EXISTS chunks;
|
|
56
|
+
DROP TABLE IF EXISTS packages;
|
|
57
|
+
`;
|
|
37
58
|
const req = createRequire(import.meta.url);
|
|
38
59
|
// Branch on the RUNTIME, not on a try/catch: each builtin exists in exactly one
|
|
39
60
|
// of them. Requiring `bun:sqlite` under node throws MODULE_NOT_FOUND, and
|
|
@@ -61,9 +82,35 @@ export function openCache(dbPath) {
|
|
|
61
82
|
db.exec('PRAGMA journal_mode = WAL;');
|
|
62
83
|
db.exec('PRAGMA synchronous = NORMAL;');
|
|
63
84
|
db.exec('PRAGMA foreign_keys = ON;');
|
|
64
|
-
|
|
85
|
+
// A blocking busy handler, not a tuned delay: research children open this
|
|
86
|
+
// cache concurrently, and the loser of the migration lock has to WAIT for the
|
|
87
|
+
// winner's COMMIT rather than throw SQLITE_BUSY on the spot. The bound only
|
|
88
|
+
// stops a wedged handle hanging the worker forever.
|
|
89
|
+
db.exec('PRAGMA busy_timeout = 30000;');
|
|
90
|
+
migrate(db);
|
|
65
91
|
return {
|
|
66
92
|
db,
|
|
67
93
|
close: () => db.close()
|
|
68
94
|
};
|
|
69
95
|
}
|
|
96
|
+
/**
|
|
97
|
+
* `CREATE TABLE IF NOT EXISTS` is a no-op against an older shape, so the version
|
|
98
|
+
* check has to run BEFORE it or a stale table survives untouched.
|
|
99
|
+
*/
|
|
100
|
+
function migrate(db) {
|
|
101
|
+
db.exec('BEGIN IMMEDIATE');
|
|
102
|
+
try {
|
|
103
|
+
const row = db.prepare('PRAGMA user_version').get();
|
|
104
|
+
// A brand-new database also reads 0, hence DROP ... IF EXISTS.
|
|
105
|
+
if ((row?.user_version ?? 0) < SCHEMA_VERSION) {
|
|
106
|
+
db.exec(DROP_SQL);
|
|
107
|
+
db.exec(`PRAGMA user_version = ${SCHEMA_VERSION};`);
|
|
108
|
+
}
|
|
109
|
+
db.exec(SCHEMA_SQL);
|
|
110
|
+
db.exec('COMMIT');
|
|
111
|
+
}
|
|
112
|
+
catch (err) {
|
|
113
|
+
db.exec('ROLLBACK');
|
|
114
|
+
throw err;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
@@ -53,15 +53,18 @@ export declare function splitAtMatches(text: string, re: RegExp): string[];
|
|
|
53
53
|
*/
|
|
54
54
|
export declare function sliceBytes(s: string, maxBytes: number): string[];
|
|
55
55
|
/**
|
|
56
|
-
* Chunk a declaration file
|
|
57
|
-
*
|
|
56
|
+
* Chunk a declaration file, one chunk per declaration, each labelled with the
|
|
57
|
+
* file it came from.
|
|
58
58
|
*
|
|
59
59
|
* `relPath` is a MODEL-FACING label and is used exactly as given. docs-index.ts
|
|
60
60
|
* normalises it to POSIX (`.replace(/\\/g, '/')`) so a package index is identical
|
|
61
61
|
* across platforms; docs-project.ts passes `path.relative` through with the native
|
|
62
62
|
* separator. It is never re-joined to the filesystem, so neither is wrong — this
|
|
63
63
|
* leaves the choice with the caller that has a reason for it.
|
|
64
|
+
*
|
|
65
|
+
* `splitRe` and `commentPrefix` default to the TypeScript pair, which is what
|
|
66
|
+
* both the project corpus and npm packages are written in.
|
|
64
67
|
*/
|
|
65
|
-
export declare function chunkDeclarations(content: string, relPath: string): string[];
|
|
68
|
+
export declare function chunkDeclarations(content: string, relPath: string, splitRe?: RegExp, commentPrefix?: string): string[];
|
|
66
69
|
/** Chunk a README, one chunk per top-level section, each labelled by heading. */
|
|
67
70
|
export declare function chunkReadme(content: string): string[];
|
|
@@ -90,22 +90,25 @@ export function sliceBytes(s, maxBytes) {
|
|
|
90
90
|
return out;
|
|
91
91
|
}
|
|
92
92
|
/**
|
|
93
|
-
* Chunk a declaration file
|
|
94
|
-
*
|
|
93
|
+
* Chunk a declaration file, one chunk per declaration, each labelled with the
|
|
94
|
+
* file it came from.
|
|
95
95
|
*
|
|
96
96
|
* `relPath` is a MODEL-FACING label and is used exactly as given. docs-index.ts
|
|
97
97
|
* normalises it to POSIX (`.replace(/\\/g, '/')`) so a package index is identical
|
|
98
98
|
* across platforms; docs-project.ts passes `path.relative` through with the native
|
|
99
99
|
* separator. It is never re-joined to the filesystem, so neither is wrong — this
|
|
100
100
|
* leaves the choice with the caller that has a reason for it.
|
|
101
|
+
*
|
|
102
|
+
* `splitRe` and `commentPrefix` default to the TypeScript pair, which is what
|
|
103
|
+
* both the project corpus and npm packages are written in.
|
|
101
104
|
*/
|
|
102
|
-
export function chunkDeclarations(content, relPath) {
|
|
105
|
+
export function chunkDeclarations(content, relPath, splitRe = DECL_SPLIT_RE, commentPrefix = '//') {
|
|
103
106
|
const chunks = [];
|
|
104
|
-
for (const part of splitAtMatches(content, new RegExp(
|
|
107
|
+
for (const part of splitAtMatches(content, new RegExp(splitRe.source, 'gm'))) {
|
|
105
108
|
const trimmed = part.trim();
|
|
106
109
|
if (!trimmed)
|
|
107
110
|
continue;
|
|
108
|
-
const prefixed =
|
|
111
|
+
const prefixed = `${commentPrefix} ${relPath}\n${trimmed}`;
|
|
109
112
|
if (Buffer.byteLength(prefixed, 'utf8') > MAX_CHUNK_BYTES) {
|
|
110
113
|
for (const slice of sliceBytes(prefixed, MAX_CHUNK_BYTES))
|
|
111
114
|
chunks.push(slice);
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { openCache as defaultOpenCache } from './docs-cache.js';
|
|
2
2
|
import { ensureIndexed as defaultEnsureIndexed } from './docs-index.js';
|
|
3
|
+
import { type EcosystemId, type EcosystemIo, type EcosystemProfile } from './docs-ecosystems.js';
|
|
3
4
|
import { resolvePackage as defaultResolvePackage, type ResolvedPackage } from './docs-resolve.js';
|
|
4
5
|
import { retrieveChunks as defaultRetrieveChunks, type RetrievedChunk } from './docs-retrieve.js';
|
|
5
6
|
import { npmVersionLookup as defaultNpmVersionLookup, type NpmVersionInfo } from './npm-version.js';
|
|
@@ -37,6 +38,8 @@ export type DocsRawResult = {
|
|
|
37
38
|
autoInstalled?: boolean;
|
|
38
39
|
autoInstallPin?: AutoInstallPin;
|
|
39
40
|
npmVersion?: NpmVersionInfo | null;
|
|
41
|
+
/** The registry the answer came from, for the block that leads it. */
|
|
42
|
+
registryLabel?: string;
|
|
40
43
|
} | {
|
|
41
44
|
kind: 'no_chunks';
|
|
42
45
|
pkg: ResolvedPackage;
|
|
@@ -46,10 +49,12 @@ export type DocsRawResult = {
|
|
|
46
49
|
autoInstalled?: boolean;
|
|
47
50
|
autoInstallPin?: AutoInstallPin;
|
|
48
51
|
npmVersion?: NpmVersionInfo | null;
|
|
52
|
+
/** The registry the answer came from, for the block that leads it. */
|
|
53
|
+
registryLabel?: string;
|
|
49
54
|
} | {
|
|
50
55
|
kind: 'error';
|
|
51
56
|
message: string;
|
|
52
|
-
resolveError?: 'not_installed' | 'invalid_name';
|
|
57
|
+
resolveError?: 'not_installed' | 'invalid_name' | 'unsupported_ecosystem' | 'ambiguous_ecosystem';
|
|
53
58
|
installError?: string;
|
|
54
59
|
version?: string;
|
|
55
60
|
hitCache?: boolean;
|
|
@@ -57,11 +62,15 @@ export type DocsRawResult = {
|
|
|
57
62
|
autoInstalled?: boolean;
|
|
58
63
|
autoInstallPin?: AutoInstallPin;
|
|
59
64
|
npmVersion?: NpmVersionInfo | null;
|
|
65
|
+
/** The registry the answer came from, for the block that leads it. */
|
|
66
|
+
registryLabel?: string;
|
|
60
67
|
};
|
|
61
68
|
export interface DocsRawInput {
|
|
62
69
|
pkg: string;
|
|
63
70
|
query: string;
|
|
64
71
|
cwd: string;
|
|
72
|
+
/** Which registry to read, for a repo holding more than one manifest. */
|
|
73
|
+
ecosystem?: EcosystemId;
|
|
65
74
|
autoInstall?: boolean;
|
|
66
75
|
resolvePackage?: typeof defaultResolvePackage;
|
|
67
76
|
ensureIndexed?: typeof defaultEnsureIndexed;
|
|
@@ -69,6 +78,8 @@ export interface DocsRawInput {
|
|
|
69
78
|
openCache?: typeof defaultOpenCache;
|
|
70
79
|
spawn?: SpawnFn;
|
|
71
80
|
npmVersionLookup?: typeof defaultNpmVersionLookup;
|
|
81
|
+
/** Overrides for the filesystem and network a non-npm row reaches through. */
|
|
82
|
+
io?: Partial<EcosystemIo>;
|
|
72
83
|
signal?: AbortSignal;
|
|
73
84
|
}
|
|
74
85
|
export interface DocsFocusedResult {
|
|
@@ -162,7 +173,7 @@ export declare function findDeclaredRange(parentPkg: string, cwd: string): strin
|
|
|
162
173
|
* `bun`, and cannot declare `bun-types`, so a sentence about what package.json
|
|
163
174
|
* does or does not say has to be a sentence about `bun`.
|
|
164
175
|
*/
|
|
165
|
-
export declare function buildVersionBanner(pin: AutoInstallPin | undefined, resolved: string, version: string, cwd: string): string;
|
|
176
|
+
export declare function buildVersionBanner(pin: AutoInstallPin | undefined, resolved: string, version: string, cwd: string, profile?: EcosystemProfile): string;
|
|
166
177
|
export declare function getDocsModulesDir(): string;
|
|
167
178
|
export declare function ensureDocsModulesDir(dir: string): void;
|
|
168
179
|
/**
|
|
@@ -207,8 +218,13 @@ export interface AcquireInput {
|
|
|
207
218
|
name: string;
|
|
208
219
|
cwd: string;
|
|
209
220
|
spawn: SpawnFn;
|
|
210
|
-
|
|
221
|
+
/** Overrides `profile.resolve` when given, for callers that inject a resolver. */
|
|
222
|
+
resolvePackage?: typeof defaultResolvePackage;
|
|
211
223
|
signal: AbortSignal | undefined;
|
|
224
|
+
/** Which registry to acquire from. */
|
|
225
|
+
profile?: EcosystemProfile;
|
|
226
|
+
/** The row's filesystem and network reach. Built from `spawn` when absent. */
|
|
227
|
+
io?: EcosystemIo;
|
|
212
228
|
}
|
|
213
229
|
/**
|
|
214
230
|
* Get a package onto disk and resolved: resolve from `cwd`, and on
|
|
@@ -231,6 +247,14 @@ export interface AcquireInput {
|
|
|
231
247
|
* sites should differ only about WHETHER to install, never about HOW.
|
|
232
248
|
*/
|
|
233
249
|
export declare function acquirePackage(input: AcquireInput): Promise<AcquireOutcome>;
|
|
250
|
+
/** The docs pipeline's adapter over the shared redirect walk (docs-resolve.ts):
|
|
251
|
+
* hops resolve through the auto-installing lookup, so a declaration package that is
|
|
252
|
+
* declared but not yet on disk is fetched rather than abandoned. */
|
|
253
|
+
export declare function resolveTypeSourceForDocs(pkg: ResolvedPackage, requested: string, cwd: string, spawn: SpawnFn, resolvePackage: typeof defaultResolvePackage, signal: AbortSignal | undefined): Promise<{
|
|
254
|
+
pkg: ResolvedPackage;
|
|
255
|
+
installed: boolean;
|
|
256
|
+
pin?: AutoInstallPin;
|
|
257
|
+
}>;
|
|
234
258
|
export declare function docsRaw(input: DocsRawInput): Promise<DocsRawResult>;
|
|
235
259
|
export declare function docsFocused(input: DocsFocusedInput): Promise<DocsFocusedResult>;
|
|
236
260
|
export declare function buildPrompt(pkg: ResolvedPackage, query: string, content: string): string;
|