@mjasnikovs/pi-task 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +125 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +6 -0
- package/dist/shared/child-output.d.ts +21 -0
- package/dist/shared/child-output.js +40 -0
- package/dist/shared/child-process.d.ts +71 -0
- package/dist/shared/child-process.js +190 -0
- package/dist/shared/pi-invocation.d.ts +7 -0
- package/dist/shared/pi-invocation.js +24 -0
- package/dist/task/child-runner.d.ts +66 -0
- package/dist/task/child-runner.js +157 -0
- package/dist/task/enrichment.d.ts +12 -0
- package/dist/task/enrichment.js +82 -0
- package/dist/task/failure-classifier.d.ts +15 -0
- package/dist/task/failure-classifier.js +63 -0
- package/dist/task/file-inventory.d.ts +9 -0
- package/dist/task/file-inventory.js +44 -0
- package/dist/task/loop-detector.d.ts +32 -0
- package/dist/task/loop-detector.js +46 -0
- package/dist/task/orchestrator.d.ts +54 -0
- package/dist/task/orchestrator.js +387 -0
- package/dist/task/parsers.d.ts +32 -0
- package/dist/task/parsers.js +172 -0
- package/dist/task/phases.d.ts +56 -0
- package/dist/task/phases.js +477 -0
- package/dist/task/prompts.d.ts +21 -0
- package/dist/task/prompts.js +346 -0
- package/dist/task/service-blocks.d.ts +3 -0
- package/dist/task/service-blocks.js +10 -0
- package/dist/task/task-file.d.ts +14 -0
- package/dist/task/task-file.js +15 -0
- package/dist/task/task-io.d.ts +19 -0
- package/dist/task/task-io.js +78 -0
- package/dist/task/task-parsers.d.ts +12 -0
- package/dist/task/task-parsers.js +75 -0
- package/dist/task/task-types.d.ts +21 -0
- package/dist/task/task-types.js +18 -0
- package/dist/task/timings.d.ts +18 -0
- package/dist/task/timings.js +36 -0
- package/dist/task/widget.d.ts +39 -0
- package/dist/task/widget.js +122 -0
- package/dist/workers/brave-search.d.ts +17 -0
- package/dist/workers/brave-search.js +77 -0
- package/dist/workers/docs-cache.d.ts +16 -0
- package/dist/workers/docs-cache.js +66 -0
- package/dist/workers/docs-core.d.ts +86 -0
- package/dist/workers/docs-core.js +329 -0
- package/dist/workers/docs-index.d.ts +9 -0
- package/dist/workers/docs-index.js +200 -0
- package/dist/workers/docs-resolve.d.ts +12 -0
- package/dist/workers/docs-resolve.js +126 -0
- package/dist/workers/docs-retrieve.d.ts +15 -0
- package/dist/workers/docs-retrieve.js +91 -0
- package/dist/workers/fetch-core.d.ts +35 -0
- package/dist/workers/fetch-core.js +91 -0
- package/dist/workers/html-clean.d.ts +17 -0
- package/dist/workers/html-clean.js +142 -0
- package/dist/workers/index.d.ts +2 -0
- package/dist/workers/index.js +10 -0
- package/dist/workers/npm-version.d.ts +32 -0
- package/dist/workers/npm-version.js +102 -0
- package/dist/workers/pi-worker-core.d.ts +28 -0
- package/dist/workers/pi-worker-core.js +29 -0
- package/dist/workers/pi-worker-docs.d.ts +16 -0
- package/dist/workers/pi-worker-docs.js +143 -0
- package/dist/workers/pi-worker-fetch.d.ts +20 -0
- package/dist/workers/pi-worker-fetch.js +72 -0
- package/dist/workers/pi-worker-search.d.ts +7 -0
- package/dist/workers/pi-worker-search.js +55 -0
- package/dist/workers/pi-worker.d.ts +10 -0
- package/dist/workers/pi-worker.js +61 -0
- package/dist/workers/search-core.d.ts +19 -0
- package/dist/workers/search-core.js +35 -0
- package/dist/workers/shared.d.ts +3 -0
- package/dist/workers/shared.js +4 -0
- package/package.json +50 -0
|
@@ -0,0 +1,329 @@
|
|
|
1
|
+
import { spawn as defaultSpawn } from 'node:child_process';
|
|
2
|
+
import * as fs from 'node:fs';
|
|
3
|
+
import * as os from 'node:os';
|
|
4
|
+
import * as path from 'node:path';
|
|
5
|
+
import { openCache as defaultOpenCache } from './docs-cache.js';
|
|
6
|
+
import { ensureIndexed as defaultEnsureIndexed } from './docs-index.js';
|
|
7
|
+
import { resolvePackage as defaultResolvePackage, ResolveError } from './docs-resolve.js';
|
|
8
|
+
import { retrieveChunks as defaultRetrieveChunks } from './docs-retrieve.js';
|
|
9
|
+
import { npmVersionLookup as defaultNpmVersionLookup } from './npm-version.js';
|
|
10
|
+
import { getPiInvocation } from '../shared/pi-invocation.js';
|
|
11
|
+
import { CHILD_BASE_ARGS, runChild } from '../shared/child-process.js';
|
|
12
|
+
import { parseChildOutput, isExcerptInContent, formatResultText as formatResultTextShared } from '../shared/child-output.js';
|
|
13
|
+
const DEFAULT_LIMIT = 8;
|
|
14
|
+
const DEFAULT_BUDGET = 24_000;
|
|
15
|
+
const NO_CACHE_HEAD = 25_000;
|
|
16
|
+
const NO_CACHE_TAIL = 5_000;
|
|
17
|
+
const NO_CACHE_TOTAL = NO_CACHE_HEAD + NO_CACHE_TAIL;
|
|
18
|
+
const NO_CACHE_MARKER = '\n\n[...content continues, truncated...]\n\n';
|
|
19
|
+
const CHILD_ARGS = [...CHILD_BASE_ARGS, '--no-tools'];
|
|
20
|
+
export function extractParentPackage(moduleName) {
|
|
21
|
+
if (moduleName.startsWith('@')) {
|
|
22
|
+
const parts = moduleName.split('/');
|
|
23
|
+
return `${parts[0]}/${parts[1]}`;
|
|
24
|
+
}
|
|
25
|
+
return moduleName.split('/')[0];
|
|
26
|
+
}
|
|
27
|
+
export function getDocsModulesDir() {
|
|
28
|
+
const base = process.env.XDG_CACHE_HOME?.trim() || path.join(os.homedir(), '.cache');
|
|
29
|
+
return path.join(base, 'pi-worker', 'docs-modules');
|
|
30
|
+
}
|
|
31
|
+
export function ensureDocsModulesDir(dir) {
|
|
32
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
33
|
+
const pkgPath = path.join(dir, 'package.json');
|
|
34
|
+
if (!fs.existsSync(pkgPath)) {
|
|
35
|
+
fs.writeFileSync(pkgPath, '{"name":"pi-worker-docs-modules","private":true}\n', 'utf8');
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
export async function runAutoInstall(spawn, packageName, signal) {
|
|
39
|
+
const installDir = getDocsModulesDir();
|
|
40
|
+
ensureDocsModulesDir(installDir);
|
|
41
|
+
const result = await runChild(spawn, {
|
|
42
|
+
command: 'npm',
|
|
43
|
+
args: ['install', '--no-audit', '--no-fund', '--loglevel=error', packageName]
|
|
44
|
+
}, installDir, signal, { mode: 'text', discardStdout: true });
|
|
45
|
+
return { success: result.exitCode === 0 && !result.aborted, installDir, stderr: result.stderr };
|
|
46
|
+
}
|
|
47
|
+
export async function docsRaw(input) {
|
|
48
|
+
const resolvePackage = input.resolvePackage ?? defaultResolvePackage;
|
|
49
|
+
const ensureIndexed = input.ensureIndexed ?? defaultEnsureIndexed;
|
|
50
|
+
const retrieveChunks = input.retrieveChunks ?? defaultRetrieveChunks;
|
|
51
|
+
const openCache = input.openCache ?? defaultOpenCache;
|
|
52
|
+
const spawn = input.spawn ?? defaultSpawn;
|
|
53
|
+
const npmVersionLookup = input.npmVersionLookup ?? defaultNpmVersionLookup;
|
|
54
|
+
// Fire the npm registry lookup in parallel with resolve/index/retrieve.
|
|
55
|
+
// It returns null on any failure, so it never blocks the local pipeline.
|
|
56
|
+
const npmVersionPromise = npmVersionLookup(extractParentPackage(input.pkg), {
|
|
57
|
+
signal: input.signal
|
|
58
|
+
}).catch(() => null);
|
|
59
|
+
// Step 1: resolve package
|
|
60
|
+
let pkg;
|
|
61
|
+
let autoInstalled = false;
|
|
62
|
+
try {
|
|
63
|
+
pkg = resolvePackage(input.pkg, input.cwd);
|
|
64
|
+
}
|
|
65
|
+
catch (firstErr) {
|
|
66
|
+
if (firstErr instanceof ResolveError && firstErr.kind === 'not_installed') {
|
|
67
|
+
// auto-install
|
|
68
|
+
const parentPkg = extractParentPackage(input.pkg);
|
|
69
|
+
const installResult = await runAutoInstall(spawn, parentPkg, undefined);
|
|
70
|
+
if (!installResult.success) {
|
|
71
|
+
return {
|
|
72
|
+
kind: 'error',
|
|
73
|
+
message: `Package "${parentPkg}" is not installed and auto-install failed.\n${installResult.stderr}`,
|
|
74
|
+
resolveError: 'not_installed',
|
|
75
|
+
installError: installResult.stderr,
|
|
76
|
+
npmVersion: await npmVersionPromise
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
autoInstalled = true;
|
|
80
|
+
try {
|
|
81
|
+
pkg = resolvePackage(input.pkg, installResult.installDir);
|
|
82
|
+
}
|
|
83
|
+
catch (retryErr) {
|
|
84
|
+
if (retryErr instanceof ResolveError) {
|
|
85
|
+
return {
|
|
86
|
+
kind: 'error',
|
|
87
|
+
message: retryErr.message,
|
|
88
|
+
resolveError: retryErr.kind,
|
|
89
|
+
autoInstalled,
|
|
90
|
+
npmVersion: await npmVersionPromise
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
return {
|
|
94
|
+
kind: 'error',
|
|
95
|
+
message: `Could not resolve "${input.pkg}" after install: ${retryErr instanceof Error ? retryErr.message : String(retryErr)}`,
|
|
96
|
+
autoInstalled,
|
|
97
|
+
npmVersion: await npmVersionPromise
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
else if (firstErr instanceof ResolveError) {
|
|
102
|
+
return {
|
|
103
|
+
kind: 'error',
|
|
104
|
+
message: firstErr.message,
|
|
105
|
+
resolveError: firstErr.kind,
|
|
106
|
+
npmVersion: await npmVersionPromise
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
else {
|
|
110
|
+
return {
|
|
111
|
+
kind: 'error',
|
|
112
|
+
message: `Could not resolve "${input.pkg}": ${firstErr instanceof Error ? firstErr.message : String(firstErr)}`,
|
|
113
|
+
npmVersion: await npmVersionPromise
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
// Step 2: open cache
|
|
118
|
+
let cache = null;
|
|
119
|
+
let cacheError;
|
|
120
|
+
try {
|
|
121
|
+
cache = openCache();
|
|
122
|
+
}
|
|
123
|
+
catch (err) {
|
|
124
|
+
cacheError = err instanceof Error ? err.message : String(err);
|
|
125
|
+
}
|
|
126
|
+
const result = cache ?
|
|
127
|
+
docsRawCached(cache, pkg, input.query, ensureIndexed, retrieveChunks, autoInstalled)
|
|
128
|
+
: docsRawUncached(pkg, cacheError ?? 'unknown cache error', autoInstalled);
|
|
129
|
+
result.npmVersion = await npmVersionPromise;
|
|
130
|
+
return result;
|
|
131
|
+
}
|
|
132
|
+
function docsRawCached(cache, pkg, query, ensureIndexed, retrieveChunks, autoInstalled) {
|
|
133
|
+
let indexResult;
|
|
134
|
+
const t0 = Date.now();
|
|
135
|
+
try {
|
|
136
|
+
indexResult = ensureIndexed(cache, pkg);
|
|
137
|
+
}
|
|
138
|
+
catch (err) {
|
|
139
|
+
return {
|
|
140
|
+
kind: 'error',
|
|
141
|
+
message: `Indexing failed for ${pkg.name}@${pkg.version}: ${err instanceof Error ? err.message : String(err)}`,
|
|
142
|
+
version: pkg.version
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
const indexingMs = indexResult.hitCache ? undefined : Date.now() - t0;
|
|
146
|
+
const chunkCount = cache.db
|
|
147
|
+
.prepare('SELECT count(*) AS c FROM chunks WHERE name = ? AND version = ?')
|
|
148
|
+
.get(pkg.name, pkg.version)?.c ?? 0;
|
|
149
|
+
if (chunkCount === 0) {
|
|
150
|
+
return {
|
|
151
|
+
kind: 'no_chunks',
|
|
152
|
+
pkg,
|
|
153
|
+
hitCache: indexResult.hitCache,
|
|
154
|
+
indexedFiles: 0,
|
|
155
|
+
autoInstalled: autoInstalled ? true : undefined
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
let chunks;
|
|
159
|
+
try {
|
|
160
|
+
chunks = retrieveChunks(cache, {
|
|
161
|
+
name: pkg.name,
|
|
162
|
+
version: pkg.version,
|
|
163
|
+
query,
|
|
164
|
+
limit: DEFAULT_LIMIT,
|
|
165
|
+
contentBudget: DEFAULT_BUDGET
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
catch (err) {
|
|
169
|
+
return {
|
|
170
|
+
kind: 'error',
|
|
171
|
+
message: `Retrieval failed for ${pkg.name}@${pkg.version}: ${err instanceof Error ? err.message : String(err)}`,
|
|
172
|
+
version: pkg.version,
|
|
173
|
+
hitCache: indexResult.hitCache
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
if (chunks.length === 0) {
|
|
177
|
+
return {
|
|
178
|
+
kind: 'no_chunks',
|
|
179
|
+
pkg,
|
|
180
|
+
hitCache: indexResult.hitCache,
|
|
181
|
+
autoInstalled: autoInstalled ? true : undefined
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
return {
|
|
185
|
+
kind: 'ok',
|
|
186
|
+
pkg,
|
|
187
|
+
chunks,
|
|
188
|
+
hitCache: indexResult.hitCache,
|
|
189
|
+
indexingMs,
|
|
190
|
+
autoInstalled: autoInstalled ? true : undefined
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
function docsRawUncached(pkg, cacheError, autoInstalled) {
|
|
194
|
+
const parts = [];
|
|
195
|
+
const dtsFiles = walkDtsAlpha(pkg.root);
|
|
196
|
+
const entryFirst = pkg.entryDts ? [pkg.entryDts, ...dtsFiles.filter(f => f !== pkg.entryDts)] : dtsFiles;
|
|
197
|
+
for (const abs of entryFirst) {
|
|
198
|
+
let raw;
|
|
199
|
+
try {
|
|
200
|
+
raw = fs.readFileSync(abs, 'utf8');
|
|
201
|
+
}
|
|
202
|
+
catch {
|
|
203
|
+
continue;
|
|
204
|
+
}
|
|
205
|
+
const rel = path.relative(pkg.root, abs);
|
|
206
|
+
parts.push(`// ${rel}\n${raw}`);
|
|
207
|
+
}
|
|
208
|
+
if (pkg.readme) {
|
|
209
|
+
const rel = path.relative(pkg.root, pkg.readme);
|
|
210
|
+
try {
|
|
211
|
+
const raw = fs.readFileSync(pkg.readme, 'utf8');
|
|
212
|
+
parts.push(`<!-- README: ${rel} -->\n${raw}`);
|
|
213
|
+
}
|
|
214
|
+
catch {
|
|
215
|
+
// skip
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
const joined = parts.join('\n\n');
|
|
219
|
+
if (joined.length === 0) {
|
|
220
|
+
return {
|
|
221
|
+
kind: 'no_chunks',
|
|
222
|
+
pkg,
|
|
223
|
+
hitCache: false,
|
|
224
|
+
indexedFiles: 0,
|
|
225
|
+
cacheError,
|
|
226
|
+
autoInstalled: autoInstalled ? true : undefined
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
const truncated = truncateHeadTail(joined);
|
|
230
|
+
const chunks = [
|
|
231
|
+
{ filePath: '<no-cache>', kind: 'dts', content: truncated, rank: 0 }
|
|
232
|
+
];
|
|
233
|
+
return {
|
|
234
|
+
kind: 'ok',
|
|
235
|
+
pkg,
|
|
236
|
+
chunks,
|
|
237
|
+
hitCache: false,
|
|
238
|
+
cacheError,
|
|
239
|
+
autoInstalled: autoInstalled ? true : undefined
|
|
240
|
+
};
|
|
241
|
+
}
|
|
242
|
+
function walkDtsAlpha(root) {
|
|
243
|
+
const out = [];
|
|
244
|
+
const stack = [root];
|
|
245
|
+
while (stack.length) {
|
|
246
|
+
const dir = stack.pop();
|
|
247
|
+
let entries;
|
|
248
|
+
try {
|
|
249
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
250
|
+
}
|
|
251
|
+
catch {
|
|
252
|
+
continue;
|
|
253
|
+
}
|
|
254
|
+
for (const entry of entries) {
|
|
255
|
+
if (entry.name === 'node_modules')
|
|
256
|
+
continue;
|
|
257
|
+
const full = path.join(dir, entry.name);
|
|
258
|
+
if (entry.isDirectory())
|
|
259
|
+
stack.push(full);
|
|
260
|
+
else if (entry.isFile() && entry.name.endsWith('.d.ts'))
|
|
261
|
+
out.push(full);
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
return out.sort();
|
|
265
|
+
}
|
|
266
|
+
function truncateHeadTail(s) {
|
|
267
|
+
if (s.length <= NO_CACHE_TOTAL)
|
|
268
|
+
return s;
|
|
269
|
+
return s.slice(0, NO_CACHE_HEAD) + NO_CACHE_MARKER + s.slice(s.length - NO_CACHE_TAIL);
|
|
270
|
+
}
|
|
271
|
+
export async function docsFocused(input) {
|
|
272
|
+
const spawn = input.spawn ?? defaultSpawn;
|
|
273
|
+
const rawResult = await docsRaw(input);
|
|
274
|
+
if (rawResult.kind === 'error') {
|
|
275
|
+
throw new Error(rawResult.message);
|
|
276
|
+
}
|
|
277
|
+
if (rawResult.kind === 'not_installed') {
|
|
278
|
+
throw new Error(`Package "${rawResult.pkg}" is not installed`);
|
|
279
|
+
}
|
|
280
|
+
if (rawResult.kind === 'no_chunks') {
|
|
281
|
+
throw new Error(`Package ${rawResult.pkg.name}@${rawResult.pkg.version} has no .d.ts files or README.`);
|
|
282
|
+
}
|
|
283
|
+
const { pkg, chunks, hitCache, indexingMs } = rawResult;
|
|
284
|
+
const concatenated = chunks.map(c => c.content).join('\n\n');
|
|
285
|
+
const prompt = buildPrompt(pkg, input.query, concatenated);
|
|
286
|
+
const invocation = getPiInvocation([...CHILD_ARGS, prompt]);
|
|
287
|
+
const child = await runChild(spawn, invocation, input.cwd, input.signal);
|
|
288
|
+
const parsed = parseChildOutput(child.stdout);
|
|
289
|
+
const excerptVerified = parsed.excerpt ? isExcerptInContent(parsed.excerpt, concatenated) : undefined;
|
|
290
|
+
return {
|
|
291
|
+
answer: parsed.answer,
|
|
292
|
+
excerpt: parsed.excerpt,
|
|
293
|
+
excerptVerified,
|
|
294
|
+
pkg,
|
|
295
|
+
version: pkg.version,
|
|
296
|
+
exitCode: child.exitCode,
|
|
297
|
+
aborted: child.aborted,
|
|
298
|
+
stderr: child.stderr,
|
|
299
|
+
hitCache,
|
|
300
|
+
indexingMs,
|
|
301
|
+
chunksRetrieved: chunks.length,
|
|
302
|
+
autoInstalled: rawResult.autoInstalled,
|
|
303
|
+
npmVersion: rawResult.npmVersion
|
|
304
|
+
};
|
|
305
|
+
}
|
|
306
|
+
export function buildPrompt(pkg, query, content) {
|
|
307
|
+
return (`You answer one question about an npm package, using only the provided content.\n`
|
|
308
|
+
+ `\n`
|
|
309
|
+
+ `Rules:\n`
|
|
310
|
+
+ `1. Output ONLY two tags, in this order, with NO text outside them:\n`
|
|
311
|
+
+ ` <answer>...your answer...</answer>\n`
|
|
312
|
+
+ ` <excerpt>...verbatim quote from <package-content>...</excerpt>\n`
|
|
313
|
+
+ `2. The <excerpt> MUST be copied character-for-character from <package-content>.\n`
|
|
314
|
+
+ ` Do not paraphrase, translate, or summarise inside <excerpt>.\n`
|
|
315
|
+
+ `3. Prefer type signatures, function declarations, and code blocks as evidence over prose.\n`
|
|
316
|
+
+ `4. If the answer is unclear, ambiguous, or absent from <package-content>, write exactly:\n`
|
|
317
|
+
+ ` <answer>unclear from this package</answer> and put the closest related text in <excerpt>.\n`
|
|
318
|
+
+ ` Do not guess.\n`
|
|
319
|
+
+ `5. Be terse. One short paragraph in <answer> max.\n`
|
|
320
|
+
+ `\n`
|
|
321
|
+
+ `<package>${pkg.name}@${pkg.version}</package>\n`
|
|
322
|
+
+ `<question>${query}</question>\n`
|
|
323
|
+
+ `<package-content>\n${content}\n</package-content>\n`);
|
|
324
|
+
}
|
|
325
|
+
// ─── Backward-compatible wrappers (thin — delegates to shared/) ─────────────
|
|
326
|
+
/** Thin wrapper so existing callers using the pkg-based signature still work. */
|
|
327
|
+
export function formatResultText(pkg, parsed, verified) {
|
|
328
|
+
return formatResultTextShared(`Per ${pkg.name}@${pkg.version}:`, parsed, verified);
|
|
329
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { CacheHandle } from './docs-cache.js';
|
|
2
|
+
import type { ResolvedPackage } from './docs-resolve.js';
|
|
3
|
+
export interface IndexResult {
|
|
4
|
+
hitCache: boolean;
|
|
5
|
+
filesIngested: number;
|
|
6
|
+
chunksWritten: number;
|
|
7
|
+
contentHash: string;
|
|
8
|
+
}
|
|
9
|
+
export declare function ensureIndexed(cache: CacheHandle, pkg: ResolvedPackage): IndexResult;
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import * as fs from 'node:fs';
|
|
3
|
+
import * as path from 'node:path';
|
|
4
|
+
const MAX_CHUNK_BYTES = 8 * 1024;
|
|
5
|
+
const ZERO_SEP = Buffer.from([0]);
|
|
6
|
+
const DECL_SPLIT_RE = /^(?:export\s+|declare\s+)?(?:default\s+)?(?:async\s+)?(?:function|class|interface|type|namespace|module|const|let|var|enum)\s+/m;
|
|
7
|
+
const README_SPLIT_RE = /^#{1,2} /m;
|
|
8
|
+
function computeContentHash(pkg) {
|
|
9
|
+
const hash = createHash('sha256');
|
|
10
|
+
hash.update(Buffer.from(`${pkg.name}@${pkg.version}`, 'utf8'));
|
|
11
|
+
hash.update(ZERO_SEP);
|
|
12
|
+
if (pkg.entryDts && fs.existsSync(pkg.entryDts)) {
|
|
13
|
+
hash.update(fs.readFileSync(pkg.entryDts));
|
|
14
|
+
}
|
|
15
|
+
hash.update(ZERO_SEP);
|
|
16
|
+
if (pkg.readme && fs.existsSync(pkg.readme)) {
|
|
17
|
+
hash.update(fs.readFileSync(pkg.readme));
|
|
18
|
+
}
|
|
19
|
+
return hash.digest('hex');
|
|
20
|
+
}
|
|
21
|
+
function walkDts(root) {
|
|
22
|
+
const out = [];
|
|
23
|
+
const stack = [root];
|
|
24
|
+
while (stack.length) {
|
|
25
|
+
const dir = stack.pop();
|
|
26
|
+
let entries;
|
|
27
|
+
try {
|
|
28
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
continue;
|
|
32
|
+
}
|
|
33
|
+
for (const entry of entries) {
|
|
34
|
+
if (entry.name === 'node_modules')
|
|
35
|
+
continue;
|
|
36
|
+
const full = path.join(dir, entry.name);
|
|
37
|
+
if (entry.isSymbolicLink()) {
|
|
38
|
+
let realPath;
|
|
39
|
+
try {
|
|
40
|
+
realPath = fs.realpathSync(full);
|
|
41
|
+
}
|
|
42
|
+
catch {
|
|
43
|
+
continue;
|
|
44
|
+
}
|
|
45
|
+
const relReal = path.relative(root, realPath);
|
|
46
|
+
if (relReal.startsWith('..'))
|
|
47
|
+
continue;
|
|
48
|
+
const stat = fs.statSync(realPath);
|
|
49
|
+
if (stat.isDirectory())
|
|
50
|
+
stack.push(realPath);
|
|
51
|
+
else if (stat.isFile() && realPath.endsWith('.d.ts'))
|
|
52
|
+
out.push(realPath);
|
|
53
|
+
continue;
|
|
54
|
+
}
|
|
55
|
+
if (entry.isDirectory())
|
|
56
|
+
stack.push(full);
|
|
57
|
+
else if (entry.isFile() && entry.name.endsWith('.d.ts'))
|
|
58
|
+
out.push(full);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
return out.sort();
|
|
62
|
+
}
|
|
63
|
+
function collectFiles(pkg) {
|
|
64
|
+
return {
|
|
65
|
+
dts: walkDts(pkg.root),
|
|
66
|
+
readme: pkg.readme
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
function chunkDts(content, relPath) {
|
|
70
|
+
const splits = splitAtMatches(content, new RegExp(DECL_SPLIT_RE.source, 'gm'));
|
|
71
|
+
const chunks = [];
|
|
72
|
+
for (const part of splits) {
|
|
73
|
+
const trimmed = part.trim();
|
|
74
|
+
if (!trimmed)
|
|
75
|
+
continue;
|
|
76
|
+
const prefixed = `// ${relPath}\n${trimmed}`;
|
|
77
|
+
if (Buffer.byteLength(prefixed, 'utf8') > MAX_CHUNK_BYTES) {
|
|
78
|
+
for (const slice of sliceBytes(prefixed, MAX_CHUNK_BYTES)) {
|
|
79
|
+
chunks.push(slice);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
else {
|
|
83
|
+
chunks.push(prefixed);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
return chunks;
|
|
87
|
+
}
|
|
88
|
+
function chunkReadme(content) {
|
|
89
|
+
const splits = splitAtMatches(content, new RegExp(README_SPLIT_RE.source, 'gm'));
|
|
90
|
+
const chunks = [];
|
|
91
|
+
for (const part of splits) {
|
|
92
|
+
const trimmed = part.replace(/\s+$/, '');
|
|
93
|
+
if (!trimmed)
|
|
94
|
+
continue;
|
|
95
|
+
const headingMatch = /^(#{1,2}) (.+)$/m.exec(trimmed);
|
|
96
|
+
const heading = headingMatch ? headingMatch[2] : '(intro)';
|
|
97
|
+
const prefixed = `<!-- README: ${heading} -->\n${trimmed}`;
|
|
98
|
+
if (Buffer.byteLength(prefixed, 'utf8') > MAX_CHUNK_BYTES) {
|
|
99
|
+
for (const slice of sliceBytes(prefixed, MAX_CHUNK_BYTES))
|
|
100
|
+
chunks.push(slice);
|
|
101
|
+
}
|
|
102
|
+
else {
|
|
103
|
+
chunks.push(prefixed);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
return chunks;
|
|
107
|
+
}
|
|
108
|
+
function splitAtMatches(text, re) {
|
|
109
|
+
const parts = [];
|
|
110
|
+
let lastIndex = 0;
|
|
111
|
+
let m;
|
|
112
|
+
while ((m = re.exec(text))) {
|
|
113
|
+
if (m.index > lastIndex)
|
|
114
|
+
parts.push(text.slice(lastIndex, m.index));
|
|
115
|
+
lastIndex = m.index;
|
|
116
|
+
re.lastIndex = m.index + 1;
|
|
117
|
+
}
|
|
118
|
+
if (lastIndex < text.length)
|
|
119
|
+
parts.push(text.slice(lastIndex));
|
|
120
|
+
return parts.length ? parts : [text];
|
|
121
|
+
}
|
|
122
|
+
function sliceBytes(s, maxBytes) {
|
|
123
|
+
const out = [];
|
|
124
|
+
let buf = Buffer.from(s, 'utf8');
|
|
125
|
+
while (buf.length > maxBytes) {
|
|
126
|
+
const slice = buf.subarray(0, maxBytes).toString('utf8');
|
|
127
|
+
out.push(slice);
|
|
128
|
+
buf = buf.subarray(Buffer.byteLength(slice, 'utf8'));
|
|
129
|
+
}
|
|
130
|
+
if (buf.length)
|
|
131
|
+
out.push(buf.toString('utf8'));
|
|
132
|
+
return out;
|
|
133
|
+
}
|
|
134
|
+
function ingestBody(cache, pkg, contentHash) {
|
|
135
|
+
const inside = cache.db
|
|
136
|
+
.prepare('SELECT content_hash FROM packages WHERE name = ? AND version = ?')
|
|
137
|
+
.get(pkg.name, pkg.version);
|
|
138
|
+
if (inside && inside.content_hash === contentHash) {
|
|
139
|
+
return { hitCache: true, filesIngested: 0, chunksWritten: 0 };
|
|
140
|
+
}
|
|
141
|
+
cache.db.prepare('DELETE FROM chunks WHERE name = ? AND version = ?').run(pkg.name, pkg.version);
|
|
142
|
+
const files = collectFiles(pkg);
|
|
143
|
+
let chunksWritten = 0;
|
|
144
|
+
let filesIngested = 0;
|
|
145
|
+
const insertChunk = cache.db.prepare('INSERT INTO chunks (name, version, file_path, kind, content) VALUES (?, ?, ?, ?, ?)');
|
|
146
|
+
for (const abs of files.dts) {
|
|
147
|
+
const rel = path.relative(pkg.root, abs);
|
|
148
|
+
let raw;
|
|
149
|
+
try {
|
|
150
|
+
raw = fs.readFileSync(abs, 'utf8');
|
|
151
|
+
}
|
|
152
|
+
catch {
|
|
153
|
+
continue;
|
|
154
|
+
}
|
|
155
|
+
const chunks = chunkDts(raw, rel);
|
|
156
|
+
if (!chunks.length)
|
|
157
|
+
continue;
|
|
158
|
+
filesIngested++;
|
|
159
|
+
for (const c of chunks) {
|
|
160
|
+
insertChunk.run(pkg.name, pkg.version, rel, 'dts', c);
|
|
161
|
+
chunksWritten++;
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
if (files.readme) {
|
|
165
|
+
const rel = path.relative(pkg.root, files.readme);
|
|
166
|
+
const raw = fs.readFileSync(files.readme, 'utf8');
|
|
167
|
+
const chunks = chunkReadme(raw);
|
|
168
|
+
if (chunks.length) {
|
|
169
|
+
filesIngested++;
|
|
170
|
+
for (const c of chunks) {
|
|
171
|
+
insertChunk.run(pkg.name, pkg.version, rel, 'readme', c);
|
|
172
|
+
chunksWritten++;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
cache.db
|
|
177
|
+
.prepare('INSERT OR REPLACE INTO packages (name, version, content_hash, indexed_at) VALUES (?, ?, ?, ?)')
|
|
178
|
+
.run(pkg.name, pkg.version, contentHash, Date.now());
|
|
179
|
+
return { hitCache: false, filesIngested, chunksWritten };
|
|
180
|
+
}
|
|
181
|
+
export function ensureIndexed(cache, pkg) {
|
|
182
|
+
const contentHash = computeContentHash(pkg);
|
|
183
|
+
const existing = cache.db
|
|
184
|
+
.prepare('SELECT content_hash FROM packages WHERE name = ? AND version = ?')
|
|
185
|
+
.get(pkg.name, pkg.version);
|
|
186
|
+
if (existing && existing.content_hash === contentHash) {
|
|
187
|
+
return { hitCache: true, filesIngested: 0, chunksWritten: 0, contentHash };
|
|
188
|
+
}
|
|
189
|
+
cache.db.exec('BEGIN IMMEDIATE');
|
|
190
|
+
let result;
|
|
191
|
+
try {
|
|
192
|
+
result = ingestBody(cache, pkg, contentHash);
|
|
193
|
+
cache.db.exec('COMMIT');
|
|
194
|
+
}
|
|
195
|
+
catch (err) {
|
|
196
|
+
cache.db.exec('ROLLBACK');
|
|
197
|
+
throw err;
|
|
198
|
+
}
|
|
199
|
+
return { ...result, contentHash };
|
|
200
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export interface ResolvedPackage {
|
|
2
|
+
name: string;
|
|
3
|
+
version: string;
|
|
4
|
+
root: string;
|
|
5
|
+
entryDts: string | null;
|
|
6
|
+
readme: string | null;
|
|
7
|
+
}
|
|
8
|
+
export declare class ResolveError extends Error {
|
|
9
|
+
readonly kind: 'not_installed' | 'invalid_name';
|
|
10
|
+
constructor(kind: 'not_installed' | 'invalid_name', message: string);
|
|
11
|
+
}
|
|
12
|
+
export declare function resolvePackage(moduleName: string, cwd: string): ResolvedPackage;
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import { createRequire } from 'node:module';
|
|
2
|
+
import * as fs from 'node:fs';
|
|
3
|
+
import * as path from 'node:path';
|
|
4
|
+
export class ResolveError extends Error {
|
|
5
|
+
kind;
|
|
6
|
+
constructor(kind, message) {
|
|
7
|
+
super(message);
|
|
8
|
+
this.kind = kind;
|
|
9
|
+
this.name = 'ResolveError';
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
const MODULE_NAME_RE = /^(?:@[a-z0-9-_.]+\/)?[a-z0-9-_.]+(?:\/[a-z0-9-_./]+)?$/i;
|
|
13
|
+
function isValidModuleName(name) {
|
|
14
|
+
if (!name || name.includes('..') || name.startsWith('/'))
|
|
15
|
+
return false;
|
|
16
|
+
return MODULE_NAME_RE.test(name);
|
|
17
|
+
}
|
|
18
|
+
function parentPackageName(moduleName) {
|
|
19
|
+
if (moduleName.startsWith('@')) {
|
|
20
|
+
const parts = moduleName.split('/');
|
|
21
|
+
return `${parts[0]}/${parts[1]}`;
|
|
22
|
+
}
|
|
23
|
+
return moduleName.split('/')[0];
|
|
24
|
+
}
|
|
25
|
+
function readPackageJson(file) {
|
|
26
|
+
return JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
27
|
+
}
|
|
28
|
+
function extractTypesFromExports(exports) {
|
|
29
|
+
if (!exports || typeof exports === 'string')
|
|
30
|
+
return null;
|
|
31
|
+
const root = exports['.'];
|
|
32
|
+
if (!root || typeof root !== 'object')
|
|
33
|
+
return null;
|
|
34
|
+
const r = root;
|
|
35
|
+
const t = r.types;
|
|
36
|
+
return typeof t === 'string' ? t : null;
|
|
37
|
+
}
|
|
38
|
+
function findReadme(root) {
|
|
39
|
+
const candidate = path.join(root, 'README.md');
|
|
40
|
+
if (fs.existsSync(candidate))
|
|
41
|
+
return candidate;
|
|
42
|
+
let entries;
|
|
43
|
+
try {
|
|
44
|
+
entries = fs.readdirSync(root);
|
|
45
|
+
}
|
|
46
|
+
catch {
|
|
47
|
+
return null;
|
|
48
|
+
}
|
|
49
|
+
const match = entries.find(e => e.toLowerCase() === 'readme.md');
|
|
50
|
+
return match ? path.join(root, match) : null;
|
|
51
|
+
}
|
|
52
|
+
function resolveEntryDts(moduleName, parent, root, pkg) {
|
|
53
|
+
if (moduleName !== parent) {
|
|
54
|
+
const subpath = moduleName.slice(parent.length + 1);
|
|
55
|
+
const candidates = [`${subpath}.d.ts`, `${subpath}/index.d.ts`, subpath];
|
|
56
|
+
for (const c of candidates) {
|
|
57
|
+
const abs = path.join(root, c);
|
|
58
|
+
if (fs.existsSync(abs) && abs.endsWith('.d.ts'))
|
|
59
|
+
return abs;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
const fromTypes = pkg.types || pkg.typings;
|
|
63
|
+
if (fromTypes) {
|
|
64
|
+
const abs = path.resolve(root, fromTypes);
|
|
65
|
+
if (fs.existsSync(abs))
|
|
66
|
+
return abs;
|
|
67
|
+
}
|
|
68
|
+
const fromExports = extractTypesFromExports(pkg.exports);
|
|
69
|
+
if (fromExports) {
|
|
70
|
+
const abs = path.resolve(root, fromExports);
|
|
71
|
+
if (fs.existsSync(abs))
|
|
72
|
+
return abs;
|
|
73
|
+
}
|
|
74
|
+
const fallback = path.join(root, 'index.d.ts');
|
|
75
|
+
if (fs.existsSync(fallback))
|
|
76
|
+
return fallback;
|
|
77
|
+
return null;
|
|
78
|
+
}
|
|
79
|
+
export function resolvePackage(moduleName, cwd) {
|
|
80
|
+
if (!isValidModuleName(moduleName)) {
|
|
81
|
+
throw new ResolveError('invalid_name', `Invalid module name: "${moduleName}"`);
|
|
82
|
+
}
|
|
83
|
+
const parent = parentPackageName(moduleName);
|
|
84
|
+
// First try: use createRequire (works when package.json is exported)
|
|
85
|
+
const requireFromCwd = createRequire(path.join(cwd, '__pi-worker-docs-sentinel__'));
|
|
86
|
+
let pkgJsonPath = null;
|
|
87
|
+
try {
|
|
88
|
+
pkgJsonPath = requireFromCwd.resolve(`${parent}/package.json`);
|
|
89
|
+
}
|
|
90
|
+
catch (err) {
|
|
91
|
+
const code = err.code;
|
|
92
|
+
if (code !== 'MODULE_NOT_FOUND' && code !== 'ERR_PACKAGE_PATH_NOT_EXPORTED')
|
|
93
|
+
throw err;
|
|
94
|
+
// Fall through to direct filesystem fallback below
|
|
95
|
+
}
|
|
96
|
+
// Second try: walk node_modules directly (handles packages that don't export package.json)
|
|
97
|
+
if (!pkgJsonPath) {
|
|
98
|
+
const direct = findPackageJsonInNodeModules(parent, cwd);
|
|
99
|
+
if (!direct) {
|
|
100
|
+
throw new ResolveError('not_installed', `Package "${parent}" is not installed in ${cwd}. Run \`npm install ${parent}\` (or \`bun add ${parent}\`) and retry.`);
|
|
101
|
+
}
|
|
102
|
+
pkgJsonPath = direct;
|
|
103
|
+
}
|
|
104
|
+
const root = path.dirname(pkgJsonPath);
|
|
105
|
+
const pkg = readPackageJson(pkgJsonPath);
|
|
106
|
+
return {
|
|
107
|
+
name: pkg.name ?? parent,
|
|
108
|
+
version: pkg.version ?? '0.0.0',
|
|
109
|
+
root,
|
|
110
|
+
entryDts: resolveEntryDts(moduleName, parent, root, pkg),
|
|
111
|
+
readme: findReadme(root)
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
function findPackageJsonInNodeModules(parent, startDir) {
|
|
115
|
+
const segments = parent.startsWith('@') ? parent.split('/').slice(0, 2) : [parent.split('/')[0]];
|
|
116
|
+
let dir = startDir;
|
|
117
|
+
while (true) {
|
|
118
|
+
const candidate = path.join(dir, 'node_modules', ...segments, 'package.json');
|
|
119
|
+
if (fs.existsSync(candidate))
|
|
120
|
+
return candidate;
|
|
121
|
+
const up = path.dirname(dir);
|
|
122
|
+
if (up === dir)
|
|
123
|
+
return null;
|
|
124
|
+
dir = up;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { CacheHandle } from './docs-cache.js';
|
|
2
|
+
export interface RetrievedChunk {
|
|
3
|
+
filePath: string;
|
|
4
|
+
kind: 'dts' | 'readme';
|
|
5
|
+
content: string;
|
|
6
|
+
rank: number;
|
|
7
|
+
}
|
|
8
|
+
export interface RetrieveOptions {
|
|
9
|
+
name: string;
|
|
10
|
+
version: string;
|
|
11
|
+
query: string;
|
|
12
|
+
limit?: number;
|
|
13
|
+
contentBudget?: number;
|
|
14
|
+
}
|
|
15
|
+
export declare function retrieveChunks(cache: CacheHandle, opts: RetrieveOptions): RetrievedChunk[];
|