@mjasnikovs/pi-task 0.40.1 → 0.40.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/task/enrichment.d.ts +14 -1
- package/dist/task/enrichment.js +16 -1
- package/dist/task/external-context.d.ts +12 -0
- package/dist/task/external-context.js +11 -4
- package/dist/workers/docs-chunk.js +14 -5
- package/dist/workers/docs-core.js +26 -2
- package/dist/workers/docs-ecosystems.d.ts +20 -0
- package/dist/workers/docs-ecosystems.js +34 -5
- package/dist/workers/docs-index.d.ts +28 -0
- package/dist/workers/docs-index.js +71 -3
- package/dist/workers/docs-project.js +5 -1
- package/dist/workers/docs-retrieve.d.ts +7 -0
- package/dist/workers/docs-retrieve.js +100 -1
- package/dist/workers/eco-cargo.d.ts +14 -3
- package/dist/workers/eco-cargo.js +44 -3
- package/dist/workers/eco-hackage.d.ts +9 -0
- package/dist/workers/eco-hackage.js +46 -0
- package/dist/workers/pi-worker-docs.js +5 -3
- package/dist/workers/typeonly-log.d.ts +13 -0
- package/package.json +1 -1
|
@@ -6,8 +6,21 @@
|
|
|
6
6
|
* denylisted shell names are dropped, a URL's trailing sentence punctuation is
|
|
7
7
|
* stripped, docs targets stop at ENRICH_CAP while version targets continue to
|
|
8
8
|
* ENRICH_VERSION_CAP, and the version list is a strict superset of the docs list.
|
|
9
|
+
*
|
|
10
|
+
* Package extraction is additionally gated on the caller's declared dependencies;
|
|
11
|
+
* see the `declared` parameter on {@link extractEnrichTargets}.
|
|
12
|
+
*/
|
|
13
|
+
export declare function extractEnrichTargets(text: string,
|
|
14
|
+
/**
|
|
15
|
+
* The project's declared dependencies. A backticked name outside this set is
|
|
16
|
+
* not enriched: the model backticks filenames (`config.ts`, `tsconfig.json`)
|
|
17
|
+
* and field names (`name`, `port`) far more often than package names, and each
|
|
18
|
+
* of those is also a real, unrelated package on the public registry — so the
|
|
19
|
+
* permissive read fetched and indexed a stranger's code under the project's
|
|
20
|
+
* own filename. Omit when the manifest is unreadable, which is not the same
|
|
21
|
+
* fact as "declares nothing".
|
|
9
22
|
*/
|
|
10
|
-
|
|
23
|
+
declared?: ReadonlySet<string>): {
|
|
11
24
|
/** Packages that get a (heavy) docs fetch — capped at ENRICH_CAP. */
|
|
12
25
|
packages: string[];
|
|
13
26
|
/**
|
package/dist/task/enrichment.js
CHANGED
|
@@ -6,6 +6,9 @@
|
|
|
6
6
|
* denylisted shell names are dropped, a URL's trailing sentence punctuation is
|
|
7
7
|
* stripped, docs targets stop at ENRICH_CAP while version targets continue to
|
|
8
8
|
* ENRICH_VERSION_CAP, and the version list is a strict superset of the docs list.
|
|
9
|
+
*
|
|
10
|
+
* Package extraction is additionally gated on the caller's declared dependencies;
|
|
11
|
+
* see the `declared` parameter on {@link extractEnrichTargets}.
|
|
9
12
|
*/
|
|
10
13
|
const ENRICH_PKG_RE = /`((?:@[a-z0-9][a-z0-9._-]*\/)?[a-z0-9][a-z0-9._-]*)`/g;
|
|
11
14
|
const ENRICH_URL_RE = /https?:\/\/[^\s)`>]+/g;
|
|
@@ -79,13 +82,25 @@ function parseServices(text) {
|
|
|
79
82
|
}
|
|
80
83
|
return out;
|
|
81
84
|
}
|
|
82
|
-
export function extractEnrichTargets(text
|
|
85
|
+
export function extractEnrichTargets(text,
|
|
86
|
+
/**
|
|
87
|
+
* The project's declared dependencies. A backticked name outside this set is
|
|
88
|
+
* not enriched: the model backticks filenames (`config.ts`, `tsconfig.json`)
|
|
89
|
+
* and field names (`name`, `port`) far more often than package names, and each
|
|
90
|
+
* of those is also a real, unrelated package on the public registry — so the
|
|
91
|
+
* permissive read fetched and indexed a stranger's code under the project's
|
|
92
|
+
* own filename. Omit when the manifest is unreadable, which is not the same
|
|
93
|
+
* fact as "declares nothing".
|
|
94
|
+
*/
|
|
95
|
+
declared) {
|
|
83
96
|
const pkgs = [];
|
|
84
97
|
const seen = new Set();
|
|
85
98
|
for (const m of text.matchAll(ENRICH_PKG_RE)) {
|
|
86
99
|
const t = m[1];
|
|
87
100
|
if (ENRICH_DENYLIST.has(t) || seen.has(t))
|
|
88
101
|
continue;
|
|
102
|
+
if (declared && !declared.has(t))
|
|
103
|
+
continue;
|
|
89
104
|
seen.add(t);
|
|
90
105
|
pkgs.push(t);
|
|
91
106
|
if (pkgs.length >= ENRICH_VERSION_CAP)
|
|
@@ -83,6 +83,18 @@ export interface ExternalContextPolicy {
|
|
|
83
83
|
targetCap?: number;
|
|
84
84
|
/** Max services fanned out. Omit for uncapped; the auto-answer path caps at 2. */
|
|
85
85
|
serviceCap?: number;
|
|
86
|
+
/**
|
|
87
|
+
* Fan named packages out to a docs body. The RESEARCH path does not, and the
|
|
88
|
+
* auto-answer path does.
|
|
89
|
+
*
|
|
90
|
+
* Research retrieves against `refined.split('\n')[0]` — the literal word
|
|
91
|
+
* "GOAL" for every refined spec — so its bodies are whatever ranks against
|
|
92
|
+
* that, pasted raw. The live run of 2026-09-05 found no research output
|
|
93
|
+
* citing one, while the model's own docs tool answered 49 real questions
|
|
94
|
+
* across the same three runs. The auto-answer path asks a focused child an
|
|
95
|
+
* actual question, which is the shape that works.
|
|
96
|
+
*/
|
|
97
|
+
packageDocs?: boolean;
|
|
86
98
|
/**
|
|
87
99
|
* A cheap live version lookup for every named dep that did NOT get a docs
|
|
88
100
|
* target, so a version block exists for ALL of them. Omit to disable, as the
|
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
* Run against a mixed source, the emitted headings come out in exactly that order:
|
|
23
23
|
* `### npm:` then `### docs:` then `### url:` then `### service:`.
|
|
24
24
|
*/
|
|
25
|
-
import { chooseEcosystem, defaultEcosystemIo } from '../workers/docs-ecosystems.js';
|
|
25
|
+
import { chooseEcosystem, declaredDepNames, defaultEcosystemIo } from '../workers/docs-ecosystems.js';
|
|
26
26
|
import { docsRaw } from '../workers/docs-core.js';
|
|
27
27
|
import { fetchRaw } from '../workers/fetch-core.js';
|
|
28
28
|
import { formatNpmVersionSection, npmVersionLookup } from '../workers/npm-version.js';
|
|
@@ -39,18 +39,24 @@ const RAW_BODY_LIMIT = 4000;
|
|
|
39
39
|
*/
|
|
40
40
|
export async function buildExternalContext(source, deps, lookups, policy = {}) {
|
|
41
41
|
const searchFn = lookups.search ?? defaultSearch;
|
|
42
|
-
const enrichTargets = extractEnrichTargets(source);
|
|
42
|
+
const enrichTargets = extractEnrichTargets(source, declaredDepNames(deps.cwd));
|
|
43
43
|
// Packages lead urls, then the combined cap applies — so a capped run spends
|
|
44
44
|
// its budget on named deps first. Uncapped, this is just "packages, then urls".
|
|
45
|
+
const docsPackages = policy.packageDocs === false ? [] : enrichTargets.packages;
|
|
45
46
|
const targets = [
|
|
46
|
-
...
|
|
47
|
+
...docsPackages.map(name => ({ kind: 'pkg', name })),
|
|
47
48
|
...enrichTargets.urls.map(name => ({ kind: 'url', name }))
|
|
48
49
|
].slice(0, policy.targetCap ?? Number.POSITIVE_INFINITY);
|
|
49
50
|
const services = enrichTargets.services.slice(0, policy.serviceCap ?? Number.POSITIVE_INFINITY);
|
|
50
51
|
const versionLookup = policy.versionLookup;
|
|
51
52
|
const docsTargets = new Set(targets.filter(t => t.kind === 'pkg').map(t => t.name));
|
|
52
53
|
const extraVersionPkgs = versionLookup ? enrichTargets.versionPackages.filter(p => !docsTargets.has(p)) : [];
|
|
53
|
-
|
|
54
|
+
// Version packages count as work here: with `packageDocs: false` a named dep
|
|
55
|
+
// produces no target at all, and returning early would drop its version block.
|
|
56
|
+
if (policy.earlyReturnOnNoTargets
|
|
57
|
+
&& targets.length === 0
|
|
58
|
+
&& services.length === 0
|
|
59
|
+
&& extraVersionPkgs.length === 0)
|
|
54
60
|
return '';
|
|
55
61
|
const startedAt = Date.now();
|
|
56
62
|
const [targetResults, serviceResults, extraVersionResults] = await Promise.all([
|
|
@@ -191,6 +197,7 @@ export async function gatherExternalContext(refined, deps) {
|
|
|
191
197
|
search: deps.searchFn
|
|
192
198
|
}, {
|
|
193
199
|
versionLookup,
|
|
200
|
+
packageDocs: false,
|
|
194
201
|
subStepLabel: 'enrichment',
|
|
195
202
|
earlyReturnOnNoTargets: true
|
|
196
203
|
});
|
|
@@ -37,16 +37,25 @@ export const README_SPLIT_RE = /^#{1,2} /m;
|
|
|
37
37
|
export function splitAtMatches(text, re) {
|
|
38
38
|
const parts = [];
|
|
39
39
|
let lastIndex = 0;
|
|
40
|
+
let acceptedEnd = 0;
|
|
40
41
|
let m;
|
|
41
42
|
while ((m = re.exec(text))) {
|
|
43
|
+
// Scan resumes ONE character in, not past the match: the regex's trailing
|
|
44
|
+
// `\s+` can span a newline, so a genuinely separate line-anchored
|
|
45
|
+
// declaration may start inside what this match consumed.
|
|
46
|
+
re.lastIndex = m.index + 1;
|
|
47
|
+
// But a match that starts inside the last ACCEPTED one is that match's own
|
|
48
|
+
// tail, not a new declaration, and cutting there severs a declaration from
|
|
49
|
+
// the modifiers and attributes that open it. `export\nfunction a(){}` is
|
|
50
|
+
// one declaration; so is a cargo `#[cfg(…)]` above its `impl`, which
|
|
51
|
+
// CARGO_DECL_SPLIT_RE absorbs on purpose and this used to hand back as a
|
|
52
|
+
// chunk holding nothing but the attribute.
|
|
53
|
+
if (m.index < acceptedEnd)
|
|
54
|
+
continue;
|
|
42
55
|
if (m.index > lastIndex)
|
|
43
56
|
parts.push(text.slice(lastIndex, m.index));
|
|
44
57
|
lastIndex = m.index;
|
|
45
|
-
|
|
46
|
-
// span a newline, so the next line-anchored declaration may start INSIDE
|
|
47
|
-
// what this match consumed: `export\nfunction a(){}` is two chunks here
|
|
48
|
-
// and one if the scan resumes past the match.
|
|
49
|
-
re.lastIndex = m.index + 1;
|
|
58
|
+
acceptedEnd = m.index + m[0].length;
|
|
50
59
|
}
|
|
51
60
|
if (lastIndex < text.length)
|
|
52
61
|
parts.push(text.slice(lastIndex));
|
|
@@ -128,9 +128,32 @@ export function findDeclaredRange(parentPkg, cwd) {
|
|
|
128
128
|
* does or does not say has to be a sentence about `bun`.
|
|
129
129
|
*/
|
|
130
130
|
export function buildVersionBanner(pin, resolved, version, cwd, profile = ECOSYSTEMS.npm) {
|
|
131
|
+
const asked = pin?.asked ?? resolved;
|
|
132
|
+
// Resolvable is not usable. A lock file, a cabal plan and `node_modules` are
|
|
133
|
+
// all the transitive CLOSURE, so the tool can answer in full confidence about
|
|
134
|
+
// a package the project may not import. That is what made the Rust run of
|
|
135
|
+
// 2026-09-05 a hard fail: a correct `tower::util::ServiceExt` answer, the
|
|
136
|
+
// import written, and E0433 "cannot find module or crate tower" from the compiler.
|
|
137
|
+
const undeclared = undeclaredNotice(asked, cwd, profile);
|
|
131
138
|
if (!pin)
|
|
139
|
+
return undeclared;
|
|
140
|
+
return undeclared + pinBanner(pin, asked, resolved, version, cwd, profile);
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* The one sentence a package present-but-not-declared needs, or `''` when it is
|
|
144
|
+
* declared or when no manifest could be read.
|
|
145
|
+
*/
|
|
146
|
+
function undeclaredNotice(asked, cwd, profile) {
|
|
147
|
+
const declared = profile.manifestDeps(cwd);
|
|
148
|
+
const root = profile.parentPackage(asked);
|
|
149
|
+
if (!declared || declared.has(root) || declared.has(asked))
|
|
132
150
|
return '';
|
|
133
|
-
|
|
151
|
+
return (`[DEPENDENCY] "${root}" is present in this project but is not a declared `
|
|
152
|
+
+ `dependency in ${profile.manifestLabel} — it resolves only because something `
|
|
153
|
+
+ `else pulled it in. Add it to ${profile.manifestLabel} before importing it, or `
|
|
154
|
+
+ `the build will not find it.\n\n`);
|
|
155
|
+
}
|
|
156
|
+
function pinBanner(pin, asked, resolved, version, cwd, profile) {
|
|
134
157
|
const grounded = resolved !== asked ? ` The types this answer reads come from ${resolved}.` : '';
|
|
135
158
|
const manifest = profile.manifestLabel;
|
|
136
159
|
const registry = profile.registryLabel;
|
|
@@ -485,7 +508,8 @@ function docsRawCached(cache, pkg, profile, query, ensureIndexed, retrieveChunks
|
|
|
485
508
|
version: pkg.version,
|
|
486
509
|
query,
|
|
487
510
|
limit: DEFAULT_LIMIT,
|
|
488
|
-
contentBudget: DEFAULT_BUDGET
|
|
511
|
+
contentBudget: DEFAULT_BUDGET,
|
|
512
|
+
typeKeywords: profile.typeKeywords
|
|
489
513
|
});
|
|
490
514
|
}
|
|
491
515
|
catch (err) {
|
|
@@ -84,6 +84,11 @@ export interface EcosystemProfile {
|
|
|
84
84
|
surface: (content: string) => string;
|
|
85
85
|
/** Where a declaration begins, so a chunk never splits a signature. */
|
|
86
86
|
declSplitRe: RegExp;
|
|
87
|
+
/**
|
|
88
|
+
* The keywords that INTRODUCE a named type in this language, for finding the
|
|
89
|
+
* chunk that defines a name rather than the many that use it.
|
|
90
|
+
*/
|
|
91
|
+
typeKeywords: readonly string[];
|
|
87
92
|
/** Line-comment marker, used to label a chunk with the file it came from. */
|
|
88
93
|
commentPrefix: string;
|
|
89
94
|
/** Directories the surface walk never descends into: tests, build output. */
|
|
@@ -105,6 +110,15 @@ export interface EcosystemProfile {
|
|
|
105
110
|
* package's version, which is a different fact from "declares nothing".
|
|
106
111
|
*/
|
|
107
112
|
declaredDeps: (cwd: string) => Record<string, string> | undefined;
|
|
113
|
+
/**
|
|
114
|
+
* The names the MANIFEST itself declares — what the project may import.
|
|
115
|
+
*
|
|
116
|
+
* Distinct from {@link declaredDeps}, which for cargo and hackage reads a
|
|
117
|
+
* lock or plan file: that is the whole transitive closure, so it answers
|
|
118
|
+
* "does this resolve" and not "may this be used". Undefined when there is no
|
|
119
|
+
* readable manifest, which is not the same fact as "declares nothing".
|
|
120
|
+
*/
|
|
121
|
+
manifestDeps: (cwd: string) => Set<string> | undefined;
|
|
108
122
|
}
|
|
109
123
|
/** Overrides a caller has already been given its own copies of. */
|
|
110
124
|
export interface NpmProfileHooks {
|
|
@@ -134,6 +148,12 @@ export declare const ECOSYSTEMS: {
|
|
|
134
148
|
};
|
|
135
149
|
/** Which ecosystems `cwd` looks like a project of, in roster order. */
|
|
136
150
|
export declare function detectEcosystems(cwd: string, roster?: readonly EcosystemProfile[]): EcosystemId[];
|
|
151
|
+
/**
|
|
152
|
+
* Every dependency `cwd`'s manifests declare, across the ecosystems it is a
|
|
153
|
+
* project of. Undefined when no detected ecosystem could read its manifest —
|
|
154
|
+
* "we cannot tell", which callers must not read as "declares nothing".
|
|
155
|
+
*/
|
|
156
|
+
export declare function declaredDepNames(cwd: string, roster?: readonly EcosystemProfile[]): Set<string> | undefined;
|
|
137
157
|
export type EcosystemChoice = {
|
|
138
158
|
ok: true;
|
|
139
159
|
profile: EcosystemProfile;
|
|
@@ -21,8 +21,8 @@ import { runAutoInstall, findDeclaredRange, extractParentPackage, resolveTypeSou
|
|
|
21
21
|
import { resolvePackage, isDtsFile, isValidModuleName } from './docs-resolve.js';
|
|
22
22
|
import { DECL_SPLIT_RE } from './docs-chunk.js';
|
|
23
23
|
import { npmVersionLookup } from './npm-version.js';
|
|
24
|
-
import { resolveCrate, cratesLatest, crateTarballUrl, crateOf, isValidCrateName, isRustFile, lockedVersion, rustSurface, cargoProjectName, childDirs, lockedDeps, CARGO_DECL_SPLIT_RE } from './eco-cargo.js';
|
|
25
|
-
import { resolveHackage, hackageLatest, hackageVersion, hackageTarballUrl, hackageExtractDir, hackageProjectName, findCabalTarball, cachedVersions, resolvedVersions, isValidHackageName, isHaskellFile, haskellSurface, HACKAGE_DECL_SPLIT_RE, HACKAGE_SKIP_DIRS } from './eco-hackage.js';
|
|
24
|
+
import { resolveCrate, cratesLatest, crateTarballUrl, crateOf, isValidCrateName, isRustFile, lockedVersion, rustSurface, cargoProjectName, childDirs, lockedDeps, manifestCrates, CARGO_DECL_SPLIT_RE } from './eco-cargo.js';
|
|
25
|
+
import { resolveHackage, hackageLatest, hackageVersion, hackageTarballUrl, hackageExtractDir, hackageProjectName, findCabalTarball, cachedVersions, resolvedVersions, manifestPackages, isValidHackageName, isHaskellFile, haskellSurface, HACKAGE_DECL_SPLIT_RE, HACKAGE_SKIP_DIRS } from './eco-hackage.js';
|
|
26
26
|
import { runChild } from '../shared/child-process.js';
|
|
27
27
|
/**
|
|
28
28
|
* Is any of `names` present at `cwd` or above it?
|
|
@@ -128,6 +128,7 @@ export function npmProfile(hooks = {}) {
|
|
|
128
128
|
isSurfaceFile: isDtsFile,
|
|
129
129
|
surface: content => content,
|
|
130
130
|
declSplitRe: DECL_SPLIT_RE,
|
|
131
|
+
typeKeywords: ['interface', 'type', 'class', 'enum'],
|
|
131
132
|
commentPrefix: '//',
|
|
132
133
|
// A nested node_modules is another package's surface, never this one's.
|
|
133
134
|
skipDirs: ['node_modules'],
|
|
@@ -135,7 +136,11 @@ export function npmProfile(hooks = {}) {
|
|
|
135
136
|
packageSubject: 'an npm package',
|
|
136
137
|
projectGlobs: ['*.ts', '*.tsx'],
|
|
137
138
|
projectName: npmProjectName,
|
|
138
|
-
declaredDeps: npmDeclaredDeps
|
|
139
|
+
declaredDeps: npmDeclaredDeps,
|
|
140
|
+
manifestDeps: cwd => {
|
|
141
|
+
const deps = npmDeclaredDeps(cwd);
|
|
142
|
+
return deps && new Set(Object.keys(deps));
|
|
143
|
+
}
|
|
139
144
|
};
|
|
140
145
|
}
|
|
141
146
|
const NPM_DEP_BLOCKS = [
|
|
@@ -266,13 +271,15 @@ const cargoProfile = {
|
|
|
266
271
|
isSurfaceFile: isRustFile,
|
|
267
272
|
surface: content => rustSurface(content),
|
|
268
273
|
declSplitRe: CARGO_DECL_SPLIT_RE,
|
|
274
|
+
typeKeywords: ['struct', 'trait', 'enum', 'type', 'union'],
|
|
269
275
|
commentPrefix: '//',
|
|
270
276
|
skipDirs: ['tests', 'benches', 'examples', 'target'],
|
|
271
277
|
surfaceLabel: '.rs source or README',
|
|
272
278
|
packageSubject: 'a Rust crate from crates.io',
|
|
273
279
|
projectGlobs: ['*.rs'],
|
|
274
280
|
projectName: cargoProjectName,
|
|
275
|
-
declaredDeps: lockedDeps
|
|
281
|
+
declaredDeps: lockedDeps,
|
|
282
|
+
manifestDeps: manifestCrates
|
|
276
283
|
};
|
|
277
284
|
/**
|
|
278
285
|
* Unpack a Hackage tarball into the tool's own directory. Whether it came from
|
|
@@ -351,13 +358,15 @@ const hackageProfile = {
|
|
|
351
358
|
isSurfaceFile: isHaskellFile,
|
|
352
359
|
surface: haskellSurface,
|
|
353
360
|
declSplitRe: HACKAGE_DECL_SPLIT_RE,
|
|
361
|
+
typeKeywords: ['type', 'data', 'newtype', 'class'],
|
|
354
362
|
commentPrefix: '--',
|
|
355
363
|
skipDirs: HACKAGE_SKIP_DIRS,
|
|
356
364
|
surfaceLabel: '.hs source or README',
|
|
357
365
|
packageSubject: 'a Haskell package from Hackage',
|
|
358
366
|
projectGlobs: ['*.hs'],
|
|
359
367
|
projectName: hackageProjectName,
|
|
360
|
-
declaredDeps: resolvedVersions
|
|
368
|
+
declaredDeps: resolvedVersions,
|
|
369
|
+
manifestDeps: manifestPackages
|
|
361
370
|
};
|
|
362
371
|
/** A cabal, stack or hpack project declares itself with one of these. */
|
|
363
372
|
function hasCabalManifest(cwd) {
|
|
@@ -385,6 +394,26 @@ export const ECOSYSTEMS = {
|
|
|
385
394
|
export function detectEcosystems(cwd, roster = Object.values(ECOSYSTEMS)) {
|
|
386
395
|
return roster.filter(p => p.detect(cwd)).map(p => p.id);
|
|
387
396
|
}
|
|
397
|
+
/**
|
|
398
|
+
* Every dependency `cwd`'s manifests declare, across the ecosystems it is a
|
|
399
|
+
* project of. Undefined when no detected ecosystem could read its manifest —
|
|
400
|
+
* "we cannot tell", which callers must not read as "declares nothing".
|
|
401
|
+
*/
|
|
402
|
+
export function declaredDepNames(cwd, roster = Object.values(ECOSYSTEMS)) {
|
|
403
|
+
let any = false;
|
|
404
|
+
const names = new Set();
|
|
405
|
+
for (const p of roster) {
|
|
406
|
+
if (!p.detect(cwd))
|
|
407
|
+
continue;
|
|
408
|
+
const deps = p.manifestDeps(cwd);
|
|
409
|
+
if (!deps)
|
|
410
|
+
continue;
|
|
411
|
+
any = true;
|
|
412
|
+
for (const name of deps)
|
|
413
|
+
names.add(name);
|
|
414
|
+
}
|
|
415
|
+
return any ? names : undefined;
|
|
416
|
+
}
|
|
388
417
|
/**
|
|
389
418
|
* Which ecosystem a lookup belongs to. The MANIFEST decides, never the model:
|
|
390
419
|
* `text`, `base`, `aeson`, `tokio` and `clap` are all real npm packages as well
|
|
@@ -7,4 +7,32 @@ export interface IndexResult {
|
|
|
7
7
|
chunksWritten: number;
|
|
8
8
|
contentHash: string;
|
|
9
9
|
}
|
|
10
|
+
/**
|
|
11
|
+
* The gate that decides whether a package needs re-indexing.
|
|
12
|
+
*
|
|
13
|
+
* The entry file goes in SURFACED, not raw. What is cached is the extractor's
|
|
14
|
+
* OUTPUT, so a build whose extractor changed has stale chunks even though every
|
|
15
|
+
* byte on disk is identical — a crate indexed before the braced-`use` fix keeps
|
|
16
|
+
* `pub use crate::runtime::;` forever, because name, version and file bytes all
|
|
17
|
+
* still match. Surfacing here costs one file and makes the hash answer the
|
|
18
|
+
* question actually being asked: would re-reading produce the same chunks?
|
|
19
|
+
*
|
|
20
|
+
* The CHUNKER counts too, for the same reason: the rows are chunks, not surface,
|
|
21
|
+
* so a fix to where a declaration is cut leaves stale rows behind on its own. So
|
|
22
|
+
* does WHICH FILES are read: dropping a package's duplicate `.d.cts` twins
|
|
23
|
+
* changes the rows without changing a byte on disk.
|
|
24
|
+
*
|
|
25
|
+
* It is not total. An extractor change that alters only files BELOW the entry
|
|
26
|
+
* goes unnoticed; deleting the cache is still the escape hatch for that.
|
|
27
|
+
*/
|
|
28
|
+
/**
|
|
29
|
+
* The chunker's own source, so a cut-point fix invalidates every cached package.
|
|
30
|
+
*
|
|
31
|
+
* `declSplitRe.source` alone does not do it: the attribute-orphaning bug was in
|
|
32
|
+
* `splitAtMatches`, not in any profile's regex, so the fingerprint sat still while
|
|
33
|
+
* the rows it describes changed. A package indexed before that fix keeps its
|
|
34
|
+
* dangling `#[cfg(...)]` chunks forever, and the hash is the only thing that would
|
|
35
|
+
* have said so.
|
|
36
|
+
*/
|
|
37
|
+
export declare function chunkerFingerprint(): string;
|
|
10
38
|
export declare function ensureIndexed(cache: CacheHandle, pkg: ResolvedPackage, profile?: EcosystemProfile): IndexResult;
|
|
@@ -2,7 +2,7 @@ import { createHash } from 'node:crypto';
|
|
|
2
2
|
import * as fs from 'node:fs';
|
|
3
3
|
import * as path from 'node:path';
|
|
4
4
|
import {} from './docs-resolve.js';
|
|
5
|
-
import { chunkDeclarations, chunkReadme } from './docs-chunk.js';
|
|
5
|
+
import { chunkDeclarations, chunkReadme, splitAtMatches } from './docs-chunk.js';
|
|
6
6
|
import { ECOSYSTEMS } from './docs-ecosystems.js';
|
|
7
7
|
const ZERO_SEP = Buffer.from([0]);
|
|
8
8
|
/**
|
|
@@ -16,17 +16,38 @@ const ZERO_SEP = Buffer.from([0]);
|
|
|
16
16
|
* question actually being asked: would re-reading produce the same chunks?
|
|
17
17
|
*
|
|
18
18
|
* The CHUNKER counts too, for the same reason: the rows are chunks, not surface,
|
|
19
|
-
* so a fix to where a declaration is cut leaves stale rows behind on its own.
|
|
19
|
+
* so a fix to where a declaration is cut leaves stale rows behind on its own. So
|
|
20
|
+
* does WHICH FILES are read: dropping a package's duplicate `.d.cts` twins
|
|
21
|
+
* changes the rows without changing a byte on disk.
|
|
20
22
|
*
|
|
21
23
|
* It is not total. An extractor change that alters only files BELOW the entry
|
|
22
24
|
* goes unnoticed; deleting the cache is still the escape hatch for that.
|
|
23
25
|
*/
|
|
26
|
+
/**
|
|
27
|
+
* The chunker's own source, so a cut-point fix invalidates every cached package.
|
|
28
|
+
*
|
|
29
|
+
* `declSplitRe.source` alone does not do it: the attribute-orphaning bug was in
|
|
30
|
+
* `splitAtMatches`, not in any profile's regex, so the fingerprint sat still while
|
|
31
|
+
* the rows it describes changed. A package indexed before that fix keeps its
|
|
32
|
+
* dangling `#[cfg(...)]` chunks forever, and the hash is the only thing that would
|
|
33
|
+
* have said so.
|
|
34
|
+
*/
|
|
35
|
+
export function chunkerFingerprint() {
|
|
36
|
+
return `${String(splitAtMatches)}\u0000${String(chunkDeclarations)}\u0000${String(chunkReadme)}`;
|
|
37
|
+
}
|
|
24
38
|
function computeContentHash(pkg, profile) {
|
|
25
39
|
const hash = createHash('sha256');
|
|
26
40
|
hash.update(Buffer.from(`${pkg.name}@${pkg.version}`, 'utf8'));
|
|
27
41
|
hash.update(ZERO_SEP);
|
|
28
42
|
hash.update(Buffer.from(`${profile.declSplitRe.source}\u0000${profile.commentPrefix}`, 'utf8'));
|
|
29
43
|
hash.update(ZERO_SEP);
|
|
44
|
+
hash.update(Buffer.from(chunkerFingerprint(), 'utf8'));
|
|
45
|
+
hash.update(ZERO_SEP);
|
|
46
|
+
// Source text, the same trick as `declSplitRe.source`: the fingerprint moves
|
|
47
|
+
// whenever the selection rule does, with nothing to remember to bump.
|
|
48
|
+
hash.update(Buffer.from(`${String(profile.isSurfaceFile)}\u0000${String(dropParallelDeclarations)}`
|
|
49
|
+
+ `\u0000${String(dropDeadMajors)}`, 'utf8'));
|
|
50
|
+
hash.update(ZERO_SEP);
|
|
30
51
|
if (pkg.entry && fs.existsSync(pkg.entry)) {
|
|
31
52
|
try {
|
|
32
53
|
hash.update(Buffer.from(profile.surface(fs.readFileSync(pkg.entry, 'utf8')), 'utf8'));
|
|
@@ -83,9 +104,56 @@ function walkSurface(root, profile) {
|
|
|
83
104
|
}
|
|
84
105
|
return out.sort();
|
|
85
106
|
}
|
|
107
|
+
/**
|
|
108
|
+
* Drop a `.d.cts` / `.d.mts` that sits beside a `.d.ts` of the same name.
|
|
109
|
+
*
|
|
110
|
+
* Modern npm packages ship parallel declarations for ESM and CJS: the same API
|
|
111
|
+
* written twice. zod 4.5.4 indexed to 2565 chunks over 1215 distinct bodies,
|
|
112
|
+
* 1280 of them from `.d.cts`; hono, which ships none, had 704 distinct of 708.
|
|
113
|
+
* The cost is the eight-chunk retrieval budget — half of it can go to text the
|
|
114
|
+
* reader already has.
|
|
115
|
+
*
|
|
116
|
+
* The sibling test, not a blanket ban on the extensions: a package shipping only
|
|
117
|
+
* `.d.cts` still has to be readable, and all 123 of zod's had a `.d.ts` twin.
|
|
118
|
+
*/
|
|
119
|
+
function dropParallelDeclarations(files) {
|
|
120
|
+
const esm = new Set(files.filter(f => f.endsWith('.d.ts')).map(f => f.slice(0, -'.d.ts'.length)));
|
|
121
|
+
return files.filter(f => {
|
|
122
|
+
const base = /\.d\.[cm]ts$/.exec(f) ? f.slice(0, -'.d.cts'.length) : null;
|
|
123
|
+
return base === null || !esm.has(base);
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* Drop a top-level `vN/` directory holding a major the package is no longer on.
|
|
128
|
+
*
|
|
129
|
+
* zod@4.5.4 ships `v3/` for back-compat, and 414 of its 2565 chunks came from
|
|
130
|
+
* it. Nothing downstream can separate them: same identifiers, same package, same
|
|
131
|
+
* version banner, and the file path is not a ranking signal. An answer went out
|
|
132
|
+
* under `Per zod@4.5.4:` carrying v3's `email(message?): ZodString` — wrong
|
|
133
|
+
* parameter, wrong return, and silent about the `@deprecated` line sitting
|
|
134
|
+
* directly above the real declaration.
|
|
135
|
+
*
|
|
136
|
+
* Only a MISMATCHING major goes. `v4/` under 4.5.4 is the current API and is
|
|
137
|
+
* most of the package; a package whose only content lives under `v1/` keeps it.
|
|
138
|
+
*/
|
|
139
|
+
function dropDeadMajors(files, root, version) {
|
|
140
|
+
const major = /^(\d+)\./.exec(version)?.[1];
|
|
141
|
+
if (major === undefined)
|
|
142
|
+
return files;
|
|
143
|
+
const kept = files.filter(abs => {
|
|
144
|
+
const top = path.relative(root, abs).replace(/\\/g, '/').split('/')[0];
|
|
145
|
+
const dir = /^v(\d+)$/.exec(top);
|
|
146
|
+
return dir === null || dir[1] === major;
|
|
147
|
+
});
|
|
148
|
+
// A package whose whole surface lives under a `vN/` that does not match its
|
|
149
|
+
// own version is not shipping a dead major — it is shipping its API there.
|
|
150
|
+
return kept.length > 0 ? kept : files;
|
|
151
|
+
}
|
|
86
152
|
function collectFiles(pkg, profile) {
|
|
153
|
+
const walked = walkSurface(pkg.root, profile);
|
|
154
|
+
const surface = dropDeadMajors(walked, pkg.root, pkg.version);
|
|
87
155
|
return {
|
|
88
|
-
surface:
|
|
156
|
+
surface: profile.id === 'npm' ? dropParallelDeclarations(surface) : surface,
|
|
89
157
|
readme: pkg.readme
|
|
90
158
|
};
|
|
91
159
|
}
|
|
@@ -234,7 +234,11 @@ listFiles = getProjectFiles) {
|
|
|
234
234
|
version,
|
|
235
235
|
query,
|
|
236
236
|
limit: DEFAULT_LIMIT,
|
|
237
|
-
contentBudget: DEFAULT_BUDGET
|
|
237
|
+
contentBudget: DEFAULT_BUDGET,
|
|
238
|
+
// Every detected ecosystem's keywords: a polyglot project's own
|
|
239
|
+
// source has no single language, and the extra keywords only widen
|
|
240
|
+
// which definition the hop can find.
|
|
241
|
+
typeKeywords: [...new Set(projectProfiles(cwd).flatMap(p => p.typeKeywords))]
|
|
238
242
|
});
|
|
239
243
|
}
|
|
240
244
|
catch (err) {
|
|
@@ -12,6 +12,13 @@ export interface RetrieveOptions {
|
|
|
12
12
|
version: string;
|
|
13
13
|
query: string;
|
|
14
14
|
limit?: number;
|
|
15
|
+
/**
|
|
16
|
+
* The keywords that introduce a named type, for the definition hop. Passed by
|
|
17
|
+
* the caller rather than read off `EcosystemProfile` here: docs-ecosystems
|
|
18
|
+
* imports docs-core, which imports this module, and reaching back for the
|
|
19
|
+
* profile closes that cycle at run time.
|
|
20
|
+
*/
|
|
21
|
+
typeKeywords?: readonly string[];
|
|
15
22
|
contentBudget?: number;
|
|
16
23
|
}
|
|
17
24
|
/**
|
|
@@ -16,6 +16,19 @@ export const RETRIEVE_CONTENT_BUDGET = 24_000;
|
|
|
16
16
|
const DEFAULT_LIMIT = PROJECT_RETRIEVE_LIMIT;
|
|
17
17
|
const DEFAULT_BUDGET = RETRIEVE_CONTENT_BUDGET;
|
|
18
18
|
const MIN_TOKEN_LEN = 2;
|
|
19
|
+
/**
|
|
20
|
+
* How many alias definitions one retrieval will chase. Three covers the observed
|
|
21
|
+
* case — hono's `get`/`json` pair plus one — without letting a chunk full of
|
|
22
|
+
* aliased members spend the whole budget on hops.
|
|
23
|
+
*/
|
|
24
|
+
const MAX_ALIAS_HOPS = 3;
|
|
25
|
+
/** Backstop for a caller that names no ecosystem; every real one passes its own. */
|
|
26
|
+
const DEFAULT_TYPE_KEYWORDS = ['interface', 'type', 'class', 'enum'];
|
|
27
|
+
/** A member declared as a bare capitalised type: `get: HandlerInterface<…>`. */
|
|
28
|
+
const MEMBER_TYPE_RE = /^\s*(?:readonly\s+)?([A-Za-z_$][\w$]*)\??\s*:\s*([A-Z][A-Za-z0-9_]*)\s*[<;,)|&]/gm;
|
|
29
|
+
const TYPE_DECL_RE = /\b(?:interface|type|class|data|newtype|struct|trait|enum)\s+([A-Z][A-Za-z0-9_]*)/g;
|
|
30
|
+
/** The `<E extends Env, BasePath extends string>` a declaration introduces itself. */
|
|
31
|
+
const TYPE_PARAMS_RE = /<([^<>]*)>/g;
|
|
19
32
|
const FALLBACK_DTS_CHARS = 12_000;
|
|
20
33
|
const FALLBACK_README_CHARS = 4_000;
|
|
21
34
|
/**
|
|
@@ -86,6 +99,77 @@ function enforceBudget(chunks, budget) {
|
|
|
86
99
|
}
|
|
87
100
|
return out;
|
|
88
101
|
}
|
|
102
|
+
/**
|
|
103
|
+
* The type names the retrieved text declares MEMBERS as, whose own definitions
|
|
104
|
+
* are not in hand and which the query itself names — by the member or by the
|
|
105
|
+
* type.
|
|
106
|
+
*
|
|
107
|
+
* This is the alias hop. A package that types its public surface through
|
|
108
|
+
* interface aliases puts every real signature one declaration away from the name
|
|
109
|
+
* a query matches: hono writes `get: HandlerInterface<…>` in hono-base.d.ts and
|
|
110
|
+
* keeps the call signatures in `HandlerInterface`, in types.d.ts. BM25 ranks
|
|
111
|
+
* chunks independently, so retrieval lands on the alias and the extraction child
|
|
112
|
+
* sees a name where a signature should be. Measured on hono 4.13.5: three real
|
|
113
|
+
* lookups, three abstentions, and the definition in one chunk of 708.
|
|
114
|
+
*
|
|
115
|
+
* Ranking hops by frequency does not work — `Response` and the English word
|
|
116
|
+
* `The` both outrank `HandlerInterface` in the same text. What the query names
|
|
117
|
+
* is the signal.
|
|
118
|
+
*/
|
|
119
|
+
function hopNames(text, tokens) {
|
|
120
|
+
const declared = new Set([...text.matchAll(TYPE_DECL_RE)].map(m => m[1]));
|
|
121
|
+
const typeParams = new Set();
|
|
122
|
+
for (const m of text.matchAll(TYPE_PARAMS_RE)) {
|
|
123
|
+
for (const part of m[1].split(',')) {
|
|
124
|
+
const name = /^\s*([A-Z][A-Za-z0-9_]*)\s*(?:extends|=|$)/.exec(part);
|
|
125
|
+
if (name)
|
|
126
|
+
typeParams.add(name[1]);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
const asked = new Set(tokens.map(t => t.toLowerCase()));
|
|
130
|
+
const out = [];
|
|
131
|
+
// A capitalised name the QUERY itself asks about. scotty's seven failures were
|
|
132
|
+
// all of this shape: `type ActionM = ActionT IO` sits in one chunk of 312
|
|
133
|
+
// while 67 chunks USE the name, and a chunk carrying BOTH query terms
|
|
134
|
+
// (`get :: RoutePattern -> ActionM () -> ScottyM ()`) outranks the definition
|
|
135
|
+
// every time. Reading the ranked output, all eight slots went to uses.
|
|
136
|
+
for (const t of tokens) {
|
|
137
|
+
if (!/^[A-Z][A-Za-z0-9_]{2,}$/.test(t) || declared.has(t) || out.includes(t))
|
|
138
|
+
continue;
|
|
139
|
+
out.push(t);
|
|
140
|
+
if (out.length >= MAX_ALIAS_HOPS)
|
|
141
|
+
return out;
|
|
142
|
+
}
|
|
143
|
+
for (const m of text.matchAll(MEMBER_TYPE_RE)) {
|
|
144
|
+
const [, member, typeName] = m;
|
|
145
|
+
if (declared.has(typeName) || typeParams.has(typeName))
|
|
146
|
+
continue;
|
|
147
|
+
if (!asked.has(member.toLowerCase()) && !asked.has(typeName.toLowerCase()))
|
|
148
|
+
continue;
|
|
149
|
+
if (out.includes(typeName))
|
|
150
|
+
continue;
|
|
151
|
+
out.push(typeName);
|
|
152
|
+
if (out.length >= MAX_ALIAS_HOPS)
|
|
153
|
+
break;
|
|
154
|
+
}
|
|
155
|
+
return out;
|
|
156
|
+
}
|
|
157
|
+
/** The smallest chunk that DECLARES `name`, or null. */
|
|
158
|
+
function definitionChunk(cache, opts, name) {
|
|
159
|
+
const keywords = opts.typeKeywords ?? DEFAULT_TYPE_KEYWORDS;
|
|
160
|
+
// Smallest first: the DEFINITION of a name is a short declaration, while the
|
|
161
|
+
// long chunks holding it are the ones that merely use it.
|
|
162
|
+
const where = keywords.map((_, i) => `content GLOB ?${i + 4}`).join(' OR ');
|
|
163
|
+
const row = cache.db
|
|
164
|
+
.prepare(`SELECT file_path, kind, content, 0 AS rank FROM chunks
|
|
165
|
+
WHERE ecosystem = ?1 AND name = ?2 AND version = ?3
|
|
166
|
+
AND (${where})
|
|
167
|
+
ORDER BY length(content) LIMIT 1`)
|
|
168
|
+
.get(opts.ecosystem, opts.name, opts.version, ...keywords.map(k => `*${k} ${name}[ <={(=]*`));
|
|
169
|
+
if (!row)
|
|
170
|
+
return null;
|
|
171
|
+
return { filePath: row.file_path, kind: row.kind, content: row.content, rank: row.rank };
|
|
172
|
+
}
|
|
89
173
|
export function retrieveChunks(cache, opts) {
|
|
90
174
|
const limit = opts.limit ?? DEFAULT_LIMIT;
|
|
91
175
|
const budget = opts.contentBudget ?? DEFAULT_BUDGET;
|
|
@@ -117,5 +201,20 @@ export function retrieveChunks(cache, opts) {
|
|
|
117
201
|
content: r.content,
|
|
118
202
|
rank: r.rank
|
|
119
203
|
}));
|
|
120
|
-
|
|
204
|
+
const kept = enforceBudget(mapped, budget);
|
|
205
|
+
const key = (c) => `${c.filePath}\u0000${c.content.length}`;
|
|
206
|
+
const have = new Set(kept.map(key));
|
|
207
|
+
const hops = [];
|
|
208
|
+
for (const name of hopNames(kept.map(c => c.content).join('\n'), tokens)) {
|
|
209
|
+
const def = definitionChunk(cache, opts, name);
|
|
210
|
+
if (!def || have.has(key(def)))
|
|
211
|
+
continue;
|
|
212
|
+
hops.push(def);
|
|
213
|
+
}
|
|
214
|
+
if (hops.length === 0)
|
|
215
|
+
return kept;
|
|
216
|
+
// Hops sit directly behind the top-ranked chunk, so re-budgeting drops the
|
|
217
|
+
// WEAKEST original rather than the definition that explains the strongest.
|
|
218
|
+
// The budget itself does not move.
|
|
219
|
+
return enforceBudget([kept[0], ...hops, ...kept.slice(1)], budget);
|
|
121
220
|
}
|
|
@@ -6,9 +6,9 @@
|
|
|
6
6
|
* bodies and private items dropped. That is what `surface` below does, and it is
|
|
7
7
|
* why this row needs code where the npm row needed none.
|
|
8
8
|
*
|
|
9
|
-
*
|
|
10
|
-
* `
|
|
11
|
-
* a TOML dependency would cost a dependency.
|
|
9
|
+
* No TOML parser. `Cargo.lock` is generated with a fixed `[[package]]` shape, and
|
|
10
|
+
* `Cargo.toml`'s dependency tables are read for their KEYS only, so a line reader
|
|
11
|
+
* covers both where a TOML dependency would cost a dependency.
|
|
12
12
|
*/
|
|
13
13
|
import { type ResolvedPackage } from './docs-resolve.js';
|
|
14
14
|
import type { NpmVersionInfo } from './npm-version.js';
|
|
@@ -112,4 +112,15 @@ export declare function rustSurface(src: string, insideTrait?: boolean, topLevel
|
|
|
112
112
|
export declare function isRustFile(name: string): boolean;
|
|
113
113
|
/** The `[package] name` of a cargo project, for labelling its own source. */
|
|
114
114
|
export declare function cargoProjectName(cwd: string): string | null;
|
|
115
|
+
/**
|
|
116
|
+
* The crate names `Cargo.toml` itself declares, under both `-` and `_` spellings.
|
|
117
|
+
*
|
|
118
|
+
* NOT {@link lockedDeps}: a lock file is the whole transitive closure, so it
|
|
119
|
+
* answers "can this resolve" and not "may this crate `use` it". The live run of
|
|
120
|
+
* 2026-09-05 answered about `tower` — in the lock via axum, absent from
|
|
121
|
+
* `[dependencies]` — and the crate did not compile.
|
|
122
|
+
*
|
|
123
|
+
* Undefined when there is no readable manifest, which is not "declares nothing".
|
|
124
|
+
*/
|
|
125
|
+
export declare function manifestCrates(cwd: string): Set<string> | undefined;
|
|
115
126
|
export {};
|
|
@@ -6,9 +6,9 @@
|
|
|
6
6
|
* bodies and private items dropped. That is what `surface` below does, and it is
|
|
7
7
|
* why this row needs code where the npm row needed none.
|
|
8
8
|
*
|
|
9
|
-
*
|
|
10
|
-
* `
|
|
11
|
-
* a TOML dependency would cost a dependency.
|
|
9
|
+
* No TOML parser. `Cargo.lock` is generated with a fixed `[[package]]` shape, and
|
|
10
|
+
* `Cargo.toml`'s dependency tables are read for their KEYS only, so a line reader
|
|
11
|
+
* covers both where a TOML dependency would cost a dependency.
|
|
12
12
|
*/
|
|
13
13
|
import * as fs from 'node:fs';
|
|
14
14
|
import * as path from 'node:path';
|
|
@@ -791,3 +791,44 @@ export function cargoProjectName(cwd) {
|
|
|
791
791
|
const match = /^\s*name\s*=\s*"([^"]+)"/m.exec(section?.[1] ?? '');
|
|
792
792
|
return match ? match[1] : null;
|
|
793
793
|
}
|
|
794
|
+
/**
|
|
795
|
+
* The crate names `Cargo.toml` itself declares, under both `-` and `_` spellings.
|
|
796
|
+
*
|
|
797
|
+
* NOT {@link lockedDeps}: a lock file is the whole transitive closure, so it
|
|
798
|
+
* answers "can this resolve" and not "may this crate `use` it". The live run of
|
|
799
|
+
* 2026-09-05 answered about `tower` — in the lock via axum, absent from
|
|
800
|
+
* `[dependencies]` — and the crate did not compile.
|
|
801
|
+
*
|
|
802
|
+
* Undefined when there is no readable manifest, which is not "declares nothing".
|
|
803
|
+
*/
|
|
804
|
+
export function manifestCrates(cwd) {
|
|
805
|
+
const text = safeRead(path.join(cwd, 'Cargo.toml'));
|
|
806
|
+
if (text === null)
|
|
807
|
+
return undefined;
|
|
808
|
+
const out = new Set();
|
|
809
|
+
const add = (name) => {
|
|
810
|
+
for (const key of new Set([name, canonical(name)]))
|
|
811
|
+
out.add(key);
|
|
812
|
+
};
|
|
813
|
+
let inDeps = false;
|
|
814
|
+
for (const raw of text.split('\n')) {
|
|
815
|
+
const line = raw.trim();
|
|
816
|
+
const header = /^\[([^\]]+)\]$/.exec(line);
|
|
817
|
+
if (header) {
|
|
818
|
+
const section = header[1];
|
|
819
|
+
// `[dependencies.serde]` and `[target.'cfg(unix)'.dependencies]` both
|
|
820
|
+
// declare, and the first names its crate in the header itself.
|
|
821
|
+
const table = /^(?:target\.[^.]*\.)?(?:dev-|build-)?dependencies(?:\.(.+))?$/.exec(section);
|
|
822
|
+
inDeps = table !== null && table[1] === undefined;
|
|
823
|
+
if (table?.[1])
|
|
824
|
+
add(table[1]);
|
|
825
|
+
continue;
|
|
826
|
+
}
|
|
827
|
+
if (!inDeps)
|
|
828
|
+
continue;
|
|
829
|
+
const key = /^([A-Za-z0-9_-]+)\s*=/.exec(line);
|
|
830
|
+
if (key)
|
|
831
|
+
add(key[1]);
|
|
832
|
+
}
|
|
833
|
+
return out;
|
|
834
|
+
}
|
|
@@ -91,3 +91,12 @@ export declare function haskellSurface(rawSrc: string): string;
|
|
|
91
91
|
export declare function isHaskellFile(name: string): boolean;
|
|
92
92
|
/** The `name:` field of the project's own `.cabal` file. */
|
|
93
93
|
export declare function hackageProjectName(cwd: string): string | null;
|
|
94
|
+
/**
|
|
95
|
+
* The package names the project's `.cabal` file declares in `build-depends`,
|
|
96
|
+
* across every stanza.
|
|
97
|
+
*
|
|
98
|
+
* NOT {@link resolvedVersions}: that reads the cabal install plan, which is the
|
|
99
|
+
* whole transitive closure, so it cannot say whether a module may be imported.
|
|
100
|
+
* Undefined when there is no readable `.cabal` file.
|
|
101
|
+
*/
|
|
102
|
+
export declare function manifestPackages(cwd: string): Set<string> | undefined;
|
|
@@ -506,3 +506,49 @@ export function hackageProjectName(cwd) {
|
|
|
506
506
|
const match = /^\s*name\s*:\s*(\S+)/m.exec(safeRead(path.join(cwd, cabal)) ?? '');
|
|
507
507
|
return match ? match[1] : null;
|
|
508
508
|
}
|
|
509
|
+
/**
|
|
510
|
+
* The package names the project's `.cabal` file declares in `build-depends`,
|
|
511
|
+
* across every stanza.
|
|
512
|
+
*
|
|
513
|
+
* NOT {@link resolvedVersions}: that reads the cabal install plan, which is the
|
|
514
|
+
* whole transitive closure, so it cannot say whether a module may be imported.
|
|
515
|
+
* Undefined when there is no readable `.cabal` file.
|
|
516
|
+
*/
|
|
517
|
+
export function manifestPackages(cwd) {
|
|
518
|
+
let entries;
|
|
519
|
+
try {
|
|
520
|
+
entries = fs.readdirSync(cwd);
|
|
521
|
+
}
|
|
522
|
+
catch {
|
|
523
|
+
return undefined;
|
|
524
|
+
}
|
|
525
|
+
const cabal = entries.find(e => e.endsWith('.cabal'));
|
|
526
|
+
if (!cabal)
|
|
527
|
+
return undefined;
|
|
528
|
+
const text = safeRead(path.join(cwd, cabal));
|
|
529
|
+
if (text === null)
|
|
530
|
+
return undefined;
|
|
531
|
+
const out = new Set();
|
|
532
|
+
let inDepends = false;
|
|
533
|
+
for (const raw of text.split('\n')) {
|
|
534
|
+
const line = raw.replace(/--.*$/, '');
|
|
535
|
+
const start = /^\s*build-depends\s*:(.*)$/i.exec(line);
|
|
536
|
+
const body = start ? start[1] : line;
|
|
537
|
+
if (start)
|
|
538
|
+
inDepends = true;
|
|
539
|
+
else if (!inDepends)
|
|
540
|
+
continue;
|
|
541
|
+
// A continuation is indented; a new field at the stanza's own indent ends
|
|
542
|
+
// the list. Both `,`-leading and `,`-trailing layouts are in the wild.
|
|
543
|
+
else if (/^\s*[A-Za-z-]+\s*:/.test(line) || line.trim() === '') {
|
|
544
|
+
inDepends = false;
|
|
545
|
+
continue;
|
|
546
|
+
}
|
|
547
|
+
for (const part of body.split(',')) {
|
|
548
|
+
const name = /^\s*([A-Za-z0-9][A-Za-z0-9_-]*)/.exec(part);
|
|
549
|
+
if (name)
|
|
550
|
+
out.add(name[1]);
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
return out;
|
|
554
|
+
}
|
|
@@ -194,7 +194,7 @@ export function registerPiWorkerDocs(pi, internals = {}) {
|
|
|
194
194
|
const r = await lookup(projectCorpus(projectName), chunks);
|
|
195
195
|
if (r.kind === 'failed')
|
|
196
196
|
return docsFailureResult(r.extraction, baseDetails, '');
|
|
197
|
-
const { extraction, excerptVerified: verified, body: text } = r;
|
|
197
|
+
const { extraction, excerptVerified: verified, body: text, content: retrieved } = r;
|
|
198
198
|
// SAME instrumentation channel as the package path below. Both branches
|
|
199
199
|
// record, or "the last docs answer before the worker stopped" is
|
|
200
200
|
// unanswerable whenever the last answer came from the branch that does
|
|
@@ -213,7 +213,8 @@ export function registerPiWorkerDocs(pi, internals = {}) {
|
|
|
213
213
|
reason: 'project-source lookup — the type-only detector is not applied here',
|
|
214
214
|
excerptVerified: verified,
|
|
215
215
|
excerptCheck: extraction.excerptCheck,
|
|
216
|
-
toolText: text
|
|
216
|
+
toolText: text,
|
|
217
|
+
retrievedText: retrieved
|
|
217
218
|
});
|
|
218
219
|
return workerAnswer(text, {
|
|
219
220
|
...baseDetails,
|
|
@@ -346,7 +347,8 @@ export function registerPiWorkerDocs(pi, internals = {}) {
|
|
|
346
347
|
reason: typeOnly.reason,
|
|
347
348
|
excerptVerified: verified,
|
|
348
349
|
excerptCheck: extraction.excerptCheck,
|
|
349
|
-
toolText: text
|
|
350
|
+
toolText: text,
|
|
351
|
+
retrievedText: concatenated
|
|
350
352
|
});
|
|
351
353
|
return workerAnswer(text, {
|
|
352
354
|
...baseDetails,
|
|
@@ -31,6 +31,19 @@ export interface TypeOnlyLogRecord {
|
|
|
31
31
|
* Optional — older records still parse.
|
|
32
32
|
*/
|
|
33
33
|
excerptCheck?: ExcerptVerification;
|
|
34
|
+
/**
|
|
35
|
+
* The retrieved chunk text handed to the extraction child, before it wrote a word.
|
|
36
|
+
*
|
|
37
|
+
* `toolText` cannot stand in for it. The tool return embeds the child's own prose,
|
|
38
|
+
* so scoring that prose against it asks whether the answer contains itself — two
|
|
39
|
+
* full runs read 13/13, 12/12, 10/10, 13/13, 4/4 and 2/2 clean, 54 answers and not
|
|
40
|
+
* one miss, while one of them shipped `decodeFile`, which aeson 2 does not have.
|
|
41
|
+
* Removing the prose leaves only the child's own CITED excerpt, a line or two, and
|
|
42
|
+
* flags `from_str` and `Context` as invented. Neither corpus is the one the question
|
|
43
|
+
* needs; this is. Optional — records written before it still parse, and a scorer
|
|
44
|
+
* must say "not computable" rather than guess when it is absent.
|
|
45
|
+
*/
|
|
46
|
+
retrievedText?: string;
|
|
34
47
|
/**
|
|
35
48
|
* The tool's ENTIRE return text — version banner, npm header, the answer prose, the cited
|
|
36
49
|
* excerpt, and (when it fires) the type-only banner. Optional so logs written before this
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mjasnikovs/pi-task",
|
|
3
|
-
"version": "0.40.
|
|
3
|
+
"version": "0.40.3",
|
|
4
4
|
"description": "Deterministic task planning and spec-orchestration for local models — crash-safe /task pipelines with verify/enforce gates, a real-time remote web view, and web/docs/fetch/worker subagent tools.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|