@mjasnikovs/pi-task 0.40.1 → 0.40.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/task/enrichment.d.ts +14 -1
- package/dist/task/enrichment.js +16 -1
- package/dist/task/external-context.d.ts +12 -0
- package/dist/task/external-context.js +11 -4
- package/dist/workers/docs-core.js +26 -2
- package/dist/workers/docs-ecosystems.d.ts +20 -0
- package/dist/workers/docs-ecosystems.js +34 -5
- package/dist/workers/docs-index.js +56 -2
- package/dist/workers/docs-project.js +5 -1
- package/dist/workers/docs-retrieve.d.ts +7 -0
- package/dist/workers/docs-retrieve.js +100 -1
- package/dist/workers/eco-cargo.d.ts +14 -3
- package/dist/workers/eco-cargo.js +44 -3
- package/dist/workers/eco-hackage.d.ts +9 -0
- package/dist/workers/eco-hackage.js +46 -0
- package/package.json +1 -1
|
@@ -6,8 +6,21 @@
|
|
|
6
6
|
* denylisted shell names are dropped, a URL's trailing sentence punctuation is
|
|
7
7
|
* stripped, docs targets stop at ENRICH_CAP while version targets continue to
|
|
8
8
|
* ENRICH_VERSION_CAP, and the version list is a strict superset of the docs list.
|
|
9
|
+
*
|
|
10
|
+
* Package extraction is additionally gated on the caller's declared dependencies;
|
|
11
|
+
* see the `declared` parameter on {@link extractEnrichTargets}.
|
|
12
|
+
*/
|
|
13
|
+
export declare function extractEnrichTargets(text: string,
|
|
14
|
+
/**
|
|
15
|
+
* The project's declared dependencies. A backticked name outside this set is
|
|
16
|
+
* not enriched: the model backticks filenames (`config.ts`, `tsconfig.json`)
|
|
17
|
+
* and field names (`name`, `port`) far more often than package names, and each
|
|
18
|
+
* of those is also a real, unrelated package on the public registry — so the
|
|
19
|
+
* permissive read fetched and indexed a stranger's code under the project's
|
|
20
|
+
* own filename. Omit when the manifest is unreadable, which is not the same
|
|
21
|
+
* fact as "declares nothing".
|
|
9
22
|
*/
|
|
10
|
-
|
|
23
|
+
declared?: ReadonlySet<string>): {
|
|
11
24
|
/** Packages that get a (heavy) docs fetch — capped at ENRICH_CAP. */
|
|
12
25
|
packages: string[];
|
|
13
26
|
/**
|
package/dist/task/enrichment.js
CHANGED
|
@@ -6,6 +6,9 @@
|
|
|
6
6
|
* denylisted shell names are dropped, a URL's trailing sentence punctuation is
|
|
7
7
|
* stripped, docs targets stop at ENRICH_CAP while version targets continue to
|
|
8
8
|
* ENRICH_VERSION_CAP, and the version list is a strict superset of the docs list.
|
|
9
|
+
*
|
|
10
|
+
* Package extraction is additionally gated on the caller's declared dependencies;
|
|
11
|
+
* see the `declared` parameter on {@link extractEnrichTargets}.
|
|
9
12
|
*/
|
|
10
13
|
const ENRICH_PKG_RE = /`((?:@[a-z0-9][a-z0-9._-]*\/)?[a-z0-9][a-z0-9._-]*)`/g;
|
|
11
14
|
const ENRICH_URL_RE = /https?:\/\/[^\s)`>]+/g;
|
|
@@ -79,13 +82,25 @@ function parseServices(text) {
|
|
|
79
82
|
}
|
|
80
83
|
return out;
|
|
81
84
|
}
|
|
82
|
-
export function extractEnrichTargets(text
|
|
85
|
+
export function extractEnrichTargets(text,
|
|
86
|
+
/**
|
|
87
|
+
* The project's declared dependencies. A backticked name outside this set is
|
|
88
|
+
* not enriched: the model backticks filenames (`config.ts`, `tsconfig.json`)
|
|
89
|
+
* and field names (`name`, `port`) far more often than package names, and each
|
|
90
|
+
* of those is also a real, unrelated package on the public registry — so the
|
|
91
|
+
* permissive read fetched and indexed a stranger's code under the project's
|
|
92
|
+
* own filename. Omit when the manifest is unreadable, which is not the same
|
|
93
|
+
* fact as "declares nothing".
|
|
94
|
+
*/
|
|
95
|
+
declared) {
|
|
83
96
|
const pkgs = [];
|
|
84
97
|
const seen = new Set();
|
|
85
98
|
for (const m of text.matchAll(ENRICH_PKG_RE)) {
|
|
86
99
|
const t = m[1];
|
|
87
100
|
if (ENRICH_DENYLIST.has(t) || seen.has(t))
|
|
88
101
|
continue;
|
|
102
|
+
if (declared && !declared.has(t))
|
|
103
|
+
continue;
|
|
89
104
|
seen.add(t);
|
|
90
105
|
pkgs.push(t);
|
|
91
106
|
if (pkgs.length >= ENRICH_VERSION_CAP)
|
|
@@ -83,6 +83,18 @@ export interface ExternalContextPolicy {
|
|
|
83
83
|
targetCap?: number;
|
|
84
84
|
/** Max services fanned out. Omit for uncapped; the auto-answer path caps at 2. */
|
|
85
85
|
serviceCap?: number;
|
|
86
|
+
/**
|
|
87
|
+
* Fan named packages out to a docs body. The RESEARCH path does not, and the
|
|
88
|
+
* auto-answer path does.
|
|
89
|
+
*
|
|
90
|
+
* Research retrieves against `refined.split('\n')[0]` — the literal word
|
|
91
|
+
* "GOAL" for every refined spec — so its bodies are whatever ranks against
|
|
92
|
+
* that, pasted raw. The live run of 2026-09-05 found no research output
|
|
93
|
+
* citing one, while the model's own docs tool answered 49 real questions
|
|
94
|
+
* across the same three runs. The auto-answer path asks a focused child an
|
|
95
|
+
* actual question, which is the shape that works.
|
|
96
|
+
*/
|
|
97
|
+
packageDocs?: boolean;
|
|
86
98
|
/**
|
|
87
99
|
* A cheap live version lookup for every named dep that did NOT get a docs
|
|
88
100
|
* target, so a version block exists for ALL of them. Omit to disable, as the
|
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
* Run against a mixed source, the emitted headings come out in exactly that order:
|
|
23
23
|
* `### npm:` then `### docs:` then `### url:` then `### service:`.
|
|
24
24
|
*/
|
|
25
|
-
import { chooseEcosystem, defaultEcosystemIo } from '../workers/docs-ecosystems.js';
|
|
25
|
+
import { chooseEcosystem, declaredDepNames, defaultEcosystemIo } from '../workers/docs-ecosystems.js';
|
|
26
26
|
import { docsRaw } from '../workers/docs-core.js';
|
|
27
27
|
import { fetchRaw } from '../workers/fetch-core.js';
|
|
28
28
|
import { formatNpmVersionSection, npmVersionLookup } from '../workers/npm-version.js';
|
|
@@ -39,18 +39,24 @@ const RAW_BODY_LIMIT = 4000;
|
|
|
39
39
|
*/
|
|
40
40
|
export async function buildExternalContext(source, deps, lookups, policy = {}) {
|
|
41
41
|
const searchFn = lookups.search ?? defaultSearch;
|
|
42
|
-
const enrichTargets = extractEnrichTargets(source);
|
|
42
|
+
const enrichTargets = extractEnrichTargets(source, declaredDepNames(deps.cwd));
|
|
43
43
|
// Packages lead urls, then the combined cap applies — so a capped run spends
|
|
44
44
|
// its budget on named deps first. Uncapped, this is just "packages, then urls".
|
|
45
|
+
const docsPackages = policy.packageDocs === false ? [] : enrichTargets.packages;
|
|
45
46
|
const targets = [
|
|
46
|
-
...
|
|
47
|
+
...docsPackages.map(name => ({ kind: 'pkg', name })),
|
|
47
48
|
...enrichTargets.urls.map(name => ({ kind: 'url', name }))
|
|
48
49
|
].slice(0, policy.targetCap ?? Number.POSITIVE_INFINITY);
|
|
49
50
|
const services = enrichTargets.services.slice(0, policy.serviceCap ?? Number.POSITIVE_INFINITY);
|
|
50
51
|
const versionLookup = policy.versionLookup;
|
|
51
52
|
const docsTargets = new Set(targets.filter(t => t.kind === 'pkg').map(t => t.name));
|
|
52
53
|
const extraVersionPkgs = versionLookup ? enrichTargets.versionPackages.filter(p => !docsTargets.has(p)) : [];
|
|
53
|
-
|
|
54
|
+
// Version packages count as work here: with `packageDocs: false` a named dep
|
|
55
|
+
// produces no target at all, and returning early would drop its version block.
|
|
56
|
+
if (policy.earlyReturnOnNoTargets
|
|
57
|
+
&& targets.length === 0
|
|
58
|
+
&& services.length === 0
|
|
59
|
+
&& extraVersionPkgs.length === 0)
|
|
54
60
|
return '';
|
|
55
61
|
const startedAt = Date.now();
|
|
56
62
|
const [targetResults, serviceResults, extraVersionResults] = await Promise.all([
|
|
@@ -191,6 +197,7 @@ export async function gatherExternalContext(refined, deps) {
|
|
|
191
197
|
search: deps.searchFn
|
|
192
198
|
}, {
|
|
193
199
|
versionLookup,
|
|
200
|
+
packageDocs: false,
|
|
194
201
|
subStepLabel: 'enrichment',
|
|
195
202
|
earlyReturnOnNoTargets: true
|
|
196
203
|
});
|
|
@@ -128,9 +128,32 @@ export function findDeclaredRange(parentPkg, cwd) {
|
|
|
128
128
|
* does or does not say has to be a sentence about `bun`.
|
|
129
129
|
*/
|
|
130
130
|
export function buildVersionBanner(pin, resolved, version, cwd, profile = ECOSYSTEMS.npm) {
|
|
131
|
+
const asked = pin?.asked ?? resolved;
|
|
132
|
+
// Resolvable is not usable. A lock file, a cabal plan and `node_modules` are
|
|
133
|
+
// all the transitive CLOSURE, so the tool can answer in full confidence about
|
|
134
|
+
// a package the project may not import. That is what made the Rust run of
|
|
135
|
+
// 2026-09-05 a hard fail: a correct `tower::util::ServiceExt` answer, the
|
|
136
|
+
// import written, and E0433 "cannot find module or crate tower" from the compiler.
|
|
137
|
+
const undeclared = undeclaredNotice(asked, cwd, profile);
|
|
131
138
|
if (!pin)
|
|
139
|
+
return undeclared;
|
|
140
|
+
return undeclared + pinBanner(pin, asked, resolved, version, cwd, profile);
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* The one sentence a package present-but-not-declared needs, or `''` when it is
|
|
144
|
+
* declared or when no manifest could be read.
|
|
145
|
+
*/
|
|
146
|
+
function undeclaredNotice(asked, cwd, profile) {
|
|
147
|
+
const declared = profile.manifestDeps(cwd);
|
|
148
|
+
const root = profile.parentPackage(asked);
|
|
149
|
+
if (!declared || declared.has(root) || declared.has(asked))
|
|
132
150
|
return '';
|
|
133
|
-
|
|
151
|
+
return (`[DEPENDENCY] "${root}" is present in this project but is not a declared `
|
|
152
|
+
+ `dependency in ${profile.manifestLabel} — it resolves only because something `
|
|
153
|
+
+ `else pulled it in. Add it to ${profile.manifestLabel} before importing it, or `
|
|
154
|
+
+ `the build will not find it.\n\n`);
|
|
155
|
+
}
|
|
156
|
+
function pinBanner(pin, asked, resolved, version, cwd, profile) {
|
|
134
157
|
const grounded = resolved !== asked ? ` The types this answer reads come from ${resolved}.` : '';
|
|
135
158
|
const manifest = profile.manifestLabel;
|
|
136
159
|
const registry = profile.registryLabel;
|
|
@@ -485,7 +508,8 @@ function docsRawCached(cache, pkg, profile, query, ensureIndexed, retrieveChunks
|
|
|
485
508
|
version: pkg.version,
|
|
486
509
|
query,
|
|
487
510
|
limit: DEFAULT_LIMIT,
|
|
488
|
-
contentBudget: DEFAULT_BUDGET
|
|
511
|
+
contentBudget: DEFAULT_BUDGET,
|
|
512
|
+
typeKeywords: profile.typeKeywords
|
|
489
513
|
});
|
|
490
514
|
}
|
|
491
515
|
catch (err) {
|
|
@@ -84,6 +84,11 @@ export interface EcosystemProfile {
|
|
|
84
84
|
surface: (content: string) => string;
|
|
85
85
|
/** Where a declaration begins, so a chunk never splits a signature. */
|
|
86
86
|
declSplitRe: RegExp;
|
|
87
|
+
/**
|
|
88
|
+
* The keywords that INTRODUCE a named type in this language, for finding the
|
|
89
|
+
* chunk that defines a name rather than the many that use it.
|
|
90
|
+
*/
|
|
91
|
+
typeKeywords: readonly string[];
|
|
87
92
|
/** Line-comment marker, used to label a chunk with the file it came from. */
|
|
88
93
|
commentPrefix: string;
|
|
89
94
|
/** Directories the surface walk never descends into: tests, build output. */
|
|
@@ -105,6 +110,15 @@ export interface EcosystemProfile {
|
|
|
105
110
|
* package's version, which is a different fact from "declares nothing".
|
|
106
111
|
*/
|
|
107
112
|
declaredDeps: (cwd: string) => Record<string, string> | undefined;
|
|
113
|
+
/**
|
|
114
|
+
* The names the MANIFEST itself declares — what the project may import.
|
|
115
|
+
*
|
|
116
|
+
* Distinct from {@link declaredDeps}, which for cargo and hackage reads a
|
|
117
|
+
* lock or plan file: that is the whole transitive closure, so it answers
|
|
118
|
+
* "does this resolve" and not "may this be used". Undefined when there is no
|
|
119
|
+
* readable manifest, which is not the same fact as "declares nothing".
|
|
120
|
+
*/
|
|
121
|
+
manifestDeps: (cwd: string) => Set<string> | undefined;
|
|
108
122
|
}
|
|
109
123
|
/** Overrides a caller has already been given its own copies of. */
|
|
110
124
|
export interface NpmProfileHooks {
|
|
@@ -134,6 +148,12 @@ export declare const ECOSYSTEMS: {
|
|
|
134
148
|
};
|
|
135
149
|
/** Which ecosystems `cwd` looks like a project of, in roster order. */
|
|
136
150
|
export declare function detectEcosystems(cwd: string, roster?: readonly EcosystemProfile[]): EcosystemId[];
|
|
151
|
+
/**
|
|
152
|
+
* Every dependency `cwd`'s manifests declare, across the ecosystems it is a
|
|
153
|
+
* project of. Undefined when no detected ecosystem could read its manifest —
|
|
154
|
+
* "we cannot tell", which callers must not read as "declares nothing".
|
|
155
|
+
*/
|
|
156
|
+
export declare function declaredDepNames(cwd: string, roster?: readonly EcosystemProfile[]): Set<string> | undefined;
|
|
137
157
|
export type EcosystemChoice = {
|
|
138
158
|
ok: true;
|
|
139
159
|
profile: EcosystemProfile;
|
|
@@ -21,8 +21,8 @@ import { runAutoInstall, findDeclaredRange, extractParentPackage, resolveTypeSou
|
|
|
21
21
|
import { resolvePackage, isDtsFile, isValidModuleName } from './docs-resolve.js';
|
|
22
22
|
import { DECL_SPLIT_RE } from './docs-chunk.js';
|
|
23
23
|
import { npmVersionLookup } from './npm-version.js';
|
|
24
|
-
import { resolveCrate, cratesLatest, crateTarballUrl, crateOf, isValidCrateName, isRustFile, lockedVersion, rustSurface, cargoProjectName, childDirs, lockedDeps, CARGO_DECL_SPLIT_RE } from './eco-cargo.js';
|
|
25
|
-
import { resolveHackage, hackageLatest, hackageVersion, hackageTarballUrl, hackageExtractDir, hackageProjectName, findCabalTarball, cachedVersions, resolvedVersions, isValidHackageName, isHaskellFile, haskellSurface, HACKAGE_DECL_SPLIT_RE, HACKAGE_SKIP_DIRS } from './eco-hackage.js';
|
|
24
|
+
import { resolveCrate, cratesLatest, crateTarballUrl, crateOf, isValidCrateName, isRustFile, lockedVersion, rustSurface, cargoProjectName, childDirs, lockedDeps, manifestCrates, CARGO_DECL_SPLIT_RE } from './eco-cargo.js';
|
|
25
|
+
import { resolveHackage, hackageLatest, hackageVersion, hackageTarballUrl, hackageExtractDir, hackageProjectName, findCabalTarball, cachedVersions, resolvedVersions, manifestPackages, isValidHackageName, isHaskellFile, haskellSurface, HACKAGE_DECL_SPLIT_RE, HACKAGE_SKIP_DIRS } from './eco-hackage.js';
|
|
26
26
|
import { runChild } from '../shared/child-process.js';
|
|
27
27
|
/**
|
|
28
28
|
* Is any of `names` present at `cwd` or above it?
|
|
@@ -128,6 +128,7 @@ export function npmProfile(hooks = {}) {
|
|
|
128
128
|
isSurfaceFile: isDtsFile,
|
|
129
129
|
surface: content => content,
|
|
130
130
|
declSplitRe: DECL_SPLIT_RE,
|
|
131
|
+
typeKeywords: ['interface', 'type', 'class', 'enum'],
|
|
131
132
|
commentPrefix: '//',
|
|
132
133
|
// A nested node_modules is another package's surface, never this one's.
|
|
133
134
|
skipDirs: ['node_modules'],
|
|
@@ -135,7 +136,11 @@ export function npmProfile(hooks = {}) {
|
|
|
135
136
|
packageSubject: 'an npm package',
|
|
136
137
|
projectGlobs: ['*.ts', '*.tsx'],
|
|
137
138
|
projectName: npmProjectName,
|
|
138
|
-
declaredDeps: npmDeclaredDeps
|
|
139
|
+
declaredDeps: npmDeclaredDeps,
|
|
140
|
+
manifestDeps: cwd => {
|
|
141
|
+
const deps = npmDeclaredDeps(cwd);
|
|
142
|
+
return deps && new Set(Object.keys(deps));
|
|
143
|
+
}
|
|
139
144
|
};
|
|
140
145
|
}
|
|
141
146
|
const NPM_DEP_BLOCKS = [
|
|
@@ -266,13 +271,15 @@ const cargoProfile = {
|
|
|
266
271
|
isSurfaceFile: isRustFile,
|
|
267
272
|
surface: content => rustSurface(content),
|
|
268
273
|
declSplitRe: CARGO_DECL_SPLIT_RE,
|
|
274
|
+
typeKeywords: ['struct', 'trait', 'enum', 'type', 'union'],
|
|
269
275
|
commentPrefix: '//',
|
|
270
276
|
skipDirs: ['tests', 'benches', 'examples', 'target'],
|
|
271
277
|
surfaceLabel: '.rs source or README',
|
|
272
278
|
packageSubject: 'a Rust crate from crates.io',
|
|
273
279
|
projectGlobs: ['*.rs'],
|
|
274
280
|
projectName: cargoProjectName,
|
|
275
|
-
declaredDeps: lockedDeps
|
|
281
|
+
declaredDeps: lockedDeps,
|
|
282
|
+
manifestDeps: manifestCrates
|
|
276
283
|
};
|
|
277
284
|
/**
|
|
278
285
|
* Unpack a Hackage tarball into the tool's own directory. Whether it came from
|
|
@@ -351,13 +358,15 @@ const hackageProfile = {
|
|
|
351
358
|
isSurfaceFile: isHaskellFile,
|
|
352
359
|
surface: haskellSurface,
|
|
353
360
|
declSplitRe: HACKAGE_DECL_SPLIT_RE,
|
|
361
|
+
typeKeywords: ['type', 'data', 'newtype', 'class'],
|
|
354
362
|
commentPrefix: '--',
|
|
355
363
|
skipDirs: HACKAGE_SKIP_DIRS,
|
|
356
364
|
surfaceLabel: '.hs source or README',
|
|
357
365
|
packageSubject: 'a Haskell package from Hackage',
|
|
358
366
|
projectGlobs: ['*.hs'],
|
|
359
367
|
projectName: hackageProjectName,
|
|
360
|
-
declaredDeps: resolvedVersions
|
|
368
|
+
declaredDeps: resolvedVersions,
|
|
369
|
+
manifestDeps: manifestPackages
|
|
361
370
|
};
|
|
362
371
|
/** A cabal, stack or hpack project declares itself with one of these. */
|
|
363
372
|
function hasCabalManifest(cwd) {
|
|
@@ -385,6 +394,26 @@ export const ECOSYSTEMS = {
|
|
|
385
394
|
export function detectEcosystems(cwd, roster = Object.values(ECOSYSTEMS)) {
|
|
386
395
|
return roster.filter(p => p.detect(cwd)).map(p => p.id);
|
|
387
396
|
}
|
|
397
|
+
/**
|
|
398
|
+
* Every dependency `cwd`'s manifests declare, across the ecosystems it is a
|
|
399
|
+
* project of. Undefined when no detected ecosystem could read its manifest —
|
|
400
|
+
* "we cannot tell", which callers must not read as "declares nothing".
|
|
401
|
+
*/
|
|
402
|
+
export function declaredDepNames(cwd, roster = Object.values(ECOSYSTEMS)) {
|
|
403
|
+
let any = false;
|
|
404
|
+
const names = new Set();
|
|
405
|
+
for (const p of roster) {
|
|
406
|
+
if (!p.detect(cwd))
|
|
407
|
+
continue;
|
|
408
|
+
const deps = p.manifestDeps(cwd);
|
|
409
|
+
if (!deps)
|
|
410
|
+
continue;
|
|
411
|
+
any = true;
|
|
412
|
+
for (const name of deps)
|
|
413
|
+
names.add(name);
|
|
414
|
+
}
|
|
415
|
+
return any ? names : undefined;
|
|
416
|
+
}
|
|
388
417
|
/**
|
|
389
418
|
* Which ecosystem a lookup belongs to. The MANIFEST decides, never the model:
|
|
390
419
|
* `text`, `base`, `aeson`, `tokio` and `clap` are all real npm packages as well
|
|
@@ -16,7 +16,9 @@ const ZERO_SEP = Buffer.from([0]);
|
|
|
16
16
|
* question actually being asked: would re-reading produce the same chunks?
|
|
17
17
|
*
|
|
18
18
|
* The CHUNKER counts too, for the same reason: the rows are chunks, not surface,
|
|
19
|
-
* so a fix to where a declaration is cut leaves stale rows behind on its own.
|
|
19
|
+
* so a fix to where a declaration is cut leaves stale rows behind on its own. So
|
|
20
|
+
* does WHICH FILES are read: dropping a package's duplicate `.d.cts` twins
|
|
21
|
+
* changes the rows without changing a byte on disk.
|
|
20
22
|
*
|
|
21
23
|
* It is not total. An extractor change that alters only files BELOW the entry
|
|
22
24
|
* goes unnoticed; deleting the cache is still the escape hatch for that.
|
|
@@ -27,6 +29,11 @@ function computeContentHash(pkg, profile) {
|
|
|
27
29
|
hash.update(ZERO_SEP);
|
|
28
30
|
hash.update(Buffer.from(`${profile.declSplitRe.source}\u0000${profile.commentPrefix}`, 'utf8'));
|
|
29
31
|
hash.update(ZERO_SEP);
|
|
32
|
+
// Source text, the same trick as `declSplitRe.source`: the fingerprint moves
|
|
33
|
+
// whenever the selection rule does, with nothing to remember to bump.
|
|
34
|
+
hash.update(Buffer.from(`${String(profile.isSurfaceFile)}\u0000${String(dropParallelDeclarations)}`
|
|
35
|
+
+ `\u0000${String(dropDeadMajors)}`, 'utf8'));
|
|
36
|
+
hash.update(ZERO_SEP);
|
|
30
37
|
if (pkg.entry && fs.existsSync(pkg.entry)) {
|
|
31
38
|
try {
|
|
32
39
|
hash.update(Buffer.from(profile.surface(fs.readFileSync(pkg.entry, 'utf8')), 'utf8'));
|
|
@@ -83,9 +90,56 @@ function walkSurface(root, profile) {
|
|
|
83
90
|
}
|
|
84
91
|
return out.sort();
|
|
85
92
|
}
|
|
93
|
+
/**
|
|
94
|
+
* Drop a `.d.cts` / `.d.mts` that sits beside a `.d.ts` of the same name.
|
|
95
|
+
*
|
|
96
|
+
* Modern npm packages ship parallel declarations for ESM and CJS: the same API
|
|
97
|
+
* written twice. zod 4.5.4 indexed to 2565 chunks over 1215 distinct bodies,
|
|
98
|
+
* 1280 of them from `.d.cts`; hono, which ships none, had 704 distinct of 708.
|
|
99
|
+
* The cost is the eight-chunk retrieval budget — half of it can go to text the
|
|
100
|
+
* reader already has.
|
|
101
|
+
*
|
|
102
|
+
* The sibling test, not a blanket ban on the extensions: a package shipping only
|
|
103
|
+
* `.d.cts` still has to be readable, and all 123 of zod's had a `.d.ts` twin.
|
|
104
|
+
*/
|
|
105
|
+
function dropParallelDeclarations(files) {
|
|
106
|
+
const esm = new Set(files.filter(f => f.endsWith('.d.ts')).map(f => f.slice(0, -'.d.ts'.length)));
|
|
107
|
+
return files.filter(f => {
|
|
108
|
+
const base = /\.d\.[cm]ts$/.exec(f) ? f.slice(0, -'.d.cts'.length) : null;
|
|
109
|
+
return base === null || !esm.has(base);
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Drop a top-level `vN/` directory holding a major the package is no longer on.
|
|
114
|
+
*
|
|
115
|
+
* zod@4.5.4 ships `v3/` for back-compat, and 414 of its 2565 chunks came from
|
|
116
|
+
* it. Nothing downstream can separate them: same identifiers, same package, same
|
|
117
|
+
* version banner, and the file path is not a ranking signal. An answer went out
|
|
118
|
+
* under `Per zod@4.5.4:` carrying v3's `email(message?): ZodString` — wrong
|
|
119
|
+
* parameter, wrong return, and silent about the `@deprecated` line sitting
|
|
120
|
+
* directly above the real declaration.
|
|
121
|
+
*
|
|
122
|
+
* Only a MISMATCHING major goes. `v4/` under 4.5.4 is the current API and is
|
|
123
|
+
* most of the package; a package whose only content lives under `v1/` keeps it.
|
|
124
|
+
*/
|
|
125
|
+
function dropDeadMajors(files, root, version) {
|
|
126
|
+
const major = /^(\d+)\./.exec(version)?.[1];
|
|
127
|
+
if (major === undefined)
|
|
128
|
+
return files;
|
|
129
|
+
const kept = files.filter(abs => {
|
|
130
|
+
const top = path.relative(root, abs).replace(/\\/g, '/').split('/')[0];
|
|
131
|
+
const dir = /^v(\d+)$/.exec(top);
|
|
132
|
+
return dir === null || dir[1] === major;
|
|
133
|
+
});
|
|
134
|
+
// A package whose whole surface lives under a `vN/` that does not match its
|
|
135
|
+
// own version is not shipping a dead major — it is shipping its API there.
|
|
136
|
+
return kept.length > 0 ? kept : files;
|
|
137
|
+
}
|
|
86
138
|
function collectFiles(pkg, profile) {
|
|
139
|
+
const walked = walkSurface(pkg.root, profile);
|
|
140
|
+
const surface = dropDeadMajors(walked, pkg.root, pkg.version);
|
|
87
141
|
return {
|
|
88
|
-
surface:
|
|
142
|
+
surface: profile.id === 'npm' ? dropParallelDeclarations(surface) : surface,
|
|
89
143
|
readme: pkg.readme
|
|
90
144
|
};
|
|
91
145
|
}
|
|
@@ -234,7 +234,11 @@ listFiles = getProjectFiles) {
|
|
|
234
234
|
version,
|
|
235
235
|
query,
|
|
236
236
|
limit: DEFAULT_LIMIT,
|
|
237
|
-
contentBudget: DEFAULT_BUDGET
|
|
237
|
+
contentBudget: DEFAULT_BUDGET,
|
|
238
|
+
// Every detected ecosystem's keywords: a polyglot project's own
|
|
239
|
+
// source has no single language, and the extra keywords only widen
|
|
240
|
+
// which definition the hop can find.
|
|
241
|
+
typeKeywords: [...new Set(projectProfiles(cwd).flatMap(p => p.typeKeywords))]
|
|
238
242
|
});
|
|
239
243
|
}
|
|
240
244
|
catch (err) {
|
|
@@ -12,6 +12,13 @@ export interface RetrieveOptions {
|
|
|
12
12
|
version: string;
|
|
13
13
|
query: string;
|
|
14
14
|
limit?: number;
|
|
15
|
+
/**
|
|
16
|
+
* The keywords that introduce a named type, for the definition hop. Passed by
|
|
17
|
+
* the caller rather than read off `EcosystemProfile` here: docs-ecosystems
|
|
18
|
+
* imports docs-core, which imports this module, and reaching back for the
|
|
19
|
+
* profile closes that cycle at run time.
|
|
20
|
+
*/
|
|
21
|
+
typeKeywords?: readonly string[];
|
|
15
22
|
contentBudget?: number;
|
|
16
23
|
}
|
|
17
24
|
/**
|
|
@@ -16,6 +16,19 @@ export const RETRIEVE_CONTENT_BUDGET = 24_000;
|
|
|
16
16
|
const DEFAULT_LIMIT = PROJECT_RETRIEVE_LIMIT;
|
|
17
17
|
const DEFAULT_BUDGET = RETRIEVE_CONTENT_BUDGET;
|
|
18
18
|
const MIN_TOKEN_LEN = 2;
|
|
19
|
+
/**
|
|
20
|
+
* How many alias definitions one retrieval will chase. Three covers the observed
|
|
21
|
+
* case — hono's `get`/`json` pair plus one — without letting a chunk full of
|
|
22
|
+
* aliased members spend the whole budget on hops.
|
|
23
|
+
*/
|
|
24
|
+
const MAX_ALIAS_HOPS = 3;
|
|
25
|
+
/** Backstop for a caller that names no ecosystem; every real one passes its own. */
|
|
26
|
+
const DEFAULT_TYPE_KEYWORDS = ['interface', 'type', 'class', 'enum'];
|
|
27
|
+
/** A member declared as a bare capitalised type: `get: HandlerInterface<…>`. */
|
|
28
|
+
const MEMBER_TYPE_RE = /^\s*(?:readonly\s+)?([A-Za-z_$][\w$]*)\??\s*:\s*([A-Z][A-Za-z0-9_]*)\s*[<;,)|&]/gm;
|
|
29
|
+
const TYPE_DECL_RE = /\b(?:interface|type|class|data|newtype|struct|trait|enum)\s+([A-Z][A-Za-z0-9_]*)/g;
|
|
30
|
+
/** The `<E extends Env, BasePath extends string>` a declaration introduces itself. */
|
|
31
|
+
const TYPE_PARAMS_RE = /<([^<>]*)>/g;
|
|
19
32
|
const FALLBACK_DTS_CHARS = 12_000;
|
|
20
33
|
const FALLBACK_README_CHARS = 4_000;
|
|
21
34
|
/**
|
|
@@ -86,6 +99,77 @@ function enforceBudget(chunks, budget) {
|
|
|
86
99
|
}
|
|
87
100
|
return out;
|
|
88
101
|
}
|
|
102
|
+
/**
|
|
103
|
+
* The type names the retrieved text declares MEMBERS as, whose own definitions
|
|
104
|
+
* are not in hand and which the query itself names — by the member or by the
|
|
105
|
+
* type.
|
|
106
|
+
*
|
|
107
|
+
* This is the alias hop. A package that types its public surface through
|
|
108
|
+
* interface aliases puts every real signature one declaration away from the name
|
|
109
|
+
* a query matches: hono writes `get: HandlerInterface<…>` in hono-base.d.ts and
|
|
110
|
+
* keeps the call signatures in `HandlerInterface`, in types.d.ts. BM25 ranks
|
|
111
|
+
* chunks independently, so retrieval lands on the alias and the extraction child
|
|
112
|
+
* sees a name where a signature should be. Measured on hono 4.13.5: three real
|
|
113
|
+
* lookups, three abstentions, and the definition in one chunk of 708.
|
|
114
|
+
*
|
|
115
|
+
* Ranking hops by frequency does not work — `Response` and the English word
|
|
116
|
+
* `The` both outrank `HandlerInterface` in the same text. What the query names
|
|
117
|
+
* is the signal.
|
|
118
|
+
*/
|
|
119
|
+
function hopNames(text, tokens) {
|
|
120
|
+
const declared = new Set([...text.matchAll(TYPE_DECL_RE)].map(m => m[1]));
|
|
121
|
+
const typeParams = new Set();
|
|
122
|
+
for (const m of text.matchAll(TYPE_PARAMS_RE)) {
|
|
123
|
+
for (const part of m[1].split(',')) {
|
|
124
|
+
const name = /^\s*([A-Z][A-Za-z0-9_]*)\s*(?:extends|=|$)/.exec(part);
|
|
125
|
+
if (name)
|
|
126
|
+
typeParams.add(name[1]);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
const asked = new Set(tokens.map(t => t.toLowerCase()));
|
|
130
|
+
const out = [];
|
|
131
|
+
// A capitalised name the QUERY itself asks about. scotty's seven failures were
|
|
132
|
+
// all of this shape: `type ActionM = ActionT IO` sits in one chunk of 312
|
|
133
|
+
// while 67 chunks USE the name, and a chunk carrying BOTH query terms
|
|
134
|
+
// (`get :: RoutePattern -> ActionM () -> ScottyM ()`) outranks the definition
|
|
135
|
+
// every time. Reading the ranked output, all eight slots went to uses.
|
|
136
|
+
for (const t of tokens) {
|
|
137
|
+
if (!/^[A-Z][A-Za-z0-9_]{2,}$/.test(t) || declared.has(t) || out.includes(t))
|
|
138
|
+
continue;
|
|
139
|
+
out.push(t);
|
|
140
|
+
if (out.length >= MAX_ALIAS_HOPS)
|
|
141
|
+
return out;
|
|
142
|
+
}
|
|
143
|
+
for (const m of text.matchAll(MEMBER_TYPE_RE)) {
|
|
144
|
+
const [, member, typeName] = m;
|
|
145
|
+
if (declared.has(typeName) || typeParams.has(typeName))
|
|
146
|
+
continue;
|
|
147
|
+
if (!asked.has(member.toLowerCase()) && !asked.has(typeName.toLowerCase()))
|
|
148
|
+
continue;
|
|
149
|
+
if (out.includes(typeName))
|
|
150
|
+
continue;
|
|
151
|
+
out.push(typeName);
|
|
152
|
+
if (out.length >= MAX_ALIAS_HOPS)
|
|
153
|
+
break;
|
|
154
|
+
}
|
|
155
|
+
return out;
|
|
156
|
+
}
|
|
157
|
+
/** The smallest chunk that DECLARES `name`, or null. */
|
|
158
|
+
function definitionChunk(cache, opts, name) {
|
|
159
|
+
const keywords = opts.typeKeywords ?? DEFAULT_TYPE_KEYWORDS;
|
|
160
|
+
// Smallest first: the DEFINITION of a name is a short declaration, while the
|
|
161
|
+
// long chunks holding it are the ones that merely use it.
|
|
162
|
+
const where = keywords.map((_, i) => `content GLOB ?${i + 4}`).join(' OR ');
|
|
163
|
+
const row = cache.db
|
|
164
|
+
.prepare(`SELECT file_path, kind, content, 0 AS rank FROM chunks
|
|
165
|
+
WHERE ecosystem = ?1 AND name = ?2 AND version = ?3
|
|
166
|
+
AND (${where})
|
|
167
|
+
ORDER BY length(content) LIMIT 1`)
|
|
168
|
+
.get(opts.ecosystem, opts.name, opts.version, ...keywords.map(k => `*${k} ${name}[ <={(=]*`));
|
|
169
|
+
if (!row)
|
|
170
|
+
return null;
|
|
171
|
+
return { filePath: row.file_path, kind: row.kind, content: row.content, rank: row.rank };
|
|
172
|
+
}
|
|
89
173
|
export function retrieveChunks(cache, opts) {
|
|
90
174
|
const limit = opts.limit ?? DEFAULT_LIMIT;
|
|
91
175
|
const budget = opts.contentBudget ?? DEFAULT_BUDGET;
|
|
@@ -117,5 +201,20 @@ export function retrieveChunks(cache, opts) {
|
|
|
117
201
|
content: r.content,
|
|
118
202
|
rank: r.rank
|
|
119
203
|
}));
|
|
120
|
-
|
|
204
|
+
const kept = enforceBudget(mapped, budget);
|
|
205
|
+
const key = (c) => `${c.filePath}\u0000${c.content.length}`;
|
|
206
|
+
const have = new Set(kept.map(key));
|
|
207
|
+
const hops = [];
|
|
208
|
+
for (const name of hopNames(kept.map(c => c.content).join('\n'), tokens)) {
|
|
209
|
+
const def = definitionChunk(cache, opts, name);
|
|
210
|
+
if (!def || have.has(key(def)))
|
|
211
|
+
continue;
|
|
212
|
+
hops.push(def);
|
|
213
|
+
}
|
|
214
|
+
if (hops.length === 0)
|
|
215
|
+
return kept;
|
|
216
|
+
// Hops sit directly behind the top-ranked chunk, so re-budgeting drops the
|
|
217
|
+
// WEAKEST original rather than the definition that explains the strongest.
|
|
218
|
+
// The budget itself does not move.
|
|
219
|
+
return enforceBudget([kept[0], ...hops, ...kept.slice(1)], budget);
|
|
121
220
|
}
|
|
@@ -6,9 +6,9 @@
|
|
|
6
6
|
* bodies and private items dropped. That is what `surface` below does, and it is
|
|
7
7
|
* why this row needs code where the npm row needed none.
|
|
8
8
|
*
|
|
9
|
-
*
|
|
10
|
-
* `
|
|
11
|
-
* a TOML dependency would cost a dependency.
|
|
9
|
+
* No TOML parser. `Cargo.lock` is generated with a fixed `[[package]]` shape, and
|
|
10
|
+
* `Cargo.toml`'s dependency tables are read for their KEYS only, so a line reader
|
|
11
|
+
* covers both where a TOML dependency would cost a dependency.
|
|
12
12
|
*/
|
|
13
13
|
import { type ResolvedPackage } from './docs-resolve.js';
|
|
14
14
|
import type { NpmVersionInfo } from './npm-version.js';
|
|
@@ -112,4 +112,15 @@ export declare function rustSurface(src: string, insideTrait?: boolean, topLevel
|
|
|
112
112
|
export declare function isRustFile(name: string): boolean;
|
|
113
113
|
/** The `[package] name` of a cargo project, for labelling its own source. */
|
|
114
114
|
export declare function cargoProjectName(cwd: string): string | null;
|
|
115
|
+
/**
|
|
116
|
+
* The crate names `Cargo.toml` itself declares, under both `-` and `_` spellings.
|
|
117
|
+
*
|
|
118
|
+
* NOT {@link lockedDeps}: a lock file is the whole transitive closure, so it
|
|
119
|
+
* answers "can this resolve" and not "may this crate `use` it". The live run of
|
|
120
|
+
* 2026-09-05 answered about `tower` — in the lock via axum, absent from
|
|
121
|
+
* `[dependencies]` — and the crate did not compile.
|
|
122
|
+
*
|
|
123
|
+
* Undefined when there is no readable manifest, which is not "declares nothing".
|
|
124
|
+
*/
|
|
125
|
+
export declare function manifestCrates(cwd: string): Set<string> | undefined;
|
|
115
126
|
export {};
|
|
@@ -6,9 +6,9 @@
|
|
|
6
6
|
* bodies and private items dropped. That is what `surface` below does, and it is
|
|
7
7
|
* why this row needs code where the npm row needed none.
|
|
8
8
|
*
|
|
9
|
-
*
|
|
10
|
-
* `
|
|
11
|
-
* a TOML dependency would cost a dependency.
|
|
9
|
+
* No TOML parser. `Cargo.lock` is generated with a fixed `[[package]]` shape, and
|
|
10
|
+
* `Cargo.toml`'s dependency tables are read for their KEYS only, so a line reader
|
|
11
|
+
* covers both where a TOML dependency would cost a dependency.
|
|
12
12
|
*/
|
|
13
13
|
import * as fs from 'node:fs';
|
|
14
14
|
import * as path from 'node:path';
|
|
@@ -791,3 +791,44 @@ export function cargoProjectName(cwd) {
|
|
|
791
791
|
const match = /^\s*name\s*=\s*"([^"]+)"/m.exec(section?.[1] ?? '');
|
|
792
792
|
return match ? match[1] : null;
|
|
793
793
|
}
|
|
794
|
+
/**
|
|
795
|
+
* The crate names `Cargo.toml` itself declares, under both `-` and `_` spellings.
|
|
796
|
+
*
|
|
797
|
+
* NOT {@link lockedDeps}: a lock file is the whole transitive closure, so it
|
|
798
|
+
* answers "can this resolve" and not "may this crate `use` it". The live run of
|
|
799
|
+
* 2026-09-05 answered about `tower` — in the lock via axum, absent from
|
|
800
|
+
* `[dependencies]` — and the crate did not compile.
|
|
801
|
+
*
|
|
802
|
+
* Undefined when there is no readable manifest, which is not "declares nothing".
|
|
803
|
+
*/
|
|
804
|
+
export function manifestCrates(cwd) {
|
|
805
|
+
const text = safeRead(path.join(cwd, 'Cargo.toml'));
|
|
806
|
+
if (text === null)
|
|
807
|
+
return undefined;
|
|
808
|
+
const out = new Set();
|
|
809
|
+
const add = (name) => {
|
|
810
|
+
for (const key of new Set([name, canonical(name)]))
|
|
811
|
+
out.add(key);
|
|
812
|
+
};
|
|
813
|
+
let inDeps = false;
|
|
814
|
+
for (const raw of text.split('\n')) {
|
|
815
|
+
const line = raw.trim();
|
|
816
|
+
const header = /^\[([^\]]+)\]$/.exec(line);
|
|
817
|
+
if (header) {
|
|
818
|
+
const section = header[1];
|
|
819
|
+
// `[dependencies.serde]` and `[target.'cfg(unix)'.dependencies]` both
|
|
820
|
+
// declare, and the first names its crate in the header itself.
|
|
821
|
+
const table = /^(?:target\.[^.]*\.)?(?:dev-|build-)?dependencies(?:\.(.+))?$/.exec(section);
|
|
822
|
+
inDeps = table !== null && table[1] === undefined;
|
|
823
|
+
if (table?.[1])
|
|
824
|
+
add(table[1]);
|
|
825
|
+
continue;
|
|
826
|
+
}
|
|
827
|
+
if (!inDeps)
|
|
828
|
+
continue;
|
|
829
|
+
const key = /^([A-Za-z0-9_-]+)\s*=/.exec(line);
|
|
830
|
+
if (key)
|
|
831
|
+
add(key[1]);
|
|
832
|
+
}
|
|
833
|
+
return out;
|
|
834
|
+
}
|
|
@@ -91,3 +91,12 @@ export declare function haskellSurface(rawSrc: string): string;
|
|
|
91
91
|
export declare function isHaskellFile(name: string): boolean;
|
|
92
92
|
/** The `name:` field of the project's own `.cabal` file. */
|
|
93
93
|
export declare function hackageProjectName(cwd: string): string | null;
|
|
94
|
+
/**
|
|
95
|
+
* The package names the project's `.cabal` file declares in `build-depends`,
|
|
96
|
+
* across every stanza.
|
|
97
|
+
*
|
|
98
|
+
* NOT {@link resolvedVersions}: that reads the cabal install plan, which is the
|
|
99
|
+
* whole transitive closure, so it cannot say whether a module may be imported.
|
|
100
|
+
* Undefined when there is no readable `.cabal` file.
|
|
101
|
+
*/
|
|
102
|
+
export declare function manifestPackages(cwd: string): Set<string> | undefined;
|
|
@@ -506,3 +506,49 @@ export function hackageProjectName(cwd) {
|
|
|
506
506
|
const match = /^\s*name\s*:\s*(\S+)/m.exec(safeRead(path.join(cwd, cabal)) ?? '');
|
|
507
507
|
return match ? match[1] : null;
|
|
508
508
|
}
|
|
509
|
+
/**
|
|
510
|
+
* The package names the project's `.cabal` file declares in `build-depends`,
|
|
511
|
+
* across every stanza.
|
|
512
|
+
*
|
|
513
|
+
* NOT {@link resolvedVersions}: that reads the cabal install plan, which is the
|
|
514
|
+
* whole transitive closure, so it cannot say whether a module may be imported.
|
|
515
|
+
* Undefined when there is no readable `.cabal` file.
|
|
516
|
+
*/
|
|
517
|
+
export function manifestPackages(cwd) {
|
|
518
|
+
let entries;
|
|
519
|
+
try {
|
|
520
|
+
entries = fs.readdirSync(cwd);
|
|
521
|
+
}
|
|
522
|
+
catch {
|
|
523
|
+
return undefined;
|
|
524
|
+
}
|
|
525
|
+
const cabal = entries.find(e => e.endsWith('.cabal'));
|
|
526
|
+
if (!cabal)
|
|
527
|
+
return undefined;
|
|
528
|
+
const text = safeRead(path.join(cwd, cabal));
|
|
529
|
+
if (text === null)
|
|
530
|
+
return undefined;
|
|
531
|
+
const out = new Set();
|
|
532
|
+
let inDepends = false;
|
|
533
|
+
for (const raw of text.split('\n')) {
|
|
534
|
+
const line = raw.replace(/--.*$/, '');
|
|
535
|
+
const start = /^\s*build-depends\s*:(.*)$/i.exec(line);
|
|
536
|
+
const body = start ? start[1] : line;
|
|
537
|
+
if (start)
|
|
538
|
+
inDepends = true;
|
|
539
|
+
else if (!inDepends)
|
|
540
|
+
continue;
|
|
541
|
+
// A continuation is indented; a new field at the stanza's own indent ends
|
|
542
|
+
// the list. Both `,`-leading and `,`-trailing layouts are in the wild.
|
|
543
|
+
else if (/^\s*[A-Za-z-]+\s*:/.test(line) || line.trim() === '') {
|
|
544
|
+
inDepends = false;
|
|
545
|
+
continue;
|
|
546
|
+
}
|
|
547
|
+
for (const part of body.split(',')) {
|
|
548
|
+
const name = /^\s*([A-Za-z0-9][A-Za-z0-9_-]*)/.exec(part);
|
|
549
|
+
if (name)
|
|
550
|
+
out.add(name[1]);
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
return out;
|
|
554
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mjasnikovs/pi-task",
|
|
3
|
-
"version": "0.40.
|
|
3
|
+
"version": "0.40.2",
|
|
4
4
|
"description": "Deterministic task planning and spec-orchestration for local models — crash-safe /task pipelines with verify/enforce gates, a real-time remote web view, and web/docs/fetch/worker subagent tools.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|