@mjasnikovs/pi-task 0.40.5 → 0.40.6

package/dist/workers/docs-core.js

@@ -465,8 +465,18 @@ export async function docsRaw(input) {
     catch (err) {
         cacheError = err instanceof Error ? err.message : String(err);
     }
+    // A facade package exports names it does not declare, and its index is a
+    // table of contents until the packages that DO declare them are folded in.
+    // Best-effort: a supplement that will not resolve leaves the index as it was.
+    let supplements;
+    try {
+        supplements = (await profile.supplements?.(pkg, input.cwd, io)) ?? [];
+    }
+    catch {
+        supplements = [];
+    }
     const result = cache ?
-        docsRawCached(cache, pkg, profile, input.query, ensureIndexed, retrieveChunks, autoInstalled)
+        docsRawCached(cache, pkg, profile, input.query, ensureIndexed, retrieveChunks, autoInstalled, supplements)
         : docsRawUncached(pkg, profile, cacheError ?? 'unknown cache error', autoInstalled);
     result.npmVersion = await npmVersionPromise;
     result.registryLabel = registryLabel;
@@ -474,11 +484,11 @@ export async function docsRaw(input) {
         result.autoInstallPin = autoInstallPin;
     return result;
 }
-function docsRawCached(cache, pkg, profile, query, ensureIndexed, retrieveChunks, autoInstalled) {
+function docsRawCached(cache, pkg, profile, query, ensureIndexed, retrieveChunks, autoInstalled, supplements = []) {
     let indexResult;
     const t0 = Date.now();
     try {
-        indexResult = ensureIndexed(cache, pkg, profile);
+        indexResult = ensureIndexed(cache, pkg, profile, supplements);
     }
     catch (err) {
         return {

package/dist/workers/docs-ecosystems.d.ts

@@ -73,6 +73,12 @@ export interface EcosystemProfile {
         installed: boolean;
         pin?: AutoInstallPin;
     }>;
+    /**
+     * Packages whose declarations belong in THIS package's index, because this
+     * package exports names it does not declare. Only hackage has facades of the
+     * `hspec`/`hspec-core` shape; see DEFECT-12-STOPPING-RULE.md.
+     */
+    supplements?: (pkg: ResolvedPackage, cwd: string, io: EcosystemIo) => Promise<ResolvedPackage[]>;
     /** The registry's own newest version, for grounding an answer in the present. */
     latest: (name: string, io: EcosystemIo) => Promise<NpmVersionInfo | null>;
     /** True for a file that carries the package's public API surface. */

package/dist/workers/docs-ecosystems.js

@@ -22,7 +22,7 @@ import { resolvePackage, isDtsFile, isValidModuleName } from './docs-resolve.js'
 import { DECL_SPLIT_RE } from './docs-chunk.js';
 import { npmVersionLookup } from './npm-version.js';
 import { resolveCrate, cratesLatest, crateTarballUrl, crateOf, isValidCrateName, isRustFile, lockedVersion, rustSurface, cargoProjectName, childDirs, lockedDeps, manifestCrates, CARGO_DECL_SPLIT_RE } from './eco-cargo.js';
-import { resolveHackage, hackageLatest, hackageVersion, hackageTarballUrl, hackageExtractDir, hackageProjectName, findCabalTarball, cachedVersions, resolvedVersions, manifestPackages, isValidHackageName, isHaskellFile, haskellSurface, HACKAGE_DECL_SPLIT_RE, HACKAGE_SKIP_DIRS } from './eco-hackage.js';
+import { resolveHackage, hackageLatest, hackageVersion, hackageTarballUrl, hackageExtractDir, hackageProjectName, supplementCandidates, findCabalTarball, cachedVersions, resolvedVersions, manifestPackages, isValidHackageName, isHaskellFile, haskellSurface, HACKAGE_DECL_SPLIT_RE, HACKAGE_SKIP_DIRS } from './eco-hackage.js';
 import { runChild } from '../shared/child-process.js';
 /**
  * Is any of `names` present at `cwd` or above it?
@@ -354,6 +354,33 @@ const hackageProfile = {
             };
         }
     },
+    supplements: async (pkg, cwd, io) => {
+        const deps = manifestPackages(pkg.root);
+        if (!deps)
+            return [];
+        const candidates = supplementCandidates(pkg.name, deps, resolvedVersions(cwd) ?? {});
+        const out = [];
+        for (const c of candidates) {
+            try {
+                out.push(resolveHackage(c.name, cwd, { modulesDir: io.modulesDir }));
+                continue;
+            }
+            catch {
+                // Not unpacked yet. `acquire` prefers the tarball cabal already
+                // downloaded as a dependency, so this is local work, not a fetch.
+            }
+            const got = await hackageProfile.acquire(c.name, c.version, io);
+            if (!got.success)
+                continue;
+            try {
+                out.push(resolveHackage(c.name, cwd, { modulesDir: io.modulesDir }));
+            }
+            catch {
+                // A supplement that will not resolve leaves the facade as it was.
+            }
+        }
+        return out;
+    },
     latest: (name, io) => hackageLatest(name, io.fetch, io.signal),
     isSurfaceFile: isHaskellFile,
     surface: haskellSurface,

package/dist/workers/docs-index.d.ts

@@ -35,4 +35,4 @@ export interface IndexResult {
  * have said so.
  */
 export declare function chunkerFingerprint(): string;
-export declare function ensureIndexed(cache: CacheHandle, pkg: ResolvedPackage, profile?: EcosystemProfile): IndexResult;
+export declare function ensureIndexed(cache: CacheHandle, pkg: ResolvedPackage, profile?: EcosystemProfile, supplements?: readonly ResolvedPackage[]): IndexResult;

package/dist/workers/docs-index.js

@@ -4,6 +4,7 @@ import * as path from 'node:path';
 import {} from './docs-resolve.js';
 import { chunkDeclarations, chunkReadme, splitAtMatches } from './docs-chunk.js';
 import { ECOSYSTEMS } from './docs-ecosystems.js';
+import { hackageExportGap, declaredInSurface } from './eco-hackage.js';
 const ZERO_SEP = Buffer.from([0]);
 /**
  * The gate that decides whether a package needs re-indexing.
@@ -35,7 +36,7 @@ const ZERO_SEP = Buffer.from([0]);
 export function chunkerFingerprint() {
     return `${String(splitAtMatches)}\u0000${String(chunkDeclarations)}\u0000${String(chunkReadme)}`;
 }
-function computeContentHash(pkg, profile) {
+function computeContentHash(pkg, profile, supplements = []) {
     const hash = createHash('sha256');
     hash.update(Buffer.from(`${pkg.name}@${pkg.version}`, 'utf8'));
     hash.update(ZERO_SEP);
@@ -48,6 +49,15 @@ function computeContentHash(pkg, profile) {
     hash.update(Buffer.from(`${String(profile.isSurfaceFile)}\u0000${String(dropParallelDeclarations)}`
         + `\u0000${String(dropDeadMajors)}`, 'utf8'));
     hash.update(ZERO_SEP);
+    // The extractor and the writer, by source. Surfacing only `pkg.entry` below
+    // leaves a package cached whenever a fix moves some OTHER module — the
+    // wrapped `instance` head is in aeson's `Types/FromJSON.hs`, never its entry
+    // — and nothing surfaced the duplicate drop in `ingestBody` at all.
+    hash.update(Buffer.from(`${String(profile.surface)}\u0000${String(ingestBody)}`, 'utf8'));
+    hash.update(ZERO_SEP);
+    // Which packages were folded in, so gaining or losing one re-indexes.
+    hash.update(Buffer.from(supplements.map(s => `${s.name}@${s.version}`).join('\u0000'), 'utf8'));
+    hash.update(ZERO_SEP);
     if (pkg.entry && fs.existsSync(pkg.entry)) {
         try {
             hash.update(Buffer.from(profile.surface(fs.readFileSync(pkg.entry, 'utf8')), 'utf8'));
@@ -157,7 +167,7 @@ function collectFiles(pkg, profile) {
         readme: pkg.readme
     };
 }
-function ingestBody(cache, pkg, profile, contentHash) {
+function ingestBody(cache, pkg, profile, contentHash, supplements = []) {
     const ecosystem = profile.id;
     const inside = cache.db
         .prepare('SELECT content_hash FROM packages WHERE ecosystem = ? AND name = ? AND version = ?')
@@ -172,6 +182,12 @@ function ingestBody(cache, pkg, profile, contentHash) {
     let chunksWritten = 0;
     let filesIngested = 0;
     const insertChunk = cache.db.prepare('INSERT INTO chunks (ecosystem, name, version, file_path, kind, content) VALUES (?, ?, ?, ?, ?, ?)');
+    // The content carries its own `<comment> <path>` header, so two rows that
+    // match on it are the same declaration from the same file, written twice.
+    // aeson's `Data.Aeson.KeyMap` declares its whole API once per `#ifdef`
+    // branch and nothing here preprocesses CPP: 43 of aeson's 55 duplicate
+    // bodies. A second copy carries no second fact and still spends a slot.
+    const seen = new Set();
     for (const abs of files.surface) {
         // Normalise the separator before storing, so the same package indexes to
         // the same rows whatever built the path. The value is a MODEL-FACING
@@ -190,10 +206,46 @@ function ingestBody(cache, pkg, profile, contentHash) {
             continue;
         filesIngested++;
         for (const c of chunks) {
+            if (seen.has(c))
+                continue;
+            seen.add(c);
             insertChunk.run(ecosystem, pkg.name, pkg.version, rel, 'dts', c);
             chunksWritten++;
         }
     }
+    // A facade package indexes to a table of contents: `hspec` is 14 chunks of
+    // export lists and every signature is in `hspec-core`. Fill only the holes —
+    // see DEFECT-12-STOPPING-RULE.md for the boundary and why it stops here.
+    const gap = supplements.length > 0 ? hackageExportGap(pkg.root) : null;
+    for (const sup of gap && (gap.unresolved.size > 0 || gap.reexportedModules.size > 0) ?
+        supplements
+        : []) {
+        for (const abs of collectFiles(sup, profile).surface) {
+            let raw;
+            try {
+                raw = fs.readFileSync(abs, 'utf8');
+            }
+            catch {
+                continue;
+            }
+            const module = /^module\s+([\w.']+)/m.exec(raw)?.[1];
+            const whole = module !== undefined && gap.reexportedModules.has(module);
+            // The path names the package the declaration really came from: the
+            // chunk header is model-facing, and a signature attributed to the
+            // wrong package is the bug this whole table exists for.
+            const rel = `${sup.name}-${sup.version}/` + path.relative(sup.root, abs).replace(/\\/g, '/');
+            for (const c of chunkDeclarations(profile.surface(raw), rel, profile.declSplitRe, profile.commentPrefix)) {
+                const declares = declaredInSurface(c.replace(/^\S.*\n/, ''));
+                if (!whole && ![...declares].some(n => gap.unresolved.has(n)))
+                    continue;
+                if (seen.has(c))
+                    continue;
+                seen.add(c);
+                insertChunk.run(ecosystem, pkg.name, pkg.version, rel, 'dts', c);
+                chunksWritten++;
+            }
+        }
+    }
     if (files.readme) {
         const rel = path.relative(pkg.root, files.readme).replace(/\\/g, '/');
         const raw = fs.readFileSync(files.readme, 'utf8');
@@ -211,9 +263,9 @@ function ingestBody(cache, pkg, profile, contentHash) {
         .run(ecosystem, pkg.name, pkg.version, contentHash, Date.now());
     return { hitCache: false, filesIngested, chunksWritten };
 }
-export function ensureIndexed(cache, pkg, profile = ECOSYSTEMS[pkg.ecosystem]) {
+export function ensureIndexed(cache, pkg, profile = ECOSYSTEMS[pkg.ecosystem], supplements = []) {
     const ecosystem = profile.id;
-    const contentHash = computeContentHash(pkg, profile);
+    const contentHash = computeContentHash(pkg, profile, supplements);
     const existing = cache.db
         .prepare('SELECT content_hash FROM packages WHERE ecosystem = ? AND name = ? AND version = ?')
         .get(ecosystem, pkg.name, pkg.version);
@@ -223,7 +275,7 @@ export function ensureIndexed(cache, pkg, profile = ECOSYSTEMS[pkg.ecosystem]) {
     cache.db.exec('BEGIN IMMEDIATE');
     let result;
     try {
-        result = ingestBody(cache, pkg, profile, contentHash);
+        result = ingestBody(cache, pkg, profile, contentHash, supplements);
         cache.db.exec('COMMIT');
     }
     catch (err) {

package/dist/workers/eco-hackage.d.ts

@@ -100,3 +100,42 @@ export declare function hackageProjectName(cwd: string): string | null;
  * Undefined when there is no readable `.cabal` file.
  */
 export declare function manifestPackages(cwd: string): Set<string> | undefined;
+/**
+ * What a package exports but does not declare.
+ *
+ * `hspec` indexes to 14 chunks of export lists: `it`, `describe` and `shouldBe`
+ * are in the corpus as bare names with no signature attached, because every
+ * signature is in `hspec-core`. The whole index is a table of contents.
+ *
+ * Both shapes are here because either alone misses half of it. A name-level
+ * re-export puts the name in the export list; a `module X` re-export puts
+ * nothing there at all, which is why `shouldBe` is invisible to the first.
+ *
+ * See DEFECT-12-STOPPING-RULE.md for why this triggers on the hole itself
+ * rather than on a fraction of the export list.
+ */
+export interface HackageExportGap {
+    /** Exported names with no declaration anywhere in the package. */
+    unresolved: Set<string>;
+    /** `module X` re-exports of modules this package does not own. */
+    reexportedModules: Set<string>;
+}
+/** Every name the extracted surface declares: signatures, heads, constructors, fields. */
+export declare function declaredInSurface(surface: string): Set<string>;
+export declare function hackageExportGap(root: string): HackageExportGap;
+/**
+ * Which declared dependencies may be opened to fill the gap.
+ *
+ * Hackage splits a facade from its implementation by name — `hspec`/`hspec-core`,
+ * `hspec`/`hspec-expectations` — and that convention is the whole bound. Without
+ * it the rule has to fetch every `build-depends` entry to find out whether it
+ * declares anything: aeson names 38 of them and would resolve none, because its
+ * nine unresolved exports are CPP macros and internal punctuation helpers.
+ *
+ * The cost of the bound is stated rather than hidden: `scotty` re-exports
+ * sixteen names from `cookie`, which shares no prefix, so that hole stays open.
+ */
+export declare function supplementCandidates(pkgName: string, declaredDeps: ReadonlySet<string>, resolved: Readonly<Record<string, string>>): Array<{
+    name: string;
+    version: string;
+}>;

package/dist/workers/eco-hackage.js

@@ -381,6 +381,40 @@ function haddockBlockAt(lines, start) {
     }
     return { text: text.filter(l => l.trim() !== '--'), next: lines.length };
 }
+/**
+ * A head line that has not finished: an open bracket, an arrow, a pragma, or the
+ * bare keyword — aeson writes the constraint block on the lines underneath.
+ */
+const INSTANCE_HEAD_CONTINUES_RE = /(?:=>|->|,|\(|\[|#-\})\s*$|^\s*instance\s*$/;
+/**
+ * The head of an `instance`, which may wrap.
+ *
+ * `instance {-# OVERLAPPING #-}` and `instance ( Selector s` are both a first
+ * line that names nothing, and aeson ships twelve chunks that are exactly that —
+ * each one still costing a retrieval slot. The head runs to `where`, or to the
+ * first line that closes its brackets without ending mid-declaration.
+ */
+function instanceHead(block) {
+    const head = [];
+    let depth = 0;
+    for (const line of block) {
+        const w = /\bwhere\b/.exec(line);
+        if (w) {
+            head.push(line.slice(0, w.index + 'where'.length).trimEnd());
+            return head;
+        }
+        head.push(line);
+        for (const c of line) {
+            if (c === '(' || c === '[')
+                depth++;
+            else if (c === ')' || c === ']')
+                depth--;
+        }
+        if (depth <= 0 && !INSTANCE_HEAD_CONTINUES_RE.test(line))
+            return head;
+    }
+    return head;
+}
 export function haskellSurface(rawSrc) {
     // Haddock blocks survive; ordinary `{- … -}` commentary does not.
     const src = stripBlockComments(rawSrc);
@@ -478,7 +512,7 @@ export function haskellSurface(rawSrc) {
         else if (TYPE_HEAD_RE.test(line)) {
             const head = TYPE_HEAD_RE.exec(line)[1];
             // An instance's indented lines are definitions, not fields.
-            out.push(...pending, ...(head === 'instance' ? [line] : block), '');
+            out.push(...pending, ...(head === 'instance' ? instanceHead(block) : block), '');
         }
         pending = [];
         i = j;
@@ -552,3 +586,132 @@ export function manifestPackages(cwd) {
     }
     return out;
 }
+const EXPORT_NAME_RE = /^[A-Za-z_][\w']*$/;
+/** `module X` inside an export list is a re-export; `Prelude` is base, and base is not fetched. */
+const REEXPORT_RE = /\bmodule\s+([\w.']+)/g;
+/** The text between `module M (` and its balancing `)`. */
+function exportListText(src) {
+    const m = /^module\s+[\w.']+\s*/m.exec(src);
+    if (!m)
+        return '';
+    const open = src.indexOf('(', m.index);
+    if (open < 0)
+        return '';
+    let depth = 0;
+    for (let i = open; i < src.length; i++) {
+        if (src[i] === '(')
+            depth++;
+        else if (src[i] === ')' && --depth === 0)
+            return src.slice(open + 1, i);
+    }
+    return '';
+}
+const SURFACE_DECL_RE = /^([a-z_][\w']*)\s*::|^\(([^)]+)\)\s*::|^(?:data|newtype|type|class)\s+(?:family\s+)?(?:[^=>]*=>\s*)?([A-Z][\w']*)/;
+/** Every name the extracted surface declares: signatures, heads, constructors, fields. */
+export function declaredInSurface(surface) {
+    const out = new Set();
+    const lines = surface.split('\n');
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        const m = SURFACE_DECL_RE.exec(line);
+        if (m) {
+            out.add((m[1] ?? m[2] ?? m[3]).trim());
+            continue;
+        }
+        if (BARE_NAME_RE.test(line) && (lines[i + 1]?.trim().startsWith('::') ?? false)) {
+            for (const n of line.split(','))
+                out.add(n.trim());
+            continue;
+        }
+        if (!/^\s/.test(line))
+            continue;
+        for (const c of line.matchAll(/(?:^|[=|])\s*([A-Z][\w']*)/g))
+            out.add(c[1]);
+        for (const f of line.matchAll(/(?:^|[,{])\s*([a-z_][\w']*)\s*::/g))
+            out.add(f[1]);
+    }
+    return out;
+}
+/** Read every `.hs` under `root`, skipping the directories no surface pass reads. */
+function haskellSources(root) {
+    const out = [];
+    const walk = (dir) => {
+        let entries;
+        try {
+            entries = fs.readdirSync(dir, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const e of entries) {
+            if (e.isDirectory()) {
+                if (!HACKAGE_SKIP_DIRS.includes(e.name))
+                    walk(path.join(dir, e.name));
+            }
+            else if (isHaskellFile(e.name))
+                out.push(path.join(dir, e.name));
+        }
+    };
+    walk(root);
+    return out;
+}
+export function hackageExportGap(root) {
+    const declared = new Set();
+    const exported = new Set();
+    const ownModules = new Set();
+    const reexportedModules = new Set();
+    for (const file of haskellSources(root)) {
+        const src = safeRead(file);
+        if (src === null)
+            continue;
+        for (const n of declaredInSurface(haskellSurface(src)))
+            declared.add(n);
+        const own = /^module\s+([\w.']+)/m.exec(src);
+        if (own)
+            ownModules.add(own[1]);
+        const list = exportListText(src);
+        for (const m of list.matchAll(REEXPORT_RE))
+            reexportedModules.add(m[1]);
+        for (const raw of list.split('\n')) {
+            const line = raw.replace(/--.*$/, '').replace(REEXPORT_RE, '');
+            for (const token of line.split(/[,\s]+/)) {
+                const name = token
+                    .replace(/\(\.\.\)$/, '')
+                    .replace(/[(),]/g, '')
+                    .trim();
+                if (EXPORT_NAME_RE.test(name))
+                    exported.add(name);
+            }
+        }
+    }
+    for (const m of ownModules)
+        reexportedModules.delete(m);
+    reexportedModules.delete('Prelude');
+    return {
+        unresolved: new Set([...exported].filter(n => !declared.has(n))),
+        reexportedModules
+    };
+}
+/**
+ * Which declared dependencies may be opened to fill the gap.
+ *
+ * Hackage splits a facade from its implementation by name — `hspec`/`hspec-core`,
+ * `hspec`/`hspec-expectations` — and that convention is the whole bound. Without
+ * it the rule has to fetch every `build-depends` entry to find out whether it
+ * declares anything: aeson names 38 of them and would resolve none, because its
+ * nine unresolved exports are CPP macros and internal punctuation helpers.
+ *
+ * The cost of the bound is stated rather than hidden: `scotty` re-exports
+ * sixteen names from `cookie`, which shares no prefix, so that hole stays open.
+ */
+export function supplementCandidates(pkgName, declaredDeps, resolved) {
+    const out = [];
+    for (const dep of declaredDeps) {
+        if (!dep.startsWith(`${pkgName}-`))
+            continue;
+        const version = resolved[dep];
+        if (version)
+            out.push({ name: dep, version });
+    }
+    return out.sort((a, b) => a.name.localeCompare(b.name));
+}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@mjasnikovs/pi-task",
-    "version": "0.40.5",
+    "version": "0.40.6",
     "description": "Deterministic task planning and spec-orchestration for local models — crash-safe /task pipelines with verify/enforce gates, a real-time remote web view, and web/docs/fetch/worker subagent tools.",
     "type": "module",
     "main": "./dist/index.js",