@mjasnikovs/pi-task 0.40.50 → 0.41.0

package/dist/workers/pi-worker-fetch.d.ts

@@ -35,8 +35,9 @@ export declare function registerPiWorkerFetch(pi: ExtensionAPI, internals?: PiWo
  * refused.
  */
 export declare function fetchCacheable(d: Pick<FetchDetails, 'answer'>): boolean;
-/** The fetch cache key. URL verbatim (path case can matter), question normalised —
- *  same page, different question is a different answer. */
+/** The fetch cache key. URL verbatim (path case can matter), question as its sorted
+ *  distinctive tokens — same page, different question is a different answer, but two
+ *  phrasings of one question are not. */
 export declare function fetchCacheKey(params: {
     url: string;
     query: string;

package/dist/workers/pi-worker-fetch.js

@@ -4,7 +4,7 @@ import { fetchAndClean as defaultFetchAndClean, FetchAndCleanError } from './htm
 import { fetchFocused } from './fetch-core.js';
 import { formatResultText } from '../shared/child-output.js';
 import { childFailureReason, makeWorkerTool, workerAnswer, workerUnavailable } from './shared.js';
-import { normalizeQuery } from './research-cache.js';
+import { queryTokenKey } from './research-cache.js';
 import { isAbstention } from './abstention.js';
 const RENDER_QUERY_MAX = 100;
 const Params = Type.Object({
@@ -125,8 +125,9 @@ export function fetchCacheable(d) {
     // ordinary shape of an abstention.
     return d.answer !== undefined && !isAbstention(d.answer);
 }
-/** The fetch cache key. URL verbatim (path case can matter), question normalised —
- *  same page, different question is a different answer. */
+/** The fetch cache key. URL verbatim (path case can matter), question as its sorted
+ *  distinctive tokens — same page, different question is a different answer, but two
+ *  phrasings of one question are not. */
 export function fetchCacheKey(params) {
-    return `${params.url.trim()}::${normalizeQuery(params.query)}`;
+    return `${params.url.trim()}\u0000${queryTokenKey(params.query)}`;
 }

package/dist/workers/pi-worker-search.js

@@ -3,7 +3,7 @@ import { Text } from '@earendil-works/pi-tui';
 import { getConfig } from '../config/config.js';
 import { search } from './search-core.js';
 import { makeWorkerTool, workerAnswer, workerUnavailable } from './shared.js';
-import { normalizeQuery } from './research-cache.js';
+import { queryTokenKey } from './research-cache.js';
 const Params = Type.Object({
     query: Type.String({ description: 'Search query.' }),
     count: Type.Optional(Type.Integer({
@@ -62,9 +62,10 @@ export function registerPiWorkerSearch(pi, internals = {}) {
         // the live web again. Three parts key the entry: the PROVIDER, because two
         // engines' result sets for one query are different answers and must not serve
         // for each other; the query, lowercased and whitespace-collapsed by
-        // `normalizeQuery` so phrasing variants share an entry; and the COUNT, because a
-        // larger request is a different result set.
-        cacheKey: params => `${provider()}::${normalizeQuery(params.query)}::${params.count ?? ''}`,
+        // `queryTokenKey` so phrasing variants share an entry; and the COUNT, because a
+        // larger request is a different result set. The tokens go LAST because that is
+        // where research-cache.ts looks for the question it may widen a hit from.
+        cacheKey: params => `${provider()}::${params.count ?? ''}\u0000${queryTokenKey(params.query)}`,
         // Only a non-empty result set is worth caching. An empty one falls through so a
         // later attempt can succeed; no-key and engine errors never reach this at all,
         // since makeWorkerTool refuses to store an `unavailable` outcome.

package/dist/workers/research-cache.d.ts

@@ -16,8 +16,11 @@ export declare function newRunToken(): string;
  * caching is enabled it stamps a FRESH token (so a long-lived host never reuses a
  * prior run's token, and planAuto + the task loop of THIS run share one id); when
  * disabled it clears any token a prior run left, so the workers cache nothing.
+ *
+ * `token` lets the caller supply the run's OWN id (task/run-context.ts) rather
+ * than a second one minted here — one run, one id.
  */
-export declare function configureResearchRun(enabled: boolean): string | undefined;
+export declare function configureResearchRun(enabled: boolean, token?: string): string | undefined;
 /**
  * Normalise a query/module string for the cache KEY: collapse whitespace and
  * lowercase, so trivially-varied phrasings of the same question share a digest. The
@@ -25,6 +28,25 @@ export declare function configureResearchRun(enabled: boolean): string | undefin
  * of asking the same thing resolve to the same (correct) result.
  */
 export declare function normalizeQuery(s: string): string;
+/**
+ * The QUESTION half of a cache key: the query's distinctive tokens, sorted.
+ *
+ * Normalised text made the key an exact-phrasing match, and two children asking
+ * one question in two sentences missed each other every time — the run's hit rate
+ * was ~0%. Sorted tokens collapse word order and phrasing; what they do NOT
+ * collapse is scope, which is why a hit also has to pass {@link coversQuery}.
+ */
+export declare function queryTokenKey(query: string): string;
+/**
+ * Is `cached` an answer to `asked`? Only when the answer was produced for a
+ * question that asked for AT LEAST as much: every token of the new query must
+ * appear in the cached one.
+ *
+ * The direction matters and is not symmetric. An answer to "hono routing and
+ * middleware" contains the answer to "hono routing"; the reverse does not hold,
+ * and serving it would quietly drop half the question.
+ */
+export declare function coversQuery(cached: string, asked: string): boolean;
 /**
  * Resume hook: keep the interrupted run's cache id and PRUNE the entries the manifest
  * has invalidated, rather than discarding the whole cache because anything moved. Returns

package/dist/workers/research-cache.js

@@ -28,6 +28,12 @@
  * A resume therefore REUSES the interrupted run's id — but only on POSITIVE evidence
  * that the digests still describe the same dependency surface.
  *
+ * KEY SHAPE: `<tool>\0<subject>\0<sorted query tokens>` — the tool name, the package
+ * or url the answer is about, and the query reduced to its distinctive tokens. The
+ * token field is both the key and the stored token set, so there is one statement of
+ * it; a lookup that misses exactly falls back to any entry for the same subject whose
+ * tokens CONTAIN all of the new query's (see `coversQuery`).
+ *
  * PER-PACKAGE INVALIDATION. One fingerprint over the whole dependency block cannot be
  * that evidence: a greenfield run ADDS dependencies every few tasks, so every resume
  * sees a moved fingerprint and keeps nothing. Adding a package invalidates nothing that
@@ -78,6 +84,7 @@
  */
 import * as fsp from 'node:fs/promises';
 import * as path from 'node:path';
+import { contentTokens } from '../shared/content-tokens.js';
 import { tasksDir } from '../task/task-io.js';
 import { ECOSYSTEMS } from './docs-ecosystems.js';
 const RESEARCH_CACHE_FILE = 'research-cache.json';
@@ -156,13 +163,15 @@ export function newRunToken() {
  * caching is enabled it stamps a FRESH token (so a long-lived host never reuses a
  * prior run's token, and planAuto + the task loop of THIS run share one id); when
  * disabled it clears any token a prior run left, so the workers cache nothing.
+ *
+ * `token` lets the caller supply the run's OWN id (task/run-context.ts) rather
+ * than a second one minted here — one run, one id.
  */
-export function configureResearchRun(enabled) {
+export function configureResearchRun(enabled, token = newRunToken()) {
     if (!enabled) {
         delete process.env[RESEARCH_RUN_ID_ENV];
         return undefined;
     }
-    const token = newRunToken();
     process.env[RESEARCH_RUN_ID_ENV] = token;
     return token;
 }
@@ -175,6 +184,40 @@ export function configureResearchRun(enabled) {
 export function normalizeQuery(s) {
     return s.replace(/\s+/g, ' ').trim().toLowerCase();
 }
+/**
+ * The QUESTION half of a cache key: the query's distinctive tokens, sorted.
+ *
+ * Normalised text made the key an exact-phrasing match, and two children asking
+ * one question in two sentences missed each other every time — the run's hit rate
+ * was ~0%. Sorted tokens collapse word order and phrasing; what they do NOT
+ * collapse is scope, which is why a hit also has to pass {@link coversQuery}.
+ */
+export function queryTokenKey(query) {
+    return [...contentTokens(query)].sort().join(' ');
+}
+/**
+ * Is `cached` an answer to `asked`? Only when the answer was produced for a
+ * question that asked for AT LEAST as much: every token of the new query must
+ * appear in the cached one.
+ *
+ * The direction matters and is not symmetric. An answer to "hono routing and
+ * middleware" contains the answer to "hono routing"; the reverse does not hold,
+ * and serving it would quietly drop half the question.
+ */
+export function coversQuery(cached, asked) {
+    const have = new Set(cached.split(' ').filter(t => t.length > 0));
+    return asked.split(' ').every(t => t.length === 0 || have.has(t));
+}
+/** The `tool\0subject\0tokens` split of a key, or null when it carries no token
+ *  field (a URL fetch, whose question is the URL itself). */
+function splitTokenKey(key) {
+    const i = key.lastIndexOf('\u0000');
+    if (i <= 0)
+        return null;
+    const scope = key.slice(0, i + 1);
+    const tokens = key.slice(i + 1);
+    return scope.split('\u0000').length < 3 ? null : { scope, tokens };
+}
 /** The same question for any ecosystem: what does its manifest pin, name to version. */
 function depsMapFor(cwd, ecosystem) {
     return ECOSYSTEMS[ecosystem].declaredDeps(cwd);
@@ -280,9 +323,26 @@ export async function lookupResearch(cwd, runId, key) {
     const file = await readCacheFile(cwd);
     if (!file || file.runId !== runId)
         return undefined;
-    const entry = file.entries[key];
+    const entry = file.entries[key] ?? coveringEntry(file, key);
     return entry ? { text: entry.text, details: entry.details } : undefined;
 }
+/**
+ * A cached answer to a WIDER question about the same subject, for a key whose last
+ * field is a token set. Exact match is tried first; this is what turns two
+ * phrasings of one question into one lookup.
+ */
+function coveringEntry(file, key) {
+    const asked = splitTokenKey(key);
+    if (!asked)
+        return undefined;
+    for (const [k, entry] of Object.entries(file.entries)) {
+        if (!k.startsWith(asked.scope))
+            continue;
+        if (coversQuery(k.slice(asked.scope.length), asked.tokens))
+            return entry;
+    }
+    return undefined;
+}
 /** Write the cache file atomic-ish, so a concurrent reader never sees it half-written. */
 async function writeCacheFile(cwd, out) {
     const tmp = `${researchCacheFile(cwd)}.${process.pid}.${Math.random().toString(36).slice(2, 8)}.tmp`;

package/dist/workers/shared.d.ts

@@ -128,5 +128,23 @@ export interface WorkerToolSpec<TParams extends TSchema, TDetails> {
      */
     cacheable?(details: TDetails): boolean;
 }
+/**
+ * The per-run research cache, as a call wrapper.
+ *
+ * Extracted from `makeWorkerTool` because the host fans out to the SAME workers
+ * outside any tool call: `gatherExternalContext` (task/external-context.ts) looks
+ * up docs, pages and live versions for every task's research phase, in-process,
+ * and paid the network for all of it again on every task of the run. The cache was
+ * never about tool registration; it is about (tool, subject, question) -> answer.
+ *
+ * A null `rawKey`, or no run id stamped, means the call is not cached at all.
+ */
+export declare function cachedWorkerCall<TDetails>(cwd: string, tool: string, rawKey: string | null, run: () => Promise<WorkerOutcome<TDetails>>, rules?: {
+    cacheable?: (details: TDetails) => boolean;
+    pkg?: (details: TDetails) => CachePackage | undefined;
+}): Promise<{
+    text: string;
+    details: TDetails;
+}>;
 /** Register a worker tool from its spec, supplying the shared registration ritual. */
 export declare function makeWorkerTool<TParams extends TSchema, TDetails>(pi: ExtensionAPI, spec: WorkerToolSpec<TParams, TDetails>): void;

package/dist/workers/shared.js

@@ -39,6 +39,40 @@ export function childFailureReason(child) {
 export function workerUnavailable(text, details, reason) {
     return { kind: 'unavailable', text, details, reason };
 }
+/**
+ * The per-run research cache, as a call wrapper.
+ *
+ * Extracted from `makeWorkerTool` because the host fans out to the SAME workers
+ * outside any tool call: `gatherExternalContext` (task/external-context.ts) looks
+ * up docs, pages and live versions for every task's research phase, in-process,
+ * and paid the network for all of it again on every task of the run. The cache was
+ * never about tool registration; it is about (tool, subject, question) -> answer.
+ *
+ * A null `rawKey`, or no run id stamped, means the call is not cached at all.
+ */
+export async function cachedWorkerCall(cwd, tool, rawKey, run, rules = {}) {
+    // The stored key is namespaced by tool name, joined with a NUL — a byte no tool
+    // name or key can contain, so no pair of them can collide by concatenation.
+    const runId = researchRunId();
+    const cacheKey = runId !== undefined && rawKey !== null ? `${tool}\u0000${rawKey}` : null;
+    if (runId !== undefined && cacheKey !== null) {
+        const hit = await lookupResearch(cwd, runId, cacheKey);
+        if (hit !== undefined)
+            return { text: hit.text, details: hit.details };
+    }
+    const outcome = await run();
+    const { text, details } = outcome;
+    // A non-answer is never stored, whatever the caller's own rule says. The rule
+    // decides answer QUALITY; this decides whether there is an answer.
+    if (outcome.kind === 'answer'
+        && runId !== undefined
+        && cacheKey !== null
+        && (rules.cacheable ? rules.cacheable(details) : true)) {
+        const provenance = rules.pkg?.(details);
+        await storeResearch(cwd, runId, cacheKey, text, details, provenance?.pkg, provenance?.ecosystem);
+    }
+    return { text, details };
+}
 /** Register a worker tool from its spec, supplying the shared registration ritual. */
 export function makeWorkerTool(pi, spec) {
     pi.registerTool({
@@ -48,28 +82,10 @@ export function makeWorkerTool(pi, spec) {
         parameters: spec.parameters,
         executionMode: 'parallel',
         async execute(_toolCallId, params, signal, _onUpdate, ctx) {
-            // Per-run research cache: serve an earlier sibling's digest for the
-            // same (tool, package/url, query) instead of re-fetching. Active only when
-            // the orchestrator stamped a run id AND this call opts in via cacheKey.
-            const runId = researchRunId();
-            const rawKey = spec.cacheKey ? spec.cacheKey(params) : null;
-            const cacheKey = runId !== undefined && rawKey !== null ? `${spec.name}\u0000${rawKey}` : null;
-            if (runId !== undefined && cacheKey !== null) {
-                const hit = await lookupResearch(ctx.cwd, runId, cacheKey);
-                if (hit !== undefined)
-                    return textResult(hit.text, hit.details);
-            }
-            const outcome = await spec.run(params, signal, ctx);
-            const { text, details } = outcome;
-            // A non-answer is never stored, whatever the tool's own rule says. The
-            // rule decides answer QUALITY; this decides whether there is an answer.
-            if (outcome.kind === 'answer'
-                && runId !== undefined
-                && cacheKey !== null
-                && (spec.cacheable ? spec.cacheable(details) : true)) {
-                const provenance = spec.cachePkg?.(params, details);
-                await storeResearch(ctx.cwd, runId, cacheKey, text, details, provenance?.pkg, provenance?.ecosystem);
-            }
+            const { text, details } = await cachedWorkerCall(ctx.cwd, spec.name, spec.cacheKey ? spec.cacheKey(params) : null, () => spec.run(params, signal, ctx), {
+                ...(spec.cacheable ? { cacheable: d => spec.cacheable(d) } : {}),
+                ...(spec.cachePkg ? { pkg: d => spec.cachePkg(params, d) } : {})
+            });
             return textResult(text, details);
         },
         renderCall: (args, theme) => spec.renderCall(args, theme)

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@mjasnikovs/pi-task",
-    "version": "0.40.50",
+    "version": "0.41.0",
     "description": "Deterministic task planning and spec-orchestration for local models — crash-safe /task pipelines with verify/enforce gates, a real-time remote web view, and web/docs/fetch/worker subagent tools.",
     "type": "module",
     "main": "./dist/index.js",