@mjasnikovs/pi-task 0.21.4 → 0.21.6

package/dist/task/phases.js

@@ -1343,6 +1343,50 @@ export const PHASES = [
         }
     }
 ];
+// INTEGRATION-DEPTH APPEND (2026-07-27): the lever proposed for this exact site —
+// deterministically append a known-runnable integration command to the VERIFY block
+// whenever a task's ACCEPTANCE claims runtime behaviour — is REFUTED at STEP 0 and was
+// NOT built. Nothing below is wired; this is the record.
+//
+// The defect is real and reproduces on four unrelated stacks. Measured with
+// scripts/verify-integration-depth-step0.ts (published metric regex in that file;
+// re-runnable, no model time): VERIFY blocks that boot or hit the real integrated
+// product — mx5 7/41, IAR1 3/10, godot-engine 2/20, runner 0/2, total 12/73 (16.4%).
+// That total OVERSTATES the truth: godot's two hits only match because the block greps
+// a URL out of CLAUDE.md, and IAR1's TASK_0006 curls a GitHub tarball to check a hash,
+// so genuine product integration is ~9/73 (~12%). The addressable class — runtime claim
+// in ACCEPTANCE, a boot command available with provenance, static-only VERIFY anyway —
+// is 29/73 (39.7%), clearing the task's 25% kill condition, but 27 of those 29 are mx5.
+//
+// What kills the lever is its input, not its premise. It needs a command that is
+// (1) provenance-bearing, (2) proven runnable, (3) terminating, and (4) integration-
+// shaped. Measured on scratch clones with scripts/integration-command-provenance.ts,
+// that intersection is EMPTY on every stack available here:
+//   mx5    `bun run dev` is the only command matching the metric and it does not
+//          terminate (probe killed it at 60s) — appending it hangs the verify child
+//          until the 15-minute command watchdog and then FAILs. Every command that DOES
+//          terminate (`bun run build` exit 0) fails the metric.
+//   godot  `godot --headless --quit` (0.8s, exit 0) and the real GUT runner (1.7s,
+//          exit 0) are both appendable — and neither matches the metric, which is
+//          web-shaped (curl/localhost/PORT=/run dev/http://). Nothing appendable can
+//          move the registered metric on a non-web stack, so the two-stack A/B the
+//          task requires is unsatisfiable by construction, not by sample size.
+//   IAR1   no candidate at all: cmake is absent from this box (the N5 finding).
+// R4 forbids swapping the metric after seeing results, so a broadened "boots the real
+// product" metric is a separate pre-registered experiment, not a rescue of this one.
+//
+// The deeper finding: on the stack holding 93% of the addressable mass, integration is
+// not a COMMAND. mx5's 7 integrated blocks are whole task-specific procedures — pick a
+// free port (3911/41234/42421/3001), export DATABASE_URL, boot, seed, log in, assert N
+// endpoints, tear down. A host-side append cannot synthesize that from provenance
+// (launch-contract.md records script NAMES only — no port, no health URL), and
+// synthesizing it from the task's own `## verified tooling` is the N5 fabrication road.
+// That machinery already exists in ONE place that owns ports, seeding and teardown: the
+// final gate's render check. The mx5-class defect belongs there (nexttask TASK 6), not
+// in per-task VERIFY blocks.
+//
+// Durable assets kept: both rigs above and their unit tests. Do NOT wire an append here
+// without a command source that satisfies all four properties at once.
 export async function postCommitPhase(phase, deps, pc, out) {
     if (phase.name !== 'refine')
         return;

package/dist/workers/research-cache.js

@@ -62,6 +62,34 @@
  * to tell its docs entries apart, so it still falls back to a fresh id: every
  * inconclusive path costs time, never correctness.
  *
+ * CONCURRENT WRITERS (nexttask TASK 4). Storing is a read-modify-write over ONE file,
+ * and its writers are concurrent on two axes at once: makeWorkerTool registers the
+ * research tools with executionMode 'parallel', so a single child can issue 4-6 docs
+ * calls in the same millisecond, and the research phase runs four worker children as
+ * separate PROCESSES. Unsynchronised, that is a classic lost update — last writer wins,
+ * and everything read before it is discarded. Measured on this box: 40 concurrent stores
+ * left 1 entry in-process and 11 of 40 across four child processes.
+ *
+ * So the read-modify-write is serialised twice over: an in-process queue per cache file
+ * (siblings inside one child never touch the filesystem lock at all) wrapped in an
+ * advisory lock directory beside the file, which is what makes it hold across processes.
+ * An atomic `mkdir` is the lock — it is the one primitive that both POSIX and Windows
+ * give us with create-or-fail semantics and no fd bookkeeping.
+ *
+ * The lock is BEST-EFFORT LIKE EVERYTHING ELSE HERE: acquisition is bounded, and a
+ * writer that cannot get in within the timeout SKIPS its store rather than waiting. A
+ * skipped store costs one re-lookup later; a blocked store would stall a worker, which
+ * this cache is never allowed to do.
+ *
+ * WHAT THE FIX IS WORTH, measured before it was built (scripts/research-cache-
+ * write-loss-step0.ts, re-runnable): the lost updates are real but nearly free. mx5 lost
+ * ~70 of 204 attempted keys, all in the docs channel — and asked ZERO of them twice, so
+ * nothing lost was ever wanted again. IAR1 and godot-engine retained more distinct keys
+ * than the logs show attempted, i.e. lost nothing at all. Estimated recovery on all three
+ * projects: 0s per run. This is a correctness fix, not a performance one; it stops the
+ * cache silently discarding work, and it will matter to a run whose research phase does
+ * repeat itself. Do not oversell it.
+ *
  * Stored under `.pi-tasks/` (sibling of env-notes.md / contracts.md), which the
  * git-state guard and discardEdits both exclude. Best-effort throughout: any I/O or
  * parse failure falls back to a live fetch — the cache only ever saves time, it can
@@ -79,6 +107,23 @@ export const RESEARCH_RUN_ID_ENV = 'PI_TASK_RUN_ID';
  * lookups (dozens), so a real run never evicts a still-useful digest.
  */
 const MAX_ENTRIES = 250;
+/** Lock directory guarding the cache file's read-modify-write, created beside it. */
+const LOCK_SUFFIX = '.lock';
+/**
+ * How long a writer waits for the lock before giving up and skipping its store. Sized
+ * well above a real critical section (one small read + one small write, sub-millisecond
+ * at these file sizes) and well below anything a worker would notice.
+ */
+const LOCK_TIMEOUT_MS = 2_000;
+/** Poll interval while the lock is held by someone else. */
+const LOCK_POLL_MS = 10;
+/**
+ * A lock older than this is treated as abandoned and removed. Two writers can both
+ * decide that and both proceed, which degrades exactly to the pre-lock behaviour (one
+ * lost update) — strictly better than a crashed child wedging the cache for the rest of
+ * the run. Sized far above the critical section, so a live holder is never stolen from.
+ */
+const LOCK_STALE_MS = 30_000;
 /**
  * Schema marker for per-entry package provenance. A file without it was written by a
  * version that stored no `pkg` on its entries, so its docs entries are indistinguishable
@@ -197,10 +242,26 @@ export async function resumeResearchRun(cwd, enabled) {
         delete process.env[RESEARCH_RUN_ID_ENV];
         return { runId: undefined, reused: false, entries: 0, dropped: 0 };
     }
-    const file = await readCacheFile(cwd);
-    if (!file || file.pkgv !== PKG_PROVENANCE_VERSION) {
+    // Prune under the same lock a store takes. In practice a resume runs before this
+    // run's first worker child exists, so there is nothing to race — but the prune is a
+    // read-modify-write over the same file, and it costs nothing to make that true by
+    // construction rather than by scheduling. Falls back to an unlocked prune if the
+    // lock is unavailable, which is exactly the pre-lock behaviour.
+    const pruned = (await withCacheLock(cwd, () => pruneCache(cwd))) ?? (await pruneCache(cwd));
+    if (!pruned)
         return { runId: configureResearchRun(true), reused: false, entries: 0, dropped: 0 };
-    }
+    process.env[RESEARCH_RUN_ID_ENV] = pruned.runId;
+    return { runId: pruned.runId, reused: true, entries: pruned.entries, dropped: pruned.dropped };
+}
+/**
+ * Drop the entries the manifest has invalidated and persist the result, returning the
+ * reusable run id — or null when there is nothing to reason with (no cache file, or one
+ * predating per-entry package provenance).
+ */
+async function pruneCache(cwd) {
+    const file = await readCacheFile(cwd);
+    if (!file || file.pkgv !== PKG_PROVENANCE_VERSION)
+        return null;
     const deps = await depsMap(cwd);
     const kept = {};
     let dropped = 0;
@@ -214,8 +275,7 @@ export async function resumeResearchRun(cwd, enabled) {
     // stale digests behind under a reused id.
     if (dropped > 0)
         await writeCacheFile(cwd, { runId: file.runId, entries: kept, pkgv: file.pkgv });
-    process.env[RESEARCH_RUN_ID_ENV] = file.runId;
-    return { runId: file.runId, reused: true, entries: Object.keys(kept).length, dropped };
+    return { runId: file.runId, entries: Object.keys(kept).length, dropped };
 }
 async function readCacheFile(cwd) {
     try {
@@ -257,6 +317,91 @@ async function writeCacheFile(cwd, out) {
         // best-effort cache
     }
 }
+/**
+ * Serialises this process's own writers per cache file, so the 4-6 parallel tool calls
+ * a single research child issues in one turn queue in memory instead of contending for
+ * the lock directory. Purely an optimisation: the cross-process lock below is what makes
+ * the store correct.
+ */
+const inProcessQueues = new Map();
+/**
+ * Take the advisory lock, or return false once `deadline` passes. Never throws: an
+ * unexpected filesystem error is reported as "not acquired", which the caller turns
+ * into a skipped store.
+ */
+async function acquireLock(lockPath, deadline) {
+    for (;;) {
+        try {
+            // mkdir is create-or-fail: exactly one caller can win it.
+            await fsp.mkdir(lockPath);
+            return true;
+        }
+        catch (err) {
+            if (err.code !== 'EEXIST')
+                return false;
+            try {
+                const st = await fsp.stat(lockPath);
+                if (Date.now() - st.mtimeMs > LOCK_STALE_MS) {
+                    await fsp.rmdir(lockPath).catch(() => { });
+                    if (Date.now() >= deadline)
+                        return false;
+                    continue;
+                }
+            }
+            catch {
+                // vanished between mkdir and stat ⇒ the holder just released it
+                if (Date.now() >= deadline)
+                    return false;
+                continue;
+            }
+            if (Date.now() >= deadline)
+                return false;
+            await new Promise(resolve => setTimeout(resolve, LOCK_POLL_MS));
+        }
+    }
+}
+/**
+ * Run `fn` as the sole writer of this project's cache file, across both the calling
+ * process's own parallel tool calls and every other pi child sharing the tree. Returns
+ * `fn`'s result, or undefined when the lock could not be taken in time — the caller then
+ * SKIPS the write (invariant I1: bounded wait, never a deadlock and never a stall).
+ */
+async function withCacheLock(cwd, fn) {
+    const file = researchCacheFile(cwd);
+    const prior = inProcessQueues.get(file) ?? Promise.resolve();
+    let release;
+    const mine = new Promise(resolve => (release = resolve));
+    // The chain is rejection-free by construction: `mine` only ever resolves, and the
+    // root is Promise.resolve(), so no waiter can inherit a rejection.
+    const tail = prior.then(() => mine);
+    inProcessQueues.set(file, tail);
+    await prior.catch(() => { });
+    try {
+        const lockPath = `${file}${LOCK_SUFFIX}`;
+        let held = false;
+        try {
+            await fsp.mkdir(tasksDir(cwd), { recursive: true });
+            held = await acquireLock(lockPath, Date.now() + LOCK_TIMEOUT_MS);
+        }
+        catch {
+            held = false;
+        }
+        if (!held)
+            return undefined;
+        try {
+            return await fn();
+        }
+        finally {
+            await fsp.rmdir(lockPath).catch(() => { });
+        }
+    }
+    finally {
+        release();
+        // Keep the map from growing one entry per project for the process's lifetime.
+        if (inProcessQueues.get(file) === tail)
+            inProcessQueues.delete(file);
+    }
+}
 /**
  * Store a successful result under `key` for the current run. A file written for a
  * different run id is discarded and started fresh (first write of a new run drops the
@@ -272,24 +417,29 @@ async function writeCacheFile(cwd, out) {
  */
 export async function storeResearch(cwd, runId, key, text, details, pkg) {
     try {
-        const existing = await readCacheFile(cwd);
-        const entries = existing && existing.runId === runId ? existing.entries : {};
+        // Resolved OUTSIDE the critical section: it reads package.json, which no other
+        // writer can be mutating, and keeping it out holds the lock for the file
+        // read/write alone. The version is still the one current at store time.
         const pkgVersion = pkg === undefined ? undefined : await declaredVersion(cwd, pkg);
-        entries[key] = {
-            text,
-            details,
-            at: Date.now(),
-            ...(pkg === undefined ? {} : { pkg }),
-            ...(pkgVersion === undefined ? {} : { pkgVersion })
-        };
-        // Evict oldest by write time if over the cap.
-        const keys = Object.keys(entries);
-        if (keys.length > MAX_ENTRIES) {
-            const ordered = keys.sort((a, b) => entries[a].at - entries[b].at);
-            for (const k of ordered.slice(0, keys.length - MAX_ENTRIES))
-                delete entries[k];
-        }
-        await writeCacheFile(cwd, { runId, entries, pkgv: PKG_PROVENANCE_VERSION });
+        await withCacheLock(cwd, async () => {
+            const existing = await readCacheFile(cwd);
+            const entries = existing && existing.runId === runId ? existing.entries : {};
+            entries[key] = {
+                text,
+                details,
+                at: Date.now(),
+                ...(pkg === undefined ? {} : { pkg }),
+                ...(pkgVersion === undefined ? {} : { pkgVersion })
+            };
+            // Evict oldest by write time if over the cap.
+            const keys = Object.keys(entries);
+            if (keys.length > MAX_ENTRIES) {
+                const ordered = keys.sort((a, b) => entries[a].at - entries[b].at);
+                for (const k of ordered.slice(0, keys.length - MAX_ENTRIES))
+                    delete entries[k];
+            }
+            await writeCacheFile(cwd, { runId, entries, pkgv: PKG_PROVENANCE_VERSION });
+        });
     }
     catch {
         // best-effort cache

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@mjasnikovs/pi-task",
-    "version": "0.21.4",
+    "version": "0.21.6",
     "description": "Deterministic task planning and spec-orchestration for local models — crash-safe /task pipelines with verify/enforce gates, a real-time remote web view, and web/docs/fetch/worker subagent tools.",
     "type": "module",
     "main": "./dist/index.js",