dorfl 0.13.3 → 0.14.0

package/dist/needs-attention.js

@@ -1306,10 +1306,17 @@ export function prepareTreelessSurfaceCommit(params) {
         rmSync(scratchIndex, { force: true });
     }
 }
-/** The terminal `work/` paths for an item, tagged by {@link TerminalKind}, so a
- * reader can tell "the work happened" from "the item was abandoned". Mirrors
- * `terminalMainPaths` in `item-lock.ts` (same folders, same per-regime split);
- * this variant carries the KIND the question-state drain branches on. */
+/**
+ * The terminal `work/` paths for an item, tagged by {@link TerminalKind}, so a
+ * reader can tell "the work happened" from "the item was abandoned".
+ *
+ * Same SHAPE as `terminalMainPaths` in `item-lock.ts`, but deliberately NOT the
+ * same folder set, and the difference must not be "tidied" away: locks treat
+ * `specs/tasked/` as terminal (correctly, a tasked spec must release its lock),
+ * whereas QUESTION state there is still live. This map is keyed to "is the
+ * question loop CLOSED at this resting place?", not "has the item stopped
+ * moving?". See the `case 'spec'` comment below.
+ */
 export function terminalMainPathsByKind(type, slug) {
     const file = `${slug}.md`;
     switch (type) {
@@ -1319,16 +1326,54 @@ export function terminalMainPathsByKind(type, slug) {
                 { path: workItemRel('cancelled', file), kind: 'wont-proceed' },
             ];
         case 'spec':
-            return [
-                { path: workItemRel('specs-tasked', file), kind: 'completed' },
-                { path: workItemRel('specs-dropped', file), kind: 'wont-proceed' },
-            ];
+            // `specs/tasked/` is deliberately NOT listed. WORK-CONTRACT ("A SPEC that
+            // has drifted AFTER it was TASKED") makes a bare `needsAnswers:true` on a
+            // tasked spec LEGAL and load-bearing: it means "tasked, but the spec has
+            // drifted, do not RE-task or rely on it until reconciled", and the
+            // contract says to set it *while the spec stays in `specs/tasked/`*
+            // (moving it back would falsely un-record a tasking that really happened
+            // and orphan the tasks it already emitted).
+            //
+            // So the reasoning that makes a task's question state moot at its terminal
+            // does NOT transfer: a tasked spec is still IN the question loop.
+            // `lifecycle-gather.ts` enumerates tasked resting specs UNCONDITIONALLY,
+            // routing a bare flag to the SURFACE rung and an answered sidecar to the
+            // APPLY rung, so BOTH halves are live inputs to a rung that WILL run.
+            // Draining either would disarm a live drift gate and let a stale spec be
+            // re-tasked. That is precisely the "clearing a live needsAnswers hands gated work
+            // to agents" harm this pass exists to avoid.
+            //
+            // `specs/dropped/` needs no such carve-out: a dropped spec is abandoned,
+            // and no rung enumerates it.
+            return [{ path: workItemRel('specs-dropped', file), kind: 'wont-proceed' }];
         case 'observation':
             // A note has no durable terminal folder: it leaves by DELETION, so there
             // is no resting record to reconcile against.
             return [];
     }
 }
+/** Every SUCCESS-terminal folder that can hold a stranded `needsAnswers` flag,
+ * paired with the item type that rests there. Derived from
+ * {@link terminalMainPathsByKind} with a sentinel slug so the folder set stays
+ * SINGLE-SOURCED: adding a regime there adds it here, and the `wont-proceed`
+ * terminals are excluded by the SAME `kind` split the drain already branches on
+ * (a cancelled item's flag is accurate history, not residue). `observation`
+ * contributes nothing, having no durable terminal. */
+function successTerminalFolders() {
+    const out = [];
+    for (const type of ['task', 'spec', 'observation']) {
+        for (const candidate of terminalMainPathsByKind(type, '__slug__')) {
+            if (candidate.kind !== 'completed') {
+                continue;
+            }
+            out.push({
+                folder: candidate.path.slice(0, candidate.path.lastIndexOf('/')),
+                type,
+            });
+        }
+    }
+    return out;
+}
 /**
  * Classify the arbiter's STRANDED QUESTION STATE, read-only (observation
  * `a-rebuilt-task-leaves-its-bounce-question-asking-to-cancel-a-merged-task`).
@@ -1364,11 +1409,21 @@ export function terminalMainPathsByKind(type, slug) {
  * folder on `main`; an item resting in a pool or staging folder keeps whatever
  * state it has, untouched.
  *
- * The enumeration is anchored on the SIDECAR SET (`work/questions/` on `main`),
- * which is small, cheap to list, and is the half that makes the residue
- * discoverable unambiguously. A terminal item carrying a bare flag and no sidecar
- * is deliberately NOT swept: that shape is the legal one above, and there is no
- * second signal to distinguish residue from a hand-authored declaration.
+ * TWO ENUMERATIONS, because the two halves can be separated in either order and
+ * a sweep anchored on one is blind to the other:
+ *   1. the SIDECAR SET (`work/questions/` on `main`), small and cheap to list,
+ *      which finds a stale sidecar and the flag paired with it; and
+ *   2. the SUCCESS-TERMINAL BODIES that are flagged with NO sidecar beside them
+ *      ({@link collectStrandedTerminalFlags}), which finds the armed gate ALONE.
+ *
+ * (2) is not optional tidiness. It is the half that actually gates work, and a
+ * sweep anchored only on (1) reports success while any gate it cannot see stays
+ * armed. The sidecar is the
+ * half a human deletes by hand (it is the visible one, in a folder they scan),
+ * and deleting it REMOVES the only handle (1) has, stranding the flag for good.
+ * The discriminator that keeps (2) safe is POSITION, exactly as for (1): a bare
+ * flag is LEGAL on a pool/staging item (the `surface` rung's input) and residue
+ * only once the item has shipped, where `surface` can never run again.
  *
  * Best-effort and never throws.
  */
@@ -1404,8 +1459,16 @@ function deriveTerminalQuestionResidue(base, cwd, env) {
     const out = {
         drainable: [],
         answeredHeld: [],
+        staleFlags: [],
         errors: [],
     };
+    // The SECOND half of the residue, enumerated from the OTHER side. The sidecar
+    // sweep below can only ever see items that still HAVE a sidecar; this one finds
+    // the SUCCESS-terminal bodies whose gate is armed with no sidecar left to point
+    // at them. Both must run: they are the same defect observed through the two
+    // halves the surface path writes atomically, and either half can outlive the
+    // other.
+    collectStrandedTerminalFlags(mainRef, cwd, env, out);
     const questionsDir = workFolderRel('questions');
     const ls = run('git', ['ls-tree', '--name-only', `${mainRef}:${questionsDir}`], cwd, { env });
     if (ls.status !== 0) {
@@ -1482,25 +1545,159 @@ function deriveTerminalQuestionResidue(base, cwd, env) {
     }
     return out;
 }
+/**
+ * Find every SUCCESS-terminal body on `base` carrying `needsAnswers:true` with NO
+ * sidecar beside it, appending them to `out.staleFlags`.
+ *
+ * ENUMERATION COST is why this is a `git grep` and not a walk. The terminal
+ * folders are the repo's largest and most monotonically growing (this repo holds
+ * 404 done tasks), and this runs on the CLAIM path, so reading every terminal
+ * body per claim would be a real tax on a hot path. One `git grep -l` returns
+ * only the candidates, and the frontmatter parse runs over that short list.
+ *
+ * The pattern is ANCHORED to match the PARSER rather than the word.
+ * `parseFrontmatter` reads keys with `/^([A-Za-z0-9_.]+)\s*:\s*(.*)$/`, so a key
+ * it will honour is always at column 0; an unanchored needle instead matches
+ * every body that merely DISCUSSES the flag, which in `work/tasks/done/` here is
+ * 77 files against 18 anchored, and the truthy form narrows it to 1.
+ *
+ * The value part is matched LOOSELY on purpose (optional quote, any case),
+ * because `toBoolean` unquotes and lower-cases before comparing, so
+ * `needsAnswers: 'True'` is a real armed gate. A needle of `:\s*true` would read
+ * tighter and be WRONG: it would silently skip those bodies for ever, which is
+ * the blind-spot class this function exists to remove. A superset is the safe
+ * direction for a shortlist; a subset is not.
+ *
+ * The grep is still only a CANDIDATE FILTER, never the decision: prose can sit
+ * at column 0 too (`needsAnswers: true?` appears in this repo's own bodies), so
+ * every hit is confirmed by actually PARSING the frontmatter.
+ *
+ * Two git-isms are pinned rather than left to the environment:
+ *   - `core.quotePath=false`, or git C-quotes any non-ASCII path
+ *     (`"work/.../caf\303\251.md"`). A quoted line still starts with the
+ *     `<base>:` prefix but then fails the folder-prefix test, so such a body
+ *     would be SILENTLY skipped for ever, a permanent blind spot of exactly the
+ *     class this function exists to remove.
+ *   - `--full-name` + `:(top,literal)` pathspecs, because `git grep`'s pathspecs
+ *     are CWD-RELATIVE (unlike the `ls-tree`/`cat-file` probes elsewhere here,
+ *     which are tree-relative) and are globs. Without these, running any dorfl
+ *     command from a SUBDIRECTORY makes this half a silent no-op while the
+ *     sidecar half keeps working.
+ *
+ * Never throws; a failed grep yields no candidates, which leaves state alone.
+ */
+function collectStrandedTerminalFlags(base, cwd, env, out) {
+    const folders = successTerminalFolders();
+    if (folders.length === 0) {
+        return;
+    }
+    // `-l` names files only, `-I` skips binaries. Exit 1 means NO MATCH, which
+    // ALSO covers "the folder does not exist on this base yet" (verified: an
+    // absent pathspec folder exits 1 with no stderr), and an absent lifecycle
+    // folder is legal per WORK-CONTRACT rule 3. Any OTHER non-zero is a genuine
+    // fault and is REPORTED rather than swallowed: degrading silently to "no
+    // candidates" would leave this half a no-op while the sidecar half keeps
+    // reporting success, which is the very "reports success while the gate stays
+    // armed" shape this change exists to correct.
+    const grep = run('git', [
+        '-c',
+        'core.quotePath=false',
+        'grep',
+        '-l',
+        '-I',
+        '--full-name',
+        '-E',
+        '^needsAnswers:[[:space:]]*[\'"]?[Tt][Rr][Uu][Ee]',
+        base,
+        '--',
+        ...folders.map((f) => `:(top,literal)${f.folder}`),
+    ], cwd, { env });
+    if (grep.status !== 0) {
+        if (grep.status !== 1) {
+            out.errors.push({
+                item: '(stranded-flag scan)',
+                message: `git grep over the terminal folders failed (exit ${grep.status}): ` +
+                    `${grep.stderr.trim() || 'no stderr'}; stranded gates were NOT scanned.`,
+            });
+        }
+        return;
+    }
+    const prefix = `${base}:`;
+    for (const line of grep.stdout.split('\n')) {
+        const raw = line.trim();
+        if (raw === '' || !raw.startsWith(prefix)) {
+            continue;
+        }
+        const path = raw.slice(prefix.length);
+        try {
+            const home = folders.find((f) => path.startsWith(`${f.folder}/`));
+            if (home === undefined) {
+                continue;
+            }
+            const name = path.slice(home.folder.length + 1);
+            // Direct children only: a nested path is not an item body.
+            if (name.includes('/') || !isWorkItemFile(name)) {
+                continue;
+            }
+            // Case-INSENSITIVE to match `isWorkItemFile` above: a `Foo.MD` body must
+            // yield the slug `Foo`, or the sidecar-existence guard below would probe
+            // the wrong path and could clear a gate whose sidecar holds an answer.
+            const slug = name.replace(/\.md$/i, '');
+            if (slug === '') {
+                continue;
+            }
+            const item = `${home.type}:${slug}`;
+            // A sidecar STILL EXISTS for this item (canonical or legacy alias) ⇒ this
+            // is the sidecar-anchored sweep's business, not ours. Skipping keeps the
+            // two enumerations DISJOINT, so an item is never planned twice in one
+            // commit and the answered-sidecar carve-out cannot be bypassed through
+            // this path (an item held for an unapplied human answer keeps its flag).
+            if (sidecarPathCandidates(item).some((c) => pathInCommit(base, c, cwd, env))) {
+                continue;
+            }
+            // CONFIRM against the parsed frontmatter: the grep only shortlisted.
+            const body = catBlob(`${base}:${path}`, cwd, env);
+            if (parseFrontmatter(body).needsAnswers !== true) {
+                continue;
+            }
+            out.staleFlags.push({ item, itemPath: path });
+        }
+        catch (err) {
+            out.errors.push({
+                item: path,
+                message: err instanceof Error ? err.message : String(err),
+            });
+        }
+    }
+}
 /**
  * Drain the stranded question state {@link classifyTerminalQuestionResidue}
  * finds, in ONE tree-less commit CAS-published to the arbiter's `main` through
  * the SAME {@link runTreelessLedgerMove} core the surface path uses (same
  * contention-retry, same lease, same write seam; there is no second mechanism).
  *
- * WHAT IT CLEARS, and the deliberate asymmetry between the two terminals:
- *   - the SIDECAR is deleted for EITHER terminal. A question asking whether to
- *     cancel an item that has already come to rest is stale in both cases, and it
- *     sits in a folder a human scans carrying a destructive default.
+ * WHAT IT CLEARS, and the deliberate asymmetry between the two terminals. Note
+ * the terminal SET first: `specs/tasked/` is deliberately NOT in this map at all
+ * (see {@link terminalMainPathsByKind}), so nothing below applies to a tasked
+ * spec, whose question state stays untouched in both halves.
+ *   - the SIDECAR is deleted for EITHER terminal in the map. A question asking
+ *     whether to cancel an item that has already come to rest is stale in both
+ *     cases, and it sits in a folder a human scans carrying a destructive
+ *     default.
  *   - the `needsAnswers` FLAG is cleared ONLY for a `completed` terminal
- *     (`tasks/done/`, `specs/tasked/`). On a `wont-proceed` terminal
+ *     (`tasks/done/`). On a `wont-proceed` terminal
  *     (`tasks/cancelled/`, `specs/dropped/`) the flag is KEPT, because an item
  *     can be cancelled precisely BECAUSE its questions were never answered: there
  *     the flag is accurate history, not residue, and the body may carry a real
  *     `## Open questions` section saying so. Keeping it is harmless, a terminal
  *     item is in no pool, so the flag gates nothing.
+ *   - a SUCCESS-terminal item whose gate is armed with NO sidecar left beside it
+ *     has that FLAG cleared and nothing deleted (there is nothing to delete).
+ *     Restricted to the `completed` terminal by the same asymmetry above.
  *
- * A sidecar with ANY answered entry is never touched (see the classifier).
+ * A sidecar with ANY answered entry is never touched (see the classifier), and
+ * an item still holding such a sidecar is excluded from the flag-only half too,
+ * so the carve-out cannot be bypassed by clearing its gate.
  *
  * Best-effort: it never throws, and any fault leaves the state exactly as it was.
  */
@@ -1532,7 +1729,7 @@ export async function reconcileTerminalQuestionResidue(params) {
     }
     result.answeredHeld = report.answeredHeld.map((r) => r.item);
     result.errors.push(...report.errors);
-    if (report.drainable.length === 0) {
+    if (report.drainable.length === 0 && report.staleFlags.length === 0) {
         return result;
     }
     // What the LANDED commit ACTUALLY did, filled in by the plan against the base
@@ -1540,6 +1737,10 @@ export async function reconcileTerminalQuestionResidue(params) {
     // anything to do?" probe; reporting from it would claim a gate was disarmed
     // when a contention retry re-derived the residue and skipped the item.
     let applied = [];
+    // Filled by the PLAN with what it actually STAGED (not what it intended), so a
+    // body the marker writer cannot annotate is never reported as unflagged.
+    const clearedSidecarFlags = [];
+    const clearedStaleFlags = [];
     // NEVER THROW. `runTreelessLedgerMove` and the git plumbing inside the plan
     // both throw on any non-zero git, and this pass runs from the CLAIM path as
     // OPPORTUNISTIC HYGIENE on unrelated items. A fault here (a stale scratch ref,
@@ -1570,6 +1771,9 @@ export async function reconcileTerminalQuestionResidue(params) {
                     cwd,
                     base,
                     residue: fresh.drainable,
+                    staleFlags: fresh.staleFlags,
+                    clearedSidecarFlags,
+                    clearedStaleFlags,
                     env,
                 });
             },
@@ -1592,12 +1796,46 @@ export async function reconcileTerminalQuestionResidue(params) {
     }
     for (const r of applied) {
         result.drained.push(r.item);
-        if (r.terminal === 'completed' && r.flagged) {
-            result.unflagged.push(r.item);
-        }
+    }
+    // `unflagged` reports what the commit ACTUALLY staged, from both halves. The
+    // flag-only half never appears in `drained`: it deletes nothing.
+    for (const r of [...clearedSidecarFlags, ...clearedStaleFlags]) {
+        result.unflagged.push(r.item);
     }
     return result;
 }
+/**
+ * Stage `itemPath` with `needsAnswers` cleared, into the scratch index the drain
+ * commit is being built in. Shared by BOTH halves of the residue (the
+ * sidecar-paired flag and the stranded flag-only one) so they can never disagree
+ * about what clearing a gate means.
+ *
+ * Defense-in-depth, mirroring the surface path's guard in the opposite
+ * direction: if the marker does not parse back as `false`, the body is left
+ * ALONE rather than written as something we cannot vouch for. Every uncertainty
+ * resolves to LEAVING STATE ALONE.
+ *
+ * RETURNS whether the gate was actually STAGED, so callers report EFFECT rather
+ * than INTENT. That distinction is load-bearing here: a body this cannot
+ * annotate (e.g. duplicate `needsAnswers` keys, where the writer replaces the
+ * FIRST and the parser reads the LAST) would otherwise be reported as unflagged
+ * on every claim for ever while its gate stayed armed, the precise
+ * "reports success while the defect remains" failure this whole change exists to
+ * correct.
+ */
+function clearNeedsAnswersInIndex(itemPath, base, cwd, env, withIndex) {
+    if (!pathInCommit(base, itemPath, cwd, env)) {
+        return false;
+    }
+    const body = catBlob(`${base}:${itemPath}`, cwd, env);
+    const cleared = setNeedsAnswersMarker(body, false);
+    if (parseFrontmatter(cleared).needsAnswers !== false) {
+        return false;
+    }
+    const blob = hashObject(cleared, cwd, env);
+    gitHard(['update-index', '--add', '--cacheinfo', `100644,${blob},${itemPath}`], cwd, withIndex);
+    return true;
+}
 /**
  * Build the ONE tree-less commit that removes every drainable sidecar and clears
  * the `needsAnswers` flag on every `completed`-terminal body, using PLUMBING on a
@@ -1609,10 +1847,13 @@ export async function reconcileTerminalQuestionResidue(params) {
  * land atomically.
  */
 function prepareTerminalQuestionDrainCommit(params) {
-    const { cwd, base, residue, env } = params;
+    const { cwd, base, residue, env, clearedSidecarFlags, clearedStaleFlags } = params;
+    clearedSidecarFlags.length = 0;
+    clearedStaleFlags.length = 0;
     // RE-DERIVE against THIS base: anything already gone is not our business.
     const live = residue.filter((r) => pathInCommit(base, r.sidecarPath, cwd, env));
-    if (live.length === 0) {
+    const liveFlags = params.staleFlags.filter((r) => pathInCommit(base, r.itemPath, cwd, env));
+    if (live.length === 0 && liveFlags.length === 0) {
         return 'already-done';
     }
     const scratchIndex = join(tmpdir(), `dorfl-question-drain-${process.pid}-${Date.now()}.index`);
@@ -1629,29 +1870,26 @@ function prepareTerminalQuestionDrainCommit(params) {
             if (r.terminal !== 'completed' || !r.flagged) {
                 continue;
             }
-            if (!pathInCommit(base, r.itemPath, cwd, env)) {
-                continue;
+            if (clearNeedsAnswersInIndex(r.itemPath, base, cwd, env, withIndex)) {
+                clearedSidecarFlags.push(r);
             }
-            const body = catBlob(`${base}:${r.itemPath}`, cwd, env);
-            const cleared = setNeedsAnswersMarker(body, false);
-            // Defense-in-depth, mirroring the surface path's guard: if the marker did
-            // not parse back as `false`, leave the body ALONE rather than write a
-            // body we cannot vouch for.
-            if (parseFrontmatter(cleared).needsAnswers !== false) {
-                continue;
+        }
+        // The FLAG-ONLY half: a SUCCESS terminal whose sidecar is already gone. No
+        // `--force-remove` here, because there is nothing to delete; the armed gate IS the
+        // whole residue.
+        // Record what was actually STAGED: a body we could not annotate is dropped
+        // from the report rather than claimed as cleared.
+        for (const r of liveFlags) {
+            if (clearNeedsAnswersInIndex(r.itemPath, base, cwd, env, withIndex)) {
+                clearedStaleFlags.push(r);
             }
-            const blob = hashObject(cleared, cwd, env);
-            gitHard([
-                'update-index',
-                '--add',
-                '--cacheinfo',
-                `100644,${blob},${r.itemPath}`,
-            ], cwd, withIndex);
         }
         const tree = runHard(['write-tree'], cwd, withIndex).stdout.trim();
-        const subject = live.length === 1
-            ? `drain stranded question state for ${live[0].item} (terminal on main)`
-            : `drain stranded question state for ${live.length} terminal items`;
+        const touched = live.length + liveFlags.length;
+        const only = live[0]?.item ?? liveFlags[0]?.item;
+        const subject = touched === 1
+            ? `drain stranded question state for ${only} (terminal on main)`
+            : `drain stranded question state for ${touched} terminal items`;
         const commit = runHard(['commit-tree', tree, '-p', base, '-m', subject], cwd, env).stdout.trim();
         const ref = 'refs/dorfl/question-drain/batch';
         gitHard(['update-ref', ref, commit], cwd, env);