@namzu/sdk 39.0.0 → 41.0.0

package/dist/runtime/query/iteration/index.js

@@ -17,6 +17,7 @@ import { DELEGATION_TIMEOUT_MS } from '../../../tools/coordinator/index.js';
 import { NamzuError } from '../../../types/errors/index.js';
 import { createAssistantMessage, createRuntimeContextMessage, createSystemMessage, } from '../../../types/message/index.js';
 import { classifyProviderError } from '../../../types/provider/errors.js';
+import { readPositiveIntEnv } from '../../../utils/env.js';
 import { toErrorMessage } from '../../../utils/error.js';
 import { stableDigest } from '../../../utils/hash.js';
 import { generateMessageId } from '../../../utils/id.js';
@@ -25,7 +26,7 @@ import { projectObservationContext } from '../observation-context.js';
 import { applyLifecycleHookResults } from '../plugin-hooks.js';
 import { diffRequestContext, snapshotRequestContext, } from '../request-context.js';
 import { DEFAULT_MAX_REQUEST_RICH_CONTENT_BYTES, markProviderRejectedImage, projectRequestRichContent, } from '../request-rich-content.js';
-import { formatSteeringNote, isOperatorUserMessage } from '../steering.js';
+import { formatJobNote, formatSteeringNote, isOperatorUserMessage } from '../steering.js';
 import { parseNativeCandidate } from './native-output.js';
 import { runAdvisoryPhase } from './phases/advisory.js';
 import { runIterationCheckpoint } from './phases/checkpoint.js';
@@ -116,6 +117,42 @@ const SETTLE_GRACE_FRACTION = 0.5;
 export function settleGraceMs(remainingBeforeFinalizeMs) {
     return Math.min(Math.floor(remainingBeforeFinalizeMs * SETTLE_GRACE_FRACTION), DELEGATION_TIMEOUT_MS);
 }
+/**
+ * The ceiling on the job half of that grace, in milliseconds.
+ *
+ * `DELEGATION_TIMEOUT_MS` is the wrong ceiling for a shell job, and the gap
+ * only opens where it matters most: a run with no `timeoutMs` — the CLI's
+ * shipping default, `No run deadline by default` — has infinite time before
+ * it must start finishing, so `settleGraceMs` returns the ceiling flat. For a
+ * delegated task that is sound, because the hour is the longest the task
+ * itself may live: the hold cannot outlast the work. A background job has no
+ * such bound. `tail -f`, a watcher and a dev server all outlive any hold, so
+ * the same arithmetic parks an interactive session for an hour on a job that
+ * was never going to exit.
+ *
+ * So the job leg gets its own bound, and it is sized to what the wait buys
+ * rather than to how long a job may live: a turn in which to use the exit.
+ * A model that already waited its `wait_for_job` bound out and saw nothing is
+ * not usually two minutes from an exit, and the run ending is not the news
+ * being lost — with no run in flight the session announces the exit itself
+ * (`docs/cli/background-jobs.md`, *Learning that it ended*), which is the
+ * cheaper of the two places to hear it.
+ */
+const DEFAULT_JOB_HOLD_MAX_MS = 2 * 60 * 1000;
+/**
+ * The same share of the run, under {@link DEFAULT_JOB_HOLD_MAX_MS}.
+ *
+ * `NAMZU_JOB_HOLD_MAX_MS` overrides the ceiling for a host that wants a
+ * longer or shorter park, the way `NAMZU_JOB_WAIT_TIMEOUT_MS` overrides
+ * `wait_for_job`'s own bound — and it is the same parse, so a value that is
+ * not a positive whole number of milliseconds leaves the default standing
+ * rather than holding a run for `NaN`. Called here rather than at module
+ * load, because a host that sets it after import is not ignored.
+ */
+export function awaitedJobGraceMs(remainingBeforeFinalizeMs) {
+    const ceiling = readPositiveIntEnv('NAMZU_JOB_HOLD_MAX_MS', DEFAULT_JOB_HOLD_MAX_MS);
+    return Math.min(settleGraceMs(remainingBeforeFinalizeMs), ceiling);
+}
 export class IterationOrchestrator {
     ctx;
     advisoryTurn;
@@ -1301,8 +1338,10 @@ export class IterationOrchestrator {
                     // returned — which is what makes a terminal submit_answer tool
                     // usable without discarding its output.
                     if (await this.shouldStop()) {
-                        // Outstanding delegated work outranks the host's stop
-                        // predicate, exactly once.
+                        // Outstanding work outranks the host's stop predicate —
+                        // a delegated task the completion inbox is expecting, or
+                        // a background job the model told `wait_for_job` it is
+                        // waiting on.
                         //
                         // This is a precedence rule chosen here, not something
                         // `stopWhen` implies — a stop predicate is a programmable
@@ -1311,13 +1350,20 @@ export class IterationOrchestrator {
                         // tool or a captured structured output. Those decide the
                         // result, so no turn follows and a hold would buy nothing.
                         // This one only says "stop", and stopping one turn later
-                        // with the worker's result in hand is a better reading of
-                        // the host's intent than stopping now and discarding it.
+                        // with the result in hand is a better reading of the
+                        // host's intent than stopping now and discarding it.
                         //
-                        // Bounded: after the notification is delivered the inbox
-                        // is drained, so the predicate fires again next turn with
-                        // nothing pending and the run stops. Exactly one extra
-                        // turn, and `maxIterations` bounds it regardless.
+                        // Bounded by what is left to deliver, not by a count.
+                        // Each delivery consumes what it delivered — the inbox is
+                        // drained, and a job exit's notice is taken with the
+                        // record of the exits it accounts for — so the predicate
+                        // is asked again next turn against whatever is still
+                        // outstanding. One task deferred it once; two awaited
+                        // jobs exiting a minute apart defer it twice, each time
+                        // for a turn the model spends on news it has not read.
+                        // `maxIterations` and the run's own deadline bound all of
+                        // it regardless, and a leg with nothing pending never
+                        // opens a hold at all.
                         if (yield* this.holdForOutstandingWork(iterationNum, true)) {
                             // Remember WHY the next turn exists, so the turn that
                             // ends the run can name the host's decision instead of
@@ -1529,27 +1575,60 @@ export class IterationOrchestrator {
         }
     }
     /**
-     * Hold the run open for a worker that has not finished, and deliver it.
+     * Hold the run open for work that has not finished, and deliver it.
      *
-     * Returns whether a completion or operator message entered the transcript —
-     * the caller continues on `true`, so the model gets a turn to respond.
-     * That turn is the entire justification for waiting, which
+     * Returns whether a completion, a job exit or an operator message entered
+     * the transcript — the caller continues on `true`, so the model gets a turn
+     * to respond. That turn is the entire justification for waiting, which
      * is why only the exits that can still take one call this.
      *
-     * Bounded by `settleGraceMs` and by `maxIterations`, so a worker that never
-     * finishes cannot keep the run open.
+     * Two kinds of work qualify and they are raced together, because a run has
+     * one settle point and one grace period to spend at it:
+     *
+     *  - a delegated task the `CompletionInbox` is still expecting;
+     *  - a background job the model told `wait_for_job` it is waiting on.
+     *
+     * The job half is deliberately narrow. Intent comes from the wait and from
+     * nothing else — a dev server the model started and never waited on is
+     * running because somebody wanted it running, and a hold for it would add
+     * the grace period to the end of every turn for the rest of the session.
+     *
+     * Each leg is opened only when it has something pending: both
+     * `waitForArrival` implementations resolve immediately when their own side
+     * is idle, so racing an idle one would end the hold before it began.
+     *
+     * Bounded by `settleGraceMs` and by `maxIterations`, so work that never
+     * finishes cannot keep the run open. On a run with a deadline the grace is
+     * a share of what is LEFT of it rather than a fresh allowance, so a
+     * `wait_for_job` call that already spent minutes has shortened this hold
+     * by the same minutes. On a run without one — the CLI's default — there is
+     * no remainder to take a share of, and the job leg's own ceiling
+     * (`awaitedJobGraceMs`) is what keeps a timed-out wait from being followed
+     * by an hour of silence.
      */
     async *holdForOutstandingWork(iterationNum, hasToolCalls) {
-        if (!this.ctx.completionInbox?.hasPendingWork)
+        const inbox = this.ctx.completionInbox?.hasPendingWork ? this.ctx.completionInbox : undefined;
+        const jobs = this.ctx.awaitedJobs?.hasPendingWork ? this.ctx.awaitedJobs : undefined;
+        if (!inbox && !jobs)
             return false;
         // Read HERE rather than from `forceFinalize`, which was sampled at the
         // top of the iteration: one that has since crossed the finalize point
         // must not open a wait against a reserve it has already entered.
-        const graceMs = settleGraceMs(this.ctx.guard.remainingBeforeFinalizeMs());
-        this.ctx.log.info('Holding the run open for a background task', {
+        const remainingMs = this.ctx.guard.remainingBeforeFinalizeMs();
+        // One deadline for the race, and it is the LONGEST ceiling any pending
+        // leg justifies. A leg resolving on its own timer ends the whole race,
+        // so handing the job leg its shorter ceiling while a task was also
+        // outstanding would cut the task's hold down to the job's — a run
+        // walking away from a worker it had time for, because a job happened
+        // to be running. A job therefore never shortens a wait, and it never
+        // lengthens one either: where a task is outstanding too, that is how
+        // long this run was waiting anyway.
+        const graceMs = inbox ? settleGraceMs(remainingMs) : awaitedJobGraceMs(remainingMs);
+        this.ctx.log.info('Holding the run open for outstanding work', {
             [NAMZU.RUN_ID]: this.ctx.runMgr.id,
             [NAMZU.ITERATION]: iterationNum,
             'namzu.runtime.grace_ms': graceMs,
+            'namzu.runtime.awaited_jobs': jobs?.outstandingJobIds ?? [],
         });
         // User input releases this wait without cancelling any child. Both waits
         // share a disposable signal so the losing arrival listener cannot leak.
@@ -1561,7 +1640,8 @@ export class IterationOrchestrator {
             cancelWait();
         try {
             await Promise.race([
-                this.ctx.completionInbox.waitForArrival(graceMs, waiting.signal),
+                ...(inbox ? [inbox.waitForArrival(graceMs, waiting.signal)] : []),
+                ...(jobs ? [jobs.waitForArrival(graceMs, waiting.signal)] : []),
                 ...(this.ctx.waitForInbound ? [this.ctx.waitForInbound(waiting.signal)] : []),
             ]);
         }
@@ -1574,12 +1654,13 @@ export class IterationOrchestrator {
             runSignal.removeEventListener('abort', cancelWait);
         }
         runSignal.throwIfAborted();
-        const arrived = this.ctx.completionInbox.drain();
+        const arrived = this.ctx.completionInbox?.drain() ?? [];
         if (arrived.length > 0) {
             this.ctx.runMgr.pushMessage(createRuntimeContextMessage(formatCompletionNotification(arrived), 'task-completion'));
         }
+        const exited = this.deliverAwaitedJobExits();
         const inbound = this.deliverInbound();
-        if (arrived.length === 0 && inbound === 0)
+        if (arrived.length === 0 && !exited && inbound === 0)
             return false;
         await this.ctx.emitEvent({
             type: 'iteration_completed',
@@ -1591,8 +1672,45 @@ export class IterationOrchestrator {
         return true;
     }
     /**
-     * Account for delegated work on the way out: deliver what arrived, and say
-     * what did not.
+     * Put the job exits this hold was waiting for in front of the model.
+     *
+     * Through `jobNotices`, which is the channel a job exit already travels on
+     * — `attachNotice` rides it out on the next tool result — rather than a
+     * second one built for this path. A turn that called no tools has no such
+     * result, so the queued text becomes a `runtime-context` message instead,
+     * exactly as `deliverInbound` does for steering that found no tool result
+     * to attach to.
+     *
+     * That drain is also what keeps one exit from being delivered twice: the
+     * channel hands its text over once, so an exit already attached to a tool
+     * result earlier in the turn leaves nothing here — and the record of it
+     * went with that delivery, so this returns `false` rather than buying a
+     * turn to re-read what the model has read.
+     *
+     * `takeDelivery` is what pairs the two. Taking the exits first and then
+     * finding no notice would discard them, which is the one way this path
+     * can lose an exit outright; neither is taken unless both are there.
+     *
+     * The channel is not per-job, so the text taken here can include a notice
+     * for a job nobody awaited that ended while the hold was open. Delivering
+     * it is right — it is unread either way, and the alternative is stranding
+     * it — but it is not a reason to WAIT, which is why what opens this hold
+     * is `AwaitedJobs`, and the two are asked separately.
+     */
+    deliverAwaitedJobExits() {
+        const delivered = this.ctx.awaitedJobs?.takeDelivery(() => this.ctx.jobNotices?.drain());
+        if (!delivered)
+            return false;
+        this.ctx.log.info('Delivering a background job exit the run held open for', {
+            [NAMZU.RUN_ID]: this.ctx.runMgr.id,
+            'namzu.runtime.jobs': delivered.exits.map((job) => job.id),
+        });
+        this.ctx.runMgr.pushMessage(createRuntimeContextMessage(formatJobNote(delivered.text), 'job-exit'));
+        return true;
+    }
+    /**
+     * Account for outstanding work on the way out: deliver what arrived, and
+     * say what did not.
      *
      * A run that ends with a worker outstanding must not leave the impression
      * that the worker's result was delivered. There are exactly two honest
@@ -1616,18 +1734,31 @@ export class IterationOrchestrator {
      */
     settleOutstandingWork() {
         this.deliverArrivedCompletions();
+        this.deliverArrivedJobExits();
         this.recordAbandonedWork();
     }
-    /** Delegated work this run walked away from. See {@link settleOutstandingWork}. */
+    /** Work this run walked away from. See {@link settleOutstandingWork}. */
     recordAbandonedWork() {
         const abandoned = this.ctx.completionInbox?.outstandingTaskIds ?? [];
-        if (abandoned.length === 0)
+        if (abandoned.length > 0) {
+            this.ctx.log.warn('Run ended with delegated work still running', {
+                [NAMZU.RUN_ID]: this.ctx.runMgr.id,
+                'namzu.runtime.tasks': abandoned,
+            });
+            this.ctx.runMgr.setAbandonedTaskIds(abandoned);
+        }
+        // The same statement for a job the model was waiting on when the grace
+        // ran out. Only awaited ones: a job nobody waited for was never work
+        // this run was holding, so naming it would report an abandonment that
+        // did not happen.
+        const abandonedJobs = this.ctx.awaitedJobs?.outstandingJobIds ?? [];
+        if (abandonedJobs.length === 0)
             return;
-        this.ctx.log.warn('Run ended with delegated work still running', {
+        this.ctx.log.warn('Run ended with an awaited background job still running', {
             [NAMZU.RUN_ID]: this.ctx.runMgr.id,
-            'namzu.runtime.tasks': abandoned,
+            'namzu.runtime.jobs': abandonedJobs,
         });
-        this.ctx.runMgr.setAbandonedTaskIds(abandoned);
+        this.ctx.runMgr.setAbandonedJobIds(abandonedJobs);
     }
     deliverArrivedCompletions() {
         const unheard = this.ctx.completionInbox?.drain() ?? [];
@@ -1658,6 +1789,40 @@ export class IterationOrchestrator {
         });
         this.ctx.runMgr.pushMessage(createRuntimeContextMessage(formatCompletionNotification(unheard), 'task-completion'));
     }
+    /**
+     * The job half of {@link deliverArrivedCompletions}: an exit that arrived
+     * too late to earn a turn is still delivered on the way out.
+     *
+     * The window this closes is one tick wide and it is nobody else's. An
+     * awaited job that exits between the hold's grace expiring and the run
+     * settling was never delivered — the hold had already looked — and is no
+     * longer named either, because the exit took it off the outstanding list
+     * on its way past, so `abandonedJobIds` would be lying to claim it. The
+     * host's own listener is no help: the CLI queues an exit for the next
+     * turn only when no run is in flight, and this one is still in flight.
+     * Delivered here it reaches `Run.messages`, so the transcript has it and
+     * a continued thread opens with it.
+     *
+     * Before `recordAbandonedWork`, which then reports only what is still
+     * running, and after `deliverArrivedCompletions`, so the two appended
+     * messages land in the order the work finished in.
+     */
+    deliverArrivedJobExits() {
+        const delivered = this.ctx.awaitedJobs?.takeDelivery(() => this.ctx.jobNotices?.drain());
+        if (!delivered)
+            return;
+        // Fix the run's answer BEFORE appending anything after it — the same
+        // `resolveResult` tail walk `deliverArrivedCompletions` explains just
+        // above, and the same guard against pinning an empty one.
+        const answer = this.ctx.runMgr.materializeResult();
+        if (answer.length > 0)
+            this.ctx.runMgr.setResult(answer);
+        this.ctx.log.info('Delivering a background job exit the run would have settled over', {
+            [NAMZU.RUN_ID]: this.ctx.runMgr.id,
+            'namzu.runtime.jobs': delivered.exits.map((job) => job.id),
+        });
+        this.ctx.runMgr.pushMessage(createRuntimeContextMessage(formatJobNote(delivered.text), 'job-exit'));
+    }
     stepContextMessage(content) {
         return createRuntimeContextMessage(`Current step context (runtime-generated; not a new user request):\n${content}`, 'step-context');
     }