@basein/runner 0.3.17 → 0.3.18

package/dist/replay/controller.js

@@ -21,20 +21,27 @@ import { bashCommandFor } from "./bash-channel.js";
 import { BuiltinNotRunnable, BuiltinTimeout, parseBashAllow, parseBuiltinTools, runBuiltin, } from "./builtins.js";
 import { BASH_CHANNEL_MAX, MAX_REPLAY_REASON, testWords, truncate } from "./bundle.js";
 import { coverageOf, modeFor, reachOf } from "./coverage.js";
-import { deriveParameters, } from "./derive.js";
+import { deriveParameters, localDay, } from "./derive.js";
 import { buildHandoverNote, MAX_HANDOVER_NOTE, } from "./handover.js";
 import { NO_PROXY, NoSessionProxyError, ProxyWorkQueue, QUEUE_CLOSED } from "./executor.js";
 import { logicTimeoutMs, MAX_LOGIC_TIMEOUT_MS } from "./logic.js";
-import { PREVIEW_BUDGET_MS, ScenarioReplayPlan, StepRunError, } from "./plan.js";
+import { PREVIEW_BUDGET_MS, RecordedValueError, ScenarioReplayPlan, StepRunError, } from "./plan.js";
 import { PRICING_VERSION } from "./pricing.js";
 import { hasShellStep, sinkOf, sinkOfPlan, unsafeParameter } from "./safety.js";
 import { SourceRunOutputs } from "./source-run.js";
 import { clampToFrameStart, flattenChain, frameEdges, } from "./flatten.js";
-import { blockingTargets } from "./targets.js";
+import { blockingTargets, testedOnlyTargets } from "./targets.js";
 import { isNoResult, toolResultError } from "./tool-error.js";
 import { OUTCOME_RANK, hasTargets, isReadyScenario, } from "./types.js";
 /** The first-party tool a `direct` plan is delivered through (§6.3). */
 export const DIRECT_TOOL_NAME = "mcp__bir__run_scenario";
+/**
+ * `bir`'s own tool for saying a replay's results do not answer the request
+ * (bin/bir-scenario.ts; the service's review of calcscenarioautocorrect.md,
+ * item 19): the one call a finished bundle lets the agent make, where the
+ * session has it ({@link ReplayState.reportTool}).
+ */
+export const REPORT_TOOL_NAME = "mcp__bir__report_wrong_result";
 /** How long a `/proxy/poll` is held open before it answers empty. */
 export const POLL_HOLD_MS = 25_000;
 /**
@@ -113,6 +120,19 @@ const DIRECT_RUN_MARGIN_MS = 5_000;
  * Each is bounded by `logicTimeoutMs()` (logic.ts).
  */
 const LOGIC_BODIES_OUTSIDE_PLAN = 6;
+/**
+ * The agent's day as a system prompt said it, and only then (issue 1212): a
+ * line that says a day, written out or in words. Claude Code's own date
+ * reminder is this machine's day, which the runner knows better itself.
+ */
+function systemDay(day) {
+    if (!day || day.source !== "system")
+        return undefined;
+    if (day.day)
+        return day;
+    const words = /\b(jan(uary)?|feb(ruary)?|mar(ch)?|apr(il)?|may|june?|july?|aug(ust)?|sep(t(ember)?)?|oct(ober)?|nov(ember)?|dec(ember)?)\b/i;
+    return words.test(day.line) && /\b\d{4}\b/.test(day.line) ? day : undefined;
+}
 /** A derivation wait longer than this is not a budget a direct run can keep. */
 const MAX_DIRECT_DERIVE_MS = 60_000;
 /** However a direct run is squeezed, its plan keeps at least this. */
@@ -506,7 +526,7 @@ export class ReplayController {
         // command nobody asked for.
         let deriveCtx = ctx;
         if (ctx.recentResults?.length && sinkOfPlan(planned, host?.sinks) === "command") {
-            deriveCtx = { liveCall: ctx.liveCall };
+            deriveCtx = { liveCall: ctx.liveCall, agentDay: ctx.agentDay };
             logLine("replay.derive_context", {
                 scenario: scenario.id,
                 shell: hasShellStep(planned, host?.sinks) || undefined,
@@ -515,11 +535,27 @@ export class ReplayController {
             });
         }
         const derivation = this.startDerivation(scenario, prompt, state, deriveCtx);
+        // The agent's own day shows in the session only after the prompt hook
+        // (issue 1212): the plan's start looks again, and reads the values once
+        // more when the day is another one.
+        state.agentDayNow = deriveCtx.agentDay;
+        state.rederive = (day) => this.startDerivation(scenario, prompt, state, { ...deriveCtx, agentDay: () => day });
         // Whether the derivation has answered yet — read, never waited on, when the
         // directive is written (below).
         let deriveAnswered = false;
         derivation.then(() => (deriveAnswered = true), () => (deriveAnswered = true));
-        const params = derivation.then((r) => r.params);
+        // The targets only one half of this plan reads (the service's review of
+        // calcscenarioautocorrect.md, item 11; targets.ts, `testedOnlyTargets`):
+        // the plan holds each as a mark when the request does not give it. A
+        // reading that did not answer carries every target at its recorded
+        // sample (derive.ts), and one of these must be the mark there too — never
+        // last week's task name under a step that would create the task.
+        const oneHalf = hasTargets(scenario.paramsObject)
+            ? testedOnlyTargets(targetKeys(scenario.paramsObject), scenario.paramsObject, planned, scenario.paramsLogic)
+            : [];
+        const params = derivation.then((r) => r.derived || oneHalf.length === 0
+            ? r.params
+            : { ...r.params, ...Object.fromEntries(oneHalf.map((key) => [key, null])) });
         // `params` is awaited by the plan, and by gate 7 below when there is a
         // target. Without this, the losing race in gate 7 leaves `derivation`
         // rejected and unhandled, which takes the process down on an unhandled
@@ -578,11 +614,22 @@ export class ReplayController {
                 // Worth a line of its own: this is the difference between a scenario
                 // that never runs and one that does, and it is the first thing to look
                 // at when a replay acts on the wrong thing.
-                logLine("replay.targets_found_by_chain", {
-                    scenario: scenario.id,
-                    unnamed: result.missing.join(","),
-                    why: "the chain computes these from its own steps; none is read from the caller",
-                });
+                const forOneHalf = result.missing.filter((key) => oneHalf.includes(key));
+                const byChain = result.missing.filter((key) => !oneHalf.includes(key));
+                if (byChain.length > 0) {
+                    logLine("replay.targets_found_by_chain", {
+                        scenario: scenario.id,
+                        unnamed: byChain.join(","),
+                        why: "the chain computes these from its own steps; none is read from the caller",
+                    });
+                }
+                if (forOneHalf.length > 0) {
+                    logLine("replay.targets_for_one_half", {
+                        scenario: scenario.id,
+                        unnamed: forOneHalf.join(","),
+                        why: "only steps that run under a test read these: the plan runs, and a step that needs one stops in front of its call for the agent",
+                    });
+                }
             }
             // The values are in hand: one this plan must not template into its
             // command (safety.ts) declines here, before any directive is injected,
@@ -1058,10 +1105,12 @@ export class ReplayController {
                 // a note, delivered in place of this call.
                 const step = plan.currentStep();
                 if (step) {
+                    // A value only the recording holds (issue 1212) is no fault of the step.
+                    const recorded = err instanceof RecordedValueError;
                     this.recordStep(state, {
                         step,
                         entry: plan.currentEntry(),
-                        outcome: "failed",
+                        outcome: recorded ? "skipped" : "failed",
                         stage: "tool_input_logic",
                         error: errText(err),
                     });
@@ -1071,6 +1120,7 @@ export class ReplayController {
                         position: plan.currentStepIndex,
                         error: errText(err),
                         delivery: "above",
+                        ...(recorded ? { stopReason: "not_run" } : {}),
                     });
                     const note = this.takeNote(state);
                     if (note)
@@ -1772,7 +1822,7 @@ export class ReplayController {
         const deadline = Date.now() + this.budgets.planMs;
         let result;
         try {
-            result = await plan.runToCompletion(this.executeStep(state, { deadline }), this.recordedSource(state), this.observeStep(state), deadline, MAX_REPLAY_REASON, { stopOnFailure: true, recordedFallback: this.recordedFallback, signal });
+            result = await plan.runToCompletion(this.executeStep(state, { deadline }), this.recordedSource(state), this.observeStep(state), deadline, MAX_REPLAY_REASON, { stopOnFailure: true, recordedFallback: this.recordedFallback, signal, reportTool: this.reportToolFor(state) });
         }
         catch (err) {
             // The scenario's own logic failed. Retire and let the model do the work.
@@ -2233,8 +2283,18 @@ export class ReplayController {
         const service = this.deriveService(scenario.id);
         const hasSchema = Object.keys(scenario.paramsObject ?? {}).length > 0;
         state.deriveOnService = !apiKey && service !== undefined && hasSchema;
+        // The day a system prompt told the agent it is — a benchmark's keeps its
+        // agent on 2023-11-30 for ever — and else this machine's (issue 1212).
+        // Never Claude Code's own date reminder: that is this machine's day, and
+        // on a new day's first prompt still yesterday's.
+        const agent = systemDay(ctx.agentDay?.());
+        const today = agent?.day ?? localDay();
+        state.deriveDay = today;
+        state.deriveLine = agent?.line;
         return derive({
             prompt,
+            today,
+            ...(agent?.line ? { todaySaid: agent.line } : {}),
             intent: scenario.intent ?? "",
             paramsObject: scenario.paramsObject,
             apiKey,
@@ -2252,6 +2312,9 @@ export class ReplayController {
         })
             .then((r) => {
             state.deriveCostUsd += r.costUsd;
+            // The day the service read from: a plan on a fixed date reads its own.
+            if (r.dayUsed)
+                state.deriveDay = r.dayUsed;
             logLine("replay.derived", {
                 scenario: scenario.id,
                 params: Object.keys(r.params).length,
@@ -2333,6 +2396,7 @@ export class ReplayController {
                 recordedFallback: this.recordedFallback,
                 deadline,
                 signal,
+                reportTool: this.reportToolFor(state),
             });
         }
         catch (err) {
@@ -2768,6 +2832,21 @@ export class ReplayController {
         }
         return serializeCapped(redact(result));
     }
+    /**
+     * The report tool a bundle may name, when this plan's session has it
+     * ({@link ReplayState.reportTool}); undefined leaves the bundle's words as
+     * they always were. Never for another host: its adapter reads no bundle.
+     */
+    reportToolFor(state) {
+        if (state.host || !state.reportTool)
+            return undefined;
+        try {
+            return state.reportTool() ? REPORT_TOOL_NAME : undefined;
+        }
+        catch {
+            return undefined;
+        }
+    }
     /** Where recorded outputs come from, when they may stand in at all. */
     recordedSource(state, force = false) {
         if (!force && !this.recordedFallback)
@@ -2801,6 +2880,37 @@ export class ReplayController {
             // carries the verdict, so the recording page says why it never runs.
             blame: true,
         });
+        // The agent's own day may show only now (issue 1212): its host writes a
+        // system prompt's "Today's date is …" just after the prompt hook. Values
+        // read for another day are read again, before the first step runs.
+        if (!opts.adHoc && state.rederive && state.agentDayNow && plan.currentStepIndex === 0) {
+            const now = systemDay(state.agentDayNow());
+            // A new line, unless it says the very day the values were read for. A
+            // line with the day in words is read by the service.
+            if (now && now.line !== state.deriveLine && (now.day === undefined || now.day !== state.deriveDay)) {
+                logLine("replay.rederive", {
+                    scenario: state.scenarioId ?? undefined,
+                    from: state.deriveDay,
+                    to: now.day ?? "said in words",
+                });
+                state.deriveLine = now.line;
+                if (now.day)
+                    state.deriveDay = now.day;
+                const targets = targetKeys(plan.paramsSchema);
+                // A reading that did not answer names no target, as at the prompt
+                // (gate 7): never the recorded person or date in its place.
+                const again = state.rederive(now).then((r) => {
+                    if (r.derived)
+                        return r;
+                    const params = { ...r.params };
+                    for (const key of targets)
+                        params[key] = null;
+                    return { ...r, params, missing: [...targets] };
+                });
+                again.then((r) => (state.rederivedMissing = r.missing), () => (state.rederivedMissing = [...targets]));
+                plan.resetParams(again.then((r) => r.params));
+            }
+        }
         if (!plan.parametersSettled) {
             let failure;
             const settled = plan.ready().then(() => true, (err) => {
@@ -2839,6 +2949,21 @@ export class ReplayController {
                 }
             }
         }
+        // The second reading left a value unnamed that this plan would act on.
+        const missingAgain = state.rederivedMissing;
+        state.rederivedMissing = undefined;
+        if (missingAgain && !opts.adHoc) {
+            const planned = plan.allEntries().slice(0, plan.stepCount);
+            const blocked = blockingTargets(missingAgain, plan.paramsSchema, planned, plan.paramsLogic);
+            if (blocked.length > 0) {
+                return {
+                    ok: false,
+                    error: `the value of ${blocked[0]} could not be read from this request for ${state.deriveDay}`,
+                    stage: "derive_params",
+                    blame: false,
+                };
+            }
+        }
         const sink = opts.adHoc
             ? undefined
             : sinkOfPlan(plan.allEntries().slice(0, plan.stepCount), state.host?.sinks);

package/dist/replay/derive.d.ts

@@ -38,10 +38,21 @@ export interface LiveCall {
     /** Redacted and capped, exactly as the probe sent it. */
     toolInput: string;
 }
+/** The day the agent was told it is (control/transcript.ts `agentDayOf`; issue 1212). */
+export interface AgentDay {
+    line: string;
+    day?: string;
+    source: "system" | "host";
+}
 /** What a turn can tell derivation beyond its prompt (segmented.md R-PARAM-2). */
 export interface DeriveContext {
     liveCall?: LiveCall;
     recentResults?: string[];
+    /**
+     * The agent's own day, read again each time it is asked: its host writes it
+     * into the session only after the prompt hook (issue 1212).
+     */
+    agentDay?: () => AgentDay | undefined;
 }
 /**
  * The service, which derives when this runner has no key of its own
@@ -62,6 +73,8 @@ export interface DeriveOptions extends DeriveContext {
     paramsObject: ParamsSchema | null;
     /** The day on this machine. Injected by tests; defaults to {@link localDay}. */
     today?: string;
+    /** The words that told the agent its day, when its host showed it any (issue 1212). */
+    todaySaid?: string;
     /** An override that keeps the reading on this machine. Absent → the service. */
     apiKey?: string;
     /** Used when there is no `apiKey`. Absent too → recorded samples. */
@@ -91,6 +104,8 @@ export interface DeriveResult {
     missing: string[];
     /** Settings filled from their recorded sample (R-PARAM-2). */
     sampled: string[];
+    /** The day the service read the turn's dates from, when it said (issue 1212). */
+    dayUsed?: string;
 }
 /**
  * The day on this machine, `YYYY-MM-DD`, in its own time zone.

package/dist/replay/derive.js

@@ -343,6 +343,10 @@ async function deriveViaService(opts, service, schema, keys) {
                 // What "tomorrow" is counted from. A service that does not know the
                 // field ignores it.
                 today: opts.today ?? localDay(),
+                // The words that told the agent its day ("Today's date is Thursday,
+                // 2023-11-30 …"), which the service turns into the day it reads from
+                // (issue 1212). Older services ignore it.
+                ...(opts.todaySaid ? { todaySaid: opts.todaySaid } : {}),
             }),
             signal: opts.signal ?? timeoutSignal(),
         });
@@ -402,6 +406,7 @@ async function deriveViaService(opts, service, schema, keys) {
         model: body.model,
         missing,
         sampled,
+        ...(typeof body.today === "string" ? { dayUsed: body.today } : {}),
     };
 }
 //# sourceMappingURL=derive.js.map

package/dist/replay/plan.d.ts

@@ -24,6 +24,48 @@ import { type FlatEntry } from "./flatten.js";
 import type { ReplayMode } from "./coverage.js";
 import type { HandoverStopReason } from "./handover.js";
 import type { ExecutionStage, FallbackKind, ParamsSchema, SerializedScenario, SerializedScenarioStep } from "./types.js";
+/**
+ * A value a step of the recording worked out — an address the directory
+ * returned, the id a search found; the service labels it `from: "step"` —
+ * is never replayed from the recording (issue 1212). The plan holds this mark
+ * in its place. A step whose input still holds it, because its earlier steps
+ * did not return the value and its code fell back on the parameter, stops the
+ * plan in front of it, for the agent; a summary that holds it is left out.
+ * Never the recorded e-mail address, never the recorded lead.
+ */
+export declare const FROM_STEP_MARK = "\u2063bir-from-step:";
+/**
+ * A value the request did not give, which only the steps of one half of an
+ * "if \u2026 otherwise" read (the service's review of calcscenarioautocorrect.md,
+ * item 11; targets.ts, `testedOnlyTargets`). The plan arms without it \u2014 the
+ * other half needs nothing it lacks \u2014 and holds this mark in its place, never
+ * `null` and never the recorded value. A step whose input reads or holds it
+ * stops in front of its call, and the agent does that half; a test that reads
+ * it decides nothing; a summary that reads it is left out.
+ */
+export declare const MISSING_VALUE_MARK = "\u2063bir-missing:";
+/**
+ * A step's code read a value a step works out, which here only the recording
+ * holds (issue 1212). Not a fault of the step: its earlier steps did not
+ * return the value this time. The plan stops in front of it, the step counts
+ * as not run, and the agent does the rest.
+ */
+export declare class RecordedValueError extends Error {
+    readonly key: string;
+    constructor(key: string, message?: string);
+}
+/**
+ * A step's code read a value the request did not give ({@link
+ * MISSING_VALUE_MARK}). The same stop as {@link RecordedValueError} \u2014 not a
+ * fault of the step: it counts as not run, nothing is parked or repaired, and
+ * the agent does the rest \u2014 so it is one, and every place that tells that
+ * stop apart tells this one apart too. Its words name the value.
+ */
+export declare class MissingValueError extends RecordedValueError {
+    constructor(key: string);
+}
+/** The value a mark stands for, when `value` holds one anywhere \u2014 either kind. */
+export declare function markedKey(value: unknown): string | undefined;
 /**
  * Runs one step's tool and returns its output as a string.
  *
@@ -76,6 +118,14 @@ export interface DriveOptions {
      * so what the run reports about it is the truth.
      */
     signal?: AbortSignal;
+    /**
+     * The tool the model may call although the bundle says to call none: `bir`'s
+     * own `mcp__bir__report_wrong_result`, when the session has it (the service's
+     * review of calcscenarioautocorrect.md, item 19). Set by the caller only
+     * where the `bir` server is connected; absent, the bundle's words are as
+     * they always were (bundle.ts, `AssembleOptions.reportTool`).
+     */
+    reportTool?: string;
 }
 /**
  * Supplies a step's recorded output when its tool cannot run here (§8.1).
@@ -289,7 +339,17 @@ export declare class ScenarioReplayPlan {
     private readonly scenario;
     /** The flat list this plan walks. One entry per step that will really run. */
     private readonly entries;
-    private readonly paramsPromise;
+    private paramsPromise;
+    /** The values a step worked out in the recording ({@link FROM_STEP_MARK}). */
+    private readonly fromStep;
+    /**
+     * The targets only the steps of one half read (targets.ts,
+     * `testedOnlyTargets`): each held as {@link MISSING_VALUE_MARK} when the
+     * request does not give it.
+     */
+    private readonly oneHalf;
+    /** Of {@link oneHalf}, the ones the request did not give: held as the mark now. */
+    private missingMarked;
     private params;
     /** The derivation's own answer, before `paramsLogic`. What safety.ts judges. */
     private derivedParams;
@@ -385,6 +445,12 @@ export declare class ScenarioReplayPlan {
      * throws.
      */
     useFallbackParams(): void;
+    /**
+     * The values read again, before any step ran (issue 1212): the agent's own
+     * day showed in the session only after the first reading. False once a step
+     * has run, when the values the steps ran on are the ones that stand.
+     */
+    resetParams(params: Promise<Record<string, unknown>>): boolean;
     /** Whether the parameters are final. */
     get parametersSettled(): boolean;
     /** The derivation's own values, before `paramsLogic`. Empty until settled. */
@@ -409,6 +475,21 @@ export declare class ScenarioReplayPlan {
     currentEntry(): FlatEntry | undefined;
     /** The parameters in force. Empty until {@link ready} resolves. */
     get parameters(): Readonly<Record<string, unknown>>;
+    /**
+     * The values a body may not use: a value a step works out ({@link
+     * FROM_STEP_MARK}) and a value the request did not give ({@link
+     * MISSING_VALUE_MARK}). A read of one is the stop {@link valueError} names.
+     */
+    private watchedKeys;
+    /** The stop a read of `key` means: a value the request did not give, or one only the recording had. */
+    private valueError;
+    /**
+     * `parameters` for one body, with every read of a value a step works out
+     * noted (issue 1212): a read is the recorded value used, however the code
+     * changes it afterwards — upper-cased, encoded, sliced or counted. A value
+     * the request did not give is watched the same way (review item 11).
+     */
+    private watched;
     /** Everything the steps have accumulated so far. */
     get accumulated(): Readonly<Record<string, unknown>>;
     /** The scenario's parameter schema — what the target and safety checks judge. */
@@ -439,6 +520,14 @@ export declare class ScenarioReplayPlan {
      * a throw as divergence.
      */
     toolInputForCurrentStep(): Record<string, unknown>;
+    /**
+     * A step's input, refused when it holds a value only the recording had: its
+     * code fell back on a parameter a step works out, and the earlier steps did
+     * not return it (issue 1212). The caller stops the plan in front of it. So
+     * is one that reads, or holds, a value the request did not give (review
+     * item 11): {@link MissingValueError}.
+     */
+    private inputOf;
     /** Whether the plan's cursor rests in front of the step it is parked before. */
     atStop(): boolean;
     /**