@basein/runner 0.2.12 → 0.3.0

package/dist/record/remote-recorder.d.ts

@@ -8,55 +8,181 @@
  *   PATCH /recordings/runs/:id/steps/:stepId
  *   POST  /recordings/runs/:id/finish
  *
- * Four properties matter, and each is load-bearing:
+ * The properties that matter, each load-bearing:
  *   - **Client-generated ids** (`run_`/`step_` uuids) so callers get an id
  *     synchronously and never await the network on the hot path.
- *   - **One ordered promise chain per run**, so steps arrive in the order they
- *     were recorded even though every send is fire-and-forget.
- *   - **Best-effort.** A failed send is logged and dropped, never retried inline
- *     and never thrown at the caller (§10).
- *   - **One token refresh on 401**, then one retry. Sessions outlive access tokens.
+ *   - **One ordered lane per run**, so steps arrive in the order they were
+ *     recorded even though every send is fire-and-forget.
+ *   - **Best-effort, and invisible when the service is not there.** A failed
+ *     send is logged once per run and dropped, never thrown at the caller
+ *     (§10). Every request has a deadline (`BIR_SERVICE_TIMEOUT_MS`), every
+ *     answer or its absence is reported to the process-wide health verdict
+ *     ({@link serviceHealth}), and while that verdict says the service is
+ *     cooling down the calls a hook waits on — `getMatch`, `runCreated`,
+ *     `matchIntent` — answer "no match" at once instead of making the user wait,
+ *     and `flush` returns at once, leaving the rest to drain in the background.
+ *     A run's create that fails its first attempt answers "no match" at once
+ *     too; its retry is for the recording, never for a hook.
+ *   - **One probe, not a stampede.** While the breaker is half-open, one
+ *     queued send finds out for all the lanes; the rest wait for its verdict
+ *     and then follow it a moment apart, not all at once.
+ *   - **Bounded.** The lanes are the real backlog (the hook server's step queue
+ *     drains instantly into them), so they carry the bound: past
+ *     {@link DEFAULT_MAX_PENDING_SENDS} sends or {@link DEFAULT_MAX_PENDING_BYTES}
+ *     the oldest unsent *step* is dropped and counted — creates and finishes go
+ *     last, so the service is not left with runs that never end.
+ *   - **One token refresh on 401**, shared by every send that meets the same
+ *     expiry, then one retry. Sessions outlive access tokens.
+ *   - **It follows credentials.json.** `bir logout` removes the file; a
+ *     recorder that loaded its session from it stops sending within seconds,
+ *     rather than recording for the rest of the access token's life.
+ *   - **It redacts and caps every free-text field it sends** — the prompt, the
+ *     answer, reasoning, errors — as the last line of defence. Structured tool
+ *     payloads are redacted by their producers; text is only ever seen here.
  *
  * SIMILAR-PROMPT HITS. `POST /recordings/runs` may answer `{ id: null, matched }`,
  * meaning the server recognised the prompt and did **not** create a run. Every
  * later step post for that id would then 404, so callers must stop recording —
- * {@link RemoteRecorder.getMatch} is how they find out. v1 only stops; replay is v2.
+ * {@link RemoteRecorder.getMatch} is how they find out — and this recorder skips
+ * them itself too, for the caller that stopped waiting before the answer came.
  */
 import { type AuthSession } from "../auth/client.js";
-import type { ExecutionReport, IntentMatchAnswer, IntentMatchRequest, Recorder, RunCreation, RunMatch, RunMetrics, StartRunOptions } from "./recorder.js";
+import { type ServiceHealth } from "../util/service-health.js";
+import type { ExecutionReport, FlushOptions, IntentMatchAnswer, IntentMatchOptions, IntentMatchRequest, Recorder, RecorderHealth, RecorderHealthReporter, RunCreation, RunMatch, RunMetrics, StartRunOptions } from "./recorder.js";
+/**
+ * Caps for the free text this recorder sends, in UTF-8 bytes. Tool payloads
+ * arrive capped at 64 KiB by their producers; text never passed through one,
+ * so a pasted 3 MB log used to go out whole — once as the run's input, again
+ * in the execution report — and spend the prompt's match budget uploading.
+ */
+export declare const TEXT_CAPS: {
+    /** The prompt: the run's `input`, and an execution report's `prompt`. */
+    readonly prompt: number;
+    /** The final answer, and the run's `finalOutput`. */
+    readonly answer: number;
+    /** A step's written reasoning. */
+    readonly context: number;
+    /** A step's thinking text, sent only to be embedded. */
+    readonly intent: number;
+    /** A tool's error text when it is not JSON. */
+    readonly toolError: number;
+    /** An execution report's error lines, rendered in a web page. */
+    readonly reportError: number;
+};
+/** Queued sends past which the oldest unsent step is dropped (`BIR_RECORDER_MAX_PENDING`). */
+export declare const DEFAULT_MAX_PENDING_SENDS = 500;
+/** Queued bytes past which the same happens (`BIR_RECORDER_MAX_PENDING_BYTES`). */
+export declare const DEFAULT_MAX_PENDING_BYTES: number;
+/** How long a probe waits when its caller did not say. Its answer is useless after a hook's budget. */
+export declare const PROBE_TIMEOUT_MS = 5000;
+/**
+ * How long {@link RemoteRecorder.flush} waits when its caller did not say
+ * (`BIR_FLUSH_TIMEOUT_MS`). Every caller is a hook or a shutdown, and a
+ * healthy service drains a turn's last few sends in well under a second.
+ */
+export declare const DEFAULT_FLUSH_TIMEOUT_MS = 2000;
+/** A send that did not succeed, with the status the service answered, if it answered. */
+export declare class ServiceError extends Error {
+    readonly status?: number | undefined;
+    constructor(message: string, status?: number | undefined);
+}
 export interface RemoteRecorderOptions {
     baseUrl: string;
     session: AuthSession;
     /** Value for the run's `framework` column. Defaults to `baseinstrunner`. */
     framework?: string;
+    /** The health verdict to report to. Defaults to the process-wide one; tests inject their own. */
+    health?: ServiceHealth;
+    /** Backlog bound, in sends. Defaults to `BIR_RECORDER_MAX_PENDING` or 500. */
+    maxPendingSends?: number;
+    /** Backlog bound, in bytes. Defaults to `BIR_RECORDER_MAX_PENDING_BYTES` or 32 MiB. */
+    maxPendingBytes?: number;
+    /** How often credentials.json is looked at. Tests shorten it. */
+    credentialsCheckMs?: number;
+    /** How long a refused refresh waits for a sibling's session before pausing. Tests shorten it. */
+    authRecheckMs?: number;
+    /** While paused as `auth_rejected`, how often the refresh is tried again. Tests shorten it. */
+    authRetryMs?: number;
+    /** The longest one queued send waits for a failing service ({@link COOLDOWN_WAIT_MAX_MS}). Tests shorten it. */
+    cooldownWaitMaxMs?: number;
 }
-export declare class RemoteRecorder implements Recorder {
+export declare class RemoteRecorder implements Recorder, RecorderHealthReporter {
     private readonly baseUrl;
     private readonly framework;
+    private readonly health;
+    /** The service these credentials are bound to; never changes for this recorder. */
+    private readonly binding;
+    private readonly maxPendingSends;
+    private readonly maxPendingBytes;
+    private readonly credentialsCheckMs;
     private session;
-    /** runId → tail of that run's ordered send chain. */
-    private readonly chains;
-    /** runId → the similar-prompt outcome of its create call. */
-    private readonly matches;
-    /** runId → what the create call said (not created: a hit, nothing to record). */
-    private readonly created;
+    private state;
+    private readonly runs;
+    private readonly lanes;
+    private seq;
+    private pendingSends;
+    private pendingBytes;
+    private droppedSends;
+    private failedSends;
+    private backlogFull;
+    private droppedWhileFull;
+    private lastSweep;
+    private refreshing?;
+    /** A failed refresh is not tried again before this (epoch ms). */
+    private refreshRetryAt;
+    private readonly authRecheckMs;
+    private readonly authRetryMs;
+    private readonly cooldownWaitMaxMs;
+    /** When the session was last refused or retried while refused. */
+    private rejectedAt;
+    private lastAheadAttempt;
     /**
-     * Execution reports ride their own chain: a matched turn has no run on the
-     * service (that is what a match means), so there is no run chain to append to.
+     * The one request out while the breaker is half-open — a lane's own send,
+     * or a hook-requested `GET /health` — and when it started.
      */
-    private readonly executionChainKey;
+    private probe?;
+    private matchIntentOffUntil;
+    /** True once this recorder's session is known to be the one in credentials.json. */
+    private follow;
+    private credMtime;
+    private credCheckedAt;
     constructor(opts: RemoteRecorderOptions);
+    /**
+     * The access token this recorder is using *now*. It changes on every refresh
+     * (and when a sibling process's refresh is adopted), so callers that need the
+     * token — replay's service derivation, the source-run fetch — must read it
+     * through this each time rather than keep the startup session. Undefined once
+     * the session is gone (`bir logout`) or was refused.
+     */
+    currentAccessToken(): string | undefined;
+    /** The backlog and sending state, for `/health` and `bir doctor`. */
+    recorderHealth(): RecorderHealth;
     startRun(input: string, metadata?: Record<string, unknown>, options?: StartRunOptions): string;
-    /** What the create call said. "Created" for an id this recorder never opened. */
+    /**
+     * What the create call said. "Created" for an id this recorder never opened,
+     * and — without waiting — while the service is cooling down or the session is
+     * gone, or once the create's first attempt has failed: a hook must never wait
+     * on a service that just failed to answer, nor on the retry after it.
+     */
     runCreated(runId: string): Promise<RunCreation>;
     /**
      * Match a ReAct iteration's reasoning against the caller's scenario intents
      * (fallbk.md §Runner 4). Awaited, not queued: the caller is a `PreToolUse`
      * holding a budget, and the answer decides what that hook says. A service that
-     * predates the route (404) or any failure is a miss.
+     * predates the route (404) or any failure is a miss — and so is a service in
+     * a cool-down, without a request: the probe runs on every tool call, so a
+     * dead service must cost it nothing.
+     *
+     * `opts.signal`/`opts.timeoutMs` end the request when the caller stops
+     * waiting, so an abandoned probe does not hold a socket for minutes.
+     */
+    matchIntent(req: IntentMatchRequest, opts?: IntentMatchOptions): Promise<IntentMatchAnswer | null>;
+    /**
+     * The similar-prompt match for a run's create call (null = fresh run). Like
+     * {@link runCreated}, answers null at once while the service is cooling down,
+     * and as soon as the create's first attempt fails — the first prompt of an
+     * outage is not held for the retry's sleep (up to 2 s behind a Retry-After).
      */
-    matchIntent(req: IntentMatchRequest): Promise<IntentMatchAnswer | null>;
-    /** Await the similar-prompt match for a run's create call (null = fresh run). */
     getMatch(runId: string): Promise<RunMatch | null>;
     recordToolSelected(runId: string, stepIndex: number, data: {
         toolName: string;
@@ -84,28 +210,161 @@ export declare class RemoteRecorder implements Recorder {
     /**
      * Report what a matched turn cost (docs/calculatedReplay.md §11.2).
      *
-     * Queued on the run's own chain so it lands after that run's last step and so
-     * `flush` waits for it — a report is the last thing a matched turn produces.
-     * Best-effort like everything else here: an unredeemed ticket leaves no
-     * execution row at all, which loses a saving but can never invent one.
+     * Queued on the executions lane — a matched turn has no run on the service —
+     * so `flush` waits for it. Best-effort like everything else here: an
+     * unredeemed ticket leaves no execution row at all, which loses a saving but
+     * can never invent one.
      *
      * A doubled report (a retry, `Stop` racing `SessionEnd`) is safe by
      * construction: the ticket *is* the row's id, so the service books once and
-     * answers `200 { duplicate: true }`.
+     * answers `200 { duplicate: true }`. That is also why a report *with* a
+     * ticket is the one kind retried after an ambiguous failure.
      */
     reportExecution(report: ExecutionReport): void;
-    /** Await every queued send for a run, including its execution report. */
-    flush(runId: string): Promise<void>;
-    /** Append to this run's ordered chain; failures are logged, not thrown. */
+    /**
+     * Await every queued send for a run, including its execution report — for at
+     * most `opts.timeoutMs` (default {@link DEFAULT_FLUSH_TIMEOUT_MS}, or
+     * `BIR_FLUSH_TIMEOUT_MS`), and not at all while the service is cooling down.
+     * Either way the sends carry on in the background; a flush behind a dead
+     * service must never be what holds a hook or a shutdown.
+     *
+     * WHY THE BREAKER ENDS THE WAIT. In the common outages — offline, refused,
+     * a 5xx — the first send fails in milliseconds and opens the breaker, and
+     * every job after it sleeps out the cool-down before it tries (the service
+     * is not hammered). A flush that waited for those sleeps paid its whole
+     * deadline on every turn of the outage, where the old fail-at-once chain
+     * paid tens of milliseconds. Nothing a cooling-down lane holds can land
+     * sooner than the cool-down, so the flush stops waiting the moment the
+     * breaker opens.
+     */
+    flush(runId: string, opts?: FlushOptions): Promise<void>;
+    private newEntry;
+    /** The entry for a run, made on first sight for an id this recorder did not open. */
+    private entryFor;
+    private enqueueRun;
+    /** Append to a lane. Synchronous and never throws; the lane drains after the caller returns. */
     private enqueue;
+    private drain;
+    private laneIsIdle;
+    /**
+     * Wait until every lane in `keys` is idle, the breaker opens ("cooling"), or
+     * `timeoutMs` passes — whichever is first. The breaker is polled (every
+     * {@link FLUSH_POLL_MS}) rather than subscribed to: the verdict is shared by
+     * the whole process and has no listeners, and a poll only runs while a flush
+     * is actually waiting.
+     */
+    private waitForLanes;
+    /** Run one job: decide whether it may still go, wait out a cool-down, send. Never throws. */
+    private execute;
+    private whyNotSend;
+    /**
+     * A job queued under one account, about to go out after another signed in:
+     * its run is closed (the rest would split it across accounts), and a report
+     * is dropped with a line. Returns the reason to retire it with.
+     */
+    private accountChanged;
+    /** A job leaves without being sent: counted (unless it was never meant to go), and a create settles. */
+    private retire;
+    /** Skip everything left for a run, once: its sends would all fail the same way. */
+    private markDead;
+    private onSendFailed;
+    /** At a run's finish: say what did not arrive, if anything did not. */
+    private summarize;
+    /** Keep the backlog inside its bound: drop the oldest unsent step first. */
+    private enforceBound;
+    /**
+     * The oldest unsent step or patch; failing that, the oldest job of any kind.
+     * Steps first because a run without its middle is still a run, and one
+     * without its create or finish is a run the service can never close.
+     */
+    private pickVictim;
+    private noteBacklogRecovered;
+    /** Forget runs nobody will ask about again, and release matches past their TTL. */
+    private sweep;
+    /** The breaker is closed: no failure since the last answer. */
+    private healthyNow;
+    /**
+     * May a hook wait on the service right now? Only when it is known to be
+     * answering. A hook is never the half-open probe: once the cool-down ends a
+     * background `GET /health` (or a queued send) finds out, and the next hook
+     * waits again.
+     */
+    private mayHotWait;
+    private probeSoon;
+    /** Take the half-open slot. The returned function gives it back, once. */
+    private beginProbe;
+    /**
+     * Is a probe out? One older than any request could take (every attempt has
+     * a deadline) is a slot somebody failed to give back, and is not honoured.
+     */
+    private probeInFlight;
+    /** Lanes let go after a successful probe spread over a fraction of the cool-down (at most 2 s). */
+    private releaseJitterMs;
+    /**
+     * Wait until a background send may go.
+     *
+     * - The breaker is closed: go at once.
+     * - It is cooling down: sleep until it ends. A send that has waited
+     *   {@link COOLDOWN_WAIT_MAX_MS} and still faces more than
+     *   {@link COOLDOWN_GRACE_MS} of it is given up (`giveUp`), unsent.
+     * - It is half-open: ONE lane takes the probe slot and goes (`endProbe`
+     *   gives the slot back once its send settles); every other lane waits for
+     *   that verdict. A success lets them go, each after its own short random
+     *   pause; a failure starts a longer cool-down, and they wait that out.
+     *
+     * Before this, every lane with a queued send woke at the same moment and
+     * sent into the dead service together, once per cool-down.
+     */
+    private waitForService;
+    /** Renew the access token shortly before it expires — in a lane, never on a hook. */
+    private refreshAhead;
     private post;
-    private patch;
+    /** A background send: every lane job is safe to repeat once (client ids, a PATCH, a ticket). */
     private send;
+    /**
+     * One request, with a deadline, reported to the health verdict, with the
+     * one-refresh-on-401 rule and — for background sends — one retry after a
+     * transient failure.
+     *
+     * WHICH FAILURES ARE RETRIED. Only those that prove the service did not take
+     * the request: a connection that never happened or a pooled socket the load
+     * balancer had already closed ({@link SAFE_RETRY_CODES}), and a 429 or 503.
+     * Not a 502/504 or a reset mid-request — the origin may have the first
+     * attempt — not a timeout (a hung service would just cost another deadline)
+     * and not while the service is already failing (the breaker's cool-down is
+     * the retry then). Every background body carries a client-generated id — the
+     * run, the step, the execution ticket — or is a PATCH of one field, so even
+     * a repeat whose first attempt did land is recognisable; a 409 on the retry
+     * is taken as that.
+     *
+     * THE ACCOUNT IS FIXED FOR THE REQUEST. A retry or a refresh leaves the event
+     * loop, and a hook may sign another account in meanwhile; the next attempt
+     * then goes nowhere ({@link AccountChanged}) instead of out under that
+     * account's token.
+     *
+     * EVERY ANSWER SAYS WHEN ITS ATTEMPT WAS SENT. `health.ok(started)`: the late
+     * answer of an attempt a hook already gave up on (`slow`) must not end the
+     * cool-down that overrun started.
+     *
+     * `onRetry` runs when the first attempt failed and a retry is about to
+     * sleep: whoever waits on this answer on a hook's behalf can stop now.
+     */
+    private request;
+    /**
+     * Refresh once for everyone. Concurrent 401s — a run's create, an execution
+     * report and a probe crossing the expiry together — used to each spend the
+     * rotating refresh token; one won, the others were rejected and their sends
+     * (a create: the whole turn) were dropped.
+     */
+    private refreshSingleFlight;
     /**
      * Adopt a newer session written by a sibling process, if there is one.
      *
      * Returns true when this recorder now holds a usable token it did not have to
-     * spend a refresh on.
+     * spend a refresh on. Only a session for the same service and the same
+     * account is adopted: the binding is what keeps a token from being sent to
+     * an address it was not issued for, and an account change is a run boundary
+     * ({@link checkCredentials}), never a mid-run swap.
      */
     private adoptStoredSession;
     /**
@@ -121,8 +380,44 @@ export declare class RemoteRecorder implements Recorder {
      * The fix is to treat the credentials file as the shared source of truth:
      * check it before spending our token, and check it again if our refresh is
      * rejected, because the process that beat us has already written a good one.
+     * A rejection with nothing to adopt pauses the recorder (`auth_rejected`)
+     * instead of paying two round trips on every later send; a new sign-in
+     * resumes it, and so does a retry every {@link AUTH_REJECTED_RETRY_MS}.
+     *
+     * WHAT COUNTS AS A REJECTION. Only the service's own JSON 400/401 — the
+     * answer that says this refresh token is spent. A 403 or a web page (a CDN
+     * challenge, a portal) says nothing about the token, and pausing on one
+     * stopped recording for the rest of a daemon's life. And a rejection is
+     * believed only after one more look at the file ~{@link AUTH_RECHECK_MS}
+     * later: a sibling that rotated the token a moment ago may not have written
+     * it yet. Any other failure is not retried for {@link REFRESH_RETRY_MS}, so a
+     * refresh route that keeps failing costs one request per half minute, not
+     * one per send.
      */
     private refresh;
+    /**
+     * While `auth_rejected`, try the refresh again now and then, in the
+     * background: a refusal the service later takes back (a bad deploy, a
+     * clock), or one the recheck missed, must not need a `bir login` to end.
+     * Never awaited by the caller — a hook's credentials check starts it.
+     */
+    private retryRejected;
     private refreshOverNetwork;
+    /** The file now holds this recorder's session: follow it from here on. */
+    private noteFileSeen;
+    /**
+     * Keep in step with credentials.json, cheaply: one `stat` per
+     * {@link CREDENTIALS_CHECK_MS}, and a read only when it changed.
+     *
+     *   - gone          → `bir logout`: pause (`signed_out`), drop what is queued.
+     *   - another service → pause (`other_service`): never send there.
+     *   - another account → adopt it for runs opened from now on; runs already
+     *     open are closed, so no run is split across two accounts.
+     *   - newer tokens  → adopt them (a sibling refreshed, or `bir login`), and
+     *     resume if paused.
+     */
+    private checkCredentials;
+    private pause;
+    private resume;
 }
 //# sourceMappingURL=remote-recorder.d.ts.map