@humwick/runner 0.0.0-stage → 0.130.1

package/src/agent-runner/session-mirror.ts

@@ -0,0 +1,278 @@
+import { errMessage, type ContextState, type MessageSessionMeta, type HumwickHttpClient, type SendMessageInput } from "@humwick/sdk";
+import type { ProviderAdapter, ProviderSessionEvent, ProviderSessionEventBatch, ProviderSessionTurnContext } from "../adapter";
+import { Outbox, type OutboxRecord } from "../outbox";
+import { deliverRunnerOutboxEvent, emitSessionPoisonNotice } from "../mcp-outbox";
+import { transcriptDerivedContext, type TranscriptContextCache } from "../transcript-context";
+
+interface SessionMirrorDeps {
+  agentId: string;
+  provider: string;
+  instanceId: string;
+  startedAt: number;
+  relayUrl: string;
+  outboxDir: string | undefined;
+  http: HumwickHttpClient;
+  adapter: ProviderAdapter;
+  getToken(): string | undefined;
+  updateOutboxPayload(seq: number, payload: unknown): void;
+  recoverRuntimeTokenAfterAuthFailure(source: string): void;
+  sessionLog(message: string): void;
+  sessionTurnContext(): ProviderSessionTurnContext;
+  lastTranscriptPath(): string | undefined;
+  currentTurnId(): string | undefined;
+  reasoningCapture(): boolean | undefined;
+  isRunnerInjectedPrompt(text: string): boolean;
+  publishProviderResponse(event: ProviderSessionEvent, body: string, turnId: string | undefined): Promise<void>;
+}
+
+const SESSION_MIRROR_BATCH_MAX = 100;
+
+export class SessionMirror {
+  // #332: a fast-lane outbox for display-only session-mirror trace events. Routing them through
+  // their own FIFO queue stops one transient trace failure from head-of-line blocking real-message
+  // delivery (and vice versa); its backoff is capped low since a stale live-mirror frame is cheap.
+  private readonly sessionOutbox: Outbox;
+  /** In-flight asynchronous turn captures — the settle waits these out before judging a turn empty. */
+  private turnCapturesInFlight = 0;
+  private sessionBatchSeq = 0;
+  // #1512: mtime-keyed cache of the last transcript-derived context usage, so the
+  // heartbeat's fallback parse doesn't re-read a large transcript that hasn't changed.
+  private transcriptContextCache?: TranscriptContextCache;
+  // #1235 — true while a monitorless (hookless) provider turn is being mirrored natively:
+  // the adapter supplied a transcript path on the provider-turn busy edge, so the runner
+  // drives the reasoning tail itself (start on busy, final-capture on idle) instead of via
+  // Stop/UserPromptSubmit hooks the lean/isolated worker never installs.
+  private nativeSessionCaptureActive = false;
+
+  constructor(private readonly deps: SessionMirrorDeps) {
+    this.sessionOutbox = new Outbox({
+      agentId: `${this.deps.agentId}-session`,
+      dir: this.deps.outboxDir,
+      send: (record) => this.deliverOutboxEvent(record),
+      // Ephemeral trace frames: cap the per-row stall low. #2433 — transient (connect-class /
+      // 5xx / 429) failures never burn this budget; it now only bounds unknown-class failures.
+      maxBackoffMs: 5_000,
+      maxAttempts: 12,
+      // #2433 — a poisoned session row is REAL loss (report-up chat rides this queue), so
+      // surface it in-session via the mirror-drop notice seam instead of only a FATAL host log.
+      onPoison: (record, reason) => emitSessionPoisonNotice({ record, http: this.deps.http }, reason),
+    });
+  }
+
+  enqueueResponseReport(record: Parameters<Outbox["enqueue"]>[0]): void { this.sessionOutbox.enqueue(record); }
+
+  start(): void { this.sessionOutbox.start(); }
+  close(): void { this.sessionOutbox.close(); }
+  flushSessionEvents(timeoutMs: number): Promise<boolean> { return this.sessionOutbox.flush(timeoutMs); }
+  pendingSessionEventCount(): number { return this.sessionOutbox.pendingCount(); }
+  capturesInFlight(): boolean { return this.turnCapturesInFlight > 0; }
+  beginTurnCapture(): void { this.turnCapturesInFlight += 1; }
+  endTurnCapture(): void { this.turnCapturesInFlight -= 1; }
+  nativeCaptureActive(): boolean { return this.nativeSessionCaptureActive; }
+  clearNativeSessionCapture(): void { this.nativeSessionCaptureActive = false; }
+
+  transcriptContext(tokensMax: number | undefined): ContextState | undefined {
+    const derived = transcriptDerivedContext({ transcriptPath: this.deps.lastTranscriptPath(), derive: this.deps.adapter.contextUsageFromTranscript, tokensMax, cache: this.transcriptContextCache, now: Date.now() });
+    this.transcriptContextCache = derived.cache;
+    return derived.context;
+  }
+
+  // Post one session-mirror event (prompt echo, assistant response, reasoning or
+  // tool step) as a `kind: "session"` relay message tagged with payload.session so
+  // the dashboard can render the live provider session faithfully. Display-only:
+  // session messages are never delivered back into a provider.
+  publishSessionEvent(input: { from: string; to: string; body: string; session: MessageSessionMeta; replyTo?: number; occurredAt?: number }): void {
+    // Durable, ordered, timestamped (#196/#1115): the actual POST happens in
+    // deliverOutboxEvent, retried until it lands. An explicit occurredAt preserves the
+    // provider-native event time; otherwise the outbox stamps enqueue time. Routed through
+    // the fast-lane sessionOutbox (#332) so a transient trace failure can't head-of-line
+    // block real messages.
+    // A stepId-bearing step (tool running to completed, streamed reasoning/response) uses a
+    // STABLE idempotency key so the server upserts the row in place instead of appending a dup.
+    const stepId = input.session.stepId;
+    this.sessionOutbox.enqueue({
+      kind: "session-message",
+      ...(input.occurredAt ? { occurredAt: input.occurredAt } : {}),
+      ...(stepId ? { idempotencyKey: `session-step:${input.from}:${input.session.turnId ?? ""}:${stepId}` } : {}),
+      payload: {
+        from: input.from,
+        to: input.to,
+        ...(input.replyTo ? { replyTo: input.replyTo } : {}),
+        kind: "session",
+        body: input.body,
+        payload: { session: { provider: this.deps.provider, ...input.session } },
+      } satisfies SendMessageInput,
+    });
+  }
+
+  publishSessionEventsBatch(inputs: Array<{ from: string; to: string; body: string; session: MessageSessionMeta; replyTo?: number; occurredAt?: number }>): void {
+    for (let i = 0; i < inputs.length; i += SESSION_MIRROR_BATCH_MAX) {
+      const chunk = inputs.slice(i, i + SESSION_MIRROR_BATCH_MAX);
+      if (chunk.length === 1) {
+        this.publishSessionEvent(chunk[0]!);
+        continue;
+      }
+      const messages = chunk.map((input) => {
+        const stepId = input.session.stepId;
+        return {
+          from: input.from,
+          to: input.to,
+          ...(input.replyTo ? { replyTo: input.replyTo } : {}),
+          kind: "session",
+          body: input.body,
+          ...(input.occurredAt ? { occurredAt: input.occurredAt } : {}),
+          ...(stepId ? { idempotencyKey: `session-step:${input.from}:${input.session.turnId ?? ""}:${stepId}` } : {}),
+          payload: { session: { provider: this.deps.provider, ...input.session } },
+        } satisfies SendMessageInput;
+      });
+      this.sessionOutbox.enqueue({
+        kind: "session-message-batch",
+        idempotencyKey: `session-batch:${this.deps.agentId}:${this.deps.instanceId}:${this.deps.startedAt}:${++this.sessionBatchSeq}:${messages.length}`,
+        payload: { messages },
+      });
+    }
+  }
+
+  // Map queued records to HTTP calls. Throw to retry, return to ack/delete.
+  async deliverOutboxEvent(record: OutboxRecord): Promise<void> {
+    await deliverRunnerOutboxEvent({
+      record,
+      http: this.deps.http,
+      relayUrl: this.deps.relayUrl,
+      getToken: () => this.deps.getToken(),
+      updatePayload: (seq, payload) => this.deps.updateOutboxPayload(seq, payload),
+      sessionLog: (message) => this.deps.sessionLog(message),
+      recoverRuntimeTokenAfterAuthFailure: (source) => this.deps.recoverRuntimeTokenAfterAuthFailure(source),
+    });
+  }
+
+  // Route a provider-emitted session event into the chat mirror.
+  // Mirrors provider session events into chat. Prompts are echoed with dedup, and
+  // response routing clears only the obligations the adapter says its capture can satisfy.
+  async publishProviderSessionEvent(event: ProviderSessionEvent): Promise<void> {
+    const body = event.body.trim();
+    if (!body) return;
+    const turnId = event.turnId ?? this.deps.currentTurnId();
+    if (event.type === "prompt") {
+      if (this.deps.isRunnerInjectedPrompt(body)) return;
+      await this.publishSessionEvent({
+        from: "user",
+        to: this.deps.agentId,
+        body,
+        session: { type: "prompt", origin: event.origin ?? "terminal", ...(turnId ? { turnId } : {}) },
+        ...(event.occurredAt ? { occurredAt: event.occurredAt } : {}),
+      });
+      return;
+    }
+    if (event.type === "response" && event.final === false) {
+      await this.publishSessionEvent({
+        from: this.deps.agentId,
+        to: "user",
+        body,
+        ...(event.occurredAt ? { occurredAt: event.occurredAt } : {}),
+        session: { type: "response", origin: event.origin ?? "provider", ...(turnId ? { turnId } : {}) },
+      });
+      return;
+    }
+    if (event.type === "response") {
+      return this.deps.publishProviderResponse(event, body, turnId);
+    }
+    if (this.deps.reasoningCapture() === false) return;
+    this.publishSessionEvent(this.providerStepSessionInput(event, body, turnId));
+  }
+
+  async publishProviderSessionEvents(events: ProviderSessionEventBatch): Promise<void> {
+    const inputs: Array<{ from: string; to: string; body: string; session: MessageSessionMeta; replyTo?: number; occurredAt?: number }> = [];
+    for (const event of events) {
+      const body = event.body.trim();
+      if (!body) continue;
+      if (event.type === "prompt" || event.type === "response") {
+        await this.publishProviderSessionEvent(event);
+        continue;
+      }
+      if (this.deps.reasoningCapture() === false) continue;
+      inputs.push(this.providerStepSessionInput(event, body, event.turnId ?? this.deps.currentTurnId()));
+    }
+    if (inputs.length) this.publishSessionEventsBatch(inputs);
+  }
+
+  private providerStepSessionInput(event: ProviderSessionEvent, body: string, turnId?: string): { from: string; to: string; body: string; session: MessageSessionMeta; occurredAt?: number } {
+    return {
+      from: this.deps.agentId,
+      to: "user",
+      body,
+      ...(event.occurredAt ? { occurredAt: event.occurredAt } : {}),
+      session: {
+        type: event.type,
+        origin: event.origin ?? "provider",
+        ...(turnId ? { turnId } : {}),
+        ...(event.label ? { label: event.label } : {}),
+        ...(event.category ? { toolCategory: event.category } : {}),
+        ...(event.status ? { status: event.status } : {}),
+        ...(event.streaming !== undefined ? { streaming: event.streaming } : {}),
+        ...(event.stepId ? { stepId: event.stepId } : {}),
+        ...(event.final ? { final: true } : {}),
+        ...(event.params !== undefined ? { params: event.params } : {}),
+      },
+    };
+  }
+
+  stopReasoningTail(): void {
+    this.deps.adapter.stopSessionTrace?.();
+  }
+
+  // #1235 — start the transcript reasoning tail for a hookless (monitorless) turn. Reuses the
+  // exact adapter capture path the UserPromptSubmit hook drives for hooked agents; only the
+  // trigger differs (native turn-start vs hook). A missing/old transcript makes the tail a
+  // graceful no-op (poll finds nothing), so this can never crash a turn.
+  async startNativeSessionCapture(transcriptPath: string): Promise<void> {
+    this.nativeSessionCaptureActive = true;
+    try {
+      await this.deps.adapter.handleUserPrompt?.({ prompt: "", transcriptPath }, this.deps.sessionTurnContext());
+    } catch (error) {
+      this.deps.sessionLog(`native session capture start failed: ${errMessage(error)}`);
+    }
+  }
+
+  // #1235 — capture the final assistant response for a hookless turn (the Stop hook's job for
+  // hooked agents), then tear down the tail. captureSessionTurn drains the tail, reads the
+  // closing turn text from the transcript, and emits the final `response` session event.
+  async finishNativeSessionCapture(transcriptPath: string | undefined, ctx: ProviderSessionTurnContext): Promise<void> {
+    // #2389 — this publishes the closing `response` event AFTER the idle edge; the empty-turn
+    // settle must not judge the turn until it has finished.
+    this.turnCapturesInFlight += 1;
+    try {
+      if (transcriptPath) {
+        await this.deps.adapter.captureSessionTurn?.({ transcriptPath, ...(ctx.turnId ? { promptId: ctx.turnId } : {}) }, ctx);
+      }
+    } catch (error) {
+      this.deps.sessionLog(`native session capture finish failed: ${errMessage(error)}`);
+    } finally {
+      this.turnCapturesInFlight -= 1;
+      this.stopReasoningTail();
+    }
+  }
+
+  // #2491 — the ONE terminal-settle seam for both capture paths: a monitorless (native) turn
+  // discovers its transcript on the provider-turn busy/idle edge; a hooked turn (Stop/
+  // UserPromptSubmit) discovers it via publishSessionTurn/handleUserPrompt. Both write the
+  // same `lastTranscriptPath`, so gating on that path alone (previously also required
+  // `nativeSessionCaptureSeen`, set ONLY by the native busy/idle edge) is what makes this
+  // fire for hooked agents too — without it every hooked turn's one-shot Stop-hook capture
+  // pass raced Claude's own JSONL flush with nothing to settle it.
+  // #2484 round 2 review (MED-1) — the transcript-presence bail used to live HERE, so an
+  // adapter with no transcript file to tail (OpenCode's capture is event-sourced off the
+  // message.updated SSE door, not a JSONL tail) silently never reached its
+  // `captureFinalTokenUsage` hook on any of this seam's three call sites (stop(),
+  // restartUnexpectedProviderExit, runPreSessionDestroy) — a Claude-shaped precondition baked
+  // into the shared seam. The bail is now adapter-owned: a transcript-tailing adapter checks
+  // `transcriptPath` itself (see claude.ts), an event-sourced one ignores it.
+  async settleTerminalTokenUsage(transcriptPath: string | undefined = this.deps.lastTranscriptPath()): Promise<void> {
+    try {
+      await this.deps.adapter.captureFinalTokenUsage?.(transcriptPath, this.deps.sessionTurnContext());
+    } catch (error) {
+      this.deps.sessionLog(`native final token-usage capture failed: ${errMessage(error)}`);
+    }
+  }
+
+}

package/src/agent-runner/task-claims.ts

@@ -0,0 +1,116 @@
+import type { Message, HumwickHttpClient, TaskStatusInput } from "@humwick/sdk";
+import type { Outbox, OutboxRecord } from "../outbox";
+import type { ClaimTracker } from "../claim-tracker";
+import { taskIdFromMessage } from "../runner-helpers";
+
+interface ActiveTaskClaim {
+  messageId: number;
+  taskId: number;
+  observedProviderBusy: boolean;
+  completionQueued?: boolean;
+}
+
+const CLAIM_RENEW_INTERVAL_MS = 5 * 60 * 1000;
+
+interface TaskClaimsDeps {
+  http: Pick<HumwickHttpClient, "renewMessageClaim">;
+  agentId: string;
+  startClaim: ClaimTracker["startClaim"];
+  finishClaim: ClaimTracker["finishClaim"];
+  isDrainingMessage(messageId: number): boolean;
+  updateTaskStatus(taskId: number, input: TaskStatusInput): Promise<void>;
+  publishStatus(): void;
+  enqueueCompletion: Outbox["enqueue"];
+}
+
+export class TaskClaims {
+  private claimRenewTimer?: Timer;
+  private readonly activeTaskClaims = new Map<number, ActiveTaskClaim>();
+  // #2619 — claimed Task messages whose provider-turn idle arrived while their own batch was still
+  // delivering; completed at settle only if the provider accepted them (see completeObservedTaskClaims).
+  private readonly completionDeferredTaskClaims = new Set<number>();
+
+  constructor(private readonly deps: TaskClaimsDeps) {}
+
+  stop(): void {
+    if (this.claimRenewTimer) clearInterval(this.claimRenewTimer);
+    this.claimRenewTimer = undefined;
+  }
+
+  noteClaim(messageId: number, taskId: number, providerAlreadyBusy: boolean, claimExpiresAt?: number): void {
+    this.activeTaskClaims.set(messageId, { messageId, taskId, observedProviderBusy: providerAlreadyBusy });
+    this.deps.startClaim("task", String(taskId), claimExpiresAt);
+  }
+
+  observeProviderBusy(): void {
+    for (const claim of this.activeTaskClaims.values()) claim.observedProviderBusy = true;
+  }
+
+  hasDeferredCompletion(): boolean { return this.completionDeferredTaskClaims.size > 0; }
+  forgetDeferredCompletion(messageId: number): void { this.completionDeferredTaskClaims.delete(messageId); }
+
+  start(): void {
+    if (this.claimRenewTimer) return;
+    this.claimRenewTimer = setInterval(() => {
+      void this.renewActiveTaskClaims();
+    }, CLAIM_RENEW_INTERVAL_MS);
+  }
+
+  private async renewActiveTaskClaims(): Promise<void> {
+    for (const claim of this.activeTaskClaims.values()) {
+      const renewed = await this.deps.http.renewMessageClaim(claim.messageId, this.deps.agentId).catch(() => ({ ok: false, claimExpiresAt: undefined }));
+      if (renewed.ok) {
+        this.deps.startClaim("message", String(claim.messageId), renewed.claimExpiresAt);
+        this.deps.startClaim("task", String(claim.taskId), renewed.claimExpiresAt);
+      }
+    }
+    this.deps.publishStatus();
+  }
+
+  async completeObservedTaskClaims(acceptedIds?: ReadonlySet<number>): Promise<void> {
+    const completed = [...this.activeTaskClaims.values()].filter((claim) => {
+      // Settle-time pass: complete only the deferred claims whose message the provider ACCEPTED.
+      if (claim.completionQueued) return false;
+      if (acceptedIds) return acceptedIds.has(claim.messageId) && this.completionDeferredTaskClaims.delete(claim.messageId);
+      if (!claim.observedProviderBusy) return false;
+      // #2619 — a claim still inside its own delivery batch: an idle here may belong to a SIBLING
+      // message's turn, and this one may yet be refused. Whether it was accepted is only known at
+      // settle (settleMessageDelivery), so defer instead of marking its Task done now.
+      if (this.deps.isDrainingMessage(claim.messageId)) {
+        this.completionDeferredTaskClaims.add(claim.messageId);
+        return false;
+      }
+      return true;
+    });
+    for (const claim of completed) {
+      // Persist the observed completion before yielding; read/delivery state alone cannot prove it.
+      this.deps.enqueueCompletion({
+        kind: "task-completion",
+        payload: { taskId: claim.taskId, messageId: claim.messageId },
+      });
+      claim.completionQueued = true;
+    }
+  }
+
+  async deliverCompletion(record: OutboxRecord): Promise<void> {
+    const { taskId, messageId } = record.payload as { taskId: number; messageId: number };
+    await this.deps.updateTaskStatus(taskId, {
+      status: "done",
+      agentId: this.deps.agentId,
+      body: `Runner observed provider completion for message #${messageId}`,
+      metadata: { messageId, completedBy: "runner" },
+    });
+    this.activeTaskClaims.delete(messageId);
+    this.deps.finishClaim("message", String(messageId));
+    this.deps.finishClaim("task", String(taskId));
+    this.deps.publishStatus();
+  }
+
+  clearActiveClaim(message: Message): void {
+    const taskId = taskIdFromMessage(message);
+    this.deps.finishClaim("message", String(message.id));
+    if (!taskId) return;
+    this.activeTaskClaims.delete(message.id);
+    this.deps.finishClaim("task", String(taskId));
+  }
+}

package/src/agent-runner/token-usage-coverage.ts

@@ -0,0 +1,54 @@
+// #3351 — the runner's view of provider usage it has observed but Relay has not stored yet.
+// Every adapter reports through this one seam, so a report that is in flight, or that failed
+// and was not yet replayed under the same dedupe key, is published as unreported spend
+// instead of looking like confirmed zero usage.
+import type { ProviderTokenUsageRecord } from "../adapter";
+
+export class TokenUsageReportCoverage {
+  private readonly inFlight = new Map<string, number>();
+  private readonly failed = new Set<string>();
+  private readonly captures = new Map<string, object>();
+
+  constructor(private readonly onChange: () => void) {}
+
+  /** Distinct reports not yet acknowledged by Relay. */
+  get unreported(): number {
+    let count = this.failed.size + this.captures.size;
+    for (const key of this.inFlight.keys()) if (!this.failed.has(key)) count++;
+    return count;
+  }
+
+  async capture(key: string, read: () => Promise<number>, pending = true): Promise<void> {
+    if (!pending && !this.captures.has(key)) { await read(); return; }
+    const generation = {};
+    const before = this.unreported;
+    this.captures.set(key, generation);
+    if (this.unreported !== before) this.onChange();
+    const reported = await read();
+    if (reported > 0 && this.captures.get(key) === generation) {
+      this.captures.delete(key);
+      this.onChange();
+    }
+  }
+
+  async report(record: ProviderTokenUsageRecord, send: (record: ProviderTokenUsageRecord) => Promise<unknown>): Promise<unknown> {
+    const key = record.dedupeKey;
+    const before = this.unreported;
+    this.inFlight.set(key, (this.inFlight.get(key) ?? 0) + 1);
+    if (this.unreported !== before) this.onChange();
+    let acknowledged = false;
+    try {
+      const result = await send(record);
+      acknowledged = true;
+      return result;
+    } finally {
+      const settledBefore = this.unreported;
+      const remaining = (this.inFlight.get(key) ?? 1) - 1;
+      if (remaining > 0) this.inFlight.set(key, remaining);
+      else this.inFlight.delete(key);
+      if (acknowledged) this.failed.delete(key);
+      else this.failed.add(key);
+      if (this.unreported !== settledBefore) this.onChange();
+    }
+  }
+}