@agentswf/engine 0.0.0-stage → 0.1.0-beta.1

package/src/run-usage.ts

@@ -0,0 +1,393 @@
+import type { Billing, SettledOperation } from "@agentswf/contract/records";
+import type {
+  AbsoluteDeadline,
+  AgentExecution,
+  AgentKey,
+  NativeSessionRef,
+  OperationRecord,
+} from "@agentswf/contract/workflow";
+import type { SessionAccounting, SessionRead, UsageRecord } from "@agentswf/harness";
+import type { HarnessTurnOutcome } from "@agentswf/harness/adapter";
+import { spendOf } from "./accounting/tokens";
+import { abortableSleep, deadlineWithin, runUntilStopped, waitForDeadline } from "./deadlines";
+
+/**
+ * How long a finished run waits for turns still ending after their answer. Closing the host kills
+ * them, and a request cut off that way is never logged; a closing message takes a few seconds.
+ * It bounds the run's end, so it is shorter than the host's own grace for a turn left finishing
+ * while the run goes on.
+ */
+const FINISHING_GRACE_MILLISECONDS = 10_000;
+/** Reading spend waits for files a released turn is still writing; it gives up after this. */
+const ACCOUNTING_GRACE_MILLISECONDS = 20_000;
+/** Beyond that, the host's bound on a billing status command, and this much more. */
+const STATUS_SLACK_MILLISECONDS = 2_000;
+
+/** One logical agent, as the run knows it. */
+export type AccountedAgent = {
+  key: AgentKey;
+  execution: AgentExecution;
+  /** Where it ran; a sandboxed agent's is set once its sandbox resolves it. */
+  cwd: string;
+  /** A sandboxed agent's own harness home, where its sessions are. */
+  home?: string;
+  /** Every native session id seen for the agent so far, from its `wf` calls and its adapter. */
+  sessions(): readonly string[];
+  /** When its host first prompted it, where that waits on the agent first; see `window`. */
+  promptedAt?(): number | undefined;
+  /** The caller handed back early by agents.stop. */
+  releasedAt?(): number | undefined;
+  /** When a fork's session was made: what it holds from before is its parent's (ADR 0009). */
+  forkedAt?(): number | undefined;
+};
+
+/** One operation's place in the run, held from before its dispatch so records keep that order. */
+type OperationEntry = {
+  /**
+   * Records what is known when the operation settles; spend and billing wait for the run's end.
+   * `finishing` is an answered turn the host left finishing, whose charges are added when it ends.
+   */
+  settle(
+    times: { deliveredAt?: number; settledAt: number },
+    charges: readonly number[],
+    finishing?: Promise<HarnessTurnOutcome>,
+  ): OperationRecord;
+};
+
+/**
+ * Where an operation ran: its workflow stage, absent between stages, its turn's label, and what
+ * caused it when the workflow did not.
+ */
+export type OperationTags = Pick<OperationRecord, "stage" | "label" | "cause">;
+
+/** One agent's part of the run's ledger. */
+export type AgentLedger = {
+  /** `execution` is read as the operation settles: the agent's settings it ran at. */
+  reserve(
+    operationId: string,
+    execution: () => AgentExecution,
+    tags?: OperationTags,
+  ): OperationEntry;
+  /** An operation's record before it settles, with the agent's sessions as of now. */
+  unsettled(
+    operationId: string,
+    execution: AgentExecution,
+    tags: OperationTags,
+    deliveredAt?: number,
+  ): Omit<OperationRecord, "settledAt">;
+};
+
+export type RunLedger = {
+  /** Called in open order, so which agent keeps a request two of them report does not vary. */
+  agent(agent: AccountedAgent): AgentLedger;
+  /** Every settled operation, with the sessions its agent has been seen with so far. */
+  records(): OperationRecord[];
+  /** Best effort: a turn still going at the deadline, or when the run is stopped, is killed. */
+  letFinish(runDeadline: AbsoluteDeadline, signal: AbortSignal): Promise<void>;
+  /**
+   * Every record, completed with what its agent spent. Called once the host has closed, because a
+   * released turn still writes its last request after the engine has moved on (story 002,
+   * experiment 14). Best effort, and after the run's own deadline: a read or status command that
+   * fails, overruns its grace or is stopped leaves each record as it settled.
+   */
+  settle(signal: AbortSignal): Promise<SettledOperation[]>;
+};
+
+type Operation = {
+  agent: AccountedAgent;
+  usage?: Omit<OperationRecord, "sessions">;
+  costs: number[];
+};
+type Settled = Operation & { usage: Omit<OperationRecord, "sessions"> };
+
+export function createRunLedger({
+  accounting,
+  startedAt,
+}: {
+  accounting: SessionAccounting | undefined;
+  startedAt: number;
+}): RunLedger {
+  const agents: AccountedAgent[] = [];
+  const operations: Operation[] = [];
+  const late = new Set<Promise<void>>();
+  const settled = () =>
+    operations.filter((operation): operation is Settled => operation.usage !== undefined);
+  /** Until no turn is left finishing, including one that began finishing during the wait. */
+  const lateEnded = async () => {
+    let waited = 0;
+    while (late.size > waited) {
+      waited = late.size;
+      await Promise.allSettled([...late]);
+    }
+  };
+
+  return {
+    agent(agent) {
+      agents.push(agent);
+      return {
+        reserve(operationId, execution, tags = {}) {
+          const operation: Operation = { agent, costs: [] };
+          operations.push(operation);
+          return {
+            settle(times, charges, finishing) {
+              operation.usage = {
+                ...recordOf(agent, operationId, execution(), tags, times.deliveredAt),
+                settledAt: new Date(times.settledAt).toISOString(),
+              };
+              operation.costs = [...charges];
+              if (finishing) {
+                late.add(
+                  finishing.then(
+                    ({ chargesUsd }) => {
+                      operation.costs.push(...chargesUsd);
+                    },
+                    () => undefined,
+                  ),
+                );
+              }
+              return withSessions(operation as Settled);
+            },
+          };
+        },
+        unsettled(operationId, execution, tags, deliveredAt) {
+          const record = recordOf(agent, operationId, execution, tags, deliveredAt);
+          return { ...record, sessions: sessionsOf(agent, execution) };
+        },
+      };
+    },
+
+    records: () => settled().map(withSessions),
+
+    async letFinish(runDeadline, signal) {
+      await runUntilStopped(
+        lateEnded,
+        signal,
+        deadlineWithin(FINISHING_GRACE_MILLISECONDS, runDeadline),
+      ).catch(() => undefined);
+    },
+
+    async settle(signal) {
+      const endedAt = Date.now();
+      const deadline = endedAt + ACCOUNTING_GRACE_MILLISECONDS;
+      // The bound below answers without stopping the work; this stops it, so no status command
+      // starts once the records are returned.
+      const settling = new AbortController();
+      try {
+        return await runUntilStopped(
+          async () => {
+            // Closing the host ended every turn left finishing; what each charged lands here.
+            await waitForDeadline(lateEnded(), { unixMilliseconds: deadline }).catch(
+              () => undefined,
+            );
+            return await settleUsage(settled(), agents, accounting, {
+              startedAt,
+              endedAt,
+              deadline,
+              signal: AbortSignal.any([signal, settling.signal]),
+            });
+          },
+          signal,
+          // The grace paces the re-reads; this bounds a read or status command that never returns.
+          { unixMilliseconds: deadline + (accounting?.statusMs ?? 0) + STATUS_SLACK_MILLISECONDS },
+        );
+      } catch {
+        return unread(settled());
+      } finally {
+        settling.abort();
+      }
+    },
+  };
+}
+
+/**
+ * Reads what every agent spent and gives each operation its share. While any session's turn is
+ * still open the files are read again, until that turn closes, it stops changing for `stalledMs`,
+ * `deadline` passes or `signal` aborts. Whatever the last read found stands.
+ */
+async function settleUsage(
+  operations: readonly Settled[],
+  agents: readonly AccountedAgent[],
+  accounting: SessionAccounting | undefined,
+  options: { startedAt: number; endedAt: number; deadline: number; signal: AbortSignal },
+): Promise<SettledOperation[]> {
+  if (!accounting) return unread(operations);
+  // An agent opened but never asked anything is not read, so it cannot claim another's requests.
+  const asked = agents.flatMap((agent) => {
+    const own = operations.filter((operation) => operation.agent === agent);
+    return own.length === 0 ? [] : [{ agent, own }];
+  });
+
+  const reads = await readUntilSettled(
+    asked.map(({ agent }) => agent),
+    accounting,
+    options,
+  );
+  // The caller has already answered with the records as they settled; no status command starts.
+  if (options.signal.aborted) return unread(operations);
+  const claimed = new Set<string>();
+  const counted = new Map<AccountedAgent, UsageRecord[] | undefined>();
+  for (const { agent } of asked) {
+    const { from, until } = window(agent, options);
+    counted.set(
+      agent,
+      reads.get(agent)?.records.filter((record) => {
+        const claim = `${agent.execution.harness}\u0000${record.key}`;
+        const at = Date.parse(record.at);
+        // A resumed session carries requests from before the run, and two agents can name one.
+        if (claimed.has(claim) || !(at >= from && at <= until)) return false;
+        claimed.add(claim);
+        return true;
+      }),
+    );
+  }
+  const settledAgents = new Map(
+    await Promise.all(
+      asked.map(async ({ agent, own }) => {
+        const records = counted.get(agent);
+        const billing: Billing =
+          (await attempt(() => accounting.billing(agent.execution, records))) ?? "unknown";
+        return [agent, { billing, shares: records && share(own, records) }] as const;
+      }),
+    ),
+  );
+
+  return operations.map((operation) => {
+    const { billing, shares } = settledAgents.get(operation.agent)!;
+    const { costs } = operation;
+    const charged = billing === "metered" && costs.length > 0 ? sum(costs) : undefined;
+    return {
+      ...withSessions(operation),
+      billing,
+      ...(shares ? { spend: spendOf(shares.get(operation.usage.operationId) ?? []) } : {}),
+      ...(charged === undefined ? {} : { charged: { amount: charged, currency: "USD" } }),
+    };
+  });
+}
+
+/**
+ * When an agent's records are the run's. A calling session's are from the run's first prompt to it
+ * until the run's own work ended: before, its turns were the operator's, the one that replied with
+ * the run's code included, and so are its turns after the hand-back (ADR 0010). That prompt is the
+ * host's, not the operation's delivery, which comes before the host waits for the operator's turn.
+ */
+function window(
+  agent: AccountedAgent,
+  options: { startedAt: number; endedAt: number },
+): { from: number; until: number } {
+  // Copied rows are its parent's even when the parent does not claim them: a calling session
+  // never asked anything, or asked only after them.
+  const forkedAt = agent.forkedAt?.();
+  if (forkedAt !== undefined) return { from: forkedAt, until: Number.POSITIVE_INFINITY };
+  if (!agent.execution.caller) return { from: options.startedAt, until: Number.POSITIVE_INFINITY };
+  return {
+    from: agent.promptedAt?.() ?? Number.POSITIVE_INFINITY,
+    until: agent.releasedAt?.() ?? options.endedAt,
+  };
+}
+
+async function readUntilSettled(
+  agents: readonly AccountedAgent[],
+  accounting: SessionAccounting,
+  options: { deadline: number; signal: AbortSignal },
+): Promise<Map<AccountedAgent, SessionRead | undefined>> {
+  const readAll = async () =>
+    new Map(
+      await Promise.all(
+        agents.map(async (agent) => {
+          const ids = agent.sessions();
+          const read =
+            // A sandboxed agent's home names its sessions even when nothing else did.
+            ids.length === 0 && !agent.home
+              ? undefined
+              : await attempt(() => accounting.read(agent.execution, ids, agent.cwd, agent.home));
+          return [agent, read] as const;
+        }),
+      ),
+    );
+  let latest = await readAll();
+  let changedAt = Date.now();
+  while (
+    [...latest.values()].some((read) => read?.open) &&
+    Date.now() - changedAt < accounting.stalledMs &&
+    Date.now() + accounting.pollMs < options.deadline &&
+    !options.signal.aborted
+  ) {
+    await abortableSleep(accounting.pollMs, options.signal);
+    const next = await readAll();
+    if (JSON.stringify([...next.values()]) !== JSON.stringify([...latest.values()])) {
+      changedAt = Date.now();
+    }
+    latest = next;
+  }
+  return latest;
+}
+
+/**
+ * An operation owns what its agent logged from its delivery until the next operation's. What came
+ * before the first delivery is the first operation's, and so is everything when none was
+ * delivered: a share never drops a record.
+ */
+function share(
+  operations: readonly Settled[],
+  records: readonly UsageRecord[],
+): Map<string, UsageRecord[]> {
+  const delivered = operations
+    .flatMap(({ usage }) =>
+      usage.deliveredAt === undefined
+        ? []
+        : [{ id: usage.operationId, from: Date.parse(usage.deliveredAt) }],
+    )
+    .sort((left, right) => left.from - right.from);
+  const first = delivered[0]?.id ?? operations[0]!.usage.operationId;
+  const shares = new Map<string, UsageRecord[]>();
+  for (const record of records) {
+    const at = Date.parse(record.at);
+    const owner = delivered.findLast((operation) => operation.from <= at)?.id ?? first;
+    shares.set(owner, [...(shares.get(owner) ?? []), record]);
+  }
+  return shares;
+}
+
+function recordOf(
+  agent: AccountedAgent,
+  operationId: string,
+  execution: AgentExecution,
+  tags: OperationTags,
+  deliveredAt: number | undefined,
+): Omit<OperationRecord, "sessions" | "settledAt"> {
+  return {
+    callPath: [],
+    agent: agent.key,
+    operationId,
+    execution,
+    ...(deliveredAt === undefined ? {} : { deliveredAt: new Date(deliveredAt).toISOString() }),
+    ...(tags.stage === undefined ? {} : { stage: tags.stage }),
+    ...(tags.label === undefined ? {} : { label: tags.label }),
+    ...(tags.cause === undefined ? {} : { cause: tags.cause }),
+  };
+}
+
+function sessionsOf(agent: AccountedAgent, execution: AgentExecution): NativeSessionRef[] {
+  return agent.sessions().map((id) => ({ harness: execution.harness, id }));
+}
+
+function withSessions({ usage, agent }: Settled): OperationRecord {
+  return { ...usage, sessions: sessionsOf(agent, usage.execution) };
+}
+
+function unread(operations: readonly Settled[]): SettledOperation[] {
+  return operations.map((operation) => ({ ...withSessions(operation), billing: "unknown" }));
+}
+
+/** A reader or status command is another package's code: a throw of any kind is "unknown". */
+async function attempt<T>(call: () => Promise<T>): Promise<T | undefined> {
+  try {
+    return await call();
+  } catch {
+    return undefined;
+  }
+}
+
+function sum(values: readonly number[]): number {
+  return values.reduce((total, value) => total + value, 0);
+}

package/src/run-view.ts

@@ -0,0 +1,225 @@
+import { join } from "node:path";
+import type {
+  Activity,
+  AttemptRecord,
+  OperationLivenessRecord,
+  RunLogLine,
+} from "@agentswf/contract/records";
+import type { AgentKey, HarnessKind, StageOutcome } from "@agentswf/contract/workflow";
+import { readLines } from "./jsonl";
+import { currentStage, estimateOf } from "./run-commands";
+import { type Cursor, type FileCache, readRun, timeOf } from "./run-read";
+import type { RunStatus } from "./run-state";
+
+/** A stage of the attempt shown: running, reused or given from an earlier one, or how it ended. */
+export type StageView = {
+  stage: string;
+  state: "running" | "interrupted" | "reused" | "provided" | StageOutcome;
+  startedAt?: number;
+  endedAt?: number;
+};
+
+/**
+ * An agent of the attempt shown, as its operations leave it: working on one, or its last one's
+ * outcome. What its liveness file and native files say of it is best effort, and kept apart.
+ */
+export type AgentView = {
+  key: AgentKey;
+  harness?: HarnessKind;
+  caller: boolean;
+  /** The stage its last operation ran in. */
+  stage?: string;
+  state: string;
+  turns: number;
+  /** When its current operation, or its last, started. */
+  since?: number;
+  /** Its operation's latest phase, where it is waiting within it, or awaiting `peer`'s reply. */
+  live?:
+    | { phase: "waiting"; reason?: string; until?: number }
+    | { phase: "awaiting-reply"; peer: string; until?: number };
+  last?: { activity: Activity; at: string };
+  sessions: { harness: HarnessKind; id: string; files: string[] }[];
+};
+
+export type DecisionView = { decision: number; key: string; askedAt: number; outcome?: string };
+
+export type RunView = {
+  status: RunStatus;
+  /** The attempt shown: the last. */
+  attempt?: AttemptRecord;
+  stages: StageView[];
+  /** The stage running now, or the one the attempt ended in. */
+  stage?: string;
+  agents: AgentView[];
+  decisions: DecisionView[];
+  /** Over the attempts that kept one, at list prices: never what was billed. */
+  estimate?: number;
+  lastEventAt?: number;
+  lines: { number: number; line: RunLogLine }[];
+  cursor: Cursor;
+};
+
+export async function viewRun(status: RunStatus): Promise<RunView> {
+  const { run, attempts } = status;
+  const attempt = attempts.at(-1);
+  const files: FileCache = new Map();
+  const read = await readRun(run.dir, run.record.cwd, undefined, {
+    ended: status.state !== "running",
+    files,
+    tail: true,
+  });
+  const own = read.log.lines
+    .map(({ line }) => line)
+    .filter((line) => attempt === undefined || line.attempt === attempt.attempt);
+
+  const stages: StageView[] = [];
+  const decisions: DecisionView[] = [];
+  const agents = new Map<AgentKey, AgentView>();
+  const open = new Map<string, AgentKey>();
+  const agent = (key: AgentKey) => {
+    const found = agents.get(key) ?? {
+      key,
+      caller: read.log.agents.get(key)?.caller ?? false,
+      state: "idle",
+      turns: 0,
+      sessions: [],
+    };
+    agents.set(key, found);
+    return found;
+  };
+  for (const line of own) {
+    const at = Date.parse(timeOf(line));
+    switch (line.kind) {
+      case "stage-started":
+        stages.push({ stage: line.stage, state: "running", startedAt: at });
+        break;
+      case "stage-ended": {
+        const started = stages.find((each) => each.stage === line.stage);
+        const state = line.source ?? line.outcome;
+        if (started) Object.assign(started, { state, endedAt: at });
+        else stages.push({ stage: line.stage, state, startedAt: at, endedAt: at });
+        break;
+      }
+      case "agent-opened":
+        agent(line.agent).harness = line.harness;
+        break;
+      case "operation-started": {
+        const started = agent(line.agent);
+        Object.assign(started, { state: "working", since: at, harness: line.execution.harness });
+        if (line.stage !== undefined) started.stage = line.stage;
+        open.set(line.operationId, line.agent);
+        break;
+      }
+      case "turn":
+      case "compact":
+      case "operation-failed": {
+        const settled = agent(line.agent);
+        open.delete(line.operationId);
+        if (line.stage !== undefined) settled.stage = line.stage;
+        settled.harness ??= line.execution.harness;
+        settled.state = line.kind === "operation-failed" ? "failed" : line.outcome;
+        if (line.kind === "turn") settled.turns += 1;
+        break;
+      }
+      case "agent-stopped":
+        if (agent(line.agent).state === "working") agent(line.agent).state = "stopped";
+        break;
+      case "decision-asked":
+        decisions.push({ decision: line.decision, key: line.key, askedAt: at });
+        break;
+      case "decision-ended": {
+        const asked = decisions.find((each) => each.decision === line.decision);
+        if (asked) asked.outcome = line.outcome;
+        break;
+      }
+    }
+  }
+  const stage = currentStage(status, read.log.lines);
+  // A run whose process is gone left what it was doing undone.
+  if (status.state === "interrupted") {
+    for (const each of agents.values()) if (each.state === "working") each.state = "was working";
+    for (const each of stages) if (each.state === "running") each.state = "interrupted";
+  }
+  if (status.state === "running") {
+    for (const [id, key] of open) {
+      const phase = await latestPhase(run.dir, id);
+      const until = phase?.until === undefined ? {} : { until: phase.until };
+      if (phase?.kind === "waiting") {
+        agent(key).live = {
+          phase: "waiting",
+          ...(phase.reason === undefined ? {} : { reason: phase.reason }),
+          ...until,
+        };
+      } else if (phase?.kind === "awaiting-reply" && phase.peer !== undefined) {
+        agent(key).live = { phase: "awaiting-reply", peer: phase.peer, ...until };
+      }
+    }
+  }
+  const since = attempt ? Date.parse(attempt.started) : Number.NEGATIVE_INFINITY;
+  for (const event of read.events) {
+    if (event.source !== "activity" || !event.agent || event.time < since) continue;
+    const seen = agents.get(event.agent);
+    if (seen) seen.last = { activity: event.activity, at: event.at };
+  }
+  for (const session of read.log.sessions) {
+    const seen = session.agent ? agents.get(session.agent) : undefined;
+    if (!seen) continue;
+    seen.sessions.push({
+      harness: session.harness,
+      id: session.id,
+      files: (files.get(session.n)?.files ?? []).map((file) => file.path),
+    });
+  }
+  const estimate = estimateOf(attempts);
+  const lastLine = read.log.lines.at(-1)?.line;
+  return {
+    status,
+    ...(attempt ? { attempt } : {}),
+    stages,
+    ...(stage === undefined ? {} : { stage }),
+    agents: [...agents.values()],
+    decisions,
+    ...(estimate === undefined ? {} : { estimate }),
+    ...(lastLine ? { lastEventAt: Date.parse(timeOf(lastLine)) } : {}),
+    lines: read.log.lines,
+    cursor: read.cursor,
+  };
+}
+
+/** The latest phase a call's liveness file holds, which is diagnostics, best effort. */
+async function latestPhase(
+  runDir: string,
+  operationId: string,
+): Promise<OperationLivenessRecord | undefined> {
+  const file = join(runDir, "calls", operationId, "liveness.jsonl");
+  return (await readLines<OperationLivenessRecord>(file).catch(() => [])).at(-1);
+}
+
+/** `45s`, `6m40s`, `1h02m`. */
+export function duration(milliseconds: number): string {
+  const seconds = Math.max(0, Math.round(milliseconds / 1000));
+  if (seconds < 60) return `${seconds}s`;
+  const minutes = Math.floor(seconds / 60);
+  if (minutes < 60) return `${minutes}m${String(seconds % 60).padStart(2, "0")}s`;
+  return `${Math.floor(minutes / 60)}h${String(minutes % 60).padStart(2, "0")}m`;
+}
+
+/** An activity in a few words: `tool Bash bun test → exit 0`, `message 412 chars`. */
+export function describeActivity(activity: Activity): string {
+  switch (activity.type) {
+    case "message":
+      return `${activity.from === "user" ? "prompt" : "message"}  ${activity.chars.toLocaleString("en-US")} chars`;
+    case "tool":
+      return `tool ${activity.tool}${activity.input ? `  ${activity.input}` : ""}${activity.background ? "  (background)" : ""}`;
+    case "tool-result":
+      return activity.exit !== undefined
+        ? `→ exit ${activity.exit}`
+        : activity.error
+          ? "→ error"
+          : "→ ok";
+    case "background-ended":
+      return "background task ended";
+    case "compaction":
+      return "compacted";
+  }
+}