@agentswf/engine 0.0.0-stage → 0.1.0-beta.1

package/src/run-commands.ts

@@ -0,0 +1,215 @@
+import { resolve } from "node:path";
+import type { AttemptRecord, RunLogLine } from "@agentswf/contract/records";
+import { maskCredentials } from "@agentswf/harness";
+import { continueCommand, shellWord } from "./attempt-view";
+import type { RunCommand } from "./run-command";
+import { type Cursor, encodeCursor } from "./run-read";
+import { RunNotFound, type RunStatus } from "./run-state";
+import { runRootOf } from "./runs";
+
+/** What `awf runs` and `awf logs` share: where they look, and how they say a run. */
+
+/** Where the commands that read runs look, and the options their suggested commands repeat. */
+export type ReadPlace = { shellCwd: string; runRoot: string; repeat: string[] };
+
+export type Out = { stdout(text: string): void; stderr(text: string): void };
+
+/** `--cwd`, `--run-root` and `--json`, taken out of `argv`; the rest is left for the command. */
+export function parsePlace(
+  argv: readonly string[],
+  shellCwd: string,
+): { place: ReadPlace; json: boolean; rest: string[] } {
+  let cwd = shellCwd;
+  let runRoot: string | undefined;
+  let json = false;
+  const repeat: string[] = [];
+  const rest: string[] = [];
+  for (let index = 0; index < argv.length; index += 1) {
+    const arg = argv[index]!;
+    if (arg === "--json") json = true;
+    else if (arg === "--cwd" || arg === "--run-root") {
+      const value = argv[index + 1];
+      if (!value || value.startsWith("--")) throw new Error(`${arg} needs a directory`);
+      index += 1;
+      if (arg === "--cwd") cwd = resolve(shellCwd, value);
+      else runRoot = resolve(shellCwd, value);
+      repeat.push(arg, value);
+    } else rest.push(arg);
+  }
+  return { place: { shellCwd, runRoot: runRoot ?? runRootOf(cwd), repeat }, json, rest };
+}
+
+/** A suggested command, with the place repeated. */
+export function command(place: ReadPlace, ...words: string[]): string[] {
+  const [first, second, ...rest] = words;
+  return [first!, second!, ...rest.slice(0, 1), ...place.repeat, ...rest.slice(1)];
+}
+
+/** A field whose name says it holds a credential, whatever its value looks like. */
+const SECRET_NAME =
+  /password|passwd|passphrase|passcode|secret|token|credential|cookie|bearer|authorization|(?:api|access|private|session|ssh|signing|encryption)[_-]?key|^(?:pwd|pass|pin|auth)$/i;
+
+/**
+ * Every string in `value` masked, and everything under a credential's name hidden: a run's text,
+ * errors and fields are its workflow's, its agents' and its tools'. Past `limit`, a string is cut.
+ */
+export function masked(value: unknown, limit = Number.POSITIVE_INFINITY, hidden = false): unknown {
+  if (typeof value === "string") {
+    if (hidden) return "***";
+    const shown = maskCredentials(value);
+    return shown.length > limit ? `${shown.slice(0, limit - 1)}…` : shown;
+  }
+  if (hidden && value !== null && typeof value !== "object") return "***";
+  if (Array.isArray(value)) return value.map((each) => masked(each, limit, hidden));
+  if (value && typeof value === "object") {
+    return Object.fromEntries(
+      Object.entries(value).map(([key, each]) => [
+        key,
+        masked(each, limit, hidden || SECRET_NAME.test(key)),
+      ]),
+    );
+  }
+  return value;
+}
+
+/**
+ * Text a run wrote, as one masked line of about `limit` characters, so it can never pass for a
+ * line of its own or fill a page. `@{handle} --full` prints it whole.
+ */
+export function said(text: string, limit = 120): string {
+  const line = maskCredentials(text).replace(/\s*[\r\n]+\s*/g, " ⏎ ");
+  return line.length > limit ? `${line.slice(0, limit - 1)}…` : line;
+}
+
+/** A suggested command as text to paste into a shell. */
+export function shellLine(words: readonly string[]): string {
+  return words.map(shellWord).join(" ");
+}
+
+/** Says a run that is not there, and exits 1; anything else is the run's files, exit 3. */
+export function failed(error: unknown, place: ReadPlace, out: Out, message: string): number {
+  if (error instanceof RunNotFound) {
+    out.stderr(`${error.message}; runs here: ${shellLine(command(place, "awf", "runs"))}`);
+    return 1;
+  }
+  out.stderr(`awf: ${message}`);
+  return 3;
+}
+
+/**
+ * What `awf run` says first where no person watches its progress: the run's reference, the one
+ * route a calling agent has to its own run. Its prefix, `awf: {workflow}/{id} started`, is stable.
+ */
+export function startedLine(
+  workflow: string,
+  id: string,
+  run: Pick<RunCommand, "cwd" | "cwdGiven" | "runRoot" | "runRootGiven">,
+): string {
+  const ref = `${workflow}/${id}`;
+  const repeat = [
+    ...(run.cwdGiven ? ["--cwd", run.cwd] : []),
+    ...(run.runRootGiven ? ["--run-root", run.runRoot] : []),
+  ];
+  return `awf: ${ref} started · watch: ${shellLine(["awf", "logs", ref, ...repeat])}`;
+}
+
+/** `awf logs` from `cursor`: with `--wait` unless the page stopped short of the newest. */
+export function nextLogs(
+  status: RunStatus,
+  place: ReadPlace,
+  cursor: Cursor,
+  filters: string[] = [],
+  wait = true,
+): string[] {
+  return [
+    ...command(place, "awf", "logs", status.ref),
+    ...filters,
+    "--after",
+    encodeCursor(cursor),
+    ...(wait ? ["--wait", "90"] : []),
+  ];
+}
+
+/** The command an interrupted run goes on with, as `awf run` would print it. */
+export function continueOf(
+  status: RunStatus,
+  lines: readonly { line: RunLogLine }[],
+  place: ReadPlace,
+): string | undefined {
+  const { run } = status;
+  const last = status.attempts.at(-1);
+  if (!last) return undefined;
+  const entered = lines.flatMap(({ line }) =>
+    line.attempt === last.attempt &&
+    (line.kind === "stage-started" || (line.kind === "stage-ended" && line.source))
+      ? [{ stage: line.stage }]
+      : [],
+  );
+  return continueCommand(
+    {
+      shellCwd: place.shellCwd,
+      cwd: run.record.cwd,
+      cwdGiven: resolve(run.record.cwd) !== resolve(place.shellCwd),
+      runRoot: run.root,
+      runRootGiven: resolve(run.root) !== resolve(runRootOf(run.record.cwd)),
+      workflowFile: last.file,
+      ...(last.flags.fromStage === undefined ? {} : { fromStage: last.flags.fromStage }),
+    },
+    run.record.id,
+    undefined,
+    entered,
+  );
+}
+
+/** The stage the last attempt is in: open, or the one it ended in. */
+export function currentStage(
+  status: RunStatus,
+  lines: readonly { line: RunLogLine }[],
+): string | undefined {
+  const last = status.attempts.at(-1);
+  if (!last) return undefined;
+  const open = new Set<string>();
+  for (const { line } of lines) {
+    if (line.attempt !== last.attempt) continue;
+    if (line.kind === "stage-started") open.add(line.stage);
+    if (line.kind === "stage-ended") open.delete(line.stage);
+  }
+  return [...open].at(-1) ?? last.stage;
+}
+
+/** `running`, or `running (unchecked, last event 09:41:07)` where its process could not be asked. */
+export function stateWord(status: RunStatus, lastEventAt?: number): string {
+  if (status.state !== "running" || status.checked) return status.state;
+  return `running (unchecked${lastEventAt === undefined ? "" : `, last event ${clockTime(lastEventAt)}`})`;
+}
+
+/** From the first attempt's start to the last's end, or to now while it runs. */
+export function span(
+  status: RunStatus,
+  now: number,
+): { started: number; ended?: number; ms: number } {
+  const started = Date.parse(status.attempts[0]?.started ?? status.run.record.created);
+  const last = status.attempts.at(-1);
+  const ended = last?.ended ? Date.parse(last.ended) : undefined;
+  return { started, ...(ended === undefined ? {} : { ended }), ms: (ended ?? now) - started };
+}
+
+/** At list prices, over the attempts that kept one: never what was billed. */
+export function estimateOf(attempts: readonly AttemptRecord[]): number | undefined {
+  const known = attempts.flatMap((each) =>
+    each.accounting?.totals.estimate === undefined ? [] : [each.accounting.totals.estimate],
+  );
+  return known.length > 0 ? known.reduce((total, each) => total + each, 0) : undefined;
+}
+
+/** `$6.10`; `<$0.01` for less, and nothing for none. */
+export function money(estimate: number | undefined): string[] {
+  if (!estimate) return [];
+  return [estimate < 0.005 ? "<$0.01" : `$${estimate.toFixed(2)}`];
+}
+
+export function clockTime(at: number | string): string {
+  const date = new Date(at);
+  const two = (n: number) => String(n).padStart(2, "0");
+  return `${two(date.getHours())}:${two(date.getMinutes())}:${two(date.getSeconds())}`;
+}

package/src/run-event-log.ts

@@ -0,0 +1,72 @@
+import { RUN_EVENT_VERSIONS, type RunEvent, type TurnRecord } from "@agentswf/contract/records";
+import { messageOf } from "./errors";
+import { appendEvent } from "./runs";
+
+type Body<E> = E extends unknown ? Omit<E, "version" | "attempt" | "at"> : never;
+/** An event as the engine writes it: the log fills in its version, attempt and time. */
+export type RunEventBody = Body<RunEvent>;
+
+/**
+ * The attempt's writer of the run's event log. Lines go out in call order, and a write that fails
+ * costs the run nothing: it is said through `failed`, which must not write to this log, for each
+ * turn and once for the other lines.
+ */
+export class RunEventLog {
+  readonly #writes: Promise<void>[] = [];
+  #failed = false;
+  #closed = false;
+
+  constructor(
+    private readonly options: {
+      runDir: string;
+      attempt: number;
+      failed(message: string): void;
+      now?: () => number;
+    },
+  ) {}
+
+  /** Whether the line was written, once it is; a caller may wait for it, and none has to. */
+  event(body: RunEventBody, at?: number): Promise<boolean> {
+    if (this.#closed) return Promise.resolve(false);
+    const time = new Date(at ?? this.options.now?.() ?? Date.now()).toISOString();
+    return this.#append({
+      version: RUN_EVENT_VERSIONS[body.kind],
+      attempt: this.options.attempt,
+      at: time,
+      ...body,
+    } as RunEvent);
+  }
+
+  turn(record: TurnRecord): void {
+    if (this.#closed) return;
+    this.#writes.push(
+      appendEvent(this.options.runDir, record).catch((error) =>
+        this.options.failed(`awf: a turn was not recorded in turns.jsonl: ${messageOf(error)}`),
+      ),
+    );
+  }
+
+  /**
+   * Once every line asked for so far is written, or has failed. Nothing is written after: the
+   * attempt's end follows, and then the next attempt's lines.
+   */
+  async close(): Promise<void> {
+    this.#closed = true;
+    await Promise.all(this.#writes);
+  }
+
+  #append(line: RunEvent): Promise<boolean> {
+    const written = appendEvent(this.options.runDir, line).then(
+      () => true,
+      (error) => {
+        if (!this.#failed) {
+          this.#failed = true;
+          this.options.failed(`awf: the run's event log was not written: ${messageOf(error)}`);
+        }
+        return false;
+      },
+    );
+    this.#writes.push(written.then(() => undefined));
+    return written;
+  }
+}

package/src/run-prepare.ts

@@ -0,0 +1,178 @@
+import { statSync } from "node:fs";
+import { type AttemptRecord, RUN_EVENT_VERSIONS } from "@agentswf/contract/records";
+import type { ExecutableWorkflow, JsonValue } from "@agentswf/contract/workflow";
+import { refusalMessage } from "./attempt-view";
+import { messageOf } from "./errors";
+import type { RunCommand } from "./run-command";
+import {
+  type Attempt,
+  appendEvent,
+  attemptRefusal,
+  checkContinue,
+  checkFree,
+  claimAttempt,
+  endTurnsLine,
+  idProblem,
+  openRun,
+  type Run,
+  RunRefused,
+  readAttempts,
+  readStageRecords,
+  readTurns,
+} from "./runs";
+import { assertJsonValue, type LoadedWorkflow, loadWorkflowFile } from "./workflow-loader";
+
+export type PreparedRun = {
+  args: JsonValue;
+  /** A new run's id, from `--id` or the workflow's `id(args)`; absent for one to generate. */
+  id?: string;
+  /** The run `--continue` names, its earlier attempts, and what to warn of before it goes on. */
+  continued?: { run: Run; attempts: AttemptRecord[]; warnings: string[] };
+};
+
+/**
+ * The command's workflow loaded and its run prepared, or why not, as awf says it: in the shell that
+ * typed the command, before anything starts.
+ */
+export async function loadAndPrepare(
+  command: RunCommand,
+  now: number,
+): Promise<{ loaded: LoadedWorkflow; prepared: PreparedRun } | { refused: string }> {
+  let loaded: LoadedWorkflow;
+  try {
+    loaded = await loadWorkflowFile(command.workflowFile, command.shellCwd);
+  } catch (error) {
+    return { refused: `load: ${messageOf(error)}` };
+  }
+  try {
+    return { loaded, prepared: await prepareRun(command, loaded, now) };
+  } catch (error) {
+    return { refused: `${error instanceof RunRefused ? "" : "prepare: "}${messageOf(error)}` };
+  }
+}
+
+/**
+ * The workflow's arguments, and the run `--continue` names, whose recorded argv is prepared again
+ * by the code as it is now. A new run is claimed after, by its caller.
+ */
+async function prepareRun(
+  command: RunCommand,
+  loaded: LoadedWorkflow,
+  now: number,
+): Promise<PreparedRun> {
+  const { executable, file } = loaded;
+  const { meta } = executable.definition;
+  const prepare = (argv: readonly string[], cwd: string) => {
+    const args = executable.prepare({ argv, cwd });
+    assertJsonValue(args, `${meta.name} arguments`);
+    return args;
+  };
+  if (command.continueId === undefined) {
+    const args = prepare(command.workflowArgs, command.cwd);
+    const id = command.id ?? derivedId(executable, args);
+    if (id !== undefined) await checkFree(command.runRoot, meta.name, id);
+    return { args, ...(id === undefined ? {} : { id }) };
+  }
+  const run = await openRun(command.runRoot, meta.name, command.continueId);
+  checkContinue(run, {
+    argv: command.workflowArgs,
+    ...(command.cwdGiven ? { cwd: command.cwd } : {}),
+    ...(command.sandbox === undefined ? {} : { sandbox: command.sandbox }),
+  });
+  const { id, argv, cwd } = run.record;
+  if (!statSync(cwd, { throwIfNoEntry: false })?.isDirectory()) {
+    throw new RunRefused(`${id} works in ${cwd}, which is no longer a directory`);
+  }
+  let args: JsonValue;
+  try {
+    args = prepare(argv, cwd);
+  } catch (error) {
+    throw new RunRefused(
+      `the recorded argv of ${id} no longer parses with ${file}: ${messageOf(error)}; start a new run`,
+    );
+  }
+  const attempts = await readAttempts(run);
+  const records = await readStageRecords(run.dir);
+  // The claim refuses it too; this says so before `--here` opens a tab, or `--session` waits.
+  const refused = attemptRefusal(attempts, records, { fromStage: command.fromStage !== undefined });
+  if (refused) throw new RunRefused(refusalMessage(id, refused, now));
+  const { fromStage } = command;
+  // Once per start point: a run started at a stage goes on to it attempt after attempt.
+  const unrecorded =
+    fromStage !== undefined &&
+    !records.has(fromStage) &&
+    attempts.at(-1)?.flags.fromStage !== fromStage
+      ? [`awf: nothing is recorded for ${fromStage}; the attempt stops if it never reaches it`]
+      : [];
+  return { args, continued: { run, attempts, warnings: unrecorded } };
+}
+
+/** The id the workflow's `id(args)` gives a new run, checked; undefined when it has none. */
+function derivedId(
+  executable: ExecutableWorkflow<JsonValue, JsonValue>,
+  args: JsonValue,
+): string | undefined {
+  if (!executable.id) return undefined;
+  let id: unknown;
+  try {
+    id = executable.id(args);
+  } catch (error) {
+    throw new RunRefused(`the workflow's id(args) failed: ${messageOf(error)}; --id gives one`);
+  }
+  const problem =
+    typeof id === "string" ? idProblem(id) : `it returned a ${typeof id}, not a string`;
+  if (problem) throw new RunRefused(`the workflow's id(args): ${problem}; --id gives one`);
+  return id as string;
+}
+
+/** Claims the run's next attempt, naming each earlier one found interrupted. */
+export async function claimNext(
+  run: Run,
+  loaded: LoadedWorkflow,
+  command: RunCommand,
+  notice: (text: string) => void,
+  now: number,
+): Promise<Attempt> {
+  const { meta } = loaded.executable.definition;
+  const claimed = await claimAttempt(
+    run,
+    {
+      file: loaded.file,
+      ...(meta.version === undefined ? {} : { workflowVersion: meta.version }),
+      flags: {
+        timeout: command.timeout,
+        ...(command.fromStage === undefined ? {} : { fromStage: command.fromStage }),
+      },
+    },
+    { fromStage: command.fromStage !== undefined },
+  );
+  if ("refused" in claimed) {
+    throw new RunRefused(refusalMessage(run.record.id, claimed.refused, now));
+  }
+  // A crash may have torn the last line an earlier attempt appended; this attempt's first line
+  // must not be glued to it.
+  await endTurnsLine(run.dir)
+    .then(() =>
+      appendEvent(run.dir, {
+        version: RUN_EVENT_VERSIONS["attempt-started"],
+        attempt: claimed.attempt.record.attempt,
+        at: claimed.attempt.record.started,
+        kind: "attempt-started",
+      }),
+    )
+    .catch((error) => notice(`awf: the run's event log was not written: ${messageOf(error)}`));
+  const turns = claimed.interrupted.length > 0 ? await readTurns(run.dir).catch(() => []) : [];
+  for (const earlier of claimed.interrupted) {
+    // Its last turn says the stage it was in, as nothing else it wrote does.
+    const stage = turns.findLast((turn) => turn.attempt === earlier.attempt)?.stage;
+    notice(
+      `awf: attempt ${earlier.attempt} of ${run.record.id} was interrupted${stage ? ` in ${stage}` : ""}; its panes may still be open in Herdr workspace "${workspaceLabel(meta.name, run.record.id, earlier.attempt)}"`,
+    );
+  }
+  return claimed.attempt;
+}
+
+/** A run's Herdr workspace, which names the attempt: a dead one's may still be open. */
+export function workspaceLabel(workflow: string, id: string, attempt: number): string {
+  return `awf ${workflow} ${id} #${attempt}`;
+}

package/src/run-progress.ts

@@ -0,0 +1,137 @@
+import type { TurnRecord } from "@agentswf/contract/records";
+import type { AgentKey, OperationRecord } from "@agentswf/contract/workflow";
+
+/** One labelled `parallel` call. Its label need not be unique; its position is. */
+export type GroupProgress = {
+  label: string;
+  total: number;
+  /** Items handed to the operation so far; the rest wait for a free slot. */
+  started: number;
+  done: number;
+  startedAt: number;
+  endedAt?: number;
+};
+
+type TurnProgress = Pick<TurnRecord, "kind" | "stage" | "label"> & {
+  startedAt: number;
+  phase?: string;
+  /** When `phase` began. */
+  phaseAt?: number;
+  waitingReason?: string;
+  checkInAt?: number;
+  settledAt?: number;
+  /** Absent while the turn runs. */
+  outcome?: TurnRecord["outcome"];
+  reason?: string;
+};
+
+/** A message admitted, as the progress view lines it. */
+export type MessageProgress = {
+  from: AgentKey;
+  to: AgentKey;
+  bytes: number;
+  at: number;
+  /** It asks: `to` owes a reply. */
+  asks?: true;
+  /** It answers a question `to` asked. */
+  answers?: true;
+};
+
+export type AgentProgress = {
+  /** Index into `groups` of the labelled `parallel` that opened the agent. */
+  group?: number;
+  turns: number;
+  stopped?: { at: number; reason: string; cancelledTurn: boolean };
+  /** The agent it was forked from. */
+  forkedFrom?: AgentKey;
+  /** The latest turn. */
+  turn?: TurnProgress;
+};
+
+/**
+ * What the engine knows about a run's progress beyond each agent's harness state and the stages,
+ * which the stage ledger keeps.
+ */
+export class RunProgress {
+  readonly #groups: GroupProgress[] = [];
+  readonly #agents = new Map<AgentKey, AgentProgress>();
+  readonly #messages: MessageProgress[] = [];
+
+  group(label: string, total: number): GroupProgress {
+    const group = { label, total, started: 0, done: 0, startedAt: Date.now() };
+    this.#groups.push(group);
+    return group;
+  }
+
+  agentOpened(key: AgentKey, group: GroupProgress | undefined): void {
+    if (this.#agents.has(key)) return;
+    const index = group ? this.#groups.indexOf(group) : -1;
+    this.#agents.set(key, { turns: 0, ...(index >= 0 ? { group: index } : {}) });
+  }
+
+  /** A fork that was not made, whose key is free again. */
+  agentDropped(key: AgentKey): void {
+    this.#agents.delete(key);
+  }
+
+  /** Once its parent's session was copied, not when it was asked for. */
+  agentForked(key: AgentKey, from: AgentKey): void {
+    const agent = this.#agents.get(key);
+    if (agent) agent.forkedFrom = from;
+  }
+
+  agentStopped(key: AgentKey, reason: string): void {
+    const agent = this.#agents.get(key);
+    if (agent && !agent.stopped) {
+      agent.stopped = {
+        at: Date.now(),
+        reason,
+        cancelledTurn: agent.turn !== undefined && agent.turn.outcome === undefined,
+      };
+    }
+  }
+
+  turnStarted(
+    key: AgentKey,
+    tags: Pick<OperationRecord, "stage" | "label"> = {},
+    kind: TurnRecord["kind"] = "turn",
+  ): void {
+    const agent = this.#agents.get(key);
+    if (!agent) return;
+    agent.turns += 1;
+    agent.turn = { kind, startedAt: Date.now(), ...tags };
+  }
+
+  turnSettled(key: AgentKey, outcome: TurnProgress["outcome"], reason?: string): void {
+    const turn = this.#agents.get(key)?.turn;
+    if (!turn || turn.outcome) return;
+    turn.settledAt = Date.now();
+    turn.outcome = outcome;
+    if (reason !== undefined) turn.reason = reason;
+  }
+
+  turnPhase(key: AgentKey, phase: string, reason?: string, until?: number): void {
+    const turn = this.#agents.get(key)?.turn;
+    if (!turn || turn.outcome) return;
+    if (turn.phase !== phase || turn.waitingReason !== reason) turn.phaseAt = Date.now();
+    turn.phase = phase;
+    turn.waitingReason = reason;
+    turn.checkInAt = until;
+  }
+
+  messageSent(message: MessageProgress): void {
+    this.#messages.push(message);
+  }
+
+  snapshot(): {
+    groups: GroupProgress[];
+    agents: Map<AgentKey, AgentProgress>;
+    messages: MessageProgress[];
+  } {
+    return {
+      groups: structuredClone(this.#groups),
+      agents: structuredClone(this.#agents),
+      messages: structuredClone(this.#messages),
+    };
+  }
+}