@spunto/build 0.22.0 → 0.23.0

package/src/agent-contract/conformance.ts

@@ -0,0 +1,315 @@
+/**
+ * The conformance suite of the Spunto agent contract: what says an agent app is one, rather than
+ * the platform finding out in a customer's task.
+ *
+ * It drives a runner the way the platform does — the three stdin verbs, a resume, closing stdin —
+ * and checks what comes back: every line valid against the contract, and each scenario's events in
+ * the order the platform relies on (a turn always ends, an interrupt leaves the session alive, a
+ * resume keeps the session id…).
+ *
+ * **What a scenario asks of the harness.** A suite cannot make a real model call a tool on cue, so
+ * each scenario's prompt is a fixed phrase (`CONFORMANCE_PROMPTS`) and an app runs the suite against
+ * a **test double of its harness** that does what the phrase says — a fake SDK replaying recorded
+ * messages, for an app built on one (`test/fixtures/fake-*-sdk.mjs` here), or nothing at all for
+ * an agent with no SDK (`examples/fake-agent.mjs`). What is tested is the runner: its reading of
+ * the harness, and its side of the contract.
+ *
+ * Pure: the process is the caller's (`StartRunner`) — with Node, `child_process.spawn` and a
+ * line splitter, see `test/agent-contract.exec.test.ts`. The suite only needs a timer.
+ */
+
+import type { RunnerInput } from "./vocabulary"
+import { contractLineProblem } from "./vocabulary"
+
+/** A runner the suite drives. `lines` grows as the runner prints; `exited` resolves with its code. */
+export type RunnerProcess = {
+  send(input: RunnerInput): void
+  /** Close stdin: the session ends once its turn is over. */
+  close(): void
+  /** Stop it outright — only when a scenario has already failed. */
+  kill(): void
+  lines: string[]
+  exited: Promise<number | null>
+}
+
+/** Start the runner: `run`, plus `--resume <id>` when asked. */
+export type StartRunner = (opts: { resume?: string }) => RunnerProcess
+
+export const CONFORMANCE_SCENARIOS = ["simple", "tool", "interrupt", "resume", "limit", "title", "files"] as const
+export type ConformanceScenario = (typeof CONFORMANCE_SCENARIOS)[number]
+
+/**
+ * What the test double of the harness must do on each phrase.
+ *
+ *  - `reply`: answer in prose, end the turn;
+ *  - `tool`: call one tool, get its result, answer, end the turn;
+ *  - `long`: keep the turn going (a long tool call, say) until interrupted;
+ *  - `limit`: end the turn on the account's usage limit, with the time it lifts;
+ *  - `screenshot`: call a tool whose result is a PNG, inline (base64), and end the turn.
+ */
+export const CONFORMANCE_PROMPTS = {
+  reply: "spunto-conformance: reply",
+  tool: "spunto-conformance: tool",
+  long: "spunto-conformance: long",
+  limit: "spunto-conformance: limit",
+  screenshot: "spunto-conformance: screenshot",
+} as const
+
+export type ConformanceResult = {
+  scenario: ConformanceScenario
+  /** `skipped`: the app said its harness cannot (`skip`) — Codex names no conversation, say. */
+  status: "passed" | "failed" | "skipped"
+  problems: string[]
+}
+
+export type ConformanceOptions = {
+  /** Scenarios the app's harness cannot do, and says so (a `title` the harness cannot produce). */
+  skip?: ConformanceScenario[]
+  /** How long one step may wait for the runner, in ms. */
+  timeoutMs?: number
+}
+
+type Line = Record<string, unknown>
+
+class ScenarioFailure extends Error {}
+
+/** Run every scenario, one runner process (or two, for `resume`) each. */
+export async function runConformance(start: StartRunner, opts: ConformanceOptions = {}): Promise<ConformanceResult[]> {
+  const results: ConformanceResult[] = []
+  for (const scenario of CONFORMANCE_SCENARIOS) {
+    if (opts.skip?.includes(scenario)) {
+      results.push({ scenario, status: "skipped", problems: [] })
+      continue
+    }
+    results.push(await runScenario(scenario, start, opts.timeoutMs ?? 10_000))
+  }
+  return results
+}
+
+async function runScenario(scenario: ConformanceScenario, start: StartRunner, timeoutMs: number): Promise<ConformanceResult> {
+  const problems: string[] = []
+  const procs: RunnerProcess[] = []
+  const launch = (o: { resume?: string } = {}) => {
+    const p = start(o)
+    procs.push(p)
+    return new Session(p, timeoutMs, problems)
+  }
+  try {
+    await SCENARIOS[scenario](launch, problems)
+  } catch (err) {
+    problems.push(err instanceof ScenarioFailure ? err.message : `the scenario threw: ${err}`)
+  }
+  for (const p of procs) {
+    for (const [i, line] of p.lines.entries()) {
+      const problem = lineProblem(line)
+      if (problem) problems.push(`line ${i + 1} is not valid: ${problem} — ${line.slice(0, 200)}`)
+    }
+    p.kill()
+  }
+  return { scenario, status: problems.length ? "failed" : "passed", problems: [...new Set(problems)] }
+}
+
+function lineProblem(line: string): string | null {
+  let parsed: unknown
+  try {
+    parsed = JSON.parse(line)
+  } catch {
+    return "not JSON"
+  }
+  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return "not a JSON object"
+  return contractLineProblem(parsed as Line)
+}
+
+/** One runner process, read as a sequence of contract lines. */
+class Session {
+  /** How many lines the scenario has already looked past. */
+  private cursor = 0
+
+  constructor(
+    readonly proc: RunnerProcess,
+    private readonly timeoutMs: number,
+    private readonly problems: string[],
+  ) {}
+
+  send(input: RunnerInput): void {
+    this.proc.send(input)
+  }
+
+  /** Every parsed line so far. */
+  all(): Line[] {
+    return this.proc.lines.flatMap((l) => {
+      try {
+        const v = JSON.parse(l)
+        return v && typeof v === "object" && !Array.isArray(v) ? [v as Line] : []
+      } catch {
+        return []
+      }
+    })
+  }
+
+  /** Wait for the next line (past the cursor) that matches, and move the cursor past it. */
+  async next(what: string, match: (l: Line) => boolean): Promise<Line> {
+    const deadline = Date.now() + this.timeoutMs
+    for (;;) {
+      const lines = this.all()
+      for (let i = this.cursor; i < lines.length; i++) {
+        if (match(lines[i]!)) {
+          this.cursor = i + 1
+          return lines[i]!
+        }
+      }
+      if (Date.now() > deadline) throw new ScenarioFailure(`timed out waiting for ${what}`)
+      await sleep(25)
+    }
+  }
+
+  /** The turn's end, and every line of the turn up to it. */
+  async turn(): Promise<{ lines: Line[]; ended: Line }> {
+    const from = this.cursor
+    const ended = await this.next("the turn's `session.ended`", (l) => l.type === "session.ended")
+    return { lines: this.all().slice(from, this.cursor), ended }
+  }
+
+  /** Close stdin and expect a clean exit. */
+  async close(): Promise<void> {
+    this.proc.close()
+    const code = await Promise.race([this.proc.exited, sleep(this.timeoutMs).then(() => "timeout" as const)])
+    if (code === "timeout") this.problems.push("did not exit after stdin was closed")
+    else if (code !== 0) this.problems.push(`exited ${code} after stdin was closed (expected 0)`)
+  }
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms))
+}
+
+function expect(problems: string[], ok: boolean, message: string): void {
+  if (!ok) problems.push(message)
+}
+
+type Launch = (opts?: { resume?: string }) => Session
+
+const SCENARIOS: Record<ConformanceScenario, (launch: Launch, problems: string[]) => Promise<void>> = {
+  // A turn: the session opens, the agent answers, the turn ends, and closing stdin exits 0.
+  async simple(launch, problems) {
+    const s = launch()
+    s.send({ type: "user", text: CONFORMANCE_PROMPTS.reply })
+    const { lines, ended } = await s.turn()
+    const started = lines.find((l) => l.type === "session.started")
+    expect(problems, !!started, "no `session.started` before the turn ended")
+    expect(problems, lines.findIndex((l) => l.type === "session.started") <= lines.findIndex((l) => l.type === "message"), "`session.started` came after the first message")
+    expect(problems, typeof started?.sessionId === "string" && !!started.sessionId, "`session.started` has no `sessionId` — the platform resumes with it")
+    expect(problems, lines.some((l) => l.type === "message" && l.role === "assistant"), "no assistant `message` in the turn")
+    expect(problems, ended.status === "succeeded", `the turn ended ${String(ended.status)} (expected succeeded)`)
+    await s.close()
+  },
+
+  // A tool call and its result, paired by `callId`.
+  async tool(launch, problems) {
+    const s = launch()
+    s.send({ type: "user", text: CONFORMANCE_PROMPTS.tool })
+    const { lines, ended } = await s.turn()
+    const call = lines.find((l) => l.type === "tool.call")
+    const result = lines.find((l) => l.type === "tool.result")
+    expect(problems, !!call, "no `tool.call`")
+    expect(problems, !!result, "no `tool.result`")
+    if (call && result) {
+      expect(problems, typeof call.callId === "string" && call.callId === result.callId, "the `tool.result` does not carry its call's `callId`")
+      expect(problems, lines.indexOf(call) < lines.indexOf(result), "the result came before its call")
+    }
+    expect(problems, ended.status === "succeeded", `the turn ended ${String(ended.status)} (expected succeeded)`)
+    await s.close()
+  },
+
+  // Interrupting ends the turn and leaves the session listening: the next message is a normal turn.
+  async interrupt(launch, problems) {
+    const s = launch()
+    s.send({ type: "user", text: CONFORMANCE_PROMPTS.long })
+    await s.next("the session to start", (l) => l.type === "session.started")
+    s.send({ type: "interrupt", id: "spunto-conformance-1" })
+    await s.turn()
+    s.send({ type: "user", text: CONFORMANCE_PROMPTS.reply })
+    const after = await s.turn()
+    expect(problems, after.lines.some((l) => l.type === "message" && l.role === "assistant"), "no answer to the message sent after the interrupt — did the session survive it?")
+    expect(problems, after.ended.status === "succeeded", "the turn after the interrupt did not succeed")
+    await s.close()
+  },
+
+  // `--resume <id>` reopens the same session, and says so.
+  async resume(launch, problems) {
+    const first = launch()
+    first.send({ type: "user", text: CONFORMANCE_PROMPTS.reply })
+    const opened = await first.next("the session to start", (l) => l.type === "session.started")
+    await first.turn()
+    await first.close()
+    const id = typeof opened.sessionId === "string" ? opened.sessionId : ""
+    if (!id) throw new ScenarioFailure("no session id to resume")
+    const second = launch({ resume: id })
+    second.send({ type: "user", text: CONFORMANCE_PROMPTS.reply })
+    const { lines, ended } = await second.turn()
+    const started = lines.find((l) => l.type === "session.started")
+    expect(problems, !!started, "a resumed session does not announce itself (`session.started`)")
+    expect(problems, started?.sessionId === id, `the resumed session says ${String(started?.sessionId)}, not ${id}`)
+    expect(problems, ended.status === "succeeded", "the resumed turn did not succeed")
+    await second.close()
+  },
+
+  // A turn the account ended: `session.ended` says so, with the time it lifts.
+  async limit(launch, problems) {
+    const s = launch()
+    s.send({ type: "user", text: CONFORMANCE_PROMPTS.limit })
+    const { ended } = await s.turn()
+    const limit = ended.limit as Line | undefined
+    expect(problems, ended.status === "failed", "a turn the account ended is not a success")
+    expect(problems, !!limit && limit.kind === "usage", "`session.ended` carries no `limit` of kind `usage`")
+    expect(problems, typeof limit?.resetsAt === "string" && !Number.isNaN(Date.parse(limit.resetsAt as string)), "the limit has no `resetsAt` the platform can wait for")
+    await s.close()
+  },
+
+  // The `title` verb is answered with a `session.title`.
+  async title(launch, problems) {
+    const s = launch()
+    s.send({ type: "user", text: CONFORMANCE_PROMPTS.reply })
+    await s.turn()
+    s.send({ type: "title", id: "spunto-title-conformance", description: "Fix the login page redirect" })
+    const named = await s.next("a `session.title`", (l) => l.type === "session.title")
+    expect(problems, typeof named.title === "string" && !!named.title.trim(), "an empty title")
+    await s.close()
+  },
+
+  // A file a tool hands back reaches the platform whole — in the line, as a base64 content block —
+  // and not as kilobytes of base64 in the event's text.
+  async files(launch, problems) {
+    const s = launch()
+    s.send({ type: "user", text: CONFORMANCE_PROMPTS.screenshot })
+    const { lines } = await s.turn()
+    const result = lines.find((l) => l.type === "tool.result" && hasFileBlock(l))
+    expect(problems, !!result, "no `tool.result` line carrying the file as a base64 `image` block (in its `source`)")
+    if (result) expect(problems, !String(result.output).includes(fileData(result) ?? "\u0000"), "the file's bytes are in the result's `output` text")
+    await s.close()
+  },
+}
+
+/** The first base64 file block's data in a line, if any. */
+function fileData(value: unknown, depth = 0): string | null {
+  if (depth > 8 || !value || typeof value !== "object") return null
+  if (Array.isArray(value)) {
+    for (const v of value) {
+      const found = fileData(v, depth + 1)
+      if (found) return found
+    }
+    return null
+  }
+  const obj = value as Line
+  const src = obj.source as Line | undefined
+  if ((obj.type === "image" || obj.type === "document") && src?.type === "base64" && typeof src.data === "string") return src.data
+  for (const v of Object.values(obj)) {
+    const found = fileData(v, depth + 1)
+    if (found) return found
+  }
+  return null
+}
+
+function hasFileBlock(line: Line): boolean {
+  return fileData(line) !== null
+}

package/src/agent-contract/examples/fake-agent.mjs

@@ -0,0 +1,46 @@
+#!/usr/bin/env node
+// A whole agent app runner, with no SDK and no import of ours: what it takes to speak the Spunto
+// agent contract (docs/agent-contract.md). It passes the conformance suite — that is its job; the
+// "harness" is the switch below, which does what each conformance phrase asks.
+//
+//   node fake-agent.mjs run [--resume <session id>]
+
+import { createInterface } from "node:readline"
+
+const out = (line) => process.stdout.write(`${JSON.stringify(line)}\n`)
+const argv = process.argv.slice(2)
+const resume = argv[argv.indexOf("--resume") + 1]
+const sessionId = argv.includes("--resume") && resume ? resume : `fake-${process.pid}`
+let started = false
+let pending = null // the turn waiting to be interrupted
+
+function turn(text) {
+  if (!started) out({ type: "session.started", harness: "fake", model: null, sessionId }), (started = true)
+  const done = (summary) => out({ type: "session.ended", status: "succeeded", summary })
+  if (text.endsWith("tool")) {
+    out({ type: "tool.call", callId: "c1", name: "Bash", input: '{"command":"ls"}' })
+    out({ type: "tool.result", callId: "c1", isError: false, output: "README.md" })
+  }
+  if (text.endsWith("screenshot")) {
+    const block = { type: "image", source: { type: "base64", media_type: "image/png", data: "iVBORw0KGgo=" } }
+    out({ type: "tool.call", callId: "c2", name: "Screenshot", input: "{}" })
+    out({ type: "tool.result", callId: "c2", isError: false, output: "[image]", source: { content: [block] } })
+  }
+  if (text.endsWith("limit")) {
+    const resetsAt = new Date(Date.now() + 3_600_000).toISOString()
+    const limit = { kind: "usage", status: 429, message: "Usage limit reached", resetsAt }
+    return out({ type: "session.ended", status: "failed", summary: limit.message, limit })
+  }
+  if (text.endsWith("long")) return (pending = () => out({ type: "session.ended", status: "failed", summary: "Interrupted." }))
+  out({ type: "message", role: "assistant", text: `You said: ${text}` })
+  done("ok")
+}
+
+createInterface({ input: process.stdin })
+  .on("line", (line) => {
+    const msg = JSON.parse(line)
+    if (msg.type === "user") turn(msg.text)
+    if (msg.type === "interrupt") pending?.(), (pending = null)
+    if (msg.type === "title") out({ type: "session.title", title: msg.description.slice(0, 60) })
+  })
+  .on("close", () => process.exit(0))

package/src/agent-contract/index.ts

@@ -0,0 +1,20 @@
+// The Spunto agent contract (RFC 0048 § 9, docs/agent-contract.md): what an agent app's runner reads
+// on stdin and prints on stdout, the one strict reader the platform has for it, its JSON Schema, and
+// the conformance suite that says whether an app speaks it.
+
+export {
+  AGENT_CONTRACT_VERSION,
+  agentContractSchema,
+  clip,
+  contractLineProblem,
+  isTransient,
+  MAX_TEXT,
+  readContractLine,
+  readContractObject,
+  TASK_EVENT_TYPES,
+  TRANSIENT_TYPES,
+} from "./vocabulary"
+export type { NormalizedEvent, RunnerInput, SourcedEvent, TaskEventType, TransientType } from "./vocabulary"
+
+export { CONFORMANCE_PROMPTS, CONFORMANCE_SCENARIOS, runConformance } from "./conformance"
+export type { ConformanceOptions, ConformanceResult, ConformanceScenario, RunnerProcess, StartRunner } from "./conformance"