@flow-engineer/flow 0.0.1 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +611 -0
  3. package/coding-agents/skill.md +353 -0
  4. package/coding-agents/test-skill.md +223 -0
  5. package/coding-agents/tools-skill.md +89 -0
  6. package/dist/flow-messaging/src/client.d.ts +40 -0
  7. package/dist/flow-messaging/src/client.js +142 -0
  8. package/dist/flow-messaging/src/index.d.ts +72 -0
  9. package/dist/flow-messaging/src/index.js +605 -0
  10. package/dist/simulators/src/index.d.ts +2 -0
  11. package/dist/simulators/src/index.js +2 -0
  12. package/dist/simulators/src/page.html +1245 -0
  13. package/dist/simulators/src/person.html +100 -0
  14. package/dist/simulators/src/platforms.d.ts +29 -0
  15. package/dist/simulators/src/platforms.js +72 -0
  16. package/dist/simulators/src/server.d.ts +201 -0
  17. package/dist/simulators/src/server.js +759 -0
  18. package/dist/src/agent.d.ts +504 -0
  19. package/dist/src/agent.js +818 -0
  20. package/dist/src/channels.d.ts +178 -0
  21. package/dist/src/channels.js +243 -0
  22. package/dist/src/conversation.d.ts +361 -0
  23. package/dist/src/conversation.js +1487 -0
  24. package/dist/src/db.d.ts +18 -0
  25. package/dist/src/db.js +57 -0
  26. package/dist/src/events.d.ts +17 -0
  27. package/dist/src/events.js +20 -0
  28. package/dist/src/http.d.ts +45 -0
  29. package/dist/src/http.js +269 -0
  30. package/dist/src/index.d.ts +14 -0
  31. package/dist/src/index.js +11 -0
  32. package/dist/src/model.d.ts +21 -0
  33. package/dist/src/model.js +96 -0
  34. package/dist/src/names.d.ts +22 -0
  35. package/dist/src/names.js +43 -0
  36. package/dist/src/nudges.d.ts +23 -0
  37. package/dist/src/nudges.js +117 -0
  38. package/dist/src/postgres.d.ts +20 -0
  39. package/dist/src/postgres.js +62 -0
  40. package/dist/src/schedules.d.ts +60 -0
  41. package/dist/src/schedules.js +249 -0
  42. package/dist/src/skills.d.ts +12 -0
  43. package/dist/src/skills.js +69 -0
  44. package/dist/src/store.d.ts +214 -0
  45. package/dist/src/store.js +448 -0
  46. package/dist/src/talk.d.ts +57 -0
  47. package/dist/src/talk.js +166 -0
  48. package/dist/src/time.d.ts +71 -0
  49. package/dist/src/time.js +170 -0
  50. package/dist/src/tools.d.ts +43 -0
  51. package/dist/src/tools.js +26 -0
  52. package/dist/src/traces.d.ts +31 -0
  53. package/dist/src/traces.js +303 -0
  54. package/dist/src/types.d.ts +260 -0
  55. package/dist/src/types.js +2 -0
  56. package/dist/testing/src/account.d.ts +4 -0
  57. package/dist/testing/src/account.js +580 -0
  58. package/dist/testing/src/chat.d.ts +3 -0
  59. package/dist/testing/src/chat.js +117 -0
  60. package/dist/testing/src/check-endpoint.d.ts +31 -0
  61. package/dist/testing/src/check-endpoint.js +158 -0
  62. package/dist/testing/src/chinwag.d.ts +2 -0
  63. package/dist/testing/src/chinwag.js +6 -0
  64. package/dist/testing/src/cli.d.ts +2 -0
  65. package/dist/testing/src/cli.js +592 -0
  66. package/dist/testing/src/client.d.ts +117 -0
  67. package/dist/testing/src/client.js +956 -0
  68. package/dist/testing/src/flow-login.d.ts +143 -0
  69. package/dist/testing/src/flow-login.js +249 -0
  70. package/dist/testing/src/http-agent.d.ts +47 -0
  71. package/dist/testing/src/http-agent.js +155 -0
  72. package/dist/testing/src/index.d.ts +3 -0
  73. package/dist/testing/src/index.js +2 -0
  74. package/dist/testing/src/new.d.ts +13 -0
  75. package/dist/testing/src/new.js +147 -0
  76. package/dist/testing/src/print-run.d.ts +3 -0
  77. package/dist/testing/src/print-run.js +22 -0
  78. package/dist/testing/src/protocol.d.ts +159 -0
  79. package/dist/testing/src/protocol.js +28 -0
  80. package/dist/testing/src/report.d.ts +74 -0
  81. package/dist/testing/src/report.js +90 -0
  82. package/dist/testing/src/show.d.ts +36 -0
  83. package/dist/testing/src/show.js +280 -0
  84. package/dist/testing/src/suite.d.ts +87 -0
  85. package/dist/testing/src/suite.js +198 -0
  86. package/docs/agents.md +85 -0
  87. package/examples/cafe/agent/AGENTS.md +10 -0
  88. package/examples/cafe/agent/skills/allergens/SKILL.md +7 -0
  89. package/examples/cafe/src/agent.ts +43 -0
  90. package/examples/cafe/src/channels.ts +20 -0
  91. package/examples/cafe/src/main.ts +30 -0
  92. package/examples/cafe/src/tools.ts +62 -0
  93. package/examples/cafe/tests/scenarios/latte-correction.yaml +30 -0
  94. package/examples/cafe/tests/scenarios/order-fails.yaml +26 -0
  95. package/examples/cafe/tests/tools/get_menu.yaml +14 -0
  96. package/examples/cafe/tests/tools/place_order.yaml +29 -0
  97. package/examples/http-agent/server.mjs +32 -0
  98. package/examples/http-agent/tests/scenarios/opening-hours.yaml +16 -0
  99. package/flow-messaging/README.md +191 -0
  100. package/package.json +63 -8
  101. package/spec/agent-endpoint.openapi.yaml +148 -0
  102. package/spec/nudges.openapi.yaml +167 -0
  103. package/spec/request.schema.json +121 -0
  104. package/spec/response.schema.json +284 -0
@@ -0,0 +1,147 @@
1
+ // `flow new <name>`: makes ./<name>/, an agent of its own started from the café example (Pip),
2
+ // that depends on the @flow-engineer/flow package. Its imports become "@flow-engineer/flow",
3
+ // "@flow-engineer/flow/simulators" and "@flow-engineer/flow/flow-messaging", and the coding-agent
4
+ // skills go in .claude/skills/, with an AGENTS.md (and a CLAUDE.md importing it) pointing at them.
5
+ // `flow init` makes the same agent in the current folder. Inside a clone, `npm run new -- <name>`
6
+ // makes agents/<name>/ instead.
7
+ import { cpSync, existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
8
+ import { basename, dirname, join } from "node:path";
9
+ import { fileURLToPath } from "node:url";
10
+ const NAME = /^[a-z0-9]+(-[a-z0-9]+)*$/;
11
+ /** Throws unless `name` can be an agent's folder and package name. */
12
+ export function checkName(name) {
13
+ if (!NAME.test(name))
14
+ throw new Error(`"${name}" is not a usable name: use lowercase letters, digits and dashes, such as my-agent.`);
15
+ }
16
+ /** The @flow-engineer/flow package's folder: the root of a clone, or the installed package. */
17
+ export function packageRoot(from = dirname(fileURLToPath(import.meta.url))) {
18
+ for (let dir = from;; dir = dirname(dir)) {
19
+ const file = join(dir, "package.json");
20
+ if (existsSync(file) && JSON.parse(readFileSync(file, "utf8")).name === "@flow-engineer/flow")
21
+ return dir;
22
+ if (dirname(dir) === dir)
23
+ throw new Error("flow: can't find its own package.json");
24
+ }
25
+ }
26
+ const TSCONFIG = {
27
+ compilerOptions: {
28
+ target: "es2023",
29
+ module: "nodenext",
30
+ moduleResolution: "nodenext",
31
+ strict: true,
32
+ noEmit: true,
33
+ allowImportingTsExtensions: true,
34
+ erasableSyntaxOnly: true,
35
+ verbatimModuleSyntax: true,
36
+ skipLibCheck: true,
37
+ types: ["node"],
38
+ },
39
+ include: ["src"],
40
+ };
41
+ const ENV_EXAMPLE = `# One model key. The first line with a value is the one used.
42
+ # Make it at the provider, then paste it here. Never commit this file.
43
+ # OPENROUTER_API_KEY https://openrouter.ai/keys
44
+ # OPENAI_API_KEY https://platform.openai.com/api-keys
45
+ # ANTHROPIC_API_KEY https://console.anthropic.com/settings/keys
46
+ # GEMINI_API_KEY https://aistudio.google.com/apikey
47
+ OPENROUTER_API_KEY=
48
+ OPENAI_API_KEY=
49
+ ANTHROPIC_API_KEY=
50
+ GEMINI_API_KEY=
51
+
52
+ # Optional. The model id for that provider; empty is that provider's default.
53
+ # CAFE_MODEL=
54
+
55
+ # The test cloud (npx @flow-engineer/flow test --synthetic) and traces. Sign in at https://flow.engineer/observe,
56
+ # add the agent, and on its Developers page make a CI key (oci_..., shown once); the trace key is
57
+ # from the same site.
58
+ FLOW_FRAMEWORK_CLOUD=https://flow.engineer/observe
59
+ FLOW_FRAMEWORK_CLOUD_KEY=
60
+ FLOW_FRAMEWORK_TRACES_URL=https://flow.engineer/observe
61
+ FLOW_FRAMEWORK_TRACES_KEY=
62
+ `;
63
+ const agentsMd = (name) => `# ${name}
64
+
65
+ A chat agent on Flow Framework (https://github.com/flow-engineer/flow), started from its café example (Pip).
66
+
67
+ - Load the skills in \`.claude/skills/\` before changing it: \`flow-framework\` to build the agent, \`flow-framework-tools\`
68
+ for its tools and their test files, \`flow-framework-tests\` for its test scenarios.
69
+ - \`agent/AGENTS.md\` is the agent's own prompt, not instructions for coding agents.
70
+ - The owner puts one model key in \`.env\` (never read or print it): \`OPENROUTER_API_KEY\`,
71
+ \`OPENAI_API_KEY\`, \`ANTHROPIC_API_KEY\` or \`GEMINI_API_KEY\`. Then \`npx @flow-engineer/flow chat\`
72
+ talks to it in the terminal, \`npm start\` runs it in the playground, and \`npx @flow-engineer/flow link\`
73
+ signs in, puts its other keys in \`.env\` and prints the link that reaches it from a phone while
74
+ \`npm start\` runs. \`npm run typecheck\` checks the types, \`npm test -- --synthetic\` runs the
75
+ tests in the test cloud at https://flow.engineer/observe with the owner's \`FLOW_FRAMEWORK_CLOUD_KEY\`
76
+ (\`npx @flow-engineer/flow --help\`).
77
+ - The guide for coding agents, with exit codes and next steps: https://github.com/flow-engineer/flow/blob/main/docs/agents.md
78
+ `;
79
+ /** Makes `<into>/<name>` (the current folder by default) and returns it. */
80
+ export function newStandalone(name, options = {}) {
81
+ checkName(name);
82
+ const dir = join(options.into ?? process.cwd(), name);
83
+ if (existsSync(dir))
84
+ throw new Error(`${dir} already exists: pick another name or remove it.`);
85
+ makeAgent(dir, name, options.root ?? packageRoot());
86
+ return dir;
87
+ }
88
+ /** What `flow init` writes into a folder; it refuses a folder that holds any of them, so nothing is overwritten. */
89
+ const MADE = ["package.json", "tsconfig.json", "src", "agent", "tests", ".env.example", "AGENTS.md", "CLAUDE.md", ".npmrc"];
90
+ /** Makes the agent in `dir` (the current folder by default), named after the folder; returns its name. */
91
+ export function initHere(dir = process.cwd(), options = {}) {
92
+ const taken = MADE.filter((f) => existsSync(join(dir, f)));
93
+ if (taken.length)
94
+ throw new Error(`${dir} has ${taken.join(", ")} already: run flow init in an empty folder, or flow new <name> to make one.`);
95
+ const name = basename(dir).toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "") || "my-agent";
96
+ makeAgent(dir, name, options.root ?? packageRoot());
97
+ return name;
98
+ }
99
+ function makeAgent(dir, name, root) {
100
+ const cafe = join(root, "examples", "cafe");
101
+ for (const part of ["agent", "src", "tests"])
102
+ cpSync(join(cafe, part), join(dir, part), { recursive: true });
103
+ for (const file of readdirSync(join(dir, "src"), { recursive: true, encoding: "utf8" }).filter((f) => f.endsWith(".ts"))) {
104
+ const path = join(dir, "src", file);
105
+ const source = readFileSync(path, "utf8")
106
+ .replace(/(["'])(\.\.\/)+src\/index\.ts\1/g, '"@flow-engineer/flow"')
107
+ .replace(/(["'])(\.\.\/)+simulators\/src\/index\.ts\1/g, '"@flow-engineer/flow/simulators"')
108
+ .replace(/(["'])(\.\.\/)+flow-messaging\/src\/index\.ts\1/g, '"@flow-engineer/flow/flow-messaging"');
109
+ if (/["']\.\.\/\.\.\//.test(source))
110
+ throw new Error(`flow: ${file} imports a file outside the agent; only Flow Framework's public entries can be rewritten.`);
111
+ writeFileSync(path, source);
112
+ }
113
+ const own = JSON.parse(readFileSync(join(root, "package.json"), "utf8"));
114
+ const pkg = {
115
+ name,
116
+ version: "0.1.0",
117
+ private: true,
118
+ description: `${name}: a Flow Framework agent, started from the café example.`,
119
+ type: "module",
120
+ engines: own.engines,
121
+ scripts: { start: "node --env-file-if-exists=.env src/main.ts", test: "flow test", typecheck: "tsc --noEmit" },
122
+ dependencies: { "@flow-engineer/flow": `^${own.version}` },
123
+ devDependencies: { "@types/node": own.devDependencies["@types/node"], typescript: own.devDependencies.typescript },
124
+ };
125
+ writeFileSync(join(dir, "package.json"), `${JSON.stringify(pkg, null, 2)}\n`);
126
+ writeFileSync(join(dir, "tsconfig.json"), `${JSON.stringify(TSCONFIG, null, 2)}\n`);
127
+ // An existing .gitignore (flow init in a fresh repository) keeps its lines and gets the missing ones.
128
+ const ignore = join(dir, ".gitignore");
129
+ const ignored = existsSync(ignore) ? readFileSync(ignore, "utf8") : "";
130
+ const missing = ["node_modules/", ".env", ".flow/"].filter((line) => !ignored.split("\n").includes(line));
131
+ if (missing.length)
132
+ writeFileSync(ignore, `${ignored}${ignored && !ignored.endsWith("\n") ? "\n" : ""}${missing.join("\n")}\n`);
133
+ // npm install then refuses a Node older than the engines field says, before .ts files fail to load.
134
+ writeFileSync(join(dir, ".npmrc"), "engine-strict=true\n");
135
+ writeFileSync(join(dir, ".env.example"), ENV_EXAMPLE);
136
+ const skills = join(root, "coding-agents");
137
+ for (const file of readdirSync(skills).filter((f) => f.endsWith(".md"))) {
138
+ const text = readFileSync(join(skills, file), "utf8");
139
+ const skill = /^name:\s*(\S+)\s*$/m.exec(text)?.[1];
140
+ if (!skill)
141
+ throw new Error(`flow: coding-agents/${file} has no name in its front matter.`);
142
+ mkdirSync(join(dir, ".claude", "skills", skill), { recursive: true });
143
+ writeFileSync(join(dir, ".claude", "skills", skill, "SKILL.md"), text);
144
+ }
145
+ writeFileSync(join(dir, "AGENTS.md"), agentsMd(name));
146
+ writeFileSync(join(dir, "CLAUDE.md"), "@AGENTS.md\n");
147
+ }
@@ -0,0 +1,3 @@
1
+ import type { RunSummary } from "./protocol.ts";
2
+ /** Prints each run once. A run already shown as it finished is not printed again with the summary. */
3
+ export declare function runPrinter(log?: (line: string) => void): (run: RunSummary) => void;
@@ -0,0 +1,22 @@
1
+ import { runVerdict } from "./report.js";
2
+ const usd = (n) => `$${n.toFixed(4)}`;
3
+ /** Prints each run once. A run already shown as it finished is not printed again with the summary. */
4
+ export function runPrinter(log = console.log) {
5
+ const printed = new Set();
6
+ return (r) => {
7
+ if (printed.has(r.runId))
8
+ return;
9
+ printed.add(r.runId);
10
+ const verdict = runVerdict(r);
11
+ const gaveUp = verdict === "passed" && r.ended === "gave_up" ? " (passed, the person gave up)" : "";
12
+ const retried = r.retried ? "; tried once more" : "";
13
+ // An errored run: the cloud could not play or judge it, so its checks say nothing about the agent.
14
+ const error = verdict === "errored" ? `; error: ${r.error ?? r.ended} (the cloud could not play or judge this run, not the agent's fault)` : "";
15
+ const label = verdict === "errored" ? "ERROR" : verdict === "passed" ? "PASS" : "FAIL";
16
+ log(`${label} ${r.scenario} #${r.run} (${r.channel}) ${r.runId}: ${r.turns} turn${r.turns === 1 ? "" : "s"}, ended: ${r.ended}${gaveUp}${retried}${error}; slowest acknowledgement ${r.slowestAckMs === null ? "never" : `${r.slowestAckMs} ms`}; cost ${usd(r.agentUsage.costUsd)} agent + ${usd(r.testUsage.costUsd)} test`);
17
+ if (verdict !== "errored")
18
+ for (const c of r.checks.filter((c) => !c.pass))
19
+ log(` ✗ ${c.kind}: ${c.text}\n ${c.reason}`);
20
+ log(` watch: ${r.watch}`);
21
+ };
22
+ }
@@ -0,0 +1,159 @@
1
+ import type { Attachment, ChannelInfo, ChannelSummary, Incoming, ToolSpec } from "../../src/index.ts";
2
+ /** One graded expect item of a run. */
3
+ export interface Check {
4
+ kind: "outcome" | "judge";
5
+ text: string;
6
+ pass: boolean;
7
+ reason: string;
8
+ }
9
+ /** Model tokens and their cost, added up over a run. */
10
+ export interface Usage {
11
+ input: number;
12
+ output: number;
13
+ costUsd: number;
14
+ }
15
+ export interface OpenSession {
16
+ /** name: a stable key (the package name) that keeps the agent's history together; title: for people. */
17
+ agent: {
18
+ name: string;
19
+ title?: string;
20
+ tools: ToolSpec[];
21
+ channels?: ChannelSummary[];
22
+ };
23
+ /** Every file in the local suite; only these tests run. */
24
+ files: Array<{
25
+ path: string;
26
+ hash: string;
27
+ }>;
28
+ /** How many times each scenario runs. */
29
+ runs: number;
30
+ /** Acknowledge each message independently of the serial event uploads. */
31
+ deliveryReceipts?: boolean;
32
+ /** The client will retry a call when this cloud stores the result under Idempotency-Key. */
33
+ retries?: boolean;
34
+ /** The client opens a lost stream again with Last-Event-ID when this cloud resumes. */
35
+ resume?: boolean;
36
+ /** The client opens one new session for the unfinished runs when this cloud says session_gone. */
37
+ reopen?: boolean;
38
+ /** Play only these runs, numbered as given: a re-open after session_gone. Sent only to a cloud whose Opened said reopen,
39
+ * and only when some run finished: the cloud checks it against the suite's files it already stores. */
40
+ only?: RunKey[];
41
+ }
42
+ /** One run of a suite: a scenario's id, one of its channels, and which of the `runs` times (from 1). */
43
+ export interface RunKey {
44
+ scenario: string;
45
+ channel: string;
46
+ run: number;
47
+ }
48
+ export interface Receipt {
49
+ conversationId: string;
50
+ messageId: string;
51
+ status: "taken" | "intercepted";
52
+ }
53
+ export interface Opened {
54
+ sessionId: string;
55
+ token: string;
56
+ /** Hashes the cloud does not have yet; upload these. */
57
+ missing: string[];
58
+ /** Where to watch this session's runs. */
59
+ dashboard: string;
60
+ /** The tunnel's absolute URL, where every later call goes (a hosted cloud's front door serves
61
+ * only the open). Absent or empty: the open's own address (a local cloud). */
62
+ tunnel?: string;
63
+ /** This cloud stores each Idempotency-Key with its result. Absent on an older cloud. */
64
+ idempotency?: boolean;
65
+ /** This cloud numbers stream commands with an SSE id, sends the ones after Last-Event-ID to a
66
+ * stream opened again, and holds them 2 minutes while none is attached. Absent on an older cloud. */
67
+ resume?: boolean;
68
+ /** This cloud answers session_gone for a session it lost, and plays `only` on an open. Absent on an older cloud. */
69
+ reopen?: boolean;
70
+ }
71
+ export interface UploadFile {
72
+ hash: string;
73
+ path: string;
74
+ content: string;
75
+ }
76
+ export type Command = {
77
+ type: "message";
78
+ conversationId: string;
79
+ messageId: string;
80
+ incoming: WireIncoming;
81
+ channel: ChannelInfo;
82
+ channelName?: string;
83
+ } | {
84
+ type: "reaction";
85
+ conversationId: string;
86
+ emoji: string;
87
+ toText: string;
88
+ } | {
89
+ type: "drain";
90
+ } | {
91
+ type: "run";
92
+ run: RunSummary;
93
+ } | {
94
+ type: "finished";
95
+ summary: SessionSummary;
96
+ }
97
+ /** The session stopped before it finished; the CLI exits with this error. */
98
+ | {
99
+ type: "failed";
100
+ error: string;
101
+ };
102
+ /**
103
+ * A person's message as it travels: a scenario's file goes by its path in the tests folder, and the
104
+ * agent's side reads it there, so the file never leaves the developer's machine.
105
+ */
106
+ export type WireIncoming = Omit<Incoming, "attachments"> & {
107
+ attachments?: Array<{
108
+ kind: Attachment["kind"];
109
+ file: string;
110
+ }>;
111
+ };
112
+ export interface ToolCallRequest {
113
+ conversationId: string;
114
+ tool: string;
115
+ input: unknown;
116
+ }
117
+ export type ToolCallResult = {
118
+ ok: true;
119
+ output: unknown;
120
+ } | {
121
+ ok: false;
122
+ error: string;
123
+ };
124
+ export interface RunSummary {
125
+ runId: string;
126
+ scenario: string;
127
+ run: number;
128
+ channel: string;
129
+ /** Every check passed. */
130
+ passed: boolean;
131
+ checks: Check[];
132
+ /** Why the person stopped: goal met, gave up, max turns, or the agent went quiet. */
133
+ ended: string;
134
+ turns: number;
135
+ messagesIn: number;
136
+ messagesOut: number;
137
+ toolCalls: number;
138
+ /** Slowest acknowledgement of a person's message, in ms (null if one was never acknowledged). */
139
+ slowestAckMs: number | null;
140
+ /** The agent's model calls in this run, from its own events, and the models that answered them. */
141
+ agentUsage: Usage;
142
+ agentModels: string[];
143
+ /** The test's own model calls in this run: the person, the tool simulator and the judge. */
144
+ testUsage: Usage;
145
+ watch: string;
146
+ /** The first try never reached the agent, so this run is the second try. */
147
+ retried?: boolean;
148
+ /** The cloud's code when it could not play or judge the run (judge_failed, spend_cap, simulator_failed, ...).
149
+ * The run then says nothing about the agent, whatever `passed` and `ended` say. */
150
+ error?: string;
151
+ }
152
+ export interface SessionSummary {
153
+ sessionId: string;
154
+ suiteHash: string;
155
+ /** The model that played the person and the tools and judged. */
156
+ testModel: string;
157
+ runs: RunSummary[];
158
+ dashboard: string;
159
+ }
@@ -0,0 +1,28 @@
1
+ // What travels through the tunnel between an agent on a developer's machine and the test cloud.
2
+ // Every connection is opened by the agent's side (plain HTTP), so it works behind any router.
3
+ //
4
+ // POST /api/sessions OpenSession → Opened start a session, learn which files to upload
5
+ // POST /api/files { files: UploadFile[] } upload the files the cloud lacks
6
+ // GET /api/sessions/:id/stream server-sent Commands the person's side, cloud → agent
7
+ // Last-Event-ID: N a resume: the commands after N
8
+ // POST /api/sessions/:id/outbound { conversationId, outbound } → { ok } | 409 { error }
9
+ // POST /api/sessions/:id/tool-calls ToolCallRequest → ToolCallResult held open while the tool is simulated
10
+ // POST /api/sessions/:id/events { seq, events: FlowEvent[] }
11
+ //
12
+ // Every POST after the open sends Idempotency-Key. An older cloud ignores it. When Opened
13
+ // says idempotency, a repeat of a key is the stored answer and the client may send it again.
14
+ // POST /api/sessions/:id/receipts Receipt → { ok }
15
+ // POST /api/sessions/:id/drained {} → { ok } flush before the terminal command
16
+ //
17
+ // When Opened says resume, every command on the stream carries `id: N` (1, 2, 3, ... per session).
18
+ // A stream that drops, or sits silent past the idle watch, is opened again with Last-Event-ID set
19
+ // to the last id the client handled (0 for none); the cloud sends the commands after it, and holds
20
+ // them for 2 minutes while no stream is attached. The client skips an id it has handled.
21
+ //
22
+ // When Opened says reopen, a tunnel call or the stream may answer 410 { error: "session_gone" }: the
23
+ // cloud lost the session (its tunnel restarted). The client opens one new session with `only`, the
24
+ // runs that have no result yet, and merges the two reports. Never more than once.
25
+ //
26
+ // The open goes to the cloud's front door; every later call goes to Opened.tunnel (else the same
27
+ // address) and carries `authorization: Bearer <token>` from Opened.
28
+ export {};
@@ -0,0 +1,74 @@
1
+ /** How a `flow test` ended, one per exit status (0 passed; 1 failed, invalid; 2 errored, transport; 3 refused; 130 interrupted). */
2
+ export type Outcome = "passed" | "failed" | "errored" | "transport" | "refused" | "interrupted" | "invalid";
3
+ export type Verdict = "passed" | "failed" | "errored" | "not_played";
4
+ export interface ReportError {
5
+ message: string;
6
+ /** The cloud's HTTP status, when it answered one. */
7
+ status: number | null;
8
+ /** The cloud's code for it, such as spend_cap or rate_limited. */
9
+ type: string | null;
10
+ retryAfterSeconds: number | null;
11
+ }
12
+ export interface ReportRun {
13
+ runId: string | null;
14
+ /** The scenario's file name without .yaml, as --only takes it. */
15
+ scenario: string;
16
+ channel: string;
17
+ run: number;
18
+ verdict: Verdict;
19
+ /** The cloud's code when it could not play or judge the run (judge_failed, spend_cap, ...); else null. */
20
+ error: string | null;
21
+ ended: string | null;
22
+ turns: number | null;
23
+ watch: string | null;
24
+ failedChecks: Array<{
25
+ kind: string;
26
+ text: string;
27
+ reason: string;
28
+ }>;
29
+ }
30
+ /** `flow test --json`: one document on stdout, whatever the exit. */
31
+ export interface JsonReport {
32
+ version: 1;
33
+ outcome: Outcome;
34
+ exitCode: number;
35
+ sessionId: string | null;
36
+ /** The cloud's id for the agent, which `flow show` reads runs under. */
37
+ agent: string | null;
38
+ dashboard: string | null;
39
+ suiteHash: string | null;
40
+ error: ReportError | null;
41
+ counts: {
42
+ runs: number;
43
+ passed: number;
44
+ failed: number;
45
+ errored: number;
46
+ notPlayed: number;
47
+ };
48
+ costUsd: {
49
+ agent: number;
50
+ test: number;
51
+ };
52
+ runs: ReportRun[];
53
+ }
54
+ /** A run's verdict: errored when the cloud names an error for it or ended it run_failed, since the cloud
55
+ * could not play or judge it and that says nothing about the agent (a judge_failed run can end goal_met). */
56
+ export declare function runVerdict(run: {
57
+ passed: boolean;
58
+ ended: string;
59
+ error?: string;
60
+ }): Exclude<Verdict, "not_played">;
61
+ export declare const EXIT: Record<Outcome, number>;
62
+ /** A session that ran to its summary: failed when a run failed, errored when the cloud could not play or judge a run (or never played it). */
63
+ export declare function finishedOutcome(runs: Array<{
64
+ verdict: Verdict;
65
+ }>): Outcome;
66
+ /** What stopped a session, or the open, as an outcome. `opened`: the session had opened. */
67
+ export declare function stoppedOutcome(error: unknown, opened: boolean): Outcome;
68
+ /** The error as the JSON report says it: the cloud's status, code and Retry-After when it answered. */
69
+ export declare function reportError(error: unknown, message: string): ReportError;
70
+ /** The connection's error code, from the error or what caused it. */
71
+ export declare function networkCode(error: unknown): string | undefined;
72
+ /** The agent folder's .env, or the --dotenv given; variables already set win, and no value is ever printed.
73
+ * Returns what is wrong when a named file cannot be read. */
74
+ export declare function loadEnv(file: string | undefined): string | undefined;
@@ -0,0 +1,90 @@
1
+ // What `flow test --json` prints, how an error that stopped a session is classified into an
2
+ // outcome and an exit status, and the pieces `flow test` and `flow show` share.
3
+ import { existsSync } from "node:fs";
4
+ import { CloudEnded, isTransportFailure, PostError, SessionStopped } from "./client.js";
5
+ /** A run's verdict: errored when the cloud names an error for it or ended it run_failed, since the cloud
6
+ * could not play or judge it and that says nothing about the agent (a judge_failed run can end goal_met). */
7
+ export function runVerdict(run) {
8
+ if (run.error || run.ended === "run_failed")
9
+ return "errored";
10
+ return run.passed ? "passed" : "failed";
11
+ }
12
+ export const EXIT = { passed: 0, failed: 1, invalid: 1, errored: 2, transport: 2, refused: 3, interrupted: 130 };
13
+ /** A session that ran to its summary: failed when a run failed, errored when the cloud could not play or judge a run (or never played it). */
14
+ export function finishedOutcome(runs) {
15
+ if (runs.some((r) => r.verdict === "failed"))
16
+ return "failed";
17
+ if (runs.some((r) => r.verdict !== "passed"))
18
+ return "errored";
19
+ return "passed";
20
+ }
21
+ /** What stopped a session, or the open, as an outcome. `opened`: the session had opened. */
22
+ export function stoppedOutcome(error, opened) {
23
+ if (error instanceof SessionStopped)
24
+ return "transport";
25
+ if (error instanceof CloudEnded)
26
+ return error.code === "spend_cap" || error.code === "capacity" ? "refused" : "errored";
27
+ const refusal = postErrorOf(error);
28
+ if (refusal) {
29
+ const why = refusal.type ?? refusal.code;
30
+ if (refusal.status === 503 && why === "capacity")
31
+ return "refused";
32
+ if (refusal.status >= 400 && refusal.status < 500 && why)
33
+ return "refused";
34
+ return "transport";
35
+ }
36
+ if (isTransportFailure(error) || networkCode(error))
37
+ return "transport";
38
+ // Something on this machine: the agent would not build before the open, or broke the session after it.
39
+ return opened ? "failed" : "invalid";
40
+ }
41
+ /** The error as the JSON report says it: the cloud's status, code and Retry-After when it answered. */
42
+ export function reportError(error, message) {
43
+ const refusal = postErrorOf(error);
44
+ const type = refusal ? refusal.type ?? refusal.code ?? null : error instanceof CloudEnded ? error.code ?? null : null;
45
+ const retry = refusal?.retryAfterMs;
46
+ return { message, status: refusal?.status ?? null, type, retryAfterSeconds: retry === undefined ? null : Math.ceil(retry / 1000) };
47
+ }
48
+ function postErrorOf(error) {
49
+ let current = error;
50
+ for (let guard = 0; current instanceof Error && guard < 6; guard++) {
51
+ if (current instanceof PostError)
52
+ return current;
53
+ current = current.cause;
54
+ }
55
+ return undefined;
56
+ }
57
+ /** The connection's error code, from the error or what caused it. */
58
+ export function networkCode(error) {
59
+ let current = error;
60
+ for (let guard = 0; current instanceof Error && guard < 6; guard++) {
61
+ const code = current.code;
62
+ if (code === "ECONNREFUSED" || code === "ENOTFOUND" || code === "EAI_AGAIN" || code === "ETIMEDOUT" || code === "ECONNRESET")
63
+ return code;
64
+ for (const inner of current instanceof AggregateError ? current.errors : []) {
65
+ const innerCode = networkCode(inner);
66
+ if (innerCode)
67
+ return innerCode;
68
+ }
69
+ current = current.cause;
70
+ }
71
+ return undefined;
72
+ }
73
+ /** The agent folder's .env, or the --dotenv given; variables already set win, and no value is ever printed.
74
+ * Returns what is wrong when a named file cannot be read. */
75
+ export function loadEnv(file) {
76
+ if (file === undefined) {
77
+ if (existsSync(".env"))
78
+ process.loadEnvFile(".env");
79
+ return undefined;
80
+ }
81
+ if (!existsSync(file))
82
+ return `--dotenv ${file}: no such file.`;
83
+ try {
84
+ process.loadEnvFile(file);
85
+ }
86
+ catch (error) {
87
+ return `--dotenv ${file} could not be read: ${error.code ?? "unreadable"}.`;
88
+ }
89
+ return undefined;
90
+ }
@@ -0,0 +1,36 @@
1
+ /** Where every session's results go, in the agent's folder; `flow new` git-ignores .flow/. */
2
+ export declare const LAST_SESSION: string;
3
+ /** The last session's file in dir: LAST_SESSION, else, before a run has written it, the one from before the rename (.chinwag/). */
4
+ export declare function lastSessionIn(dir: string): string;
5
+ /** .flow/last-session.json: what each run ended as, for `--failed`, `flow show` and coding agents. */
6
+ export interface LastSession {
7
+ sessionId: string;
8
+ /** The cloud's id for the agent, from the open (an older cloud names none). */
9
+ agent?: string;
10
+ dashboard: string;
11
+ suiteHash: string;
12
+ /** When the session ended, as an ISO time. */
13
+ endedAt: string;
14
+ ended: "finished" | "stopped" | "interrupted";
15
+ /** The scenarios this session was limited to (--only, --failed); null when it ran the whole suite. */
16
+ only: string[] | null;
17
+ runs: Array<{
18
+ /** The scenario's file name without .yaml, as --only takes it. */
19
+ scenario: string;
20
+ channel: string;
21
+ run: number;
22
+ /** The cloud's id for the run, for `flow show`; null when it was not played. */
23
+ runId?: string | null;
24
+ verdict: "passed" | "failed" | "errored" | "not_played";
25
+ /** The checks as the test file says them, with the judge's reason. The file is local and
26
+ * git-ignored, and the suite is synthetic by attestation; the messages themselves are never kept. */
27
+ failedChecks: Array<{
28
+ kind: string;
29
+ text: string;
30
+ reason?: string;
31
+ }>;
32
+ }>;
33
+ }
34
+ export declare const SHOW_LINE = " flow show <run-id|last> [--json]\n Print one run's record from the test cloud: its checks with the\n judge's reasons, the transcript and the tool calls (flow show --help)";
35
+ /** `flow show`: returns the exit status. */
36
+ export declare function showCli(args: string[]): Promise<number>;