@agentswf/engine 0.1.0-beta.4 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +16 -0
- package/package.json +13 -9
- package/src/accounting/format.ts +4 -3
- package/src/accounting/spent.ts +128 -0
- package/src/accounting/summary.ts +40 -32
- package/src/agent-cli/cli.ts +35 -22
- package/src/agent-cli/client.ts +1 -27
- package/src/allowance-command.ts +8 -10
- package/src/attempt-ending.ts +5 -7
- package/src/attempt-output.ts +15 -32
- package/src/attempt-view.ts +34 -33
- package/src/author-typebox.ts +13 -0
- package/src/client/checks.ts +94 -0
- package/src/client/exported.ts +17 -0
- package/src/client/index.ts +122 -0
- package/src/client/locate.ts +34 -0
- package/src/client/published.ts +3 -0
- package/src/client/runs.ts +337 -0
- package/src/command-line.ts +64 -0
- package/src/commands.ts +7 -1
- package/src/control-plane.ts +41 -11
- package/src/coordinator-prompt.ts +4 -3
- package/src/coordinator-socket.ts +41 -33
- package/src/deadlines.ts +40 -7
- package/src/decisions/directory.ts +51 -40
- package/src/decisions/fake.ts +1 -1
- package/src/decisions/openrouter.ts +4 -2
- package/src/decisions/seam.ts +4 -4
- package/src/display-path.ts +3 -3
- package/src/duration.ts +43 -27
- package/src/emit-command.ts +88 -58
- package/src/emit.ts +254 -56
- package/src/event-ids.ts +31 -0
- package/src/files.ts +44 -4
- package/src/hand-over.ts +150 -0
- package/src/help/allowance.txt +4 -3
- package/src/help/emit.txt +18 -6
- package/src/help/inbox.txt +6 -4
- package/src/help/logs.txt +9 -6
- package/src/help/open.txt +29 -0
- package/src/help/reply.txt +6 -8
- package/src/help/run.txt +30 -11
- package/src/help/runs.txt +4 -2
- package/src/help/tick.txt +9 -4
- package/src/herdr-placement.ts +3 -33
- package/src/herdr-run-session.ts +7 -15
- package/src/herdr-workspace-marks.ts +7 -17
- package/src/holder-command.ts +56 -133
- package/src/in-lines.ts +28 -0
- package/src/inbox-command.ts +40 -57
- package/src/index.ts +1 -1
- package/src/init-command.ts +8 -17
- package/src/init-template/hello/workflow.test.ts.template +3 -3
- package/src/init-template/hello/workflow.ts.template +5 -4
- package/src/jsonl.ts +10 -6
- package/src/keep-alive.ts +15 -12
- package/src/logs-command.ts +128 -47
- package/src/message-operation.ts +6 -3
- package/src/messaging/limits.ts +4 -4
- package/src/messaging/messaging.ts +31 -28
- package/src/messaging/routes.ts +1 -1
- package/src/messaging/text.ts +9 -7
- package/src/open-command.ts +770 -0
- package/src/operation-events.ts +4 -6
- package/src/operation-liveness.ts +19 -4
- package/src/operator-cli.ts +224 -94
- package/src/operator-runtime.ts +29 -7
- package/src/ops.ts +270 -0
- package/src/pane-layout.ts +105 -59
- package/src/progress-view.ts +10 -10
- package/src/project-engine.ts +79 -0
- package/src/projects.ts +13 -2
- package/src/refusals.ts +119 -0
- package/src/result-slots.ts +11 -31
- package/src/result-validation.ts +2 -2
- package/src/run-command.ts +45 -28
- package/src/run-commands.ts +43 -115
- package/src/run-event-log.ts +1 -1
- package/src/run-lines.ts +159 -0
- package/src/run-lock.ts +57 -1
- package/src/{stage-plan.ts → run-policy/stage-plan.ts} +46 -42
- package/src/run-policy/stages.ts +113 -0
- package/src/run-policy/waits.ts +452 -0
- package/src/run-prepare.ts +205 -43
- package/src/run-read.ts +109 -41
- package/src/run-state.ts +170 -50
- package/src/run-text.ts +54 -0
- package/src/run-unreachable.ts +18 -0
- package/src/run-usage.ts +18 -12
- package/src/run-view.ts +81 -17
- package/src/runs-command.ts +83 -63
- package/src/runs.ts +310 -212
- package/src/sandbox-homes.ts +65 -13
- package/src/sandboxes.ts +182 -28
- package/src/session-lines.ts +93 -0
- package/src/skills/run-skills.ts +8 -1
- package/src/sources.ts +28 -29
- package/src/spend.ts +333 -0
- package/src/stage-ledger.ts +119 -131
- package/src/stop-crashed.ts +6 -5
- package/src/stopped.ts +4 -4
- package/src/test-command.ts +36 -14
- package/src/testing/operation-events.ts +3 -23
- package/src/testing.ts +73 -26
- package/src/tick-command.ts +145 -15
- package/src/tools.ts +25 -16
- package/src/turn-ends.ts +37 -0
- package/src/waits.ts +651 -0
- package/src/wake.ts +84 -54
- package/src/workflow-loader.ts +54 -4
- package/src/workflow-runner.ts +805 -580
- package/src/workflow-test-preload.ts +4 -1
- package/src/workflow-testing/coordinators.ts +3 -3
- package/src/workflow-testing/fake-client.ts +387 -0
- package/src/workflow-testing/host.ts +121 -53
- package/src/workflow-testing/index.ts +33 -842
- package/src/workflow-testing/run-test.ts +1128 -0
- package/src/workflow-testing/script.ts +74 -45
- package/src/hold-run-lock.fixture.ts +0 -8
- package/src/mailbox.ts +0 -990
- package/src/workflow-testing/coordinator.typecheck.ts +0 -54
- package/src/workflow-testing/workflow-testing.typecheck.ts +0 -64
package/src/run-text.ts
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
// How the commands that read runs show what a run wrote, what it cost, and when.
|
|
2
|
+
import { maskCredentials } from "@agentswf/harness";
|
|
3
|
+
import { usd } from "./accounting/format";
|
|
4
|
+
|
|
5
|
+
/** A field whose name says it holds a credential, whatever its value looks like. */
|
|
6
|
+
const SECRET_NAME =
|
|
7
|
+
/password|passwd|passphrase|passcode|secret|token|credential|cookie|bearer|authorization|(?:api|access|private|session|ssh|signing|encryption)[_-]?key|^(?:pwd|pass|pin|auth)$/i;
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Every string in `value` masked, and everything under a credential's name hidden: a run's text,
|
|
11
|
+
* errors and fields are its workflow's, its agents' and its tools'. Past `limit`, a string is cut.
|
|
12
|
+
*/
|
|
13
|
+
export function masked(value: unknown, limit = Number.POSITIVE_INFINITY, hidden = false): unknown {
|
|
14
|
+
if (typeof value === "string") {
|
|
15
|
+
if (hidden) return "***";
|
|
16
|
+
const shown = maskCredentials(value);
|
|
17
|
+
return shown.length > limit ? `${shown.slice(0, limit - 1)}…` : shown;
|
|
18
|
+
}
|
|
19
|
+
if (hidden && value !== null && typeof value !== "object") return "***";
|
|
20
|
+
if (Array.isArray(value)) return value.map((each) => masked(each, limit, hidden));
|
|
21
|
+
if (value && typeof value === "object") {
|
|
22
|
+
return Object.fromEntries(
|
|
23
|
+
Object.entries(value).map(([key, each]) => [
|
|
24
|
+
key,
|
|
25
|
+
masked(each, limit, hidden || SECRET_NAME.test(key)),
|
|
26
|
+
]),
|
|
27
|
+
);
|
|
28
|
+
}
|
|
29
|
+
return value;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Text a run wrote, as one masked line of about `limit` characters, so it can never pass for a
|
|
34
|
+
* line of its own or fill a page. `@{handle} --full` prints it whole.
|
|
35
|
+
*/
|
|
36
|
+
export function said(text: string, limit = 120): string {
|
|
37
|
+
const line = maskCredentials(text).replace(/\s*[\r\n]+\s*/g, " ⏎ ");
|
|
38
|
+
return line.length > limit ? `${line.slice(0, limit - 1)}…` : line;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* As `usd` says it, and nothing for none; `?` after it when some cost is unknown, which makes it a
|
|
43
|
+
* floor.
|
|
44
|
+
*/
|
|
45
|
+
export function money(estimate: number | undefined, unknown = 0): string[] {
|
|
46
|
+
if (!estimate) return [];
|
|
47
|
+
return [`${usd(estimate)}${unknown > 0 ? "?" : ""}`];
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export function clockTime(at: number | string): string {
|
|
51
|
+
const date = new Date(at);
|
|
52
|
+
const two = (n: number) => String(n).padStart(2, "0");
|
|
53
|
+
return `${two(date.getHours())}:${two(date.getMinutes())}:${two(date.getSeconds())}`;
|
|
54
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/** A run's files that can't be read, or a run that doesn't answer: refused `unreachable`. */
|
|
2
|
+
export class RunUnreachable extends Error {
|
|
3
|
+
readonly code = "unreachable";
|
|
4
|
+
constructor(message: string) {
|
|
5
|
+
super(message);
|
|
6
|
+
this.name = "RunUnreachable";
|
|
7
|
+
}
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* An error the system gave, `ENOENT`, `EACCES` and the like, or one awf wrapped around it as its
|
|
12
|
+
* cause: a run's file that couldn't be read or written.
|
|
13
|
+
*/
|
|
14
|
+
export function isSystemError(error: unknown): boolean {
|
|
15
|
+
const { code, cause } = (error ?? {}) as { code?: unknown; cause?: unknown };
|
|
16
|
+
if (typeof code === "string" && /^E[A-Z]+$/.test(code)) return true;
|
|
17
|
+
return cause !== undefined && cause !== error && isSystemError(cause);
|
|
18
|
+
}
|
package/src/run-usage.ts
CHANGED
|
@@ -19,9 +19,9 @@ import { abortableSleep, deadlineWithin, runUntilStopped, waitForDeadline } from
|
|
|
19
19
|
*/
|
|
20
20
|
const FINISHING_GRACE_MILLISECONDS = 10_000;
|
|
21
21
|
/** Reading spend waits for files a released turn is still writing; it gives up after this. */
|
|
22
|
-
const ACCOUNTING_GRACE_MILLISECONDS = 20_000;
|
|
22
|
+
export const ACCOUNTING_GRACE_MILLISECONDS = 20_000;
|
|
23
23
|
/** Beyond that, the host's bound on a billing status command, and this much more. */
|
|
24
|
-
const STATUS_SLACK_MILLISECONDS = 2_000;
|
|
24
|
+
export const STATUS_SLACK_MILLISECONDS = 2_000;
|
|
25
25
|
|
|
26
26
|
/** One logical agent, as the run knows it. */
|
|
27
27
|
export type AccountedAgent = {
|
|
@@ -35,7 +35,7 @@ export type AccountedAgent = {
|
|
|
35
35
|
sessions(): readonly string[];
|
|
36
36
|
/** When its host first prompted it, where that waits on the agent first; see `window`. */
|
|
37
37
|
promptedAt?(): number | undefined;
|
|
38
|
-
/** The caller handed back early by
|
|
38
|
+
/** The caller handed back early by its close(). */
|
|
39
39
|
releasedAt?(): number | undefined;
|
|
40
40
|
/** When a fork's session was made: what it holds from before is its parent's (ADR 0009). */
|
|
41
41
|
forkedAt?(): number | undefined;
|
|
@@ -45,11 +45,12 @@ export type AccountedAgent = {
|
|
|
45
45
|
type OperationEntry = {
|
|
46
46
|
/**
|
|
47
47
|
* Records what is known when the operation settles; spend and billing wait for the run's end.
|
|
48
|
-
* `finishing` is an answered turn the host left finishing, whose
|
|
48
|
+
* `finishing` is an answered turn the host left finishing, whose reported costs are added when
|
|
49
|
+
* it ends.
|
|
49
50
|
*/
|
|
50
51
|
settle(
|
|
51
52
|
times: { deliveredAt?: number; settledAt: number },
|
|
52
|
-
|
|
53
|
+
reportedCosts: readonly number[],
|
|
53
54
|
finishing?: Promise<HarnessTurnOutcome>,
|
|
54
55
|
): OperationRecord;
|
|
55
56
|
};
|
|
@@ -62,6 +63,8 @@ export type OperationTags = Pick<OperationRecord, "stage" | "label" | "cause">;
|
|
|
62
63
|
|
|
63
64
|
/** One agent's part of the run's ledger. */
|
|
64
65
|
export type AgentLedger = {
|
|
66
|
+
/** Where it runs now: its sandbox's folder once it has one. */
|
|
67
|
+
cwd(): string;
|
|
65
68
|
/** `execution` is read as the operation settles: the agent's settings it ran at. */
|
|
66
69
|
reserve(
|
|
67
70
|
operationId: string,
|
|
@@ -125,21 +128,22 @@ export function createRunLedger({
|
|
|
125
128
|
agent(agent) {
|
|
126
129
|
agents.push(agent);
|
|
127
130
|
return {
|
|
131
|
+
cwd: () => agent.cwd,
|
|
128
132
|
reserve(operationId, execution, tags = {}) {
|
|
129
133
|
const operation: Operation = { agent, costs: [] };
|
|
130
134
|
operations.push(operation);
|
|
131
135
|
return {
|
|
132
|
-
settle(times,
|
|
136
|
+
settle(times, reportedCosts, finishing) {
|
|
133
137
|
operation.usage = {
|
|
134
138
|
...recordOf(agent, operationId, execution(), tags, times.deliveredAt),
|
|
135
139
|
settledAt: new Date(times.settledAt).toISOString(),
|
|
136
140
|
};
|
|
137
|
-
operation.costs = [...
|
|
141
|
+
operation.costs = [...reportedCosts];
|
|
138
142
|
if (finishing) {
|
|
139
143
|
late.add(
|
|
140
144
|
finishing.then(
|
|
141
|
-
({
|
|
142
|
-
operation.costs.push(...
|
|
145
|
+
({ reportedCostsUsd }) => {
|
|
146
|
+
operation.costs.push(...reportedCostsUsd);
|
|
143
147
|
},
|
|
144
148
|
() => undefined,
|
|
145
149
|
),
|
|
@@ -175,7 +179,7 @@ export function createRunLedger({
|
|
|
175
179
|
try {
|
|
176
180
|
return await runUntilStopped(
|
|
177
181
|
async () => {
|
|
178
|
-
// Closing the host ended every turn left finishing; what each
|
|
182
|
+
// Closing the host ended every turn left finishing; what each reported lands here.
|
|
179
183
|
await waitForDeadline(lateEnded(), { unixMilliseconds: deadline }).catch(
|
|
180
184
|
() => undefined,
|
|
181
185
|
);
|
|
@@ -254,12 +258,14 @@ async function settleUsage(
|
|
|
254
258
|
return operations.map((operation) => {
|
|
255
259
|
const { billing, shares } = settledAgents.get(operation.agent)!;
|
|
256
260
|
const { costs } = operation;
|
|
257
|
-
const
|
|
261
|
+
const reportedCost = billing === "metered" && costs.length > 0 ? sum(costs) : undefined;
|
|
258
262
|
return {
|
|
259
263
|
...withSessions(operation),
|
|
260
264
|
billing,
|
|
261
265
|
...(shares ? { spend: spendOf(shares.get(operation.usage.operationId) ?? []) } : {}),
|
|
262
|
-
...(
|
|
266
|
+
...(reportedCost === undefined
|
|
267
|
+
? {}
|
|
268
|
+
: { reportedCost: { amount: reportedCost, currency: "USD" } }),
|
|
263
269
|
};
|
|
264
270
|
});
|
|
265
271
|
}
|
package/src/run-view.ts
CHANGED
|
@@ -1,22 +1,30 @@
|
|
|
1
|
-
import { join } from "node:path";
|
|
2
1
|
import type {
|
|
3
2
|
Activity,
|
|
4
|
-
|
|
3
|
+
AgentStateJson,
|
|
4
|
+
ClaimedAttempt,
|
|
5
5
|
OperationLivenessRecord,
|
|
6
6
|
RunLogLine,
|
|
7
|
+
StageStateJson,
|
|
7
8
|
} from "@agentswf/contract/records";
|
|
8
|
-
import type { AgentKey, HarnessKind,
|
|
9
|
+
import type { AgentKey, HarnessKind, JsonValue } from "@agentswf/contract/workflow";
|
|
10
|
+
import { messageOf } from "./errors";
|
|
9
11
|
import { readLines } from "./jsonl";
|
|
12
|
+
import { liveFile } from "./ops";
|
|
10
13
|
import { currentStage, estimateOf } from "./run-commands";
|
|
14
|
+
import { readRunLines } from "./run-lines";
|
|
11
15
|
import { type Cursor, type FileCache, readRun, timeOf } from "./run-read";
|
|
12
16
|
import type { RunStatus } from "./run-state";
|
|
13
17
|
|
|
14
|
-
/**
|
|
18
|
+
/**
|
|
19
|
+
* A stage of the attempt shown: running, reused or given from an earlier one, or how it ended. Its
|
|
20
|
+
* outcome and value are its own `stage` line's, a reused one's the line it reused.
|
|
21
|
+
*/
|
|
15
22
|
export type StageView = {
|
|
16
23
|
stage: string;
|
|
17
|
-
state:
|
|
24
|
+
state: StageStateJson;
|
|
18
25
|
startedAt?: number;
|
|
19
26
|
endedAt?: number;
|
|
27
|
+
value?: JsonValue;
|
|
20
28
|
};
|
|
21
29
|
|
|
22
30
|
/**
|
|
@@ -29,7 +37,7 @@ export type AgentView = {
|
|
|
29
37
|
caller: boolean;
|
|
30
38
|
/** The stage its last operation ran in. */
|
|
31
39
|
stage?: string;
|
|
32
|
-
state:
|
|
40
|
+
state: AgentStateJson;
|
|
33
41
|
turns: number;
|
|
34
42
|
/** When its current operation, or its last, started. */
|
|
35
43
|
since?: number;
|
|
@@ -41,19 +49,23 @@ export type AgentView = {
|
|
|
41
49
|
sessions: { harness: HarnessKind; id: string; files: string[] }[];
|
|
42
50
|
};
|
|
43
51
|
|
|
44
|
-
|
|
52
|
+
type DecisionView = { decision: number; key: string; askedAt: number; outcome?: string };
|
|
45
53
|
|
|
46
54
|
export type RunView = {
|
|
47
55
|
status: RunStatus;
|
|
48
56
|
/** The attempt shown: the last. */
|
|
49
|
-
attempt?:
|
|
57
|
+
attempt?: ClaimedAttempt;
|
|
50
58
|
stages: StageView[];
|
|
59
|
+
/** Why the stages show no values: `run.jsonl` didn't read. */
|
|
60
|
+
unread?: string;
|
|
51
61
|
/** The stage running now, or the one the attempt ended in. */
|
|
52
62
|
stage?: string;
|
|
53
63
|
agents: AgentView[];
|
|
54
64
|
decisions: DecisionView[];
|
|
55
65
|
/** Over the attempts that kept one, at list prices: never what was billed. */
|
|
56
66
|
estimate?: number;
|
|
67
|
+
/** How many agent operations' usage isn't known: `estimate` is a floor. */
|
|
68
|
+
unknownCost: number;
|
|
57
69
|
lastEventAt?: number;
|
|
58
70
|
lines: { number: number; line: RunLogLine }[];
|
|
59
71
|
cursor: Cursor;
|
|
@@ -72,7 +84,7 @@ export async function viewRun(status: RunStatus): Promise<RunView> {
|
|
|
72
84
|
.map(({ line }) => line)
|
|
73
85
|
.filter((line) => attempt === undefined || line.attempt === attempt.attempt);
|
|
74
86
|
|
|
75
|
-
const stages: StageView[] = [];
|
|
87
|
+
const stages: (StageView & { started?: string })[] = [];
|
|
76
88
|
const decisions: DecisionView[] = [];
|
|
77
89
|
const agents = new Map<AgentKey, AgentView>();
|
|
78
90
|
const open = new Map<string, AgentKey>();
|
|
@@ -91,13 +103,17 @@ export async function viewRun(status: RunStatus): Promise<RunView> {
|
|
|
91
103
|
const at = Date.parse(timeOf(line));
|
|
92
104
|
switch (line.kind) {
|
|
93
105
|
case "stage-started":
|
|
94
|
-
stages.push({ stage: line.stage, state: "running", startedAt: at });
|
|
106
|
+
stages.push({ stage: line.stage, state: "running", startedAt: at, started: line.at });
|
|
95
107
|
break;
|
|
96
108
|
case "stage-ended": {
|
|
97
|
-
|
|
109
|
+
// An event run enters a failed stage again in the same attempt: the end is the open one's.
|
|
110
|
+
const started = stages.findLast(
|
|
111
|
+
(each) => each.stage === line.stage && each.state === "running",
|
|
112
|
+
);
|
|
98
113
|
const state = line.source ?? line.outcome;
|
|
99
|
-
if (started) Object.assign(started, { state, endedAt: at });
|
|
100
|
-
else
|
|
114
|
+
if (started && line.source === undefined) Object.assign(started, { state, endedAt: at });
|
|
115
|
+
else
|
|
116
|
+
stages.push({ stage: line.stage, state, startedAt: at, endedAt: at, started: line.at });
|
|
101
117
|
break;
|
|
102
118
|
}
|
|
103
119
|
case "agent-opened":
|
|
@@ -134,10 +150,11 @@ export async function viewRun(status: RunStatus): Promise<RunView> {
|
|
|
134
150
|
}
|
|
135
151
|
}
|
|
136
152
|
}
|
|
153
|
+
const unread = await withLines(stages, run.dir, attempt?.attempt);
|
|
137
154
|
const stage = currentStage(status, read.log.lines);
|
|
138
155
|
// A run whose process is gone left what it was doing undone.
|
|
139
156
|
if (status.state === "interrupted") {
|
|
140
|
-
for (const each of agents.values()) if (each.state === "working") each.state = "
|
|
157
|
+
for (const each of agents.values()) if (each.state === "working") each.state = "interrupted";
|
|
141
158
|
for (const each of stages) if (each.state === "running") each.state = "interrupted";
|
|
142
159
|
}
|
|
143
160
|
if (status.state === "running") {
|
|
@@ -153,6 +170,9 @@ export async function viewRun(status: RunStatus): Promise<RunView> {
|
|
|
153
170
|
} else if (phase?.kind === "awaiting-reply" && phase.peer !== undefined) {
|
|
154
171
|
agent(key).live = { phase: "awaiting-reply", peer: phase.peer, ...until };
|
|
155
172
|
}
|
|
173
|
+
// One word with `wfa peers`: the phase is the state; `live` keeps its detail.
|
|
174
|
+
const { live } = agent(key);
|
|
175
|
+
if (live && agent(key).state === "working") agent(key).state = live.phase;
|
|
156
176
|
}
|
|
157
177
|
}
|
|
158
178
|
const since = attempt ? Date.parse(attempt.started) : Number.NEGATIVE_INFINITY;
|
|
@@ -170,28 +190,72 @@ export async function viewRun(status: RunStatus): Promise<RunView> {
|
|
|
170
190
|
files: (files.get(session.n)?.files ?? []).map((file) => file.path),
|
|
171
191
|
});
|
|
172
192
|
}
|
|
173
|
-
const estimate = estimateOf(
|
|
193
|
+
const { estimate, unknown } = await estimateOf(status);
|
|
174
194
|
const lastLine = read.log.lines.at(-1)?.line;
|
|
175
195
|
return {
|
|
176
196
|
status,
|
|
177
197
|
...(attempt ? { attempt } : {}),
|
|
178
|
-
stages,
|
|
198
|
+
stages: stages.map(({ started: _started, ...each }) => each),
|
|
199
|
+
...(unread === undefined ? {} : { unread }),
|
|
179
200
|
...(stage === undefined ? {} : { stage }),
|
|
180
201
|
agents: [...agents.values()],
|
|
181
202
|
decisions,
|
|
182
203
|
...(estimate === undefined ? {} : { estimate }),
|
|
204
|
+
unknownCost: unknown,
|
|
183
205
|
...(lastLine ? { lastEventAt: Date.parse(timeOf(lastLine)) } : {}),
|
|
184
206
|
lines: read.log.lines,
|
|
185
207
|
cursor: read.cursor,
|
|
186
208
|
};
|
|
187
209
|
}
|
|
188
210
|
|
|
211
|
+
/**
|
|
212
|
+
* Each stage's outcome and value from `run.jsonl`. One that ran or was given takes its own line, of
|
|
213
|
+
* attempt `attempt` with its name and `started`, ties by the order entered; a reused one takes the
|
|
214
|
+
* line it reused, its stage's settled one. A stage running, or one the plan stopped, has no line,
|
|
215
|
+
* and keeps what the log says.
|
|
216
|
+
*/
|
|
217
|
+
async function withLines(
|
|
218
|
+
stages: (StageView & { started?: string })[],
|
|
219
|
+
runDir: string,
|
|
220
|
+
attempt: number | undefined,
|
|
221
|
+
): Promise<string | undefined> {
|
|
222
|
+
let view: Awaited<ReturnType<typeof readRunLines>>["view"];
|
|
223
|
+
try {
|
|
224
|
+
({ view } = await readRunLines(runDir));
|
|
225
|
+
} catch (error) {
|
|
226
|
+
// A run whose lines can't be read still shows its story from the log, and says what it lacks.
|
|
227
|
+
return `run.jsonl could not be read, so stages show no values: ${messageOf(error)}`;
|
|
228
|
+
}
|
|
229
|
+
if (attempt === undefined) return undefined;
|
|
230
|
+
const taken = new Set<unknown>();
|
|
231
|
+
for (const entry of stages) {
|
|
232
|
+
const stage = view.stages[entry.stage];
|
|
233
|
+
if (!stage) continue;
|
|
234
|
+
if (entry.state === "reused") {
|
|
235
|
+
// The attempt shown is the latest, and a stage it reused it can't run again: no later
|
|
236
|
+
// success than the one it reused.
|
|
237
|
+
if (stage.settled?.value !== undefined) entry.value = stage.settled.value;
|
|
238
|
+
continue;
|
|
239
|
+
}
|
|
240
|
+
const line = stage.history.find(
|
|
241
|
+
(each) => each.attempt === attempt && each.started === entry.started && !taken.has(each),
|
|
242
|
+
);
|
|
243
|
+
if (!line) continue;
|
|
244
|
+
taken.add(line);
|
|
245
|
+
if (entry.state !== "provided") entry.state = line.outcome;
|
|
246
|
+
// The log is best effort: an end it lost is the line's.
|
|
247
|
+
entry.endedAt ??= Date.parse(line.at);
|
|
248
|
+
if (line.value !== undefined) entry.value = line.value;
|
|
249
|
+
}
|
|
250
|
+
return undefined;
|
|
251
|
+
}
|
|
252
|
+
|
|
189
253
|
/** The latest phase a call's liveness file holds, which is diagnostics, best effort. */
|
|
190
254
|
async function latestPhase(
|
|
191
255
|
runDir: string,
|
|
192
256
|
operationId: string,
|
|
193
257
|
): Promise<OperationLivenessRecord | undefined> {
|
|
194
|
-
const file =
|
|
258
|
+
const file = liveFile(runDir, operationId);
|
|
195
259
|
return (await readLines<OperationLivenessRecord>(file).catch(() => [])).at(-1);
|
|
196
260
|
}
|
|
197
261
|
|
package/src/runs-command.ts
CHANGED
|
@@ -1,28 +1,21 @@
|
|
|
1
1
|
import { homedir } from "node:os";
|
|
2
2
|
import type { RunJson, RunsJson } from "@agentswf/contract/records";
|
|
3
|
-
import {
|
|
3
|
+
import type { JsonValue } from "@agentswf/contract/workflow";
|
|
4
|
+
import { duration } from "./accounting/format";
|
|
5
|
+
import { command, type Out, parsePlace, type ReadPlace, shellLine } from "./command-line";
|
|
4
6
|
import { messageOf } from "./errors";
|
|
5
7
|
import help from "./help/runs.txt" with { type: "text" };
|
|
6
|
-
import {
|
|
7
|
-
|
|
8
|
-
command,
|
|
9
|
-
continueOf,
|
|
10
|
-
estimateOf,
|
|
11
|
-
failed,
|
|
12
|
-
masked,
|
|
13
|
-
money,
|
|
14
|
-
nextLogs,
|
|
15
|
-
type Out,
|
|
16
|
-
parsePlace,
|
|
17
|
-
type ReadPlace,
|
|
18
|
-
said,
|
|
19
|
-
shellLine,
|
|
20
|
-
span,
|
|
21
|
-
stateWord,
|
|
22
|
-
} from "./run-commands";
|
|
8
|
+
import { asksJson, printRefusal } from "./refusals";
|
|
9
|
+
import { continueOf, estimateOf, failed, nextLogs, span, stateWord } from "./run-commands";
|
|
23
10
|
import { encodeCursor } from "./run-read";
|
|
24
|
-
import { findRunByRef,
|
|
11
|
+
import { findRunByRef, listRunEntries, type RunStatus, runStatus } from "./run-state";
|
|
12
|
+
import { clockTime, masked, money, said } from "./run-text";
|
|
25
13
|
import { type AgentView, describeActivity, type RunView, viewRun } from "./run-view";
|
|
14
|
+
import { isClaimed, RunUnreadable } from "./runs";
|
|
15
|
+
|
|
16
|
+
/** `awf runs --json`'s `"v"`, the list's, and `awf runs {run} --json`'s. */
|
|
17
|
+
export const RUNS_V = 1 satisfies RunsJson["v"];
|
|
18
|
+
export const RUNS_ONE_V = 2 satisfies RunJson["v"];
|
|
26
19
|
|
|
27
20
|
export const runsUsage = help.trimEnd();
|
|
28
21
|
|
|
@@ -35,10 +28,6 @@ export async function runsCommand(
|
|
|
35
28
|
now: number = Date.now(),
|
|
36
29
|
home: string = homedir(),
|
|
37
30
|
): Promise<number> {
|
|
38
|
-
if (argv.includes("-h") || argv.includes("--help")) {
|
|
39
|
-
out.stdout(runsUsage);
|
|
40
|
-
return 0;
|
|
41
|
-
}
|
|
42
31
|
let parsed: ReturnType<typeof parsePlace>;
|
|
43
32
|
let limit: number | undefined;
|
|
44
33
|
let ref: string | undefined;
|
|
@@ -57,7 +46,7 @@ export async function runsCommand(
|
|
|
57
46
|
if (ref !== undefined && limit !== undefined)
|
|
58
47
|
throw new Error("--limit lists runs; it takes no run");
|
|
59
48
|
} catch (error) {
|
|
60
|
-
out
|
|
49
|
+
printRefusal(out, { json: asksJson(argv), v: runsVersion(argv) }, error, "usage", runsUsage);
|
|
61
50
|
return 2;
|
|
62
51
|
}
|
|
63
52
|
const { place, json } = parsed;
|
|
@@ -69,34 +58,52 @@ export async function runsCommand(
|
|
|
69
58
|
);
|
|
70
59
|
return 0;
|
|
71
60
|
} catch (error) {
|
|
72
|
-
return failed(error, place, out, messageOf(error));
|
|
61
|
+
return failed(error, place, out, messageOf(error), { json, v: runsVersion(argv) });
|
|
73
62
|
}
|
|
74
63
|
}
|
|
75
64
|
|
|
65
|
+
/**
|
|
66
|
+
* `awf runs {run} --json`'s `"v"` is 2, the list's 1: by whether a word names a run, read before
|
|
67
|
+
* any parse, so a usage error after it or before it carries the same.
|
|
68
|
+
*/
|
|
69
|
+
function runsVersion(argv: readonly string[]): typeof RUNS_V | typeof RUNS_ONE_V {
|
|
70
|
+
for (let index = 0; index < argv.length; index++) {
|
|
71
|
+
const word = argv[index]!;
|
|
72
|
+
if (word === "--cwd" || word === "--run-root" || word === "--limit") index++;
|
|
73
|
+
else if (!word.startsWith("-")) return RUNS_ONE_V;
|
|
74
|
+
}
|
|
75
|
+
return RUNS_V;
|
|
76
|
+
}
|
|
77
|
+
|
|
76
78
|
/** A run listed, or why it could not be read: one bad run does not hide the others. */
|
|
77
|
-
type Listed =
|
|
79
|
+
type Listed =
|
|
80
|
+
| { status: RunStatus; estimate?: number; unknown: number }
|
|
81
|
+
| { ref: string; unreadable: string };
|
|
78
82
|
|
|
79
83
|
async function list(place: ReadPlace, limit: number, json: boolean, out: Out, now: number) {
|
|
80
|
-
const runs = await
|
|
84
|
+
const runs = await listRunEntries(place.runRoot);
|
|
81
85
|
const shown = runs.slice(0, limit);
|
|
82
86
|
const listed: Listed[] = await Promise.all(
|
|
83
87
|
shown.map((run) =>
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
88
|
+
run instanceof RunUnreadable
|
|
89
|
+
? { ref: run.ref, unreadable: run.why }
|
|
90
|
+
: runStatus(run, undefined, now).then(
|
|
91
|
+
async (status) => {
|
|
92
|
+
return { status, ...(await estimateOf(status)) };
|
|
93
|
+
},
|
|
94
|
+
(error: unknown) => ({
|
|
95
|
+
ref: `${run.record.workflow}/${run.record.id}`,
|
|
96
|
+
unreadable: said(messageOf(error), 400),
|
|
97
|
+
}),
|
|
98
|
+
),
|
|
91
99
|
),
|
|
92
100
|
);
|
|
93
101
|
const older = runs.length - shown.length;
|
|
94
102
|
if (json) {
|
|
95
103
|
const runsJson: RunsJson["runs"] = listed.map((each) => {
|
|
96
104
|
if (!("status" in each)) return each;
|
|
97
|
-
const { status } = each;
|
|
105
|
+
const { status, estimate } = each;
|
|
98
106
|
const { started, ended } = span(status, now);
|
|
99
|
-
const estimate = estimateOf(status.attempts);
|
|
100
107
|
return {
|
|
101
108
|
ref: status.ref,
|
|
102
109
|
state: status.state,
|
|
@@ -108,7 +115,7 @@ async function list(place: ReadPlace, limit: number, json: boolean, out: Out, no
|
|
|
108
115
|
};
|
|
109
116
|
});
|
|
110
117
|
const document: RunsJson = {
|
|
111
|
-
v:
|
|
118
|
+
v: RUNS_V,
|
|
112
119
|
runs: runsJson,
|
|
113
120
|
...(older > 0 ? { more: { older: true } } : {}),
|
|
114
121
|
};
|
|
@@ -125,16 +132,15 @@ async function list(place: ReadPlace, limit: number, json: boolean, out: Out, no
|
|
|
125
132
|
const stateWidth = Math.max(13, ...states.map((each) => each.length + 2));
|
|
126
133
|
const lines = listed.map((each, index) => {
|
|
127
134
|
if (!("status" in each)) return `${each.ref.padEnd(width)}unreadable: ${each.unreadable}`;
|
|
128
|
-
const { status } = each;
|
|
135
|
+
const { status, estimate, unknown } = each;
|
|
129
136
|
const { started, ms } = span(status, now);
|
|
130
|
-
const estimate = estimateOf(status.attempts);
|
|
131
137
|
const waiting = waitingText(status);
|
|
132
138
|
return [
|
|
133
139
|
status.ref.padEnd(width),
|
|
134
140
|
states[index]!.padEnd(stateWidth),
|
|
135
141
|
dayTime(started).padEnd(13),
|
|
136
|
-
duration(ms).padEnd(
|
|
137
|
-
money(estimate)
|
|
142
|
+
duration(ms).padEnd(9),
|
|
143
|
+
money(estimate, unknown)
|
|
138
144
|
.join("")
|
|
139
145
|
.concat(waiting ? " " : ""),
|
|
140
146
|
waiting,
|
|
@@ -150,7 +156,7 @@ async function list(place: ReadPlace, limit: number, json: boolean, out: Out, no
|
|
|
150
156
|
/** A run at its wait: its label, and until when; a returned one, the events it never took. */
|
|
151
157
|
function waitingText(status: RunStatus): string {
|
|
152
158
|
const { waiting } = status;
|
|
153
|
-
const never = status.
|
|
159
|
+
const never = status.unacknowledged;
|
|
154
160
|
if (!waiting) return never ? `${never} event${never === 1 ? "" : "s"} never taken` : "";
|
|
155
161
|
const until = waiting.until ? `until ${dayTime(Date.parse(waiting.until))}` : "";
|
|
156
162
|
return [waiting.label, until].filter(Boolean).join(" · ") || "waiting";
|
|
@@ -162,13 +168,15 @@ function dayTime(at: number): string {
|
|
|
162
168
|
return `${two(date.getMonth() + 1)}-${two(date.getDate())} ${clockTime(at).slice(0, 5)}`;
|
|
163
169
|
}
|
|
164
170
|
|
|
165
|
-
const
|
|
171
|
+
const inTurn = (agent: AgentView) =>
|
|
172
|
+
agent.state === "working" || agent.state === "waiting" || agent.state === "awaiting-reply";
|
|
173
|
+
const busy = (agent: AgentView) => inTurn(agent) || agent.state === "interrupted";
|
|
166
174
|
|
|
167
175
|
function agentLine(agent: AgentView, indent: string, now: number, width: number): string {
|
|
168
176
|
const { live } = agent;
|
|
169
177
|
const parts = [
|
|
170
178
|
said(agent.key, 60).padEnd(width),
|
|
171
|
-
|
|
179
|
+
agent.state.padEnd(14),
|
|
172
180
|
busy(agent) ? `turn ${agent.turns + 1}` : `${agent.turns} turn${agent.turns === 1 ? "" : "s"}`,
|
|
173
181
|
...(live?.phase === "waiting"
|
|
174
182
|
? [
|
|
@@ -181,10 +189,10 @@ function agentLine(agent: AgentView, indent: string, now: number, width: number)
|
|
|
181
189
|
]
|
|
182
190
|
: live
|
|
183
191
|
? [
|
|
184
|
-
|
|
192
|
+
`from ${said(live.peer, 60)}${live.until ? ` (until ${clockTime(live.until).slice(0, 5)})` : ""}`,
|
|
185
193
|
]
|
|
186
194
|
: []),
|
|
187
|
-
...(agent.since && agent
|
|
195
|
+
...(agent.since && inTurn(agent) ? [duration(now - agent.since)] : []),
|
|
188
196
|
...(agent.last
|
|
189
197
|
? [`last: ${describeActivity(agent.last.activity)} (${clockTime(agent.last.at)})`]
|
|
190
198
|
: []),
|
|
@@ -228,7 +236,7 @@ function describe(view: RunView, place: ReadPlace, now: number): string {
|
|
|
228
236
|
[
|
|
229
237
|
status.ref,
|
|
230
238
|
stateWord(status, view.lastEventAt),
|
|
231
|
-
...(attempt && status.
|
|
239
|
+
...(attempt && status.files.length > 1
|
|
232
240
|
? [
|
|
233
241
|
`attempt ${attempt.attempt}${attempt.flags.fromStage ? ` (from stage ${attempt.flags.fromStage})` : ""}`,
|
|
234
242
|
]
|
|
@@ -237,17 +245,22 @@ function describe(view: RunView, place: ReadPlace, now: number): string {
|
|
|
237
245
|
status.state === "interrupted" && view.lastEventAt
|
|
238
246
|
? `last event ${clockTime(view.lastEventAt)}`
|
|
239
247
|
: duration(span(status, now).ms),
|
|
240
|
-
...money(view.estimate),
|
|
248
|
+
...money(view.estimate, view.unknownCost),
|
|
241
249
|
].join(" "),
|
|
242
250
|
];
|
|
243
|
-
|
|
251
|
+
// Every other attempt file, a launch that never claimed its number included.
|
|
252
|
+
for (const earlier of status.files) {
|
|
253
|
+
if (earlier.attempt === attempt?.attempt) continue;
|
|
244
254
|
const took = earlier.ended
|
|
245
255
|
? ` ${duration(Date.parse(earlier.ended) - Date.parse(earlier.started))}`
|
|
246
256
|
: "";
|
|
247
|
-
const
|
|
257
|
+
const stage = isClaimed(earlier) ? earlier.stage : undefined;
|
|
258
|
+
const how = isClaimed(earlier)
|
|
259
|
+
? `${earlier.outcome ?? "interrupted"}${stage ? ` in ${stage}` : ""}`
|
|
260
|
+
: "failed to launch";
|
|
248
261
|
lines.push(` attempt ${earlier.attempt} ${how}${took}`);
|
|
249
262
|
}
|
|
250
|
-
const indent = status.
|
|
263
|
+
const indent = status.files.length > 1 ? " " : "";
|
|
251
264
|
if (indent && attempt) lines.push(` attempt ${attempt.attempt}`);
|
|
252
265
|
const shown = new Set<string>();
|
|
253
266
|
for (const stage of view.stages) {
|
|
@@ -262,6 +275,7 @@ function describe(view: RunView, place: ReadPlace, now: number): string {
|
|
|
262
275
|
for (const agent of own) shown.add(agent.key);
|
|
263
276
|
lines.push(...agentLines(own, indent, now));
|
|
264
277
|
}
|
|
278
|
+
if (view.unread) lines.push(`${indent} awf: ${view.unread}`);
|
|
265
279
|
lines.push(
|
|
266
280
|
...agentLines(
|
|
267
281
|
view.agents.filter((each) => !shown.has(each.key)),
|
|
@@ -287,7 +301,7 @@ function trailer(view: RunView, place: ReadPlace): string[] {
|
|
|
287
301
|
if (status.state === "interrupted") return go ? [`continue: ${go}`] : [];
|
|
288
302
|
const lines: string[] = [];
|
|
289
303
|
if (view.attempt?.reason) lines.push(`reason: ${said(view.attempt.reason, 400)}`);
|
|
290
|
-
lines.push(`records: ${status.run.dir} (
|
|
304
|
+
lines.push(`records: ${status.run.dir} (run.jsonl, log.jsonl, ops/, out.json, report.md)`);
|
|
291
305
|
if (status.state === "failed") {
|
|
292
306
|
const stuck = view.agents.find((each) => each.state === "failed" || each.state === "timed-out");
|
|
293
307
|
const look = command(place, "awf", "logs", status.ref);
|
|
@@ -309,7 +323,7 @@ function runJson(view: RunView, place: ReadPlace, now: number): RunJson {
|
|
|
309
323
|
? undefined
|
|
310
324
|
: continueOf(status, view.lines, place);
|
|
311
325
|
return {
|
|
312
|
-
v:
|
|
326
|
+
v: RUNS_ONE_V,
|
|
313
327
|
run: {
|
|
314
328
|
ref: status.ref,
|
|
315
329
|
state: status.state,
|
|
@@ -320,20 +334,26 @@ function runJson(view: RunView, place: ReadPlace, now: number): RunJson {
|
|
|
320
334
|
...(status.waiting ? { waiting: status.waiting } : {}),
|
|
321
335
|
dir: status.run.dir,
|
|
322
336
|
},
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
started,
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
337
|
+
// Every attempt file, a launch that never claimed its number included, `failed` with its reason.
|
|
338
|
+
attempts: status.files.map((each) => {
|
|
339
|
+
const { attempt, started, ended, outcome, reason } = each;
|
|
340
|
+
const stage = isClaimed(each) ? each.stage : undefined;
|
|
341
|
+
return {
|
|
342
|
+
attempt,
|
|
343
|
+
started,
|
|
344
|
+
...(ended ? { ended } : {}),
|
|
345
|
+
state:
|
|
346
|
+
outcome ?? (attempt === status.attempts.at(-1)?.attempt ? status.state : "interrupted"),
|
|
347
|
+
...(stage ? { stage } : {}),
|
|
348
|
+
...(reason ? { reason: masked(reason, 1_000) as string } : {}),
|
|
349
|
+
};
|
|
350
|
+
}),
|
|
351
|
+
stages: view.stages.map(({ stage, state, startedAt, endedAt, value }) => ({
|
|
333
352
|
stage,
|
|
334
353
|
state,
|
|
335
354
|
...(startedAt ? { started: new Date(startedAt).toISOString() } : {}),
|
|
336
355
|
...(endedAt ? { ended: new Date(endedAt).toISOString() } : {}),
|
|
356
|
+
...(value === undefined ? {} : { value: masked(value) as JsonValue }),
|
|
337
357
|
})),
|
|
338
358
|
agents: view.agents.map((agent) => ({
|
|
339
359
|
key: agent.key,
|