@agentswf/engine 0.1.0-beta.5 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +16 -0
- package/package.json +13 -9
- package/src/accounting/format.ts +4 -3
- package/src/accounting/spent.ts +128 -0
- package/src/accounting/summary.ts +40 -32
- package/src/agent-cli/cli.ts +28 -55
- package/src/agent-cli/client.ts +1 -27
- package/src/allowance-command.ts +8 -10
- package/src/attempt-ending.ts +4 -4
- package/src/attempt-output.ts +13 -31
- package/src/attempt-view.ts +33 -32
- package/src/author-typebox.ts +13 -0
- package/src/client/checks.ts +94 -0
- package/src/client/exported.ts +17 -0
- package/src/client/index.ts +122 -0
- package/src/client/locate.ts +34 -0
- package/src/client/published.ts +3 -0
- package/src/client/runs.ts +337 -0
- package/src/command-line.ts +64 -0
- package/src/commands.ts +7 -1
- package/src/control-plane.ts +4 -6
- package/src/coordinator-prompt.ts +4 -3
- package/src/coordinator-socket.ts +27 -28
- package/src/deadlines.ts +6 -40
- package/src/decisions/directory.ts +50 -37
- package/src/decisions/fake.ts +1 -1
- package/src/decisions/openrouter.ts +4 -2
- package/src/decisions/seam.ts +4 -4
- package/src/display-path.ts +3 -3
- package/src/duration.ts +19 -33
- package/src/emit-command.ts +38 -22
- package/src/emit.ts +148 -59
- package/src/event-ids.ts +31 -0
- package/src/files.ts +44 -4
- package/src/hand-over.ts +150 -0
- package/src/help/allowance.txt +4 -3
- package/src/help/emit.txt +11 -7
- package/src/help/inbox.txt +5 -3
- package/src/help/logs.txt +7 -5
- package/src/help/open.txt +29 -0
- package/src/help/reply.txt +6 -8
- package/src/help/run.txt +13 -8
- package/src/help/runs.txt +4 -2
- package/src/help/tick.txt +9 -4
- package/src/herdr-placement.ts +3 -33
- package/src/herdr-run-session.ts +5 -13
- package/src/herdr-workspace-marks.ts +7 -17
- package/src/holder-command.ts +54 -136
- package/src/in-lines.ts +28 -0
- package/src/inbox-command.ts +42 -56
- package/src/index.ts +1 -1
- package/src/init-command.ts +8 -17
- package/src/init-template/hello/workflow.ts.template +3 -2
- package/src/jsonl.ts +10 -6
- package/src/keep-alive.ts +7 -19
- package/src/logs-command.ts +134 -53
- package/src/message-operation.ts +6 -3
- package/src/messaging/messaging.ts +8 -10
- package/src/messaging/text.ts +2 -2
- package/src/open-command.ts +770 -0
- package/src/operation-events.ts +4 -6
- package/src/operation-liveness.ts +19 -4
- package/src/operator-cli.ts +206 -90
- package/src/operator-runtime.ts +24 -2
- package/src/ops.ts +270 -0
- package/src/pane-layout.ts +49 -96
- package/src/progress-view.ts +2 -2
- package/src/project-engine.ts +79 -0
- package/src/projects.ts +13 -2
- package/src/refusals.ts +119 -0
- package/src/result-slots.ts +11 -31
- package/src/result-validation.ts +2 -2
- package/src/run-command.ts +27 -31
- package/src/run-commands.ts +42 -114
- package/src/run-event-log.ts +1 -1
- package/src/run-lines.ts +159 -0
- package/src/run-lock.ts +57 -1
- package/src/run-policy/stage-plan.ts +154 -0
- package/src/run-policy/stages.ts +113 -0
- package/src/run-policy/waits.ts +452 -0
- package/src/run-prepare.ts +165 -41
- package/src/run-read.ts +109 -41
- package/src/run-state.ts +170 -50
- package/src/run-text.ts +54 -0
- package/src/run-unreachable.ts +18 -0
- package/src/run-usage.ts +17 -11
- package/src/run-view.ts +73 -14
- package/src/runs-command.ts +81 -63
- package/src/runs.ts +284 -222
- package/src/sandbox-homes.ts +65 -13
- package/src/sandboxes.ts +182 -28
- package/src/session-lines.ts +93 -0
- package/src/skills/run-skills.ts +8 -1
- package/src/sources.ts +2 -39
- package/src/spend.ts +333 -0
- package/src/stage-ledger.ts +111 -153
- package/src/stop-crashed.ts +6 -5
- package/src/stopped.ts +4 -4
- package/src/test-command.ts +37 -19
- package/src/testing/operation-events.ts +3 -23
- package/src/testing.ts +66 -18
- package/src/tick-command.ts +145 -15
- package/src/tools.ts +11 -30
- package/src/turn-ends.ts +37 -0
- package/src/waits.ts +651 -0
- package/src/wake.ts +84 -54
- package/src/workflow-loader.ts +49 -2
- package/src/workflow-runner.ts +525 -618
- package/src/workflow-test-preload.ts +4 -1
- package/src/workflow-testing/coordinators.ts +1 -1
- package/src/workflow-testing/fake-client.ts +387 -0
- package/src/workflow-testing/host.ts +28 -29
- package/src/workflow-testing/index.ts +9 -27
- package/src/workflow-testing/run-test.ts +201 -286
- package/src/workflow-testing/script.ts +7 -39
- package/src/deprecations.ts +0 -177
- package/src/hold-run-lock.fixture.ts +0 -8
- package/src/mailbox.ts +0 -972
- package/src/stage-plan.ts +0 -186
- package/src/workflow-testing/coordinator.typecheck.ts +0 -54
- package/src/workflow-testing/workflow-testing.typecheck.ts +0 -67
package/src/run-text.ts
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
// How the commands that read runs show what a run wrote, what it cost, and when.
|
|
2
|
+
import { maskCredentials } from "@agentswf/harness";
|
|
3
|
+
import { usd } from "./accounting/format";
|
|
4
|
+
|
|
5
|
+
/** A field whose name says it holds a credential, whatever its value looks like. */
|
|
6
|
+
const SECRET_NAME =
|
|
7
|
+
/password|passwd|passphrase|passcode|secret|token|credential|cookie|bearer|authorization|(?:api|access|private|session|ssh|signing|encryption)[_-]?key|^(?:pwd|pass|pin|auth)$/i;
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Every string in `value` masked, and everything under a credential's name hidden: a run's text,
|
|
11
|
+
* errors and fields are its workflow's, its agents' and its tools'. Past `limit`, a string is cut.
|
|
12
|
+
*/
|
|
13
|
+
export function masked(value: unknown, limit = Number.POSITIVE_INFINITY, hidden = false): unknown {
|
|
14
|
+
if (typeof value === "string") {
|
|
15
|
+
if (hidden) return "***";
|
|
16
|
+
const shown = maskCredentials(value);
|
|
17
|
+
return shown.length > limit ? `${shown.slice(0, limit - 1)}…` : shown;
|
|
18
|
+
}
|
|
19
|
+
if (hidden && value !== null && typeof value !== "object") return "***";
|
|
20
|
+
if (Array.isArray(value)) return value.map((each) => masked(each, limit, hidden));
|
|
21
|
+
if (value && typeof value === "object") {
|
|
22
|
+
return Object.fromEntries(
|
|
23
|
+
Object.entries(value).map(([key, each]) => [
|
|
24
|
+
key,
|
|
25
|
+
masked(each, limit, hidden || SECRET_NAME.test(key)),
|
|
26
|
+
]),
|
|
27
|
+
);
|
|
28
|
+
}
|
|
29
|
+
return value;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Text a run wrote, as one masked line of about `limit` characters, so it can never pass for a
|
|
34
|
+
* line of its own or fill a page. `@{handle} --full` prints it whole.
|
|
35
|
+
*/
|
|
36
|
+
export function said(text: string, limit = 120): string {
|
|
37
|
+
const line = maskCredentials(text).replace(/\s*[\r\n]+\s*/g, " ⏎ ");
|
|
38
|
+
return line.length > limit ? `${line.slice(0, limit - 1)}…` : line;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* As `usd` says it, and nothing for none; `?` after it when some cost is unknown, which makes it a
|
|
43
|
+
* floor.
|
|
44
|
+
*/
|
|
45
|
+
export function money(estimate: number | undefined, unknown = 0): string[] {
|
|
46
|
+
if (!estimate) return [];
|
|
47
|
+
return [`${usd(estimate)}${unknown > 0 ? "?" : ""}`];
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export function clockTime(at: number | string): string {
|
|
51
|
+
const date = new Date(at);
|
|
52
|
+
const two = (n: number) => String(n).padStart(2, "0");
|
|
53
|
+
return `${two(date.getHours())}:${two(date.getMinutes())}:${two(date.getSeconds())}`;
|
|
54
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/** A run's files that can't be read, or a run that doesn't answer: refused `unreachable`. */
|
|
2
|
+
export class RunUnreachable extends Error {
|
|
3
|
+
readonly code = "unreachable";
|
|
4
|
+
constructor(message: string) {
|
|
5
|
+
super(message);
|
|
6
|
+
this.name = "RunUnreachable";
|
|
7
|
+
}
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* An error the system gave, `ENOENT`, `EACCES` and the like, or one awf wrapped around it as its
|
|
12
|
+
* cause: a run's file that couldn't be read or written.
|
|
13
|
+
*/
|
|
14
|
+
export function isSystemError(error: unknown): boolean {
|
|
15
|
+
const { code, cause } = (error ?? {}) as { code?: unknown; cause?: unknown };
|
|
16
|
+
if (typeof code === "string" && /^E[A-Z]+$/.test(code)) return true;
|
|
17
|
+
return cause !== undefined && cause !== error && isSystemError(cause);
|
|
18
|
+
}
|
package/src/run-usage.ts
CHANGED
|
@@ -19,9 +19,9 @@ import { abortableSleep, deadlineWithin, runUntilStopped, waitForDeadline } from
|
|
|
19
19
|
*/
|
|
20
20
|
const FINISHING_GRACE_MILLISECONDS = 10_000;
|
|
21
21
|
/** Reading spend waits for files a released turn is still writing; it gives up after this. */
|
|
22
|
-
const ACCOUNTING_GRACE_MILLISECONDS = 20_000;
|
|
22
|
+
export const ACCOUNTING_GRACE_MILLISECONDS = 20_000;
|
|
23
23
|
/** Beyond that, the host's bound on a billing status command, and this much more. */
|
|
24
|
-
const STATUS_SLACK_MILLISECONDS = 2_000;
|
|
24
|
+
export const STATUS_SLACK_MILLISECONDS = 2_000;
|
|
25
25
|
|
|
26
26
|
/** One logical agent, as the run knows it. */
|
|
27
27
|
export type AccountedAgent = {
|
|
@@ -45,11 +45,12 @@ export type AccountedAgent = {
|
|
|
45
45
|
type OperationEntry = {
|
|
46
46
|
/**
|
|
47
47
|
* Records what is known when the operation settles; spend and billing wait for the run's end.
|
|
48
|
-
* `finishing` is an answered turn the host left finishing, whose
|
|
48
|
+
* `finishing` is an answered turn the host left finishing, whose reported costs are added when
|
|
49
|
+
* it ends.
|
|
49
50
|
*/
|
|
50
51
|
settle(
|
|
51
52
|
times: { deliveredAt?: number; settledAt: number },
|
|
52
|
-
|
|
53
|
+
reportedCosts: readonly number[],
|
|
53
54
|
finishing?: Promise<HarnessTurnOutcome>,
|
|
54
55
|
): OperationRecord;
|
|
55
56
|
};
|
|
@@ -62,6 +63,8 @@ export type OperationTags = Pick<OperationRecord, "stage" | "label" | "cause">;
|
|
|
62
63
|
|
|
63
64
|
/** One agent's part of the run's ledger. */
|
|
64
65
|
export type AgentLedger = {
|
|
66
|
+
/** Where it runs now: its sandbox's folder once it has one. */
|
|
67
|
+
cwd(): string;
|
|
65
68
|
/** `execution` is read as the operation settles: the agent's settings it ran at. */
|
|
66
69
|
reserve(
|
|
67
70
|
operationId: string,
|
|
@@ -125,21 +128,22 @@ export function createRunLedger({
|
|
|
125
128
|
agent(agent) {
|
|
126
129
|
agents.push(agent);
|
|
127
130
|
return {
|
|
131
|
+
cwd: () => agent.cwd,
|
|
128
132
|
reserve(operationId, execution, tags = {}) {
|
|
129
133
|
const operation: Operation = { agent, costs: [] };
|
|
130
134
|
operations.push(operation);
|
|
131
135
|
return {
|
|
132
|
-
settle(times,
|
|
136
|
+
settle(times, reportedCosts, finishing) {
|
|
133
137
|
operation.usage = {
|
|
134
138
|
...recordOf(agent, operationId, execution(), tags, times.deliveredAt),
|
|
135
139
|
settledAt: new Date(times.settledAt).toISOString(),
|
|
136
140
|
};
|
|
137
|
-
operation.costs = [...
|
|
141
|
+
operation.costs = [...reportedCosts];
|
|
138
142
|
if (finishing) {
|
|
139
143
|
late.add(
|
|
140
144
|
finishing.then(
|
|
141
|
-
({
|
|
142
|
-
operation.costs.push(...
|
|
145
|
+
({ reportedCostsUsd }) => {
|
|
146
|
+
operation.costs.push(...reportedCostsUsd);
|
|
143
147
|
},
|
|
144
148
|
() => undefined,
|
|
145
149
|
),
|
|
@@ -175,7 +179,7 @@ export function createRunLedger({
|
|
|
175
179
|
try {
|
|
176
180
|
return await runUntilStopped(
|
|
177
181
|
async () => {
|
|
178
|
-
// Closing the host ended every turn left finishing; what each
|
|
182
|
+
// Closing the host ended every turn left finishing; what each reported lands here.
|
|
179
183
|
await waitForDeadline(lateEnded(), { unixMilliseconds: deadline }).catch(
|
|
180
184
|
() => undefined,
|
|
181
185
|
);
|
|
@@ -254,12 +258,14 @@ async function settleUsage(
|
|
|
254
258
|
return operations.map((operation) => {
|
|
255
259
|
const { billing, shares } = settledAgents.get(operation.agent)!;
|
|
256
260
|
const { costs } = operation;
|
|
257
|
-
const
|
|
261
|
+
const reportedCost = billing === "metered" && costs.length > 0 ? sum(costs) : undefined;
|
|
258
262
|
return {
|
|
259
263
|
...withSessions(operation),
|
|
260
264
|
billing,
|
|
261
265
|
...(shares ? { spend: spendOf(shares.get(operation.usage.operationId) ?? []) } : {}),
|
|
262
|
-
...(
|
|
266
|
+
...(reportedCost === undefined
|
|
267
|
+
? {}
|
|
268
|
+
: { reportedCost: { amount: reportedCost, currency: "USD" } }),
|
|
263
269
|
};
|
|
264
270
|
});
|
|
265
271
|
}
|
package/src/run-view.ts
CHANGED
|
@@ -1,24 +1,30 @@
|
|
|
1
|
-
import { join } from "node:path";
|
|
2
1
|
import type {
|
|
3
2
|
Activity,
|
|
4
3
|
AgentStateJson,
|
|
5
|
-
|
|
4
|
+
ClaimedAttempt,
|
|
6
5
|
OperationLivenessRecord,
|
|
7
6
|
RunLogLine,
|
|
8
7
|
StageStateJson,
|
|
9
8
|
} from "@agentswf/contract/records";
|
|
10
|
-
import type { AgentKey, HarnessKind } from "@agentswf/contract/workflow";
|
|
9
|
+
import type { AgentKey, HarnessKind, JsonValue } from "@agentswf/contract/workflow";
|
|
10
|
+
import { messageOf } from "./errors";
|
|
11
11
|
import { readLines } from "./jsonl";
|
|
12
|
+
import { liveFile } from "./ops";
|
|
12
13
|
import { currentStage, estimateOf } from "./run-commands";
|
|
14
|
+
import { readRunLines } from "./run-lines";
|
|
13
15
|
import { type Cursor, type FileCache, readRun, timeOf } from "./run-read";
|
|
14
16
|
import type { RunStatus } from "./run-state";
|
|
15
17
|
|
|
16
|
-
/**
|
|
18
|
+
/**
|
|
19
|
+
* A stage of the attempt shown: running, reused or given from an earlier one, or how it ended. Its
|
|
20
|
+
* outcome and value are its own `stage` line's, a reused one's the line it reused.
|
|
21
|
+
*/
|
|
17
22
|
export type StageView = {
|
|
18
23
|
stage: string;
|
|
19
24
|
state: StageStateJson;
|
|
20
25
|
startedAt?: number;
|
|
21
26
|
endedAt?: number;
|
|
27
|
+
value?: JsonValue;
|
|
22
28
|
};
|
|
23
29
|
|
|
24
30
|
/**
|
|
@@ -43,19 +49,23 @@ export type AgentView = {
|
|
|
43
49
|
sessions: { harness: HarnessKind; id: string; files: string[] }[];
|
|
44
50
|
};
|
|
45
51
|
|
|
46
|
-
|
|
52
|
+
type DecisionView = { decision: number; key: string; askedAt: number; outcome?: string };
|
|
47
53
|
|
|
48
54
|
export type RunView = {
|
|
49
55
|
status: RunStatus;
|
|
50
56
|
/** The attempt shown: the last. */
|
|
51
|
-
attempt?:
|
|
57
|
+
attempt?: ClaimedAttempt;
|
|
52
58
|
stages: StageView[];
|
|
59
|
+
/** Why the stages show no values: `run.jsonl` didn't read. */
|
|
60
|
+
unread?: string;
|
|
53
61
|
/** The stage running now, or the one the attempt ended in. */
|
|
54
62
|
stage?: string;
|
|
55
63
|
agents: AgentView[];
|
|
56
64
|
decisions: DecisionView[];
|
|
57
65
|
/** Over the attempts that kept one, at list prices: never what was billed. */
|
|
58
66
|
estimate?: number;
|
|
67
|
+
/** How many agent operations' usage isn't known: `estimate` is a floor. */
|
|
68
|
+
unknownCost: number;
|
|
59
69
|
lastEventAt?: number;
|
|
60
70
|
lines: { number: number; line: RunLogLine }[];
|
|
61
71
|
cursor: Cursor;
|
|
@@ -74,7 +84,7 @@ export async function viewRun(status: RunStatus): Promise<RunView> {
|
|
|
74
84
|
.map(({ line }) => line)
|
|
75
85
|
.filter((line) => attempt === undefined || line.attempt === attempt.attempt);
|
|
76
86
|
|
|
77
|
-
const stages: StageView[] = [];
|
|
87
|
+
const stages: (StageView & { started?: string })[] = [];
|
|
78
88
|
const decisions: DecisionView[] = [];
|
|
79
89
|
const agents = new Map<AgentKey, AgentView>();
|
|
80
90
|
const open = new Map<string, AgentKey>();
|
|
@@ -93,13 +103,17 @@ export async function viewRun(status: RunStatus): Promise<RunView> {
|
|
|
93
103
|
const at = Date.parse(timeOf(line));
|
|
94
104
|
switch (line.kind) {
|
|
95
105
|
case "stage-started":
|
|
96
|
-
stages.push({ stage: line.stage, state: "running", startedAt: at });
|
|
106
|
+
stages.push({ stage: line.stage, state: "running", startedAt: at, started: line.at });
|
|
97
107
|
break;
|
|
98
108
|
case "stage-ended": {
|
|
99
|
-
|
|
109
|
+
// An event run enters a failed stage again in the same attempt: the end is the open one's.
|
|
110
|
+
const started = stages.findLast(
|
|
111
|
+
(each) => each.stage === line.stage && each.state === "running",
|
|
112
|
+
);
|
|
100
113
|
const state = line.source ?? line.outcome;
|
|
101
|
-
if (started) Object.assign(started, { state, endedAt: at });
|
|
102
|
-
else
|
|
114
|
+
if (started && line.source === undefined) Object.assign(started, { state, endedAt: at });
|
|
115
|
+
else
|
|
116
|
+
stages.push({ stage: line.stage, state, startedAt: at, endedAt: at, started: line.at });
|
|
103
117
|
break;
|
|
104
118
|
}
|
|
105
119
|
case "agent-opened":
|
|
@@ -136,6 +150,7 @@ export async function viewRun(status: RunStatus): Promise<RunView> {
|
|
|
136
150
|
}
|
|
137
151
|
}
|
|
138
152
|
}
|
|
153
|
+
const unread = await withLines(stages, run.dir, attempt?.attempt);
|
|
139
154
|
const stage = currentStage(status, read.log.lines);
|
|
140
155
|
// A run whose process is gone left what it was doing undone.
|
|
141
156
|
if (status.state === "interrupted") {
|
|
@@ -175,28 +190,72 @@ export async function viewRun(status: RunStatus): Promise<RunView> {
|
|
|
175
190
|
files: (files.get(session.n)?.files ?? []).map((file) => file.path),
|
|
176
191
|
});
|
|
177
192
|
}
|
|
178
|
-
const estimate = estimateOf(
|
|
193
|
+
const { estimate, unknown } = await estimateOf(status);
|
|
179
194
|
const lastLine = read.log.lines.at(-1)?.line;
|
|
180
195
|
return {
|
|
181
196
|
status,
|
|
182
197
|
...(attempt ? { attempt } : {}),
|
|
183
|
-
stages,
|
|
198
|
+
stages: stages.map(({ started: _started, ...each }) => each),
|
|
199
|
+
...(unread === undefined ? {} : { unread }),
|
|
184
200
|
...(stage === undefined ? {} : { stage }),
|
|
185
201
|
agents: [...agents.values()],
|
|
186
202
|
decisions,
|
|
187
203
|
...(estimate === undefined ? {} : { estimate }),
|
|
204
|
+
unknownCost: unknown,
|
|
188
205
|
...(lastLine ? { lastEventAt: Date.parse(timeOf(lastLine)) } : {}),
|
|
189
206
|
lines: read.log.lines,
|
|
190
207
|
cursor: read.cursor,
|
|
191
208
|
};
|
|
192
209
|
}
|
|
193
210
|
|
|
211
|
+
/**
|
|
212
|
+
* Each stage's outcome and value from `run.jsonl`. One that ran or was given takes its own line, of
|
|
213
|
+
* attempt `attempt` with its name and `started`, ties by the order entered; a reused one takes the
|
|
214
|
+
* line it reused, its stage's settled one. A stage running, or one the plan stopped, has no line,
|
|
215
|
+
* and keeps what the log says.
|
|
216
|
+
*/
|
|
217
|
+
async function withLines(
|
|
218
|
+
stages: (StageView & { started?: string })[],
|
|
219
|
+
runDir: string,
|
|
220
|
+
attempt: number | undefined,
|
|
221
|
+
): Promise<string | undefined> {
|
|
222
|
+
let view: Awaited<ReturnType<typeof readRunLines>>["view"];
|
|
223
|
+
try {
|
|
224
|
+
({ view } = await readRunLines(runDir));
|
|
225
|
+
} catch (error) {
|
|
226
|
+
// A run whose lines can't be read still shows its story from the log, and says what it lacks.
|
|
227
|
+
return `run.jsonl could not be read, so stages show no values: ${messageOf(error)}`;
|
|
228
|
+
}
|
|
229
|
+
if (attempt === undefined) return undefined;
|
|
230
|
+
const taken = new Set<unknown>();
|
|
231
|
+
for (const entry of stages) {
|
|
232
|
+
const stage = view.stages[entry.stage];
|
|
233
|
+
if (!stage) continue;
|
|
234
|
+
if (entry.state === "reused") {
|
|
235
|
+
// The attempt shown is the latest, and a stage it reused it can't run again: no later
|
|
236
|
+
// success than the one it reused.
|
|
237
|
+
if (stage.settled?.value !== undefined) entry.value = stage.settled.value;
|
|
238
|
+
continue;
|
|
239
|
+
}
|
|
240
|
+
const line = stage.history.find(
|
|
241
|
+
(each) => each.attempt === attempt && each.started === entry.started && !taken.has(each),
|
|
242
|
+
);
|
|
243
|
+
if (!line) continue;
|
|
244
|
+
taken.add(line);
|
|
245
|
+
if (entry.state !== "provided") entry.state = line.outcome;
|
|
246
|
+
// The log is best effort: an end it lost is the line's.
|
|
247
|
+
entry.endedAt ??= Date.parse(line.at);
|
|
248
|
+
if (line.value !== undefined) entry.value = line.value;
|
|
249
|
+
}
|
|
250
|
+
return undefined;
|
|
251
|
+
}
|
|
252
|
+
|
|
194
253
|
/** The latest phase a call's liveness file holds, which is diagnostics, best effort. */
|
|
195
254
|
async function latestPhase(
|
|
196
255
|
runDir: string,
|
|
197
256
|
operationId: string,
|
|
198
257
|
): Promise<OperationLivenessRecord | undefined> {
|
|
199
|
-
const file =
|
|
258
|
+
const file = liveFile(runDir, operationId);
|
|
200
259
|
return (await readLines<OperationLivenessRecord>(file).catch(() => [])).at(-1);
|
|
201
260
|
}
|
|
202
261
|
|
package/src/runs-command.ts
CHANGED
|
@@ -1,28 +1,21 @@
|
|
|
1
1
|
import { homedir } from "node:os";
|
|
2
2
|
import type { RunJson, RunsJson } from "@agentswf/contract/records";
|
|
3
|
-
import {
|
|
3
|
+
import type { JsonValue } from "@agentswf/contract/workflow";
|
|
4
|
+
import { duration } from "./accounting/format";
|
|
5
|
+
import { command, type Out, parsePlace, type ReadPlace, shellLine } from "./command-line";
|
|
4
6
|
import { messageOf } from "./errors";
|
|
5
7
|
import help from "./help/runs.txt" with { type: "text" };
|
|
6
|
-
import {
|
|
7
|
-
|
|
8
|
-
command,
|
|
9
|
-
continueOf,
|
|
10
|
-
estimateOf,
|
|
11
|
-
failed,
|
|
12
|
-
masked,
|
|
13
|
-
money,
|
|
14
|
-
nextLogs,
|
|
15
|
-
type Out,
|
|
16
|
-
parsePlace,
|
|
17
|
-
type ReadPlace,
|
|
18
|
-
said,
|
|
19
|
-
shellLine,
|
|
20
|
-
span,
|
|
21
|
-
stateWord,
|
|
22
|
-
} from "./run-commands";
|
|
8
|
+
import { asksJson, printRefusal } from "./refusals";
|
|
9
|
+
import { continueOf, estimateOf, failed, nextLogs, span, stateWord } from "./run-commands";
|
|
23
10
|
import { encodeCursor } from "./run-read";
|
|
24
|
-
import { findRunByRef,
|
|
11
|
+
import { findRunByRef, listRunEntries, type RunStatus, runStatus } from "./run-state";
|
|
12
|
+
import { clockTime, masked, money, said } from "./run-text";
|
|
25
13
|
import { type AgentView, describeActivity, type RunView, viewRun } from "./run-view";
|
|
14
|
+
import { isClaimed, RunUnreadable } from "./runs";
|
|
15
|
+
|
|
16
|
+
/** `awf runs --json`'s `"v"`, the list's, and `awf runs {run} --json`'s. */
|
|
17
|
+
export const RUNS_V = 1 satisfies RunsJson["v"];
|
|
18
|
+
export const RUNS_ONE_V = 2 satisfies RunJson["v"];
|
|
26
19
|
|
|
27
20
|
export const runsUsage = help.trimEnd();
|
|
28
21
|
|
|
@@ -35,10 +28,6 @@ export async function runsCommand(
|
|
|
35
28
|
now: number = Date.now(),
|
|
36
29
|
home: string = homedir(),
|
|
37
30
|
): Promise<number> {
|
|
38
|
-
if (argv.includes("-h") || argv.includes("--help")) {
|
|
39
|
-
out.stdout(runsUsage);
|
|
40
|
-
return 0;
|
|
41
|
-
}
|
|
42
31
|
let parsed: ReturnType<typeof parsePlace>;
|
|
43
32
|
let limit: number | undefined;
|
|
44
33
|
let ref: string | undefined;
|
|
@@ -57,7 +46,7 @@ export async function runsCommand(
|
|
|
57
46
|
if (ref !== undefined && limit !== undefined)
|
|
58
47
|
throw new Error("--limit lists runs; it takes no run");
|
|
59
48
|
} catch (error) {
|
|
60
|
-
out
|
|
49
|
+
printRefusal(out, { json: asksJson(argv), v: runsVersion(argv) }, error, "usage", runsUsage);
|
|
61
50
|
return 2;
|
|
62
51
|
}
|
|
63
52
|
const { place, json } = parsed;
|
|
@@ -69,34 +58,52 @@ export async function runsCommand(
|
|
|
69
58
|
);
|
|
70
59
|
return 0;
|
|
71
60
|
} catch (error) {
|
|
72
|
-
return failed(error, place, out, messageOf(error));
|
|
61
|
+
return failed(error, place, out, messageOf(error), { json, v: runsVersion(argv) });
|
|
73
62
|
}
|
|
74
63
|
}
|
|
75
64
|
|
|
65
|
+
/**
|
|
66
|
+
* `awf runs {run} --json`'s `"v"` is 2, the list's 1: by whether a word names a run, read before
|
|
67
|
+
* any parse, so a usage error after it or before it carries the same.
|
|
68
|
+
*/
|
|
69
|
+
function runsVersion(argv: readonly string[]): typeof RUNS_V | typeof RUNS_ONE_V {
|
|
70
|
+
for (let index = 0; index < argv.length; index++) {
|
|
71
|
+
const word = argv[index]!;
|
|
72
|
+
if (word === "--cwd" || word === "--run-root" || word === "--limit") index++;
|
|
73
|
+
else if (!word.startsWith("-")) return RUNS_ONE_V;
|
|
74
|
+
}
|
|
75
|
+
return RUNS_V;
|
|
76
|
+
}
|
|
77
|
+
|
|
76
78
|
/** A run listed, or why it could not be read: one bad run does not hide the others. */
|
|
77
|
-
type Listed =
|
|
79
|
+
type Listed =
|
|
80
|
+
| { status: RunStatus; estimate?: number; unknown: number }
|
|
81
|
+
| { ref: string; unreadable: string };
|
|
78
82
|
|
|
79
83
|
async function list(place: ReadPlace, limit: number, json: boolean, out: Out, now: number) {
|
|
80
|
-
const runs = await
|
|
84
|
+
const runs = await listRunEntries(place.runRoot);
|
|
81
85
|
const shown = runs.slice(0, limit);
|
|
82
86
|
const listed: Listed[] = await Promise.all(
|
|
83
87
|
shown.map((run) =>
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
88
|
+
run instanceof RunUnreadable
|
|
89
|
+
? { ref: run.ref, unreadable: run.why }
|
|
90
|
+
: runStatus(run, undefined, now).then(
|
|
91
|
+
async (status) => {
|
|
92
|
+
return { status, ...(await estimateOf(status)) };
|
|
93
|
+
},
|
|
94
|
+
(error: unknown) => ({
|
|
95
|
+
ref: `${run.record.workflow}/${run.record.id}`,
|
|
96
|
+
unreadable: said(messageOf(error), 400),
|
|
97
|
+
}),
|
|
98
|
+
),
|
|
91
99
|
),
|
|
92
100
|
);
|
|
93
101
|
const older = runs.length - shown.length;
|
|
94
102
|
if (json) {
|
|
95
103
|
const runsJson: RunsJson["runs"] = listed.map((each) => {
|
|
96
104
|
if (!("status" in each)) return each;
|
|
97
|
-
const { status } = each;
|
|
105
|
+
const { status, estimate } = each;
|
|
98
106
|
const { started, ended } = span(status, now);
|
|
99
|
-
const estimate = estimateOf(status.attempts);
|
|
100
107
|
return {
|
|
101
108
|
ref: status.ref,
|
|
102
109
|
state: status.state,
|
|
@@ -108,7 +115,7 @@ async function list(place: ReadPlace, limit: number, json: boolean, out: Out, no
|
|
|
108
115
|
};
|
|
109
116
|
});
|
|
110
117
|
const document: RunsJson = {
|
|
111
|
-
v:
|
|
118
|
+
v: RUNS_V,
|
|
112
119
|
runs: runsJson,
|
|
113
120
|
...(older > 0 ? { more: { older: true } } : {}),
|
|
114
121
|
};
|
|
@@ -125,16 +132,15 @@ async function list(place: ReadPlace, limit: number, json: boolean, out: Out, no
|
|
|
125
132
|
const stateWidth = Math.max(13, ...states.map((each) => each.length + 2));
|
|
126
133
|
const lines = listed.map((each, index) => {
|
|
127
134
|
if (!("status" in each)) return `${each.ref.padEnd(width)}unreadable: ${each.unreadable}`;
|
|
128
|
-
const { status } = each;
|
|
135
|
+
const { status, estimate, unknown } = each;
|
|
129
136
|
const { started, ms } = span(status, now);
|
|
130
|
-
const estimate = estimateOf(status.attempts);
|
|
131
137
|
const waiting = waitingText(status);
|
|
132
138
|
return [
|
|
133
139
|
status.ref.padEnd(width),
|
|
134
140
|
states[index]!.padEnd(stateWidth),
|
|
135
141
|
dayTime(started).padEnd(13),
|
|
136
|
-
|
|
137
|
-
money(estimate)
|
|
142
|
+
duration(ms).padEnd(9),
|
|
143
|
+
money(estimate, unknown)
|
|
138
144
|
.join("")
|
|
139
145
|
.concat(waiting ? " " : ""),
|
|
140
146
|
waiting,
|
|
@@ -150,7 +156,7 @@ async function list(place: ReadPlace, limit: number, json: boolean, out: Out, no
|
|
|
150
156
|
/** A run at its wait: its label, and until when; a returned one, the events it never took. */
|
|
151
157
|
function waitingText(status: RunStatus): string {
|
|
152
158
|
const { waiting } = status;
|
|
153
|
-
const never = status.
|
|
159
|
+
const never = status.unacknowledged;
|
|
154
160
|
if (!waiting) return never ? `${never} event${never === 1 ? "" : "s"} never taken` : "";
|
|
155
161
|
const until = waiting.until ? `until ${dayTime(Date.parse(waiting.until))}` : "";
|
|
156
162
|
return [waiting.label, until].filter(Boolean).join(" · ") || "waiting";
|
|
@@ -186,7 +192,7 @@ function agentLine(agent: AgentView, indent: string, now: number, width: number)
|
|
|
186
192
|
`from ${said(live.peer, 60)}${live.until ? ` (until ${clockTime(live.until).slice(0, 5)})` : ""}`,
|
|
187
193
|
]
|
|
188
194
|
: []),
|
|
189
|
-
...(agent.since && inTurn(agent) ? [
|
|
195
|
+
...(agent.since && inTurn(agent) ? [duration(now - agent.since)] : []),
|
|
190
196
|
...(agent.last
|
|
191
197
|
? [`last: ${describeActivity(agent.last.activity)} (${clockTime(agent.last.at)})`]
|
|
192
198
|
: []),
|
|
@@ -230,7 +236,7 @@ function describe(view: RunView, place: ReadPlace, now: number): string {
|
|
|
230
236
|
[
|
|
231
237
|
status.ref,
|
|
232
238
|
stateWord(status, view.lastEventAt),
|
|
233
|
-
...(attempt && status.
|
|
239
|
+
...(attempt && status.files.length > 1
|
|
234
240
|
? [
|
|
235
241
|
`attempt ${attempt.attempt}${attempt.flags.fromStage ? ` (from stage ${attempt.flags.fromStage})` : ""}`,
|
|
236
242
|
]
|
|
@@ -238,25 +244,30 @@ function describe(view: RunView, place: ReadPlace, now: number): string {
|
|
|
238
244
|
...(view.stage ? [`stage ${view.stage}`] : []),
|
|
239
245
|
status.state === "interrupted" && view.lastEventAt
|
|
240
246
|
? `last event ${clockTime(view.lastEventAt)}`
|
|
241
|
-
:
|
|
242
|
-
...money(view.estimate),
|
|
247
|
+
: duration(span(status, now).ms),
|
|
248
|
+
...money(view.estimate, view.unknownCost),
|
|
243
249
|
].join(" "),
|
|
244
250
|
];
|
|
245
|
-
|
|
251
|
+
// Every other attempt file, a launch that never claimed its number included.
|
|
252
|
+
for (const earlier of status.files) {
|
|
253
|
+
if (earlier.attempt === attempt?.attempt) continue;
|
|
246
254
|
const took = earlier.ended
|
|
247
|
-
? ` ${
|
|
255
|
+
? ` ${duration(Date.parse(earlier.ended) - Date.parse(earlier.started))}`
|
|
248
256
|
: "";
|
|
249
|
-
const
|
|
257
|
+
const stage = isClaimed(earlier) ? earlier.stage : undefined;
|
|
258
|
+
const how = isClaimed(earlier)
|
|
259
|
+
? `${earlier.outcome ?? "interrupted"}${stage ? ` in ${stage}` : ""}`
|
|
260
|
+
: "failed to launch";
|
|
250
261
|
lines.push(` attempt ${earlier.attempt} ${how}${took}`);
|
|
251
262
|
}
|
|
252
|
-
const indent = status.
|
|
263
|
+
const indent = status.files.length > 1 ? " " : "";
|
|
253
264
|
if (indent && attempt) lines.push(` attempt ${attempt.attempt}`);
|
|
254
265
|
const shown = new Set<string>();
|
|
255
266
|
for (const stage of view.stages) {
|
|
256
267
|
const took =
|
|
257
268
|
stage.state === "reused" || stage.state === "provided" || stage.startedAt === undefined
|
|
258
269
|
? ""
|
|
259
|
-
:
|
|
270
|
+
: duration((stage.endedAt ?? now) - stage.startedAt);
|
|
260
271
|
lines.push(
|
|
261
272
|
`${indent} stage ${stage.stage.padEnd(8)} ${stage.state.padEnd(10)} ${took}`.trimEnd(),
|
|
262
273
|
);
|
|
@@ -264,6 +275,7 @@ function describe(view: RunView, place: ReadPlace, now: number): string {
|
|
|
264
275
|
for (const agent of own) shown.add(agent.key);
|
|
265
276
|
lines.push(...agentLines(own, indent, now));
|
|
266
277
|
}
|
|
278
|
+
if (view.unread) lines.push(`${indent} awf: ${view.unread}`);
|
|
267
279
|
lines.push(
|
|
268
280
|
...agentLines(
|
|
269
281
|
view.agents.filter((each) => !shown.has(each.key)),
|
|
@@ -289,7 +301,7 @@ function trailer(view: RunView, place: ReadPlace): string[] {
|
|
|
289
301
|
if (status.state === "interrupted") return go ? [`continue: ${go}`] : [];
|
|
290
302
|
const lines: string[] = [];
|
|
291
303
|
if (view.attempt?.reason) lines.push(`reason: ${said(view.attempt.reason, 400)}`);
|
|
292
|
-
lines.push(`records: ${status.run.dir} (
|
|
304
|
+
lines.push(`records: ${status.run.dir} (run.jsonl, log.jsonl, ops/, out.json, report.md)`);
|
|
293
305
|
if (status.state === "failed") {
|
|
294
306
|
const stuck = view.agents.find((each) => each.state === "failed" || each.state === "timed-out");
|
|
295
307
|
const look = command(place, "awf", "logs", status.ref);
|
|
@@ -311,7 +323,7 @@ function runJson(view: RunView, place: ReadPlace, now: number): RunJson {
|
|
|
311
323
|
? undefined
|
|
312
324
|
: continueOf(status, view.lines, place);
|
|
313
325
|
return {
|
|
314
|
-
v:
|
|
326
|
+
v: RUNS_ONE_V,
|
|
315
327
|
run: {
|
|
316
328
|
ref: status.ref,
|
|
317
329
|
state: status.state,
|
|
@@ -322,20 +334,26 @@ function runJson(view: RunView, place: ReadPlace, now: number): RunJson {
|
|
|
322
334
|
...(status.waiting ? { waiting: status.waiting } : {}),
|
|
323
335
|
dir: status.run.dir,
|
|
324
336
|
},
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
started,
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
337
|
+
// Every attempt file, a launch that never claimed its number included, `failed` with its reason.
|
|
338
|
+
attempts: status.files.map((each) => {
|
|
339
|
+
const { attempt, started, ended, outcome, reason } = each;
|
|
340
|
+
const stage = isClaimed(each) ? each.stage : undefined;
|
|
341
|
+
return {
|
|
342
|
+
attempt,
|
|
343
|
+
started,
|
|
344
|
+
...(ended ? { ended } : {}),
|
|
345
|
+
state:
|
|
346
|
+
outcome ?? (attempt === status.attempts.at(-1)?.attempt ? status.state : "interrupted"),
|
|
347
|
+
...(stage ? { stage } : {}),
|
|
348
|
+
...(reason ? { reason: masked(reason, 1_000) as string } : {}),
|
|
349
|
+
};
|
|
350
|
+
}),
|
|
351
|
+
stages: view.stages.map(({ stage, state, startedAt, endedAt, value }) => ({
|
|
335
352
|
stage,
|
|
336
353
|
state,
|
|
337
354
|
...(startedAt ? { started: new Date(startedAt).toISOString() } : {}),
|
|
338
355
|
...(endedAt ? { ended: new Date(endedAt).toISOString() } : {}),
|
|
356
|
+
...(value === undefined ? {} : { value: masked(value) as JsonValue }),
|
|
339
357
|
})),
|
|
340
358
|
agents: view.agents.map((agent) => ({
|
|
341
359
|
key: agent.key,
|