@niwa-labs/workflow 0.0.0-stage → 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +13 -2
- package/package.json +55 -4
- package/src/actions.ts +408 -0
- package/src/agent.ts +211 -0
- package/src/ask.ts +84 -0
- package/src/bridge.ts +710 -0
- package/src/catalogue.ts +74 -0
- package/src/check.ts +880 -0
- package/src/child.ts +1378 -0
- package/src/cockpit/actions.ts +128 -0
- package/src/cockpit/catalogue-actions.ts +97 -0
- package/src/cockpit/dashboard.tsx +134 -0
- package/src/cockpit/groups.ts +45 -0
- package/src/cockpit/index.tsx +50 -0
- package/src/cockpit/launch.tsx +136 -0
- package/src/cockpit/plan.ts +101 -0
- package/src/cockpit/run-commands.tsx +102 -0
- package/src/cockpit/run-page.tsx +140 -0
- package/src/cockpit/run-summary.ts +16 -0
- package/src/cockpit/runs.tsx +73 -0
- package/src/cockpit/statistics.tsx +176 -0
- package/src/cockpit/track.tsx +159 -0
- package/src/cockpit/workflows.tsx +383 -0
- package/src/describe-cache.ts +122 -0
- package/src/describe.ts +174 -0
- package/src/guide.ts +213 -0
- package/src/index.ts +2813 -0
- package/src/inspect.ts +235 -0
- package/src/installation.ts +90 -0
- package/src/language.ts +217 -0
- package/src/records-entry.ts +63 -0
- package/src/records.ts +574 -0
- package/src/registry.ts +250 -0
- package/src/revise.ts +89 -0
- package/src/runner.ts +339 -0
- package/src/script-types.ts +549 -0
- package/src/stats.ts +460 -0
- package/src/status.ts +512 -0
- package/src/store.ts +259 -0
- package/src/subscriptions.ts +102 -0
- package/src/waiting.ts +61 -0
- package/test/cockpit-actions.test.ts +20 -0
- package/test/cockpit-groups.test.ts +42 -0
- package/test/cockpit-plan.test.ts +118 -0
- package/test/fixtures/branches-host.ts +58 -0
- package/test/fixtures/branches-script.js +23 -0
- package/test/fixtures/bridge-host.ts +23 -0
- package/test/fixtures/bridge-keys-script.js +15 -0
- package/test/fixtures/bridge-wait-script.js +8 -0
- package/test/fixtures/cancel-host.ts +58 -0
- package/test/fixtures/cancel-script.js +12 -0
- package/test/fixtures/crash-host.ts +61 -0
- package/test/fixtures/crash-script.js +14 -0
- package/test/fixtures/ping-records.ts +20 -0
- package/test/fixtures/revision-host.ts +57 -0
- package/test/fixtures/revision-script.js +11 -0
- package/test/fixtures/running-host.ts +61 -0
- package/test/fixtures/running-script.js +12 -0
- package/test/fixtures/with-data/flow.ts +2 -0
- package/test/fixtures/with-data/table.json +1 -0
- package/test/git-repo.ts +45 -0
- package/test/git-worktree.test.ts +413 -0
- package/test/question-format.test.ts +279 -0
- package/test/workflow-agent-caps.test.ts +91 -0
- package/test/workflow-agent-lifetime.test.ts +93 -0
- package/test/workflow-agent-options.test.ts +216 -0
- package/test/workflow-agent-schema.test.ts +386 -0
- package/test/workflow-agent-typings.test.ts +241 -0
- package/test/workflow-ask-resume.test.ts +145 -0
- package/test/workflow-branch-keys.test.ts +133 -0
- package/test/workflow-bridge-inproc.test.ts +230 -0
- package/test/workflow-bridge.test.ts +141 -0
- package/test/workflow-builtin-candidate.test.ts +230 -0
- package/test/workflow-builtin.test.ts +197 -0
- package/test/workflow-cancel-agent.test.ts +144 -0
- package/test/workflow-cancel.test.ts +166 -0
- package/test/workflow-chain.test.ts +84 -0
- package/test/workflow-check-options.test.ts +144 -0
- package/test/workflow-check-uses.test.ts +148 -0
- package/test/workflow-check.test.ts +282 -0
- package/test/workflow-contract.test.ts +295 -0
- package/test/workflow-crash-resume.test.ts +141 -0
- package/test/workflow-deadline.test.ts +102 -0
- package/test/workflow-describe-cache.test.ts +240 -0
- package/test/workflow-directory.test.ts +73 -0
- package/test/workflow-emits.test.ts +261 -0
- package/test/workflow-group-keys.test.ts +72 -0
- package/test/workflow-harnesses.test.ts +262 -0
- package/test/workflow-inspect.test.ts +152 -0
- package/test/workflow-language-types.test.ts +148 -0
- package/test/workflow-manifest.test.ts +93 -0
- package/test/workflow-manual-pause.test.ts +116 -0
- package/test/workflow-owners.test.ts +154 -0
- package/test/workflow-pause.test.ts +200 -0
- package/test/workflow-phases.test.ts +445 -0
- package/test/workflow-plugin-entries.test.ts +59 -0
- package/test/workflow-record-refs.test.ts +310 -0
- package/test/workflow-replay.test.ts +206 -0
- package/test/workflow-resume-once.test.ts +188 -0
- package/test/workflow-revision.test.ts +216 -0
- package/test/workflow-scope.test.ts +505 -0
- package/test/workflow-script-defaults.test.ts +119 -0
- package/test/workflow-script-error.test.ts +228 -0
- package/test/workflow-script-libraries.test.ts +88 -0
- package/test/workflow-script-packages.test.ts +119 -0
- package/test/workflow-stdout.test.ts +89 -0
- package/test/workflow-stop.test.ts +299 -0
- package/test/workflow-store.test.ts +220 -0
- package/test/workflow-visible.test.ts +136 -0
- package/test/workflow-wait-version.test.ts +54 -0
- package/test/workflow-wait.test.ts +224 -0
- package/test/workflow-waiting-agent.test.ts +64 -0
- package/tsconfig.json +15 -0
package/src/stats.ts
ADDED
|
@@ -0,0 +1,460 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `workflow.stats`: how the workflows did, folded from the journal.
|
|
3
|
+
*
|
|
4
|
+
* Nothing is written: the fold lives in memory, reads the journal by `seq` and catches up before
|
|
5
|
+
* each answer, so a restart folds it again. Without an id it measures every workflow; with one it
|
|
6
|
+
* measures that workflow version by version, or the two versions `compare` names, from `since` on.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { CORE_TYPES, IdText, WAIT_ACTION, actionRef, seq } from "@niwa-labs/core";
|
|
10
|
+
import type { JournalReader, RecordTypeName, Seq, StoredRecord } from "@niwa-labs/core";
|
|
11
|
+
import { requestAsk } from "@niwa-labs/requests/records";
|
|
12
|
+
import { SessionModel, SessionReverseAnswered, SessionReverseAsked, SessionReverseCancelled, sessionStart } from "@niwa-labs/sessions/records";
|
|
13
|
+
import { z } from "zod";
|
|
14
|
+
|
|
15
|
+
import { PhaseEntered, RunFinished, RunId, RunOutcome, RunPaused, RunRequested, RunStarted, WorkflowId, WorkflowVersion } from "./records.ts";
|
|
16
|
+
|
|
17
|
+
/** The share of runs that ended `done` with no human input, or null when no run ended. */
|
|
18
|
+
const Rate = z.number().min(0).max(1).nullable();
|
|
19
|
+
|
|
20
|
+
/** A spread of durations in milliseconds. */
|
|
21
|
+
export const Spread = z.object({ n: z.number().int(), p50: z.number(), p90: z.number(), max: z.number() });
|
|
22
|
+
export type Spread = z.infer<typeof Spread>;
|
|
23
|
+
|
|
24
|
+
/** What the runs of a workflow, or of one version of it, came to. */
|
|
25
|
+
export const Measures = z.object({
|
|
26
|
+
runs: z.number().int(),
|
|
27
|
+
outcomes: z.object({
|
|
28
|
+
done: z.number().int(),
|
|
29
|
+
failed: z.number().int(),
|
|
30
|
+
cancelled: z.number().int(),
|
|
31
|
+
paused: z.number().int(),
|
|
32
|
+
running: z.number().int(),
|
|
33
|
+
}),
|
|
34
|
+
/** The runs that ended `done` with no question to a human, no answered pause and no permission asked of the starter. */
|
|
35
|
+
clean: z.number().int(),
|
|
36
|
+
/** `clean` over the runs that ended (done, failed, cancelled). */
|
|
37
|
+
cleanRate: Rate,
|
|
38
|
+
/** From the request of a run to its end, over the runs that ended. Absent when none ended. */
|
|
39
|
+
duration: Spread.optional(),
|
|
40
|
+
/** Where the time of the runs that ended went, summed. `other` is what the rest leaves of the wall time. */
|
|
41
|
+
time: z.object({
|
|
42
|
+
agentMs: z.number(),
|
|
43
|
+
humanMs: z.number(),
|
|
44
|
+
/** Of `humanMs`: the questions of the script (`ask`). */
|
|
45
|
+
askMs: z.number(),
|
|
46
|
+
/** Of `humanMs`: what the sessions asked their starter (permissions, questions). */
|
|
47
|
+
reverseMs: z.number(),
|
|
48
|
+
timerMs: z.number(),
|
|
49
|
+
otherMs: z.number(),
|
|
50
|
+
}),
|
|
51
|
+
/** The steps that failed and paused a run, and the runs that ended `failed`, by the phase the run was in. */
|
|
52
|
+
failures: z.array(z.object({ phase: z.string().nullable(), count: z.number().int() })),
|
|
53
|
+
/** The `ask()` calls, by harness and the model the session ran (null until the harness named it). */
|
|
54
|
+
agentCalls: z.array(
|
|
55
|
+
z.object({ harness: z.string(), model: z.string().nullable(), calls: z.number().int(), failed: z.number().int(), ms: z.number() }),
|
|
56
|
+
),
|
|
57
|
+
});
|
|
58
|
+
export type Measures = z.infer<typeof Measures>;
|
|
59
|
+
|
|
60
|
+
/** What a row of the general view is about: a registered workflow, or a script file run without registration. */
|
|
61
|
+
export const StatsSubject = z.discriminatedUnion("kind", [
|
|
62
|
+
z.object({ kind: z.literal("workflow"), id: WorkflowId }),
|
|
63
|
+
z.object({ kind: z.literal("script"), file: z.string() }),
|
|
64
|
+
]);
|
|
65
|
+
export type StatsSubject = z.infer<typeof StatsSubject>;
|
|
66
|
+
|
|
67
|
+
/** The measures of one version of a workflow, with when its runs were asked for. */
|
|
68
|
+
export const VersionStats = z.object({
|
|
69
|
+
version: WorkflowVersion,
|
|
70
|
+
/** Absent for a version `compare` named that has no runs. */
|
|
71
|
+
firstRunAt: z.number().int().optional(),
|
|
72
|
+
lastRunAt: z.number().int().optional(),
|
|
73
|
+
measures: Measures,
|
|
74
|
+
});
|
|
75
|
+
export type VersionStats = z.infer<typeof VersionStats>;
|
|
76
|
+
|
|
77
|
+
export const StatsInput = z.object({
|
|
78
|
+
/** One registered workflow, by its id or a text that names it. Absent: every workflow. */
|
|
79
|
+
id: IdText.optional(),
|
|
80
|
+
/** With `id`: only these two versions (a version or a unique start of one), in this order. */
|
|
81
|
+
compare: z.tuple([z.string().min(1), z.string().min(1)]).optional(),
|
|
82
|
+
/** Only the runs asked for at or after this moment, in milliseconds since the epoch. */
|
|
83
|
+
since: z.number().int().nonnegative().optional(),
|
|
84
|
+
});
|
|
85
|
+
export type StatsInput = z.infer<typeof StatsInput>;
|
|
86
|
+
|
|
87
|
+
export const StatsOutput = z.discriminatedUnion("scope", [
|
|
88
|
+
z.object({
|
|
89
|
+
scope: z.literal("all"),
|
|
90
|
+
total: Measures,
|
|
91
|
+
workflows: z.array(z.object({ subject: StatsSubject, measures: Measures })),
|
|
92
|
+
caveats: z.array(z.string()),
|
|
93
|
+
}),
|
|
94
|
+
z.object({
|
|
95
|
+
scope: z.literal("workflow"),
|
|
96
|
+
id: WorkflowId,
|
|
97
|
+
total: Measures,
|
|
98
|
+
/** The versions that ran, the earliest first; with `compare`, the two it names. */
|
|
99
|
+
versions: z.array(VersionStats),
|
|
100
|
+
caveats: z.array(z.string()),
|
|
101
|
+
}),
|
|
102
|
+
]);
|
|
103
|
+
export type StatsOutput = z.infer<typeof StatsOutput>;
|
|
104
|
+
|
|
105
|
+
export const workflowStats = actionRef("workflow.stats", {
|
|
106
|
+
description:
|
|
107
|
+
"How the workflows did, from the journal: runs, outcomes (done, failed, cancelled, paused, running), durations, the time split between agents, humans (questions, permissions), timers and the rest, failures by phase, agent calls by harness and model, and the clean rate (done with no human input). Without id every workflow; with id one workflow version by version (a version is the hash of its code), or the two versions `compare` names. `since` keeps the runs asked for from then on.",
|
|
108
|
+
input: StatsInput,
|
|
109
|
+
output: StatsOutput,
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
/** What the numbers cannot say exactly. */
|
|
113
|
+
const CAVEATS: readonly string[] = [
|
|
114
|
+
"A run revised or resumed under another version counts whole under the version it was last asked for.",
|
|
115
|
+
"Time is the difference of the moments records were written; a pause waiting for a decision counts as other time.",
|
|
116
|
+
"Human time of a session counts inside the agent call that asked it and is taken out of agent time.",
|
|
117
|
+
"Scripts run by file, not registered, have no version: they show in the general view by file only.",
|
|
118
|
+
"No token or cost counts: the harnesses do not record them yet.",
|
|
119
|
+
];
|
|
120
|
+
|
|
121
|
+
/** The steps and records the fold reads. */
|
|
122
|
+
const FOLDED: readonly RecordTypeName[] = [
|
|
123
|
+
RunRequested.name,
|
|
124
|
+
RunStarted.name,
|
|
125
|
+
PhaseEntered.name,
|
|
126
|
+
RunPaused.name,
|
|
127
|
+
RunFinished.name,
|
|
128
|
+
CORE_TYPES.invocationState,
|
|
129
|
+
CORE_TYPES.stepIntent,
|
|
130
|
+
CORE_TYPES.stepOutcome,
|
|
131
|
+
SessionModel.name,
|
|
132
|
+
SessionReverseAsked.name,
|
|
133
|
+
SessionReverseAnswered.name,
|
|
134
|
+
SessionReverseCancelled.name,
|
|
135
|
+
];
|
|
136
|
+
|
|
137
|
+
/** How many records one read of the journal takes. */
|
|
138
|
+
const CHUNK = 5000;
|
|
139
|
+
|
|
140
|
+
const StepIntent = z.object({ invocation: z.string(), key: z.string(), action: z.string(), input: z.unknown().optional() });
|
|
141
|
+
const StepOutcome = z.object({
|
|
142
|
+
invocation: z.string(),
|
|
143
|
+
key: z.string(),
|
|
144
|
+
outcome: z.object({ status: z.enum(["completed", "denied", "failed", "unknown"]), value: z.unknown().optional() }),
|
|
145
|
+
});
|
|
146
|
+
const InvocationState = z.object({ invocation: z.string(), state: z.string() });
|
|
147
|
+
/** What `session.start` answers: the pair the harness and its id make, `ref`, which the records of the session carry (`session` is the niwa id). */
|
|
148
|
+
const StartedSession = z.object({ ref: z.string() });
|
|
149
|
+
|
|
150
|
+
/** The step key of a call made inside another call: `<outer>.step-<n>`, or `<outer>.<attempt>.step-<n>`. */
|
|
151
|
+
const NESTED = /^(.*?)(?:\.\d+)?\.step-\d+$/;
|
|
152
|
+
|
|
153
|
+
/** What a top-level step of a run spends its time on. */
|
|
154
|
+
type StepKind = "agent" | "ask" | "timer" | "other";
|
|
155
|
+
|
|
156
|
+
function kindOf(action: string): StepKind {
|
|
157
|
+
if (action === sessionStart.name || action.startsWith(`${sessionStart.name}.`)) return "agent";
|
|
158
|
+
if (action === requestAsk.name) return "ask";
|
|
159
|
+
if (action === WAIT_ACTION) return "timer";
|
|
160
|
+
return "other";
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/** One `ask()` call of a run. */
|
|
164
|
+
interface AgentCall {
|
|
165
|
+
harness?: string;
|
|
166
|
+
/** The ref of its session, the pair its records carry. */
|
|
167
|
+
session?: string;
|
|
168
|
+
ms: number;
|
|
169
|
+
failed: boolean;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/** A run as the fold knows it. */
|
|
173
|
+
interface RunFact {
|
|
174
|
+
run: RunId;
|
|
175
|
+
workflow?: WorkflowId;
|
|
176
|
+
version?: WorkflowVersion;
|
|
177
|
+
file: string;
|
|
178
|
+
requestedAt: number;
|
|
179
|
+
phase?: string;
|
|
180
|
+
outcome?: RunOutcome["status"] | "paused";
|
|
181
|
+
endedAt?: number;
|
|
182
|
+
time: { agent: number; ask: number; reverse: number; timer: number };
|
|
183
|
+
/** Whether a human answered something for it: a question, a pause, a permission. */
|
|
184
|
+
human: boolean;
|
|
185
|
+
failures: Array<string | undefined>;
|
|
186
|
+
calls: Map<string, AgentCall>;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/** The fold over the journal: run facts by run, and what links steps and sessions to them. */
|
|
190
|
+
export class StatsFold {
|
|
191
|
+
private next: Seq = seq(0);
|
|
192
|
+
private readonly runs = new Map<string, RunFact>();
|
|
193
|
+
private readonly runOfInvocation = new Map<string, RunFact>();
|
|
194
|
+
private readonly intents = new Map<string, { at: number; action: string; input: unknown }>();
|
|
195
|
+
private readonly runOfSession = new Map<string, RunFact>();
|
|
196
|
+
private readonly modelOf = new Map<string, string>();
|
|
197
|
+
private readonly reverseAsked = new Map<string, { at: number; session: string }>();
|
|
198
|
+
|
|
199
|
+
/** Reads what the journal holds past the fold. */
|
|
200
|
+
catchUp(journal: JournalReader): void {
|
|
201
|
+
for (;;) {
|
|
202
|
+
const rows = journal.since(this.next, [...FOLDED], CHUNK);
|
|
203
|
+
for (const row of rows) this.absorb(row);
|
|
204
|
+
if (rows.length < CHUNK) return;
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/** Folds one record in. */
|
|
209
|
+
absorb(row: StoredRecord): void {
|
|
210
|
+
this.next = row.seq;
|
|
211
|
+
const at = Number(row.at);
|
|
212
|
+
switch (row.type) {
|
|
213
|
+
case RunRequested.name: {
|
|
214
|
+
const data = RunRequested.schema.parse(row.data);
|
|
215
|
+
const known = this.runs.get(data.run);
|
|
216
|
+
if (known === undefined) {
|
|
217
|
+
this.runs.set(data.run, {
|
|
218
|
+
run: data.run,
|
|
219
|
+
...(data.workflow === undefined ? {} : { workflow: data.workflow }),
|
|
220
|
+
...(data.version === undefined ? {} : { version: data.version }),
|
|
221
|
+
file: data.file,
|
|
222
|
+
requestedAt: at,
|
|
223
|
+
time: { agent: 0, ask: 0, reverse: 0, timer: 0 },
|
|
224
|
+
human: false,
|
|
225
|
+
failures: [],
|
|
226
|
+
calls: new Map(),
|
|
227
|
+
});
|
|
228
|
+
return;
|
|
229
|
+
}
|
|
230
|
+
// A resume or a revision asks for the run again: it goes on, under the version it is asked for now.
|
|
231
|
+
if (data.version !== undefined) known.version = data.version;
|
|
232
|
+
known.file = data.file;
|
|
233
|
+
if (known.outcome === "paused") known.human = true;
|
|
234
|
+
delete known.outcome;
|
|
235
|
+
delete known.endedAt;
|
|
236
|
+
return;
|
|
237
|
+
}
|
|
238
|
+
case RunStarted.name: {
|
|
239
|
+
const fact = this.runs.get(RunStarted.schema.parse(row.data).run);
|
|
240
|
+
if (fact !== undefined && row.actor.kind === "handler") this.runOfInvocation.set(String(row.actor.invocation), fact);
|
|
241
|
+
return;
|
|
242
|
+
}
|
|
243
|
+
case PhaseEntered.name: {
|
|
244
|
+
const data = PhaseEntered.schema.parse(row.data);
|
|
245
|
+
const fact = this.runs.get(data.run);
|
|
246
|
+
if (fact !== undefined) fact.phase = data.phase;
|
|
247
|
+
return;
|
|
248
|
+
}
|
|
249
|
+
case RunPaused.name: {
|
|
250
|
+
const fact = this.runs.get(RunPaused.schema.parse(row.data).run);
|
|
251
|
+
if (fact !== undefined && fact.outcome === undefined) fact.outcome = "paused";
|
|
252
|
+
return;
|
|
253
|
+
}
|
|
254
|
+
case RunFinished.name: {
|
|
255
|
+
const data = RunFinished.schema.parse(row.data);
|
|
256
|
+
const fact = this.runs.get(data.run);
|
|
257
|
+
if (fact === undefined) return;
|
|
258
|
+
fact.outcome = data.outcome.status;
|
|
259
|
+
fact.endedAt = at;
|
|
260
|
+
if (data.outcome.status === "failed") fact.failures.push(fact.phase);
|
|
261
|
+
return;
|
|
262
|
+
}
|
|
263
|
+
case CORE_TYPES.invocationState: {
|
|
264
|
+
const data = InvocationState.parse(row.data);
|
|
265
|
+
const fact = this.runOfInvocation.get(data.invocation);
|
|
266
|
+
if (fact === undefined) return;
|
|
267
|
+
if (data.state === "paused") {
|
|
268
|
+
fact.outcome = "paused";
|
|
269
|
+
fact.failures.push(fact.phase);
|
|
270
|
+
} else if (data.state === "running" && fact.outcome === "paused") {
|
|
271
|
+
// A pause answered with retry or skip: a human decided.
|
|
272
|
+
fact.human = true;
|
|
273
|
+
delete fact.outcome;
|
|
274
|
+
} else if (data.state === "done" && fact.outcome === undefined) {
|
|
275
|
+
// The invocation ended without an end of the run: it was cancelled while it paused.
|
|
276
|
+
fact.outcome = "cancelled";
|
|
277
|
+
fact.endedAt = at;
|
|
278
|
+
}
|
|
279
|
+
return;
|
|
280
|
+
}
|
|
281
|
+
case CORE_TYPES.stepIntent: {
|
|
282
|
+
const data = StepIntent.parse(row.data);
|
|
283
|
+
if (this.runOfInvocation.has(data.invocation)) this.intents.set(`${data.invocation} ${data.key}`, { at, action: data.action, input: data.input });
|
|
284
|
+
return;
|
|
285
|
+
}
|
|
286
|
+
case CORE_TYPES.stepOutcome:
|
|
287
|
+
this.stepEnded(StepOutcome.parse(row.data), at);
|
|
288
|
+
return;
|
|
289
|
+
case SessionModel.name: {
|
|
290
|
+
const data = SessionModel.schema.parse(row.data);
|
|
291
|
+
this.modelOf.set(data.session, data.model);
|
|
292
|
+
return;
|
|
293
|
+
}
|
|
294
|
+
case SessionReverseAsked.name: {
|
|
295
|
+
const data = SessionReverseAsked.schema.parse(row.data);
|
|
296
|
+
if (data.route === "starter") this.reverseAsked.set(data.id, { at, session: data.session });
|
|
297
|
+
return;
|
|
298
|
+
}
|
|
299
|
+
case SessionReverseAnswered.name:
|
|
300
|
+
case SessionReverseCancelled.name: {
|
|
301
|
+
const id = row.type === SessionReverseAnswered.name ? SessionReverseAnswered.schema.parse(row.data).id : SessionReverseCancelled.schema.parse(row.data).id;
|
|
302
|
+
const asked = this.reverseAsked.get(id);
|
|
303
|
+
if (asked === undefined) return;
|
|
304
|
+
this.reverseAsked.delete(id);
|
|
305
|
+
const fact = this.runOfSession.get(asked.session);
|
|
306
|
+
if (fact === undefined) return;
|
|
307
|
+
fact.time.reverse += at - asked.at;
|
|
308
|
+
if (row.type === SessionReverseAnswered.name && SessionReverseAnswered.schema.parse(row.data).by === "starter") fact.human = true;
|
|
309
|
+
return;
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
private stepEnded(data: z.infer<typeof StepOutcome>, at: number): void {
|
|
315
|
+
const id = `${data.invocation} ${data.key}`;
|
|
316
|
+
const intent = this.intents.get(id);
|
|
317
|
+
const fact = this.runOfInvocation.get(data.invocation);
|
|
318
|
+
if (intent === undefined || fact === undefined) return;
|
|
319
|
+
this.intents.delete(id);
|
|
320
|
+
const ms = at - intent.at;
|
|
321
|
+
const outer = NESTED.exec(data.key)?.[1];
|
|
322
|
+
if (outer !== undefined) {
|
|
323
|
+
// The start of the session inside an agent call names the harness and the session; it ends before the call does.
|
|
324
|
+
if (intent.action !== sessionStart.name || kindOf(this.intents.get(`${data.invocation} ${outer}`)?.action ?? "") !== "agent") return;
|
|
325
|
+
const call = fact.calls.get(outer) ?? { ms: 0, failed: false };
|
|
326
|
+
fact.calls.set(outer, call);
|
|
327
|
+
const input = sessionStart.input.safeParse(intent.input);
|
|
328
|
+
const started = StartedSession.safeParse(data.outcome.value);
|
|
329
|
+
if (input.success) call.harness = input.data.harness;
|
|
330
|
+
if (started.success) {
|
|
331
|
+
call.session = started.data.ref;
|
|
332
|
+
this.runOfSession.set(started.data.ref, fact);
|
|
333
|
+
}
|
|
334
|
+
return;
|
|
335
|
+
}
|
|
336
|
+
const kind = kindOf(intent.action);
|
|
337
|
+
switch (kind) {
|
|
338
|
+
case "agent": {
|
|
339
|
+
const call = fact.calls.get(data.key) ?? { ms: 0, failed: false };
|
|
340
|
+
call.ms = ms;
|
|
341
|
+
call.failed = data.outcome.status !== "completed";
|
|
342
|
+
fact.calls.set(data.key, call);
|
|
343
|
+
fact.time.agent += ms;
|
|
344
|
+
return;
|
|
345
|
+
}
|
|
346
|
+
case "ask":
|
|
347
|
+
fact.time.ask += ms;
|
|
348
|
+
if (data.outcome.status === "completed") fact.human = true;
|
|
349
|
+
return;
|
|
350
|
+
case "timer":
|
|
351
|
+
fact.time.timer += ms;
|
|
352
|
+
return;
|
|
353
|
+
case "other":
|
|
354
|
+
return;
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
/** Every run asked for at or after `since`, in the order they were first asked for. */
|
|
359
|
+
private factsSince(since: number | undefined): RunFact[] {
|
|
360
|
+
return [...this.runs.values()].filter((fact) => since === undefined || fact.requestedAt >= since);
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
/** The versions a workflow ran, the earliest first. */
|
|
364
|
+
versionsOf(id: WorkflowId): WorkflowVersion[] {
|
|
365
|
+
return [...new Set([...this.runs.values()].flatMap((fact) => (fact.workflow === id && fact.version !== undefined ? [fact.version] : [])))];
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
/** The answer to `workflow.stats`, for a workflow already resolved to its id. */
|
|
369
|
+
answer(input: { id?: WorkflowId; compare?: readonly [WorkflowVersion, WorkflowVersion]; since?: number }): StatsOutput {
|
|
370
|
+
const facts = this.factsSince(input.since);
|
|
371
|
+
if (input.id === undefined) {
|
|
372
|
+
const groups = new Map<string, { subject: StatsSubject; facts: RunFact[] }>();
|
|
373
|
+
for (const fact of facts) {
|
|
374
|
+
const subject: StatsSubject = fact.workflow === undefined ? { kind: "script", file: fact.file } : { kind: "workflow", id: fact.workflow };
|
|
375
|
+
const key = subject.kind === "workflow" ? subject.id : `file ${subject.file}`;
|
|
376
|
+
const group = groups.get(key) ?? { subject, facts: [] };
|
|
377
|
+
group.facts.push(fact);
|
|
378
|
+
groups.set(key, group);
|
|
379
|
+
}
|
|
380
|
+
return {
|
|
381
|
+
scope: "all",
|
|
382
|
+
total: this.measure(facts),
|
|
383
|
+
workflows: [...groups.values()].map((group) => ({ subject: group.subject, measures: this.measure(group.facts) })),
|
|
384
|
+
caveats: [...CAVEATS],
|
|
385
|
+
};
|
|
386
|
+
}
|
|
387
|
+
const id = input.id;
|
|
388
|
+
const mine = facts.filter((fact) => fact.workflow === id);
|
|
389
|
+
const versions = input.compare ?? this.versionsOf(id).filter((version) => mine.some((fact) => fact.version === version));
|
|
390
|
+
return {
|
|
391
|
+
scope: "workflow",
|
|
392
|
+
id,
|
|
393
|
+
total: this.measure(mine),
|
|
394
|
+
versions: versions.map((version) => {
|
|
395
|
+
const of = mine.filter((fact) => fact.version === version);
|
|
396
|
+
const first = of[0];
|
|
397
|
+
const last = of.at(-1);
|
|
398
|
+
return {
|
|
399
|
+
version,
|
|
400
|
+
...(first === undefined ? {} : { firstRunAt: first.requestedAt }),
|
|
401
|
+
...(last === undefined ? {} : { lastRunAt: last.requestedAt }),
|
|
402
|
+
measures: this.measure(of),
|
|
403
|
+
};
|
|
404
|
+
}),
|
|
405
|
+
caveats: [...CAVEATS],
|
|
406
|
+
};
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
private measure(facts: readonly RunFact[]): Measures {
|
|
410
|
+
const outcomes = { done: 0, failed: 0, cancelled: 0, paused: 0, running: 0 };
|
|
411
|
+
const time = { agentMs: 0, humanMs: 0, askMs: 0, reverseMs: 0, timerMs: 0, otherMs: 0 };
|
|
412
|
+
const durations: number[] = [];
|
|
413
|
+
const failures = new Map<string | null, number>();
|
|
414
|
+
const calls = new Map<string, { harness: string; model: string | null; calls: number; failed: number; ms: number }>();
|
|
415
|
+
let clean = 0;
|
|
416
|
+
for (const fact of facts) {
|
|
417
|
+
outcomes[fact.outcome ?? "running"] += 1;
|
|
418
|
+
for (const phase of fact.failures) failures.set(phase ?? null, (failures.get(phase ?? null) ?? 0) + 1);
|
|
419
|
+
for (const call of fact.calls.values()) {
|
|
420
|
+
const harness = call.harness ?? "unknown";
|
|
421
|
+
const model = call.session === undefined ? null : (this.modelOf.get(call.session) ?? null);
|
|
422
|
+
const key = `${harness} ${model ?? ""}`;
|
|
423
|
+
const row = calls.get(key) ?? { harness, model, calls: 0, failed: 0, ms: 0 };
|
|
424
|
+
row.calls += 1;
|
|
425
|
+
row.failed += call.failed ? 1 : 0;
|
|
426
|
+
row.ms += call.ms;
|
|
427
|
+
calls.set(key, row);
|
|
428
|
+
}
|
|
429
|
+
if (fact.endedAt === undefined) continue;
|
|
430
|
+
if (fact.outcome === "done" && !fact.human) clean += 1;
|
|
431
|
+
const wall = fact.endedAt - fact.requestedAt;
|
|
432
|
+
durations.push(wall);
|
|
433
|
+
const agent = Math.max(0, fact.time.agent - fact.time.reverse);
|
|
434
|
+
const human = fact.time.ask + fact.time.reverse;
|
|
435
|
+
time.agentMs += agent;
|
|
436
|
+
time.askMs += fact.time.ask;
|
|
437
|
+
time.reverseMs += fact.time.reverse;
|
|
438
|
+
time.humanMs += human;
|
|
439
|
+
time.timerMs += fact.time.timer;
|
|
440
|
+
time.otherMs += Math.max(0, wall - agent - human - fact.time.timer);
|
|
441
|
+
}
|
|
442
|
+
const ended = outcomes.done + outcomes.failed + outcomes.cancelled;
|
|
443
|
+
return {
|
|
444
|
+
runs: facts.length,
|
|
445
|
+
outcomes,
|
|
446
|
+
clean,
|
|
447
|
+
cleanRate: ended === 0 ? null : clean / ended,
|
|
448
|
+
...(durations.length === 0 ? {} : { duration: spreadOf(durations) }),
|
|
449
|
+
time,
|
|
450
|
+
failures: [...failures].map(([phase, count]) => ({ phase, count })),
|
|
451
|
+
agentCalls: [...calls.values()],
|
|
452
|
+
};
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
function spreadOf(values: number[]): Spread {
|
|
457
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
458
|
+
const at = (share: number): number => sorted[Math.min(sorted.length - 1, Math.ceil(share * sorted.length) - 1)] ?? 0;
|
|
459
|
+
return { n: sorted.length, p50: at(0.5), p90: at(0.9), max: sorted.at(-1) ?? 0 };
|
|
460
|
+
}
|