@niwa-labs/watch 0.0.0-stage → 0.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,579 @@
1
+ import { describe, expect, test } from "bun:test";
2
+
3
+ import { recordId, recordTypeName, resolveQuery, selectsType, seq, streamId, timestamp } from "@niwa-labs/core";
4
+ import type { Caller, StoredRecord } from "@niwa-labs/core";
5
+
6
+ import { DEFAULT_SETTINGS, Model, SOURCES, transitions } from "../src/readings.ts";
7
+ import type { OpenFlag, Settings, Transition } from "../src/readings.ts";
8
+ import { runSubject, sessionSubject } from "../src/records.ts";
9
+ import { Reading } from "../src/records.ts";
10
+
11
+ const MINUTE = 60_000;
12
+ const HOST: Caller = { kind: "host" };
13
+
14
+ let counter = 0;
15
+ function row(type: string, data: unknown, at: number, actor: Caller = HOST): StoredRecord {
16
+ counter += 1;
17
+ const id = recordId(`r${counter}`);
18
+ return {
19
+ id,
20
+ seq: seq(counter),
21
+ stream: streamId("test"),
22
+ type: recordTypeName(type),
23
+ schema: 1,
24
+ data,
25
+ actor,
26
+ cause: null,
27
+ root: id,
28
+ at: timestamp(at),
29
+ };
30
+ }
31
+
32
+ function handler(invocation: string): Caller {
33
+ return { kind: "handler", handler: "workflow/run" as never, invocation: invocation as never };
34
+ }
35
+
36
+ function modelOf(rows: StoredRecord[], floor = 0): Model {
37
+ const model = new Model(floor);
38
+ for (const one of rows) model.apply(one);
39
+ return model;
40
+ }
41
+
42
+ function read(model: Model, subject: ReturnType<typeof sessionSubject>, now: number, settings: Settings = DEFAULT_SETTINGS): Reading | undefined {
43
+ return model.readingOf(subject, now, settings);
44
+ }
45
+
46
+ const ME = "claude abc";
47
+ const me = sessionSubject("claude", "abc");
48
+
49
+ describe("sessions", () => {
50
+ const turn = (at: number) => [row("session.started", { session: ME, id: "s-1" }, 0), row("session.turn-started", { session: ME, turn: 1 }, at)];
51
+
52
+ test("a session in a turn is working until it has been quiet for too long, then stuck since that moment", () => {
53
+ const model = modelOf(turn(0));
54
+ expect(read(model, me, 9 * MINUTE)).toMatchObject({ state: "working", since: 0, last: 0 });
55
+ expect(read(model, me, 10 * MINUTE)).toEqual({ subject: me, state: "stuck", kind: "silent", since: 10 * MINUTE, last: 0 });
56
+ });
57
+
58
+ test("a sign of life moves the end of the quiet", () => {
59
+ const model = modelOf([...turn(0), row("session.activity", { session: ME, tool: "Bash" }, 6 * MINUTE)]);
60
+ expect(read(model, me, 15 * MINUTE)).toMatchObject({ state: "working", last: 6 * MINUTE });
61
+ expect(read(model, me, 16 * MINUTE)).toMatchObject({ state: "stuck", kind: "silent", since: 16 * MINUTE });
62
+ });
63
+
64
+ test("a session that says how long a turn may go quiet is held to that, not to the default", () => {
65
+ const model = modelOf([row("session.started", { session: ME, id: "s-1", stuckAfter: 2 * MINUTE }, 0), row("session.turn-started", { session: ME, turn: 1 }, 0)]);
66
+ expect(read(model, me, 2 * MINUTE)).toMatchObject({ state: "stuck", kind: "silent", since: 2 * MINUTE });
67
+ });
68
+
69
+ test("a session between turns is idle for as long as it likes", () => {
70
+ const model = modelOf([...turn(0), row("session.turn-ended", { session: ME, turn: 1, how: "completed" }, MINUTE)]);
71
+ expect(read(model, me, 5 * 60 * MINUTE)).toMatchObject({ state: "idle", since: MINUTE });
72
+ });
73
+
74
+ test("a turn that ended nobody knows how leaves the session unknown", () => {
75
+ const model = modelOf([...turn(0), row("session.turn-ended", { session: ME, turn: 1, how: "unknown" }, MINUTE)]);
76
+ expect(read(model, me, 2 * MINUTE)).toMatchObject({ state: "unknown", since: MINUTE });
77
+ // The next turn makes it working again.
78
+ model.apply(row("session.turn-started", { session: ME, turn: 2 }, 3 * MINUTE));
79
+ expect(read(model, me, 4 * MINUTE)).toMatchObject({ state: "working", since: 3 * MINUTE });
80
+ });
81
+
82
+ test("the same activity over and over with no progress is looping, since the first of them", () => {
83
+ const rows = [...turn(0)];
84
+ for (let n = 1; n <= 25; n += 1) rows.push(row("session.activity", { session: ME, tool: "Edit" }, n * 1000));
85
+ const model = modelOf(rows);
86
+ expect(read(model, me, 26_000)).toEqual({ subject: me, state: "looping", kind: "looping", since: 1000, last: 25_000 });
87
+ // Progress breaks the loop.
88
+ model.apply(row("session.progress", { session: { harness: "claude", native: "abc" }, done: 1, total: 3 }, 27_000));
89
+ expect(read(model, me, 28_000)?.state).toBe("working");
90
+ });
91
+
92
+ test("a different tool in between, or one fewer repeat, is not a loop", () => {
93
+ const rows = [...turn(0)];
94
+ for (let n = 1; n <= 24; n += 1) rows.push(row("session.activity", { session: ME, tool: "Edit" }, n * 1000));
95
+ expect(read(modelOf(rows), me, 25_000)?.state).toBe("working");
96
+ rows.push(row("session.activity", { session: ME, tool: "Read" }, 25_000), row("session.activity", { session: ME, tool: "Edit" }, 26_000));
97
+ expect(read(modelOf(rows), me, 27_000)?.state).toBe("working");
98
+ });
99
+
100
+ test("activity that names no tool is a sign of life and not a repeated operation", () => {
101
+ const rows = [...turn(0)];
102
+ for (let n = 1; n <= 40; n += 1) rows.push(row("session.activity", { session: ME }, n * 1000));
103
+ expect(read(modelOf(rows), me, 41_000)).toMatchObject({ state: "working", last: 40_000 });
104
+ // Between calls of one tool it neither adds to the repeats nor breaks them.
105
+ const calls = [...turn(0)];
106
+ for (let n = 1; n <= 25; n += 1) calls.push(row("session.activity", { session: ME, tool: "Edit" }, n * 2000), row("session.activity", { session: ME }, n * 2000 + 1000));
107
+ expect(read(modelOf(calls), me, 60_000)).toMatchObject({ state: "looping", since: 2000 });
108
+ });
109
+
110
+ test("a head that moved for the session counts as progress and breaks a loop", () => {
111
+ const rows = [...turn(0)];
112
+ for (let n = 1; n <= 25; n += 1) rows.push(row("session.activity", { session: ME, tool: "Edit" }, n * 1000));
113
+ rows.push(row("git.head.moved", { session: ME, head: "abc123" }, 26_000));
114
+ expect(read(modelOf(rows), me, 27_000)).toMatchObject({ state: "working", last: 26_000 });
115
+ });
116
+
117
+ test("an open question from the session means waiting for a person, not silence", () => {
118
+ const asker = { kind: "session", harness: "claude", id: "abc" };
119
+ const model = modelOf([...turn(0), row("request.created", { id: "req-1", to: { kind: "owner" }, from: asker }, MINUTE)]);
120
+ expect(read(model, me, 30 * MINUTE)).toMatchObject({ state: "waiting-human" });
121
+ model.apply(row("request.answered", { id: "req-1", answer: {} }, 31 * MINUTE));
122
+ expect(read(model, me, 32 * MINUTE)).toMatchObject({ state: "working", last: 31 * MINUTE });
123
+ });
124
+
125
+ test("a question to another session is waiting for an event, not for a person", () => {
126
+ const asker = { kind: "session", harness: "claude", id: "abc" };
127
+ const model = modelOf([...turn(0), row("request.created", { id: "req-1", to: { kind: "session", harness: "codex", id: "x1" }, from: asker }, MINUTE)]);
128
+ expect(read(model, me, 30 * MINUTE)).toMatchObject({ state: "waiting-event" });
129
+ // A person's question beside it makes the session wait for the person.
130
+ model.apply(row("request.created", { id: "req-2", to: { kind: "owner" }, from: asker }, 2 * MINUTE));
131
+ expect(read(model, me, 30 * MINUTE)).toMatchObject({ state: "waiting-human" });
132
+ model.apply(row("request.cancelled", { id: "req-2" }, 3 * MINUTE));
133
+ expect(read(model, me, 30 * MINUTE)).toMatchObject({ state: "waiting-event" });
134
+ model.apply(row("request.answered", { id: "req-1", answer: {} }, 4 * MINUTE));
135
+ expect(read(model, me, 5 * MINUTE)).toMatchObject({ state: "working" });
136
+ });
137
+
138
+ test("a question puts the session in waiting-human from its own moment, and the answer ends it", () => {
139
+ const asker = { kind: "session", harness: "claude", id: "abc" };
140
+ const model = modelOf([...turn(0), row("request.created", { id: "req-1", to: { kind: "owner" }, from: asker }, MINUTE)]);
141
+ expect(read(model, me, 30 * MINUTE)).toMatchObject({ state: "waiting-human", since: MINUTE });
142
+ model.apply(row("request.answered", { id: "req-1", answer: {} }, 31 * MINUTE));
143
+ expect(read(model, me, 32 * MINUTE)).toMatchObject({ state: "working", since: 31 * MINUTE });
144
+ });
145
+
146
+ test("a session of another harness is unknown when first seen, working with its progress, idle when it pauses", () => {
147
+ const named = { harness: "codex", native: "x1" };
148
+ const subject = sessionSubject("codex", "x1");
149
+ const model = modelOf([row("history.session.seen", { session: named, harness: "codex" }, 0)]);
150
+ expect(read(model, subject, MINUTE)).toMatchObject({ state: "unknown" });
151
+ model.apply(row("session.progress", { session: named, done: 0, total: 2 }, 2 * MINUTE));
152
+ expect(read(model, subject, 3 * MINUTE)).toMatchObject({ state: "working", since: 2 * MINUTE });
153
+ model.apply(row("history.session.paused", { session: named, upto: "x", why: "idle" }, 4 * MINUTE));
154
+ expect(read(model, subject, 99 * MINUTE)).toMatchObject({ state: "idle", since: 4 * MINUTE });
155
+ });
156
+
157
+ test("the history beginning to watch a session niwa runs does not make it unknown", () => {
158
+ const model = modelOf([...turn(0), row("history.session.seen", { session: { harness: "claude", native: "abc" }, harness: "claude" }, 1000)]);
159
+ expect(read(model, me, 2000)).toMatchObject({ state: "working", since: 0 });
160
+ });
161
+
162
+ test("an ended session is not followed, and a stray sign after the end does not bring it back", () => {
163
+ const model = modelOf([...turn(0), row("session.ended", { session: ME }, MINUTE)]);
164
+ expect(read(model, me, 2 * MINUTE)).toBeUndefined();
165
+ model.apply(row("session.activity", { session: ME, tool: "Bash" }, 3 * MINUTE));
166
+ expect(model.readings(4 * MINUTE)).toEqual([]);
167
+ // A new turn is a reason.
168
+ model.apply(row("session.turn-started", { session: ME, turn: 2 }, 5 * MINUTE));
169
+ expect(read(model, me, 6 * MINUTE)?.state).toBe("working");
170
+ });
171
+
172
+ test("a session whose name cannot be a subject is passed over, not thrown at", () => {
173
+ const rows = [
174
+ row("session.started", { session: "claude", id: "s-1" }, 0),
175
+ row("session.turn-started", { session: "claude ", turn: 1 }, 0),
176
+ row("session.activity", { session: { harness: "cl aude", native: "x" } }, 0),
177
+ row("session.progress", { session: { harness: "claude", native: "" }, done: 0, total: 1 }, 0),
178
+ row("history.session.seen", { session: " x", harness: "claude" }, 0),
179
+ row("request.created", { id: "req-1", to: { kind: "owner" }, from: { kind: "session", harness: "a b", id: "c" } }, 0),
180
+ ...turn(0),
181
+ ];
182
+ expect(() => modelOf(rows)).not.toThrow();
183
+ expect(modelOf(rows).readings(MINUTE).map((one) => one.subject)).toEqual([me]);
184
+ });
185
+
186
+ test("a limit that is not a whole number of milliseconds is ignored, so a flag can always be dated", () => {
187
+ const model = modelOf([row("session.started", { session: ME, id: "s-1", stuckAfter: 1500.5 }, 0), row("session.turn-started", { session: ME, turn: 1 }, 0)]);
188
+ const reading = read(model, me, 20 * MINUTE);
189
+ expect(reading).toMatchObject({ state: "stuck", kind: "silent", since: 10 * MINUTE });
190
+ expect(Reading.safeParse(reading).success).toBe(true);
191
+ });
192
+
193
+ test("a record that cannot be read is passed over", () => {
194
+ const model = modelOf([row("session.started", { nonsense: true }, 0), row("session.activity", 7, 0), ...turn(0)]);
195
+ expect(model.readings(MINUTE).map((one) => one.subject)).toEqual([me]);
196
+ });
197
+ });
198
+
199
+ describe("sessions the history also follows", () => {
200
+ const named = { harness: "claude", native: "abc" };
201
+ const managed = (at = 0) => [row("session.started", { session: ME, id: "s-1" }, 0), row("session.turn-started", { session: ME, turn: 1 }, at)];
202
+
203
+ test("a pause of the transcript does not make a turn niwa runs idle: a hung turn goes on to be stuck", () => {
204
+ const model = modelOf([...managed(), row("history.session.seen", { session: named, harness: "claude" }, 1000), row("history.session.paused", { session: named, upto: "x", why: "idle" }, MINUTE)]);
205
+ expect(read(model, me, 5 * MINUTE)).toMatchObject({ state: "working", since: 0 });
206
+ expect(read(model, me, 10 * MINUTE)).toMatchObject({ state: "stuck", kind: "silent", since: 10 * MINUTE });
207
+ });
208
+
209
+ test("a guessed end from the history does not end a session niwa runs; an exact one does", () => {
210
+ const model = modelOf([...managed(), row("history.session.ended", { session: named, certainty: "guessed", why: "idle" }, MINUTE)]);
211
+ expect(read(model, me, 2 * MINUTE)).toMatchObject({ state: "working" });
212
+ model.apply(row("history.session.ended", { session: named, certainty: "exact", why: "session-ended" }, 3 * MINUTE));
213
+ expect(read(model, me, 4 * MINUTE)).toBeUndefined();
214
+ });
215
+
216
+ test("a session only the history follows is not followed once it is unwatched, and its flag is closed", () => {
217
+ const other = { harness: "codex", native: "x1" };
218
+ const subject = sessionSubject("codex", "x1");
219
+ const model = modelOf([
220
+ row("history.session.seen", { session: other, harness: "codex" }, 0),
221
+ row("session.progress", { session: other, done: 1, total: 3 }, MINUTE),
222
+ row("watch.stuck", { subject, since: 11 * MINUTE, last: MINUTE, kind: "silent" }, 11 * MINUTE),
223
+ row("history.session.unwatched", { session: other }, 12 * MINUTE),
224
+ ]);
225
+ expect(read(model, subject, 60 * MINUTE)).toBeUndefined();
226
+ expect(transitions(model.flags(), model.readings(60 * MINUTE)).map((one) => one.type)).toEqual(["recovered"]);
227
+ // A sign from it afterwards does not bring it back.
228
+ model.apply(row("session.progress", { session: other, done: 2, total: 3 }, 13 * MINUTE));
229
+ expect(model.readings(14 * MINUTE)).toEqual([]);
230
+ });
231
+
232
+ test("a session niwa runs is still watched by its own records after the history stops watching it", () => {
233
+ const model = modelOf([...managed(), row("history.session.unwatched", { session: named }, MINUTE)]);
234
+ expect(read(model, me, 2 * MINUTE)).toMatchObject({ state: "working" });
235
+ expect(read(model, me, 10 * MINUTE)).toMatchObject({ state: "stuck", kind: "silent" });
236
+ });
237
+ });
238
+
239
+ describe("runs", () => {
240
+ const run = runSubject("run-1");
241
+ const inv = handler("inv-1");
242
+ const started = (at = 0) => [row("workflow.run.requested", { run: "run-1", file: "f.ts" }, at), row("workflow.run.started", { run: "run-1", file: "f.ts" }, at, inv)];
243
+ const intent = (key: string, input: unknown, at: number, action = "git.status") =>
244
+ row("niwa.step.intent", { invocation: "inv-1", key, action, input, operationId: `op-${key}` }, at);
245
+
246
+ test("a run asked for and not started is idle", () => {
247
+ const model = modelOf([row("workflow.run.requested", { run: "run-1", file: "f.ts" }, 0)]);
248
+ expect(model.readingOf(run, 99 * MINUTE)).toMatchObject({ state: "idle", since: 0 });
249
+ });
250
+
251
+ test("a started run is working, and stuck when its steps stop for too long", () => {
252
+ const model = modelOf([...started(), intent("a", 1, MINUTE)]);
253
+ expect(model.readingOf(run, 5 * MINUTE)).toMatchObject({ state: "working", last: MINUTE });
254
+ expect(model.readingOf(run, 11 * MINUTE)).toEqual({ subject: run, state: "stuck", kind: "silent", since: 11 * MINUTE, last: MINUTE });
255
+ });
256
+
257
+ test("a run suspended on a record waits for an event; on a timer it sleeps; on a question it waits for a person", () => {
258
+ const model = modelOf([...started(), row("niwa.invocation.state", { invocation: "inv-1", state: "waiting" }, MINUTE)]);
259
+ expect(model.readingOf(run, 60 * MINUTE)).toMatchObject({ state: "waiting-event" });
260
+
261
+ model.apply(intent("nap", null, 2 * MINUTE, "niwa.wait"));
262
+ model.apply(row("timer.sleep.requested", { at: 90 * MINUTE, key: "inv-1:nap" }, 2 * MINUTE, inv));
263
+ expect(model.readingOf(run, 60 * MINUTE)).toMatchObject({ state: "sleeping" });
264
+ model.apply(row("timer.slept", { key: "inv-1:nap" }, 90 * MINUTE));
265
+ model.apply(row("niwa.step.outcome", { invocation: "inv-1", key: "nap", outcome: {} }, 90 * MINUTE));
266
+ expect(model.readingOf(run, 91 * MINUTE)).toMatchObject({ state: "waiting-event" });
267
+
268
+ model.apply(row("request.created", { id: "req-9", to: { kind: "owner" } }, 92 * MINUTE, inv));
269
+ expect(model.readingOf(run, 120 * MINUTE)).toMatchObject({ state: "waiting-human" });
270
+ model.apply(row("request.expired", { id: "req-9" }, 121 * MINUTE));
271
+ model.apply(row("niwa.invocation.state", { invocation: "inv-1", state: "running" }, 121 * MINUTE));
272
+ expect(model.readingOf(run, 122 * MINUTE)).toMatchObject({ state: "working" });
273
+ });
274
+
275
+ test("a wait with a timeout waits for an event, and a timer left behind does not make a later wait a sleep", () => {
276
+ const outcome = (key: string, at: number) => row("niwa.step.outcome", { invocation: "inv-1", key, outcome: {} }, at);
277
+ const model = modelOf([
278
+ ...started(),
279
+ intent("w1", { type: "x.done", place: "inv-1:w1" }, MINUTE, "workflow.script.wait"),
280
+ row("timer.sleep.requested", { at: 90 * MINUTE, key: "inv-1:w1" }, MINUTE, inv),
281
+ row("niwa.invocation.state", { invocation: "inv-1", state: "waiting" }, MINUTE),
282
+ ]);
283
+ expect(model.readingOf(run, 30 * MINUTE)).toMatchObject({ state: "waiting-event" });
284
+ // The record came before the timer ran out; its timer is still armed when the run waits again.
285
+ model.apply(outcome("w1", 2 * MINUTE));
286
+ model.apply(row("niwa.invocation.state", { invocation: "inv-1", state: "running" }, 2 * MINUTE));
287
+ model.apply(intent("w2", { type: "y.done" }, 3 * MINUTE, "workflow.script.wait"));
288
+ model.apply(row("niwa.invocation.state", { invocation: "inv-1", state: "waiting" }, 3 * MINUTE));
289
+ expect(model.readingOf(run, 30 * MINUTE)).toMatchObject({ state: "waiting-event" });
290
+ });
291
+
292
+ test("a sleep that is cancelled with its branch is no longer what the run waits for", () => {
293
+ const model = modelOf([
294
+ ...started(),
295
+ intent("nap", null, MINUTE, "niwa.wait"),
296
+ row("timer.sleep.requested", { at: 90 * MINUTE, key: "inv-1:nap" }, MINUTE, inv),
297
+ intent("w", { type: "x.done" }, MINUTE, "workflow.script.wait"),
298
+ row("niwa.step.cancelled", { invocation: "inv-1", prefixes: ["w"], reason: "enough" }, 2 * MINUTE),
299
+ row("niwa.invocation.state", { invocation: "inv-1", state: "waiting" }, 2 * MINUTE),
300
+ ]);
301
+ expect(model.readingOf(run, 30 * MINUTE)).toMatchObject({ state: "sleeping" });
302
+ });
303
+
304
+ test("a run that asked a session waits for an event; one that asked the owner waits for a person", () => {
305
+ const model = modelOf([...started(), row("niwa.invocation.state", { invocation: "inv-1", state: "waiting" }, MINUTE)]);
306
+ model.apply(row("request.created", { id: "req-s", to: { kind: "session", harness: "codex", id: "x1" } }, 2 * MINUTE, inv));
307
+ expect(model.readingOf(run, 30 * MINUTE)).toMatchObject({ state: "waiting-event" });
308
+ model.apply(row("request.created", { id: "req-o", to: { kind: "owner" } }, 3 * MINUTE, inv));
309
+ expect(model.readingOf(run, 30 * MINUTE)).toMatchObject({ state: "waiting-human" });
310
+ });
311
+
312
+ const state = (value: string, at: number, invocation = "inv-1") => row("niwa.invocation.state", { invocation, state: value }, at);
313
+ const resolved = (at: number, choice = "retry") => row("niwa.invocation.resolved", { invocation: "inv-1", key: "a", choice }, at);
314
+
315
+ test("a run paused on an error waits for a person until the answer, whichever it is", () => {
316
+ const paused = [...started(), row("workflow.run.paused", { run: "run-1", reason: "a step failed", key: "a", choices: ["retry", "skip", "cancel"] }, MINUTE, inv), state("paused", MINUTE)];
317
+ const model = modelOf(paused);
318
+ expect(model.readingOf(run, 60 * MINUTE)).toMatchObject({ state: "waiting-human" });
319
+ // A retry or a skip is an answer of the core; no `workflow.run.continued` comes with it.
320
+ model.apply(resolved(61 * MINUTE));
321
+ model.apply(state("running", 61 * MINUTE));
322
+ expect(model.readingOf(run, 62 * MINUTE)).toMatchObject({ state: "working" });
323
+ });
324
+
325
+ test("a run cancelled from a pause is over, and one that waits for its script to be revised is not", () => {
326
+ const failed = [...started(), row("workflow.run.paused", { run: "run-1", reason: "a step failed", key: "a", choices: ["retry", "skip", "cancel"] }, MINUTE, inv), state("paused", MINUTE)];
327
+ const cancelled = modelOf([...failed, resolved(2 * MINUTE, "cancel"), state("done", 2 * MINUTE)]);
328
+ expect(cancelled.readings(3 * MINUTE)).toEqual([]);
329
+
330
+ const edited = modelOf([...started(), row("workflow.run.paused", { run: "run-1", reason: "the script was edited", choices: ["cancel"] }, MINUTE, inv), state("done", MINUTE)]);
331
+ expect(edited.readingOf(run, 60 * MINUTE)).toMatchObject({ state: "waiting-human" });
332
+ });
333
+
334
+ test("a revised run starts clean: the pause of the attempt before does not carry over", () => {
335
+ const model = modelOf([...started(), row("workflow.run.paused", { run: "run-1", reason: "the script was edited", choices: ["cancel"] }, MINUTE, inv), state("done", MINUTE)]);
336
+ model.apply(row("workflow.run.requested", { run: "run-1", file: "f.ts" }, 2 * MINUTE));
337
+ expect(model.readingOf(run, 3 * MINUTE)).toMatchObject({ state: "idle" });
338
+ const next = handler("inv-2");
339
+ model.apply(row("workflow.run.started", { run: "run-1", file: "f.ts" }, 4 * MINUTE, next));
340
+ expect(model.readingOf(run, 5 * MINUTE)).toMatchObject({ state: "working" });
341
+ // What the first attempt's invocation writes afterwards is not the new attempt's.
342
+ model.apply(state("paused", 6 * MINUTE));
343
+ model.apply(row("workflow.run.paused", { run: "run-1", reason: "late" }, 6 * MINUTE, inv));
344
+ expect(model.readingOf(run, 7 * MINUTE)).toMatchObject({ state: "working" });
345
+ });
346
+
347
+ test("a pause someone asked for holds the run at its next boundary, and only the matching continue releases it", () => {
348
+ const begun = [...started(), intent("a", 1, 30_000)];
349
+ const pause = row("workflow.run.pause-requested", { run: "run-1" }, MINUTE);
350
+ const model = modelOf([...begun, pause]);
351
+ // Not yet at a boundary: the run goes on.
352
+ expect(model.readingOf(run, 2 * MINUTE)).toMatchObject({ state: "working" });
353
+ model.apply(state("waiting", 3 * MINUTE));
354
+ expect(model.readingOf(run, 60 * MINUTE)).toMatchObject({ state: "waiting-human" });
355
+ // Another pause's release (a hold shares the record) does not free it.
356
+ model.apply(row("workflow.run.continued", { run: "run-1", pause: "some-hold" }, 4 * MINUTE));
357
+ expect(model.readingOf(run, 60 * MINUTE)).toMatchObject({ state: "waiting-human" });
358
+ model.apply(row("workflow.run.continued", { run: "run-1", pause: String(pause.id) }, 5 * MINUTE));
359
+ model.apply(state("running", 5 * MINUTE));
360
+ expect(model.readingOf(run, 6 * MINUTE)).toMatchObject({ state: "working" });
361
+ });
362
+
363
+ test("a run that reached its deadline waits for a person from the moment it says so, until it is extended", () => {
364
+ const begun = started();
365
+ const pause = row("workflow.run.pause-requested", { run: "run-1", kind: "deadline", deadline: MINUTE }, MINUTE);
366
+ const model = modelOf([...begun, pause]);
367
+ expect(model.readingOf(run, 2 * MINUTE)).toMatchObject({ state: "waiting-human" });
368
+ model.apply(row("workflow.run.continued", { run: "run-1", pause: String(pause.id) }, 3 * MINUTE));
369
+ expect(model.readingOf(run, 4 * MINUTE)).toMatchObject({ state: "working" });
370
+ });
371
+
372
+ test("the same step six times in a row is a loop; another step or a moved head ends it", () => {
373
+ const rows = [...started()];
374
+ for (let n = 1; n <= 6; n += 1) rows.push(intent(`k${n}`, { repo: "r" }, n * 1000));
375
+ const model = modelOf(rows);
376
+ expect(model.readingOf(run, 7000)).toEqual({ subject: run, state: "looping", kind: "looping", since: 1000, last: 6000 });
377
+
378
+ model.apply(row("git.head.moved", { repo: "r", head: "h2" }, 8000, inv));
379
+ expect(model.readingOf(run, 9000)?.state).toBe("working");
380
+
381
+ const other = modelOf([...rows.slice(0, 5), intent("odd", { repo: "x" }, 6000), intent("k7", { repo: "r" }, 7000)]);
382
+ expect(other.readingOf(run, 8000)?.state).toBe("working");
383
+ });
384
+
385
+ test("the steps the bridge adds around a call do not hide a loop of the calls", () => {
386
+ const rows = [...started()];
387
+ for (let n = 1; n <= 6; n += 1) {
388
+ rows.push(intent(`k${n}~auth`, { verb: "call", name: "git.status" }, n * 1000, "workflow.authorize"), intent(`k${n}`, { repo: "r" }, n * 1000 + 1));
389
+ }
390
+ expect(modelOf(rows).readingOf(run, 7000)).toEqual({ subject: run, state: "looping", kind: "looping", since: 1001, last: 6001 });
391
+ // They are signs of life all the same.
392
+ const lone = modelOf([...started(), intent("k1~auth", { verb: "call", name: "x" }, 5 * MINUTE, "workflow.authorize")]);
393
+ expect(lone.readingOf(run, 6 * MINUTE)).toMatchObject({ state: "working", last: 5 * MINUTE });
394
+ });
395
+
396
+ test("a loop of the same sleep is not counted by the clock and the arming around it", () => {
397
+ const rows = [...started()];
398
+ for (let n = 1; n <= 6; n += 1) {
399
+ rows.push(
400
+ intent(`nap${n}~now`, null, n * 1000, "niwa.now"),
401
+ intent(`nap${n}~arm`, { at: n * 5000, key: `inv-1:nap${n}` }, n * 1000, "timer.sleep.request"),
402
+ intent(`nap${n}`, null, n * 1000, "niwa.wait"),
403
+ );
404
+ }
405
+ expect(modelOf(rows).readingOf(run, 7000)).toMatchObject({ state: "working" });
406
+ });
407
+
408
+ test("a line the run wrote to its log is a sign of life, a line of an attempt before is not", () => {
409
+ const model = modelOf([...started(), intent("a", 1, 0), row("workflow.run.log", { run: "run-1", text: "halfway" }, 9 * MINUTE + 50_000, inv)]);
410
+ expect(model.readingOf(run, 19 * MINUTE)).toMatchObject({ state: "working", last: 9 * MINUTE + 50_000 });
411
+ expect(model.readingOf(run, 20 * MINUTE)).toMatchObject({ state: "stuck", kind: "silent" });
412
+ model.apply(row("workflow.run.log", { run: "run-1", text: "late" }, 15 * MINUTE, handler("inv-0")));
413
+ expect(model.readingOf(run, 20 * MINUTE)).toMatchObject({ last: 9 * MINUTE + 50_000 });
414
+ });
415
+
416
+ test("a phase the script enters or leaves is progress: a loop before it is over", () => {
417
+ const rows = [...started()];
418
+ for (let n = 1; n <= 6; n += 1) rows.push(intent(`k${n}`, { repo: "r" }, n * 1000));
419
+ const model = modelOf(rows);
420
+ expect(model.readingOf(run, 7000)?.state).toBe("looping");
421
+ model.apply(row("workflow.phase.entered", { run: "run-1", branch: "", phase: "Review" }, 8000, inv));
422
+ expect(model.readingOf(run, 9000)).toMatchObject({ state: "working", last: 8000 });
423
+ for (let n = 7; n <= 12; n += 1) model.apply(intent(`k${n}`, { repo: "r" }, n * 1000));
424
+ expect(model.readingOf(run, 13_000)?.state).toBe("looping");
425
+ model.apply(row("workflow.phase.left", { run: "run-1", branch: "" }, 14_000, inv));
426
+ expect(model.readingOf(run, 15_000)).toMatchObject({ state: "working", last: 14_000 });
427
+ });
428
+
429
+ test("since is when the state last changed: before a wait, inside it, and after it", () => {
430
+ const model = modelOf(started(10));
431
+ expect(model.readingOf(run, 50)).toMatchObject({ state: "working", since: 10 });
432
+ model.apply(row("niwa.invocation.state", { invocation: "inv-1", state: "waiting" }, 100));
433
+ expect(model.readingOf(run, 150)).toMatchObject({ state: "waiting-event", since: 100 });
434
+ model.apply(row("request.created", { id: "req-o", to: { kind: "owner" } }, 120, inv));
435
+ expect(model.readingOf(run, 150)).toMatchObject({ state: "waiting-human", since: 120 });
436
+ model.apply(row("request.answered", { id: "req-o", answer: {} }, 180));
437
+ expect(model.readingOf(run, 190)).toMatchObject({ state: "waiting-event", since: 180 });
438
+ model.apply(row("niwa.invocation.state", { invocation: "inv-1", state: "running" }, 200));
439
+ expect(model.readingOf(run, 210)).toMatchObject({ state: "working", since: 200 });
440
+ });
441
+
442
+ test("the time a pause begins is the time it is read from", () => {
443
+ const begun = started();
444
+ const pause = row("workflow.run.pause-requested", { run: "run-1", kind: "deadline", deadline: MINUTE }, MINUTE);
445
+ const model = modelOf([...begun, pause]);
446
+ expect(model.readingOf(run, 2 * MINUTE)).toMatchObject({ state: "waiting-human", since: MINUTE });
447
+ model.apply(row("workflow.run.continued", { run: "run-1", pause: String(pause.id) }, 3 * MINUTE));
448
+ expect(model.readingOf(run, 4 * MINUTE)).toMatchObject({ state: "working", since: 3 * MINUTE });
449
+ });
450
+
451
+ test("a record read twice is counted once", () => {
452
+ const rows = [...started()];
453
+ for (let n = 1; n <= 5; n += 1) rows.push(intent(`k${n}`, { repo: "r" }, n * 1000));
454
+ const model = new Model();
455
+ for (const one of rows) model.apply(one);
456
+ for (const one of rows) model.apply(one);
457
+ expect(model.readingOf(run, 6000)).toMatchObject({ state: "working" });
458
+ model.apply(intent("k6", { repo: "r" }, 6000));
459
+ expect(model.readingOf(run, 7000)).toMatchObject({ state: "looping" });
460
+ });
461
+
462
+ test("the input is compared as data, not as text: key order does not hide a loop", () => {
463
+ const rows = [...started()];
464
+ for (let n = 1; n <= 6; n += 1) rows.push(intent(`k${n}`, n % 2 === 0 ? { a: 1, b: 2 } : { b: 2, a: 1 }, n * 1000));
465
+ expect(modelOf(rows).readingOf(run, 7000)?.state).toBe("looping");
466
+ });
467
+
468
+ const shield = (scope: string, at: number) => row("workflow.scope.opened", { run: "run-1", scope, onExit: "wait", shield: true }, at);
469
+ const cancel = (at: number, except?: string[]) =>
470
+ row("niwa.step.cancelled", { invocation: "inv-1", prefixes: [""], ...(except === undefined ? {} : { except }), reason: "enough" }, at);
471
+
472
+ test("a cancel that spares a shielded scope makes the run stuck once the grace has passed, until the scope closes", () => {
473
+ const model = modelOf([...started(), shield("safe", 1000), cancel(5000, ["safe"])]);
474
+ expect(model.readingOf(run, 5000 + 59_999)?.kind).toBeUndefined();
475
+ expect(model.readingOf(run, 5000 + MINUTE)).toEqual({ subject: run, state: "stuck", kind: "shielded-cancel", since: 5000, last: 5000 });
476
+ model.apply(row("workflow.scope.closed", { run: "run-1", scope: "safe", outcome: "done" }, 70_000));
477
+ expect(model.readingOf(run, 80_000)?.kind).toBeUndefined();
478
+ });
479
+
480
+ test("a cancel that spares nothing is not held against the run, even when a shield opens long after it", () => {
481
+ const model = modelOf([...started(), cancel(5000), shield("late", 10 * MINUTE)]);
482
+ expect(model.readingOf(run, 15 * MINUTE)).toMatchObject({ state: "working", last: 10 * MINUTE });
483
+ });
484
+
485
+ test("only the shields a cancel spared are blamed, each from its own cancel", () => {
486
+ const model = modelOf([...started(), shield("a", 1000), cancel(5000, ["a"]), shield("b", 6000), cancel(10_000, ["a", "b"])]);
487
+ expect(model.readingOf(run, 20_000)?.kind).toBeUndefined();
488
+ expect(model.readingOf(run, 5000 + MINUTE)).toMatchObject({ kind: "shielded-cancel", since: 5000 });
489
+ // "a" closes: "b", spared by the later cancel, is what is left.
490
+ model.apply(row("workflow.scope.closed", { run: "run-1", scope: "a", outcome: "done" }, 62_000));
491
+ expect(model.readingOf(run, 63_000)?.kind).toBeUndefined();
492
+ expect(model.readingOf(run, 10_000 + MINUTE)).toMatchObject({ kind: "shielded-cancel", since: 10_000 });
493
+ });
494
+
495
+ test("a finished run is not followed", () => {
496
+ const model = modelOf([...started(), row("workflow.run.finished", { run: "run-1", outcome: { status: "done", result: null } }, MINUTE)]);
497
+ expect(model.readings(2 * MINUTE)).toEqual([]);
498
+ });
499
+
500
+ test("steps of an invocation that is no run of the model's are passed over", () => {
501
+ const model = modelOf([...started(), row("niwa.step.intent", { invocation: "inv-other", key: "a", action: "x", input: 1 }, 5000)]);
502
+ expect(model.readingOf(run, 6000)).toMatchObject({ last: 0 });
503
+ });
504
+ });
505
+
506
+ describe("the time the watch was not looking", () => {
507
+ test("silence is counted from the moment the watch began, not from a record written before it", () => {
508
+ const rows = [row("session.started", { session: ME, id: "s-1" }, 0), row("session.turn-started", { session: ME, turn: 1 }, 0)];
509
+ const model = modelOf(rows, 100 * MINUTE);
510
+ expect(read(model, me, 105 * MINUTE)?.state).toBe("working");
511
+ expect(read(model, me, 110 * MINUTE)).toMatchObject({ state: "stuck", since: 110 * MINUTE, last: 0 });
512
+ });
513
+
514
+ test("a silence the watch had already flagged stays stuck from where it began", () => {
515
+ const rows = [
516
+ row("session.started", { session: ME, id: "s-1" }, 0),
517
+ row("session.turn-started", { session: ME, turn: 1 }, 0),
518
+ row("watch.stuck", { subject: me, since: 10 * MINUTE, last: 0, kind: "silent" }, 10 * MINUTE),
519
+ ];
520
+ const model = modelOf(rows, 100 * MINUTE);
521
+ expect(read(model, me, 101 * MINUTE)).toMatchObject({ state: "stuck", kind: "silent", since: 10 * MINUTE });
522
+ expect(transitions(model.flags(), model.readings(101 * MINUTE))).toEqual([]);
523
+ });
524
+ });
525
+
526
+ describe("flags and transitions", () => {
527
+ const reading = (over: Partial<Reading>): Reading => ({ subject: me, state: "stuck", since: 10, last: 0, kind: "silent", ...over });
528
+ const flag = (over: Partial<OpenFlag> = {}): OpenFlag => ({ since: 10, last: 0, kind: "silent", ...over });
529
+ const flags = (one?: OpenFlag) => new Map(one === undefined ? [] : [[me, one]]);
530
+
531
+ test("a stuck subject with no flag gets one, and with the same flag gets nothing", () => {
532
+ expect(transitions(flags(), [reading({})])).toEqual([{ type: "stuck", data: { subject: me, since: 10, last: 0, kind: "silent" } }]);
533
+ expect(transitions(flags(flag()), [reading({})])).toEqual([]);
534
+ });
535
+
536
+ test("a flag whose subject works again, is gone, or is stuck another way is closed first", () => {
537
+ const closes: Transition = { type: "recovered", data: { subject: me, since: 10, kind: "silent" } };
538
+ expect(transitions(flags(flag()), [reading({ state: "working", kind: undefined })])).toEqual([closes]);
539
+ expect(transitions(flags(flag()), [])).toEqual([closes]);
540
+ expect(transitions(flags(flag()), [reading({ state: "looping", kind: "looping", since: 12 })])).toEqual([
541
+ closes,
542
+ { type: "stuck", data: { subject: me, since: 12, last: 0, kind: "looping" } },
543
+ ]);
544
+ // The same kind from a new moment is a new condition.
545
+ expect(transitions(flags(flag()), [reading({ since: 99 })]).map((one) => one.type)).toEqual(["recovered", "stuck"]);
546
+ });
547
+
548
+ test("a recovered record closes the flag it names and not a newer one", () => {
549
+ const model = modelOf([
550
+ row("watch.stuck", { subject: me, since: 10, last: 0, kind: "silent" }, 10),
551
+ row("watch.recovered", { subject: me, since: 10, kind: "silent" }, 20),
552
+ row("watch.stuck", { subject: me, since: 30, last: 25, kind: "silent" }, 30),
553
+ row("watch.recovered", { subject: me, since: 10, kind: "silent" }, 31),
554
+ ]);
555
+ expect([...model.flags().values()]).toEqual([{ since: 30, last: 25, kind: "silent" }]);
556
+ });
557
+ });
558
+
559
+ describe("what the service reads", () => {
560
+ test("the sources hold every record type the model reads", () => {
561
+ const query = resolveQuery({ types: SOURCES });
562
+ for (const type of [
563
+ "session.activity",
564
+ "history.session.unwatched",
565
+ "workflow.run.pause-requested",
566
+ "workflow.run.continued",
567
+ "workflow.run.log",
568
+ "workflow.phase.entered",
569
+ "workflow.phase.left",
570
+ "niwa.invocation.state",
571
+ "niwa.invocation.resolved",
572
+ "niwa.step.intent",
573
+ "niwa.step.outcome",
574
+ "niwa.step.cancelled",
575
+ ]) {
576
+ expect(selectsType(query, type), type).toBe(true);
577
+ }
578
+ });
579
+ });