@niwa-labs/watch 0.0.0-stage → 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +75 -2
- package/package.json +29 -4
- package/src/cockpit/actions.ts +29 -0
- package/src/cockpit/decisions.ts +35 -0
- package/src/cockpit/index.tsx +19 -0
- package/src/cockpit/page.tsx +116 -0
- package/src/cockpit/subjects.ts +138 -0
- package/src/index.ts +160 -0
- package/src/readings.ts +709 -0
- package/src/records.ts +180 -0
- package/test/cockpit-actions.test.ts +29 -0
- package/test/cockpit-subjects.test.ts +89 -0
- package/test/queries.test.ts +27 -0
- package/test/readings.test.ts +579 -0
- package/test/watch.test.ts +287 -0
- package/tsconfig.json +15 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Niwa Labs contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,76 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @niwa-labs/watch
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
A plugin that derives the state of every session and run from the journal and records when something is stuck and when it has recovered.
|
|
4
|
+
|
|
5
|
+
A subject is a session (`session:<harness> <native id>`) or a run (`run:<id>`). Each is in one state:
|
|
6
|
+
|
|
7
|
+
| State | Meaning |
|
|
8
|
+
| --- | --- |
|
|
9
|
+
| `working` | A turn or a run is going on and signs of life keep coming. |
|
|
10
|
+
| `waiting-human` | A question the subject put to the owner is open, or the run is paused for a decision: after a failed step, at a pause someone asked for once the run reaches a boundary, or at a deadline. |
|
|
11
|
+
| `waiting-event` | A run is suspended on a record or on other work, or a subject waits for the answer of a session or another address that is not the owner. |
|
|
12
|
+
| `sleeping` | A run is in a sleep. A wait for a record with a timeout is `waiting-event`: a record can end it. |
|
|
13
|
+
| `idle` | A session between turns, a run asked for and not started. |
|
|
14
|
+
| `looping` | Working, but repeating the same step with nothing changing. |
|
|
15
|
+
| `stuck` | Working and silent for too long, or cancelled and held up by a shielded scope. |
|
|
16
|
+
| `unknown` | The journal does not say: a session another harness runs was just seen, or a turn ended with no known outcome. |
|
|
17
|
+
|
|
18
|
+
`watch.get { subject }` and `watch.list { state? }` read the states now (effect `read`, world `niwa`). A grant names
|
|
19
|
+
the subject of `watch.get` as `session/<harness>/<id>` or `run/<id>`, each segment encoded, so
|
|
20
|
+
`watch.get@session/claude/*` covers one harness.
|
|
21
|
+
|
|
22
|
+
## Records
|
|
23
|
+
|
|
24
|
+
- `watch.stuck { subject, since, last, kind }` — `kind` is `silent` (no sign of life for `silentAfter`; a session that
|
|
25
|
+
says its own `stuckAfter` is held to that), `looping` (`loopRepeats` identical steps of a run, or `sessionLoopRepeats`
|
|
26
|
+
identical activities of a session, with no progress between them) or `shielded-cancel` (a cancel spared a
|
|
27
|
+
shielded scope of the run, as its `except` says, and the scope was still open `shieldGrace` later). `since` is when the condition began, `last` the moment of
|
|
28
|
+
the last sign of life.
|
|
29
|
+
- `watch.recovered { subject, since, kind }` — closes the `watch.stuck` with that subject, kind and `since`: the subject
|
|
30
|
+
gave a sign of life, stopped repeating itself, ended, or its shielded scope closed. Another `watch.stuck` for the
|
|
31
|
+
subject follows only after it.
|
|
32
|
+
|
|
33
|
+
The watch does not react. A workflow does, by subscribing to the records:
|
|
34
|
+
|
|
35
|
+
```ts
|
|
36
|
+
on: [{ type: "watch.stuck", filter: { kind: "silent" } }]
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
A script waits for or reads them with the data typed, through the builders of `@niwa-labs/watch/records`:
|
|
40
|
+
`await wait(watchStuck(subject), { timeout: "1h" })` and `watchRecovered(subject, kind?)`.
|
|
41
|
+
|
|
42
|
+
## Sources
|
|
43
|
+
|
|
44
|
+
The model reads by type name and imports none of the plugins that write them, so it works with whichever are installed:
|
|
45
|
+
`session.started`, `.turn-started`, `.turn-ended`, `.activity`, `.progress`, `.ended`; `history.session.*`;
|
|
46
|
+
`workflow.run.*` (including its log lines and pauses), `workflow.phase.*` and `workflow.scope.*`; the steps, cancels, answers
|
|
47
|
+
and state of the run's invocation (`niwa.step.*`, `niwa.invocation.resolved`, `niwa.invocation.state`); `request.*`;
|
|
48
|
+
`timer.sleep.requested` and `timer.slept`; and `git.head.moved`, which counts as progress for the session its data names or
|
|
49
|
+
the run whose handler wrote it.
|
|
50
|
+
|
|
51
|
+
A turn of a session niwa runs is open until `session.turn-ended` or `session.ended`: a quiet transcript (`history.session.paused`)
|
|
52
|
+
or a guessed end from the history does not close it. Only a session that the history alone follows goes idle on a pause, and
|
|
53
|
+
it is dropped when it is unwatched. Activity that names no tool is a sign of life but not a repeated operation, and the steps
|
|
54
|
+
the workflow bridge writes around a call (its permission check, the clock, the arming of a wake) are signs of life that
|
|
55
|
+
neither add to a loop nor break one.
|
|
56
|
+
|
|
57
|
+
A service reads new records on a clock (`tick`, 15 seconds) rather than through `api.on`. Silence is the absence of a
|
|
58
|
+
record, which no handler can be woken by, and a handler is an invocation that writes records of its own for every sign
|
|
59
|
+
of life.
|
|
60
|
+
|
|
61
|
+
## Cockpit
|
|
62
|
+
|
|
63
|
+
`./cockpit` is the plugin's page: every subject with its state, troubles first, and the recent `watch.stuck` and
|
|
64
|
+
`watch.recovered` records as incidents. The incidents come from the live feed, so they are the recent ones.
|
|
65
|
+
|
|
66
|
+
## Options
|
|
67
|
+
|
|
68
|
+
`watch({ silentAfter, loopRepeats, sessionLoopRepeats, shieldGrace, tick })`, all in milliseconds except the repeat
|
|
69
|
+
counts, and all positive whole numbers. Defaults: 10 minutes, 6, 25, 1 minute, 15 seconds.
|
|
70
|
+
|
|
71
|
+
## Limits
|
|
72
|
+
|
|
73
|
+
- Time the host was down is not counted as silence: it is counted from the moment the watch began looking. A silence
|
|
74
|
+
already flagged stays flagged across a restart.
|
|
75
|
+
- A long step that writes nothing while it runs looks silent. Raise `silentAfter` for hosts that run such steps.
|
|
76
|
+
- The model is rebuilt from the journal when the host starts.
|
package/package.json
CHANGED
|
@@ -1,6 +1,31 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@niwa-labs/watch",
|
|
3
|
-
"version": "0.0.0
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.0.0",
|
|
4
|
+
"license": "MIT",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"exports": {
|
|
7
|
+
".": "./src/index.ts",
|
|
8
|
+
"./records": "./src/records.ts",
|
|
9
|
+
"./cockpit": "./src/cockpit/index.tsx"
|
|
10
|
+
},
|
|
11
|
+
"dependencies": {
|
|
12
|
+
"@niwa-labs/cockpit-sdk": "*",
|
|
13
|
+
"@niwa-labs/core": "*",
|
|
14
|
+
"@niwa-labs/plugin": "*",
|
|
15
|
+
"@niwa-labs/ui-kit": "*",
|
|
16
|
+
"zod": "^4.6.7"
|
|
17
|
+
},
|
|
18
|
+
"devDependencies": {
|
|
19
|
+
"@niwa-labs/testing": "*",
|
|
20
|
+
"@tanstack/react-query": "^5.108.2",
|
|
21
|
+
"@types/react": "^19.3.0",
|
|
22
|
+
"@types/react-dom": "^19.3.0",
|
|
23
|
+
"react": "^19.3.1",
|
|
24
|
+
"react-dom": "^19.3.1"
|
|
25
|
+
},
|
|
26
|
+
"peerDependencies": {
|
|
27
|
+
"@tanstack/react-query": "^5.108.2",
|
|
28
|
+
"react": "^19.3.1",
|
|
29
|
+
"react-dom": "^19.3.1"
|
|
30
|
+
}
|
|
31
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The watch actions as the cockpit calls them: name and the part of each schema the page reads.
|
|
3
|
+
*
|
|
4
|
+
* The module runs in the browser and cannot import the host-side schemas, which reach for Node. A field the host
|
|
5
|
+
* stopped sending fails the page that reads it, which `test/cockpit-actions.test.ts` checks against the real actions.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { z } from "zod";
|
|
9
|
+
|
|
10
|
+
export const WatchState = z.enum(["working", "waiting-human", "waiting-event", "sleeping", "idle", "looping", "stuck", "unknown"]);
|
|
11
|
+
export type WatchState = z.infer<typeof WatchState>;
|
|
12
|
+
|
|
13
|
+
export const StuckKind = z.enum(["silent", "looping", "shielded-cancel"]);
|
|
14
|
+
export type StuckKind = z.infer<typeof StuckKind>;
|
|
15
|
+
|
|
16
|
+
export const Reading = z.object({
|
|
17
|
+
subject: z.string(),
|
|
18
|
+
state: WatchState,
|
|
19
|
+
since: z.number(),
|
|
20
|
+
last: z.number(),
|
|
21
|
+
kind: StuckKind.optional(),
|
|
22
|
+
});
|
|
23
|
+
export type Reading = z.infer<typeof Reading>;
|
|
24
|
+
|
|
25
|
+
export const listAction = {
|
|
26
|
+
name: "watch.list",
|
|
27
|
+
input: z.object({ state: WatchState.optional() }),
|
|
28
|
+
output: z.object({ subjects: z.array(Reading) }),
|
|
29
|
+
} as const;
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
|
|
3
|
+
const caller = z.object({ kind: z.string(), id: z.string().optional(), handler: z.string().optional() });
|
|
4
|
+
const refused = z.object({ recordType: z.string(), entity: z.string(), reason: z.string() });
|
|
5
|
+
const merge = z.object({
|
|
6
|
+
operation: z.string(),
|
|
7
|
+
pr: z.object({ repo: z.object({ host: z.string(), path: z.string() }), number: z.number() }),
|
|
8
|
+
reason: z.string().optional(), outcome: z.string().optional(),
|
|
9
|
+
});
|
|
10
|
+
|
|
11
|
+
const labels: Record<string, string> = {
|
|
12
|
+
"forge.pr.merge.started": "Запрошено слияние",
|
|
13
|
+
"forge.pr.merge.cancel.started": "Запрошена отмена",
|
|
14
|
+
"forge.pr.merge.cancel.refused": "В отмене отказано",
|
|
15
|
+
"forge.pr.merge.armed": "Слияние отложено",
|
|
16
|
+
"forge.pr.merge.settled": "Подтверждён итог слияния",
|
|
17
|
+
"forge.pr.merge.disarmed": "Отложенное слияние отменено",
|
|
18
|
+
"forge.pr.merge.refused": "В слиянии отказано",
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
export const decisionTypes = ["niwa.record.refused", ...Object.keys(labels)];
|
|
22
|
+
|
|
23
|
+
export function decisionText(row: { type: string; data: unknown; actor?: unknown }): string | undefined {
|
|
24
|
+
if (row.type === "niwa.record.refused") {
|
|
25
|
+
const data = refused.safeParse(row.data).data;
|
|
26
|
+
const actor = caller.safeParse(row.actor).data;
|
|
27
|
+
const who = actor && (actor.handler ?? actor.id ?? actor.kind);
|
|
28
|
+
if (data) return `${who ? `${who} · ` : ""}${data.recordType} · ${data.entity} · ${data.reason}`;
|
|
29
|
+
}
|
|
30
|
+
const label = labels[row.type];
|
|
31
|
+
const data = merge.safeParse(row.data).data;
|
|
32
|
+
if (!label || !data) return undefined;
|
|
33
|
+
const detail = data.reason ?? data.outcome;
|
|
34
|
+
return `${label} · ${data.pr.repo.host}/${data.pr.repo.path}!${data.pr.number}${detail ? ` · ${detail}` : ""}`;
|
|
35
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/** The cockpit module of the watch plugin: what every session and run is doing, and what was flagged as stuck. */
|
|
2
|
+
|
|
3
|
+
import { definePlugin } from "@niwa-labs/cockpit-sdk";
|
|
4
|
+
|
|
5
|
+
import { WatchPage } from "./page.tsx";
|
|
6
|
+
|
|
7
|
+
export default definePlugin({
|
|
8
|
+
id: "watch",
|
|
9
|
+
setup(ctx) {
|
|
10
|
+
ctx.pages.add({
|
|
11
|
+
path: "/watch",
|
|
12
|
+
title: "Наблюдение",
|
|
13
|
+
commandHeading: true,
|
|
14
|
+
nav: { order: 50 },
|
|
15
|
+
commands: [{ action: "watch.get", label: "Проверить объект" }],
|
|
16
|
+
component: WatchPage,
|
|
17
|
+
});
|
|
18
|
+
},
|
|
19
|
+
});
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
/** What every session and run is doing, which of them the watch flagged, and how the flags came and went. */
|
|
2
|
+
|
|
3
|
+
import { EntityLink, recordList, useCall, useProjection } from "@niwa-labs/cockpit-sdk";
|
|
4
|
+
import { CopyValue, StateRing } from "@niwa-labs/ui-kit";
|
|
5
|
+
|
|
6
|
+
import { decisionText, decisionTypes } from "./decisions.ts";
|
|
7
|
+
import { listAction, type Reading } from "./actions.ts";
|
|
8
|
+
import { KIND_LABEL, REFRESH_ON, RING, STATE_LABEL, agoText, incidents, ordered, subjectName, troubled, type Incident } from "./subjects.ts";
|
|
9
|
+
|
|
10
|
+
/** The flags the live feed holds: recent ones, not the whole history. */
|
|
11
|
+
const FLAGS = recordList({ types: ["watch.stuck", "watch.recovered"] }, 200);
|
|
12
|
+
|
|
13
|
+
const DECISIONS = recordList({ types: decisionTypes }, 200);
|
|
14
|
+
|
|
15
|
+
function SubjectLink({ subject }: { subject: string }) {
|
|
16
|
+
const name = subjectName(subject);
|
|
17
|
+
return (
|
|
18
|
+
<span className="flex min-w-0 flex-wrap items-center gap-2 text-sm font-medium">
|
|
19
|
+
{name.kind === "run" ? <>Запуск <EntityLink entity={{ kind: "run", id: name.id }} short /></> :
|
|
20
|
+
<>Сессия {name.harness} <CopyValue value={name.id} /></>}
|
|
21
|
+
</span>
|
|
22
|
+
);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function SubjectRow({ reading, now }: { reading: Reading; now: number }) {
|
|
26
|
+
return (
|
|
27
|
+
<li data-slot="subject-row" data-state={reading.state} className="min-w-0" style={{ display: "grid", gridTemplateColumns: "auto minmax(0, 1fr)", alignItems: "start", columnGap: "0.75rem", paddingBlock: "0.6rem" }}>
|
|
28
|
+
<StateRing state={RING[reading.state]} label={STATE_LABEL[reading.state]} />
|
|
29
|
+
<div className="flex min-w-0 flex-col gap-1">
|
|
30
|
+
<SubjectLink subject={reading.subject} />
|
|
31
|
+
<p className="desk-tools text-xs text-muted-foreground">
|
|
32
|
+
<span className={troubled(reading.state) ? "font-medium text-destructive" : "font-medium text-foreground"}>{STATE_LABEL[reading.state]}</span>
|
|
33
|
+
{reading.kind === undefined ? null : <span>{KIND_LABEL[reading.kind]}</span>}
|
|
34
|
+
<span>Состояние отмечено {agoText(reading.since, now)}</span>
|
|
35
|
+
<span>Последняя активность {agoText(reading.last, now)}</span>
|
|
36
|
+
</p>
|
|
37
|
+
</div>
|
|
38
|
+
</li>
|
|
39
|
+
);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function IncidentRow({ incident, now }: { incident: Incident; now: number }) {
|
|
43
|
+
const recovered = incident.recoveredAt !== undefined;
|
|
44
|
+
return (
|
|
45
|
+
<li data-slot="incident-row" data-recovered={recovered} className="min-w-0" style={{ display: "grid", gridTemplateColumns: "auto minmax(0, 1fr)", alignItems: "start", columnGap: "0.75rem", paddingBlock: "0.6rem" }}>
|
|
46
|
+
<StateRing state={recovered ? "done" : "failed"} label={recovered ? "Восстановилось" : "Застряло"} />
|
|
47
|
+
<div className="flex min-w-0 flex-col gap-1">
|
|
48
|
+
<SubjectLink subject={incident.subject} />
|
|
49
|
+
<p className="desk-tools text-xs text-muted-foreground">
|
|
50
|
+
<span>{KIND_LABEL[incident.kind]}</span>
|
|
51
|
+
<span>Замечено {agoText(incident.since, now)}</span>
|
|
52
|
+
{incident.recoveredAt === undefined ? null : <span>Восстановилось {agoText(incident.recoveredAt, now)}</span>}
|
|
53
|
+
</p>
|
|
54
|
+
</div>
|
|
55
|
+
</li>
|
|
56
|
+
);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export function WatchPage() {
|
|
60
|
+
const subjects = useCall(listAction, {}, { refreshOn: { types: REFRESH_ON.types } });
|
|
61
|
+
const flags = incidents(useProjection(FLAGS));
|
|
62
|
+
const decisions = useProjection(DECISIONS).flatMap(row => {
|
|
63
|
+
const text = decisionText(row);
|
|
64
|
+
return text ? [{ id: row.id, at: row.at, text }] : [];
|
|
65
|
+
});
|
|
66
|
+
const now = Date.now();
|
|
67
|
+
const rows = ordered(subjects.data?.subjects ?? []);
|
|
68
|
+
const needsAttention = (one: Reading) => troubled(one.state) || one.state === "waiting-human";
|
|
69
|
+
const urgent = rows.filter(needsAttention);
|
|
70
|
+
const rest = rows.filter((one) => !needsAttention(one));
|
|
71
|
+
const list = (items: Reading[]) => (
|
|
72
|
+
<ul data-slot="subjects" className="divide-y rounded-lg border bg-card px-3">
|
|
73
|
+
{items.map((reading) => <SubjectRow key={reading.subject} reading={reading} now={now} />)}
|
|
74
|
+
</ul>
|
|
75
|
+
);
|
|
76
|
+
return (
|
|
77
|
+
<div className="desk-page">
|
|
78
|
+
{subjects.isPending ? (
|
|
79
|
+
<p className="text-sm text-muted-foreground">Загружаем сессии и запуски…</p>
|
|
80
|
+
) : subjects.isError ? (
|
|
81
|
+
<p role="alert" className="text-sm text-destructive">Не удалось загрузить наблюдения. {subjects.error.message}</p>
|
|
82
|
+
) : rows.length === 0 ? (
|
|
83
|
+
<p className="text-sm text-muted-foreground">Хост ещё не наблюдает ни одной сессии или запуска.</p>
|
|
84
|
+
) : (
|
|
85
|
+
<>
|
|
86
|
+
<section className="flex min-w-0 flex-col gap-2">
|
|
87
|
+
<h3 className="text-sm font-semibold">Требуют внимания</h3>
|
|
88
|
+
{urgent.length ? list(urgent) : <p className="text-sm text-muted-foreground">Ничего не застряло и не ждёт ответа.</p>}
|
|
89
|
+
</section>
|
|
90
|
+
{rest.length ? <section className="flex min-w-0 flex-col gap-2">
|
|
91
|
+
<h3 className="text-sm font-semibold">Остальные сессии и запуски</h3>
|
|
92
|
+
{list(rest)}
|
|
93
|
+
</section> : null}
|
|
94
|
+
</>
|
|
95
|
+
)}
|
|
96
|
+
{decisions.length > 0 && <section>
|
|
97
|
+
<h3 className="text-sm font-semibold">Недавние слияния и отказы записи</h3>
|
|
98
|
+
<ul className="divide-y rounded-lg border px-3">
|
|
99
|
+
{decisions.slice().reverse().map(row => <li key={row.id} className="py-2 text-sm">{row.text} · {agoText(row.at, now)}</li>)}
|
|
100
|
+
</ul>
|
|
101
|
+
</section>}
|
|
102
|
+
<section className="flex min-w-0 flex-col gap-2">
|
|
103
|
+
<h3 className="text-sm font-semibold">История сбоев</h3>
|
|
104
|
+
{flags.length === 0 ? (
|
|
105
|
+
<p className="text-sm text-muted-foreground">В недавних записях сбоев нет.</p>
|
|
106
|
+
) : (
|
|
107
|
+
<ul data-slot="incidents" className="divide-y rounded-lg border bg-card px-3 text-card-foreground">
|
|
108
|
+
{flags.map((incident) => (
|
|
109
|
+
<IncidentRow key={`${incident.subject}/${incident.kind}/${String(incident.since)}`} incident={incident} now={now} />
|
|
110
|
+
))}
|
|
111
|
+
</ul>
|
|
112
|
+
)}
|
|
113
|
+
</section>
|
|
114
|
+
</div>
|
|
115
|
+
);
|
|
116
|
+
}
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
/** What the watch page makes of the subjects and of the records that flag them. */
|
|
2
|
+
|
|
3
|
+
import type { WorkState } from "@niwa-labs/ui-kit";
|
|
4
|
+
|
|
5
|
+
import type { Reading, StuckKind, WatchState } from "./actions.ts";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* The records after which the list of subjects is read again: those that move a subject between states. A session's
|
|
9
|
+
* activity is left out, as it moves nothing and arrives many times a second.
|
|
10
|
+
*/
|
|
11
|
+
export const REFRESH_ON = {
|
|
12
|
+
types: ["watch.*", "session.started", "session.turn-started", "session.turn-ended", "session.ended", "workflow.run.*", "request.*", "timer.*"],
|
|
13
|
+
} as const;
|
|
14
|
+
|
|
15
|
+
/** The ring a state is drawn with: a stuck or looping subject is a failure the owner should see. */
|
|
16
|
+
export const RING: Readonly<Record<WatchState, WorkState>> = {
|
|
17
|
+
working: "running",
|
|
18
|
+
"waiting-human": "waiting",
|
|
19
|
+
"waiting-event": "waiting",
|
|
20
|
+
sleeping: "waiting",
|
|
21
|
+
idle: "pending",
|
|
22
|
+
looping: "failed",
|
|
23
|
+
stuck: "failed",
|
|
24
|
+
unknown: "pending",
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
/** What a state is called on the page. */
|
|
28
|
+
export const STATE_LABEL: Readonly<Record<WatchState, string>> = {
|
|
29
|
+
working: "В работе",
|
|
30
|
+
"waiting-human": "Ждёт ответа",
|
|
31
|
+
"waiting-event": "Ждёт события",
|
|
32
|
+
sleeping: "Пауза по таймеру",
|
|
33
|
+
idle: "Простаивает",
|
|
34
|
+
looping: "Зациклилось",
|
|
35
|
+
stuck: "Застряло",
|
|
36
|
+
unknown: "Нет данных",
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
/** What a stuck kind says about the subject. */
|
|
40
|
+
export const KIND_LABEL: Readonly<Record<StuckKind, string>> = {
|
|
41
|
+
silent: "Нет активности",
|
|
42
|
+
looping: "Повторяет один и тот же шаг",
|
|
43
|
+
"shielded-cancel": "После отмены продолжает защищённую работу",
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
/** The order states are listed in: what needs the owner first. */
|
|
47
|
+
const ORDER: readonly WatchState[] = ["stuck", "looping", "waiting-human", "unknown", "working", "waiting-event", "sleeping", "idle"];
|
|
48
|
+
|
|
49
|
+
/** Whether a state is a problem the watch flagged. */
|
|
50
|
+
export function troubled(state: WatchState): boolean {
|
|
51
|
+
return state === "stuck" || state === "looping";
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** The subjects in the order of the page: troubles first, then by how long they have been in their state, longest first. */
|
|
55
|
+
export function ordered(readings: readonly Reading[]): Reading[] {
|
|
56
|
+
return [...readings].sort((a, b) => ORDER.indexOf(a.state) - ORDER.indexOf(b.state) || a.since - b.since || a.subject.localeCompare(b.subject));
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** How many subjects are in each state, the states with none left out. */
|
|
60
|
+
export function countByState(readings: readonly Reading[]): Array<{ state: WatchState; count: number }> {
|
|
61
|
+
return ORDER.flatMap((state) => {
|
|
62
|
+
const count = readings.filter((one) => one.state === state).length;
|
|
63
|
+
return count === 0 ? [] : [{ state, count }];
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** A subject taken apart: a run is `run:<id>`, a session `session:<harness> <native id>`. */
|
|
68
|
+
export type SubjectName = { kind: "run"; id: string } | { kind: "session"; harness: string; id: string };
|
|
69
|
+
|
|
70
|
+
export function subjectName(subject: string): SubjectName {
|
|
71
|
+
const colon = subject.indexOf(":");
|
|
72
|
+
const rest = subject.slice(colon + 1);
|
|
73
|
+
if (subject.slice(0, colon) === "run") return { kind: "run", id: rest };
|
|
74
|
+
const space = rest.indexOf(" ");
|
|
75
|
+
return space < 0 ? { kind: "session", harness: "", id: rest } : { kind: "session", harness: rest.slice(0, space), id: rest.slice(space + 1) };
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** A subject in the words of the page. */
|
|
79
|
+
export function subjectText(subject: string): string {
|
|
80
|
+
const name = subjectName(subject);
|
|
81
|
+
return name.kind === "run" ? `run ${name.id}` : `session ${name.harness === "" ? "" : `${name.harness}/`}${name.id}`;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** Relative time, with future timestamps shown as just now. */
|
|
85
|
+
export function agoText(at: number, now: number): string {
|
|
86
|
+
const seconds = Math.max(0, Math.round((now - at) / 1000));
|
|
87
|
+
if (seconds < 5) return "только что";
|
|
88
|
+
if (seconds < 90) return `${String(seconds)} с назад`;
|
|
89
|
+
const minutes = Math.round(seconds / 60);
|
|
90
|
+
if (minutes < 90) return `${String(minutes)} мин назад`;
|
|
91
|
+
const hours = Math.round(minutes / 60);
|
|
92
|
+
if (hours < 36) return `${String(hours)} ч назад`;
|
|
93
|
+
return `${String(Math.round(hours / 24))} дн назад`;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** A record as the feed gives it: the part the incidents read. */
|
|
97
|
+
export type FlagRecord = { readonly type: string; readonly data: unknown; readonly at: number };
|
|
98
|
+
|
|
99
|
+
/** A time a subject was flagged, and whether it has recovered. */
|
|
100
|
+
export type Incident = {
|
|
101
|
+
subject: string;
|
|
102
|
+
kind: StuckKind;
|
|
103
|
+
/** When the condition began. */
|
|
104
|
+
since: number;
|
|
105
|
+
/** The last sign of life, known from the flag. */
|
|
106
|
+
last: number;
|
|
107
|
+
/** When the watch wrote the recovery; undefined while the subject is still flagged. */
|
|
108
|
+
recoveredAt: number | undefined;
|
|
109
|
+
};
|
|
110
|
+
|
|
111
|
+
function fieldsOf(data: unknown): { subject: string; kind: StuckKind; since: number; last: number | undefined } | undefined {
|
|
112
|
+
if (data === null || typeof data !== "object") return undefined;
|
|
113
|
+
const found = data as Record<string, unknown>;
|
|
114
|
+
const { subject, kind, since } = found;
|
|
115
|
+
if (typeof subject !== "string" || typeof since !== "number") return undefined;
|
|
116
|
+
if (kind !== "silent" && kind !== "looping" && kind !== "shielded-cancel") return undefined;
|
|
117
|
+
return { subject, kind, since, last: typeof found["last"] === "number" ? found["last"] : undefined };
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* The incidents the records tell of, the latest flag first. A `watch.recovered` closes the flag with its subject,
|
|
122
|
+
* kind and `since`; a recovery whose flag the feed no longer holds is not an incident of its own.
|
|
123
|
+
*/
|
|
124
|
+
export function incidents(records: readonly FlagRecord[]): Incident[] {
|
|
125
|
+
const found = new Map<string, Incident>();
|
|
126
|
+
const keyOf = (one: { subject: string; kind: StuckKind; since: number }): string => `${one.subject}\0${one.kind}\0${String(one.since)}`;
|
|
127
|
+
for (const record of records) {
|
|
128
|
+
const fields = fieldsOf(record.data);
|
|
129
|
+
if (fields === undefined) continue;
|
|
130
|
+
if (record.type === "watch.stuck") {
|
|
131
|
+
found.set(keyOf(fields), { subject: fields.subject, kind: fields.kind, since: fields.since, last: fields.last ?? fields.since, recoveredAt: undefined });
|
|
132
|
+
} else if (record.type === "watch.recovered") {
|
|
133
|
+
const open = found.get(keyOf(fields));
|
|
134
|
+
if (open !== undefined) open.recoveredAt = record.at;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
return [...found.values()].sort((a, b) => b.since - a.since);
|
|
138
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The watch plugin: what every session and run is doing, read from the journal, and a record when
|
|
3
|
+
* one of them is stuck and when it is not any more.
|
|
4
|
+
*
|
|
5
|
+
* - **watch.get** and **watch.list** read the state of subjects now.
|
|
6
|
+
* - **watch.stuck** and **watch.recovered** are written by a service that looks at the journal on
|
|
7
|
+
* a clock. Reacting is not the watch's business: a workflow subscribes to those records
|
|
8
|
+
* (`on: [{ type: "watch.stuck", filter: { kind: "silent" } }]`) and decides what to do.
|
|
9
|
+
*
|
|
10
|
+
* Records are read with a query by type name (`readings.ts`) and not through `api.on`: silence is
|
|
11
|
+
* the absence of a record, so a handler per record could not see it, and a handler is an
|
|
12
|
+
* invocation, which would write several records for every sign of life it was woken by. The
|
|
13
|
+
* sessions, the workflows and the history are not imported, so the watch works with whichever of
|
|
14
|
+
* them the host has installed.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { fileURLToPath } from "node:url";
|
|
18
|
+
|
|
19
|
+
import { action, resolveQuery } from "@niwa-labs/core";
|
|
20
|
+
import type { StoredRecord } from "@niwa-labs/core";
|
|
21
|
+
import { COCKPIT_MODULE, plugin } from "@niwa-labs/plugin";
|
|
22
|
+
import type { Plugin } from "@niwa-labs/plugin";
|
|
23
|
+
|
|
24
|
+
import { DEFAULT_SETTINGS, Model, SOURCES, transitions } from "./readings.ts";
|
|
25
|
+
import type { Settings } from "./readings.ts";
|
|
26
|
+
import { WATCH_TYPES, WatchRecovered, WatchStuck, watchGet, watchList } from "./records.ts";
|
|
27
|
+
|
|
28
|
+
export * from "./records.ts";
|
|
29
|
+
export type { Settings } from "./readings.ts";
|
|
30
|
+
|
|
31
|
+
/** The limits of the watch; what is not given is the default. Durations are milliseconds. */
|
|
32
|
+
export interface WatchOptions extends Partial<Settings> {
|
|
33
|
+
/** How often the service looks. Default: 15 seconds. */
|
|
34
|
+
readonly tick?: number;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** A subject the watch does not follow: no record named it, or it has ended. */
|
|
38
|
+
export class UnknownSubject extends Error {
|
|
39
|
+
override name = "UnknownSubject";
|
|
40
|
+
constructor(readonly subject: string) {
|
|
41
|
+
super(`the watch does not follow ${subject}: no session or run of that name is working`);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const DEFAULT_TICK = 15_000;
|
|
46
|
+
|
|
47
|
+
function settingsOf(options: WatchOptions): Settings {
|
|
48
|
+
const settings: Settings = { ...DEFAULT_SETTINGS, ...stripped(options) };
|
|
49
|
+
for (const [name, value] of Object.entries({ ...settings, tick: options.tick ?? DEFAULT_TICK })) {
|
|
50
|
+
// The times of a flag are whole milliseconds, which the records of a flag require.
|
|
51
|
+
if (!Number.isSafeInteger(value) || value <= 0) throw new RangeError(`the watch setting "${name}" must be a positive whole number, got ${String(value)}`);
|
|
52
|
+
}
|
|
53
|
+
return settings;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** The options that were given, without the ones left undefined. */
|
|
57
|
+
function stripped(options: WatchOptions): Partial<Settings> {
|
|
58
|
+
const { tick: _tick, ...rest } = options;
|
|
59
|
+
return Object.fromEntries(Object.entries(rest).filter(([, value]) => value !== undefined)) as Partial<Settings>;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** The watch plugin. */
|
|
63
|
+
export function watch(options: WatchOptions = {}): Plugin {
|
|
64
|
+
const settings = settingsOf(options);
|
|
65
|
+
const tick = options.tick ?? DEFAULT_TICK;
|
|
66
|
+
|
|
67
|
+
return plugin("watch", (api) => {
|
|
68
|
+
api.declare(...WATCH_TYPES);
|
|
69
|
+
// The page of this plugin in the cockpit: a client module, offered by the path of its source.
|
|
70
|
+
api.offer(COCKPIT_MODULE, { id: "watch", entry: fileURLToPath(new URL("./cockpit/index.tsx", import.meta.url)) });
|
|
71
|
+
|
|
72
|
+
const model = new Model();
|
|
73
|
+
let cursor = 0;
|
|
74
|
+
let clock: (() => number) | undefined;
|
|
75
|
+
const now = (): number => clock?.() ?? Date.now();
|
|
76
|
+
|
|
77
|
+
/** Fold the records written since the last look. */
|
|
78
|
+
const sync = (): void => {
|
|
79
|
+
const rows: StoredRecord[] = api.journal?.()?.query(resolveQuery({ types: SOURCES, from: cursor })) ?? [];
|
|
80
|
+
for (const row of rows) {
|
|
81
|
+
model.apply(row);
|
|
82
|
+
cursor = row.seq;
|
|
83
|
+
}
|
|
84
|
+
};
|
|
85
|
+
|
|
86
|
+
api.service((ctx) => {
|
|
87
|
+
clock = () => ctx.now();
|
|
88
|
+
model.startedAt(ctx.now());
|
|
89
|
+
let stopped = false;
|
|
90
|
+
let cancel = (): void => {};
|
|
91
|
+
|
|
92
|
+
const look = async (): Promise<void> => {
|
|
93
|
+
sync();
|
|
94
|
+
for (const next of transitions(model.flags(), model.readings(ctx.now(), settings))) {
|
|
95
|
+
if (next.type === "stuck") {
|
|
96
|
+
const data = next.data;
|
|
97
|
+
await ctx.emit(WatchStuck, data);
|
|
98
|
+
} else {
|
|
99
|
+
const data = next.data;
|
|
100
|
+
await ctx.emit(WatchRecovered, data);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
// What was just written is read back, like any other record: the flags are those the journal holds,
|
|
104
|
+
// which is also what keeps a restart from writing one twice.
|
|
105
|
+
sync();
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
const arm = (): void => {
|
|
109
|
+
cancel = ctx.schedule(ctx.now() + tick, async () => {
|
|
110
|
+
try {
|
|
111
|
+
await look();
|
|
112
|
+
} catch (error) {
|
|
113
|
+
// A host that is closing has no journal to write to; anything else is a real failure.
|
|
114
|
+
if (api.journal?.() !== undefined && !stopped) throw error;
|
|
115
|
+
} finally {
|
|
116
|
+
if (!stopped) arm();
|
|
117
|
+
}
|
|
118
|
+
});
|
|
119
|
+
};
|
|
120
|
+
arm();
|
|
121
|
+
return () => {
|
|
122
|
+
stopped = true;
|
|
123
|
+
cancel();
|
|
124
|
+
};
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
api.action(
|
|
128
|
+
action(watchGet, {
|
|
129
|
+
recovery: "repeat",
|
|
130
|
+
effect: "read",
|
|
131
|
+
world: "niwa",
|
|
132
|
+
run: (input) => {
|
|
133
|
+
sync();
|
|
134
|
+
const reading = model.readingOf(input.subject, now(), settings);
|
|
135
|
+
if (reading === undefined) throw new UnknownSubject(input.subject);
|
|
136
|
+
return reading;
|
|
137
|
+
},
|
|
138
|
+
}),
|
|
139
|
+
);
|
|
140
|
+
|
|
141
|
+
api.action(
|
|
142
|
+
action(watchList, {
|
|
143
|
+
recovery: "repeat",
|
|
144
|
+
effect: "read",
|
|
145
|
+
world: "niwa",
|
|
146
|
+
run: (input) => {
|
|
147
|
+
sync();
|
|
148
|
+
return { subjects: model.readings(now(), settings).filter((one) => input.state === undefined || one.state === input.state) };
|
|
149
|
+
},
|
|
150
|
+
}),
|
|
151
|
+
);
|
|
152
|
+
|
|
153
|
+
api.summarize(() => {
|
|
154
|
+
sync();
|
|
155
|
+
const counts: Record<string, number> = {};
|
|
156
|
+
for (const one of model.readings(now(), settings)) counts[one.state] = (counts[one.state] ?? 0) + 1;
|
|
157
|
+
return counts;
|
|
158
|
+
});
|
|
159
|
+
});
|
|
160
|
+
}
|