omp-conductor 0.3.15 → 0.3.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -31
- package/package.json +2 -2
- package/skills/conductor-update/SKILL.md +179 -0
- package/src/cli.ts +31 -7
- package/src/fleet.ts +211 -40
- package/src/orchestrator-tick.ts +63 -2
package/README.md
CHANGED
|
@@ -7,17 +7,16 @@ tiers: first to an orchestrator session that can re-brief the worker, then to yo
|
|
|
7
7
|
## What it is
|
|
8
8
|
|
|
9
9
|
You label an issue. Within one tick the conductor claims it on the tracker, cuts a
|
|
10
|
-
worktree, hands one omp
|
|
11
|
-
|
|
10
|
+
worktree, hands one omp worker a self-contained brief, and watches it to a green
|
|
11
|
+
PR. The worker then stops: it never merges, tags, publishes or deploys.
|
|
12
12
|
|
|
13
13
|
### Scope
|
|
14
14
|
|
|
15
|
-
One issue, one green PR. That is the whole remit.
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
asked to. A worker whose change needs releasing reports that and stops.
|
|
15
|
+
One issue, one green PR. That is the worker's whole remit. A change that needs a
|
|
16
|
+
merge or release is reported to the orchestrator, whose `merge` and `release`
|
|
17
|
+
authority are chosen during setup and default to `human`. Granting either action
|
|
18
|
+
to the orchestrator never grants it to a worker or the dispatch daemon. Releases
|
|
19
|
+
remain batched from coherent groups of merged work, never cut one per worker PR.
|
|
21
20
|
|
|
22
21
|
Code counts every limit that decides whether work starts: concurrency, dollars per
|
|
23
22
|
day, turns and wall clock per worker, attempts per issue. None of it is left to the
|
|
@@ -29,10 +28,10 @@ When a run does get stuck, the first responder is not you. A tier-1 escalation i
|
|
|
29
28
|
injected into a long-lived **orchestrator session** that can read the issue and the
|
|
30
29
|
run's transcript and then either re-brief the worker or decide the problem genuinely
|
|
31
30
|
needs a human. It never edits product code and never pushes a branch; whether it
|
|
32
|
-
may
|
|
33
|
-
pages you directly.
|
|
31
|
+
may merge or release is a setup answer (`authority`), and both default to no. Only
|
|
32
|
+
tier 2 pages you directly.
|
|
34
33
|
|
|
35
|
-
The package ships three deployables, plus
|
|
34
|
+
The package ships three deployables, plus two skills:
|
|
36
35
|
|
|
37
36
|
| Deployable | Entry | What it is for |
|
|
38
37
|
| --- | --- | --- |
|
|
@@ -40,6 +39,7 @@ The package ships three deployables, plus one skill:
|
|
|
40
39
|
| Standalone daemon | `omp-conductor` binary | The dispatch loop, managed as a background process (`start` / `stop` / `restart`) with a `/healthz` endpoint for a supervisor. |
|
|
41
40
|
| Orchestrator heartbeat | omp extension, activated by `.conductor-tick.json` | Prompts a 24/7 orchestrator session on a fixed interval so its standing loop actually runs, and marks the session stalled when its prompts stop being consumed. Inert in every other session — including a second session opened in the fleet's own directory. See [Orchestrator tick](#orchestrator-tick). |
|
|
42
41
|
| Onboarding skill | `skill://conductor-onboarding` | Directs an omp session to interview you, read your repos for real CI gates, and tailor `ORCHESTRATOR.md` — then finish through the wizard. Discovered automatically once the plugin is installed. See [Onboarding](#onboarding). |
|
|
42
|
+
| Update skill | `skill://conductor-update` | Treats the Bun-global CLI, omp plugin, and Herdr plugin as one operation: pause claims, drain, install one pinned release, reload, verify twice, and restore the prior pause state. See [Updating](#updating). |
|
|
43
43
|
|
|
44
44
|
The first two are thin wrappers over the same `daemon.ts`, so the plugin and the
|
|
45
45
|
CLI cannot disagree about what a cap means or where the state lives. Claiming is
|
|
@@ -166,6 +166,28 @@ Also required on the host:
|
|
|
166
166
|
With neither, tier 2 degrades to a comment on the issue. Nothing is broken in
|
|
167
167
|
that configuration: it is supported, just slower to reach you.
|
|
168
168
|
|
|
169
|
+
## Updating
|
|
170
|
+
|
|
171
|
+
Say “update conductor” from an operator shell or maintenance omp session outside
|
|
172
|
+
the target `herdr-fleet.service`. That is the whole operator interface. The bundled
|
|
173
|
+
`skill://conductor-update` discovers the installed and registry versions, pauses
|
|
174
|
+
new claims while active work drains, and pins one release across the Bun-global
|
|
175
|
+
CLI, omp plugin, and Herdr plugin. It converts an old local Herdr link to a managed
|
|
176
|
+
checkout when necessary, reloads the Herdr service and dispatch daemon, verifies
|
|
177
|
+
the layered status twice, and restores the original dispatch state.
|
|
178
|
+
|
|
179
|
+
Ticks remain in their existing armed or disarmed state, so an ordinary update
|
|
180
|
+
does not halt the exact pane or require another Telegram arm challenge. Any
|
|
181
|
+
installation, reload, or verification failure leaves dispatch paused instead of
|
|
182
|
+
bringing up a mixed fleet. The skill does not publish npm or edit an install root.
|
|
183
|
+
It also refuses to run from the fleet pane that Herdr must restart: an updater
|
|
184
|
+
that kills itself cannot verify the result. With skill commands enabled, invoke
|
|
185
|
+
it directly with:
|
|
186
|
+
|
|
187
|
+
```text
|
|
188
|
+
/skill:conductor-update
|
|
189
|
+
```
|
|
190
|
+
|
|
169
191
|
## Onboarding
|
|
170
192
|
|
|
171
193
|
Onboarding this package has two layers, and installing it gives you both.
|
|
@@ -225,7 +247,7 @@ the file behind instead. So a re-run against a project that is already configure
|
|
|
225
247
|
opens with one question:
|
|
226
248
|
|
|
227
249
|
```text
|
|
228
|
-
"
|
|
250
|
+
"platform" is already configured — what would you like to do?
|
|
229
251
|
> Change one area
|
|
230
252
|
asks one area's questions; every other answer is carried through from the saved config
|
|
231
253
|
Walk every question again
|
|
@@ -237,14 +259,14 @@ says right now, so the row you want is the row you can see:
|
|
|
237
259
|
|
|
238
260
|
```text
|
|
239
261
|
Which area? Each row shows what it says now
|
|
240
|
-
tracker & repos —
|
|
241
|
-
gates —
|
|
262
|
+
tracker & repos — acme/platform, queue "ready-for-agent", "repo:" → platform, api, web, worker
|
|
263
|
+
gates — platform: bun run check; api: ruff check . @ backend; web: pnpm lint…
|
|
242
264
|
caps & worker model — 2 workers, 120 turns, 90m, $25/day, 2 attempts (all defaults) — harness default model
|
|
243
265
|
code graph — not configured — workers grep
|
|
244
266
|
authority — merge=orchestrator, release=orchestrator
|
|
245
|
-
escalation & triage — tier 2 pages Telegram
|
|
267
|
+
escalation & triage — tier 2 pages Telegram 123456789, comments too, triage external
|
|
246
268
|
reporting scope — material — escalations, plus green PRs, second failures, and anything that stops the fleet
|
|
247
|
-
orchestrator brief — none at
|
|
269
|
+
orchestrator brief — none at ~/.omp/conductor/worktrees/ORCHESTRATOR.md
|
|
248
270
|
```
|
|
249
271
|
|
|
250
272
|
Only that area's questions are asked. Every other answer is read back out of
|
|
@@ -254,11 +276,11 @@ writes a config, and it still writes nothing before you agree. The consent scree
|
|
|
254
276
|
leads with the delta and then shows the whole project as it would be written:
|
|
255
277
|
|
|
256
278
|
```text
|
|
257
|
-
amending code graph — project
|
|
279
|
+
amending code graph — project platform
|
|
258
280
|
was not configured — workers grep
|
|
259
|
-
now
|
|
281
|
+
now ~/.cache/conductor-graph/acme — 4 clone(s): platform, api, web, worker
|
|
260
282
|
carried over tracker & repos, gates, caps & worker model, authority, escalation & triage, reporting scope, orchestrator brief
|
|
261
|
-
read back from
|
|
283
|
+
read back from ~/.omp/conductor/config.json and rewritten unchanged
|
|
262
284
|
```
|
|
263
285
|
|
|
264
286
|
A first run, or a project name this config has never seen, never sees either
|
|
@@ -331,12 +353,22 @@ The **Learning loop** proposes diffs against `POLICY.md` for you to approve over
|
|
|
331
353
|
omp-conductor start
|
|
332
354
|
```
|
|
333
355
|
|
|
334
|
-
`start`
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
356
|
+
`start` first starts `herdr-fleet.service` when that optional unit is installed,
|
|
357
|
+
clearing a previous `halt --pane` recovery pin so Herdr can resume the exact
|
|
358
|
+
conductor pane. Hosts without systemd or without that unit keep the standalone
|
|
359
|
+
daemon behaviour. It then waits until the daemon actually answers
|
|
360
|
+
`GET /healthz`; spawning is not starting. A daemon whose config is broken,
|
|
361
|
+
whose port is taken or whose database is locked exits within a second, and the
|
|
362
|
+
command fails with the tail of `daemon.log` instead of printing a false
|
|
363
|
+
success. It refuses to start a second daemon, naming the live pid. Starting
|
|
364
|
+
processes does not clear `pause` or arm ticks; those remain explicit operator
|
|
365
|
+
decisions.
|
|
366
|
+
|
|
367
|
+
Pane recovery spans two separately installed plugins: npm ships the omp
|
|
368
|
+
heartbeat/status half, while `herdr-conductor` supplies `recover.sh`. After an
|
|
369
|
+
npm upgrade, refresh the Herdr plugin from `TerrifiedBug/conductor/herdr` as
|
|
370
|
+
well; publishing or installing npm alone cannot add the recovery-side tick
|
|
371
|
+
request.
|
|
340
372
|
|
|
341
373
|
For a first run, take a single tick in the foreground and watch it:
|
|
342
374
|
|
|
@@ -366,7 +398,7 @@ Four control planes used to answer "stop" differently. The package verbs:
|
|
|
366
398
|
|
|
367
399
|
`resume` clears pause only and **never re-arms**. `arm` is proof-gated: it sends a Telegram challenge and writes the arm marker only after your reply appears as a *user* turn in the orchestrator transcript. `halt --pane` targets the configured conductor agent only — it does **not** run `systemctl stop herdr-fleet`.
|
|
368
400
|
|
|
369
|
-
`status` prints a layered header (`dispatch` / `ticks` / `pane` / `recovery` / `herdr` / `daemon`) so a paused fleet cannot hide an armed orchestrator still spending turns.
|
|
401
|
+
`status` prints a layered header (`dispatch` / `ticks` / next tick time / `pane` / `recovery` / `herdr` / `telegram` / `daemon`) so a paused fleet cannot hide an armed orchestrator still spending turns. The Telegram line calls the official `getMe` endpoint to prove the token and API are usable without sending a message, then separately reports whether the inbound bridge is configured.
|
|
370
402
|
|
|
371
403
|
`halt --pane` is **fail-closed**: it exits `0` only when the conductor agent is
|
|
372
404
|
*proven* gone. It writes the recovery pin first, so a failed stop still cannot be
|
|
@@ -1043,7 +1075,7 @@ indistinguishable from the first:
|
|
|
1043
1075
|
```text
|
|
1044
1076
|
[omp-conductor] orchestrator tick inactive: pane w1:p1 (agent "fleet") owns the fleet tick here — this session will not tick
|
|
1045
1077
|
[omp-conductor] orchestrator tick inactive: this pane is agent "scratch", not the fleet agent "fleet" — this session will not tick
|
|
1046
|
-
[omp-conductor] orchestrator tick inactive: pid
|
|
1078
|
+
[omp-conductor] orchestrator tick inactive: pid 12345 (claimed 2026-01-02T03:04:05.000Z, session …/fleet.jsonl) owns the fleet tick in /home/conductor/.omp/conductor — this session will not tick
|
|
1047
1079
|
```
|
|
1048
1080
|
|
|
1049
1081
|
A `herdr agent list` that does not answer also declines, for the same reason the
|
|
@@ -1164,6 +1196,7 @@ least of all on a fleet whose session lives somewhere else.
|
|
|
1164
1196
|
|
|
1165
1197
|
```bash
|
|
1166
1198
|
omp-conductor start [--port N] [--project NAME]
|
|
1199
|
+
omp-conductor --version
|
|
1167
1200
|
omp-conductor stop
|
|
1168
1201
|
omp-conductor restart [--port N] [--project NAME]
|
|
1169
1202
|
omp-conductor status [--project NAME]
|
|
@@ -1184,10 +1217,10 @@ omp-conductor help
|
|
|
1184
1217
|
|
|
1185
1218
|
| Command | Behaviour |
|
|
1186
1219
|
| --- | --- |
|
|
1187
|
-
| `start` |
|
|
1220
|
+
| `start` | Start `herdr-fleet.service` when that optional unit is installed, clearing a previous pane-recovery pin, then spawn the dispatch loop in the background and wait until it answers `GET /healthz` on `:8787`. Without systemd or that unit it keeps the standalone daemon behaviour. It never clears pause or arms ticks. Refuses if a daemon is already live, naming its pid; if the process dies or never serves, it cleans up and quotes the tail of `daemon.log`. |
|
|
1188
1221
|
| `stop` | Prefer `systemctl stop omp-conductor.service` when that unit's MainPID is the live daemon — systemd then owns the stop and will not schedule a restart for the exit it just requested. Otherwise `SIGTERM`, then `SIGKILL` after a 10-second grace period. Prints `not running` when there is nothing to stop, and tags the confirmation with `(via systemctl)` when the unit path was used. |
|
|
1189
1222
|
| `restart` | Prefer `systemctl restart` when the unit owns the live pid so the replacement stays supervised; otherwise `stop` then `start`, inheriting the running daemon's port and project unless a flag overrides them. The new process **salvages dirty live worktrees before orphaning** those rows — see [Deploying a new package onto a busy fleet](#deploying-a-new-package-onto-a-busy-fleet). |
|
|
1190
|
-
| `status [--project NAME]` | Layered fleet report first: `dispatch` / `ticks` / `pane` / `recovery` / `herdr` / `daemon`, then the project body
|
|
1223
|
+
| `status [--project NAME]` | Layered fleet report first: `dispatch` / `ticks` / next scheduled tick / `pane` / `recovery` / `herdr` / `telegram` / `daemon`, then the project body. The next time comes from the live heartbeat process, not a guess from log timestamps. Telegram health uses `getMe` to prove API authentication without sending a message and reports inbound bridge configuration separately. The daemon block includes `rss` from `/healthz`; live workers add a busy-deploy warning. A `.conductor-stalled` marker adds an `orchestrator STALLED since …` line. |
|
|
1191
1224
|
| `hold [--project NAME]` | Soft stop: pause claiming **and** disarm ticks. Daemon and pane stay up. Prefer this over `pause` when the intent is "stop the conductor" without killing processes. See [Stop the conductor](#stop-the-conductor-hold--halt). |
|
|
1192
1225
|
| `halt [--pane] [--project NAME]` | `hold`, then stop the dispatch daemon (systemctl-aware). Pane stays up unless `--pane` is passed. `halt --pane` also pins herdr-conductor recovery off for the conductor agent only — it does **not** stop `herdr-fleet.service` or any other herdr session. Fail-closed: exits nonzero unless the agent is proven gone. |
|
|
1193
1226
|
| `arm [--project NAME]` | Proof-gated: send a Telegram challenge and write the arm marker only after your reply appears as a user turn in the orchestrator transcript. Never auto-armed by `resume` / `hold`. |
|
|
@@ -1201,6 +1234,7 @@ omp-conductor help
|
|
|
1201
1234
|
| `--project NAME` | Pick the project to service. One daemon process serves exactly one project; with several configured projects the name is required. |
|
|
1202
1235
|
| `pause` | Stop claiming new work only. The running daemon notices on its next tick; runs already in flight finish. The orchestrator heartbeat keeps ticking if armed — its gate is the arm marker, not this flag. Prefer `hold` to silence both. |
|
|
1203
1236
|
| `resume` | Clear pause only — does **not** re-arm. Run `arm` after an inbound Telegram proof to resume ticks. |
|
|
1237
|
+
| `--version`, `-V`, `version` | Print the installed `omp-conductor` package version and exit `0`. Works from the global binary and npm/plugin install because it reads the package metadata beside the shipped CLI. |
|
|
1204
1238
|
| `graph-setup` | Print how to set up the code-graph indexes workers query instead of grepping: a `git clone` for every index-only clone that does not exist yet, the one-shot index command per repo, and a `cbm-reindex.service` + `cbm-reindex.timer` pair generated from the project's own repos and branches. Reads only, so it is safe on a host where you are not root. Exits `1` when no repo in the project has [`graphProject`](#configuration) set, because the fix is a wizard answer rather than a flag. See [Code-graph discovery](#code-graph-discovery). |
|
|
1205
1239
|
| `--write` | Only for `graph-setup`. Writes the refresh script into the state directory and the two units into `/etc/systemd/system`, then prints the exact `systemctl daemon-reload && systemctl enable --now cbm-reindex.timer` to run. It never runs `systemctl` itself and never enables anything: that needs root, and a package that enables system timers behind your back is one you cannot audit by reading its output. |
|
|
1206
1240
|
| `brief-upgrade` | Inspect the package-floor + `POLICY.md` overlay. Reports by default; see [Keeping a brief current](#keeping-a-brief-current). |
|
|
@@ -1368,8 +1402,9 @@ Known and deliberate in this version:
|
|
|
1368
1402
|
`slots = maxConcurrentWorkers - live workers`, admits at most that many issues
|
|
1369
1403
|
per tick, and dispatches them together. To see them, read `omp-conductor
|
|
1370
1404
|
status`, which lists every occupied issue, or follow `daemon.log`.
|
|
1371
|
-
- **
|
|
1372
|
-
|
|
1405
|
+
- **Workers stop at green PRs.** They never merge, release or deploy. Those
|
|
1406
|
+
actions default to a human, but setup may grant either to the orchestrator;
|
|
1407
|
+
`authority` never grants them to a worker or the dispatch daemon.
|
|
1373
1408
|
- **Worker confinement is partial.** Structured `write` / `edit` / `read` /
|
|
1374
1409
|
`grep` / `glob` calls are blocked outside the worktree by an inline harness
|
|
1375
1410
|
extension (`confineToCwd`). `bash` is not: a shell one-liner can still leave
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "omp-conductor",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.17",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"description": "A 24/7 dispatcher that takes ready GitHub issues to green, mergeable PRs using omp coding sessions, with tiered escalation first to an orchestrator session and then to a human.",
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
]
|
|
18
18
|
},
|
|
19
19
|
"bin": {
|
|
20
|
-
"omp-conductor": "
|
|
20
|
+
"omp-conductor": "src/cli.ts"
|
|
21
21
|
},
|
|
22
22
|
"files": [
|
|
23
23
|
"src",
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: conductor-update
|
|
3
|
+
description: Update an installed conductor fleet through one operator request. Use when the user asks to update, upgrade, refresh, reinstall, or deploy conductor. Pins one npm release across the Bun-global CLI, omp plugin, and Herdr recovery plugin; converts a local Herdr link when needed; pauses claims, reloads processes, verifies the fleet, and restores the operator's prior pause state.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Update a conductor fleet
|
|
7
|
+
|
|
8
|
+
The operator interface is one request: **“update conductor.”** Do not hand them
|
|
9
|
+
the implementation as a checklist unless execution is blocked.
|
|
10
|
+
|
|
11
|
+
One release has three installed surfaces:
|
|
12
|
+
|
|
13
|
+
1. Bun-global `omp-conductor`: CLI and systemd daemon source.
|
|
14
|
+
2. omp npm plugin `omp-conductor`: slash command and heartbeat loaded by sessions.
|
|
15
|
+
3. Herdr plugin `herdr-conductor`: exact-pane recovery, pinned to the npm
|
|
16
|
+
release's `gitHead`.
|
|
17
|
+
|
|
18
|
+
This skill performs the whole swap. It does not publish npm, merge, tag, edit an
|
|
19
|
+
install root, or update unrelated Bun packages.
|
|
20
|
+
|
|
21
|
+
## Safety boundary
|
|
22
|
+
|
|
23
|
+
Run from an operator shell or maintenance omp session that is not hosted by the
|
|
24
|
+
target `herdr-fleet.service`. If restarting that unit would kill this updater,
|
|
25
|
+
move to an external session first. Never print npm, Telegram, or bot credentials.
|
|
26
|
+
|
|
27
|
+
The normal update is:
|
|
28
|
+
|
|
29
|
+
```text
|
|
30
|
+
inspect → pause claims → drain → pinned installs → reload → verify → restore pause state
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Ticks stay in their existing armed or disarmed state. Do not use `hold`,
|
|
34
|
+
`halt --pane`, `disarm`, `arm`, recovery pins, `pkill`, or manual install-root
|
|
35
|
+
edits for a healthy update.
|
|
36
|
+
|
|
37
|
+
## 1. Resolve one release and inspect every surface
|
|
38
|
+
|
|
39
|
+
Use the registry latest unless the operator names another published version:
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
version=$(npm view omp-conductor version)
|
|
43
|
+
gitHead=$(npm view "omp-conductor@$version" gitHead)
|
|
44
|
+
npm view "omp-conductor@$version" version gitHead --json
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Require `$version` to be nonempty and `$gitHead` to be a full commit SHA before
|
|
48
|
+
changing state.
|
|
49
|
+
|
|
50
|
+
Inspect, without mutating:
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
omp-conductor --version
|
|
54
|
+
omp plugin list --json
|
|
55
|
+
herdr --session "$session" plugin list
|
|
56
|
+
omp-conductor status [--project NAME]
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
Honor the installation's existing Herdr session and `HERDR_CONFIG_PATH`; discover
|
|
60
|
+
them from the running unit/config rather than assuming `fleet` or a path.
|
|
61
|
+
|
|
62
|
+
Record:
|
|
63
|
+
|
|
64
|
+
- Bun-global CLI version;
|
|
65
|
+
- omp plugin version;
|
|
66
|
+
- Herdr plugin source: GitHub revision, `local:<path>`, or missing;
|
|
67
|
+
- initial dispatch state: running, paused, or stopped;
|
|
68
|
+
- initial ticks state, which this update must not change;
|
|
69
|
+
- active runs and layered health.
|
|
70
|
+
|
|
71
|
+
Report “already current” only when **all** of these are true:
|
|
72
|
+
|
|
73
|
+
- Bun-global CLI version equals `$version`;
|
|
74
|
+
- omp plugin version equals `$version`;
|
|
75
|
+
- Herdr source revision equals `$gitHead`;
|
|
76
|
+
- layered status is healthy for the fleet's configured topology.
|
|
77
|
+
|
|
78
|
+
A current npm version with a linked, missing, or stale Herdr plugin is a partial
|
|
79
|
+
update, not a no-op. If all versions match but status is unhealthy, diagnose the
|
|
80
|
+
reported layer; never call an unhealthy fleet current.
|
|
81
|
+
|
|
82
|
+
## 2. Pause claims and drain
|
|
83
|
+
|
|
84
|
+
If dispatch was running:
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
omp-conductor pause [--project NAME]
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
If it was already paused, preserve that state. If it was stopped, do not start it
|
|
91
|
+
later merely because packages were updated.
|
|
92
|
+
|
|
93
|
+
Wait for `active runs (none)`. Never kill workers for an update. If a run does not
|
|
94
|
+
drain, stop and report it; leave any pause this skill added in place.
|
|
95
|
+
|
|
96
|
+
## 3. Replace all three surfaces with the pinned release
|
|
97
|
+
|
|
98
|
+
Use exact versions. A broad `bun update` is prohibited.
|
|
99
|
+
|
|
100
|
+
```bash
|
|
101
|
+
bun add -g "omp-conductor@$version"
|
|
102
|
+
omp plugin install "omp-conductor@$version"
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
For Herdr, inspect the source found in step 1:
|
|
106
|
+
|
|
107
|
+
- `local:<path>`: unlink the plugin id first;
|
|
108
|
+
- GitHub-managed or missing: do not unlink.
|
|
109
|
+
|
|
110
|
+
Then install the exact npm release commit. Herdr requires the source argument
|
|
111
|
+
before its options:
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
herdr plugin unlink herdr-conductor # only when step 1 reported local:<path>
|
|
115
|
+
herdr plugin install TerrifiedBug/conductor/herdr --ref "$gitHead" --yes
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
If any install fails, do not reload processes and do not restore dispatch.
|
|
119
|
+
Leave the fleet paused, report the failed surface, and give the one retry command.
|
|
120
|
+
The still-running processes keep their already-loaded code until a successful
|
|
121
|
+
reload.
|
|
122
|
+
|
|
123
|
+
## 4. Reload the installed code
|
|
124
|
+
|
|
125
|
+
When Herdr is systemd-managed, restart its fleet unit. This reloads the managed
|
|
126
|
+
recovery plugin and causes exact-identity recovery to resume the orchestrator pane
|
|
127
|
+
with the new omp extension:
|
|
128
|
+
|
|
129
|
+
```bash
|
|
130
|
+
systemctl restart herdr-fleet.service
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
Wait until Herdr reports its unit active and the exact configured agent live.
|
|
134
|
+
|
|
135
|
+
If the dispatch daemon was running or paused initially, restart it so its process
|
|
136
|
+
and source-integrity baseline use the new Bun-global package:
|
|
137
|
+
|
|
138
|
+
```bash
|
|
139
|
+
omp-conductor restart [--project NAME]
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
Do not substitute `start` when the daemon was initially stopped.
|
|
143
|
+
|
|
144
|
+
## 5. Verify before restoring claims
|
|
145
|
+
|
|
146
|
+
Check every surface again:
|
|
147
|
+
|
|
148
|
+
```bash
|
|
149
|
+
omp-conductor --version
|
|
150
|
+
omp plugin list --json
|
|
151
|
+
herdr --session "$session" plugin list
|
|
152
|
+
omp-conductor status [--project NAME]
|
|
153
|
+
systemctl is-active herdr-fleet.service
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
Require:
|
|
157
|
+
|
|
158
|
+
- global CLI and omp plugin both equal `$version`;
|
|
159
|
+
- Herdr source equals `TerrifiedBug/conductor` at `$gitHead`;
|
|
160
|
+
- exact pane is live and recovery is clear;
|
|
161
|
+
- Herdr and daemon health are OK when managed;
|
|
162
|
+
- ticks equal their initial state;
|
|
163
|
+
- active runs remain empty;
|
|
164
|
+
- dispatch remains paused if this skill paused it.
|
|
165
|
+
|
|
166
|
+
Run status a second time after recovery settles. Any failed check leaves dispatch
|
|
167
|
+
paused and is reported as a partial update.
|
|
168
|
+
|
|
169
|
+
Only if dispatch was initially running and every check passed:
|
|
170
|
+
|
|
171
|
+
```bash
|
|
172
|
+
omp-conductor resume [--project NAME]
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
Verify `dispatch running` once more. If dispatch was initially paused or stopped,
|
|
176
|
+
preserve that state and report it.
|
|
177
|
+
|
|
178
|
+
Finish with one compact result: old → new version, all three installed surfaces,
|
|
179
|
+
final layered status, and whether the original dispatch state was restored.
|
package/src/cli.ts
CHANGED
|
@@ -31,6 +31,7 @@ import {
|
|
|
31
31
|
hold,
|
|
32
32
|
releaseHold,
|
|
33
33
|
renderStatus,
|
|
34
|
+
startHerdrFleet,
|
|
34
35
|
} from "./fleet.ts";
|
|
35
36
|
import { formatGraphSetup, graphRepos, writeGraphSetup, type GraphSetupWrite } from "./graph.ts";
|
|
36
37
|
import {
|
|
@@ -55,10 +56,21 @@ import { makeTracker } from "./tracker/github.ts";
|
|
|
55
56
|
import type { ProjectConfig } from "./types.ts";
|
|
56
57
|
import { formatUnblock, unblockIssue } from "./unblock.ts";
|
|
57
58
|
|
|
59
|
+
function packageVersion(): string {
|
|
60
|
+
const parsed = JSON.parse(readFileSync(join(import.meta.dir, "..", "package.json"), "utf8")) as {
|
|
61
|
+
version?: unknown;
|
|
62
|
+
};
|
|
63
|
+
if (typeof parsed.version !== "string" || parsed.version.length === 0) {
|
|
64
|
+
throw new Error("installed package.json has no version");
|
|
65
|
+
}
|
|
66
|
+
return parsed.version;
|
|
67
|
+
}
|
|
68
|
+
|
|
58
69
|
const USAGE = `omp-conductor — dispatch ready issues to omp coding sessions
|
|
59
70
|
|
|
60
71
|
usage:
|
|
61
72
|
omp-conductor start [--port N] [--project NAME]
|
|
73
|
+
omp-conductor --version
|
|
62
74
|
omp-conductor stop
|
|
63
75
|
omp-conductor restart [--port N] [--project NAME]
|
|
64
76
|
omp-conductor status [--project NAME]
|
|
@@ -76,9 +88,9 @@ usage:
|
|
|
76
88
|
omp-conductor brief-upgrade [--migrate|--retrofit] [--apply] [--file PATH] [--project NAME]
|
|
77
89
|
omp-conductor help
|
|
78
90
|
|
|
79
|
-
start
|
|
80
|
-
|
|
81
|
-
|
|
91
|
+
start start the installed herdr-fleet.service when present, then run the
|
|
92
|
+
dispatch loop in the background and wait until it answers GET
|
|
93
|
+
/healthz on :8787 (override with --port). Refuses if one is running.
|
|
82
94
|
stop stop the running daemon. Uses systemctl when the omp-conductor
|
|
83
95
|
unit owns the process (so Restart=on-failure cannot bring it back);
|
|
84
96
|
otherwise SIGTERM then SIGKILL.
|
|
@@ -87,8 +99,9 @@ usage:
|
|
|
87
99
|
dirty live worktrees before orphaning those rows — see README
|
|
88
100
|
"Deploying a new package onto a busy fleet". Goes through systemctl
|
|
89
101
|
when the unit owns the live pid.
|
|
90
|
-
status layered fleet report: dispatch (running|paused|stopped), ticks
|
|
91
|
-
|
|
102
|
+
status layered fleet report: dispatch (running|paused|stopped), ticks and
|
|
103
|
+
next due time, pane, herdr, Telegram bot/API health, daemon, caps
|
|
104
|
+
and active runs.
|
|
92
105
|
hold soft stop: pause claiming AND disarm ticks. Daemon and pane stay up.
|
|
93
106
|
This is "stop the conductor overnight" without killing processes.
|
|
94
107
|
halt hold, then stop the dispatch daemon (systemctl-aware). Pane stays up
|
|
@@ -133,6 +146,8 @@ usage:
|
|
|
133
146
|
--apply still merges a bannered single-file brief. --file checks a
|
|
134
147
|
brief that is not where the wizard would have put it.
|
|
135
148
|
help print this text (also --help, -h).
|
|
149
|
+
--version
|
|
150
|
+
print the installed omp-conductor package version (also -V, version).
|
|
136
151
|
|
|
137
152
|
Pause is a flag file under the state directory, so it applies to every project
|
|
138
153
|
and survives a daemon restart. Hold also removes the arm marker the heartbeat
|
|
@@ -378,6 +393,11 @@ const cmd = argv[0];
|
|
|
378
393
|
|
|
379
394
|
try {
|
|
380
395
|
switch (cmd) {
|
|
396
|
+
case "--version":
|
|
397
|
+
case "-V":
|
|
398
|
+
case "version":
|
|
399
|
+
process.stdout.write(`${packageVersion()}\n`);
|
|
400
|
+
break;
|
|
381
401
|
case "daemon": {
|
|
382
402
|
// Until now only `lifecycle.startDaemon()` — the spawn path — wrote the
|
|
383
403
|
// pidfile, which left a daemon started in the foreground (which is how
|
|
@@ -429,10 +449,14 @@ try {
|
|
|
429
449
|
}
|
|
430
450
|
|
|
431
451
|
case "start": {
|
|
432
|
-
const
|
|
452
|
+
const project = flag(argv, "project");
|
|
453
|
+
const herdr = startHerdrFleet(project);
|
|
454
|
+
const rec = await startDaemon({ port: portFlag(argv), project });
|
|
433
455
|
process.stdout.write(
|
|
434
456
|
`started — pid ${rec.pid}, /healthz on :${rec.port}` +
|
|
435
|
-
`${rec.project === undefined ? "" : `, project ${rec.project}`}\
|
|
457
|
+
`${rec.project === undefined ? "" : `, project ${rec.project}`}\n` +
|
|
458
|
+
`herdr ${herdr.kind === "active" ? `active (${herdr.unit})${herdr.recoveryReleased ? "; recovery pin cleared" : ""}` : `unmanaged (${herdr.reason})`}\n` +
|
|
459
|
+
`log ${rec.logFile}\n`,
|
|
436
460
|
);
|
|
437
461
|
break;
|
|
438
462
|
}
|
package/src/fleet.ts
CHANGED
|
@@ -31,6 +31,7 @@ import { findProject, loadConfig, stateDir } from "./config.ts";
|
|
|
31
31
|
import { isPaused, setPaused, statusSnapshot, type StatusSnapshot } from "./daemon.ts";
|
|
32
32
|
import {
|
|
33
33
|
healthCheck,
|
|
34
|
+
isAlive,
|
|
34
35
|
livingDaemon,
|
|
35
36
|
stopDaemon,
|
|
36
37
|
type StopResult,
|
|
@@ -39,6 +40,7 @@ import {
|
|
|
39
40
|
import { formatRss, rssBytesFromHealthz } from "./host.ts";
|
|
40
41
|
import {
|
|
41
42
|
readTickConfig,
|
|
43
|
+
readTickRuntimeStatus,
|
|
42
44
|
TICK_CONFIG_FILE,
|
|
43
45
|
type TickConfig,
|
|
44
46
|
type TickConfigResult,
|
|
@@ -304,6 +306,66 @@ export function clearPaneHalt(projectName?: string): { path: string; wasHalted:
|
|
|
304
306
|
return { path, wasHalted };
|
|
305
307
|
}
|
|
306
308
|
|
|
309
|
+
export type HerdrStartResult =
|
|
310
|
+
| { kind: "active"; unit: string; recoveryReleased: boolean }
|
|
311
|
+
| { kind: "unmanaged"; unit: string; reason: string };
|
|
312
|
+
|
|
313
|
+
export interface HerdrStartDeps {
|
|
314
|
+
systemctl?: (args: string[]) => { ok: boolean; stdout: string; stderr: string; missing?: boolean };
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/**
|
|
318
|
+
* Starts the dedicated Herdr fleet unit when it is installed. Hosts without
|
|
319
|
+
* systemd or without that optional unit keep the standalone daemon behaviour.
|
|
320
|
+
*/
|
|
321
|
+
export function startHerdrFleet(projectName?: string, deps: HerdrStartDeps = {}): HerdrStartResult {
|
|
322
|
+
const run: NonNullable<HerdrStartDeps["systemctl"]> =
|
|
323
|
+
deps.systemctl ??
|
|
324
|
+
((args: string[]) => {
|
|
325
|
+
const res = spawnSync("systemctl", args, { encoding: "utf8", timeout: 15_000, env: process.env });
|
|
326
|
+
if (res.error) {
|
|
327
|
+
const err = res.error as NodeJS.ErrnoException;
|
|
328
|
+
return {
|
|
329
|
+
ok: false,
|
|
330
|
+
stdout: "",
|
|
331
|
+
stderr: err.message,
|
|
332
|
+
...(err.code === "ENOENT" ? { missing: true } : {}),
|
|
333
|
+
};
|
|
334
|
+
}
|
|
335
|
+
return { ok: res.status === 0, stdout: res.stdout ?? "", stderr: res.stderr ?? "" };
|
|
336
|
+
});
|
|
337
|
+
|
|
338
|
+
const shown = run(["show", DEFAULT_HERDR_UNIT, "--property=LoadState", "--value"]);
|
|
339
|
+
if (shown.missing) return { kind: "unmanaged", unit: DEFAULT_HERDR_UNIT, reason: "no systemctl" };
|
|
340
|
+
if (!shown.ok) {
|
|
341
|
+
const detail = (shown.stderr.trim() || shown.stdout.trim() || "systemctl show failed").split("\n")[0]!;
|
|
342
|
+
throw new Error(`cannot inspect ${DEFAULT_HERDR_UNIT}: ${detail}`);
|
|
343
|
+
}
|
|
344
|
+
const loadState = shown.stdout.trim();
|
|
345
|
+
if (loadState === "not-found" || loadState === "") {
|
|
346
|
+
return { kind: "unmanaged", unit: DEFAULT_HERDR_UNIT, reason: "unit not installed" };
|
|
347
|
+
}
|
|
348
|
+
if (loadState !== "loaded") {
|
|
349
|
+
throw new Error(`${DEFAULT_HERDR_UNIT} is ${loadState}, not startable`);
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
const halt = resolvePaneHaltPath(projectName);
|
|
353
|
+
const recoveryReleased = halt.kind === "ok" && existsSync(halt.path);
|
|
354
|
+
if (halt.kind === "ok") rmSync(halt.path, { force: true });
|
|
355
|
+
|
|
356
|
+
const started = run(["start", DEFAULT_HERDR_UNIT]);
|
|
357
|
+
if (!started.ok) {
|
|
358
|
+
const detail = (started.stderr.trim() || started.stdout.trim() || "systemctl start failed").split("\n")[0]!;
|
|
359
|
+
throw new Error(`systemctl start ${DEFAULT_HERDR_UNIT} failed: ${detail}`);
|
|
360
|
+
}
|
|
361
|
+
const active = run(["is-active", DEFAULT_HERDR_UNIT]);
|
|
362
|
+
if (!active.ok || active.stdout.trim() !== "active") {
|
|
363
|
+
const detail = (active.stderr.trim() || active.stdout.trim() || "not active").split("\n")[0]!;
|
|
364
|
+
throw new Error(`${DEFAULT_HERDR_UNIT} did not become active: ${detail}`);
|
|
365
|
+
}
|
|
366
|
+
return { kind: "active", unit: DEFAULT_HERDR_UNIT, recoveryReleased };
|
|
367
|
+
}
|
|
368
|
+
|
|
307
369
|
export interface HerdrAgent {
|
|
308
370
|
name: string;
|
|
309
371
|
paneId: string;
|
|
@@ -565,11 +627,13 @@ async function herdrAgentList(deps: PaneStopDeps): Promise<HerdrAgent[]> {
|
|
|
565
627
|
if (res.status !== 0) {
|
|
566
628
|
throw new Error((res.stderr ?? res.stdout ?? `herdr exit ${String(res.status)}`).trim());
|
|
567
629
|
}
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
630
|
+
return parseHerdrAgentList(res.stdout ?? "");
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
export function parseHerdrAgentList(rawOutput: string): HerdrAgent[] {
|
|
634
|
+
// Only an explicit `agents: []` means no agents. Every malformed answer is
|
|
635
|
+
// uncertainty, never permission to report the conductor pane missing.
|
|
636
|
+
const raw = rawOutput.trim();
|
|
573
637
|
if (raw.length === 0) {
|
|
574
638
|
throw new Error("herdr agent list printed nothing — cannot tell whether the pane is running");
|
|
575
639
|
}
|
|
@@ -590,38 +654,35 @@ async function herdrAgentList(deps: PaneStopDeps): Promise<HerdrAgent[]> {
|
|
|
590
654
|
if (row === null || typeof row !== "object") {
|
|
591
655
|
throw new Error("herdr agent list contains a non-object agent row — unrecognized schema");
|
|
592
656
|
}
|
|
593
|
-
const
|
|
594
|
-
const name =
|
|
595
|
-
const paneId =
|
|
657
|
+
const agent = row as { readonly [key: string]: unknown };
|
|
658
|
+
const name = agent["name"];
|
|
659
|
+
const paneId = agent["pane_id"];
|
|
596
660
|
if (typeof name !== "string" || typeof paneId !== "string") {
|
|
597
661
|
throw new Error(
|
|
598
662
|
"herdr agent list row is missing a string `name`/`pane_id` — " +
|
|
599
663
|
"cannot tell whether it is the conductor pane",
|
|
600
664
|
);
|
|
601
665
|
}
|
|
602
|
-
|
|
603
|
-
// agent has exited — recover.sh reads the same field as `(.agent // "")`,
|
|
604
|
-
// and the released-agent fixture omits the key entirely. A *present* value
|
|
605
|
-
// of any other type is a row we cannot read: coercing it to `undefined`
|
|
606
|
-
// would report a live pane as already-gone.
|
|
607
|
-
const rawAgent = a["agent"];
|
|
666
|
+
const rawAgent = agent["agent"];
|
|
608
667
|
if (rawAgent !== undefined && rawAgent !== null && typeof rawAgent !== "string") {
|
|
609
668
|
throw new Error(
|
|
610
669
|
`herdr agent list row for ${name} has a non-string \`agent\` (${typeof rawAgent}) — ` +
|
|
611
670
|
`cannot tell whether an agent is live`,
|
|
612
671
|
);
|
|
613
672
|
}
|
|
614
|
-
const
|
|
673
|
+
const liveAgent = typeof rawAgent === "string" ? rawAgent : undefined;
|
|
615
674
|
let sessionPath: string | undefined;
|
|
616
|
-
const
|
|
617
|
-
if (
|
|
618
|
-
const
|
|
619
|
-
if (
|
|
675
|
+
const session = agent["agent_session"];
|
|
676
|
+
if (session !== null && typeof session === "object") {
|
|
677
|
+
const value = session as { readonly [key: string]: unknown };
|
|
678
|
+
if (value["source"] === "herdr:omp" && typeof value["value"] === "string") {
|
|
679
|
+
sessionPath = value["value"];
|
|
680
|
+
}
|
|
620
681
|
}
|
|
621
682
|
out.push({
|
|
622
683
|
name,
|
|
623
684
|
paneId,
|
|
624
|
-
...(
|
|
685
|
+
...(liveAgent === undefined ? {} : { agent: liveAgent }),
|
|
625
686
|
...(sessionPath === undefined ? {} : { sessionPath }),
|
|
626
687
|
});
|
|
627
688
|
}
|
|
@@ -709,11 +770,18 @@ export type PaneLayer = "live" | "missing" | "unknown";
|
|
|
709
770
|
/** `unpinnable`: no tick config, so FLEET_CWD — the only path recovery reads — is unknown. */
|
|
710
771
|
export type RecoveryLayer = "pinned" | "clear" | "unpinnable";
|
|
711
772
|
export type HerdrLayer = "active" | "inactive" | "unknown";
|
|
773
|
+
export type TelegramLayer = "ok" | "degraded" | "down" | "unconfigured" | "unprobed";
|
|
774
|
+
|
|
775
|
+
export interface TelegramHealth {
|
|
776
|
+
kind: TelegramLayer;
|
|
777
|
+
detail?: string;
|
|
778
|
+
}
|
|
712
779
|
|
|
713
780
|
export interface FleetLayers {
|
|
714
781
|
dispatch: DispatchLayer;
|
|
715
782
|
ticks: TicksLayer;
|
|
716
783
|
ticksDetail?: string;
|
|
784
|
+
nextTickAt?: string;
|
|
717
785
|
pane: PaneLayer;
|
|
718
786
|
paneDetail?: string;
|
|
719
787
|
recovery: RecoveryLayer;
|
|
@@ -735,6 +803,7 @@ export function fleetLayers(projectName?: string): FleetLayers {
|
|
|
735
803
|
const tick = resolveTickConfig(projectName);
|
|
736
804
|
let ticks: TicksLayer;
|
|
737
805
|
let ticksDetail: string | undefined;
|
|
806
|
+
let nextTickAt: string | undefined;
|
|
738
807
|
let armedPath: string | undefined;
|
|
739
808
|
let tickConfigPath: string | undefined;
|
|
740
809
|
if (tick.kind === "absent") {
|
|
@@ -756,6 +825,8 @@ export function fleetLayers(projectName?: string): FleetLayers {
|
|
|
756
825
|
ticks = "disarmed";
|
|
757
826
|
ticksDetail = tick.config.armedFile;
|
|
758
827
|
}
|
|
828
|
+
const runtime = readTickRuntimeStatus(tick.cwd);
|
|
829
|
+
if (runtime !== undefined && isAlive(runtime.pid)) nextTickAt = runtime.nextTickAt;
|
|
759
830
|
}
|
|
760
831
|
if (armedPath === undefined) armedPath = armedMarkerPath(projectName);
|
|
761
832
|
|
|
@@ -764,7 +835,7 @@ export function fleetLayers(projectName?: string): FleetLayers {
|
|
|
764
835
|
const resolvedHalt = resolvePaneHaltPath(projectName);
|
|
765
836
|
const haltPath = resolvedHalt.kind === "ok" ? resolvedHalt.path : undefined;
|
|
766
837
|
const recoveryPinned = haltPath !== undefined && existsSync(haltPath);
|
|
767
|
-
const omp = probeOmpPane();
|
|
838
|
+
const omp = probeOmpPane(projectName);
|
|
768
839
|
let pane: PaneLayer;
|
|
769
840
|
let paneDetail: string | undefined;
|
|
770
841
|
if (omp.kind === "live") {
|
|
@@ -783,6 +854,7 @@ export function fleetLayers(projectName?: string): FleetLayers {
|
|
|
783
854
|
dispatch,
|
|
784
855
|
ticks,
|
|
785
856
|
...(ticksDetail === undefined ? {} : { ticksDetail }),
|
|
857
|
+
...(nextTickAt === undefined ? {} : { nextTickAt }),
|
|
786
858
|
pane,
|
|
787
859
|
...(paneDetail === undefined ? {} : { paneDetail }),
|
|
788
860
|
recovery: haltPath === undefined ? "unpinnable" : recoveryPinned ? "pinned" : "clear",
|
|
@@ -805,11 +877,21 @@ export function formatFleetStatus(
|
|
|
805
877
|
s: StatusSnapshot,
|
|
806
878
|
layers: FleetLayers,
|
|
807
879
|
daemonHealth?: { ok: boolean; body?: string },
|
|
880
|
+
telegram: TelegramHealth = { kind: "unprobed" },
|
|
881
|
+
now = Date.now(),
|
|
808
882
|
): string {
|
|
809
883
|
const tickLine =
|
|
810
884
|
layers.ticksDetail === undefined
|
|
811
885
|
? `ticks ${layers.ticks}`
|
|
812
886
|
: `ticks ${layers.ticks} (${layers.ticksDetail})`;
|
|
887
|
+
let nextTickLine: string | undefined;
|
|
888
|
+
if (layers.nextTickAt !== undefined) {
|
|
889
|
+
const delta = Date.parse(layers.nextTickAt) - now;
|
|
890
|
+
const minutes = Math.max(1, Math.ceil(Math.abs(delta) / 60_000));
|
|
891
|
+
nextTickLine =
|
|
892
|
+
`next tick ${layers.nextTickAt} ` +
|
|
893
|
+
`(${delta >= 0 ? `in ${minutes}m` : `overdue by ${minutes}m`})`;
|
|
894
|
+
}
|
|
813
895
|
const paneLine =
|
|
814
896
|
layers.paneDetail === undefined
|
|
815
897
|
? `pane ${layers.pane}`
|
|
@@ -822,6 +904,10 @@ export function formatFleetStatus(
|
|
|
822
904
|
layers.herdrDetail === undefined
|
|
823
905
|
? `herdr ${layers.herdr}`
|
|
824
906
|
: `herdr ${layers.herdr} (${layers.herdrDetail})`;
|
|
907
|
+
const telegramLine =
|
|
908
|
+
telegram.detail === undefined
|
|
909
|
+
? `telegram ${telegram.kind}`
|
|
910
|
+
: `telegram ${telegram.kind} (${telegram.detail})`;
|
|
825
911
|
|
|
826
912
|
let daemonBlock: string;
|
|
827
913
|
if (!layers.daemon.running || layers.daemon.pid === undefined) {
|
|
@@ -847,9 +933,11 @@ export function formatFleetStatus(
|
|
|
847
933
|
return [
|
|
848
934
|
`dispatch ${layers.dispatch}`,
|
|
849
935
|
tickLine,
|
|
936
|
+
...(nextTickLine === undefined ? [] : [nextTickLine]),
|
|
850
937
|
paneLine,
|
|
851
938
|
recoveryLine,
|
|
852
939
|
herdrLine,
|
|
940
|
+
telegramLine,
|
|
853
941
|
daemonBlock,
|
|
854
942
|
"",
|
|
855
943
|
formatProjectBody(s),
|
|
@@ -898,10 +986,12 @@ function formatProjectBody(s: StatusSnapshot): string {
|
|
|
898
986
|
export async function renderStatus(projectName?: string): Promise<string> {
|
|
899
987
|
const s = statusSnapshot(projectName);
|
|
900
988
|
const layers = fleetLayers(projectName);
|
|
901
|
-
let health: { ok: boolean; body?: string } | undefined;
|
|
902
989
|
const rec = livingDaemon();
|
|
903
|
-
|
|
904
|
-
|
|
990
|
+
const [health, telegram] = await Promise.all([
|
|
991
|
+
rec === undefined ? undefined : healthCheck(rec.port),
|
|
992
|
+
probeTelegramHealth(projectName),
|
|
993
|
+
]);
|
|
994
|
+
return formatFleetStatus(s, layers, health, telegram);
|
|
905
995
|
}
|
|
906
996
|
|
|
907
997
|
// ---------------------------------------------------------------------------
|
|
@@ -949,6 +1039,57 @@ function readBotToken(): string | undefined {
|
|
|
949
1039
|
return undefined;
|
|
950
1040
|
}
|
|
951
1041
|
|
|
1042
|
+
export async function probeTelegramHealth(
|
|
1043
|
+
projectName?: string,
|
|
1044
|
+
request: (input: string, init?: RequestInit) => Promise<Response> = fetch,
|
|
1045
|
+
): Promise<TelegramHealth> {
|
|
1046
|
+
const tick = resolveTickConfig(projectName);
|
|
1047
|
+
const accessPath =
|
|
1048
|
+
tick.kind === "ok" && tick.config.accessFile !== undefined
|
|
1049
|
+
? tick.config.accessFile
|
|
1050
|
+
: join(telegramStateDir(), "access.json");
|
|
1051
|
+
const channel = readPairedChannel(accessPath);
|
|
1052
|
+
const token = readBotToken();
|
|
1053
|
+
if (token === undefined) {
|
|
1054
|
+
return {
|
|
1055
|
+
kind: "unconfigured",
|
|
1056
|
+
detail: `no TELEGRAM_BOT_TOKEN; inbound ${channel.kind === "up" ? "configured" : channel.reason}`,
|
|
1057
|
+
};
|
|
1058
|
+
}
|
|
1059
|
+
|
|
1060
|
+
let response: Response;
|
|
1061
|
+
let body: unknown;
|
|
1062
|
+
try {
|
|
1063
|
+
response = await request(`https://api.telegram.org/bot${token}/getMe`, {
|
|
1064
|
+
signal: AbortSignal.timeout(5_000),
|
|
1065
|
+
});
|
|
1066
|
+
body = await response.json();
|
|
1067
|
+
} catch {
|
|
1068
|
+
return {
|
|
1069
|
+
kind: "down",
|
|
1070
|
+
detail: `Telegram API request failed; inbound ${channel.kind === "up" ? "configured" : channel.reason}`,
|
|
1071
|
+
};
|
|
1072
|
+
}
|
|
1073
|
+
|
|
1074
|
+
if (body === null || typeof body !== "object" || Array.isArray(body)) {
|
|
1075
|
+
return { kind: "down", detail: "Telegram API returned an invalid response" };
|
|
1076
|
+
}
|
|
1077
|
+
const result = body as Record<string, unknown>;
|
|
1078
|
+
if (!response.ok || result["ok"] !== true) {
|
|
1079
|
+
const description =
|
|
1080
|
+
typeof result["description"] === "string" ? result["description"] : `HTTP ${response.status}`;
|
|
1081
|
+
return { kind: "down", detail: `${description}; inbound ${channel.kind === "up" ? "configured" : channel.reason}` };
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
const user =
|
|
1085
|
+
result["result"] !== null && typeof result["result"] === "object" && !Array.isArray(result["result"])
|
|
1086
|
+
? (result["result"] as Record<string, unknown>)
|
|
1087
|
+
: undefined;
|
|
1088
|
+
const username = typeof user?.["username"] === "string" ? `@${user["username"]}` : "authenticated";
|
|
1089
|
+
if (channel.kind === "down") return { kind: "degraded", detail: `${username}; inbound ${channel.reason}` };
|
|
1090
|
+
return { kind: "ok", detail: `${username}; inbound configured` };
|
|
1091
|
+
}
|
|
1092
|
+
|
|
952
1093
|
export function sessionDirForCwd(cwd: string): string {
|
|
953
1094
|
const home = homedir();
|
|
954
1095
|
const slug = cwd.startsWith(home) ? cwd.slice(home.length) : cwd;
|
|
@@ -1067,33 +1208,63 @@ function probeHerdrUnit(unit = DEFAULT_HERDR_UNIT): { kind: HerdrLayer; detail?:
|
|
|
1067
1208
|
}
|
|
1068
1209
|
}
|
|
1069
1210
|
|
|
1070
|
-
function
|
|
1211
|
+
export function paneLayerFromAgents(
|
|
1212
|
+
agents: HerdrAgent[],
|
|
1213
|
+
agentName: string,
|
|
1214
|
+
):
|
|
1215
|
+
| { kind: "live"; summary: string }
|
|
1216
|
+
| { kind: "missing" }
|
|
1217
|
+
| { kind: "unknown"; reason: string } {
|
|
1218
|
+
const matches = agents.filter((agent) => agent.name === agentName);
|
|
1219
|
+
if (matches.length === 0) return { kind: "missing" };
|
|
1220
|
+
if (matches.length > 1) {
|
|
1221
|
+
return { kind: "unknown", reason: `${matches.length} herdr agents named ${agentName}` };
|
|
1222
|
+
}
|
|
1223
|
+
const match = matches[0]!;
|
|
1224
|
+
if (match.agent === "omp") {
|
|
1225
|
+
return { kind: "live", summary: `agent ${agentName} pane ${match.paneId}` };
|
|
1226
|
+
}
|
|
1227
|
+
if (match.agent === undefined || match.agent === "") return { kind: "missing" };
|
|
1228
|
+
return {
|
|
1229
|
+
kind: "unknown",
|
|
1230
|
+
reason: `agent ${agentName} pane ${match.paneId} is ${match.agent}, not omp`,
|
|
1231
|
+
};
|
|
1232
|
+
}
|
|
1233
|
+
|
|
1234
|
+
function probeOmpPane(
|
|
1235
|
+
projectName?: string,
|
|
1236
|
+
):
|
|
1071
1237
|
| { kind: "live"; summary: string }
|
|
1072
1238
|
| { kind: "missing" }
|
|
1073
1239
|
| { kind: "unknown"; reason: string } {
|
|
1240
|
+
const tick = resolveTickConfig(projectName);
|
|
1241
|
+
if (tick.kind === "invalid") {
|
|
1242
|
+
return {
|
|
1243
|
+
kind: "unknown",
|
|
1244
|
+
reason:
|
|
1245
|
+
`tick config invalid at ${tick.path} (${tick.problem}) — ` +
|
|
1246
|
+
"the conductor agent name cannot be read",
|
|
1247
|
+
};
|
|
1248
|
+
}
|
|
1249
|
+
const agentName =
|
|
1250
|
+
tick.kind === "ok" ? (tick.config.agentName ?? DEFAULT_FLEET_AGENT_NAME) : DEFAULT_FLEET_AGENT_NAME;
|
|
1251
|
+
const session = process.env["HERDR_SESSION"] ?? "fleet";
|
|
1074
1252
|
try {
|
|
1075
|
-
const res = spawnSync("
|
|
1253
|
+
const res = spawnSync("herdr", ["--session", session, "agent", "list"], {
|
|
1076
1254
|
encoding: "utf8",
|
|
1077
|
-
timeout:
|
|
1255
|
+
timeout: 8_000,
|
|
1078
1256
|
env: process.env,
|
|
1079
1257
|
});
|
|
1080
1258
|
if (res.error) {
|
|
1081
1259
|
const err = res.error as NodeJS.ErrnoException;
|
|
1082
|
-
if (err.code === "ENOENT") return { kind: "unknown", reason: "no
|
|
1260
|
+
if (err.code === "ENOENT") return { kind: "unknown", reason: "no herdr" };
|
|
1083
1261
|
return { kind: "unknown", reason: err.message };
|
|
1084
1262
|
}
|
|
1085
|
-
|
|
1086
|
-
.
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
if (lines.length === 0) return { kind: "missing" };
|
|
1091
|
-
const first = lines[0]!;
|
|
1092
|
-
const summary = first.length > 100 ? `${first.slice(0, 97)}…` : first;
|
|
1093
|
-
return {
|
|
1094
|
-
kind: "live",
|
|
1095
|
-
summary: lines.length === 1 ? summary : `${summary} (+${lines.length - 1} more)`,
|
|
1096
|
-
};
|
|
1263
|
+
if (res.status !== 0) {
|
|
1264
|
+
const detail = (res.stderr ?? res.stdout ?? `herdr exit ${String(res.status)}`).trim();
|
|
1265
|
+
return { kind: "unknown", reason: detail };
|
|
1266
|
+
}
|
|
1267
|
+
return paneLayerFromAgents(parseHerdrAgentList(res.stdout ?? ""), agentName);
|
|
1097
1268
|
} catch (err) {
|
|
1098
1269
|
return { kind: "unknown", reason: err instanceof Error ? err.message : String(err) };
|
|
1099
1270
|
}
|
package/src/orchestrator-tick.ts
CHANGED
|
@@ -41,7 +41,7 @@
|
|
|
41
41
|
*/
|
|
42
42
|
|
|
43
43
|
import { spawnSync } from "node:child_process";
|
|
44
|
-
import { existsSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
44
|
+
import { existsSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
45
45
|
import { isAbsolute, join, resolve } from "node:path";
|
|
46
46
|
import { findProject, loadConfig } from "./config.ts";
|
|
47
47
|
import {
|
|
@@ -107,6 +107,43 @@ export const STALL_TICKS = 2;
|
|
|
107
107
|
*/
|
|
108
108
|
export const TICK_REQUESTED_FILE = ".conductor-tick-requested";
|
|
109
109
|
|
|
110
|
+
/** Runtime heartbeat schedule consumed by `omp-conductor status`. */
|
|
111
|
+
export const TICK_STATUS_FILE = ".conductor-tick-status.json";
|
|
112
|
+
|
|
113
|
+
export interface TickRuntimeStatus {
|
|
114
|
+
pid: number;
|
|
115
|
+
intervalSeconds: number;
|
|
116
|
+
nextTickAt: string;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
export function readTickRuntimeStatus(cwd: string): TickRuntimeStatus | undefined {
|
|
120
|
+
let parsed: unknown;
|
|
121
|
+
try {
|
|
122
|
+
parsed = JSON.parse(readFileSync(join(cwd, TICK_STATUS_FILE), "utf8"));
|
|
123
|
+
} catch {
|
|
124
|
+
return undefined;
|
|
125
|
+
}
|
|
126
|
+
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) return undefined;
|
|
127
|
+
const row = parsed as Record<string, unknown>;
|
|
128
|
+
if (
|
|
129
|
+
typeof row["pid"] !== "number" ||
|
|
130
|
+
!Number.isInteger(row["pid"]) ||
|
|
131
|
+
row["pid"] <= 1 ||
|
|
132
|
+
typeof row["intervalSeconds"] !== "number" ||
|
|
133
|
+
!Number.isInteger(row["intervalSeconds"]) ||
|
|
134
|
+
row["intervalSeconds"] < MIN_INTERVAL_SECONDS ||
|
|
135
|
+
typeof row["nextTickAt"] !== "string" ||
|
|
136
|
+
!Number.isFinite(Date.parse(row["nextTickAt"]))
|
|
137
|
+
) {
|
|
138
|
+
return undefined;
|
|
139
|
+
}
|
|
140
|
+
return {
|
|
141
|
+
pid: row["pid"],
|
|
142
|
+
intervalSeconds: row["intervalSeconds"],
|
|
143
|
+
nextTickAt: row["nextTickAt"],
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
|
|
110
147
|
/**
|
|
111
148
|
* The marker's one line after its ISO timestamp, and the middle of the error
|
|
112
149
|
* log. Shared so the file and the log can never describe different failures.
|
|
@@ -971,6 +1008,23 @@ function tick(pi: TickApi, ctx: TickContext, config: TickConfig, session: TickSe
|
|
|
971
1008
|
clearTickRequest(pi, ctx.cwd);
|
|
972
1009
|
}
|
|
973
1010
|
|
|
1011
|
+
function writeTickRuntimeStatus(pi: TickApi, cwd: string, intervalSeconds: number): void {
|
|
1012
|
+
const path = join(cwd, TICK_STATUS_FILE);
|
|
1013
|
+
const tmp = `${path}.${process.pid}.tmp`;
|
|
1014
|
+
const status: TickRuntimeStatus = {
|
|
1015
|
+
pid: process.pid,
|
|
1016
|
+
intervalSeconds,
|
|
1017
|
+
nextTickAt: new Date(Date.now() + intervalSeconds * 1000).toISOString(),
|
|
1018
|
+
};
|
|
1019
|
+
try {
|
|
1020
|
+
writeFileSync(tmp, `${JSON.stringify(status)}\n`);
|
|
1021
|
+
renameSync(tmp, path);
|
|
1022
|
+
} catch (err) {
|
|
1023
|
+
rmSync(tmp, { force: true });
|
|
1024
|
+
pi.logger.error(`[omp-conductor] could not write ${path}: ${err instanceof Error ? err.message : String(err)}`);
|
|
1025
|
+
}
|
|
1026
|
+
}
|
|
1027
|
+
|
|
974
1028
|
/**
|
|
975
1029
|
* Arm the interval heartbeat, then honour a recover poke if one is waiting.
|
|
976
1030
|
* Extracted so the ownership-retry path and the immediate-accept path cannot
|
|
@@ -978,7 +1032,14 @@ function tick(pi: TickApi, ctx: TickContext, config: TickConfig, session: TickSe
|
|
|
978
1032
|
* behaviour.
|
|
979
1033
|
*/
|
|
980
1034
|
function armTickHeartbeat(pi: TickApi, ctx: TickContext, config: TickConfig, session: TickSession): void {
|
|
981
|
-
|
|
1035
|
+
writeTickRuntimeStatus(pi, ctx.cwd, config.intervalSeconds);
|
|
1036
|
+
ctx.setInterval(() => {
|
|
1037
|
+
try {
|
|
1038
|
+
tick(pi, ctx, config, session);
|
|
1039
|
+
} finally {
|
|
1040
|
+
writeTickRuntimeStatus(pi, ctx.cwd, config.intervalSeconds);
|
|
1041
|
+
}
|
|
1042
|
+
}, config.intervalSeconds * 1000);
|
|
982
1043
|
if (!existsSync(join(ctx.cwd, TICK_REQUESTED_FILE))) return;
|
|
983
1044
|
pi.logger.info("[omp-conductor] tick requested by recover — firing without waiting for the interval");
|
|
984
1045
|
tick(pi, ctx, config, session);
|