omp-conductor 0.3.15 → 0.3.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +61 -31
- package/package.json +2 -2
- package/skills/conductor-update/SKILL.md +157 -0
- package/src/cli.ts +31 -7
- package/src/fleet.ts +211 -40
- package/src/orchestrator-tick.ts +63 -2
package/README.md
CHANGED
|
@@ -7,17 +7,16 @@ tiers: first to an orchestrator session that can re-brief the worker, then to yo
|
|
|
7
7
|
## What it is
|
|
8
8
|
|
|
9
9
|
You label an issue. Within one tick the conductor claims it on the tracker, cuts a
|
|
10
|
-
worktree, hands one omp
|
|
11
|
-
|
|
10
|
+
worktree, hands one omp worker a self-contained brief, and watches it to a green
|
|
11
|
+
PR. The worker then stops: it never merges, tags, publishes or deploys.
|
|
12
12
|
|
|
13
13
|
### Scope
|
|
14
14
|
|
|
15
|
-
One issue, one green PR. That is the whole remit.
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
asked to. A worker whose change needs releasing reports that and stops.
|
|
15
|
+
One issue, one green PR. That is the worker's whole remit. A change that needs a
|
|
16
|
+
merge or release is reported to the orchestrator, whose `merge` and `release`
|
|
17
|
+
authority are chosen during setup and default to `human`. Granting either action
|
|
18
|
+
to the orchestrator never grants it to a worker or the dispatch daemon. Releases
|
|
19
|
+
remain batched from coherent groups of merged work, never cut one per worker PR.
|
|
21
20
|
|
|
22
21
|
Code counts every limit that decides whether work starts: concurrency, dollars per
|
|
23
22
|
day, turns and wall clock per worker, attempts per issue. None of it is left to the
|
|
@@ -29,10 +28,10 @@ When a run does get stuck, the first responder is not you. A tier-1 escalation i
|
|
|
29
28
|
injected into a long-lived **orchestrator session** that can read the issue and the
|
|
30
29
|
run's transcript and then either re-brief the worker or decide the problem genuinely
|
|
31
30
|
needs a human. It never edits product code and never pushes a branch; whether it
|
|
32
|
-
may
|
|
33
|
-
pages you directly.
|
|
31
|
+
may merge or release is a setup answer (`authority`), and both default to no. Only
|
|
32
|
+
tier 2 pages you directly.
|
|
34
33
|
|
|
35
|
-
The package ships three deployables, plus
|
|
34
|
+
The package ships three deployables, plus two skills:
|
|
36
35
|
|
|
37
36
|
| Deployable | Entry | What it is for |
|
|
38
37
|
| --- | --- | --- |
|
|
@@ -40,6 +39,7 @@ The package ships three deployables, plus one skill:
|
|
|
40
39
|
| Standalone daemon | `omp-conductor` binary | The dispatch loop, managed as a background process (`start` / `stop` / `restart`) with a `/healthz` endpoint for a supervisor. |
|
|
41
40
|
| Orchestrator heartbeat | omp extension, activated by `.conductor-tick.json` | Prompts a 24/7 orchestrator session on a fixed interval so its standing loop actually runs, and marks the session stalled when its prompts stop being consumed. Inert in every other session — including a second session opened in the fleet's own directory. See [Orchestrator tick](#orchestrator-tick). |
|
|
42
41
|
| Onboarding skill | `skill://conductor-onboarding` | Directs an omp session to interview you, read your repos for real CI gates, and tailor `ORCHESTRATOR.md` — then finish through the wizard. Discovered automatically once the plugin is installed. See [Onboarding](#onboarding). |
|
|
42
|
+
| Update skill | `skill://conductor-update` | Treats the npm and Herdr plugins as one maintenance operation: drain, halt, replace both halves, restart, re-arm through Telegram proof, and verify twice. See [Updating](#updating). |
|
|
43
43
|
|
|
44
44
|
The first two are thin wrappers over the same `daemon.ts`, so the plugin and the
|
|
45
45
|
CLI cannot disagree about what a cap means or where the state lives. Claiming is
|
|
@@ -166,6 +166,23 @@ Also required on the host:
|
|
|
166
166
|
With neither, tier 2 degrades to a comment on the issue. Nothing is broken in
|
|
167
167
|
that configuration: it is supported, just slower to reach you.
|
|
168
168
|
|
|
169
|
+
## Updating
|
|
170
|
+
|
|
171
|
+
Say “update conductor” from an operator shell or maintenance omp session outside
|
|
172
|
+
the target `herdr-fleet.service`. The bundled `skill://conductor-update` discovers
|
|
173
|
+
the installed and registry versions, drains active work, pins exact-pane recovery,
|
|
174
|
+
replaces both independently installed plugins, restarts through
|
|
175
|
+
`omp-conductor start`, re-arms through the existing Telegram proof, and verifies
|
|
176
|
+
the layered status twice.
|
|
177
|
+
|
|
178
|
+
The skill deliberately does not publish npm or edit an install root. It also
|
|
179
|
+
refuses to run from the fleet pane it must replace: an updater that kills itself
|
|
180
|
+
cannot verify the result. With skill commands enabled, invoke it directly with:
|
|
181
|
+
|
|
182
|
+
```text
|
|
183
|
+
/skill:conductor-update
|
|
184
|
+
```
|
|
185
|
+
|
|
169
186
|
## Onboarding
|
|
170
187
|
|
|
171
188
|
Onboarding this package has two layers, and installing it gives you both.
|
|
@@ -225,7 +242,7 @@ the file behind instead. So a re-run against a project that is already configure
|
|
|
225
242
|
opens with one question:
|
|
226
243
|
|
|
227
244
|
```text
|
|
228
|
-
"
|
|
245
|
+
"platform" is already configured — what would you like to do?
|
|
229
246
|
> Change one area
|
|
230
247
|
asks one area's questions; every other answer is carried through from the saved config
|
|
231
248
|
Walk every question again
|
|
@@ -237,14 +254,14 @@ says right now, so the row you want is the row you can see:
|
|
|
237
254
|
|
|
238
255
|
```text
|
|
239
256
|
Which area? Each row shows what it says now
|
|
240
|
-
tracker & repos —
|
|
241
|
-
gates —
|
|
257
|
+
tracker & repos — acme/platform, queue "ready-for-agent", "repo:" → platform, api, web, worker
|
|
258
|
+
gates — platform: bun run check; api: ruff check . @ backend; web: pnpm lint…
|
|
242
259
|
caps & worker model — 2 workers, 120 turns, 90m, $25/day, 2 attempts (all defaults) — harness default model
|
|
243
260
|
code graph — not configured — workers grep
|
|
244
261
|
authority — merge=orchestrator, release=orchestrator
|
|
245
|
-
escalation & triage — tier 2 pages Telegram
|
|
262
|
+
escalation & triage — tier 2 pages Telegram 123456789, comments too, triage external
|
|
246
263
|
reporting scope — material — escalations, plus green PRs, second failures, and anything that stops the fleet
|
|
247
|
-
orchestrator brief — none at
|
|
264
|
+
orchestrator brief — none at ~/.omp/conductor/worktrees/ORCHESTRATOR.md
|
|
248
265
|
```
|
|
249
266
|
|
|
250
267
|
Only that area's questions are asked. Every other answer is read back out of
|
|
@@ -254,11 +271,11 @@ writes a config, and it still writes nothing before you agree. The consent scree
|
|
|
254
271
|
leads with the delta and then shows the whole project as it would be written:
|
|
255
272
|
|
|
256
273
|
```text
|
|
257
|
-
amending code graph — project
|
|
274
|
+
amending code graph — project platform
|
|
258
275
|
was not configured — workers grep
|
|
259
|
-
now
|
|
276
|
+
now ~/.cache/conductor-graph/acme — 4 clone(s): platform, api, web, worker
|
|
260
277
|
carried over tracker & repos, gates, caps & worker model, authority, escalation & triage, reporting scope, orchestrator brief
|
|
261
|
-
read back from
|
|
278
|
+
read back from ~/.omp/conductor/config.json and rewritten unchanged
|
|
262
279
|
```
|
|
263
280
|
|
|
264
281
|
A first run, or a project name this config has never seen, never sees either
|
|
@@ -331,12 +348,22 @@ The **Learning loop** proposes diffs against `POLICY.md` for you to approve over
|
|
|
331
348
|
omp-conductor start
|
|
332
349
|
```
|
|
333
350
|
|
|
334
|
-
`start`
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
351
|
+
`start` first starts `herdr-fleet.service` when that optional unit is installed,
|
|
352
|
+
clearing a previous `halt --pane` recovery pin so Herdr can resume the exact
|
|
353
|
+
conductor pane. Hosts without systemd or without that unit keep the standalone
|
|
354
|
+
daemon behaviour. It then waits until the daemon actually answers
|
|
355
|
+
`GET /healthz`; spawning is not starting. A daemon whose config is broken,
|
|
356
|
+
whose port is taken or whose database is locked exits within a second, and the
|
|
357
|
+
command fails with the tail of `daemon.log` instead of printing a false
|
|
358
|
+
success. It refuses to start a second daemon, naming the live pid. Starting
|
|
359
|
+
processes does not clear `pause` or arm ticks; those remain explicit operator
|
|
360
|
+
decisions.
|
|
361
|
+
|
|
362
|
+
Pane recovery spans two separately installed plugins: npm ships the omp
|
|
363
|
+
heartbeat/status half, while `herdr-conductor` supplies `recover.sh`. After an
|
|
364
|
+
npm upgrade, refresh the Herdr plugin from `TerrifiedBug/conductor/herdr` as
|
|
365
|
+
well; publishing or installing npm alone cannot add the recovery-side tick
|
|
366
|
+
request.
|
|
340
367
|
|
|
341
368
|
For a first run, take a single tick in the foreground and watch it:
|
|
342
369
|
|
|
@@ -366,7 +393,7 @@ Four control planes used to answer "stop" differently. The package verbs:
|
|
|
366
393
|
|
|
367
394
|
`resume` clears pause only and **never re-arms**. `arm` is proof-gated: it sends a Telegram challenge and writes the arm marker only after your reply appears as a *user* turn in the orchestrator transcript. `halt --pane` targets the configured conductor agent only — it does **not** run `systemctl stop herdr-fleet`.
|
|
368
395
|
|
|
369
|
-
`status` prints a layered header (`dispatch` / `ticks` / `pane` / `recovery` / `herdr` / `daemon`) so a paused fleet cannot hide an armed orchestrator still spending turns.
|
|
396
|
+
`status` prints a layered header (`dispatch` / `ticks` / next tick time / `pane` / `recovery` / `herdr` / `telegram` / `daemon`) so a paused fleet cannot hide an armed orchestrator still spending turns. The Telegram line calls the official `getMe` endpoint to prove the token and API are usable without sending a message, then separately reports whether the inbound bridge is configured.
|
|
370
397
|
|
|
371
398
|
`halt --pane` is **fail-closed**: it exits `0` only when the conductor agent is
|
|
372
399
|
*proven* gone. It writes the recovery pin first, so a failed stop still cannot be
|
|
@@ -1043,7 +1070,7 @@ indistinguishable from the first:
|
|
|
1043
1070
|
```text
|
|
1044
1071
|
[omp-conductor] orchestrator tick inactive: pane w1:p1 (agent "fleet") owns the fleet tick here — this session will not tick
|
|
1045
1072
|
[omp-conductor] orchestrator tick inactive: this pane is agent "scratch", not the fleet agent "fleet" — this session will not tick
|
|
1046
|
-
[omp-conductor] orchestrator tick inactive: pid
|
|
1073
|
+
[omp-conductor] orchestrator tick inactive: pid 12345 (claimed 2026-01-02T03:04:05.000Z, session …/fleet.jsonl) owns the fleet tick in /home/conductor/.omp/conductor — this session will not tick
|
|
1047
1074
|
```
|
|
1048
1075
|
|
|
1049
1076
|
A `herdr agent list` that does not answer also declines, for the same reason the
|
|
@@ -1164,6 +1191,7 @@ least of all on a fleet whose session lives somewhere else.
|
|
|
1164
1191
|
|
|
1165
1192
|
```bash
|
|
1166
1193
|
omp-conductor start [--port N] [--project NAME]
|
|
1194
|
+
omp-conductor --version
|
|
1167
1195
|
omp-conductor stop
|
|
1168
1196
|
omp-conductor restart [--port N] [--project NAME]
|
|
1169
1197
|
omp-conductor status [--project NAME]
|
|
@@ -1184,10 +1212,10 @@ omp-conductor help
|
|
|
1184
1212
|
|
|
1185
1213
|
| Command | Behaviour |
|
|
1186
1214
|
| --- | --- |
|
|
1187
|
-
| `start` |
|
|
1215
|
+
| `start` | Start `herdr-fleet.service` when that optional unit is installed, clearing a previous pane-recovery pin, then spawn the dispatch loop in the background and wait until it answers `GET /healthz` on `:8787`. Without systemd or that unit it keeps the standalone daemon behaviour. It never clears pause or arms ticks. Refuses if a daemon is already live, naming its pid; if the process dies or never serves, it cleans up and quotes the tail of `daemon.log`. |
|
|
1188
1216
|
| `stop` | Prefer `systemctl stop omp-conductor.service` when that unit's MainPID is the live daemon — systemd then owns the stop and will not schedule a restart for the exit it just requested. Otherwise `SIGTERM`, then `SIGKILL` after a 10-second grace period. Prints `not running` when there is nothing to stop, and tags the confirmation with `(via systemctl)` when the unit path was used. |
|
|
1189
1217
|
| `restart` | Prefer `systemctl restart` when the unit owns the live pid so the replacement stays supervised; otherwise `stop` then `start`, inheriting the running daemon's port and project unless a flag overrides them. The new process **salvages dirty live worktrees before orphaning** those rows — see [Deploying a new package onto a busy fleet](#deploying-a-new-package-onto-a-busy-fleet). |
|
|
1190
|
-
| `status [--project NAME]` | Layered fleet report first: `dispatch` / `ticks` / `pane` / `recovery` / `herdr` / `daemon`, then the project body
|
|
1218
|
+
| `status [--project NAME]` | Layered fleet report first: `dispatch` / `ticks` / next scheduled tick / `pane` / `recovery` / `herdr` / `telegram` / `daemon`, then the project body. The next time comes from the live heartbeat process, not a guess from log timestamps. Telegram health uses `getMe` to prove API authentication without sending a message and reports inbound bridge configuration separately. The daemon block includes `rss` from `/healthz`; live workers add a busy-deploy warning. A `.conductor-stalled` marker adds an `orchestrator STALLED since …` line. |
|
|
1191
1219
|
| `hold [--project NAME]` | Soft stop: pause claiming **and** disarm ticks. Daemon and pane stay up. Prefer this over `pause` when the intent is "stop the conductor" without killing processes. See [Stop the conductor](#stop-the-conductor-hold--halt). |
|
|
1192
1220
|
| `halt [--pane] [--project NAME]` | `hold`, then stop the dispatch daemon (systemctl-aware). Pane stays up unless `--pane` is passed. `halt --pane` also pins herdr-conductor recovery off for the conductor agent only — it does **not** stop `herdr-fleet.service` or any other herdr session. Fail-closed: exits nonzero unless the agent is proven gone. |
|
|
1193
1221
|
| `arm [--project NAME]` | Proof-gated: send a Telegram challenge and write the arm marker only after your reply appears as a user turn in the orchestrator transcript. Never auto-armed by `resume` / `hold`. |
|
|
@@ -1201,6 +1229,7 @@ omp-conductor help
|
|
|
1201
1229
|
| `--project NAME` | Pick the project to service. One daemon process serves exactly one project; with several configured projects the name is required. |
|
|
1202
1230
|
| `pause` | Stop claiming new work only. The running daemon notices on its next tick; runs already in flight finish. The orchestrator heartbeat keeps ticking if armed — its gate is the arm marker, not this flag. Prefer `hold` to silence both. |
|
|
1203
1231
|
| `resume` | Clear pause only — does **not** re-arm. Run `arm` after an inbound Telegram proof to resume ticks. |
|
|
1232
|
+
| `--version`, `-V`, `version` | Print the installed `omp-conductor` package version and exit `0`. Works from the global binary and npm/plugin install because it reads the package metadata beside the shipped CLI. |
|
|
1204
1233
|
| `graph-setup` | Print how to set up the code-graph indexes workers query instead of grepping: a `git clone` for every index-only clone that does not exist yet, the one-shot index command per repo, and a `cbm-reindex.service` + `cbm-reindex.timer` pair generated from the project's own repos and branches. Reads only, so it is safe on a host where you are not root. Exits `1` when no repo in the project has [`graphProject`](#configuration) set, because the fix is a wizard answer rather than a flag. See [Code-graph discovery](#code-graph-discovery). |
|
|
1205
1234
|
| `--write` | Only for `graph-setup`. Writes the refresh script into the state directory and the two units into `/etc/systemd/system`, then prints the exact `systemctl daemon-reload && systemctl enable --now cbm-reindex.timer` to run. It never runs `systemctl` itself and never enables anything: that needs root, and a package that enables system timers behind your back is one you cannot audit by reading its output. |
|
|
1206
1235
|
| `brief-upgrade` | Inspect the package-floor + `POLICY.md` overlay. Reports by default; see [Keeping a brief current](#keeping-a-brief-current). |
|
|
@@ -1368,8 +1397,9 @@ Known and deliberate in this version:
|
|
|
1368
1397
|
`slots = maxConcurrentWorkers - live workers`, admits at most that many issues
|
|
1369
1398
|
per tick, and dispatches them together. To see them, read `omp-conductor
|
|
1370
1399
|
status`, which lists every occupied issue, or follow `daemon.log`.
|
|
1371
|
-
- **
|
|
1372
|
-
|
|
1400
|
+
- **Workers stop at green PRs.** They never merge, release or deploy. Those
|
|
1401
|
+
actions default to a human, but setup may grant either to the orchestrator;
|
|
1402
|
+
`authority` never grants them to a worker or the dispatch daemon.
|
|
1373
1403
|
- **Worker confinement is partial.** Structured `write` / `edit` / `read` /
|
|
1374
1404
|
`grep` / `glob` calls are blocked outside the worktree by an inline harness
|
|
1375
1405
|
extension (`confineToCwd`). `bash` is not: a shell one-liner can still leave
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "omp-conductor",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.16",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"description": "A 24/7 dispatcher that takes ready GitHub issues to green, mergeable PRs using omp coding sessions, with tiered escalation first to an orchestrator session and then to a human.",
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
]
|
|
18
18
|
},
|
|
19
19
|
"bin": {
|
|
20
|
-
"omp-conductor": "
|
|
20
|
+
"omp-conductor": "src/cli.ts"
|
|
21
21
|
},
|
|
22
22
|
"files": [
|
|
23
23
|
"src",
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: conductor-update
|
|
3
|
+
description: Safely update an installed omp-conductor fleet as one maintenance operation across both separately installed halves: the npm omp plugin and the GitHub herdr-conductor plugin. Use when the user asks to update, upgrade, refresh, reinstall, or deploy a newly published conductor version, or when installed conductor code is behind npm/main. Quiesces work, preserves exact pane identity, performs a whole-version swap, restarts through the supported lifecycle, re-arms through Telegram proof, and verifies the live fleet.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Update a conductor fleet
|
|
7
|
+
|
|
8
|
+
Treat an update as one operation across two independently installed plugins:
|
|
9
|
+
|
|
10
|
+
- `omp-conductor` comes from npm and owns dispatch, CLI, status, and heartbeat.
|
|
11
|
+
- `herdr-conductor` comes from `TerrifiedBug/conductor/herdr` and owns exact-pane recovery.
|
|
12
|
+
|
|
13
|
+
Updating only one half can leave a version that starts but cannot recover correctly.
|
|
14
|
+
This skill updates an installed fleet. It does **not** publish npm, merge a PR, tag a
|
|
15
|
+
release, or edit files in an install directory.
|
|
16
|
+
|
|
17
|
+
## Safety boundary
|
|
18
|
+
|
|
19
|
+
Run this procedure from an operator shell or maintenance omp session that is not
|
|
20
|
+
hosted by the target `herdr-fleet.service`. If the current working directory
|
|
21
|
+
contains `.conductor-tick.json`, or stopping that unit would kill the session
|
|
22
|
+
executing this skill, stop and move the update to an external maintenance
|
|
23
|
+
session. A self-terminating updater cannot produce trustworthy verification.
|
|
24
|
+
|
|
25
|
+
Use the configured target host. Do not assume a hostname, state directory, Herdr
|
|
26
|
+
session name, project name, or systemd availability. Read them from the existing
|
|
27
|
+
installation and `omp-conductor status`. Never print bot tokens, npm tokens, or
|
|
28
|
+
authentication files.
|
|
29
|
+
|
|
30
|
+
## 1. Establish the desired version
|
|
31
|
+
|
|
32
|
+
Read, do not guess:
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
omp-conductor --version
|
|
36
|
+
version=$(npm view omp-conductor version)
|
|
37
|
+
gitHead=$(npm view "omp-conductor@$version" gitHead)
|
|
38
|
+
npm view "omp-conductor@$version" version gitHead --json
|
|
39
|
+
omp plugin list --json
|
|
40
|
+
omp-conductor status [--project NAME]
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
Use the registry latest as `$version` unless the user explicitly names another
|
|
44
|
+
published version, then query that exact version as shown. Record its `gitHead`
|
|
45
|
+
and require a full commit SHA: the npm spec and Herdr ref below are both pinned,
|
|
46
|
+
so a concurrent release cannot mix two versions. If npm has no newer version, do
|
|
47
|
+
not churn the fleet: report that it is current. Confirm that `herdr` and
|
|
48
|
+
`systemctl` are present before taking anything down when status says Herdr
|
|
49
|
+
manages the pane.
|
|
50
|
+
|
|
51
|
+
## 2. Quiesce without losing work
|
|
52
|
+
|
|
53
|
+
First stop new claims and heartbeat prompts:
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
omp-conductor hold [--project NAME]
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
Read status until `active runs (none)`. Do not kill workers to make the update
|
|
60
|
+
faster. If a run does not drain, report the run and stop; ordinary update authority
|
|
61
|
+
does not include discarding work.
|
|
62
|
+
|
|
63
|
+
Then stop the exact conductor pane and dispatch daemon through the supported,
|
|
64
|
+
fail-closed path:
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
omp-conductor halt --pane [--project NAME]
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
This writes the recovery pin before stopping the exact configured Herdr agent. If
|
|
71
|
+
identity is invalid, missing, or ambiguous, it refuses rather than killing a guess.
|
|
72
|
+
Do not bypass that refusal with `pkill`.
|
|
73
|
+
|
|
74
|
+
When `herdr-fleet.service` is installed, stop it before replacing either plugin:
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
systemctl stop herdr-fleet.service
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
At this point the daemon is stopped, the pane is stopped, and recovery remains
|
|
81
|
+
pinned. If any of those statements is false, do not modify the install.
|
|
82
|
+
|
|
83
|
+
## 3. Replace both installed halves
|
|
84
|
+
|
|
85
|
+
Refresh the exact npm version using omp's package installer:
|
|
86
|
+
|
|
87
|
+
```bash
|
|
88
|
+
omp plugin install "omp-conductor@$version"
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
Then refresh the Herdr managed checkout from the exact npm release commit. Herdr
|
|
92
|
+
requires the GitHub source before its options:
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
herdr plugin install TerrifiedBug/conductor/herdr --ref "$gitHead" --yes
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
Do not use `git pull`, `scp`, a linked checkout, or edits under either plugin's
|
|
99
|
+
install root. This is a whole-version swap. If either install fails, leave the
|
|
100
|
+
fleet stopped and recovery pinned; report the failed command and do not continue
|
|
101
|
+
with a mixed live version.
|
|
102
|
+
|
|
103
|
+
## 4. Start through the new lifecycle
|
|
104
|
+
|
|
105
|
+
Run the newly installed CLI:
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
omp-conductor --version
|
|
109
|
+
omp-conductor start [--project NAME]
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
`start` clears the recovery pin, starts the optional `herdr-fleet.service`, and
|
|
113
|
+
starts the dispatch daemon after a real `/healthz` check. Herdr then recovers the
|
|
114
|
+
exact orchestrator pane and requests an immediate tick; the verification below,
|
|
115
|
+
not the `start` command alone, proves that recovery completed.
|
|
116
|
+
|
|
117
|
+
Wait until `omp-conductor status` reports the exact pane `live`; `start` returning
|
|
118
|
+
only proves the service and dispatch daemon. `hold` deliberately disarmed ticks,
|
|
119
|
+
so restore unattended operation through the existing Telegram proof:
|
|
120
|
+
|
|
121
|
+
```bash
|
|
122
|
+
omp-conductor arm [--project NAME]
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
Wait for the operator's inbound Telegram reply. Never create the arm marker by
|
|
126
|
+
hand and never treat an outbound challenge as proof.
|
|
127
|
+
|
|
128
|
+
## 5. Verify the live result
|
|
129
|
+
|
|
130
|
+
Require all of the following before reporting success:
|
|
131
|
+
|
|
132
|
+
```bash
|
|
133
|
+
omp-conductor --version
|
|
134
|
+
npm view "omp-conductor@$version" version gitHead --json
|
|
135
|
+
omp plugin list --json
|
|
136
|
+
herdr plugin list
|
|
137
|
+
omp-conductor status [--project NAME]
|
|
138
|
+
systemctl is-active herdr-fleet.service
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
- installed version and the npm package entry both equal `$version`;
|
|
142
|
+
- the Herdr plugin source revision equals the npm release `gitHead`;
|
|
143
|
+
- `dispatch` is `running`;
|
|
144
|
+
- `ticks` is `armed` and a next tick is shown;
|
|
145
|
+
- `pane` is `live` for the exact configured Herdr agent;
|
|
146
|
+
- `recovery` is `clear`;
|
|
147
|
+
- `herdr` is `active` when managed;
|
|
148
|
+
- Telegram is `ok` (or its exact supported degraded state is reported);
|
|
149
|
+
- daemon `/healthz` is `ok`;
|
|
150
|
+
- no unexpected active runs appeared during maintenance.
|
|
151
|
+
|
|
152
|
+
Run a second status check after the immediate recovery tick is consumed. A single
|
|
153
|
+
healthy snapshot is not proof that recovery and heartbeat scheduling survived the
|
|
154
|
+
swap.
|
|
155
|
+
|
|
156
|
+
Report the old and new versions, both plugin refreshes, the final layered status,
|
|
157
|
+
and any supported degraded state. Do not report success for a partial update.
|
package/src/cli.ts
CHANGED
|
@@ -31,6 +31,7 @@ import {
|
|
|
31
31
|
hold,
|
|
32
32
|
releaseHold,
|
|
33
33
|
renderStatus,
|
|
34
|
+
startHerdrFleet,
|
|
34
35
|
} from "./fleet.ts";
|
|
35
36
|
import { formatGraphSetup, graphRepos, writeGraphSetup, type GraphSetupWrite } from "./graph.ts";
|
|
36
37
|
import {
|
|
@@ -55,10 +56,21 @@ import { makeTracker } from "./tracker/github.ts";
|
|
|
55
56
|
import type { ProjectConfig } from "./types.ts";
|
|
56
57
|
import { formatUnblock, unblockIssue } from "./unblock.ts";
|
|
57
58
|
|
|
59
|
+
function packageVersion(): string {
|
|
60
|
+
const parsed = JSON.parse(readFileSync(join(import.meta.dir, "..", "package.json"), "utf8")) as {
|
|
61
|
+
version?: unknown;
|
|
62
|
+
};
|
|
63
|
+
if (typeof parsed.version !== "string" || parsed.version.length === 0) {
|
|
64
|
+
throw new Error("installed package.json has no version");
|
|
65
|
+
}
|
|
66
|
+
return parsed.version;
|
|
67
|
+
}
|
|
68
|
+
|
|
58
69
|
const USAGE = `omp-conductor — dispatch ready issues to omp coding sessions
|
|
59
70
|
|
|
60
71
|
usage:
|
|
61
72
|
omp-conductor start [--port N] [--project NAME]
|
|
73
|
+
omp-conductor --version
|
|
62
74
|
omp-conductor stop
|
|
63
75
|
omp-conductor restart [--port N] [--project NAME]
|
|
64
76
|
omp-conductor status [--project NAME]
|
|
@@ -76,9 +88,9 @@ usage:
|
|
|
76
88
|
omp-conductor brief-upgrade [--migrate|--retrofit] [--apply] [--file PATH] [--project NAME]
|
|
77
89
|
omp-conductor help
|
|
78
90
|
|
|
79
|
-
start
|
|
80
|
-
|
|
81
|
-
|
|
91
|
+
start start the installed herdr-fleet.service when present, then run the
|
|
92
|
+
dispatch loop in the background and wait until it answers GET
|
|
93
|
+
/healthz on :8787 (override with --port). Refuses if one is running.
|
|
82
94
|
stop stop the running daemon. Uses systemctl when the omp-conductor
|
|
83
95
|
unit owns the process (so Restart=on-failure cannot bring it back);
|
|
84
96
|
otherwise SIGTERM then SIGKILL.
|
|
@@ -87,8 +99,9 @@ usage:
|
|
|
87
99
|
dirty live worktrees before orphaning those rows — see README
|
|
88
100
|
"Deploying a new package onto a busy fleet". Goes through systemctl
|
|
89
101
|
when the unit owns the live pid.
|
|
90
|
-
status layered fleet report: dispatch (running|paused|stopped), ticks
|
|
91
|
-
|
|
102
|
+
status layered fleet report: dispatch (running|paused|stopped), ticks and
|
|
103
|
+
next due time, pane, herdr, Telegram bot/API health, daemon, caps
|
|
104
|
+
and active runs.
|
|
92
105
|
hold soft stop: pause claiming AND disarm ticks. Daemon and pane stay up.
|
|
93
106
|
This is "stop the conductor overnight" without killing processes.
|
|
94
107
|
halt hold, then stop the dispatch daemon (systemctl-aware). Pane stays up
|
|
@@ -133,6 +146,8 @@ usage:
|
|
|
133
146
|
--apply still merges a bannered single-file brief. --file checks a
|
|
134
147
|
brief that is not where the wizard would have put it.
|
|
135
148
|
help print this text (also --help, -h).
|
|
149
|
+
--version
|
|
150
|
+
print the installed omp-conductor package version (also -V, version).
|
|
136
151
|
|
|
137
152
|
Pause is a flag file under the state directory, so it applies to every project
|
|
138
153
|
and survives a daemon restart. Hold also removes the arm marker the heartbeat
|
|
@@ -378,6 +393,11 @@ const cmd = argv[0];
|
|
|
378
393
|
|
|
379
394
|
try {
|
|
380
395
|
switch (cmd) {
|
|
396
|
+
case "--version":
|
|
397
|
+
case "-V":
|
|
398
|
+
case "version":
|
|
399
|
+
process.stdout.write(`${packageVersion()}\n`);
|
|
400
|
+
break;
|
|
381
401
|
case "daemon": {
|
|
382
402
|
// Until now only `lifecycle.startDaemon()` — the spawn path — wrote the
|
|
383
403
|
// pidfile, which left a daemon started in the foreground (which is how
|
|
@@ -429,10 +449,14 @@ try {
|
|
|
429
449
|
}
|
|
430
450
|
|
|
431
451
|
case "start": {
|
|
432
|
-
const
|
|
452
|
+
const project = flag(argv, "project");
|
|
453
|
+
const herdr = startHerdrFleet(project);
|
|
454
|
+
const rec = await startDaemon({ port: portFlag(argv), project });
|
|
433
455
|
process.stdout.write(
|
|
434
456
|
`started — pid ${rec.pid}, /healthz on :${rec.port}` +
|
|
435
|
-
`${rec.project === undefined ? "" : `, project ${rec.project}`}\
|
|
457
|
+
`${rec.project === undefined ? "" : `, project ${rec.project}`}\n` +
|
|
458
|
+
`herdr ${herdr.kind === "active" ? `active (${herdr.unit})${herdr.recoveryReleased ? "; recovery pin cleared" : ""}` : `unmanaged (${herdr.reason})`}\n` +
|
|
459
|
+
`log ${rec.logFile}\n`,
|
|
436
460
|
);
|
|
437
461
|
break;
|
|
438
462
|
}
|
package/src/fleet.ts
CHANGED
|
@@ -31,6 +31,7 @@ import { findProject, loadConfig, stateDir } from "./config.ts";
|
|
|
31
31
|
import { isPaused, setPaused, statusSnapshot, type StatusSnapshot } from "./daemon.ts";
|
|
32
32
|
import {
|
|
33
33
|
healthCheck,
|
|
34
|
+
isAlive,
|
|
34
35
|
livingDaemon,
|
|
35
36
|
stopDaemon,
|
|
36
37
|
type StopResult,
|
|
@@ -39,6 +40,7 @@ import {
|
|
|
39
40
|
import { formatRss, rssBytesFromHealthz } from "./host.ts";
|
|
40
41
|
import {
|
|
41
42
|
readTickConfig,
|
|
43
|
+
readTickRuntimeStatus,
|
|
42
44
|
TICK_CONFIG_FILE,
|
|
43
45
|
type TickConfig,
|
|
44
46
|
type TickConfigResult,
|
|
@@ -304,6 +306,66 @@ export function clearPaneHalt(projectName?: string): { path: string; wasHalted:
|
|
|
304
306
|
return { path, wasHalted };
|
|
305
307
|
}
|
|
306
308
|
|
|
309
|
+
export type HerdrStartResult =
|
|
310
|
+
| { kind: "active"; unit: string; recoveryReleased: boolean }
|
|
311
|
+
| { kind: "unmanaged"; unit: string; reason: string };
|
|
312
|
+
|
|
313
|
+
export interface HerdrStartDeps {
|
|
314
|
+
systemctl?: (args: string[]) => { ok: boolean; stdout: string; stderr: string; missing?: boolean };
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/**
|
|
318
|
+
* Starts the dedicated Herdr fleet unit when it is installed. Hosts without
|
|
319
|
+
* systemd or without that optional unit keep the standalone daemon behaviour.
|
|
320
|
+
*/
|
|
321
|
+
export function startHerdrFleet(projectName?: string, deps: HerdrStartDeps = {}): HerdrStartResult {
|
|
322
|
+
const run: NonNullable<HerdrStartDeps["systemctl"]> =
|
|
323
|
+
deps.systemctl ??
|
|
324
|
+
((args: string[]) => {
|
|
325
|
+
const res = spawnSync("systemctl", args, { encoding: "utf8", timeout: 15_000, env: process.env });
|
|
326
|
+
if (res.error) {
|
|
327
|
+
const err = res.error as NodeJS.ErrnoException;
|
|
328
|
+
return {
|
|
329
|
+
ok: false,
|
|
330
|
+
stdout: "",
|
|
331
|
+
stderr: err.message,
|
|
332
|
+
...(err.code === "ENOENT" ? { missing: true } : {}),
|
|
333
|
+
};
|
|
334
|
+
}
|
|
335
|
+
return { ok: res.status === 0, stdout: res.stdout ?? "", stderr: res.stderr ?? "" };
|
|
336
|
+
});
|
|
337
|
+
|
|
338
|
+
const shown = run(["show", DEFAULT_HERDR_UNIT, "--property=LoadState", "--value"]);
|
|
339
|
+
if (shown.missing) return { kind: "unmanaged", unit: DEFAULT_HERDR_UNIT, reason: "no systemctl" };
|
|
340
|
+
if (!shown.ok) {
|
|
341
|
+
const detail = (shown.stderr.trim() || shown.stdout.trim() || "systemctl show failed").split("\n")[0]!;
|
|
342
|
+
throw new Error(`cannot inspect ${DEFAULT_HERDR_UNIT}: ${detail}`);
|
|
343
|
+
}
|
|
344
|
+
const loadState = shown.stdout.trim();
|
|
345
|
+
if (loadState === "not-found" || loadState === "") {
|
|
346
|
+
return { kind: "unmanaged", unit: DEFAULT_HERDR_UNIT, reason: "unit not installed" };
|
|
347
|
+
}
|
|
348
|
+
if (loadState !== "loaded") {
|
|
349
|
+
throw new Error(`${DEFAULT_HERDR_UNIT} is ${loadState}, not startable`);
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
const halt = resolvePaneHaltPath(projectName);
|
|
353
|
+
const recoveryReleased = halt.kind === "ok" && existsSync(halt.path);
|
|
354
|
+
if (halt.kind === "ok") rmSync(halt.path, { force: true });
|
|
355
|
+
|
|
356
|
+
const started = run(["start", DEFAULT_HERDR_UNIT]);
|
|
357
|
+
if (!started.ok) {
|
|
358
|
+
const detail = (started.stderr.trim() || started.stdout.trim() || "systemctl start failed").split("\n")[0]!;
|
|
359
|
+
throw new Error(`systemctl start ${DEFAULT_HERDR_UNIT} failed: ${detail}`);
|
|
360
|
+
}
|
|
361
|
+
const active = run(["is-active", DEFAULT_HERDR_UNIT]);
|
|
362
|
+
if (!active.ok || active.stdout.trim() !== "active") {
|
|
363
|
+
const detail = (active.stderr.trim() || active.stdout.trim() || "not active").split("\n")[0]!;
|
|
364
|
+
throw new Error(`${DEFAULT_HERDR_UNIT} did not become active: ${detail}`);
|
|
365
|
+
}
|
|
366
|
+
return { kind: "active", unit: DEFAULT_HERDR_UNIT, recoveryReleased };
|
|
367
|
+
}
|
|
368
|
+
|
|
307
369
|
export interface HerdrAgent {
|
|
308
370
|
name: string;
|
|
309
371
|
paneId: string;
|
|
@@ -565,11 +627,13 @@ async function herdrAgentList(deps: PaneStopDeps): Promise<HerdrAgent[]> {
|
|
|
565
627
|
if (res.status !== 0) {
|
|
566
628
|
throw new Error((res.stderr ?? res.stdout ?? `herdr exit ${String(res.status)}`).trim());
|
|
567
629
|
}
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
630
|
+
return parseHerdrAgentList(res.stdout ?? "");
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
export function parseHerdrAgentList(rawOutput: string): HerdrAgent[] {
|
|
634
|
+
// Only an explicit `agents: []` means no agents. Every malformed answer is
|
|
635
|
+
// uncertainty, never permission to report the conductor pane missing.
|
|
636
|
+
const raw = rawOutput.trim();
|
|
573
637
|
if (raw.length === 0) {
|
|
574
638
|
throw new Error("herdr agent list printed nothing — cannot tell whether the pane is running");
|
|
575
639
|
}
|
|
@@ -590,38 +654,35 @@ async function herdrAgentList(deps: PaneStopDeps): Promise<HerdrAgent[]> {
|
|
|
590
654
|
if (row === null || typeof row !== "object") {
|
|
591
655
|
throw new Error("herdr agent list contains a non-object agent row — unrecognized schema");
|
|
592
656
|
}
|
|
593
|
-
const
|
|
594
|
-
const name =
|
|
595
|
-
const paneId =
|
|
657
|
+
const agent = row as { readonly [key: string]: unknown };
|
|
658
|
+
const name = agent["name"];
|
|
659
|
+
const paneId = agent["pane_id"];
|
|
596
660
|
if (typeof name !== "string" || typeof paneId !== "string") {
|
|
597
661
|
throw new Error(
|
|
598
662
|
"herdr agent list row is missing a string `name`/`pane_id` — " +
|
|
599
663
|
"cannot tell whether it is the conductor pane",
|
|
600
664
|
);
|
|
601
665
|
}
|
|
602
|
-
|
|
603
|
-
// agent has exited — recover.sh reads the same field as `(.agent // "")`,
|
|
604
|
-
// and the released-agent fixture omits the key entirely. A *present* value
|
|
605
|
-
// of any other type is a row we cannot read: coercing it to `undefined`
|
|
606
|
-
// would report a live pane as already-gone.
|
|
607
|
-
const rawAgent = a["agent"];
|
|
666
|
+
const rawAgent = agent["agent"];
|
|
608
667
|
if (rawAgent !== undefined && rawAgent !== null && typeof rawAgent !== "string") {
|
|
609
668
|
throw new Error(
|
|
610
669
|
`herdr agent list row for ${name} has a non-string \`agent\` (${typeof rawAgent}) — ` +
|
|
611
670
|
`cannot tell whether an agent is live`,
|
|
612
671
|
);
|
|
613
672
|
}
|
|
614
|
-
const
|
|
673
|
+
const liveAgent = typeof rawAgent === "string" ? rawAgent : undefined;
|
|
615
674
|
let sessionPath: string | undefined;
|
|
616
|
-
const
|
|
617
|
-
if (
|
|
618
|
-
const
|
|
619
|
-
if (
|
|
675
|
+
const session = agent["agent_session"];
|
|
676
|
+
if (session !== null && typeof session === "object") {
|
|
677
|
+
const value = session as { readonly [key: string]: unknown };
|
|
678
|
+
if (value["source"] === "herdr:omp" && typeof value["value"] === "string") {
|
|
679
|
+
sessionPath = value["value"];
|
|
680
|
+
}
|
|
620
681
|
}
|
|
621
682
|
out.push({
|
|
622
683
|
name,
|
|
623
684
|
paneId,
|
|
624
|
-
...(
|
|
685
|
+
...(liveAgent === undefined ? {} : { agent: liveAgent }),
|
|
625
686
|
...(sessionPath === undefined ? {} : { sessionPath }),
|
|
626
687
|
});
|
|
627
688
|
}
|
|
@@ -709,11 +770,18 @@ export type PaneLayer = "live" | "missing" | "unknown";
|
|
|
709
770
|
/** `unpinnable`: no tick config, so FLEET_CWD — the only path recovery reads — is unknown. */
|
|
710
771
|
export type RecoveryLayer = "pinned" | "clear" | "unpinnable";
|
|
711
772
|
export type HerdrLayer = "active" | "inactive" | "unknown";
|
|
773
|
+
export type TelegramLayer = "ok" | "degraded" | "down" | "unconfigured" | "unprobed";
|
|
774
|
+
|
|
775
|
+
export interface TelegramHealth {
|
|
776
|
+
kind: TelegramLayer;
|
|
777
|
+
detail?: string;
|
|
778
|
+
}
|
|
712
779
|
|
|
713
780
|
export interface FleetLayers {
|
|
714
781
|
dispatch: DispatchLayer;
|
|
715
782
|
ticks: TicksLayer;
|
|
716
783
|
ticksDetail?: string;
|
|
784
|
+
nextTickAt?: string;
|
|
717
785
|
pane: PaneLayer;
|
|
718
786
|
paneDetail?: string;
|
|
719
787
|
recovery: RecoveryLayer;
|
|
@@ -735,6 +803,7 @@ export function fleetLayers(projectName?: string): FleetLayers {
|
|
|
735
803
|
const tick = resolveTickConfig(projectName);
|
|
736
804
|
let ticks: TicksLayer;
|
|
737
805
|
let ticksDetail: string | undefined;
|
|
806
|
+
let nextTickAt: string | undefined;
|
|
738
807
|
let armedPath: string | undefined;
|
|
739
808
|
let tickConfigPath: string | undefined;
|
|
740
809
|
if (tick.kind === "absent") {
|
|
@@ -756,6 +825,8 @@ export function fleetLayers(projectName?: string): FleetLayers {
|
|
|
756
825
|
ticks = "disarmed";
|
|
757
826
|
ticksDetail = tick.config.armedFile;
|
|
758
827
|
}
|
|
828
|
+
const runtime = readTickRuntimeStatus(tick.cwd);
|
|
829
|
+
if (runtime !== undefined && isAlive(runtime.pid)) nextTickAt = runtime.nextTickAt;
|
|
759
830
|
}
|
|
760
831
|
if (armedPath === undefined) armedPath = armedMarkerPath(projectName);
|
|
761
832
|
|
|
@@ -764,7 +835,7 @@ export function fleetLayers(projectName?: string): FleetLayers {
|
|
|
764
835
|
const resolvedHalt = resolvePaneHaltPath(projectName);
|
|
765
836
|
const haltPath = resolvedHalt.kind === "ok" ? resolvedHalt.path : undefined;
|
|
766
837
|
const recoveryPinned = haltPath !== undefined && existsSync(haltPath);
|
|
767
|
-
const omp = probeOmpPane();
|
|
838
|
+
const omp = probeOmpPane(projectName);
|
|
768
839
|
let pane: PaneLayer;
|
|
769
840
|
let paneDetail: string | undefined;
|
|
770
841
|
if (omp.kind === "live") {
|
|
@@ -783,6 +854,7 @@ export function fleetLayers(projectName?: string): FleetLayers {
|
|
|
783
854
|
dispatch,
|
|
784
855
|
ticks,
|
|
785
856
|
...(ticksDetail === undefined ? {} : { ticksDetail }),
|
|
857
|
+
...(nextTickAt === undefined ? {} : { nextTickAt }),
|
|
786
858
|
pane,
|
|
787
859
|
...(paneDetail === undefined ? {} : { paneDetail }),
|
|
788
860
|
recovery: haltPath === undefined ? "unpinnable" : recoveryPinned ? "pinned" : "clear",
|
|
@@ -805,11 +877,21 @@ export function formatFleetStatus(
|
|
|
805
877
|
s: StatusSnapshot,
|
|
806
878
|
layers: FleetLayers,
|
|
807
879
|
daemonHealth?: { ok: boolean; body?: string },
|
|
880
|
+
telegram: TelegramHealth = { kind: "unprobed" },
|
|
881
|
+
now = Date.now(),
|
|
808
882
|
): string {
|
|
809
883
|
const tickLine =
|
|
810
884
|
layers.ticksDetail === undefined
|
|
811
885
|
? `ticks ${layers.ticks}`
|
|
812
886
|
: `ticks ${layers.ticks} (${layers.ticksDetail})`;
|
|
887
|
+
let nextTickLine: string | undefined;
|
|
888
|
+
if (layers.nextTickAt !== undefined) {
|
|
889
|
+
const delta = Date.parse(layers.nextTickAt) - now;
|
|
890
|
+
const minutes = Math.max(1, Math.ceil(Math.abs(delta) / 60_000));
|
|
891
|
+
nextTickLine =
|
|
892
|
+
`next tick ${layers.nextTickAt} ` +
|
|
893
|
+
`(${delta >= 0 ? `in ${minutes}m` : `overdue by ${minutes}m`})`;
|
|
894
|
+
}
|
|
813
895
|
const paneLine =
|
|
814
896
|
layers.paneDetail === undefined
|
|
815
897
|
? `pane ${layers.pane}`
|
|
@@ -822,6 +904,10 @@ export function formatFleetStatus(
|
|
|
822
904
|
layers.herdrDetail === undefined
|
|
823
905
|
? `herdr ${layers.herdr}`
|
|
824
906
|
: `herdr ${layers.herdr} (${layers.herdrDetail})`;
|
|
907
|
+
const telegramLine =
|
|
908
|
+
telegram.detail === undefined
|
|
909
|
+
? `telegram ${telegram.kind}`
|
|
910
|
+
: `telegram ${telegram.kind} (${telegram.detail})`;
|
|
825
911
|
|
|
826
912
|
let daemonBlock: string;
|
|
827
913
|
if (!layers.daemon.running || layers.daemon.pid === undefined) {
|
|
@@ -847,9 +933,11 @@ export function formatFleetStatus(
|
|
|
847
933
|
return [
|
|
848
934
|
`dispatch ${layers.dispatch}`,
|
|
849
935
|
tickLine,
|
|
936
|
+
...(nextTickLine === undefined ? [] : [nextTickLine]),
|
|
850
937
|
paneLine,
|
|
851
938
|
recoveryLine,
|
|
852
939
|
herdrLine,
|
|
940
|
+
telegramLine,
|
|
853
941
|
daemonBlock,
|
|
854
942
|
"",
|
|
855
943
|
formatProjectBody(s),
|
|
@@ -898,10 +986,12 @@ function formatProjectBody(s: StatusSnapshot): string {
|
|
|
898
986
|
export async function renderStatus(projectName?: string): Promise<string> {
|
|
899
987
|
const s = statusSnapshot(projectName);
|
|
900
988
|
const layers = fleetLayers(projectName);
|
|
901
|
-
let health: { ok: boolean; body?: string } | undefined;
|
|
902
989
|
const rec = livingDaemon();
|
|
903
|
-
|
|
904
|
-
|
|
990
|
+
const [health, telegram] = await Promise.all([
|
|
991
|
+
rec === undefined ? undefined : healthCheck(rec.port),
|
|
992
|
+
probeTelegramHealth(projectName),
|
|
993
|
+
]);
|
|
994
|
+
return formatFleetStatus(s, layers, health, telegram);
|
|
905
995
|
}
|
|
906
996
|
|
|
907
997
|
// ---------------------------------------------------------------------------
|
|
@@ -949,6 +1039,57 @@ function readBotToken(): string | undefined {
|
|
|
949
1039
|
return undefined;
|
|
950
1040
|
}
|
|
951
1041
|
|
|
1042
|
+
export async function probeTelegramHealth(
|
|
1043
|
+
projectName?: string,
|
|
1044
|
+
request: (input: string, init?: RequestInit) => Promise<Response> = fetch,
|
|
1045
|
+
): Promise<TelegramHealth> {
|
|
1046
|
+
const tick = resolveTickConfig(projectName);
|
|
1047
|
+
const accessPath =
|
|
1048
|
+
tick.kind === "ok" && tick.config.accessFile !== undefined
|
|
1049
|
+
? tick.config.accessFile
|
|
1050
|
+
: join(telegramStateDir(), "access.json");
|
|
1051
|
+
const channel = readPairedChannel(accessPath);
|
|
1052
|
+
const token = readBotToken();
|
|
1053
|
+
if (token === undefined) {
|
|
1054
|
+
return {
|
|
1055
|
+
kind: "unconfigured",
|
|
1056
|
+
detail: `no TELEGRAM_BOT_TOKEN; inbound ${channel.kind === "up" ? "configured" : channel.reason}`,
|
|
1057
|
+
};
|
|
1058
|
+
}
|
|
1059
|
+
|
|
1060
|
+
let response: Response;
|
|
1061
|
+
let body: unknown;
|
|
1062
|
+
try {
|
|
1063
|
+
response = await request(`https://api.telegram.org/bot${token}/getMe`, {
|
|
1064
|
+
signal: AbortSignal.timeout(5_000),
|
|
1065
|
+
});
|
|
1066
|
+
body = await response.json();
|
|
1067
|
+
} catch {
|
|
1068
|
+
return {
|
|
1069
|
+
kind: "down",
|
|
1070
|
+
detail: `Telegram API request failed; inbound ${channel.kind === "up" ? "configured" : channel.reason}`,
|
|
1071
|
+
};
|
|
1072
|
+
}
|
|
1073
|
+
|
|
1074
|
+
if (body === null || typeof body !== "object" || Array.isArray(body)) {
|
|
1075
|
+
return { kind: "down", detail: "Telegram API returned an invalid response" };
|
|
1076
|
+
}
|
|
1077
|
+
const result = body as Record<string, unknown>;
|
|
1078
|
+
if (!response.ok || result["ok"] !== true) {
|
|
1079
|
+
const description =
|
|
1080
|
+
typeof result["description"] === "string" ? result["description"] : `HTTP ${response.status}`;
|
|
1081
|
+
return { kind: "down", detail: `${description}; inbound ${channel.kind === "up" ? "configured" : channel.reason}` };
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
const user =
|
|
1085
|
+
result["result"] !== null && typeof result["result"] === "object" && !Array.isArray(result["result"])
|
|
1086
|
+
? (result["result"] as Record<string, unknown>)
|
|
1087
|
+
: undefined;
|
|
1088
|
+
const username = typeof user?.["username"] === "string" ? `@${user["username"]}` : "authenticated";
|
|
1089
|
+
if (channel.kind === "down") return { kind: "degraded", detail: `${username}; inbound ${channel.reason}` };
|
|
1090
|
+
return { kind: "ok", detail: `${username}; inbound configured` };
|
|
1091
|
+
}
|
|
1092
|
+
|
|
952
1093
|
export function sessionDirForCwd(cwd: string): string {
|
|
953
1094
|
const home = homedir();
|
|
954
1095
|
const slug = cwd.startsWith(home) ? cwd.slice(home.length) : cwd;
|
|
@@ -1067,33 +1208,63 @@ function probeHerdrUnit(unit = DEFAULT_HERDR_UNIT): { kind: HerdrLayer; detail?:
|
|
|
1067
1208
|
}
|
|
1068
1209
|
}
|
|
1069
1210
|
|
|
1070
|
-
function
|
|
1211
|
+
export function paneLayerFromAgents(
|
|
1212
|
+
agents: HerdrAgent[],
|
|
1213
|
+
agentName: string,
|
|
1214
|
+
):
|
|
1215
|
+
| { kind: "live"; summary: string }
|
|
1216
|
+
| { kind: "missing" }
|
|
1217
|
+
| { kind: "unknown"; reason: string } {
|
|
1218
|
+
const matches = agents.filter((agent) => agent.name === agentName);
|
|
1219
|
+
if (matches.length === 0) return { kind: "missing" };
|
|
1220
|
+
if (matches.length > 1) {
|
|
1221
|
+
return { kind: "unknown", reason: `${matches.length} herdr agents named ${agentName}` };
|
|
1222
|
+
}
|
|
1223
|
+
const match = matches[0]!;
|
|
1224
|
+
if (match.agent === "omp") {
|
|
1225
|
+
return { kind: "live", summary: `agent ${agentName} pane ${match.paneId}` };
|
|
1226
|
+
}
|
|
1227
|
+
if (match.agent === undefined || match.agent === "") return { kind: "missing" };
|
|
1228
|
+
return {
|
|
1229
|
+
kind: "unknown",
|
|
1230
|
+
reason: `agent ${agentName} pane ${match.paneId} is ${match.agent}, not omp`,
|
|
1231
|
+
};
|
|
1232
|
+
}
|
|
1233
|
+
|
|
1234
|
+
function probeOmpPane(
|
|
1235
|
+
projectName?: string,
|
|
1236
|
+
):
|
|
1071
1237
|
| { kind: "live"; summary: string }
|
|
1072
1238
|
| { kind: "missing" }
|
|
1073
1239
|
| { kind: "unknown"; reason: string } {
|
|
1240
|
+
const tick = resolveTickConfig(projectName);
|
|
1241
|
+
if (tick.kind === "invalid") {
|
|
1242
|
+
return {
|
|
1243
|
+
kind: "unknown",
|
|
1244
|
+
reason:
|
|
1245
|
+
`tick config invalid at ${tick.path} (${tick.problem}) — ` +
|
|
1246
|
+
"the conductor agent name cannot be read",
|
|
1247
|
+
};
|
|
1248
|
+
}
|
|
1249
|
+
const agentName =
|
|
1250
|
+
tick.kind === "ok" ? (tick.config.agentName ?? DEFAULT_FLEET_AGENT_NAME) : DEFAULT_FLEET_AGENT_NAME;
|
|
1251
|
+
const session = process.env["HERDR_SESSION"] ?? "fleet";
|
|
1074
1252
|
try {
|
|
1075
|
-
const res = spawnSync("
|
|
1253
|
+
const res = spawnSync("herdr", ["--session", session, "agent", "list"], {
|
|
1076
1254
|
encoding: "utf8",
|
|
1077
|
-
timeout:
|
|
1255
|
+
timeout: 8_000,
|
|
1078
1256
|
env: process.env,
|
|
1079
1257
|
});
|
|
1080
1258
|
if (res.error) {
|
|
1081
1259
|
const err = res.error as NodeJS.ErrnoException;
|
|
1082
|
-
if (err.code === "ENOENT") return { kind: "unknown", reason: "no
|
|
1260
|
+
if (err.code === "ENOENT") return { kind: "unknown", reason: "no herdr" };
|
|
1083
1261
|
return { kind: "unknown", reason: err.message };
|
|
1084
1262
|
}
|
|
1085
|
-
|
|
1086
|
-
.
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
if (lines.length === 0) return { kind: "missing" };
|
|
1091
|
-
const first = lines[0]!;
|
|
1092
|
-
const summary = first.length > 100 ? `${first.slice(0, 97)}…` : first;
|
|
1093
|
-
return {
|
|
1094
|
-
kind: "live",
|
|
1095
|
-
summary: lines.length === 1 ? summary : `${summary} (+${lines.length - 1} more)`,
|
|
1096
|
-
};
|
|
1263
|
+
if (res.status !== 0) {
|
|
1264
|
+
const detail = (res.stderr ?? res.stdout ?? `herdr exit ${String(res.status)}`).trim();
|
|
1265
|
+
return { kind: "unknown", reason: detail };
|
|
1266
|
+
}
|
|
1267
|
+
return paneLayerFromAgents(parseHerdrAgentList(res.stdout ?? ""), agentName);
|
|
1097
1268
|
} catch (err) {
|
|
1098
1269
|
return { kind: "unknown", reason: err instanceof Error ? err.message : String(err) };
|
|
1099
1270
|
}
|
package/src/orchestrator-tick.ts
CHANGED
|
@@ -41,7 +41,7 @@
|
|
|
41
41
|
*/
|
|
42
42
|
|
|
43
43
|
import { spawnSync } from "node:child_process";
|
|
44
|
-
import { existsSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
44
|
+
import { existsSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
45
45
|
import { isAbsolute, join, resolve } from "node:path";
|
|
46
46
|
import { findProject, loadConfig } from "./config.ts";
|
|
47
47
|
import {
|
|
@@ -107,6 +107,43 @@ export const STALL_TICKS = 2;
|
|
|
107
107
|
*/
|
|
108
108
|
export const TICK_REQUESTED_FILE = ".conductor-tick-requested";
|
|
109
109
|
|
|
110
|
+
/** Runtime heartbeat schedule consumed by `omp-conductor status`. */
|
|
111
|
+
export const TICK_STATUS_FILE = ".conductor-tick-status.json";
|
|
112
|
+
|
|
113
|
+
export interface TickRuntimeStatus {
|
|
114
|
+
pid: number;
|
|
115
|
+
intervalSeconds: number;
|
|
116
|
+
nextTickAt: string;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
export function readTickRuntimeStatus(cwd: string): TickRuntimeStatus | undefined {
|
|
120
|
+
let parsed: unknown;
|
|
121
|
+
try {
|
|
122
|
+
parsed = JSON.parse(readFileSync(join(cwd, TICK_STATUS_FILE), "utf8"));
|
|
123
|
+
} catch {
|
|
124
|
+
return undefined;
|
|
125
|
+
}
|
|
126
|
+
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) return undefined;
|
|
127
|
+
const row = parsed as Record<string, unknown>;
|
|
128
|
+
if (
|
|
129
|
+
typeof row["pid"] !== "number" ||
|
|
130
|
+
!Number.isInteger(row["pid"]) ||
|
|
131
|
+
row["pid"] <= 1 ||
|
|
132
|
+
typeof row["intervalSeconds"] !== "number" ||
|
|
133
|
+
!Number.isInteger(row["intervalSeconds"]) ||
|
|
134
|
+
row["intervalSeconds"] < MIN_INTERVAL_SECONDS ||
|
|
135
|
+
typeof row["nextTickAt"] !== "string" ||
|
|
136
|
+
!Number.isFinite(Date.parse(row["nextTickAt"]))
|
|
137
|
+
) {
|
|
138
|
+
return undefined;
|
|
139
|
+
}
|
|
140
|
+
return {
|
|
141
|
+
pid: row["pid"],
|
|
142
|
+
intervalSeconds: row["intervalSeconds"],
|
|
143
|
+
nextTickAt: row["nextTickAt"],
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
|
|
110
147
|
/**
|
|
111
148
|
* The marker's one line after its ISO timestamp, and the middle of the error
|
|
112
149
|
* log. Shared so the file and the log can never describe different failures.
|
|
@@ -971,6 +1008,23 @@ function tick(pi: TickApi, ctx: TickContext, config: TickConfig, session: TickSe
|
|
|
971
1008
|
clearTickRequest(pi, ctx.cwd);
|
|
972
1009
|
}
|
|
973
1010
|
|
|
1011
|
+
function writeTickRuntimeStatus(pi: TickApi, cwd: string, intervalSeconds: number): void {
|
|
1012
|
+
const path = join(cwd, TICK_STATUS_FILE);
|
|
1013
|
+
const tmp = `${path}.${process.pid}.tmp`;
|
|
1014
|
+
const status: TickRuntimeStatus = {
|
|
1015
|
+
pid: process.pid,
|
|
1016
|
+
intervalSeconds,
|
|
1017
|
+
nextTickAt: new Date(Date.now() + intervalSeconds * 1000).toISOString(),
|
|
1018
|
+
};
|
|
1019
|
+
try {
|
|
1020
|
+
writeFileSync(tmp, `${JSON.stringify(status)}\n`);
|
|
1021
|
+
renameSync(tmp, path);
|
|
1022
|
+
} catch (err) {
|
|
1023
|
+
rmSync(tmp, { force: true });
|
|
1024
|
+
pi.logger.error(`[omp-conductor] could not write ${path}: ${err instanceof Error ? err.message : String(err)}`);
|
|
1025
|
+
}
|
|
1026
|
+
}
|
|
1027
|
+
|
|
974
1028
|
/**
|
|
975
1029
|
* Arm the interval heartbeat, then honour a recover poke if one is waiting.
|
|
976
1030
|
* Extracted so the ownership-retry path and the immediate-accept path cannot
|
|
@@ -978,7 +1032,14 @@ function tick(pi: TickApi, ctx: TickContext, config: TickConfig, session: TickSe
|
|
|
978
1032
|
* behaviour.
|
|
979
1033
|
*/
|
|
980
1034
|
function armTickHeartbeat(pi: TickApi, ctx: TickContext, config: TickConfig, session: TickSession): void {
|
|
981
|
-
|
|
1035
|
+
writeTickRuntimeStatus(pi, ctx.cwd, config.intervalSeconds);
|
|
1036
|
+
ctx.setInterval(() => {
|
|
1037
|
+
try {
|
|
1038
|
+
tick(pi, ctx, config, session);
|
|
1039
|
+
} finally {
|
|
1040
|
+
writeTickRuntimeStatus(pi, ctx.cwd, config.intervalSeconds);
|
|
1041
|
+
}
|
|
1042
|
+
}, config.intervalSeconds * 1000);
|
|
982
1043
|
if (!existsSync(join(ctx.cwd, TICK_REQUESTED_FILE))) return;
|
|
983
1044
|
pi.logger.info("[omp-conductor] tick requested by recover — firing without waiting for the interval");
|
|
984
1045
|
tick(pi, ctx, config, session);
|