omp-conductor 0.19.6 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/REFERENCE.md +27 -2
- package/agents/to-spec.md +76 -9
- package/package.json +1 -1
- package/schema/config.schema.json +4 -0
- package/src/arm-challenge.ts +204 -85
- package/src/ask.ts +130 -615
- package/src/board.ts +7 -1
- package/src/brief-upgrade.ts +24 -0
- package/src/briefs/console.md +253 -0
- package/src/briefs/correction.md +203 -0
- package/src/briefs/orchestrator.md +167 -97
- package/src/briefs/policy.md +19 -16
- package/src/briefs/to-spec.md +76 -9
- package/src/briefs/worker.md +50 -16
- package/src/cli.ts +4 -0
- package/src/command-manifest.ts +54 -8
- package/src/commands/arm.ts +113 -49
- package/src/commands/console.ts +70 -0
- package/src/commands/context.ts +2 -0
- package/src/commands/epic.ts +132 -0
- package/src/commands/extend.ts +9 -1
- package/src/commands/intake.ts +44 -14
- package/src/commands/stats.ts +19 -4
- package/src/commands/worker.ts +9 -1
- package/src/config-schema.ts +13 -0
- package/src/config.ts +27 -0
- package/src/daemon/ack.ts +159 -0
- package/src/daemon/admission-pass.ts +135 -0
- package/src/daemon/brief.ts +461 -0
- package/src/daemon/deps.ts +539 -0
- package/src/daemon/dispatch.ts +1779 -0
- package/src/daemon/drain.ts +185 -0
- package/src/daemon/groom-pass.ts +412 -0
- package/src/daemon/http.ts +417 -0
- package/src/daemon/integrity.ts +108 -0
- package/src/daemon/panes.ts +180 -0
- package/src/daemon/review.ts +1888 -0
- package/src/daemon/runtime.ts +736 -0
- package/src/daemon/settle-pass.ts +589 -0
- package/src/daemon/supervision.ts +438 -0
- package/src/daemon/tick.ts +968 -0
- package/src/daemon/views.ts +751 -0
- package/src/daemon.ts +105 -7832
- package/src/dashboard/app.js +58 -0
- package/src/dashboard/controls.ts +22 -3
- package/src/dashboard/server.ts +4 -0
- package/src/diff-flags.ts +24 -3
- package/src/doctor.ts +17 -12
- package/src/escalate.ts +39 -21
- package/src/failure-class.ts +75 -1
- package/src/fleet.ts +1218 -304
- package/src/groom.ts +461 -0
- package/src/http-token.ts +142 -0
- package/src/knowledge.ts +229 -0
- package/src/mining.ts +316 -0
- package/src/orchestrator-tick.ts +428 -1681
- package/src/ready-gate.ts +267 -0
- package/src/settlement.ts +72 -6
- package/src/setup-host.ts +32 -9
- package/src/setup-wizard.ts +55 -7
- package/src/setup.ts +229 -3
- package/src/stats.ts +257 -2
- package/src/status-render.ts +158 -7
- package/src/store.ts +646 -26
- package/src/to-spec.ts +194 -21
- package/src/tracker/github.ts +50 -0
- package/src/types.ts +435 -15
- package/src/verbs/protocol.ts +28 -0
- package/src/verbs/server.ts +384 -12
- package/src/wake.ts +19 -2
- package/src/worker.ts +456 -1
package/src/fleet.ts
CHANGED
|
@@ -27,8 +27,10 @@ import { findProject, loadConfig, resolveArmProof, stateDir } from "./config.ts"
|
|
|
27
27
|
import {
|
|
28
28
|
clearArmTransaction,
|
|
29
29
|
FLEET_ARM_KEY,
|
|
30
|
-
readArmAcknowledgement,
|
|
31
30
|
recordArmChallenge,
|
|
31
|
+
resolveArmReply,
|
|
32
|
+
type ArmReplyVerdict,
|
|
33
|
+
type ArmTarget,
|
|
32
34
|
} from "./arm-challenge.ts";
|
|
33
35
|
import {
|
|
34
36
|
claimedTelegramTopics,
|
|
@@ -283,19 +285,53 @@ export function disarmTicks(projectName?: string): { path: string; wasArmed: boo
|
|
|
283
285
|
return { path, wasArmed };
|
|
284
286
|
}
|
|
285
287
|
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
alreadyArmed: boolean;
|
|
289
|
-
owner: string;
|
|
290
|
-
/** The challenge code that proved arming, present only for `challenge` proof. */
|
|
291
|
-
challenge?: string;
|
|
288
|
+
/** One project whose arm marker was written. */
|
|
289
|
+
export interface ArmedProject {
|
|
292
290
|
/**
|
|
293
|
-
*
|
|
294
|
-
*
|
|
291
|
+
* The configured project name, absent only for a legacy unstamped
|
|
292
|
+
* single-project fleet.
|
|
295
293
|
*/
|
|
294
|
+
project?: string;
|
|
295
|
+
path: string;
|
|
296
|
+
alreadyArmed: boolean;
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
/** Markers written without any reply: the `claim-only` policy proof (#613). */
|
|
300
|
+
export interface ArmMarkersWritten {
|
|
301
|
+
outcome: "armed";
|
|
296
302
|
proof: ArmProof;
|
|
303
|
+
owner: string;
|
|
304
|
+
armed: ArmedProject[];
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
/**
|
|
308
|
+
* A challenge filed and sent, with nothing armed yet.
|
|
309
|
+
*
|
|
310
|
+
* Arming used to block here for up to five minutes on an in-session
|
|
311
|
+
* acknowledgement. With the console owning the operator DM, the reply lands in
|
|
312
|
+
* a session that runs no tick extension, so that wait could never be satisfied
|
|
313
|
+
* — it is now two mechanical steps, and this is the first one's receipt.
|
|
314
|
+
*/
|
|
315
|
+
export interface ArmChallengeSent {
|
|
316
|
+
outcome: "challenge-sent";
|
|
317
|
+
/** Always `challenge`: `claim-only` never sends, so it never reaches here. */
|
|
318
|
+
proof: "challenge";
|
|
319
|
+
/** The chat the challenge went to, recorded with the transaction. */
|
|
320
|
+
owner: string;
|
|
321
|
+
/** The pending transaction the reply must prove. */
|
|
322
|
+
challengeId: string;
|
|
323
|
+
/** Unix ms after which the code stops being a proof. */
|
|
324
|
+
expiresAt: number;
|
|
325
|
+
/** mm:ss the code stays good for — the same clock the message quotes. */
|
|
326
|
+
validFor: string;
|
|
327
|
+
/** Exactly what a matching reply will arm, as recorded with the challenge. */
|
|
328
|
+
targets: ArmTarget[];
|
|
329
|
+
/** The verbatim command that completes the ceremony, copy-pasteable as-is. */
|
|
330
|
+
followUp: string;
|
|
297
331
|
}
|
|
298
332
|
|
|
333
|
+
export type ArmResult = ArmMarkersWritten | ArmChallengeSent;
|
|
334
|
+
|
|
299
335
|
export interface ArmDeps {
|
|
300
336
|
sendChallenge?: (token: string, owner: string, text: string, topicId?: number) => Promise<void>;
|
|
301
337
|
/**
|
|
@@ -307,7 +343,7 @@ export interface ArmDeps {
|
|
|
307
343
|
*/
|
|
308
344
|
claimedSessionFile?: () => string | undefined;
|
|
309
345
|
now?: () => number;
|
|
310
|
-
|
|
346
|
+
/** How long a sent code stays a proof. */
|
|
311
347
|
timeoutMs?: number;
|
|
312
348
|
/**
|
|
313
349
|
* Liveness seams for the claim-only verdict (#613), with omp-telegram's own
|
|
@@ -321,11 +357,10 @@ export interface ArmDeps {
|
|
|
321
357
|
lockPidAlive?: (pid: number) => boolean;
|
|
322
358
|
lockFresh?: (mtimeMs: number) => boolean;
|
|
323
359
|
/**
|
|
324
|
-
* Where the
|
|
325
|
-
*
|
|
326
|
-
*
|
|
327
|
-
*
|
|
328
|
-
* (the CLI, the wizard) pass theirs.
|
|
360
|
+
* Where the ceremony's one-line narration goes. Nothing waits any more, so
|
|
361
|
+
* this is no longer a liveness heartbeat (#861) — it is the send receipt and
|
|
362
|
+
* the follow-up instruction, for a surface that prints as it goes. Absent
|
|
363
|
+
* means no reporting.
|
|
329
364
|
*/
|
|
330
365
|
progress?: (line: string) => void;
|
|
331
366
|
}
|
|
@@ -367,31 +402,32 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
|
|
|
367
402
|
// one pane that can ask. The challenge names which one, or the operator is
|
|
368
403
|
// answering a question they cannot attribute.
|
|
369
404
|
const named = tick.config.project ?? projectName;
|
|
370
|
-
// The handshake state key must be exactly what the
|
|
371
|
-
//
|
|
372
|
-
// config. `named` may fall back to the CLI argument for the
|
|
373
|
-
// and config lookups; the state key must not
|
|
374
|
-
//
|
|
375
|
-
// an acknowledgement that can never be written.
|
|
405
|
+
// The handshake state key must be exactly what the reply step recomputes
|
|
406
|
+
// from this same tick config: TickConfig.project, undefined for a legacy
|
|
407
|
+
// unstamped config. `named` may fall back to the CLI argument for the
|
|
408
|
+
// challenge text and config lookups; the state key must not, or the reply
|
|
409
|
+
// would look for the challenge under a key nothing recorded.
|
|
376
410
|
const stateKey = tick.config.project;
|
|
377
411
|
|
|
378
412
|
// The arming proof is a declared per-project policy (#613). A config that
|
|
379
|
-
// cannot name the project fails safe to `challenge` —
|
|
413
|
+
// cannot name the project fails safe to `challenge` — the authenticated
|
|
380
414
|
// round-trip — so a missing or unreadable config never silently weakens the
|
|
381
415
|
// gate.
|
|
382
416
|
let proof: ArmProof = DEFAULT_ARM_PROOF;
|
|
383
417
|
try {
|
|
384
418
|
proof = resolveArmProof(findProject(loadConfig(), named));
|
|
385
419
|
} catch {
|
|
386
|
-
/* no project config — keep
|
|
420
|
+
/* no project config — keep the challenge behaviour */
|
|
387
421
|
}
|
|
388
422
|
|
|
389
423
|
// The orchestrator's live session file per omp-telegram's claim (#600) —
|
|
390
424
|
// input to the claim-only verdict's session-identity checks below. The
|
|
391
|
-
// challenge proof never reads it: its
|
|
392
|
-
//
|
|
393
|
-
|
|
394
|
-
|
|
425
|
+
// challenge proof never reads it: its proof is conductor state, so where (or
|
|
426
|
+
// whether) a transcript lives is no longer part of arming (#614).
|
|
427
|
+
const claimed =
|
|
428
|
+
deps.claimedSessionFile !== undefined
|
|
429
|
+
? deps.claimedSessionFile()
|
|
430
|
+
: claimedOrchestratorSessionFile(named, deps.pidAlive ?? pidAlive);
|
|
395
431
|
|
|
396
432
|
// Prefer the project's live forum topic so arm challenges land where
|
|
397
433
|
// escalations already do (#318), following the bridge's current claim when the
|
|
@@ -399,24 +435,19 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
|
|
|
399
435
|
let sendTopic: number | undefined;
|
|
400
436
|
if (named !== undefined) {
|
|
401
437
|
try {
|
|
402
|
-
sendTopic = resolveProjectTopicId(findProject(loadConfig(), named));
|
|
438
|
+
sendTopic = resolveProjectTopicId(findProject(loadConfig(), named), deps.pidAlive ?? pidAlive);
|
|
403
439
|
} catch {
|
|
404
440
|
/* no project config */
|
|
405
441
|
}
|
|
406
442
|
}
|
|
407
443
|
|
|
408
444
|
const path = tick.config.armedFile;
|
|
409
|
-
// The gate as the heartbeat reads it, so "replaced previous marker" is not a
|
|
410
|
-
// lie about a fleet the shared marker was arming, and so the write below knows
|
|
411
|
-
// whether it is superseding that marker.
|
|
412
|
-
const arm = resolveArmState(path, named);
|
|
413
|
-
const alreadyArmed = arm.armed;
|
|
414
445
|
|
|
415
446
|
if (proof === "claim-only") {
|
|
416
447
|
// The human-intent gate is declared satisfied by policy, so #612's shared
|
|
417
448
|
// verdict is the whole proof: the same state reads and the same liveness
|
|
418
449
|
// rules the doctor's "telegram-plumbing" finding applies, on the route a
|
|
419
|
-
// challenge would have ridden. No Telegram send, no
|
|
450
|
+
// challenge would have ridden. No Telegram send, no reply step, no
|
|
420
451
|
// pending-challenge record. A failed fact refuses arming by name — never
|
|
421
452
|
// a silent pass from file existence, and never a marker.
|
|
422
453
|
const scan = armVerdictScanDirs(tick.cwd, claimed);
|
|
@@ -439,13 +470,23 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
|
|
|
439
470
|
`NOT armed; no marker was written`,
|
|
440
471
|
);
|
|
441
472
|
}
|
|
473
|
+
// The gate as the heartbeat reads it, so "replaced previous marker" is not
|
|
474
|
+
// a lie about a fleet the shared marker was arming, and so the write knows
|
|
475
|
+
// whether it is superseding that marker.
|
|
476
|
+
const arm = resolveArmState(path, named);
|
|
442
477
|
writeArmedMarker(path, channel.owner, arm);
|
|
443
|
-
return {
|
|
478
|
+
return {
|
|
479
|
+
outcome: "armed",
|
|
480
|
+
proof,
|
|
481
|
+
owner: channel.owner,
|
|
482
|
+
armed: [{ ...(named === undefined ? {} : { project: named }), path, alreadyArmed: arm.armed }],
|
|
483
|
+
};
|
|
444
484
|
}
|
|
445
485
|
|
|
446
486
|
const send = deps.sendChallenge ?? sendTelegramMessage;
|
|
447
487
|
const timeoutMs = deps.timeoutMs ?? ARM_CHALLENGE_TIMEOUT_MS;
|
|
448
488
|
const code = makeChallengeCode();
|
|
489
|
+
const followUp = armReplyCommand(projectName);
|
|
449
490
|
// Self-describing (#991): with two live challenges in one chat the operator
|
|
450
491
|
// was working out which was which from message order, and nothing said how
|
|
451
492
|
// long a code stayed good — so the safe move was to scroll for the newest,
|
|
@@ -454,15 +495,18 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
|
|
|
454
495
|
`Fleet arming check${named === undefined ? "" : ` — project ${named}`}. ` +
|
|
455
496
|
`Reply to this chat with exactly:\n${code}\n` +
|
|
456
497
|
`Valid for ${armClock(timeoutMs)}. ` +
|
|
457
|
-
`Nothing
|
|
498
|
+
`Nothing is dispatched until that reply is verified.`;
|
|
458
499
|
const sentAt = (deps.now ?? Date.now)();
|
|
459
|
-
//
|
|
460
|
-
// challenge
|
|
461
|
-
//
|
|
462
|
-
//
|
|
463
|
-
//
|
|
464
|
-
// a
|
|
465
|
-
const challengeId = recordArmChallenge(stateKey, code, sentAt, sentAt + timeoutMs
|
|
500
|
+
// Recorded BEFORE the send, so a reply that beats this process's own return
|
|
501
|
+
// still finds an active challenge — and so a send that fails has a
|
|
502
|
+
// transaction to settle rather than a code loose in a chat (#415). The record
|
|
503
|
+
// carries what the reply will arm: the config could change before the
|
|
504
|
+
// operator answers, and arming anything but what the challenge named would
|
|
505
|
+
// arm a fleet nobody was asked about.
|
|
506
|
+
const challengeId = recordArmChallenge(stateKey, code, sentAt, sentAt + timeoutMs, {
|
|
507
|
+
targets: [{ ...(named === undefined ? {} : { project: named }), armedFile: path }],
|
|
508
|
+
owner: channel.owner,
|
|
509
|
+
});
|
|
466
510
|
try {
|
|
467
511
|
await send(token, channel.owner, text, sendTopic);
|
|
468
512
|
} catch (err) {
|
|
@@ -473,50 +517,33 @@ export async function armTicks(projectName?: string, deps: ArmDeps = {}): Promis
|
|
|
473
517
|
`arm: outbound sendMessage failed — NOT armed: ${err instanceof Error ? err.message : String(err)}`,
|
|
474
518
|
);
|
|
475
519
|
}
|
|
476
|
-
// What is being waited on, before the wait starts: the window, the chat the
|
|
477
|
-
// reply has to land in, and the fact that dispatch is held until it does
|
|
478
|
-
// (#861). One line, so a five-minute wait opens with an explanation rather
|
|
479
|
-
// than with silence.
|
|
480
520
|
deps.progress?.(
|
|
481
521
|
`arm: challenge sent to ${channel.owner}${sendTopic === undefined ? "" : ` (topic ${sendTopic})`} — ` +
|
|
482
|
-
`
|
|
522
|
+
`valid for ${armClock(timeoutMs)}. Nothing is armed yet: run ${followUp}`,
|
|
483
523
|
);
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
`arm: the challenge was never acknowledged in time — NOT armed.\n` +
|
|
495
|
-
`The orchestrator's inbound adapter acknowledges the reply when it lands as a user turn; ` +
|
|
496
|
-
`no acknowledgement for challenge ${challengeId} arrived.\n` +
|
|
497
|
-
`Inbound Telegram is not reaching the omp session. Check, in order:\n` +
|
|
498
|
-
` * is the bridge polling? attach and run: /telegram status\n` +
|
|
499
|
-
` * is another process holding this bot token? Telegram allows exactly one\n` +
|
|
500
|
-
` getUpdates consumer and rejects the second with HTTP 409.\n` +
|
|
501
|
-
` * did you reply in the DM with the bot, not another chat?\n`,
|
|
502
|
-
);
|
|
503
|
-
}
|
|
504
|
-
|
|
505
|
-
writeArmedMarker(path, channel.owner, arm);
|
|
506
|
-
// The acknowledgement landed and this project is armed: settling clears this
|
|
507
|
-
// transaction's pending record and acknowledgement — never a newer
|
|
508
|
-
// replacement's — so a later unsolicited lookalike stays inert past this
|
|
509
|
-
// handshake.
|
|
510
|
-
clearArmTransaction(stateKey, challengeId);
|
|
511
|
-
return { path, alreadyArmed, owner: channel.owner, challenge: code, proof };
|
|
524
|
+
return {
|
|
525
|
+
outcome: "challenge-sent",
|
|
526
|
+
proof: "challenge",
|
|
527
|
+
owner: channel.owner,
|
|
528
|
+
challengeId,
|
|
529
|
+
expiresAt: sentAt + timeoutMs,
|
|
530
|
+
validFor: armClock(timeoutMs),
|
|
531
|
+
targets: [{ ...(named === undefined ? {} : { project: named }), armedFile: path }],
|
|
532
|
+
followUp,
|
|
533
|
+
};
|
|
512
534
|
}
|
|
513
535
|
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
536
|
+
/**
|
|
537
|
+
* The second half of the ceremony, verbatim. Printed by every surface that
|
|
538
|
+
* issues a challenge, and quoted in the refusals, because an agent reading a
|
|
539
|
+
* console pane has to be able to paste it without composing anything: the
|
|
540
|
+
* operator's message is the only variable.
|
|
541
|
+
*/
|
|
542
|
+
function armReplyCommand(projectName?: string): string {
|
|
543
|
+
return (
|
|
544
|
+
`omp-conductor arm --reply "<the operator's reply, verbatim>"` +
|
|
545
|
+
`${projectName === undefined ? "" : ` --project ${projectName}`}`
|
|
546
|
+
);
|
|
520
547
|
}
|
|
521
548
|
|
|
522
549
|
/**
|
|
@@ -524,10 +551,9 @@ export interface FleetArmResult {
|
|
|
524
551
|
*
|
|
525
552
|
* Arming was per project: a two-project fleet meant two sequential handshakes
|
|
526
553
|
* with two codes in one chat, though nothing about the fleet's state differed
|
|
527
|
-
* between them. This sends **one** challenge
|
|
528
|
-
*
|
|
529
|
-
*
|
|
530
|
-
* current friction with one command wrapped around it.
|
|
554
|
+
* between them. This sends **one** challenge whose single matching reply arms
|
|
555
|
+
* every configured project — one fleet-wide pending record, not a loop that
|
|
556
|
+
* sends N challenges and collects N replies.
|
|
531
557
|
*
|
|
532
558
|
* Every project is validated before anything is sent, so a fleet whose second
|
|
533
559
|
* project has no `armedFile` refuses the ceremony instead of arming the first
|
|
@@ -538,11 +564,14 @@ export interface FleetArmResult {
|
|
|
538
564
|
* authenticated round-trip is strictly stronger than the plumbing verdict that
|
|
539
565
|
* policy would have accepted, so satisfying the weaker gate with the stronger
|
|
540
566
|
* proof cannot weaken it.
|
|
567
|
+
*
|
|
568
|
+
* Like {@link armTicks}, this returns as soon as the challenge is filed and
|
|
569
|
+
* sent: `omp-conductor arm --reply` writes the markers.
|
|
541
570
|
*/
|
|
542
571
|
export async function armFleet(
|
|
543
572
|
projectNames: readonly string[],
|
|
544
573
|
deps: ArmDeps = {},
|
|
545
|
-
): Promise<
|
|
574
|
+
): Promise<ArmChallengeSent> {
|
|
546
575
|
if (projectNames.length === 0) throw new Error("arm: no projects are configured");
|
|
547
576
|
|
|
548
577
|
// Resolve and validate every project first. Nothing is sent and no marker is
|
|
@@ -593,8 +622,8 @@ export async function armFleet(
|
|
|
593
622
|
|
|
594
623
|
// The first project that resolves a live topic carries the ceremony, and the
|
|
595
624
|
// message says so: a fleet-wide question still has to land somewhere an
|
|
596
|
-
// operator is reading, and
|
|
597
|
-
//
|
|
625
|
+
// operator is reading, and the reply step reaches the record from any project
|
|
626
|
+
// because it is fleet-wide rather than topic-scoped.
|
|
598
627
|
let sendTopic: number | undefined;
|
|
599
628
|
for (const target of targets) {
|
|
600
629
|
try {
|
|
@@ -609,14 +638,28 @@ export async function armFleet(
|
|
|
609
638
|
const timeoutMs = deps.timeoutMs ?? ARM_CHALLENGE_TIMEOUT_MS;
|
|
610
639
|
const code = makeChallengeCode();
|
|
611
640
|
const names = targets.map((t) => t.project).join(", ");
|
|
641
|
+
// No `--project`: the reply resolves the fleet record, which is the one this
|
|
642
|
+
// ceremony wrote. Naming a project here would be a narrower command than the
|
|
643
|
+
// challenge the operator answered.
|
|
644
|
+
const followUp = armReplyCommand();
|
|
612
645
|
const text =
|
|
613
646
|
`Fleet arming check — ${String(targets.length)} project(s): ${names}. ` +
|
|
614
647
|
`Reply to this chat with exactly:\n${code}\n` +
|
|
615
648
|
`Valid for ${armClock(timeoutMs)}, and one reply arms all of them. ` +
|
|
616
|
-
`Nothing
|
|
649
|
+
`Nothing is dispatched until that reply is verified.`;
|
|
617
650
|
|
|
618
651
|
const sentAt = (deps.now ?? Date.now)();
|
|
619
|
-
|
|
652
|
+
// The record carries every project the message named, so the reply arms
|
|
653
|
+
// exactly the fleet the operator was asked about — not whatever the config
|
|
654
|
+
// says minutes later.
|
|
655
|
+
const armTargets: ArmTarget[] = targets.map((target) => ({
|
|
656
|
+
project: target.project,
|
|
657
|
+
armedFile: target.armedFile,
|
|
658
|
+
}));
|
|
659
|
+
const challengeId = recordArmChallenge(FLEET_ARM_KEY, code, sentAt, sentAt + timeoutMs, {
|
|
660
|
+
targets: armTargets,
|
|
661
|
+
owner: channel.owner,
|
|
662
|
+
});
|
|
620
663
|
try {
|
|
621
664
|
await send(token, channel.owner, text, sendTopic);
|
|
622
665
|
} catch (err) {
|
|
@@ -627,31 +670,169 @@ export async function armFleet(
|
|
|
627
670
|
}
|
|
628
671
|
deps.progress?.(
|
|
629
672
|
`arm: one challenge sent to ${channel.owner}${sendTopic === undefined ? "" : ` (topic ${sendTopic})`} for ${names} — ` +
|
|
630
|
-
`
|
|
673
|
+
`valid for ${armClock(timeoutMs)}. Nothing is armed yet: run ${followUp}`,
|
|
631
674
|
);
|
|
675
|
+
return {
|
|
676
|
+
outcome: "challenge-sent",
|
|
677
|
+
proof: "challenge",
|
|
678
|
+
owner: channel.owner,
|
|
679
|
+
challengeId,
|
|
680
|
+
expiresAt: sentAt + timeoutMs,
|
|
681
|
+
validFor: armClock(timeoutMs),
|
|
682
|
+
targets: armTargets,
|
|
683
|
+
followUp,
|
|
684
|
+
};
|
|
685
|
+
}
|
|
632
686
|
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
` getUpdates consumer and rejects the second with HTTP 409.\n` +
|
|
642
|
-
` * did you reply in the chat the challenge names, not another one?\n`,
|
|
643
|
-
);
|
|
644
|
-
}
|
|
687
|
+
/** What a reply that armed nothing was, and what to do about it. */
|
|
688
|
+
export interface ArmReplyRefused {
|
|
689
|
+
outcome: "refused";
|
|
690
|
+
/** Never `matched`: a match is the armed outcome. */
|
|
691
|
+
verdict: Exclude<ArmReplyVerdict, "matched">;
|
|
692
|
+
/** One operator-facing sentence naming the verdict and the next move. */
|
|
693
|
+
message: string;
|
|
694
|
+
}
|
|
645
695
|
|
|
646
|
-
|
|
647
|
-
|
|
696
|
+
export interface ArmReplyAccepted {
|
|
697
|
+
outcome: "armed";
|
|
698
|
+
/** The chat the proved challenge was sent to — the marker's `owner=`. */
|
|
699
|
+
owner: string;
|
|
700
|
+
/** The transaction the reply proved and this call settled. */
|
|
701
|
+
challengeId: string;
|
|
702
|
+
/** Every project the challenge recorded, now armed. */
|
|
703
|
+
armed: ArmedProject[];
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
export type ArmReplyResult = ArmReplyAccepted | ArmReplyRefused;
|
|
707
|
+
|
|
708
|
+
/**
|
|
709
|
+
* The verification half of the ceremony: classify the operator's verbatim
|
|
710
|
+
* message and, on a match, arm exactly the projects the challenge recorded.
|
|
711
|
+
*
|
|
712
|
+
* This runs in the console session, which is where the operator's reply lands
|
|
713
|
+
* now that it owns the Telegram DM. No session waits for anything: the
|
|
714
|
+
* challenge is durable state, so the two halves are ordinary commands that can
|
|
715
|
+
* run minutes apart in different processes.
|
|
716
|
+
*
|
|
717
|
+
* The security properties are the send half's, unchanged. Only the project's
|
|
718
|
+
* own record and the fleet record are read (no other project's ceremony can be
|
|
719
|
+
* revealed or consumed), an expired record is never a proof, and a marker is
|
|
720
|
+
* written only for a `matched` verdict — every other verdict returns a refusal
|
|
721
|
+
* having written nothing.
|
|
722
|
+
*/
|
|
723
|
+
export function armReply(
|
|
724
|
+
replyText: string,
|
|
725
|
+
projectName?: string,
|
|
726
|
+
deps: Pick<ArmDeps, "now"> = {},
|
|
727
|
+
): ArmReplyResult {
|
|
728
|
+
const tick = resolveTickConfig(projectName);
|
|
729
|
+
if (tick.kind === "invalid") {
|
|
730
|
+
// Every derivation below would come from this file. A config that does not
|
|
731
|
+
// parse cannot be trusted to name a state key or a marker.
|
|
732
|
+
throw new Error(`tick config invalid at ${tick.path}: ${tick.problem}`);
|
|
733
|
+
}
|
|
734
|
+
// An absent config is not fatal here, which is the difference between the two
|
|
735
|
+
// halves: a fleet ceremony's record carries its own targets, and on a
|
|
736
|
+
// multi-project host the search roots for an unnamed project resolve no
|
|
737
|
+
// config at all (the fleet cwds each carry their own). Refusing here would
|
|
738
|
+
// make `arm --reply` unusable for exactly the ceremony that needs it most.
|
|
739
|
+
const config = tick.kind === "ok" ? tick.config : undefined;
|
|
740
|
+
// The same derivation the send half used, from the same file: the state key
|
|
741
|
+
// is TickConfig.project (undefined for a legacy unstamped config), never the
|
|
742
|
+
// CLI argument, or the reply would look under a key nothing recorded.
|
|
743
|
+
const stateKey = config?.project;
|
|
744
|
+
const now = (deps.now ?? Date.now)();
|
|
745
|
+
const resolved = resolveArmReply(stateKey, replyText, now);
|
|
746
|
+
if (resolved.verdict !== "matched") {
|
|
747
|
+
return {
|
|
748
|
+
outcome: "refused",
|
|
749
|
+
verdict: resolved.verdict,
|
|
750
|
+
message: armRefusalText(resolved.verdict, projectName),
|
|
751
|
+
};
|
|
752
|
+
}
|
|
753
|
+
const match = resolved.match;
|
|
754
|
+
// Recorded targets are the contract. A record written before targets existed
|
|
755
|
+
// (one release of overlap) can only be read as naming the project it is keyed
|
|
756
|
+
// under — or, for the fleet record, the project this command was given, since
|
|
757
|
+
// re-deriving the fleet from today's config could arm a project the operator
|
|
758
|
+
// was never asked about.
|
|
759
|
+
const targets: ArmTarget[] = match.targets ?? [legacyArmTarget(config, projectName)];
|
|
760
|
+
// The owner the challenge was actually sent to. Only a pre-targets record
|
|
761
|
+
// lacks it, and then the paired channel is the only other honest source.
|
|
762
|
+
const owner = match.owner ?? pairedChannelOwner(config?.accessFile);
|
|
648
763
|
const armed = targets.map((target) => {
|
|
649
764
|
const state = resolveArmState(target.armedFile, target.project);
|
|
650
|
-
writeArmedMarker(target.armedFile,
|
|
651
|
-
return {
|
|
765
|
+
writeArmedMarker(target.armedFile, owner, state);
|
|
766
|
+
return {
|
|
767
|
+
...(target.project === undefined ? {} : { project: target.project }),
|
|
768
|
+
path: target.armedFile,
|
|
769
|
+
alreadyArmed: state.armed,
|
|
770
|
+
};
|
|
652
771
|
});
|
|
653
|
-
|
|
654
|
-
|
|
772
|
+
// Settled by id against the key that matched, so a replayed reply cannot arm
|
|
773
|
+
// a second time and a newer replacement's record is never removed.
|
|
774
|
+
clearArmTransaction(match.key, match.id);
|
|
775
|
+
return { outcome: "armed", owner, challengeId: match.id, armed };
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
/**
|
|
779
|
+
* What a pre-targets pending record arms. The tick config that resolved the
|
|
780
|
+
* state key also names this project's marker, which is exactly what the send
|
|
781
|
+
* half would have recorded. Without such a config there is nothing to fall back
|
|
782
|
+
* to, and guessing a marker path is not an option — a challenge from before the
|
|
783
|
+
* upgrade is simply re-run.
|
|
784
|
+
*/
|
|
785
|
+
function legacyArmTarget(config: TickConfig | undefined, projectName?: string): ArmTarget {
|
|
786
|
+
const named = config?.project ?? projectName;
|
|
787
|
+
if (config?.armedFile === undefined) {
|
|
788
|
+
throw new Error(
|
|
789
|
+
`arm: the pending challenge names no targets and no readable ${TICK_CONFIG_FILE} names an armedFile — ` +
|
|
790
|
+
`nothing to arm; re-run \`omp-conductor arm\` for a challenge that records its own targets`,
|
|
791
|
+
);
|
|
792
|
+
}
|
|
793
|
+
return { ...(named === undefined ? {} : { project: named }), armedFile: config.armedFile };
|
|
794
|
+
}
|
|
795
|
+
|
|
796
|
+
/** The paired owner, for the one record shape that does not carry its own. */
|
|
797
|
+
function pairedChannelOwner(accessFile: string | undefined): string {
|
|
798
|
+
if (accessFile === undefined) {
|
|
799
|
+
throw new Error(`${TICK_CONFIG_FILE} has no accessFile — the challenge's owner cannot be established`);
|
|
800
|
+
}
|
|
801
|
+
const channel = readPairedChannel(accessFile);
|
|
802
|
+
if (channel.kind === "down") {
|
|
803
|
+
throw new Error(
|
|
804
|
+
`escalation channel is not up (${accessFile}): ${channel.reason} — ` +
|
|
805
|
+
`this challenge predates owner recording, so the marker cannot name whom it armed`,
|
|
806
|
+
);
|
|
807
|
+
}
|
|
808
|
+
return channel.owner;
|
|
809
|
+
}
|
|
810
|
+
|
|
811
|
+
/**
|
|
812
|
+
* Each non-matching verdict in the operator's words, with the next move. A
|
|
813
|
+
* silent no-op was the old failure mode: the operator could not tell a wrong
|
|
814
|
+
* code from an expired one from a command that never looked.
|
|
815
|
+
*/
|
|
816
|
+
function armRefusalText(verdict: Exclude<ArmReplyVerdict, "matched">, projectName?: string): string {
|
|
817
|
+
const reissue = `omp-conductor arm${projectName === undefined ? "" : ` --project ${projectName}`}`;
|
|
818
|
+
switch (verdict) {
|
|
819
|
+
case "expired":
|
|
820
|
+
return (
|
|
821
|
+
`arm: that code belongs to a challenge whose window has closed — NOT armed, no marker written. ` +
|
|
822
|
+
`An expired code is never a proof: send a fresh challenge with \`${reissue}\` and verify that one.`
|
|
823
|
+
);
|
|
824
|
+
case "unknown":
|
|
825
|
+
return (
|
|
826
|
+
`arm: that message carries an arming code, but no live challenge here matches it — NOT armed, ` +
|
|
827
|
+
`no marker written. Only this project's own challenge and the fleet ceremony are ever consulted. ` +
|
|
828
|
+
`Send a challenge with \`${reissue}\` and reply to that one.`
|
|
829
|
+
);
|
|
830
|
+
case "none":
|
|
831
|
+
return (
|
|
832
|
+
`arm: that message contains no arming code at all — NOT armed, no marker written. ` +
|
|
833
|
+
`Pass the operator's reply verbatim, including the FLEET-… code; if none was sent, run \`${reissue}\` first.`
|
|
834
|
+
);
|
|
835
|
+
}
|
|
655
836
|
}
|
|
656
837
|
|
|
657
838
|
export interface HoldResult {
|
|
@@ -907,10 +1088,17 @@ export interface WorkerPaneIdentity {
|
|
|
907
1088
|
sessionFile?: string;
|
|
908
1089
|
}
|
|
909
1090
|
|
|
910
|
-
/** A tracked pane, or the explicit reason there is none. Never a silent claim.
|
|
1091
|
+
/** A tracked pane, or the explicit reason there is none. Never a silent claim.
|
|
1092
|
+
* The tracked variant names the marked workspace the representation lives in,
|
|
1093
|
+
* so a caller that just created one can protect it from the same pass's
|
|
1094
|
+
* empty-workspace cleanup. `phase` names the step that refused — `"split"`
|
|
1095
|
+
* marks the one failure the re-establishment budget spends whole (#998): a
|
|
1096
|
+
* terminal that refuses to split will refuse every retry within the pass, so
|
|
1097
|
+
* burning the budget once beats leaking an attempt per reconciliation pass
|
|
1098
|
+
* against a wall. */
|
|
911
1099
|
export type WorkerPaneOutcome =
|
|
912
|
-
| { kind: "tracked"; paneId: string; label: string; pid: number }
|
|
913
|
-
| { kind: "unavailable"; reason: string };
|
|
1100
|
+
| { kind: "tracked"; paneId: string; label: string; pid: number; workspaceId?: string }
|
|
1101
|
+
| { kind: "unavailable"; reason: string; phase?: "split" };
|
|
914
1102
|
|
|
915
1103
|
/** One `herdr` invocation, injected so every path is testable with no terminal. */
|
|
916
1104
|
export type HerdrRun = (args: readonly string[]) => { ok: boolean; stdout: string; stderr: string };
|
|
@@ -920,6 +1108,9 @@ export interface WorkerPaneDeps {
|
|
|
920
1108
|
session?: string;
|
|
921
1109
|
/** The follower command the pane displays; the CLI by default. */
|
|
922
1110
|
viewer?: (identity: WorkerPaneIdentity) => readonly string[];
|
|
1111
|
+
/** The durable ownership half of workspace discovery (#1035 review);
|
|
1112
|
+
* production wires the conductor store, tests a recorder. */
|
|
1113
|
+
ownership?: WorkspaceOwnership;
|
|
923
1114
|
}
|
|
924
1115
|
|
|
925
1116
|
/**
|
|
@@ -932,9 +1123,13 @@ export interface WorkerPaneDeps {
|
|
|
932
1123
|
/**
|
|
933
1124
|
* The external-supervisor source every conductor pane report carries.
|
|
934
1125
|
*
|
|
935
|
-
*
|
|
936
|
-
*
|
|
937
|
-
*
|
|
1126
|
+
* Written onto every pane conductor reports against. It is no longer what
|
|
1127
|
+
* proves a pane is conductor's: production repeatedly handed back panes whose
|
|
1128
|
+
* `agent_session` record Herdr had dropped (#1035), so filtering by reported
|
|
1129
|
+
* source quietly exempted exactly the panes that needed cleaning. Ownership is
|
|
1130
|
+
* structural now — conductor creates representations only inside a workspace
|
|
1131
|
+
* it marked itself (see {@link WORKER_WORKSPACE_TOKEN}) — and this source
|
|
1132
|
+
* remains on the reports as the display identity of who speaks for a pane.
|
|
938
1133
|
*/
|
|
939
1134
|
export const WORKER_PANE_SOURCE = "omp-conductor";
|
|
940
1135
|
|
|
@@ -950,6 +1145,26 @@ export const WORKER_PANE_SOURCE = "omp-conductor";
|
|
|
950
1145
|
*/
|
|
951
1146
|
export const PANE_REATTEMPT_MAX = 3;
|
|
952
1147
|
|
|
1148
|
+
/**
|
|
1149
|
+
* The workspace metadata token that marks a workspace as conductor's
|
|
1150
|
+
* per-project worker surface (#1035).
|
|
1151
|
+
*
|
|
1152
|
+
* Written once at creation (`workspace report-metadata --token
|
|
1153
|
+
* conductor_workers=<project>`) and read back from `workspace list`, so
|
|
1154
|
+
* rediscovery survives daemon and Herdr restarts without trusting the label,
|
|
1155
|
+
* the sidebar position, or whichever workspace happens to be focused. Only
|
|
1156
|
+
* panes inside a marked workspace are ever reconciled here, which is what makes
|
|
1157
|
+
* the operator's own panes invisible to this surface by construction — and
|
|
1158
|
+
* what makes every pane inside a marked workspace conductor's, however Herdr
|
|
1159
|
+
* feels about reporting `agent_session` back.
|
|
1160
|
+
*/
|
|
1161
|
+
export const WORKER_WORKSPACE_TOKEN = "conductor_workers";
|
|
1162
|
+
|
|
1163
|
+
/** The dedicated sibling workspace's display label. Never identity. */
|
|
1164
|
+
export function workerWorkspaceLabel(project: string): string {
|
|
1165
|
+
return `${project}-workers`;
|
|
1166
|
+
}
|
|
1167
|
+
|
|
953
1168
|
export function workerPaneLabel(identity: WorkerPaneIdentity): string {
|
|
954
1169
|
return `worker-${identity.project}-${identity.issue}-a${identity.attempt}-${identity.runId.slice(0, 8)}`;
|
|
955
1170
|
}
|
|
@@ -997,41 +1212,402 @@ function parsePaneId(stdout: string): string | undefined {
|
|
|
997
1212
|
return typeof paneId === "string" && paneId.trim() !== "" ? paneId.trim() : undefined;
|
|
998
1213
|
}
|
|
999
1214
|
|
|
1215
|
+
/**
|
|
1216
|
+
* The workspace and root-pane ids inside `herdr workspace create`'s answer
|
|
1217
|
+
* (`{"id":"cli:workspace:create","result":{"workspace":{…},"root_pane":{…}}}`).
|
|
1218
|
+
* The root pane is the anchor every later split targets, so a create answer
|
|
1219
|
+
* without one is only half a workspace — the caller cleans it up rather than
|
|
1220
|
+
* guessing at a pane to split.
|
|
1221
|
+
*/
|
|
1222
|
+
function parseCreatedWorkspace(stdout: string): { workspaceId: string; rootPaneId?: string } | undefined {
|
|
1223
|
+
const result = envelopeOf(stdout);
|
|
1224
|
+
if (result === undefined) return undefined;
|
|
1225
|
+
const workspace = result["workspace"];
|
|
1226
|
+
if (workspace === null || typeof workspace !== "object") return undefined;
|
|
1227
|
+
const workspaceId = (workspace as Record<string, unknown>)["workspace_id"];
|
|
1228
|
+
if (typeof workspaceId !== "string" || workspaceId.trim() === "") return undefined;
|
|
1229
|
+
const root = (result as Record<string, unknown>)["root_pane"];
|
|
1230
|
+
const rootPaneId =
|
|
1231
|
+
root !== null && typeof root === "object"
|
|
1232
|
+
? (root as Record<string, unknown>)["pane_id"]
|
|
1233
|
+
: undefined;
|
|
1234
|
+
return {
|
|
1235
|
+
workspaceId: workspaceId.trim(),
|
|
1236
|
+
...(typeof rootPaneId === "string" && rootPaneId.trim() !== "" ? { rootPaneId: rootPaneId.trim() } : {}),
|
|
1237
|
+
};
|
|
1238
|
+
}
|
|
1239
|
+
|
|
1240
|
+
/** The `{result: …}` object of a herdr CLI envelope, or nothing. */
|
|
1241
|
+
|
|
1242
|
+
function envelopeOf(stdout: string): Record<string, unknown> | undefined {
|
|
1243
|
+
const line = firstLine(stdout);
|
|
1244
|
+
if (line === "") return undefined;
|
|
1245
|
+
let payload: unknown;
|
|
1246
|
+
try {
|
|
1247
|
+
payload = JSON.parse(line);
|
|
1248
|
+
} catch {
|
|
1249
|
+
return undefined;
|
|
1250
|
+
}
|
|
1251
|
+
if (payload === null || typeof payload !== "object") return undefined;
|
|
1252
|
+
const result = (payload as Record<string, unknown>)["result"];
|
|
1253
|
+
return result === null || typeof result !== "object" ? undefined : (result as Record<string, unknown>);
|
|
1254
|
+
}
|
|
1255
|
+
|
|
1256
|
+
/** The root-pane id inside a `tab create` answer (`result.root_pane.pane_id`):
|
|
1257
|
+
* a tab answer carries no workspace object, so {@link parseCreatedWorkspace}
|
|
1258
|
+
* cannot read it. */
|
|
1259
|
+
function parseRootPane(stdout: string): string | undefined {
|
|
1260
|
+
const result = envelopeOf(stdout);
|
|
1261
|
+
const root = result?.["root_pane"];
|
|
1262
|
+
const paneId =
|
|
1263
|
+
root !== null && typeof root === "object" ? (root as Record<string, unknown>)["pane_id"] : undefined;
|
|
1264
|
+
return typeof paneId === "string" && paneId.trim() !== "" ? paneId.trim() : undefined;
|
|
1265
|
+
}
|
|
1266
|
+
|
|
1267
|
+
/** Every workspace in a `workspace list` answer, with the tokens ownership is read from. */
|
|
1268
|
+
function parseWorkspaceList(stdout: string): { workspaces: { workspaceId: string; label?: string; tokens: Record<string, string> }[] } | undefined {
|
|
1269
|
+
const result = envelopeOf(stdout);
|
|
1270
|
+
const raw = result?.["workspaces"];
|
|
1271
|
+
if (!Array.isArray(raw)) return undefined;
|
|
1272
|
+
const workspaces: { workspaceId: string; label?: string; tokens: Record<string, string> }[] = [];
|
|
1273
|
+
for (const entry of raw) {
|
|
1274
|
+
if (entry === null || typeof entry !== "object") continue;
|
|
1275
|
+
const w = entry as Record<string, unknown>;
|
|
1276
|
+
if (typeof w["workspace_id"] !== "string" || w["workspace_id"].trim() === "") continue;
|
|
1277
|
+
const tokens: Record<string, string> = {};
|
|
1278
|
+
if (w["tokens"] !== null && typeof w["tokens"] === "object") {
|
|
1279
|
+
for (const [name, value] of Object.entries(w["tokens"] as Record<string, unknown>)) {
|
|
1280
|
+
if (typeof value === "string") tokens[name] = value;
|
|
1281
|
+
}
|
|
1282
|
+
}
|
|
1283
|
+
workspaces.push({
|
|
1284
|
+
workspaceId: w["workspace_id"],
|
|
1285
|
+
...(typeof w["label"] === "string" ? { label: w["label"] } : {}),
|
|
1286
|
+
tokens,
|
|
1287
|
+
});
|
|
1288
|
+
}
|
|
1289
|
+
return { workspaces };
|
|
1290
|
+
}
|
|
1291
|
+
|
|
1292
|
+
/** One pane record out of any `pane list` answer — global or `--workspace`-scoped. */
|
|
1293
|
+
export interface ListedPane {
|
|
1294
|
+
paneId: string;
|
|
1295
|
+
/** The run id Herdr still reports for this pane, or "" when it reports none. */
|
|
1296
|
+
runId: string;
|
|
1297
|
+
agent?: string;
|
|
1298
|
+
workspaceId?: string;
|
|
1299
|
+
}
|
|
1300
|
+
|
|
1301
|
+
/** The panes of a `pane list` answer, carrying whichever fields reconciliation reads. */
|
|
1302
|
+
function parsePaneRecords(stdout: string): { panes: ListedPane[] } | undefined {
|
|
1303
|
+
const result = envelopeOf(stdout);
|
|
1304
|
+
const raw = result?.["panes"];
|
|
1305
|
+
if (!Array.isArray(raw)) return undefined;
|
|
1306
|
+
const panes: ListedPane[] = [];
|
|
1307
|
+
for (const entry of raw) {
|
|
1308
|
+
if (entry === null || typeof entry !== "object") continue;
|
|
1309
|
+
const p = entry as Record<string, unknown>;
|
|
1310
|
+
if (typeof p["pane_id"] !== "string" || p["pane_id"].trim() === "") continue;
|
|
1311
|
+
const session = p["agent_session"];
|
|
1312
|
+
const source = session !== null && typeof session === "object" ? (session as Record<string, unknown>)["source"] : undefined;
|
|
1313
|
+
const value = session !== null && typeof session === "object" ? (session as Record<string, unknown>)["value"] : undefined;
|
|
1314
|
+
panes.push({
|
|
1315
|
+
paneId: p["pane_id"].trim(),
|
|
1316
|
+
// Ours by construction inside a marked workspace; the reported session id
|
|
1317
|
+
// is a secondary signal only (#1035), because Herdr has been observed to
|
|
1318
|
+
// drop it while the pane lives on.
|
|
1319
|
+
runId: typeof value === "string" && source === WORKER_PANE_SOURCE ? value : "",
|
|
1320
|
+
...(typeof p["agent"] === "string" ? { agent: p["agent"] } : {}),
|
|
1321
|
+
...(typeof p["workspace_id"] === "string" ? { workspaceId: p["workspace_id"] } : {}),
|
|
1322
|
+
});
|
|
1323
|
+
}
|
|
1324
|
+
return { panes };
|
|
1325
|
+
}
|
|
1326
|
+
|
|
1327
|
+
/** One conductor-marked worker workspace, as discovery returns it. */
|
|
1328
|
+
export interface WorkerWorkspace {
|
|
1329
|
+
workspaceId: string;
|
|
1330
|
+
/** The project the marking token binds it to — never derived from the label. */
|
|
1331
|
+
project: string;
|
|
1332
|
+
label?: string;
|
|
1333
|
+
}
|
|
1334
|
+
|
|
1335
|
+
/**
|
|
1336
|
+
* The durable half of worker-workspace discovery (#1035 review).
|
|
1337
|
+
*
|
|
1338
|
+
* Herdr restores a session's workspaces and panes across a server restart but
|
|
1339
|
+
* drops `report-metadata` tokens — probed live: the restored worker workspace
|
|
1340
|
+
* comes back carrying no tokens at all. Token-only discovery would then ignore
|
|
1341
|
+
* the surface conductor created, build a duplicate beside it, and never be
|
|
1342
|
+
* able to reconcile or remove the restored one's stale panes — exactly the
|
|
1343
|
+
* accumulation this issue exists to end. Ownership is therefore two-legged:
|
|
1344
|
+
* the live token AND the conductor-store record of workspace ids the project
|
|
1345
|
+
* created. Either leg proves ownership; the store is the leg that survives.
|
|
1346
|
+
*/
|
|
1347
|
+
export interface WorkspaceOwnership {
|
|
1348
|
+
/** Store-recorded workspace ids for a project, oldest first. */
|
|
1349
|
+
recordedWorkspaces(project: string): readonly string[];
|
|
1350
|
+
rememberWorkspace(project: string, workspaceId: string): void;
|
|
1351
|
+
forgetWorkspace(project: string, workspaceId: string): void;
|
|
1352
|
+
}
|
|
1353
|
+
|
|
1354
|
+
/** Every workspace in a `workspace list` answer, marked or not. */
|
|
1355
|
+
function listWorkspacesRaw(
|
|
1356
|
+
deps: { run?: HerdrRun; session?: string } = {},
|
|
1357
|
+
): {
|
|
1358
|
+
ok: true;
|
|
1359
|
+
workspaces: { workspaceId: string; label?: string; tokens: Record<string, string> }[];
|
|
1360
|
+
} | { ok: false; reason: string } {
|
|
1361
|
+
const run = deps.run ?? realHerdrRun;
|
|
1362
|
+
const session = deps.session ?? resolveHerdrSession();
|
|
1363
|
+
const res = run(["--session", session, "workspace", "list"]);
|
|
1364
|
+
if (!res.ok) {
|
|
1365
|
+
return { ok: false, reason: `herdr workspace list failed: ${firstLine(res.stderr) || "no output"}` };
|
|
1366
|
+
}
|
|
1367
|
+
const parsed = parseWorkspaceList(res.stdout);
|
|
1368
|
+
if (parsed === undefined) return { ok: false, reason: "herdr workspace list was unreadable" };
|
|
1369
|
+
return { ok: true, workspaces: parsed.workspaces };
|
|
1370
|
+
}
|
|
1371
|
+
|
|
1372
|
+
/**
|
|
1373
|
+
* Every conductor-marked worker workspace Herdr currently has, keyed by nothing:
|
|
1374
|
+
* the caller filters and scopes. Discovery reads the token, not the label — a
|
|
1375
|
+
* renamed workspace is still ours, and a lookalike label without the token is
|
|
1376
|
+
* not (#1035).
|
|
1377
|
+
*/
|
|
1378
|
+
export function listWorkerWorkspaces(deps: { run?: HerdrRun; session?: string } = {}): {
|
|
1379
|
+
ok: true;
|
|
1380
|
+
workspaces: WorkerWorkspace[];
|
|
1381
|
+
} | { ok: false; reason: string } {
|
|
1382
|
+
const listed = listWorkspacesRaw(deps);
|
|
1383
|
+
if (!listed.ok) return listed;
|
|
1384
|
+
const workspaces: WorkerWorkspace[] = [];
|
|
1385
|
+
for (const w of listed.workspaces) {
|
|
1386
|
+
const project = w.tokens[WORKER_WORKSPACE_TOKEN];
|
|
1387
|
+
if (project === undefined || project.trim() === "") continue;
|
|
1388
|
+
workspaces.push({
|
|
1389
|
+
workspaceId: w.workspaceId,
|
|
1390
|
+
project: project.trim(),
|
|
1391
|
+
...(w.label === undefined ? {} : { label: w.label }),
|
|
1392
|
+
});
|
|
1393
|
+
}
|
|
1394
|
+
return { ok: true, workspaces };
|
|
1395
|
+
}
|
|
1396
|
+
|
|
1397
|
+
/**
|
|
1398
|
+
* Every worker workspace these projects own on the live Herdr (#1035 review):
|
|
1399
|
+
* token-marked ones plus store-recorded ids that still exist. A recorded id
|
|
1400
|
+
* Herdr no longer lists is dead weight rather than evidence — its workspace is
|
|
1401
|
+
* gone, and reconciliation's removal path is what forgets it.
|
|
1402
|
+
*/
|
|
1403
|
+
export function ownedWorkerWorkspaces(
|
|
1404
|
+
projects: readonly string[],
|
|
1405
|
+
deps: { run?: HerdrRun; session?: string; ownership?: WorkspaceOwnership } = {},
|
|
1406
|
+
): { ok: true; workspaces: WorkerWorkspace[] } | { ok: false; reason: string } {
|
|
1407
|
+
const raw = listWorkspacesRaw(deps);
|
|
1408
|
+
if (!raw.ok) return raw;
|
|
1409
|
+
const scope = new Set(projects);
|
|
1410
|
+
const owned = new Map<string, WorkerWorkspace>();
|
|
1411
|
+
for (const w of raw.workspaces) {
|
|
1412
|
+
const project = w.tokens[WORKER_WORKSPACE_TOKEN]?.trim();
|
|
1413
|
+
if (project === undefined || project === "" || !scope.has(project)) continue;
|
|
1414
|
+
owned.set(w.workspaceId, {
|
|
1415
|
+
workspaceId: w.workspaceId,
|
|
1416
|
+
project,
|
|
1417
|
+
...(w.label === undefined ? {} : { label: w.label }),
|
|
1418
|
+
});
|
|
1419
|
+
// Adopt token-owned surfaces into the durable leg even when no pane needs
|
|
1420
|
+
// opening this pass. Otherwise an upgrade followed by a Herdr restart can
|
|
1421
|
+
// drop the token before ensureWorkerWorkspace ever records the workspace.
|
|
1422
|
+
deps.ownership?.rememberWorkspace(project, w.workspaceId);
|
|
1423
|
+
}
|
|
1424
|
+
for (const project of projects) {
|
|
1425
|
+
for (const id of deps.ownership?.recordedWorkspaces(project) ?? []) {
|
|
1426
|
+
if (owned.has(id)) continue;
|
|
1427
|
+
const hit = raw.workspaces.find((w) => w.workspaceId === id);
|
|
1428
|
+
if (hit === undefined) {
|
|
1429
|
+
// The workspace disappeared outside conductor. Forget the stale id now
|
|
1430
|
+
// so a future Herdr session cannot reuse it as false ownership.
|
|
1431
|
+
deps.ownership?.forgetWorkspace(project, id);
|
|
1432
|
+
continue;
|
|
1433
|
+
}
|
|
1434
|
+
owned.set(id, { workspaceId: id, project, ...(hit.label === undefined ? {} : { label: hit.label }) });
|
|
1435
|
+
}
|
|
1436
|
+
}
|
|
1437
|
+
return { ok: true, workspaces: [...owned.values()] };
|
|
1438
|
+
}
|
|
1439
|
+
|
|
1440
|
+
export type WorkerWorkspaceResolution =
|
|
1441
|
+
| { kind: "ready"; workspaceId: string; anchorPaneId: string; created?: true }
|
|
1442
|
+
| { kind: "unavailable"; reason: string };
|
|
1443
|
+
|
|
1444
|
+
/**
|
|
1445
|
+
* Find this project's worker workspace or create it — the single door every
|
|
1446
|
+
* split goes through (#1035).
|
|
1447
|
+
*
|
|
1448
|
+
* Identity is two-legged now (#1035 review): the marking token OR the
|
|
1449
|
+
* conductor-store record, so rediscovery after any restart finds the same
|
|
1450
|
+
* workspace whatever Herdr preserved. Duplicates are deliberately NOT
|
|
1451
|
+
* resolved here: closing one blind — without listing its panes or checking
|
|
1452
|
+
* the result — could destroy the only live representation and strand the
|
|
1453
|
+
* durable row pointing at a dead pane. Convergence belongs to
|
|
1454
|
+
* {@link reconcileWorkerPanes}, which attributes every pane first and removes
|
|
1455
|
+
* a duplicate only once nothing living holds it. A discovered workspace with
|
|
1456
|
+
* no panes gets a fresh tab rather than failing forever; a workspace CREATED
|
|
1457
|
+
* by this call is closed again on every later failure, because newly-made
|
|
1458
|
+
* residue is exactly what the acceptance criterion forbids.
|
|
1459
|
+
*/
|
|
1460
|
+
export function ensureWorkerWorkspace(
|
|
1461
|
+
project: string,
|
|
1462
|
+
deps: { run?: HerdrRun; session?: string; ownership?: WorkspaceOwnership } = {},
|
|
1463
|
+
): WorkerWorkspaceResolution {
|
|
1464
|
+
const run = deps.run ?? realHerdrRun;
|
|
1465
|
+
const session = deps.session ?? resolveHerdrSession();
|
|
1466
|
+
const base = ["--session", session];
|
|
1467
|
+
|
|
1468
|
+
let listed = ownedWorkerWorkspaces([project], deps);
|
|
1469
|
+
if (!listed.ok) return { kind: "unavailable", reason: listed.reason };
|
|
1470
|
+
|
|
1471
|
+
for (const existing of [...listed.workspaces].sort((a, b) => (a.workspaceId < b.workspaceId ? -1 : 1))) {
|
|
1472
|
+
const res = run([...base, "pane", "list", "--workspace", existing.workspaceId]);
|
|
1473
|
+
if (!res.ok) {
|
|
1474
|
+
return { kind: "unavailable", reason: `herdr pane list failed: ${firstLine(res.stderr) || "no output"}` };
|
|
1475
|
+
}
|
|
1476
|
+
const parsed = parsePaneRecords(res.stdout);
|
|
1477
|
+
if (parsed === undefined) return { kind: "unavailable", reason: "herdr pane list was unreadable" };
|
|
1478
|
+
let anchor = parsed.panes[0]?.paneId;
|
|
1479
|
+
if (anchor === undefined) {
|
|
1480
|
+
// No pane to split: give the workspace a fresh tab and use its root pane.
|
|
1481
|
+
const tabbed = run([...base, "tab", "create", "--workspace", existing.workspaceId, "--no-focus"]);
|
|
1482
|
+
if (!tabbed.ok) {
|
|
1483
|
+
return { kind: "unavailable", reason: `herdr tab create failed: ${firstLine(tabbed.stderr) || "no output"}` };
|
|
1484
|
+
}
|
|
1485
|
+
anchor = parseRootPane(tabbed.stdout);
|
|
1486
|
+
if (anchor === undefined) {
|
|
1487
|
+
return { kind: "unavailable", reason: "herdr tab create reported no root pane" };
|
|
1488
|
+
}
|
|
1489
|
+
}
|
|
1490
|
+
// Adopted into the store: if this workspace's token is what Herdr drops
|
|
1491
|
+
// next restart, the record below is what still finds it.
|
|
1492
|
+
deps.ownership?.rememberWorkspace(project, existing.workspaceId);
|
|
1493
|
+
return { kind: "ready", workspaceId: existing.workspaceId, anchorPaneId: anchor };
|
|
1494
|
+
}
|
|
1495
|
+
|
|
1496
|
+
const made = run([
|
|
1497
|
+
...base,
|
|
1498
|
+
"workspace",
|
|
1499
|
+
"create",
|
|
1500
|
+
"--label",
|
|
1501
|
+
workerWorkspaceLabel(project),
|
|
1502
|
+
"--no-focus",
|
|
1503
|
+
]);
|
|
1504
|
+
if (!made.ok) {
|
|
1505
|
+
return { kind: "unavailable", reason: `herdr workspace create failed: ${firstLine(made.stderr) || "no output"}` };
|
|
1506
|
+
}
|
|
1507
|
+
const parsed = parseCreatedWorkspace(made.stdout);
|
|
1508
|
+
if (parsed === undefined) {
|
|
1509
|
+
return { kind: "unavailable", reason: "herdr workspace create reported no workspace id" };
|
|
1510
|
+
}
|
|
1511
|
+
/** This attempt made the workspace, so every exit from here takes it back. */
|
|
1512
|
+
const discardMade = (): void => {
|
|
1513
|
+
run([...base, "workspace", "close", parsed.workspaceId]);
|
|
1514
|
+
deps.ownership?.forgetWorkspace(project, parsed.workspaceId);
|
|
1515
|
+
};
|
|
1516
|
+
// Mark before use: an unmarked workspace would be invisible to token-side
|
|
1517
|
+
// discovery on the next pass — an empty shell accumulating precisely as
|
|
1518
|
+
// #1035 describes. A failure here takes the half-made workspace with it.
|
|
1519
|
+
const marked = run([
|
|
1520
|
+
...base,
|
|
1521
|
+
"workspace",
|
|
1522
|
+
"report-metadata",
|
|
1523
|
+
parsed.workspaceId,
|
|
1524
|
+
"--source",
|
|
1525
|
+
WORKER_PANE_SOURCE,
|
|
1526
|
+
"--token",
|
|
1527
|
+
`${WORKER_WORKSPACE_TOKEN}=${project}`,
|
|
1528
|
+
]);
|
|
1529
|
+
if (!marked.ok) {
|
|
1530
|
+
discardMade();
|
|
1531
|
+
return {
|
|
1532
|
+
kind: "unavailable",
|
|
1533
|
+
reason: `herdr workspace report-metadata failed: ${firstLine(marked.stderr) || "no output"}`,
|
|
1534
|
+
};
|
|
1535
|
+
}
|
|
1536
|
+
if (parsed.rootPaneId === undefined) {
|
|
1537
|
+
// No anchor means no split can ever be targeted; a fresh-made workspace
|
|
1538
|
+
// without one is residue this very attempt must not leave behind
|
|
1539
|
+
// (#1035 review) — so it goes now, not on some later pass.
|
|
1540
|
+
discardMade();
|
|
1541
|
+
return { kind: "unavailable", reason: "herdr workspace create reported no root pane" };
|
|
1542
|
+
}
|
|
1543
|
+
deps.ownership?.rememberWorkspace(project, parsed.workspaceId);
|
|
1544
|
+
return { kind: "ready", workspaceId: parsed.workspaceId, anchorPaneId: parsed.rootPaneId, created: true };
|
|
1545
|
+
}
|
|
1546
|
+
|
|
1000
1547
|
/**
|
|
1001
1548
|
* Create the pane, run the follower in it, and report the child's identity and
|
|
1002
1549
|
* initial state — or say exactly why it could not.
|
|
1003
1550
|
*
|
|
1004
|
-
*
|
|
1005
|
-
*
|
|
1006
|
-
*
|
|
1007
|
-
*
|
|
1008
|
-
*
|
|
1009
|
-
*
|
|
1010
|
-
*
|
|
1551
|
+
* The representation lives in the project's worker workspace (#1035), found or
|
|
1552
|
+
* created first, and the split targets an anchor pane inside it explicitly, so
|
|
1553
|
+
* the operator's own layout is never touched. Every step is checked, and the
|
|
1554
|
+
* first failure returns `unavailable` with the reason and leaves ZERO residue
|
|
1555
|
+
* of this attempt (#1035 review): the pane the split had already created is
|
|
1556
|
+
* closed, and a workspace this call CREATED is closed with it — only surfaces
|
|
1557
|
+
* that already existed are preserved. A partially established representation
|
|
1558
|
+
* is worse than none: it is a pane the operator would read as a tracked
|
|
1559
|
+
* worker, and before #992 it was left behind on every single launch. What
|
|
1560
|
+
* this deliberately does NOT do is decide what a failure means for the launch
|
|
1561
|
+
* — failing closed versus running degraded is #841's policy, and inventing it
|
|
1562
|
+
* here would pre-empt it.
|
|
1011
1563
|
*/
|
|
1012
1564
|
export function openWorkerPane(identity: WorkerPaneIdentity, deps: WorkerPaneDeps = {}): WorkerPaneOutcome {
|
|
1013
1565
|
const run = deps.run ?? realHerdrRun;
|
|
1014
1566
|
const session = deps.session ?? resolveHerdrSession();
|
|
1015
1567
|
const label = workerPaneLabel(identity);
|
|
1568
|
+
const ensured = ensureWorkerWorkspace(identity.project, deps);
|
|
1569
|
+
if (ensured.kind === "unavailable") return ensured;
|
|
1016
1570
|
const base = ["--session", session, "pane"];
|
|
1017
1571
|
|
|
1018
|
-
|
|
1572
|
+
/** Undo everything this attempt built so far — the split's pane once there
|
|
1573
|
+
* is one, and always a workspace this call itself created (#1035 review). */
|
|
1574
|
+
const abandon = (reason: string, phase?: "split", paneId?: string): WorkerPaneOutcome => {
|
|
1575
|
+
if (paneId !== undefined) run([...base, "close", paneId]);
|
|
1576
|
+
if (ensured.created === true) {
|
|
1577
|
+
run(["--session", session, "workspace", "close", ensured.workspaceId]);
|
|
1578
|
+
deps.ownership?.forgetWorkspace(identity.project, ensured.workspaceId);
|
|
1579
|
+
}
|
|
1580
|
+
return { kind: "unavailable", reason, ...(phase === undefined ? {} : { phase }) };
|
|
1581
|
+
};
|
|
1582
|
+
|
|
1583
|
+
// The explicit target is the whole point (#1035): the split names a pane
|
|
1584
|
+
// inside the worker workspace, so whichever pane the operator happens to be
|
|
1585
|
+
// looking at is never touched.
|
|
1586
|
+
const split = run([
|
|
1587
|
+
...base,
|
|
1588
|
+
"split",
|
|
1589
|
+
"--pane",
|
|
1590
|
+
ensured.anchorPaneId,
|
|
1591
|
+
"--direction",
|
|
1592
|
+
"down",
|
|
1593
|
+
"--ratio",
|
|
1594
|
+
"0.3",
|
|
1595
|
+
"--no-focus",
|
|
1596
|
+
]);
|
|
1019
1597
|
if (!split.ok) {
|
|
1020
|
-
return
|
|
1598
|
+
return abandon(
|
|
1599
|
+
`herdr pane split failed: ${firstLine(split.stderr) || "no output"}`,
|
|
1600
|
+
"split",
|
|
1601
|
+
);
|
|
1021
1602
|
}
|
|
1022
1603
|
const paneId = parsePaneId(split.stdout);
|
|
1023
1604
|
if (paneId === undefined) {
|
|
1024
|
-
return
|
|
1605
|
+
return abandon("herdr pane split reported no pane id");
|
|
1025
1606
|
}
|
|
1026
|
-
// The split has created a pane. From here every exit must take it with it.
|
|
1027
|
-
const abandon = (reason: string): WorkerPaneOutcome => {
|
|
1028
|
-
run([...base, "close", paneId]);
|
|
1029
|
-
return { kind: "unavailable", reason };
|
|
1030
|
-
};
|
|
1031
1607
|
|
|
1032
1608
|
const named = run([...base, "rename", paneId, label]);
|
|
1033
1609
|
if (!named.ok) {
|
|
1034
|
-
return abandon(`herdr pane rename failed: ${firstLine(named.stderr) || "no output"}
|
|
1610
|
+
return abandon(`herdr pane rename failed: ${firstLine(named.stderr) || "no output"}`, undefined, paneId);
|
|
1035
1611
|
}
|
|
1036
1612
|
|
|
1037
1613
|
// Identity before display: the pane must be attributable to this exact run
|
|
@@ -1052,12 +1628,16 @@ export function openWorkerPane(identity: WorkerPaneIdentity, deps: WorkerPaneDep
|
|
|
1052
1628
|
WORKER_PANE_SOURCE,
|
|
1053
1629
|
]);
|
|
1054
1630
|
if (!identified.ok) {
|
|
1055
|
-
return abandon(
|
|
1631
|
+
return abandon(
|
|
1632
|
+
`herdr pane report-agent-session failed: ${firstLine(identified.stderr) || "no output"}`,
|
|
1633
|
+
undefined,
|
|
1634
|
+
paneId,
|
|
1635
|
+
);
|
|
1056
1636
|
}
|
|
1057
1637
|
|
|
1058
1638
|
const started = run([...base, "run", paneId, ...deps.viewer?.(identity) ?? defaultViewer(identity)]);
|
|
1059
1639
|
if (!started.ok) {
|
|
1060
|
-
return abandon(`herdr pane run failed: ${firstLine(started.stderr) || "no output"}
|
|
1640
|
+
return abandon(`herdr pane run failed: ${firstLine(started.stderr) || "no output"}`, undefined, paneId);
|
|
1061
1641
|
}
|
|
1062
1642
|
|
|
1063
1643
|
// A worker that has just spawned is working by definition. The ongoing
|
|
@@ -1065,9 +1645,9 @@ export function openWorkerPane(identity: WorkerPaneIdentity, deps: WorkerPaneDep
|
|
|
1065
1645
|
// events — never from the pane's output.
|
|
1066
1646
|
const reported = reportWorkerPaneState(paneId, label, "working", { run, session });
|
|
1067
1647
|
if (!reported.ok) {
|
|
1068
|
-
return abandon(reported.reason);
|
|
1648
|
+
return abandon(reported.reason, undefined, paneId);
|
|
1069
1649
|
}
|
|
1070
|
-
return { kind: "tracked", paneId, label, pid: identity.pid };
|
|
1650
|
+
return { kind: "tracked", paneId, label, pid: identity.pid, workspaceId: ensured.workspaceId };
|
|
1071
1651
|
}
|
|
1072
1652
|
|
|
1073
1653
|
/** Report one lifecycle state for a tracked pane. Monotonic `seq` is the caller's. */
|
|
@@ -1100,12 +1680,11 @@ export function reportWorkerPaneState(
|
|
|
1100
1680
|
}
|
|
1101
1681
|
|
|
1102
1682
|
/**
|
|
1103
|
-
* Hand lifecycle authority back when
|
|
1104
|
-
*
|
|
1105
|
-
*
|
|
1106
|
-
*
|
|
1107
|
-
*
|
|
1108
|
-
* no longer speaks for this agent", which is exactly what is true.
|
|
1683
|
+
* Hand lifecycle authority back when conductor no longer speaks for a pane's
|
|
1684
|
+
* agent. Half of {@link retireWorkerPane}: it says "this agent record is no
|
|
1685
|
+
* longer ours" without removing anything, which is what reconciliation wants
|
|
1686
|
+
* for a pane it is about to close anyway but also stands alone for callers
|
|
1687
|
+
* that only re-report identity (#842's adopt path).
|
|
1109
1688
|
*/
|
|
1110
1689
|
export function releaseWorkerPane(
|
|
1111
1690
|
paneId: string,
|
|
@@ -1131,71 +1710,153 @@ export function releaseWorkerPane(
|
|
|
1131
1710
|
: { ok: false, reason: `herdr pane release-agent failed: ${firstLine(res.stderr) || "no output"}` };
|
|
1132
1711
|
}
|
|
1133
1712
|
|
|
1713
|
+
/** Close one pane outright. The pane holds only ever a follower, so closing
|
|
1714
|
+
* it cannot reach the authoritative worker — that child belongs to the
|
|
1715
|
+
* daemon's process tree, not to any terminal (#1035). */
|
|
1716
|
+
export function closeWorkerPane(
|
|
1717
|
+
paneId: string,
|
|
1718
|
+
deps: { run?: HerdrRun; session?: string } = {},
|
|
1719
|
+
): { ok: true } | { ok: false; reason: string } {
|
|
1720
|
+
const run = deps.run ?? realHerdrRun;
|
|
1721
|
+
const session = deps.session ?? resolveHerdrSession();
|
|
1722
|
+
const res = run(["--session", session, "pane", "close", paneId]);
|
|
1723
|
+
return res.ok
|
|
1724
|
+
? { ok: true }
|
|
1725
|
+
: { ok: false, reason: `herdr pane close failed: ${firstLine(res.stderr) || "no output"}` };
|
|
1726
|
+
}
|
|
1727
|
+
|
|
1728
|
+
/**
|
|
1729
|
+
* Retire one run's visual representation (#1035): hand lifecycle authority
|
|
1730
|
+
* back to Herdr, then close the pane.
|
|
1731
|
+
*
|
|
1732
|
+
* #841 originally stopped at release, and production showed what that leaves:
|
|
1733
|
+
* panes advertising `working` for runs that had settled hours earlier, their
|
|
1734
|
+
* follower long exited, a bare shell where a worker used to be. A settled run
|
|
1735
|
+
* keeps no representation. The close reaches only the follower — the
|
|
1736
|
+
* authoritative child was never in the pane, so nothing here can signal it.
|
|
1737
|
+
*
|
|
1738
|
+
* A close failure is reported, not swallowed: the next reconciliation pass
|
|
1739
|
+
* sees the pane as unclaimed junk and closes it again, so one refusal costs a
|
|
1740
|
+
* pass rather than the representation.
|
|
1741
|
+
*/
|
|
1742
|
+
export function retireWorkerPane(
|
|
1743
|
+
paneId: string,
|
|
1744
|
+
label: string,
|
|
1745
|
+
deps: { run?: HerdrRun; session?: string; seq?: number } = {},
|
|
1746
|
+
): { ok: true } | { ok: false; reason: string } {
|
|
1747
|
+
const released = releaseWorkerPane(paneId, label, deps);
|
|
1748
|
+
if (!released.ok) return released;
|
|
1749
|
+
return closeWorkerPane(paneId, deps);
|
|
1750
|
+
}
|
|
1751
|
+
|
|
1134
1752
|
/**
|
|
1135
|
-
* Hand back the
|
|
1753
|
+
* Hand back the representation a dead worker left behind (#842), retired for
|
|
1754
|
+
* good by closing it (#1035) — but only once the recorded pane is PROVEN still
|
|
1755
|
+
* conductor's (#1035 review).
|
|
1136
1756
|
*
|
|
1137
1757
|
* Called for every run a restart reaps. The recorded pid is deliberately NOT
|
|
1138
1758
|
* consulted for liveness: a `session-host` child dies with the daemon that owned
|
|
1139
1759
|
* its verb socket, so an orphaned row's worker is gone whatever pid it carries —
|
|
1140
1760
|
* and pids are reused, so checking one is how a stranger's process comes to read
|
|
1141
|
-
* as a live worker.
|
|
1142
|
-
*
|
|
1761
|
+
* as a live worker. The same reuse argument applies to pane ids: after a Herdr
|
|
1762
|
+
* restart the stored `w2:pT` may belong to a pane in somebody else's workspace,
|
|
1763
|
+
* so a blind release+close could destroy a stranger's terminal. Ownership is
|
|
1764
|
+
* therefore proven before anything is closed:
|
|
1765
|
+
*
|
|
1766
|
+
* - the pane must be listed inside a workspace conductor OWNS — a
|
|
1767
|
+
* {@link WORKER_WORKSPACE_TOKEN}-marked one, or one the store recorded for
|
|
1768
|
+
* the run's project (#1035 review: Herdr restarts restore workspaces without
|
|
1769
|
+
* their tokens, and a token-only check would leave the restored pane
|
|
1770
|
+
* unretired forever); and
|
|
1143
1771
|
*/
|
|
1144
1772
|
export function releaseOrphanedWorkerPane(
|
|
1145
|
-
run: { paneId?: string; paneLabel?: string },
|
|
1146
|
-
deps: { run?: HerdrRun; session?: string; seq?: number } = {},
|
|
1147
|
-
):
|
|
1773
|
+
run: { paneId?: string; paneLabel?: string; runId?: string; project?: string },
|
|
1774
|
+
deps: { run?: HerdrRun; session?: string; seq?: number; ownership?: WorkspaceOwnership } = {},
|
|
1775
|
+
):
|
|
1776
|
+
| { kind: "none" }
|
|
1777
|
+
| { kind: "released"; paneId: string }
|
|
1778
|
+
| { kind: "failed"; paneId: string; reason: string }
|
|
1779
|
+
| { kind: "unowned"; paneId: string; reason: string } {
|
|
1148
1780
|
if (run.paneId === undefined || run.paneLabel === undefined) return { kind: "none" };
|
|
1149
|
-
const
|
|
1150
|
-
|
|
1781
|
+
const discovered = ownedWorkerWorkspaces(run.project === undefined ? [] : [run.project], deps);
|
|
1782
|
+
if (!discovered.ok) {
|
|
1783
|
+
return { kind: "failed", paneId: run.paneId, reason: discovered.reason };
|
|
1784
|
+
}
|
|
1785
|
+
const runFn = deps.run ?? realHerdrRun;
|
|
1786
|
+
const session = deps.session ?? resolveHerdrSession();
|
|
1787
|
+
let home: string | undefined;
|
|
1788
|
+
let reportedAgent: string | undefined;
|
|
1789
|
+
let reportedSession: string | undefined;
|
|
1790
|
+
for (const ws of discovered.workspaces) {
|
|
1791
|
+
const res = runFn(["--session", session, "pane", "list", "--workspace", ws.workspaceId]);
|
|
1792
|
+
if (!res.ok) {
|
|
1793
|
+
return {
|
|
1794
|
+
kind: "failed",
|
|
1795
|
+
paneId: run.paneId,
|
|
1796
|
+
reason: `herdr pane list failed: ${firstLine(res.stderr) || "no output"}`,
|
|
1797
|
+
};
|
|
1798
|
+
}
|
|
1799
|
+
const parsed = parsePaneRecords(res.stdout);
|
|
1800
|
+
if (parsed === undefined) {
|
|
1801
|
+
return { kind: "failed", paneId: run.paneId, reason: "herdr pane list was unreadable" };
|
|
1802
|
+
}
|
|
1803
|
+
const found = parsed.panes.find((pane) => pane.paneId === run.paneId);
|
|
1804
|
+
if (found !== undefined && home === undefined) {
|
|
1805
|
+
home = ws.workspaceId;
|
|
1806
|
+
if (found.agent !== undefined) reportedAgent = found.agent;
|
|
1807
|
+
if (found.runId !== "") reportedSession = found.runId;
|
|
1808
|
+
}
|
|
1809
|
+
}
|
|
1810
|
+
if (home === undefined) {
|
|
1811
|
+
// Not in any marked workspace: either already retired or never ours to
|
|
1812
|
+
// touch. Both answers close nothing.
|
|
1813
|
+
return { kind: "none" };
|
|
1814
|
+
}
|
|
1815
|
+
if (
|
|
1816
|
+
(reportedAgent !== undefined && reportedAgent !== run.paneLabel) ||
|
|
1817
|
+
(reportedSession !== undefined && run.runId !== undefined && reportedSession !== run.runId)
|
|
1818
|
+
) {
|
|
1819
|
+
return {
|
|
1820
|
+
kind: "unowned",
|
|
1821
|
+
paneId: run.paneId,
|
|
1822
|
+
reason: `pane ${run.paneId} in marked workspace ${home} reports another identity (agent ${reportedAgent ?? "none"}, session ${reportedSession ?? "none"})`,
|
|
1823
|
+
};
|
|
1824
|
+
}
|
|
1825
|
+
const retired = retireWorkerPane(run.paneId, run.paneLabel, deps);
|
|
1826
|
+
return retired.ok
|
|
1151
1827
|
? { kind: "released", paneId: run.paneId }
|
|
1152
|
-
: { kind: "failed", paneId: run.paneId, reason:
|
|
1828
|
+
: { kind: "failed", paneId: run.paneId, reason: retired.reason };
|
|
1153
1829
|
}
|
|
1154
1830
|
|
|
1155
1831
|
/**
|
|
1156
|
-
* Every
|
|
1157
|
-
* reported under (#841).
|
|
1832
|
+
* Every pane inside a conductor-marked worker workspace (#1035).
|
|
1158
1833
|
*
|
|
1159
|
-
*
|
|
1160
|
-
*
|
|
1161
|
-
*
|
|
1162
|
-
*
|
|
1834
|
+
* Ownership here is structural: conductor created the workspace, marked it,
|
|
1835
|
+
* and splits into it exclusively, so every pane inside it is conductor's
|
|
1836
|
+
* however Herdr feels about reporting `agent_session` back — production has
|
|
1837
|
+
* repeatedly handed back worker panes with that record dropped to null, which
|
|
1838
|
+
* is why the previous source-filtered listing could not see the very panes
|
|
1839
|
+
* that needed cleaning. The reported session id still rides along as a
|
|
1840
|
+
* secondary attribution signal; an empty run id means "unattributable", and
|
|
1841
|
+
* unattributable panes in a marked workspace are exactly the stale shells
|
|
1842
|
+
* this surface exists to converge away.
|
|
1163
1843
|
*/
|
|
1164
1844
|
export function listWorkerPanes(
|
|
1165
1845
|
deps: { run?: HerdrRun; session?: string } = {},
|
|
1166
|
-
): { ok: true; panes:
|
|
1846
|
+
): { ok: true; panes: (ListedPane & { project: string })[] } | { ok: false; reason: string } {
|
|
1847
|
+
const workspaces = listWorkerWorkspaces(deps);
|
|
1848
|
+
if (!workspaces.ok) return workspaces;
|
|
1167
1849
|
const run = deps.run ?? realHerdrRun;
|
|
1168
1850
|
const session = deps.session ?? resolveHerdrSession();
|
|
1169
|
-
const
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
parsed =
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
}
|
|
1179
|
-
const panes = (parsed as { result?: { panes?: unknown[] } }).result?.panes;
|
|
1180
|
-
if (!Array.isArray(panes)) return { ok: false, reason: "herdr pane list carried no pane array" };
|
|
1181
|
-
const owned: { paneId: string; runId: string; label?: string }[] = [];
|
|
1182
|
-
for (const pane of panes) {
|
|
1183
|
-
const p = pane as {
|
|
1184
|
-
pane_id?: unknown;
|
|
1185
|
-
agent?: unknown;
|
|
1186
|
-
agent_session?: { source?: unknown; value?: unknown };
|
|
1187
|
-
};
|
|
1188
|
-
if (typeof p.pane_id !== "string") continue;
|
|
1189
|
-
if (p.agent_session?.source !== WORKER_PANE_SOURCE) continue;
|
|
1190
|
-
const runId = p.agent_session.value;
|
|
1191
|
-
// Ours by source but carrying no run id: an identity nobody can resolve is
|
|
1192
|
-
// not an identity. Reported as a pane with an empty run id so the caller
|
|
1193
|
-
// treats it as stale rather than silently ignoring it.
|
|
1194
|
-
owned.push({
|
|
1195
|
-
paneId: p.pane_id,
|
|
1196
|
-
runId: typeof runId === "string" ? runId : "",
|
|
1197
|
-
...(typeof p.agent === "string" ? { label: p.agent } : {}),
|
|
1198
|
-
});
|
|
1851
|
+
const owned: (ListedPane & { project: string })[] = [];
|
|
1852
|
+
for (const ws of workspaces.workspaces) {
|
|
1853
|
+
const res = run(["--session", session, "pane", "list", "--workspace", ws.workspaceId]);
|
|
1854
|
+
if (!res.ok) {
|
|
1855
|
+
return { ok: false, reason: `herdr pane list failed: ${firstLine(res.stderr) || "no output"}` };
|
|
1856
|
+
}
|
|
1857
|
+
const parsed = parsePaneRecords(res.stdout);
|
|
1858
|
+
if (parsed === undefined) return { ok: false, reason: "herdr pane list carried no pane array" };
|
|
1859
|
+
for (const pane of parsed.panes) owned.push({ ...pane, project: ws.project });
|
|
1199
1860
|
}
|
|
1200
1861
|
return { ok: true, panes: owned };
|
|
1201
1862
|
}
|
|
@@ -1213,43 +1874,147 @@ export interface LiveWorkerPane {
|
|
|
1213
1874
|
sessionFile?: string;
|
|
1214
1875
|
}
|
|
1215
1876
|
|
|
1877
|
+
export type StaleCause = "settled" | "duplicate" | "unidentified";
|
|
1878
|
+
|
|
1216
1879
|
export type WorkerPaneReconciliation =
|
|
1217
|
-
/** The recorded pane is still there and
|
|
1880
|
+
/** The recorded pane is still there and its follower is alive — nothing done. */
|
|
1218
1881
|
| { kind: "intact"; runId: string; paneId: string }
|
|
1219
|
-
/**
|
|
1882
|
+
/** The pane stands but its follower table could not decide life or death:
|
|
1883
|
+
* no evidence, no destruction (#1035 review). The visual may be stale; the
|
|
1884
|
+
* caller says so once instead of gambling a live worker's representation. */
|
|
1885
|
+
| { kind: "visual-unreadable"; runId: string; paneId: string; reason: string }
|
|
1886
|
+
/** The run had no usable representation; a new one now represents the same
|
|
1887
|
+
* child — a Herdr restart, a missing pane, or a follower that had exited. */
|
|
1220
1888
|
| { kind: "reassociated"; runId: string; paneId: string; label: string }
|
|
1221
1889
|
/** No representation, and the reason. The run keeps working regardless.
|
|
1222
1890
|
* `attempted` marks the ones that spent a re-establishment attempt (#998),
|
|
1223
|
-
* so the caller can bound them; the no-pid case costs nothing.
|
|
1224
|
-
|
|
1891
|
+
* so the caller can bound them; the no-pid case costs nothing. `phase`
|
|
1892
|
+
* carries {@link WorkerPaneOutcome}'s split marker through. */
|
|
1893
|
+
| { kind: "untracked"; runId: string; reason: string; attempted?: true; phase?: "split" }
|
|
1225
1894
|
/** The attempt budget for this run is spent, so nothing was tried this pass
|
|
1226
1895
|
* (#998). Before this existed, a run whose pane could not be created was
|
|
1227
1896
|
* retried on every reconciliation pass forever — and each attempt leaked a
|
|
1228
1897
|
* pane until #992, ending in `ghostty error -2` once enough had piled up. */
|
|
1229
1898
|
| { kind: "attempts-exhausted"; runId: string; attempts: number }
|
|
1230
|
-
/** A conductor pane
|
|
1231
|
-
|
|
1232
|
-
|
|
1899
|
+
/** A conductor-workspace pane with no live run behind it: authority handed
|
|
1900
|
+
* back and the pane closed. `settled` — the run finished and its retirement
|
|
1901
|
+
* never landed; `duplicate` — a second representation of one live run, or
|
|
1902
|
+
* the one a fresh replacement displaced; `unidentified` — a representation
|
|
1903
|
+
* whose run id nobody can resolve any more, the bare shells production
|
|
1904
|
+
* accumulated (#1035). */
|
|
1905
|
+
| { kind: "stale-released"; paneId: string; runId: string; cause: StaleCause }
|
|
1906
|
+
| { kind: "stale-release-failed"; paneId: string; runId: string; reason: string; cause: StaleCause }
|
|
1907
|
+
/** A scoped marked workspace nobody's live run holds any more, emptied by
|
|
1908
|
+
* the cleanup above, is closed (#1035). Every marked workspace converges,
|
|
1909
|
+
* not just one per project (#1035 review). */
|
|
1910
|
+
| { kind: "workspace-removed"; project: string; workspaceId: string }
|
|
1911
|
+
| { kind: "workspace-remove-failed"; project: string; workspaceId: string; reason: string };
|
|
1912
|
+
|
|
1913
|
+
/** The verdict the foreground process table supports about one pane's
|
|
1914
|
+
* read-only follower. Terminal output is never read: this says whether the
|
|
1915
|
+
* *visual* is alive, nothing more, and settlement stays the store's business
|
|
1916
|
+
* (#1035). `unknown` is a real verdict, not an error — recognition of
|
|
1917
|
+
* `omp-conductor` in the table is positive evidence of life, but its absence
|
|
1918
|
+
* is only evidence of death when every entry carries enough identity (name or
|
|
1919
|
+
* argv) to rule it out. Real Herdr answers name + argv; a table with blanked
|
|
1920
|
+
* or missing fields, no shell pid, or no entries at all cannot decide, and a
|
|
1921
|
+
* pane destroyed on a guess we did not have is a live worker gone blind. */
|
|
1922
|
+
export type FollowerPulse =
|
|
1923
|
+
| { pulse: "alive" }
|
|
1924
|
+
| { pulse: "dead" }
|
|
1925
|
+
| { pulse: "unknown"; reason: string };
|
|
1926
|
+
|
|
1927
|
+
export function followerPulseFromProcessInfo(info: ProcessInfo): FollowerPulse {
|
|
1928
|
+
const procs = Array.isArray(info.foreground_processes) ? info.foreground_processes : undefined;
|
|
1929
|
+
if (procs === undefined || procs.length === 0) {
|
|
1930
|
+
return { pulse: "unknown", reason: "process-info carried no foreground process table" };
|
|
1931
|
+
}
|
|
1932
|
+
const shell = typeof info.shell_pid === "number" ? info.shell_pid : undefined;
|
|
1933
|
+
let nonShell = false;
|
|
1934
|
+
for (const proc of procs) {
|
|
1935
|
+
const parts = [proc.name ?? "", proc.argv0 ?? "", ...(proc.argv ?? [])];
|
|
1936
|
+
const isShell = shell !== undefined && proc.pid === shell;
|
|
1937
|
+
if (!isShell && !parts.some((a) => a !== "")) {
|
|
1938
|
+
return { pulse: "unknown", reason: `foreground pid ${proc.pid ?? "unknown"} carries no name or argv` };
|
|
1939
|
+
}
|
|
1940
|
+
if (!isShell) {
|
|
1941
|
+
nonShell = true;
|
|
1942
|
+
if (parts.some((a) => a.toLowerCase() === "omp-conductor" || a.toLowerCase().endsWith("/omp-conductor"))) {
|
|
1943
|
+
return { pulse: "alive" };
|
|
1944
|
+
}
|
|
1945
|
+
}
|
|
1946
|
+
}
|
|
1947
|
+
if (!nonShell && shell === undefined) {
|
|
1948
|
+
return { pulse: "unknown", reason: "process-info named neither a shell nor any identifiable process" };
|
|
1949
|
+
}
|
|
1950
|
+
return { pulse: "dead" };
|
|
1951
|
+
}
|
|
1952
|
+
|
|
1953
|
+
/**
|
|
1954
|
+
* Probe one pane's follower liveness. A probe that cannot be read, or a table
|
|
1955
|
+
* that cannot decide, answers `unknown` rather than dead: destroying a
|
|
1956
|
+
* representation on evidence we do not have is how a healthy worker goes blind
|
|
1957
|
+
* mid-run.
|
|
1958
|
+
*/
|
|
1959
|
+
export function paneFollowerAlive(
|
|
1960
|
+
paneId: string,
|
|
1961
|
+
deps: { run?: HerdrRun; session?: string } = {},
|
|
1962
|
+
): FollowerPulse {
|
|
1963
|
+
const run = deps.run ?? realHerdrRun;
|
|
1964
|
+
const session = deps.session ?? resolveHerdrSession();
|
|
1965
|
+
const res = run(["--session", session, "pane", "process-info", "--pane", paneId]);
|
|
1966
|
+
if (!res.ok) {
|
|
1967
|
+
return {
|
|
1968
|
+
pulse: "unknown",
|
|
1969
|
+
reason: `herdr pane process-info failed: ${firstLine(res.stderr) || "no output"}`,
|
|
1970
|
+
};
|
|
1971
|
+
}
|
|
1972
|
+
try {
|
|
1973
|
+
return followerPulseFromProcessInfo(parseHerdrProcessInfo(res.stdout, paneId));
|
|
1974
|
+
} catch (err) {
|
|
1975
|
+
return { pulse: "unknown", reason: err instanceof Error ? err.message : String(err) };
|
|
1976
|
+
}
|
|
1977
|
+
}
|
|
1978
|
+
|
|
1979
|
+
function firstLine(text: string): string {
|
|
1980
|
+
return text.split("\n", 1)[0]?.trim() ?? "";
|
|
1981
|
+
}
|
|
1982
|
+
/** The kill syscall, injectable so error mapping is testable without one. */
|
|
1983
|
+
export type KillFn = (pid: number, sig: NodeJS.Signals | 0) => void;
|
|
1984
|
+
|
|
1985
|
+
const realKill: KillFn = (pid, sig) => {
|
|
1986
|
+
process.kill(pid, sig);
|
|
1987
|
+
};
|
|
1233
1988
|
|
|
1234
1989
|
/**
|
|
1235
|
-
* Make
|
|
1990
|
+
* Make the conductor-owned worker workspaces agree with the live run set
|
|
1991
|
+
* (#841, reworked by #1035).
|
|
1236
1992
|
*
|
|
1237
1993
|
* Idempotent by construction: a second pass over an already-reconciled fleet
|
|
1238
|
-
* returns `intact` for every live run and finds
|
|
1239
|
-
*
|
|
1994
|
+
* returns `intact` for every live run and finds nothing left to close, so
|
|
1995
|
+
* repeated launch/settlement/restart cycles converge instead of accumulating.
|
|
1240
1996
|
*
|
|
1241
|
-
*
|
|
1997
|
+
* Four rules, and each exists to refuse a specific way this goes wrong:
|
|
1242
1998
|
*
|
|
1243
|
-
* - **
|
|
1244
|
-
*
|
|
1245
|
-
*
|
|
1246
|
-
*
|
|
1247
|
-
*
|
|
1248
|
-
*
|
|
1249
|
-
*
|
|
1250
|
-
*
|
|
1251
|
-
*
|
|
1252
|
-
*
|
|
1999
|
+
* - **Only owned workspaces are touched.** Scope is the projects of the live
|
|
2000
|
+
* set plus the caller's managed list; ownership is the marking token OR the
|
|
2001
|
+
* conductor-store record (#1035 review), so a Herdr restart that drops
|
|
2002
|
+
* tokens still finds the restored surface. A pane in another project's
|
|
2003
|
+
* worker workspace is invisible to this pass, which is what keeps conductor
|
|
2004
|
+
* and veltrosecurity from discovering, reusing, or cleaning each other.
|
|
2005
|
+
* - **Attribution is exact, structural first.** A pane belongs to the live run
|
|
2006
|
+
* the store recorded it under; the session id Herdr reports is the fallback.
|
|
2007
|
+
* Everything else in an owned workspace — a settled run's leftover, a
|
|
2008
|
+
* duplicate of a live run, a representation whose identity Herdr dropped —
|
|
2009
|
+
* is closed, because inside an owned workspace "ours" and "junk" are the
|
|
2010
|
+
* only options and leaving junk is what filled the operator's layout.
|
|
2011
|
+
* - **A live run keeps exactly one living representation.** Duplicates fold
|
|
2012
|
+
* into the recorded pane; a pane whose follower has exited is replaced while
|
|
2013
|
+
* the run is live; a missing pane is recreated — all bounded by the same
|
|
2014
|
+
* re-establishment budget (#998).
|
|
2015
|
+
* - **Nothing here can stop a worker.** Panes hold followers only; the
|
|
2016
|
+
* authoritative child lives in the daemon's process tree and is never
|
|
2017
|
+
* signalled, and worker liveness comes from the caller's live set alone.
|
|
1253
2018
|
*/
|
|
1254
2019
|
export function reconcileWorkerPanes(
|
|
1255
2020
|
live: readonly LiveWorkerPane[],
|
|
@@ -1258,39 +2023,184 @@ export function reconcileWorkerPanes(
|
|
|
1258
2023
|
* and a daemon restart legitimately gets a fresh budget (#998). */
|
|
1259
2024
|
attempts: ReadonlyMap<string, number> = new Map(),
|
|
1260
2025
|
maxAttempts = PANE_REATTEMPT_MAX,
|
|
2026
|
+
/** Projects this pass manages even when they have no live runs: their stale
|
|
2027
|
+
* panes are cleaned and a workspace left empty by that cleanup is removed. */
|
|
2028
|
+
managedProjects: readonly string[] = [],
|
|
1261
2029
|
): { ok: true; outcomes: WorkerPaneReconciliation[] } | { ok: false; reason: string } {
|
|
1262
|
-
const
|
|
1263
|
-
|
|
1264
|
-
const
|
|
1265
|
-
|
|
2030
|
+
const run = deps.run ?? realHerdrRun;
|
|
2031
|
+
const session = deps.session ?? resolveHerdrSession();
|
|
2032
|
+
const base = ["--session", session];
|
|
2033
|
+
|
|
2034
|
+
const scope = new Set([...managedProjects, ...live.map((worker) => worker.project)]);
|
|
2035
|
+
const discovered = ownedWorkerWorkspaces([...scope], deps);
|
|
2036
|
+
if (!discovered.ok) return { ok: false, reason: discovered.reason };
|
|
2037
|
+
// Deterministic order so outcomes and closes are reproducible pass to pass.
|
|
2038
|
+
const scoped = [...discovered.workspaces].sort((a, b) => (a.workspaceId < b.workspaceId ? -1 : 1));
|
|
2039
|
+
|
|
2040
|
+
// Every marked workspace of every scoped project converges (#1035 review):
|
|
2041
|
+
// duplicate token-marked workspaces are cleaned like any other, so a settled
|
|
2042
|
+
// project cannot keep stale shells in a second marked surface nobody tracks.
|
|
2043
|
+
const entries: { ws: WorkerWorkspace; panes: ListedPane[] }[] = [];
|
|
2044
|
+
for (const ws of scoped) {
|
|
2045
|
+
const res = run([...base, "pane", "list", "--workspace", ws.workspaceId]);
|
|
2046
|
+
if (!res.ok) {
|
|
2047
|
+
// An unreadable workspace is not evidence that anything is stale, so the
|
|
2048
|
+
// pass mutates nothing rather than closing on a guess.
|
|
2049
|
+
return { ok: false, reason: `herdr pane list failed: ${firstLine(res.stderr) || "no output"}` };
|
|
2050
|
+
}
|
|
2051
|
+
const parsed = parsePaneRecords(res.stdout);
|
|
2052
|
+
if (parsed === undefined) return { ok: false, reason: "herdr pane list carried no pane array" };
|
|
2053
|
+
entries.push({ ws, panes: parsed.panes });
|
|
2054
|
+
}
|
|
2055
|
+
|
|
2056
|
+
const liveByRun = new Map(live.map((worker) => [worker.runId, worker]));
|
|
1266
2057
|
const outcomes: WorkerPaneReconciliation[] = [];
|
|
2058
|
+
/** Panes this pass will close, with the identity the outcome names. */
|
|
2059
|
+
const junk = new Map<string, { runId: string; cause: StaleCause; agent?: string }>();
|
|
2060
|
+
/** Panes still standing when cleanup ends, keyed to their workspace — an
|
|
2061
|
+
* occupied workspace survives, whichever marked surface it sits in. */
|
|
2062
|
+
const standingWs = new Map<string, string>();
|
|
2063
|
+
/** Workspaces that gained a representation this pass, read from the tracked
|
|
2064
|
+
* outcomes: their listing predates the new pane, so removal never judges
|
|
2065
|
+
* them on it. */
|
|
2066
|
+
const openedWs = new Set<string>();
|
|
2067
|
+
|
|
2068
|
+
/** One representation per live run: the pane plus the marked workspace that
|
|
2069
|
+
* holds it. Attribution runs across ALL of a project's marked workspaces,
|
|
2070
|
+
* because the recorded pane may sit in any of them. */
|
|
2071
|
+
const held = new Map<string, { pane: ListedPane; wsId: string }>();
|
|
2072
|
+
for (const entry of entries) {
|
|
2073
|
+
const projectLive = live.filter((worker) => worker.project === entry.ws.project);
|
|
2074
|
+
for (const pane of entry.panes) {
|
|
2075
|
+
standingWs.set(pane.paneId, entry.ws.workspaceId);
|
|
2076
|
+
const owner =
|
|
2077
|
+
projectLive.find((worker) => worker.paneId === pane.paneId) ??
|
|
2078
|
+
(pane.runId === "" ? undefined : liveByRun.get(pane.runId));
|
|
2079
|
+
if (owner === undefined || owner.project !== entry.ws.project) {
|
|
2080
|
+
// Not attributable to any live run of this project: a settled run's
|
|
2081
|
+
// leftover (still carrying its old id) or an unidentified shell.
|
|
2082
|
+
junk.set(pane.paneId, {
|
|
2083
|
+
runId: pane.runId,
|
|
2084
|
+
cause: pane.runId === "" ? "unidentified" : "settled",
|
|
2085
|
+
...(pane.agent === undefined ? {} : { agent: pane.agent }),
|
|
2086
|
+
});
|
|
2087
|
+
continue;
|
|
2088
|
+
}
|
|
2089
|
+
const bucket = held.get(owner.runId);
|
|
2090
|
+
if (bucket === undefined) {
|
|
2091
|
+
held.set(owner.runId, { pane, wsId: entry.ws.workspaceId });
|
|
2092
|
+
continue;
|
|
2093
|
+
}
|
|
2094
|
+
// Exactly one focusable representation per live run: the recorded pane
|
|
2095
|
+
// wins so the durable row stays true without a write; the other is a
|
|
2096
|
+
// duplicate (#1035).
|
|
2097
|
+
const keep =
|
|
2098
|
+
bucket.pane.paneId === owner.paneId
|
|
2099
|
+
? bucket
|
|
2100
|
+
: pane.paneId === owner.paneId
|
|
2101
|
+
? { pane, wsId: entry.ws.workspaceId }
|
|
2102
|
+
: bucket;
|
|
2103
|
+
const drop = keep.pane.paneId === bucket.pane.paneId ? pane : bucket.pane;
|
|
2104
|
+
held.set(owner.runId, keep);
|
|
2105
|
+
junk.set(drop.paneId, {
|
|
2106
|
+
runId: owner.runId,
|
|
2107
|
+
cause: "duplicate",
|
|
2108
|
+
...(drop.agent === undefined ? {} : { agent: drop.agent }),
|
|
2109
|
+
});
|
|
2110
|
+
}
|
|
2111
|
+
}
|
|
1267
2112
|
|
|
1268
2113
|
for (const worker of live) {
|
|
1269
|
-
const
|
|
1270
|
-
if (
|
|
1271
|
-
|
|
2114
|
+
const hold = held.get(worker.runId);
|
|
2115
|
+
if (hold === undefined) {
|
|
2116
|
+
// No representation anywhere in the marked workspaces. Only the pid makes
|
|
2117
|
+
// a replacement honest: the pane represents an exact child, so without
|
|
2118
|
+
// one there is nothing to represent and inventing a pane would be the
|
|
2119
|
+
// silent claim this whole surface exists to avoid.
|
|
2120
|
+
if (worker.pid === undefined) {
|
|
2121
|
+
outcomes.push({
|
|
2122
|
+
kind: "untracked",
|
|
2123
|
+
runId: worker.runId,
|
|
2124
|
+
reason: "no session-host pid was ever recorded for this run",
|
|
2125
|
+
});
|
|
2126
|
+
continue;
|
|
2127
|
+
}
|
|
2128
|
+
// The budget, checked before the attempt: an exhausted run is skipped
|
|
2129
|
+
// silently here and reported once by the caller (#998).
|
|
2130
|
+
const spent = attempts.get(worker.runId) ?? 0;
|
|
2131
|
+
if (spent >= maxAttempts) {
|
|
2132
|
+
outcomes.push({ kind: "attempts-exhausted", runId: worker.runId, attempts: spent });
|
|
2133
|
+
continue;
|
|
2134
|
+
}
|
|
2135
|
+
const opened = openWorkerPane(
|
|
2136
|
+
{
|
|
2137
|
+
project: worker.project,
|
|
2138
|
+
issue: worker.issue,
|
|
2139
|
+
attempt: worker.attempt,
|
|
2140
|
+
runId: worker.runId,
|
|
2141
|
+
pid: worker.pid,
|
|
2142
|
+
...(worker.sessionFile === undefined ? {} : { sessionFile: worker.sessionFile }),
|
|
2143
|
+
},
|
|
2144
|
+
deps,
|
|
2145
|
+
);
|
|
2146
|
+
if (opened.kind === "tracked") {
|
|
2147
|
+
if (opened.workspaceId !== undefined) openedWs.add(opened.workspaceId);
|
|
2148
|
+
outcomes.push({
|
|
2149
|
+
kind: "reassociated",
|
|
2150
|
+
runId: worker.runId,
|
|
2151
|
+
paneId: opened.paneId,
|
|
2152
|
+
label: opened.label,
|
|
2153
|
+
});
|
|
2154
|
+
} else {
|
|
2155
|
+
outcomes.push({
|
|
2156
|
+
kind: "untracked",
|
|
2157
|
+
runId: worker.runId,
|
|
2158
|
+
reason: opened.reason,
|
|
2159
|
+
attempted: true,
|
|
2160
|
+
...(opened.phase === undefined ? {} : { phase: opened.phase }),
|
|
2161
|
+
});
|
|
2162
|
+
}
|
|
2163
|
+
continue;
|
|
2164
|
+
}
|
|
2165
|
+
// The pane stands. Whether it still shows anything is the follower's
|
|
2166
|
+
// question, asked only to decide the *visual* — never settlement (#1035).
|
|
2167
|
+
// A probe that cannot be read, or a table that cannot decide, is no
|
|
2168
|
+
// evidence the visual died: it is reported and the pane keeps standing,
|
|
2169
|
+
// because a live worker's representation is never destroyed on a guess.
|
|
2170
|
+
const pulse = paneFollowerAlive(hold.pane.paneId, deps);
|
|
2171
|
+
if (pulse.pulse !== "dead") {
|
|
2172
|
+
outcomes.push(
|
|
2173
|
+
pulse.pulse === "alive"
|
|
2174
|
+
? { kind: "intact", runId: worker.runId, paneId: hold.pane.paneId }
|
|
2175
|
+
: {
|
|
2176
|
+
kind: "visual-unreadable",
|
|
2177
|
+
runId: worker.runId,
|
|
2178
|
+
paneId: hold.pane.paneId,
|
|
2179
|
+
reason: pulse.reason,
|
|
2180
|
+
},
|
|
2181
|
+
);
|
|
1272
2182
|
continue;
|
|
1273
2183
|
}
|
|
1274
|
-
//
|
|
1275
|
-
//
|
|
1276
|
-
//
|
|
1277
|
-
//
|
|
2184
|
+
// The follower has provably exited. A replacement represents the exact
|
|
2185
|
+
// child, so a run with no recorded pid keeps its dead pane rather than
|
|
2186
|
+
// gaining a representation of nobody — reported so the record drops and
|
|
2187
|
+
// the next pass reads the orphan as junk.
|
|
1278
2188
|
if (worker.pid === undefined) {
|
|
1279
2189
|
outcomes.push({
|
|
1280
2190
|
kind: "untracked",
|
|
1281
2191
|
runId: worker.runId,
|
|
1282
|
-
reason: "no session-host pid was ever recorded
|
|
2192
|
+
reason: "follower exited but no session-host pid was ever recorded; not replacing",
|
|
1283
2193
|
});
|
|
1284
2194
|
continue;
|
|
1285
2195
|
}
|
|
1286
|
-
// The budget, checked before the attempt: an exhausted run is skipped
|
|
1287
|
-
// silently here and reported once by the caller (#998).
|
|
1288
2196
|
const spent = attempts.get(worker.runId) ?? 0;
|
|
1289
2197
|
if (spent >= maxAttempts) {
|
|
2198
|
+
// Out of budget: the dead pane keeps its last state until a restart
|
|
2199
|
+
// refreshes the budget or the run settles and retires it outright.
|
|
1290
2200
|
outcomes.push({ kind: "attempts-exhausted", runId: worker.runId, attempts: spent });
|
|
1291
2201
|
continue;
|
|
1292
2202
|
}
|
|
1293
|
-
const
|
|
2203
|
+
const replaced = openWorkerPane(
|
|
1294
2204
|
{
|
|
1295
2205
|
project: worker.project,
|
|
1296
2206
|
issue: worker.issue,
|
|
@@ -1301,36 +2211,89 @@ export function reconcileWorkerPanes(
|
|
|
1301
2211
|
},
|
|
1302
2212
|
deps,
|
|
1303
2213
|
);
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
2214
|
+
if (replaced.kind === "tracked") {
|
|
2215
|
+
if (replaced.workspaceId !== undefined) openedWs.add(replaced.workspaceId);
|
|
2216
|
+
outcomes.push({
|
|
2217
|
+
kind: "reassociated",
|
|
2218
|
+
runId: worker.runId,
|
|
2219
|
+
paneId: replaced.paneId,
|
|
2220
|
+
label: replaced.label,
|
|
2221
|
+
});
|
|
2222
|
+
junk.set(hold.pane.paneId, {
|
|
2223
|
+
runId: worker.runId,
|
|
2224
|
+
cause: "duplicate",
|
|
2225
|
+
...(hold.pane.agent === undefined ? {} : { agent: hold.pane.agent }),
|
|
2226
|
+
});
|
|
2227
|
+
} else {
|
|
2228
|
+
outcomes.push({
|
|
2229
|
+
kind: "untracked",
|
|
2230
|
+
runId: worker.runId,
|
|
2231
|
+
reason: replaced.reason,
|
|
2232
|
+
attempted: true,
|
|
2233
|
+
...(replaced.phase === undefined ? {} : { phase: replaced.phase }),
|
|
2234
|
+
});
|
|
2235
|
+
// The replacement failed; the dead pane has nothing left to show, so it
|
|
2236
|
+
// goes now rather than advertising a worker for another pass. Its record
|
|
2237
|
+
// is dropped either way, so a survivor is junk next pass.
|
|
2238
|
+
closeWorkerPane(hold.pane.paneId, deps);
|
|
2239
|
+
standingWs.delete(hold.pane.paneId);
|
|
2240
|
+
}
|
|
2241
|
+
}
|
|
2242
|
+
|
|
2243
|
+
for (const [paneId, info] of junk) {
|
|
2244
|
+
const released =
|
|
2245
|
+
info.agent === undefined ? ({ ok: true } as const) : releaseWorkerPane(paneId, info.agent, deps);
|
|
2246
|
+
if (released.ok) {
|
|
2247
|
+
const closed = closeWorkerPane(paneId, deps);
|
|
2248
|
+
if (closed.ok) {
|
|
2249
|
+
standingWs.delete(paneId);
|
|
2250
|
+
outcomes.push({ kind: "stale-released", paneId, runId: info.runId, cause: info.cause });
|
|
2251
|
+
continue;
|
|
2252
|
+
}
|
|
2253
|
+
outcomes.push({
|
|
2254
|
+
kind: "stale-release-failed",
|
|
2255
|
+
paneId,
|
|
2256
|
+
runId: info.runId,
|
|
2257
|
+
reason: closed.reason,
|
|
2258
|
+
cause: info.cause,
|
|
2259
|
+
});
|
|
2260
|
+
continue;
|
|
2261
|
+
}
|
|
2262
|
+
outcomes.push({
|
|
2263
|
+
kind: "stale-release-failed",
|
|
2264
|
+
paneId,
|
|
2265
|
+
runId: info.runId,
|
|
2266
|
+
reason: released.reason,
|
|
2267
|
+
cause: info.cause,
|
|
2268
|
+
});
|
|
1309
2269
|
}
|
|
1310
2270
|
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
2271
|
+
// Every scoped marked workspace whose panes the cleanup above emptied, and
|
|
2272
|
+
// that gained no representation this pass, is closed — however many marked
|
|
2273
|
+
// surfaces a project accumulated (#1035 review). A live project's primary
|
|
2274
|
+
// workspace survives because its representation keeps it standing; extras
|
|
2275
|
+
// converge away; repeated cycles hold at zero shells.
|
|
2276
|
+
for (const entry of entries) {
|
|
2277
|
+
if (openedWs.has(entry.ws.workspaceId)) continue;
|
|
2278
|
+
if ([...standingWs.values()].some((wsId) => wsId === entry.ws.workspaceId)) continue;
|
|
2279
|
+
const res = run([...base, "workspace", "close", entry.ws.workspaceId]);
|
|
2280
|
+
if (res.ok) deps.ownership?.forgetWorkspace(entry.ws.project, entry.ws.workspaceId);
|
|
2281
|
+
// A failed close keeps the record: the workspace may still exist, and
|
|
2282
|
+
// forgetting ownership of a live surface is how orphans are manufactured.
|
|
1314
2283
|
outcomes.push(
|
|
1315
|
-
|
|
1316
|
-
? { kind: "
|
|
1317
|
-
: {
|
|
2284
|
+
res.ok
|
|
2285
|
+
? { kind: "workspace-removed", project: entry.ws.project, workspaceId: entry.ws.workspaceId }
|
|
2286
|
+
: {
|
|
2287
|
+
kind: "workspace-remove-failed",
|
|
2288
|
+
project: entry.ws.project,
|
|
2289
|
+
workspaceId: entry.ws.workspaceId,
|
|
2290
|
+
reason: `herdr workspace close failed: ${firstLine(res.stderr) || "no output"}`,
|
|
2291
|
+
},
|
|
1318
2292
|
);
|
|
1319
2293
|
}
|
|
1320
2294
|
return { ok: true, outcomes };
|
|
1321
2295
|
}
|
|
1322
2296
|
|
|
1323
|
-
function firstLine(text: string): string {
|
|
1324
|
-
return text.split("\n", 1)[0]?.trim() ?? "";
|
|
1325
|
-
}
|
|
1326
|
-
|
|
1327
|
-
/** The kill syscall, injectable so error mapping is testable without one. */
|
|
1328
|
-
export type KillFn = (pid: number, sig: NodeJS.Signals | 0) => void;
|
|
1329
|
-
|
|
1330
|
-
const realKill: KillFn = (pid, sig) => {
|
|
1331
|
-
process.kill(pid, sig);
|
|
1332
|
-
};
|
|
1333
|
-
|
|
1334
2297
|
/**
|
|
1335
2298
|
* What a failed `kill` proves.
|
|
1336
2299
|
*
|
|
@@ -2556,10 +3519,13 @@ export function sessionsRoot(): string {
|
|
|
2556
3519
|
* claim-only verdict consumes this — the challenge proof reads conductor
|
|
2557
3520
|
* state, not transcripts (#614).
|
|
2558
3521
|
*/
|
|
2559
|
-
function claimedOrchestratorSessionFile(
|
|
3522
|
+
function claimedOrchestratorSessionFile(
|
|
3523
|
+
named: string | undefined,
|
|
3524
|
+
alive?: (pid: number) => boolean,
|
|
3525
|
+
): string | undefined {
|
|
2560
3526
|
if (named === undefined) return undefined;
|
|
2561
3527
|
try {
|
|
2562
|
-
return resolveClaimedSessionFile(findProject(loadConfig(), named));
|
|
3528
|
+
return resolveClaimedSessionFile(findProject(loadConfig(), named), alive);
|
|
2563
3529
|
} catch {
|
|
2564
3530
|
return undefined;
|
|
2565
3531
|
}
|
|
@@ -2573,11 +3539,11 @@ function makeChallengeCode(): string {
|
|
|
2573
3539
|
}
|
|
2574
3540
|
|
|
2575
3541
|
/**
|
|
2576
|
-
* mm:ss for
|
|
3542
|
+
* mm:ss for the window a challenge code stays good for. Deliberately not
|
|
2577
3543
|
* `formatDownDuration`, which rounds to whole minutes because it reports
|
|
2578
|
-
* hours-to-days outages: rounded to the minute,
|
|
2579
|
-
*
|
|
2580
|
-
*
|
|
3544
|
+
* hours-to-days outages: rounded to the minute, a code with 30 seconds left
|
|
3545
|
+
* would read "5m", which is the exact ambiguity these lines exist to remove
|
|
3546
|
+
* (#861).
|
|
2581
3547
|
*/
|
|
2582
3548
|
function armClock(ms: number): string {
|
|
2583
3549
|
const total = Math.max(0, Math.round(ms / 1000));
|
|
@@ -2586,58 +3552,6 @@ function armClock(ms: number): string {
|
|
|
2586
3552
|
return minutes === 0 ? `${seconds}s` : `${minutes}m${String(seconds).padStart(2, "0")}s`;
|
|
2587
3553
|
}
|
|
2588
3554
|
|
|
2589
|
-
/** How often the wait says it is still waiting. Six lines across the window. */
|
|
2590
|
-
const ARM_PROGRESS_INTERVAL_MS = 30_000;
|
|
2591
|
-
|
|
2592
|
-
/**
|
|
2593
|
-
* Polls the acknowledgement record for one exact challenge id until the
|
|
2594
|
-
* orchestrator's inbound adapter writes it or the deadline passes (#614).
|
|
2595
|
-
*
|
|
2596
|
-
* The state is a small JSON file re-read every pass, never snapshotted: the
|
|
2597
|
-
* acknowledgement may land at any point in the window, written by the live
|
|
2598
|
-
* orchestrator process. Only a record naming this exact id satisfies the
|
|
2599
|
-
* wait — an acknowledgement cut for a replaced challenge is inert here by
|
|
2600
|
-
* construction, and no transcript anywhere is opened.
|
|
2601
|
-
*
|
|
2602
|
-
* It also *says* it is waiting (#861). Measured 2026-08-21: an arm proof sat
|
|
2603
|
-
* silent for five minutes and was reported as a hung setup — the process was
|
|
2604
|
-
* healthy and the operator had no way to tell. A silent five-minute wait
|
|
2605
|
-
* inside a fence that holds dispatch is indistinguishable from a dead one, so
|
|
2606
|
-
* the elapsed/remaining line lands every {@link ARM_PROGRESS_INTERVAL_MS}
|
|
2607
|
-
* regardless of the (much shorter) poll cadence, and the terminal outcome is
|
|
2608
|
-
* always printed.
|
|
2609
|
-
*/
|
|
2610
|
-
async function waitForArmAcknowledgement(
|
|
2611
|
-
challengeId: string,
|
|
2612
|
-
timeoutMs: number,
|
|
2613
|
-
deps: ArmDeps,
|
|
2614
|
-
): Promise<boolean> {
|
|
2615
|
-
const now = deps.now ?? Date.now;
|
|
2616
|
-
const sleep = deps.sleep ?? ((ms: number) => new Promise<void>((r) => setTimeout(r, ms)));
|
|
2617
|
-
const report = deps.progress;
|
|
2618
|
-
const startedAt = now();
|
|
2619
|
-
const deadline = startedAt + timeoutMs;
|
|
2620
|
-
let nextReportAt = startedAt + ARM_PROGRESS_INTERVAL_MS;
|
|
2621
|
-
for (;;) {
|
|
2622
|
-
if (readArmAcknowledgement(challengeId) !== undefined) {
|
|
2623
|
-
report?.(`arm: reply acknowledged after ${armClock(now() - startedAt)} — the fleet is armed.`);
|
|
2624
|
-
return true;
|
|
2625
|
-
}
|
|
2626
|
-
const at = now();
|
|
2627
|
-
if (at >= deadline) return false;
|
|
2628
|
-
if (report !== undefined && at >= nextReportAt) {
|
|
2629
|
-
report(
|
|
2630
|
-
`arm: still waiting for the reply — ${armClock(at - startedAt)} elapsed, ` +
|
|
2631
|
-
`${armClock(deadline - at)} left. Nothing is stuck: reply in the Telegram chat with the code.`,
|
|
2632
|
-
);
|
|
2633
|
-
// Anchored to the clock, not to this pass, so a slow pass cannot make the
|
|
2634
|
-
// cadence drift into silence.
|
|
2635
|
-
while (nextReportAt <= at) nextReportAt += ARM_PROGRESS_INTERVAL_MS;
|
|
2636
|
-
}
|
|
2637
|
-
await sleep(5_000);
|
|
2638
|
-
}
|
|
2639
|
-
}
|
|
2640
|
-
|
|
2641
3555
|
/**
|
|
2642
3556
|
* The one armed-marker write both proofs share: same content, same mode, and
|
|
2643
3557
|
* the same restamp of the pre-per-project shared marker the heartbeat still
|