@ours.network/fleet 0.17.1 → 0.17.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -2
- package/dist/application/role-removal-service.js +1 -1
- package/dist/application/session-control.d.ts +14 -10
- package/dist/application/session-control.js +14 -3
- package/dist/atomic-file.d.ts +7 -1
- package/dist/atomic-file.js +33 -5
- package/dist/build-info.json +10 -0
- package/dist/capabilities.d.ts +20 -0
- package/dist/capabilities.js +21 -0
- package/dist/cli.js +98 -10
- package/dist/config.d.ts +9 -2
- package/dist/config.js +16 -2
- package/dist/creation.d.ts +16 -0
- package/dist/creation.js +28 -0
- package/dist/docs.d.ts +1 -1
- package/dist/docs.js +70 -4
- package/dist/doctor.d.ts +5 -0
- package/dist/doctor.js +87 -2
- package/dist/harness/acp-agent.d.ts +3 -0
- package/dist/harness/acp-agent.js +4 -1
- package/dist/harness/codex-app-server-proxy.d.ts +4 -0
- package/dist/harness/codex-app-server-proxy.js +133 -0
- package/dist/harness/codex.js +79 -11
- package/dist/index.d.ts +2 -1
- package/dist/index.js +1 -0
- package/dist/loops/manager.d.ts +42 -1
- package/dist/loops/manager.js +115 -16
- package/dist/loops/state.d.ts +46 -2
- package/dist/loops/state.js +81 -3
- package/dist/monitor.d.ts +21 -0
- package/dist/monitor.js +42 -0
- package/dist/ops.d.ts +6 -0
- package/dist/ops.js +46 -1
- package/dist/owner-channel/channel.d.ts +18 -2
- package/dist/owner-channel/channel.js +146 -2
- package/dist/owner-channel/commands.d.ts +2 -2
- package/dist/owner-channel/commands.js +7 -2
- package/dist/owner-channel/notices.d.ts +2 -0
- package/dist/owner-channel/notices.js +3 -0
- package/dist/provenance.d.ts +77 -0
- package/dist/provenance.js +283 -0
- package/dist/runner.d.ts +7 -1
- package/dist/runner.js +100 -14
- package/dist/session/acp.d.ts +40 -4
- package/dist/session/acp.js +157 -30
- package/dist/session/arbiter.d.ts +28 -2
- package/dist/session/arbiter.js +75 -4
- package/dist/session/control.js +12 -6
- package/dist/session/event-log.d.ts +109 -0
- package/dist/session/event-log.js +247 -0
- package/dist/session/events.d.ts +21 -0
- package/dist/session/events.js +105 -26
- package/dist/session/tmux.d.ts +3 -2
- package/dist/session/tmux.js +2 -0
- package/dist/session/types.d.ts +39 -2
- package/dist/session/types.js +11 -1
- package/dist/spawn.d.ts +3 -3
- package/dist/spawn.js +40 -14
- package/dist/temp-lifecycle.d.ts +62 -0
- package/dist/temp-lifecycle.js +437 -0
- package/package.json +5 -3
package/dist/monitor.js
CHANGED
|
@@ -109,6 +109,48 @@ export function resolveEndpoint(env) {
|
|
|
109
109
|
headers: token ? { 'x-ours-api-token': token } : {},
|
|
110
110
|
};
|
|
111
111
|
}
|
|
112
|
+
/**
|
|
113
|
+
* Ask the daemon's authoritative identity index. The notifications endpoint is
|
|
114
|
+
* intentionally unsuitable for lifecycle: it serves an empty 200 page for a
|
|
115
|
+
* valid but missing identity, which made a closed temp identity look healthy.
|
|
116
|
+
*/
|
|
117
|
+
export async function probeIdentityPresence(name, fetch, env) {
|
|
118
|
+
const ep = resolveEndpoint(env);
|
|
119
|
+
let response;
|
|
120
|
+
try {
|
|
121
|
+
response = await fetch(`${ep.origin}/identities`, { headers: ep.headers });
|
|
122
|
+
}
|
|
123
|
+
catch (error) {
|
|
124
|
+
return { state: 'unknown', detail: `identity index unreachable (${msg(error)})` };
|
|
125
|
+
}
|
|
126
|
+
if (!response.ok)
|
|
127
|
+
return {
|
|
128
|
+
state: 'unknown',
|
|
129
|
+
detail: response.status === 401
|
|
130
|
+
? `daemon rejected the API token (401) — ${authResolutionHint(ep)}`
|
|
131
|
+
: `identity index returned HTTP ${response.status}`,
|
|
132
|
+
};
|
|
133
|
+
try {
|
|
134
|
+
const body = await response.json();
|
|
135
|
+
if (!Array.isArray(body.identities))
|
|
136
|
+
return { state: 'unknown', detail: 'identity index response is malformed' };
|
|
137
|
+
// A healthy daemon normally has at least its Human identity. During daemon
|
|
138
|
+
// restart, however, the authenticated endpoint can briefly serve a valid
|
|
139
|
+
// but empty index while state is still loading. Empty is therefore not
|
|
140
|
+
// enough authority to retire a live temporary role.
|
|
141
|
+
if (body.identities.length === 0)
|
|
142
|
+
return { state: 'unknown', detail: 'identity index is temporarily empty' };
|
|
143
|
+
const found = body.identities.find(identity => (typeof identity === 'string' ? identity : identity?.name) === name);
|
|
144
|
+
if (!found)
|
|
145
|
+
return { state: 'absent' };
|
|
146
|
+
return typeof found === 'string'
|
|
147
|
+
? { state: 'present', temporary: false, stale: false }
|
|
148
|
+
: { state: 'present', temporary: found.temporary === true, stale: found.stale === true };
|
|
149
|
+
}
|
|
150
|
+
catch (error) {
|
|
151
|
+
return { state: 'unknown', detail: `identity index response is unreadable (${msg(error)})` };
|
|
152
|
+
}
|
|
153
|
+
}
|
|
112
154
|
/** Actionable, secret-free description of every token source for this profile. */
|
|
113
155
|
export function authResolutionHint(ep) {
|
|
114
156
|
const tokenPath = join(ep.stateDir, 'daemon-token');
|
package/dist/ops.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { FleetConfig, ResolvedRole } from './config.js';
|
|
2
2
|
import type { InstallOutcome as BackendInstallOutcome, SupervisorBackend } from './supervisor/types.js';
|
|
3
|
+
import { type Exec } from './exec.js';
|
|
3
4
|
/** An install outcome tagged with the role it belongs to. */
|
|
4
5
|
export interface InstallOutcome extends BackendInstallOutcome {
|
|
5
6
|
role: string;
|
|
@@ -8,6 +9,11 @@ export interface OpsDeps {
|
|
|
8
9
|
backend: SupervisorBackend;
|
|
9
10
|
binPath: string;
|
|
10
11
|
log(line: string): void;
|
|
12
|
+
/** Process/service inspection for exact temporary-supervisor lifecycle commands. */
|
|
13
|
+
exec?: Exec;
|
|
14
|
+
/** Test seam for exact detached-supervisor signaling/liveness. */
|
|
15
|
+
kill?(pid: number, signal: NodeJS.Signals | 0): void;
|
|
16
|
+
sleep?(ms: number): Promise<void>;
|
|
11
17
|
/**
|
|
12
18
|
* Called the INSTANT a registration is created, before anything else can
|
|
13
19
|
* fail. A creation transaction that learns about registrations only from
|
package/dist/ops.js
CHANGED
|
@@ -6,6 +6,8 @@ import { findRole } from './config.js';
|
|
|
6
6
|
import { getAdapter } from './harness/registry.js';
|
|
7
7
|
import { generateBriefing } from './briefing.js';
|
|
8
8
|
import { resetRestartLedger } from './runner.js';
|
|
9
|
+
import { archiveTempState, stopTempSupervisor, tempSupervisorLiveness, } from './temp-lifecycle.js';
|
|
10
|
+
import { realExec } from './exec.js';
|
|
9
11
|
// Launch staggering now lives at the harness-launch point (the runner's start
|
|
10
12
|
// gate, driven by `start_stagger_ms`), so it covers systemd host-boot too — not
|
|
11
13
|
// just the `up`/`restart` command loop below. The old in-loop FLEET_START_STAGGER
|
|
@@ -174,7 +176,23 @@ async function reconcileWatchdogScheduler(cfg, deps, configPath) {
|
|
|
174
176
|
}
|
|
175
177
|
}
|
|
176
178
|
export async function down(cfg, names, deps) {
|
|
177
|
-
|
|
179
|
+
const roles = names.length ? names.map(name => cfg.roles.find(role => role.name === name)) : cfg.roles;
|
|
180
|
+
for (let index = 0; index < roles.length; index++) {
|
|
181
|
+
const role = roles[index];
|
|
182
|
+
const requested = names[index];
|
|
183
|
+
if (!role && requested && /^[A-Za-z0-9_-]+$/.test(requested)
|
|
184
|
+
&& existsSync(agentDir(requested, true))) {
|
|
185
|
+
try {
|
|
186
|
+
const outcome = await stopTempSupervisor(requested, { exec: deps.exec ?? realExec });
|
|
187
|
+
deps.log(`■ ${outcome === 'stopped' ? 'stopping' : 'stopped'} temporary role ${requested}`);
|
|
188
|
+
}
|
|
189
|
+
catch (e) {
|
|
190
|
+
deps.log(` ! could not stop temporary role ${requested}: ${e instanceof Error ? e.message : String(e)}`);
|
|
191
|
+
}
|
|
192
|
+
continue;
|
|
193
|
+
}
|
|
194
|
+
if (!role)
|
|
195
|
+
throw new Error(`no such role '${requested}'`);
|
|
178
196
|
// Never swallow the backend's reason. "maybe not running" hid real stop
|
|
179
197
|
// failures — a wedged unit, an unreachable user bus — behind a guess.
|
|
180
198
|
try {
|
|
@@ -213,6 +231,33 @@ export async function restartRoles(cfg, names, deps, mode, configPath) {
|
|
|
213
231
|
}
|
|
214
232
|
/** Stop + forget a role: unit, state dir, and its fleet.d file when spawned. */
|
|
215
233
|
export async function rmRole(cfg, name, deps) {
|
|
234
|
+
const temporaryDir = /^[A-Za-z0-9_-]+$/.test(name) ? agentDir(name, true) : '';
|
|
235
|
+
const configured = cfg.roles.find(role => role.name === name);
|
|
236
|
+
if (!configured && temporaryDir && existsSync(temporaryDir)) {
|
|
237
|
+
const lifecycleDeps = { exec: deps.exec ?? realExec, ...(deps.kill ? { kill: deps.kill } : {}) };
|
|
238
|
+
const stopOutcome = await stopTempSupervisor(name, lifecycleDeps);
|
|
239
|
+
const sleep = deps.sleep ?? ((ms) => new Promise(resolve => setTimeout(resolve, ms)));
|
|
240
|
+
let liveness = stopOutcome === 'already-stopped' ? 'stopped' : 'unknown';
|
|
241
|
+
// The supervisor normally archives itself. A detached fallback may keep
|
|
242
|
+
// running after SIGTERM, so directory age or a fixed delay is never
|
|
243
|
+
// cleanup authority: poll exact ownership and archive only after a proven
|
|
244
|
+
// stop. Unknown and still-running both fail closed with evidence in place.
|
|
245
|
+
for (let i = 0; i < 50 && existsSync(temporaryDir) && liveness !== 'stopped'; i++) {
|
|
246
|
+
liveness = await tempSupervisorLiveness(temporaryDir, lifecycleDeps);
|
|
247
|
+
if (liveness === 'stopped')
|
|
248
|
+
break;
|
|
249
|
+
if (i < 49)
|
|
250
|
+
await sleep(100);
|
|
251
|
+
}
|
|
252
|
+
if (existsSync(temporaryDir) && liveness !== 'stopped')
|
|
253
|
+
throw new Error(`temporary role '${name}' supervisor is ${liveness}; refusing to archive live or ambiguous state`);
|
|
254
|
+
const archived = existsSync(temporaryDir)
|
|
255
|
+
? archiveTempState(name, 'operator-stop', 'retired', 'operator removal completed after supervisor liveness was proven stopped; evidence preserved')
|
|
256
|
+
: undefined;
|
|
257
|
+
deps.log(`removed temporary role '${name}'`
|
|
258
|
+
+ `${archived ? `; evidence archived at ${archived}` : '; supervisor archived its evidence'}`);
|
|
259
|
+
return;
|
|
260
|
+
}
|
|
216
261
|
const role = findRole(cfg, name);
|
|
217
262
|
await deps.backend.uninstall(name);
|
|
218
263
|
rmSync(agentDir(name), { recursive: true, force: true });
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { type ChildProcessWithoutNullStreams } from 'node:child_process';
|
|
2
2
|
import { type OwnerChannelConfig } from '../config.js';
|
|
3
|
-
import type
|
|
3
|
+
import { type FetchLike } from '../monitor.js';
|
|
4
|
+
import { type SessionHandle } from '../session/types.js';
|
|
4
5
|
import { type OwnerFleetOps } from './commands.js';
|
|
5
6
|
import type { ManagedFleetSpawnResult } from '../fleet-proxy.js';
|
|
6
7
|
import { type OursToolClient } from './mcp.js';
|
|
@@ -19,8 +20,12 @@ export interface OwnerChannelOptions {
|
|
|
19
20
|
command?: string;
|
|
20
21
|
log(line: string): void;
|
|
21
22
|
client?: OursToolClient;
|
|
22
|
-
/**
|
|
23
|
+
/** Legacy child-process test seam; production uses the direct notification API. */
|
|
23
24
|
watch?: (identity: string) => ChildProcessWithoutNullStreams;
|
|
25
|
+
/** Test seam for the production direct notification long-poll. */
|
|
26
|
+
watchFetch?: FetchLike;
|
|
27
|
+
/** Test seam for the long-poll stall bound; production uses OWNER_WATCH_STALL_MS. */
|
|
28
|
+
watchStallMs?: number;
|
|
24
29
|
/** Test seam; production uses the detached ours-fleet CLI (`fleetCliOps`). */
|
|
25
30
|
fleet?: OwnerFleetOps;
|
|
26
31
|
/** Forwarded to fleet CLI invocations spawned for owner commands. */
|
|
@@ -152,6 +157,7 @@ export declare class OwnerChannel implements OwnerChannelHandle {
|
|
|
152
157
|
private stopping;
|
|
153
158
|
private watchProcess?;
|
|
154
159
|
private watchTask?;
|
|
160
|
+
private watchAbort?;
|
|
155
161
|
private drainTask?;
|
|
156
162
|
private drainRequested;
|
|
157
163
|
private readonly completionTasks;
|
|
@@ -250,6 +256,16 @@ export declare class OwnerChannel implements OwnerChannelHandle {
|
|
|
250
256
|
/** Map only event shape and allowlisted status to owner-safe phase text. */
|
|
251
257
|
private progressPhase;
|
|
252
258
|
private watchLoop;
|
|
259
|
+
/**
|
|
260
|
+
* `recovered` distinguishes a first-ever start (no state, nothing lost) from a
|
|
261
|
+
* cursor we HAD and can no longer read. Only the latter is a recovery, and the
|
|
262
|
+
* caller needs to know because the reason it reports is the only evidence a
|
|
263
|
+
* durable cursor was ever lost.
|
|
264
|
+
*/
|
|
265
|
+
private readWatchState;
|
|
266
|
+
private writeWatchState;
|
|
267
|
+
/** Compatibility path for injected child-process tests; production is direct. */
|
|
268
|
+
private legacyWatchLoop;
|
|
253
269
|
private errorText;
|
|
254
270
|
private logError;
|
|
255
271
|
}
|
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
import { spawn } from 'node:child_process';
|
|
2
2
|
import { createHash, randomUUID } from 'node:crypto';
|
|
3
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
3
4
|
import { mkdir, readFile, readdir, rm } from 'node:fs/promises';
|
|
4
5
|
import { createInterface } from 'node:readline';
|
|
5
6
|
import { join } from 'node:path';
|
|
6
7
|
import { DEFAULT_OWNER_ATTACHMENT_MIME, canonicalCid, } from '../config.js';
|
|
8
|
+
import { replaceFileAtomically } from '../atomic-file.js';
|
|
9
|
+
import { resolveEndpoint } from '../monitor.js';
|
|
10
|
+
import { ACP_CANCEL_DEADLINE_EXCEEDED, SessionControlError, interruptOutcome, } from '../session/types.js';
|
|
7
11
|
import { VERSION } from '../version.js';
|
|
8
12
|
import { dispatchOwnerCommand, fleetCliOps, isOwnerCommandText, } from './commands.js';
|
|
9
13
|
import { OursMcpClient } from './mcp.js';
|
|
@@ -24,6 +28,15 @@ const COMMENTARY_MAX_CHARS = 1_600;
|
|
|
24
28
|
const COMMENTARY_MAX_BYTES = 6_400;
|
|
25
29
|
const COMMENTARY_MAX_UPDATES = 32;
|
|
26
30
|
const COMMENTARY_DEDUPE_LIMIT = 512;
|
|
31
|
+
const OWNER_WATCH_STALL_MS = 120_000;
|
|
32
|
+
const OWNER_WATCH_BACKOFF_MAX_MS = 30_000;
|
|
33
|
+
/**
|
|
34
|
+
* Credentials that are wrong now are wrong on the next attempt too. Retrying a
|
|
35
|
+
* permanent 401 forever burns the daemon and hides the real fault behind an
|
|
36
|
+
* endless reconnect log, so the watch stops after this many consecutive auth
|
|
37
|
+
* rejections and records a terminal reason instead.
|
|
38
|
+
*/
|
|
39
|
+
const OWNER_WATCH_AUTH_FATAL_ATTEMPTS = 5;
|
|
27
40
|
/** A relay attempt that failed only because no owner route exists yet. */
|
|
28
41
|
class RelayUnroutableError extends Error {
|
|
29
42
|
}
|
|
@@ -59,6 +72,7 @@ export class OwnerChannel {
|
|
|
59
72
|
stopping = false;
|
|
60
73
|
watchProcess;
|
|
61
74
|
watchTask;
|
|
75
|
+
watchAbort;
|
|
62
76
|
drainTask;
|
|
63
77
|
drainRequested = false;
|
|
64
78
|
completionTasks = new Set();
|
|
@@ -136,7 +150,7 @@ export class OwnerChannel {
|
|
|
136
150
|
void cleanupAttachmentRoot(this.attachmentRoot, Date.now(), this.attachmentConfig.retention_ms).catch(error => this.logError('attachment crash cleanup failed', error));
|
|
137
151
|
}
|
|
138
152
|
this.ready = true;
|
|
139
|
-
this.watchTask = this.watchLoop();
|
|
153
|
+
this.watchTask = this.options.watch ? this.legacyWatchLoop() : this.watchLoop();
|
|
140
154
|
// Do not make role startup wait for an old owner request to finish a turn.
|
|
141
155
|
void this.drain().catch(error => this.logError('initial drain failed', error));
|
|
142
156
|
}
|
|
@@ -159,6 +173,10 @@ export class OwnerChannel {
|
|
|
159
173
|
this.watchProcess = undefined;
|
|
160
174
|
if (watch && watch.exitCode === null)
|
|
161
175
|
watch.kill('SIGTERM');
|
|
176
|
+
this.watchAbort?.abort();
|
|
177
|
+
if (!this.options.watch)
|
|
178
|
+
await this.watchTask?.catch(error => this.logError('watch shutdown failed', error));
|
|
179
|
+
this.watchTask = undefined;
|
|
162
180
|
await this.managementTail;
|
|
163
181
|
try {
|
|
164
182
|
await this.client.close();
|
|
@@ -825,6 +843,15 @@ export class OwnerChannel {
|
|
|
825
843
|
}
|
|
826
844
|
catch (error) {
|
|
827
845
|
await rm(outbox, { recursive: true, force: true });
|
|
846
|
+
if (error instanceof SessionControlError
|
|
847
|
+
&& error.reasonCode === ACP_CANCEL_DEADLINE_EXCEEDED) {
|
|
848
|
+
// drainAll deferred this authenticated message before delivery. The
|
|
849
|
+
// adapter generation is terminating, so leave the wire unhandled and
|
|
850
|
+
// body-free: the resumed owner channel will replay it exactly once.
|
|
851
|
+
this.options.log(`[${this.options.role}] owner request ${requestId.slice(0, 12)} `
|
|
852
|
+
+ `held for adapter resume reason=${ACP_CANCEL_DEADLINE_EXCEEDED}`);
|
|
853
|
+
return false;
|
|
854
|
+
}
|
|
828
855
|
this.logError('request delivery failed', error);
|
|
829
856
|
await this.send(sender.id, ownerNotices.deliveryFailed(this.options.role), wireId);
|
|
830
857
|
this.state.remember(wireId);
|
|
@@ -874,7 +901,7 @@ export class OwnerChannel {
|
|
|
874
901
|
harness: this.options.harness,
|
|
875
902
|
version: VERSION,
|
|
876
903
|
snapshot: () => this.options.session.snapshot(),
|
|
877
|
-
interrupt: () => this.options.session.interrupt('owner'),
|
|
904
|
+
interrupt: async () => interruptOutcome(await this.options.session.interrupt('owner')),
|
|
878
905
|
runHarnessCommand: command => this.runHarnessCommand(sender, command, wireId),
|
|
879
906
|
restart: mode => this.restartSelf(sender, mode, wireId),
|
|
880
907
|
comments: () => this.commentsState(),
|
|
@@ -1549,6 +1576,123 @@ export class OwnerChannel {
|
|
|
1549
1576
|
return undefined;
|
|
1550
1577
|
}
|
|
1551
1578
|
async watchLoop() {
|
|
1579
|
+
const endpoint = resolveEndpoint({ ...process.env, ...(this.options.env ?? {}) });
|
|
1580
|
+
const fetch = this.options.watchFetch
|
|
1581
|
+
?? ((url, init) => globalThis.fetch(url, init));
|
|
1582
|
+
const sleep = this.options.binderDeps?.sleep
|
|
1583
|
+
?? (ms => new Promise(resolve => setTimeout(resolve, ms)));
|
|
1584
|
+
const restored = this.readWatchState();
|
|
1585
|
+
let state = restored.state;
|
|
1586
|
+
// An unreadable cursor is the ONLY reason to restart at the tip. Say so on
|
|
1587
|
+
// the next successful connect, and drain first: everything the lost cursor
|
|
1588
|
+
// would have pointed at is still in the inbox, which is the authority.
|
|
1589
|
+
let recovering = restored.recovered;
|
|
1590
|
+
let cursor = state?.cursor ?? 'tip';
|
|
1591
|
+
let delayMs = 1_000;
|
|
1592
|
+
let authFailures = 0;
|
|
1593
|
+
if (recovering)
|
|
1594
|
+
await this.drain().catch(error => this.logError('cursor recovery drain failed', error));
|
|
1595
|
+
while (!this.stopping) {
|
|
1596
|
+
const ctrl = new AbortController();
|
|
1597
|
+
this.watchAbort = ctrl;
|
|
1598
|
+
let stalled = false;
|
|
1599
|
+
let authRejected = false;
|
|
1600
|
+
const timer = setTimeout(() => { stalled = true; ctrl.abort(); }, this.options.watchStallMs ?? OWNER_WATCH_STALL_MS);
|
|
1601
|
+
timer.unref?.();
|
|
1602
|
+
try {
|
|
1603
|
+
const response = await fetch(`${endpoint.url(this.options.config.identity)}?since=${cursor}`, { headers: endpoint.headers, signal: ctrl.signal });
|
|
1604
|
+
if (response.status === 401) {
|
|
1605
|
+
authRejected = true;
|
|
1606
|
+
authFailures++;
|
|
1607
|
+
const at = cursor === 'tip' ? state?.cursor ?? 0 : cursor;
|
|
1608
|
+
if (authFailures >= OWNER_WATCH_AUTH_FATAL_ATTEMPTS) {
|
|
1609
|
+
state = this.writeWatchState(state, at, 'OWNER_WATCH_AUTH_FATAL', true);
|
|
1610
|
+
this.options.log(`[${this.options.role}] owner watch stopped `
|
|
1611
|
+
+ `reason=OWNER_WATCH_AUTH_FATAL after ${authFailures} consecutive HTTP 401 responses; `
|
|
1612
|
+
+ 'owner notifications require re-authorization and will not be retried');
|
|
1613
|
+
return;
|
|
1614
|
+
}
|
|
1615
|
+
state = this.writeWatchState(state, at, 'OWNER_WATCH_AUTH_FAILED', true);
|
|
1616
|
+
throw new Error('OWNER_WATCH_AUTH_FAILED: daemon rejected notification credentials');
|
|
1617
|
+
}
|
|
1618
|
+
if (!response.ok)
|
|
1619
|
+
throw new Error(`daemon returned HTTP ${response.status}`);
|
|
1620
|
+
const body = await response.json();
|
|
1621
|
+
const next = typeof body.cursor === 'number' ? body.cursor : cursor === 'tip' ? 0 : cursor;
|
|
1622
|
+
const reconnect = (state?.consecutiveFailures ?? 0) > 0;
|
|
1623
|
+
state = this.writeWatchState(state, next, recovering ? 'OWNER_WATCH_CURSOR_RECOVERED' : 'OWNER_WATCH_CONNECTED', false, reconnect);
|
|
1624
|
+
recovering = false;
|
|
1625
|
+
authFailures = 0;
|
|
1626
|
+
cursor = next;
|
|
1627
|
+
delayMs = 1_000;
|
|
1628
|
+
// Notification events are content-free hints. The inbox remains the
|
|
1629
|
+
// authority and its wire-level durable dedupe prevents duplicate turns.
|
|
1630
|
+
if ((body.events?.length ?? 0) > 0)
|
|
1631
|
+
await this.drain();
|
|
1632
|
+
}
|
|
1633
|
+
catch (error) {
|
|
1634
|
+
if (this.stopping)
|
|
1635
|
+
return;
|
|
1636
|
+
const reason = stalled
|
|
1637
|
+
? 'OWNER_WATCH_STALLED' : 'OWNER_WATCH_STREAM_ERROR';
|
|
1638
|
+
const current = cursor === 'tip' ? state?.cursor ?? 0 : cursor;
|
|
1639
|
+
cursor = current;
|
|
1640
|
+
// Only THIS iteration's auth rejection is already written. A stale
|
|
1641
|
+
// AUTH_FAILED from an earlier attempt must never suppress the cursor,
|
|
1642
|
+
// the failure counter, or a later STALLED transition.
|
|
1643
|
+
if (!authRejected)
|
|
1644
|
+
state = this.writeWatchState(state, current, reason, true);
|
|
1645
|
+
this.options.log(`[${this.options.role}] owner watch reconnect `
|
|
1646
|
+
+ `reason=${authRejected ? 'OWNER_WATCH_AUTH_FAILED' : reason} `
|
|
1647
|
+
+ `delay_ms=${delayMs} cursor=${current}`);
|
|
1648
|
+
await sleep(delayMs);
|
|
1649
|
+
delayMs = Math.min(delayMs * 2, OWNER_WATCH_BACKOFF_MAX_MS);
|
|
1650
|
+
}
|
|
1651
|
+
finally {
|
|
1652
|
+
clearTimeout(timer);
|
|
1653
|
+
if (this.watchAbort === ctrl)
|
|
1654
|
+
this.watchAbort = undefined;
|
|
1655
|
+
}
|
|
1656
|
+
}
|
|
1657
|
+
}
|
|
1658
|
+
/**
|
|
1659
|
+
* `recovered` distinguishes a first-ever start (no state, nothing lost) from a
|
|
1660
|
+
* cursor we HAD and can no longer read. Only the latter is a recovery, and the
|
|
1661
|
+
* caller needs to know because the reason it reports is the only evidence a
|
|
1662
|
+
* durable cursor was ever lost.
|
|
1663
|
+
*/
|
|
1664
|
+
readWatchState() {
|
|
1665
|
+
const path = join(this.options.stateDir, '.owner-channel-watch.json');
|
|
1666
|
+
if (!existsSync(path))
|
|
1667
|
+
return { recovered: false };
|
|
1668
|
+
try {
|
|
1669
|
+
const value = JSON.parse(readFileSync(path, 'utf8'));
|
|
1670
|
+
if (value.version !== 1 || !Number.isSafeInteger(value.cursor) || value.cursor < 0
|
|
1671
|
+
|| !Number.isSafeInteger(value.reconnects) || value.reconnects < 0
|
|
1672
|
+
|| !Number.isSafeInteger(value.consecutiveFailures) || value.consecutiveFailures < 0)
|
|
1673
|
+
throw new Error('invalid owner watch state');
|
|
1674
|
+
return { state: value, recovered: false };
|
|
1675
|
+
}
|
|
1676
|
+
catch {
|
|
1677
|
+
this.options.log(`[${this.options.role}] owner watch `
|
|
1678
|
+
+ 'reason=OWNER_WATCH_CURSOR_RECOVERED invalid cursor state; draining inbox and starting at tip');
|
|
1679
|
+
return { recovered: true };
|
|
1680
|
+
}
|
|
1681
|
+
}
|
|
1682
|
+
writeWatchState(previous, cursor, reason, failed, reconnected = false) {
|
|
1683
|
+
const state = {
|
|
1684
|
+
version: 1,
|
|
1685
|
+
cursor,
|
|
1686
|
+
reconnects: (previous?.reconnects ?? 0) + (reconnected ? 1 : 0),
|
|
1687
|
+
consecutiveFailures: failed ? (previous?.consecutiveFailures ?? 0) + 1 : 0,
|
|
1688
|
+
reason,
|
|
1689
|
+
updatedAt: new Date(this.options.binderDeps?.now?.() ?? Date.now()).toISOString(),
|
|
1690
|
+
};
|
|
1691
|
+
replaceFileAtomically(join(this.options.stateDir, '.owner-channel-watch.json'), `${JSON.stringify(state)}\n`, 0o600);
|
|
1692
|
+
return state;
|
|
1693
|
+
}
|
|
1694
|
+
/** Compatibility path for injected child-process tests; production is direct. */
|
|
1695
|
+
async legacyWatchLoop() {
|
|
1552
1696
|
let delayMs = 1_000;
|
|
1553
1697
|
while (!this.stopping) {
|
|
1554
1698
|
try {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { SessionEvent, SessionSnapshot } from '../session/types.js';
|
|
1
|
+
import type { InterruptOutcome, SessionEvent, SessionSnapshot } from '../session/types.js';
|
|
2
2
|
import { type OwnerCommentsState } from './notices.js';
|
|
3
3
|
/**
|
|
4
4
|
* Fleet-level effects a deterministic owner command may trigger. Production
|
|
@@ -22,7 +22,7 @@ export interface OwnerCommandContext {
|
|
|
22
22
|
harness: string;
|
|
23
23
|
version: string;
|
|
24
24
|
snapshot(): SessionSnapshot;
|
|
25
|
-
interrupt(): Promise<
|
|
25
|
+
interrupt(): Promise<InterruptOutcome>;
|
|
26
26
|
/**
|
|
27
27
|
* Deliver raw slash text to the agent harness. Only commands the bundled
|
|
28
28
|
* ACP adapter for `harness` verifiably executes locally may be forwarded
|
|
@@ -71,13 +71,18 @@ export const ownerCommands = [
|
|
|
71
71
|
{
|
|
72
72
|
name: 'interrupt', summary: "cancel the agent's active turn",
|
|
73
73
|
execute: noArgs('/interrupt', async (ctx) => {
|
|
74
|
+
let outcome;
|
|
75
|
+
// Only a cancellation that never reached the session is a failure. A
|
|
76
|
+
// forced recovery stopped the turn: say so, and say it plainly.
|
|
74
77
|
try {
|
|
75
|
-
await ctx.interrupt();
|
|
78
|
+
outcome = await ctx.interrupt();
|
|
76
79
|
}
|
|
77
80
|
catch {
|
|
78
81
|
return ctx.reply(ownerNotices.interruptFailed(ctx.role));
|
|
79
82
|
}
|
|
80
|
-
await ctx.reply(
|
|
83
|
+
await ctx.reply(outcome?.state === 'forced'
|
|
84
|
+
? ownerNotices.interruptForced(ctx.role)
|
|
85
|
+
: ownerNotices.interrupted(ctx.role));
|
|
81
86
|
}),
|
|
82
87
|
},
|
|
83
88
|
{
|
|
@@ -24,6 +24,8 @@ export declare const ownerNotices: {
|
|
|
24
24
|
receivedInterrupting: () => string;
|
|
25
25
|
status: (role: string, snapshot: SessionSnapshot) => string;
|
|
26
26
|
interrupted: (role: string) => string;
|
|
27
|
+
/** The turn IS cancelled — say how, without implying the owner must retry. */
|
|
28
|
+
interruptForced: (role: string) => string;
|
|
27
29
|
interruptFailed: (role: string) => string;
|
|
28
30
|
commandStarted: (command: string) => string;
|
|
29
31
|
commandOutcome: (command: string, outcome: TurnOutcome, output?: string) => string;
|
|
@@ -28,6 +28,9 @@ export const ownerNotices = {
|
|
|
28
28
|
+ 'The response will arrive in this channel when ready.',
|
|
29
29
|
status: (role, snapshot) => `📊 ${role} status: ${snapshot.readiness}; session is ${snapshot.alive ? 'online' : 'offline'}.`,
|
|
30
30
|
interrupted: (role) => `🛑 Interrupt sent to ${role}'s active turn.`,
|
|
31
|
+
/** The turn IS cancelled — say how, without implying the owner must retry. */
|
|
32
|
+
interruptForced: (role) => `🛑 Interrupt enforced: ${role}'s turn ignored the cancellation, so the session `
|
|
33
|
+
+ 'was stopped and is restarting. It resumes automatically.',
|
|
31
34
|
interruptFailed: (role) => `⚠️ Could not interrupt ${role}'s active turn.`,
|
|
32
35
|
commandStarted: (command) => `⏳ Running ${command} — the result will follow in this channel.`,
|
|
33
36
|
commandOutcome: (command, outcome, output) => {
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
export declare const PACKAGE_NAME = "@ours.network/fleet";
|
|
2
|
+
export declare const BIN_NAME = "ours-fleet";
|
|
3
|
+
/** Stand-in build id for artifacts built before this module existed. */
|
|
4
|
+
export declare const UNKNOWN_BUILD = "unknown";
|
|
5
|
+
/** Identity of one built artifact, written into dist/ at build time. */
|
|
6
|
+
export interface BuildInfo {
|
|
7
|
+
version: string;
|
|
8
|
+
/** First 12 hex of a sha256 over every other file in dist/. */
|
|
9
|
+
buildId: string;
|
|
10
|
+
/** Commit the build was cut from, when git was available. */
|
|
11
|
+
commit?: string;
|
|
12
|
+
/** Whether that working tree had uncommitted changes. */
|
|
13
|
+
dirty?: boolean;
|
|
14
|
+
builtAt?: string;
|
|
15
|
+
capabilities: string[];
|
|
16
|
+
}
|
|
17
|
+
/** One @ours.network/fleet package directory on this host. */
|
|
18
|
+
export interface Install {
|
|
19
|
+
packageRoot: string;
|
|
20
|
+
version: string;
|
|
21
|
+
/** Absent for a pre-provenance build — its identity is unknowable. */
|
|
22
|
+
build?: BuildInfo;
|
|
23
|
+
}
|
|
24
|
+
export interface InstallRecord extends Install {
|
|
25
|
+
/** The PATH candidate that reaches this install, if any. */
|
|
26
|
+
bin?: string;
|
|
27
|
+
realBin?: string;
|
|
28
|
+
/** Position of `bin`'s directory in PATH; absent when off PATH. */
|
|
29
|
+
pathIndex?: number;
|
|
30
|
+
/** Whether this is the install executing right now. */
|
|
31
|
+
running: boolean;
|
|
32
|
+
}
|
|
33
|
+
export type SkewKind = 'version-build-conflict' | 'shadowed-runtime' | 'unknown-build-identity';
|
|
34
|
+
export interface InstallSkew {
|
|
35
|
+
kind: SkewKind;
|
|
36
|
+
severity: 'error' | 'warn';
|
|
37
|
+
message: string;
|
|
38
|
+
}
|
|
39
|
+
/** Read the install rooted at `packageRoot`, or undefined if that is not one. */
|
|
40
|
+
export declare function readInstall(packageRoot: string): Install | undefined;
|
|
41
|
+
/** Walk up from `from` to the @ours.network/fleet package directory containing it. */
|
|
42
|
+
export declare function findPackageRoot(from: string): string | undefined;
|
|
43
|
+
/**
|
|
44
|
+
* Every install reachable from PATH, plus the one executing right now.
|
|
45
|
+
* PATH order is preserved; an install reached by several PATH entries is listed
|
|
46
|
+
* once, at its earliest position.
|
|
47
|
+
*/
|
|
48
|
+
export declare function discoverInstalls(opts?: {
|
|
49
|
+
path?: string;
|
|
50
|
+
argv1?: string;
|
|
51
|
+
binName?: string;
|
|
52
|
+
platform?: NodeJS.Platform;
|
|
53
|
+
}): InstallRecord[];
|
|
54
|
+
/**
|
|
55
|
+
* sha256 over an install's dist/ tree, first 12 hex — the same bytes and order
|
|
56
|
+
* `scripts/build-info.mjs` hashes, so a stamped install's digest equals its
|
|
57
|
+
* buildId. This is what tells two PRE-provenance installs apart: they both
|
|
58
|
+
* report `unknown`, but they are not the same artifact, and the host that
|
|
59
|
+
* motivated this module had exactly that pair.
|
|
60
|
+
*/
|
|
61
|
+
export declare function contentDigest(packageRoot: string): string | undefined;
|
|
62
|
+
/** `0.17.0+9f1c2a3b4d5e`, or `…+unknown` for a pre-provenance build. */
|
|
63
|
+
export declare const buildLabel: (install: Pick<Install, "version" | "build">) => string;
|
|
64
|
+
/** What an install says it can do — never guessed from its version. */
|
|
65
|
+
export declare const capabilitySummary: (install: Install) => string;
|
|
66
|
+
/**
|
|
67
|
+
* Conflicts an operator must know about. `version-build-conflict` is the one
|
|
68
|
+
* that bit this host: same semver, different artifact, silently different rules.
|
|
69
|
+
*
|
|
70
|
+
* `digest` is only consulted inside a version group of two or more, so the
|
|
71
|
+
* common single-install case never hashes anything.
|
|
72
|
+
*/
|
|
73
|
+
export declare function analyzeInstalls(records: InstallRecord[], digest?: (packageRoot: string) => string | undefined): InstallSkew[];
|
|
74
|
+
/** Identity of the build executing right now. */
|
|
75
|
+
export declare function buildInfo(): BuildInfo;
|
|
76
|
+
/** `ours-fleet 0.17.0+9f1c2a3b4d5e` — one line, safe for any operator output. */
|
|
77
|
+
export declare const runningLabel: () => string;
|