@ours.network/fleet 1.1.0-nightly.6 → 1.1.0-nightly.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-recovery-gate.d.ts +17 -0
- package/dist/agent-recovery-gate.js +135 -0
- package/dist/application/fleet-query-service.js +13 -1
- package/dist/build-info.json +4 -4
- package/dist/daemon-recovery.d.ts +93 -0
- package/dist/daemon-recovery.js +328 -0
- package/dist/doctor.js +14 -0
- package/dist/owner-channel/channel.d.ts +40 -0
- package/dist/owner-channel/channel.js +210 -16
- package/dist/runner.d.ts +8 -1
- package/dist/runner.js +74 -3
- package/dist/temp-lifecycle.d.ts +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { ConversationEventV1 } from './session/conversation-types.js';
|
|
2
|
+
import type { AgentSession } from './session/types.js';
|
|
3
|
+
export interface AgentRecoveryEvidence {
|
|
4
|
+
ok: boolean;
|
|
5
|
+
reason: 'RECOVERY_TOOLS_VERIFIED' | 'RECOVERY_PROMPT_MISSING' | 'RECOVERY_TURN_INCOMPLETE' | 'RECOVERY_TURN_FAILED' | 'RECOVERY_CHOOSE_MISSING' | 'RECOVERY_CURRENT_MISSING' | 'RECOVERY_GET_MESSAGES_MISSING' | 'RECOVERY_TOOL_ORDER_INVALID';
|
|
6
|
+
chooseIdentity: boolean;
|
|
7
|
+
currentIdentity: boolean;
|
|
8
|
+
getMessages: boolean;
|
|
9
|
+
turnCompleted: boolean;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Verify one exact recovery turn from its durable conversation ledger. Tool
|
|
13
|
+
* arguments/results are inspected only to derive these booleans and are never
|
|
14
|
+
* returned or persisted by this gate.
|
|
15
|
+
*/
|
|
16
|
+
export declare function evaluateAgentRecovery(events: readonly ConversationEventV1[], promptId: string, identity: string): AgentRecoveryEvidence;
|
|
17
|
+
export declare function recoverAgentIdentity(session: AgentSession, identity: string): Promise<AgentRecoveryEvidence>;
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
const TOOL_NAMES = {
|
|
2
|
+
choose: new Set(['choose_identity', 'ours.choose_identity', 'mcp__ours__choose_identity']),
|
|
3
|
+
current: new Set(['current_identity', 'ours.current_identity', 'mcp__ours__current_identity']),
|
|
4
|
+
messages: new Set(['get_messages', 'ours.get_messages', 'mcp__ours__get_messages']),
|
|
5
|
+
};
|
|
6
|
+
function recordFor(records, event) {
|
|
7
|
+
if (event.kind !== 'tool.upsert' || !event.toolCallId)
|
|
8
|
+
return undefined;
|
|
9
|
+
const payload = event.payload;
|
|
10
|
+
const previous = records.get(event.toolCallId) ?? {};
|
|
11
|
+
const next = {
|
|
12
|
+
...previous,
|
|
13
|
+
...(payload.title !== undefined ? { title: payload.title } : {}),
|
|
14
|
+
...(payload.status !== undefined ? { status: payload.status } : {}),
|
|
15
|
+
...(payload.rawInput !== undefined ? { rawInput: payload.rawInput } : {}),
|
|
16
|
+
...(payload.status === 'completed' ? { completedSeq: event.seq } : {}),
|
|
17
|
+
};
|
|
18
|
+
records.set(event.toolCallId, next);
|
|
19
|
+
return next;
|
|
20
|
+
}
|
|
21
|
+
function objectInput(input) {
|
|
22
|
+
if (!input || input.truncated || input.redacted || !input.json
|
|
23
|
+
|| typeof input.json !== 'object' || Array.isArray(input.json))
|
|
24
|
+
return undefined;
|
|
25
|
+
return input.json;
|
|
26
|
+
}
|
|
27
|
+
function safeChoose(record, identity) {
|
|
28
|
+
if (!record.title || !TOOL_NAMES.choose.has(record.title) || record.status !== 'completed')
|
|
29
|
+
return false;
|
|
30
|
+
const input = objectInput(record.rawInput);
|
|
31
|
+
if (!input)
|
|
32
|
+
return false;
|
|
33
|
+
const keys = Object.keys(input).sort();
|
|
34
|
+
if (keys.some(key => key !== 'force' && key !== 'name'))
|
|
35
|
+
return false;
|
|
36
|
+
return input.name === identity && (input.force === undefined || input.force === false);
|
|
37
|
+
}
|
|
38
|
+
function safeNoOrBoundedInput(record, names, allowed) {
|
|
39
|
+
if (!record.title || !names.has(record.title) || record.status !== 'completed')
|
|
40
|
+
return false;
|
|
41
|
+
if (!record.rawInput)
|
|
42
|
+
return allowed({});
|
|
43
|
+
const input = objectInput(record.rawInput);
|
|
44
|
+
return input !== undefined && allowed(input);
|
|
45
|
+
}
|
|
46
|
+
function safeCurrent(record) {
|
|
47
|
+
return safeNoOrBoundedInput(record, TOOL_NAMES.current, input => Object.keys(input).length === 0);
|
|
48
|
+
}
|
|
49
|
+
function safeMessages(record) {
|
|
50
|
+
return safeNoOrBoundedInput(record, TOOL_NAMES.messages, input => {
|
|
51
|
+
const keys = Object.keys(input);
|
|
52
|
+
if (keys.some(key => key !== 'limit'))
|
|
53
|
+
return false;
|
|
54
|
+
return input.limit === undefined
|
|
55
|
+
|| (Number.isSafeInteger(input.limit) && input.limit >= 1 && input.limit <= 200);
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Verify one exact recovery turn from its durable conversation ledger. Tool
|
|
60
|
+
* arguments/results are inspected only to derive these booleans and are never
|
|
61
|
+
* returned or persisted by this gate.
|
|
62
|
+
*/
|
|
63
|
+
export function evaluateAgentRecovery(events, promptId, identity) {
|
|
64
|
+
const ordered = [...events].sort((a, b) => a.seq - b.seq);
|
|
65
|
+
const admitted = ordered.find(event => event.kind === 'prompt.admitted' && event.promptId === promptId);
|
|
66
|
+
const empty = (reason) => ({
|
|
67
|
+
ok: false, reason, chooseIdentity: false, currentIdentity: false,
|
|
68
|
+
getMessages: false, turnCompleted: false,
|
|
69
|
+
});
|
|
70
|
+
if (!admitted)
|
|
71
|
+
return empty('RECOVERY_PROMPT_MISSING');
|
|
72
|
+
const terminal = ordered.find(event => event.seq > admitted.seq
|
|
73
|
+
&& event.kind === 'turn.completed'
|
|
74
|
+
&& event.promptId === promptId
|
|
75
|
+
&& event.sessionGeneration === admitted.sessionGeneration);
|
|
76
|
+
if (!terminal)
|
|
77
|
+
return empty('RECOVERY_TURN_INCOMPLETE');
|
|
78
|
+
const outcome = terminal.payload.outcome;
|
|
79
|
+
if (outcome !== 'completed')
|
|
80
|
+
return empty('RECOVERY_TURN_FAILED');
|
|
81
|
+
const records = new Map();
|
|
82
|
+
for (const event of ordered) {
|
|
83
|
+
if (event.seq <= admitted.seq || event.seq >= terminal.seq
|
|
84
|
+
|| event.promptId !== promptId
|
|
85
|
+
|| event.sessionGeneration !== admitted.sessionGeneration)
|
|
86
|
+
continue;
|
|
87
|
+
recordFor(records, event);
|
|
88
|
+
}
|
|
89
|
+
const values = [...records.values()];
|
|
90
|
+
const chooses = values.filter(record => safeChoose(record, identity));
|
|
91
|
+
const currents = values.filter(safeCurrent);
|
|
92
|
+
const messages = values.filter(safeMessages);
|
|
93
|
+
const chooseIdentity = chooses.length > 0;
|
|
94
|
+
const currentIdentity = currents.length > 0;
|
|
95
|
+
const getMessages = messages.length > 0;
|
|
96
|
+
const base = { chooseIdentity, currentIdentity, getMessages, turnCompleted: true };
|
|
97
|
+
if (!chooseIdentity)
|
|
98
|
+
return { ok: false, reason: 'RECOVERY_CHOOSE_MISSING', ...base };
|
|
99
|
+
if (!currentIdentity)
|
|
100
|
+
return { ok: false, reason: 'RECOVERY_CURRENT_MISSING', ...base };
|
|
101
|
+
if (!getMessages)
|
|
102
|
+
return { ok: false, reason: 'RECOVERY_GET_MESSAGES_MISSING', ...base };
|
|
103
|
+
const orderedChain = chooses.some(choose => currents.some(current => messages.some(message => choose.completedSeq !== undefined && current.completedSeq !== undefined
|
|
104
|
+
&& message.completedSeq !== undefined
|
|
105
|
+
&& choose.completedSeq < current.completedSeq
|
|
106
|
+
&& current.completedSeq < message.completedSeq)));
|
|
107
|
+
if (!orderedChain)
|
|
108
|
+
return { ok: false, reason: 'RECOVERY_TOOL_ORDER_INVALID', ...base };
|
|
109
|
+
return { ok: true, reason: 'RECOVERY_TOOLS_VERIFIED', ...base };
|
|
110
|
+
}
|
|
111
|
+
export async function recoverAgentIdentity(session, identity) {
|
|
112
|
+
if (!session.subscribeConversation)
|
|
113
|
+
return {
|
|
114
|
+
ok: false, reason: 'RECOVERY_PROMPT_MISSING', chooseIdentity: false,
|
|
115
|
+
currentIdentity: false, getMessages: false, turnCompleted: false,
|
|
116
|
+
};
|
|
117
|
+
const events = [];
|
|
118
|
+
const unsubscribe = session.subscribeConversation(event => events.push(event));
|
|
119
|
+
try {
|
|
120
|
+
const queued = await session.queuePrompt([
|
|
121
|
+
'[fleet-recovery] The shared ours daemon restarted.',
|
|
122
|
+
`Call ours choose_identity with name ${JSON.stringify(identity)} and force false.`,
|
|
123
|
+
'Then call current_identity, then get_messages. Complete all three in that order.',
|
|
124
|
+
'Do not create/delete identities, force-bind, interrupt, or restart any service/session.',
|
|
125
|
+
].join('\n'), { origin: { kind: 'fleet-monitor' } });
|
|
126
|
+
await queued.completion;
|
|
127
|
+
// Conversation publication is synchronous with terminal settlement in the
|
|
128
|
+
// in-tree ACP store; filtering by exact promptId/sessionGeneration remains
|
|
129
|
+
// the authority even if unrelated events arrived concurrently.
|
|
130
|
+
return evaluateAgentRecovery(events, queued.promptId, identity);
|
|
131
|
+
}
|
|
132
|
+
finally {
|
|
133
|
+
unsubscribe();
|
|
134
|
+
}
|
|
135
|
+
}
|
|
@@ -4,6 +4,7 @@ import { classifyActivity } from '../session/activity.js';
|
|
|
4
4
|
import { controlRequest } from '../session/control.js';
|
|
5
5
|
import { SessionControlError } from '../session/types.js';
|
|
6
6
|
import { readExitRecord, readRestartLedger } from '../runner.js';
|
|
7
|
+
import { readDaemonRecoveryStatus } from '../daemon-recovery.js';
|
|
7
8
|
import { roleCapabilities } from './capabilities.js';
|
|
8
9
|
import { FleetError } from './errors.js';
|
|
9
10
|
const clean = (value, max = 512) => value.replace(/[\0-\x08\x0b\x0c\x0e-\x1f\x7f]/g, '').trim().slice(0, max);
|
|
@@ -46,7 +47,8 @@ function readIsolation(dir) {
|
|
|
46
47
|
function sessionOverall(supervisor, session, restart, monitor, isolation, problems) {
|
|
47
48
|
if (restart.circuit === 'open' || monitor.health === 'failed' || monitor.health === 'degraded'
|
|
48
49
|
|| isolation.degraded
|
|
49
|
-
|| problems.some(problem => problem.severity === 'error'
|
|
50
|
+
|| problems.some(problem => problem.severity === 'error'
|
|
51
|
+
|| problem.source === 'watchdog' || problem.source === 'daemon-recovery')
|
|
50
52
|
|| session.readiness === 'failed')
|
|
51
53
|
return 'attention';
|
|
52
54
|
// A reachable agent session is the user's live interaction surface. Its
|
|
@@ -103,6 +105,16 @@ export class FleetQueryService {
|
|
|
103
105
|
const lastExit = dir ? readExitRecord(join(dir, '.exit-status')) ?? undefined : undefined;
|
|
104
106
|
const session = await this.session(role, dir, live.state);
|
|
105
107
|
const problems = [...role.problems];
|
|
108
|
+
const recovery = dir ? readDaemonRecoveryStatus(dir) : undefined;
|
|
109
|
+
if (recovery && recovery.state !== 'recovered') {
|
|
110
|
+
const paths = ['agent', 'owner']
|
|
111
|
+
.filter(name => recovery.paths[name].state !== 'recovered')
|
|
112
|
+
.map(name => `${name}:${recovery.paths[name].state}`);
|
|
113
|
+
problems.push({
|
|
114
|
+
code: 'daemon_recovery', severity: 'warning', source: 'daemon-recovery',
|
|
115
|
+
detail: `${recovery.state}; ${paths.join(', ') || 'no degraded path'}; epoch ${recovery.epoch || 'unavailable'}`,
|
|
116
|
+
});
|
|
117
|
+
}
|
|
106
118
|
if (live.state === 'running' && session.reachability !== 'online')
|
|
107
119
|
problems.push({
|
|
108
120
|
code: 'supervisor_session_disagreement', severity: 'warning',
|
package/dist/build-info.json
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
{
|
|
2
|
-
"version": "1.1.0-nightly.
|
|
3
|
-
"buildId": "
|
|
4
|
-
"commit": "
|
|
2
|
+
"version": "1.1.0-nightly.7",
|
|
3
|
+
"buildId": "a09aa98e92aa",
|
|
4
|
+
"commit": "d79160ae66847305a109bed339d8c8830a20bba7",
|
|
5
5
|
"dirty": true,
|
|
6
|
-
"builtAt": "2026-08-
|
|
6
|
+
"builtAt": "2026-08-31T07:32:26.852Z",
|
|
7
7
|
"capabilities": [
|
|
8
8
|
"monitor.interrupt.after_tool"
|
|
9
9
|
]
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { type FetchLike } from './monitor.js';
|
|
2
|
+
export declare const DAEMON_RECOVERY_MAX_ATTEMPTS = 6;
|
|
3
|
+
export declare const DAEMON_RECOVERY_INITIAL_BACKOFF_MS = 1000;
|
|
4
|
+
export declare const DAEMON_RECOVERY_MAX_BACKOFF_MS = 5000;
|
|
5
|
+
export declare const DAEMON_RECOVERY_DEADLINE_MS = 60000;
|
|
6
|
+
export interface DaemonGeneration {
|
|
7
|
+
bootId: string;
|
|
8
|
+
pid: number;
|
|
9
|
+
startedAt: number;
|
|
10
|
+
stateDir: string;
|
|
11
|
+
}
|
|
12
|
+
export type DaemonGenerationProbe = {
|
|
13
|
+
state: 'ready';
|
|
14
|
+
generation: DaemonGeneration;
|
|
15
|
+
} | {
|
|
16
|
+
state: 'unavailable';
|
|
17
|
+
reason: string;
|
|
18
|
+
};
|
|
19
|
+
export type DaemonGenerationObservation = {
|
|
20
|
+
kind: 'baseline' | 'stable' | 'available';
|
|
21
|
+
generation: DaemonGeneration;
|
|
22
|
+
} | {
|
|
23
|
+
kind: 'changed';
|
|
24
|
+
previous: DaemonGeneration;
|
|
25
|
+
generation: DaemonGeneration;
|
|
26
|
+
} | {
|
|
27
|
+
kind: 'lost' | 'unavailable';
|
|
28
|
+
previous?: DaemonGeneration;
|
|
29
|
+
reason: string;
|
|
30
|
+
};
|
|
31
|
+
interface GenerationProbeDeps {
|
|
32
|
+
readText?(path: string): string;
|
|
33
|
+
canonicalize?(path: string): string;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Corroborate the loopback daemon's unauthenticated `/info`, its credentialed
|
|
37
|
+
* identity-index readiness route, and its local boot-generation record. The
|
|
38
|
+
* index enforces auth when the daemon visibility requires it; open visibility
|
|
39
|
+
* deliberately does not. No source is sufficient
|
|
40
|
+
* alone: `/info` has no boot id, the identity index has no generation, and a
|
|
41
|
+
* stale `ready` file can outlive the process that wrote it.
|
|
42
|
+
*/
|
|
43
|
+
export declare function probeDaemonGeneration(fetch: FetchLike, env: NodeJS.ProcessEnv, deps?: GenerationProbeDeps): Promise<DaemonGenerationProbe>;
|
|
44
|
+
export declare class DaemonGenerationObserver {
|
|
45
|
+
private current?;
|
|
46
|
+
private unavailable;
|
|
47
|
+
observe(probe: DaemonGenerationProbe): DaemonGenerationObservation;
|
|
48
|
+
}
|
|
49
|
+
export declare function daemonRecoveryBackoff(attempt: number): number;
|
|
50
|
+
export type RecoveryPath = 'agent' | 'owner';
|
|
51
|
+
export type RecoveryPathResult = {
|
|
52
|
+
ok: true;
|
|
53
|
+
} | {
|
|
54
|
+
ok: false;
|
|
55
|
+
reason: string;
|
|
56
|
+
};
|
|
57
|
+
export interface RoleRecoveryControllerOptions {
|
|
58
|
+
role: string;
|
|
59
|
+
identity: string;
|
|
60
|
+
stateDir: string;
|
|
61
|
+
now(): number;
|
|
62
|
+
sleep(ms: number): Promise<void>;
|
|
63
|
+
recoverAgent(epoch: string): Promise<RecoveryPathResult>;
|
|
64
|
+
recoverOwner(epoch: string): Promise<RecoveryPathResult>;
|
|
65
|
+
log(line: string): void;
|
|
66
|
+
}
|
|
67
|
+
export interface RecoveryStatus {
|
|
68
|
+
version: 1;
|
|
69
|
+
identity: string;
|
|
70
|
+
epoch: string;
|
|
71
|
+
state: 'recovering' | 'recovered' | 'degraded' | 'cancelled';
|
|
72
|
+
paths: Record<RecoveryPath, {
|
|
73
|
+
state: 'pending' | 'recovered' | 'degraded';
|
|
74
|
+
attempts: number;
|
|
75
|
+
reason?: string;
|
|
76
|
+
}>;
|
|
77
|
+
updatedAt: string;
|
|
78
|
+
}
|
|
79
|
+
export declare function readDaemonRecoveryStatus(dir: string): RecoveryStatus | undefined;
|
|
80
|
+
/** Per-role, per-generation bounded recovery with path-level fault isolation. */
|
|
81
|
+
export declare class RoleRecoveryController {
|
|
82
|
+
private readonly options;
|
|
83
|
+
private token;
|
|
84
|
+
private activeEpoch?;
|
|
85
|
+
private active?;
|
|
86
|
+
private status?;
|
|
87
|
+
constructor(options: RoleRecoveryControllerOptions);
|
|
88
|
+
recover(generation: DaemonGeneration): Promise<RecoveryStatus>;
|
|
89
|
+
cancel(): void;
|
|
90
|
+
noteLoss(reason: string): void;
|
|
91
|
+
private write;
|
|
92
|
+
}
|
|
93
|
+
export {};
|
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
import { readFileSync, realpathSync } from 'node:fs';
|
|
2
|
+
import { join, resolve } from 'node:path';
|
|
3
|
+
import { createHash } from 'node:crypto';
|
|
4
|
+
import { resolveEndpoint } from './monitor.js';
|
|
5
|
+
import { replaceFileAtomically } from './atomic-file.js';
|
|
6
|
+
export const DAEMON_RECOVERY_MAX_ATTEMPTS = 6;
|
|
7
|
+
export const DAEMON_RECOVERY_INITIAL_BACKOFF_MS = 1_000;
|
|
8
|
+
export const DAEMON_RECOVERY_MAX_BACKOFF_MS = 5_000;
|
|
9
|
+
export const DAEMON_RECOVERY_DEADLINE_MS = 60_000;
|
|
10
|
+
const STARTUP_PHASES = new Set([
|
|
11
|
+
'initializing', 'wrapper', 'registrar', 'identities', 'reconciliation',
|
|
12
|
+
'server', 'ready', 'failed',
|
|
13
|
+
]);
|
|
14
|
+
function positiveInteger(value) {
|
|
15
|
+
return Number.isSafeInteger(value) && value > 0;
|
|
16
|
+
}
|
|
17
|
+
function finiteTimestamp(value) {
|
|
18
|
+
return typeof value === 'number' && Number.isFinite(value) && value > 0;
|
|
19
|
+
}
|
|
20
|
+
function startupProgress(value) {
|
|
21
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
22
|
+
return undefined;
|
|
23
|
+
const row = value;
|
|
24
|
+
if (row.version !== 1 || !positiveInteger(row.pid)
|
|
25
|
+
|| typeof row.bootId !== 'string' || row.bootId.length === 0
|
|
26
|
+
|| typeof row.phase !== 'string' || !STARTUP_PHASES.has(row.phase)
|
|
27
|
+
|| !finiteTimestamp(row.startedAt) || !finiteTimestamp(row.updatedAt))
|
|
28
|
+
return undefined;
|
|
29
|
+
if (row.updatedAt < row.startedAt)
|
|
30
|
+
return undefined;
|
|
31
|
+
if ((row.completed === undefined) !== (row.total === undefined))
|
|
32
|
+
return undefined;
|
|
33
|
+
if (row.completed !== undefined
|
|
34
|
+
&& (!Number.isSafeInteger(row.completed) || row.completed < 0
|
|
35
|
+
|| !Number.isSafeInteger(row.total) || row.total < 0
|
|
36
|
+
|| row.completed > row.total))
|
|
37
|
+
return undefined;
|
|
38
|
+
return row;
|
|
39
|
+
}
|
|
40
|
+
function canonical(path, deps) {
|
|
41
|
+
const absolute = resolve(path);
|
|
42
|
+
return deps.canonicalize?.(absolute) ?? realpathSync.native(absolute);
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Corroborate the loopback daemon's unauthenticated `/info`, its credentialed
|
|
46
|
+
* identity-index readiness route, and its local boot-generation record. The
|
|
47
|
+
* index enforces auth when the daemon visibility requires it; open visibility
|
|
48
|
+
* deliberately does not. No source is sufficient
|
|
49
|
+
* alone: `/info` has no boot id, the identity index has no generation, and a
|
|
50
|
+
* stale `ready` file can outlive the process that wrote it.
|
|
51
|
+
*/
|
|
52
|
+
export async function probeDaemonGeneration(fetch, env, deps = {}) {
|
|
53
|
+
const endpoint = resolveEndpoint(env);
|
|
54
|
+
let response;
|
|
55
|
+
try {
|
|
56
|
+
response = await fetch(`${endpoint.origin}/info`, { headers: endpoint.headers });
|
|
57
|
+
}
|
|
58
|
+
catch {
|
|
59
|
+
return { state: 'unavailable', reason: 'DAEMON_INFO_UNREACHABLE' };
|
|
60
|
+
}
|
|
61
|
+
if (!response.ok)
|
|
62
|
+
return {
|
|
63
|
+
state: 'unavailable',
|
|
64
|
+
reason: response.status === 401 ? 'DAEMON_INFO_UNAUTHORIZED' : 'DAEMON_INFO_HTTP_ERROR',
|
|
65
|
+
};
|
|
66
|
+
let info;
|
|
67
|
+
try {
|
|
68
|
+
const value = await response.json();
|
|
69
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
70
|
+
throw new Error('shape');
|
|
71
|
+
info = value;
|
|
72
|
+
}
|
|
73
|
+
catch {
|
|
74
|
+
return { state: 'unavailable', reason: 'DAEMON_INFO_INVALID' };
|
|
75
|
+
}
|
|
76
|
+
if (info.name !== 'ours' || !positiveInteger(info.pid)
|
|
77
|
+
|| typeof info.stateDir !== 'string' || info.stateDir.length === 0)
|
|
78
|
+
return { state: 'unavailable', reason: 'DAEMON_INFO_INVALID' };
|
|
79
|
+
let reportedStateDir;
|
|
80
|
+
let expectedStateDir;
|
|
81
|
+
try {
|
|
82
|
+
reportedStateDir = canonical(info.stateDir, deps);
|
|
83
|
+
expectedStateDir = canonical(endpoint.stateDir, deps);
|
|
84
|
+
}
|
|
85
|
+
catch {
|
|
86
|
+
return { state: 'unavailable', reason: 'DAEMON_STATE_DIR_UNAVAILABLE' };
|
|
87
|
+
}
|
|
88
|
+
if (reportedStateDir !== expectedStateDir)
|
|
89
|
+
return { state: 'unavailable', reason: 'DAEMON_STATE_DIR_MISMATCH' };
|
|
90
|
+
// `/identities` is the daemon's credential-capable readiness route. A
|
|
91
|
+
// valid-but-empty index is a documented intermediate startup state, never
|
|
92
|
+
// proof of readiness.
|
|
93
|
+
let identitiesResponse;
|
|
94
|
+
try {
|
|
95
|
+
identitiesResponse = await fetch(`${endpoint.origin}/identities`, { headers: endpoint.headers });
|
|
96
|
+
}
|
|
97
|
+
catch {
|
|
98
|
+
return { state: 'unavailable', reason: 'DAEMON_IDENTITIES_UNREACHABLE' };
|
|
99
|
+
}
|
|
100
|
+
if (!identitiesResponse.ok)
|
|
101
|
+
return {
|
|
102
|
+
state: 'unavailable',
|
|
103
|
+
reason: identitiesResponse.status === 401
|
|
104
|
+
? 'DAEMON_IDENTITIES_UNAUTHORIZED' : 'DAEMON_IDENTITIES_HTTP_ERROR',
|
|
105
|
+
};
|
|
106
|
+
try {
|
|
107
|
+
const body = await identitiesResponse.json();
|
|
108
|
+
if (!body || typeof body !== 'object' || Array.isArray(body)
|
|
109
|
+
|| !Array.isArray(body.identities))
|
|
110
|
+
return { state: 'unavailable', reason: 'DAEMON_IDENTITIES_INVALID' };
|
|
111
|
+
if (body.identities.length === 0)
|
|
112
|
+
return { state: 'unavailable', reason: 'DAEMON_IDENTITIES_NOT_READY' };
|
|
113
|
+
}
|
|
114
|
+
catch {
|
|
115
|
+
return { state: 'unavailable', reason: 'DAEMON_IDENTITIES_INVALID' };
|
|
116
|
+
}
|
|
117
|
+
let progress;
|
|
118
|
+
try {
|
|
119
|
+
const text = deps.readText?.(join(reportedStateDir, 'startup-progress.json'))
|
|
120
|
+
?? readFileSync(join(reportedStateDir, 'startup-progress.json'), 'utf8');
|
|
121
|
+
progress = startupProgress(JSON.parse(text));
|
|
122
|
+
}
|
|
123
|
+
catch {
|
|
124
|
+
return { state: 'unavailable', reason: 'DAEMON_PROGRESS_UNAVAILABLE' };
|
|
125
|
+
}
|
|
126
|
+
if (!progress)
|
|
127
|
+
return { state: 'unavailable', reason: 'DAEMON_PROGRESS_INVALID' };
|
|
128
|
+
if (progress.phase !== 'ready')
|
|
129
|
+
return { state: 'unavailable', reason: 'DAEMON_PROGRESS_NOT_READY' };
|
|
130
|
+
if (progress.pid !== info.pid)
|
|
131
|
+
return { state: 'unavailable', reason: 'DAEMON_GENERATION_MISMATCH' };
|
|
132
|
+
return {
|
|
133
|
+
state: 'ready',
|
|
134
|
+
generation: {
|
|
135
|
+
// Keep bootId opaque. Its current writer uses `${pid}-${startedAt}`, but
|
|
136
|
+
// equality—not its formatting—is the generation contract Fleet needs.
|
|
137
|
+
bootId: progress.bootId,
|
|
138
|
+
pid: progress.pid,
|
|
139
|
+
startedAt: progress.startedAt,
|
|
140
|
+
stateDir: reportedStateDir,
|
|
141
|
+
},
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
export class DaemonGenerationObserver {
|
|
145
|
+
current;
|
|
146
|
+
unavailable = false;
|
|
147
|
+
observe(probe) {
|
|
148
|
+
if (probe.state === 'unavailable') {
|
|
149
|
+
// Cold startup has no generation to lose. Do not manufacture a recovery
|
|
150
|
+
// epoch before the first corroborated ready observation.
|
|
151
|
+
if (!this.current)
|
|
152
|
+
return { kind: 'unavailable', reason: probe.reason };
|
|
153
|
+
const kind = this.unavailable ? 'unavailable' : 'lost';
|
|
154
|
+
this.unavailable = true;
|
|
155
|
+
return { kind, previous: this.current, reason: probe.reason };
|
|
156
|
+
}
|
|
157
|
+
const next = probe.generation;
|
|
158
|
+
const previous = this.current;
|
|
159
|
+
this.current = next;
|
|
160
|
+
if (!previous) {
|
|
161
|
+
this.unavailable = false;
|
|
162
|
+
return { kind: 'baseline', generation: next };
|
|
163
|
+
}
|
|
164
|
+
const changed = previous.bootId !== next.bootId
|
|
165
|
+
|| previous.pid !== next.pid
|
|
166
|
+
|| previous.startedAt !== next.startedAt
|
|
167
|
+
|| previous.stateDir !== next.stateDir;
|
|
168
|
+
const wasUnavailable = this.unavailable;
|
|
169
|
+
this.unavailable = false;
|
|
170
|
+
if (changed)
|
|
171
|
+
return { kind: 'changed', previous, generation: next };
|
|
172
|
+
return { kind: wasUnavailable ? 'available' : 'stable', generation: next };
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
export function daemonRecoveryBackoff(attempt) {
|
|
176
|
+
const exponent = Math.max(0, Math.min(attempt - 1, 30));
|
|
177
|
+
return Math.min(DAEMON_RECOVERY_INITIAL_BACKOFF_MS * 2 ** exponent, DAEMON_RECOVERY_MAX_BACKOFF_MS);
|
|
178
|
+
}
|
|
179
|
+
function safeRecoveryReason(reason, fallback = 'RECOVERY_PATH_FAILED') {
|
|
180
|
+
return /^[A-Z][A-Z0-9_]{0,95}$/.test(reason) ? reason : fallback;
|
|
181
|
+
}
|
|
182
|
+
export function readDaemonRecoveryStatus(dir) {
|
|
183
|
+
try {
|
|
184
|
+
const raw = readFileSync(join(dir, '.daemon-recovery.json'), 'utf8');
|
|
185
|
+
if (raw.length > 16 * 1024)
|
|
186
|
+
return undefined;
|
|
187
|
+
const value = JSON.parse(raw);
|
|
188
|
+
if (value.version !== 1 || typeof value.epoch !== 'string'
|
|
189
|
+
|| !['recovering', 'recovered', 'degraded', 'cancelled'].includes(value.state ?? '')
|
|
190
|
+
|| !value.paths || typeof value.paths !== 'object')
|
|
191
|
+
return undefined;
|
|
192
|
+
for (const name of ['agent', 'owner']) {
|
|
193
|
+
const path = value.paths[name];
|
|
194
|
+
if (!path || !['pending', 'recovered', 'degraded'].includes(path.state)
|
|
195
|
+
|| !Number.isSafeInteger(path.attempts) || path.attempts < 0
|
|
196
|
+
|| (path.reason !== undefined && !/^[A-Z0-9_]{1,96}$/.test(path.reason)))
|
|
197
|
+
return undefined;
|
|
198
|
+
}
|
|
199
|
+
return value;
|
|
200
|
+
}
|
|
201
|
+
catch {
|
|
202
|
+
return undefined;
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
/** Per-role, per-generation bounded recovery with path-level fault isolation. */
|
|
206
|
+
export class RoleRecoveryController {
|
|
207
|
+
options;
|
|
208
|
+
token = 0;
|
|
209
|
+
activeEpoch;
|
|
210
|
+
active;
|
|
211
|
+
status;
|
|
212
|
+
constructor(options) {
|
|
213
|
+
this.options = options;
|
|
214
|
+
}
|
|
215
|
+
recover(generation) {
|
|
216
|
+
const epoch = createHash('sha256').update([
|
|
217
|
+
generation.bootId, generation.pid, generation.startedAt, generation.stateDir,
|
|
218
|
+
].join('\0')).digest('hex').slice(0, 24);
|
|
219
|
+
if (this.activeEpoch === epoch && this.active)
|
|
220
|
+
return this.active;
|
|
221
|
+
const token = ++this.token;
|
|
222
|
+
this.activeEpoch = epoch;
|
|
223
|
+
const deadline = this.options.now() + DAEMON_RECOVERY_DEADLINE_MS;
|
|
224
|
+
const status = {
|
|
225
|
+
version: 1, identity: this.options.identity, epoch, state: 'recovering',
|
|
226
|
+
paths: {
|
|
227
|
+
agent: { state: 'pending', attempts: 0 },
|
|
228
|
+
owner: { state: 'pending', attempts: 0 },
|
|
229
|
+
},
|
|
230
|
+
updatedAt: new Date(this.options.now()).toISOString(),
|
|
231
|
+
};
|
|
232
|
+
this.status = status;
|
|
233
|
+
this.write(status, token);
|
|
234
|
+
const runPath = async (path, operation) => {
|
|
235
|
+
let lastReason = 'RECOVERY_ATTEMPTS_EXHAUSTED';
|
|
236
|
+
for (let attempt = 1; attempt <= DAEMON_RECOVERY_MAX_ATTEMPTS; attempt++) {
|
|
237
|
+
if (token !== this.token)
|
|
238
|
+
return;
|
|
239
|
+
if (this.options.now() >= deadline) {
|
|
240
|
+
lastReason = 'RECOVERY_DEADLINE_EXCEEDED';
|
|
241
|
+
break;
|
|
242
|
+
}
|
|
243
|
+
status.paths[path] = { state: 'pending', attempts: attempt };
|
|
244
|
+
this.write(status, token);
|
|
245
|
+
try {
|
|
246
|
+
const remaining = Math.max(0, deadline - this.options.now());
|
|
247
|
+
let timer;
|
|
248
|
+
const result = await Promise.race([
|
|
249
|
+
operation(epoch),
|
|
250
|
+
new Promise(resolve => {
|
|
251
|
+
timer = setTimeout(() => resolve({ ok: false, reason: 'RECOVERY_DEADLINE_EXCEEDED' }), remaining);
|
|
252
|
+
timer.unref?.();
|
|
253
|
+
}),
|
|
254
|
+
]).finally(() => { if (timer)
|
|
255
|
+
clearTimeout(timer); });
|
|
256
|
+
if (token !== this.token)
|
|
257
|
+
return;
|
|
258
|
+
if (result.ok) {
|
|
259
|
+
status.paths[path] = { state: 'recovered', attempts: attempt };
|
|
260
|
+
this.write(status, token);
|
|
261
|
+
return;
|
|
262
|
+
}
|
|
263
|
+
lastReason = safeRecoveryReason(result.reason);
|
|
264
|
+
}
|
|
265
|
+
catch (error) {
|
|
266
|
+
lastReason = safeRecoveryReason(error instanceof Error && error.name
|
|
267
|
+
? `RECOVERY_${error.name.toUpperCase()}` : 'RECOVERY_UNKNOWN_ERROR', 'RECOVERY_UNKNOWN_ERROR');
|
|
268
|
+
}
|
|
269
|
+
if (attempt < DAEMON_RECOVERY_MAX_ATTEMPTS) {
|
|
270
|
+
const wait = Math.min(daemonRecoveryBackoff(attempt), Math.max(0, deadline - this.options.now()));
|
|
271
|
+
await this.options.sleep(wait);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
if (token !== this.token)
|
|
275
|
+
return;
|
|
276
|
+
status.paths[path] = {
|
|
277
|
+
state: 'degraded', attempts: status.paths[path].attempts, reason: lastReason,
|
|
278
|
+
};
|
|
279
|
+
this.write(status, token);
|
|
280
|
+
};
|
|
281
|
+
const active = Promise.all([
|
|
282
|
+
runPath('agent', this.options.recoverAgent),
|
|
283
|
+
runPath('owner', this.options.recoverOwner),
|
|
284
|
+
]).then(() => {
|
|
285
|
+
if (token !== this.token) {
|
|
286
|
+
status.state = 'cancelled';
|
|
287
|
+
return status;
|
|
288
|
+
}
|
|
289
|
+
status.state = Object.values(status.paths).every(path => path.state === 'recovered')
|
|
290
|
+
? 'recovered' : 'degraded';
|
|
291
|
+
this.write(status, token);
|
|
292
|
+
return status;
|
|
293
|
+
});
|
|
294
|
+
this.active = active;
|
|
295
|
+
return active;
|
|
296
|
+
}
|
|
297
|
+
cancel() {
|
|
298
|
+
const token = ++this.token;
|
|
299
|
+
this.activeEpoch = undefined;
|
|
300
|
+
this.active = undefined;
|
|
301
|
+
if (this.status?.state === 'recovering') {
|
|
302
|
+
this.status.state = 'cancelled';
|
|
303
|
+
this.write(this.status, token);
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
noteLoss(reason) {
|
|
307
|
+
const token = ++this.token;
|
|
308
|
+
this.activeEpoch = undefined;
|
|
309
|
+
this.active = undefined;
|
|
310
|
+
const status = {
|
|
311
|
+
version: 1, identity: this.options.identity, epoch: '', state: 'degraded',
|
|
312
|
+
paths: {
|
|
313
|
+
agent: { state: 'degraded', attempts: 0, reason: safeRecoveryReason(reason, 'DAEMON_UNAVAILABLE') },
|
|
314
|
+
owner: { state: 'degraded', attempts: 0, reason: safeRecoveryReason(reason, 'DAEMON_UNAVAILABLE') },
|
|
315
|
+
},
|
|
316
|
+
updatedAt: new Date(this.options.now()).toISOString(),
|
|
317
|
+
};
|
|
318
|
+
this.status = status;
|
|
319
|
+
this.write(status, token);
|
|
320
|
+
}
|
|
321
|
+
write(status, token) {
|
|
322
|
+
if (token !== this.token)
|
|
323
|
+
return;
|
|
324
|
+
status.updatedAt = new Date(this.options.now()).toISOString();
|
|
325
|
+
replaceFileAtomically(join(this.options.stateDir, '.daemon-recovery.json'), JSON.stringify(status, null, 2) + '\n', 0o600);
|
|
326
|
+
this.options.log(`[${this.options.role}] daemon recovery ${status.state} epoch=${status.epoch || 'unavailable'}`);
|
|
327
|
+
}
|
|
328
|
+
}
|
package/dist/doctor.js
CHANGED
|
@@ -13,6 +13,7 @@ import { authResolutionHint, resolveEndpoint, } from './monitor.js';
|
|
|
13
13
|
import { readScheduledLoops, storedLoopHealth } from './loops/state.js';
|
|
14
14
|
import { controlSocketPath } from './session/control.js';
|
|
15
15
|
import { analyzeInstalls, buildInfo, buildLabel, discoverInstalls, BIN_NAME, } from './provenance.js';
|
|
16
|
+
import { readDaemonRecoveryStatus } from './daemon-recovery.js';
|
|
16
17
|
/** Which cgroup-v2 controllers are delegated to this user manager (advisory). */
|
|
17
18
|
function cgroupDelegationDetail() {
|
|
18
19
|
try {
|
|
@@ -106,6 +107,19 @@ export async function doctor(opts = {}, exec = realExec, platform = process.plat
|
|
|
106
107
|
ok: true,
|
|
107
108
|
detail: `warning: ${diagnostic.message}`,
|
|
108
109
|
});
|
|
110
|
+
for (const role of roles) {
|
|
111
|
+
const recovery = readDaemonRecoveryStatus(agentDir(role.name));
|
|
112
|
+
if (!recovery)
|
|
113
|
+
continue;
|
|
114
|
+
const paths = ['agent', 'owner']
|
|
115
|
+
.map(name => `${name}=${recovery.paths[name].state}`)
|
|
116
|
+
.join(', ');
|
|
117
|
+
checks.push({
|
|
118
|
+
name: `daemon recovery: ${role.name}`,
|
|
119
|
+
ok: recovery.state === 'recovered',
|
|
120
|
+
detail: `${recovery.state}; ${paths}; epoch ${recovery.epoch || 'unavailable'}`,
|
|
121
|
+
});
|
|
122
|
+
}
|
|
109
123
|
// ── Rooms/tasks checks ────────────────────────────────────────
|
|
110
124
|
if (loaded.ok && loaded.rooms) {
|
|
111
125
|
const rooms = loaded.rooms;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { type OwnerChannelConfig } from '../config.js';
|
|
2
2
|
import { type AgentSession } from '../session/types.js';
|
|
3
3
|
import { type OwnerFleetOps } from './commands.js';
|
|
4
|
+
import type { ManagedFleetSpawnResult } from '../fleet-proxy.js';
|
|
4
5
|
import { type FleetAuditAttempt, type FleetAuditPresentation, type FleetCommandOutcomeClass } from '../fleet-command-audit.js';
|
|
5
6
|
import { type OursOps } from './ours-client.js';
|
|
6
7
|
import { type OwnerUpdatePhase } from './notices.js';
|
|
@@ -27,12 +28,24 @@ export interface OwnerChannelOptions {
|
|
|
27
28
|
binderDeps?: OwnerBinderDeps;
|
|
28
29
|
/** Pre-acquired by the runner so the predecessor control socket remains reachable while waiting. */
|
|
29
30
|
binderLease?: OwnerBinderLease;
|
|
31
|
+
recoveryDeps?: {
|
|
32
|
+
now(): number;
|
|
33
|
+
setTimer(fn: () => void, ms: number): ReturnType<typeof setTimeout>;
|
|
34
|
+
clearTimer(timer: ReturnType<typeof setTimeout>): void;
|
|
35
|
+
deadlineMs?: number;
|
|
36
|
+
};
|
|
30
37
|
}
|
|
31
38
|
export interface OwnerChannelHandle {
|
|
32
39
|
start(): Promise<void>;
|
|
33
40
|
drain(): Promise<void>;
|
|
34
41
|
close(): Promise<void>;
|
|
42
|
+
/** Reattach this same supervisor-owned channel after a daemon generation change. */
|
|
43
|
+
recover?(epoch: string): Promise<void>;
|
|
44
|
+
/** False when shutdown skipped client disposal because quiescence was unproven. */
|
|
45
|
+
binderReleaseSafe?(): boolean;
|
|
35
46
|
manage(request: OwnerChannelManagementRequest): Promise<OwnerChannelManagementResult>;
|
|
47
|
+
/** Fleet-owned deterministic lifecycle notice; absent on legacy test doubles. */
|
|
48
|
+
notifyFleetSpawn?(event: ManagedFleetSpawnResult): Promise<void>;
|
|
36
49
|
beginFleetCommandAudit?(requestId: string, argv: string[]): Promise<FleetAuditAttempt>;
|
|
37
50
|
finishFleetCommandAudit?(input: {
|
|
38
51
|
correlationId: string;
|
|
@@ -43,6 +56,17 @@ export interface OwnerChannelHandle {
|
|
|
43
56
|
presentation?: FleetAuditPresentation;
|
|
44
57
|
}): Promise<FleetAuditAttempt>;
|
|
45
58
|
}
|
|
59
|
+
export declare const OWNER_RECOVERY_QUEUE_CAPACITY = 32;
|
|
60
|
+
export declare const OWNER_RECOVERY_DEGRADED = "OWNER_RECOVERY_DEGRADED";
|
|
61
|
+
export declare class OwnerRecoveryDegradedError extends Error {
|
|
62
|
+
readonly code = "OWNER_RECOVERY_DEGRADED";
|
|
63
|
+
constructor();
|
|
64
|
+
}
|
|
65
|
+
export declare class OwnerRecoveryTimeoutError extends Error {
|
|
66
|
+
readonly stage: string;
|
|
67
|
+
readonly code = "OWNER_RECOVERY_DEGRADED";
|
|
68
|
+
constructor(stage: string);
|
|
69
|
+
}
|
|
46
70
|
export type OwnerChannelManagementRequest = {
|
|
47
71
|
action: 'contact_list';
|
|
48
72
|
} | {
|
|
@@ -169,7 +193,17 @@ export declare class OwnerChannel implements OwnerChannelHandle {
|
|
|
169
193
|
private readonly completionTasks;
|
|
170
194
|
private readonly activeRequests;
|
|
171
195
|
private managementTail;
|
|
196
|
+
private recoveryEpoch?;
|
|
197
|
+
private recoveryTask?;
|
|
198
|
+
private recoveryToken;
|
|
199
|
+
private recoveryQueued;
|
|
200
|
+
private watchGeneration;
|
|
201
|
+
/** A timed-out client operation that must settle before replacement is safe. */
|
|
202
|
+
private recoveryQuiescence?;
|
|
203
|
+
private releaseSafe;
|
|
172
204
|
private ready;
|
|
205
|
+
private startedOnce;
|
|
206
|
+
private shutdownDegraded;
|
|
173
207
|
private binder?;
|
|
174
208
|
private binderOwnedInternally;
|
|
175
209
|
private readonly fleetOps;
|
|
@@ -179,7 +213,9 @@ export declare class OwnerChannel implements OwnerChannelHandle {
|
|
|
179
213
|
start(): Promise<void>;
|
|
180
214
|
drain(): Promise<void>;
|
|
181
215
|
close(): Promise<void>;
|
|
216
|
+
binderReleaseSafe(): boolean;
|
|
182
217
|
manage(request: OwnerChannelManagementRequest): Promise<OwnerChannelManagementResult>;
|
|
218
|
+
notifyFleetSpawn(event: ManagedFleetSpawnResult): Promise<void>;
|
|
183
219
|
beginFleetCommandAudit(requestId: string, argv: string[]): Promise<FleetAuditAttempt>;
|
|
184
220
|
finishFleetCommandAudit(input: {
|
|
185
221
|
correlationId: string;
|
|
@@ -189,6 +225,10 @@ export declare class OwnerChannel implements OwnerChannelHandle {
|
|
|
189
225
|
resourceIds?: Record<string, string>;
|
|
190
226
|
presentation?: FleetAuditPresentation;
|
|
191
227
|
}): Promise<FleetAuditAttempt>;
|
|
228
|
+
recover(epoch: string): Promise<void>;
|
|
229
|
+
private recoveryStage;
|
|
230
|
+
private writeShutdownState;
|
|
231
|
+
private queueManagement;
|
|
192
232
|
private manageNow;
|
|
193
233
|
/**
|
|
194
234
|
* The daemon reports established contacts and pending introductions as two
|
|
@@ -3,6 +3,7 @@ import { existsSync, readFileSync } from 'node:fs';
|
|
|
3
3
|
import { mkdir, readFile, readdir, rm } from 'node:fs/promises';
|
|
4
4
|
import { join } from 'node:path';
|
|
5
5
|
import { canonicalCid } from '../config.js';
|
|
6
|
+
import { DAEMON_RECOVERY_DEADLINE_MS } from '../daemon-recovery.js';
|
|
6
7
|
import { replaceFileAtomically } from '../atomic-file.js';
|
|
7
8
|
import { TaskRoomApplicationService } from '../application/task-room-service.js';
|
|
8
9
|
import { RoleLifecycleService } from '../application/role-command-service.js';
|
|
@@ -22,6 +23,24 @@ import { OwnerTaskState, ownerTaskAuditId, ownerTaskDigest, } from './tasks.js';
|
|
|
22
23
|
import { AttachmentRecoveryState, admitAttachments, cleanupAttachmentRoot, parseIncomingAttachments, parseRetrievedAttachments, prepareAttachmentDirectory, recoveredAttachment, removeRequestDirectory, safeField, validateAttachmentSelection, validateAttachmentRelaySelection, writeRecoveredAttachment, } from './attachments.js';
|
|
23
24
|
import { acquireOwnerBinderLease, OWNER_BIND_HANDOFF_TIMEOUT_MS, } from './binder.js';
|
|
24
25
|
import { MessageRecoveryState } from './message-recovery.js';
|
|
26
|
+
export const OWNER_RECOVERY_QUEUE_CAPACITY = 32;
|
|
27
|
+
export const OWNER_RECOVERY_DEGRADED = 'OWNER_RECOVERY_DEGRADED';
|
|
28
|
+
export class OwnerRecoveryDegradedError extends Error {
|
|
29
|
+
code = OWNER_RECOVERY_DEGRADED;
|
|
30
|
+
constructor() {
|
|
31
|
+
super('owner channel recovery is degraded');
|
|
32
|
+
this.name = 'OwnerRecoveryDegradedError';
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
export class OwnerRecoveryTimeoutError extends Error {
|
|
36
|
+
stage;
|
|
37
|
+
code = OWNER_RECOVERY_DEGRADED;
|
|
38
|
+
constructor(stage) {
|
|
39
|
+
super(`owner channel recovery timed out at ${stage}`);
|
|
40
|
+
this.stage = stage;
|
|
41
|
+
this.name = 'OwnerRecoveryTimeoutError';
|
|
42
|
+
}
|
|
43
|
+
}
|
|
25
44
|
const OWNER_UPDATE_MIN_INTERVAL_MS = 5_000;
|
|
26
45
|
const OWNER_UPDATE_MAX_COUNT = 20;
|
|
27
46
|
const OWNER_UPDATE_MAX_CHARS = 280;
|
|
@@ -77,7 +96,17 @@ export class OwnerChannel {
|
|
|
77
96
|
completionTasks = new Set();
|
|
78
97
|
activeRequests = new Map();
|
|
79
98
|
managementTail = Promise.resolve();
|
|
99
|
+
recoveryEpoch;
|
|
100
|
+
recoveryTask;
|
|
101
|
+
recoveryToken = 0;
|
|
102
|
+
recoveryQueued = 0;
|
|
103
|
+
watchGeneration = 0;
|
|
104
|
+
/** A timed-out client operation that must settle before replacement is safe. */
|
|
105
|
+
recoveryQuiescence;
|
|
106
|
+
releaseSafe = true;
|
|
80
107
|
ready = false;
|
|
108
|
+
startedOnce = false;
|
|
109
|
+
shutdownDegraded = false;
|
|
81
110
|
binder;
|
|
82
111
|
binderOwnedInternally = false;
|
|
83
112
|
fleetOps;
|
|
@@ -127,6 +156,7 @@ export class OwnerChannel {
|
|
|
127
156
|
}
|
|
128
157
|
async start() {
|
|
129
158
|
this.stopping = false;
|
|
159
|
+
this.shutdownDegraded = false;
|
|
130
160
|
this.binderOwnedInternally = !this.options.binderLease;
|
|
131
161
|
this.binder = this.options.binderLease ?? await acquireOwnerBinderLease(this.options.stateDir, this.options.role, this.options.config.identity, this.options.binderDeps);
|
|
132
162
|
try {
|
|
@@ -167,11 +197,12 @@ export class OwnerChannel {
|
|
|
167
197
|
this.attachmentRecovery.cleanup(Date.now(), this.attachmentConfig.retention_ms);
|
|
168
198
|
void cleanupAttachmentRoot(this.attachmentRoot, Date.now(), this.attachmentConfig.retention_ms).catch(error => this.logError('attachment crash cleanup failed', error));
|
|
169
199
|
}
|
|
200
|
+
this.startedOnce = true;
|
|
170
201
|
this.ready = true;
|
|
171
202
|
// Do not make role startup wait for an old owner request to finish a turn.
|
|
172
203
|
// watchLoop itself drains before every establishment, including this first
|
|
173
204
|
// one, so there is no drain-to-tip race.
|
|
174
|
-
this.watchTask = this.watchLoop();
|
|
205
|
+
this.watchTask = this.watchLoop(++this.watchGeneration);
|
|
175
206
|
}
|
|
176
207
|
drain() {
|
|
177
208
|
this.drainRequested = true;
|
|
@@ -186,25 +217,65 @@ export class OwnerChannel {
|
|
|
186
217
|
return this.drainTask;
|
|
187
218
|
}
|
|
188
219
|
async close() {
|
|
220
|
+
if (this.recoveryTask)
|
|
221
|
+
this.shutdownDegraded = true;
|
|
189
222
|
this.stopping = true;
|
|
190
223
|
this.ready = false;
|
|
191
|
-
this.
|
|
192
|
-
|
|
224
|
+
this.recoveryToken++;
|
|
225
|
+
this.watchGeneration++;
|
|
226
|
+
const staleWatch = this.watchTask;
|
|
193
227
|
this.watchTask = undefined;
|
|
194
|
-
|
|
228
|
+
this.watchAbort?.abort();
|
|
229
|
+
const now = this.options.recoveryDeps?.now ?? (() => Date.now());
|
|
230
|
+
const deadlineAt = now()
|
|
231
|
+
+ (this.options.recoveryDeps?.deadlineMs ?? DAEMON_RECOVERY_DEADLINE_MS);
|
|
232
|
+
let disposed = false;
|
|
195
233
|
try {
|
|
196
|
-
await this.
|
|
234
|
+
await this.recoveryStage('shutdown_quiescence', Promise.all([
|
|
235
|
+
staleWatch?.catch(error => this.logError('watch shutdown failed', error)),
|
|
236
|
+
this.drainTask?.catch(error => this.logError('drain shutdown failed', error)),
|
|
237
|
+
this.managementTail.then(() => undefined),
|
|
238
|
+
this.recoveryQuiescence,
|
|
239
|
+
]), deadlineAt);
|
|
240
|
+
await this.recoveryStage('shutdown_close', this.client.close(), deadlineAt);
|
|
241
|
+
disposed = true;
|
|
242
|
+
this.writeShutdownState('closed', 'OWNER_SHUTDOWN_CLOSED', now());
|
|
243
|
+
}
|
|
244
|
+
catch (error) {
|
|
245
|
+
this.shutdownDegraded = true;
|
|
246
|
+
this.releaseSafe = false;
|
|
247
|
+
const stage = error instanceof OwnerRecoveryTimeoutError ? error.stage : 'shutdown_unknown';
|
|
248
|
+
this.writeShutdownState('degraded', `OWNER_${stage.toUpperCase()}_TIMEOUT`, now());
|
|
249
|
+
this.options.log(`[${this.options.role}] owner shutdown degraded stage=${stage}`);
|
|
197
250
|
}
|
|
198
251
|
finally {
|
|
199
|
-
if (this.binderOwnedInternally)
|
|
252
|
+
if (disposed && this.binderOwnedInternally)
|
|
200
253
|
this.binder?.release();
|
|
201
|
-
|
|
254
|
+
if (disposed)
|
|
255
|
+
this.binder = undefined;
|
|
202
256
|
}
|
|
203
257
|
}
|
|
258
|
+
binderReleaseSafe() { return this.releaseSafe; }
|
|
204
259
|
manage(request) {
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
260
|
+
return this.queueManagement(() => this.manageNow(request));
|
|
261
|
+
}
|
|
262
|
+
notifyFleetSpawn(event) {
|
|
263
|
+
return this.queueManagement(async () => {
|
|
264
|
+
if (!this.ready || this.stopping)
|
|
265
|
+
throw new Error('owner-channel MCP client is unavailable');
|
|
266
|
+
const model = event.model ? `, model ${event.model}` : '';
|
|
267
|
+
const monitorPolicy = event.monitor.interrupt === true
|
|
268
|
+
? ' with interruption'
|
|
269
|
+
: event.monitor.interrupt === 'after_tool' ? ' with after-tool steering' : '';
|
|
270
|
+
const monitor = `${event.monitor.mode} monitor${monitorPolicy}`;
|
|
271
|
+
const permission = event.permissionMode
|
|
272
|
+
? `; permission ${event.permissionMode.fleetMode}, native ${event.permissionMode.nativeMode}`
|
|
273
|
+
: '';
|
|
274
|
+
const inherited = event.inherited.length
|
|
275
|
+
? ` Supervisor inherited omitted defaults: ${event.inherited.join(', ')}.` : '';
|
|
276
|
+
await this.sendProactiveMessage(`🧑💻 ${event.caller} spawned ${event.lifetime} agent ${event.role} `
|
|
277
|
+
+ `(${event.harness}/${event.session}${model}; ${monitor}${permission}).${inherited}`, `fleet-spawn\0${event.creationActionId}`, 0);
|
|
278
|
+
});
|
|
208
279
|
}
|
|
209
280
|
beginFleetCommandAudit(requestId, argv) {
|
|
210
281
|
const run = this.managementTail.then(async () => {
|
|
@@ -263,6 +334,129 @@ export class OwnerChannel {
|
|
|
263
334
|
this.managementTail = run.then(() => undefined, () => undefined);
|
|
264
335
|
return run;
|
|
265
336
|
}
|
|
337
|
+
recover(epoch) {
|
|
338
|
+
if (this.stopping)
|
|
339
|
+
return Promise.reject(new Error('owner channel is stopping'));
|
|
340
|
+
if (this.recoveryEpoch === epoch && this.recoveryTask)
|
|
341
|
+
return this.recoveryTask;
|
|
342
|
+
this.recoveryEpoch = epoch;
|
|
343
|
+
this.ready = false;
|
|
344
|
+
const token = ++this.recoveryToken;
|
|
345
|
+
const watchGeneration = ++this.watchGeneration;
|
|
346
|
+
const staleWatch = this.watchTask;
|
|
347
|
+
this.watchTask = undefined;
|
|
348
|
+
this.watchAbort?.abort();
|
|
349
|
+
const superseded = () => this.stopping || token !== this.recoveryToken;
|
|
350
|
+
const now = this.options.recoveryDeps?.now ?? (() => Date.now());
|
|
351
|
+
const deadlineAt = now()
|
|
352
|
+
+ (this.options.recoveryDeps?.deadlineMs ?? DAEMON_RECOVERY_DEADLINE_MS);
|
|
353
|
+
const run = this.managementTail.then(async () => {
|
|
354
|
+
if (superseded())
|
|
355
|
+
throw new Error('owner recovery epoch superseded');
|
|
356
|
+
if (this.recoveryQuiescence) {
|
|
357
|
+
const debt = this.recoveryQuiescence;
|
|
358
|
+
await this.recoveryStage('prior_quiescence', debt, deadlineAt);
|
|
359
|
+
if (this.recoveryQuiescence === debt)
|
|
360
|
+
this.recoveryQuiescence = undefined;
|
|
361
|
+
}
|
|
362
|
+
// Quiesce every stale-client reader before replacing its transport. A
|
|
363
|
+
// watch can be inside the shared drain task when its abort lands.
|
|
364
|
+
await this.recoveryStage('stale_quiescence', Promise.all([
|
|
365
|
+
staleWatch?.catch(error => this.logError('stale watch shutdown failed', error)),
|
|
366
|
+
this.drainTask?.catch(error => this.logError('stale drain shutdown failed', error)),
|
|
367
|
+
]), deadlineAt);
|
|
368
|
+
if (superseded())
|
|
369
|
+
throw new Error('owner recovery epoch superseded');
|
|
370
|
+
await this.recoveryStage('close', this.client.close(), deadlineAt);
|
|
371
|
+
if (superseded())
|
|
372
|
+
throw new Error('owner recovery epoch superseded');
|
|
373
|
+
try {
|
|
374
|
+
await this.recoveryStage('start', this.client.start(), deadlineAt);
|
|
375
|
+
if (superseded())
|
|
376
|
+
throw new Error('owner recovery epoch superseded');
|
|
377
|
+
await this.recoveryStage('bind', this.client.bindIdentity(this.options.config.identity), deadlineAt);
|
|
378
|
+
if (superseded())
|
|
379
|
+
throw new Error('owner recovery epoch superseded');
|
|
380
|
+
// Journal-aware drain is the only operation allowed to mark owner mail
|
|
381
|
+
// read. The final listing is a typed, read-only channel probe.
|
|
382
|
+
await this.recoveryStage('drain', this.drain(), deadlineAt);
|
|
383
|
+
await this.recoveryStage('list', this.client.listIncomingMessages(), deadlineAt);
|
|
384
|
+
if (superseded())
|
|
385
|
+
throw new Error('owner recovery epoch superseded');
|
|
386
|
+
this.ready = true;
|
|
387
|
+
this.watchTask = this.watchLoop(watchGeneration);
|
|
388
|
+
}
|
|
389
|
+
catch (error) {
|
|
390
|
+
this.ready = false;
|
|
391
|
+
if (token === this.recoveryToken)
|
|
392
|
+
this.recoveryToken++;
|
|
393
|
+
// A timed-out operation may still be using this client. Never overlap
|
|
394
|
+
// its disposal; the recorded quiescence debt gates the next retry.
|
|
395
|
+
if (!this.recoveryQuiescence) {
|
|
396
|
+
try {
|
|
397
|
+
await this.recoveryStage('close_after_failure', this.client.close(), deadlineAt);
|
|
398
|
+
}
|
|
399
|
+
catch (closeError) {
|
|
400
|
+
this.logError('recovery client close failed', closeError);
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
throw error;
|
|
404
|
+
}
|
|
405
|
+
});
|
|
406
|
+
this.managementTail = run.then(() => undefined, () => undefined);
|
|
407
|
+
const task = run.finally(() => {
|
|
408
|
+
if (this.recoveryTask === task)
|
|
409
|
+
this.recoveryTask = undefined;
|
|
410
|
+
});
|
|
411
|
+
this.recoveryTask = task;
|
|
412
|
+
return task;
|
|
413
|
+
}
|
|
414
|
+
recoveryStage(stage, operation, deadlineAt) {
|
|
415
|
+
const now = this.options.recoveryDeps?.now ?? (() => Date.now());
|
|
416
|
+
const remaining = Math.max(0, deadlineAt - now());
|
|
417
|
+
const setTimer = this.options.recoveryDeps?.setTimer ?? ((fn, ms) => setTimeout(fn, ms));
|
|
418
|
+
const clearTimer = this.options.recoveryDeps?.clearTimer ?? (timer => clearTimeout(timer));
|
|
419
|
+
let timer;
|
|
420
|
+
const timeout = new Promise((_resolve, reject) => {
|
|
421
|
+
timer = setTimer(() => reject(new OwnerRecoveryTimeoutError(stage)), remaining);
|
|
422
|
+
});
|
|
423
|
+
return Promise.race([operation, timeout]).catch(error => {
|
|
424
|
+
if (error instanceof OwnerRecoveryTimeoutError) {
|
|
425
|
+
const debt = operation.then(() => undefined, () => undefined);
|
|
426
|
+
this.recoveryQuiescence = debt;
|
|
427
|
+
}
|
|
428
|
+
throw error;
|
|
429
|
+
}).finally(() => clearTimer(timer));
|
|
430
|
+
}
|
|
431
|
+
writeShutdownState(state, reason, at) {
|
|
432
|
+
try {
|
|
433
|
+
replaceFileAtomically(join(this.options.stateDir, '.owner-channel-shutdown.json'), JSON.stringify({ version: 1, state, reason, at: new Date(at).toISOString() }, null, 2) + '\n');
|
|
434
|
+
}
|
|
435
|
+
catch (error) {
|
|
436
|
+
this.logError('shutdown status write failed', error);
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
queueManagement(operation) {
|
|
440
|
+
const deferred = this.recoveryTask !== undefined && !this.ready;
|
|
441
|
+
if (!this.ready && !deferred)
|
|
442
|
+
return Promise.reject(!this.startedOnce || (this.stopping && !this.shutdownDegraded)
|
|
443
|
+
? new Error('owner-channel MCP client is unavailable')
|
|
444
|
+
: new OwnerRecoveryDegradedError());
|
|
445
|
+
if (deferred && this.recoveryQueued >= OWNER_RECOVERY_QUEUE_CAPACITY)
|
|
446
|
+
return Promise.reject(new Error('owner recovery queue is full'));
|
|
447
|
+
if (deferred)
|
|
448
|
+
this.recoveryQueued++;
|
|
449
|
+
const run = this.managementTail.then(() => {
|
|
450
|
+
if (!this.ready || this.stopping)
|
|
451
|
+
throw new OwnerRecoveryDegradedError();
|
|
452
|
+
return operation();
|
|
453
|
+
});
|
|
454
|
+
this.managementTail = run.then(() => undefined, () => undefined);
|
|
455
|
+
return run.finally(() => {
|
|
456
|
+
if (deferred)
|
|
457
|
+
this.recoveryQueued = Math.max(0, this.recoveryQueued - 1);
|
|
458
|
+
});
|
|
459
|
+
}
|
|
266
460
|
async manageNow(request) {
|
|
267
461
|
if (!this.ready || this.stopping)
|
|
268
462
|
throw new Error('owner-channel MCP client is unavailable');
|
|
@@ -1916,7 +2110,7 @@ export class OwnerChannel {
|
|
|
1916
2110
|
}
|
|
1917
2111
|
return undefined;
|
|
1918
2112
|
}
|
|
1919
|
-
async watchLoop() {
|
|
2113
|
+
async watchLoop(generation) {
|
|
1920
2114
|
const sleep = this.options.binderDeps?.sleep
|
|
1921
2115
|
?? (ms => new Promise(resolve => setTimeout(resolve, ms)));
|
|
1922
2116
|
const restored = this.readWatchState();
|
|
@@ -1925,7 +2119,7 @@ export class OwnerChannel {
|
|
|
1925
2119
|
state = this.writeWatchState(state, 'OWNER_WATCH_STATE_RECOVERED');
|
|
1926
2120
|
let delayMs = 1_000;
|
|
1927
2121
|
let attempts = 0;
|
|
1928
|
-
while (!this.stopping) {
|
|
2122
|
+
while (!this.stopping && generation === this.watchGeneration) {
|
|
1929
2123
|
const ctrl = new AbortController();
|
|
1930
2124
|
this.watchAbort = ctrl;
|
|
1931
2125
|
try {
|
|
@@ -1935,12 +2129,12 @@ export class OwnerChannel {
|
|
|
1935
2129
|
// fall into a gap. Persistent history plus fleet's body-free claim
|
|
1936
2130
|
// journal is authoritative; durable wire-ID dedupe makes hints harmless.
|
|
1937
2131
|
await this.drain();
|
|
1938
|
-
if (this.stopping)
|
|
2132
|
+
if (this.stopping || generation !== this.watchGeneration)
|
|
1939
2133
|
return;
|
|
1940
2134
|
state = this.writeWatchState(state, 'OWNER_WATCH_CONNECTING', { reconnected: attempts > 0 });
|
|
1941
2135
|
attempts++;
|
|
1942
2136
|
for await (const _event of this.client.watchNotifications(this.options.config.identity, { since: 0, signal: ctrl.signal })) {
|
|
1943
|
-
if (this.stopping)
|
|
2137
|
+
if (this.stopping || generation !== this.watchGeneration)
|
|
1944
2138
|
return;
|
|
1945
2139
|
state = this.writeWatchState(state, 'OWNER_WATCH_CONNECTED', { resetFailures: true });
|
|
1946
2140
|
delayMs = 1_000;
|
|
@@ -1949,11 +2143,11 @@ export class OwnerChannel {
|
|
|
1949
2143
|
// recorded before a managed request is dispatched.
|
|
1950
2144
|
await this.drain();
|
|
1951
2145
|
}
|
|
1952
|
-
if (!this.stopping)
|
|
2146
|
+
if (!this.stopping && generation === this.watchGeneration)
|
|
1953
2147
|
throw new Error('ours SDK notification stream ended');
|
|
1954
2148
|
}
|
|
1955
2149
|
catch (error) {
|
|
1956
|
-
if (this.stopping)
|
|
2150
|
+
if (this.stopping || generation !== this.watchGeneration)
|
|
1957
2151
|
return;
|
|
1958
2152
|
// SDK 2 deliberately hides transport status behind its typed stream.
|
|
1959
2153
|
// Do not parse error prose to rediscover it: every failure follows the
|
package/dist/runner.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type ResolvedRole } from './config.js';
|
|
2
2
|
import { type MonitorHandle, type MonitorOpts, type FetchLike } from './monitor.js';
|
|
3
|
+
import { type DaemonGenerationProbe } from './daemon-recovery.js';
|
|
3
4
|
import { type Exec } from './exec.js';
|
|
4
5
|
import { RoleControlServer } from './session/control.js';
|
|
5
6
|
import type { AgentSession, ExitRecord, TurnResult } from './session/types.js';
|
|
@@ -15,6 +16,7 @@ export interface RunnerDeps {
|
|
|
15
16
|
log(line: string): void;
|
|
16
17
|
/** HTTP transport for the monitor's daemon long-poll (injectable for tests). */
|
|
17
18
|
fetch: FetchLike;
|
|
19
|
+
probeGeneration(env: NodeJS.ProcessEnv): Promise<DaemonGenerationProbe>;
|
|
18
20
|
/** Construct the supervisor mail monitor (injectable so tests stub it out). */
|
|
19
21
|
createMonitor(opts: MonitorOpts): MonitorHandle;
|
|
20
22
|
/** Construct trusted owner ingress (injectable for lifecycle tests). */
|
|
@@ -31,6 +33,11 @@ export interface RunnerDeps {
|
|
|
31
33
|
/** Lets a test (or a shutdown path) end the supervised restart loop. */
|
|
32
34
|
shouldStop?(): boolean;
|
|
33
35
|
}
|
|
36
|
+
export declare const SUPERVISOR_RECYCLE_REQUIRED = "OWNER_CHANNEL_SUPERVISOR_RECYCLE_REQUIRED";
|
|
37
|
+
export declare class SupervisorRecycleRequiredError extends Error {
|
|
38
|
+
readonly code = "OWNER_CHANNEL_SUPERVISOR_RECYCLE_REQUIRED";
|
|
39
|
+
constructor();
|
|
40
|
+
}
|
|
34
41
|
/** Environment injected only into the managed harness process. */
|
|
35
42
|
export declare function managedFleetProxyEnv(role: ResolvedRole, stateDir: string): Record<string, string>;
|
|
36
43
|
/**
|
|
@@ -176,4 +183,4 @@ export declare function runSupervised(name: string, opts?: {
|
|
|
176
183
|
allowResumeRotation?: boolean;
|
|
177
184
|
}, d: Partial<RunnerDeps>) => Promise<AttemptResult>): Promise<RestartLedger>;
|
|
178
185
|
/** Temp-agent entrypoint: run once, journal why it ended, then archive its evidence. */
|
|
179
|
-
export declare function runTemp(name: string, deps?: Partial<RunnerDeps
|
|
186
|
+
export declare function runTemp(name: string, deps?: Partial<RunnerDeps>, attempt?: typeof runOnce): Promise<void>;
|
package/dist/runner.js
CHANGED
|
@@ -6,6 +6,8 @@ import { agentDir, stateRoot } from './paths.js';
|
|
|
6
6
|
import { loadConfig, findRole, isolationContextFor, resolveMonitorConfig, resolvePermissions, } from './config.js';
|
|
7
7
|
import { getAdapter } from './harness/registry.js';
|
|
8
8
|
import { createMonitor, probeIdentityPresence, } from './monitor.js';
|
|
9
|
+
import { DaemonGenerationObserver, RoleRecoveryController, probeDaemonGeneration, } from './daemon-recovery.js';
|
|
10
|
+
import { recoverAgentIdentity } from './agent-recovery-gate.js';
|
|
9
11
|
import { realExec } from './exec.js';
|
|
10
12
|
import { resolveIsolation } from './isolation/policy.js';
|
|
11
13
|
import { selectIsolationBackend } from './isolation/registry.js';
|
|
@@ -24,6 +26,14 @@ import { FLEET_PROXY_CALLER_ENV, FLEET_PROXY_STATE_DIR_ENV, } from './fleet-prox
|
|
|
24
26
|
import { effectivePermissionMode } from './permissions.js';
|
|
25
27
|
import { assertModelPinReachesChild, effectiveRoleModel, repinModelEnv } from './model-env.js';
|
|
26
28
|
import { archiveTempState, markTempSupervisorActive, requestedTempStopReason, } from './temp-lifecycle.js';
|
|
29
|
+
export const SUPERVISOR_RECYCLE_REQUIRED = 'OWNER_CHANNEL_SUPERVISOR_RECYCLE_REQUIRED';
|
|
30
|
+
export class SupervisorRecycleRequiredError extends Error {
|
|
31
|
+
code = SUPERVISOR_RECYCLE_REQUIRED;
|
|
32
|
+
constructor() {
|
|
33
|
+
super('owner channel requires a fresh supervisor process after unproven client quiescence');
|
|
34
|
+
this.name = 'SupervisorRecycleRequiredError';
|
|
35
|
+
}
|
|
36
|
+
}
|
|
27
37
|
const defaultDeps = () => ({
|
|
28
38
|
exec: realExec,
|
|
29
39
|
cpuDelegated: () => cpuControllerDelegated(),
|
|
@@ -38,6 +48,7 @@ const defaultDeps = () => ({
|
|
|
38
48
|
now: () => Date.now(),
|
|
39
49
|
log: line => process.stderr.write(line + '\n'),
|
|
40
50
|
fetch: (url, init) => globalThis.fetch(url, init),
|
|
51
|
+
probeGeneration: env => probeDaemonGeneration((url, init) => globalThis.fetch(url, init), env),
|
|
41
52
|
createMonitor: opts => createMonitor(opts),
|
|
42
53
|
createOwnerChannel: opts => new OwnerChannel(opts),
|
|
43
54
|
startAgentSession: (adapter, options) => adapter.start(options),
|
|
@@ -532,6 +543,9 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
532
543
|
name, identity: role.identity, agentDir: dir, cfg: role.monitor,
|
|
533
544
|
deps: resolvedMonitorDeps,
|
|
534
545
|
}) : null;
|
|
546
|
+
const daemonObserver = new DaemonGenerationObserver();
|
|
547
|
+
const initialDaemon = daemonObserver.observe(await deps.probeGeneration(resolvedMonitorDeps.env));
|
|
548
|
+
let sessionStartedWithoutDaemonBaseline = initialDaemon.kind !== 'baseline';
|
|
535
549
|
if (monitor)
|
|
536
550
|
await monitor.prime({ resetCursor: resetMonitorCursor });
|
|
537
551
|
rmSync(exitFile, { force: true });
|
|
@@ -546,6 +560,7 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
546
560
|
let ownerBinder;
|
|
547
561
|
let loopManager;
|
|
548
562
|
let arbiter;
|
|
563
|
+
let recoveryController;
|
|
549
564
|
let reloadLoopConfig;
|
|
550
565
|
let loopGeneration = JSON.stringify((role.loops ?? []).map(loop => [
|
|
551
566
|
loop.name, loop.definitionHash, loop.promptHash,
|
|
@@ -782,16 +797,49 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
782
797
|
// The monitor loop lives exactly as long as the session: it starts once the
|
|
783
798
|
// pane pid is known and is stopped when that pid dies (task dies with runner).
|
|
784
799
|
monitorLoop ??= monitor?.run(pid);
|
|
800
|
+
recoveryController = new RoleRecoveryController({
|
|
801
|
+
role: name, identity: role.identity, stateDir: dir, now: deps.now, sleep: deps.sleep,
|
|
802
|
+
log: deps.log,
|
|
803
|
+
recoverAgent: async () => {
|
|
804
|
+
const evidence = await recoverAgentIdentity(arbiter, role.identity);
|
|
805
|
+
return evidence.ok ? { ok: true } : { ok: false, reason: evidence.reason };
|
|
806
|
+
},
|
|
807
|
+
recoverOwner: async (epoch) => {
|
|
808
|
+
if (!ownerChannel?.recover)
|
|
809
|
+
return { ok: true };
|
|
810
|
+
try {
|
|
811
|
+
await ownerChannel.recover(epoch);
|
|
812
|
+
return { ok: true };
|
|
813
|
+
}
|
|
814
|
+
catch (error) {
|
|
815
|
+
return { ok: false, reason: error instanceof Error
|
|
816
|
+
? `OWNER_${error.name.toUpperCase()}` : 'OWNER_UNKNOWN_ERROR' };
|
|
817
|
+
}
|
|
818
|
+
},
|
|
819
|
+
});
|
|
785
820
|
const start = deps.now();
|
|
786
821
|
let nextLoopReloadAt = deps.now() + 30_000;
|
|
787
822
|
let nextIdentityPollAt = deps.now();
|
|
823
|
+
let nextDaemonProbeAt = deps.now();
|
|
788
824
|
let lastReloadError = '';
|
|
789
825
|
let identityObserved = false;
|
|
790
826
|
let identityAbsentSince;
|
|
791
827
|
let retirementReason;
|
|
828
|
+
let supervisorRecycleRequired = false;
|
|
792
829
|
while (sessionHandle.isAlive()) {
|
|
793
830
|
await deps.sleep(temp ? 500 : 2000);
|
|
794
831
|
const now = deps.now();
|
|
832
|
+
if (now >= nextDaemonProbeAt) {
|
|
833
|
+
nextDaemonProbeAt = now + 2_000;
|
|
834
|
+
const observation = daemonObserver.observe(await deps.probeGeneration(resolvedMonitorDeps.env));
|
|
835
|
+
if (observation.kind === 'lost')
|
|
836
|
+
recoveryController.noteLoss(observation.reason);
|
|
837
|
+
if (observation.kind === 'changed' || observation.kind === 'available'
|
|
838
|
+
|| (observation.kind === 'baseline' && sessionStartedWithoutDaemonBaseline)) {
|
|
839
|
+
sessionStartedWithoutDaemonBaseline = false;
|
|
840
|
+
void recoveryController.recover(observation.generation).catch(error => deps.log(`[${name}] daemon recovery controller failed: ${error?.name ?? 'Error'}`));
|
|
841
|
+
}
|
|
842
|
+
}
|
|
795
843
|
if (temp && deps.shouldStop?.()) {
|
|
796
844
|
retirementReason = requestedTempStopReason(dir) ?? 'supervisor-signal';
|
|
797
845
|
deps.log(`[${name}] temporary supervisor retirement requested (${retirementReason})`);
|
|
@@ -842,6 +890,11 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
842
890
|
control?.setLoopManager(undefined);
|
|
843
891
|
await loopManager.stop();
|
|
844
892
|
}
|
|
893
|
+
// Let already-settled path operations publish their aggregate result before
|
|
894
|
+
// the shutdown fence invalidates the epoch. This never waits on I/O.
|
|
895
|
+
await Promise.resolve();
|
|
896
|
+
await Promise.resolve();
|
|
897
|
+
recoveryController?.cancel();
|
|
845
898
|
control?.setConfigReloader(undefined);
|
|
846
899
|
// Close the authenticated control route before releasing the binder lease;
|
|
847
900
|
// otherwise the predecessor can unlink the replacement's new socket.
|
|
@@ -852,7 +905,12 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
852
905
|
}
|
|
853
906
|
if (ownerChannel)
|
|
854
907
|
await ownerChannel.close();
|
|
855
|
-
|
|
908
|
+
if (ownerChannel?.binderReleaseSafe?.() === false) {
|
|
909
|
+
deps.log(`[${name}] owner binder retained because client quiescence was not proven at shutdown`);
|
|
910
|
+
supervisorRecycleRequired = true;
|
|
911
|
+
}
|
|
912
|
+
else
|
|
913
|
+
ownerBinder?.release();
|
|
856
914
|
if (monitor) {
|
|
857
915
|
monitor.stop();
|
|
858
916
|
await monitorLoop;
|
|
@@ -901,6 +959,8 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
901
959
|
}
|
|
902
960
|
else
|
|
903
961
|
deps.log(`[${name}] ${exitRecord.detail} (${elapsed.toFixed(0)}s) -> next start RESUMES context`);
|
|
962
|
+
if (supervisorRecycleRequired)
|
|
963
|
+
throw new SupervisorRecycleRequiredError();
|
|
904
964
|
return {
|
|
905
965
|
elapsedSecs: elapsed, exit: exitRecord, rotated, mode, modelRecovery,
|
|
906
966
|
...(retirementReason ? { retirementReason } : {}),
|
|
@@ -977,6 +1037,16 @@ export async function runSupervised(name, opts = {}, partialDeps = {}, attempt =
|
|
|
977
1037
|
result = await attempt(name, { configPath: opts.configPath, allowResumeRotation: !ledger.resumeDiscarded }, deps);
|
|
978
1038
|
}
|
|
979
1039
|
catch (e) {
|
|
1040
|
+
if (e instanceof SupervisorRecycleRequiredError) {
|
|
1041
|
+
writeRestartLedger(dir, {
|
|
1042
|
+
...ledger,
|
|
1043
|
+
lastReason: SUPERVISOR_RECYCLE_REQUIRED,
|
|
1044
|
+
nextDelayMs: 0,
|
|
1045
|
+
updatedAt: stamp(),
|
|
1046
|
+
});
|
|
1047
|
+
deps.log(`[${name}] supervisor recycle required: ${SUPERVISOR_RECYCLE_REQUIRED}`);
|
|
1048
|
+
throw e;
|
|
1049
|
+
}
|
|
980
1050
|
// A session that could not even start is an immediate failure like any
|
|
981
1051
|
// other; it must count, or an unstartable role loops forever.
|
|
982
1052
|
result = {
|
|
@@ -1070,7 +1140,7 @@ function fastFailSecsFor(name, configPath) {
|
|
|
1070
1140
|
}
|
|
1071
1141
|
}
|
|
1072
1142
|
/** Temp-agent entrypoint: run once, journal why it ended, then archive its evidence. */
|
|
1073
|
-
export async function runTemp(name, deps = {}) {
|
|
1143
|
+
export async function runTemp(name, deps = {}, attempt = runOnce) {
|
|
1074
1144
|
const dir = agentDir(name, true);
|
|
1075
1145
|
await markTempSupervisorActive(dir);
|
|
1076
1146
|
let signal;
|
|
@@ -1081,7 +1151,7 @@ export async function runTemp(name, deps = {}) {
|
|
|
1081
1151
|
let result;
|
|
1082
1152
|
let failure;
|
|
1083
1153
|
try {
|
|
1084
|
-
result = await
|
|
1154
|
+
result = await attempt(name, { temp: true }, {
|
|
1085
1155
|
...deps,
|
|
1086
1156
|
shouldStop: () => Boolean(signal) || (deps.shouldStop?.() ?? false),
|
|
1087
1157
|
});
|
|
@@ -1095,6 +1165,7 @@ export async function runTemp(name, deps = {}) {
|
|
|
1095
1165
|
const requested = requestedTempStopReason(dir);
|
|
1096
1166
|
const reason = requested
|
|
1097
1167
|
?? result?.retirementReason
|
|
1168
|
+
?? (failure instanceof SupervisorRecycleRequiredError ? 'supervisor-recycle' : undefined)
|
|
1098
1169
|
?? (signal ? 'supervisor-signal' : failure ? 'startup-failure' : 'session-ended');
|
|
1099
1170
|
// A service-manager stop can make the child connection close before the
|
|
1100
1171
|
// runner reaches its normal loop. That is still a successful requested
|
package/dist/temp-lifecycle.d.ts
CHANGED
|
@@ -5,7 +5,7 @@ export declare const TEMP_STOP_REQUEST_FILE = ".temp-stop-request.json";
|
|
|
5
5
|
export declare const TEMP_RECLAIM_BATCH = 32;
|
|
6
6
|
export declare const TEMP_LAUNCH_GRACE_MS = 60000;
|
|
7
7
|
export type TempSupervisorKind = 'systemd-transient' | 'launchd-transient' | 'detached';
|
|
8
|
-
export type TempTerminationReason = 'identity-closed' | 'session-ended' | 'operator-stop' | 'supervisor-signal' | 'startup-failure' | 'stale-supervisor';
|
|
8
|
+
export type TempTerminationReason = 'identity-closed' | 'session-ended' | 'operator-stop' | 'supervisor-signal' | 'supervisor-recycle' | 'startup-failure' | 'stale-supervisor';
|
|
9
9
|
export interface TempSupervisorRecord {
|
|
10
10
|
version: 1;
|
|
11
11
|
role: string;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ours.network/fleet",
|
|
3
|
-
"version": "1.1.0-nightly.
|
|
3
|
+
"version": "1.1.0-nightly.7",
|
|
4
4
|
"description": "Harness-agnostic fleet of persistent, identity-bound AI agents. Declarative fleet.yaml, ACP sessions, supervision, and ours.network messaging.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "FSL-1.1-Apache-2.0",
|