@ours.network/fleet 1.2.0-nightly.54 → 1.2.0-nightly.55
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -12
- package/dist/agent-ours/service.d.ts +4 -1
- package/dist/agent-ours/service.js +31 -7
- package/dist/application/log-service.js +8 -8
- package/dist/application/role-creation-service.js +2 -1
- package/dist/application/role-removal-service.js +7 -1
- package/dist/application/role-repository.js +17 -0
- package/dist/application/task-layouts.js +12 -2
- package/dist/application/task-room-service.js +19 -17
- package/dist/application/types.d.ts +2 -0
- package/dist/build-info.json +4 -4
- package/dist/cli.js +31 -3
- package/dist/config.d.ts +2 -0
- package/dist/harness/agent-session.d.ts +2 -0
- package/dist/harness/claude-code-session.js +1 -1
- package/dist/harness/codex-session.js +2 -2
- package/dist/managed-recovery.d.ts +13 -0
- package/dist/managed-recovery.js +31 -0
- package/dist/ops.d.ts +2 -0
- package/dist/ops.js +15 -2
- package/dist/rooms-tasks/cli.js +21 -0
- package/dist/rooms-tasks/close.d.ts +2 -0
- package/dist/rooms-tasks/close.js +66 -1
- package/dist/rooms-tasks/deletion.js +10 -0
- package/dist/rooms-tasks/erasure.js +4 -2
- package/dist/rooms-tasks/layout-member-retirement.d.ts +10 -0
- package/dist/rooms-tasks/layout-member-retirement.js +90 -0
- package/dist/rooms-tasks/layout-service.d.ts +1 -0
- package/dist/rooms-tasks/layout-service.js +31 -13
- package/dist/rooms-tasks/layout.d.ts +7 -0
- package/dist/rooms-tasks/layout.js +35 -0
- package/dist/rooms-tasks/legacy-task-member.d.ts +19 -0
- package/dist/rooms-tasks/legacy-task-member.js +116 -0
- package/dist/rooms-tasks/member-ownership.d.ts +5 -0
- package/dist/rooms-tasks/member-ownership.js +40 -0
- package/dist/rooms-tasks/provision.js +26 -1
- package/dist/rooms-tasks/recovery.d.ts +19 -0
- package/dist/rooms-tasks/recovery.js +110 -0
- package/dist/rooms-tasks/task-service-retirement.d.ts +11 -0
- package/dist/rooms-tasks/task-service-retirement.js +21 -0
- package/dist/rooms-tasks/task-state.d.ts +1 -0
- package/dist/rooms-tasks/task-state.js +16 -1
- package/dist/rooms-tasks/types.d.ts +3 -0
- package/dist/rooms-tasks/workspace.d.ts +2 -0
- package/dist/rooms-tasks/workspace.js +5 -0
- package/dist/runner.d.ts +6 -0
- package/dist/runner.js +160 -22
- package/dist/session/acp.d.ts +1 -0
- package/dist/session/acp.js +7 -0
- package/dist/session/arbiter.d.ts +5 -0
- package/dist/session/arbiter.js +16 -2
- package/dist/session/codex-app-server.d.ts +1 -0
- package/dist/session/codex-app-server.js +5 -0
- package/dist/session/types.d.ts +2 -0
- package/dist/spawn.d.ts +5 -1
- package/dist/spawn.js +32 -3
- package/dist/startup-readiness.d.ts +1 -1
- package/dist/startup-readiness.js +3 -3
- package/dist/supervisor/adoption.d.ts +8 -0
- package/dist/supervisor/adoption.js +78 -0
- package/dist/supervisor/catalog.d.ts +62 -0
- package/dist/supervisor/catalog.js +254 -0
- package/dist/supervisor/fleet.d.ts +8 -0
- package/dist/supervisor/fleet.js +89 -0
- package/dist/supervisor/index.js +3 -6
- package/dist/supervisor/launchd.d.ts +1 -1
- package/dist/supervisor/launchd.js +24 -12
- package/dist/supervisor/legacy.d.ts +35 -0
- package/dist/supervisor/legacy.js +524 -0
- package/dist/supervisor/manager.d.ts +15 -0
- package/dist/supervisor/manager.js +258 -0
- package/dist/supervisor/scope.d.ts +4 -0
- package/dist/supervisor/scope.js +10 -0
- package/dist/supervisor/systemd.d.ts +2 -2
- package/dist/supervisor/systemd.js +14 -14
- package/dist/task-supervision.d.ts +9 -0
- package/dist/task-supervision.js +145 -0
- package/dist/temp-lifecycle.d.ts +17 -7
- package/dist/temp-lifecycle.js +85 -53
- package/dist/temp-supervisor-recovery.d.ts +1 -1
- package/dist/temp-supervisor-recovery.js +2 -2
- package/docs/fleet-migration.md +54 -0
- package/docs/task-workspaces.md +26 -0
- package/package.json +4 -1
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { existsSync, lstatSync, readFileSync, readdirSync } from 'node:fs';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { agentDir, defaultConfigPath, stateRoot } from '../paths.js';
|
|
4
|
+
import { replaceFileAtomically, withFileLock } from '../atomic-file.js';
|
|
5
|
+
import { loadConfig, findRole } from '../config.js';
|
|
6
|
+
import { realExec } from '../exec.js';
|
|
7
|
+
import { readMember, memberKey, unregisterMember, registerMember } from './catalog.js';
|
|
8
|
+
import { migrateLegacyPermanentMembers } from './legacy.js';
|
|
9
|
+
import { ensureFleetParent } from './fleet.js';
|
|
10
|
+
export async function adoptConfiguredFleet(binPath, exec, platform, configPath, selectedRole) {
|
|
11
|
+
const path = configPath || defaultConfigPath();
|
|
12
|
+
if (existsSync(path)) {
|
|
13
|
+
const config = loadConfig(path);
|
|
14
|
+
await migrateLegacyPermanentMembers({
|
|
15
|
+
roles: selectedRole ? [findRole(config, selectedRole)] : config.roles, binPath, configPath: path, exec, platform,
|
|
16
|
+
prepareParent: () => ensureFleetParent(exec, platform),
|
|
17
|
+
register: async (name, desired, selectedConfig) => {
|
|
18
|
+
await registerMember({ name, kind: 'permanent', dir: agentDir(name), configPath: selectedConfig }, { initialDesired: desired, preserveExisting: true });
|
|
19
|
+
},
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
else
|
|
23
|
+
await ensureFleetParent(exec, platform);
|
|
24
|
+
}
|
|
25
|
+
/** Boot resumes recorded transfers only. It does not rediscover removed agents
|
|
26
|
+
* or decide that every configured role should be started. */
|
|
27
|
+
export async function resumeFleetTransfers(binPath, exec = realExec, platform = process.platform) {
|
|
28
|
+
const root = join(stateRoot(), 'supervisor/legacy-permanent');
|
|
29
|
+
let names;
|
|
30
|
+
try {
|
|
31
|
+
names = readdirSync(root).filter(name => name.endsWith('.json'));
|
|
32
|
+
}
|
|
33
|
+
catch (error) {
|
|
34
|
+
if (error.code === 'ENOENT')
|
|
35
|
+
names = [];
|
|
36
|
+
else
|
|
37
|
+
throw error;
|
|
38
|
+
}
|
|
39
|
+
for (const name of names) {
|
|
40
|
+
const path = join(root, name), stat = lstatSync(path);
|
|
41
|
+
if (!stat.isFile() || stat.isSymbolicLink() || stat.size > 256 * 1024)
|
|
42
|
+
throw Error('LEGACY_PERMANENT_UNSAFE_PROOF');
|
|
43
|
+
const receipt = JSON.parse(readFileSync(path, 'utf8'));
|
|
44
|
+
if (receipt.phase === 'registered')
|
|
45
|
+
continue;
|
|
46
|
+
if (name !== receipt.name + '.json')
|
|
47
|
+
throw Error('LEGACY_PERMANENT_RECEIPT_MISMATCH');
|
|
48
|
+
const configPath = receipt.configPath || defaultConfigPath();
|
|
49
|
+
const role = findRole(loadConfig(configPath), receipt.name);
|
|
50
|
+
await migrateLegacyPermanentMembers({
|
|
51
|
+
roles: [role], binPath, configPath, exec, platform,
|
|
52
|
+
// This runs inside the parent. No recursive OS start, unit writes or reload.
|
|
53
|
+
prepareParent: async () => { },
|
|
54
|
+
register: async (name, desired, selectedConfig) => {
|
|
55
|
+
await registerMember({ name, kind: 'permanent', dir: agentDir(name), configPath: selectedConfig }, { initialDesired: desired, preserveExisting: true });
|
|
56
|
+
},
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
/** A remove racing the native-retired → registered seam must not be undone by
|
|
61
|
+
* boot receipt recovery. Serialize with that seam and close its cursor first. */
|
|
62
|
+
export async function retirePermanentRegistration(name, exec) {
|
|
63
|
+
memberKey(name, 'permanent');
|
|
64
|
+
return withFileLock(join(stateRoot(), 'locks/legacy-permanent', name), async () => {
|
|
65
|
+
const member = readMember(memberKey(name, 'permanent'));
|
|
66
|
+
const path = join(stateRoot(), 'supervisor/legacy-permanent', name + '.json');
|
|
67
|
+
if (member && existsSync(path)) {
|
|
68
|
+
const stat = lstatSync(path);
|
|
69
|
+
if (!stat.isFile() || stat.isSymbolicLink() || stat.size > 256 * 1024)
|
|
70
|
+
throw Error('LEGACY_PERMANENT_UNSAFE_PROOF');
|
|
71
|
+
const receipt = JSON.parse(readFileSync(path, 'utf8'));
|
|
72
|
+
if (receipt.version !== 1 || receipt.name !== name || !['native-retired', 'registered'].includes(receipt.phase))
|
|
73
|
+
throw Error('LEGACY_PERMANENT_RECEIPT_MISMATCH');
|
|
74
|
+
replaceFileAtomically(path, JSON.stringify({ ...receipt, phase: 'registered' }) + '\n');
|
|
75
|
+
}
|
|
76
|
+
return unregisterMember(memberKey(name, 'permanent'), { exec });
|
|
77
|
+
});
|
|
78
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { type Exec } from '../exec.js';
|
|
2
|
+
import type { TaskSupervisorOwner } from '../temp-lifecycle.js';
|
|
3
|
+
import type { Liveness } from './types.js';
|
|
4
|
+
export interface FleetMember {
|
|
5
|
+
version: 1;
|
|
6
|
+
key: string;
|
|
7
|
+
name: string;
|
|
8
|
+
kind: 'permanent' | 'task' | 'temporary' | 'watchdogs';
|
|
9
|
+
dir: string;
|
|
10
|
+
desired: 'running' | 'stopped';
|
|
11
|
+
retiring?: boolean;
|
|
12
|
+
incarnation: string;
|
|
13
|
+
taskOwner?: TaskSupervisorOwner;
|
|
14
|
+
launchId?: string;
|
|
15
|
+
configPath?: string;
|
|
16
|
+
environment?: Record<string, string>;
|
|
17
|
+
generation?: string;
|
|
18
|
+
pid?: number;
|
|
19
|
+
retryAt?: number;
|
|
20
|
+
}
|
|
21
|
+
/** Private 0600 member state retains exactly the creator's profile selection.
|
|
22
|
+
* Pin the resolved profile path even when the caller relied on HOME defaults. */
|
|
23
|
+
export declare function captureMemberEnvironment(env?: NodeJS.ProcessEnv): Promise<Record<string, string>>;
|
|
24
|
+
export declare function managedMemberEnvironment(member: FleetMember, parent?: NodeJS.ProcessEnv): NodeJS.ProcessEnv;
|
|
25
|
+
export declare const catalogRoot: () => string;
|
|
26
|
+
export declare function memberKey(name: string, kind: FleetMember['kind']): string;
|
|
27
|
+
export declare function memberPath(key: string): string;
|
|
28
|
+
export declare const withMemberLock: <T>(key: string, work: () => T | Promise<T>) => Promise<T>;
|
|
29
|
+
export declare function readMember(key: string): FleetMember | undefined;
|
|
30
|
+
export declare function writeMember(record: FleetMember): void;
|
|
31
|
+
export declare function listMemberKeys(): string[];
|
|
32
|
+
export declare function listMembers(): FleetMember[];
|
|
33
|
+
export type MemberRegistration = Pick<FleetMember, 'name' | 'kind' | 'dir' | 'taskOwner' | 'launchId' | 'configPath' | 'environment'>;
|
|
34
|
+
/** Persist intent before starting anything. The parent never discovers members from tmp directories. */
|
|
35
|
+
export declare function registerMember(input: MemberRegistration, options?: {
|
|
36
|
+
initialDesired?: FleetMember['desired'];
|
|
37
|
+
preserveExisting?: boolean;
|
|
38
|
+
}): Promise<{
|
|
39
|
+
created: boolean;
|
|
40
|
+
member: FleetMember;
|
|
41
|
+
}>;
|
|
42
|
+
export interface CatalogDeps {
|
|
43
|
+
exec?: Exec;
|
|
44
|
+
sleep?(ms: number): Promise<void>;
|
|
45
|
+
timeoutMs?: number;
|
|
46
|
+
}
|
|
47
|
+
/** A unique generation in argv fences PID reuse, including the spawn-to-pid-write crash seam. */
|
|
48
|
+
export declare function memberProcess(record: FleetMember, exec?: Exec): Promise<Liveness & {
|
|
49
|
+
pid?: number;
|
|
50
|
+
}>;
|
|
51
|
+
export declare function catalogLiveness(key: string, exec?: Exec): Promise<Liveness>;
|
|
52
|
+
export declare function waitMember(key: string, desired: 'running' | 'stopped', deps?: CatalogDeps): Promise<void>;
|
|
53
|
+
export declare function stopMember(key: string, deps?: CatalogDeps): Promise<void>;
|
|
54
|
+
export declare function unregisterMember(key: string, deps?: CatalogDeps): Promise<boolean>;
|
|
55
|
+
export declare function unregisterTaskMember(role: string, expected: {
|
|
56
|
+
taskId: string;
|
|
57
|
+
creationActionId?: string;
|
|
58
|
+
launchId?: string;
|
|
59
|
+
}, deps?: CatalogDeps): Promise<void>;
|
|
60
|
+
export declare function hasTaskMemberRegistration(role: string): boolean;
|
|
61
|
+
export declare function assertTaskRegistrationsAbsent(taskId: string): void;
|
|
62
|
+
export declare const permanentRegistration: (name: string) => MemberRegistration;
|
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
import { chmodSync, mkdirSync, lstatSync, readFileSync, readdirSync, rmSync } from 'node:fs';
|
|
2
|
+
import { join, resolve } from 'node:path';
|
|
3
|
+
import { randomUUID } from 'node:crypto';
|
|
4
|
+
import { replaceFileAtomically, withFileLock } from '../atomic-file.js';
|
|
5
|
+
import { agentDir, stateRoot, watchdogsRoot } from '../paths.js';
|
|
6
|
+
import { realExec } from '../exec.js';
|
|
7
|
+
import { assertSafeAncestors } from '../rooms-tasks/workspace.js';
|
|
8
|
+
const MEMBER_ENV_KEYS = ['HOME', 'PATH', 'XDG_RUNTIME_DIR', 'CODEX_HOME', 'CODEX_PATH', 'OURS_PORT', 'OURS_STATE_DIR', 'OURS_API_TOKEN', 'OURS_CONFIG'];
|
|
9
|
+
/** Private 0600 member state retains exactly the creator's profile selection.
|
|
10
|
+
* Pin the resolved profile path even when the caller relied on HOME defaults. */
|
|
11
|
+
export async function captureMemberEnvironment(env = process.env) {
|
|
12
|
+
const { clientConfigPath } = await import('../client-profile.js');
|
|
13
|
+
return Object.fromEntries([...MEMBER_ENV_KEYS.flatMap(key => env[key] === undefined ? [] : [[key, env[key]]]), ['OURS_CONFIG', clientConfigPath(env)]]);
|
|
14
|
+
}
|
|
15
|
+
export function managedMemberEnvironment(member, parent = process.env) {
|
|
16
|
+
const env = { ...parent };
|
|
17
|
+
if (member.environment) {
|
|
18
|
+
for (const key of MEMBER_ENV_KEYS)
|
|
19
|
+
delete env[key];
|
|
20
|
+
Object.assign(env, member.environment);
|
|
21
|
+
}
|
|
22
|
+
return env;
|
|
23
|
+
}
|
|
24
|
+
export const catalogRoot = () => join(stateRoot(), 'supervisor', 'members');
|
|
25
|
+
export function memberKey(name, kind) {
|
|
26
|
+
if (!/^[A-Za-z0-9_-]{1,160}$/.test(name))
|
|
27
|
+
throw Error('FLEET_MEMBER_INVALID_NAME');
|
|
28
|
+
return `${kind}-${name}`;
|
|
29
|
+
}
|
|
30
|
+
export function memberPath(key) {
|
|
31
|
+
if (!/^(permanent|task|temporary|watchdogs)-[A-Za-z0-9_-]{1,160}$/.test(key))
|
|
32
|
+
throw Error('FLEET_MEMBER_INVALID_KEY');
|
|
33
|
+
return join(catalogRoot(), `${key}.json`);
|
|
34
|
+
}
|
|
35
|
+
export const withMemberLock = (key, work) => withFileLock(join(stateRoot(), 'locks', 'fleet-members', memberKeyFromValidatedKey(key)), work);
|
|
36
|
+
function memberKeyFromValidatedKey(key) { memberPath(key); return key; }
|
|
37
|
+
export function readMember(key) {
|
|
38
|
+
const path = memberPath(key);
|
|
39
|
+
assertSafeAncestors(catalogRoot());
|
|
40
|
+
let stat;
|
|
41
|
+
try {
|
|
42
|
+
stat = lstatSync(path);
|
|
43
|
+
}
|
|
44
|
+
catch (error) {
|
|
45
|
+
if (error.code === 'ENOENT')
|
|
46
|
+
return;
|
|
47
|
+
throw error;
|
|
48
|
+
}
|
|
49
|
+
if (!stat.isFile() || stat.isSymbolicLink() || stat.size > 256 * 1024)
|
|
50
|
+
throw Error('FLEET_MEMBER_UNSAFE_RECORD');
|
|
51
|
+
const record = JSON.parse(readFileSync(path, 'utf8'));
|
|
52
|
+
validateMember(record, key);
|
|
53
|
+
return record;
|
|
54
|
+
}
|
|
55
|
+
function validateMember(record, key) {
|
|
56
|
+
const expectedDir = record.kind === 'watchdogs' ? watchdogsRoot()
|
|
57
|
+
: agentDir(record.name, record.kind !== 'permanent');
|
|
58
|
+
if (record.version !== 1 || !['permanent', 'task', 'temporary', 'watchdogs'].includes(record.kind)
|
|
59
|
+
|| memberKey(record.name, record.kind) !== key || record.key !== key
|
|
60
|
+
|| !['running', 'stopped'].includes(record.desired)
|
|
61
|
+
|| !/^[a-f0-9-]{36}$/.test(record.incarnation)
|
|
62
|
+
|| record.dir !== resolve(record.dir) || record.dir !== expectedDir
|
|
63
|
+
|| (record.environment !== undefined && (typeof record.environment !== 'object' || record.environment === null
|
|
64
|
+
|| Object.entries(record.environment).some(([key, value]) => !MEMBER_ENV_KEYS.includes(key) || typeof value !== 'string' || value.includes('\0'))))
|
|
65
|
+
|| (record.pid !== undefined && (!Number.isSafeInteger(record.pid) || record.pid <= 0))
|
|
66
|
+
|| (record.retryAt !== undefined && (!Number.isFinite(record.retryAt) || record.retryAt < 0))
|
|
67
|
+
|| (record.kind === 'task' && (!record.taskOwner?.taskId || !record.taskOwner.creationActionId || !record.launchId)))
|
|
68
|
+
throw Error('FLEET_MEMBER_INVALID_RECORD');
|
|
69
|
+
}
|
|
70
|
+
export function writeMember(record) {
|
|
71
|
+
validateMember(record, record.key);
|
|
72
|
+
assertSafeAncestors(catalogRoot());
|
|
73
|
+
mkdirSync(catalogRoot(), { recursive: true, mode: 0o700 });
|
|
74
|
+
chmodSync(catalogRoot(), 0o700);
|
|
75
|
+
replaceFileAtomically(memberPath(record.key), JSON.stringify(record) + '\n');
|
|
76
|
+
}
|
|
77
|
+
export function listMemberKeys() {
|
|
78
|
+
try {
|
|
79
|
+
return readdirSync(catalogRoot()).filter(name => name.endsWith('.json')).map(name => name.slice(0, -5));
|
|
80
|
+
}
|
|
81
|
+
catch (error) {
|
|
82
|
+
if (error.code === 'ENOENT')
|
|
83
|
+
return [];
|
|
84
|
+
throw error;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
export function listMembers() {
|
|
88
|
+
let entries;
|
|
89
|
+
try {
|
|
90
|
+
entries = readdirSync(catalogRoot(), { withFileTypes: true });
|
|
91
|
+
}
|
|
92
|
+
catch (error) {
|
|
93
|
+
if (error.code === 'ENOENT')
|
|
94
|
+
return [];
|
|
95
|
+
throw error;
|
|
96
|
+
}
|
|
97
|
+
return entries.filter(entry => entry.name.endsWith('.json')).map(entry => {
|
|
98
|
+
const value = readMember(entry.name.slice(0, -5));
|
|
99
|
+
if (!value)
|
|
100
|
+
throw Error('FLEET_MEMBER_DISAPPEARED');
|
|
101
|
+
return value;
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
/** Persist intent before starting anything. The parent never discovers members from tmp directories. */
|
|
105
|
+
export async function registerMember(input, options = {}) {
|
|
106
|
+
const key = memberKey(input.name, input.kind);
|
|
107
|
+
return withMemberLock(key, () => {
|
|
108
|
+
const previous = readMember(key);
|
|
109
|
+
if (previous && (previous.dir !== input.dir || previous.launchId !== input.launchId
|
|
110
|
+
|| JSON.stringify(previous.taskOwner) !== JSON.stringify(input.taskOwner)))
|
|
111
|
+
throw Error('FLEET_MEMBER_OWNER_COLLISION');
|
|
112
|
+
if (previous && options.preserveExisting)
|
|
113
|
+
return { created: false, member: previous };
|
|
114
|
+
if (previous?.retiring)
|
|
115
|
+
throw Error('FLEET_MEMBER_RETIRING');
|
|
116
|
+
const desired = options.initialDesired ?? 'running';
|
|
117
|
+
const member = previous
|
|
118
|
+
? { ...previous, desired, configPath: input.configPath ?? previous.configPath,
|
|
119
|
+
environment: previous.kind === 'task' ? previous.environment : input.environment ?? previous.environment, retryAt: 0 }
|
|
120
|
+
: { ...input, version: 1, key, desired, incarnation: randomUUID() };
|
|
121
|
+
writeMember(member);
|
|
122
|
+
return { created: !previous, member };
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
/** A unique generation in argv fences PID reuse, including the spawn-to-pid-write crash seam. */
|
|
126
|
+
export async function memberProcess(record, exec = realExec) {
|
|
127
|
+
if (!record.generation)
|
|
128
|
+
return { state: 'stopped', detail: 'no child generation' };
|
|
129
|
+
if (!/^[a-f0-9-]{36}$/.test(record.generation))
|
|
130
|
+
return { state: 'unknown', detail: 'invalid child generation' };
|
|
131
|
+
const matches = (command) => command.split(/\s+/).some((part, index, words) => part === '_run-managed' && words[index + 1] === record.key && words[index + 2] === record.generation);
|
|
132
|
+
if (record.pid) {
|
|
133
|
+
try {
|
|
134
|
+
process.kill(record.pid, 0);
|
|
135
|
+
}
|
|
136
|
+
catch (error) {
|
|
137
|
+
return error.code === 'ESRCH'
|
|
138
|
+
? { state: 'stopped', detail: 'child pid absent' }
|
|
139
|
+
: { state: 'unknown', detail: 'child pid probe failed' };
|
|
140
|
+
}
|
|
141
|
+
try {
|
|
142
|
+
const command = readFileSync(`/proc/${record.pid}/cmdline`, 'utf8').split('\0').join(' ');
|
|
143
|
+
return matches(command) ? { state: 'running', detail: 'exact child generation', pid: record.pid }
|
|
144
|
+
: { state: 'stopped', detail: 'pid belongs to another generation' };
|
|
145
|
+
}
|
|
146
|
+
catch { /* non-/proc hosts use ps */ }
|
|
147
|
+
}
|
|
148
|
+
const result = await exec('ps', record.pid ? ['-p', String(record.pid), '-o', 'pid=', '-o', 'command=']
|
|
149
|
+
: ['-ax', '-o', 'pid=', '-o', 'command=']);
|
|
150
|
+
if (result.code !== 0)
|
|
151
|
+
return result.code === 1 && record.pid
|
|
152
|
+
? { state: 'stopped', detail: 'child absent' } : { state: 'unknown', detail: 'process probe unavailable' };
|
|
153
|
+
const pids = result.stdout.split('\n').flatMap(line => {
|
|
154
|
+
const row = /^\s*(\d+)\s+(.*)$/.exec(line);
|
|
155
|
+
return row && matches(row[2]) ? [Number(row[1])] : [];
|
|
156
|
+
});
|
|
157
|
+
if (pids.length > 1)
|
|
158
|
+
return { state: 'unknown', detail: 'ambiguous child generation' };
|
|
159
|
+
return pids.length ? { state: 'running', detail: 'exact child generation', pid: pids[0] }
|
|
160
|
+
: { state: 'stopped', detail: 'child generation absent' };
|
|
161
|
+
}
|
|
162
|
+
export async function catalogLiveness(key, exec = realExec) {
|
|
163
|
+
try {
|
|
164
|
+
const record = readMember(key);
|
|
165
|
+
if (!record)
|
|
166
|
+
return { state: 'stopped', detail: 'member unregistered' };
|
|
167
|
+
const live = await memberProcess(record, exec);
|
|
168
|
+
if (live.state === 'stopped' && record.desired === 'running') {
|
|
169
|
+
const { readRestartLedger } = await import('../runner.js');
|
|
170
|
+
const ledger = readRestartLedger(record.dir);
|
|
171
|
+
if (ledger.circuit === 'open')
|
|
172
|
+
return { ...live, detail: `held after ${ledger.consecutiveImmediateFailures} failures: ${ledger.lastReason}; use up/restart or task recovery to reset` };
|
|
173
|
+
}
|
|
174
|
+
return live;
|
|
175
|
+
}
|
|
176
|
+
catch {
|
|
177
|
+
return { state: 'unknown', detail: 'member record unavailable' };
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
export async function waitMember(key, desired, deps = {}) {
|
|
181
|
+
const deadline = Date.now() + (deps.timeoutMs ?? 30_000);
|
|
182
|
+
do {
|
|
183
|
+
const live = await catalogLiveness(key, deps.exec);
|
|
184
|
+
if (live.state === desired)
|
|
185
|
+
return;
|
|
186
|
+
await (deps.sleep ?? (ms => new Promise(resolve => setTimeout(resolve, ms))))(100);
|
|
187
|
+
} while (Date.now() < deadline);
|
|
188
|
+
throw Error(desired === 'stopped' ? 'FLEET_MEMBER_STOP_UNPROVEN' : 'FLEET_MEMBER_START_UNPROVEN');
|
|
189
|
+
}
|
|
190
|
+
export async function stopMember(key, deps = {}) {
|
|
191
|
+
await withMemberLock(key, () => {
|
|
192
|
+
const record = readMember(key);
|
|
193
|
+
if (record)
|
|
194
|
+
writeMember({ ...record, desired: 'stopped' });
|
|
195
|
+
});
|
|
196
|
+
// A stopped parent cannot consume intent, but its IPC children stop on disconnect.
|
|
197
|
+
// Never signal an arbitrary recorded pid or erase a record on timeout.
|
|
198
|
+
await waitMember(key, 'stopped', deps);
|
|
199
|
+
}
|
|
200
|
+
export async function unregisterMember(key, deps = {}) {
|
|
201
|
+
const original = readMember(key);
|
|
202
|
+
if (!original)
|
|
203
|
+
return false;
|
|
204
|
+
await withMemberLock(key, () => {
|
|
205
|
+
const current = readMember(key);
|
|
206
|
+
if (!current || current.incarnation !== original.incarnation)
|
|
207
|
+
throw Error('FLEET_MEMBER_RETIREMENT_CONFLICT');
|
|
208
|
+
writeMember({ ...current, desired: 'stopped', retiring: true });
|
|
209
|
+
});
|
|
210
|
+
await waitMember(key, 'stopped', deps);
|
|
211
|
+
return withMemberLock(key, async () => {
|
|
212
|
+
const current = readMember(key);
|
|
213
|
+
if (!current)
|
|
214
|
+
return false;
|
|
215
|
+
if (current.incarnation !== original.incarnation || current.desired !== 'stopped')
|
|
216
|
+
throw Error('FLEET_MEMBER_RETIREMENT_CONFLICT');
|
|
217
|
+
if ((await memberProcess(current, deps.exec)).state !== 'stopped')
|
|
218
|
+
throw Error('FLEET_MEMBER_STOP_UNPROVEN');
|
|
219
|
+
rmSync(memberPath(key));
|
|
220
|
+
return true;
|
|
221
|
+
});
|
|
222
|
+
}
|
|
223
|
+
export async function unregisterTaskMember(role, expected, deps = {}) {
|
|
224
|
+
const key = memberKey(role, 'task');
|
|
225
|
+
await withMemberLock(key, () => {
|
|
226
|
+
const current = readMember(key);
|
|
227
|
+
if (!current)
|
|
228
|
+
return;
|
|
229
|
+
if (current.kind !== 'task' || current.taskOwner?.taskId !== expected.taskId
|
|
230
|
+
|| (!expected.creationActionId && !expected.launchId)
|
|
231
|
+
|| (expected.creationActionId !== undefined && current.taskOwner.creationActionId !== expected.creationActionId)
|
|
232
|
+
|| (expected.launchId !== undefined && current.launchId !== expected.launchId))
|
|
233
|
+
throw Error('TASK_SERVICE_OWNER_MISMATCH');
|
|
234
|
+
writeMember({ ...current, desired: 'stopped', retiring: true });
|
|
235
|
+
});
|
|
236
|
+
await unregisterMember(key, deps);
|
|
237
|
+
}
|
|
238
|
+
export function hasTaskMemberRegistration(role) {
|
|
239
|
+
const path = memberPath(memberKey(role, 'task'));
|
|
240
|
+
try {
|
|
241
|
+
lstatSync(path);
|
|
242
|
+
return true;
|
|
243
|
+
}
|
|
244
|
+
catch (error) {
|
|
245
|
+
if (error.code === 'ENOENT')
|
|
246
|
+
return false;
|
|
247
|
+
throw error;
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
export function assertTaskRegistrationsAbsent(taskId) {
|
|
251
|
+
if (listMembers().some(record => record.taskOwner?.taskId === taskId))
|
|
252
|
+
throw Error('TASK_SERVICES_REMAIN');
|
|
253
|
+
}
|
|
254
|
+
export const permanentRegistration = (name) => ({ name, kind: 'permanent', dir: agentDir(name) });
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { type Exec } from '../exec.js';
|
|
2
|
+
import type { SupervisorBackend } from './types.js';
|
|
3
|
+
export declare function fleetHostBackend(exec?: Exec, platform?: NodeJS.Platform): SupervisorBackend;
|
|
4
|
+
/** Registration only starts an explicitly installed parent; it cannot repoint
|
|
5
|
+
* the fleet at the invoking task/development binary or rewrite user settings. */
|
|
6
|
+
export declare function ensureFleetParent(exec?: Exec, platform?: NodeJS.Platform): Promise<void>;
|
|
7
|
+
/** OS operations refer only to the Fleet parent; member operations refer only to its catalog. */
|
|
8
|
+
export declare function makeFleetBackend(exec?: Exec, platform?: NodeJS.Platform): SupervisorBackend;
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { join } from 'node:path';
|
|
2
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
3
|
+
import { agentDir } from '../paths.js';
|
|
4
|
+
import { realExec } from '../exec.js';
|
|
5
|
+
import { home } from '../paths.js';
|
|
6
|
+
import { makeSystemdBackend } from './systemd.js';
|
|
7
|
+
import { makeLaunchdBackend } from './launchd.js';
|
|
8
|
+
import { catalogLiveness, memberKey, readMember, registerMember, stopMember, unregisterMember, waitMember } from './catalog.js';
|
|
9
|
+
export function fleetHostBackend(exec = realExec, platform = process.platform) {
|
|
10
|
+
if (platform === 'linux')
|
|
11
|
+
return makeSystemdBackend(exec);
|
|
12
|
+
if (platform === 'darwin')
|
|
13
|
+
return makeLaunchdBackend(exec);
|
|
14
|
+
throw Error('Fleet service requires Linux or macOS');
|
|
15
|
+
}
|
|
16
|
+
/** Registration only starts an explicitly installed parent; it cannot repoint
|
|
17
|
+
* the fleet at the invoking task/development binary or rewrite user settings. */
|
|
18
|
+
export async function ensureFleetParent(exec = realExec, platform = process.platform) {
|
|
19
|
+
const host = fleetHostBackend(exec, platform);
|
|
20
|
+
const path = platform === 'linux' ? join(home(), '.config/systemd/user/ours-fleet.service')
|
|
21
|
+
: join(home(), 'Library/LaunchAgents/network.ours.fleet.plist');
|
|
22
|
+
if (!existsSync(path))
|
|
23
|
+
throw Error('FLEET_SERVICE_NOT_INSTALLED: run ours-fleet init');
|
|
24
|
+
const live = await host.liveness('fleet');
|
|
25
|
+
if (live.state === 'unknown')
|
|
26
|
+
throw Error('FLEET_SERVICE_LIVENESS_UNKNOWN');
|
|
27
|
+
if (live.state === 'stopped')
|
|
28
|
+
await host.install('fleet', '');
|
|
29
|
+
}
|
|
30
|
+
/** OS operations refer only to the Fleet parent; member operations refer only to its catalog. */
|
|
31
|
+
export function makeFleetBackend(exec = realExec, platform = process.platform) {
|
|
32
|
+
const host = fleetHostBackend(exec, platform);
|
|
33
|
+
return {
|
|
34
|
+
id: host.id,
|
|
35
|
+
async init(binPath) {
|
|
36
|
+
const messages = await host.init(binPath);
|
|
37
|
+
const { adoptConfiguredFleet } = await import('./adoption.js');
|
|
38
|
+
await adoptConfiguredFleet(binPath, exec, platform);
|
|
39
|
+
return messages;
|
|
40
|
+
},
|
|
41
|
+
async install(name, binPath) {
|
|
42
|
+
const { adoptConfiguredFleet } = await import('./adoption.js');
|
|
43
|
+
let configPath;
|
|
44
|
+
try {
|
|
45
|
+
configPath = readFileSync(join(agentDir(name), '.config-path'), 'utf8').trim() || undefined;
|
|
46
|
+
}
|
|
47
|
+
catch { /* a new state directory may not carry a config marker yet */ }
|
|
48
|
+
await adoptConfiguredFleet(binPath, exec, platform, configPath, name);
|
|
49
|
+
const result = await registerMember({ name, kind: 'permanent', dir: agentDir(name) });
|
|
50
|
+
try {
|
|
51
|
+
await waitMember(result.member.key, 'running', { exec });
|
|
52
|
+
}
|
|
53
|
+
catch (error) {
|
|
54
|
+
// Preserve existing intent. New registration rollback is exact and waits
|
|
55
|
+
// for process quiescence before its creation evidence can be discarded.
|
|
56
|
+
if (result.created)
|
|
57
|
+
await unregisterMember(result.member.key, { exec });
|
|
58
|
+
throw error;
|
|
59
|
+
}
|
|
60
|
+
return { created: result.created, detail: `Fleet member ${name} registered` };
|
|
61
|
+
},
|
|
62
|
+
async start(name) {
|
|
63
|
+
const key = memberKey(name, 'permanent'), member = readMember(key);
|
|
64
|
+
if (!member)
|
|
65
|
+
throw Error('Fleet member not registered; use up');
|
|
66
|
+
await registerMember(member);
|
|
67
|
+
await ensureFleetParent(exec, platform);
|
|
68
|
+
await waitMember(key, 'running', { exec });
|
|
69
|
+
},
|
|
70
|
+
async stop(name) { await stopMember(memberKey(name, 'permanent'), { exec }); },
|
|
71
|
+
async restart(name) {
|
|
72
|
+
await this.stop(name);
|
|
73
|
+
await this.start(name);
|
|
74
|
+
},
|
|
75
|
+
async status(name) { return (await this.liveness(name)).detail; },
|
|
76
|
+
liveness(name) {
|
|
77
|
+
const key = memberKey(name, 'permanent');
|
|
78
|
+
if (!readMember(key) && existsSync(agentDir(name)))
|
|
79
|
+
return Promise.resolve({ state: 'unknown', detail: 'unmigrated member; preserving context until legacy inspection' });
|
|
80
|
+
return catalogLiveness(key, exec);
|
|
81
|
+
},
|
|
82
|
+
async inspect(name) { return { backend: host.id, ...await this.liveness(name), nativeState: 'fleet-member' }; },
|
|
83
|
+
async uninstall(name) {
|
|
84
|
+
const { retirePermanentRegistration } = await import('./adoption.js');
|
|
85
|
+
return { removed: await retirePermanentRegistration(name, exec), detail: `Fleet member ${name} unregistered` };
|
|
86
|
+
},
|
|
87
|
+
logsArgs(name, follow) { return { cmd: 'tail', args: [follow ? '-f' : '-n', ...(follow ? [] : ['200']), join(agentDir(name), 'supervisor.log')] }; },
|
|
88
|
+
};
|
|
89
|
+
}
|
package/dist/supervisor/index.js
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { realExec } from '../exec.js';
|
|
2
|
-
import {
|
|
3
|
-
import { makeLaunchdBackend } from './launchd.js';
|
|
2
|
+
import { makeFleetBackend } from './fleet.js';
|
|
4
3
|
import { makeNoneBackend } from './none.js';
|
|
5
4
|
export { makeSystemdBackend, unitFor } from './systemd.js';
|
|
6
5
|
export { makeLaunchdBackend, labelFor } from './launchd.js';
|
|
@@ -8,9 +7,7 @@ export { makeNoneBackend } from './none.js';
|
|
|
8
7
|
export function pickBackend(exec = realExec, platform = process.platform) {
|
|
9
8
|
if (process.env.OURS_FLEET_SUPERVISOR === 'none')
|
|
10
9
|
return makeNoneBackend(exec);
|
|
11
|
-
if (platform === 'darwin')
|
|
12
|
-
return
|
|
13
|
-
if (platform === 'linux')
|
|
14
|
-
return makeSystemdBackend(exec);
|
|
10
|
+
if (platform === 'darwin' || platform === 'linux')
|
|
11
|
+
return makeFleetBackend(exec, platform);
|
|
15
12
|
throw new Error(`unsupported platform '${platform}' (linux and darwin only)`);
|
|
16
13
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type Exec } from '../exec.js';
|
|
2
2
|
import type { SupervisorBackend } from './types.js';
|
|
3
|
-
export declare const labelFor: (
|
|
3
|
+
export declare const labelFor: (_name: string) => string;
|
|
4
4
|
/**
|
|
5
5
|
* What `launchctl print` said about a job, parsed ONCE so that the two questions
|
|
6
6
|
* asked of it cannot drift apart. They are not the same question:
|
|
@@ -1,9 +1,10 @@
|
|
|
1
|
+
import { assertNativeFleetScope } from './scope.js';
|
|
1
2
|
import { clientConfigPath } from '../client-profile.js';
|
|
2
3
|
import { existsSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
|
|
3
4
|
import { join } from 'node:path';
|
|
4
5
|
import { home, logsRoot } from '../paths.js';
|
|
5
6
|
import { realExec } from '../exec.js';
|
|
6
|
-
export const labelFor = (
|
|
7
|
+
export const labelFor = (_name) => 'network.ours.fleet';
|
|
7
8
|
/**
|
|
8
9
|
* The spelling of the last-exit line is not stable across macOS releases
|
|
9
10
|
* (`last exit code`, `last exit status`, `last exit reason`), so match the
|
|
@@ -22,16 +23,17 @@ const xml = (value) => value
|
|
|
22
23
|
.replaceAll('"', '"')
|
|
23
24
|
.replaceAll("'", ''');
|
|
24
25
|
function plist(name, binPath) {
|
|
25
|
-
const log = join(logsRoot(),
|
|
26
|
+
const log = join(logsRoot(), 'fleet.log');
|
|
26
27
|
const config = clientConfigPath(process.env);
|
|
28
|
+
const env = { OURS_CONFIG: config, ...Object.fromEntries(['PATH', 'OURS_FLEET_HOME', 'OURS_FLEET_SOCKET_ROOT', 'CODEX_HOME', 'CODEX_PATH'].flatMap(key => process.env[key] !== undefined ? [[key, process.env[key]]] : [])) };
|
|
27
29
|
return `<?xml version="1.0" encoding="UTF-8"?>
|
|
28
30
|
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
|
29
31
|
<plist version="1.0">
|
|
30
32
|
<dict>
|
|
31
33
|
<key>Label</key><string>${labelFor(name)}</string>
|
|
32
34
|
<key>ProgramArguments</key>
|
|
33
|
-
<array><string>${
|
|
34
|
-
${
|
|
35
|
+
<array><string>${xml(process.execPath)}</string><string>${xml(binPath)}</string><string>_run-fleet</string></array>
|
|
36
|
+
${` <key>EnvironmentVariables</key><dict>${Object.entries(env).map(([key, value]) => `<key>${key}</key><string>${xml(value)}</string>`).join('')}</dict>\n`} <!-- The runner owns the child-session restart loop. launchd must only
|
|
35
37
|
recover the runner PROCESS crashing: a bare KeepAlive would resume the
|
|
36
38
|
uncounted relaunch loop and restart a deliberately held-down agent. -->
|
|
37
39
|
<key>KeepAlive</key><dict><key>SuccessfulExit</key><false/></dict>
|
|
@@ -80,6 +82,7 @@ export function makeLaunchdBackend(exec = realExec, uid = process.getuid?.() ??
|
|
|
80
82
|
const domain = `gui/${uid}`;
|
|
81
83
|
/** Read `launchctl print` once; both callers classify it for their own question. */
|
|
82
84
|
const printJob = async (name) => {
|
|
85
|
+
assertNativeFleetScope(exec);
|
|
83
86
|
const r = await exec('launchctl', ['print', `${domain}/${labelFor(name)}`]);
|
|
84
87
|
const out = `${r.stdout}\n${r.stderr}`;
|
|
85
88
|
if (r.code !== 0)
|
|
@@ -97,21 +100,26 @@ export function makeLaunchdBackend(exec = realExec, uid = process.getuid?.() ??
|
|
|
97
100
|
};
|
|
98
101
|
return {
|
|
99
102
|
id: 'launchd',
|
|
100
|
-
async init() {
|
|
103
|
+
async init(binPath) {
|
|
104
|
+
assertNativeFleetScope(exec);
|
|
101
105
|
mkdirSync(agentsDir(), { recursive: true });
|
|
102
106
|
mkdirSync(logsRoot(), { recursive: true });
|
|
107
|
+
writeFileSync(plistPath('fleet'), plist('fleet', binPath));
|
|
103
108
|
return [
|
|
104
109
|
`LaunchAgents dir ready: ${agentsDir()}`,
|
|
105
110
|
'note: launchd agents start at login (macOS has no linger equivalent)',
|
|
106
111
|
];
|
|
107
112
|
},
|
|
108
113
|
async install(name, binPath) {
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
const existed =
|
|
113
|
-
|
|
114
|
-
|
|
114
|
+
assertNativeFleetScope(exec);
|
|
115
|
+
if (!existsSync(plistPath(name)))
|
|
116
|
+
throw Error('FLEET_SERVICE_NOT_INSTALLED: run ours-fleet init');
|
|
117
|
+
const existed = true;
|
|
118
|
+
const loaded = await printJob(name);
|
|
119
|
+
if (loaded.loaded)
|
|
120
|
+
return { created: !existed, detail: 'Fleet service already loaded' };
|
|
121
|
+
if (!loaded.notFound)
|
|
122
|
+
throw new Error('Fleet launchd liveness unknown');
|
|
115
123
|
// Undo only what WE wrote. A plist that was already there belongs to
|
|
116
124
|
// whoever put it there, and rollback may never remove it.
|
|
117
125
|
const undo = async () => {
|
|
@@ -154,16 +162,19 @@ export function makeLaunchdBackend(exec = realExec, uid = process.getuid?.() ??
|
|
|
154
162
|
: { created: true, detail: `installed ${labelFor(name)} (${start.detail})` };
|
|
155
163
|
},
|
|
156
164
|
async start(name) {
|
|
165
|
+
assertNativeFleetScope(exec);
|
|
157
166
|
const r = await exec('launchctl', ['bootstrap', domain, plistPath(name)]);
|
|
158
167
|
if (r.code !== 0)
|
|
159
168
|
await exec('launchctl', ['kickstart', `${domain}/${labelFor(name)}`]);
|
|
160
169
|
},
|
|
161
170
|
async stop(name) {
|
|
171
|
+
assertNativeFleetScope(exec);
|
|
162
172
|
const r = await exec('launchctl', ['bootout', `${domain}/${labelFor(name)}`]);
|
|
163
173
|
if (r.code !== 0)
|
|
164
174
|
throw new Error(`launchctl bootout ${labelFor(name)} failed: ${r.stderr.trim()}`);
|
|
165
175
|
},
|
|
166
176
|
async restart(name) {
|
|
177
|
+
assertNativeFleetScope(exec);
|
|
167
178
|
const r = await exec('launchctl', ['kickstart', '-k', `${domain}/${labelFor(name)}`]);
|
|
168
179
|
if (r.code !== 0)
|
|
169
180
|
throw new Error(`launchctl kickstart ${labelFor(name)} failed: ${r.stderr.trim()}`);
|
|
@@ -203,6 +214,7 @@ export function makeLaunchdBackend(exec = realExec, uid = process.getuid?.() ??
|
|
|
203
214
|
};
|
|
204
215
|
},
|
|
205
216
|
async uninstall(name) {
|
|
217
|
+
assertNativeFleetScope(exec);
|
|
206
218
|
const existed = existsSync(plistPath(name));
|
|
207
219
|
await exec('launchctl', ['bootout', `${domain}/${labelFor(name)}`]); // idempotent
|
|
208
220
|
rmSync(plistPath(name), { force: true });
|
|
@@ -211,7 +223,7 @@ export function makeLaunchdBackend(exec = realExec, uid = process.getuid?.() ??
|
|
|
211
223
|
: { removed: false, detail: `${labelFor(name)} was not installed` };
|
|
212
224
|
},
|
|
213
225
|
logsArgs(name, follow) {
|
|
214
|
-
const log = join(logsRoot(),
|
|
226
|
+
const log = join(logsRoot(), 'fleet.log');
|
|
215
227
|
return { cmd: 'tail', args: follow ? ['-f', log] : ['-n', '200', log] };
|
|
216
228
|
},
|
|
217
229
|
};
|