@omercnet/paseo-omp 5.6.3-next.481.1 → 5.6.3-next.482.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/server/hub.ts +271 -28
- package/server/operational-failure-diagnostics.ts +2 -0
- package/server/package-version.ts +1 -1
- package/server/paths.ts +23 -0
- package/server/provider/omp-rpc.ts +115 -3
- package/server/provider/profile-providers.ts +11 -0
- package/server/provider/session.ts +41 -2
package/package.json
CHANGED
package/server/hub.ts
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import type { Dirent } from "node:fs";
|
|
2
|
+
import { readdir, readFile, realpath } from "node:fs/promises";
|
|
2
3
|
import { homedir } from "node:os";
|
|
3
|
-
import { join } from "node:path";
|
|
4
|
+
import { join, resolve } from "node:path";
|
|
4
5
|
import type { RpcInput } from "@getpaseo/plugin";
|
|
5
6
|
import { z } from "zod";
|
|
6
7
|
import type { HubProcess, listHubProcesses, tailHubLog } from "../shared/hub";
|
|
8
|
+
import { waitMs } from "./provider/omp-rpc-process";
|
|
7
9
|
|
|
8
10
|
const MAX_LOG_BYTES = 64 * 1024;
|
|
9
11
|
|
|
@@ -46,32 +48,70 @@ function ompRunDir(): string {
|
|
|
46
48
|
return process.env.PASEO_OMP_RUN_DIR ?? join(homedir(), ".omp", "run", "daemons");
|
|
47
49
|
}
|
|
48
50
|
|
|
49
|
-
|
|
51
|
+
type Read<T> = { value: T } | { missing: true } | { unreadable: true };
|
|
52
|
+
|
|
53
|
+
export function readFailure(error: unknown): { missing: true } | { unreadable: true } {
|
|
54
|
+
const code = (error as NodeJS.ErrnoException | undefined)?.code;
|
|
55
|
+
// Only ENOENT proves a record is gone; anything else (ENOTDIR included) leaves it unknown.
|
|
56
|
+
return code === "ENOENT" ? { missing: true } : { unreadable: true };
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
async function readJson(path: string): Promise<Read<unknown>> {
|
|
60
|
+
let text: string;
|
|
50
61
|
try {
|
|
51
|
-
|
|
62
|
+
text = await readFile(path, "utf8");
|
|
63
|
+
} catch (error) {
|
|
64
|
+
return readFailure(error);
|
|
65
|
+
}
|
|
66
|
+
try {
|
|
67
|
+
return { value: JSON.parse(text) };
|
|
52
68
|
} catch {
|
|
53
|
-
return
|
|
69
|
+
return { unreadable: true };
|
|
54
70
|
}
|
|
55
71
|
}
|
|
56
72
|
|
|
57
|
-
async function
|
|
73
|
+
async function readDirEntries(path: string): Promise<Read<Dirent[]>> {
|
|
58
74
|
try {
|
|
59
|
-
return await readdir(path);
|
|
75
|
+
return { value: await readdir(path, { withFileTypes: true }) };
|
|
76
|
+
} catch (error) {
|
|
77
|
+
return readFailure(error);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// omp keys a project's broker by its realpath (`canonicalProjectDir`), so a symlinked or
|
|
82
|
+
// trailing-slash cwd must be resolved the same way to find its `scope.json`.
|
|
83
|
+
async function canonicalProjectDir(cwd: string): Promise<string> {
|
|
84
|
+
const resolved = resolve(cwd);
|
|
85
|
+
try {
|
|
86
|
+
return await realpath(resolved);
|
|
60
87
|
} catch {
|
|
61
|
-
return
|
|
88
|
+
return resolved;
|
|
62
89
|
}
|
|
63
90
|
}
|
|
64
91
|
|
|
65
|
-
async function findProjectDaemonRoots(
|
|
66
|
-
|
|
67
|
-
|
|
92
|
+
async function findProjectDaemonRoots(
|
|
93
|
+
root: string,
|
|
94
|
+
cwd: string,
|
|
95
|
+
): Promise<{ roots: string[]; complete: boolean }> {
|
|
96
|
+
const [hashes, projectDir] = await Promise.all([readDirEntries(root), canonicalProjectDir(cwd)]);
|
|
97
|
+
if ("unreadable" in hashes) return { roots: [], complete: false };
|
|
98
|
+
if (!("value" in hashes)) return { roots: [], complete: true };
|
|
99
|
+
const roots: string[] = [];
|
|
100
|
+
let complete = true;
|
|
68
101
|
await Promise.all(
|
|
69
|
-
hashes.map(async (
|
|
70
|
-
|
|
71
|
-
if (
|
|
102
|
+
hashes.value.map(async (entry) => {
|
|
103
|
+
// Only directories can be broker scopes; a stray file (`.DS_Store`) is not one.
|
|
104
|
+
if (!entry.isDirectory()) return;
|
|
105
|
+
const hash = entry.name;
|
|
106
|
+
const read = await readJson(join(root, hash, "scope.json"));
|
|
107
|
+
if ("unreadable" in read) complete = false;
|
|
108
|
+
if (!("value" in read)) return;
|
|
109
|
+
const scope = ScopeFileSchema.safeParse(read.value);
|
|
110
|
+
if (scope.success && (scope.data.projectDir === projectDir || scope.data.projectDir === cwd))
|
|
111
|
+
roots.push(join(root, hash));
|
|
72
112
|
}),
|
|
73
113
|
);
|
|
74
|
-
return
|
|
114
|
+
return { roots, complete };
|
|
75
115
|
}
|
|
76
116
|
|
|
77
117
|
function toHubProcess(name: string, value: unknown, specFile: unknown): HubProcess | undefined {
|
|
@@ -98,27 +138,61 @@ function toHubProcess(name: string, value: unknown, specFile: unknown): HubProce
|
|
|
98
138
|
};
|
|
99
139
|
}
|
|
100
140
|
|
|
101
|
-
export
|
|
102
|
-
|
|
103
|
-
|
|
141
|
+
export interface HubProcessScan {
|
|
142
|
+
processes: HubProcess[];
|
|
143
|
+
/** Records that exist but could not be read or parsed: their state is unknown. */
|
|
144
|
+
unreadable: string[];
|
|
145
|
+
/**
|
|
146
|
+
* Entries under `daemons/` that are not directories. omp skips them too (`readPersistedDaemons`),
|
|
147
|
+
* so they are not services, but an owned service whose record turned into one is unknown.
|
|
148
|
+
*/
|
|
149
|
+
strays: string[];
|
|
150
|
+
/** False when a directory that could hold this project's records could not be read. */
|
|
151
|
+
complete: boolean;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
export async function scanHubProcessesFrom(root: string, cwd: string): Promise<HubProcessScan> {
|
|
155
|
+
const found = await findProjectDaemonRoots(root, cwd);
|
|
156
|
+
const scan: HubProcessScan = {
|
|
157
|
+
processes: [],
|
|
158
|
+
unreadable: [],
|
|
159
|
+
strays: [],
|
|
160
|
+
complete: found.complete,
|
|
161
|
+
};
|
|
104
162
|
await Promise.all(
|
|
105
|
-
roots.map(async (projectRoot) => {
|
|
106
|
-
const
|
|
163
|
+
found.roots.map(async (projectRoot) => {
|
|
164
|
+
const entries = await readDirEntries(join(projectRoot, "daemons"));
|
|
165
|
+
if ("unreadable" in entries) scan.complete = false;
|
|
166
|
+
if (!("value" in entries)) return;
|
|
107
167
|
await Promise.all(
|
|
108
|
-
|
|
168
|
+
entries.value.map(async (entry) => {
|
|
169
|
+
const name = entry.name;
|
|
170
|
+
if (!entry.isDirectory()) {
|
|
171
|
+
scan.strays.push(name);
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
109
174
|
const dir = join(projectRoot, "daemons", name);
|
|
110
175
|
const [meta, spec] = await Promise.all([
|
|
111
|
-
|
|
112
|
-
|
|
176
|
+
readJson(join(dir, "meta.json")),
|
|
177
|
+
readJson(join(dir, "spec.json")),
|
|
113
178
|
]);
|
|
114
|
-
|
|
115
|
-
|
|
179
|
+
if ("missing" in meta) return;
|
|
180
|
+
const process =
|
|
181
|
+
"value" in meta
|
|
182
|
+
? toHubProcess(name, meta.value, "value" in spec ? spec.value : undefined)
|
|
183
|
+
: undefined;
|
|
184
|
+
if (process) scan.processes.push(process);
|
|
185
|
+
else scan.unreadable.push(name);
|
|
116
186
|
}),
|
|
117
187
|
);
|
|
118
188
|
}),
|
|
119
189
|
);
|
|
120
|
-
processes.sort((a, b) => (b.startedAt ?? 0) - (a.startedAt ?? 0));
|
|
121
|
-
return
|
|
190
|
+
scan.processes.sort((a, b) => (b.startedAt ?? 0) - (a.startedAt ?? 0));
|
|
191
|
+
return scan;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
export async function listHubProcessesFrom(root: string, cwd: string): Promise<HubProcess[]> {
|
|
195
|
+
return (await scanHubProcessesFrom(root, cwd)).processes;
|
|
122
196
|
}
|
|
123
197
|
|
|
124
198
|
export async function resolveListHubProcesses({
|
|
@@ -127,12 +201,181 @@ export async function resolveListHubProcesses({
|
|
|
127
201
|
return { processes: await listHubProcessesFrom(ompRunDir(), cwd) };
|
|
128
202
|
}
|
|
129
203
|
|
|
204
|
+
// omp 18.18.9 made launch services persistent in RPC mode (`launch.persist` is forced on for
|
|
205
|
+
// every non-interactive host), so they no longer end with the omp process that started them.
|
|
206
|
+
// A Paseo agent owns its OMP session: once that session's process tree is gone, stop the
|
|
207
|
+
// services it still owns. Detached services were explicitly handed off and are left alone, and
|
|
208
|
+
// non-persistent ones are still stopped by omp's own broker lifecycle.
|
|
209
|
+
const LIVE_HUB_STATES: Readonly<Record<string, true>> = {
|
|
210
|
+
starting: true,
|
|
211
|
+
running: true,
|
|
212
|
+
ready: true,
|
|
213
|
+
restarting: true,
|
|
214
|
+
};
|
|
215
|
+
// The broker records `stopping` before it signals the tree and the terminal state after it
|
|
216
|
+
// settles, so a service is only confirmed gone once its metadata leaves both.
|
|
217
|
+
const UNSETTLED_HUB_STATES: Readonly<Record<string, true>> = { ...LIVE_HUB_STATES, stopping: true };
|
|
218
|
+
const HUB_CLEANUP_POLL_MS = 100;
|
|
219
|
+
// One deadline for the whole cleanup, so a hung CLI, a service that ignores SIGTERM or many
|
|
220
|
+
// services cannot hold the agent's close. omp's own stop grace is 5s plus 1s to settle. Stops
|
|
221
|
+
// leave the last stretch for verification, and the broker stops each record on its own, so a
|
|
222
|
+
// few stops run at once.
|
|
223
|
+
const HUB_CLEANUP_MS = 12_000;
|
|
224
|
+
const HUB_CLEANUP_VERIFY_RESERVE_MS = 2_000;
|
|
225
|
+
const HUB_CLEANUP_STOP_CONCURRENCY = 4;
|
|
226
|
+
|
|
227
|
+
// omp's broker rejects any other service name; anything else read from disk is not ours to stop,
|
|
228
|
+
// and the leading-alphanumeric rule keeps a name from ever being parsed as a CLI flag.
|
|
229
|
+
const HUB_SERVICE_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,47}$/u;
|
|
230
|
+
|
|
231
|
+
export interface OwnedHubCleanupDependencies {
|
|
232
|
+
scan(cwd: string): Promise<HubProcessScan>;
|
|
233
|
+
/** A timer that fires after `ms`; scans that outlive it count as incomplete. */
|
|
234
|
+
expire(ms: number): { fired: Promise<void>; cancel(): void };
|
|
235
|
+
/**
|
|
236
|
+
* Asks omp's broker to stop one service within `timeoutMs`; resolves true only when the
|
|
237
|
+
* command succeeded.
|
|
238
|
+
*/
|
|
239
|
+
stop(name: string, cwd: string, timeoutMs: number): Promise<boolean>;
|
|
240
|
+
sleep(ms: number): Promise<void>;
|
|
241
|
+
now(): number;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
export interface OwnedHubCleanupResult {
|
|
245
|
+
stopped: string[];
|
|
246
|
+
failed: string[];
|
|
247
|
+
/** False when omp's records could not all be read, so an owned service may have been missed. */
|
|
248
|
+
complete: boolean;
|
|
249
|
+
/** Set when the launcher wraps OMP's environment, so the broker could not be located. */
|
|
250
|
+
unsupportedLauncher?: true;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/**
|
|
254
|
+
* OMP session ids whose services an agent may stop, each mapped to the last `createdAt` it can
|
|
255
|
+
* claim: `Infinity` for the live id, the hand-off time for an id the agent left (a rewind or
|
|
256
|
+
* branch releases it, so another agent may resume it and start services of its own).
|
|
257
|
+
*/
|
|
258
|
+
export type HubServiceOwners = ReadonlyMap<string, number>;
|
|
259
|
+
|
|
260
|
+
function startedWhileOwned(process: HubProcess, ownedUntil: number | undefined): boolean {
|
|
261
|
+
if (ownedUntil === undefined) return false;
|
|
262
|
+
if (ownedUntil === Number.POSITIVE_INFINITY) return true;
|
|
263
|
+
return process.createdAt !== null && process.createdAt <= ownedUntil;
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
function ownedPersistentServices(
|
|
267
|
+
processes: readonly HubProcess[],
|
|
268
|
+
owners: HubServiceOwners,
|
|
269
|
+
states: Readonly<Record<string, true>>,
|
|
270
|
+
): string[] {
|
|
271
|
+
return processes
|
|
272
|
+
.filter(
|
|
273
|
+
(process) =>
|
|
274
|
+
HUB_SERVICE_NAME.test(process.name) &&
|
|
275
|
+
process.owner !== null &&
|
|
276
|
+
startedWhileOwned(process, owners.get(process.owner)) &&
|
|
277
|
+
process.persist &&
|
|
278
|
+
!process.detached &&
|
|
279
|
+
states[process.state] === true,
|
|
280
|
+
)
|
|
281
|
+
.map((process) => process.name);
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
async function boundedScan(
|
|
285
|
+
dependencies: OwnedHubCleanupDependencies,
|
|
286
|
+
cwd: string,
|
|
287
|
+
ms: number,
|
|
288
|
+
): Promise<HubProcessScan | undefined> {
|
|
289
|
+
if (ms <= 0) return undefined;
|
|
290
|
+
const timer = dependencies.expire(ms);
|
|
291
|
+
try {
|
|
292
|
+
return await Promise.race([
|
|
293
|
+
dependencies.scan(cwd).catch(() => undefined),
|
|
294
|
+
timer.fired.then(() => undefined),
|
|
295
|
+
]);
|
|
296
|
+
} finally {
|
|
297
|
+
timer.cancel();
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
export async function stopOwnedHubProcesses(
|
|
302
|
+
cwd: string,
|
|
303
|
+
owners: HubServiceOwners,
|
|
304
|
+
dependencies: OwnedHubCleanupDependencies,
|
|
305
|
+
): Promise<OwnedHubCleanupResult> {
|
|
306
|
+
if (owners.size === 0) return { stopped: [], failed: [], complete: true };
|
|
307
|
+
// The deadline covers discovery too, so a hung filesystem cannot hold the close either.
|
|
308
|
+
const deadline = dependencies.now() + HUB_CLEANUP_MS;
|
|
309
|
+
const initial = await boundedScan(dependencies, cwd, deadline - dependencies.now());
|
|
310
|
+
if (!initial) return { stopped: [], failed: [], complete: false };
|
|
311
|
+
const owned = ownedPersistentServices(initial.processes, owners, LIVE_HUB_STATES);
|
|
312
|
+
// A record that cannot be read may be one of ours, so the lookup is not complete.
|
|
313
|
+
const complete = initial.complete && initial.unreadable.length === 0;
|
|
314
|
+
if (owned.length === 0) return { stopped: [], failed: [], complete };
|
|
315
|
+
const failed = new Set<string>();
|
|
316
|
+
const queue = [...owned];
|
|
317
|
+
const stopDeadline = deadline - HUB_CLEANUP_VERIFY_RESERVE_MS;
|
|
318
|
+
const stopNext = async (): Promise<void> => {
|
|
319
|
+
for (let name = queue.shift(); name !== undefined; name = queue.shift()) {
|
|
320
|
+
const remaining = stopDeadline - dependencies.now();
|
|
321
|
+
const stopped =
|
|
322
|
+
remaining > 0 && (await dependencies.stop(name, cwd, remaining).catch(() => false));
|
|
323
|
+
if (!stopped) failed.add(name);
|
|
324
|
+
}
|
|
325
|
+
};
|
|
326
|
+
await Promise.all(
|
|
327
|
+
Array.from({ length: Math.min(HUB_CLEANUP_STOP_CONCURRENCY, owned.length) }, stopNext),
|
|
328
|
+
);
|
|
329
|
+
// Report success only for services omp's own state confirms have settled. A record that cannot
|
|
330
|
+
// be read is unknown, not stopped.
|
|
331
|
+
let unsettled: ReadonlySet<string>;
|
|
332
|
+
while (true) {
|
|
333
|
+
const current = await boundedScan(dependencies, cwd, deadline - dependencies.now());
|
|
334
|
+
unsettled = new Set(
|
|
335
|
+
current?.complete
|
|
336
|
+
? [
|
|
337
|
+
...ownedPersistentServices(current.processes, owners, UNSETTLED_HUB_STATES),
|
|
338
|
+
...current.unreadable,
|
|
339
|
+
...current.strays,
|
|
340
|
+
].filter((name) => owned.includes(name))
|
|
341
|
+
: owned,
|
|
342
|
+
);
|
|
343
|
+
if (unsettled.size === 0 || dependencies.now() >= deadline) break;
|
|
344
|
+
await dependencies.sleep(HUB_CLEANUP_POLL_MS);
|
|
345
|
+
// Keep the last observation rather than scan with no time left.
|
|
346
|
+
if (dependencies.now() >= deadline) break;
|
|
347
|
+
}
|
|
348
|
+
for (const name of unsettled) failed.add(name);
|
|
349
|
+
return {
|
|
350
|
+
stopped: owned.filter((name) => !failed.has(name)),
|
|
351
|
+
failed: owned.filter((name) => failed.has(name)),
|
|
352
|
+
complete,
|
|
353
|
+
};
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
export function ownedHubCleanupDependencies(
|
|
357
|
+
runOmp: (args: readonly string[], timeoutMs: number) => Promise<boolean>,
|
|
358
|
+
root = ompRunDir(),
|
|
359
|
+
): OwnedHubCleanupDependencies {
|
|
360
|
+
return {
|
|
361
|
+
scan: (cwd) => scanHubProcessesFrom(root, cwd),
|
|
362
|
+
expire: (ms) => {
|
|
363
|
+
const fired = Promise.withResolvers<void>();
|
|
364
|
+
const timer = setTimeout(fired.resolve, ms);
|
|
365
|
+
return { fired: fired.promise, cancel: () => clearTimeout(timer) };
|
|
366
|
+
},
|
|
367
|
+
stop: (name, cwd, timeoutMs) => runOmp(["ps", "stop", "--dir", cwd, "--", name], timeoutMs),
|
|
368
|
+
sleep: waitMs,
|
|
369
|
+
now: () => Date.now(),
|
|
370
|
+
};
|
|
371
|
+
}
|
|
372
|
+
|
|
130
373
|
export async function tailHubLogFrom(
|
|
131
374
|
root: string,
|
|
132
375
|
cwd: string,
|
|
133
376
|
name: string,
|
|
134
377
|
): Promise<{ content: string; truncated: boolean }> {
|
|
135
|
-
const roots = await findProjectDaemonRoots(root, cwd);
|
|
378
|
+
const { roots } = await findProjectDaemonRoots(root, cwd);
|
|
136
379
|
for (const projectRoot of roots) {
|
|
137
380
|
try {
|
|
138
381
|
const buffer = await readFile(join(projectRoot, "daemons", name, "output.log"));
|
|
@@ -15,6 +15,8 @@ export const OMP_OPERATIONAL_FAILURES = [
|
|
|
15
15
|
{ category: "tool-projector", stage: "subsession-projector" },
|
|
16
16
|
{ category: "terminal-outcome", stage: "failed" },
|
|
17
17
|
{ category: "terminal-outcome", stage: "unresolved" },
|
|
18
|
+
{ category: "session-close", stage: "hub-cleanup" },
|
|
19
|
+
{ category: "session-close", stage: "hub-cleanup-unsupported-launcher" },
|
|
18
20
|
] as const;
|
|
19
21
|
|
|
20
22
|
export type OmpOperationalFailure = (typeof OMP_OPERATIONAL_FAILURES)[number];
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
/** Release Please keeps this bundle-safe constant aligned with package.json. */
|
|
2
|
-
export const PASEO_OMP_PACKAGE_VERSION = "5.6.3-next.
|
|
2
|
+
export const PASEO_OMP_PACKAGE_VERSION = "5.6.3-next.482.1"; // x-release-please-version
|
package/server/paths.ts
CHANGED
|
@@ -160,6 +160,29 @@ export function ompStateDir(
|
|
|
160
160
|
return ompStorageDir("state", environment, platform);
|
|
161
161
|
}
|
|
162
162
|
|
|
163
|
+
/**
|
|
164
|
+
* OMP's service broker root (`run/daemons`). It hangs off the profile's config root, not the
|
|
165
|
+
* agent directory, so every named profile has its own brokers (pi-utils `getDaemonRuntimeRoot`).
|
|
166
|
+
*/
|
|
167
|
+
export function ompHubRunDir(
|
|
168
|
+
environment: NodeJS.ProcessEnv = currentOmpEnvironment(),
|
|
169
|
+
platform: NodeJS.Platform = process.platform,
|
|
170
|
+
): string {
|
|
171
|
+
const profile = ompProfile(environment);
|
|
172
|
+
const defaultAgentDir = defaultOmpAgentDir(environment, profile);
|
|
173
|
+
let root = dirname(defaultAgentDir);
|
|
174
|
+
const xdgHome = environment.XDG_STATE_HOME;
|
|
175
|
+
if (
|
|
176
|
+
xdgHome &&
|
|
177
|
+
(platform === "linux" || platform === "darwin") &&
|
|
178
|
+
ompAgentDir(environment) === defaultAgentDir
|
|
179
|
+
) {
|
|
180
|
+
const xdgRoot = profile ? join(xdgHome, "omp", "profiles", profile) : join(xdgHome, "omp");
|
|
181
|
+
if (existsSync(xdgRoot)) root = xdgRoot;
|
|
182
|
+
}
|
|
183
|
+
return join(root, "run", "daemons");
|
|
184
|
+
}
|
|
185
|
+
|
|
163
186
|
function configuredSessionDir(agentDir: string): string | undefined {
|
|
164
187
|
for (const settingsPath of [
|
|
165
188
|
join(agentDir, "settings.json"),
|
|
@@ -1,8 +1,19 @@
|
|
|
1
1
|
import type { ChildProcessWithoutNullStreams } from "node:child_process";
|
|
2
|
-
import { join } from "node:path";
|
|
2
|
+
import { basename, join } from "node:path";
|
|
3
3
|
import { z } from "zod";
|
|
4
|
-
import {
|
|
5
|
-
|
|
4
|
+
import {
|
|
5
|
+
type HubServiceOwners,
|
|
6
|
+
type OwnedHubCleanupResult,
|
|
7
|
+
ownedHubCleanupDependencies,
|
|
8
|
+
stopOwnedHubProcesses,
|
|
9
|
+
} from "../hub";
|
|
10
|
+
import { ompDataDir, ompHubRunDir } from "../paths";
|
|
11
|
+
import { defaultSpawn, runBounded } from "../provider-diagnostics";
|
|
12
|
+
import {
|
|
13
|
+
buildOmpSpawnRequest,
|
|
14
|
+
type OmpSpawnRequest,
|
|
15
|
+
type OmpStartOptions,
|
|
16
|
+
} from "./omp-rpc-environment";
|
|
6
17
|
import { waitMs } from "./omp-rpc-process";
|
|
7
18
|
import {
|
|
8
19
|
inspectJsonBounds,
|
|
@@ -153,6 +164,11 @@ export interface OmpRuntime {
|
|
|
153
164
|
cwd: string;
|
|
154
165
|
signal?: AbortSignal;
|
|
155
166
|
}): Promise<OmpPersistedSubagentMessages>;
|
|
167
|
+
/** Stops the live Hub services these OMP sessions own once their process tree is gone. */
|
|
168
|
+
stopOwnedServices?(
|
|
169
|
+
options: OmpStartOptions,
|
|
170
|
+
owners: HubServiceOwners,
|
|
171
|
+
): Promise<OwnedHubCleanupResult>;
|
|
156
172
|
}
|
|
157
173
|
|
|
158
174
|
export interface OmpRpcRuntimeOptions {
|
|
@@ -164,6 +180,73 @@ export interface OmpRpcRuntimeOptions {
|
|
|
164
180
|
listSessions?: (
|
|
165
181
|
options: OmpSessionListOptions,
|
|
166
182
|
) => OmpSessionDescriptor[] | Promise<OmpSessionDescriptor[]>;
|
|
183
|
+
/** Runs one OMP CLI command bounded by `timeoutMs`; resolves true only on a clean zero exit. */
|
|
184
|
+
runOmpCommand?: (
|
|
185
|
+
command: string,
|
|
186
|
+
args: readonly string[],
|
|
187
|
+
env: NodeJS.ProcessEnv,
|
|
188
|
+
cwd: string,
|
|
189
|
+
timeoutMs: number,
|
|
190
|
+
) => Promise<boolean>;
|
|
191
|
+
hubRunDir?: string;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
const OMP_COMMAND_TIMEOUT_MS = 15_000;
|
|
195
|
+
const OMP_COMMAND_KILL_GRACE_MS = 1_000;
|
|
196
|
+
const OMP_COMMAND_MAX_OUTPUT_BYTES = 64 * 1024;
|
|
197
|
+
|
|
198
|
+
async function runOmpCommand(
|
|
199
|
+
command: string,
|
|
200
|
+
args: readonly string[],
|
|
201
|
+
env: NodeJS.ProcessEnv,
|
|
202
|
+
cwd: string,
|
|
203
|
+
timeoutMs: number,
|
|
204
|
+
): Promise<boolean> {
|
|
205
|
+
const run = await runBounded(
|
|
206
|
+
defaultSpawn,
|
|
207
|
+
command,
|
|
208
|
+
args,
|
|
209
|
+
env,
|
|
210
|
+
Math.min(timeoutMs, OMP_COMMAND_TIMEOUT_MS),
|
|
211
|
+
OMP_COMMAND_KILL_GRACE_MS,
|
|
212
|
+
OMP_COMMAND_MAX_OUTPUT_BYTES,
|
|
213
|
+
cwd,
|
|
214
|
+
);
|
|
215
|
+
return run.outcome === "exited" && run.exitCode === 0 && !run.cleanupFailed;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/**
|
|
219
|
+
* The configured command prefix as an `omp ps` launcher. OMP strips `--profile` before it looks
|
|
220
|
+
* for a subcommand, so a bare flag left in front of `ps` (`--resume`, an unknown long flag) would
|
|
221
|
+
* take `ps` as its value: the profile moves to `OMP_PROFILE`, which OMP reads when no `--profile`
|
|
222
|
+
* is given, and every bare flag left at the end of the prefix is dropped. The last `--profile`
|
|
223
|
+
* wins, as in OMP.
|
|
224
|
+
*/
|
|
225
|
+
function psLauncher(args: readonly string[]): { prefix: string[]; profile?: string } {
|
|
226
|
+
const prefix: string[] = [];
|
|
227
|
+
let profile: string | undefined;
|
|
228
|
+
for (let index = 0; index < args.length; index += 1) {
|
|
229
|
+
const argument = args[index] ?? "";
|
|
230
|
+
if (argument === "--profile") profile = args[++index];
|
|
231
|
+
else if (argument.startsWith("--profile=")) profile = argument.slice(10);
|
|
232
|
+
else prefix.push(argument);
|
|
233
|
+
}
|
|
234
|
+
while (/^-[^=]*$/u.test(prefix.at(-1) ?? "")) prefix.pop();
|
|
235
|
+
return profile === undefined ? { prefix } : { prefix, profile };
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* An `env` wrapper or a `NAME=value` word in the launcher can change OMP's profile or store root
|
|
240
|
+
* in ways the plugin cannot tell apart reliably (`env -i`, `-u`, `-S`, PATH changes), so Hub
|
|
241
|
+
* cleanup fails closed for such launchers instead of guessing the broker.
|
|
242
|
+
*/
|
|
243
|
+
function wrapsEnvironment(command: string, args: readonly string[]): boolean {
|
|
244
|
+
return [command, ...args].some((word) => {
|
|
245
|
+
const executable = basename(word).toLowerCase();
|
|
246
|
+
return (
|
|
247
|
+
executable === "env" || executable === "env.exe" || /^[A-Za-z_][A-Za-z0-9_]*=/u.test(word)
|
|
248
|
+
);
|
|
249
|
+
});
|
|
167
250
|
}
|
|
168
251
|
|
|
169
252
|
// OMP accepts an empty message when images carry the prompt.
|
|
@@ -601,6 +684,35 @@ export class OmpRpcRuntime implements OmpRuntime {
|
|
|
601
684
|
listOmpSessionDescriptors(options, this.options.environment ?? process.env),
|
|
602
685
|
);
|
|
603
686
|
}
|
|
687
|
+
async stopOwnedServices(
|
|
688
|
+
options: OmpStartOptions,
|
|
689
|
+
owners: HubServiceOwners,
|
|
690
|
+
): Promise<OwnedHubCleanupResult> {
|
|
691
|
+
// Same executable, launcher prefix, profile and environment as the RPC process, so the CLI
|
|
692
|
+
// reaches the broker that session used; brokers live under each profile's own config root.
|
|
693
|
+
const request = buildOmpSpawnRequest({
|
|
694
|
+
...options,
|
|
695
|
+
environment: options.environment ?? this.options.environment,
|
|
696
|
+
});
|
|
697
|
+
const { prefix, profile } = psLauncher(
|
|
698
|
+
request.args.slice(0, Math.max(0, (options.command?.length ?? 1) - 1)),
|
|
699
|
+
);
|
|
700
|
+
if (wrapsEnvironment(request.command, prefix)) {
|
|
701
|
+
return { stopped: [], failed: [], complete: false, unsupportedLauncher: true };
|
|
702
|
+
}
|
|
703
|
+
// OMP prefers `--profile` over `OMP_PROFILE`.
|
|
704
|
+
const env = profile === undefined ? request.env : { ...request.env, OMP_PROFILE: profile };
|
|
705
|
+
const run = this.options.runOmpCommand ?? runOmpCommand;
|
|
706
|
+
return stopOwnedHubProcesses(
|
|
707
|
+
request.cwd,
|
|
708
|
+
owners,
|
|
709
|
+
ownedHubCleanupDependencies(
|
|
710
|
+
(args, timeoutMs) =>
|
|
711
|
+
run(request.command, [...prefix, ...args], env, request.cwd, timeoutMs),
|
|
712
|
+
this.options.hubRunDir ?? process.env.PASEO_OMP_RUN_DIR ?? ompHubRunDir(env),
|
|
713
|
+
),
|
|
714
|
+
);
|
|
715
|
+
}
|
|
604
716
|
async readPersistedSessionTranscript(options: {
|
|
605
717
|
sessionFile: string;
|
|
606
718
|
sessionId: string;
|
|
@@ -3,6 +3,7 @@ import { opendir } from "node:fs/promises";
|
|
|
3
3
|
import { homedir } from "node:os";
|
|
4
4
|
import { basename, join, resolve } from "node:path";
|
|
5
5
|
import { isOmpProfileName } from "../../shared/omp-store";
|
|
6
|
+
import type { HubServiceOwners } from "../hub";
|
|
6
7
|
import { ompAgentDir, ompCacheDir, ompDataDir, ompSessionDir, ompStateDir } from "../paths";
|
|
7
8
|
import { OmpRpcRuntime, type OmpRuntime } from "./omp-rpc";
|
|
8
9
|
import type { OmpStartOptions } from "./omp-rpc-environment";
|
|
@@ -159,6 +160,7 @@ export function createProfileOmpProvider(profile: string, options: OmpProviderOp
|
|
|
159
160
|
reportProtocolViolation: options.reportProtocolViolation,
|
|
160
161
|
});
|
|
161
162
|
const readPersistedSessionTranscript = runtime.readPersistedSessionTranscript?.bind(runtime);
|
|
163
|
+
const stopOwnedServices = runtime.stopOwnedServices?.bind(runtime);
|
|
162
164
|
const assertSessionDir = (requested?: string) => {
|
|
163
165
|
if (requested !== undefined && resolve(requested) !== sessionDir) {
|
|
164
166
|
throw new OmpPublicError("OMP session directory conflicts with the selected profile");
|
|
@@ -203,6 +205,15 @@ export function createProfileOmpProvider(profile: string, options: OmpProviderOp
|
|
|
203
205
|
readPersistedSubagentTranscript(input) {
|
|
204
206
|
return runtime.readPersistedSubagentTranscript(input);
|
|
205
207
|
},
|
|
208
|
+
...(stopOwnedServices
|
|
209
|
+
? {
|
|
210
|
+
stopOwnedServices(start: OmpStartOptions, owners: HubServiceOwners) {
|
|
211
|
+
assertSessionDir(start.sessionDir);
|
|
212
|
+
assertEnvironment(start.env);
|
|
213
|
+
return stopOwnedServices({ ...start, environment, sessionDir }, owners);
|
|
214
|
+
},
|
|
215
|
+
}
|
|
216
|
+
: {}),
|
|
206
217
|
};
|
|
207
218
|
const provider = createOmpProvider({
|
|
208
219
|
...options,
|
|
@@ -347,6 +347,8 @@ export class OmpProviderSession implements OmpOpenedSession, OmpSubagentControls
|
|
|
347
347
|
private unsupportedThinkingNoticePending = false;
|
|
348
348
|
private unsupportedThinkingNoticePublished = false;
|
|
349
349
|
private degradedMcpServers: readonly OmpDegradedMcpServer[] = [];
|
|
350
|
+
/** Every OMP session ID this agent's runtimes have used; OMP records each as a service owner. */
|
|
351
|
+
private readonly hubServiceOwners = new Map<string, number>();
|
|
350
352
|
|
|
351
353
|
private constructor(
|
|
352
354
|
id: string,
|
|
@@ -376,6 +378,7 @@ export class OmpProviderSession implements OmpOpenedSession, OmpSubagentControls
|
|
|
376
378
|
) {
|
|
377
379
|
this.id = id;
|
|
378
380
|
this.cwd = config.cwd;
|
|
381
|
+
this.hubServiceOwners.set(nativeSessionId, Number.POSITIVE_INFINITY);
|
|
379
382
|
this.scheduler = scheduler;
|
|
380
383
|
this.usage = new OmpSessionUsage(
|
|
381
384
|
id,
|
|
@@ -993,7 +996,11 @@ export class OmpProviderSession implements OmpOpenedSession, OmpSubagentControls
|
|
|
993
996
|
const nextNativeSessionId = validateNativeSessionId(state.sessionId);
|
|
994
997
|
if (nextNativeSessionId !== this.nativeSessionId) {
|
|
995
998
|
this.transitionNativeSession(this.nativeSessionId, nextNativeSessionId);
|
|
999
|
+
// The released id may be resumed by another agent; only services it started before now
|
|
1000
|
+
// remain this agent's.
|
|
1001
|
+
this.hubServiceOwners.set(this.nativeSessionId, Date.now());
|
|
996
1002
|
this.nativeSessionId = nextNativeSessionId;
|
|
1003
|
+
this.hubServiceOwners.set(nextNativeSessionId, Number.POSITIVE_INFINITY);
|
|
997
1004
|
if (this.persistSession) {
|
|
998
1005
|
this.emit({
|
|
999
1006
|
type: "session.persistence",
|
|
@@ -1075,10 +1082,16 @@ export class OmpProviderSession implements OmpOpenedSession, OmpSubagentControls
|
|
|
1075
1082
|
this.hostTools.detach();
|
|
1076
1083
|
this.runtimeDisposal ??= runtime.close();
|
|
1077
1084
|
this.hostToolsDisposal ??= this.hostTools.close();
|
|
1085
|
+
// Same gate as a normal close: only once this OMP tree is verified gone.
|
|
1086
|
+
const hubCleanup = this.runtimeDisposal.then(
|
|
1087
|
+
() => this.stopOwnedHubServices(),
|
|
1088
|
+
() => undefined,
|
|
1089
|
+
);
|
|
1078
1090
|
const cleanup = settleSessionCleanup(
|
|
1079
1091
|
[
|
|
1080
1092
|
this.runtimeDisposal,
|
|
1081
1093
|
this.hostToolsDisposal,
|
|
1094
|
+
hubCleanup,
|
|
1082
1095
|
this.recoveryPromise,
|
|
1083
1096
|
configRefresh,
|
|
1084
1097
|
this.configRefreshInFlight,
|
|
@@ -2031,8 +2044,9 @@ export class OmpProviderSession implements OmpOpenedSession, OmpSubagentControls
|
|
|
2031
2044
|
const deferredCleanup: Promise<void>[] = [];
|
|
2032
2045
|
const seenCleanup = new Set<Promise<void>>();
|
|
2033
2046
|
const seenCoordination = new Set<Promise<void>>();
|
|
2047
|
+
let ompTreeUnverified = false;
|
|
2034
2048
|
while (true) {
|
|
2035
|
-
const cleanup = [this.runtimeDisposal, this.hostToolsDisposal].filter(
|
|
2049
|
+
const cleanup: Promise<void>[] = [this.runtimeDisposal, this.hostToolsDisposal].filter(
|
|
2036
2050
|
(promise): promise is Promise<void> => promise !== null && !seenCleanup.has(promise),
|
|
2037
2051
|
);
|
|
2038
2052
|
const coordination = [this.recoveryPromise, configRefresh, this.configRefreshInFlight].filter(
|
|
@@ -2042,8 +2056,10 @@ export class OmpProviderSession implements OmpOpenedSession, OmpSubagentControls
|
|
|
2042
2056
|
for (const promise of cleanup) seenCleanup.add(promise);
|
|
2043
2057
|
for (const promise of coordination) seenCoordination.add(promise);
|
|
2044
2058
|
const results = await Promise.allSettled([...cleanup, ...coordination]);
|
|
2045
|
-
for (const result of results.slice(0, cleanup.length)) {
|
|
2059
|
+
for (const [index, result] of results.slice(0, cleanup.length).entries()) {
|
|
2046
2060
|
if (result.status !== "rejected") continue;
|
|
2061
|
+
// MCP host-tool trees are the plugin's own children and never own an OMP service.
|
|
2062
|
+
if (cleanup[index] !== this.hostToolsDisposal) ompTreeUnverified = true;
|
|
2047
2063
|
if (isOmpCleanupFailure(result.reason)) {
|
|
2048
2064
|
if (!seenCleanup.has(result.reason.cleanup)) deferredCleanup.push(result.reason.cleanup);
|
|
2049
2065
|
continue;
|
|
@@ -2051,6 +2067,8 @@ export class OmpProviderSession implements OmpOpenedSession, OmpSubagentControls
|
|
|
2051
2067
|
cleanupErrors.push(result.reason);
|
|
2052
2068
|
}
|
|
2053
2069
|
}
|
|
2070
|
+
// Only after the OMP tree is verified gone, so no live session can start another service.
|
|
2071
|
+
if (!ompTreeUnverified) await this.stopOwnedHubServices();
|
|
2054
2072
|
if (deferredCleanup.length > 0) {
|
|
2055
2073
|
const failed =
|
|
2056
2074
|
cleanupErrors.length > 0
|
|
@@ -2067,6 +2085,27 @@ export class OmpProviderSession implements OmpOpenedSession, OmpSubagentControls
|
|
|
2067
2085
|
}
|
|
2068
2086
|
}
|
|
2069
2087
|
|
|
2088
|
+
private async stopOwnedHubServices(): Promise<void> {
|
|
2089
|
+
if (!this.runtimeFactory.stopOwnedServices) return;
|
|
2090
|
+
try {
|
|
2091
|
+
const result = await this.runtimeFactory.stopOwnedServices(
|
|
2092
|
+
this.recoveryOptions,
|
|
2093
|
+
this.hubServiceOwners,
|
|
2094
|
+
);
|
|
2095
|
+
if (result.unsupportedLauncher) {
|
|
2096
|
+
this.recordOperationalFailure({
|
|
2097
|
+
category: "session-close",
|
|
2098
|
+
stage: "hub-cleanup-unsupported-launcher",
|
|
2099
|
+
});
|
|
2100
|
+
return;
|
|
2101
|
+
}
|
|
2102
|
+
if (result.complete && result.failed.length === 0) return;
|
|
2103
|
+
} catch {
|
|
2104
|
+
// Reported below; a leftover service must never fail the agent's close.
|
|
2105
|
+
}
|
|
2106
|
+
this.recordOperationalFailure({ category: "session-close", stage: "hub-cleanup" });
|
|
2107
|
+
}
|
|
2108
|
+
|
|
2070
2109
|
private publishSessionClosed(error?: { message: string }): void {
|
|
2071
2110
|
if (this.sessionClosedPublished) return;
|
|
2072
2111
|
this.sessionClosedPublished = true;
|