@ours.network/fleet 1.2.0-nightly.70 → 1.2.0-nightly.71
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/application/conversation-history.d.ts +7 -0
- package/dist/application/conversation-history.js +65 -0
- package/dist/application/fleet-query-service.d.ts +1 -1
- package/dist/application/fleet-query-service.js +2 -2
- package/dist/application/resource-observer.d.ts +40 -0
- package/dist/application/resource-observer.js +278 -0
- package/dist/application/role-repository.d.ts +1 -1
- package/dist/application/role-repository.js +11 -5
- package/dist/application/task-layouts.js +2 -0
- package/dist/application/task-room-service.d.ts +2 -0
- package/dist/application/task-room-service.js +2 -2
- package/dist/build-info.json +4 -4
- package/dist/daemon-recovery.d.ts +13 -1
- package/dist/daemon-recovery.js +117 -3
- package/dist/monitor.d.ts +1 -0
- package/dist/monitor.js +1 -1
- package/dist/notifications/fleet-producer.d.ts +4 -1
- package/dist/notifications/fleet-producer.js +36 -7
- package/dist/notifications/target-cleanup.d.ts +1 -0
- package/dist/notifications/target-cleanup.js +1 -1
- package/dist/notifications/task-producer.d.ts +3 -1
- package/dist/notifications/task-producer.js +38 -5
- package/dist/rooms-tasks/add-member.js +2 -1
- package/dist/rooms-tasks/layout.js +2 -1
- package/dist/rooms-tasks/task-state.d.ts +1 -0
- package/dist/rooms-tasks/task-state.js +6 -1
- package/dist/runner.js +22 -7
- package/dist/subscriptions/login.d.ts +4 -3
- package/dist/subscriptions/login.js +14 -6
- package/dist/subscriptions/service.d.ts +1 -1
- package/dist/subscriptions/service.js +4 -4
- package/dist/subscriptions/store.js +3 -0
- package/dist/watchdog/query.d.ts +3 -1
- package/dist/watchdog/query.js +2 -1
- package/dist/web/events.d.ts +2 -0
- package/dist/web/events.js +9 -0
- package/dist/web/prefix-gateway.js +20 -10
- package/dist/web/resource-views.d.ts +124 -0
- package/dist/web/resource-views.js +424 -0
- package/dist/web/runtime.js +4 -1
- package/dist/web/server.d.ts +3 -0
- package/dist/web/server.js +170 -19
- package/docs/conversation-stream.md +6 -0
- package/package.json +1 -1
package/dist/daemon-recovery.js
CHANGED
|
@@ -2,6 +2,7 @@ import { readFileSync } from 'node:fs';
|
|
|
2
2
|
import { join } from 'node:path';
|
|
3
3
|
import { createHash, randomUUID } from 'node:crypto';
|
|
4
4
|
import { attachOursClient, } from '@ours.network/sdk/client';
|
|
5
|
+
import { classifyIdentityPresence } from './monitor.js';
|
|
5
6
|
import { readClientProfile } from './client-profile.js';
|
|
6
7
|
import { replaceFileAtomically } from './atomic-file.js';
|
|
7
8
|
export const DAEMON_RECOVERY_MAX_ATTEMPTS = 6;
|
|
@@ -41,7 +42,8 @@ function startupProgress(value) {
|
|
|
41
42
|
/** Read generation and readiness through the selected gateway. Server paths are opaque. */
|
|
42
43
|
export async function probeDaemonGeneration(fetch, env, deps = {}) {
|
|
43
44
|
try {
|
|
44
|
-
|
|
45
|
+
const { identities: _identities, ...generation } = await probeSelectedDaemon(deps, readClientProfile(env));
|
|
46
|
+
return generation;
|
|
45
47
|
}
|
|
46
48
|
catch {
|
|
47
49
|
return { state: 'unavailable', reason: 'CLIENT_GATEWAY_PROFILE_INVALID' };
|
|
@@ -56,7 +58,7 @@ async function probeSelectedDaemon(deps, profile) {
|
|
|
56
58
|
const options = {
|
|
57
59
|
endpoint: profile.endpoint, expectedInstanceId: profile.expectedInstanceId,
|
|
58
60
|
credentialPath: profile.credentialPath,
|
|
59
|
-
sessionMode: 'external', leaseToken: randomUUID(), env: {},
|
|
61
|
+
sessionMode: 'external', leaseToken: randomUUID(), env: {}, requestSignal: deps.requestSignal,
|
|
60
62
|
};
|
|
61
63
|
client = await (deps.attachClient?.(options) ?? attachOursClient(options));
|
|
62
64
|
const info = await client.version({ startup: true });
|
|
@@ -75,7 +77,7 @@ async function probeSelectedDaemon(deps, profile) {
|
|
|
75
77
|
return { state: 'unavailable', reason: 'DAEMON_IDENTITIES_INVALID' };
|
|
76
78
|
if (!identities.length)
|
|
77
79
|
return { state: 'unavailable', reason: 'DAEMON_IDENTITIES_NOT_READY' };
|
|
78
|
-
return { state: 'ready', generation: {
|
|
80
|
+
return { state: 'ready', identities, generation: {
|
|
79
81
|
bootId: progress.bootId, pid: progress.pid, startedAt: progress.startedAt, stateDir: info.stateDir,
|
|
80
82
|
} };
|
|
81
83
|
}
|
|
@@ -87,6 +89,118 @@ async function probeSelectedDaemon(deps, profile) {
|
|
|
87
89
|
await client?.close().catch(() => undefined);
|
|
88
90
|
}
|
|
89
91
|
}
|
|
92
|
+
const selectedHealthOwners = new Map();
|
|
93
|
+
const unavailableHealth = (reason) => ({
|
|
94
|
+
generation: { state: 'unavailable', reason },
|
|
95
|
+
});
|
|
96
|
+
/** Retained runtime health, independent of browser delivery and stream readiness. */
|
|
97
|
+
export function acquireSelectedDaemonHealth(env, deps = {}) {
|
|
98
|
+
let profile;
|
|
99
|
+
try {
|
|
100
|
+
profile = readClientProfile(env);
|
|
101
|
+
}
|
|
102
|
+
catch {
|
|
103
|
+
return {
|
|
104
|
+
sample: async () => unavailableHealth('CLIENT_GATEWAY_PROFILE_INVALID'),
|
|
105
|
+
presence: () => ({ state: 'unknown', detail: 'selected daemon unavailable' }),
|
|
106
|
+
release: () => { },
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
// Credentials are part of the selection boundary, not merely endpoint identity.
|
|
110
|
+
const key = JSON.stringify([profile.endpoint, profile.expectedInstanceId, profile.credentialPath]);
|
|
111
|
+
let owner = selectedHealthOwners.get(key);
|
|
112
|
+
if (!owner) {
|
|
113
|
+
let stopped = false;
|
|
114
|
+
let timer;
|
|
115
|
+
let current;
|
|
116
|
+
let nextProbeAt = 0;
|
|
117
|
+
let pending;
|
|
118
|
+
let cancelProbe;
|
|
119
|
+
const run = () => {
|
|
120
|
+
let client;
|
|
121
|
+
let closed = false;
|
|
122
|
+
let cancelled = false;
|
|
123
|
+
const request = new AbortController();
|
|
124
|
+
const close = async () => {
|
|
125
|
+
if (client && !closed) {
|
|
126
|
+
closed = true;
|
|
127
|
+
await client.close().catch(() => undefined);
|
|
128
|
+
}
|
|
129
|
+
};
|
|
130
|
+
let deadline;
|
|
131
|
+
const interrupted = new Promise(resolve => {
|
|
132
|
+
cancelProbe = () => {
|
|
133
|
+
cancelled = true;
|
|
134
|
+
request.abort();
|
|
135
|
+
void close();
|
|
136
|
+
resolve(unavailableHealth('DAEMON_SELECTED_PROBE_UNAVAILABLE'));
|
|
137
|
+
};
|
|
138
|
+
deadline = setTimeout(cancelProbe, 10_000);
|
|
139
|
+
deadline.unref?.();
|
|
140
|
+
});
|
|
141
|
+
const probe = probeSelectedDaemon({ ...deps, requestSignal: request.signal, attachClient: async (options) => {
|
|
142
|
+
client = await (deps.attachClient?.(options) ?? attachOursClient(options));
|
|
143
|
+
if (cancelled) {
|
|
144
|
+
void close();
|
|
145
|
+
throw new Error('health probe cancelled');
|
|
146
|
+
}
|
|
147
|
+
return { version: (...args) => client.version(...args),
|
|
148
|
+
identities: () => client.identities(), close };
|
|
149
|
+
} }, profile).then(({ identities, ...generation }) => ({ generation, identities }));
|
|
150
|
+
pending = Promise.race([probe, interrupted]).then(snapshot => {
|
|
151
|
+
if (!stopped)
|
|
152
|
+
current = snapshot;
|
|
153
|
+
return snapshot;
|
|
154
|
+
}).finally(() => {
|
|
155
|
+
if (deadline)
|
|
156
|
+
clearTimeout(deadline);
|
|
157
|
+
cancelProbe = undefined;
|
|
158
|
+
pending = undefined;
|
|
159
|
+
if (!stopped) {
|
|
160
|
+
nextProbeAt = Date.now() + 2_000;
|
|
161
|
+
timer = setTimeout(() => { void run(); }, 2_000);
|
|
162
|
+
timer.unref?.();
|
|
163
|
+
}
|
|
164
|
+
});
|
|
165
|
+
return pending;
|
|
166
|
+
};
|
|
167
|
+
owner = {
|
|
168
|
+
consumers: 0,
|
|
169
|
+
sample: () => {
|
|
170
|
+
if (pending)
|
|
171
|
+
return pending;
|
|
172
|
+
if (current && Date.now() < nextProbeAt)
|
|
173
|
+
return Promise.resolve(current);
|
|
174
|
+
// A delayed timer after process suspension cannot certify old absence.
|
|
175
|
+
if (timer)
|
|
176
|
+
clearTimeout(timer);
|
|
177
|
+
return run();
|
|
178
|
+
},
|
|
179
|
+
stop: () => { stopped = true; if (timer)
|
|
180
|
+
clearTimeout(timer); cancelProbe?.(); },
|
|
181
|
+
};
|
|
182
|
+
selectedHealthOwners.set(key, owner);
|
|
183
|
+
void run();
|
|
184
|
+
}
|
|
185
|
+
const retained = owner;
|
|
186
|
+
retained.consumers++;
|
|
187
|
+
let released = false;
|
|
188
|
+
return {
|
|
189
|
+
sample: () => released ? Promise.resolve(unavailableHealth('HEALTH_OWNER_RELEASED')) : retained.sample(),
|
|
190
|
+
presence: (name, snapshot) => snapshot.generation.state === 'ready'
|
|
191
|
+
? classifyIdentityPresence(name, snapshot.identities)
|
|
192
|
+
: { state: 'unknown', detail: 'selected daemon unavailable' },
|
|
193
|
+
release: () => {
|
|
194
|
+
if (released)
|
|
195
|
+
return;
|
|
196
|
+
released = true;
|
|
197
|
+
if (--retained.consumers === 0) {
|
|
198
|
+
retained.stop();
|
|
199
|
+
selectedHealthOwners.delete(key);
|
|
200
|
+
}
|
|
201
|
+
},
|
|
202
|
+
};
|
|
203
|
+
}
|
|
90
204
|
export class DaemonGenerationObserver {
|
|
91
205
|
current;
|
|
92
206
|
unavailable = false;
|
package/dist/monitor.d.ts
CHANGED
|
@@ -108,6 +108,7 @@ export type IdentityPresence = {
|
|
|
108
108
|
* valid but missing identity, which made a closed temp identity look healthy.
|
|
109
109
|
*/
|
|
110
110
|
export declare function probeIdentityPresence(name: string, fetch: FetchLike, env: NodeJS.ProcessEnv, deps?: IdentityProbeDeps): Promise<IdentityPresence>;
|
|
111
|
+
export declare function classifyIdentityPresence(name: string, identities: unknown): IdentityPresence;
|
|
111
112
|
/** Keep only the events whose type the role asked to wake on. */
|
|
112
113
|
export declare function filterEvents(events: NotifyEvent[], wakeSources: string[]): NotifyEvent[];
|
|
113
114
|
/**
|
package/dist/monitor.js
CHANGED
|
@@ -65,7 +65,7 @@ export async function probeIdentityPresence(name, fetch, env, deps = {}) {
|
|
|
65
65
|
await client?.close().catch(() => undefined);
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
|
-
function classifyIdentityPresence(name, identities) {
|
|
68
|
+
export function classifyIdentityPresence(name, identities) {
|
|
69
69
|
if (!Array.isArray(identities))
|
|
70
70
|
return { state: 'unknown', detail: 'identity index response is malformed' };
|
|
71
71
|
// A healthy daemon normally has at least its Human identity. During daemon
|
|
@@ -8,12 +8,15 @@ export declare class FleetNotificationProducer {
|
|
|
8
8
|
private readonly config;
|
|
9
9
|
private readonly warn;
|
|
10
10
|
private readonly queues;
|
|
11
|
-
private readonly
|
|
11
|
+
private readonly observation;
|
|
12
|
+
private continuation?;
|
|
13
|
+
private stopped;
|
|
12
14
|
private readonly existing;
|
|
13
15
|
private cleanup?;
|
|
14
16
|
constructor(roots: string[], stateDir: string, config: ProducerConfig, warn?: (line: string) => void);
|
|
15
17
|
private names;
|
|
16
18
|
poll(): void;
|
|
19
|
+
private resume;
|
|
17
20
|
drain(): Promise<void>;
|
|
18
21
|
close(): Promise<void>;
|
|
19
22
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
+
import { observeFleetResources } from '../application/resource-observer.js';
|
|
1
2
|
import { roomsDir } from '../rooms-tasks/room-state.js';
|
|
2
|
-
import { drainNotificationTargetCleanup, notificationAgentId } from './target-cleanup.js';
|
|
3
|
+
import { cleanupDir, drainNotificationTargetCleanup, notificationAgentId } from './target-cleanup.js';
|
|
3
4
|
import { closeSync, existsSync, lstatSync, openSync, readSync, readFileSync, readdirSync, statSync } from 'node:fs';
|
|
4
5
|
import { join } from 'node:path';
|
|
5
6
|
import { createHash } from 'node:crypto';
|
|
@@ -46,7 +47,9 @@ export class FleetNotificationProducer {
|
|
|
46
47
|
config;
|
|
47
48
|
warn;
|
|
48
49
|
queues = new Map();
|
|
49
|
-
|
|
50
|
+
observation = new AbortController();
|
|
51
|
+
continuation;
|
|
52
|
+
stopped = false;
|
|
50
53
|
existing = new Set();
|
|
51
54
|
cleanup;
|
|
52
55
|
constructor(roots, stateDir, config, warn = () => { }) {
|
|
@@ -54,12 +57,16 @@ export class FleetNotificationProducer {
|
|
|
54
57
|
this.stateDir = stateDir;
|
|
55
58
|
this.config = config;
|
|
56
59
|
this.warn = warn;
|
|
60
|
+
observeFleetResources({ retryWatchFailures: true, roots, paths: [...roots, roomsDir(), cleanupDir()] }, state => {
|
|
61
|
+
if (state.health === 'healthy')
|
|
62
|
+
this.poll();
|
|
63
|
+
else
|
|
64
|
+
this.warn('Notification discovery observation unavailable; checkpoint retained');
|
|
65
|
+
}, this.observation.signal);
|
|
57
66
|
for (const root of roots)
|
|
58
67
|
for (const name of this.names(root))
|
|
59
68
|
this.existing.add(join(root, name));
|
|
60
69
|
this.poll();
|
|
61
|
-
this.timer = setInterval(() => this.poll(), 1000);
|
|
62
|
-
this.timer.unref();
|
|
63
70
|
}
|
|
64
71
|
names(root) {
|
|
65
72
|
try {
|
|
@@ -70,6 +77,8 @@ export class FleetNotificationProducer {
|
|
|
70
77
|
}
|
|
71
78
|
}
|
|
72
79
|
poll() {
|
|
80
|
+
if (this.stopped)
|
|
81
|
+
return;
|
|
73
82
|
this.cleanup ??= drainNotificationTargetCleanup(this.config, undefined, undefined, { beforeDelete: async (url) => {
|
|
74
83
|
const name = notificationAgentId(url);
|
|
75
84
|
if (!name || !/^[A-Za-z0-9_-]+$/.test(name))
|
|
@@ -105,7 +114,14 @@ export class FleetNotificationProducer {
|
|
|
105
114
|
}
|
|
106
115
|
}
|
|
107
116
|
return true;
|
|
108
|
-
} }).catch(() => { }).finally(() => {
|
|
117
|
+
} }).catch(() => { }).finally(() => {
|
|
118
|
+
this.cleanup = undefined;
|
|
119
|
+
try {
|
|
120
|
+
if (readdirSync(cleanupDir()).some(name => /^[a-f0-9]{64}\.json$/.test(name)))
|
|
121
|
+
this.resume(3000);
|
|
122
|
+
}
|
|
123
|
+
catch { }
|
|
124
|
+
});
|
|
109
125
|
let taskRoles;
|
|
110
126
|
try {
|
|
111
127
|
taskRoles = taskRoleAssociations();
|
|
@@ -157,6 +173,8 @@ export class FleetNotificationProducer {
|
|
|
157
173
|
finally {
|
|
158
174
|
closeSync(fd);
|
|
159
175
|
}
|
|
176
|
+
if (current.offset + count < statSync(join(dir, current.segment)).size || index < segments.length - 1)
|
|
177
|
+
this.resume();
|
|
160
178
|
let position = 0;
|
|
161
179
|
while (position < count) {
|
|
162
180
|
const end = buffer.indexOf(10, position);
|
|
@@ -187,8 +205,10 @@ export class FleetNotificationProducer {
|
|
|
187
205
|
if (['message.chunk', 'message.replace'].includes(event.kind) && event.promptId && event.messageId && current.prompts.includes(prompt))
|
|
188
206
|
current.messages[prompt] = event.messageId;
|
|
189
207
|
if (event.kind === 'turn.completed' && event.promptId && current.prompts.includes(prompt)) {
|
|
190
|
-
if (!outbox.enqueue(`${roleId}:${event.sessionGeneration}:${event.seq}`, { ...(taskRoles ? { taskId: taskRoles.get(roleId) ?? null } : {}), url: `/fleet/chats/agents/${encodeURIComponent(roleId)}#fleet-message-${encodeURIComponent(current.messages[prompt] ?? event.promptId)}` }))
|
|
208
|
+
if (!outbox.enqueue(`${roleId}:${event.sessionGeneration}:${event.seq}`, { ...(taskRoles ? { taskId: taskRoles.get(roleId) ?? null } : {}), url: `/fleet/chats/agents/${encodeURIComponent(roleId)}#fleet-message-${encodeURIComponent(current.messages[prompt] ?? event.promptId)}` })) {
|
|
209
|
+
this.resume(1000);
|
|
191
210
|
break;
|
|
211
|
+
}
|
|
192
212
|
current.prompts = current.prompts.filter(p => p !== prompt);
|
|
193
213
|
delete current.messages[prompt];
|
|
194
214
|
}
|
|
@@ -200,9 +220,18 @@ export class FleetNotificationProducer {
|
|
|
200
220
|
}
|
|
201
221
|
catch {
|
|
202
222
|
this.warn('Notification ledger admission pending; checkpoint will replay');
|
|
223
|
+
// Retained, not-yet-admitted ledger work needs its own next-due wake;
|
|
224
|
+
// the outbox sender can retry only events that were already admitted.
|
|
225
|
+
this.resume(1000);
|
|
203
226
|
}
|
|
204
227
|
}
|
|
205
228
|
}
|
|
229
|
+
resume(delay = 0) {
|
|
230
|
+
if (this.continuation || this.stopped)
|
|
231
|
+
return;
|
|
232
|
+
this.continuation = setTimeout(() => { this.continuation = undefined; this.poll(); }, delay);
|
|
233
|
+
this.continuation.unref();
|
|
234
|
+
}
|
|
206
235
|
async drain() { await Promise.all([...this.queues.values()].map(q => q.drain())); }
|
|
207
|
-
async close() {
|
|
236
|
+
async close() { this.stopped = true; this.observation.abort(); clearTimeout(this.continuation); await this.cleanup; await Promise.all([...this.queues.values()].map(q => q.close())); }
|
|
208
237
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { type ProducerConfig } from './outbox.js';
|
|
2
|
+
export declare const cleanupDir: () => string;
|
|
2
3
|
/** Only the canonical agent namespace can authorize role-target cleanup. */
|
|
3
4
|
export declare function notificationAgentId(path: string): string | null;
|
|
4
5
|
/** Queue only after the owning lifecycle has established deletion, never on a failed inventory read. */
|
|
@@ -4,7 +4,7 @@ import { join } from 'node:path';
|
|
|
4
4
|
import { replaceFileAtomically } from '../atomic-file.js';
|
|
5
5
|
import { stateRoot } from '../paths.js';
|
|
6
6
|
import { producerConfig } from './outbox.js';
|
|
7
|
-
const cleanupDir = () => join(stateRoot(), 'notifications', 'removed-targets');
|
|
7
|
+
export const cleanupDir = () => join(stateRoot(), 'notifications', 'removed-targets');
|
|
8
8
|
/** Only the canonical agent namespace can authorize role-target cleanup. */
|
|
9
9
|
export function notificationAgentId(path) {
|
|
10
10
|
try {
|
|
@@ -6,13 +6,15 @@ export declare class TaskNotificationProducer {
|
|
|
6
6
|
private readonly config;
|
|
7
7
|
private readonly warn;
|
|
8
8
|
private readonly request;
|
|
9
|
-
private
|
|
9
|
+
private timer?;
|
|
10
|
+
private dirty;
|
|
10
11
|
private running?;
|
|
11
12
|
private offset;
|
|
12
13
|
private stopped;
|
|
13
14
|
private readonly shutdown;
|
|
14
15
|
constructor(config: ProducerConfig, warn?: (line: string) => void, request?: typeof fetch);
|
|
15
16
|
drain(): Promise<void>;
|
|
17
|
+
private retry;
|
|
16
18
|
private drainOnce;
|
|
17
19
|
close(): Promise<void>;
|
|
18
20
|
}
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { observeFleetResources } from '../application/resource-observer.js';
|
|
2
|
+
import { tasksDir } from '../rooms-tasks/task-state.js';
|
|
1
3
|
import { acknowledgeTaskNotification, pendingTaskNotifications } from '../rooms-tasks/task-state.js';
|
|
2
4
|
/** The Fleet manager owns this sender even when the web console is stopped.
|
|
3
5
|
* Events are already durable in their owning task commit, so no second outbox
|
|
@@ -7,6 +9,7 @@ export class TaskNotificationProducer {
|
|
|
7
9
|
warn;
|
|
8
10
|
request;
|
|
9
11
|
timer;
|
|
12
|
+
dirty = false;
|
|
10
13
|
running;
|
|
11
14
|
offset = 0;
|
|
12
15
|
stopped = false;
|
|
@@ -15,14 +18,42 @@ export class TaskNotificationProducer {
|
|
|
15
18
|
this.config = config;
|
|
16
19
|
this.warn = warn;
|
|
17
20
|
this.request = request;
|
|
18
|
-
|
|
19
|
-
|
|
21
|
+
// Subscribe before startup replay: a commit while fetch is pending needs a second pass.
|
|
22
|
+
observeFleetResources({ retryWatchFailures: true, paths: [tasksDir()], roots: [] }, state => {
|
|
23
|
+
if (state.health !== 'healthy')
|
|
24
|
+
this.warn('Task notification observation unavailable; committed events retained');
|
|
25
|
+
else {
|
|
26
|
+
this.dirty = true;
|
|
27
|
+
void this.drain();
|
|
28
|
+
}
|
|
29
|
+
}, this.shutdown.signal);
|
|
20
30
|
void this.drain();
|
|
21
31
|
}
|
|
22
32
|
drain() {
|
|
23
|
-
|
|
33
|
+
if (this.stopped)
|
|
34
|
+
return Promise.resolve();
|
|
35
|
+
if (this.running)
|
|
36
|
+
return this.running;
|
|
37
|
+
this.dirty = false;
|
|
38
|
+
clearTimeout(this.timer);
|
|
39
|
+
this.timer = undefined;
|
|
40
|
+
this.running = this.drainOnce().catch(() => {
|
|
24
41
|
this.warn('Task notification snapshot unavailable; committed events retained');
|
|
25
|
-
|
|
42
|
+
this.retry();
|
|
43
|
+
}).finally(() => {
|
|
44
|
+
this.running = undefined;
|
|
45
|
+
if (this.dirty && !this.stopped) {
|
|
46
|
+
this.dirty = false;
|
|
47
|
+
void this.drain();
|
|
48
|
+
}
|
|
49
|
+
});
|
|
50
|
+
return this.running;
|
|
51
|
+
}
|
|
52
|
+
retry() {
|
|
53
|
+
if (this.stopped || this.timer)
|
|
54
|
+
return;
|
|
55
|
+
this.timer = setTimeout(() => { this.timer = undefined; void this.drain(); }, 3000);
|
|
56
|
+
this.timer.unref();
|
|
26
57
|
}
|
|
27
58
|
async drainOnce() {
|
|
28
59
|
if (this.stopped)
|
|
@@ -33,6 +64,8 @@ export class TaskNotificationProducer {
|
|
|
33
64
|
const start = this.offset % pending.length;
|
|
34
65
|
const batch = [...pending.slice(start), ...pending.slice(0, start)].slice(0, 32);
|
|
35
66
|
this.offset = start + batch.length;
|
|
67
|
+
// Retry only outstanding work; never run an idle discovery timer.
|
|
68
|
+
this.retry();
|
|
36
69
|
for (let index = 0; index < batch.length; index += 4) {
|
|
37
70
|
if (this.stopped)
|
|
38
71
|
break;
|
|
@@ -63,5 +96,5 @@ export class TaskNotificationProducer {
|
|
|
63
96
|
}));
|
|
64
97
|
}
|
|
65
98
|
}
|
|
66
|
-
async close() { this.stopped = true;
|
|
99
|
+
async close() { this.stopped = true; clearTimeout(this.timer); this.shutdown.abort(); await this.running; }
|
|
67
100
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { publishFleetResourceCommit } from '../application/resource-observer.js';
|
|
1
2
|
import { createHash } from 'node:crypto';
|
|
2
3
|
import { existsSync, mkdirSync, readFileSync } from 'node:fs';
|
|
3
4
|
import { join } from 'node:path';
|
|
@@ -62,7 +63,7 @@ export async function requestMemberAddition(input) {
|
|
|
62
63
|
})();
|
|
63
64
|
const template = sealTemplateSnapshot(plan.snapshot, input.cfg.agentTemplates ?? {}, plan.launchDefinitions);
|
|
64
65
|
const receipt = { taskId, requestId: r.requestId, roleId, hash, state: 'running' };
|
|
65
|
-
const save = () => replaceFileAtomically(path, JSON.stringify(receipt) + '\n', 0o600);
|
|
66
|
+
const save = () => { replaceFileAtomically(path, JSON.stringify(receipt) + '\n', 0o600); publishFleetResourceCommit(path); };
|
|
66
67
|
save();
|
|
67
68
|
running.add(path);
|
|
68
69
|
void (input.provision ?? provisionMembers)({ cfg: input.cfg, cowork: input.cowork, configPath: input.configPath, binPath: input.binPath,
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { publishFleetResourceCommit } from '../application/resource-observer.js';
|
|
1
2
|
/** Durable local room-layout controller.
|
|
2
3
|
* Uses native Cowork operations and a supervising Fleet's exact-instance port.
|
|
3
4
|
* Unknown mutation outcomes stop for reconciliation; they are never replayed.
|
|
@@ -45,7 +46,7 @@ export class RoomLayout {
|
|
|
45
46
|
}
|
|
46
47
|
get cowork() { return this.coworkAdapter ??= this.coworkFactory(); }
|
|
47
48
|
snapshot() { return JSON.parse(readFileSync(this.file, 'utf8')); }
|
|
48
|
-
save(s) { replaceFileAtomically(this.file, JSON.stringify(s, null, 2)); }
|
|
49
|
+
save(s) { replaceFileAtomically(this.file, JSON.stringify(s, null, 2)); publishFleetResourceCommit(this.file); }
|
|
49
50
|
async verify(i) {
|
|
50
51
|
need(i.supervisor === this.supervisor.id || i.remote, 'remote binding unsupported');
|
|
51
52
|
need(i.launch && i.cid && i.session, 'incomplete instance reference');
|
|
@@ -39,6 +39,7 @@ export declare function listTasks(filter?: {
|
|
|
39
39
|
state?: TaskState | TaskState[];
|
|
40
40
|
listId?: string;
|
|
41
41
|
includeDeleting?: boolean;
|
|
42
|
+
strict?: boolean;
|
|
42
43
|
}): TaskRecord[];
|
|
43
44
|
export declare function moveTaskToList(id: string, listId: string): TaskRecord;
|
|
44
45
|
export declare function findByIdempotencyKey(key: string): TaskRecord | undefined;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { publishFleetResourceCommit } from '../application/resource-observer.js';
|
|
1
2
|
import { existsSync, lstatSync, mkdirSync, readdirSync, readFileSync, unlinkSync, writeFileSync, rmSync, renameSync } from 'node:fs';
|
|
2
3
|
import { join } from 'node:path';
|
|
3
4
|
import { randomUUID } from 'node:crypto';
|
|
@@ -107,6 +108,7 @@ function writeTask(record) {
|
|
|
107
108
|
else
|
|
108
109
|
delete persisted.notification_events;
|
|
109
110
|
replaceFileAtomically(taskPath(record.task_id), JSON.stringify(persisted, null, 2) + '\n');
|
|
111
|
+
publishFleetResourceCommit(taskPath(record.task_id));
|
|
110
112
|
}
|
|
111
113
|
const retiredNotificationsDir = () => join(tasksDir(), '.notification-retired');
|
|
112
114
|
/** Atomic snapshots contain the events, including intermediate states between polls. */
|
|
@@ -280,7 +282,10 @@ export function listTasks(filter) {
|
|
|
280
282
|
if ((!states || states.includes(t.state)) && (!filter?.listId || t.list_id === filter.listId))
|
|
281
283
|
tasks.push(t);
|
|
282
284
|
}
|
|
283
|
-
catch {
|
|
285
|
+
catch (error) {
|
|
286
|
+
if (filter?.strict)
|
|
287
|
+
throw error; /* legacy CLI inventory tolerates corrupt files */
|
|
288
|
+
}
|
|
284
289
|
}
|
|
285
290
|
const order = new Map(readTaskLists().map((item, index) => [item.list_id, index]));
|
|
286
291
|
return tasks.sort((a, b) => (order.get(a.list_id) - order.get(b.list_id))
|
package/dist/runner.js
CHANGED
|
@@ -17,7 +17,7 @@ import { agentDir, stateRoot } from './paths.js';
|
|
|
17
17
|
import { loadConfig, findRole, isolationContextFor, resolveSnapshotMonitor, resolvePermissions, } from './config.js';
|
|
18
18
|
import { getAdapter } from './harness/registry.js';
|
|
19
19
|
import { createMonitor, probeIdentityPresence, } from './monitor.js';
|
|
20
|
-
import { DaemonGenerationObserver, RoleRecoveryController, probeDaemonGeneration, } from './daemon-recovery.js';
|
|
20
|
+
import { DaemonGenerationObserver, RoleRecoveryController, probeDaemonGeneration, acquireSelectedDaemonHealth, } from './daemon-recovery.js';
|
|
21
21
|
import { realExec } from './exec.js';
|
|
22
22
|
import { resolveIsolation } from './isolation/policy.js';
|
|
23
23
|
import { pinSubscriptionForLaunch } from './subscriptions/launch.js';
|
|
@@ -48,6 +48,7 @@ export class SupervisorRecycleRequiredError extends Error {
|
|
|
48
48
|
this.name = 'SupervisorRecycleRequiredError';
|
|
49
49
|
}
|
|
50
50
|
}
|
|
51
|
+
const defaultProbeGeneration = (env) => probeDaemonGeneration((url, init) => globalThis.fetch(url, init), env);
|
|
51
52
|
const defaultDeps = () => ({
|
|
52
53
|
prepareAgentOurs: prepareManagedAgent,
|
|
53
54
|
releaseAgentOurs: releaseManagedAgent,
|
|
@@ -64,7 +65,7 @@ const defaultDeps = () => ({
|
|
|
64
65
|
now: () => Date.now(),
|
|
65
66
|
log: line => process.stderr.write(line + '\n'),
|
|
66
67
|
fetch: (url, init) => globalThis.fetch(url, init),
|
|
67
|
-
probeGeneration:
|
|
68
|
+
probeGeneration: defaultProbeGeneration,
|
|
68
69
|
createMonitor: opts => createMonitor(opts),
|
|
69
70
|
createOwnerChannel: opts => new OwnerChannel(opts),
|
|
70
71
|
startAgentSession: (adapter, options) => adapter.start(options),
|
|
@@ -622,6 +623,7 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
622
623
|
// Self-reminders live and die with this attempt's runtime binding: every exit
|
|
623
624
|
// path below stops the dispatcher before that binding is released.
|
|
624
625
|
let reminderManager;
|
|
626
|
+
let selectedHealth;
|
|
625
627
|
try {
|
|
626
628
|
const managedHarness = prepareManagedHarness(role, dir, runCwd, managedService.descriptor, { ...harnessChildEnv(role, launch.env, dir), ...subscription?.env });
|
|
627
629
|
// Start-stagger: space this launch at least start_stagger_ms after the previous
|
|
@@ -669,8 +671,15 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
669
671
|
name, identity: role.identity, agentDir: dir, cfg: role.monitor,
|
|
670
672
|
deps: resolvedMonitorDeps,
|
|
671
673
|
}) : null;
|
|
674
|
+
// Injected legacy probes keep their test/embedding contract. Production uses
|
|
675
|
+
// one retained owner for generation and identity health in this process.
|
|
676
|
+
const retainHealth = deps.probeGeneration === defaultProbeGeneration
|
|
677
|
+
&& !deps.identityProbeDeps ? acquireSelectedDaemonHealth : undefined;
|
|
678
|
+
selectedHealth = retainHealth?.(resolvedMonitorDeps.env);
|
|
672
679
|
const daemonObserver = new DaemonGenerationObserver();
|
|
673
|
-
const initialDaemon = daemonObserver.observe(
|
|
680
|
+
const initialDaemon = daemonObserver.observe(selectedHealth
|
|
681
|
+
? (await selectedHealth.sample()).generation
|
|
682
|
+
: await deps.probeGeneration(resolvedMonitorDeps.env));
|
|
674
683
|
let sessionStartedWithoutDaemonBaseline = initialDaemon.kind !== 'baseline';
|
|
675
684
|
if (monitor)
|
|
676
685
|
await monitor.prime({ resetCursor: resetMonitorCursor });
|
|
@@ -1113,7 +1122,9 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
1113
1122
|
const now = deps.now();
|
|
1114
1123
|
if (now >= nextDaemonProbeAt) {
|
|
1115
1124
|
nextDaemonProbeAt = now + 2_000;
|
|
1116
|
-
const observation = daemonObserver.observe(
|
|
1125
|
+
const observation = daemonObserver.observe(selectedHealth
|
|
1126
|
+
? (await selectedHealth.sample()).generation
|
|
1127
|
+
: await deps.probeGeneration(resolvedMonitorDeps.env));
|
|
1117
1128
|
if (observation.kind === 'lost')
|
|
1118
1129
|
recoveryController.noteLoss(observation.reason);
|
|
1119
1130
|
if (observation.kind === 'changed' || observation.kind === 'available'
|
|
@@ -1147,19 +1158,22 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
1147
1158
|
}
|
|
1148
1159
|
if (temp && now >= nextIdentityPollAt) {
|
|
1149
1160
|
nextIdentityPollAt = now + TEMP_IDENTITY_POLL_MS;
|
|
1150
|
-
const presence =
|
|
1161
|
+
const presence = selectedHealth
|
|
1162
|
+
? selectedHealth.presence(role.identity, await selectedHealth.sample())
|
|
1163
|
+
: await probeIdentityPresence(role.identity, deps.fetch, resolvedMonitorDeps.env, deps.identityProbeDeps);
|
|
1164
|
+
const identityObservedAt = deps.now();
|
|
1151
1165
|
if (presence.state === 'present') {
|
|
1152
1166
|
identityObserved = true;
|
|
1153
1167
|
identityAbsentSince = undefined;
|
|
1154
1168
|
}
|
|
1155
1169
|
else if (presence.state === 'absent' && identityObserved) {
|
|
1156
|
-
identityAbsentSince ??=
|
|
1170
|
+
identityAbsentSince ??= identityObservedAt;
|
|
1157
1171
|
// Require a continuous, time-bounded run of authoritative absence.
|
|
1158
1172
|
// The first positive observation is the readiness gate: cold harness
|
|
1159
1173
|
// starts may spend minutes loading the harness and briefing before the
|
|
1160
1174
|
// agent creates/binds its identity, and absence before then is not a
|
|
1161
1175
|
// close event. After readiness, debounce a real disappearance.
|
|
1162
|
-
if (
|
|
1176
|
+
if (identityObservedAt - identityAbsentSince >= TEMP_IDENTITY_CLOSE_DEBOUNCE_MS) {
|
|
1163
1177
|
retirementReason = 'identity-closed';
|
|
1164
1178
|
deps.log(`[${name}] temporary identity '${role.identity}' closed; retiring session and supervisor`);
|
|
1165
1179
|
reminderManager?.stop();
|
|
@@ -1273,6 +1287,7 @@ export async function runOnce(name, opts = {}, partialDeps = {}) {
|
|
|
1273
1287
|
};
|
|
1274
1288
|
}
|
|
1275
1289
|
finally {
|
|
1290
|
+
selectedHealth?.release();
|
|
1276
1291
|
reminderManager?.stop();
|
|
1277
1292
|
managedService.setReminders?.(undefined);
|
|
1278
1293
|
await managedService.close(layoutRetiring);
|
|
@@ -33,6 +33,7 @@ declare abstract class BaseLogin implements LoginHandle {
|
|
|
33
33
|
readonly provider: SubscriptionProvider;
|
|
34
34
|
readonly owner: string;
|
|
35
35
|
readonly kind: LoginView['kind'];
|
|
36
|
+
protected readonly changed: () => void;
|
|
36
37
|
readonly loginId: string;
|
|
37
38
|
readonly expiresAt: number;
|
|
38
39
|
state: LoginState;
|
|
@@ -42,7 +43,7 @@ declare abstract class BaseLogin implements LoginHandle {
|
|
|
42
43
|
done: Promise<boolean>;
|
|
43
44
|
protected finish: (ok: boolean) => void;
|
|
44
45
|
private readonly timer;
|
|
45
|
-
constructor(provider: SubscriptionProvider, owner: string, kind: LoginView['kind'], now?: number);
|
|
46
|
+
constructor(provider: SubscriptionProvider, owner: string, kind: LoginView['kind'], now?: number, changed?: () => void);
|
|
46
47
|
view(): LoginView;
|
|
47
48
|
abstract submitCode(code: string): void;
|
|
48
49
|
protected abstract stop(): void;
|
|
@@ -59,7 +60,7 @@ export declare class ClaudeLogin extends BaseLogin {
|
|
|
59
60
|
private continued;
|
|
60
61
|
/** Resolves with the CLI exit status once it exits. */
|
|
61
62
|
exited: Promise<number | null>;
|
|
62
|
-
constructor(owner: string, bins: SubscriptionBinaries, home: string, profileId: string);
|
|
63
|
+
constructor(owner: string, bins: SubscriptionBinaries, home: string, profileId: string, changed?: () => void);
|
|
63
64
|
submitCode(code: string): void;
|
|
64
65
|
protected stop(): void;
|
|
65
66
|
}
|
|
@@ -69,7 +70,7 @@ export declare class CodexDeviceLogin extends BaseLogin {
|
|
|
69
70
|
private codexLoginId?;
|
|
70
71
|
/** Resolves true when the app-server reports a successful login. */
|
|
71
72
|
completed: Promise<boolean>;
|
|
72
|
-
constructor(owner: string, bins: SubscriptionBinaries, home: string, profileId: string);
|
|
73
|
+
constructor(owner: string, bins: SubscriptionBinaries, home: string, profileId: string, changed?: () => void);
|
|
73
74
|
submitCode(): void;
|
|
74
75
|
protected stop(): void;
|
|
75
76
|
}
|