@ours.network/fleet 1.2.0-nightly.70 → 1.2.0-nightly.72
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/account-origin.d.ts +4 -1
- package/dist/account-origin.js +27 -2
- package/dist/application/conversation-history.d.ts +7 -0
- package/dist/application/conversation-history.js +65 -0
- package/dist/application/fleet-query-service.d.ts +1 -1
- package/dist/application/fleet-query-service.js +2 -2
- package/dist/application/resource-observer.d.ts +40 -0
- package/dist/application/resource-observer.js +278 -0
- package/dist/application/role-removal-service.js +1 -1
- package/dist/application/role-repository.d.ts +1 -1
- package/dist/application/role-repository.js +11 -5
- package/dist/application/task-layouts.js +2 -0
- package/dist/application/task-room-service.d.ts +2 -0
- package/dist/application/task-room-service.js +2 -2
- package/dist/build-info.json +4 -4
- package/dist/daemon-recovery.d.ts +13 -1
- package/dist/daemon-recovery.js +117 -3
- package/dist/monitor.d.ts +1 -0
- package/dist/monitor.js +1 -1
- package/dist/notifications/fleet-producer.d.ts +4 -1
- package/dist/notifications/fleet-producer.js +39 -10
- package/dist/notifications/outbox.d.ts +14 -1
- package/dist/notifications/outbox.js +51 -10
- package/dist/notifications/target-cleanup.d.ts +10 -4
- package/dist/notifications/target-cleanup.js +35 -23
- package/dist/notifications/task-producer.d.ts +3 -1
- package/dist/notifications/task-producer.js +38 -5
- package/dist/ops.js +1 -1
- package/dist/paths.d.ts +3 -0
- package/dist/paths.js +3 -0
- package/dist/rooms-tasks/add-member.js +2 -1
- package/dist/rooms-tasks/layout.js +2 -1
- package/dist/rooms-tasks/task-state.d.ts +1 -0
- package/dist/rooms-tasks/task-state.js +27 -11
- package/dist/rooms-tasks/types.d.ts +4 -1
- package/dist/runner.js +22 -7
- package/dist/subscriptions/login.d.ts +4 -3
- package/dist/subscriptions/login.js +14 -6
- package/dist/subscriptions/service.d.ts +1 -1
- package/dist/subscriptions/service.js +4 -4
- package/dist/subscriptions/store.js +3 -0
- package/dist/tunnel-setup-input.js +7 -1
- package/dist/watchdog/query.d.ts +3 -1
- package/dist/watchdog/query.js +2 -1
- package/dist/web/auth.d.ts +2 -2
- package/dist/web/auth.js +4 -4
- package/dist/web/events.d.ts +2 -0
- package/dist/web/events.js +9 -0
- package/dist/web/prefix-gateway.js +20 -10
- package/dist/web/resource-views.d.ts +124 -0
- package/dist/web/resource-views.js +424 -0
- package/dist/web/runtime.js +6 -3
- package/dist/web/server.d.ts +3 -0
- package/dist/web/server.js +171 -20
- package/dist/workspace-enrollment.js +53 -28
- package/dist/workspace-replacement.js +28 -10
- package/docs/conversation-stream.md +6 -0
- package/package.json +1 -1
package/dist/daemon-recovery.js
CHANGED
|
@@ -2,6 +2,7 @@ import { readFileSync } from 'node:fs';
|
|
|
2
2
|
import { join } from 'node:path';
|
|
3
3
|
import { createHash, randomUUID } from 'node:crypto';
|
|
4
4
|
import { attachOursClient, } from '@ours.network/sdk/client';
|
|
5
|
+
import { classifyIdentityPresence } from './monitor.js';
|
|
5
6
|
import { readClientProfile } from './client-profile.js';
|
|
6
7
|
import { replaceFileAtomically } from './atomic-file.js';
|
|
7
8
|
export const DAEMON_RECOVERY_MAX_ATTEMPTS = 6;
|
|
@@ -41,7 +42,8 @@ function startupProgress(value) {
|
|
|
41
42
|
/** Read generation and readiness through the selected gateway. Server paths are opaque. */
|
|
42
43
|
export async function probeDaemonGeneration(fetch, env, deps = {}) {
|
|
43
44
|
try {
|
|
44
|
-
|
|
45
|
+
const { identities: _identities, ...generation } = await probeSelectedDaemon(deps, readClientProfile(env));
|
|
46
|
+
return generation;
|
|
45
47
|
}
|
|
46
48
|
catch {
|
|
47
49
|
return { state: 'unavailable', reason: 'CLIENT_GATEWAY_PROFILE_INVALID' };
|
|
@@ -56,7 +58,7 @@ async function probeSelectedDaemon(deps, profile) {
|
|
|
56
58
|
const options = {
|
|
57
59
|
endpoint: profile.endpoint, expectedInstanceId: profile.expectedInstanceId,
|
|
58
60
|
credentialPath: profile.credentialPath,
|
|
59
|
-
sessionMode: 'external', leaseToken: randomUUID(), env: {},
|
|
61
|
+
sessionMode: 'external', leaseToken: randomUUID(), env: {}, requestSignal: deps.requestSignal,
|
|
60
62
|
};
|
|
61
63
|
client = await (deps.attachClient?.(options) ?? attachOursClient(options));
|
|
62
64
|
const info = await client.version({ startup: true });
|
|
@@ -75,7 +77,7 @@ async function probeSelectedDaemon(deps, profile) {
|
|
|
75
77
|
return { state: 'unavailable', reason: 'DAEMON_IDENTITIES_INVALID' };
|
|
76
78
|
if (!identities.length)
|
|
77
79
|
return { state: 'unavailable', reason: 'DAEMON_IDENTITIES_NOT_READY' };
|
|
78
|
-
return { state: 'ready', generation: {
|
|
80
|
+
return { state: 'ready', identities, generation: {
|
|
79
81
|
bootId: progress.bootId, pid: progress.pid, startedAt: progress.startedAt, stateDir: info.stateDir,
|
|
80
82
|
} };
|
|
81
83
|
}
|
|
@@ -87,6 +89,118 @@ async function probeSelectedDaemon(deps, profile) {
|
|
|
87
89
|
await client?.close().catch(() => undefined);
|
|
88
90
|
}
|
|
89
91
|
}
|
|
92
|
+
const selectedHealthOwners = new Map();
|
|
93
|
+
const unavailableHealth = (reason) => ({
|
|
94
|
+
generation: { state: 'unavailable', reason },
|
|
95
|
+
});
|
|
96
|
+
/** Retained runtime health, independent of browser delivery and stream readiness. */
|
|
97
|
+
export function acquireSelectedDaemonHealth(env, deps = {}) {
|
|
98
|
+
let profile;
|
|
99
|
+
try {
|
|
100
|
+
profile = readClientProfile(env);
|
|
101
|
+
}
|
|
102
|
+
catch {
|
|
103
|
+
return {
|
|
104
|
+
sample: async () => unavailableHealth('CLIENT_GATEWAY_PROFILE_INVALID'),
|
|
105
|
+
presence: () => ({ state: 'unknown', detail: 'selected daemon unavailable' }),
|
|
106
|
+
release: () => { },
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
// Credentials are part of the selection boundary, not merely endpoint identity.
|
|
110
|
+
const key = JSON.stringify([profile.endpoint, profile.expectedInstanceId, profile.credentialPath]);
|
|
111
|
+
let owner = selectedHealthOwners.get(key);
|
|
112
|
+
if (!owner) {
|
|
113
|
+
let stopped = false;
|
|
114
|
+
let timer;
|
|
115
|
+
let current;
|
|
116
|
+
let nextProbeAt = 0;
|
|
117
|
+
let pending;
|
|
118
|
+
let cancelProbe;
|
|
119
|
+
const run = () => {
|
|
120
|
+
let client;
|
|
121
|
+
let closed = false;
|
|
122
|
+
let cancelled = false;
|
|
123
|
+
const request = new AbortController();
|
|
124
|
+
const close = async () => {
|
|
125
|
+
if (client && !closed) {
|
|
126
|
+
closed = true;
|
|
127
|
+
await client.close().catch(() => undefined);
|
|
128
|
+
}
|
|
129
|
+
};
|
|
130
|
+
let deadline;
|
|
131
|
+
const interrupted = new Promise(resolve => {
|
|
132
|
+
cancelProbe = () => {
|
|
133
|
+
cancelled = true;
|
|
134
|
+
request.abort();
|
|
135
|
+
void close();
|
|
136
|
+
resolve(unavailableHealth('DAEMON_SELECTED_PROBE_UNAVAILABLE'));
|
|
137
|
+
};
|
|
138
|
+
deadline = setTimeout(cancelProbe, 10_000);
|
|
139
|
+
deadline.unref?.();
|
|
140
|
+
});
|
|
141
|
+
const probe = probeSelectedDaemon({ ...deps, requestSignal: request.signal, attachClient: async (options) => {
|
|
142
|
+
client = await (deps.attachClient?.(options) ?? attachOursClient(options));
|
|
143
|
+
if (cancelled) {
|
|
144
|
+
void close();
|
|
145
|
+
throw new Error('health probe cancelled');
|
|
146
|
+
}
|
|
147
|
+
return { version: (...args) => client.version(...args),
|
|
148
|
+
identities: () => client.identities(), close };
|
|
149
|
+
} }, profile).then(({ identities, ...generation }) => ({ generation, identities }));
|
|
150
|
+
pending = Promise.race([probe, interrupted]).then(snapshot => {
|
|
151
|
+
if (!stopped)
|
|
152
|
+
current = snapshot;
|
|
153
|
+
return snapshot;
|
|
154
|
+
}).finally(() => {
|
|
155
|
+
if (deadline)
|
|
156
|
+
clearTimeout(deadline);
|
|
157
|
+
cancelProbe = undefined;
|
|
158
|
+
pending = undefined;
|
|
159
|
+
if (!stopped) {
|
|
160
|
+
nextProbeAt = Date.now() + 2_000;
|
|
161
|
+
timer = setTimeout(() => { void run(); }, 2_000);
|
|
162
|
+
timer.unref?.();
|
|
163
|
+
}
|
|
164
|
+
});
|
|
165
|
+
return pending;
|
|
166
|
+
};
|
|
167
|
+
owner = {
|
|
168
|
+
consumers: 0,
|
|
169
|
+
sample: () => {
|
|
170
|
+
if (pending)
|
|
171
|
+
return pending;
|
|
172
|
+
if (current && Date.now() < nextProbeAt)
|
|
173
|
+
return Promise.resolve(current);
|
|
174
|
+
// A delayed timer after process suspension cannot certify old absence.
|
|
175
|
+
if (timer)
|
|
176
|
+
clearTimeout(timer);
|
|
177
|
+
return run();
|
|
178
|
+
},
|
|
179
|
+
stop: () => { stopped = true; if (timer)
|
|
180
|
+
clearTimeout(timer); cancelProbe?.(); },
|
|
181
|
+
};
|
|
182
|
+
selectedHealthOwners.set(key, owner);
|
|
183
|
+
void run();
|
|
184
|
+
}
|
|
185
|
+
const retained = owner;
|
|
186
|
+
retained.consumers++;
|
|
187
|
+
let released = false;
|
|
188
|
+
return {
|
|
189
|
+
sample: () => released ? Promise.resolve(unavailableHealth('HEALTH_OWNER_RELEASED')) : retained.sample(),
|
|
190
|
+
presence: (name, snapshot) => snapshot.generation.state === 'ready'
|
|
191
|
+
? classifyIdentityPresence(name, snapshot.identities)
|
|
192
|
+
: { state: 'unknown', detail: 'selected daemon unavailable' },
|
|
193
|
+
release: () => {
|
|
194
|
+
if (released)
|
|
195
|
+
return;
|
|
196
|
+
released = true;
|
|
197
|
+
if (--retained.consumers === 0) {
|
|
198
|
+
retained.stop();
|
|
199
|
+
selectedHealthOwners.delete(key);
|
|
200
|
+
}
|
|
201
|
+
},
|
|
202
|
+
};
|
|
203
|
+
}
|
|
90
204
|
export class DaemonGenerationObserver {
|
|
91
205
|
current;
|
|
92
206
|
unavailable = false;
|
package/dist/monitor.d.ts
CHANGED
|
@@ -108,6 +108,7 @@ export type IdentityPresence = {
|
|
|
108
108
|
* valid but missing identity, which made a closed temp identity look healthy.
|
|
109
109
|
*/
|
|
110
110
|
export declare function probeIdentityPresence(name: string, fetch: FetchLike, env: NodeJS.ProcessEnv, deps?: IdentityProbeDeps): Promise<IdentityPresence>;
|
|
111
|
+
export declare function classifyIdentityPresence(name: string, identities: unknown): IdentityPresence;
|
|
111
112
|
/** Keep only the events whose type the role asked to wake on. */
|
|
112
113
|
export declare function filterEvents(events: NotifyEvent[], wakeSources: string[]): NotifyEvent[];
|
|
113
114
|
/**
|
package/dist/monitor.js
CHANGED
|
@@ -65,7 +65,7 @@ export async function probeIdentityPresence(name, fetch, env, deps = {}) {
|
|
|
65
65
|
await client?.close().catch(() => undefined);
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
|
-
function classifyIdentityPresence(name, identities) {
|
|
68
|
+
export function classifyIdentityPresence(name, identities) {
|
|
69
69
|
if (!Array.isArray(identities))
|
|
70
70
|
return { state: 'unknown', detail: 'identity index response is malformed' };
|
|
71
71
|
// A healthy daemon normally has at least its Human identity. During daemon
|
|
@@ -8,12 +8,15 @@ export declare class FleetNotificationProducer {
|
|
|
8
8
|
private readonly config;
|
|
9
9
|
private readonly warn;
|
|
10
10
|
private readonly queues;
|
|
11
|
-
private readonly
|
|
11
|
+
private readonly observation;
|
|
12
|
+
private continuation?;
|
|
13
|
+
private stopped;
|
|
12
14
|
private readonly existing;
|
|
13
15
|
private cleanup?;
|
|
14
16
|
constructor(roots: string[], stateDir: string, config: ProducerConfig, warn?: (line: string) => void);
|
|
15
17
|
private names;
|
|
16
18
|
poll(): void;
|
|
19
|
+
private resume;
|
|
17
20
|
drain(): Promise<void>;
|
|
18
21
|
close(): Promise<void>;
|
|
19
22
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
+
import { observeFleetResources } from '../application/resource-observer.js';
|
|
1
2
|
import { roomsDir } from '../rooms-tasks/room-state.js';
|
|
2
|
-
import { drainNotificationTargetCleanup, notificationAgentId } from './target-cleanup.js';
|
|
3
|
+
import { cleanupDir, drainNotificationTargetCleanup, notificationAgentId, hasPendingNotificationTargetCleanup } from './target-cleanup.js';
|
|
3
4
|
import { closeSync, existsSync, lstatSync, openSync, readSync, readFileSync, readdirSync, statSync } from 'node:fs';
|
|
4
5
|
import { join } from 'node:path';
|
|
5
6
|
import { createHash } from 'node:crypto';
|
|
@@ -46,7 +47,9 @@ export class FleetNotificationProducer {
|
|
|
46
47
|
config;
|
|
47
48
|
warn;
|
|
48
49
|
queues = new Map();
|
|
49
|
-
|
|
50
|
+
observation = new AbortController();
|
|
51
|
+
continuation;
|
|
52
|
+
stopped = false;
|
|
50
53
|
existing = new Set();
|
|
51
54
|
cleanup;
|
|
52
55
|
constructor(roots, stateDir, config, warn = () => { }) {
|
|
@@ -54,12 +57,16 @@ export class FleetNotificationProducer {
|
|
|
54
57
|
this.stateDir = stateDir;
|
|
55
58
|
this.config = config;
|
|
56
59
|
this.warn = warn;
|
|
60
|
+
observeFleetResources({ retryWatchFailures: true, roots, paths: [...roots, roomsDir(), cleanupDir()] }, state => {
|
|
61
|
+
if (state.health === 'healthy')
|
|
62
|
+
this.poll();
|
|
63
|
+
else
|
|
64
|
+
this.warn('Notification discovery observation unavailable; checkpoint retained');
|
|
65
|
+
}, this.observation.signal);
|
|
57
66
|
for (const root of roots)
|
|
58
67
|
for (const name of this.names(root))
|
|
59
68
|
this.existing.add(join(root, name));
|
|
60
69
|
this.poll();
|
|
61
|
-
this.timer = setInterval(() => this.poll(), 1000);
|
|
62
|
-
this.timer.unref();
|
|
63
70
|
}
|
|
64
71
|
names(root) {
|
|
65
72
|
try {
|
|
@@ -70,8 +77,10 @@ export class FleetNotificationProducer {
|
|
|
70
77
|
}
|
|
71
78
|
}
|
|
72
79
|
poll() {
|
|
73
|
-
|
|
74
|
-
|
|
80
|
+
if (this.stopped)
|
|
81
|
+
return;
|
|
82
|
+
this.cleanup ??= drainNotificationTargetCleanup(this.config, undefined, undefined, { beforeDelete: async (target) => {
|
|
83
|
+
const name = notificationAgentId(target);
|
|
75
84
|
if (!name || !/^[A-Za-z0-9_-]+$/.test(name))
|
|
76
85
|
return false;
|
|
77
86
|
// A recreated or retained role still has a valid target. Filesystem errors cannot prove removal.
|
|
@@ -105,7 +114,14 @@ export class FleetNotificationProducer {
|
|
|
105
114
|
}
|
|
106
115
|
}
|
|
107
116
|
return true;
|
|
108
|
-
} }).catch(() => { }).finally(() => {
|
|
117
|
+
} }).catch(() => { }).finally(() => {
|
|
118
|
+
this.cleanup = undefined;
|
|
119
|
+
try {
|
|
120
|
+
if (hasPendingNotificationTargetCleanup())
|
|
121
|
+
this.resume(3000);
|
|
122
|
+
}
|
|
123
|
+
catch { }
|
|
124
|
+
});
|
|
109
125
|
let taskRoles;
|
|
110
126
|
try {
|
|
111
127
|
taskRoles = taskRoleAssociations();
|
|
@@ -126,7 +142,7 @@ export class FleetNotificationProducer {
|
|
|
126
142
|
if (!outbox) {
|
|
127
143
|
const key = createHash('sha256').update(roleDir).digest('hex');
|
|
128
144
|
outbox = new NotificationOutbox(join(this.stateDir, key + '.json'), this.config, async (value, eventId) => ({ eventId, title: `${roleId}: agent completed`, body: 'Your agent finished responding. Open the chat to view the result.',
|
|
129
|
-
|
|
145
|
+
...value, target: { kind: 'agent', agentId: roleId } }), this.warn);
|
|
130
146
|
if (!outbox.checkpoint) {
|
|
131
147
|
const segment = this.existing.has(roleDir) ? segments.at(-1) : segments[0];
|
|
132
148
|
outbox.saveCheckpoint({ segment, offset: this.existing.has(roleDir) ? statSync(join(dir, segment)).size : 0, prompts: [] });
|
|
@@ -157,6 +173,8 @@ export class FleetNotificationProducer {
|
|
|
157
173
|
finally {
|
|
158
174
|
closeSync(fd);
|
|
159
175
|
}
|
|
176
|
+
if (current.offset + count < statSync(join(dir, current.segment)).size || index < segments.length - 1)
|
|
177
|
+
this.resume();
|
|
160
178
|
let position = 0;
|
|
161
179
|
while (position < count) {
|
|
162
180
|
const end = buffer.indexOf(10, position);
|
|
@@ -187,8 +205,10 @@ export class FleetNotificationProducer {
|
|
|
187
205
|
if (['message.chunk', 'message.replace'].includes(event.kind) && event.promptId && event.messageId && current.prompts.includes(prompt))
|
|
188
206
|
current.messages[prompt] = event.messageId;
|
|
189
207
|
if (event.kind === 'turn.completed' && event.promptId && current.prompts.includes(prompt)) {
|
|
190
|
-
if (!outbox.enqueue(`${roleId}:${event.sessionGeneration}:${event.seq}`, { ...(taskRoles ? { taskId: taskRoles.get(roleId) ?? null } : {}),
|
|
208
|
+
if (!outbox.enqueue(`${roleId}:${event.sessionGeneration}:${event.seq}`, { ...(taskRoles ? { taskId: taskRoles.get(roleId) ?? null } : {}), anchor: { kind: 'fleet-message', messageId: current.messages[prompt] ?? event.promptId } })) {
|
|
209
|
+
this.resume(1000);
|
|
191
210
|
break;
|
|
211
|
+
}
|
|
192
212
|
current.prompts = current.prompts.filter(p => p !== prompt);
|
|
193
213
|
delete current.messages[prompt];
|
|
194
214
|
}
|
|
@@ -200,9 +220,18 @@ export class FleetNotificationProducer {
|
|
|
200
220
|
}
|
|
201
221
|
catch {
|
|
202
222
|
this.warn('Notification ledger admission pending; checkpoint will replay');
|
|
223
|
+
// Retained, not-yet-admitted ledger work needs its own next-due wake;
|
|
224
|
+
// the outbox sender can retry only events that were already admitted.
|
|
225
|
+
this.resume(1000);
|
|
203
226
|
}
|
|
204
227
|
}
|
|
205
228
|
}
|
|
229
|
+
resume(delay = 0) {
|
|
230
|
+
if (this.continuation || this.stopped)
|
|
231
|
+
return;
|
|
232
|
+
this.continuation = setTimeout(() => { this.continuation = undefined; this.poll(); }, delay);
|
|
233
|
+
this.continuation.unref();
|
|
234
|
+
}
|
|
206
235
|
async drain() { await Promise.all([...this.queues.values()].map(q => q.drain())); }
|
|
207
|
-
async close() {
|
|
236
|
+
async close() { this.stopped = true; this.observation.abort(); clearTimeout(this.continuation); await this.cleanup; await Promise.all([...this.queues.values()].map(q => q.close())); }
|
|
208
237
|
}
|
|
@@ -1,10 +1,23 @@
|
|
|
1
|
+
export type NotificationTarget = {
|
|
2
|
+
kind: 'agent';
|
|
3
|
+
agentId: string;
|
|
4
|
+
} | {
|
|
5
|
+
kind: 'task' | 'task-deleted';
|
|
6
|
+
taskId: string;
|
|
7
|
+
};
|
|
1
8
|
export interface NotificationInput {
|
|
2
9
|
eventId: string;
|
|
3
10
|
title: string;
|
|
4
11
|
body: string;
|
|
5
|
-
|
|
12
|
+
target: NotificationTarget;
|
|
13
|
+
anchor?: {
|
|
14
|
+
kind: 'fleet-message';
|
|
15
|
+
messageId: string;
|
|
16
|
+
};
|
|
6
17
|
taskId?: string | null;
|
|
7
18
|
}
|
|
19
|
+
/** Only the closed typed contract is sent; any other payload is never sent or converted. */
|
|
20
|
+
export declare function isNotificationInput(value: unknown): value is NotificationInput;
|
|
8
21
|
export interface ProducerConfig {
|
|
9
22
|
/** Service base URL. With a gateway credential it is the server gateway's notifications route. */
|
|
10
23
|
origin: string;
|
|
@@ -2,6 +2,38 @@ import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, re
|
|
|
2
2
|
import { dirname, join } from 'node:path';
|
|
3
3
|
import { readPrivateFile } from '@ours.network/sdk/connector';
|
|
4
4
|
import { readClientProfile } from '../client-profile.js';
|
|
5
|
+
const object = (value) => !!value && typeof value === 'object' && !Array.isArray(value);
|
|
6
|
+
const closed = (value, keys) => Object.keys(value).every(key => keys.includes(key));
|
|
7
|
+
const domainId = (id) => typeof id === 'string' && /^[A-Za-z0-9_-]{1,120}$/.test(id);
|
|
8
|
+
const text = (value, max) => typeof value === 'string' && !!value.trim() && value.length <= max;
|
|
9
|
+
/** Only the closed typed contract is sent; any other payload is never sent or converted. */
|
|
10
|
+
export function isNotificationInput(value) {
|
|
11
|
+
if (!object(value) || !closed(value, ['eventId', 'title', 'body', 'target', 'anchor', 'taskId'])
|
|
12
|
+
|| !text(value.eventId, 512) || !text(value.title, 160) || !text(value.body, 512) || !object(value.target))
|
|
13
|
+
return false;
|
|
14
|
+
const target = value.target;
|
|
15
|
+
if (target.kind === 'agent') {
|
|
16
|
+
if (!closed(target, ['kind', 'agentId']) || !domainId(target.agentId))
|
|
17
|
+
return false;
|
|
18
|
+
}
|
|
19
|
+
else if (target.kind === 'task' || target.kind === 'task-deleted') {
|
|
20
|
+
if (!closed(target, ['kind', 'taskId']) || !domainId(target.taskId) || (value.taskId !== undefined && value.taskId !== target.taskId))
|
|
21
|
+
return false;
|
|
22
|
+
}
|
|
23
|
+
else
|
|
24
|
+
return false;
|
|
25
|
+
if (value.taskId !== undefined && value.taskId !== null && !domainId(value.taskId))
|
|
26
|
+
return false;
|
|
27
|
+
return value.anchor === undefined || (target.kind === 'agent' && object(value.anchor)
|
|
28
|
+
&& closed(value.anchor, ['kind', 'messageId']) && value.anchor.kind === 'fleet-message' && text(value.anchor.messageId, 1024) && !/[\x00-\x1f\x7f]/.test(value.anchor.messageId));
|
|
29
|
+
}
|
|
30
|
+
const has = (entry, key) => Object.hasOwn(entry, key);
|
|
31
|
+
// An entry is sent only when it is an object with a string ID and its frozen payload is the closed
|
|
32
|
+
// typed contract or, with no payload field at all, it carries the current version marker. Anything
|
|
33
|
+
// else, including null or scalar entries, is never read further, projected, sent or changed.
|
|
34
|
+
const supported = (entry) => object(entry) && typeof entry.id === 'string' && !has(entry, 'operation')
|
|
35
|
+
&& (has(entry, 'payload') ? isNotificationInput(entry.payload) : entry.targetVersion === 1);
|
|
36
|
+
const same = (id) => (entry) => supported(entry) && entry.id === id;
|
|
5
37
|
function defaultProfile() {
|
|
6
38
|
try {
|
|
7
39
|
return readClientProfile();
|
|
@@ -75,11 +107,12 @@ export class NotificationOutbox {
|
|
|
75
107
|
this.warn = warn;
|
|
76
108
|
mkdirSync(dirname(file), { recursive: true, mode: 0o700 });
|
|
77
109
|
const saved = existsSync(file) ? JSON.parse(readFileSync(file, 'utf8')) : { entries: [] };
|
|
78
|
-
|
|
110
|
+
// Only the current state shape is read; anything else is refused and left unchanged on disk.
|
|
111
|
+
if (!saved || typeof saved !== 'object' || Array.isArray(saved) || !Array.isArray(saved.entries))
|
|
112
|
+
throw new Error('Unsupported notification outbox state');
|
|
113
|
+
this.entries = saved.entries;
|
|
79
114
|
this.cursor = saved.cursor;
|
|
80
115
|
this.checkpoint = saved.checkpoint;
|
|
81
|
-
if (!Array.isArray(this.entries))
|
|
82
|
-
throw new Error('invalid notification outbox');
|
|
83
116
|
this.timer = setInterval(() => { void this.drain(); }, 3000);
|
|
84
117
|
this.timer.unref();
|
|
85
118
|
}
|
|
@@ -109,14 +142,14 @@ export class NotificationOutbox {
|
|
|
109
142
|
enqueue(id, value, cursor = this.cursor) {
|
|
110
143
|
if (this.retired)
|
|
111
144
|
return false;
|
|
112
|
-
if (this.entries.some(
|
|
145
|
+
if (this.entries.some(same(id))) {
|
|
113
146
|
if (cursor !== this.cursor)
|
|
114
147
|
this.advance(cursor);
|
|
115
148
|
return true;
|
|
116
149
|
}
|
|
117
|
-
if (this.entries.length >= 2000)
|
|
150
|
+
if (this.entries.filter(supported).length >= 2000)
|
|
118
151
|
return false;
|
|
119
|
-
this.persist([...this.entries, { id, value }], cursor);
|
|
152
|
+
this.persist([...this.entries, { id, value, targetVersion: 1 }], cursor);
|
|
120
153
|
return true;
|
|
121
154
|
}
|
|
122
155
|
drain() {
|
|
@@ -125,15 +158,23 @@ export class NotificationOutbox {
|
|
|
125
158
|
return this.running ??= this.drainOnce().finally(() => { this.running = undefined; });
|
|
126
159
|
}
|
|
127
160
|
async drainOnce() {
|
|
128
|
-
|
|
161
|
+
let attempts = 0;
|
|
162
|
+
for (const entry of [...this.entries]) {
|
|
163
|
+
if (!supported(entry))
|
|
164
|
+
continue;
|
|
165
|
+
if (attempts >= 32)
|
|
166
|
+
break;
|
|
129
167
|
if (this.retired)
|
|
130
168
|
break;
|
|
131
169
|
try {
|
|
132
170
|
const value = entry.payload ?? await this.project(entry.value, entry.id);
|
|
171
|
+
if (!isNotificationInput(value))
|
|
172
|
+
continue;
|
|
173
|
+
attempts++;
|
|
133
174
|
// Freeze the canonical projection before any network attempt. A crash
|
|
134
175
|
// after service acceptance must replay the identical dedupe payload.
|
|
135
176
|
if (!entry.payload)
|
|
136
|
-
this.persist(this.entries.map(e =>
|
|
177
|
+
this.persist(this.entries.map(e => same(entry.id)(e) ? { ...e, payload: value } : e));
|
|
137
178
|
const response = await fetch(this.config.origin + '/api/v1/send', {
|
|
138
179
|
method: 'POST', redirect: 'error', signal: AbortSignal.timeout(10_000),
|
|
139
180
|
headers: { ...(this.config.gatewayCredential
|
|
@@ -144,7 +185,7 @@ export class NotificationOutbox {
|
|
|
144
185
|
await response.body?.cancel();
|
|
145
186
|
if (!response.ok)
|
|
146
187
|
throw new Error('notification service rejected event');
|
|
147
|
-
this.persist(this.entries.filter(e =>
|
|
188
|
+
this.persist(this.entries.filter(e => !same(entry.id)(e)));
|
|
148
189
|
}
|
|
149
190
|
catch {
|
|
150
191
|
this.warn('Notification producer delivery pending; retrying from durable outbox');
|
|
@@ -153,6 +194,6 @@ export class NotificationOutbox {
|
|
|
153
194
|
}
|
|
154
195
|
}
|
|
155
196
|
/** Wait for the current send before clearing queued sends and retiring its target. */
|
|
156
|
-
async retire() { this.retired = true; clearInterval(this.timer); await this.running; this.persist(
|
|
197
|
+
async retire() { this.retired = true; clearInterval(this.timer); await this.running; this.persist(this.entries.filter(entry => !supported(entry))); }
|
|
157
198
|
async close() { clearInterval(this.timer); await this.running; }
|
|
158
199
|
}
|
|
@@ -1,10 +1,16 @@
|
|
|
1
1
|
import { type ProducerConfig } from './outbox.js';
|
|
2
|
-
|
|
3
|
-
export
|
|
2
|
+
export declare const cleanupDir: () => string;
|
|
3
|
+
export type AgentTarget = {
|
|
4
|
+
kind: 'agent';
|
|
5
|
+
agentId: string;
|
|
6
|
+
};
|
|
7
|
+
/** Only a closed agent identity authorizes this producer's role-target cleanup. */
|
|
8
|
+
export declare function notificationAgentId(target: unknown): string | null;
|
|
9
|
+
export declare function hasPendingNotificationTargetCleanup(dir?: string): boolean;
|
|
4
10
|
/** Queue only after the owning lifecycle has established deletion, never on a failed inventory read. */
|
|
5
|
-
export declare function retireNotificationTarget(
|
|
11
|
+
export declare function retireNotificationTarget(target: AgentTarget, config?: ProducerConfig | undefined, dir?: string, request?: typeof fetch): Promise<void>;
|
|
6
12
|
/** Persisted retries also run from the ordinary web notification producer after restart. */
|
|
7
13
|
export declare function drainNotificationTargetCleanup(config: ProducerConfig, dir?: string, request?: typeof fetch, options?: {
|
|
8
14
|
retain?: boolean;
|
|
9
|
-
beforeDelete?: (
|
|
15
|
+
beforeDelete?: (target: AgentTarget) => Promise<boolean>;
|
|
10
16
|
}): Promise<void>;
|
|
@@ -4,31 +4,36 @@ import { join } from 'node:path';
|
|
|
4
4
|
import { replaceFileAtomically } from '../atomic-file.js';
|
|
5
5
|
import { stateRoot } from '../paths.js';
|
|
6
6
|
import { producerConfig } from './outbox.js';
|
|
7
|
-
const cleanupDir = () => join(stateRoot(), 'notifications', 'removed-targets');
|
|
8
|
-
/** Only
|
|
9
|
-
export function notificationAgentId(
|
|
10
|
-
|
|
11
|
-
if (typeof path !== 'string' || /[\\\x00-\x20]|%(?:2f|5c|00|0[ad])/i.test(path))
|
|
12
|
-
return null;
|
|
13
|
-
const match = /^\/fleet\/chats\/agents\/([^/?#]+)(?:[?#]|$)/.exec(path);
|
|
14
|
-
if (!match)
|
|
15
|
-
return null;
|
|
16
|
-
const url = new URL(path, 'https://ours.invalid');
|
|
17
|
-
if (['source', 'chat', 'task', 'detail'].some(key => url.searchParams.has(key)))
|
|
18
|
-
return null;
|
|
19
|
-
const id = decodeURIComponent(match[1]);
|
|
20
|
-
return /^[A-Za-z0-9_-]+$/.test(id) ? id : null;
|
|
21
|
-
}
|
|
22
|
-
catch {
|
|
7
|
+
export const cleanupDir = () => join(stateRoot(), 'notifications', 'removed-targets');
|
|
8
|
+
/** Only a closed agent identity authorizes this producer's role-target cleanup. */
|
|
9
|
+
export function notificationAgentId(target) {
|
|
10
|
+
if (!target || typeof target !== 'object' || Array.isArray(target))
|
|
23
11
|
return null;
|
|
24
|
-
|
|
12
|
+
const value = target;
|
|
13
|
+
return Object.keys(value).every(key => ['kind', 'agentId'].includes(key)) && value.kind === 'agent'
|
|
14
|
+
&& typeof value.agentId === 'string' && /^[A-Za-z0-9_-]{1,120}$/.test(value.agentId) ? value.agentId : null;
|
|
15
|
+
}
|
|
16
|
+
export function hasPendingNotificationTargetCleanup(dir = cleanupDir()) {
|
|
17
|
+
if (!existsSync(dir))
|
|
18
|
+
return false;
|
|
19
|
+
return readdirSync(dir).filter(file => /^[a-f0-9]{64}\.json$/.test(file)).some(file => {
|
|
20
|
+
try {
|
|
21
|
+
const value = JSON.parse(readFileSync(join(dir, file), 'utf8'));
|
|
22
|
+
return !!value && typeof value === 'object' && Object.keys(value).length === 1 && !!notificationAgentId(value.target);
|
|
23
|
+
}
|
|
24
|
+
catch (error) {
|
|
25
|
+
return !(error instanceof SyntaxError);
|
|
26
|
+
}
|
|
27
|
+
});
|
|
25
28
|
}
|
|
26
29
|
/** Queue only after the owning lifecycle has established deletion, never on a failed inventory read. */
|
|
27
|
-
export async function retireNotificationTarget(
|
|
30
|
+
export async function retireNotificationTarget(target, config = producerConfig(), dir = cleanupDir(), request = fetch) {
|
|
28
31
|
if (!config)
|
|
29
32
|
return;
|
|
33
|
+
if (!notificationAgentId(target))
|
|
34
|
+
throw new Error('Invalid notification target');
|
|
30
35
|
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
31
|
-
replaceFileAtomically(join(dir, createHash('sha256').update(
|
|
36
|
+
replaceFileAtomically(join(dir, createHash('sha256').update(JSON.stringify({ kind: 'agent', agentId: target.agentId })).digest('hex') + '.json'), JSON.stringify({ target }) + '\n', 0o600);
|
|
32
37
|
await drainNotificationTargetCleanup(config, dir, request, { retain: true });
|
|
33
38
|
}
|
|
34
39
|
/** Persisted retries also run from the ordinary web notification producer after restart. */
|
|
@@ -39,12 +44,19 @@ export async function drainNotificationTargetCleanup(config, dir = cleanupDir(),
|
|
|
39
44
|
for (const file of readdirSync(dir).filter(file => /^[a-f0-9]{64}\.json$/.test(file))) {
|
|
40
45
|
if (attempts >= 32)
|
|
41
46
|
break;
|
|
47
|
+
const path = join(dir, file);
|
|
48
|
+
let value;
|
|
49
|
+
try {
|
|
50
|
+
value = JSON.parse(readFileSync(path, 'utf8'));
|
|
51
|
+
}
|
|
52
|
+
catch {
|
|
53
|
+
continue;
|
|
54
|
+
}
|
|
55
|
+
if (!value || typeof value !== 'object' || Array.isArray(value) || Object.keys(value).length !== 1 || !notificationAgentId(value.target))
|
|
56
|
+
continue;
|
|
42
57
|
try {
|
|
43
|
-
const path = join(dir, file), value = JSON.parse(readFileSync(path, 'utf8'));
|
|
44
|
-
if (!notificationAgentId(value.url))
|
|
45
|
-
continue;
|
|
46
58
|
attempts++;
|
|
47
|
-
if (options.beforeDelete && !(await options.beforeDelete(value.
|
|
59
|
+
if (options.beforeDelete && !(await options.beforeDelete(value.target))) {
|
|
48
60
|
rmSync(path, { force: true });
|
|
49
61
|
continue;
|
|
50
62
|
}
|
|
@@ -6,13 +6,15 @@ export declare class TaskNotificationProducer {
|
|
|
6
6
|
private readonly config;
|
|
7
7
|
private readonly warn;
|
|
8
8
|
private readonly request;
|
|
9
|
-
private
|
|
9
|
+
private timer?;
|
|
10
|
+
private dirty;
|
|
10
11
|
private running?;
|
|
11
12
|
private offset;
|
|
12
13
|
private stopped;
|
|
13
14
|
private readonly shutdown;
|
|
14
15
|
constructor(config: ProducerConfig, warn?: (line: string) => void, request?: typeof fetch);
|
|
15
16
|
drain(): Promise<void>;
|
|
17
|
+
private retry;
|
|
16
18
|
private drainOnce;
|
|
17
19
|
close(): Promise<void>;
|
|
18
20
|
}
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { observeFleetResources } from '../application/resource-observer.js';
|
|
2
|
+
import { tasksDir } from '../rooms-tasks/task-state.js';
|
|
1
3
|
import { acknowledgeTaskNotification, pendingTaskNotifications } from '../rooms-tasks/task-state.js';
|
|
2
4
|
/** The Fleet manager owns this sender even when the web console is stopped.
|
|
3
5
|
* Events are already durable in their owning task commit, so no second outbox
|
|
@@ -7,6 +9,7 @@ export class TaskNotificationProducer {
|
|
|
7
9
|
warn;
|
|
8
10
|
request;
|
|
9
11
|
timer;
|
|
12
|
+
dirty = false;
|
|
10
13
|
running;
|
|
11
14
|
offset = 0;
|
|
12
15
|
stopped = false;
|
|
@@ -15,14 +18,42 @@ export class TaskNotificationProducer {
|
|
|
15
18
|
this.config = config;
|
|
16
19
|
this.warn = warn;
|
|
17
20
|
this.request = request;
|
|
18
|
-
|
|
19
|
-
|
|
21
|
+
// Subscribe before startup replay: a commit while fetch is pending needs a second pass.
|
|
22
|
+
observeFleetResources({ retryWatchFailures: true, paths: [tasksDir()], roots: [] }, state => {
|
|
23
|
+
if (state.health !== 'healthy')
|
|
24
|
+
this.warn('Task notification observation unavailable; committed events retained');
|
|
25
|
+
else {
|
|
26
|
+
this.dirty = true;
|
|
27
|
+
void this.drain();
|
|
28
|
+
}
|
|
29
|
+
}, this.shutdown.signal);
|
|
20
30
|
void this.drain();
|
|
21
31
|
}
|
|
22
32
|
drain() {
|
|
23
|
-
|
|
33
|
+
if (this.stopped)
|
|
34
|
+
return Promise.resolve();
|
|
35
|
+
if (this.running)
|
|
36
|
+
return this.running;
|
|
37
|
+
this.dirty = false;
|
|
38
|
+
clearTimeout(this.timer);
|
|
39
|
+
this.timer = undefined;
|
|
40
|
+
this.running = this.drainOnce().catch(() => {
|
|
24
41
|
this.warn('Task notification snapshot unavailable; committed events retained');
|
|
25
|
-
|
|
42
|
+
this.retry();
|
|
43
|
+
}).finally(() => {
|
|
44
|
+
this.running = undefined;
|
|
45
|
+
if (this.dirty && !this.stopped) {
|
|
46
|
+
this.dirty = false;
|
|
47
|
+
void this.drain();
|
|
48
|
+
}
|
|
49
|
+
});
|
|
50
|
+
return this.running;
|
|
51
|
+
}
|
|
52
|
+
retry() {
|
|
53
|
+
if (this.stopped || this.timer)
|
|
54
|
+
return;
|
|
55
|
+
this.timer = setTimeout(() => { this.timer = undefined; void this.drain(); }, 3000);
|
|
56
|
+
this.timer.unref();
|
|
26
57
|
}
|
|
27
58
|
async drainOnce() {
|
|
28
59
|
if (this.stopped)
|
|
@@ -33,6 +64,8 @@ export class TaskNotificationProducer {
|
|
|
33
64
|
const start = this.offset % pending.length;
|
|
34
65
|
const batch = [...pending.slice(start), ...pending.slice(0, start)].slice(0, 32);
|
|
35
66
|
this.offset = start + batch.length;
|
|
67
|
+
// Retry only outstanding work; never run an idle discovery timer.
|
|
68
|
+
this.retry();
|
|
36
69
|
for (let index = 0; index < batch.length; index += 4) {
|
|
37
70
|
if (this.stopped)
|
|
38
71
|
break;
|
|
@@ -63,5 +96,5 @@ export class TaskNotificationProducer {
|
|
|
63
96
|
}));
|
|
64
97
|
}
|
|
65
98
|
}
|
|
66
|
-
async close() { this.stopped = true;
|
|
99
|
+
async close() { this.stopped = true; clearTimeout(this.timer); this.shutdown.abort(); await this.running; }
|
|
67
100
|
}
|
package/dist/ops.js
CHANGED
|
@@ -271,7 +271,7 @@ export async function rmRole(cfg, name, deps) {
|
|
|
271
271
|
rmSync(agentDir(name), { recursive: true, force: true });
|
|
272
272
|
if (generatedSource && existsSync(role.sourceFile)) {
|
|
273
273
|
unlinkSync(role.sourceFile);
|
|
274
|
-
await retireNotificationTarget(
|
|
274
|
+
await retireNotificationTarget({ kind: 'agent', agentId: name }).catch(() => { });
|
|
275
275
|
deps.log(`removed ${role.sourceFile}`);
|
|
276
276
|
}
|
|
277
277
|
deps.log(`removed '${name}' (its ours identity is left intact)`);
|