@ours.network/fleet 2.1.2 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/application/conversation-history.d.ts +8 -0
- package/dist/application/conversation-history.js +37 -2
- package/dist/application/task-room-service.d.ts +7 -0
- package/dist/application/task-room-service.js +19 -0
- package/dist/build-info.json +4 -4
- package/dist/notifications/fleet-producer.d.ts +0 -1
- package/dist/notifications/fleet-producer.js +65 -47
- package/dist/notifications/mission-ownership.d.ts +31 -0
- package/dist/notifications/mission-ownership.js +148 -0
- package/dist/notifications/outbox.d.ts +10 -0
- package/dist/notifications/outbox.js +28 -2
- package/dist/notifications/task-producer.d.ts +11 -1
- package/dist/notifications/task-producer.js +34 -5
- package/dist/rooms-tasks/layout-service.d.ts +2 -0
- package/dist/rooms-tasks/layout-service.js +7 -2
- package/dist/rooms-tasks/layout.d.ts +5 -1
- package/dist/rooms-tasks/layout.js +7 -1
- package/dist/rooms-tasks/notification-manifest.d.ts +29 -0
- package/dist/rooms-tasks/notification-manifest.js +65 -0
- package/dist/rooms-tasks/task-review.d.ts +2 -0
- package/dist/rooms-tasks/task-review.js +7 -5
- package/dist/rooms-tasks/task-state.d.ts +35 -0
- package/dist/rooms-tasks/task-state.js +151 -17
- package/dist/rooms-tasks/types.d.ts +10 -0
- package/dist/session/acp.d.ts +3 -0
- package/dist/session/acp.js +58 -1
- package/dist/session/codex-app-server.js +22 -1
- package/dist/session/conversation-types.d.ts +8 -2
- package/dist/session/session-ownership.d.ts +18 -0
- package/dist/session/session-ownership.js +74 -0
- package/dist/supervisor/manager.js +9 -2
- package/dist/web/resource-views.d.ts +2 -0
- package/dist/web/resource-views.js +4 -2
- package/dist/web/server.js +29 -8
- package/package.json +5 -5
|
@@ -25,6 +25,14 @@ export declare function resumedConversationPage(stateDir: string, request: {
|
|
|
25
25
|
}, live: ConversationPageView, expectedSessionId: string): Promise<ConversationPageView>;
|
|
26
26
|
/** Download authorization requires an actual published attachment, never an orphan copy. */
|
|
27
27
|
export declare function conversationHasAttachment(stateDir: string, attachment: import('../file-delivery/types.js').DeliveredFile): Promise<boolean>;
|
|
28
|
+
/**
|
|
29
|
+
* The newest assistant message's time alone: the same event the text preview ends with,
|
|
30
|
+
* found by reading backwards only until it, within the same event and byte bounds.
|
|
31
|
+
* Records older than it are not read, so they are not validated either.
|
|
32
|
+
*/
|
|
33
|
+
export declare function conversationActivity(stateDir: string, signal: AbortSignal): Promise<{
|
|
34
|
+
at: string;
|
|
35
|
+
}>;
|
|
28
36
|
export declare function conversationPreview(stateDir: string, signal: AbortSignal): Promise<{
|
|
29
37
|
text: string;
|
|
30
38
|
at: string;
|
|
@@ -243,6 +243,8 @@ export async function conversationHasAttachment(stateDir, attachment) {
|
|
|
243
243
|
/** Passive, bounded last assistant text for directory previews. Never opens the
|
|
244
244
|
* ledger writer/recovery path or acquires a session/controller. An incomplete
|
|
245
245
|
* newest append is not committed yet; completed malformed records still fail. */
|
|
246
|
+
/** The assistant message events a preview is made of. */
|
|
247
|
+
const previewAssistant = (event) => ['message.chunk', 'message.replace'].includes(event.kind) && event.payload?.role === 'assistant' && typeof event.payload?.content?.type === 'string';
|
|
246
248
|
/** Bytes of a segment's tail a preview may consider, and the newest events it reads. */
|
|
247
249
|
const PREVIEW_WINDOW_BYTES = 2 * 1024 * 1024;
|
|
248
250
|
const PREVIEW_EVENTS = 200;
|
|
@@ -252,7 +254,7 @@ const PREVIEW_EVENTS = 200;
|
|
|
252
254
|
* cut by the window start is not complete and is skipped. Only the newest
|
|
253
255
|
* segment can have an in-flight append; elsewhere a partial record fails.
|
|
254
256
|
*/
|
|
255
|
-
async function previewSegmentEvents(path, newest, need, signal) {
|
|
257
|
+
async function previewSegmentEvents(path, newest, need, signal, until) {
|
|
256
258
|
const events = [];
|
|
257
259
|
const file = await open(path, 'r');
|
|
258
260
|
try {
|
|
@@ -263,6 +265,8 @@ async function previewSegmentEvents(path, newest, need, signal) {
|
|
|
263
265
|
if (event.schemaVersion !== 1 || !Number.isSafeInteger(event.seq))
|
|
264
266
|
throw Error('preview ledger invalid');
|
|
265
267
|
events.push(event);
|
|
268
|
+
if (until?.(event))
|
|
269
|
+
need = events.length;
|
|
266
270
|
};
|
|
267
271
|
while (position > floor && events.length < need) {
|
|
268
272
|
signal.throwIfAborted();
|
|
@@ -297,6 +301,37 @@ async function previewSegmentEvents(path, newest, need, signal) {
|
|
|
297
301
|
}
|
|
298
302
|
return events;
|
|
299
303
|
}
|
|
304
|
+
/**
|
|
305
|
+
* The newest assistant message's time alone: the same event the text preview ends with,
|
|
306
|
+
* found by reading backwards only until it, within the same event and byte bounds.
|
|
307
|
+
* Records older than it are not read, so they are not validated either.
|
|
308
|
+
*/
|
|
309
|
+
export async function conversationActivity(stateDir, signal) {
|
|
310
|
+
signal.throwIfAborted();
|
|
311
|
+
const dir = join(stateDir, '.conversation');
|
|
312
|
+
let files;
|
|
313
|
+
try {
|
|
314
|
+
files = (await readdir(dir)).filter(name => /^events-\d{6}\.jsonl$/.test(name)).sort().reverse();
|
|
315
|
+
}
|
|
316
|
+
catch (error) {
|
|
317
|
+
if (error.code === 'ENOENT')
|
|
318
|
+
return { at: '' };
|
|
319
|
+
throw error;
|
|
320
|
+
}
|
|
321
|
+
let read = 0;
|
|
322
|
+
for (const name of files) {
|
|
323
|
+
signal.throwIfAborted();
|
|
324
|
+
const events = await previewSegmentEvents(join(dir, name), name === files[0], PREVIEW_EVENTS - read, signal, previewAssistant);
|
|
325
|
+
const found = events.find(previewAssistant);
|
|
326
|
+
if (found)
|
|
327
|
+
return { at: found.at ?? '' };
|
|
328
|
+
read += events.length;
|
|
329
|
+
if (read >= PREVIEW_EVENTS)
|
|
330
|
+
break;
|
|
331
|
+
}
|
|
332
|
+
signal.throwIfAborted();
|
|
333
|
+
return { at: '' };
|
|
334
|
+
}
|
|
300
335
|
export async function conversationPreview(stateDir, signal) {
|
|
301
336
|
signal.throwIfAborted();
|
|
302
337
|
const dir = join(stateDir, '.conversation');
|
|
@@ -317,7 +352,7 @@ export async function conversationPreview(stateDir, signal) {
|
|
|
317
352
|
break;
|
|
318
353
|
}
|
|
319
354
|
signal.throwIfAborted();
|
|
320
|
-
const assistant =
|
|
355
|
+
const assistant = previewAssistant;
|
|
321
356
|
const end = events.findIndex(assistant);
|
|
322
357
|
if (end < 0)
|
|
323
358
|
return { text: 'No recent agent output', at: '' };
|
|
@@ -106,6 +106,8 @@ export interface TaskRoomServiceDeps {
|
|
|
106
106
|
launchDeletionWorker?(taskId: string): Promise<void>;
|
|
107
107
|
sleep?(ms: number): Promise<void>;
|
|
108
108
|
now?(): number;
|
|
109
|
+
/** Test seam: register a task room with the notification service before the Owner joins. */
|
|
110
|
+
associateRoomNotifications?(taskId: string, roomCid: string): Promise<void>;
|
|
109
111
|
}
|
|
110
112
|
/** Exact extraction of the previously CLI-owned task create/start behavior. */
|
|
111
113
|
export declare class TaskRoomApplicationService {
|
|
@@ -325,6 +327,11 @@ export declare class TaskRoomApplicationService {
|
|
|
325
327
|
};
|
|
326
328
|
taskId: string;
|
|
327
329
|
}): Promise<TaskProvisioningContinuationResult>;
|
|
330
|
+
/**
|
|
331
|
+
* Before the Owner can join, register the room as this task's with the notification service, so own-room message
|
|
332
|
+
* preferences apply to its first message. An unreachable service stops here and the saga retries from this step.
|
|
333
|
+
*/
|
|
334
|
+
private associateRoomNotifications;
|
|
328
335
|
private setOwnerRoomCommands;
|
|
329
336
|
private reconcileProvisioningOwner;
|
|
330
337
|
private acceptTerminal;
|
|
@@ -20,6 +20,7 @@ import { withFileLock } from '../atomic-file.js';
|
|
|
20
20
|
import { launchFleetWorker } from '../rooms-tasks/external-worker.js';
|
|
21
21
|
import { storedRoomLaunchPolicy, TASK_CANCELLABLE_STATES, TASK_TERMINAL_STATES } from '../rooms-tasks/types.js';
|
|
22
22
|
import { deriveTaskRoomName } from '../rooms-tasks/task-room-name.js';
|
|
23
|
+
import { associateRoomNotifications } from '../notifications/mission-ownership.js';
|
|
23
24
|
// Cowork 1.3.0+ stores this selector verbatim and matches future room commands.
|
|
24
25
|
const OWNER_ROOM_COMMANDS = ['*'];
|
|
25
26
|
/** Repairable terminal observation derived entirely from durable Task/Room state. */
|
|
@@ -705,6 +706,21 @@ export class TaskRoomApplicationService {
|
|
|
705
706
|
return { kind: 'provisioning_resume_failed', task, room, issues };
|
|
706
707
|
}
|
|
707
708
|
}
|
|
709
|
+
/**
|
|
710
|
+
* Before the Owner can join, register the room as this task's with the notification service, so own-room message
|
|
711
|
+
* preferences apply to its first message. An unreachable service stops here and the saga retries from this step.
|
|
712
|
+
*/
|
|
713
|
+
async associateRoomNotifications(room) {
|
|
714
|
+
if (!room.task_id || !room.room_identity_cid)
|
|
715
|
+
return;
|
|
716
|
+
try {
|
|
717
|
+
await (this.deps.associateRoomNotifications ?? associateRoomNotifications)(room.task_id, room.room_identity_cid);
|
|
718
|
+
}
|
|
719
|
+
catch (error) {
|
|
720
|
+
setSagaError(room.room_id, error instanceof Error ? error.message : String(error), `Start or repair the notification service, then run ours-fleet task start ${room.task_id}.`, 'waiting_cowork');
|
|
721
|
+
throw error;
|
|
722
|
+
}
|
|
723
|
+
}
|
|
708
724
|
async setOwnerRoomCommands(cowork, roomId, role) {
|
|
709
725
|
try {
|
|
710
726
|
await cowork.setRoleCommands(roomId, { role, commands: [...OWNER_ROOM_COMMANDS] });
|
|
@@ -748,6 +764,8 @@ export class TaskRoomApplicationService {
|
|
|
748
764
|
if (!existing && !cfg.ownerInvite)
|
|
749
765
|
throw new ConfigError('rooms.owner: configure public_invite or public_invite_file before continuing task provisioning');
|
|
750
766
|
await this.setOwnerRoomCommands(cowork, roomId, existing?.role ?? cfg.rooms.owner.role);
|
|
767
|
+
if (!existing)
|
|
768
|
+
await this.associateRoomNotifications(room);
|
|
751
769
|
const acceptedCid = existing?.identity_cid ?? (await cowork.acceptInvite(roomId, cfg.ownerInvite, {
|
|
752
770
|
role: cfg.rooms.owner.role, expected_cid: cfg.rooms.owner.expected_cid,
|
|
753
771
|
})).seat_cid;
|
|
@@ -1182,6 +1200,7 @@ export class TaskRoomApplicationService {
|
|
|
1182
1200
|
if (attachOwner) {
|
|
1183
1201
|
room = advanceSaga(room.room_id, 'attach_owner', 2);
|
|
1184
1202
|
await this.setOwnerRoomCommands(cowork, room.room_id, rooms.owner.role);
|
|
1203
|
+
await this.associateRoomNotifications(room);
|
|
1185
1204
|
try {
|
|
1186
1205
|
const accepted = await cowork.acceptInvite(room.room_id, cfg.ownerInvite, {
|
|
1187
1206
|
role: rooms.owner.role, expected_cid: rooms.owner.expected_cid,
|
package/dist/build-info.json
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
{
|
|
2
|
-
"version": "2.
|
|
3
|
-
"buildId": "
|
|
4
|
-
"commit": "
|
|
2
|
+
"version": "2.2.0",
|
|
3
|
+
"buildId": "31d6e72bc212",
|
|
4
|
+
"commit": "f70ffa3f08be7346d3be96e5380d11667d306df6",
|
|
5
5
|
"dirty": false,
|
|
6
|
-
"builtAt": "2026-10-
|
|
6
|
+
"builtAt": "2026-10-28T21:45:52.624Z",
|
|
7
7
|
"capabilities": [
|
|
8
8
|
"service.refresh-v1",
|
|
9
9
|
"managed-cli.codex-reasoning-effort-v1",
|
|
@@ -1,45 +1,35 @@
|
|
|
1
1
|
import { observeFleetResources } from '../application/resource-observer.js';
|
|
2
|
-
import { roomsDir } from '../rooms-tasks/room-state.js';
|
|
3
2
|
import { cleanupDir, drainNotificationTargetCleanup, notificationAgentId, hasPendingNotificationTargetCleanup } from './target-cleanup.js';
|
|
4
|
-
import { closeSync, existsSync, lstatSync, openSync, readSync,
|
|
3
|
+
import { closeSync, existsSync, lstatSync, openSync, readSync, readdirSync, statSync } from 'node:fs';
|
|
5
4
|
import { basename, join, relative, resolve, sep } from 'node:path';
|
|
6
5
|
import { createHash } from 'node:crypto';
|
|
7
6
|
import { NotificationOutbox } from './outbox.js';
|
|
7
|
+
import { readSessionOwnership, validOwnerTask } from '../session/session-ownership.js';
|
|
8
|
+
const MAX_BINDINGS = 16;
|
|
8
9
|
const BATCH_BYTES = 256 * 1024;
|
|
9
10
|
const TURN_ROLES = 4, TURN_MS = 8;
|
|
10
|
-
const RUNTIME_FILES = new Set(['.monitor-status', '.monitor-state.json', '.notify-cursor', '.session-events.jsonl', '.session-events.jsonl.1', '.subscription-usage.json']);
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
const
|
|
14
|
-
|
|
11
|
+
const RUNTIME_FILES = new Set(['.session-ownership.json', '.monitor-status', '.monitor-state.json', '.notify-cursor', '.session-events.jsonl', '.session-events.jsonl.1', '.subscription-usage.json']);
|
|
12
|
+
function bind(bindings, event) {
|
|
13
|
+
const payload = event.payload;
|
|
14
|
+
const taskId = payload?.taskId;
|
|
15
|
+
if (!(payload?.launchId === null || typeof payload?.launchId === 'string') || !(taskId === null || validOwnerTask(taskId))
|
|
16
|
+
|| (taskId !== null && payload?.launchId === null))
|
|
17
|
+
return;
|
|
18
|
+
bindings[event.sessionGeneration] = taskId;
|
|
19
|
+
const keys = Object.keys(bindings);
|
|
20
|
+
for (const key of keys.slice(0, Math.max(0, keys.length - MAX_BINDINGS)))
|
|
21
|
+
delete bindings[key];
|
|
22
|
+
}
|
|
23
|
+
/** Startup skips earlier history, where a live generation's binding may lie: seed from the durable sidecar, keyed by
|
|
24
|
+
* exact generation. An absent sidecar is legacy history (generations stay unknown); an unreadable or invalid one is
|
|
25
|
+
* unavailable evidence and returns undefined, so the caller holds its checkpoint instead of emitting unowned work. */
|
|
26
|
+
function seedBindings(roleDir) {
|
|
15
27
|
try {
|
|
16
|
-
|
|
17
|
-
}
|
|
18
|
-
catch (error) {
|
|
19
|
-
if (error.code === 'ENOENT')
|
|
20
|
-
return result;
|
|
21
|
-
throw error;
|
|
28
|
+
return Object.fromEntries(Object.entries(readSessionOwnership(roleDir)).map(([generation, owner]) => [generation, owner.taskId]));
|
|
22
29
|
}
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
if (!stat.isFile() || stat.isSymbolicLink() || stat.size > 1024 * 1024)
|
|
26
|
-
throw Error('Unreadable room association');
|
|
27
|
-
const room = JSON.parse(readFileSync(file, 'utf8'));
|
|
28
|
-
if (!room || typeof room.room_id !== 'string' || !Array.isArray(room.member_seats))
|
|
29
|
-
throw Error('Invalid room association');
|
|
30
|
-
if (room.task_id === undefined)
|
|
31
|
-
continue;
|
|
32
|
-
if (typeof room.task_id !== 'string' || !/^[A-Za-z0-9_-]{1,120}$/.test(room.task_id))
|
|
33
|
-
throw Error('Invalid task association');
|
|
34
|
-
for (const seat of room.member_seats) {
|
|
35
|
-
if (typeof seat.role_name !== 'string')
|
|
36
|
-
throw Error('Invalid member association');
|
|
37
|
-
if (result.has(seat.role_name) && result.get(seat.role_name) !== room.task_id)
|
|
38
|
-
throw Error('Ambiguous task association');
|
|
39
|
-
result.set(seat.role_name, room.task_id);
|
|
40
|
-
}
|
|
30
|
+
catch {
|
|
31
|
+
return undefined;
|
|
41
32
|
}
|
|
42
|
-
return result;
|
|
43
33
|
}
|
|
44
34
|
/** Read durable supervisor ledgers without restarting agents or attaching a
|
|
45
35
|
* conversation controller. Each tick reads at most 256 KiB per role, from its
|
|
@@ -56,7 +46,6 @@ export class FleetNotificationProducer {
|
|
|
56
46
|
dirty = new Set();
|
|
57
47
|
retryAt = new Map();
|
|
58
48
|
discoveryNeeded = true;
|
|
59
|
-
taskRoles;
|
|
60
49
|
stopped = false;
|
|
61
50
|
existing = new Map();
|
|
62
51
|
cleanup;
|
|
@@ -66,7 +55,7 @@ export class FleetNotificationProducer {
|
|
|
66
55
|
this.config = config;
|
|
67
56
|
this.warn = warn;
|
|
68
57
|
let healthy = true;
|
|
69
|
-
observeFleetResources({ retryWatchFailures: true, roots, paths: [...roots,
|
|
58
|
+
observeFleetResources({ retryWatchFailures: true, roots, paths: [...roots, cleanupDir()],
|
|
70
59
|
onInvalidated: path => this.accept(path) }, state => {
|
|
71
60
|
if (state.health !== 'healthy') {
|
|
72
61
|
healthy = false;
|
|
@@ -191,13 +180,6 @@ export class FleetNotificationProducer {
|
|
|
191
180
|
});
|
|
192
181
|
if (this.discoveryNeeded) {
|
|
193
182
|
this.discoveryNeeded = false;
|
|
194
|
-
try {
|
|
195
|
-
this.taskRoles = taskRoleAssociations();
|
|
196
|
-
}
|
|
197
|
-
catch {
|
|
198
|
-
this.taskRoles = undefined;
|
|
199
|
-
this.warn('Notification task association unavailable; delivering unclassified events');
|
|
200
|
-
}
|
|
201
183
|
for (const root of this.roots)
|
|
202
184
|
for (const name of this.names(root))
|
|
203
185
|
this.dirty.add(join(root, name));
|
|
@@ -242,10 +224,23 @@ export class FleetNotificationProducer {
|
|
|
242
224
|
const initial = this.existing.get(roleDir);
|
|
243
225
|
if (initial === null)
|
|
244
226
|
throw Error('Notification startup ledger boundary unavailable');
|
|
245
|
-
|
|
227
|
+
// History before the startup boundary is skipped, but the live generation's binding may lie in it:
|
|
228
|
+
// seed it from the role's durable ownership sidecar.
|
|
229
|
+
if (initial) {
|
|
230
|
+
const bindings = seedBindings(roleDir);
|
|
231
|
+
if (!bindings) {
|
|
232
|
+
this.warn('Notification ownership evidence unavailable; ledger admission held');
|
|
233
|
+
this.retryAt.set(roleDir, Date.now() + 30_000);
|
|
234
|
+
this.dirty.add(roleDir);
|
|
235
|
+
return;
|
|
236
|
+
}
|
|
237
|
+
outbox.saveCheckpoint({ ...initial, bindings });
|
|
238
|
+
}
|
|
239
|
+
else
|
|
240
|
+
outbox.saveCheckpoint({ segment: segments[0], offset: 0, prompts: [] });
|
|
246
241
|
}
|
|
247
242
|
const saved = outbox.checkpoint;
|
|
248
|
-
const current = { ...saved, prompts: [...saved.prompts], messages: { ...saved.messages } };
|
|
243
|
+
const current = { ...saved, prompts: [...saved.prompts], messages: { ...saved.messages }, bindings: { ...saved.bindings } };
|
|
249
244
|
let index = segments.indexOf(current.segment);
|
|
250
245
|
if (index < 0) {
|
|
251
246
|
this.warn('Notification ledger segment unavailable; checkpoint retained');
|
|
@@ -275,7 +270,7 @@ export class FleetNotificationProducer {
|
|
|
275
270
|
finally {
|
|
276
271
|
closeSync(fd);
|
|
277
272
|
}
|
|
278
|
-
let position = 0, blocked = false;
|
|
273
|
+
let position = 0, blocked = false, admitted = false, unavailable = false, sidecar;
|
|
279
274
|
while (position < count) {
|
|
280
275
|
const end = buffer.indexOf(10, position);
|
|
281
276
|
if (end < 0 || end >= count) {
|
|
@@ -298,6 +293,8 @@ export class FleetNotificationProducer {
|
|
|
298
293
|
if (event.schemaVersion !== 1 || event.roleId !== roleId)
|
|
299
294
|
throw new Error('invalid ledger event');
|
|
300
295
|
const prompt = `${event.sessionGeneration}:${event.promptId}`;
|
|
296
|
+
if (event.kind === 'session.ownership')
|
|
297
|
+
bind(current.bindings, event);
|
|
301
298
|
if (event.kind === 'prompt.admitted' && event.promptId && ['browser', 'owner_admin_console', 'owner_channel'].includes(event.source ?? '')) {
|
|
302
299
|
if (!current.prompts.includes(prompt))
|
|
303
300
|
current.prompts.push(prompt);
|
|
@@ -305,10 +302,25 @@ export class FleetNotificationProducer {
|
|
|
305
302
|
if (['message.chunk', 'message.replace'].includes(event.kind) && event.promptId && event.messageId && current.prompts.includes(prompt))
|
|
306
303
|
current.messages[prompt] = event.messageId;
|
|
307
304
|
if (event.kind === 'turn.completed' && event.promptId && current.prompts.includes(prompt)) {
|
|
308
|
-
|
|
305
|
+
// Task association only from this generation's own binding: a task, null (non-task role), or unknown (omitted).
|
|
306
|
+
// A generation bound before this checkpoint (an older checkpoint, or startup history) is recovered from the
|
|
307
|
+
// durable sidecar by exact generation; other generations are never relabelled.
|
|
308
|
+
// Unavailable sidecar evidence holds the checkpoint before this completion: nothing unowned is emitted.
|
|
309
|
+
if (!Object.hasOwn(current.bindings, event.sessionGeneration)) {
|
|
310
|
+
sidecar ??= seedBindings(roleDir);
|
|
311
|
+
if (!sidecar) {
|
|
312
|
+
unavailable = true;
|
|
313
|
+
break;
|
|
314
|
+
}
|
|
315
|
+
if (Object.hasOwn(sidecar, event.sessionGeneration))
|
|
316
|
+
current.bindings[event.sessionGeneration] = sidecar[event.sessionGeneration];
|
|
317
|
+
}
|
|
318
|
+
const binding = Object.hasOwn(current.bindings, event.sessionGeneration) ? { taskId: current.bindings[event.sessionGeneration] } : {};
|
|
319
|
+
if (!outbox.enqueue(`${roleId}:${event.sessionGeneration}:${event.seq}`, { ...binding, anchor: { kind: 'fleet-message', messageId: current.messages[prompt] ?? event.promptId } })) {
|
|
309
320
|
blocked = true;
|
|
310
321
|
break;
|
|
311
322
|
}
|
|
323
|
+
admitted = true;
|
|
312
324
|
current.prompts = current.prompts.filter(p => p !== prompt);
|
|
313
325
|
delete current.messages[prompt];
|
|
314
326
|
}
|
|
@@ -319,8 +331,14 @@ export class FleetNotificationProducer {
|
|
|
319
331
|
}
|
|
320
332
|
if (current.segment !== saved.segment || current.offset !== saved.offset)
|
|
321
333
|
outbox.saveCheckpoint(current);
|
|
322
|
-
|
|
323
|
-
|
|
334
|
+
// Prompt only after both completion and checkpoint have been durably recorded.
|
|
335
|
+
if (admitted)
|
|
336
|
+
outbox.prompt();
|
|
337
|
+
if (unavailable)
|
|
338
|
+
this.warn('Notification ownership evidence unavailable; ledger admission held');
|
|
339
|
+
if (blocked || unavailable) {
|
|
340
|
+
// Corrupt evidence needs an operator repair: re-check it slowly rather than spin.
|
|
341
|
+
this.retryAt.set(roleDir, Date.now() + (unavailable ? 30_000 : 1000));
|
|
324
342
|
this.dirty.add(roleDir);
|
|
325
343
|
}
|
|
326
344
|
else if ((current.segment !== saved.segment || current.offset !== saved.offset)
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { TaskRecord } from '../rooms-tasks/types.js';
|
|
2
|
+
import { type ProducerConfig } from './outbox.js';
|
|
3
|
+
/** The service predates mission ownership: it has no such route. Work is retained, never reported as done. */
|
|
4
|
+
export declare class MissionOwnershipUnsupported extends Error {
|
|
5
|
+
}
|
|
6
|
+
export declare class MissionOwnershipUnavailable extends Error {
|
|
7
|
+
}
|
|
8
|
+
export declare function missionRequest(config: ProducerConfig, path: string, body: unknown, request?: typeof fetch): Promise<{
|
|
9
|
+
status: number;
|
|
10
|
+
data: any;
|
|
11
|
+
}>;
|
|
12
|
+
/**
|
|
13
|
+
* Register a room as this task's before the Owner can join it: with own-room messages off, the service must know
|
|
14
|
+
* the room before the room can message the Owner. Unreachable service or unconfirmed coverage → throws, so room
|
|
15
|
+
* activation waits and retries. No producer configured → nothing to gate. An older service without the route →
|
|
16
|
+
* admitted with a warning (it cannot suppress own-room messages either).
|
|
17
|
+
*/
|
|
18
|
+
export declare function associateRoomNotifications(taskId: string, roomCid: string, options?: {
|
|
19
|
+
config?: ProducerConfig | undefined;
|
|
20
|
+
warn?: (line: string) => void;
|
|
21
|
+
request?: typeof fetch;
|
|
22
|
+
}): Promise<void>;
|
|
23
|
+
/** Retire every queued deleted task's notifications; an intent is removed only after the service acknowledged it. */
|
|
24
|
+
export declare function drainMissionCleanup(config: ProducerConfig, warn: (line: string) => void, request?: typeof fetch): Promise<boolean>;
|
|
25
|
+
/**
|
|
26
|
+
* Register every existing task's rooms under one numbered run, then mark the run complete. Until a run completes
|
|
27
|
+
* the service refuses own-room OFF, so no room that existed before this release can escape the preference.
|
|
28
|
+
* The inventory is read in full before the run starts: an unreadable task or layout throws, and the run never
|
|
29
|
+
* completes on partial evidence. Every task must be confirmed by the service before completion.
|
|
30
|
+
*/
|
|
31
|
+
export declare function bootstrapMissionRooms(config: ProducerConfig, tasks: () => TaskRecord[], layoutRooms: (task: TaskRecord) => string[], request?: typeof fetch): Promise<void>;
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
import { mkdirSync, readFileSync } from 'node:fs';
|
|
2
|
+
import { dirname, join } from 'node:path';
|
|
3
|
+
import { replaceFileAtomically } from '../atomic-file.js';
|
|
4
|
+
import { stateRoot } from '../paths.js';
|
|
5
|
+
import { canonicalCid, pendingMissionCleanups, settleMissionCleanup, unavailableMissionCleanups, validRoomCid } from '../rooms-tasks/notification-manifest.js';
|
|
6
|
+
import { advanceNotificationRevision, notificationRoomCids, recordNotificationRooms } from '../rooms-tasks/task-state.js';
|
|
7
|
+
import { producerConfig } from './outbox.js';
|
|
8
|
+
/** The service predates mission ownership: it has no such route. Work is retained, never reported as done. */
|
|
9
|
+
export class MissionOwnershipUnsupported extends Error {
|
|
10
|
+
}
|
|
11
|
+
export class MissionOwnershipUnavailable extends Error {
|
|
12
|
+
}
|
|
13
|
+
export async function missionRequest(config, path, body, request = fetch) {
|
|
14
|
+
let response;
|
|
15
|
+
try {
|
|
16
|
+
response = await request(config.origin + path, {
|
|
17
|
+
method: 'POST', redirect: 'error', signal: AbortSignal.timeout(10_000),
|
|
18
|
+
headers: { ...(config.gatewayCredential
|
|
19
|
+
? { 'X-Ours-Api-Token': config.gatewayCredential, 'X-Ours-Notifications-Producer': config.token }
|
|
20
|
+
: { Authorization: `Bearer ${config.token}` }), 'Content-Type': 'application/json' },
|
|
21
|
+
body: JSON.stringify(body),
|
|
22
|
+
});
|
|
23
|
+
}
|
|
24
|
+
catch {
|
|
25
|
+
throw new MissionOwnershipUnavailable('notification service unreachable');
|
|
26
|
+
}
|
|
27
|
+
const data = await response.json().catch(() => undefined);
|
|
28
|
+
if (response.status === 404)
|
|
29
|
+
throw new MissionOwnershipUnsupported('notification service does not support mission ownership');
|
|
30
|
+
return { status: response.status, data };
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Register the task's current manifest. Only a 2xx response whose `contactIds` include every requested room proves
|
|
34
|
+
* coverage. A 409 (mission deleted, stale or conflicting revision) proves nothing about these rooms: the local revision
|
|
35
|
+
* is moved past the refused one, so a later retry registers the full union under a newer revision, and this attempt fails.
|
|
36
|
+
*/
|
|
37
|
+
async function registerMission(config, taskId, manifest, request) {
|
|
38
|
+
const { status, data } = await missionRequest(config, '/api/v1/missions/associate', { taskId, revision: manifest.revision, contactIds: manifest.room_cids }, request);
|
|
39
|
+
if (status >= 200 && status < 300) {
|
|
40
|
+
const registered = Array.isArray(data?.contactIds) ? new Set(data.contactIds.filter(validRoomCid).map(canonicalCid)) : undefined;
|
|
41
|
+
if (registered && manifest.room_cids.every(cid => registered.has(cid)))
|
|
42
|
+
return;
|
|
43
|
+
throw new MissionOwnershipUnavailable('notification service did not confirm every mission room');
|
|
44
|
+
}
|
|
45
|
+
if (status === 409)
|
|
46
|
+
advanceNotificationRevision(taskId, manifest.revision);
|
|
47
|
+
throw new MissionOwnershipUnavailable(`notification service refused room registration (HTTP ${status})`);
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Register a room as this task's before the Owner can join it: with own-room messages off, the service must know
|
|
51
|
+
* the room before the room can message the Owner. Unreachable service or unconfirmed coverage → throws, so room
|
|
52
|
+
* activation waits and retries. No producer configured → nothing to gate. An older service without the route →
|
|
53
|
+
* admitted with a warning (it cannot suppress own-room messages either).
|
|
54
|
+
*/
|
|
55
|
+
export async function associateRoomNotifications(taskId, roomCid, options = {}) {
|
|
56
|
+
const manifest = recordNotificationRooms(taskId, [roomCid]);
|
|
57
|
+
const config = 'config' in options ? options.config : producerConfig();
|
|
58
|
+
if (!config)
|
|
59
|
+
return;
|
|
60
|
+
try {
|
|
61
|
+
await registerMission(config, taskId, manifest, options.request);
|
|
62
|
+
}
|
|
63
|
+
catch (error) {
|
|
64
|
+
if (error instanceof MissionOwnershipUnsupported) {
|
|
65
|
+
options.warn?.('Notification service does not support mission rooms; own-room message preferences are unavailable');
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
if (error instanceof MissionOwnershipUnavailable)
|
|
69
|
+
throw new Error('Waiting for notification service to register the mission room');
|
|
70
|
+
throw error;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
/** Retire every queued deleted task's notifications; an intent is removed only after the service acknowledged it. */
|
|
74
|
+
export async function drainMissionCleanup(config, warn, request = fetch) {
|
|
75
|
+
let pending = false;
|
|
76
|
+
for (const intent of pendingMissionCleanups().slice(0, 32)) {
|
|
77
|
+
try {
|
|
78
|
+
const { status } = await missionRequest(config, '/api/v1/missions/delete', { taskId: intent.taskId, contactIds: intent.contactIds }, request);
|
|
79
|
+
if (status >= 200 && status < 300) {
|
|
80
|
+
if (!settleMissionCleanup(intent))
|
|
81
|
+
pending = true;
|
|
82
|
+
continue;
|
|
83
|
+
}
|
|
84
|
+
warn(`Mission notification cleanup pending (HTTP ${status}); intent retained`);
|
|
85
|
+
}
|
|
86
|
+
catch (error) {
|
|
87
|
+
warn(error instanceof MissionOwnershipUnsupported
|
|
88
|
+
? 'Notification service does not support mission cleanup; cleanup pending'
|
|
89
|
+
: 'Mission notification cleanup pending; intent retained');
|
|
90
|
+
}
|
|
91
|
+
return true;
|
|
92
|
+
}
|
|
93
|
+
// An unreadable intent stays pending (and its file retained): overall cleanup is never reported done around it.
|
|
94
|
+
const unavailable = unavailableMissionCleanups();
|
|
95
|
+
if (unavailable.length)
|
|
96
|
+
warn(`Mission notification cleanup unavailable for ${unavailable.length} unreadable intent(s); files retained`);
|
|
97
|
+
return pending || unavailable.length > 0 || pendingMissionCleanups().length > 32;
|
|
98
|
+
}
|
|
99
|
+
const bootstrapFile = () => join(stateRoot(), 'notifications', 'mission-bootstrap.json');
|
|
100
|
+
/** Only a missing file starts at 0. Unreadable, corrupt or out-of-range state is refused and its bytes are kept. */
|
|
101
|
+
function nextGeneration() {
|
|
102
|
+
let text;
|
|
103
|
+
try {
|
|
104
|
+
text = readFileSync(bootstrapFile(), 'utf8');
|
|
105
|
+
}
|
|
106
|
+
catch (error) {
|
|
107
|
+
if (error.code !== 'ENOENT')
|
|
108
|
+
throw new MissionOwnershipUnavailable('mission bootstrap state unreadable');
|
|
109
|
+
}
|
|
110
|
+
let generation = 0;
|
|
111
|
+
if (text !== undefined) {
|
|
112
|
+
let value;
|
|
113
|
+
try {
|
|
114
|
+
value = JSON.parse(text);
|
|
115
|
+
}
|
|
116
|
+
catch {
|
|
117
|
+
throw new MissionOwnershipUnavailable('mission bootstrap state corrupt; file retained');
|
|
118
|
+
}
|
|
119
|
+
const stored = value && typeof value === 'object' && !Array.isArray(value) ? value.generation : undefined;
|
|
120
|
+
if (typeof stored !== 'number' || !Number.isSafeInteger(stored) || stored < 0)
|
|
121
|
+
throw new MissionOwnershipUnavailable('mission bootstrap state invalid; file retained');
|
|
122
|
+
generation = stored;
|
|
123
|
+
}
|
|
124
|
+
if (!Number.isSafeInteger(generation + 1))
|
|
125
|
+
throw new MissionOwnershipUnavailable('mission bootstrap generation exhausted');
|
|
126
|
+
mkdirSync(dirname(bootstrapFile()), { recursive: true, mode: 0o700 });
|
|
127
|
+
replaceFileAtomically(bootstrapFile(), JSON.stringify({ generation: generation + 1 }) + '\n', 0o600);
|
|
128
|
+
return generation + 1;
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Register every existing task's rooms under one numbered run, then mark the run complete. Until a run completes
|
|
132
|
+
* the service refuses own-room OFF, so no room that existed before this release can escape the preference.
|
|
133
|
+
* The inventory is read in full before the run starts: an unreadable task or layout throws, and the run never
|
|
134
|
+
* completes on partial evidence. Every task must be confirmed by the service before completion.
|
|
135
|
+
*/
|
|
136
|
+
export async function bootstrapMissionRooms(config, tasks, layoutRooms, request = fetch) {
|
|
137
|
+
const inventory = tasks().filter(task => task.deletion?.status !== 'pending')
|
|
138
|
+
.map(task => ({ task, cids: [...notificationRoomCids(task), ...layoutRooms(task)] })).filter(entry => entry.cids.length);
|
|
139
|
+
const generation = nextGeneration();
|
|
140
|
+
const begin = await missionRequest(config, '/api/v1/missions/bootstrap', { generation, complete: false }, request);
|
|
141
|
+
if (begin.status < 200 || begin.status >= 300)
|
|
142
|
+
throw new MissionOwnershipUnavailable(`bootstrap refused (HTTP ${begin.status})`);
|
|
143
|
+
for (const { task, cids } of inventory)
|
|
144
|
+
await registerMission(config, task.task_id, recordNotificationRooms(task.task_id, cids), request);
|
|
145
|
+
const done = await missionRequest(config, '/api/v1/missions/bootstrap', { generation, complete: true }, request);
|
|
146
|
+
if (done.status < 200 || done.status >= 300)
|
|
147
|
+
throw new MissionOwnershipUnavailable(`bootstrap completion refused (HTTP ${done.status})`);
|
|
148
|
+
}
|
|
@@ -44,12 +44,22 @@ export declare class NotificationOutbox {
|
|
|
44
44
|
checkpoint?: unknown;
|
|
45
45
|
private running?;
|
|
46
46
|
private retired;
|
|
47
|
+
private again;
|
|
48
|
+
private failed;
|
|
49
|
+
private closed;
|
|
47
50
|
private timer;
|
|
48
51
|
constructor(file: string, config: ProducerConfig, project: (value: unknown, id: string) => Promise<NotificationInput>, warn?: (message: string) => void);
|
|
49
52
|
private persist;
|
|
50
53
|
advance(cursor: number): void;
|
|
51
54
|
saveCheckpoint(checkpoint: unknown): void;
|
|
52
55
|
enqueue(id: string, value: unknown, cursor?: number | undefined): boolean;
|
|
56
|
+
/**
|
|
57
|
+
* Deliver soon after a durable enqueue instead of at the next recovery tick. A request
|
|
58
|
+
* during a pass (which iterates its own snapshot) runs one follow-up pass after it;
|
|
59
|
+
* after a failed pass only the recovery timer retries, so arrivals during an outage
|
|
60
|
+
* add no attempts.
|
|
61
|
+
*/
|
|
62
|
+
prompt(): void;
|
|
53
63
|
drain(): Promise<void>;
|
|
54
64
|
private drainOnce;
|
|
55
65
|
/** Wait for the current send before clearing queued sends and retiring its target. */
|
|
@@ -99,6 +99,9 @@ export class NotificationOutbox {
|
|
|
99
99
|
checkpoint;
|
|
100
100
|
running;
|
|
101
101
|
retired = false;
|
|
102
|
+
again = false;
|
|
103
|
+
failed = false;
|
|
104
|
+
closed = false;
|
|
102
105
|
timer;
|
|
103
106
|
constructor(file, config, project, warn = () => { }) {
|
|
104
107
|
this.file = file;
|
|
@@ -152,12 +155,34 @@ export class NotificationOutbox {
|
|
|
152
155
|
this.persist([...this.entries, { id, value, targetVersion: 1 }], cursor);
|
|
153
156
|
return true;
|
|
154
157
|
}
|
|
158
|
+
/**
|
|
159
|
+
* Deliver soon after a durable enqueue instead of at the next recovery tick. A request
|
|
160
|
+
* during a pass (which iterates its own snapshot) runs one follow-up pass after it;
|
|
161
|
+
* after a failed pass only the recovery timer retries, so arrivals during an outage
|
|
162
|
+
* add no attempts.
|
|
163
|
+
*/
|
|
164
|
+
prompt() {
|
|
165
|
+
if (this.closed || this.retired || this.failed)
|
|
166
|
+
return;
|
|
167
|
+
if (this.running) {
|
|
168
|
+
this.again = true;
|
|
169
|
+
return;
|
|
170
|
+
}
|
|
171
|
+
void this.drain();
|
|
172
|
+
}
|
|
155
173
|
drain() {
|
|
156
174
|
if (this.retired)
|
|
157
175
|
return Promise.resolve();
|
|
158
|
-
return this.running ??= this.drainOnce().finally(() => {
|
|
176
|
+
return this.running ??= this.drainOnce().finally(() => {
|
|
177
|
+
this.running = undefined;
|
|
178
|
+
if (this.again) {
|
|
179
|
+
this.again = false;
|
|
180
|
+
this.prompt();
|
|
181
|
+
}
|
|
182
|
+
});
|
|
159
183
|
}
|
|
160
184
|
async drainOnce() {
|
|
185
|
+
this.failed = false;
|
|
161
186
|
let attempts = 0;
|
|
162
187
|
for (const entry of [...this.entries]) {
|
|
163
188
|
if (!supported(entry))
|
|
@@ -188,6 +213,7 @@ export class NotificationOutbox {
|
|
|
188
213
|
this.persist(this.entries.filter(e => !same(entry.id)(e)));
|
|
189
214
|
}
|
|
190
215
|
catch {
|
|
216
|
+
this.failed = true;
|
|
191
217
|
this.warn('Notification producer delivery pending; retrying from durable outbox');
|
|
192
218
|
break;
|
|
193
219
|
}
|
|
@@ -195,5 +221,5 @@ export class NotificationOutbox {
|
|
|
195
221
|
}
|
|
196
222
|
/** Wait for the current send before clearing queued sends and retiring its target. */
|
|
197
223
|
async retire() { this.retired = true; clearInterval(this.timer); await this.running; this.persist(this.entries.filter(entry => !supported(entry))); }
|
|
198
|
-
async close() { clearInterval(this.timer); await this.running; }
|
|
224
|
+
async close() { this.closed = true; clearInterval(this.timer); await this.running; }
|
|
199
225
|
}
|