@stratusagent/gateway 0.13.7 → 0.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/event-intake.d.ts +159 -0
- package/dist/event-intake.d.ts.map +1 -0
- package/dist/event-intake.js +445 -0
- package/dist/event-intake.js.map +1 -0
- package/dist/events.d.ts +256 -0
- package/dist/events.d.ts.map +1 -0
- package/dist/events.js +462 -0
- package/dist/events.js.map +1 -0
- package/dist/index.d.ts +108 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +451 -24
- package/dist/index.js.map +1 -1
- package/dist/schedules.d.ts +84 -1
- package/dist/schedules.d.ts.map +1 -1
- package/dist/schedules.js +550 -49
- package/dist/schedules.js.map +1 -1
- package/dist/sessions.d.ts +15 -7
- package/dist/sessions.d.ts.map +1 -1
- package/dist/sessions.js +89 -19
- package/dist/sessions.js.map +1 -1
- package/package.json +7 -7
package/dist/schedules.js
CHANGED
|
@@ -2,38 +2,52 @@ import { randomUUID } from 'node:crypto';
|
|
|
2
2
|
import { chmodSync, mkdirSync } from 'node:fs';
|
|
3
3
|
import path from 'node:path';
|
|
4
4
|
import { DatabaseSync } from 'node:sqlite';
|
|
5
|
-
import { SENDER_TRUST_METADATA_KEY } from '@stratusagent/core';
|
|
6
|
-
import { canonicalDestination, nextFireAfter, SCHEDULE_ID_METADATA_KEY, SCHEDULED_TURN_METADATA_KEY, } from '@stratusagent/agents';
|
|
5
|
+
import { leastTrusted, SENDER_TRUST_METADATA_KEY, } from '@stratusagent/core';
|
|
6
|
+
import { canonicalDestination, DEFAULT_EVENT_MAX_PER_HOUR, isWithinQuietHours, nextFireAfter, SCHEDULE_ID_METADATA_KEY, SCHEDULED_TURN_METADATA_KEY, } from '@stratusagent/agents';
|
|
7
7
|
import { assertDerivedStatePathSync } from '@stratusagent/state';
|
|
8
|
+
import { tightenSqliteFile } from "./sessions.js";
|
|
9
|
+
/**
|
|
10
|
+
* Open `fleet.db` the way every store sharing it must: the derived-path
|
|
11
|
+
* check, an owner-only directory and file, WAL with a busy timeout, then
|
|
12
|
+
* `schema`, then the sidecars tightened. One implementation, because the
|
|
13
|
+
* schedule store and the event store (card 590) each hold their own
|
|
14
|
+
* connection to this file, and a second copy of this sequence is the one
|
|
15
|
+
* that drifts.
|
|
16
|
+
*/
|
|
17
|
+
export const openFleetDatabase = (filePath, options, schema) => {
|
|
18
|
+
if (options.stateHome !== undefined) {
|
|
19
|
+
assertDerivedStatePathSync(options.stateHome, filePath, 'file');
|
|
20
|
+
}
|
|
21
|
+
// Same posture as the session store sharing this file: the directory
|
|
22
|
+
// and the database must not be readable by other local users, and the
|
|
23
|
+
// sidecar chmods cover databases created under a looser umask.
|
|
24
|
+
mkdirSync(path.dirname(filePath), { recursive: true, mode: 0o700 });
|
|
25
|
+
const db = new DatabaseSync(filePath);
|
|
26
|
+
try {
|
|
27
|
+
chmodSync(filePath, 0o600);
|
|
28
|
+
}
|
|
29
|
+
catch (error) {
|
|
30
|
+
if (error.code !== 'ENOENT') {
|
|
31
|
+
throw error;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
db.exec('PRAGMA journal_mode = WAL');
|
|
35
|
+
// WAL serializes writers to one at a time across the whole file, and
|
|
36
|
+
// this database is shared between two processes: the daemon claims
|
|
37
|
+
// slots and saves sessions while `stratus schedules cancel` deletes a
|
|
38
|
+
// row from another connection. Without a busy timeout the loser of that
|
|
39
|
+
// lock throws `database is locked` immediately — a cancel that silently
|
|
40
|
+
// failed would leave the schedule (and its destination grant) live.
|
|
41
|
+
// Wait for the writer instead.
|
|
42
|
+
db.exec('PRAGMA busy_timeout = 5000');
|
|
43
|
+
db.exec(schema);
|
|
44
|
+
tightenSqliteFile(filePath);
|
|
45
|
+
return db;
|
|
46
|
+
};
|
|
8
47
|
export class SqliteScheduleStore {
|
|
9
48
|
db;
|
|
10
49
|
constructor(filePath, options = {}) {
|
|
11
|
-
|
|
12
|
-
assertDerivedStatePathSync(options.stateHome, filePath, 'file');
|
|
13
|
-
}
|
|
14
|
-
// Same posture as the session store sharing this file: the directory
|
|
15
|
-
// and the database must not be readable by other local users, and the
|
|
16
|
-
// sidecar chmods cover databases created under a looser umask.
|
|
17
|
-
mkdirSync(path.dirname(filePath), { recursive: true, mode: 0o700 });
|
|
18
|
-
this.db = new DatabaseSync(filePath);
|
|
19
|
-
try {
|
|
20
|
-
chmodSync(filePath, 0o600);
|
|
21
|
-
}
|
|
22
|
-
catch (error) {
|
|
23
|
-
if (error.code !== 'ENOENT') {
|
|
24
|
-
throw error;
|
|
25
|
-
}
|
|
26
|
-
}
|
|
27
|
-
this.db.exec('PRAGMA journal_mode = WAL');
|
|
28
|
-
// WAL serializes writers to one at a time across the whole file, and
|
|
29
|
-
// this database is shared between two processes: the daemon claims
|
|
30
|
-
// slots and saves sessions while `stratus schedules cancel` deletes a
|
|
31
|
-
// row from another connection. Without a busy timeout the loser of that
|
|
32
|
-
// lock throws `database is locked` immediately — a cancel that silently
|
|
33
|
-
// failed would leave the schedule (and its destination grant) live.
|
|
34
|
-
// Wait for the writer instead.
|
|
35
|
-
this.db.exec('PRAGMA busy_timeout = 5000');
|
|
36
|
-
this.db.exec(`
|
|
50
|
+
this.db = openFleetDatabase(filePath, options, `
|
|
37
51
|
CREATE TABLE IF NOT EXISTS schedules (
|
|
38
52
|
id TEXT PRIMARY KEY,
|
|
39
53
|
agent_id TEXT NOT NULL,
|
|
@@ -42,16 +56,6 @@ export class SqliteScheduleStore {
|
|
|
42
56
|
created_at TEXT NOT NULL
|
|
43
57
|
)
|
|
44
58
|
`);
|
|
45
|
-
for (const sensitive of [filePath, `${filePath}-wal`, `${filePath}-shm`, `${filePath}-journal`]) {
|
|
46
|
-
try {
|
|
47
|
-
chmodSync(sensitive, 0o600);
|
|
48
|
-
}
|
|
49
|
-
catch (error) {
|
|
50
|
-
if (error.code !== 'ENOENT') {
|
|
51
|
-
throw error;
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
59
|
}
|
|
56
60
|
insert(record) {
|
|
57
61
|
this.db
|
|
@@ -115,11 +119,67 @@ export class SqliteScheduleStore {
|
|
|
115
119
|
export const SCHEDULE_SESSION_ID_PREFIX = 'schedule:';
|
|
116
120
|
/** Whether a session id belongs to the reserved scheduled-firing namespace. */
|
|
117
121
|
export const isScheduleSessionId = (sessionId) => sessionId.startsWith(SCHEDULE_SESSION_ID_PREFIX);
|
|
122
|
+
/** The longest prefix of `text` that fits in `bytes` of UTF-8, never splitting a character. */
|
|
123
|
+
export const utf8Prefix = (text, bytes) => {
|
|
124
|
+
const { read } = new TextEncoder().encodeInto(text, new Uint8Array(Math.max(0, bytes)));
|
|
125
|
+
return text.slice(0, read);
|
|
126
|
+
};
|
|
127
|
+
/** A resumed firing carries at most this much of the stopped run's summary. */
|
|
128
|
+
export const RESUME_SUMMARY_MAX_BYTES = 4_096;
|
|
129
|
+
const STOPPED_EARLY_WHY = {
|
|
130
|
+
'turn-limit': 'it used all its steps',
|
|
131
|
+
'repeated-failure': 'it kept hitting the same failure',
|
|
132
|
+
};
|
|
133
|
+
const ordinal = (n) => {
|
|
134
|
+
const tens = n % 100;
|
|
135
|
+
const suffix = tens >= 11 && tens <= 13 ? 'th' : (['th', 'st', 'nd', 'rd'][n % 10] ?? 'th');
|
|
136
|
+
return `${n}${suffix}`;
|
|
137
|
+
};
|
|
138
|
+
/**
|
|
139
|
+
* What a firing that resumes sees after its schedule's prompt: when the
|
|
140
|
+
* stopped run ended and why, its own summary of what was done and left
|
|
141
|
+
* (capped), and, from the second stop in a row, that it's a repeat. The
|
|
142
|
+
* runtime posts nothing about a repeat: only the agent knows whether a
|
|
143
|
+
* person is waiting, so it's told it may post one line to the destination
|
|
144
|
+
* its schedule was approved with, when there is one.
|
|
145
|
+
*/
|
|
146
|
+
export const renderResumeBlock = (stoppedEarly, reply, streak, reportsSomewhere) => {
|
|
147
|
+
const text = reply?.trim() ?? '';
|
|
148
|
+
const summary = text === ''
|
|
149
|
+
? 'It left no summary.'
|
|
150
|
+
: Buffer.byteLength(text) <= RESUME_SUMMARY_MAX_BYTES
|
|
151
|
+
? text
|
|
152
|
+
: `${utf8Prefix(text, RESUME_SUMMARY_MAX_BYTES)}\n[summary cut at ${Math.round(RESUME_SUMMARY_MAX_BYTES / 1_024)} KB]`;
|
|
153
|
+
const when = typeof stoppedEarly.at === 'string' ? `, at ${stoppedEarly.at},` : '';
|
|
154
|
+
const why = STOPPED_EARLY_WHY[stoppedEarly.reason] ?? 'it stopped before finishing';
|
|
155
|
+
const lines = [
|
|
156
|
+
`Your last run of this schedule${when} stopped early: ${why}. What it said was done and what was left:`,
|
|
157
|
+
'',
|
|
158
|
+
summary,
|
|
159
|
+
'',
|
|
160
|
+
"Carry on from there, and don't redo finished work.",
|
|
161
|
+
];
|
|
162
|
+
if (streak >= 2) {
|
|
163
|
+
lines.push('', `That was the ${ordinal(streak)} run in a row to stop early.`
|
|
164
|
+
+ (reportsSomewhere ? " If that blocks someone, post one line to this schedule's report destination saying what's stuck." : ''));
|
|
165
|
+
}
|
|
166
|
+
return lines.join('\n');
|
|
167
|
+
};
|
|
118
168
|
const DEFAULT_MIN_INTERVAL_MS = 60_000;
|
|
119
169
|
const DEFAULT_MAX_CONCURRENT_PER_AGENT = 1;
|
|
120
170
|
const DEFAULT_TICK_MS = 1_000;
|
|
171
|
+
/** A held event firing older than this expires, so a long pause never ends in a burst. */
|
|
172
|
+
const EVENT_HOLD_MAX_MS = 24 * 60 * 60 * 1_000;
|
|
173
|
+
/** Held firings kept per coalescing key; older ones are superseded. */
|
|
174
|
+
const EVENT_HELD_PER_KEY = 5;
|
|
175
|
+
/** How often the event tables are pruned to their 7-day window and row cap. */
|
|
176
|
+
const EVENT_PRUNE_INTERVAL_MS = 60 * 60 * 1_000;
|
|
121
177
|
export const createSchedulerRuntime = (options) => {
|
|
122
178
|
const { store, dispatch, validateDestination, sessionStatus, hasAgent, log, warn } = options;
|
|
179
|
+
// Late-bindable once: see `attachEvents`.
|
|
180
|
+
let events = options.events;
|
|
181
|
+
const clock = options.now ?? (() => new Date());
|
|
182
|
+
let maxEventsPerAgent = events?.maxConcurrentPerAgent ?? 1;
|
|
123
183
|
const minIntervalMs = options.limits?.minIntervalMs ?? DEFAULT_MIN_INTERVAL_MS;
|
|
124
184
|
const maxConcurrentPerAgent = options.limits?.maxConcurrentPerAgent ?? DEFAULT_MAX_CONCURRENT_PER_AGENT;
|
|
125
185
|
const tickMs = options.limits?.tickMs ?? DEFAULT_TICK_MS;
|
|
@@ -171,11 +231,48 @@ export const createSchedulerRuntime = (options) => {
|
|
|
171
231
|
}
|
|
172
232
|
}
|
|
173
233
|
};
|
|
234
|
+
/**
|
|
235
|
+
* An event trigger: no slot, so the due scan never sees it; it fires when
|
|
236
|
+
* a delivery matches. Refused for a source the operator hasn't configured,
|
|
237
|
+
* now, while the person approving it can hear why.
|
|
238
|
+
*/
|
|
239
|
+
const createTrigger = async (input) => {
|
|
240
|
+
if (input.cadence.kind !== 'event') {
|
|
241
|
+
throw new Error('createTrigger takes an event cadence.');
|
|
242
|
+
}
|
|
243
|
+
if (!events) {
|
|
244
|
+
throw new Error('This daemon takes no events: its config has no "events" sources and no plugin feeds one, so a trigger could never fire. The operator adds a source first.');
|
|
245
|
+
}
|
|
246
|
+
const configured = events.sources();
|
|
247
|
+
if (!configured.includes(input.cadence.source)) {
|
|
248
|
+
throw new Error(`No event source "${input.cadence.source}" is configured on this daemon. `
|
|
249
|
+
+ (configured.length > 0 ? `Configured: ${configured.join(', ')}.` : 'None is.'));
|
|
250
|
+
}
|
|
251
|
+
if (input.destination) {
|
|
252
|
+
await validateDestination(input.agentId, input.destination);
|
|
253
|
+
}
|
|
254
|
+
const record = {
|
|
255
|
+
id: randomUUID(),
|
|
256
|
+
agentId: input.agentId,
|
|
257
|
+
cadence: input.cadence,
|
|
258
|
+
prompt: input.prompt,
|
|
259
|
+
...(input.destination ? { destination: input.destination } : {}),
|
|
260
|
+
createdAt: clock().toISOString(),
|
|
261
|
+
...(input.createdBy ? { createdBy: input.createdBy } : {}),
|
|
262
|
+
...(input.trust ? { trust: input.trust } : {}),
|
|
263
|
+
};
|
|
264
|
+
store.insert(record);
|
|
265
|
+
log(`event trigger ${record.id} created for ${record.agentId} on source ${input.cadence.source}${record.destination ? `, reports to ${canonicalDestination(record.destination)}` : ''}`);
|
|
266
|
+
return record;
|
|
267
|
+
};
|
|
174
268
|
const handle = {
|
|
175
269
|
async create(input) {
|
|
176
270
|
if (!hasAgent(input.agentId)) {
|
|
177
271
|
throw new Error(`Agent not found: ${input.agentId}`);
|
|
178
272
|
}
|
|
273
|
+
if (input.cadence.kind === 'event') {
|
|
274
|
+
return createTrigger(input);
|
|
275
|
+
}
|
|
179
276
|
enforceFloor(input);
|
|
180
277
|
const now = new Date();
|
|
181
278
|
const next = nextFireAfter(input.cadence, now);
|
|
@@ -215,6 +312,7 @@ export const createSchedulerRuntime = (options) => {
|
|
|
215
312
|
const deleted = store.delete(scheduleId);
|
|
216
313
|
if (deleted) {
|
|
217
314
|
log(`schedule ${scheduleId} cancelled by ${agentId}`);
|
|
315
|
+
expireTriggerFirings(scheduleId);
|
|
218
316
|
}
|
|
219
317
|
return deleted;
|
|
220
318
|
},
|
|
@@ -240,6 +338,128 @@ export const createSchedulerRuntime = (options) => {
|
|
|
240
338
|
const { nextFireAt: _spent, ...rest } = record;
|
|
241
339
|
return { ...rest, ...extra, ...(next ? { nextFireAt: next.toISOString() } : {}) };
|
|
242
340
|
};
|
|
341
|
+
/**
|
|
342
|
+
* Card 858: a readiness check that failed after an await inside a scan.
|
|
343
|
+
* Carries the reason out to `tick`, which treats it like the check at the
|
|
344
|
+
* top of the tick: the scheduler stops, and at start the failure is
|
|
345
|
+
* `start()`'s rejection.
|
|
346
|
+
*/
|
|
347
|
+
class NotReady extends Error {
|
|
348
|
+
scheduleId;
|
|
349
|
+
reason;
|
|
350
|
+
constructor(scheduleId, reason) {
|
|
351
|
+
super(reason instanceof Error ? reason.message : String(reason));
|
|
352
|
+
this.scheduleId = scheduleId;
|
|
353
|
+
this.reason = reason;
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
const stillReady = async (scheduleId) => {
|
|
357
|
+
if (!options.ready) {
|
|
358
|
+
return;
|
|
359
|
+
}
|
|
360
|
+
try {
|
|
361
|
+
await options.ready();
|
|
362
|
+
}
|
|
363
|
+
catch (error) {
|
|
364
|
+
stopped = true;
|
|
365
|
+
throw new NotReady(scheduleId, error);
|
|
366
|
+
}
|
|
367
|
+
};
|
|
368
|
+
/**
|
|
369
|
+
* What the firing about to be claimed carries from the one in
|
|
370
|
+
* `lastSessionId` (card 858): `'defer'` while that run is still going (a
|
|
371
|
+
* schedule never claims past a run whose outcome it hasn't read, or that
|
|
372
|
+
* run's summary is lost and the streak breaks), otherwise the new streak
|
|
373
|
+
* and the stopped run to resume from, if any. Undefined when the daemon
|
|
374
|
+
* gave no `previousRun`: the claim then leaves both fields as they are.
|
|
375
|
+
*/
|
|
376
|
+
const planResume = async (record, last, slot) => {
|
|
377
|
+
const deferKey = `${record.id}:${slot}:previous`;
|
|
378
|
+
const holdBack = () => {
|
|
379
|
+
if (!deferralsWarned.has(deferKey)) {
|
|
380
|
+
deferralsWarned.add(deferKey);
|
|
381
|
+
warn(`schedule ${record.id}: its previous run ${last} is still going; deferring this firing (slot ${slot})`);
|
|
382
|
+
}
|
|
383
|
+
return 'defer';
|
|
384
|
+
};
|
|
385
|
+
// Started by this daemon and not settled: its session may not exist yet.
|
|
386
|
+
if (activeFirings.has(last)) {
|
|
387
|
+
return holdBack();
|
|
388
|
+
}
|
|
389
|
+
const read = options.previousRun;
|
|
390
|
+
if (!read) {
|
|
391
|
+
return undefined;
|
|
392
|
+
}
|
|
393
|
+
const streak = record.stoppedEarlyStreak ?? 0;
|
|
394
|
+
const readRun = async (sessionId) => {
|
|
395
|
+
try {
|
|
396
|
+
return await read(sessionId);
|
|
397
|
+
}
|
|
398
|
+
catch (error) {
|
|
399
|
+
warn(`schedule ${record.id}: could not read its previous run ${sessionId} (${error instanceof Error ? error.message : String(error)}); firing without it`);
|
|
400
|
+
return 'unreadable';
|
|
401
|
+
}
|
|
402
|
+
};
|
|
403
|
+
const previous = await readRun(last);
|
|
404
|
+
if (previous === 'unreadable') {
|
|
405
|
+
// No outcome to count: the streak stays, and there's no block.
|
|
406
|
+
return { streak };
|
|
407
|
+
}
|
|
408
|
+
if (previous !== undefined) {
|
|
409
|
+
if (previous.status === 'running' || previous.status === 'pending_approval') {
|
|
410
|
+
return holdBack();
|
|
411
|
+
}
|
|
412
|
+
deferralsWarned.delete(deferKey);
|
|
413
|
+
return previous.stoppedEarly !== undefined
|
|
414
|
+
? { streak: streak + 1, from: { sessionId: last, run: previous } }
|
|
415
|
+
: { streak: 0 };
|
|
416
|
+
}
|
|
417
|
+
deferralsWarned.delete(deferKey);
|
|
418
|
+
const fallback = record.resumeFrom;
|
|
419
|
+
if (fallback !== undefined && fallback.firing === last) {
|
|
420
|
+
// That firing was claimed and never started: the daemon died before
|
|
421
|
+
// its session existed, or refused it on the way down. It has no
|
|
422
|
+
// outcome, so the streak stays, and it carried a checkpoint, which
|
|
423
|
+
// this firing carries instead.
|
|
424
|
+
const from = await readRun(fallback.from);
|
|
425
|
+
if (from === 'unreadable') {
|
|
426
|
+
return { streak };
|
|
427
|
+
}
|
|
428
|
+
if (from?.stoppedEarly !== undefined) {
|
|
429
|
+
return { streak, from: { sessionId: fallback.from, run: from } };
|
|
430
|
+
}
|
|
431
|
+
log(`schedule ${record.id}: previous run ${last} never started, and the run it resumed from (${fallback.from}) is gone; starting fresh`);
|
|
432
|
+
return { streak: 0 };
|
|
433
|
+
}
|
|
434
|
+
// Removed after it ran, a home restored without it, or a fallback left
|
|
435
|
+
// for another firing (an older build moved `lastSessionId` without it).
|
|
436
|
+
// Replaying an older checkpoint here could redo finished work.
|
|
437
|
+
log(`schedule ${record.id}: previous run ${last} not found; starting fresh`);
|
|
438
|
+
return { streak: 0 };
|
|
439
|
+
};
|
|
440
|
+
/**
|
|
441
|
+
* A resumed firing whose session exists no longer needs its fallback: the
|
|
442
|
+
* next claim reads that session. Only while the row still names this
|
|
443
|
+
* firing, through the slot claim's own conditional write, so a later
|
|
444
|
+
* claim's fallback is never the one removed. Best effort: one left
|
|
445
|
+
* behind is used only while `lastSessionId` names no session.
|
|
446
|
+
*/
|
|
447
|
+
const dropFallback = async (scheduleId, sessionId) => {
|
|
448
|
+
try {
|
|
449
|
+
if ((await sessionStatus(sessionId)) === undefined) {
|
|
450
|
+
return;
|
|
451
|
+
}
|
|
452
|
+
const current = store.get(scheduleId);
|
|
453
|
+
if (current?.resumeFrom?.firing !== sessionId) {
|
|
454
|
+
return;
|
|
455
|
+
}
|
|
456
|
+
const { resumeFrom: _done, ...rest } = current;
|
|
457
|
+
store.claimSlot(rest, current.nextFireAt ?? '');
|
|
458
|
+
}
|
|
459
|
+
catch (error) {
|
|
460
|
+
warn(`schedule ${scheduleId}: could not clear the resume fallback for ${sessionId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
461
|
+
}
|
|
462
|
+
};
|
|
243
463
|
/**
|
|
244
464
|
* Fire one due schedule: consume the slot durably, then dispatch.
|
|
245
465
|
*
|
|
@@ -249,7 +469,7 @@ export const createSchedulerRuntime = (options) => {
|
|
|
249
469
|
* advance past it with a log line, never replay. Anything else fires,
|
|
250
470
|
* which on time is the normal case and late is the single catch-up.
|
|
251
471
|
*/
|
|
252
|
-
const fireOrSkip = (record, now) => {
|
|
472
|
+
const fireOrSkip = async (record, now) => {
|
|
253
473
|
const slot = record.nextFireAt;
|
|
254
474
|
if (!slot) {
|
|
255
475
|
return;
|
|
@@ -270,11 +490,15 @@ export const createSchedulerRuntime = (options) => {
|
|
|
270
490
|
// and the line that blamed downtime for it sent an operator to look
|
|
271
491
|
// for a restart that never happened.
|
|
272
492
|
const deferralKey = `${record.id}:${slot}`;
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
493
|
+
const skippingTo = next ? next.toISOString() : 'never';
|
|
494
|
+
log(deferralsWarned.has(`${deferralKey}:previous`)
|
|
495
|
+
? `schedule ${record.id}: the firing deferred while its previous run was still going (slot ${slot}) outlasted its window; skipping to ${skippingTo}`
|
|
496
|
+
: deferralsWarned.has(deferralKey)
|
|
497
|
+
? `schedule ${record.id}: the firing deferred by the per-agent cap (slot ${slot}) outlasted its window; skipping to ${skippingTo}`
|
|
498
|
+
: `schedule ${record.id}: missed firing(s) while the daemon was down; skipping to ${skippingTo}`);
|
|
276
499
|
store.claimSlot(withNextFire(record, next), slot);
|
|
277
500
|
deferralsWarned.delete(deferralKey);
|
|
501
|
+
deferralsWarned.delete(`${deferralKey}:previous`);
|
|
278
502
|
return;
|
|
279
503
|
}
|
|
280
504
|
if (options.paused?.() === true) {
|
|
@@ -307,6 +531,25 @@ export const createSchedulerRuntime = (options) => {
|
|
|
307
531
|
return;
|
|
308
532
|
}
|
|
309
533
|
deferralsWarned.delete(`${record.id}:${slot}`);
|
|
534
|
+
// Card 858: what this firing carries from the one before it. Read here,
|
|
535
|
+
// at the claim, so a daemon killed after a wrap-up still leaves the
|
|
536
|
+
// stopped session for this read; it's the one copy of the checkpoint.
|
|
537
|
+
let plan;
|
|
538
|
+
if (record.cadence.kind !== 'at' && record.lastSessionId !== undefined) {
|
|
539
|
+
const planned = await planResume(record, record.lastSessionId, slot);
|
|
540
|
+
if (planned === 'defer') {
|
|
541
|
+
return;
|
|
542
|
+
}
|
|
543
|
+
if (planned !== undefined) {
|
|
544
|
+
// An await sat between the readiness check and the claim below,
|
|
545
|
+
// which writes into the schedules table: check both again.
|
|
546
|
+
if (stopped) {
|
|
547
|
+
return;
|
|
548
|
+
}
|
|
549
|
+
await stillReady(record.id);
|
|
550
|
+
plan = planned;
|
|
551
|
+
}
|
|
552
|
+
}
|
|
310
553
|
const firedAt = now.toISOString();
|
|
311
554
|
const sessionId = `${SCHEDULE_SESSION_ID_PREFIX}${record.id}:${slot}`;
|
|
312
555
|
// Advance from the scheduled slot, not from now: `following` is the
|
|
@@ -320,28 +563,53 @@ export const createSchedulerRuntime = (options) => {
|
|
|
320
563
|
// and spent ATOMICALLY against the slot this tick read: a schedule
|
|
321
564
|
// cancelled between the due scan and here is gone, and its prompt
|
|
322
565
|
// must not run on a grant that was just revoked.
|
|
323
|
-
|
|
566
|
+
let claimed = withNextFire(record, next, { lastFiredAt: firedAt, lastSessionId: sessionId });
|
|
567
|
+
if (plan !== undefined) {
|
|
568
|
+
// The streak and the fallback are written in the same claim as the
|
|
569
|
+
// slot, so a crash can't leave one without the other.
|
|
570
|
+
const { stoppedEarlyStreak: _streak, resumeFrom: _resumeFrom, ...rest } = claimed;
|
|
571
|
+
claimed = {
|
|
572
|
+
...rest,
|
|
573
|
+
...(plan.streak > 0 ? { stoppedEarlyStreak: plan.streak } : {}),
|
|
574
|
+
...(plan.from !== undefined ? { resumeFrom: { firing: sessionId, from: plan.from.sessionId } } : {}),
|
|
575
|
+
};
|
|
576
|
+
}
|
|
577
|
+
if (!store.claimSlot(claimed, slot)) {
|
|
324
578
|
log(`schedule ${record.id}: slot ${slot} was cancelled or already claimed; not firing`);
|
|
325
579
|
return;
|
|
326
580
|
}
|
|
327
|
-
|
|
581
|
+
const resumeFrom = plan?.from;
|
|
582
|
+
// Re-read: a firing of this agent's may have settled during the await above.
|
|
583
|
+
runningPerAgent.set(record.agentId, (runningPerAgent.get(record.agentId) ?? 0) + 1);
|
|
328
584
|
activeFirings.add(sessionId);
|
|
329
585
|
const firing = dispatch({
|
|
330
586
|
sessionId,
|
|
331
587
|
agentId: record.agentId,
|
|
332
|
-
userMessage:
|
|
588
|
+
userMessage: resumeFrom === undefined
|
|
589
|
+
? record.prompt
|
|
590
|
+
: [record.prompt, renderResumeBlock(resumeFrom.run.stoppedEarly, resumeFrom.run.reply, plan.streak, record.destination !== undefined)].join('\n\n'),
|
|
333
591
|
metadata: {
|
|
334
592
|
[SCHEDULED_TURN_METADATA_KEY]: true,
|
|
335
593
|
[SCHEDULE_ID_METADATA_KEY]: record.id,
|
|
336
594
|
// The firing's "sender" is the session that wrote the prompt, at
|
|
337
595
|
// the label it had then; a schedule set before labels existed is
|
|
338
|
-
// `unknown`, like every other record from before.
|
|
339
|
-
|
|
596
|
+
// `unknown`, like every other record from before. A resumed firing
|
|
597
|
+
// also carries the stopped run's own words, so it's sent at the
|
|
598
|
+
// lower of the two, the rule `agent.delegate` uses for a child: a
|
|
599
|
+
// run that read a web page can't launder that page's text into a
|
|
600
|
+
// turn at the schedule's label.
|
|
601
|
+
[SENDER_TRUST_METADATA_KEY]: resumeFrom === undefined
|
|
602
|
+
? record.trust ?? 'unknown'
|
|
603
|
+
: leastTrusted(record.trust ?? 'unknown', resumeFrom.run.trust),
|
|
340
604
|
},
|
|
341
605
|
}).then(() => {
|
|
342
606
|
log(`schedule ${record.id} fired — session ${sessionId} completed`);
|
|
343
607
|
}, (error) => {
|
|
344
608
|
warn(`schedule ${record.id} firing failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
609
|
+
}).then(async () => {
|
|
610
|
+
if (resumeFrom !== undefined) {
|
|
611
|
+
await dropFallback(record.id, sessionId);
|
|
612
|
+
}
|
|
345
613
|
}).finally(() => {
|
|
346
614
|
activeFirings.delete(sessionId);
|
|
347
615
|
const count = (runningPerAgent.get(record.agentId) ?? 1) - 1;
|
|
@@ -360,6 +628,204 @@ export const createSchedulerRuntime = (options) => {
|
|
|
360
628
|
inflight.add(firing);
|
|
361
629
|
void firing.finally(() => inflight.delete(firing));
|
|
362
630
|
};
|
|
631
|
+
// ---- event triggers (card 590) ------------------------------------------
|
|
632
|
+
let started = false;
|
|
633
|
+
/** When the event tables were last pruned; unset until the first pass after start. */
|
|
634
|
+
let eventsPrunedAt;
|
|
635
|
+
/** Batches this process dispatched and has not seen settle. Empty after a restart, which is what makes a claimed batch recoverable. */
|
|
636
|
+
const inflightBatches = new Set();
|
|
637
|
+
const runningEventsPerAgent = new Map();
|
|
638
|
+
/** Groups whose deferral under the per-agent slot was already logged. */
|
|
639
|
+
const eventDeferralsWarned = new Set();
|
|
640
|
+
/** A trigger's waiting firings never go out once it is cancelled. */
|
|
641
|
+
const expireTriggerFirings = (triggerId) => {
|
|
642
|
+
if (!events) {
|
|
643
|
+
return;
|
|
644
|
+
}
|
|
645
|
+
const ids = events.store.open()
|
|
646
|
+
.filter((row) => row.triggerId === triggerId && !(row.batchKey !== undefined && inflightBatches.has(row.batchKey)))
|
|
647
|
+
.map((row) => row.id);
|
|
648
|
+
if (ids.length > 0) {
|
|
649
|
+
events.store.retire(ids, 'expired', 'Its trigger was cancelled.', clock().toISOString());
|
|
650
|
+
}
|
|
651
|
+
};
|
|
652
|
+
const fireEventBatch = (record, rows, batchKey) => {
|
|
653
|
+
const sessionId = rows[0].sessionId;
|
|
654
|
+
const ids = rows.map((row) => row.id);
|
|
655
|
+
inflightBatches.add(batchKey);
|
|
656
|
+
runningEventsPerAgent.set(record.agentId, (runningEventsPerAgent.get(record.agentId) ?? 0) + 1);
|
|
657
|
+
activeFirings.add(sessionId);
|
|
658
|
+
const firing = dispatch({
|
|
659
|
+
sessionId,
|
|
660
|
+
agentId: record.agentId,
|
|
661
|
+
// The trigger's prompt, then each event in the order it arrived.
|
|
662
|
+
userMessage: [record.prompt, ...rows.map((row) => row.block ?? '')].join('\n\n'),
|
|
663
|
+
idempotencyKey: batchKey,
|
|
664
|
+
metadata: {
|
|
665
|
+
[SCHEDULED_TURN_METADATA_KEY]: true,
|
|
666
|
+
[SCHEDULE_ID_METADATA_KEY]: record.id,
|
|
667
|
+
// The event text came from outside: a valid signature proves who
|
|
668
|
+
// delivered it, not who wrote it (anyone can comment on a PR). So
|
|
669
|
+
// the turn runs at `external`, under the gate that applies after
|
|
670
|
+
// reading a web page, whatever the trigger's creator was trusted at.
|
|
671
|
+
[SENDER_TRUST_METADATA_KEY]: 'external',
|
|
672
|
+
},
|
|
673
|
+
}).then(() => {
|
|
674
|
+
events?.store.finish(ids, { status: 'fired' }, clock().toISOString());
|
|
675
|
+
log(`event trigger ${record.id} fired — session ${sessionId} completed (${ids.length} event${ids.length === 1 ? '' : 's'})`);
|
|
676
|
+
}, (error) => {
|
|
677
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
678
|
+
// Refused by a daemon on its way down, or cut short by its stop and
|
|
679
|
+
// left for the next start: the batch stays claimed, and the next
|
|
680
|
+
// start sends it again under its key, which continues that turn
|
|
681
|
+
// (or runs it, if it never started) rather than losing the events.
|
|
682
|
+
// Matched by name, as a channel matches it. Only once this
|
|
683
|
+
// scheduler has stopped, which the gateway does before it refuses
|
|
684
|
+
// or aborts anything: a batch left claimed while ticks still ran
|
|
685
|
+
// would go straight out again.
|
|
686
|
+
if (stopped && error instanceof Error && error.name === 'GatewayStoppingError') {
|
|
687
|
+
log(`event trigger ${record.id}: ${reason} The batch stays claimed for the next start.`);
|
|
688
|
+
return;
|
|
689
|
+
}
|
|
690
|
+
events?.store.finish(ids, { status: 'failed', reason }, clock().toISOString());
|
|
691
|
+
warn(`event trigger ${record.id} firing failed: ${reason}`);
|
|
692
|
+
}).finally(() => {
|
|
693
|
+
activeFirings.delete(sessionId);
|
|
694
|
+
inflightBatches.delete(batchKey);
|
|
695
|
+
const count = (runningEventsPerAgent.get(record.agentId) ?? 1) - 1;
|
|
696
|
+
if (count > 0) {
|
|
697
|
+
runningEventsPerAgent.set(record.agentId, count);
|
|
698
|
+
}
|
|
699
|
+
else {
|
|
700
|
+
runningEventsPerAgent.delete(record.agentId);
|
|
701
|
+
}
|
|
702
|
+
});
|
|
703
|
+
inflight.add(firing);
|
|
704
|
+
void firing.finally(() => inflight.delete(firing));
|
|
705
|
+
};
|
|
706
|
+
/**
|
|
707
|
+
* The hold-or-fire step, for every waiting event firing: a fresh match, a
|
|
708
|
+
* hold whose window opened, and at startup whatever a crash left. One
|
|
709
|
+
* path, so a recovered firing still waits out quiet hours and counts
|
|
710
|
+
* against its trigger's hour like any other.
|
|
711
|
+
*/
|
|
712
|
+
const processEvents = (now) => {
|
|
713
|
+
// Not before start(): a held daemon's scheduler never starts, and its
|
|
714
|
+
// waiting firings must not go out through a delivery's kick either.
|
|
715
|
+
// Read once: the closures below would not see a narrowed `let`.
|
|
716
|
+
const current = events;
|
|
717
|
+
if (!current || stopped || !started) {
|
|
718
|
+
return;
|
|
719
|
+
}
|
|
720
|
+
const nowIso = now.toISOString();
|
|
721
|
+
// The 7-day window and the row cap hold only if something enforces
|
|
722
|
+
// them, and this is the loop that runs for the daemon's whole life: at
|
|
723
|
+
// the start-up pass, then hourly. Kept off the hold-or-fire path's
|
|
724
|
+
// failure: a prune that can't get the lock waits for the next hour.
|
|
725
|
+
if (eventsPrunedAt === undefined || now.getTime() - eventsPrunedAt >= EVENT_PRUNE_INTERVAL_MS) {
|
|
726
|
+
eventsPrunedAt = now.getTime();
|
|
727
|
+
try {
|
|
728
|
+
current.store.prune(nowIso);
|
|
729
|
+
}
|
|
730
|
+
catch (error) {
|
|
731
|
+
warn(`events: could not prune old deliveries: ${error instanceof Error ? error.message : String(error)}`);
|
|
732
|
+
}
|
|
733
|
+
}
|
|
734
|
+
const triggers = new Map();
|
|
735
|
+
const triggerFor = (id) => {
|
|
736
|
+
if (!triggers.has(id)) {
|
|
737
|
+
const record = store.get(id);
|
|
738
|
+
triggers.set(id, record?.cadence.kind === 'event' ? record : undefined);
|
|
739
|
+
}
|
|
740
|
+
return triggers.get(id);
|
|
741
|
+
};
|
|
742
|
+
// A claimed batch stays itself; free firings group by trigger and key.
|
|
743
|
+
// Map order is first appearance in `open()`, oldest first.
|
|
744
|
+
const groups = new Map();
|
|
745
|
+
for (const row of current.store.open()) {
|
|
746
|
+
if (row.batchKey !== undefined && inflightBatches.has(row.batchKey)) {
|
|
747
|
+
continue;
|
|
748
|
+
}
|
|
749
|
+
const trigger = triggerFor(row.triggerId);
|
|
750
|
+
if (!trigger || trigger.agentId !== row.agentId) {
|
|
751
|
+
current.store.retire([row.id], 'expired', 'Its trigger was cancelled.', nowIso);
|
|
752
|
+
continue;
|
|
753
|
+
}
|
|
754
|
+
// Holds expire; a claim does not. A batch claimed before a crash may
|
|
755
|
+
// already have reached the gateway, and its key either continues that
|
|
756
|
+
// turn or sends an event that never went out, however long the
|
|
757
|
+
// daemon was down.
|
|
758
|
+
if (row.status !== 'firing' && now.getTime() - Date.parse(row.createdAt) > EVENT_HOLD_MAX_MS) {
|
|
759
|
+
current.store.retire([row.id], 'expired', 'Held for more than 24 hours.', nowIso);
|
|
760
|
+
continue;
|
|
761
|
+
}
|
|
762
|
+
const key = row.batchKey !== undefined ? `batch\n${row.batchKey}` : `key\n${row.triggerId}\n${row.coalescingKey}`;
|
|
763
|
+
const group = groups.get(key) ?? [];
|
|
764
|
+
group.push(row);
|
|
765
|
+
groups.set(key, group);
|
|
766
|
+
}
|
|
767
|
+
const holdAs = (rows, status) => {
|
|
768
|
+
const ids = rows.filter((row) => row.status !== status).map((row) => row.id);
|
|
769
|
+
if (ids.length > 0) {
|
|
770
|
+
current.store.hold(ids, status, nowIso);
|
|
771
|
+
}
|
|
772
|
+
};
|
|
773
|
+
for (const [groupKey, all] of groups) {
|
|
774
|
+
const trigger = triggerFor(all[0].triggerId);
|
|
775
|
+
if (trigger.cadence.kind !== 'event') {
|
|
776
|
+
continue;
|
|
777
|
+
}
|
|
778
|
+
let rows = all;
|
|
779
|
+
const batched = rows[0].batchKey !== undefined;
|
|
780
|
+
if (!batched && rows.length > EVENT_HELD_PER_KEY) {
|
|
781
|
+
const dropped = rows.slice(0, rows.length - EVENT_HELD_PER_KEY);
|
|
782
|
+
current.store.retire(dropped.map((row) => row.id), 'superseded', `Over ${EVENT_HELD_PER_KEY} waiting for one subject; newer ones were kept.`, nowIso);
|
|
783
|
+
rows = rows.slice(-EVENT_HELD_PER_KEY);
|
|
784
|
+
}
|
|
785
|
+
const { quietHours } = trigger.cadence;
|
|
786
|
+
if (options.paused?.() === true || (quietHours !== undefined && isWithinQuietHours(quietHours, now))) {
|
|
787
|
+
holdAs(rows, 'held');
|
|
788
|
+
continue;
|
|
789
|
+
}
|
|
790
|
+
const running = runningEventsPerAgent.get(trigger.agentId) ?? 0;
|
|
791
|
+
if (running >= maxEventsPerAgent) {
|
|
792
|
+
if (!eventDeferralsWarned.has(groupKey)) {
|
|
793
|
+
eventDeferralsWarned.add(groupKey);
|
|
794
|
+
log(`event trigger ${trigger.id}: ${trigger.agentId} already has ${running} event firing(s) running; this one waits`);
|
|
795
|
+
}
|
|
796
|
+
continue;
|
|
797
|
+
}
|
|
798
|
+
eventDeferralsWarned.delete(groupKey);
|
|
799
|
+
const maxPerHour = trigger.cadence.maxPerHour ?? DEFAULT_EVENT_MAX_PER_HOUR;
|
|
800
|
+
const hourAgo = new Date(now.getTime() - 3_600_000).toISOString();
|
|
801
|
+
if (current.store.turnsStartedSince(trigger.id, hourAgo, rows[0].batchKey) >= maxPerHour) {
|
|
802
|
+
if (!batched && rows.length > 1) {
|
|
803
|
+
current.store.retire(rows.slice(0, -1).map((row) => row.id), 'superseded', 'A newer event for the same subject replaced it while over the hourly limit.', nowIso);
|
|
804
|
+
rows = rows.slice(-1);
|
|
805
|
+
}
|
|
806
|
+
holdAs(rows, 'rate_limited');
|
|
807
|
+
continue;
|
|
808
|
+
}
|
|
809
|
+
const batchKey = rows[0].batchKey ?? rows[rows.length - 1].dispatchKey;
|
|
810
|
+
if (!current.store.claim(rows.map((row) => row.id), batchKey, nowIso, { triggerId: trigger.id })) {
|
|
811
|
+
continue;
|
|
812
|
+
}
|
|
813
|
+
// When it last went out, on the trigger's own row, so `schedule.list`,
|
|
814
|
+
// `stratus schedules` and the control API show it the way they show a
|
|
815
|
+
// timed schedule's. The same conditional write a slot claim makes; an
|
|
816
|
+
// event trigger's slot is always empty, so it only requires the row.
|
|
817
|
+
// A cancel landing after the claim above makes this a no-op, and the
|
|
818
|
+
// claimed batch still goes, as a claimed slot does. An audit field, so
|
|
819
|
+
// a write that fails is warned and never keeps a claimed batch back.
|
|
820
|
+
try {
|
|
821
|
+
store.claimSlot(withNextFire(trigger, undefined, { lastFiredAt: nowIso, lastSessionId: rows[0].sessionId }), '');
|
|
822
|
+
}
|
|
823
|
+
catch (error) {
|
|
824
|
+
warn(`event trigger ${trigger.id}: could not record its last firing: ${error instanceof Error ? error.message : String(error)}`);
|
|
825
|
+
}
|
|
826
|
+
fireEventBatch(trigger, rows, batchKey);
|
|
827
|
+
}
|
|
828
|
+
};
|
|
363
829
|
/**
|
|
364
830
|
* `propagateReadiness` is the startup tick's: a readiness failure there
|
|
365
831
|
* has a caller to reach — `start()`, whose rejection is the gateway's
|
|
@@ -383,15 +849,23 @@ export const createSchedulerRuntime = (options) => {
|
|
|
383
849
|
}
|
|
384
850
|
}
|
|
385
851
|
try {
|
|
386
|
-
const now =
|
|
852
|
+
const now = clock();
|
|
387
853
|
for (const record of store.due(now.toISOString())) {
|
|
388
854
|
if (stopped) {
|
|
389
855
|
break;
|
|
390
856
|
}
|
|
391
|
-
fireOrSkip(record, now);
|
|
857
|
+
await fireOrSkip(record, now);
|
|
392
858
|
}
|
|
859
|
+
processEvents(now);
|
|
393
860
|
}
|
|
394
861
|
catch (error) {
|
|
862
|
+
if (error instanceof NotReady) {
|
|
863
|
+
if (propagateReadiness) {
|
|
864
|
+
throw error.reason;
|
|
865
|
+
}
|
|
866
|
+
warn(`scheduler stopped before claiming schedule ${error.scheduleId}'s firing: ${error.message}`);
|
|
867
|
+
return;
|
|
868
|
+
}
|
|
395
869
|
warn(`scheduler tick failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
396
870
|
}
|
|
397
871
|
if (!stopped) {
|
|
@@ -406,6 +880,31 @@ export const createSchedulerRuntime = (options) => {
|
|
|
406
880
|
};
|
|
407
881
|
return {
|
|
408
882
|
handle,
|
|
883
|
+
eventTriggers(sourceId) {
|
|
884
|
+
return store.list()
|
|
885
|
+
.filter((record) => record.cadence.kind === 'event' && record.cadence.source === sourceId)
|
|
886
|
+
.map((record) => ({
|
|
887
|
+
triggerId: record.id,
|
|
888
|
+
agentId: record.agentId,
|
|
889
|
+
source: sourceId,
|
|
890
|
+
filter: record.cadence.filter,
|
|
891
|
+
}));
|
|
892
|
+
},
|
|
893
|
+
attachEvents(attached) {
|
|
894
|
+
if (events) {
|
|
895
|
+
throw new Error('This scheduler already takes events; its sources are read live, so add a source there instead.');
|
|
896
|
+
}
|
|
897
|
+
events = attached;
|
|
898
|
+
maxEventsPerAgent = attached.maxConcurrentPerAgent ?? 1;
|
|
899
|
+
},
|
|
900
|
+
kickEvents() {
|
|
901
|
+
try {
|
|
902
|
+
processEvents(clock());
|
|
903
|
+
}
|
|
904
|
+
catch (error) {
|
|
905
|
+
warn(`event triggers: processing failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
906
|
+
}
|
|
907
|
+
},
|
|
409
908
|
isPreauthorized(session, destination) {
|
|
410
909
|
// A live firing of this scheduler's, first: session metadata is
|
|
411
910
|
// written by whoever dispatches, so on its own it proves nothing —
|
|
@@ -463,6 +962,7 @@ export const createSchedulerRuntime = (options) => {
|
|
|
463
962
|
const deleted = store.delete(scheduleId);
|
|
464
963
|
if (deleted) {
|
|
465
964
|
log(`schedule ${scheduleId} cancelled`);
|
|
965
|
+
expireTriggerFirings(scheduleId);
|
|
466
966
|
}
|
|
467
967
|
return deleted;
|
|
468
968
|
},
|
|
@@ -497,6 +997,7 @@ export const createSchedulerRuntime = (options) => {
|
|
|
497
997
|
// The first tick's readiness check propagates too: the stamp can
|
|
498
998
|
// advance between the check above and here, and a `start()` that
|
|
499
999
|
// resolved after noticing would let the gateway's sweeps run.
|
|
1000
|
+
started = true;
|
|
500
1001
|
await tick(true);
|
|
501
1002
|
},
|
|
502
1003
|
stop() {
|