instar 1.3.1646 → 1.3.1647
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/SessionManager.d.ts +1 -1
- package/dist/core/SessionManager.d.ts.map +1 -1
- package/dist/core/SessionManager.js +15 -7
- package/dist/core/SessionManager.js.map +1 -1
- package/dist/data/standards-guard-index.json +1 -1
- package/dist/data/standards-guard-index.meta.json +2 -2
- package/dist/data/standards-registry.meta.json +1 -1
- package/dist/messaging/userNotices.d.ts +3 -0
- package/dist/messaging/userNotices.d.ts.map +1 -1
- package/dist/messaging/userNotices.js +25 -0
- package/dist/messaging/userNotices.js.map +1 -1
- package/dist/native/InstarWedgeCanary.app/Contents/MacOS/InstarWedgeCanary +0 -0
- package/dist/native/instar-desktop-host +0 -0
- package/dist/scheduler/JobClaimManager.d.ts +3 -3
- package/dist/scheduler/JobClaimManager.d.ts.map +1 -1
- package/dist/scheduler/JobClaimManager.js.map +1 -1
- package/dist/scheduler/JobLeaseClaimStore.d.ts +2 -2
- package/dist/scheduler/JobLeaseClaimStore.d.ts.map +1 -1
- package/dist/scheduler/JobLeaseClaimStore.js.map +1 -1
- package/dist/scheduler/JobScheduler.d.ts +10 -6
- package/dist/scheduler/JobScheduler.d.ts.map +1 -1
- package/dist/scheduler/JobScheduler.js +194 -64
- package/dist/scheduler/JobScheduler.js.map +1 -1
- package/dist/testing/selfActionRegistry.d.ts.map +1 -1
- package/dist/testing/selfActionRegistry.js +31 -0
- package/dist/testing/selfActionRegistry.js.map +1 -1
- package/package.json +1 -1
- package/src/data/builtin-manifest.json +4 -4
- package/src/data/standards-guard-index.json +1 -1
- package/src/data/standards-guard-index.meta.json +2 -2
- package/src/data/standards-registry.meta.json +1 -1
- package/upgrades/1.3.1647.md +20 -0
- package/upgrades/job-alert-capacity-episode.eli16.md +13 -0
- package/upgrades/side-effects/job-alert-capacity-episode.md +65 -0
|
@@ -40,6 +40,8 @@ import { classifySessionDeath } from '../monitoring/QuotaExhaustionDetector.js';
|
|
|
40
40
|
import { TOPIC_STYLE } from '../messaging/TelegramAdapter.js';
|
|
41
41
|
import { TelegramOriginHoldError } from '../messaging/telegram-origin/types.js';
|
|
42
42
|
import { reportDeliverySinkFailure } from '../messaging/DeliverySinkFailure.js';
|
|
43
|
+
import { RerouteRefusedError } from '../core/rerouteRefusedError.js';
|
|
44
|
+
import { jobCapacityNotice, jobFailureNotice } from '../messaging/userNotices.js';
|
|
43
45
|
import { decideClaimPath } from './JobLeaseCutoverGate.js';
|
|
44
46
|
/**
|
|
45
47
|
* Fallback expected duration (minutes) used when a run is encountered for a
|
|
@@ -47,6 +49,9 @@ import { decideClaimPath } from './JobLeaseCutoverGate.js';
|
|
|
47
49
|
* or reloaded mid-sleep). Reaper threshold = this × thresholdMultiplier.
|
|
48
50
|
*/
|
|
49
51
|
const DEFAULT_EXPECTED_MINUTES = 30;
|
|
52
|
+
const CAPACITY_POLL_MS = 15_000;
|
|
53
|
+
const CRITICAL_CAPACITY_ALERT_MS = 10 * 60_000;
|
|
54
|
+
const OTHER_CAPACITY_ALERT_MS = 30 * 60_000;
|
|
50
55
|
const PRIORITY_ORDER = {
|
|
51
56
|
critical: 0,
|
|
52
57
|
high: 1,
|
|
@@ -144,6 +149,11 @@ export class JobScheduler {
|
|
|
144
149
|
failureAlertsInFlight = new Set();
|
|
145
150
|
/** Scheduler-owned wakeups for delivery retries and unresolved reminders. */
|
|
146
151
|
failureAlertTimers = new Map();
|
|
152
|
+
/** One local admission attempt per poll; persisted episode survives restart. */
|
|
153
|
+
capacityPoll = null;
|
|
154
|
+
capacityAlertInFlight = false;
|
|
155
|
+
queueAdmissionInFlight = false;
|
|
156
|
+
activeSessionSpawns = new Set();
|
|
147
157
|
/** Startup composition has completed, even when its result is "no sink". */
|
|
148
158
|
failureAlertDeliveryActive = false;
|
|
149
159
|
/** Optional quota tracker for death classification cross-reference */
|
|
@@ -600,6 +610,16 @@ export class JobScheduler {
|
|
|
600
610
|
return;
|
|
601
611
|
this.refreshJobs();
|
|
602
612
|
this.running = true;
|
|
613
|
+
const persistedWait = this.state.get('job-capacity-episode');
|
|
614
|
+
if (persistedWait?.version === 1) {
|
|
615
|
+
for (const slug of persistedWait.jobs) {
|
|
616
|
+
const job = this.jobs.find(j => j.slug === slug);
|
|
617
|
+
if (job?.enabled && this.isJobScopedToThisMachine(job))
|
|
618
|
+
this.enqueue(slug, 'capacity-recovery', true);
|
|
619
|
+
else
|
|
620
|
+
this.markCapacityJobStarted(slug);
|
|
621
|
+
}
|
|
622
|
+
}
|
|
603
623
|
const scopedJobs = this.rebuildCronTasks();
|
|
604
624
|
const enabledJobs = this.jobs.filter(j => j.enabled);
|
|
605
625
|
const skippedByScope = enabledJobs.length - scopedJobs.length;
|
|
@@ -656,6 +676,9 @@ export class JobScheduler {
|
|
|
656
676
|
this.missedJobStartupTimer = null;
|
|
657
677
|
}
|
|
658
678
|
this.queue = [];
|
|
679
|
+
if (this.capacityPoll)
|
|
680
|
+
clearInterval(this.capacityPoll);
|
|
681
|
+
this.capacityPoll = null;
|
|
659
682
|
// Clear all retry timers
|
|
660
683
|
for (const [, state] of this.retryState) {
|
|
661
684
|
if (state.timer)
|
|
@@ -857,6 +880,8 @@ export class JobScheduler {
|
|
|
857
880
|
timestamp: new Date().toISOString(),
|
|
858
881
|
metadata: { slug, reason, claimedBy, claimPath: claimPath?.path ?? 'bus' },
|
|
859
882
|
});
|
|
883
|
+
if (reason.startsWith('queued:'))
|
|
884
|
+
this.markCapacityJobStarted(slug);
|
|
860
885
|
return 'skipped';
|
|
861
886
|
}
|
|
862
887
|
// Per-slug double-run guard (june15-headless-spawn-reroute, PR2 finding O3).
|
|
@@ -869,7 +894,7 @@ export class JobScheduler {
|
|
|
869
894
|
// that outran its cron interval was always at risk; this closes it generally.
|
|
870
895
|
// The boot-reconcile in SessionManager.purgeDeadSessions kills surviving
|
|
871
896
|
// rerouted job sessions, so the steady state is one-run-per-slug.
|
|
872
|
-
const liveWithSlug = this.sessionManager.listRunningSessions().some(s => s.jobSlug === slug);
|
|
897
|
+
const liveWithSlug = this.activeSessionSpawns.has(slug) || this.sessionManager.listRunningSessions().some(s => s.jobSlug === slug);
|
|
873
898
|
if (liveWithSlug) {
|
|
874
899
|
this.skipLedger.recordSkip(slug, 'already-running');
|
|
875
900
|
console.log(`[scheduler] Job "${slug}" skipped — a live session already holds this slug (avoiding double-run).`);
|
|
@@ -879,6 +904,8 @@ export class JobScheduler {
|
|
|
879
904
|
timestamp: new Date().toISOString(),
|
|
880
905
|
metadata: { slug, reason, gateReason: 'already-running' },
|
|
881
906
|
});
|
|
907
|
+
if (reason.startsWith('queued:'))
|
|
908
|
+
this.markCapacityJobStarted(slug);
|
|
882
909
|
return 'skipped';
|
|
883
910
|
}
|
|
884
911
|
// Script jobs bypass quota gating — they don't consume LLM tokens
|
|
@@ -990,13 +1017,16 @@ export class JobScheduler {
|
|
|
990
1017
|
// Check session capacity
|
|
991
1018
|
const runningSessions = this.sessionManager.listRunningSessions();
|
|
992
1019
|
const jobSessions = runningSessions.filter(s => s.jobSlug);
|
|
993
|
-
if (jobSessions.length >= this.config.maxParallelJobs) {
|
|
1020
|
+
if (jobSessions.length + this.activeSessionSpawns.size >= this.config.maxParallelJobs) {
|
|
994
1021
|
this.enqueue(slug, reason);
|
|
995
1022
|
return 'queued';
|
|
996
1023
|
}
|
|
997
1024
|
const timeoutMs = (job.expectedDurationMinutes ?? 30) * 2 * 60_000;
|
|
998
|
-
if (!this.takeClaim(job, reason, claimPath, timeoutMs))
|
|
1025
|
+
if (!this.takeClaim(job, reason, claimPath, timeoutMs)) {
|
|
1026
|
+
if (reason.startsWith('queued:'))
|
|
1027
|
+
this.markCapacityJobStarted(slug);
|
|
999
1028
|
return 'skipped';
|
|
1029
|
+
}
|
|
1000
1030
|
// Clear retry state on successful trigger
|
|
1001
1031
|
this.clearRetryState(slug);
|
|
1002
1032
|
this.spawnJobSession(job, reason);
|
|
@@ -1072,7 +1102,7 @@ export class JobScheduler {
|
|
|
1072
1102
|
* Process the queue — dequeue and run next job if a slot is available.
|
|
1073
1103
|
*/
|
|
1074
1104
|
processQueue() {
|
|
1075
|
-
if (this.paused || this.queue.length === 0)
|
|
1105
|
+
if (this.paused || this.queue.length === 0 || this.queueAdmissionInFlight)
|
|
1076
1106
|
return;
|
|
1077
1107
|
try {
|
|
1078
1108
|
this.refreshJobs();
|
|
@@ -1083,14 +1113,24 @@ export class JobScheduler {
|
|
|
1083
1113
|
}
|
|
1084
1114
|
const runningSessions = this.sessionManager.listRunningSessions();
|
|
1085
1115
|
const jobSessions = runningSessions.filter(s => s.jobSlug);
|
|
1086
|
-
if (jobSessions.length >= this.config.maxParallelJobs)
|
|
1116
|
+
if (jobSessions.length + this.activeSessionSpawns.size >= this.config.maxParallelJobs)
|
|
1117
|
+
return;
|
|
1118
|
+
// The normal queue drain checks only maxParallelJobs. The force-mode
|
|
1119
|
+
// reroute has a separate host-wide cap and a live memory-pressure gate.
|
|
1120
|
+
// Keep the queued job in place until that gate opens; no failed spawn is
|
|
1121
|
+
// recorded and only one queued job is admitted per poll.
|
|
1122
|
+
if (this.queue[0]?.capacityWait && this.sessionManager.confinedRerouteCapacity() === 'saturated') {
|
|
1123
|
+
void this.maybeAlertCapacityEpisode();
|
|
1087
1124
|
return;
|
|
1125
|
+
}
|
|
1088
1126
|
const next = this.queue.shift();
|
|
1089
1127
|
if (!next)
|
|
1090
1128
|
return;
|
|
1091
1129
|
const job = this.jobs.find(j => j.slug === next.slug);
|
|
1092
1130
|
if (!job) {
|
|
1093
1131
|
this.skipLedger.recordSkip(next.slug, 'disabled');
|
|
1132
|
+
if (next.capacityWait)
|
|
1133
|
+
this.markCapacityJobStarted(next.slug);
|
|
1094
1134
|
this.state.appendEvent({
|
|
1095
1135
|
type: 'job_skipped',
|
|
1096
1136
|
summary: `Job "${next.slug}" skipped — no longer present in the live job set`,
|
|
@@ -1101,6 +1141,8 @@ export class JobScheduler {
|
|
|
1101
1141
|
}
|
|
1102
1142
|
if (!job.enabled) {
|
|
1103
1143
|
this.skipLedger.recordSkip(next.slug, 'disabled');
|
|
1144
|
+
if (next.capacityWait)
|
|
1145
|
+
this.markCapacityJobStarted(next.slug);
|
|
1104
1146
|
this.state.appendEvent({
|
|
1105
1147
|
type: 'job_skipped',
|
|
1106
1148
|
summary: `Job "${next.slug}" skipped — disabled in the live job set`,
|
|
@@ -1131,7 +1173,20 @@ export class JobScheduler {
|
|
|
1131
1173
|
this.queue.unshift(next);
|
|
1132
1174
|
return;
|
|
1133
1175
|
}
|
|
1134
|
-
|
|
1176
|
+
// Re-enter the normal admission path so a recovered wait takes a fresh
|
|
1177
|
+
// machine claim; the old refused attempt released its claim as "waited".
|
|
1178
|
+
this.queueAdmissionInFlight = true;
|
|
1179
|
+
void this.triggerJob(job.slug, `queued:${next.reason}`).then(result => {
|
|
1180
|
+
if (result === 'skipped' && next.capacityWait) {
|
|
1181
|
+
const episode = this.state.get('job-capacity-episode');
|
|
1182
|
+
if (episode?.jobs.includes(job.slug))
|
|
1183
|
+
this.enqueue(job.slug, next.reason, true);
|
|
1184
|
+
}
|
|
1185
|
+
}).catch(err => {
|
|
1186
|
+
console.error(`[scheduler] Queued trigger failed for "${job.slug}": ${err instanceof Error ? err.message : String(err)}`);
|
|
1187
|
+
if (next.capacityWait)
|
|
1188
|
+
this.enqueue(job.slug, next.reason, true);
|
|
1189
|
+
}).finally(() => { this.queueAdmissionInFlight = false; });
|
|
1135
1190
|
}
|
|
1136
1191
|
/**
|
|
1137
1192
|
* Pause — cron tasks keep ticking but triggers are skipped.
|
|
@@ -1144,6 +1199,10 @@ export class JobScheduler {
|
|
|
1144
1199
|
*/
|
|
1145
1200
|
clearQueue() {
|
|
1146
1201
|
this.queue.length = 0;
|
|
1202
|
+
this.state.set('job-capacity-episode', null);
|
|
1203
|
+
if (this.capacityPoll)
|
|
1204
|
+
clearInterval(this.capacityPoll);
|
|
1205
|
+
this.capacityPoll = null;
|
|
1147
1206
|
}
|
|
1148
1207
|
/**
|
|
1149
1208
|
* Resume — triggers start executing again.
|
|
@@ -1364,22 +1423,126 @@ export class JobScheduler {
|
|
|
1364
1423
|
console.log(`[scheduler] Job "${job.slug}" body changed on disk after scheduler start; ` +
|
|
1365
1424
|
'reloaded the validated body for this run.');
|
|
1366
1425
|
}
|
|
1367
|
-
enqueue(slug, reason) {
|
|
1368
|
-
// Don't queue duplicates
|
|
1369
|
-
|
|
1426
|
+
enqueue(slug, reason, capacityWait = false) {
|
|
1427
|
+
// Don't queue duplicates; a capacity refusal upgrades an ordinary wait.
|
|
1428
|
+
const existing = this.queue.find(q => q.slug === slug);
|
|
1429
|
+
if (existing) {
|
|
1430
|
+
if (capacityWait && !existing.capacityWait) {
|
|
1431
|
+
existing.capacityWait = true;
|
|
1432
|
+
this.recordCapacityWait(slug);
|
|
1433
|
+
}
|
|
1370
1434
|
return;
|
|
1435
|
+
}
|
|
1371
1436
|
// Cap queue size to prevent unbounded growth
|
|
1372
1437
|
if (this.queue.length >= 50) {
|
|
1373
1438
|
console.warn(`[scheduler] Queue full (50 items), dropping enqueue for "${slug}"`);
|
|
1374
1439
|
return;
|
|
1375
1440
|
}
|
|
1376
|
-
this.queue.push({ slug, reason, queuedAt: new Date().toISOString() });
|
|
1441
|
+
this.queue.push({ slug, reason, queuedAt: new Date().toISOString(), capacityWait });
|
|
1377
1442
|
// Sort by priority — critical first
|
|
1378
1443
|
this.queue.sort((a, b) => {
|
|
1379
1444
|
const jobA = this.jobs.find(j => j.slug === a.slug);
|
|
1380
1445
|
const jobB = this.jobs.find(j => j.slug === b.slug);
|
|
1381
1446
|
return (PRIORITY_ORDER[jobA?.priority ?? 'low']) - (PRIORITY_ORDER[jobB?.priority ?? 'low']);
|
|
1382
1447
|
});
|
|
1448
|
+
if (capacityWait)
|
|
1449
|
+
this.recordCapacityWait(slug);
|
|
1450
|
+
}
|
|
1451
|
+
recordCapacityWait(slug) {
|
|
1452
|
+
const key = 'job-capacity-episode';
|
|
1453
|
+
const prior = this.state.get(key);
|
|
1454
|
+
const episode = prior?.version === 1
|
|
1455
|
+
? { ...prior, jobs: [...new Set([...prior.jobs, slug])], blockedAtByJob: { ...prior.blockedAtByJob, [slug]: prior.blockedAtByJob?.[slug] ?? new Date().toISOString() } }
|
|
1456
|
+
: { version: 1, firstBlockedAt: new Date().toISOString(), jobs: [slug], blockedAtByJob: { [slug]: new Date().toISOString() } };
|
|
1457
|
+
this.state.set(key, episode);
|
|
1458
|
+
this.state.appendEvent({
|
|
1459
|
+
type: 'job_waited_for_room',
|
|
1460
|
+
summary: `Job "${slug}" waited for reroute capacity`,
|
|
1461
|
+
timestamp: new Date().toISOString(),
|
|
1462
|
+
metadata: { slug, reason: 'reroute-capacity' },
|
|
1463
|
+
});
|
|
1464
|
+
this.armCapacityPoll();
|
|
1465
|
+
}
|
|
1466
|
+
/* @self-action-controller: job-capacity-episode */
|
|
1467
|
+
armCapacityPoll() {
|
|
1468
|
+
if (this.capacityPoll)
|
|
1469
|
+
return;
|
|
1470
|
+
this.capacityPoll = setInterval(() => {
|
|
1471
|
+
if (!this.running || this.paused)
|
|
1472
|
+
return;
|
|
1473
|
+
this.processQueue();
|
|
1474
|
+
void this.maybeAlertCapacityEpisode();
|
|
1475
|
+
}, CAPACITY_POLL_MS);
|
|
1476
|
+
this.capacityPoll.unref?.();
|
|
1477
|
+
}
|
|
1478
|
+
markCapacityJobStarted(slug) {
|
|
1479
|
+
const key = 'job-capacity-episode';
|
|
1480
|
+
const episode = this.state.get(key);
|
|
1481
|
+
if (!episode || !episode.jobs.includes(slug))
|
|
1482
|
+
return;
|
|
1483
|
+
this.queue = this.queue.filter(q => q.slug !== slug);
|
|
1484
|
+
const jobs = episode.jobs.filter(s => s !== slug);
|
|
1485
|
+
if (jobs.length) {
|
|
1486
|
+
const blockedAtByJob = { ...episode.blockedAtByJob };
|
|
1487
|
+
delete blockedAtByJob[slug];
|
|
1488
|
+
this.state.set(key, { ...episode, jobs, blockedAtByJob });
|
|
1489
|
+
}
|
|
1490
|
+
else {
|
|
1491
|
+
this.state.set(key, null);
|
|
1492
|
+
if (this.capacityPoll)
|
|
1493
|
+
clearInterval(this.capacityPoll);
|
|
1494
|
+
this.capacityPoll = null;
|
|
1495
|
+
}
|
|
1496
|
+
}
|
|
1497
|
+
async maybeAlertCapacityEpisode() {
|
|
1498
|
+
const episode = this.state.get('job-capacity-episode');
|
|
1499
|
+
if (!episode || episode.alertedAt || this.capacityAlertInFlight || !episode.jobs.length)
|
|
1500
|
+
return;
|
|
1501
|
+
const waiting = episode.jobs.map(s => this.jobs.find(j => j.slug === s)).filter((j) => Boolean(j));
|
|
1502
|
+
if (!waiting.length) {
|
|
1503
|
+
for (const slug of episode.jobs)
|
|
1504
|
+
this.markCapacityJobStarted(slug);
|
|
1505
|
+
return;
|
|
1506
|
+
}
|
|
1507
|
+
const ageMs = Date.now() - Date.parse(episode.firstBlockedAt);
|
|
1508
|
+
if (!Number.isFinite(ageMs))
|
|
1509
|
+
return;
|
|
1510
|
+
const eligible = waiting.some(j => {
|
|
1511
|
+
const since = Date.parse(episode.blockedAtByJob?.[j.slug] ?? episode.firstBlockedAt);
|
|
1512
|
+
const threshold = j.priority === 'critical' ? CRITICAL_CAPACITY_ALERT_MS : OTHER_CAPACITY_ALERT_MS;
|
|
1513
|
+
return Number.isFinite(since) && Date.now() - since >= threshold;
|
|
1514
|
+
});
|
|
1515
|
+
if (!eligible)
|
|
1516
|
+
return;
|
|
1517
|
+
this.capacityAlertInFlight = true;
|
|
1518
|
+
try {
|
|
1519
|
+
// Existing session monitoring owns liveness. Give its completion sweep
|
|
1520
|
+
// one chance to free room before an operator-facing alert.
|
|
1521
|
+
await this.sessionManager.purgeDeadSessions(true);
|
|
1522
|
+
if (this.sessionManager.confinedRerouteCapacity() === 'available') {
|
|
1523
|
+
this.processQueue();
|
|
1524
|
+
return;
|
|
1525
|
+
}
|
|
1526
|
+
const names = waiting.sort((a, b) => PRIORITY_ORDER[a.priority] - PRIORITY_ORDER[b.priority]).map(j => j.name);
|
|
1527
|
+
const notice = jobCapacityNotice(names, ageMs);
|
|
1528
|
+
const hub = this.state.get('agent-attention-topic');
|
|
1529
|
+
if (this.telegram && hub)
|
|
1530
|
+
await this.telegram.sendToTopic(hub, notice);
|
|
1531
|
+
else if (!this.telegram && this.messenger)
|
|
1532
|
+
await this.messenger.send({ userId: 'system', content: notice });
|
|
1533
|
+
else
|
|
1534
|
+
return;
|
|
1535
|
+
const current = this.state.get('job-capacity-episode');
|
|
1536
|
+
if (current?.firstBlockedAt === episode.firstBlockedAt && current.jobs.length) {
|
|
1537
|
+
this.state.set('job-capacity-episode', { ...current, alertedAt: new Date().toISOString() });
|
|
1538
|
+
}
|
|
1539
|
+
}
|
|
1540
|
+
catch (err) {
|
|
1541
|
+
console.error(`[scheduler] Capacity episode notice failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
1542
|
+
}
|
|
1543
|
+
finally {
|
|
1544
|
+
this.capacityAlertInFlight = false;
|
|
1545
|
+
}
|
|
1383
1546
|
}
|
|
1384
1547
|
/**
|
|
1385
1548
|
* Execute a `script`-type job directly in a bounded subprocess instead of
|
|
@@ -2354,6 +2517,7 @@ export class JobScheduler {
|
|
|
2354
2517
|
// spawn-error path. Hashing the body/frontmatter is cheap (a few µs for
|
|
2355
2518
|
// typical body sizes) — see spec performance budget.
|
|
2356
2519
|
const observability = JobScheduler.computeRunObservability(job, allowlistResolution);
|
|
2520
|
+
this.activeSessionSpawns.add(job.slug);
|
|
2357
2521
|
this.sessionManager.spawnSession({
|
|
2358
2522
|
name: sessionName,
|
|
2359
2523
|
prompt,
|
|
@@ -2369,6 +2533,7 @@ export class JobScheduler {
|
|
|
2369
2533
|
// (docs/specs/LOOP-SESSION-NO-MCP-SPEC.md). Absent/'project' → unchanged.
|
|
2370
2534
|
disableProjectMcp: job.mcpAccess === 'none' ? true : undefined,
|
|
2371
2535
|
}).then((session) => {
|
|
2536
|
+
this.markCapacityJobStarted(job.slug);
|
|
2372
2537
|
this.recordLedgerSession(session?.id ?? sessionName, job.slug, new Date().toISOString(), 'job');
|
|
2373
2538
|
// Record in run history with full Phase 1b observability payload
|
|
2374
2539
|
const runId = this.runHistory.recordStart({
|
|
@@ -2411,6 +2576,14 @@ export class JobScheduler {
|
|
|
2411
2576
|
metadata: { slug: job.slug, reason, model: job.model },
|
|
2412
2577
|
});
|
|
2413
2578
|
}).catch((err) => {
|
|
2579
|
+
if (err instanceof RerouteRefusedError) {
|
|
2580
|
+
this.releaseClaim(job.slug, 'waited');
|
|
2581
|
+
this.enqueue(job.slug, reason, true);
|
|
2582
|
+
return;
|
|
2583
|
+
}
|
|
2584
|
+
// A real spawn failure ends this job's capacity wait; it is now handled
|
|
2585
|
+
// by the ordinary failure path, not the capacity episode.
|
|
2586
|
+
this.markCapacityJobStarted(job.slug);
|
|
2414
2587
|
// Record spawn error in run history
|
|
2415
2588
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
2416
2589
|
this.runHistory.recordSpawnError({
|
|
@@ -2443,6 +2616,12 @@ export class JobScheduler {
|
|
|
2443
2616
|
},
|
|
2444
2617
|
});
|
|
2445
2618
|
void this.alertOnConsecutiveFailures(job, failures, errorMsg);
|
|
2619
|
+
}).finally(() => {
|
|
2620
|
+
this.activeSessionSpawns.delete(job.slug);
|
|
2621
|
+
// A spawn may settle after its session has already finished. Recheck
|
|
2622
|
+
// ordinary queued work then; capacity waits keep their 15-second pace.
|
|
2623
|
+
if (this.running && this.queue.length && !this.queue[0].capacityWait)
|
|
2624
|
+
this.processQueue();
|
|
2446
2625
|
});
|
|
2447
2626
|
return sessionName;
|
|
2448
2627
|
}
|
|
@@ -3138,21 +3317,15 @@ export class JobScheduler {
|
|
|
3138
3317
|
return;
|
|
3139
3318
|
this.failureAlertsInFlight.add(job.slug);
|
|
3140
3319
|
try {
|
|
3141
|
-
const
|
|
3142
|
-
let alertText;
|
|
3143
|
-
if (isSessionBlocked) {
|
|
3144
|
-
alertText = this.buildSessionBlockedAlert(job, failures, error);
|
|
3145
|
-
}
|
|
3146
|
-
else {
|
|
3147
|
-
alertText = `*Job Alert: ${job.name}*\n\n${failures} consecutive failures.\nLast error: ${error}\nPriority: ${job.priority}`;
|
|
3148
|
-
}
|
|
3320
|
+
const alertText = jobFailureNotice(job.name, failures, error);
|
|
3149
3321
|
alertState.lastAttemptAt = new Date(now).toISOString();
|
|
3150
3322
|
const attemptId = crypto.randomUUID();
|
|
3151
3323
|
alertState.activeAttemptId = attemptId;
|
|
3152
3324
|
this.state.set(stateKey, alertState);
|
|
3153
|
-
const
|
|
3154
|
-
|
|
3155
|
-
|
|
3325
|
+
const noticeTopic = job.topicId ?? this.state.get('agent-attention-topic');
|
|
3326
|
+
const send = this.telegram && noticeTopic
|
|
3327
|
+
? () => this.telegram.sendToTopic(noticeTopic, alertText).then(() => undefined)
|
|
3328
|
+
: !this.telegram && this.messenger
|
|
3156
3329
|
? () => this.messenger.send({ userId: 'system', content: alertText }).then(() => undefined)
|
|
3157
3330
|
: null;
|
|
3158
3331
|
if (!send) {
|
|
@@ -3248,49 +3421,6 @@ export class JobScheduler {
|
|
|
3248
3421
|
impact: `Job ${job.slug} remains failed and its operator alert remains undelivered; retry is scheduled for ${alertState.nextEligibleAt}`,
|
|
3249
3422
|
});
|
|
3250
3423
|
}
|
|
3251
|
-
/**
|
|
3252
|
-
* Build an intelligent "Job Blocked" notification when a job can't start
|
|
3253
|
-
* because all session slots are occupied. Includes session diagnostics,
|
|
3254
|
-
* staleness detection, memory pressure, and actionable suggestions.
|
|
3255
|
-
*/
|
|
3256
|
-
buildSessionBlockedAlert(job, failures, error) {
|
|
3257
|
-
const diagnostics = this.sessionManager.getSessionDiagnostics();
|
|
3258
|
-
const lines = [];
|
|
3259
|
-
lines.push(`*Job Blocked: ${job.name}*`);
|
|
3260
|
-
lines.push('');
|
|
3261
|
-
lines.push(`Could not start — all ${diagnostics.maxSessions} session slots are in use.`);
|
|
3262
|
-
lines.push(`(${failures} consecutive attempts blocked)`);
|
|
3263
|
-
lines.push('');
|
|
3264
|
-
// Session list with ages
|
|
3265
|
-
lines.push('*Running sessions:*');
|
|
3266
|
-
for (const s of diagnostics.sessions) {
|
|
3267
|
-
const age = s.ageMinutes >= 60
|
|
3268
|
-
? `${Math.floor(s.ageMinutes / 60)}h ${s.ageMinutes % 60}m`
|
|
3269
|
-
: `${s.ageMinutes}m`;
|
|
3270
|
-
const staleFlag = s.isStale ? ' ⚠️' : '';
|
|
3271
|
-
const jobTag = s.jobSlug ? ` (${s.jobSlug})` : '';
|
|
3272
|
-
lines.push(`• ${s.name}${jobTag} — ${age}${staleFlag}`);
|
|
3273
|
-
if (s.staleReason) {
|
|
3274
|
-
lines.push(` └ ${s.staleReason}`);
|
|
3275
|
-
}
|
|
3276
|
-
}
|
|
3277
|
-
lines.push('');
|
|
3278
|
-
// Memory pressure
|
|
3279
|
-
const memEmoji = diagnostics.memoryPressure === 'critical' ? '🔴'
|
|
3280
|
-
: diagnostics.memoryPressure === 'high' ? '🟠'
|
|
3281
|
-
: diagnostics.memoryPressure === 'moderate' ? '🟡'
|
|
3282
|
-
: '🟢';
|
|
3283
|
-
lines.push(`Memory: ${memEmoji} ${diagnostics.memoryUsedPercent}% used (${diagnostics.freeMemoryMB} MB free)`);
|
|
3284
|
-
lines.push('');
|
|
3285
|
-
// Suggestions
|
|
3286
|
-
if (diagnostics.suggestions.length > 0) {
|
|
3287
|
-
lines.push('*Suggestions:*');
|
|
3288
|
-
for (const suggestion of diagnostics.suggestions) {
|
|
3289
|
-
lines.push(`→ ${suggestion}`);
|
|
3290
|
-
}
|
|
3291
|
-
}
|
|
3292
|
-
return lines.join('\n');
|
|
3293
|
-
}
|
|
3294
3424
|
async checkMissedJobs(enabledJobs) {
|
|
3295
3425
|
const now = Date.now();
|
|
3296
3426
|
// Collect all missed jobs first, then sort by priority before triggering.
|