instar 1.3.553 → 1.3.555
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/server.d.ts.map +1 -1
- package/dist/commands/server.js +56 -11
- package/dist/commands/server.js.map +1 -1
- package/dist/config/ConfigDefaults.d.ts.map +1 -1
- package/dist/config/ConfigDefaults.js +78 -85
- package/dist/config/ConfigDefaults.js.map +1 -1
- package/dist/core/PostUpdateMigrator.d.ts +23 -0
- package/dist/core/PostUpdateMigrator.d.ts.map +1 -1
- package/dist/core/PostUpdateMigrator.js +69 -0
- package/dist/core/PostUpdateMigrator.js.map +1 -1
- package/dist/core/devAgentGate.d.ts +27 -0
- package/dist/core/devAgentGate.d.ts.map +1 -1
- package/dist/core/devAgentGate.js +32 -0
- package/dist/core/devAgentGate.js.map +1 -1
- package/dist/core/devGatedFeatures.d.ts.map +1 -1
- package/dist/core/devGatedFeatures.js +66 -35
- package/dist/core/devGatedFeatures.js.map +1 -1
- package/dist/core/types.d.ts +26 -0
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js.map +1 -1
- package/dist/scheduler/JobLeaseClaimStore.d.ts +105 -0
- package/dist/scheduler/JobLeaseClaimStore.d.ts.map +1 -0
- package/dist/scheduler/JobLeaseClaimStore.js +165 -0
- package/dist/scheduler/JobLeaseClaimStore.js.map +1 -0
- package/dist/scheduler/JobLeaseCutoverGate.d.ts +73 -0
- package/dist/scheduler/JobLeaseCutoverGate.d.ts.map +1 -0
- package/dist/scheduler/JobLeaseCutoverGate.js +66 -0
- package/dist/scheduler/JobLeaseCutoverGate.js.map +1 -0
- package/dist/scheduler/JobScheduler.d.ts +53 -0
- package/dist/scheduler/JobScheduler.d.ts.map +1 -1
- package/dist/scheduler/JobScheduler.js +136 -20
- package/dist/scheduler/JobScheduler.js.map +1 -1
- package/dist/server/routes.d.ts.map +1 -1
- package/dist/server/routes.js +10 -2
- package/dist/server/routes.js.map +1 -1
- package/package.json +1 -1
- package/src/data/builtin-manifest.json +64 -64
- package/upgrades/1.3.554.md +79 -0
- package/upgrades/1.3.555.md +71 -0
- package/upgrades/side-effects/mm-stores-devgate.md +92 -0
- package/upgrades/side-effects/ws43-journal-lease-cutover.md +132 -0
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* JobLeaseClaimStore — WS4.3 durable, epoch-fenced job-claim leases.
|
|
3
|
+
*
|
|
4
|
+
* (MULTI-MACHINE-SEAMLESSNESS-SPEC §WS4.3, "Cutover discipline".)
|
|
5
|
+
*
|
|
6
|
+
* The journal-backed upgrade of the best-effort `JobClaimManager` bus
|
|
7
|
+
* broadcast. A claim is a LEASE with a fenced ownership epoch: a receiver
|
|
8
|
+
* rejects a stale-epoch claim (invariant-4 epoch fencing), so a demoted
|
|
9
|
+
* machine's late claim cannot steal a job from the current lease-holder, and a
|
|
10
|
+
* partition double-run is structurally prevented when the journal is reachable.
|
|
11
|
+
*
|
|
12
|
+
* Claim records store METADATA ONLY — machine id, job slug, epoch, timestamps —
|
|
13
|
+
* never job payloads (the spec's explicit privacy bound). The store is the
|
|
14
|
+
* local materialized view of the replicated claim kind; the journal carries
|
|
15
|
+
* each record to peers and feeds them back through `applyRemote`.
|
|
16
|
+
*
|
|
17
|
+
* This store does NOT decide WHEN to use the journal path — that is the
|
|
18
|
+
* `JobLeaseCutoverGate`. The store only manages the leases once the gate has
|
|
19
|
+
* selected the journal path.
|
|
20
|
+
*/
|
|
21
|
+
import crypto from 'node:crypto';
|
|
22
|
+
import fs from 'node:fs';
|
|
23
|
+
import path from 'node:path';
|
|
24
|
+
const DEFAULT_LEASE_TIMEOUT_MS = 30 * 60_000;
|
|
25
|
+
const CLAIMS_FILE = 'job-lease-claims.json';
|
|
26
|
+
export class JobLeaseClaimStore {
|
|
27
|
+
machineId;
|
|
28
|
+
claimsDir;
|
|
29
|
+
defaultLeaseTimeoutMs;
|
|
30
|
+
now;
|
|
31
|
+
claims = new Map(); // keyed by jobSlug
|
|
32
|
+
constructor(config) {
|
|
33
|
+
this.machineId = config.machineId;
|
|
34
|
+
this.defaultLeaseTimeoutMs = config.defaultLeaseTimeoutMs ?? DEFAULT_LEASE_TIMEOUT_MS;
|
|
35
|
+
this.now = config.now ?? (() => Date.now());
|
|
36
|
+
this.claimsDir = path.join(config.stateDir, 'state');
|
|
37
|
+
if (!fs.existsSync(this.claimsDir)) {
|
|
38
|
+
fs.mkdirSync(this.claimsDir, { recursive: true });
|
|
39
|
+
}
|
|
40
|
+
this.load();
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Attempt to take a lease on a job at the given ownership epoch.
|
|
44
|
+
*
|
|
45
|
+
* Returns the new lease on success, or a rejection when a peer holds a live,
|
|
46
|
+
* non-stale lease. Idempotent: re-claiming a slug this machine already leases
|
|
47
|
+
* (at the same-or-newer epoch) returns the existing lease.
|
|
48
|
+
*/
|
|
49
|
+
tryClaim(jobSlug, epoch, timeoutMs) {
|
|
50
|
+
this.pruneExpired();
|
|
51
|
+
const existing = this.claims.get(jobSlug);
|
|
52
|
+
if (existing && !existing.completed && !this.isExpired(existing)) {
|
|
53
|
+
if (existing.machineId === this.machineId) {
|
|
54
|
+
return { ok: false, reason: 'already-own', claim: existing };
|
|
55
|
+
}
|
|
56
|
+
// A peer holds a live lease. We may only supersede it with a STRICTLY
|
|
57
|
+
// newer epoch (the peer was demoted; our epoch advanced). A same/older
|
|
58
|
+
// epoch is fenced out — never steal from a current lease-holder.
|
|
59
|
+
if (epoch <= existing.epoch) {
|
|
60
|
+
return { ok: false, reason: 'held-by-peer', heldBy: existing.machineId };
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
const claim = {
|
|
64
|
+
claimId: `lease_${crypto.randomBytes(8).toString('hex')}`,
|
|
65
|
+
jobSlug,
|
|
66
|
+
machineId: this.machineId,
|
|
67
|
+
epoch,
|
|
68
|
+
claimedAt: new Date(this.now()).toISOString(),
|
|
69
|
+
expiresAt: new Date(this.now() + (timeoutMs ?? this.defaultLeaseTimeoutMs)).toISOString(),
|
|
70
|
+
completed: false,
|
|
71
|
+
};
|
|
72
|
+
this.claims.set(jobSlug, claim);
|
|
73
|
+
this.save();
|
|
74
|
+
return { ok: true, claim };
|
|
75
|
+
}
|
|
76
|
+
/** Mark our lease on a slug complete (releases it for the next tick). */
|
|
77
|
+
completeClaim(jobSlug, result) {
|
|
78
|
+
const claim = this.claims.get(jobSlug);
|
|
79
|
+
if (!claim || claim.machineId !== this.machineId)
|
|
80
|
+
return;
|
|
81
|
+
claim.completed = true;
|
|
82
|
+
claim.result = result;
|
|
83
|
+
claim.completedAt = new Date(this.now()).toISOString();
|
|
84
|
+
this.save();
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Apply a claim record received from a peer over the journal. Epoch fencing:
|
|
88
|
+
* a record whose epoch is OLDER than the one we already hold for the slug is
|
|
89
|
+
* rejected (returns false) — a demoted machine's stale claim never overwrites
|
|
90
|
+
* a fresher lease. Returns true if the record was applied.
|
|
91
|
+
*/
|
|
92
|
+
applyRemote(record) {
|
|
93
|
+
// Never let a remote record masquerade as our own machine's authority.
|
|
94
|
+
if (record.machineId === this.machineId)
|
|
95
|
+
return false;
|
|
96
|
+
const existing = this.claims.get(record.jobSlug);
|
|
97
|
+
if (existing && !existing.completed && !this.isExpired(existing)) {
|
|
98
|
+
// Strict epoch fence — equal epoch keeps the incumbent (first-writer-wins
|
|
99
|
+
// is resolved upstream by the journal's ordering; we never flip on a tie).
|
|
100
|
+
if (record.epoch <= existing.epoch)
|
|
101
|
+
return false;
|
|
102
|
+
}
|
|
103
|
+
this.claims.set(record.jobSlug, { ...record });
|
|
104
|
+
this.save();
|
|
105
|
+
return true;
|
|
106
|
+
}
|
|
107
|
+
/** True if a peer holds a live, non-stale lease on the slug. */
|
|
108
|
+
hasRemoteClaim(jobSlug) {
|
|
109
|
+
this.pruneExpired();
|
|
110
|
+
const claim = this.claims.get(jobSlug);
|
|
111
|
+
if (!claim || claim.completed || claim.machineId === this.machineId)
|
|
112
|
+
return false;
|
|
113
|
+
return !this.isExpired(claim);
|
|
114
|
+
}
|
|
115
|
+
getClaim(jobSlug) {
|
|
116
|
+
this.pruneExpired();
|
|
117
|
+
return this.claims.get(jobSlug);
|
|
118
|
+
}
|
|
119
|
+
getActiveClaims() {
|
|
120
|
+
this.pruneExpired();
|
|
121
|
+
return Array.from(this.claims.values()).filter((c) => !c.completed);
|
|
122
|
+
}
|
|
123
|
+
getAllClaims() {
|
|
124
|
+
return Array.from(this.claims.values());
|
|
125
|
+
}
|
|
126
|
+
/** Remove expired (uncompleted past expiry) + old completed records. */
|
|
127
|
+
pruneExpired() {
|
|
128
|
+
const now = this.now();
|
|
129
|
+
let pruned = 0;
|
|
130
|
+
for (const [slug, claim] of this.claims) {
|
|
131
|
+
if (claim.completed) {
|
|
132
|
+
const completedAt = claim.completedAt ? new Date(claim.completedAt).getTime() : now;
|
|
133
|
+
if (now - completedAt > 60 * 60_000) {
|
|
134
|
+
this.claims.delete(slug);
|
|
135
|
+
pruned++;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
else if (this.isExpired(claim)) {
|
|
139
|
+
this.claims.delete(slug);
|
|
140
|
+
pruned++;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
if (pruned > 0)
|
|
144
|
+
this.save();
|
|
145
|
+
return pruned;
|
|
146
|
+
}
|
|
147
|
+
isExpired(claim) {
|
|
148
|
+
return this.now() > new Date(claim.expiresAt).getTime();
|
|
149
|
+
}
|
|
150
|
+
load() {
|
|
151
|
+
try {
|
|
152
|
+
const data = JSON.parse(fs.readFileSync(path.join(this.claimsDir, CLAIMS_FILE), 'utf-8'));
|
|
153
|
+
this.claims.clear();
|
|
154
|
+
for (const c of data)
|
|
155
|
+
this.claims.set(c.jobSlug, c);
|
|
156
|
+
}
|
|
157
|
+
catch {
|
|
158
|
+
/* @silent-fallback-ok — no ledger yet; start empty */
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
save() {
|
|
162
|
+
fs.writeFileSync(path.join(this.claimsDir, CLAIMS_FILE), JSON.stringify(Array.from(this.claims.values()), null, 2) + '\n');
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
//# sourceMappingURL=JobLeaseClaimStore.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"JobLeaseClaimStore.js","sourceRoot":"","sources":["../../src/scheduler/JobLeaseClaimStore.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAoC7B,MAAM,wBAAwB,GAAG,EAAE,GAAG,MAAM,CAAC;AAC7C,MAAM,WAAW,GAAG,uBAAuB,CAAC;AAU5C,MAAM,OAAO,kBAAkB;IACrB,SAAS,CAAS;IAClB,SAAS,CAAS;IAClB,qBAAqB,CAAS;IAC9B,GAAG,CAAe;IAClB,MAAM,GAA+B,IAAI,GAAG,EAAE,CAAC,CAAC,mBAAmB;IAE3E,YAAY,MAAgC;QAC1C,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;QAClC,IAAI,CAAC,qBAAqB,GAAG,MAAM,CAAC,qBAAqB,IAAI,wBAAwB,CAAC;QACtF,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAC5C,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACrD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YACnC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACpD,CAAC;QACD,IAAI,CAAC,IAAI,EAAE,CAAC;IACd,CAAC;IAED;;;;;;OAMG;IACH,QAAQ,CAAC,OAAe,EAAE,KAAa,EAAE,SAAkB;QACzD,IAAI,CAAC,YAAY,EAAE,CAAC;QACpB,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAE1C,IAAI,QAAQ,IAAI,CAAC,QAAQ,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjE,IAAI,QAAQ,CAAC,SAAS,KAAK,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC1C,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;YAC/D,CAAC;YACD,sEAAsE;YACtE,uEAAuE;YACvE,iEAAiE;YACjE,IAAI,KAAK,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;gBAC5B,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC3E,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAkB;YAC3B,OAAO,EAAE,SAAS,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;YACzD,OAAO;YACP,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,KAAK;YACL,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,WAAW,EAAE;YAC7C,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,SAAS,IAAI,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC,WAAW,EAAE;YACzF,SAAS,EAAE,KAAK;SACjB,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAChC,IAAI,CAAC,IAAI,EAAE,CAAC;QACZ,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IAC7B,CAAC;IAED,yEAAyE;IACzE,aAAa,CAAC,OAAe,EAAE,MAA6B;QAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACvC,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,SAAS,KAAK,IAAI,CAAC,SAAS;YAAE,OAAO;QACzD,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC;QACvB,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC;QACtB,KAAK,CAAC,WAAW,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QACvD,IAAI,CAAC,IAAI,EAAE,CAAC;IACd,CAAC;IAED;;;;;OAKG;IACH,WAAW,CAAC,MAAqB;QAC/B,uEAAuE;QACvE,IAAI,MAAM,CAAC,SAAS,KAAK,IAAI,CAAC,SAAS;YAAE,OAAO,KAAK,CAAC;QACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACjD,IAAI,QAAQ,IAAI,CAAC,QAAQ,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjE,0EAA0E;YAC1E,2EAA2E;YAC3E,IAAI,MAAM,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK;gBAAE,OAAO,KAAK,CAAC;QACnD,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;QAC/C,IAAI,CAAC,IAAI,EAAE,CAAC;QACZ,OAAO,IAAI,CAAC;IACd,CAAC;IAED,gEAAgE;IAChE,cAAc,CAAC,OAAe;QAC5B,IAAI,CAAC,YAAY,EAAE,CAAC;QACpB,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACvC,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,SAAS,KAAK,IAAI,CAAC,SAAS;YAAE,OAAO,KAAK,CAAC;QAClF,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,QAAQ,CAAC,OAAe;QACtB,IAAI,CAAC,YAAY,EAAE,CAAC;QACpB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAClC,CAAC;IAED,eAAe;QACb,IAAI,CAAC,YAAY,EAAE,CAAC;QACpB,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACtE,CAAC;IAED,YAAY;QACV,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAC1C,CAAC;IAED,wEAAwE;IACxE,YAAY;QACV,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACxC,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;gBACpB,MAAM,WAAW,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;gBACpF,IAAI,GAAG,GAAG,WAAW,GAAG,EAAE,GAAG,MAAM,EAAE,CAAC;oBACpC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;oBACzB,MAAM,EAAE,CAAC;gBACX,CAAC;YACH,CAAC;iBAAM,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;gBACjC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBACzB,MAAM,EAAE,CAAC;YACX,CAAC;QACH,CAAC;QACD,IAAI,MAAM,GAAG,CAAC;YAAE,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,SAAS,CAAC,KAAoB;QACpC,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC;IAC1D,CAAC;IAEO,IAAI;QACV,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CACrB,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,EAAE,OAAO,CAAC,CAC9C,CAAC;YACrB,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YACpB,KAAK,MAAM,CAAC,IAAI,IAAI;gBAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACtD,CAAC;QAAC,MAAM,CAAC;YACP,sDAAsD;QACxD,CAAC;IACH,CAAC;IAEO,IAAI;QACV,EAAE,CAAC,aAAa,CACd,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,EACtC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CACjE,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* JobLeaseCutoverGate — WS4.3 journal-lease cutover discipline.
|
|
3
|
+
*
|
|
4
|
+
* (MULTI-MACHINE-SEAMLESSNESS-SPEC §WS4.3, "Cutover discipline".)
|
|
5
|
+
*
|
|
6
|
+
* Job claims upgrade from the legacy best-effort AgentBus broadcast
|
|
7
|
+
* (`JobClaimManager`) to a durable, epoch-fenced lease carried over the
|
|
8
|
+
* replicated journal. The named migration hazard the spec closes is two
|
|
9
|
+
* NON-INTEROPERATING claim mechanisms running at once: one machine leasing
|
|
10
|
+
* via the journal while a peer broadcasts via the bus for the SAME job set.
|
|
11
|
+
* There must never be a window where both are live for a job.
|
|
12
|
+
*
|
|
13
|
+
* This gate is the SINGLE decision point that resolves which path a job uses.
|
|
14
|
+
* It engages the journal-lease path ONLY when invariant-5 flag coherence holds:
|
|
15
|
+
* 1. the dark flag `multiMachine.seamlessness.ws43JournalLease` is on;
|
|
16
|
+
* 2. the pool has at least one peer (single-machine = strict no-op, the
|
|
17
|
+
* legacy path is byte-for-byte today's behavior);
|
|
18
|
+
* 3. EVERY online peer advertises the `ws43JournalLease` capability in its
|
|
19
|
+
* `seamlessnessFlags` heartbeat — a peer that does not advertise it (an
|
|
20
|
+
* older version, or the flag off there) keeps the WHOLE pool on the bus.
|
|
21
|
+
*
|
|
22
|
+
* A peer with NO seamlessnessFlags field is an older version → treated as NOT
|
|
23
|
+
* advertising (the conservative side). Absent = non-participant, per the
|
|
24
|
+
* spec's invariant-5 ("absent = the peer predates this spec = flag-state-off").
|
|
25
|
+
*
|
|
26
|
+
* Pure + deterministic: the gate holds no state and performs no IO, so it is
|
|
27
|
+
* trivially testable and can be re-read live at every spawn boundary (a
|
|
28
|
+
* mid-run flag flip or a peer going dark cuts the pool back to the bus
|
|
29
|
+
* immediately, never stranding a half-migrated job set).
|
|
30
|
+
*/
|
|
31
|
+
/** A peer's advertised seamlessness capability, read from its heartbeat. */
|
|
32
|
+
export interface CutoverPeerAdvert {
|
|
33
|
+
machineId: string;
|
|
34
|
+
/** Whether the peer is currently online. An offline peer is not a participant
|
|
35
|
+
* in coherence (it cannot lease or broadcast right now) and is excluded. */
|
|
36
|
+
online?: boolean;
|
|
37
|
+
/** The peer's advertised WS4.3 journal-lease capability. ABSENT/false = the
|
|
38
|
+
* peer does not advertise it (older version or flag off there). */
|
|
39
|
+
ws43JournalLease?: boolean;
|
|
40
|
+
}
|
|
41
|
+
/** Inputs to the cutover decision, all read LIVE at each evaluation. */
|
|
42
|
+
export interface CutoverGateInput {
|
|
43
|
+
/** `multiMachine.seamlessness.ws43JournalLease` resolved live. */
|
|
44
|
+
enabled: boolean;
|
|
45
|
+
/** `multiMachine.seamlessness.ws43JournalLeaseDryRun` resolved live. When
|
|
46
|
+
* true AND the gate would otherwise select 'journal', the journal claim is
|
|
47
|
+
* LOGGED as intended but the legacy bus path still runs (the WS-wide dry-run
|
|
48
|
+
* posture: "log intended refusals/claims"). */
|
|
49
|
+
dryRun: boolean;
|
|
50
|
+
/** Every KNOWN peer (excluding self) with its advertised capability. An empty
|
|
51
|
+
* list means single-machine (no peers) → strict no-op. */
|
|
52
|
+
peers: ReadonlyArray<CutoverPeerAdvert>;
|
|
53
|
+
}
|
|
54
|
+
/** Which claim mechanism a job must use this evaluation. */
|
|
55
|
+
export type ClaimPath = 'journal' | 'bus';
|
|
56
|
+
export interface CutoverDecision {
|
|
57
|
+
/** The claim path the scheduler must use. Exactly one — never both. */
|
|
58
|
+
path: ClaimPath;
|
|
59
|
+
/** When true, the journal claim should be LOGGED as intended but the bus path
|
|
60
|
+
* actually runs (dry-run). Only ever true alongside `path: 'bus'`. */
|
|
61
|
+
journalDryRun: boolean;
|
|
62
|
+
/** Machine-readable reason, for audit + the status surface. */
|
|
63
|
+
reason: 'flag-off' | 'single-machine' | 'peers-incoherent' | 'dry-run' | 'journal-coherent';
|
|
64
|
+
/** Online peers that do NOT advertise the capability (empty ⇒ coherent). The
|
|
65
|
+
* reason a coherent-pool cutover was withheld, surfaced for the operator. */
|
|
66
|
+
incoherentPeers: string[];
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Resolve the claim path for the current pool state. Pure: same input ⇒ same
|
|
70
|
+
* output, no IO, no clock.
|
|
71
|
+
*/
|
|
72
|
+
export declare function decideClaimPath(input: CutoverGateInput): CutoverDecision;
|
|
73
|
+
//# sourceMappingURL=JobLeaseCutoverGate.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"JobLeaseCutoverGate.d.ts","sourceRoot":"","sources":["../../src/scheduler/JobLeaseCutoverGate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,4EAA4E;AAC5E,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB;iFAC6E;IAC7E,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;wEACoE;IACpE,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,wEAAwE;AACxE,MAAM,WAAW,gBAAgB;IAC/B,kEAAkE;IAClE,OAAO,EAAE,OAAO,CAAC;IACjB;;;oDAGgD;IAChD,MAAM,EAAE,OAAO,CAAC;IAChB;+DAC2D;IAC3D,KAAK,EAAE,aAAa,CAAC,iBAAiB,CAAC,CAAC;CACzC;AAED,4DAA4D;AAC5D,MAAM,MAAM,SAAS,GAAG,SAAS,GAAG,KAAK,CAAC;AAE1C,MAAM,WAAW,eAAe;IAC9B,uEAAuE;IACvE,IAAI,EAAE,SAAS,CAAC;IAChB;2EACuE;IACvE,aAAa,EAAE,OAAO,CAAC;IACvB,+DAA+D;IAC/D,MAAM,EACF,UAAU,GACV,gBAAgB,GAChB,kBAAkB,GAClB,SAAS,GACT,kBAAkB,CAAC;IACvB;kFAC8E;IAC9E,eAAe,EAAE,MAAM,EAAE,CAAC;CAC3B;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,gBAAgB,GAAG,eAAe,CAoCxE"}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* JobLeaseCutoverGate — WS4.3 journal-lease cutover discipline.
|
|
3
|
+
*
|
|
4
|
+
* (MULTI-MACHINE-SEAMLESSNESS-SPEC §WS4.3, "Cutover discipline".)
|
|
5
|
+
*
|
|
6
|
+
* Job claims upgrade from the legacy best-effort AgentBus broadcast
|
|
7
|
+
* (`JobClaimManager`) to a durable, epoch-fenced lease carried over the
|
|
8
|
+
* replicated journal. The named migration hazard the spec closes is two
|
|
9
|
+
* NON-INTEROPERATING claim mechanisms running at once: one machine leasing
|
|
10
|
+
* via the journal while a peer broadcasts via the bus for the SAME job set.
|
|
11
|
+
* There must never be a window where both are live for a job.
|
|
12
|
+
*
|
|
13
|
+
* This gate is the SINGLE decision point that resolves which path a job uses.
|
|
14
|
+
* It engages the journal-lease path ONLY when invariant-5 flag coherence holds:
|
|
15
|
+
* 1. the dark flag `multiMachine.seamlessness.ws43JournalLease` is on;
|
|
16
|
+
* 2. the pool has at least one peer (single-machine = strict no-op, the
|
|
17
|
+
* legacy path is byte-for-byte today's behavior);
|
|
18
|
+
* 3. EVERY online peer advertises the `ws43JournalLease` capability in its
|
|
19
|
+
* `seamlessnessFlags` heartbeat — a peer that does not advertise it (an
|
|
20
|
+
* older version, or the flag off there) keeps the WHOLE pool on the bus.
|
|
21
|
+
*
|
|
22
|
+
* A peer with NO seamlessnessFlags field is an older version → treated as NOT
|
|
23
|
+
* advertising (the conservative side). Absent = non-participant, per the
|
|
24
|
+
* spec's invariant-5 ("absent = the peer predates this spec = flag-state-off").
|
|
25
|
+
*
|
|
26
|
+
* Pure + deterministic: the gate holds no state and performs no IO, so it is
|
|
27
|
+
* trivially testable and can be re-read live at every spawn boundary (a
|
|
28
|
+
* mid-run flag flip or a peer going dark cuts the pool back to the bus
|
|
29
|
+
* immediately, never stranding a half-migrated job set).
|
|
30
|
+
*/
|
|
31
|
+
/**
|
|
32
|
+
* Resolve the claim path for the current pool state. Pure: same input ⇒ same
|
|
33
|
+
* output, no IO, no clock.
|
|
34
|
+
*/
|
|
35
|
+
export function decideClaimPath(input) {
|
|
36
|
+
// (1) Flag off → legacy bus, byte-for-byte today's behavior.
|
|
37
|
+
if (!input.enabled) {
|
|
38
|
+
return { path: 'bus', journalDryRun: false, reason: 'flag-off', incoherentPeers: [] };
|
|
39
|
+
}
|
|
40
|
+
// Consider only ONLINE peers — an offline peer cannot claim/broadcast right
|
|
41
|
+
// now, so it neither blocks coherence nor participates in it (it will be
|
|
42
|
+
// re-evaluated when it returns and re-advertises).
|
|
43
|
+
const onlinePeers = input.peers.filter((p) => p.online !== false);
|
|
44
|
+
// (2) No online peers → single-machine (or all-peers-dark) strict no-op. The
|
|
45
|
+
// legacy path is the no-op baseline; with no peers there is no one to
|
|
46
|
+
// double-run against, so the cutover is irrelevant.
|
|
47
|
+
if (onlinePeers.length === 0) {
|
|
48
|
+
return { path: 'bus', journalDryRun: false, reason: 'single-machine', incoherentPeers: [] };
|
|
49
|
+
}
|
|
50
|
+
// (3) Flag coherence: EVERY online peer must advertise ws43JournalLease.
|
|
51
|
+
// Absent/false = does not advertise (the conservative side — invariant-5).
|
|
52
|
+
const incoherentPeers = onlinePeers
|
|
53
|
+
.filter((p) => p.ws43JournalLease !== true)
|
|
54
|
+
.map((p) => p.machineId);
|
|
55
|
+
if (incoherentPeers.length > 0) {
|
|
56
|
+
// Mixed pool → stay on the bus for the WHOLE job set. Never lease while a
|
|
57
|
+
// peer broadcasts: the named migration hazard. Degrade conservatively.
|
|
58
|
+
return { path: 'bus', journalDryRun: false, reason: 'peers-incoherent', incoherentPeers };
|
|
59
|
+
}
|
|
60
|
+
// Coherent pool. Dry-run still runs the bus path but logs the intended claim.
|
|
61
|
+
if (input.dryRun) {
|
|
62
|
+
return { path: 'bus', journalDryRun: true, reason: 'dry-run', incoherentPeers: [] };
|
|
63
|
+
}
|
|
64
|
+
return { path: 'journal', journalDryRun: false, reason: 'journal-coherent', incoherentPeers: [] };
|
|
65
|
+
}
|
|
66
|
+
//# sourceMappingURL=JobLeaseCutoverGate.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"JobLeaseCutoverGate.js","sourceRoot":"","sources":["../../src/scheduler/JobLeaseCutoverGate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAgDH;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,KAAuB;IACrD,6DAA6D;IAC7D,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;QACnB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,eAAe,EAAE,EAAE,EAAE,CAAC;IACxF,CAAC;IAED,4EAA4E;IAC5E,yEAAyE;IACzE,mDAAmD;IACnD,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC;IAElE,6EAA6E;IAC7E,sEAAsE;IACtE,oDAAoD;IACpD,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,EAAE,gBAAgB,EAAE,eAAe,EAAE,EAAE,EAAE,CAAC;IAC9F,CAAC;IAED,yEAAyE;IACzE,2EAA2E;IAC3E,MAAM,eAAe,GAAG,WAAW;SAChC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB,KAAK,IAAI,CAAC;SAC1C,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAE3B,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/B,0EAA0E;QAC1E,uEAAuE;QACvE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,EAAE,kBAAkB,EAAE,eAAe,EAAE,CAAC;IAC5F,CAAC;IAED,8EAA8E;IAC9E,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACjB,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,eAAe,EAAE,EAAE,EAAE,CAAC;IACtF,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,EAAE,kBAAkB,EAAE,eAAe,EAAE,EAAE,EAAE,CAAC;AACpG,CAAC"}
|
|
@@ -17,6 +17,8 @@ import type { CanRunJobResult, IntelligenceProvider, MessagingAdapter } from '..
|
|
|
17
17
|
import type { JobDefinition, JobSchedulerConfig, JobPriority } from '../core/types.js';
|
|
18
18
|
import type { TelegramAdapter } from '../messaging/TelegramAdapter.js';
|
|
19
19
|
import type { JobClaimManager } from './JobClaimManager.js';
|
|
20
|
+
import type { JobLeaseClaimStore } from './JobLeaseClaimStore.js';
|
|
21
|
+
import { type CutoverGateInput, type CutoverDecision } from './JobLeaseCutoverGate.js';
|
|
20
22
|
import type { TopicMemory } from '../memory/TopicMemory.js';
|
|
21
23
|
interface QueuedJob {
|
|
22
24
|
slug: string;
|
|
@@ -66,6 +68,26 @@ export declare class JobScheduler {
|
|
|
66
68
|
private quotaTracker;
|
|
67
69
|
/** Optional job claim manager for multi-machine deduplication (Phase 4C) */
|
|
68
70
|
private claimManager;
|
|
71
|
+
/**
|
|
72
|
+
* MULTI-MACHINE-SEAMLESSNESS-SPEC §WS4.3 journal-lease cutover. The durable,
|
|
73
|
+
* epoch-fenced lease store — the journal-backed UPGRADE of the best-effort
|
|
74
|
+
* AgentBus broadcast in `claimManager`. It is consulted for a job ONLY when
|
|
75
|
+
* `journalLeaseGate` selects the 'journal' path (flag on + pool coherent). The
|
|
76
|
+
* gate guarantees the two mechanisms are never both live for a job set (the
|
|
77
|
+
* named migration hazard). When unset, the scheduler stays on the legacy bus
|
|
78
|
+
* path (byte-for-byte today's behavior).
|
|
79
|
+
*/
|
|
80
|
+
private leaseClaimStore;
|
|
81
|
+
/**
|
|
82
|
+
* Live provider for the cutover-gate input (flag + dry-run + the pool peers'
|
|
83
|
+
* advertised ws43JournalLease capability + this machine's lease epoch). Read
|
|
84
|
+
* LIVE at each claim boundary so a flag flip / a peer going dark cuts the pool
|
|
85
|
+
* back to the bus immediately. When unset, the gate is never consulted and the
|
|
86
|
+
* legacy bus path runs (strict no-op).
|
|
87
|
+
*/
|
|
88
|
+
private journalLeaseGate;
|
|
89
|
+
/** Optional sink for the most-recent cutover decision (status/audit surface). */
|
|
90
|
+
private onCutoverDecision;
|
|
69
91
|
/**
|
|
70
92
|
* UNIFIED-SESSION-LIFECYCLE §P0 #9 (SE-8): function returning cumulative
|
|
71
93
|
* wall-time-asleep in milliseconds during a half-open window [startMs, endMs).
|
|
@@ -130,6 +152,37 @@ export declare class JobScheduler {
|
|
|
130
152
|
* and skip jobs already claimed by other machines.
|
|
131
153
|
*/
|
|
132
154
|
setJobClaimManager(manager: JobClaimManager): void;
|
|
155
|
+
/**
|
|
156
|
+
* MULTI-MACHINE-SEAMLESSNESS-SPEC §WS4.3 — inject the journal-lease cutover.
|
|
157
|
+
*
|
|
158
|
+
* `store` is the durable epoch-fenced lease store; `gateInput` returns the
|
|
159
|
+
* LIVE cutover-gate input (flag, dry-run, pool peers' advertised capability,
|
|
160
|
+
* and this machine's current lease epoch) — read fresh at each claim boundary,
|
|
161
|
+
* never cached. `onDecision` (optional) receives the resolved decision per job
|
|
162
|
+
* for the status/audit surface.
|
|
163
|
+
*
|
|
164
|
+
* When omitted, the scheduler never consults the journal path — the legacy
|
|
165
|
+
* AgentBus `claimManager` runs unchanged (single-machine / flag-off no-op).
|
|
166
|
+
*/
|
|
167
|
+
setJournalLeaseCutover(store: JobLeaseClaimStore, gateInput: () => CutoverGateInput & {
|
|
168
|
+
epoch: number;
|
|
169
|
+
}, onDecision?: (slug: string, decision: CutoverDecision) => void): void;
|
|
170
|
+
/**
|
|
171
|
+
* Resolve which claim mechanism a job uses RIGHT NOW. Pure read of the live
|
|
172
|
+
* gate input. Returns null when the cutover is not wired (legacy bus only).
|
|
173
|
+
* Exposed for the scheduler's claim seams (check + claim + complete) so a
|
|
174
|
+
* single evaluation drives all three — never two mechanisms for one job.
|
|
175
|
+
*/
|
|
176
|
+
private resolveClaimPath;
|
|
177
|
+
/**
|
|
178
|
+
* Release a claim for a completed job across BOTH mechanisms. The path that
|
|
179
|
+
* actually took the claim may differ from the path the gate would pick at
|
|
180
|
+
* completion time (a mid-run flag flip), so we release both — each store's
|
|
181
|
+
* completeClaim is a strict no-op when this machine does not own the slug's
|
|
182
|
+
* live record, so releasing the unused store is harmless. The journal release
|
|
183
|
+
* is synchronous (durable); the bus release is best-effort async.
|
|
184
|
+
*/
|
|
185
|
+
private releaseClaim;
|
|
133
186
|
/**
|
|
134
187
|
* Set local machine identity for machine-scoped job filtering.
|
|
135
188
|
* Jobs with a `machines` field will only run on machines whose ID or name matches.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"JobScheduler.d.ts","sourceRoot":"","sources":["../../src/scheduler/JobScheduler.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAaH,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAGvD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAE7C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,KAAK,EAAE,eAAe,EAAE,oBAAoB,EAAE,gBAAgB,EAAc,MAAM,kBAAkB,CAAC;AAE5G,OAAO,KAAK,EAAE,aAAa,EAAE,kBAAkB,EAAY,WAAW,EAAE,MAAM,kBAAkB,CAAC;AACjG,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,iCAAiC,CAAC;AACvE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAS5D,UAAU,SAAS;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,eAAe;IACvB,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AASD,qBAAa,YAAY;IACvB,OAAO,CAAC,MAAM,CAAqB;IACnC,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,UAAU,CAAgB;IAClC,OAAO,CAAC,IAAI,CAAuB;IACnC,OAAO,CAAC,SAAS,CAAgC;IACjD,OAAO,CAAC,KAAK,CAAmB;IAChC,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,MAAM,CAAS;IAEvB,2DAA2D;IAC3D,OAAO,CAAC,YAAY,CAAkC;IAEtD,2EAA2E;IAC3E,OAAO,CAAC,UAAU,CAAqF;IAEvG,2DAA2D;IAC3D,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAOrC;IAEF,qEAAqE;IACrE,OAAO,CAAC,SAAS,CAAuB;IACxC,OAAO,CAAC,WAAW,CAAuB;IAE1C;;;;OAIG;IACH,SAAS,EAAE,CAAC,QAAQ,EAAE,WAAW,KAAK,OAAO,GAAG,eAAe,CAAc;IAE7E,uDAAuD;IACvD,OAAO,CAAC,SAAS,CAAiC;IAElD,uDAAuD;IACvD,OAAO,CAAC,QAAQ,CAAgC;IAEhD,sEAAsE;IACtE,OAAO,CAAC,YAAY,CAA6B;IAEjD,4EAA4E;IAC5E,OAAO,CAAC,YAAY,CAAgC;
|
|
1
|
+
{"version":3,"file":"JobScheduler.d.ts","sourceRoot":"","sources":["../../src/scheduler/JobScheduler.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAaH,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAGvD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAE7C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,KAAK,EAAE,eAAe,EAAE,oBAAoB,EAAE,gBAAgB,EAAc,MAAM,kBAAkB,CAAC;AAE5G,OAAO,KAAK,EAAE,aAAa,EAAE,kBAAkB,EAAY,WAAW,EAAE,MAAM,kBAAkB,CAAC;AACjG,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,iCAAiC,CAAC;AACvE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAClE,OAAO,EAAmB,KAAK,gBAAgB,EAAE,KAAK,eAAe,EAAE,MAAM,0BAA0B,CAAC;AACxG,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAS5D,UAAU,SAAS;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,eAAe;IACvB,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AASD,qBAAa,YAAY;IACvB,OAAO,CAAC,MAAM,CAAqB;IACnC,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,UAAU,CAAgB;IAClC,OAAO,CAAC,IAAI,CAAuB;IACnC,OAAO,CAAC,SAAS,CAAgC;IACjD,OAAO,CAAC,KAAK,CAAmB;IAChC,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,MAAM,CAAS;IAEvB,2DAA2D;IAC3D,OAAO,CAAC,YAAY,CAAkC;IAEtD,2EAA2E;IAC3E,OAAO,CAAC,UAAU,CAAqF;IAEvG,2DAA2D;IAC3D,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAOrC;IAEF,qEAAqE;IACrE,OAAO,CAAC,SAAS,CAAuB;IACxC,OAAO,CAAC,WAAW,CAAuB;IAE1C;;;;OAIG;IACH,SAAS,EAAE,CAAC,QAAQ,EAAE,WAAW,KAAK,OAAO,GAAG,eAAe,CAAc;IAE7E,uDAAuD;IACvD,OAAO,CAAC,SAAS,CAAiC;IAElD,uDAAuD;IACvD,OAAO,CAAC,QAAQ,CAAgC;IAEhD,sEAAsE;IACtE,OAAO,CAAC,YAAY,CAA6B;IAEjD,4EAA4E;IAC5E,OAAO,CAAC,YAAY,CAAgC;IAEpD;;;;;;;;OAQG;IACH,OAAO,CAAC,eAAe,CAAmC;IAC1D;;;;;;OAMG;IACH,OAAO,CAAC,gBAAgB,CAA6D;IACrF,iFAAiF;IACjF,OAAO,CAAC,iBAAiB,CAAoE;IAC7F;;;;;;OAMG;IACH,OAAO,CAAC,uBAAuB,CAA6D;IAE5F,2EAA2E;IAC3E,OAAO,CAAC,YAAY,CAAqC;IAEzD,0EAA0E;IAC1E,OAAO,CAAC,eAAe,CAAgC;IAEvD,wDAAwD;IACxD,OAAO,CAAC,WAAW,CAA4B;IAE/C;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,iBAAiB,CAAkE;IAE3F;;;;;;OAMG;IACH,OAAO,CAAC,kBAAkB,CAAmE;gBAG3F,MAAM,EAAE,kBAAkB,EAC1B,cAAc,EAAE,cAAc,EAC9B,KAAK,EAAE,YAAY,EACnB,QAAQ,EAAE,MAAM;IAUlB;;OAEG;IACH,YAAY,CAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI;IAI7C;;;;OAIG;IACH,0BAA0B,CAAC,EAAE,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI;IAIhF;;;OAGG;IACH,WAAW,CAAC,OAAO,EAAE,eAAe,GAAG,IAAI;IAa3C;;;;OAIG;IACH,eAAe,CAAC,OAAO,EAAE,YAAY,GAAG,IAAI;IAI5C;;;;OAIG;IACH,kBAAkB,CAAC,OAAO,EAAE,eAAe,GAAG,IAAI;IAIlD;;;;;;;;;;;OAWG;IACH,sBAAsB,CACpB,KAAK,EAAE,kBAAkB,EACzB,SAAS,EAAE,MAAM,gBAAgB,GAAG;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,EACrD,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,eAAe,KAAK,IAAI,GAC7D,IAAI;IAMP;;;;;OAKG;IACH,OAAO,CAAC,gBAAgB;IAexB;;;;;;;OAOG;IACH,OAAO,CAAC,YAAY;IAWpB;;;OAGG;IACH,kBAAkB,CAAC,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,IAAI;IAMhE;;OAEG;IACH,eAAe,CAAC,QAAQ,EAAE,oBAAoB,GAAG,IAAI;IAIrD;;;;OAIG;IACH,kBAAkB,CAAC,IAAI,EAAE,eAAe,GAAG,IAAI;IAI/C;;;OAGG;IACH,cAAc,CAAC,WAAW,EAAE,WAAW,GAAG,IAAI;IAI9C;;;;;;OAMG;IACH,YAAY,CACV,QAAQ,EAAE,MAAM;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,UAAU,EAAE,OAAO,CAAA;KAAE,EACzD,SAAS,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,IAAI,KAAK,IAAI,GAC3D,IAAI;IAKP;;OAEG;IACH,KAAK,IAAI,IAAI;IA8Db;;OAEG;IACH,IAAI,IAAI,IAAI;IAsBZ;;OAEG;IACG,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,QAAQ,GAAG,SAAS,CAAC;IAsM3F;;;;OAIG;IACH,OAAO,CAAC,wBAAwB;IAahC;;OAEG;IACH,YAAY,IAAI,IAAI;IAuBpB;;OAEG;IACH,KAAK,IAAI,IAAI;IAIb;;OAEG;IACH,UAAU,IAAI,IAAI;IAIlB;;OAEG;IACH,MAAM,IAAI,IAAI;IAKd;;OAEG;IACH,SAAS,IAAI,eAAe;IAY5B;;OAEG;IACH,OAAO,IAAI,aAAa,EAAE;IAI1B;;OAEG;IACH,QAAQ,IAAI,SAAS,EAAE;IAIvB;;;;OAIG;IACH,eAAe,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAQrD;;OAEG;IACH,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAKjC;;OAEG;IACH,aAAa,IAAI,UAAU;IAI3B;;OAEG;IACH,aAAa,IAAI,aAAa;IAI9B;;;;;;;;;;;;;;;;OAgBG;IACG,aAAa,CAAC,UAAU,EAAE;QAAE,oBAAoB,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,EAAE,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IAoGjH,OAAO,CAAC,OAAO;IAoBf;;;;;;OAMG;IACH,OAAO,CAAC,YAAY;IA2FpB,OAAO,CAAC,eAAe;IA4IvB;;;;;;;;OAQG;IACH,OAAO,CAAC,gBAAgB;IAaxB,OAAO,CAAC,WAAW;IAsGnB;;;OAGG;IACH,OAAO,CAAC,aAAa;IAOrB;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAI1B;;;;OAIG;IACH,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;IAKpD,OAAO,CAAC,sBAAsB;IAI9B,OAAO,CAAC,UAAU;IAOlB;;OAEG;IACG,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA2O9E;;;;OAIG;YACW,eAAe;IAsE7B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAM3B;;;;;;;;OAQG;YACW,YAAY;IAwD1B;;;;OAIG;IACH,OAAO,CAAC,qBAAqB;IAK7B;;;;OAIG;IACH,OAAO,CAAC,aAAa;IAkCrB;;OAEG;IACH,OAAO,CAAC,eAAe;IAQvB;;;;;;;;;;OAUG;IACH,OAAO,CAAC,0BAA0B;IA0BlC;;;;OAIG;IACH,OAAO,CAAC,wBAAwB;YA4ClB,eAAe;IAmD7B;;;;OAIG;YACW,gBAAgB;IAyD9B;;;;;;;;;;;;;;;OAeG;IACH,MAAM,CAAC,gBAAgB,CAAC,GAAG,EAAE,aAAa,GAAG,mBAAmB;IA4FhE;;;8EAG0E;IAC1E,MAAM,CAAC,qBAAqB,CAAC,SAAS,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,OAAO;IAQ5E;;;;OAIG;IACH,MAAM,CAAC,uBAAuB,CAC5B,GAAG,EAAE,aAAa,EAClB,UAAU,EAAE,mBAAmB,GAC9B,gBAAgB;IAqCnB;;;;;OAKG;IACH,MAAM,CAAC,YAAY,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM;IAc3C;;;;;;;;;;;;;OAaG;IACH,OAAO,CAAC,oBAAoB;IA8D5B;;;;wCAIoC;IACpC,WAAW,IAAI;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,cAAc,EAAE,MAAM,CAAA;KAAE;CAO9E;AAED,4EAA4E;AAC5E,MAAM,WAAW,mBAAmB;IAClC,IAAI,EACA,QAAQ,GACR,OAAO,GACP,cAAc,GACd,SAAS,GACT,cAAc,GACd,qBAAqB,GACrB,wBAAwB,CAAC;IAC7B;;;2FAGuF;IACvF,SAAS,EAAE,MAAM,EAAE,GAAG,GAAG,GAAG,IAAI,CAAC;IACjC,iBAAiB,EAAE,OAAO,CAAC;IAC3B,gBAAgB,EAAE,OAAO,CAAC;IAC1B;;;;mEAI+D;IAC/D,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED,iDAAiD;AACjD,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,QAAQ,GAAG,MAAM,GAAG,QAAQ,CAAC;IACrC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,aAAa,EAAE,MAAM,EAAE,GAAG,GAAG,GAAG,IAAI,CAAC;IACrC,iBAAiB,EAAE,OAAO,CAAC;IAC3B,gBAAgB,EAAE,OAAO,CAAC;CAC3B"}
|
|
@@ -23,6 +23,7 @@ import { JobRunHistory } from './JobRunHistory.js';
|
|
|
23
23
|
import { SkipLedger } from './SkipLedger.js';
|
|
24
24
|
import { classifySessionDeath } from '../monitoring/QuotaExhaustionDetector.js';
|
|
25
25
|
import { TOPIC_STYLE } from '../messaging/TelegramAdapter.js';
|
|
26
|
+
import { decideClaimPath } from './JobLeaseCutoverGate.js';
|
|
26
27
|
/**
|
|
27
28
|
* Fallback expected duration (minutes) used when a run is encountered for a
|
|
28
29
|
* slug that's no longer present in `jobs` (e.g., job config was removed
|
|
@@ -77,6 +78,26 @@ export class JobScheduler {
|
|
|
77
78
|
quotaTracker = null;
|
|
78
79
|
/** Optional job claim manager for multi-machine deduplication (Phase 4C) */
|
|
79
80
|
claimManager = null;
|
|
81
|
+
/**
|
|
82
|
+
* MULTI-MACHINE-SEAMLESSNESS-SPEC §WS4.3 journal-lease cutover. The durable,
|
|
83
|
+
* epoch-fenced lease store — the journal-backed UPGRADE of the best-effort
|
|
84
|
+
* AgentBus broadcast in `claimManager`. It is consulted for a job ONLY when
|
|
85
|
+
* `journalLeaseGate` selects the 'journal' path (flag on + pool coherent). The
|
|
86
|
+
* gate guarantees the two mechanisms are never both live for a job set (the
|
|
87
|
+
* named migration hazard). When unset, the scheduler stays on the legacy bus
|
|
88
|
+
* path (byte-for-byte today's behavior).
|
|
89
|
+
*/
|
|
90
|
+
leaseClaimStore = null;
|
|
91
|
+
/**
|
|
92
|
+
* Live provider for the cutover-gate input (flag + dry-run + the pool peers'
|
|
93
|
+
* advertised ws43JournalLease capability + this machine's lease epoch). Read
|
|
94
|
+
* LIVE at each claim boundary so a flag flip / a peer going dark cuts the pool
|
|
95
|
+
* back to the bus immediately. When unset, the gate is never consulted and the
|
|
96
|
+
* legacy bus path runs (strict no-op).
|
|
97
|
+
*/
|
|
98
|
+
journalLeaseGate = null;
|
|
99
|
+
/** Optional sink for the most-recent cutover decision (status/audit surface). */
|
|
100
|
+
onCutoverDecision = null;
|
|
80
101
|
/**
|
|
81
102
|
* UNIFIED-SESSION-LIFECYCLE §P0 #9 (SE-8): function returning cumulative
|
|
82
103
|
* wall-time-asleep in milliseconds during a half-open window [startMs, endMs).
|
|
@@ -166,6 +187,64 @@ export class JobScheduler {
|
|
|
166
187
|
setJobClaimManager(manager) {
|
|
167
188
|
this.claimManager = manager;
|
|
168
189
|
}
|
|
190
|
+
/**
|
|
191
|
+
* MULTI-MACHINE-SEAMLESSNESS-SPEC §WS4.3 — inject the journal-lease cutover.
|
|
192
|
+
*
|
|
193
|
+
* `store` is the durable epoch-fenced lease store; `gateInput` returns the
|
|
194
|
+
* LIVE cutover-gate input (flag, dry-run, pool peers' advertised capability,
|
|
195
|
+
* and this machine's current lease epoch) — read fresh at each claim boundary,
|
|
196
|
+
* never cached. `onDecision` (optional) receives the resolved decision per job
|
|
197
|
+
* for the status/audit surface.
|
|
198
|
+
*
|
|
199
|
+
* When omitted, the scheduler never consults the journal path — the legacy
|
|
200
|
+
* AgentBus `claimManager` runs unchanged (single-machine / flag-off no-op).
|
|
201
|
+
*/
|
|
202
|
+
setJournalLeaseCutover(store, gateInput, onDecision) {
|
|
203
|
+
this.leaseClaimStore = store;
|
|
204
|
+
this.journalLeaseGate = gateInput;
|
|
205
|
+
this.onCutoverDecision = onDecision ?? null;
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* Resolve which claim mechanism a job uses RIGHT NOW. Pure read of the live
|
|
209
|
+
* gate input. Returns null when the cutover is not wired (legacy bus only).
|
|
210
|
+
* Exposed for the scheduler's claim seams (check + claim + complete) so a
|
|
211
|
+
* single evaluation drives all three — never two mechanisms for one job.
|
|
212
|
+
*/
|
|
213
|
+
resolveClaimPath(slug) {
|
|
214
|
+
if (!this.journalLeaseGate || !this.leaseClaimStore)
|
|
215
|
+
return null;
|
|
216
|
+
let input;
|
|
217
|
+
try {
|
|
218
|
+
input = this.journalLeaseGate();
|
|
219
|
+
}
|
|
220
|
+
catch {
|
|
221
|
+
// @silent-fallback-ok — fail toward the legacy bus path (the conservative
|
|
222
|
+
// side) if the live gate read throws; never strand a job on a broken gate.
|
|
223
|
+
return null;
|
|
224
|
+
}
|
|
225
|
+
const decision = decideClaimPath(input);
|
|
226
|
+
this.onCutoverDecision?.(slug, decision);
|
|
227
|
+
return { path: decision.path, epoch: input.epoch, journalDryRun: decision.journalDryRun };
|
|
228
|
+
}
|
|
229
|
+
/**
|
|
230
|
+
* Release a claim for a completed job across BOTH mechanisms. The path that
|
|
231
|
+
* actually took the claim may differ from the path the gate would pick at
|
|
232
|
+
* completion time (a mid-run flag flip), so we release both — each store's
|
|
233
|
+
* completeClaim is a strict no-op when this machine does not own the slug's
|
|
234
|
+
* live record, so releasing the unused store is harmless. The journal release
|
|
235
|
+
* is synchronous (durable); the bus release is best-effort async.
|
|
236
|
+
*/
|
|
237
|
+
releaseClaim(slug, result) {
|
|
238
|
+
try {
|
|
239
|
+
this.leaseClaimStore?.completeClaim(slug, result);
|
|
240
|
+
}
|
|
241
|
+
catch {
|
|
242
|
+
/* @silent-fallback-ok — best-effort; lease expires on its own otherwise */
|
|
243
|
+
}
|
|
244
|
+
this.claimManager?.completeClaim(slug, result).catch(err => {
|
|
245
|
+
console.error(`[scheduler] Failed to broadcast claim completion for "${slug}": ${err}`);
|
|
246
|
+
});
|
|
247
|
+
}
|
|
169
248
|
/**
|
|
170
249
|
* Set local machine identity for machine-scoped job filtering.
|
|
171
250
|
* Jobs with a `machines` field will only run on machines whose ID or name matches.
|
|
@@ -356,15 +435,27 @@ export class JobScheduler {
|
|
|
356
435
|
return 'skipped';
|
|
357
436
|
}
|
|
358
437
|
}
|
|
359
|
-
// Multi-machine claim check (Phase 4C — Gap 5)
|
|
360
|
-
//
|
|
361
|
-
|
|
438
|
+
// Multi-machine claim check (Phase 4C — Gap 5; WS4.3 journal-lease cutover).
|
|
439
|
+
// The cutover gate resolves WHICH mechanism owns dedup for this job RIGHT
|
|
440
|
+
// NOW — the durable epoch-fenced journal lease (flag on + pool coherent) or
|
|
441
|
+
// the legacy AgentBus broadcast. Exactly one is consulted per job, so the
|
|
442
|
+
// two never run simultaneously for the same job set (the named migration
|
|
443
|
+
// hazard). The decision is recomputed live at each seam; in the steady
|
|
444
|
+
// coherent state it is stable across the same tick.
|
|
445
|
+
const claimPath = this.resolveClaimPath(slug);
|
|
446
|
+
const remoteHeld = claimPath?.path === 'journal'
|
|
447
|
+
? this.leaseClaimStore.hasRemoteClaim(slug)
|
|
448
|
+
: (this.claimManager?.hasRemoteClaim(slug) ?? false);
|
|
449
|
+
if (remoteHeld) {
|
|
450
|
+
const claimedBy = claimPath?.path === 'journal'
|
|
451
|
+
? this.leaseClaimStore.getClaim(slug)?.machineId
|
|
452
|
+
: this.claimManager?.getClaim(slug)?.machineId;
|
|
362
453
|
this.skipLedger.recordSkip(slug, 'claimed');
|
|
363
454
|
this.state.appendEvent({
|
|
364
455
|
type: 'job_skipped',
|
|
365
456
|
summary: `Job "${slug}" skipped — claimed by another machine`,
|
|
366
457
|
timestamp: new Date().toISOString(),
|
|
367
|
-
metadata: { slug, reason, claimedBy:
|
|
458
|
+
metadata: { slug, reason, claimedBy, claimPath: claimPath?.path ?? 'bus' },
|
|
368
459
|
});
|
|
369
460
|
return 'skipped';
|
|
370
461
|
}
|
|
@@ -431,12 +522,39 @@ export class JobScheduler {
|
|
|
431
522
|
this.enqueue(slug, reason);
|
|
432
523
|
return 'queued';
|
|
433
524
|
}
|
|
434
|
-
//
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
525
|
+
// Take the claim before spawning. The cutover gate (resolved at the check
|
|
526
|
+
// seam above) decides the mechanism: a durable epoch-fenced journal lease
|
|
527
|
+
// when the pool is coherent, else the legacy best-effort AgentBus broadcast.
|
|
528
|
+
// DRY-RUN: when the gate is coherent-but-dry-run, the intended journal claim
|
|
529
|
+
// is LOGGED (the WS-wide "log intended claims" posture) but the bus path
|
|
530
|
+
// still runs — so a dry-run pool never half-migrates.
|
|
531
|
+
const timeoutMs = (job.expectedDurationMinutes ?? 30) * 2 * 60_000;
|
|
532
|
+
if (claimPath?.path === 'journal') {
|
|
533
|
+
// Synchronous, durable: the lease is persisted before the spawn, so a
|
|
534
|
+
// crash between claim and spawn leaves the lease (fenced, expiring) rather
|
|
535
|
+
// than a silent double-run window.
|
|
536
|
+
const attempt = this.leaseClaimStore.tryClaim(slug, claimPath.epoch, timeoutMs);
|
|
537
|
+
if (!attempt.ok && attempt.reason === 'held-by-peer') {
|
|
538
|
+
// A peer leased it between the check and here — yield (rare race).
|
|
539
|
+
this.skipLedger.recordSkip(slug, 'claimed');
|
|
540
|
+
this.state.appendEvent({
|
|
541
|
+
type: 'job_skipped',
|
|
542
|
+
summary: `Job "${slug}" skipped — leased by another machine`,
|
|
543
|
+
timestamp: new Date().toISOString(),
|
|
544
|
+
metadata: { slug, reason, claimedBy: attempt.heldBy, claimPath: 'journal' },
|
|
545
|
+
});
|
|
546
|
+
return 'skipped';
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
else {
|
|
550
|
+
if (claimPath?.journalDryRun) {
|
|
551
|
+
console.log(`[scheduler] WS4.3 journal-lease DRY-RUN — would lease "${slug}" via journal (epoch ${claimPath.epoch}); running legacy bus path.`);
|
|
552
|
+
}
|
|
553
|
+
if (this.claimManager) {
|
|
554
|
+
this.claimManager.tryClaim(slug, timeoutMs).catch(err => {
|
|
555
|
+
console.error(`[scheduler] Failed to broadcast claim for "${slug}": ${err}`);
|
|
556
|
+
});
|
|
557
|
+
}
|
|
440
558
|
}
|
|
441
559
|
// Clear retry state on successful trigger
|
|
442
560
|
this.clearRetryState(slug);
|
|
@@ -652,8 +770,9 @@ export class JobScheduler {
|
|
|
652
770
|
try {
|
|
653
771
|
// Note: claim is finalized as 'failure' (slug is unblocked for the next tick)
|
|
654
772
|
// while run history records 'timeout' (preserves the cause for diagnostics).
|
|
655
|
-
// The asymmetry is intentional — don't "fix" it.
|
|
656
|
-
|
|
773
|
+
// The asymmetry is intentional — don't "fix" it. Releases both the journal
|
|
774
|
+
// lease and the legacy bus claim (whichever took it).
|
|
775
|
+
this.releaseClaim(run.slug, 'failure');
|
|
657
776
|
}
|
|
658
777
|
catch {
|
|
659
778
|
// Best-effort — claim may have already cleared.
|
|
@@ -754,7 +873,7 @@ export class JobScheduler {
|
|
|
754
873
|
consecutiveFailures: 0,
|
|
755
874
|
nextScheduled: this.getNextRun(job.slug),
|
|
756
875
|
});
|
|
757
|
-
this.
|
|
876
|
+
this.releaseClaim(job.slug, 'success');
|
|
758
877
|
}).catch((err) => {
|
|
759
878
|
const errorMsg = err instanceof Error ? err.message : String(err);
|
|
760
879
|
const output = [
|
|
@@ -772,7 +891,7 @@ export class JobScheduler {
|
|
|
772
891
|
consecutiveFailures: (previous?.consecutiveFailures ?? 0) + 1,
|
|
773
892
|
nextScheduled: this.getNextRun(job.slug),
|
|
774
893
|
});
|
|
775
|
-
this.
|
|
894
|
+
this.releaseClaim(job.slug, 'failure');
|
|
776
895
|
this.scheduleRetry(job.slug, 'error');
|
|
777
896
|
});
|
|
778
897
|
}
|
|
@@ -1098,12 +1217,9 @@ export class JobScheduler {
|
|
|
1098
1217
|
}
|
|
1099
1218
|
this.activeRunIds.delete(session.name);
|
|
1100
1219
|
}
|
|
1101
|
-
// Signal claim completion (Phase 4C — Gap 5)
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
console.error(`[scheduler] Failed to broadcast claim completion for "${job.slug}": ${err}`);
|
|
1105
|
-
});
|
|
1106
|
-
}
|
|
1220
|
+
// Signal claim completion (Phase 4C — Gap 5; WS4.3 journal-lease cutover).
|
|
1221
|
+
// Releases whichever mechanism took the claim (journal lease or legacy bus).
|
|
1222
|
+
this.releaseClaim(job.slug, failed ? 'failure' : 'success');
|
|
1107
1223
|
// Clear active-job.json now that the job is done
|
|
1108
1224
|
const activeJob = this.state.get('active-job');
|
|
1109
1225
|
if (activeJob?.slug === job.slug) {
|