@yugabytedb/meko-mcp 2.0.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +183 -33
- package/assets/hooks-handlers/codex-session-end.sh +13 -0
- package/assets/hooks-handlers/cursor-session-start.sh +19 -1
- package/assets/hooks-handlers/lib/capture-notices.js +321 -0
- package/assets/hooks-handlers/lib/capture-state.js +2079 -0
- package/assets/hooks-handlers/lib/capture-status-cli.js +82 -0
- package/assets/hooks-handlers/lib/capture.js +1653 -225
- package/assets/hooks-handlers/lib/checkpoint-timer.js +455 -50
- package/assets/hooks-handlers/lib/kiro-handler.js +738 -0
- package/assets/hooks-handlers/lib/terminal-handoff.js +418 -0
- package/assets/hooks-handlers/session-end.sh +3 -24
- package/assets/hooks-handlers/session-start.sh +8 -1
- package/assets/hooks-handlers/test_checkpoint_timer.sh +162 -0
- package/assets/scripts/meko-statusline.sh +41 -1
- package/assets/scripts/package-desktop-skill.sh +32 -1
- package/assets/skills/meko-mcp-tools/SKILL.md +90 -137
- package/assets/skills/meko-mcp-tools/references/_sections.md +3 -5
- package/assets/skills/meko-mcp-tools/references/tools-agent-id-conventions.md +31 -28
- package/assets/skills/meko-mcp-tools/references/tools-conversation-dedup.md +5 -7
- package/assets/skills/meko-mcp-tools/references/tools-conversation-reasoning.md +50 -0
- package/assets/skills/meko-mcp-tools/references/tools-cookbook.md +69 -49
- package/assets/skills/meko-mcp-tools/references/tools-datapack-workflow.md +6 -6
- package/assets/skills/meko-mcp-tools/references/tools-known-limitations.md +29 -5
- package/assets/skills/meko-mcp-tools/references/tools-memory-vs-conversation.md +29 -92
- package/assets/skills/meko-mcp-tools/references/tools-overview.md +42 -40
- package/assets/skills/meko-mcp-tools/references/tools-rag-workflow.md +1 -2
- package/assets/skills/meko-mcp-tools/references/tools-troubleshooting.md +18 -11
- package/assets/skills/meko-mcp-tools/references/tools-user-id-conventions.md +10 -9
- package/assets/skills/meko-mcp-tools-desktop/SKILL.md +51 -44
- package/assets/skills/meko-mcp-tools-desktop/desktop/README.md +6 -6
- package/assets/skills/meko-mcp-tools-desktop/references/_sections.md +2 -2
- package/assets/skills/meko-mcp-tools-desktop/references/tools-agent-id-conventions.md +34 -25
- package/assets/skills/meko-mcp-tools-desktop/references/tools-conversation-reasoning.md +50 -0
- package/assets/skills/meko-mcp-tools-desktop/references/tools-cookbook.md +63 -37
- package/assets/skills/meko-mcp-tools-desktop/references/tools-known-limitations.md +4 -4
- package/assets/skills/meko-mcp-tools-desktop/references/tools-memory-vs-conversation.md +19 -22
- package/assets/skills/meko-mcp-tools-desktop/references/tools-overview.md +42 -40
- package/assets/skills/meko-mcp-tools-desktop/references/tools-rag-workflow.md +1 -2
- package/assets/skills/meko-mcp-tools-desktop/references/tools-troubleshooting.md +3 -17
- package/assets/skills/meko-select-datapack-desktop/SKILL.md +31 -21
- package/assets/skills/meko-select-datapack-desktop/references/cookbook.md +8 -8
- package/assets/skills/meko-select-datapack-desktop/references/role-display-future.md +41 -0
- package/assets/skills/meko-select-datapack-desktop/references/troubleshooting.md +1 -1
- package/bin/create-meko-setup.mjs +93 -16
- package/lib/capture-status.mjs +67 -0
- package/lib/clients/aliases.mjs +5 -0
- package/lib/clients/claude-code.mjs +8 -4
- package/lib/clients/claude-desktop.mjs +133 -4
- package/lib/clients/codex.mjs +29 -11
- package/lib/clients/common.mjs +2 -0
- package/lib/clients/cursor.mjs +7 -2
- package/lib/clients/index.mjs +2 -0
- package/lib/clients/kiro.mjs +705 -0
- package/lib/config.mjs +90 -0
- package/lib/detect.mjs +7 -2
- package/lib/installer.mjs +1 -0
- package/lib/migrate/adapters/codex.mjs +490 -0
- package/lib/migrate/id.mjs +28 -3
- package/lib/migrate/orchestrator.mjs +34 -7
- package/lib/migrate.mjs +4 -2
- package/lib/prompt.mjs +17 -2
- package/lib/url-selection.mjs +55 -0
- package/package.json +4 -4
- package/assets/skills/meko-mcp-tools/references/tools-scope-permissions.md +0 -71
- package/assets/skills/meko-mcp-tools-desktop/references/tools-scope-permissions.md +0 -71
|
@@ -0,0 +1,2079 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* capture-state.js — Durable outbox / lease / health helpers for Meko Capture V2.
|
|
3
|
+
*
|
|
4
|
+
* CommonJS, zero npm deps. Workstream A surface consumed by capture.js (B),
|
|
5
|
+
* checkpoint-timer.js (C), and status/CLI (D). See:
|
|
6
|
+
* docs/plans/meko-capture-v2-contracts.md
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
"use strict";
|
|
10
|
+
|
|
11
|
+
const crypto = require("crypto");
|
|
12
|
+
const fs = require("fs");
|
|
13
|
+
const os = require("os");
|
|
14
|
+
const path = require("path");
|
|
15
|
+
|
|
16
|
+
// ---------------------------------------------------------------------------
|
|
17
|
+
// Defaults (contracts env table) + readers
|
|
18
|
+
// ---------------------------------------------------------------------------
|
|
19
|
+
|
|
20
|
+
const DEFAULTS = Object.freeze({
|
|
21
|
+
WATERMARK_DIR: null, // resolved via watermarkDir()
|
|
22
|
+
MAX_CAPTURE_BATCH_SIZE: 5,
|
|
23
|
+
CHECKPOINT_INTERVAL: 600,
|
|
24
|
+
CHECKPOINT_TIMEOUT: 120,
|
|
25
|
+
RECOVERY_MAX_SESSIONS: 3,
|
|
26
|
+
RECOVERY_MAX_EXCHANGES: 20,
|
|
27
|
+
RECOVERY_MIN_IDLE: 900,
|
|
28
|
+
LEASE_TTL_SECONDS: 120,
|
|
29
|
+
RETRY_BASE_SECONDS: 30,
|
|
30
|
+
RETRY_MAX_SECONDS: 600,
|
|
31
|
+
RETRY_JITTER: 0.2,
|
|
32
|
+
PERSISTENT_RETRY_SECONDS: 21600,
|
|
33
|
+
WORKER_MAX_AGE_SECONDS: 86400,
|
|
34
|
+
TIMER_MAX_AGE: 86400,
|
|
35
|
+
HEALTH_CACHE: "capture-health.json",
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
const FAILURE_CLASSES = Object.freeze([
|
|
39
|
+
"transient",
|
|
40
|
+
"persistent",
|
|
41
|
+
"state_corrupt",
|
|
42
|
+
"ownership",
|
|
43
|
+
"datapack",
|
|
44
|
+
]);
|
|
45
|
+
|
|
46
|
+
const LIFECYCLES = Object.freeze(["active", "closing", "closed"]);
|
|
47
|
+
const DELIVERIES = Object.freeze([
|
|
48
|
+
"needs_conversation",
|
|
49
|
+
"idle",
|
|
50
|
+
"pending",
|
|
51
|
+
"draining",
|
|
52
|
+
"retry_wait",
|
|
53
|
+
"blocked",
|
|
54
|
+
"complete",
|
|
55
|
+
]);
|
|
56
|
+
|
|
57
|
+
const HEALTH_STATUSES = Object.freeze([
|
|
58
|
+
"blocked_action_required",
|
|
59
|
+
"degraded_retrying",
|
|
60
|
+
"catching_up",
|
|
61
|
+
"healthy",
|
|
62
|
+
]);
|
|
63
|
+
|
|
64
|
+
const HEALTH_PRECEDENCE = Object.freeze({
|
|
65
|
+
blocked_action_required: 4,
|
|
66
|
+
degraded_retrying: 3,
|
|
67
|
+
catching_up: 2,
|
|
68
|
+
healthy: 1,
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
function envInt(name, fallback) {
|
|
72
|
+
const raw = process.env[name];
|
|
73
|
+
if (raw == null || raw === "") return fallback;
|
|
74
|
+
const n = Number.parseInt(String(raw), 10);
|
|
75
|
+
return Number.isFinite(n) ? n : fallback;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function envFloat(name, fallback) {
|
|
79
|
+
const raw = process.env[name];
|
|
80
|
+
if (raw == null || raw === "") return fallback;
|
|
81
|
+
const n = Number.parseFloat(String(raw));
|
|
82
|
+
return Number.isFinite(n) ? n : fallback;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function envString(name, fallback) {
|
|
86
|
+
const raw = process.env[name];
|
|
87
|
+
if (raw == null || raw === "") return fallback;
|
|
88
|
+
return String(raw);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function maxCaptureBatchSize() {
|
|
92
|
+
return Math.max(1, envInt("MEKO_MAX_CAPTURE_BATCH_SIZE", DEFAULTS.MAX_CAPTURE_BATCH_SIZE));
|
|
93
|
+
}
|
|
94
|
+
function checkpointInterval() {
|
|
95
|
+
return envInt("MEKO_CHECKPOINT_INTERVAL", DEFAULTS.CHECKPOINT_INTERVAL);
|
|
96
|
+
}
|
|
97
|
+
function checkpointTimeout() {
|
|
98
|
+
return envInt("MEKO_CHECKPOINT_TIMEOUT", DEFAULTS.CHECKPOINT_TIMEOUT);
|
|
99
|
+
}
|
|
100
|
+
function recoveryMaxSessions() {
|
|
101
|
+
return envInt("MEKO_RECOVERY_MAX_SESSIONS", DEFAULTS.RECOVERY_MAX_SESSIONS);
|
|
102
|
+
}
|
|
103
|
+
function recoveryMaxExchanges() {
|
|
104
|
+
return envInt("MEKO_RECOVERY_MAX_EXCHANGES", DEFAULTS.RECOVERY_MAX_EXCHANGES);
|
|
105
|
+
}
|
|
106
|
+
function recoveryMinIdle() {
|
|
107
|
+
return envInt("MEKO_RECOVERY_MIN_IDLE", DEFAULTS.RECOVERY_MIN_IDLE);
|
|
108
|
+
}
|
|
109
|
+
function leaseTtlSeconds() {
|
|
110
|
+
return envInt("MEKO_LEASE_TTL_SECONDS", DEFAULTS.LEASE_TTL_SECONDS);
|
|
111
|
+
}
|
|
112
|
+
function retryBaseSeconds() {
|
|
113
|
+
return envInt("MEKO_RETRY_BASE_SECONDS", DEFAULTS.RETRY_BASE_SECONDS);
|
|
114
|
+
}
|
|
115
|
+
function retryMaxSeconds() {
|
|
116
|
+
return envInt("MEKO_RETRY_MAX_SECONDS", DEFAULTS.RETRY_MAX_SECONDS);
|
|
117
|
+
}
|
|
118
|
+
function retryJitter() {
|
|
119
|
+
return envFloat("MEKO_RETRY_JITTER", DEFAULTS.RETRY_JITTER);
|
|
120
|
+
}
|
|
121
|
+
function persistentRetrySeconds() {
|
|
122
|
+
return envInt("MEKO_PERSISTENT_RETRY_SECONDS", DEFAULTS.PERSISTENT_RETRY_SECONDS);
|
|
123
|
+
}
|
|
124
|
+
function workerMaxAgeSeconds() {
|
|
125
|
+
return envInt("MEKO_WORKER_MAX_AGE_SECONDS", DEFAULTS.WORKER_MAX_AGE_SECONDS);
|
|
126
|
+
}
|
|
127
|
+
function timerMaxAge() {
|
|
128
|
+
return envInt("MEKO_TIMER_MAX_AGE", DEFAULTS.TIMER_MAX_AGE);
|
|
129
|
+
}
|
|
130
|
+
function healthCacheFilename() {
|
|
131
|
+
return envString("MEKO_HEALTH_CACHE", DEFAULTS.HEALTH_CACHE);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// ---------------------------------------------------------------------------
|
|
135
|
+
// Path helpers
|
|
136
|
+
// ---------------------------------------------------------------------------
|
|
137
|
+
|
|
138
|
+
function watermarkDir() {
|
|
139
|
+
return (
|
|
140
|
+
process.env.MEKO_WATERMARK_DIR ||
|
|
141
|
+
path.join(os.homedir(), ".claude", "meko-capture")
|
|
142
|
+
);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
function watermarkPath(sessionId) {
|
|
146
|
+
return path.join(watermarkDir(), `${sessionId}.watermark.json`);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
function leasePath(sessionId) {
|
|
150
|
+
return path.join(watermarkDir(), `${sessionId}.drain.lease`);
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
function mutationLockPath(sessionId) {
|
|
154
|
+
return path.join(watermarkDir(), `${sessionId}.state.lock`);
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function timerPidPath(sessionId) {
|
|
158
|
+
return path.join(watermarkDir(), `${sessionId}.timer.pid`);
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function healthCachePath() {
|
|
162
|
+
return path.join(watermarkDir(), healthCacheFilename());
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* Convert an agent_id into the filesystem slug used by legacy error files and
|
|
167
|
+
* datapack pins (matches capture.js datapackPinSlug).
|
|
168
|
+
*/
|
|
169
|
+
function agentSlug(agentId) {
|
|
170
|
+
const trimmed = (typeof agentId === "string" ? agentId : "").trim();
|
|
171
|
+
if (!trimmed) return "meko_agent";
|
|
172
|
+
return (
|
|
173
|
+
trimmed.replace(/[^A-Za-z0-9._-]+/g, "_").replace(/^_+|_+$/g, "") ||
|
|
174
|
+
"meko_agent"
|
|
175
|
+
);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
function legacyErrorPath(agentSlugOrId) {
|
|
179
|
+
// Accept either a pre-slugged name or a raw agent_id; agentSlug is idempotent
|
|
180
|
+
// for already-safe slugs.
|
|
181
|
+
return path.join(
|
|
182
|
+
watermarkDir(),
|
|
183
|
+
`last-capture-error-${agentSlug(agentSlugOrId)}.json`,
|
|
184
|
+
);
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
// ---------------------------------------------------------------------------
|
|
188
|
+
// Time / PID / atomic IO helpers
|
|
189
|
+
// ---------------------------------------------------------------------------
|
|
190
|
+
|
|
191
|
+
function nowIso(nowMs) {
|
|
192
|
+
return new Date(nowMs != null ? nowMs : Date.now()).toISOString();
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
function parseIsoMs(value) {
|
|
196
|
+
if (value == null || value === "") return null;
|
|
197
|
+
const ms = Date.parse(String(value));
|
|
198
|
+
return Number.isFinite(ms) ? ms : null;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
function isPidAlive(pid) {
|
|
202
|
+
if (!Number.isInteger(pid) || pid <= 0) return false;
|
|
203
|
+
try {
|
|
204
|
+
process.kill(pid, 0);
|
|
205
|
+
return true;
|
|
206
|
+
} catch {
|
|
207
|
+
return false;
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
function ensureDir(dir) {
|
|
212
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/**
|
|
216
|
+
* Persist JSON atomically: temp (mode 0600) → fsync → rename.
|
|
217
|
+
* Never writes the destination in place.
|
|
218
|
+
*/
|
|
219
|
+
function atomicWriteJson(destPath, value, options) {
|
|
220
|
+
const opts = options && typeof options === "object" ? options : {};
|
|
221
|
+
ensureDir(path.dirname(destPath));
|
|
222
|
+
const payload = `${JSON.stringify(value)}\n`;
|
|
223
|
+
const tmp = `${destPath}.${process.pid}.${Date.now()}.${crypto
|
|
224
|
+
.randomBytes(4)
|
|
225
|
+
.toString("hex")}.tmp`;
|
|
226
|
+
let fd;
|
|
227
|
+
try {
|
|
228
|
+
fd = fs.openSync(tmp, "w", 0o600);
|
|
229
|
+
fs.writeFileSync(fd, payload, "utf8");
|
|
230
|
+
fs.fsyncSync(fd);
|
|
231
|
+
} finally {
|
|
232
|
+
if (fd != null) {
|
|
233
|
+
try {
|
|
234
|
+
fs.closeSync(fd);
|
|
235
|
+
} catch {
|
|
236
|
+
// ignore
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
try {
|
|
241
|
+
if (typeof opts.beforeRename === "function") opts.beforeRename();
|
|
242
|
+
fs.renameSync(tmp, destPath);
|
|
243
|
+
} catch (err) {
|
|
244
|
+
try {
|
|
245
|
+
fs.unlinkSync(tmp);
|
|
246
|
+
} catch {
|
|
247
|
+
// ignore
|
|
248
|
+
}
|
|
249
|
+
throw err;
|
|
250
|
+
}
|
|
251
|
+
try {
|
|
252
|
+
fs.chmodSync(destPath, 0o600);
|
|
253
|
+
} catch {
|
|
254
|
+
// Best-effort; some platforms ignore mode bits.
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
function emptyInFlight() {
|
|
259
|
+
return { seed: null, user_line: null, user_turn_id: null };
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
function hasConversationId(value) {
|
|
263
|
+
return typeof value === "string" && value.trim() !== "";
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
function deliveryFromConversation(conversationId) {
|
|
267
|
+
return hasConversationId(conversationId) ? "idle" : "needs_conversation";
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
function cloneInFlight(raw) {
|
|
271
|
+
if (!raw || typeof raw !== "object") return emptyInFlight();
|
|
272
|
+
return {
|
|
273
|
+
seed: raw.seed == null ? null : String(raw.seed),
|
|
274
|
+
user_line:
|
|
275
|
+
raw.user_line == null || raw.user_line === ""
|
|
276
|
+
? null
|
|
277
|
+
: Number(raw.user_line),
|
|
278
|
+
user_turn_id: raw.user_turn_id == null ? null : String(raw.user_turn_id),
|
|
279
|
+
};
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
// ---------------------------------------------------------------------------
|
|
283
|
+
// V2 validation / V1 normalize
|
|
284
|
+
// ---------------------------------------------------------------------------
|
|
285
|
+
|
|
286
|
+
function blockedOutcome(failureClass, error, extras) {
|
|
287
|
+
return {
|
|
288
|
+
ok: false,
|
|
289
|
+
delivery: "blocked",
|
|
290
|
+
failure_class: failureClass || "state_corrupt",
|
|
291
|
+
error: error || "state blocked",
|
|
292
|
+
...(extras || {}),
|
|
293
|
+
};
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
/**
|
|
297
|
+
* Validate a candidate v2 state object.
|
|
298
|
+
* @returns {{ ok:true, state:Object } | { ok:false, delivery:"blocked", failure_class:"state_corrupt", error:string }}
|
|
299
|
+
*/
|
|
300
|
+
function validateV2(state) {
|
|
301
|
+
if (!state || typeof state !== "object" || Array.isArray(state)) {
|
|
302
|
+
return blockedOutcome("state_corrupt", "state is not an object");
|
|
303
|
+
}
|
|
304
|
+
if (state.schema_version !== 2) {
|
|
305
|
+
return blockedOutcome(
|
|
306
|
+
"state_corrupt",
|
|
307
|
+
`unsupported schema_version: ${state.schema_version}`,
|
|
308
|
+
);
|
|
309
|
+
}
|
|
310
|
+
if (typeof state.session_id !== "string" || !state.session_id.trim()) {
|
|
311
|
+
return blockedOutcome("state_corrupt", "session_id must be a non-empty string");
|
|
312
|
+
}
|
|
313
|
+
if (
|
|
314
|
+
typeof state.conversation_epoch !== "number" ||
|
|
315
|
+
!Number.isInteger(state.conversation_epoch) ||
|
|
316
|
+
state.conversation_epoch < 0
|
|
317
|
+
) {
|
|
318
|
+
return blockedOutcome("state_corrupt", "conversation_epoch must be a non-negative integer");
|
|
319
|
+
}
|
|
320
|
+
if (
|
|
321
|
+
typeof state.last_line_number !== "number" ||
|
|
322
|
+
!Number.isInteger(state.last_line_number) ||
|
|
323
|
+
state.last_line_number < 0
|
|
324
|
+
) {
|
|
325
|
+
return blockedOutcome("state_corrupt", "last_line_number must be a non-negative integer");
|
|
326
|
+
}
|
|
327
|
+
if (!LIFECYCLES.includes(state.lifecycle)) {
|
|
328
|
+
return blockedOutcome("state_corrupt", `invalid lifecycle: ${state.lifecycle}`);
|
|
329
|
+
}
|
|
330
|
+
if (!DELIVERIES.includes(state.delivery)) {
|
|
331
|
+
return blockedOutcome("state_corrupt", `invalid delivery: ${state.delivery}`);
|
|
332
|
+
}
|
|
333
|
+
if (!state.in_flight || typeof state.in_flight !== "object") {
|
|
334
|
+
return blockedOutcome("state_corrupt", "in_flight must be an object");
|
|
335
|
+
}
|
|
336
|
+
if (
|
|
337
|
+
state.failure_class != null &&
|
|
338
|
+
!FAILURE_CLASSES.includes(state.failure_class)
|
|
339
|
+
) {
|
|
340
|
+
return blockedOutcome(
|
|
341
|
+
"state_corrupt",
|
|
342
|
+
`invalid failure_class: ${state.failure_class}`,
|
|
343
|
+
);
|
|
344
|
+
}
|
|
345
|
+
return { ok: true, state };
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
function coerceNonNegInt(value, fallback) {
|
|
349
|
+
const n = Number(value);
|
|
350
|
+
if (Number.isInteger(n) && n >= 0) return n;
|
|
351
|
+
return fallback;
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
/**
|
|
355
|
+
* Lazy v1→v2 normalize without advancing/resetting the cursor.
|
|
356
|
+
* Malformed / unsupported schema → typed blocked outcome (does not throw).
|
|
357
|
+
*
|
|
358
|
+
* @param {any} raw
|
|
359
|
+
* @param {object} [context]
|
|
360
|
+
* @returns {{ ok:true, state:Object, migrated?:boolean } | blocked outcome}
|
|
361
|
+
*/
|
|
362
|
+
function normalizeState(raw, context) {
|
|
363
|
+
const ctx = context && typeof context === "object" ? context : {};
|
|
364
|
+
if (raw == null) {
|
|
365
|
+
return blockedOutcome("state_corrupt", "state is null or undefined");
|
|
366
|
+
}
|
|
367
|
+
if (typeof raw !== "object" || Array.isArray(raw)) {
|
|
368
|
+
return blockedOutcome("state_corrupt", "state is not an object");
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
const schemaVersion = raw.schema_version;
|
|
372
|
+
if (schemaVersion != null && schemaVersion !== 1 && schemaVersion !== 2) {
|
|
373
|
+
return blockedOutcome(
|
|
374
|
+
"state_corrupt",
|
|
375
|
+
`unsupported schema_version: ${schemaVersion}`,
|
|
376
|
+
{ raw },
|
|
377
|
+
);
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
if (schemaVersion === 2) {
|
|
381
|
+
const validated = validateV2(raw);
|
|
382
|
+
if (!validated.ok) return validated;
|
|
383
|
+
return { ok: true, state: validated.state, migrated: false };
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
// V1 (no schema_version) or explicit schema_version:1 — migrate in memory.
|
|
387
|
+
const conversationId =
|
|
388
|
+
raw.conversation_id == null || raw.conversation_id === ""
|
|
389
|
+
? null
|
|
390
|
+
: String(raw.conversation_id);
|
|
391
|
+
const now = nowIso(ctx.now);
|
|
392
|
+
const sessionId =
|
|
393
|
+
(typeof ctx.session_id === "string" && ctx.session_id) ||
|
|
394
|
+
(typeof raw.session_id === "string" && raw.session_id) ||
|
|
395
|
+
"";
|
|
396
|
+
if (!sessionId) {
|
|
397
|
+
return blockedOutcome(
|
|
398
|
+
"state_corrupt",
|
|
399
|
+
"cannot normalize v1 state without session_id",
|
|
400
|
+
{ raw },
|
|
401
|
+
);
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
const lastLine = coerceNonNegInt(raw.last_line_number, 0);
|
|
405
|
+
const agentId =
|
|
406
|
+
raw.agent_id != null
|
|
407
|
+
? String(raw.agent_id)
|
|
408
|
+
: ctx.agent_id != null
|
|
409
|
+
? String(ctx.agent_id)
|
|
410
|
+
: "";
|
|
411
|
+
|
|
412
|
+
const state = {
|
|
413
|
+
schema_version: 2,
|
|
414
|
+
session_id: sessionId,
|
|
415
|
+
client:
|
|
416
|
+
(typeof raw.client === "string" && raw.client) ||
|
|
417
|
+
(typeof ctx.client === "string" && ctx.client) ||
|
|
418
|
+
"",
|
|
419
|
+
agent_id: agentId,
|
|
420
|
+
// Preserve provenance used by capture.js owner reconciliation.
|
|
421
|
+
agent_id_source:
|
|
422
|
+
raw.agent_id_source != null
|
|
423
|
+
? String(raw.agent_id_source)
|
|
424
|
+
: ctx.agent_id_source != null
|
|
425
|
+
? String(ctx.agent_id_source)
|
|
426
|
+
: "",
|
|
427
|
+
transcript_path:
|
|
428
|
+
raw.transcript_path != null
|
|
429
|
+
? String(raw.transcript_path)
|
|
430
|
+
: ctx.transcript_path != null
|
|
431
|
+
? String(ctx.transcript_path)
|
|
432
|
+
: null,
|
|
433
|
+
datapack_id:
|
|
434
|
+
raw.datapack_id == null || raw.datapack_id === ""
|
|
435
|
+
? null
|
|
436
|
+
: String(raw.datapack_id),
|
|
437
|
+
datapack_name:
|
|
438
|
+
raw.datapack_name == null || raw.datapack_name === ""
|
|
439
|
+
? null
|
|
440
|
+
: String(raw.datapack_name),
|
|
441
|
+
created_at:
|
|
442
|
+
(typeof raw.created_at === "string" && raw.created_at) ||
|
|
443
|
+
(typeof raw.updated_at === "string" && raw.updated_at) ||
|
|
444
|
+
now,
|
|
445
|
+
updated_at: (typeof raw.updated_at === "string" && raw.updated_at) || now,
|
|
446
|
+
last_activity_at:
|
|
447
|
+
(typeof raw.last_activity_at === "string" && raw.last_activity_at) ||
|
|
448
|
+
(typeof raw.updated_at === "string" && raw.updated_at) ||
|
|
449
|
+
now,
|
|
450
|
+
conversation_id: conversationId,
|
|
451
|
+
conversation_epoch: 0,
|
|
452
|
+
last_line_number: lastLine,
|
|
453
|
+
in_flight: emptyInFlight(),
|
|
454
|
+
lifecycle: "active",
|
|
455
|
+
delivery: deliveryFromConversation(conversationId),
|
|
456
|
+
queued_exchanges: coerceNonNegInt(raw.queued_exchanges, 0),
|
|
457
|
+
attempt_count: coerceNonNegInt(raw.attempt_count, 0),
|
|
458
|
+
next_retry_at: raw.next_retry_at == null ? null : String(raw.next_retry_at),
|
|
459
|
+
failure_class: null,
|
|
460
|
+
last_error: null,
|
|
461
|
+
last_success_at:
|
|
462
|
+
raw.last_success_at == null ? null : String(raw.last_success_at),
|
|
463
|
+
blocked_reason: null,
|
|
464
|
+
};
|
|
465
|
+
|
|
466
|
+
const validated = validateV2(state);
|
|
467
|
+
if (!validated.ok) return validated;
|
|
468
|
+
return { ok: true, state: validated.state, migrated: true };
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
// ---------------------------------------------------------------------------
|
|
472
|
+
// Atomic persistence
|
|
473
|
+
// ---------------------------------------------------------------------------
|
|
474
|
+
|
|
475
|
+
/**
|
|
476
|
+
* Read and normalize a session watermark.
|
|
477
|
+
* Corrupt / truncated / unsupported → typed failure; file left intact.
|
|
478
|
+
* Missing file → { ok:false, missing:true } (not state_corrupt).
|
|
479
|
+
*/
|
|
480
|
+
function readState(sessionId) {
|
|
481
|
+
const filePath = watermarkPath(sessionId);
|
|
482
|
+
let rawText;
|
|
483
|
+
try {
|
|
484
|
+
rawText = fs.readFileSync(filePath, "utf8");
|
|
485
|
+
} catch (err) {
|
|
486
|
+
if (err && err.code === "ENOENT") {
|
|
487
|
+
return {
|
|
488
|
+
ok: false,
|
|
489
|
+
missing: true,
|
|
490
|
+
session_id: sessionId,
|
|
491
|
+
path: filePath,
|
|
492
|
+
failure_class: null,
|
|
493
|
+
error: "watermark missing",
|
|
494
|
+
};
|
|
495
|
+
}
|
|
496
|
+
return blockedOutcome("state_corrupt", `read failed: ${err.message}`, {
|
|
497
|
+
session_id: sessionId,
|
|
498
|
+
path: filePath,
|
|
499
|
+
});
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
let parsed;
|
|
503
|
+
try {
|
|
504
|
+
parsed = JSON.parse(rawText);
|
|
505
|
+
} catch (err) {
|
|
506
|
+
return blockedOutcome(
|
|
507
|
+
"state_corrupt",
|
|
508
|
+
`malformed JSON: ${err.message}`,
|
|
509
|
+
{ session_id: sessionId, path: filePath },
|
|
510
|
+
);
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
const normalized = normalizeState(parsed, { session_id: sessionId });
|
|
514
|
+
if (!normalized.ok) {
|
|
515
|
+
return {
|
|
516
|
+
...normalized,
|
|
517
|
+
session_id: sessionId,
|
|
518
|
+
path: filePath,
|
|
519
|
+
};
|
|
520
|
+
}
|
|
521
|
+
return {
|
|
522
|
+
ok: true,
|
|
523
|
+
session_id: sessionId,
|
|
524
|
+
path: filePath,
|
|
525
|
+
state: normalized.state,
|
|
526
|
+
migrated: Boolean(normalized.migrated),
|
|
527
|
+
};
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
const MUTATION_LOCK_TTL_MS = 5000;
|
|
531
|
+
const MUTATION_LOCK_RETRIES = 100;
|
|
532
|
+
const MUTATION_LOCK_RETRY_MS = 10;
|
|
533
|
+
|
|
534
|
+
function sleepSync(ms) {
|
|
535
|
+
const cell = new Int32Array(new SharedArrayBuffer(4));
|
|
536
|
+
Atomics.wait(cell, 0, 0, ms);
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
function readMutationLockSnapshot(sessionId) {
|
|
540
|
+
try {
|
|
541
|
+
const raw = fs.readFileSync(mutationLockPath(sessionId), "utf8");
|
|
542
|
+
let value = null;
|
|
543
|
+
try {
|
|
544
|
+
value = JSON.parse(raw);
|
|
545
|
+
} catch {
|
|
546
|
+
/* malformed locks are reclaimable evidence */
|
|
547
|
+
}
|
|
548
|
+
return {
|
|
549
|
+
raw,
|
|
550
|
+
lock: value && typeof value === "object" ? value : null,
|
|
551
|
+
};
|
|
552
|
+
} catch (err) {
|
|
553
|
+
return {
|
|
554
|
+
raw: null,
|
|
555
|
+
lock: null,
|
|
556
|
+
missing: Boolean(err && err.code === "ENOENT"),
|
|
557
|
+
error: err && err.code !== "ENOENT" ? err.message : null,
|
|
558
|
+
};
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
function readMutationLock(sessionId) {
|
|
563
|
+
return readMutationLockSnapshot(sessionId).lock;
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
function mutationLockReclaimable(snapshot, now) {
|
|
567
|
+
if (!snapshot || snapshot.raw == null) return true;
|
|
568
|
+
const existing = snapshot.lock;
|
|
569
|
+
if (!existing) return true;
|
|
570
|
+
const pid = Number(existing.pid);
|
|
571
|
+
if (Number.isInteger(pid) && pid > 0) {
|
|
572
|
+
// Never evict a live owner merely because its short TTL elapsed. The lock
|
|
573
|
+
// protects synchronous local I/O, but a paused process must still retain
|
|
574
|
+
// mutual exclusion until it exits.
|
|
575
|
+
return !isPidAlive(pid);
|
|
576
|
+
}
|
|
577
|
+
const expiry = parseIsoMs(existing.expires_at);
|
|
578
|
+
return expiry == null || expiry <= now;
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
function mutationLockQuarantinePath(sessionId, action) {
|
|
582
|
+
return (
|
|
583
|
+
`${mutationLockPath(sessionId)}.${action}.${process.pid}.` +
|
|
584
|
+
crypto.randomBytes(8).toString("hex")
|
|
585
|
+
);
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
function restoreQuarantinedMutationLock(filePath, quarantinePath) {
|
|
589
|
+
try {
|
|
590
|
+
// A hard link restores the exact inode without overwriting a lock that a
|
|
591
|
+
// competing waiter may already have created in the rename gap.
|
|
592
|
+
fs.linkSync(quarantinePath, filePath);
|
|
593
|
+
} catch (err) {
|
|
594
|
+
if (!err || err.code !== "EEXIST") return false;
|
|
595
|
+
}
|
|
596
|
+
try {
|
|
597
|
+
fs.unlinkSync(quarantinePath);
|
|
598
|
+
} catch {
|
|
599
|
+
/* best effort; the uniquely named quarantine is never treated as a lock */
|
|
600
|
+
}
|
|
601
|
+
return true;
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
/**
|
|
605
|
+
* Atomically move the canonical lock aside and verify that the moved bytes are
|
|
606
|
+
* exactly the snapshot the caller classified. A competing stale taker can no
|
|
607
|
+
* longer unlink a freshly created owner lock and then claim success.
|
|
608
|
+
*/
|
|
609
|
+
function quarantineMutationLock(sessionId, snapshot, action) {
|
|
610
|
+
const filePath = mutationLockPath(sessionId);
|
|
611
|
+
const quarantinePath = mutationLockQuarantinePath(sessionId, action);
|
|
612
|
+
try {
|
|
613
|
+
fs.renameSync(filePath, quarantinePath);
|
|
614
|
+
} catch (err) {
|
|
615
|
+
if (err && err.code === "ENOENT") return { ok: false, raced: true };
|
|
616
|
+
return { ok: false, error: `state lock ${action} failed: ${err.message}` };
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
let movedRaw = null;
|
|
620
|
+
try {
|
|
621
|
+
movedRaw = fs.readFileSync(quarantinePath, "utf8");
|
|
622
|
+
} catch (err) {
|
|
623
|
+
const restored = restoreQuarantinedMutationLock(filePath, quarantinePath);
|
|
624
|
+
return {
|
|
625
|
+
ok: false,
|
|
626
|
+
error:
|
|
627
|
+
`state lock ${action} verify failed: ${err.message}` +
|
|
628
|
+
(restored ? "" : "; original lock could not be restored"),
|
|
629
|
+
};
|
|
630
|
+
}
|
|
631
|
+
if (!snapshot || movedRaw !== snapshot.raw) {
|
|
632
|
+
const restored = restoreQuarantinedMutationLock(filePath, quarantinePath);
|
|
633
|
+
return restored
|
|
634
|
+
? { ok: false, raced: true }
|
|
635
|
+
: {
|
|
636
|
+
ok: false,
|
|
637
|
+
error: `state lock ${action} raced and could not be restored`,
|
|
638
|
+
};
|
|
639
|
+
}
|
|
640
|
+
return { ok: true, path: quarantinePath };
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
function acquireMutationLock(sessionId) {
|
|
644
|
+
ensureDir(watermarkDir());
|
|
645
|
+
const filePath = mutationLockPath(sessionId);
|
|
646
|
+
let stabilizeAfterReclaim = false;
|
|
647
|
+
for (let attempt = 0; attempt < MUTATION_LOCK_RETRIES; attempt++) {
|
|
648
|
+
const now = Date.now();
|
|
649
|
+
const lock = {
|
|
650
|
+
token: crypto.randomBytes(16).toString("hex"),
|
|
651
|
+
pid: process.pid,
|
|
652
|
+
expires_at: new Date(now + MUTATION_LOCK_TTL_MS).toISOString(),
|
|
653
|
+
session_id: sessionId,
|
|
654
|
+
};
|
|
655
|
+
try {
|
|
656
|
+
fs.writeFileSync(filePath, `${JSON.stringify(lock)}\n`, {
|
|
657
|
+
flag: "wx",
|
|
658
|
+
mode: 0o600,
|
|
659
|
+
});
|
|
660
|
+
if (stabilizeAfterReclaim) {
|
|
661
|
+
// Give stale contenders that read the old owner before quarantine one
|
|
662
|
+
// retry interval to finish their verified rename. If one displaced and
|
|
663
|
+
// restored this lock, the token remains ours; if another owner won the
|
|
664
|
+
// create race, retry instead of entering the critical section.
|
|
665
|
+
sleepSync(MUTATION_LOCK_RETRY_MS);
|
|
666
|
+
const owned = readMutationLock(sessionId);
|
|
667
|
+
if (!owned || String(owned.token) !== String(lock.token)) continue;
|
|
668
|
+
}
|
|
669
|
+
return { ok: true, lock };
|
|
670
|
+
} catch (err) {
|
|
671
|
+
if (!err || err.code !== "EEXIST") {
|
|
672
|
+
return blockedOutcome("transient", `state lock failed: ${err.message}`, {
|
|
673
|
+
session_id: sessionId,
|
|
674
|
+
});
|
|
675
|
+
}
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
const snapshot = readMutationLockSnapshot(sessionId);
|
|
679
|
+
if (mutationLockReclaimable(snapshot, now)) {
|
|
680
|
+
const quarantined = quarantineMutationLock(
|
|
681
|
+
sessionId,
|
|
682
|
+
snapshot,
|
|
683
|
+
"stale",
|
|
684
|
+
);
|
|
685
|
+
if (quarantined.ok) {
|
|
686
|
+
try {
|
|
687
|
+
fs.unlinkSync(quarantined.path);
|
|
688
|
+
} catch {
|
|
689
|
+
/* unique quarantine is inert; acquisition may still continue */
|
|
690
|
+
}
|
|
691
|
+
stabilizeAfterReclaim = true;
|
|
692
|
+
continue;
|
|
693
|
+
}
|
|
694
|
+
if (quarantined.error) {
|
|
695
|
+
return blockedOutcome("transient", quarantined.error, {
|
|
696
|
+
session_id: sessionId,
|
|
697
|
+
});
|
|
698
|
+
}
|
|
699
|
+
sleepSync(MUTATION_LOCK_RETRY_MS);
|
|
700
|
+
continue;
|
|
701
|
+
}
|
|
702
|
+
if (snapshot.error) {
|
|
703
|
+
return blockedOutcome(
|
|
704
|
+
"transient",
|
|
705
|
+
`state lock read failed: ${snapshot.error}`,
|
|
706
|
+
{ session_id: sessionId },
|
|
707
|
+
);
|
|
708
|
+
}
|
|
709
|
+
sleepSync(MUTATION_LOCK_RETRY_MS);
|
|
710
|
+
}
|
|
711
|
+
return blockedOutcome("transient", "state mutation lock busy", {
|
|
712
|
+
session_id: sessionId,
|
|
713
|
+
path: mutationLockPath(sessionId),
|
|
714
|
+
});
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
function releaseMutationLock(sessionId, token) {
|
|
718
|
+
const snapshot = readMutationLockSnapshot(sessionId);
|
|
719
|
+
const existing = snapshot.lock;
|
|
720
|
+
if (!existing || String(existing.token) !== String(token)) return false;
|
|
721
|
+
const quarantined = quarantineMutationLock(sessionId, snapshot, "release");
|
|
722
|
+
if (!quarantined.ok) return false;
|
|
723
|
+
try {
|
|
724
|
+
fs.unlinkSync(quarantined.path);
|
|
725
|
+
return true;
|
|
726
|
+
} catch {
|
|
727
|
+
return false;
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
|
|
731
|
+
function lifecycleRank(value) {
|
|
732
|
+
return value === "closed" ? 2 : value === "closing" ? 1 : 0;
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
const STATE_LOCK_LOST_CODE = "MEKO_STATE_LOCK_LOST";
|
|
736
|
+
|
|
737
|
+
function assertMutationLockOwner(sessionId, token) {
|
|
738
|
+
const existing = readMutationLock(sessionId);
|
|
739
|
+
if (existing && String(existing.token) === String(token)) return;
|
|
740
|
+
const err = new Error("state mutation lock ownership lost before commit");
|
|
741
|
+
err.code = STATE_LOCK_LOST_CODE;
|
|
742
|
+
throw err;
|
|
743
|
+
}
|
|
744
|
+
|
|
745
|
+
function writeStateUnlocked(sessionId, state, options) {
|
|
746
|
+
const opts = options && typeof options === "object" ? options : {};
|
|
747
|
+
if (!opts.lockToken) {
|
|
748
|
+
return blockedOutcome(
|
|
749
|
+
"transient",
|
|
750
|
+
"state mutation lock token missing before commit",
|
|
751
|
+
{ session_id: sessionId },
|
|
752
|
+
);
|
|
753
|
+
}
|
|
754
|
+
const validated = validateV2(state);
|
|
755
|
+
if (!validated.ok) return { ...validated, session_id: sessionId };
|
|
756
|
+
|
|
757
|
+
let next = { ...validated.state, session_id: sessionId };
|
|
758
|
+
const current = readState(sessionId);
|
|
759
|
+
if (!current.ok && !current.missing) return current;
|
|
760
|
+
if (current.ok && current.state) {
|
|
761
|
+
const disk = current.state;
|
|
762
|
+
if (next.conversation_epoch < disk.conversation_epoch) {
|
|
763
|
+
return blockedOutcome(
|
|
764
|
+
"ownership",
|
|
765
|
+
`stale conversation_epoch write: ${next.conversation_epoch} < ${disk.conversation_epoch}`,
|
|
766
|
+
{ session_id: sessionId },
|
|
767
|
+
);
|
|
768
|
+
}
|
|
769
|
+
if (next.conversation_epoch === disk.conversation_epoch) {
|
|
770
|
+
const candidateMissingConversation =
|
|
771
|
+
!next.conversation_id && Boolean(disk.conversation_id);
|
|
772
|
+
if (
|
|
773
|
+
next.conversation_id &&
|
|
774
|
+
disk.conversation_id &&
|
|
775
|
+
next.conversation_id !== disk.conversation_id
|
|
776
|
+
) {
|
|
777
|
+
return blockedOutcome(
|
|
778
|
+
"ownership",
|
|
779
|
+
"conversation_id changed without advancing conversation_epoch",
|
|
780
|
+
{ session_id: sessionId },
|
|
781
|
+
);
|
|
782
|
+
}
|
|
783
|
+
if (candidateMissingConversation) {
|
|
784
|
+
// Conversation creation may complete while another process still
|
|
785
|
+
// holds the pre-create snapshot. Clearing ownership requires an epoch
|
|
786
|
+
// transition; a same-epoch stale write must retain the new ID.
|
|
787
|
+
next.conversation_id = disk.conversation_id;
|
|
788
|
+
if (next.delivery === "needs_conversation") {
|
|
789
|
+
next.delivery = disk.delivery;
|
|
790
|
+
}
|
|
791
|
+
}
|
|
792
|
+
if (next.last_line_number < disk.last_line_number) {
|
|
793
|
+
return blockedOutcome(
|
|
794
|
+
"ownership",
|
|
795
|
+
`non-monotonic persisted cursor: ${next.last_line_number} < ${disk.last_line_number}`,
|
|
796
|
+
{ session_id: sessionId },
|
|
797
|
+
);
|
|
798
|
+
}
|
|
799
|
+
// Preserve extension fields (for example legacy import markers) and a
|
|
800
|
+
// concurrent SessionEnd lifecycle transition when a checkpoint writes a
|
|
801
|
+
// state snapshot it loaded before the transition.
|
|
802
|
+
next = { ...disk, ...next };
|
|
803
|
+
if (
|
|
804
|
+
!opts.allowLifecycleReset &&
|
|
805
|
+
lifecycleRank(next.lifecycle) < lifecycleRank(disk.lifecycle)
|
|
806
|
+
) {
|
|
807
|
+
const candidateDelivery = next.delivery;
|
|
808
|
+
next.lifecycle = disk.lifecycle;
|
|
809
|
+
if (["idle", "complete"].includes(candidateDelivery)) {
|
|
810
|
+
next.delivery = disk.delivery;
|
|
811
|
+
next.queued_exchanges = Math.max(
|
|
812
|
+
coerceNonNegInt(next.queued_exchanges, 0),
|
|
813
|
+
coerceNonNegInt(disk.queued_exchanges, 0),
|
|
814
|
+
);
|
|
815
|
+
}
|
|
816
|
+
}
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
next.updated_at = nowIso();
|
|
820
|
+
const filePath = watermarkPath(sessionId);
|
|
821
|
+
try {
|
|
822
|
+
atomicWriteJson(filePath, next, {
|
|
823
|
+
// The temp file is fully written and fsynced before this callback. Check
|
|
824
|
+
// ownership at the narrowest dependency-free commit boundary: directly
|
|
825
|
+
// before the atomic rename that replaces durable session state.
|
|
826
|
+
beforeRename: () => assertMutationLockOwner(sessionId, opts.lockToken),
|
|
827
|
+
});
|
|
828
|
+
} catch (err) {
|
|
829
|
+
if (err && err.code === STATE_LOCK_LOST_CODE) {
|
|
830
|
+
return blockedOutcome("transient", err.message, {
|
|
831
|
+
session_id: sessionId,
|
|
832
|
+
path: filePath,
|
|
833
|
+
});
|
|
834
|
+
}
|
|
835
|
+
return blockedOutcome("transient", `write failed: ${err.message}`, {
|
|
836
|
+
session_id: sessionId,
|
|
837
|
+
path: filePath,
|
|
838
|
+
});
|
|
839
|
+
}
|
|
840
|
+
return { ok: true, session_id: sessionId, path: filePath, state: next };
|
|
841
|
+
}
|
|
842
|
+
|
|
843
|
+
/**
|
|
844
|
+
* Persist a v2 state object atomically under the short-lived state mutation
|
|
845
|
+
* lock. Cursor/epoch regressions are rejected against the fresh on-disk state.
|
|
846
|
+
*/
|
|
847
|
+
function writeState(sessionId, state, options) {
|
|
848
|
+
const acquired = acquireMutationLock(sessionId);
|
|
849
|
+
if (!acquired.ok) return acquired;
|
|
850
|
+
try {
|
|
851
|
+
return writeStateUnlocked(sessionId, state, {
|
|
852
|
+
...(options && typeof options === "object" ? options : {}),
|
|
853
|
+
lockToken: acquired.lock.token,
|
|
854
|
+
});
|
|
855
|
+
} finally {
|
|
856
|
+
releaseMutationLock(sessionId, acquired.lock.token);
|
|
857
|
+
}
|
|
858
|
+
}
|
|
859
|
+
|
|
860
|
+
/**
|
|
861
|
+
* Serialize a read-modify-write against fresh state. The mutator receives the
|
|
862
|
+
* readState outcome and returns either a candidate state or a typed failure.
|
|
863
|
+
*/
|
|
864
|
+
function mutateState(sessionId, mutator, options) {
|
|
865
|
+
if (typeof mutator !== "function") {
|
|
866
|
+
return blockedOutcome("persistent", "state mutator must be a function", {
|
|
867
|
+
session_id: sessionId,
|
|
868
|
+
});
|
|
869
|
+
}
|
|
870
|
+
const acquired = acquireMutationLock(sessionId);
|
|
871
|
+
if (!acquired.ok) return acquired;
|
|
872
|
+
try {
|
|
873
|
+
const current = readState(sessionId);
|
|
874
|
+
const result = mutator(current);
|
|
875
|
+
if (result && result.ok === false) return result;
|
|
876
|
+
const candidate = result && result.state ? result.state : result;
|
|
877
|
+
return writeStateUnlocked(sessionId, candidate, {
|
|
878
|
+
...(options && typeof options === "object" ? options : {}),
|
|
879
|
+
lockToken: acquired.lock.token,
|
|
880
|
+
});
|
|
881
|
+
} catch (err) {
|
|
882
|
+
return blockedOutcome("transient", `state mutation failed: ${err.message}`, {
|
|
883
|
+
session_id: sessionId,
|
|
884
|
+
});
|
|
885
|
+
} finally {
|
|
886
|
+
releaseMutationLock(sessionId, acquired.lock.token);
|
|
887
|
+
}
|
|
888
|
+
}
|
|
889
|
+
|
|
890
|
+
/**
|
|
891
|
+
* SessionStart intent write — durable before any network call.
|
|
892
|
+
* lifecycle=active; delivery=needs_conversation when conversation_id absent.
|
|
893
|
+
*/
|
|
894
|
+
function writeSessionIntent(sessionId, fields) {
|
|
895
|
+
const f = fields && typeof fields === "object" ? fields : {};
|
|
896
|
+
return mutateState(
|
|
897
|
+
sessionId,
|
|
898
|
+
(existing) => {
|
|
899
|
+
// Intent must not invent a cursor on corrupt files.
|
|
900
|
+
if (
|
|
901
|
+
!existing.ok &&
|
|
902
|
+
!existing.missing &&
|
|
903
|
+
existing.failure_class === "state_corrupt"
|
|
904
|
+
) {
|
|
905
|
+
return {
|
|
906
|
+
...existing,
|
|
907
|
+
error:
|
|
908
|
+
existing.error ||
|
|
909
|
+
"refusing writeSessionIntent over corrupt watermark",
|
|
910
|
+
};
|
|
911
|
+
}
|
|
912
|
+
|
|
913
|
+
const base =
|
|
914
|
+
existing.ok && existing.state
|
|
915
|
+
? { ...existing.state }
|
|
916
|
+
: {
|
|
917
|
+
schema_version: 2,
|
|
918
|
+
session_id: sessionId,
|
|
919
|
+
client: "",
|
|
920
|
+
agent_id: "",
|
|
921
|
+
agent_id_source: "",
|
|
922
|
+
transcript_path: null,
|
|
923
|
+
datapack_id: null,
|
|
924
|
+
datapack_name: null,
|
|
925
|
+
created_at: nowIso(),
|
|
926
|
+
updated_at: nowIso(),
|
|
927
|
+
last_activity_at: nowIso(),
|
|
928
|
+
conversation_id: null,
|
|
929
|
+
conversation_epoch: 0,
|
|
930
|
+
last_line_number: 0,
|
|
931
|
+
in_flight: emptyInFlight(),
|
|
932
|
+
lifecycle: "active",
|
|
933
|
+
delivery: "needs_conversation",
|
|
934
|
+
queued_exchanges: 0,
|
|
935
|
+
attempt_count: 0,
|
|
936
|
+
next_retry_at: null,
|
|
937
|
+
failure_class: null,
|
|
938
|
+
last_error: null,
|
|
939
|
+
last_success_at: null,
|
|
940
|
+
blocked_reason: null,
|
|
941
|
+
};
|
|
942
|
+
|
|
943
|
+
const conversationId =
|
|
944
|
+
f.conversation_id !== undefined
|
|
945
|
+
? f.conversation_id == null || f.conversation_id === ""
|
|
946
|
+
? null
|
|
947
|
+
: String(f.conversation_id)
|
|
948
|
+
: base.conversation_id;
|
|
949
|
+
|
|
950
|
+
const merged = {
|
|
951
|
+
...base,
|
|
952
|
+
session_id: sessionId,
|
|
953
|
+
schema_version: 2,
|
|
954
|
+
client: f.client != null ? String(f.client) : base.client || "",
|
|
955
|
+
agent_id: f.agent_id != null ? String(f.agent_id) : base.agent_id || "",
|
|
956
|
+
agent_id_source:
|
|
957
|
+
f.agent_id_source != null
|
|
958
|
+
? String(f.agent_id_source)
|
|
959
|
+
: base.agent_id_source || "",
|
|
960
|
+
transcript_path:
|
|
961
|
+
f.transcript_path !== undefined
|
|
962
|
+
? f.transcript_path
|
|
963
|
+
: base.transcript_path,
|
|
964
|
+
datapack_id:
|
|
965
|
+
f.datapack_id !== undefined ? f.datapack_id : base.datapack_id,
|
|
966
|
+
datapack_name:
|
|
967
|
+
f.datapack_name !== undefined ? f.datapack_name : base.datapack_name,
|
|
968
|
+
conversation_id: conversationId,
|
|
969
|
+
lifecycle: "active",
|
|
970
|
+
delivery: hasConversationId(conversationId)
|
|
971
|
+
? base.delivery === "needs_conversation"
|
|
972
|
+
? "idle"
|
|
973
|
+
: base.delivery || "idle"
|
|
974
|
+
: "needs_conversation",
|
|
975
|
+
last_activity_at: nowIso(),
|
|
976
|
+
updated_at: nowIso(),
|
|
977
|
+
};
|
|
978
|
+
|
|
979
|
+
if (f.last_line_number != null && existing.missing) {
|
|
980
|
+
merged.last_line_number = coerceNonNegInt(f.last_line_number, 0);
|
|
981
|
+
}
|
|
982
|
+
return merged;
|
|
983
|
+
},
|
|
984
|
+
{ allowLifecycleReset: true },
|
|
985
|
+
);
|
|
986
|
+
}
|
|
987
|
+
|
|
988
|
+
// ---------------------------------------------------------------------------
|
|
989
|
+
// Cursor / epoch
|
|
990
|
+
// ---------------------------------------------------------------------------
|
|
991
|
+
|
|
992
|
+
/**
|
|
993
|
+
* Advance last_line_number monotonically within the current conversation_epoch.
|
|
994
|
+
* Rejects non-monotonic advances. Optionally clears in_flight seed.
|
|
995
|
+
*
|
|
996
|
+
* @param {object} state
|
|
997
|
+
* @param {number} nextLine
|
|
998
|
+
* @param {{ clearSeed?: boolean, seed?: null }} [opts]
|
|
999
|
+
* @returns {{ ok:true, state } | { ok:false, failure_class, error }}
|
|
1000
|
+
*/
|
|
1001
|
+
function advanceCursor(state, nextLine, opts) {
|
|
1002
|
+
const options = opts && typeof opts === "object" ? opts : {};
|
|
1003
|
+
const validated = validateV2(state);
|
|
1004
|
+
if (!validated.ok) return validated;
|
|
1005
|
+
|
|
1006
|
+
const next = Number(nextLine);
|
|
1007
|
+
if (!Number.isInteger(next) || next < 0) {
|
|
1008
|
+
return {
|
|
1009
|
+
ok: false,
|
|
1010
|
+
failure_class: "persistent",
|
|
1011
|
+
error: `nextLine must be a non-negative integer, got ${nextLine}`,
|
|
1012
|
+
};
|
|
1013
|
+
}
|
|
1014
|
+
if (next < validated.state.last_line_number) {
|
|
1015
|
+
return {
|
|
1016
|
+
ok: false,
|
|
1017
|
+
failure_class: "persistent",
|
|
1018
|
+
error: `non-monotonic cursor advance: ${next} < ${validated.state.last_line_number}`,
|
|
1019
|
+
};
|
|
1020
|
+
}
|
|
1021
|
+
|
|
1022
|
+
const out = {
|
|
1023
|
+
...validated.state,
|
|
1024
|
+
last_line_number: next,
|
|
1025
|
+
updated_at: nowIso(),
|
|
1026
|
+
last_activity_at: nowIso(),
|
|
1027
|
+
in_flight: cloneInFlight(validated.state.in_flight),
|
|
1028
|
+
};
|
|
1029
|
+
|
|
1030
|
+
const clearSeed =
|
|
1031
|
+
options.clearSeed === true ||
|
|
1032
|
+
options["seed clear"] === true ||
|
|
1033
|
+
Object.prototype.hasOwnProperty.call(options, "seed") &&
|
|
1034
|
+
options.seed == null;
|
|
1035
|
+
|
|
1036
|
+
if (clearSeed) {
|
|
1037
|
+
out.in_flight = emptyInFlight();
|
|
1038
|
+
}
|
|
1039
|
+
|
|
1040
|
+
return { ok: true, state: out };
|
|
1041
|
+
}
|
|
1042
|
+
|
|
1043
|
+
/**
|
|
1044
|
+
* Explicit rebucket: increment conversation_epoch and reset cursor.
|
|
1045
|
+
*/
|
|
1046
|
+
function rebucket(state, newConversationId) {
|
|
1047
|
+
const validated = validateV2(state);
|
|
1048
|
+
if (!validated.ok) return validated;
|
|
1049
|
+
if (!hasConversationId(newConversationId)) {
|
|
1050
|
+
return {
|
|
1051
|
+
ok: false,
|
|
1052
|
+
failure_class: "persistent",
|
|
1053
|
+
error: "rebucket requires a non-empty conversation_id",
|
|
1054
|
+
};
|
|
1055
|
+
}
|
|
1056
|
+
const out = {
|
|
1057
|
+
...validated.state,
|
|
1058
|
+
conversation_id: String(newConversationId),
|
|
1059
|
+
conversation_epoch: validated.state.conversation_epoch + 1,
|
|
1060
|
+
last_line_number: 0,
|
|
1061
|
+
in_flight: emptyInFlight(),
|
|
1062
|
+
delivery: "idle",
|
|
1063
|
+
failure_class: null,
|
|
1064
|
+
last_error: null,
|
|
1065
|
+
blocked_reason: null,
|
|
1066
|
+
next_retry_at: null,
|
|
1067
|
+
attempt_count: 0,
|
|
1068
|
+
updated_at: nowIso(),
|
|
1069
|
+
last_activity_at: nowIso(),
|
|
1070
|
+
};
|
|
1071
|
+
return { ok: true, state: out };
|
|
1072
|
+
}
|
|
1073
|
+
|
|
1074
|
+
// ---------------------------------------------------------------------------
|
|
1075
|
+
// Leases
|
|
1076
|
+
// ---------------------------------------------------------------------------
|
|
1077
|
+
|
|
1078
|
+
function readLeaseFile(sessionId) {
|
|
1079
|
+
const filePath = leasePath(sessionId);
|
|
1080
|
+
try {
|
|
1081
|
+
const raw = JSON.parse(fs.readFileSync(filePath, "utf8"));
|
|
1082
|
+
if (!raw || typeof raw !== "object") return null;
|
|
1083
|
+
return raw;
|
|
1084
|
+
} catch (err) {
|
|
1085
|
+
if (err && err.code === "ENOENT") return null;
|
|
1086
|
+
return null;
|
|
1087
|
+
}
|
|
1088
|
+
}
|
|
1089
|
+
|
|
1090
|
+
function leaseExpired(lease, nowMs) {
|
|
1091
|
+
if (!lease) return true;
|
|
1092
|
+
const exp = parseIsoMs(lease.expires_at);
|
|
1093
|
+
if (exp == null) return true;
|
|
1094
|
+
return nowMs >= exp;
|
|
1095
|
+
}
|
|
1096
|
+
|
|
1097
|
+
function leaseReclaimable(lease, nowMs) {
|
|
1098
|
+
if (!lease) return true;
|
|
1099
|
+
if (leaseExpired(lease, nowMs)) return true;
|
|
1100
|
+
const pid = Number(lease.pid);
|
|
1101
|
+
if (Number.isInteger(pid) && pid > 0 && !isPidAlive(pid)) return true;
|
|
1102
|
+
return false;
|
|
1103
|
+
}
|
|
1104
|
+
|
|
1105
|
+
function buildLease(sessionId, ttlSeconds, pid, nowMs) {
|
|
1106
|
+
const acquired = nowIso(nowMs);
|
|
1107
|
+
const expires = nowIso(nowMs + ttlSeconds * 1000);
|
|
1108
|
+
return {
|
|
1109
|
+
token: crypto.randomBytes(16).toString("hex"),
|
|
1110
|
+
pid: Number.isInteger(pid) && pid > 0 ? pid : process.pid,
|
|
1111
|
+
acquired_at: acquired,
|
|
1112
|
+
expires_at: expires,
|
|
1113
|
+
session_id: sessionId,
|
|
1114
|
+
};
|
|
1115
|
+
}
|
|
1116
|
+
|
|
1117
|
+
/**
|
|
1118
|
+
* Acquire a tokenized drain lease (atomic create wx, or reclaim if expired/dead).
|
|
1119
|
+
* @returns {{ ok:true, lease } | { ok:false, failure_class:"ownership", error, lease? }}
|
|
1120
|
+
*/
|
|
1121
|
+
function acquireLease(sessionId, options) {
|
|
1122
|
+
const opts = options && typeof options === "object" ? options : {};
|
|
1123
|
+
const ttl = Math.max(
|
|
1124
|
+
1,
|
|
1125
|
+
opts.ttlSeconds != null ? Number(opts.ttlSeconds) : leaseTtlSeconds(),
|
|
1126
|
+
);
|
|
1127
|
+
const pid =
|
|
1128
|
+
opts.pid != null && Number.isInteger(Number(opts.pid))
|
|
1129
|
+
? Number(opts.pid)
|
|
1130
|
+
: process.pid;
|
|
1131
|
+
const nowMs = opts.now != null ? Number(opts.now) : Date.now();
|
|
1132
|
+
const filePath = leasePath(sessionId);
|
|
1133
|
+
ensureDir(watermarkDir());
|
|
1134
|
+
|
|
1135
|
+
const lease = buildLease(sessionId, ttl, pid, nowMs);
|
|
1136
|
+
const payload = `${JSON.stringify(lease)}\n`;
|
|
1137
|
+
|
|
1138
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
1139
|
+
try {
|
|
1140
|
+
fs.writeFileSync(filePath, payload, { flag: "wx", mode: 0o600 });
|
|
1141
|
+
return { ok: true, lease };
|
|
1142
|
+
} catch (err) {
|
|
1143
|
+
if (!err || err.code !== "EEXIST") {
|
|
1144
|
+
return {
|
|
1145
|
+
ok: false,
|
|
1146
|
+
failure_class: "transient",
|
|
1147
|
+
error: `lease acquire failed: ${err.message}`,
|
|
1148
|
+
};
|
|
1149
|
+
}
|
|
1150
|
+
const existing = readLeaseFile(sessionId);
|
|
1151
|
+
if (!leaseReclaimable(existing, nowMs)) {
|
|
1152
|
+
return {
|
|
1153
|
+
ok: false,
|
|
1154
|
+
failure_class: "ownership",
|
|
1155
|
+
error: "lease held by another worker",
|
|
1156
|
+
lease: existing,
|
|
1157
|
+
};
|
|
1158
|
+
}
|
|
1159
|
+
try {
|
|
1160
|
+
fs.unlinkSync(filePath);
|
|
1161
|
+
} catch (unlinkErr) {
|
|
1162
|
+
if (unlinkErr && unlinkErr.code !== "ENOENT") {
|
|
1163
|
+
return {
|
|
1164
|
+
ok: false,
|
|
1165
|
+
failure_class: "ownership",
|
|
1166
|
+
error: `could not reclaim lease: ${unlinkErr.message}`,
|
|
1167
|
+
lease: existing,
|
|
1168
|
+
};
|
|
1169
|
+
}
|
|
1170
|
+
}
|
|
1171
|
+
}
|
|
1172
|
+
}
|
|
1173
|
+
return {
|
|
1174
|
+
ok: false,
|
|
1175
|
+
failure_class: "ownership",
|
|
1176
|
+
error: "lease acquire raced; giving up",
|
|
1177
|
+
};
|
|
1178
|
+
}
|
|
1179
|
+
|
|
1180
|
+
function renewLease(sessionId, token, options) {
|
|
1181
|
+
const opts = options && typeof options === "object" ? options : {};
|
|
1182
|
+
const ttl = Math.max(
|
|
1183
|
+
1,
|
|
1184
|
+
opts.ttlSeconds != null ? Number(opts.ttlSeconds) : leaseTtlSeconds(),
|
|
1185
|
+
);
|
|
1186
|
+
const nowMs = opts.now != null ? Number(opts.now) : Date.now();
|
|
1187
|
+
const existing = readLeaseFile(sessionId);
|
|
1188
|
+
if (!existing) {
|
|
1189
|
+
return {
|
|
1190
|
+
ok: false,
|
|
1191
|
+
failure_class: "ownership",
|
|
1192
|
+
error: "lease missing",
|
|
1193
|
+
};
|
|
1194
|
+
}
|
|
1195
|
+
if (String(existing.token) !== String(token)) {
|
|
1196
|
+
return {
|
|
1197
|
+
ok: false,
|
|
1198
|
+
failure_class: "ownership",
|
|
1199
|
+
error: "lease token mismatch",
|
|
1200
|
+
lease: existing,
|
|
1201
|
+
};
|
|
1202
|
+
}
|
|
1203
|
+
const renewed = {
|
|
1204
|
+
...existing,
|
|
1205
|
+
expires_at: nowIso(nowMs + ttl * 1000),
|
|
1206
|
+
};
|
|
1207
|
+
try {
|
|
1208
|
+
atomicWriteJson(leasePath(sessionId), renewed);
|
|
1209
|
+
} catch (err) {
|
|
1210
|
+
return {
|
|
1211
|
+
ok: false,
|
|
1212
|
+
failure_class: "transient",
|
|
1213
|
+
error: `lease renew failed: ${err.message}`,
|
|
1214
|
+
};
|
|
1215
|
+
}
|
|
1216
|
+
return { ok: true, lease: renewed };
|
|
1217
|
+
}
|
|
1218
|
+
|
|
1219
|
+
function releaseLease(sessionId, token) {
|
|
1220
|
+
const filePath = leasePath(sessionId);
|
|
1221
|
+
const existing = readLeaseFile(sessionId);
|
|
1222
|
+
if (!existing) {
|
|
1223
|
+
return { ok: true, released: false, missing: true };
|
|
1224
|
+
}
|
|
1225
|
+
if (String(existing.token) !== String(token)) {
|
|
1226
|
+
return {
|
|
1227
|
+
ok: false,
|
|
1228
|
+
failure_class: "ownership",
|
|
1229
|
+
error: "lease token mismatch; not released",
|
|
1230
|
+
lease: existing,
|
|
1231
|
+
};
|
|
1232
|
+
}
|
|
1233
|
+
try {
|
|
1234
|
+
fs.unlinkSync(filePath);
|
|
1235
|
+
} catch (err) {
|
|
1236
|
+
if (err && err.code === "ENOENT") {
|
|
1237
|
+
return { ok: true, released: false, missing: true };
|
|
1238
|
+
}
|
|
1239
|
+
return {
|
|
1240
|
+
ok: false,
|
|
1241
|
+
failure_class: "transient",
|
|
1242
|
+
error: `lease release failed: ${err.message}`,
|
|
1243
|
+
};
|
|
1244
|
+
}
|
|
1245
|
+
return { ok: true, released: true };
|
|
1246
|
+
}
|
|
1247
|
+
|
|
1248
|
+
// ---------------------------------------------------------------------------
|
|
1249
|
+
// Retry classification / backoff
|
|
1250
|
+
// ---------------------------------------------------------------------------
|
|
1251
|
+
|
|
1252
|
+
const PERSISTENT_PATTERNS = [
|
|
1253
|
+
["free_tier_limit_reached", /free[_\s-]?tier.*limit|free_tier_limit_reached/i],
|
|
1254
|
+
["quota_exceeded", /quota[_\s-]?(reached|exceeded)|(?:reached|exceeded).*quota/i],
|
|
1255
|
+
["usage_limit_reached", /usage[_\s-]?limit[_\s-]?(reached|exceeded)/i],
|
|
1256
|
+
["plan_limit_reached", /plan[_\s-]?limit[_\s-]?(reached|exceeded)/i],
|
|
1257
|
+
["payment_required", /payment[_\s-]?required/i],
|
|
1258
|
+
["subscription_required", /subscription[_\s-]?(required|inactive|expired)/i],
|
|
1259
|
+
["entitlement_denied", /entitlement.*(denied|required|missing)|(?:denied|required|missing).*entitlement/i],
|
|
1260
|
+
];
|
|
1261
|
+
|
|
1262
|
+
/**
|
|
1263
|
+
* Classify an Error, message string, or known failure_class token.
|
|
1264
|
+
* @returns {"transient"|"persistent"|"state_corrupt"|"ownership"|"datapack"}
|
|
1265
|
+
*/
|
|
1266
|
+
function classifyFailure(errOrClass) {
|
|
1267
|
+
if (errOrClass == null) return "transient";
|
|
1268
|
+
|
|
1269
|
+
if (typeof errOrClass === "string") {
|
|
1270
|
+
const trimmed = errOrClass.trim();
|
|
1271
|
+
if (FAILURE_CLASSES.includes(trimmed)) return trimmed;
|
|
1272
|
+
return classifyFailure({ message: trimmed });
|
|
1273
|
+
}
|
|
1274
|
+
|
|
1275
|
+
if (typeof errOrClass === "object") {
|
|
1276
|
+
if (
|
|
1277
|
+
typeof errOrClass.failure_class === "string" &&
|
|
1278
|
+
FAILURE_CLASSES.includes(errOrClass.failure_class)
|
|
1279
|
+
) {
|
|
1280
|
+
return errOrClass.failure_class;
|
|
1281
|
+
}
|
|
1282
|
+
const message = String(
|
|
1283
|
+
errOrClass.message != null ? errOrClass.message : errOrClass,
|
|
1284
|
+
);
|
|
1285
|
+
const normalized = message.toLowerCase();
|
|
1286
|
+
const statusCode = Number(errOrClass.statusCode || errOrClass.status);
|
|
1287
|
+
|
|
1288
|
+
if (
|
|
1289
|
+
/state[_\s-]?corrupt|malformed json|unsupported schema/i.test(message)
|
|
1290
|
+
) {
|
|
1291
|
+
return "state_corrupt";
|
|
1292
|
+
}
|
|
1293
|
+
if (
|
|
1294
|
+
/ownership|agent_id_mismatch|wrong owner|lease (held|token)/i.test(
|
|
1295
|
+
normalized,
|
|
1296
|
+
)
|
|
1297
|
+
) {
|
|
1298
|
+
return "ownership";
|
|
1299
|
+
}
|
|
1300
|
+
if (/datapack/i.test(normalized) && /mismatch|denied|invalid|missing/i.test(normalized)) {
|
|
1301
|
+
return "datapack";
|
|
1302
|
+
}
|
|
1303
|
+
|
|
1304
|
+
if (statusCode === 402 || statusCode === 403) return "persistent";
|
|
1305
|
+
if (/\bmcp http (402|403)\b/.test(normalized)) return "persistent";
|
|
1306
|
+
if (/\bhttp[_\s-]?(402|403)\b/.test(normalized)) return "persistent";
|
|
1307
|
+
if (/^http_(402|403)$/i.test(message.trim())) return "persistent";
|
|
1308
|
+
for (const [, pattern] of PERSISTENT_PATTERNS) {
|
|
1309
|
+
if (pattern.test(normalized)) return "persistent";
|
|
1310
|
+
}
|
|
1311
|
+
}
|
|
1312
|
+
|
|
1313
|
+
return "transient";
|
|
1314
|
+
}
|
|
1315
|
+
|
|
1316
|
+
/**
|
|
1317
|
+
* Compute next_retry_at ISO timestamp.
|
|
1318
|
+
* transient: exponential 30s→600s with ±jitter; persistent/blocked: 6h.
|
|
1319
|
+
*/
|
|
1320
|
+
function nextRetryAt(args) {
|
|
1321
|
+
const a = args && typeof args === "object" ? args : {};
|
|
1322
|
+
const failureClass = classifyFailure(a.failure_class || a);
|
|
1323
|
+
const attemptCount = Math.max(0, coerceNonNegInt(a.attempt_count, 0));
|
|
1324
|
+
const nowMs = a.now != null ? Number(a.now) : Date.now();
|
|
1325
|
+
const jitterFrac =
|
|
1326
|
+
a.jitter != null ? Number(a.jitter) : retryJitter();
|
|
1327
|
+
const random =
|
|
1328
|
+
typeof a.random === "number" ? a.random : Math.random();
|
|
1329
|
+
|
|
1330
|
+
let delaySec;
|
|
1331
|
+
if (failureClass === "transient") {
|
|
1332
|
+
const base = retryBaseSeconds();
|
|
1333
|
+
const max = retryMaxSeconds();
|
|
1334
|
+
const exp = Math.max(0, attemptCount <= 0 ? 0 : attemptCount - 1);
|
|
1335
|
+
const raw = Math.min(max, base * Math.pow(2, exp));
|
|
1336
|
+
const jitter = Number.isFinite(jitterFrac) ? jitterFrac : 0.2;
|
|
1337
|
+
const factor = 1 + (random * 2 - 1) * jitter;
|
|
1338
|
+
delaySec = Math.max(1, Math.round(raw * factor));
|
|
1339
|
+
} else {
|
|
1340
|
+
// persistent | state_corrupt | ownership | datapack
|
|
1341
|
+
delaySec = persistentRetrySeconds();
|
|
1342
|
+
}
|
|
1343
|
+
|
|
1344
|
+
return nowIso(nowMs + delaySec * 1000);
|
|
1345
|
+
}
|
|
1346
|
+
|
|
1347
|
+
/**
|
|
1348
|
+
* Pure delay (seconds) helper for tests — same policy as nextRetryAt without ISO.
|
|
1349
|
+
*/
|
|
1350
|
+
function retryDelaySeconds(args) {
|
|
1351
|
+
const a = args && typeof args === "object" ? args : {};
|
|
1352
|
+
const failureClass = classifyFailure(a.failure_class || a);
|
|
1353
|
+
const attemptCount = Math.max(0, coerceNonNegInt(a.attempt_count, 0));
|
|
1354
|
+
const jitterFrac =
|
|
1355
|
+
a.jitter != null ? Number(a.jitter) : retryJitter();
|
|
1356
|
+
const random =
|
|
1357
|
+
typeof a.random === "number" ? a.random : Math.random();
|
|
1358
|
+
|
|
1359
|
+
if (failureClass === "transient") {
|
|
1360
|
+
const base = retryBaseSeconds();
|
|
1361
|
+
const max = retryMaxSeconds();
|
|
1362
|
+
const exp = Math.max(0, attemptCount <= 0 ? 0 : attemptCount - 1);
|
|
1363
|
+
const raw = Math.min(max, base * Math.pow(2, exp));
|
|
1364
|
+
const jitter = Number.isFinite(jitterFrac) ? jitterFrac : 0.2;
|
|
1365
|
+
const factor = 1 + (random * 2 - 1) * jitter;
|
|
1366
|
+
return Math.max(1, Math.round(raw * factor));
|
|
1367
|
+
}
|
|
1368
|
+
return persistentRetrySeconds();
|
|
1369
|
+
}
|
|
1370
|
+
|
|
1371
|
+
// ---------------------------------------------------------------------------
|
|
1372
|
+
// Scan
|
|
1373
|
+
// ---------------------------------------------------------------------------
|
|
1374
|
+
|
|
1375
|
+
function timerLooksLive(sessionId) {
|
|
1376
|
+
try {
|
|
1377
|
+
const pid = Number.parseInt(
|
|
1378
|
+
fs.readFileSync(timerPidPath(sessionId), "utf8"),
|
|
1379
|
+
10,
|
|
1380
|
+
);
|
|
1381
|
+
return isPidAlive(pid);
|
|
1382
|
+
} catch {
|
|
1383
|
+
return false;
|
|
1384
|
+
}
|
|
1385
|
+
}
|
|
1386
|
+
|
|
1387
|
+
function buildEligibility(state, nowMs, options) {
|
|
1388
|
+
const opts = options && typeof options === "object" ? options : {};
|
|
1389
|
+
const now = nowMs != null ? nowMs : Date.now();
|
|
1390
|
+
const activityMs =
|
|
1391
|
+
parseIsoMs(state.last_activity_at) ||
|
|
1392
|
+
parseIsoMs(state.updated_at) ||
|
|
1393
|
+
null;
|
|
1394
|
+
const idleSeconds =
|
|
1395
|
+
activityMs == null ? null : Math.max(0, Math.floor((now - activityMs) / 1000));
|
|
1396
|
+
const minIdle = recoveryMinIdle();
|
|
1397
|
+
const abandoned =
|
|
1398
|
+
state.lifecycle === "active" &&
|
|
1399
|
+
idleSeconds != null &&
|
|
1400
|
+
idleSeconds >= minIdle &&
|
|
1401
|
+
!timerLooksLive(state.session_id);
|
|
1402
|
+
const closingEligible =
|
|
1403
|
+
state.lifecycle === "closing" || state.lifecycle === "closed";
|
|
1404
|
+
const hasConv = hasConversationId(state.conversation_id);
|
|
1405
|
+
const hasAgent =
|
|
1406
|
+
typeof state.agent_id === "string" && state.agent_id.trim() !== "";
|
|
1407
|
+
const retryMs = parseIsoMs(state.next_retry_at);
|
|
1408
|
+
const retryDue = retryMs == null || retryMs <= now;
|
|
1409
|
+
const persistentStartupOverride =
|
|
1410
|
+
Boolean(opts.forcePersistentRetry) &&
|
|
1411
|
+
state.failure_class === "persistent" &&
|
|
1412
|
+
!retryDue;
|
|
1413
|
+
const pendingWork =
|
|
1414
|
+
coerceNonNegInt(state.queued_exchanges, 0) > 0 ||
|
|
1415
|
+
["needs_conversation", "pending", "draining", "retry_wait", "blocked"].includes(
|
|
1416
|
+
state.delivery,
|
|
1417
|
+
);
|
|
1418
|
+
const ownerReady = hasAgent && (hasConv || state.delivery === "needs_conversation");
|
|
1419
|
+
|
|
1420
|
+
const eligibleForRecovery =
|
|
1421
|
+
pendingWork &&
|
|
1422
|
+
ownerReady &&
|
|
1423
|
+
(retryDue || persistentStartupOverride) &&
|
|
1424
|
+
state.delivery !== "complete" &&
|
|
1425
|
+
(closingEligible || abandoned || state.lifecycle !== "active");
|
|
1426
|
+
|
|
1427
|
+
return {
|
|
1428
|
+
has_conversation: hasConv,
|
|
1429
|
+
has_agent: hasAgent,
|
|
1430
|
+
delivery: state.delivery,
|
|
1431
|
+
lifecycle: state.lifecycle,
|
|
1432
|
+
queued_exchanges: coerceNonNegInt(state.queued_exchanges, 0),
|
|
1433
|
+
next_retry_at: state.next_retry_at,
|
|
1434
|
+
retry_due: retryDue,
|
|
1435
|
+
persistent_startup_override: persistentStartupOverride,
|
|
1436
|
+
last_activity_at: state.last_activity_at || null,
|
|
1437
|
+
idle_seconds: idleSeconds,
|
|
1438
|
+
timer_live: timerLooksLive(state.session_id),
|
|
1439
|
+
abandoned,
|
|
1440
|
+
eligible_for_recovery: eligibleForRecovery,
|
|
1441
|
+
sort_key: state.updated_at || state.last_activity_at || "",
|
|
1442
|
+
};
|
|
1443
|
+
}
|
|
1444
|
+
|
|
1445
|
+
function sessionHealthBucket(entry) {
|
|
1446
|
+
if (!entry || entry.ok === false) {
|
|
1447
|
+
const fc = entry && entry.failure_class;
|
|
1448
|
+
if (
|
|
1449
|
+
fc === "state_corrupt" ||
|
|
1450
|
+
fc === "ownership" ||
|
|
1451
|
+
fc === "datapack" ||
|
|
1452
|
+
(entry && entry.delivery === "blocked")
|
|
1453
|
+
) {
|
|
1454
|
+
return "blocked_action_required";
|
|
1455
|
+
}
|
|
1456
|
+
return "degraded_retrying";
|
|
1457
|
+
}
|
|
1458
|
+
const state = entry.state || entry;
|
|
1459
|
+
if (!state || typeof state !== "object") return "healthy";
|
|
1460
|
+
|
|
1461
|
+
const fc = state.failure_class;
|
|
1462
|
+
if (
|
|
1463
|
+
state.delivery === "blocked" ||
|
|
1464
|
+
fc === "state_corrupt" ||
|
|
1465
|
+
fc === "ownership" ||
|
|
1466
|
+
fc === "datapack"
|
|
1467
|
+
) {
|
|
1468
|
+
return "blocked_action_required";
|
|
1469
|
+
}
|
|
1470
|
+
if (
|
|
1471
|
+
state.delivery === "retry_wait" ||
|
|
1472
|
+
fc === "transient" ||
|
|
1473
|
+
fc === "persistent"
|
|
1474
|
+
) {
|
|
1475
|
+
return "degraded_retrying";
|
|
1476
|
+
}
|
|
1477
|
+
if (
|
|
1478
|
+
state.delivery === "needs_conversation" ||
|
|
1479
|
+
state.delivery === "pending" ||
|
|
1480
|
+
state.delivery === "draining" ||
|
|
1481
|
+
coerceNonNegInt(state.queued_exchanges, 0) > 0
|
|
1482
|
+
) {
|
|
1483
|
+
return "catching_up";
|
|
1484
|
+
}
|
|
1485
|
+
return "healthy";
|
|
1486
|
+
}
|
|
1487
|
+
|
|
1488
|
+
/**
|
|
1489
|
+
* Uncapped discovery of all watermark files under MEKO_WATERMARK_DIR.
|
|
1490
|
+
* Returns normalized/blocked entries with eligibility metadata, oldest first.
|
|
1491
|
+
*/
|
|
1492
|
+
function scanStates(options) {
|
|
1493
|
+
const opts = options && typeof options === "object" ? options : {};
|
|
1494
|
+
const includeBlocked = opts.includeBlocked !== false;
|
|
1495
|
+
const nowMs = opts.now != null ? Number(opts.now) : Date.now();
|
|
1496
|
+
const dir = watermarkDir();
|
|
1497
|
+
let names = [];
|
|
1498
|
+
try {
|
|
1499
|
+
names = fs.readdirSync(dir);
|
|
1500
|
+
} catch (err) {
|
|
1501
|
+
if (err && err.code === "ENOENT") {
|
|
1502
|
+
return { ok: true, entries: [], dir };
|
|
1503
|
+
}
|
|
1504
|
+
return {
|
|
1505
|
+
ok: false,
|
|
1506
|
+
failure_class: "transient",
|
|
1507
|
+
error: `scan failed: ${err.message}`,
|
|
1508
|
+
entries: [],
|
|
1509
|
+
dir,
|
|
1510
|
+
};
|
|
1511
|
+
}
|
|
1512
|
+
|
|
1513
|
+
const entries = [];
|
|
1514
|
+
for (const name of names) {
|
|
1515
|
+
if (!name.endsWith(".watermark.json")) continue;
|
|
1516
|
+
const sessionId = name.slice(0, -".watermark.json".length);
|
|
1517
|
+
if (!sessionId) continue;
|
|
1518
|
+
const read = readState(sessionId);
|
|
1519
|
+
if (!read.ok) {
|
|
1520
|
+
if (!includeBlocked) continue;
|
|
1521
|
+
entries.push({
|
|
1522
|
+
...read,
|
|
1523
|
+
eligibility: null,
|
|
1524
|
+
health: sessionHealthBucket(read),
|
|
1525
|
+
});
|
|
1526
|
+
continue;
|
|
1527
|
+
}
|
|
1528
|
+
const eligibility = buildEligibility(read.state, nowMs, {
|
|
1529
|
+
forcePersistentRetry: Boolean(opts.forcePersistentRetry),
|
|
1530
|
+
});
|
|
1531
|
+
entries.push({
|
|
1532
|
+
ok: true,
|
|
1533
|
+
session_id: sessionId,
|
|
1534
|
+
path: read.path,
|
|
1535
|
+
state: read.state,
|
|
1536
|
+
migrated: read.migrated,
|
|
1537
|
+
eligibility,
|
|
1538
|
+
health: sessionHealthBucket(read),
|
|
1539
|
+
});
|
|
1540
|
+
}
|
|
1541
|
+
|
|
1542
|
+
entries.sort((a, b) => {
|
|
1543
|
+
const ak =
|
|
1544
|
+
(a.eligibility && a.eligibility.sort_key) ||
|
|
1545
|
+
(a.state && a.state.updated_at) ||
|
|
1546
|
+
"";
|
|
1547
|
+
const bk =
|
|
1548
|
+
(b.eligibility && b.eligibility.sort_key) ||
|
|
1549
|
+
(b.state && b.state.updated_at) ||
|
|
1550
|
+
"";
|
|
1551
|
+
return String(ak).localeCompare(String(bk));
|
|
1552
|
+
});
|
|
1553
|
+
|
|
1554
|
+
return { ok: true, entries, dir };
|
|
1555
|
+
}
|
|
1556
|
+
|
|
1557
|
+
function sortOldestFirst(entries) {
|
|
1558
|
+
return [...(entries || [])].sort((a, b) => {
|
|
1559
|
+
const ak =
|
|
1560
|
+
(a.eligibility && a.eligibility.sort_key) ||
|
|
1561
|
+
(a.state && a.state.updated_at) ||
|
|
1562
|
+
"";
|
|
1563
|
+
const bk =
|
|
1564
|
+
(b.eligibility && b.eligibility.sort_key) ||
|
|
1565
|
+
(b.state && b.state.updated_at) ||
|
|
1566
|
+
"";
|
|
1567
|
+
return String(ak).localeCompare(String(bk));
|
|
1568
|
+
});
|
|
1569
|
+
}
|
|
1570
|
+
|
|
1571
|
+
// ---------------------------------------------------------------------------
|
|
1572
|
+
// Aggregate health
|
|
1573
|
+
// ---------------------------------------------------------------------------
|
|
1574
|
+
|
|
1575
|
+
function computeAggregateHealth(states, options) {
|
|
1576
|
+
const opts = options && typeof options === "object" ? options : {};
|
|
1577
|
+
const nowMs = opts.now != null ? Number(opts.now) : Date.now();
|
|
1578
|
+
const list = Array.isArray(states)
|
|
1579
|
+
? states
|
|
1580
|
+
: states && Array.isArray(states.entries)
|
|
1581
|
+
? states.entries
|
|
1582
|
+
: [];
|
|
1583
|
+
|
|
1584
|
+
const counts = {
|
|
1585
|
+
healthy: 0,
|
|
1586
|
+
catching_up: 0,
|
|
1587
|
+
degraded_retrying: 0,
|
|
1588
|
+
blocked_action_required: 0,
|
|
1589
|
+
};
|
|
1590
|
+
let queued = 0;
|
|
1591
|
+
let oldestPendingAge = null;
|
|
1592
|
+
let lastSuccessAt = null;
|
|
1593
|
+
const blockedReasons = [];
|
|
1594
|
+
const sessions = [];
|
|
1595
|
+
let status = "healthy";
|
|
1596
|
+
|
|
1597
|
+
for (const entry of list) {
|
|
1598
|
+
const bucket = sessionHealthBucket(entry);
|
|
1599
|
+
counts[bucket] = (counts[bucket] || 0) + 1;
|
|
1600
|
+
if (HEALTH_PRECEDENCE[bucket] > HEALTH_PRECEDENCE[status]) {
|
|
1601
|
+
status = bucket;
|
|
1602
|
+
}
|
|
1603
|
+
|
|
1604
|
+
const state = entry && (entry.state || (entry.ok !== false ? entry : null));
|
|
1605
|
+
const sessionId =
|
|
1606
|
+
(entry && entry.session_id) ||
|
|
1607
|
+
(state && state.session_id) ||
|
|
1608
|
+
null;
|
|
1609
|
+
const q =
|
|
1610
|
+
state && typeof state === "object"
|
|
1611
|
+
? coerceNonNegInt(state.queued_exchanges, 0)
|
|
1612
|
+
: 0;
|
|
1613
|
+
queued += q;
|
|
1614
|
+
|
|
1615
|
+
if (state && state.last_success_at) {
|
|
1616
|
+
if (
|
|
1617
|
+
!lastSuccessAt ||
|
|
1618
|
+
String(state.last_success_at) > String(lastSuccessAt)
|
|
1619
|
+
) {
|
|
1620
|
+
lastSuccessAt = state.last_success_at;
|
|
1621
|
+
}
|
|
1622
|
+
}
|
|
1623
|
+
|
|
1624
|
+
const pendingLike =
|
|
1625
|
+
bucket !== "healthy" ||
|
|
1626
|
+
q > 0 ||
|
|
1627
|
+
(state &&
|
|
1628
|
+
["pending", "draining", "retry_wait", "blocked"].includes(
|
|
1629
|
+
state.delivery,
|
|
1630
|
+
));
|
|
1631
|
+
if (pendingLike && state) {
|
|
1632
|
+
const activityMs =
|
|
1633
|
+
parseIsoMs(state.last_activity_at) || parseIsoMs(state.updated_at);
|
|
1634
|
+
if (activityMs != null) {
|
|
1635
|
+
const age = Math.max(0, Math.floor((nowMs - activityMs) / 1000));
|
|
1636
|
+
if (oldestPendingAge == null || age > oldestPendingAge) {
|
|
1637
|
+
oldestPendingAge = age;
|
|
1638
|
+
}
|
|
1639
|
+
}
|
|
1640
|
+
}
|
|
1641
|
+
|
|
1642
|
+
if (bucket === "blocked_action_required") {
|
|
1643
|
+
const reason =
|
|
1644
|
+
(state && (state.blocked_reason || state.last_error)) ||
|
|
1645
|
+
(entry && entry.error) ||
|
|
1646
|
+
(entry && entry.failure_class) ||
|
|
1647
|
+
"blocked";
|
|
1648
|
+
if (reason && !blockedReasons.includes(String(reason))) {
|
|
1649
|
+
blockedReasons.push(String(reason));
|
|
1650
|
+
}
|
|
1651
|
+
}
|
|
1652
|
+
|
|
1653
|
+
sessions.push({
|
|
1654
|
+
session_id: sessionId,
|
|
1655
|
+
status: bucket,
|
|
1656
|
+
delivery: state ? state.delivery : entry && entry.delivery,
|
|
1657
|
+
failure_class:
|
|
1658
|
+
(state && state.failure_class) ||
|
|
1659
|
+
(entry && entry.failure_class) ||
|
|
1660
|
+
null,
|
|
1661
|
+
queued_exchanges: q,
|
|
1662
|
+
lifecycle: state ? state.lifecycle : null,
|
|
1663
|
+
});
|
|
1664
|
+
}
|
|
1665
|
+
|
|
1666
|
+
return {
|
|
1667
|
+
schema_version: 1,
|
|
1668
|
+
status,
|
|
1669
|
+
updated_at: nowIso(nowMs),
|
|
1670
|
+
counts,
|
|
1671
|
+
queued_exchanges: queued,
|
|
1672
|
+
oldest_pending_age_seconds: oldestPendingAge,
|
|
1673
|
+
last_success_at: lastSuccessAt,
|
|
1674
|
+
blocked_reasons: blockedReasons,
|
|
1675
|
+
sessions,
|
|
1676
|
+
};
|
|
1677
|
+
}
|
|
1678
|
+
|
|
1679
|
+
function writeHealthCache(health) {
|
|
1680
|
+
const existing = readHealthCache();
|
|
1681
|
+
const merged = {
|
|
1682
|
+
...(health && typeof health === "object" ? health : {}),
|
|
1683
|
+
schema_version: 1,
|
|
1684
|
+
updated_at: nowIso(),
|
|
1685
|
+
};
|
|
1686
|
+
if (
|
|
1687
|
+
merged.last_notified_status === undefined &&
|
|
1688
|
+
existing &&
|
|
1689
|
+
existing.last_notified_status !== undefined
|
|
1690
|
+
) {
|
|
1691
|
+
merged.last_notified_status = existing.last_notified_status;
|
|
1692
|
+
}
|
|
1693
|
+
try {
|
|
1694
|
+
atomicWriteJson(healthCachePath(), merged);
|
|
1695
|
+
return { ok: true, path: healthCachePath(), health: merged };
|
|
1696
|
+
} catch (err) {
|
|
1697
|
+
return {
|
|
1698
|
+
ok: false,
|
|
1699
|
+
failure_class: "transient",
|
|
1700
|
+
error: `health cache write failed: ${err.message}`,
|
|
1701
|
+
};
|
|
1702
|
+
}
|
|
1703
|
+
}
|
|
1704
|
+
|
|
1705
|
+
function readHealthCache() {
|
|
1706
|
+
const filePath = healthCachePath();
|
|
1707
|
+
try {
|
|
1708
|
+
const raw = JSON.parse(fs.readFileSync(filePath, "utf8"));
|
|
1709
|
+
if (!raw || typeof raw !== "object") return null;
|
|
1710
|
+
return raw;
|
|
1711
|
+
} catch (err) {
|
|
1712
|
+
if (err && err.code === "ENOENT") return null;
|
|
1713
|
+
return null;
|
|
1714
|
+
}
|
|
1715
|
+
}
|
|
1716
|
+
|
|
1717
|
+
// ---------------------------------------------------------------------------
|
|
1718
|
+
// Legacy last-capture-error-* import
|
|
1719
|
+
// ---------------------------------------------------------------------------
|
|
1720
|
+
|
|
1721
|
+
function listLegacyErrorFiles() {
|
|
1722
|
+
const dir = watermarkDir();
|
|
1723
|
+
let names = [];
|
|
1724
|
+
try {
|
|
1725
|
+
names = fs.readdirSync(dir);
|
|
1726
|
+
} catch {
|
|
1727
|
+
return [];
|
|
1728
|
+
}
|
|
1729
|
+
const out = [];
|
|
1730
|
+
for (const name of names) {
|
|
1731
|
+
const m = /^last-capture-error-(.+)\.json$/.exec(name);
|
|
1732
|
+
if (!m) continue;
|
|
1733
|
+
out.push({
|
|
1734
|
+
path: path.join(dir, name),
|
|
1735
|
+
slug: m[1],
|
|
1736
|
+
name,
|
|
1737
|
+
});
|
|
1738
|
+
}
|
|
1739
|
+
return out;
|
|
1740
|
+
}
|
|
1741
|
+
|
|
1742
|
+
function readLegacyErrorFile(filePath) {
|
|
1743
|
+
try {
|
|
1744
|
+
const raw = JSON.parse(fs.readFileSync(filePath, "utf8"));
|
|
1745
|
+
return raw && typeof raw === "object" ? raw : null;
|
|
1746
|
+
} catch {
|
|
1747
|
+
return null;
|
|
1748
|
+
}
|
|
1749
|
+
}
|
|
1750
|
+
|
|
1751
|
+
/**
|
|
1752
|
+
* Import legacy last-capture-error-*.json into matching v2 watermark fields.
|
|
1753
|
+
* Does NOT delete legacy files — callers must use legacyErrorsSafeToRemove
|
|
1754
|
+
* after an uncapped scan proves no matching failed/pending sessions remain.
|
|
1755
|
+
*/
|
|
1756
|
+
function importLegacyCaptureErrors(options) {
|
|
1757
|
+
const opts = options && typeof options === "object" ? options : {};
|
|
1758
|
+
const dryRun = Boolean(opts.dryRun);
|
|
1759
|
+
const scan = scanStates({ includeBlocked: true, now: opts.now });
|
|
1760
|
+
const byAgentSlug = new Map();
|
|
1761
|
+
for (const entry of scan.entries || []) {
|
|
1762
|
+
if (!entry.ok || !entry.state) continue;
|
|
1763
|
+
const slug = agentSlug(entry.state.agent_id);
|
|
1764
|
+
if (!byAgentSlug.has(slug)) byAgentSlug.set(slug, []);
|
|
1765
|
+
byAgentSlug.get(slug).push(entry);
|
|
1766
|
+
}
|
|
1767
|
+
|
|
1768
|
+
const imported = [];
|
|
1769
|
+
const skipped = [];
|
|
1770
|
+
|
|
1771
|
+
for (const file of listLegacyErrorFiles()) {
|
|
1772
|
+
const error = readLegacyErrorFile(file.path);
|
|
1773
|
+
if (!error) {
|
|
1774
|
+
skipped.push({ path: file.path, reason: "unreadable" });
|
|
1775
|
+
continue;
|
|
1776
|
+
}
|
|
1777
|
+
const agentMatches = byAgentSlug.get(file.slug) || [];
|
|
1778
|
+
let matches = agentMatches;
|
|
1779
|
+
if (error.session_id) {
|
|
1780
|
+
matches = matches.filter(
|
|
1781
|
+
(entry) => entry.session_id === String(error.session_id),
|
|
1782
|
+
);
|
|
1783
|
+
}
|
|
1784
|
+
if (error.conversation_id) {
|
|
1785
|
+
matches = matches.filter(
|
|
1786
|
+
(entry) =>
|
|
1787
|
+
entry.state.conversation_id === String(error.conversation_id),
|
|
1788
|
+
);
|
|
1789
|
+
}
|
|
1790
|
+
if (matches.length === 0) {
|
|
1791
|
+
skipped.push({
|
|
1792
|
+
path: file.path,
|
|
1793
|
+
reason:
|
|
1794
|
+
error.session_id || error.conversation_id
|
|
1795
|
+
? "no_exact_matching_session"
|
|
1796
|
+
: "no_matching_session",
|
|
1797
|
+
});
|
|
1798
|
+
continue;
|
|
1799
|
+
}
|
|
1800
|
+
|
|
1801
|
+
for (const entry of matches) {
|
|
1802
|
+
if (typeof opts.beforeAcquire === "function") {
|
|
1803
|
+
opts.beforeAcquire(entry.session_id, file, error);
|
|
1804
|
+
}
|
|
1805
|
+
const lease = dryRun ? null : acquireLease(entry.session_id);
|
|
1806
|
+
if (!dryRun && !lease.ok) {
|
|
1807
|
+
skipped.push({
|
|
1808
|
+
path: file.path,
|
|
1809
|
+
session_id: entry.session_id,
|
|
1810
|
+
reason: "state_busy",
|
|
1811
|
+
});
|
|
1812
|
+
continue;
|
|
1813
|
+
}
|
|
1814
|
+
const leaseToken = lease && lease.lease && lease.lease.token;
|
|
1815
|
+
try {
|
|
1816
|
+
// The discovery scan is only for candidate selection. Reread after
|
|
1817
|
+
// acquiring the same per-session lease used by drainSession so a
|
|
1818
|
+
// concurrent cursor advance can never be replaced by stale state.
|
|
1819
|
+
const fresh = readState(entry.session_id);
|
|
1820
|
+
if (!fresh.ok) {
|
|
1821
|
+
skipped.push({
|
|
1822
|
+
path: file.path,
|
|
1823
|
+
session_id: entry.session_id,
|
|
1824
|
+
reason: fresh.missing ? "state_missing_after_lock" : "state_unreadable_after_lock",
|
|
1825
|
+
});
|
|
1826
|
+
continue;
|
|
1827
|
+
}
|
|
1828
|
+
const state = { ...fresh.state };
|
|
1829
|
+
// Revalidate exact identity against the fresh state, not the scan.
|
|
1830
|
+
if (
|
|
1831
|
+
(error.session_id &&
|
|
1832
|
+
state.session_id !== String(error.session_id)) ||
|
|
1833
|
+
(error.conversation_id &&
|
|
1834
|
+
state.conversation_id !== String(error.conversation_id))
|
|
1835
|
+
) {
|
|
1836
|
+
skipped.push({
|
|
1837
|
+
path: file.path,
|
|
1838
|
+
session_id: entry.session_id,
|
|
1839
|
+
reason: "identity_changed_after_lock",
|
|
1840
|
+
});
|
|
1841
|
+
continue;
|
|
1842
|
+
}
|
|
1843
|
+
const marker = state.legacy_error_import;
|
|
1844
|
+
if (marker && marker.file === file.name) {
|
|
1845
|
+
skipped.push({
|
|
1846
|
+
path: file.path,
|
|
1847
|
+
session_id: entry.session_id,
|
|
1848
|
+
reason: "already_imported",
|
|
1849
|
+
});
|
|
1850
|
+
continue;
|
|
1851
|
+
}
|
|
1852
|
+
const already =
|
|
1853
|
+
state.failure_class ||
|
|
1854
|
+
state.delivery === "blocked" ||
|
|
1855
|
+
state.delivery === "retry_wait";
|
|
1856
|
+
// Prefer existing v2 diagnostics when already set.
|
|
1857
|
+
if (!already) {
|
|
1858
|
+
// PR #262 used class: "transient" | omitted for persistent.
|
|
1859
|
+
let classified;
|
|
1860
|
+
const legacyClass = error.kind || error.class;
|
|
1861
|
+
if (legacyClass === "transient") {
|
|
1862
|
+
classified = "transient";
|
|
1863
|
+
} else if (legacyClass === "persistent") {
|
|
1864
|
+
classified = "persistent";
|
|
1865
|
+
} else {
|
|
1866
|
+
classified = classifyFailure({
|
|
1867
|
+
message: error.reason || error.code || "persistent",
|
|
1868
|
+
code: error.code,
|
|
1869
|
+
statusCode: /^http_(402|403)$/i.test(String(error.code || ""))
|
|
1870
|
+
? Number(String(error.code).slice(5))
|
|
1871
|
+
: undefined,
|
|
1872
|
+
});
|
|
1873
|
+
if (classified !== "transient") classified = "persistent";
|
|
1874
|
+
}
|
|
1875
|
+
state.failure_class =
|
|
1876
|
+
classified === "transient" ? "transient" : "persistent";
|
|
1877
|
+
state.last_error =
|
|
1878
|
+
error.reason || error.code || state.last_error || "legacy capture error";
|
|
1879
|
+
state.blocked_reason =
|
|
1880
|
+
state.failure_class === "persistent"
|
|
1881
|
+
? state.last_error
|
|
1882
|
+
: state.blocked_reason;
|
|
1883
|
+
state.delivery =
|
|
1884
|
+
state.failure_class === "persistent" ? "blocked" : "retry_wait";
|
|
1885
|
+
if (error.held_exchanges != null || error.failed_exchanges != null) {
|
|
1886
|
+
state.queued_exchanges = Math.max(
|
|
1887
|
+
coerceNonNegInt(state.queued_exchanges, 0),
|
|
1888
|
+
coerceNonNegInt(
|
|
1889
|
+
error.held_exchanges != null
|
|
1890
|
+
? error.held_exchanges
|
|
1891
|
+
: error.failed_exchanges,
|
|
1892
|
+
0,
|
|
1893
|
+
),
|
|
1894
|
+
);
|
|
1895
|
+
}
|
|
1896
|
+
if (!state.next_retry_at) {
|
|
1897
|
+
state.next_retry_at = nextRetryAt({
|
|
1898
|
+
failure_class: state.failure_class,
|
|
1899
|
+
attempt_count: state.attempt_count || 1,
|
|
1900
|
+
now: opts.now,
|
|
1901
|
+
random: 0.5,
|
|
1902
|
+
});
|
|
1903
|
+
}
|
|
1904
|
+
}
|
|
1905
|
+
// Mark the artifact consumed even when equivalent/newer V2
|
|
1906
|
+
// diagnostics already exist. Otherwise a later healthy transition can
|
|
1907
|
+
// make the stale sidecar eligible for import again.
|
|
1908
|
+
state.legacy_error_import = {
|
|
1909
|
+
file: file.name,
|
|
1910
|
+
session_id: state.session_id,
|
|
1911
|
+
conversation_id: state.conversation_id || null,
|
|
1912
|
+
imported_at: nowIso(opts.now),
|
|
1913
|
+
};
|
|
1914
|
+
if (!dryRun) {
|
|
1915
|
+
const written = writeState(entry.session_id, state);
|
|
1916
|
+
imported.push({
|
|
1917
|
+
session_id: entry.session_id,
|
|
1918
|
+
legacy_path: file.path,
|
|
1919
|
+
wrote: Boolean(written.ok),
|
|
1920
|
+
failure_class: state.failure_class,
|
|
1921
|
+
diagnostics_preserved: Boolean(already),
|
|
1922
|
+
});
|
|
1923
|
+
} else {
|
|
1924
|
+
imported.push({
|
|
1925
|
+
session_id: entry.session_id,
|
|
1926
|
+
legacy_path: file.path,
|
|
1927
|
+
wrote: false,
|
|
1928
|
+
dry_run: true,
|
|
1929
|
+
failure_class: state.failure_class,
|
|
1930
|
+
diagnostics_preserved: Boolean(already),
|
|
1931
|
+
});
|
|
1932
|
+
}
|
|
1933
|
+
} finally {
|
|
1934
|
+
if (leaseToken) releaseLease(entry.session_id, leaseToken);
|
|
1935
|
+
}
|
|
1936
|
+
}
|
|
1937
|
+
}
|
|
1938
|
+
|
|
1939
|
+
return {
|
|
1940
|
+
ok: true,
|
|
1941
|
+
imported,
|
|
1942
|
+
skipped,
|
|
1943
|
+
note:
|
|
1944
|
+
"Legacy last-capture-error-*.json files are NOT removed here. " +
|
|
1945
|
+
"Call legacyErrorsSafeToRemove(scanResult) only after an uncapped scan " +
|
|
1946
|
+
"proves every exact imported target is healthy, then unlink those paths.",
|
|
1947
|
+
};
|
|
1948
|
+
}
|
|
1949
|
+
|
|
1950
|
+
/**
|
|
1951
|
+
* Given an uncapped scan result, return legacy error file paths that are safe
|
|
1952
|
+
* to remove: the artifact is readable, every exact target carries this file's
|
|
1953
|
+
* import marker, and every such state is now healthy/complete.
|
|
1954
|
+
*
|
|
1955
|
+
* Removal of legacy files is ONLY permitted for paths returned here.
|
|
1956
|
+
*/
|
|
1957
|
+
function legacyErrorsSafeToRemove(scanResult) {
|
|
1958
|
+
const entries =
|
|
1959
|
+
(scanResult && scanResult.entries) ||
|
|
1960
|
+
(Array.isArray(scanResult) ? scanResult : []);
|
|
1961
|
+
const safe = [];
|
|
1962
|
+
for (const file of listLegacyErrorFiles()) {
|
|
1963
|
+
const error = readLegacyErrorFile(file.path);
|
|
1964
|
+
// Unreadable artifacts are evidence, not garbage. Preserve them.
|
|
1965
|
+
if (!error) continue;
|
|
1966
|
+
const candidates = entries.filter((entry) => {
|
|
1967
|
+
const state = entry && entry.state;
|
|
1968
|
+
if (!state || agentSlug(state.agent_id) !== file.slug) return false;
|
|
1969
|
+
if (error.session_id && state.session_id !== String(error.session_id)) {
|
|
1970
|
+
return false;
|
|
1971
|
+
}
|
|
1972
|
+
if (
|
|
1973
|
+
error.conversation_id &&
|
|
1974
|
+
state.conversation_id !== String(error.conversation_id)
|
|
1975
|
+
) {
|
|
1976
|
+
return false;
|
|
1977
|
+
}
|
|
1978
|
+
const marker = state.legacy_error_import;
|
|
1979
|
+
return marker && marker.file === file.name;
|
|
1980
|
+
});
|
|
1981
|
+
// Never delete an unmatched artifact or one that was skipped rather than
|
|
1982
|
+
// imported. Every exact imported target must now be healthy/complete.
|
|
1983
|
+
if (candidates.length === 0) continue;
|
|
1984
|
+
const allHealthy = candidates.every((entry) => {
|
|
1985
|
+
const state = entry.state;
|
|
1986
|
+
return (
|
|
1987
|
+
coerceNonNegInt(state.queued_exchanges, 0) === 0 &&
|
|
1988
|
+
!["needs_conversation", "pending", "draining", "retry_wait", "blocked"].includes(
|
|
1989
|
+
state.delivery,
|
|
1990
|
+
) &&
|
|
1991
|
+
!state.failure_class
|
|
1992
|
+
);
|
|
1993
|
+
});
|
|
1994
|
+
if (allHealthy) safe.push(file.path);
|
|
1995
|
+
}
|
|
1996
|
+
return safe;
|
|
1997
|
+
}
|
|
1998
|
+
|
|
1999
|
+
// ---------------------------------------------------------------------------
|
|
2000
|
+
// Exports
|
|
2001
|
+
// ---------------------------------------------------------------------------
|
|
2002
|
+
|
|
2003
|
+
module.exports = {
|
|
2004
|
+
// constants / defaults / readers
|
|
2005
|
+
DEFAULTS,
|
|
2006
|
+
FAILURE_CLASSES,
|
|
2007
|
+
LIFECYCLES,
|
|
2008
|
+
DELIVERIES,
|
|
2009
|
+
HEALTH_STATUSES,
|
|
2010
|
+
HEALTH_PRECEDENCE,
|
|
2011
|
+
envInt,
|
|
2012
|
+
envFloat,
|
|
2013
|
+
envString,
|
|
2014
|
+
maxCaptureBatchSize,
|
|
2015
|
+
checkpointInterval,
|
|
2016
|
+
checkpointTimeout,
|
|
2017
|
+
recoveryMaxSessions,
|
|
2018
|
+
recoveryMaxExchanges,
|
|
2019
|
+
recoveryMinIdle,
|
|
2020
|
+
leaseTtlSeconds,
|
|
2021
|
+
retryBaseSeconds,
|
|
2022
|
+
retryMaxSeconds,
|
|
2023
|
+
retryJitter,
|
|
2024
|
+
persistentRetrySeconds,
|
|
2025
|
+
workerMaxAgeSeconds,
|
|
2026
|
+
timerMaxAge,
|
|
2027
|
+
healthCacheFilename,
|
|
2028
|
+
|
|
2029
|
+
// paths
|
|
2030
|
+
watermarkDir,
|
|
2031
|
+
watermarkPath,
|
|
2032
|
+
leasePath,
|
|
2033
|
+
mutationLockPath,
|
|
2034
|
+
timerPidPath,
|
|
2035
|
+
healthCachePath,
|
|
2036
|
+
legacyErrorPath,
|
|
2037
|
+
agentSlug,
|
|
2038
|
+
|
|
2039
|
+
// normalize / validate
|
|
2040
|
+
normalizeState,
|
|
2041
|
+
validateV2,
|
|
2042
|
+
|
|
2043
|
+
// persistence
|
|
2044
|
+
readState,
|
|
2045
|
+
writeState,
|
|
2046
|
+
mutateState,
|
|
2047
|
+
writeSessionIntent,
|
|
2048
|
+
atomicWriteJson,
|
|
2049
|
+
|
|
2050
|
+
// cursor / epoch
|
|
2051
|
+
advanceCursor,
|
|
2052
|
+
rebucket,
|
|
2053
|
+
|
|
2054
|
+
// leases
|
|
2055
|
+
acquireLease,
|
|
2056
|
+
renewLease,
|
|
2057
|
+
releaseLease,
|
|
2058
|
+
|
|
2059
|
+
// retry
|
|
2060
|
+
classifyFailure,
|
|
2061
|
+
nextRetryAt,
|
|
2062
|
+
retryDelaySeconds,
|
|
2063
|
+
|
|
2064
|
+
// scan
|
|
2065
|
+
scanStates,
|
|
2066
|
+
sortOldestFirst,
|
|
2067
|
+
sessionHealthBucket,
|
|
2068
|
+
buildEligibility,
|
|
2069
|
+
|
|
2070
|
+
// health
|
|
2071
|
+
computeAggregateHealth,
|
|
2072
|
+
writeHealthCache,
|
|
2073
|
+
readHealthCache,
|
|
2074
|
+
|
|
2075
|
+
// legacy
|
|
2076
|
+
importLegacyCaptureErrors,
|
|
2077
|
+
legacyErrorsSafeToRemove,
|
|
2078
|
+
listLegacyErrorFiles,
|
|
2079
|
+
};
|