@joe-broadhead/opencode-mem 0.9.5 → 0.9.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1408 -0
- package/README.md +120 -41
- package/docs/agent-instructions.md +17 -0
- package/docs/agent-protocol.md +28 -27
- package/docs/permissions.md +150 -0
- package/package.json +386 -17
- package/skills/mem.pages/SKILL.md +40 -16
- package/skills/mem.sharing/SKILL.md +39 -6
- package/skills/mem.sources/SKILL.md +41 -10
- package/skills/mem.usage/SKILL.md +37 -16
- package/src/backend/records.js +8 -0
- package/src/backend/v3.js +15 -10
- package/src/call-shapes.js +18 -25
- package/src/capture-pause.js +136 -0
- package/src/capture.js +7 -5
- package/src/collector-service.js +212 -35
- package/src/collector.js +187 -47
- package/src/commands.js +37 -0
- package/src/compaction.js +4 -2
- package/src/config.js +151 -1
- package/src/curation.js +6 -10
- package/src/events.js +144 -36
- package/src/forget-purge.js +377 -0
- package/src/forgotten.js +152 -0
- package/src/hooks.js +9 -4
- package/src/index.js +5 -3
- package/src/permissions.js +116 -38
- package/src/protocol.js +32 -11
- package/src/recall.js +13 -1
- package/src/records.js +4 -1
- package/src/runtime.js +202 -15
- package/src/shapes.js +63 -0
- package/src/sources.js +205 -12
- package/src/spaces.js +58 -1
- package/src/spool.js +442 -64
- package/src/stated.js +334 -0
- package/src/storable.js +477 -0
- package/src/tools/aliases.js +36 -147
- package/src/tools/commitments.js +79 -13
- package/src/tools/held-write.js +5 -1
- package/src/tools/index.js +25 -43
- package/src/tools/ingest.js +33 -128
- package/src/tools/memory.js +525 -911
- package/src/tools/page-sections.js +147 -0
- package/src/tools/page.js +314 -82
- package/src/tools/recall-browse.js +173 -0
- package/src/tools/recall.js +36 -8
- package/src/tools/records-kit.js +2 -1
- package/src/tools/share.js +494 -131
- package/src/tools/shared.js +97 -19
- package/src/tools/spaces.js +194 -101
- package/src/tools/status.js +31 -3
- package/src/tools/wiki.js +5 -13
- package/src/util.js +10 -59
- package/src/views/contracts.js +258 -20
- package/src/views/overdue.js +1 -1
- package/src/views/profile.js +2 -3
- package/src/views/receipt.js +81 -0
- package/src/views/records-reads.js +545 -25
- package/src/views/rpc.js +41 -3
- package/views/memory.html +16 -3
- package/views/remembered.html +130 -0
- package/src/erase.js +0 -36
package/src/spool.js
CHANGED
|
@@ -14,11 +14,36 @@ const BACKOFF_MAX_MS = 60 * 60_000
|
|
|
14
14
|
const CLAIM_STALE_MS = 5 * 60_000
|
|
15
15
|
/** Stop a flush after this many consecutive "backend unreachable" failures. */
|
|
16
16
|
const UNREACHABLE_STOP = 3
|
|
17
|
+
/** A quarantined entry is deleted this long after it was quarantined. */
|
|
18
|
+
export const QUARANTINE_TTL_MS = 14 * 86_400_000
|
|
19
|
+
/** A temp file older than this was left by a crashed write. */
|
|
20
|
+
const TEMP_STALE_MS = CLAIM_STALE_MS
|
|
17
21
|
|
|
18
22
|
const PENDING = ".json"
|
|
19
23
|
const INFLIGHT = ".inflight"
|
|
20
24
|
const FAILED = ".failed"
|
|
25
|
+
/** `<name>.json.state`: an entry's retry and quarantine state, beside its payload. */
|
|
26
|
+
const STATE = ".state"
|
|
27
|
+
const TEMP = ".tmp"
|
|
21
28
|
const CLAIM_SUFFIX = /\.(\d+)-(\d+)\.inflight$/
|
|
29
|
+
/** The retry fields a 0.9.7 or older entry kept inside its payload. */
|
|
30
|
+
const LEGACY_STATE_FIELDS = ["attempts", "lastError", "lastStatus", "lastAttemptAt", "nextAttemptAt", "quarantine"]
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* A captured turn's idempotency key: its session, its message, and a hash of
|
|
34
|
+
* the payload. The live send and every replay of one spooled turn send the
|
|
35
|
+
* same key, and the gateway stores a key once (docs/capture-delivery.md).
|
|
36
|
+
* Only characters the gateway accepts are kept.
|
|
37
|
+
*
|
|
38
|
+
* @param {string} sessionID
|
|
39
|
+
* @param {string | null | undefined} messageID
|
|
40
|
+
* @param {unknown} payload
|
|
41
|
+
*/
|
|
42
|
+
export function turnKey(sessionID, messageID, payload) {
|
|
43
|
+
const safe = (value) => String(value ?? "").replace(/[^A-Za-z0-9._:-]/g, "_").slice(0, 190)
|
|
44
|
+
const session = safe(sessionID) || "session"
|
|
45
|
+
return `${/^[A-Za-z0-9]/.test(session) ? "" : "s"}${session}:${safe(messageID) || "-"}:${shortHash(JSON.stringify(payload ?? null), 16)}`
|
|
46
|
+
}
|
|
22
47
|
|
|
23
48
|
/**
|
|
24
49
|
* Classify a failed send.
|
|
@@ -65,9 +90,27 @@ function backoffMs(attempts) {
|
|
|
65
90
|
* atomic rename, so two processes, or a flush racing a live capture, never
|
|
66
91
|
* send the same entry twice. A claim older than five minutes is returned to
|
|
67
92
|
* pending.
|
|
68
|
-
* - `<name>.json.failed` is quarantined
|
|
69
|
-
*
|
|
70
|
-
*
|
|
93
|
+
* - `<name>.json.failed` is quarantined; the reason is in its state file.
|
|
94
|
+
* Rename it back to `.json` to retry it; the attempt count restarts. A
|
|
95
|
+
* quarantined entry is deleted 14 days after it was quarantined
|
|
96
|
+
* (`QUARANTINE_TTL_MS`), and `spoolStats` lists the reasons for status.
|
|
97
|
+
* - `<name>.json.state` holds the entry's retry state (attempts, next attempt,
|
|
98
|
+
* last error) and, once quarantined, `quarantine: { reason, at }`.
|
|
99
|
+
*
|
|
100
|
+
* **Durability.** A payload file is written once and then only renamed: to
|
|
101
|
+
* a claim, back to pending, or to quarantine. Retry state never touches it.
|
|
102
|
+
* Every write (a payload or a state file) goes to a temp file that is fsynced
|
|
103
|
+
* and then renamed into place, and the directory is fsynced, so a crash leaves
|
|
104
|
+
* the old file or the new one, never a torn one. A temp file a crash left
|
|
105
|
+
* behind is ignored and later deleted.
|
|
106
|
+
*
|
|
107
|
+
* **Delivery.** At least once: a send whose answer is lost (a timeout, a
|
|
108
|
+
* crash, a claim recovered from a sleeping process) is sent again. A turn
|
|
109
|
+
* carries an idempotency key (`turnKey`), so the gateway answers a repeat
|
|
110
|
+
* from its receipt instead of storing it twice. After a successful send the
|
|
111
|
+
* entry is deleted in its own step: a delete that fails (another process
|
|
112
|
+
* already recovered the claim) never puts the entry back in the queue
|
|
113
|
+
* (docs/capture-delivery.md).
|
|
71
114
|
*
|
|
72
115
|
* In control mode a turn entry records the session, agent, task, and the
|
|
73
116
|
* visibility it was captured with, never a container. Its replay goes through
|
|
@@ -91,7 +134,10 @@ function backoffMs(attempts) {
|
|
|
91
134
|
* replay (another person spoke there since), it is kept for later, never sent
|
|
92
135
|
* with that token.
|
|
93
136
|
*
|
|
94
|
-
*
|
|
137
|
+
* **Forget.** `purgeMatching` deletes entries that hold forgotten text, and
|
|
138
|
+
* `sendable` (src/forget-purge.js) is asked again just before each replay.
|
|
139
|
+
*
|
|
140
|
+
* @param {{ config: any, log: any, backend: any, captureTurn?: Function | null, now?: () => number, sendable?: (entry: any) => boolean | "hold" }} input
|
|
95
141
|
*/
|
|
96
142
|
export function createSpool(input) {
|
|
97
143
|
const { config, log, backend } = input
|
|
@@ -101,6 +147,12 @@ export function createSpool(input) {
|
|
|
101
147
|
const subject = typeof input.subject === "function" ? input.subject : () => ""
|
|
102
148
|
/** The account the current session's token names now, asking Jelly when needed: checked before a replay. */
|
|
103
149
|
const speaker = typeof input.speaker === "function" ? input.speaker : async () => ""
|
|
150
|
+
/**
|
|
151
|
+
* Whether an entry may be sent: false when it holds text the person forgot
|
|
152
|
+
* (src/forget-purge.js), deleted, never sent; "hold" when its space has a
|
|
153
|
+
* forget pending, kept for a later replay. Checked just before each replay.
|
|
154
|
+
*/
|
|
155
|
+
const sendable = typeof input.sendable === "function" ? input.sendable : () => true
|
|
104
156
|
const spoolDir = path.join(config.stateDir, "spool")
|
|
105
157
|
const claimsDir = path.join(config.stateDir, "claims")
|
|
106
158
|
let sequence = 0
|
|
@@ -114,12 +166,97 @@ export function createSpool(input) {
|
|
|
114
166
|
}
|
|
115
167
|
}
|
|
116
168
|
|
|
117
|
-
|
|
118
|
-
|
|
169
|
+
/** Make a rename or a new file in the spool survive a crash. Best effort. */
|
|
170
|
+
function syncDir() {
|
|
171
|
+
let fd
|
|
119
172
|
try {
|
|
120
|
-
fs.
|
|
173
|
+
fd = fs.openSync(spoolDir, "r")
|
|
174
|
+
fs.fsyncSync(fd)
|
|
121
175
|
} catch {
|
|
122
|
-
/*
|
|
176
|
+
/* not supported on every platform (Windows) */
|
|
177
|
+
} finally {
|
|
178
|
+
if (fd !== undefined) {
|
|
179
|
+
try {
|
|
180
|
+
fs.closeSync(fd)
|
|
181
|
+
} catch {
|
|
182
|
+
/* ignore */
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* Write a file so a crash leaves either the old content or the new, never
|
|
190
|
+
* a torn file: a private temp file in the same directory, written in full,
|
|
191
|
+
* fsynced, then renamed over the target, then the directory fsynced.
|
|
192
|
+
*/
|
|
193
|
+
function writeDurable(file, record) {
|
|
194
|
+
const bytes = Buffer.from(JSON.stringify(record, null, 2))
|
|
195
|
+
const temp = path.join(path.dirname(file), `.${path.basename(file)}.${process.pid}-${Math.random().toString(36).slice(2, 10)}${TEMP}`)
|
|
196
|
+
let fd
|
|
197
|
+
try {
|
|
198
|
+
fd = fs.openSync(temp, "wx", 0o600)
|
|
199
|
+
try {
|
|
200
|
+
fs.fchmodSync(fd, 0o600)
|
|
201
|
+
} catch {
|
|
202
|
+
/* best effort on platforms without POSIX modes */
|
|
203
|
+
}
|
|
204
|
+
let offset = 0
|
|
205
|
+
while (offset < bytes.length) offset += fs.writeSync(fd, bytes, offset, bytes.length - offset)
|
|
206
|
+
fs.fsyncSync(fd)
|
|
207
|
+
fs.closeSync(fd)
|
|
208
|
+
fd = undefined
|
|
209
|
+
fs.renameSync(temp, file)
|
|
210
|
+
} catch (error) {
|
|
211
|
+
if (fd !== undefined) {
|
|
212
|
+
try {
|
|
213
|
+
fs.closeSync(fd)
|
|
214
|
+
} catch {
|
|
215
|
+
/* ignore */
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
try {
|
|
219
|
+
fs.unlinkSync(temp)
|
|
220
|
+
} catch {
|
|
221
|
+
/* never created, or already renamed */
|
|
222
|
+
}
|
|
223
|
+
throw error
|
|
224
|
+
}
|
|
225
|
+
syncDir()
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
function stateFile(pending) {
|
|
229
|
+
return `${pending}${STATE}`
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/** An entry's sidecar state, or null when it has none (or it is unreadable). */
|
|
233
|
+
function readState(pending) {
|
|
234
|
+
try {
|
|
235
|
+
const state = JSON.parse(fs.readFileSync(stateFile(pending), "utf8"))
|
|
236
|
+
return state && typeof state === "object" && !Array.isArray(state) ? state : null
|
|
237
|
+
} catch {
|
|
238
|
+
return null
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/** The retry fields an entry from 0.9.7 or older kept in its payload. */
|
|
243
|
+
function legacyState(entry) {
|
|
244
|
+
const state = {}
|
|
245
|
+
if (!entry || typeof entry !== "object") return state
|
|
246
|
+
for (const field of LEGACY_STATE_FIELDS) if (entry[field] !== undefined) state[field] = entry[field]
|
|
247
|
+
return state
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/** The state that counts: the sidecar when there is one, else the legacy fields. */
|
|
251
|
+
function stateOf(pending, entry) {
|
|
252
|
+
return readState(pending) ?? legacyState(entry)
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
function removeState(pending) {
|
|
256
|
+
try {
|
|
257
|
+
fs.unlinkSync(stateFile(pending))
|
|
258
|
+
} catch {
|
|
259
|
+
/* none */
|
|
123
260
|
}
|
|
124
261
|
}
|
|
125
262
|
|
|
@@ -153,7 +290,7 @@ export function createSpool(input) {
|
|
|
153
290
|
try {
|
|
154
291
|
ensureDir()
|
|
155
292
|
const file = path.join(spoolDir, newName(data))
|
|
156
|
-
|
|
293
|
+
writeDurable(file, entryOf(kind, data, scope, meta))
|
|
157
294
|
return file
|
|
158
295
|
} catch (error) {
|
|
159
296
|
log.error("spool write failed", errorMessage(error))
|
|
@@ -187,7 +324,7 @@ export function createSpool(input) {
|
|
|
187
324
|
try {
|
|
188
325
|
ensureDir()
|
|
189
326
|
const file = claimName(path.join(spoolDir, newName(data)))
|
|
190
|
-
|
|
327
|
+
writeDurable(file, entryOf(kind, data, scope, meta))
|
|
191
328
|
return file
|
|
192
329
|
} catch (error) {
|
|
193
330
|
log.error("spool write failed", errorMessage(error))
|
|
@@ -195,13 +332,31 @@ export function createSpool(input) {
|
|
|
195
332
|
}
|
|
196
333
|
}
|
|
197
334
|
|
|
335
|
+
/**
|
|
336
|
+
* Delete a sent entry and its state. A missing file is not an error: another
|
|
337
|
+
* process may have recovered the claim (and will send it again, which the
|
|
338
|
+
* gateway dedupes by key). Never re-creates anything.
|
|
339
|
+
*
|
|
340
|
+
* @returns {boolean} whether the entry is gone
|
|
341
|
+
*/
|
|
198
342
|
function removeSpool(file) {
|
|
199
|
-
if (!file) return
|
|
343
|
+
if (!file) return true
|
|
344
|
+
let gone = true
|
|
200
345
|
try {
|
|
201
346
|
fs.unlinkSync(file)
|
|
202
|
-
} catch {
|
|
203
|
-
|
|
347
|
+
} catch (error) {
|
|
348
|
+
if (error?.code === "ENOENT") {
|
|
349
|
+
log.debug("sent spool entry was already gone: another process recovered its claim", { file: path.basename(pendingOf(file)) })
|
|
350
|
+
} else {
|
|
351
|
+
gone = false
|
|
352
|
+
log.warn("sent spool entry could not be deleted; a later replay is deduplicated by its key", {
|
|
353
|
+
file: path.basename(pendingOf(file)),
|
|
354
|
+
error: error?.code ?? errorMessage(error),
|
|
355
|
+
})
|
|
356
|
+
}
|
|
204
357
|
}
|
|
358
|
+
if (gone) removeState(pendingOf(file))
|
|
359
|
+
return gone
|
|
205
360
|
}
|
|
206
361
|
|
|
207
362
|
/** Atomically take a pending entry. Null when another sender got it first. */
|
|
@@ -216,23 +371,53 @@ export function createSpool(input) {
|
|
|
216
371
|
}
|
|
217
372
|
}
|
|
218
373
|
|
|
219
|
-
/**
|
|
220
|
-
|
|
221
|
-
|
|
374
|
+
/**
|
|
375
|
+
* Move an entry aside with the reason so the rest of the queue drains. The
|
|
376
|
+
* payload is renamed as it is (an unreadable one keeps its bytes); the
|
|
377
|
+
* reason goes to its state file. An entry that is no longer ours (another
|
|
378
|
+
* process recovered the claim) is left alone.
|
|
379
|
+
*
|
|
380
|
+
* **The order matters.** The payload moves first and the reason follows. A
|
|
381
|
+
* `quarantine` field in an entry's state means the entry was in quarantine,
|
|
382
|
+
* and an operator who renames it back to `.json` restarts its attempt count.
|
|
383
|
+
* Written the other way round, a crash between the two steps leaves a claim
|
|
384
|
+
* whose state says "quarantined": the next start recovers the claim, sees
|
|
385
|
+
* that field, and takes the entry for a manual retry, resetting its attempts.
|
|
386
|
+
* In this order, a crash between the steps leaves a `.failed` entry without a
|
|
387
|
+
* recorded reason (status says so), and a crash before them leaves a claim
|
|
388
|
+
* with the state it had, which resumes where it was.
|
|
389
|
+
*/
|
|
390
|
+
function quarantine(claimed, reason, state = {}) {
|
|
391
|
+
const pending = pendingOf(claimed)
|
|
392
|
+
const target = `${pending}${FAILED}`
|
|
393
|
+
if (!fs.existsSync(claimed)) {
|
|
394
|
+
log.debug("spool entry not quarantined: another process holds it now", { file: path.basename(pending) })
|
|
395
|
+
return false
|
|
396
|
+
}
|
|
222
397
|
const at = new Date(now()).toISOString()
|
|
223
|
-
const record =
|
|
224
|
-
entry && typeof entry === "object" && !Array.isArray(entry)
|
|
225
|
-
? { ...entry, quarantine: { reason, at } }
|
|
226
|
-
: { quarantine: { reason, at }, raw: String(raw ?? "") }
|
|
227
398
|
try {
|
|
228
|
-
writePrivate(claimed, record)
|
|
229
399
|
fs.renameSync(claimed, target)
|
|
230
|
-
log.warn("spool entry quarantined", { file: path.basename(target), reason })
|
|
231
|
-
return true
|
|
232
400
|
} catch (error) {
|
|
233
401
|
log.warn("spool quarantine failed", { file: path.basename(claimed), error: errorMessage(error) })
|
|
234
402
|
return false
|
|
235
403
|
}
|
|
404
|
+
try {
|
|
405
|
+
// Permission and time changes, not rewrites: the entry holds memory
|
|
406
|
+
// text, and its age in quarantine is its file's time if its reason
|
|
407
|
+
// never gets written.
|
|
408
|
+
fs.chmodSync(target, 0o600)
|
|
409
|
+
fs.utimesSync(target, now() / 1000, now() / 1000)
|
|
410
|
+
} catch {
|
|
411
|
+
/* best effort on platforms without POSIX modes */
|
|
412
|
+
}
|
|
413
|
+
try {
|
|
414
|
+
writeDurable(stateFile(pending), { ...state, quarantine: { reason, at } })
|
|
415
|
+
} catch (error) {
|
|
416
|
+
log.warn("spool quarantine reason not written", { file: path.basename(pending), error: errorMessage(error) })
|
|
417
|
+
}
|
|
418
|
+
syncDir()
|
|
419
|
+
log.warn("spool entry quarantined", { file: path.basename(target), reason })
|
|
420
|
+
return true
|
|
236
421
|
}
|
|
237
422
|
|
|
238
423
|
/**
|
|
@@ -240,17 +425,20 @@ export function createSpool(input) {
|
|
|
240
425
|
* an exhausted entry; otherwise schedule the next attempt and return the
|
|
241
426
|
* entry to the queue.
|
|
242
427
|
*
|
|
243
|
-
*
|
|
428
|
+
* The payload is never rewritten: the new state goes to the entry's state
|
|
429
|
+
* file (written atomically), then the claim is renamed back to pending. A
|
|
430
|
+
* claim that is gone (another process recovered it) is not re-created.
|
|
431
|
+
*
|
|
432
|
+
* @returns {"quarantined" | "kept" | "gone"}
|
|
244
433
|
*/
|
|
245
|
-
function recordFailure(claimed, entry, error) {
|
|
434
|
+
function recordFailure(claimed, entry, error, state = {}) {
|
|
246
435
|
const kind = classifyFailure(error)
|
|
247
436
|
const status = Number(error?.status) || 0
|
|
248
437
|
const message = oneLine(redactSecrets(errorMessage(error)), 300)
|
|
249
|
-
const attempts = (Number(
|
|
438
|
+
const attempts = (Number(state.attempts) || 0) + 1
|
|
250
439
|
const queuedAt = Date.parse(entry.queuedAt ?? "")
|
|
251
440
|
const age = Number.isFinite(queuedAt) ? now() - queuedAt : 0
|
|
252
441
|
const updated = {
|
|
253
|
-
...entry,
|
|
254
442
|
attempts,
|
|
255
443
|
lastError: message,
|
|
256
444
|
lastStatus: status,
|
|
@@ -262,12 +450,26 @@ export function createSpool(input) {
|
|
|
262
450
|
if (kind === "permanent") reason = `rejected (${status}): ${message}`
|
|
263
451
|
else if (attempts >= MAX_ATTEMPTS) reason = `gave up after ${attempts} attempts: ${message}`
|
|
264
452
|
else if (age >= MAX_AGE_MS) reason = `gave up after ${Math.floor(age / 86_400_000)} days: ${message}`
|
|
265
|
-
if (reason && quarantine(claimed,
|
|
453
|
+
if (reason && quarantine(claimed, reason, updated)) return "quarantined"
|
|
266
454
|
|
|
455
|
+
const pending = pendingOf(claimed)
|
|
456
|
+
if (!fs.existsSync(claimed)) {
|
|
457
|
+
log.warn("spool send failed; another process holds the entry now, so it is not re-queued here", { kind, status })
|
|
458
|
+
return "gone"
|
|
459
|
+
}
|
|
267
460
|
try {
|
|
268
|
-
|
|
269
|
-
|
|
461
|
+
writeDurable(stateFile(pending), updated)
|
|
462
|
+
} catch (stateError) {
|
|
463
|
+
// The entry is still returned to the queue, only without its backoff.
|
|
464
|
+
log.warn("spool retry state not written", { file: path.basename(pending), error: errorMessage(stateError) })
|
|
465
|
+
}
|
|
466
|
+
try {
|
|
467
|
+
fs.renameSync(claimed, pending)
|
|
270
468
|
} catch (releaseError) {
|
|
469
|
+
if (releaseError?.code === "ENOENT") {
|
|
470
|
+
log.warn("spool send failed; another process holds the entry now, so it is not re-queued here", { kind, status })
|
|
471
|
+
return "gone"
|
|
472
|
+
}
|
|
271
473
|
log.warn("spool release failed", { file: path.basename(claimed), error: errorMessage(releaseError) })
|
|
272
474
|
}
|
|
273
475
|
log.warn("spool send failed; kept", { kind, status, attempts, error: message })
|
|
@@ -284,6 +486,7 @@ export function createSpool(input) {
|
|
|
284
486
|
} catch {
|
|
285
487
|
/* already gone */
|
|
286
488
|
}
|
|
489
|
+
removeState(pendingOf(file))
|
|
287
490
|
log.warn("spooled entry dropped: the person speaking in its session hasn't linked a Jelly account on this host, and no other credential may send it", {
|
|
288
491
|
file: path.basename(pendingOf(file)),
|
|
289
492
|
kind: entry?.kind ?? null,
|
|
@@ -305,7 +508,7 @@ export function createSpool(input) {
|
|
|
305
508
|
}
|
|
306
509
|
try {
|
|
307
510
|
const entry = JSON.parse(fs.readFileSync(claimed, "utf8"))
|
|
308
|
-
return recordFailure(claimed, entry, error)
|
|
511
|
+
return recordFailure(claimed, entry, error, stateOf(pendingOf(claimed), entry))
|
|
309
512
|
} catch (readError) {
|
|
310
513
|
log.warn("spool release failed", { file: path.basename(claimed), error: errorMessage(readError) })
|
|
311
514
|
return null
|
|
@@ -437,6 +640,113 @@ export function createSpool(input) {
|
|
|
437
640
|
}
|
|
438
641
|
}
|
|
439
642
|
|
|
643
|
+
/** When a quarantined entry was quarantined: its state, a legacy record, else the file's time. */
|
|
644
|
+
function quarantinedAt(failed) {
|
|
645
|
+
const pending = pendingOf(failed)
|
|
646
|
+
let at = Date.parse(readState(pending)?.quarantine?.at ?? "")
|
|
647
|
+
if (!Number.isFinite(at)) {
|
|
648
|
+
try {
|
|
649
|
+
at = Date.parse(JSON.parse(fs.readFileSync(failed, "utf8"))?.quarantine?.at ?? "")
|
|
650
|
+
} catch {
|
|
651
|
+
at = Number.NaN
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
if (!Number.isFinite(at)) {
|
|
655
|
+
try {
|
|
656
|
+
at = fs.statSync(failed).mtimeMs
|
|
657
|
+
} catch {
|
|
658
|
+
at = now()
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
return at
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
/**
|
|
665
|
+
* Delete what nothing will read again: quarantined entries past
|
|
666
|
+
* `QUARANTINE_TTL_MS`, state files whose entry is gone, and temp files a
|
|
667
|
+
* crashed write left behind.
|
|
668
|
+
*/
|
|
669
|
+
function housekeep(names) {
|
|
670
|
+
const present = new Set()
|
|
671
|
+
for (const name of names) {
|
|
672
|
+
if (name.endsWith(PENDING) || name.endsWith(FAILED) || name.endsWith(INFLIGHT)) present.add(path.basename(pendingOf(name)))
|
|
673
|
+
}
|
|
674
|
+
let expired = 0
|
|
675
|
+
for (const name of names) {
|
|
676
|
+
const file = path.join(spoolDir, name)
|
|
677
|
+
try {
|
|
678
|
+
if (name.endsWith(FAILED)) {
|
|
679
|
+
if (now() - quarantinedAt(file) < QUARANTINE_TTL_MS) continue
|
|
680
|
+
fs.unlinkSync(file)
|
|
681
|
+
removeState(pendingOf(file))
|
|
682
|
+
expired += 1
|
|
683
|
+
} else if (name.endsWith(STATE)) {
|
|
684
|
+
if (!present.has(name.slice(0, -STATE.length))) fs.unlinkSync(file)
|
|
685
|
+
} else if (name.startsWith(".") && name.endsWith(TEMP)) {
|
|
686
|
+
if (now() - fs.statSync(file).mtimeMs >= TEMP_STALE_MS) fs.unlinkSync(file)
|
|
687
|
+
}
|
|
688
|
+
} catch {
|
|
689
|
+
/* another process got there first */
|
|
690
|
+
}
|
|
691
|
+
}
|
|
692
|
+
if (expired > 0) log.info("expired quarantined spool entries deleted", { count: expired, afterDays: QUARANTINE_TTL_MS / 86_400_000 })
|
|
693
|
+
return expired
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
/**
|
|
697
|
+
* Delete every queued, in-flight, or quarantined entry the predicate
|
|
698
|
+
* matches, with its state. The hook for forget (slice 6b): it decides what
|
|
699
|
+
* matches; this only finds and deletes. An entry that is being sent is
|
|
700
|
+
* deleted too; its sender then neither re-queues nor quarantines it.
|
|
701
|
+
*
|
|
702
|
+
* @param {(entry: any, info: { state: "pending" | "inflight" | "quarantined", file: string, raw: string }) => boolean} predicate
|
|
703
|
+
* `entry` is null when the file is not readable JSON; `raw` is its text.
|
|
704
|
+
* @returns {{ purged: number, failed: number }}
|
|
705
|
+
*/
|
|
706
|
+
function purgeMatching(predicate) {
|
|
707
|
+
const result = { purged: 0, failed: 0 }
|
|
708
|
+
if (typeof predicate !== "function") return result
|
|
709
|
+
let names = []
|
|
710
|
+
try {
|
|
711
|
+
names = fs.readdirSync(spoolDir)
|
|
712
|
+
} catch {
|
|
713
|
+
return result
|
|
714
|
+
}
|
|
715
|
+
for (const name of names) {
|
|
716
|
+
const state = name.endsWith(PENDING) ? "pending" : name.endsWith(INFLIGHT) ? "inflight" : name.endsWith(FAILED) ? "quarantined" : null
|
|
717
|
+
if (!state) continue
|
|
718
|
+
const file = path.join(spoolDir, name)
|
|
719
|
+
let raw = ""
|
|
720
|
+
let entry = null
|
|
721
|
+
try {
|
|
722
|
+
raw = fs.readFileSync(file, "utf8")
|
|
723
|
+
entry = JSON.parse(raw)
|
|
724
|
+
} catch (error) {
|
|
725
|
+
if (error?.code === "ENOENT") continue
|
|
726
|
+
}
|
|
727
|
+
let matched = false
|
|
728
|
+
try {
|
|
729
|
+
matched = Boolean(predicate(entry, { state, file: name, raw }))
|
|
730
|
+
} catch (error) {
|
|
731
|
+
result.failed += 1
|
|
732
|
+
log.warn("spool purge predicate failed", { file: path.basename(pendingOf(name)), error: errorMessage(error) })
|
|
733
|
+
continue
|
|
734
|
+
}
|
|
735
|
+
if (!matched) continue
|
|
736
|
+
try {
|
|
737
|
+
fs.unlinkSync(file)
|
|
738
|
+
removeState(pendingOf(file))
|
|
739
|
+
result.purged += 1
|
|
740
|
+
} catch (error) {
|
|
741
|
+
if (error?.code === "ENOENT") continue
|
|
742
|
+
result.failed += 1
|
|
743
|
+
log.warn("spool purge failed", { file: path.basename(pendingOf(name)), error: error?.code ?? errorMessage(error) })
|
|
744
|
+
}
|
|
745
|
+
}
|
|
746
|
+
if (result.purged > 0) syncDir()
|
|
747
|
+
return result
|
|
748
|
+
}
|
|
749
|
+
|
|
440
750
|
async function send(entry) {
|
|
441
751
|
// Someone else may speak in the session now. An entry captured under one
|
|
442
752
|
// person's token is never sent with another's: it waits, with backoff,
|
|
@@ -456,12 +766,15 @@ export function createSpool(input) {
|
|
|
456
766
|
// was captured with. An entry without one (written before visibility
|
|
457
767
|
// was recorded) replays as private, the narrower choice. A container
|
|
458
768
|
// in an older entry's scope is never sent.
|
|
769
|
+
// The key the turn was captured with; an entry from before 0.9.8
|
|
770
|
+
// gets one from its content, the same on every replay.
|
|
459
771
|
await captureTurn({
|
|
460
772
|
sessionID: turnSessionOf(entry),
|
|
461
773
|
agent: entry.agent ?? entry.scope?.agent_id,
|
|
462
774
|
visibility: entry.visibility === "team" ? "team" : "private",
|
|
463
775
|
task: entry.task ?? entry.scope?.task_id,
|
|
464
776
|
payload: entry.data,
|
|
777
|
+
key: typeof entry.key === "string" && entry.key ? entry.key : turnKey(turnSessionOf(entry), null, entry.data),
|
|
465
778
|
})
|
|
466
779
|
return
|
|
467
780
|
}
|
|
@@ -488,6 +801,7 @@ export function createSpool(input) {
|
|
|
488
801
|
return result
|
|
489
802
|
}
|
|
490
803
|
recoverStaleClaims(names)
|
|
804
|
+
housekeep(names)
|
|
491
805
|
try {
|
|
492
806
|
names = fs.readdirSync(spoolDir).filter((name) => name.endsWith(PENDING)).sort()
|
|
493
807
|
} catch {
|
|
@@ -507,8 +821,10 @@ export function createSpool(input) {
|
|
|
507
821
|
} catch (error) {
|
|
508
822
|
if (error?.code === "ENOENT") continue
|
|
509
823
|
}
|
|
510
|
-
|
|
511
|
-
|
|
824
|
+
let state = stateOf(file, peeked)
|
|
825
|
+
// Quarantine state on a pending entry: an operator renamed it back.
|
|
826
|
+
const requeued = Boolean(state.quarantine)
|
|
827
|
+
if (!requeued && !options.force && state.nextAttemptAt && Date.parse(state.nextAttemptAt) > now()) {
|
|
512
828
|
result.deferred += 1
|
|
513
829
|
continue
|
|
514
830
|
}
|
|
@@ -516,61 +832,102 @@ export function createSpool(input) {
|
|
|
516
832
|
const claimed = claim(file)
|
|
517
833
|
if (!claimed) continue
|
|
518
834
|
|
|
519
|
-
let raw = ""
|
|
520
835
|
let entry
|
|
521
836
|
try {
|
|
522
|
-
|
|
523
|
-
entry = JSON.parse(raw)
|
|
524
|
-
if (entry?.quarantine) {
|
|
525
|
-
// An operator renamed a quarantined entry back: start it afresh.
|
|
526
|
-
entry = { ...entry, attempts: 0 }
|
|
527
|
-
delete entry.quarantine
|
|
528
|
-
delete entry.nextAttemptAt
|
|
529
|
-
}
|
|
837
|
+
entry = JSON.parse(fs.readFileSync(claimed, "utf8"))
|
|
530
838
|
} catch (error) {
|
|
531
839
|
result.failed += 1
|
|
532
|
-
if (quarantine(claimed,
|
|
840
|
+
if (quarantine(claimed, `unreadable: ${errorMessage(error)}`, state)) result.quarantined += 1
|
|
533
841
|
continue
|
|
534
842
|
}
|
|
843
|
+
// A requeued entry starts afresh.
|
|
844
|
+
if (requeued) state = { attempts: 0 }
|
|
535
845
|
const invalid = invalidEntry(entry)
|
|
536
846
|
if (invalid) {
|
|
537
847
|
result.failed += 1
|
|
538
|
-
if (quarantine(claimed,
|
|
848
|
+
if (quarantine(claimed, `invalid: ${invalid}`, state)) result.quarantined += 1
|
|
849
|
+
continue
|
|
850
|
+
}
|
|
851
|
+
|
|
852
|
+
let keep = true
|
|
853
|
+
try {
|
|
854
|
+
keep = sendable(entry)
|
|
855
|
+
} catch (error) {
|
|
856
|
+
log.warn("spool forget check failed; the entry is kept for later", { file: path.basename(pendingOf(claimed)), error: errorMessage(error) })
|
|
857
|
+
recordFailure(claimed, entry, Object.assign(new Error("forget check failed"), { status: 0 }), state)
|
|
858
|
+
continue
|
|
859
|
+
}
|
|
860
|
+
if (keep === "hold") {
|
|
861
|
+
// Its space has a forget whose entries are not read yet: it waits,
|
|
862
|
+
// with no attempt counted, and the rest of the queue drains.
|
|
863
|
+
try {
|
|
864
|
+
fs.renameSync(claimed, pendingOf(claimed))
|
|
865
|
+
} catch {
|
|
866
|
+
/* another process holds it now */
|
|
867
|
+
}
|
|
868
|
+
result.deferred += 1
|
|
869
|
+
continue
|
|
870
|
+
}
|
|
871
|
+
if (keep === false) {
|
|
872
|
+
// Text the person forgot after it was queued: deleted, never sent.
|
|
873
|
+
removeSpool(claimed)
|
|
874
|
+
result.purged = (result.purged ?? 0) + 1
|
|
875
|
+
log.info("spooled entry holding forgotten text deleted before replay", { file: path.basename(pendingOf(claimed)) })
|
|
539
876
|
continue
|
|
540
877
|
}
|
|
541
878
|
|
|
542
879
|
sends += 1
|
|
880
|
+
let failure = null
|
|
543
881
|
try {
|
|
544
882
|
// As the session the entry was captured in: its token, no other.
|
|
545
883
|
await asSession(credentialSessionOf(entry), () => send(entry))
|
|
546
|
-
|
|
884
|
+
} catch (error) {
|
|
885
|
+
failure = error ?? new Error("send failed")
|
|
886
|
+
}
|
|
887
|
+
|
|
888
|
+
if (!failure) {
|
|
889
|
+
// The send succeeded. The delete is its own step: if it fails (another
|
|
890
|
+
// process recovered the claim) the entry is not re-queued or rewritten
|
|
891
|
+
// here; a later replay is deduplicated by its key.
|
|
892
|
+
removeSpool(claimed)
|
|
547
893
|
result.flushed += 1
|
|
548
894
|
unreachable = 0
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
}
|
|
563
|
-
|
|
564
|
-
unreachable = 0
|
|
895
|
+
continue
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
if (isUnlinked(failure)) {
|
|
899
|
+
dropUnlinked(claimed, entry)
|
|
900
|
+
result.dropped = (result.dropped ?? 0) + 1
|
|
901
|
+
continue
|
|
902
|
+
}
|
|
903
|
+
result.failed += 1
|
|
904
|
+
if (recordFailure(claimed, entry, failure, state) === "quarantined") result.quarantined += 1
|
|
905
|
+
if (classifyFailure(failure) === "unreachable") {
|
|
906
|
+
unreachable += 1
|
|
907
|
+
if (unreachable >= UNREACHABLE_STOP) {
|
|
908
|
+
log.warn("spool flush stopped: backend unreachable", { sent: sends })
|
|
909
|
+
break
|
|
565
910
|
}
|
|
911
|
+
} else {
|
|
912
|
+
unreachable = 0
|
|
566
913
|
}
|
|
567
914
|
}
|
|
568
915
|
return result
|
|
569
916
|
}
|
|
570
917
|
|
|
571
|
-
/**
|
|
918
|
+
/**
|
|
919
|
+
* Queue depth by state, for mem_status and /mem-sync. `reasons` lists the
|
|
920
|
+
* newest quarantine reasons (at most five), each with when it expires.
|
|
921
|
+
*/
|
|
572
922
|
function spoolStats() {
|
|
573
|
-
const stats = {
|
|
923
|
+
const stats = {
|
|
924
|
+
pending: 0,
|
|
925
|
+
inflight: 0,
|
|
926
|
+
quarantined: 0,
|
|
927
|
+
oldest: null,
|
|
928
|
+
reasons: [],
|
|
929
|
+
quarantineDays: QUARANTINE_TTL_MS / 86_400_000,
|
|
930
|
+
}
|
|
574
931
|
let names = []
|
|
575
932
|
try {
|
|
576
933
|
names = fs.readdirSync(spoolDir)
|
|
@@ -578,15 +935,35 @@ export function createSpool(input) {
|
|
|
578
935
|
return stats
|
|
579
936
|
}
|
|
580
937
|
let oldest = Number.POSITIVE_INFINITY
|
|
938
|
+
const quarantined = []
|
|
581
939
|
for (const name of names) {
|
|
582
940
|
if (name.endsWith(PENDING)) {
|
|
583
941
|
stats.pending += 1
|
|
584
942
|
const written = Number.parseInt(name, 10)
|
|
585
943
|
if (Number.isFinite(written) && written < oldest) oldest = written
|
|
586
944
|
} else if (name.endsWith(INFLIGHT)) stats.inflight += 1
|
|
587
|
-
else if (name.endsWith(FAILED))
|
|
945
|
+
else if (name.endsWith(FAILED)) {
|
|
946
|
+
const file = path.join(spoolDir, name)
|
|
947
|
+
const at = quarantinedAt(file)
|
|
948
|
+
if (now() - at >= QUARANTINE_TTL_MS) continue
|
|
949
|
+
stats.quarantined += 1
|
|
950
|
+
let reason = readState(pendingOf(file))?.quarantine?.reason
|
|
951
|
+
if (typeof reason !== "string") {
|
|
952
|
+
try {
|
|
953
|
+
reason = JSON.parse(fs.readFileSync(file, "utf8"))?.quarantine?.reason
|
|
954
|
+
} catch {
|
|
955
|
+
reason = undefined
|
|
956
|
+
}
|
|
957
|
+
}
|
|
958
|
+
quarantined.push({
|
|
959
|
+
at: new Date(at).toISOString(),
|
|
960
|
+
reason: oneLine(typeof reason === "string" ? reason : "reason not recorded (the process stopped while quarantining it)", 160),
|
|
961
|
+
expires: new Date(at + QUARANTINE_TTL_MS).toISOString(),
|
|
962
|
+
})
|
|
963
|
+
}
|
|
588
964
|
}
|
|
589
965
|
if (Number.isFinite(oldest)) stats.oldest = new Date(oldest).toISOString()
|
|
966
|
+
stats.reasons = quarantined.sort((a, b) => b.at.localeCompare(a.at)).slice(0, 5)
|
|
590
967
|
return stats
|
|
591
968
|
}
|
|
592
969
|
|
|
@@ -603,5 +980,6 @@ export function createSpool(input) {
|
|
|
603
980
|
enqueueSpool,
|
|
604
981
|
flushSpool,
|
|
605
982
|
spoolStats,
|
|
983
|
+
purgeMatching,
|
|
606
984
|
}
|
|
607
985
|
}
|