@joe-broadhead/opencode-mem 0.11.2 → 0.11.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,270 @@
1
+ import { errorMessage } from "./util.js"
2
+ import { wikiWriteDigest } from "./wiki-queue.js"
3
+
4
+ /** How long a write waits for the rest of its burst before one ingest starts. */
5
+ export const WIKI_INGEST_DELAY_MS = 3000
6
+ /** The longest wait between attempts while a wiki is ingesting another write. */
7
+ const RETRY_MAX_MS = 60_000
8
+ /** Attempts a queued write makes on an error that is not "the wiki is ingesting". */
9
+ const WRITE_ATTEMPTS = 3
10
+
11
+ /** True when the hub refused because a wiki is ingesting (409 "wiki is processing"). */
12
+ export function isWikiBusy(error) {
13
+ return Number(error?.status) === 409 || /wiki is processing/i.test(String(error?.what ?? errorMessage(error)))
14
+ }
15
+
16
+ /**
17
+ * Automatic ingest (jelly #89 A3): a write is never left unread, and the agent
18
+ * never has to pass `ingest: true`.
19
+ *
20
+ * - **Debounced.** Each successful write restarts one timer for its wiki
21
+ * (`delay`, 3 seconds by default); a burst of writes makes one ingest. Three
22
+ * seconds is longer than the gap between the calls of one writing burst and
23
+ * short next to an ingest, which takes minutes.
24
+ * - **Queued, not refused, and durable.** The hub refuses a write while it
25
+ * ingests (409 "wiki is processing"), so such a write is saved to the
26
+ * durable queue (`src/wiki-queue.js`) before the answer says `queued`,
27
+ * written when the ingest finishes, and one ingest follows it. After a
28
+ * restart the saved entries are replayed in order, and one whose body the
29
+ * wiki already holds is skipped (its filename and digest), so a crash
30
+ * between the hub's accept and the entry's removal never writes it twice.
31
+ * - **One ingest per wiki at a time here.** All of a wiki's queue work is
32
+ * serialised (`running`), and the hub's 409 is the lock across engines.
33
+ *
34
+ * Plain OpenCode and Jelly run the same code: nothing here knows the host.
35
+ *
36
+ * @param {{
37
+ * backend: any,
38
+ * log?: any,
39
+ * delay?: number,
40
+ * queue: { append: Function, entries: Function, remove: Function },
41
+ * rememberError?: (wikiID: string, message: string) => void,
42
+ * clearError?: (wikiID: string) => void,
43
+ * rememberPending?: (wikiID: string, scope: object) => void,
44
+ * clearPending?: (wikiID: string) => void,
45
+ * pending?: Array<{ wikiID: string, scope: object }> | (() => Array<{ wikiID: string, scope: object }>),
46
+ * }} input
47
+ */
48
+ export function createWikiIngest({
49
+ backend,
50
+ log = null,
51
+ delay = WIKI_INGEST_DELAY_MS,
52
+ queue: durable,
53
+ rememberError = () => {},
54
+ clearError = () => {},
55
+ rememberPending = () => {},
56
+ clearPending = () => {},
57
+ pending = [],
58
+ } = {}) {
59
+ if (!durable || typeof durable.append !== "function" || typeof durable.entries !== "function") {
60
+ // A held write is never taken without a durable record (jelly #89, 0.11.4).
61
+ throw new Error("the wiki ingest queue needs its durable store")
62
+ }
63
+
64
+ /** `${team}/${space}/${user}/${wiki}` -> one queue, so one ingest at a time. */
65
+ const queues = new Map()
66
+ let stopped = false
67
+ let startTimer = null
68
+
69
+ /**
70
+ * What an earlier engine start left: the writes it queued, replayed in the
71
+ * order they were saved, and the ingests its debounce did not finish. Read
72
+ * after setup returns, so a start never waits on the state files.
73
+ */
74
+ function recover() {
75
+ startTimer = null
76
+ if (stopped) return
77
+ let saved = []
78
+ try {
79
+ saved = typeof durable.entries === "function" ? durable.entries() : []
80
+ } catch (error) {
81
+ log?.warn?.("the wiki write queue could not be read", { error: errorMessage(error) })
82
+ }
83
+ for (const entry of saved) {
84
+ if (!entry?.wiki || !entry?.file) continue
85
+ const queue = queueFor(entry.scope, entry.wiki)
86
+ queue.writes.push(entry)
87
+ // Replay promptly: a busy hub answers 409, and the retry waits.
88
+ arm(queue, 10)
89
+ }
90
+ const waiting = typeof pending === "function" ? pending() : pending
91
+ for (const entry of waiting ?? []) {
92
+ if (!entry?.wikiID || !entry?.scope) continue
93
+ const queue = queueFor(entry.scope, entry.wikiID)
94
+ queue.dirty = true
95
+ arm(queue, delay)
96
+ }
97
+ }
98
+ startTimer = setTimeout(recover, 0)
99
+ startTimer?.unref?.()
100
+
101
+ function queueFor(scope, wikiID) {
102
+ const key = [scope?.team_id ?? "", scope?.space_id ?? "", scope?.user_id ?? "", wikiID].join("\u0000")
103
+ let queue = queues.get(key)
104
+ if (!queue) {
105
+ queue = { scope, wikiID, timer: null, running: false, dirty: false, writes: [], failures: 0 }
106
+ queues.set(key, queue)
107
+ }
108
+ return queue
109
+ }
110
+
111
+ function arm(queue, ms) {
112
+ if (stopped) return
113
+ if (queue.timer) clearTimeout(queue.timer)
114
+ queue.timer = setTimeout(() => {
115
+ queue.timer = null
116
+ flush(queue).catch((error) => log?.warn?.("wiki ingest queue failed", error))
117
+ }, Math.max(10, ms))
118
+ // As the collector's timers: never hold the host open.
119
+ queue.timer?.unref?.()
120
+ }
121
+
122
+ /** The wait before another attempt, doubling while the wiki stays busy. */
123
+ function retryDelay(queue) {
124
+ const factor = 2 ** Math.min(queue.failures, 6)
125
+ return Math.min(RETRY_MAX_MS, Math.max(delay, 100) * factor)
126
+ }
127
+
128
+ /**
129
+ * Save a write the hub refused because it is ingesting, before answering.
130
+ * It is written when the ingest finishes, in the order it arrived, and one
131
+ * ingest follows. A disk that refuses the save throws, so the answer is
132
+ * `refused`, never `queued` (jelly #89, 0.11.4).
133
+ */
134
+ function hold(queue, filename, content) {
135
+ const entry = durable.append({ wikiID: queue.wikiID, scope: queue.scope, filename, content })
136
+ queue.writes.push(entry)
137
+ queue.failures += 1
138
+ arm(queue, retryDelay(queue))
139
+ return entry
140
+ }
141
+
142
+ /** True when the wiki already holds exactly this body, so a replay skips it. */
143
+ async function alreadyLanded(queue, entry) {
144
+ try {
145
+ const data = await backend.readWikiRaw({ scope: queue.scope, wikiID: queue.wikiID, filenames: [entry.file] })
146
+ const item = (data?.items ?? []).find((row) => row?.filename === entry.file)
147
+ return typeof item?.content === "string" && wikiWriteDigest(item.content) === entry.digest
148
+ } catch {
149
+ // An unreadable file is not proof it landed: attempt the write.
150
+ return false
151
+ }
152
+ }
153
+
154
+ async function flush(queue) {
155
+ if (stopped || queue.running) return
156
+ queue.running = true
157
+ let landed = false
158
+ try {
159
+ while (queue.writes.length > 0) {
160
+ const next = queue.writes[0]
161
+ if (await alreadyLanded(queue, next)) {
162
+ // The hub accepted it and the engine stopped before the removal:
163
+ // the same body is never written twice.
164
+ queue.writes.shift()
165
+ durable.remove(next.id)
166
+ continue
167
+ }
168
+ try {
169
+ await backend.writeWikiRaw({ scope: queue.scope, wikiID: queue.wikiID, files: [{ filename: next.file, content: next.content }] })
170
+ } catch (error) {
171
+ if (isWikiBusy(error)) {
172
+ queue.dirty = true
173
+ queue.failures += 1
174
+ return
175
+ }
176
+ next.attempts += 1
177
+ if (next.attempts < WRITE_ATTEMPTS) {
178
+ queue.failures += 1
179
+ return
180
+ }
181
+ // The same refusal three times is not going to pass: say so, and do
182
+ // not repeat it forever.
183
+ queue.writes.shift()
184
+ durable.remove(next.id)
185
+ rememberError(queue.wikiID, `A queued write to ${next.file} failed: ${errorMessage(error)}`)
186
+ log?.warn?.("a queued wiki write was dropped", { wikiID: queue.wikiID, filename: next.file, error: errorMessage(error) })
187
+ continue
188
+ }
189
+ queue.writes.shift()
190
+ durable.remove(next.id)
191
+ queue.dirty = true
192
+ landed = true
193
+ }
194
+ // One ingest, only once every held write has landed: it covers them all.
195
+ if (queue.dirty) {
196
+ queue.dirty = false
197
+ try {
198
+ const answer = await backend.ingestWiki({ scope: queue.scope, wikiID: queue.wikiID })
199
+ clearPending(queue.wikiID)
200
+ if (String(answer?.status ?? "") === "failed") rememberError(queue.wikiID, "The wiki's last ingest failed.")
201
+ else clearError(queue.wikiID)
202
+ } catch (error) {
203
+ if (isWikiBusy(error)) {
204
+ queue.dirty = true
205
+ queue.failures += 1
206
+ return
207
+ }
208
+ rememberError(queue.wikiID, `The wiki's ingest did not start: ${errorMessage(error)}`)
209
+ log?.warn?.("the automatic wiki ingest failed", { wikiID: queue.wikiID, error: errorMessage(error) })
210
+ return
211
+ }
212
+ }
213
+ queue.failures = 0
214
+ if (landed) clearError(queue.wikiID)
215
+ } finally {
216
+ queue.running = false
217
+ if (!stopped && (queue.writes.length > 0 || queue.dirty)) arm(queue, retryDelay(queue))
218
+ }
219
+ }
220
+
221
+ return {
222
+ /**
223
+ * Write one source file now, or hold it when the wiki is ingesting. The
224
+ * ingest that reads it starts on its own after the last write of the burst.
225
+ *
226
+ * @returns {Promise<{ queued: boolean }>}
227
+ */
228
+ async write({ scope, wikiID, filename, content }) {
229
+ const queue = queueFor(scope, wikiID)
230
+ try {
231
+ await backend.writeWikiRaw({ scope, wikiID, files: [{ filename, content }] })
232
+ } catch (error) {
233
+ if (!isWikiBusy(error)) throw error
234
+ try {
235
+ hold(queue, filename, content)
236
+ } catch (queueError) {
237
+ // No durable record, so this is not queued: the answer is refused.
238
+ log?.warn?.("a queued wiki write could not be saved", { wikiID, filename, error: errorMessage(queueError) })
239
+ return { queued: false, refused: true, reason: String(queueError?.code ?? "") }
240
+ }
241
+ return { queued: true }
242
+ }
243
+ queue.dirty = true
244
+ queue.failures = 0
245
+ clearError(wikiID)
246
+ rememberPending(wikiID, scope)
247
+ arm(queue, delay)
248
+ return { queued: false }
249
+ },
250
+
251
+ /** How many writes wait for their wiki to finish ingesting (tests, health). */
252
+ held(wikiID) {
253
+ let count = 0
254
+ for (const queue of queues.values()) if (queue.wikiID === wikiID) count += queue.writes.length
255
+ return count
256
+ },
257
+
258
+ /** Stop every timer; what waited is on disk, and the next start replays it. */
259
+ stop() {
260
+ stopped = true
261
+ if (startTimer) clearTimeout(startTimer)
262
+ startTimer = null
263
+ for (const queue of queues.values()) {
264
+ if (queue.timer) clearTimeout(queue.timer)
265
+ queue.timer = null
266
+ }
267
+ queues.clear()
268
+ },
269
+ }
270
+ }
@@ -0,0 +1,252 @@
1
+ import fs from "node:fs"
2
+ import path from "node:path"
3
+ import { createHash, randomBytes } from "node:crypto"
4
+
5
+ /**
6
+ * The durable queue of wiki writes the hub refused because it was ingesting
7
+ * (jelly #89, 0.11.4). A write held only in the engine's memory dies with it,
8
+ * and that is the silent loss the ask exists to end: each one is written here
9
+ * **before** the tool answers `state: queued`, and replayed at the next start.
10
+ *
11
+ * - **Where.** One file per write in `wiki-queue/` under the plugin's own state
12
+ * directory (the one `config.stateDir` names, beside the spool and the pause
13
+ * file). Each file holds the write's body, so it is owner-only (`0600`), in a
14
+ * `0700` directory, and its content never goes into a log line.
15
+ * - **Bounds.** At most `WIKI_QUEUE_MAX_ENTRIES` writes and
16
+ * `WIKI_QUEUE_MAX_BYTES` of bodies (8 MiB); a write that would pass either
17
+ * is not saved, and the answer is `state: refused` rather than `queued`.
18
+ * - **Atomic.** Each file is written as a private temp file in the same
19
+ * directory, fsynced, renamed into place, and the directory is fsynced, so a
20
+ * crash leaves no file or the whole one, never a torn one. One file per
21
+ * write also means two engine processes on one machine never overwrite each
22
+ * other's queue: an append only ever creates its own file.
23
+ * - **Once.** A file is removed only after the hub accepted the write. A
24
+ * crash between the accept and the removal leaves it, and the replay checks
25
+ * the wiki's own file (its filename and a digest of the body) before
26
+ * writing, so the same body is never written twice.
27
+ */
28
+
29
+ export const WIKI_QUEUE_DIR = "wiki-queue"
30
+ /** Writes the queue holds. */
31
+ export const WIKI_QUEUE_MAX_ENTRIES = 100
32
+ /** Bodies the queue holds, together. */
33
+ export const WIKI_QUEUE_MAX_BYTES = 8 * 1024 * 1024
34
+
35
+ const ENTRY = /^wq_[a-f0-9]{16}\.json$/
36
+
37
+ /** A queue failure the answer can name without repeating the file system's words. */
38
+ export class WikiQueueError extends Error {
39
+ constructor(code, message) {
40
+ super(message)
41
+ this.name = "WikiQueueError"
42
+ this.code = code
43
+ }
44
+ }
45
+
46
+ /** A body's key, so a replay never writes one the wiki already holds. */
47
+ export function wikiWriteDigest(content) {
48
+ return createHash("sha256").update(String(content ?? ""), "utf8").digest("hex")
49
+ }
50
+
51
+ /** The scope a replay needs, without a session or any text. */
52
+ function scopeOf(scope) {
53
+ const kept = {}
54
+ for (const key of ["team_id", "space_id", "user_id", "agent_id"]) {
55
+ const value = String(scope?.[key] ?? "").trim()
56
+ if (value) kept[key] = value
57
+ }
58
+ return kept
59
+ }
60
+
61
+ function validEntry(entry) {
62
+ return Boolean(
63
+ entry &&
64
+ typeof entry.id === "string" &&
65
+ ENTRY.test(`${entry.id}.json`) &&
66
+ typeof entry.wiki === "string" &&
67
+ entry.wiki &&
68
+ typeof entry.file === "string" &&
69
+ entry.file &&
70
+ typeof entry.content === "string" &&
71
+ typeof entry.digest === "string" &&
72
+ entry.digest,
73
+ )
74
+ }
75
+
76
+ /**
77
+ * @param {{ stateDir?: string, log?: any }} input
78
+ */
79
+ export function createWikiQueue({ stateDir = "", log = null } = {}) {
80
+ const base = String(stateDir ?? "")
81
+ const dir = base ? path.join(base, WIKI_QUEUE_DIR) : ""
82
+ let ensured = false
83
+ /** This process's append counter, so entries of one millisecond keep their order. */
84
+ let sequence = 0
85
+
86
+ /** Create the queue directory when it is missing; never override an operator's mode. */
87
+ function ensureDir() {
88
+ if (ensured) return
89
+ const existed = fs.existsSync(dir)
90
+ fs.mkdirSync(dir, { recursive: true, mode: 0o700 })
91
+ if (!existed) {
92
+ try {
93
+ fs.chmodSync(dir, 0o700)
94
+ } catch {
95
+ /* best effort on platforms without POSIX modes */
96
+ }
97
+ }
98
+ ensured = true
99
+ }
100
+
101
+ /** Make a rename or a new file in the directory survive a crash. Best effort. */
102
+ function syncDir() {
103
+ let fd
104
+ try {
105
+ fd = fs.openSync(dir, "r")
106
+ fs.fsyncSync(fd)
107
+ } catch {
108
+ /* not supported on every platform (Windows) */
109
+ } finally {
110
+ if (fd !== undefined) {
111
+ try {
112
+ fs.closeSync(fd)
113
+ } catch {
114
+ /* ignore */
115
+ }
116
+ }
117
+ }
118
+ }
119
+
120
+ /** Every entry file, newest last, read from disk so a second process's are seen. */
121
+ function read() {
122
+ if (!dir) return []
123
+ ensureDir()
124
+ let names
125
+ try {
126
+ names = fs.readdirSync(dir)
127
+ } catch (error) {
128
+ if (error?.code !== "ENOENT") log?.debug?.("the wiki write queue is unreadable", { error: String(error?.code ?? error) })
129
+ return []
130
+ }
131
+ const entries = []
132
+ for (const name of names) {
133
+ if (!ENTRY.test(name)) continue
134
+ let parsed
135
+ try {
136
+ parsed = JSON.parse(fs.readFileSync(path.join(dir, name), "utf8"))
137
+ } catch {
138
+ log?.warn?.("a wiki write queue entry is unreadable and is left alone", { entry: name })
139
+ continue
140
+ }
141
+ if (!validEntry(parsed)) {
142
+ log?.warn?.("a wiki write queue entry is not one, and is left alone", { entry: name })
143
+ continue
144
+ }
145
+ entries.push({
146
+ id: parsed.id,
147
+ wiki: parsed.wiki,
148
+ file: parsed.file,
149
+ digest: parsed.digest,
150
+ bytes: Number(parsed.bytes) || Buffer.byteLength(parsed.content),
151
+ scope: scopeOf(parsed.scope),
152
+ content: parsed.content,
153
+ at: typeof parsed.at === "string" ? parsed.at : "",
154
+ seq: Number(parsed.seq) || 0,
155
+ })
156
+ }
157
+ // The order the writes were queued: time, then this process's own counter.
158
+ entries.sort((a, b) => String(a.at).localeCompare(String(b.at)) || a.seq - b.seq || a.id.localeCompare(b.id))
159
+ return entries.map(({ seq: _seq, ...entry }) => entry)
160
+ }
161
+
162
+ /** One entry, atomically: temp file, fsync, rename, directory fsync. */
163
+ function saveFile(entry) {
164
+ ensureDir()
165
+ const target = path.join(dir, `${entry.id}.json`)
166
+ const temp = path.join(dir, `.${entry.id}.${process.pid}-${randomBytes(4).toString("hex")}.tmp`)
167
+ const body = Buffer.from(JSON.stringify(entry, null, 2))
168
+ let fd
169
+ try {
170
+ fd = fs.openSync(temp, "wx", 0o600)
171
+ try {
172
+ fs.fchmodSync(fd, 0o600)
173
+ } catch {
174
+ /* best effort on platforms without POSIX modes */
175
+ }
176
+ let offset = 0
177
+ while (offset < body.length) offset += fs.writeSync(fd, body, offset, body.length - offset)
178
+ fs.fsyncSync(fd)
179
+ fs.closeSync(fd)
180
+ fd = undefined
181
+ fs.renameSync(temp, target)
182
+ syncDir()
183
+ } catch (error) {
184
+ if (fd !== undefined) {
185
+ try {
186
+ fs.closeSync(fd)
187
+ } catch {
188
+ /* ignore */
189
+ }
190
+ }
191
+ try {
192
+ fs.unlinkSync(temp)
193
+ } catch {
194
+ /* the temp file never existed */
195
+ }
196
+ throw new WikiQueueError("queue_io", `the queue could not be written (${String(error?.code ?? error?.message ?? error)})`)
197
+ }
198
+ }
199
+
200
+ return {
201
+ /** Every waiting write, in the order it was queued. */
202
+ entries() {
203
+ return read()
204
+ },
205
+
206
+ /**
207
+ * Save one write, or throw `queue_full` / `queue_io`. The entry is on disk
208
+ * when this returns.
209
+ */
210
+ append({ wikiID, scope, filename, content }) {
211
+ const body = String(content ?? "")
212
+ const bytes = Buffer.byteLength(body)
213
+ const waiting = read()
214
+ if (waiting.length >= WIKI_QUEUE_MAX_ENTRIES) throw new WikiQueueError("queue_full", "the wiki write queue is full")
215
+ if (waiting.reduce((sum, entry) => sum + entry.bytes, 0) + bytes > WIKI_QUEUE_MAX_BYTES) {
216
+ throw new WikiQueueError("queue_full", "the wiki write queue holds too many bytes")
217
+ }
218
+ const entry = {
219
+ id: `wq_${randomBytes(8).toString("hex")}`,
220
+ wiki: String(wikiID),
221
+ file: String(filename),
222
+ digest: wikiWriteDigest(body),
223
+ bytes,
224
+ scope: scopeOf(scope),
225
+ content: body,
226
+ at: new Date().toISOString(),
227
+ seq: (sequence += 1),
228
+ }
229
+ saveFile(entry)
230
+ const { seq: _seq, ...saved } = entry
231
+ return saved
232
+ },
233
+
234
+ /**
235
+ * Forget a write the hub accepted. A removal the disk refuses is logged
236
+ * and left to the replay's digest check, which writes nothing twice.
237
+ */
238
+ remove(id) {
239
+ if (!dir || !ENTRY.test(`${id}.json`)) return
240
+ try {
241
+ fs.unlinkSync(path.join(dir, `${id}.json`))
242
+ } catch (error) {
243
+ if (error?.code !== "ENOENT") log?.warn?.("a wiki write could not be removed from the queue; the replay will skip it by its content", { error: String(error?.code ?? error) })
244
+ }
245
+ },
246
+
247
+ /** A write with the same file and body already waits here. */
248
+ has(wikiID, filename, digest) {
249
+ return read().some((entry) => entry.wiki === wikiID && entry.file === filename && entry.digest === digest)
250
+ },
251
+ }
252
+ }