@joe-broadhead/opencode-mem 0.11.2 → 0.11.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +63 -0
- package/README.md +1 -0
- package/package.json +504 -2
- package/src/config.js +9 -1
- package/src/tools/commitments.js +50 -32
- package/src/tools/held-write.js +15 -7
- package/src/tools/index.js +1 -0
- package/src/tools/ingest.js +11 -0
- package/src/tools/memory.js +26 -18
- package/src/tools/page.js +118 -87
- package/src/tools/recall.js +12 -2
- package/src/tools/shared.js +40 -2
- package/src/tools/spaces.js +8 -1
- package/src/tools/wiki.js +208 -113
- package/src/views/contracts.js +152 -2
- package/src/views/receipt.js +26 -1
- package/src/views/records-reads.js +165 -3
- package/src/views/rpc.js +35 -17
- package/src/views/write-state.js +89 -0
- package/src/wiki-ingest.js +270 -0
- package/src/wiki-queue.js +252 -0
- package/src/wiki-state.js +228 -0
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
import fs from "node:fs"
|
|
2
|
+
import path from "node:path"
|
|
3
|
+
import { oneLine } from "./util.js"
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* A wiki's state in the declared reads (jelly #89 A1) and the last ready page
|
|
7
|
+
* set kept while an ingest runs (A2).
|
|
8
|
+
*
|
|
9
|
+
* The hub lists no pages while a wiki ingests (`/v3/wiki/page/ls` answers []
|
|
10
|
+
* unless the wiki is ready), so the 129 pages a person already had vanished
|
|
11
|
+
* from `wiki.pages` for the minutes an ingest took. The set a ready listing
|
|
12
|
+
* returned is kept here, in this process and best-effort on disk, and a
|
|
13
|
+
* listing that comes back empty because of an ingest answers that set,
|
|
14
|
+
* marked stale. Nothing here detects a host: plain OpenCode keeps working.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
/** The three states the declared read carries; any other hub status reads as processing. */
|
|
18
|
+
export const WIKI_STATES = Object.freeze(["processing", "ready", "failed"])
|
|
19
|
+
|
|
20
|
+
/** The most an `error` says, as jelly's ask declares. */
|
|
21
|
+
export const WIKI_ERROR_CHARS = 400
|
|
22
|
+
|
|
23
|
+
/** The page sets kept: one per wiki, the least recently written falls out. */
|
|
24
|
+
const MAX_WIKIS = 100
|
|
25
|
+
/** Pages kept per wiki; the view lists at most this many anyway. */
|
|
26
|
+
const MAX_PAGES = 100
|
|
27
|
+
/** The file's name in the state directory. */
|
|
28
|
+
const FILE_NAME = "wiki-pages.json"
|
|
29
|
+
|
|
30
|
+
const READY = /^(?:ready|complete|completed|succeeded|indexed|done)$/
|
|
31
|
+
const FAILED = /(?:fail|error)/
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* The state the declared reads carry for one hub wiki item: `ready` only when
|
|
35
|
+
* the hub says so, `failed` when it says the ingest failed, and `processing`
|
|
36
|
+
* for everything else (draft, processing, queued, unknown), because nothing
|
|
37
|
+
* else means "readable as ready".
|
|
38
|
+
*
|
|
39
|
+
* @param {any} item a `/v3/wiki/list` or `/v3/wiki/get` item
|
|
40
|
+
* @returns {"processing" | "ready" | "failed"}
|
|
41
|
+
*/
|
|
42
|
+
export function wikiStateOf(item) {
|
|
43
|
+
const raw = String(item?.status ?? "").trim().toLowerCase()
|
|
44
|
+
if (FAILED.test(raw)) return "failed"
|
|
45
|
+
if (READY.test(raw)) return "ready"
|
|
46
|
+
return "processing"
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** The words a failed wiki carries, from the hub or from the last ingest this process saw. */
|
|
50
|
+
export function wikiErrorOf(item, remembered = "", max = WIKI_ERROR_CHARS) {
|
|
51
|
+
if (wikiStateOf(item) !== "failed") return ""
|
|
52
|
+
const given = [item?.error, item?.ingest_error, item?.sync_error, item?.last_error].find(
|
|
53
|
+
(value) => typeof value === "string" && value.trim(),
|
|
54
|
+
)
|
|
55
|
+
const words = String(given ?? remembered ?? "").trim() || "The wiki's last ingest failed. Write a page again to start another ingest."
|
|
56
|
+
return oneLine(words, max)
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* The ready page sets of the wikis this engine has listed, and the last ingest
|
|
61
|
+
* error of each. Best-effort on disk (`wiki-pages.json` in the state
|
|
62
|
+
* directory), so a restart in the middle of an ingest still answers the set
|
|
63
|
+
* the person had; a file that cannot be read or written is ignored, and the
|
|
64
|
+
* process keeps its own copy.
|
|
65
|
+
*
|
|
66
|
+
* @param {{ stateDir?: string, log?: any }} [options]
|
|
67
|
+
*/
|
|
68
|
+
export function createWikiCache({ stateDir = "", log = null } = {}) {
|
|
69
|
+
const file = stateDir ? path.join(stateDir, FILE_NAME) : ""
|
|
70
|
+
/** wiki id -> { pages: [{ref,title}], error: string, pending: scope|null } */
|
|
71
|
+
let entries = new Map()
|
|
72
|
+
let loaded = false
|
|
73
|
+
let writable = Boolean(file)
|
|
74
|
+
|
|
75
|
+
function load() {
|
|
76
|
+
if (loaded) return
|
|
77
|
+
loaded = true
|
|
78
|
+
if (!file) return
|
|
79
|
+
let raw
|
|
80
|
+
try {
|
|
81
|
+
raw = fs.readFileSync(file, "utf8")
|
|
82
|
+
} catch (error) {
|
|
83
|
+
if (error?.code !== "ENOENT") log?.debug?.("wiki page cache is unreadable; this process keeps its own copy", { error: String(error?.code ?? error) })
|
|
84
|
+
return
|
|
85
|
+
}
|
|
86
|
+
try {
|
|
87
|
+
const parsed = JSON.parse(raw)
|
|
88
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return
|
|
89
|
+
for (const [wikiID, entry] of Object.entries(parsed)) {
|
|
90
|
+
const pages = Array.isArray(entry?.pages)
|
|
91
|
+
? entry.pages
|
|
92
|
+
.filter((page) => page && typeof page.ref === "string" && page.ref)
|
|
93
|
+
.slice(0, MAX_PAGES)
|
|
94
|
+
.map((page) => ({ ref: page.ref, title: String(page.title ?? page.ref).slice(0, 200) }))
|
|
95
|
+
: []
|
|
96
|
+
const error = typeof entry?.error === "string" ? oneLine(entry.error, WIKI_ERROR_CHARS) : ""
|
|
97
|
+
const pending = entry?.pending && typeof entry.pending === "object" ? scopeOf(entry.pending) : null
|
|
98
|
+
if (pages.length || error || pending) entries.set(String(wikiID), { pages, error, pending })
|
|
99
|
+
}
|
|
100
|
+
} catch {
|
|
101
|
+
/* a corrupt cache is no cache: the next ready listing writes a new one */
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function save() {
|
|
106
|
+
if (!writable || !file) return
|
|
107
|
+
try {
|
|
108
|
+
const out = {}
|
|
109
|
+
for (const [wikiID, entry] of entries) {
|
|
110
|
+
const saved = {}
|
|
111
|
+
if (entry.pages.length) saved.pages = entry.pages
|
|
112
|
+
else if (entry.error) saved.error = entry.error
|
|
113
|
+
if (entry.pending) saved.pending = entry.pending
|
|
114
|
+
if (saved.pages || saved.error || saved.pending) out[wikiID] = saved
|
|
115
|
+
}
|
|
116
|
+
fs.mkdirSync(path.dirname(file), { recursive: true })
|
|
117
|
+
fs.writeFileSync(file, JSON.stringify(out))
|
|
118
|
+
} catch (error) {
|
|
119
|
+
// A read must never fail for a cache: keep this process's copy only.
|
|
120
|
+
writable = false
|
|
121
|
+
log?.debug?.("wiki page cache is not writable; this process keeps its own copy", { error: String(error?.code ?? error) })
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function touch(wikiID) {
|
|
126
|
+
load()
|
|
127
|
+
const id = String(wikiID ?? "")
|
|
128
|
+
if (!id) return null
|
|
129
|
+
const entry = entries.get(id) ?? { pages: [], error: "", pending: null }
|
|
130
|
+
entries.delete(id)
|
|
131
|
+
entries.set(id, entry)
|
|
132
|
+
while (entries.size > MAX_WIKIS) entries.delete(entries.keys().next().value)
|
|
133
|
+
return entry
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
return {
|
|
137
|
+
/** Keep the pages a ready listing returned; [] is never recorded (it means an ingest, not emptiness). */
|
|
138
|
+
rememberPages(wikiID, pages) {
|
|
139
|
+
if (!Array.isArray(pages) || pages.length === 0) return
|
|
140
|
+
const entry = touch(wikiID)
|
|
141
|
+
entry.pages = pages.slice(0, MAX_PAGES).map((page) => ({ ref: String(page.ref), title: String(page.title ?? page.ref).slice(0, 200) }))
|
|
142
|
+
entry.error = ""
|
|
143
|
+
save()
|
|
144
|
+
},
|
|
145
|
+
/** The last ready set of a wiki, or [] when this engine never listed it. */
|
|
146
|
+
pages(wikiID) {
|
|
147
|
+
const entry = touch(wikiID)
|
|
148
|
+
return entry ? entry.pages.map((page) => ({ ...page })) : []
|
|
149
|
+
},
|
|
150
|
+
/** Remember why an ingest failed, for the `error` a failed wiki carries. */
|
|
151
|
+
rememberError(wikiID, message) {
|
|
152
|
+
const entry = touch(wikiID)
|
|
153
|
+
if (!entry) return
|
|
154
|
+
entry.error = oneLine(String(message ?? "").trim(), WIKI_ERROR_CHARS)
|
|
155
|
+
if (!entry.error) return
|
|
156
|
+
save()
|
|
157
|
+
},
|
|
158
|
+
/** The last ingest error recorded for a wiki, or "". */
|
|
159
|
+
error(wikiID) {
|
|
160
|
+
const entry = touch(wikiID)
|
|
161
|
+
return entry?.error ?? ""
|
|
162
|
+
},
|
|
163
|
+
/** Forget a wiki's error (its own ready listing says it is healthy again). */
|
|
164
|
+
clearError(wikiID) {
|
|
165
|
+
const entry = touch(wikiID)
|
|
166
|
+
if (!entry || !entry.error) return
|
|
167
|
+
entry.error = ""
|
|
168
|
+
save()
|
|
169
|
+
},
|
|
170
|
+
/**
|
|
171
|
+
* Note that a write waits for its ingest (jelly #89 A3), so an engine that
|
|
172
|
+
* stops inside the debounce still ingests on its next start. Only the
|
|
173
|
+
* scope's ids are kept, never the text written.
|
|
174
|
+
*/
|
|
175
|
+
rememberPending(wikiID, scope) {
|
|
176
|
+
const entry = touch(wikiID)
|
|
177
|
+
if (!entry) return
|
|
178
|
+
const kept = scopeOf(scope)
|
|
179
|
+
if (!kept) return
|
|
180
|
+
entry.pending = kept
|
|
181
|
+
save()
|
|
182
|
+
},
|
|
183
|
+
/** The ingest started: nothing waits for it any more. */
|
|
184
|
+
clearPending(wikiID) {
|
|
185
|
+
const entry = touch(wikiID)
|
|
186
|
+
if (!entry?.pending) return
|
|
187
|
+
entry.pending = null
|
|
188
|
+
save()
|
|
189
|
+
},
|
|
190
|
+
/** Every wiki whose write still waits for an ingest, as `{ wikiID, scope }`. */
|
|
191
|
+
pending() {
|
|
192
|
+
load()
|
|
193
|
+
const list = []
|
|
194
|
+
for (const [wikiID, entry] of entries) if (entry.pending) list.push({ wikiID, scope: { ...entry.pending } })
|
|
195
|
+
return list
|
|
196
|
+
},
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/** The ids of a scope, as the cache keeps them: never a session or any text. */
|
|
201
|
+
function scopeOf(scope) {
|
|
202
|
+
if (!scope || typeof scope !== "object") return null
|
|
203
|
+
const kept = {}
|
|
204
|
+
for (const key of ["team_id", "space_id", "user_id", "agent_id"]) {
|
|
205
|
+
const value = String(scope[key] ?? "").trim()
|
|
206
|
+
if (value) kept[key] = value
|
|
207
|
+
}
|
|
208
|
+
return Object.keys(kept).length > 0 ? kept : null
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/** One cache per state directory: the views and the tools read the same wiki's set. */
|
|
212
|
+
const sharedCaches = new Map()
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* The wiki cache of one engine's state directory: both the views and the tools
|
|
216
|
+
* run in that engine, with the same directory.
|
|
217
|
+
*
|
|
218
|
+
* @param {{ stateDir?: string, log?: any }} [options]
|
|
219
|
+
*/
|
|
220
|
+
export function wikiCache(options = {}) {
|
|
221
|
+
const key = String(options.stateDir ?? "")
|
|
222
|
+
let cache = sharedCaches.get(key)
|
|
223
|
+
if (!cache) {
|
|
224
|
+
cache = createWikiCache(options)
|
|
225
|
+
sharedCaches.set(key, cache)
|
|
226
|
+
}
|
|
227
|
+
return cache
|
|
228
|
+
}
|