@joe-broadhead/opencode-mem 0.9.14 → 0.11.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1856 -0
- package/README.md +15 -7
- package/SECURITY.md +3 -1
- package/docs/agent-instructions.md +1 -1
- package/docs/agent-protocol.md +7 -7
- package/docs/permissions.md +16 -0
- package/package.json +8921 -1070
- package/shared/page-paths.js +44 -0
- package/skills/mem.pages/SKILL.md +3 -1
- package/skills/mem.usage/SKILL.md +15 -15
- package/src/action-grant-keys.js +159 -0
- package/src/action-grant.js +439 -0
- package/src/actions.js +1690 -0
- package/src/backend/records.js +25 -2
- package/src/backend/v3.js +76 -22
- package/src/call-shapes.js +17 -16
- package/src/collector-lock.js +533 -62
- package/src/collector-service.js +153 -26
- package/src/collector.js +623 -30
- package/src/config.js +88 -7
- package/src/connect.js +106 -8
- package/src/credentials.js +123 -25
- package/src/events.js +174 -3
- package/src/forget-turn.js +141 -0
- package/src/host-refusal.js +78 -0
- package/src/index.js +209 -22
- package/src/jobs/host.js +557 -0
- package/src/ops/codes.js +110 -0
- package/src/ops/copy-lineage.js +42 -0
- package/src/ops/index.js +28 -0
- package/src/ops/space-ops.js +1936 -0
- package/src/outlinks.js +103 -0
- package/src/p2-reservation.js +151 -0
- package/src/permissions.js +30 -0
- package/src/plugins/descriptor.js +62 -0
- package/src/plugins/discovery.js +318 -0
- package/src/plugins/grants.js +178 -0
- package/src/plugins/locate.js +319 -0
- package/src/plugins/manifest.js +1162 -0
- package/src/plugins/source-contract/index.js +34 -0
- package/src/plugins/source-contract/policy.js +102 -0
- package/src/plugins/source-contract/vendor/PROVENANCE.json +17 -0
- package/src/plugins/source-contract/vendor/RELEASE-SHA256SUMS +3 -0
- package/src/plugins/source-contract/vendor/RELEASE.json +17 -0
- package/src/plugins/source-contract/vendor/SHA256SUMS +41 -0
- package/src/plugins/source-contract/vendor/THIRD-PARTY-NOTICES.md +35 -0
- package/src/plugins/source-contract/vendor/broker-client.js +93 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-3M5PAKSS.js +84 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-4ERCA4TK.js +185 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-6HDIB2SA.js +183 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-ELPJD3ND.js +10 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-HLVAKAYG.js +10 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-I2RXJJJM.js +81 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-KWEWSZ4V.js +505 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-KYWBTAF4.js +182 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-Q3RQA5J2.js +249 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-RRUAIQ4I.js +156 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-RYNKNCPP.js +165 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-W5SHR7DQ.js +9477 -0
- package/src/plugins/source-contract/vendor/chunks/chunk-ZUKF7P6U.js +653 -0
- package/src/plugins/source-contract/vendor/cursor.js +26 -0
- package/src/plugins/source-contract/vendor/descriptor.js +34 -0
- package/src/plugins/source-contract/vendor/events.js +257 -0
- package/src/plugins/source-contract/vendor/index.js +10 -0
- package/src/plugins/source-contract/vendor/params.js +22 -0
- package/src/plugins/source-contract/vendor/permission.js +28 -0
- package/src/plugins/source-contract/vendor/pick.js +22 -0
- package/src/plugins/source-contract/vendor/read.js +29 -0
- package/src/plugins/source-contract/vendor/registry.js +23 -0
- package/src/plugins/source-contract/vendor/schema.js +17 -0
- package/src/plugins/source-contract/vendor/schemas/output.agenda.json +96 -0
- package/src/plugins/source-contract/vendor/schemas/output.countdown.json +28 -0
- package/src/plugins/source-contract/vendor/schemas/output.image.json +28 -0
- package/src/plugins/source-contract/vendor/schemas/output.inbox.json +84 -0
- package/src/plugins/source-contract/vendor/schemas/output.links.json +46 -0
- package/src/plugins/source-contract/vendor/schemas/output.list.json +57 -0
- package/src/plugins/source-contract/vendor/schemas/output.metric.json +61 -0
- package/src/plugins/source-contract/vendor/schemas/output.note.json +14 -0
- package/src/plugins/source-contract/vendor/schemas/output.progress.json +50 -0
- package/src/plugins/source-contract/vendor/schemas/output.record.json +38 -0
- package/src/plugins/source-contract/vendor/schemas/output.table.json +61 -0
- package/src/plugins/source-contract/vendor/schemas/output.thread.json +62 -0
- package/src/plugins/source-contract/vendor/schemas/sources.json +819 -0
- package/src/plugins/source-contract/vendor/schemas.js +19 -0
- package/src/plugins/source-contract/vendor/tools-rpc.js +42 -0
- package/src/plugins/verify.js +56 -0
- package/src/protocol.js +11 -6
- package/src/recall.js +92 -5
- package/src/records.js +61 -2
- package/src/requests.js +216 -0
- package/src/runtime.js +89 -7
- package/src/setup-time.js +68 -0
- package/src/skills.js +230 -8
- package/src/source-ids.js +25 -0
- package/src/sources/declaration.js +365 -0
- package/src/sources/reads.js +772 -0
- package/src/sources/rpc.js +151 -0
- package/src/sources.js +389 -92
- package/src/spaces.js +283 -11
- package/src/spool.js +61 -4
- package/src/stated.js +10 -5
- package/src/storable.js +139 -0
- package/src/tools/commitments.js +74 -23
- package/src/tools/held-write.js +46 -19
- package/src/tools/index.js +28 -0
- package/src/tools/ingest.js +38 -2
- package/src/tools/memory.js +56 -25
- package/src/tools/page.js +212 -95
- package/src/tools/recall.js +218 -4
- package/src/tools/share.js +42 -3
- package/src/tools/shared.js +84 -6
- package/src/tools/space-lifecycle.js +278 -0
- package/src/tools/spaces.js +381 -30
- package/src/tools/status.js +9 -2
- package/src/tools/wiki.js +231 -116
- package/src/util.js +110 -6
- package/src/vendor/source-contract/README.md +24 -0
- package/src/vendor/source-contract/THIRD-PARTY-NOTICES.md +35 -0
- package/src/vendor/source-contract/VENDOR.json +81 -0
- package/src/vendor/source-contract/chunks/chunk-3M5PAKSS.js +84 -0
- package/src/vendor/source-contract/chunks/chunk-6HDIB2SA.js +183 -0
- package/src/vendor/source-contract/chunks/chunk-HLVAKAYG.js +10 -0
- package/src/vendor/source-contract/chunks/chunk-I2RXJJJM.js +81 -0
- package/src/vendor/source-contract/chunks/chunk-KWEWSZ4V.js +505 -0
- package/src/vendor/source-contract/chunks/chunk-W5SHR7DQ.js +9477 -0
- package/src/vendor/source-contract/chunks/chunk-ZUKF7P6U.js +653 -0
- package/src/vendor/source-contract/cursor.js +26 -0
- package/src/vendor/source-contract/events.js +257 -0
- package/src/vendor/source-contract/registry.js +23 -0
- package/src/vendor/source-contract/tools-rpc.js +42 -0
- package/src/views/access-reads.js +256 -0
- package/src/views/communities.js +77 -0
- package/src/views/contracts.js +1082 -23
- package/src/views/inbox-bytes.js +103 -0
- package/src/views/profile.js +31 -0
- package/src/views/receipt.js +26 -1
- package/src/views/records-reads.js +520 -41
- package/src/views/rpc.js +153 -25
- package/src/views/shape.js +8 -5
- package/src/views/write-state.js +89 -0
- package/src/wiki-ingest.js +215 -0
- package/src/wiki-state.js +228 -0
- package/views/pages.html +2 -2
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
import { errorMessage } from "./util.js"
|
|
2
|
+
|
|
3
|
+
/** How long a write waits for the rest of its burst before one ingest starts. */
|
|
4
|
+
export const WIKI_INGEST_DELAY_MS = 3000
|
|
5
|
+
/** The longest wait between attempts while a wiki is ingesting another write. */
|
|
6
|
+
const RETRY_MAX_MS = 60_000
|
|
7
|
+
/** Attempts a queued write makes on an error that is not "the wiki is ingesting". */
|
|
8
|
+
const WRITE_ATTEMPTS = 3
|
|
9
|
+
|
|
10
|
+
/** True when the hub refused because a wiki is ingesting (409 "wiki is processing"). */
|
|
11
|
+
export function isWikiBusy(error) {
|
|
12
|
+
return Number(error?.status) === 409 || /wiki is processing/i.test(String(error?.what ?? errorMessage(error)))
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Automatic ingest (jelly #89 A3): a write is never left unread, and the agent
|
|
17
|
+
* never has to pass `ingest: true`.
|
|
18
|
+
*
|
|
19
|
+
* - **Debounced.** Each successful write restarts one timer for its wiki
|
|
20
|
+
* (`delay`, 3 seconds by default); a burst of writes makes one ingest. Three
|
|
21
|
+
* seconds is longer than the gap between the calls of one writing burst and
|
|
22
|
+
* short next to an ingest, which takes minutes.
|
|
23
|
+
* - **Queued, not refused.** The hub refuses a write while it ingests (409
|
|
24
|
+
* "wiki is processing"), so such a write is held here in order, written when
|
|
25
|
+
* the ingest finishes, and one ingest follows it. A queued write lives in
|
|
26
|
+
* this process only: it does not survive a restart of the engine.
|
|
27
|
+
* - **One ingest per wiki at a time here.** All of a wiki's queue work is
|
|
28
|
+
* serialised (`running`), and the hub's 409 is the lock across engines.
|
|
29
|
+
*
|
|
30
|
+
* Plain OpenCode and Jelly run the same code: nothing here knows the host.
|
|
31
|
+
*
|
|
32
|
+
* @param {{
|
|
33
|
+
* backend: any,
|
|
34
|
+
* log?: any,
|
|
35
|
+
* delay?: number,
|
|
36
|
+
* rememberError?: (wikiID: string, message: string) => void,
|
|
37
|
+
* clearError?: (wikiID: string) => void,
|
|
38
|
+
* rememberPending?: (wikiID: string, scope: object) => void,
|
|
39
|
+
* clearPending?: (wikiID: string) => void,
|
|
40
|
+
* pending?: Array<{ wikiID: string, scope: object }> | (() => Array<{ wikiID: string, scope: object }>),
|
|
41
|
+
* }} input
|
|
42
|
+
*/
|
|
43
|
+
export function createWikiIngest({
|
|
44
|
+
backend,
|
|
45
|
+
log = null,
|
|
46
|
+
delay = WIKI_INGEST_DELAY_MS,
|
|
47
|
+
rememberError = () => {},
|
|
48
|
+
clearError = () => {},
|
|
49
|
+
rememberPending = () => {},
|
|
50
|
+
clearPending = () => {},
|
|
51
|
+
pending = [],
|
|
52
|
+
} = {}) {
|
|
53
|
+
/** `${team}/${space}/${user}/${wiki}` -> one queue, so one ingest at a time. */
|
|
54
|
+
const queues = new Map()
|
|
55
|
+
let stopped = false
|
|
56
|
+
let startTimer = null
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* A write whose debounce an earlier engine start did not finish: its source
|
|
60
|
+
* file is durable, and this start runs the ingest it was waiting for. Read
|
|
61
|
+
* after setup returns, so a start never waits on the state file.
|
|
62
|
+
*/
|
|
63
|
+
function recover() {
|
|
64
|
+
startTimer = null
|
|
65
|
+
if (stopped) return
|
|
66
|
+
const waiting = typeof pending === "function" ? pending() : pending
|
|
67
|
+
for (const entry of waiting ?? []) {
|
|
68
|
+
if (!entry?.wikiID || !entry?.scope) continue
|
|
69
|
+
const queue = queueFor(entry.scope, entry.wikiID)
|
|
70
|
+
queue.dirty = true
|
|
71
|
+
arm(queue, delay)
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
startTimer = setTimeout(recover, 0)
|
|
75
|
+
startTimer?.unref?.()
|
|
76
|
+
|
|
77
|
+
function queueFor(scope, wikiID) {
|
|
78
|
+
const key = [scope?.team_id ?? "", scope?.space_id ?? "", scope?.user_id ?? "", wikiID].join("\u0000")
|
|
79
|
+
let queue = queues.get(key)
|
|
80
|
+
if (!queue) {
|
|
81
|
+
queue = { scope, wikiID, timer: null, running: false, dirty: false, writes: [], failures: 0 }
|
|
82
|
+
queues.set(key, queue)
|
|
83
|
+
}
|
|
84
|
+
return queue
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function arm(queue, ms) {
|
|
88
|
+
if (stopped) return
|
|
89
|
+
if (queue.timer) clearTimeout(queue.timer)
|
|
90
|
+
queue.timer = setTimeout(() => {
|
|
91
|
+
queue.timer = null
|
|
92
|
+
flush(queue).catch((error) => log?.warn?.("wiki ingest queue failed", error))
|
|
93
|
+
}, Math.max(10, ms))
|
|
94
|
+
// As the collector's timers: never hold the host open.
|
|
95
|
+
queue.timer?.unref?.()
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/** The wait before another attempt, doubling while the wiki stays busy. */
|
|
99
|
+
function retryDelay(queue) {
|
|
100
|
+
const factor = 2 ** Math.min(queue.failures, 6)
|
|
101
|
+
return Math.min(RETRY_MAX_MS, Math.max(delay, 100) * factor)
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Hold a write the hub refused because it is ingesting. It is written when
|
|
106
|
+
* the ingest finishes, in the order it arrived, and one ingest follows.
|
|
107
|
+
*/
|
|
108
|
+
function hold(queue, filename, content) {
|
|
109
|
+
queue.writes.push({ filename, content, attempts: 0 })
|
|
110
|
+
queue.failures += 1
|
|
111
|
+
arm(queue, retryDelay(queue))
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
async function flush(queue) {
|
|
115
|
+
if (stopped || queue.running) return
|
|
116
|
+
queue.running = true
|
|
117
|
+
let landed = false
|
|
118
|
+
try {
|
|
119
|
+
while (queue.writes.length > 0) {
|
|
120
|
+
const next = queue.writes[0]
|
|
121
|
+
try {
|
|
122
|
+
await backend.writeWikiRaw({ scope: queue.scope, wikiID: queue.wikiID, files: [{ filename: next.filename, content: next.content }] })
|
|
123
|
+
} catch (error) {
|
|
124
|
+
if (isWikiBusy(error)) {
|
|
125
|
+
queue.dirty = true
|
|
126
|
+
queue.failures += 1
|
|
127
|
+
return
|
|
128
|
+
}
|
|
129
|
+
next.attempts += 1
|
|
130
|
+
if (next.attempts < WRITE_ATTEMPTS) {
|
|
131
|
+
queue.failures += 1
|
|
132
|
+
return
|
|
133
|
+
}
|
|
134
|
+
// The same refusal three times is not going to pass: say so, and do
|
|
135
|
+
// not repeat it forever.
|
|
136
|
+
queue.writes.shift()
|
|
137
|
+
rememberError(queue.wikiID, `A queued write to ${next.filename} failed: ${errorMessage(error)}`)
|
|
138
|
+
log?.warn?.("a queued wiki write was dropped", { wikiID: queue.wikiID, filename: next.filename, error: errorMessage(error) })
|
|
139
|
+
continue
|
|
140
|
+
}
|
|
141
|
+
queue.writes.shift()
|
|
142
|
+
queue.dirty = true
|
|
143
|
+
landed = true
|
|
144
|
+
}
|
|
145
|
+
// One ingest, only once every held write has landed: it covers them all.
|
|
146
|
+
if (queue.dirty) {
|
|
147
|
+
queue.dirty = false
|
|
148
|
+
try {
|
|
149
|
+
const answer = await backend.ingestWiki({ scope: queue.scope, wikiID: queue.wikiID })
|
|
150
|
+
clearPending(queue.wikiID)
|
|
151
|
+
if (String(answer?.status ?? "") === "failed") rememberError(queue.wikiID, "The wiki's last ingest failed.")
|
|
152
|
+
else clearError(queue.wikiID)
|
|
153
|
+
} catch (error) {
|
|
154
|
+
if (isWikiBusy(error)) {
|
|
155
|
+
queue.dirty = true
|
|
156
|
+
queue.failures += 1
|
|
157
|
+
return
|
|
158
|
+
}
|
|
159
|
+
rememberError(queue.wikiID, `The wiki's ingest did not start: ${errorMessage(error)}`)
|
|
160
|
+
log?.warn?.("the automatic wiki ingest failed", { wikiID: queue.wikiID, error: errorMessage(error) })
|
|
161
|
+
return
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
queue.failures = 0
|
|
165
|
+
if (landed) clearError(queue.wikiID)
|
|
166
|
+
} finally {
|
|
167
|
+
queue.running = false
|
|
168
|
+
if (!stopped && (queue.writes.length > 0 || queue.dirty)) arm(queue, retryDelay(queue))
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
return {
|
|
173
|
+
/**
|
|
174
|
+
* Write one source file now, or hold it when the wiki is ingesting. The
|
|
175
|
+
* ingest that reads it starts on its own after the last write of the burst.
|
|
176
|
+
*
|
|
177
|
+
* @returns {Promise<{ queued: boolean }>}
|
|
178
|
+
*/
|
|
179
|
+
async write({ scope, wikiID, filename, content }) {
|
|
180
|
+
const queue = queueFor(scope, wikiID)
|
|
181
|
+
try {
|
|
182
|
+
await backend.writeWikiRaw({ scope, wikiID, files: [{ filename, content }] })
|
|
183
|
+
} catch (error) {
|
|
184
|
+
if (!isWikiBusy(error)) throw error
|
|
185
|
+
hold(queue, filename, content)
|
|
186
|
+
return { queued: true }
|
|
187
|
+
}
|
|
188
|
+
queue.dirty = true
|
|
189
|
+
queue.failures = 0
|
|
190
|
+
clearError(wikiID)
|
|
191
|
+
rememberPending(wikiID, scope)
|
|
192
|
+
arm(queue, delay)
|
|
193
|
+
return { queued: false }
|
|
194
|
+
},
|
|
195
|
+
|
|
196
|
+
/** How many writes wait for their wiki to finish ingesting (tests, health). */
|
|
197
|
+
held(wikiID) {
|
|
198
|
+
let count = 0
|
|
199
|
+
for (const queue of queues.values()) if (queue.wikiID === wikiID) count += queue.writes.length
|
|
200
|
+
return count
|
|
201
|
+
},
|
|
202
|
+
|
|
203
|
+
/** Stop every timer; a held write is lost with the process, as it always would be. */
|
|
204
|
+
stop() {
|
|
205
|
+
stopped = true
|
|
206
|
+
if (startTimer) clearTimeout(startTimer)
|
|
207
|
+
startTimer = null
|
|
208
|
+
for (const queue of queues.values()) {
|
|
209
|
+
if (queue.timer) clearTimeout(queue.timer)
|
|
210
|
+
queue.timer = null
|
|
211
|
+
}
|
|
212
|
+
queues.clear()
|
|
213
|
+
},
|
|
214
|
+
}
|
|
215
|
+
}
|
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
import fs from "node:fs"
|
|
2
|
+
import path from "node:path"
|
|
3
|
+
import { oneLine } from "./util.js"
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* A wiki's state in the declared reads (jelly #89 A1) and the last ready page
|
|
7
|
+
* set kept while an ingest runs (A2).
|
|
8
|
+
*
|
|
9
|
+
* The hub lists no pages while a wiki ingests (`/v3/wiki/page/ls` answers []
|
|
10
|
+
* unless the wiki is ready), so the 129 pages a person already had vanished
|
|
11
|
+
* from `wiki.pages` for the minutes an ingest took. The set a ready listing
|
|
12
|
+
* returned is kept here, in this process and best-effort on disk, and a
|
|
13
|
+
* listing that comes back empty because of an ingest answers that set,
|
|
14
|
+
* marked stale. Nothing here detects a host: plain OpenCode keeps working.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
/** The three states the declared read carries; any other hub status reads as processing. */
|
|
18
|
+
export const WIKI_STATES = Object.freeze(["processing", "ready", "failed"])
|
|
19
|
+
|
|
20
|
+
/** The most an `error` says, as jelly's ask declares. */
|
|
21
|
+
export const WIKI_ERROR_CHARS = 400
|
|
22
|
+
|
|
23
|
+
/** The page sets kept: one per wiki, the least recently written falls out. */
|
|
24
|
+
const MAX_WIKIS = 100
|
|
25
|
+
/** Pages kept per wiki; the view lists at most this many anyway. */
|
|
26
|
+
const MAX_PAGES = 100
|
|
27
|
+
/** The file's name in the state directory. */
|
|
28
|
+
const FILE_NAME = "wiki-pages.json"
|
|
29
|
+
|
|
30
|
+
const READY = /^(?:ready|complete|completed|succeeded|indexed|done)$/
|
|
31
|
+
const FAILED = /(?:fail|error)/
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* The state the declared reads carry for one hub wiki item: `ready` only when
|
|
35
|
+
* the hub says so, `failed` when it says the ingest failed, and `processing`
|
|
36
|
+
* for everything else (draft, processing, queued, unknown), because nothing
|
|
37
|
+
* else means "readable as ready".
|
|
38
|
+
*
|
|
39
|
+
* @param {any} item a `/v3/wiki/list` or `/v3/wiki/get` item
|
|
40
|
+
* @returns {"processing" | "ready" | "failed"}
|
|
41
|
+
*/
|
|
42
|
+
export function wikiStateOf(item) {
|
|
43
|
+
const raw = String(item?.status ?? "").trim().toLowerCase()
|
|
44
|
+
if (FAILED.test(raw)) return "failed"
|
|
45
|
+
if (READY.test(raw)) return "ready"
|
|
46
|
+
return "processing"
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** The words a failed wiki carries, from the hub or from the last ingest this process saw. */
|
|
50
|
+
export function wikiErrorOf(item, remembered = "", max = WIKI_ERROR_CHARS) {
|
|
51
|
+
if (wikiStateOf(item) !== "failed") return ""
|
|
52
|
+
const given = [item?.error, item?.ingest_error, item?.sync_error, item?.last_error].find(
|
|
53
|
+
(value) => typeof value === "string" && value.trim(),
|
|
54
|
+
)
|
|
55
|
+
const words = String(given ?? remembered ?? "").trim() || "The wiki's last ingest failed. Write a page again to start another ingest."
|
|
56
|
+
return oneLine(words, max)
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* The ready page sets of the wikis this engine has listed, and the last ingest
|
|
61
|
+
* error of each. Best-effort on disk (`wiki-pages.json` in the state
|
|
62
|
+
* directory), so a restart in the middle of an ingest still answers the set
|
|
63
|
+
* the person had; a file that cannot be read or written is ignored, and the
|
|
64
|
+
* process keeps its own copy.
|
|
65
|
+
*
|
|
66
|
+
* @param {{ stateDir?: string, log?: any }} [options]
|
|
67
|
+
*/
|
|
68
|
+
export function createWikiCache({ stateDir = "", log = null } = {}) {
|
|
69
|
+
const file = stateDir ? path.join(stateDir, FILE_NAME) : ""
|
|
70
|
+
/** wiki id -> { pages: [{ref,title}], error: string, pending: scope|null } */
|
|
71
|
+
let entries = new Map()
|
|
72
|
+
let loaded = false
|
|
73
|
+
let writable = Boolean(file)
|
|
74
|
+
|
|
75
|
+
function load() {
|
|
76
|
+
if (loaded) return
|
|
77
|
+
loaded = true
|
|
78
|
+
if (!file) return
|
|
79
|
+
let raw
|
|
80
|
+
try {
|
|
81
|
+
raw = fs.readFileSync(file, "utf8")
|
|
82
|
+
} catch (error) {
|
|
83
|
+
if (error?.code !== "ENOENT") log?.debug?.("wiki page cache is unreadable; this process keeps its own copy", { error: String(error?.code ?? error) })
|
|
84
|
+
return
|
|
85
|
+
}
|
|
86
|
+
try {
|
|
87
|
+
const parsed = JSON.parse(raw)
|
|
88
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return
|
|
89
|
+
for (const [wikiID, entry] of Object.entries(parsed)) {
|
|
90
|
+
const pages = Array.isArray(entry?.pages)
|
|
91
|
+
? entry.pages
|
|
92
|
+
.filter((page) => page && typeof page.ref === "string" && page.ref)
|
|
93
|
+
.slice(0, MAX_PAGES)
|
|
94
|
+
.map((page) => ({ ref: page.ref, title: String(page.title ?? page.ref).slice(0, 200) }))
|
|
95
|
+
: []
|
|
96
|
+
const error = typeof entry?.error === "string" ? oneLine(entry.error, WIKI_ERROR_CHARS) : ""
|
|
97
|
+
const pending = entry?.pending && typeof entry.pending === "object" ? scopeOf(entry.pending) : null
|
|
98
|
+
if (pages.length || error || pending) entries.set(String(wikiID), { pages, error, pending })
|
|
99
|
+
}
|
|
100
|
+
} catch {
|
|
101
|
+
/* a corrupt cache is no cache: the next ready listing writes a new one */
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function save() {
|
|
106
|
+
if (!writable || !file) return
|
|
107
|
+
try {
|
|
108
|
+
const out = {}
|
|
109
|
+
for (const [wikiID, entry] of entries) {
|
|
110
|
+
const saved = {}
|
|
111
|
+
if (entry.pages.length) saved.pages = entry.pages
|
|
112
|
+
else if (entry.error) saved.error = entry.error
|
|
113
|
+
if (entry.pending) saved.pending = entry.pending
|
|
114
|
+
if (saved.pages || saved.error || saved.pending) out[wikiID] = saved
|
|
115
|
+
}
|
|
116
|
+
fs.mkdirSync(path.dirname(file), { recursive: true })
|
|
117
|
+
fs.writeFileSync(file, JSON.stringify(out))
|
|
118
|
+
} catch (error) {
|
|
119
|
+
// A read must never fail for a cache: keep this process's copy only.
|
|
120
|
+
writable = false
|
|
121
|
+
log?.debug?.("wiki page cache is not writable; this process keeps its own copy", { error: String(error?.code ?? error) })
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function touch(wikiID) {
|
|
126
|
+
load()
|
|
127
|
+
const id = String(wikiID ?? "")
|
|
128
|
+
if (!id) return null
|
|
129
|
+
const entry = entries.get(id) ?? { pages: [], error: "", pending: null }
|
|
130
|
+
entries.delete(id)
|
|
131
|
+
entries.set(id, entry)
|
|
132
|
+
while (entries.size > MAX_WIKIS) entries.delete(entries.keys().next().value)
|
|
133
|
+
return entry
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
return {
|
|
137
|
+
/** Keep the pages a ready listing returned; [] is never recorded (it means an ingest, not emptiness). */
|
|
138
|
+
rememberPages(wikiID, pages) {
|
|
139
|
+
if (!Array.isArray(pages) || pages.length === 0) return
|
|
140
|
+
const entry = touch(wikiID)
|
|
141
|
+
entry.pages = pages.slice(0, MAX_PAGES).map((page) => ({ ref: String(page.ref), title: String(page.title ?? page.ref).slice(0, 200) }))
|
|
142
|
+
entry.error = ""
|
|
143
|
+
save()
|
|
144
|
+
},
|
|
145
|
+
/** The last ready set of a wiki, or [] when this engine never listed it. */
|
|
146
|
+
pages(wikiID) {
|
|
147
|
+
const entry = touch(wikiID)
|
|
148
|
+
return entry ? entry.pages.map((page) => ({ ...page })) : []
|
|
149
|
+
},
|
|
150
|
+
/** Remember why an ingest failed, for the `error` a failed wiki carries. */
|
|
151
|
+
rememberError(wikiID, message) {
|
|
152
|
+
const entry = touch(wikiID)
|
|
153
|
+
if (!entry) return
|
|
154
|
+
entry.error = oneLine(String(message ?? "").trim(), WIKI_ERROR_CHARS)
|
|
155
|
+
if (!entry.error) return
|
|
156
|
+
save()
|
|
157
|
+
},
|
|
158
|
+
/** The last ingest error recorded for a wiki, or "". */
|
|
159
|
+
error(wikiID) {
|
|
160
|
+
const entry = touch(wikiID)
|
|
161
|
+
return entry?.error ?? ""
|
|
162
|
+
},
|
|
163
|
+
/** Forget a wiki's error (its own ready listing says it is healthy again). */
|
|
164
|
+
clearError(wikiID) {
|
|
165
|
+
const entry = touch(wikiID)
|
|
166
|
+
if (!entry || !entry.error) return
|
|
167
|
+
entry.error = ""
|
|
168
|
+
save()
|
|
169
|
+
},
|
|
170
|
+
/**
|
|
171
|
+
* Note that a write waits for its ingest (jelly #89 A3), so an engine that
|
|
172
|
+
* stops inside the debounce still ingests on its next start. Only the
|
|
173
|
+
* scope's ids are kept, never the text written.
|
|
174
|
+
*/
|
|
175
|
+
rememberPending(wikiID, scope) {
|
|
176
|
+
const entry = touch(wikiID)
|
|
177
|
+
if (!entry) return
|
|
178
|
+
const kept = scopeOf(scope)
|
|
179
|
+
if (!kept) return
|
|
180
|
+
entry.pending = kept
|
|
181
|
+
save()
|
|
182
|
+
},
|
|
183
|
+
/** The ingest started: nothing waits for it any more. */
|
|
184
|
+
clearPending(wikiID) {
|
|
185
|
+
const entry = touch(wikiID)
|
|
186
|
+
if (!entry?.pending) return
|
|
187
|
+
entry.pending = null
|
|
188
|
+
save()
|
|
189
|
+
},
|
|
190
|
+
/** Every wiki whose write still waits for an ingest, as `{ wikiID, scope }`. */
|
|
191
|
+
pending() {
|
|
192
|
+
load()
|
|
193
|
+
const list = []
|
|
194
|
+
for (const [wikiID, entry] of entries) if (entry.pending) list.push({ wikiID, scope: { ...entry.pending } })
|
|
195
|
+
return list
|
|
196
|
+
},
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/** The ids of a scope, as the cache keeps them: never a session or any text. */
|
|
201
|
+
function scopeOf(scope) {
|
|
202
|
+
if (!scope || typeof scope !== "object") return null
|
|
203
|
+
const kept = {}
|
|
204
|
+
for (const key of ["team_id", "space_id", "user_id", "agent_id"]) {
|
|
205
|
+
const value = String(scope[key] ?? "").trim()
|
|
206
|
+
if (value) kept[key] = value
|
|
207
|
+
}
|
|
208
|
+
return Object.keys(kept).length > 0 ? kept : null
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/** One cache per state directory: the views and the tools read the same wiki's set. */
|
|
212
|
+
const sharedCaches = new Map()
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* The wiki cache of one engine's state directory: both the views and the tools
|
|
216
|
+
* run in that engine, with the same directory.
|
|
217
|
+
*
|
|
218
|
+
* @param {{ stateDir?: string, log?: any }} [options]
|
|
219
|
+
*/
|
|
220
|
+
export function wikiCache(options = {}) {
|
|
221
|
+
const key = String(options.stateDir ?? "")
|
|
222
|
+
let cache = sharedCaches.get(key)
|
|
223
|
+
if (!cache) {
|
|
224
|
+
cache = createWikiCache(options)
|
|
225
|
+
sharedCaches.set(key, cache)
|
|
226
|
+
}
|
|
227
|
+
return cache
|
|
228
|
+
}
|
package/views/pages.html
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
<html lang="en-GB">
|
|
3
3
|
<head>
|
|
4
4
|
<meta charset="utf-8">
|
|
5
|
-
<meta http-equiv="Content-Security-Policy" content="default-src 'none'; script-src 'sha256-5dtPKkjegOPG8bawdSzGBjixnIaSCLYcZOUAwbRWFAc=' 'sha256-/0aqZ5aL9ON4+6IAVSSwbjH6b/AakzUoE2N5RwMn9BM=' 'sha256-
|
|
5
|
+
<meta http-equiv="Content-Security-Policy" content="default-src 'none'; script-src 'sha256-5dtPKkjegOPG8bawdSzGBjixnIaSCLYcZOUAwbRWFAc=' 'sha256-/0aqZ5aL9ON4+6IAVSSwbjH6b/AakzUoE2N5RwMn9BM=' 'sha256-jqMVE0B1Xg8TD+ZhKc6LhMN5HHe1r/kmn3M7tEmufZE='; style-src 'unsafe-inline'; base-uri 'none'; form-action 'none'">
|
|
6
6
|
<title>Pages</title>
|
|
7
7
|
<style>
|
|
8
8
|
:root {
|
|
@@ -620,7 +620,7 @@ async function drawPage(read, folder) {
|
|
|
620
620
|
if (record.clipped) nodes.push(el("p", "muted", "This page is longer than a view shows: ask your agent to read the rest."));
|
|
621
621
|
nodes.push(
|
|
622
622
|
linkList("Links to", record.outlinks, folder, "This page links to nothing."),
|
|
623
|
-
linkList(`Linked from${record.backlinksMore ? " (the newest)" : ""}`, record.backlinks, folder, "Nothing links here yet."),
|
|
623
|
+
linkList(`Linked from${record.backlinksMore ? " (the newest)" : ""}`, record.backlinks, folder, record.backlinksPartial ? "Nothing you can see links here." : "Nothing links here yet."),
|
|
624
624
|
history,
|
|
625
625
|
);
|
|
626
626
|
show(...nodes);
|