@joe-broadhead/opencode-mem 0.11.3 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/docs/agent-instructions.md +3 -2
- package/docs/agent-protocol.md +6 -5
- package/package.json +264 -2
- package/src/call-shapes.js +0 -4
- package/src/protocol.js +20 -11
- package/src/tools/commitments.js +1 -1
- package/src/tools/index.js +11 -18
- package/src/tools/page.js +40 -14
- package/src/tools/recall.js +1 -1
- package/src/tools/share.js +1 -1
- package/src/tools/wiki.js +13 -3
- package/src/views/contracts.js +29 -0
- package/src/views/receipt.js +28 -11
- package/src/views/records-reads.js +102 -0
- package/src/wiki-ingest.js +70 -15
- package/src/wiki-queue.js +252 -0
- package/views/remembered.html +7 -4
package/src/tools/wiki.js
CHANGED
|
@@ -3,6 +3,7 @@ import { RECORDS_FILE, RECORDS_WIKI, isRecordsText } from "../records.js"
|
|
|
3
3
|
import { codedMessage } from "../errors.js"
|
|
4
4
|
import { errorMessage, neutralizeFenceClosers, oneLine, redactSecrets } from "../util.js"
|
|
5
5
|
import { createWikiIngest, WIKI_INGEST_DELAY_MS } from "../wiki-ingest.js"
|
|
6
|
+
import { createWikiQueue } from "../wiki-queue.js"
|
|
6
7
|
import { wikiCache, wikiErrorOf, wikiStateOf } from "../wiki-state.js"
|
|
7
8
|
import { discoverKnowledge, isRecordsWikiName, searchKnowledge } from "./knowledge.js"
|
|
8
9
|
import { failedAnswer, refusedAnswer, wikiPlace, withWriteState } from "../views/write-state.js"
|
|
@@ -103,6 +104,10 @@ export function registerWiki(editor, toolkit) {
|
|
|
103
104
|
backend,
|
|
104
105
|
log,
|
|
105
106
|
delay: config.wikiIngestMs ?? WIKI_INGEST_DELAY_MS,
|
|
107
|
+
// Durable before the answer says queued (jelly #89, 0.11.4): the entry
|
|
108
|
+
// outlives the engine, and the file holds the write's body, owner-only,
|
|
109
|
+
// inside Memory's state directory.
|
|
110
|
+
queue: createWikiQueue({ stateDir: config.stateDir, log }),
|
|
106
111
|
rememberError: (wikiID, message) => cache.rememberError(wikiID, message),
|
|
107
112
|
clearError: (wikiID) => cache.clearError(wikiID),
|
|
108
113
|
rememberPending: (wikiID, scope) => cache.rememberPending(wikiID, scope),
|
|
@@ -295,7 +300,7 @@ export function registerWiki(editor, toolkit) {
|
|
|
295
300
|
*/
|
|
296
301
|
function ingestNote(written) {
|
|
297
302
|
if (written.queued) {
|
|
298
|
-
return "the wiki is ingesting an earlier write, so this write is
|
|
303
|
+
return "the wiki is ingesting an earlier write, so this write is written after that ingest finishes, even after a restart"
|
|
299
304
|
}
|
|
300
305
|
return "ingestion starts on its own a moment after the last write, so a burst of writes makes one ingest; search finds the changes once it finishes"
|
|
301
306
|
}
|
|
@@ -650,7 +655,7 @@ export function registerWiki(editor, toolkit) {
|
|
|
650
655
|
const wiki = {
|
|
651
656
|
name: "wiki",
|
|
652
657
|
description:
|
|
653
|
-
"
|
|
658
|
+
"Documents: uploads, repositories and their pages. Person, project, decision and runbook pages are mem_page, not here. Call list, search and read for documentation, architecture, ownership and call-graph questions about what was uploaded or indexed; for a named document read it or search to find it, even if mem_recall might answer. Write only documents the person gave you, as flat source files (\"notes.md\"; a name with a folder is refused). A space has a knowledge wiki only once action create makes one (name it \"My knowledge\" in the personal space); list shows which wikis exist. A write starts one ingest by itself a moment after the last one; a write during an ingest is queued, not refused. Like mem_recall it uses the session's space by default: your personal space, or a shared space bound to the session. A write to a shared space asks the person and shows the file and an excerpt; a write to your personal space does not. delete removes one source file (filename), one generated page (ref), or a whole wiki (wiki_id alone): without confirm it only previews; with confirm: true it always asks the person, then deletes and reads it back. Never put things to do (mem_commitments) or where a project stands here.",
|
|
654
659
|
input: {
|
|
655
660
|
type: "object",
|
|
656
661
|
properties: {
|
|
@@ -812,7 +817,12 @@ export function registerWiki(editor, toolkit) {
|
|
|
812
817
|
}
|
|
813
818
|
// Say what the write guarantees, so the agent does not read its own
|
|
814
819
|
// write back to check it: the source file now holds exactly this.
|
|
815
|
-
// A queued write guarantees nothing yet: the wiki is ingesting
|
|
820
|
+
// A queued write guarantees nothing yet: the wiki is ingesting, and
|
|
821
|
+
// it is on disk until the hub accepts it. A queue the disk refused
|
|
822
|
+
// saved nothing, so the answer is `refused` (jelly #89, 0.11.4).
|
|
823
|
+
if (written.refused) {
|
|
824
|
+
return withWriteState(text(`${note}not written: the wiki is busy, nothing was saved, try again in about a minute`), { state: "refused" })
|
|
825
|
+
}
|
|
816
826
|
if (written.queued) {
|
|
817
827
|
const what = action === "append" ? "the append to" : ""
|
|
818
828
|
return withWriteState(
|
package/src/views/contracts.js
CHANGED
|
@@ -562,6 +562,28 @@ const recentOutput = {
|
|
|
562
562
|
},
|
|
563
563
|
}
|
|
564
564
|
|
|
565
|
+
/**
|
|
566
|
+
* `memories.list` (0.12.1, jelly's #89 ask B7): the same rows as
|
|
567
|
+
* `memories.recent`, paged by this read's own opaque cursor so the list never
|
|
568
|
+
* stops at 100. The row schema is recent's, unchanged; the answer carries the
|
|
569
|
+
* cursor of the next page when there is one.
|
|
570
|
+
*/
|
|
571
|
+
const memoriesListOutput = {
|
|
572
|
+
...recentOutput,
|
|
573
|
+
properties: { ...recentOutput.properties, cursor: line(CURSOR_CHARS) },
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
const memoriesListInput = viewInput({
|
|
577
|
+
/** A space from `spaces.list`; omitted, every space the person can read. */
|
|
578
|
+
space: id,
|
|
579
|
+
/** Only what this agent wrote, as the records service names it; a trailing `*` matches every agent that starts with it. */
|
|
580
|
+
agent: line(200),
|
|
581
|
+
/** Only what was written for this task key ("checkout-v3"). */
|
|
582
|
+
task: line(200),
|
|
583
|
+
/** The `cursor` a previous page answered, to read on from the row after its last; omitted, the top. */
|
|
584
|
+
cursor: line(CURSOR_CHARS),
|
|
585
|
+
})
|
|
586
|
+
|
|
565
587
|
const wikiListOutput = {
|
|
566
588
|
type: "object",
|
|
567
589
|
additionalProperties: false,
|
|
@@ -1659,6 +1681,13 @@ export const VIEW_METHODS = Object.freeze([
|
|
|
1659
1681
|
input: recentInput,
|
|
1660
1682
|
output: recentOutput,
|
|
1661
1683
|
},
|
|
1684
|
+
{
|
|
1685
|
+
name: "memories.list",
|
|
1686
|
+
label: "list your memories, page by page",
|
|
1687
|
+
permission: "mem_recall",
|
|
1688
|
+
input: memoriesListInput,
|
|
1689
|
+
output: memoriesListOutput,
|
|
1690
|
+
},
|
|
1662
1691
|
{ name: "wiki.list", label: "list your wikis", permission: "mem_wiki", input: empty, output: wikiListOutput },
|
|
1663
1692
|
{
|
|
1664
1693
|
name: "wiki.pages",
|
package/src/views/receipt.js
CHANGED
|
@@ -19,11 +19,14 @@ import { idOf, SPACE_KINDS } from "./shape.js"
|
|
|
19
19
|
* or changed one record, so its state is always `stored`.
|
|
20
20
|
*/
|
|
21
21
|
|
|
22
|
-
/**
|
|
23
|
-
|
|
22
|
+
/**
|
|
23
|
+
* The writes that answer a receipt, in the view's own words. `written` is any
|
|
24
|
+
* `mem_page` write that leaves a page to open (0.12.1, jelly's #89 ask B7).
|
|
25
|
+
*/
|
|
26
|
+
export const RECEIPT_ACTIONS = Object.freeze(["remembered", "corrected", "confirmed", "retired", "restored", "committed", "written"])
|
|
24
27
|
|
|
25
|
-
/** The tool verb of each receipt (`mem_remember`, `mem_revise`'s actions, `mem_commitments add`). */
|
|
26
|
-
const VERB_ACTIONS = Object.freeze({ remember: "remembered", correct: "corrected", confirm: "confirmed", retire: "retired", restore: "restored", commit: "committed" })
|
|
28
|
+
/** The tool verb of each receipt (`mem_remember`, `mem_revise`'s actions, `mem_commitments add`, `mem_page`'s writes). */
|
|
29
|
+
const VERB_ACTIONS = Object.freeze({ remember: "remembered", correct: "corrected", confirm: "confirmed", retire: "retired", restore: "restored", commit: "committed", write: "written" })
|
|
27
30
|
|
|
28
31
|
/**
|
|
29
32
|
* The receipt for one write, named by the tool's own verb: the tools call this
|
|
@@ -43,13 +46,19 @@ export const RECEIPT_TITLE_CHARS = 200
|
|
|
43
46
|
/** Characters of a receipt's text: enough for the card, not the whole page. */
|
|
44
47
|
export const RECEIPT_TEXT_CHARS = 2000
|
|
45
48
|
|
|
46
|
-
/**
|
|
47
|
-
|
|
49
|
+
/**
|
|
50
|
+
* A receipt's place, as the write's first line carries it (`link`) and as the
|
|
51
|
+
* tab jelly opens (`place`): one of `memory:<id>`, `commitment:<id>`,
|
|
52
|
+
* `page:<path>` or `wiki:<wiki id>/<filename>`, the four kinds 0.11.3 declares.
|
|
53
|
+
*/
|
|
54
|
+
function placesOf(record) {
|
|
48
55
|
const id = line(record?.id, 240)
|
|
49
56
|
const kind = line(record?.kind, 20)
|
|
50
|
-
|
|
51
|
-
if (kind === "
|
|
52
|
-
return
|
|
57
|
+
const places = (name, address) => ({ link: `${name}:${address}`, place: { kind: name, id: address } })
|
|
58
|
+
if (kind === "page") return places("page", line(record?.path, 240) || id)
|
|
59
|
+
if (kind === "commitment") return places("commitment", id)
|
|
60
|
+
if (kind === "wiki") return places("wiki", line(record?.path ?? record?.filename, 240) || id)
|
|
61
|
+
return places("memory", id)
|
|
53
62
|
}
|
|
54
63
|
|
|
55
64
|
/** One line, at most `max` characters, with no control character left in it. */
|
|
@@ -65,7 +74,7 @@ const line = (value, max) =>
|
|
|
65
74
|
* report as saved (no record id, or an action the view does not know).
|
|
66
75
|
*
|
|
67
76
|
* @param {{ action?: string, record?: any, space?: { id?: string, name?: string, kind?: string } }} input
|
|
68
|
-
* @returns {{ v: 1, action: string, id: string, kind: string, type: string, status: string, state: string, link: string, space: { id: string, name: string, kind: string }, title: string } | undefined}
|
|
77
|
+
* @returns {{ v: 1, action: string, id: string, kind: string, type: string, status: string, state: string, link: string, place: { kind: string, id: string }, version: number, path?: string, space: { id: string, name: string, kind: string }, title: string } | undefined}
|
|
69
78
|
*/
|
|
70
79
|
export function receiptView({ action, record, space } = {}) {
|
|
71
80
|
if (!RECEIPT_ACTIONS.includes(action)) return undefined
|
|
@@ -75,6 +84,8 @@ export function receiptView({ action, record, space } = {}) {
|
|
|
75
84
|
? record.body
|
|
76
85
|
: typeof record?.fields?.what === "string" ? record.fields.what : typeof record?.what === "string" ? record.what : ""
|
|
77
86
|
const text = fullText.replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/g, " ").slice(0, RECEIPT_TEXT_CHARS)
|
|
87
|
+
const places = placesOf(record)
|
|
88
|
+
const path = line(record?.path, 240)
|
|
78
89
|
return {
|
|
79
90
|
v: 1,
|
|
80
91
|
action,
|
|
@@ -84,7 +95,13 @@ export function receiptView({ action, record, space } = {}) {
|
|
|
84
95
|
status: line(record?.status, 20),
|
|
85
96
|
// A receipt is a saved record: the same state and place a first line names.
|
|
86
97
|
state: "stored",
|
|
87
|
-
link:
|
|
98
|
+
link: places.link,
|
|
99
|
+
/** The tab and id the receipt card opens (jelly's #89 ask B7). */
|
|
100
|
+
place: places.place,
|
|
101
|
+
/** The record's version; 0 when the store gave none. */
|
|
102
|
+
version: Number.isSafeInteger(record?.version) ? record.version : 0,
|
|
103
|
+
/** The page's path, when the record is a page. */
|
|
104
|
+
...(path ? { path } : {}),
|
|
88
105
|
space: {
|
|
89
106
|
id: idOf(space?.id) || "unknown",
|
|
90
107
|
name: line(space?.name, 200),
|
|
@@ -382,6 +382,40 @@ function addCommunities(answer) {
|
|
|
382
382
|
for (const [at, index] of canonical.entries()) answer.nodes[index].community = groups[at]
|
|
383
383
|
}
|
|
384
384
|
|
|
385
|
+
/** The tag `memories.list`'s cursor carries, so no other read's cursor is taken for it. */
|
|
386
|
+
const MEMORIES_CURSOR = "memories.list"
|
|
387
|
+
|
|
388
|
+
/**
|
|
389
|
+
* The opaque cursor of `memories.list`: the last row shown (`at`, `id`) and how
|
|
390
|
+
* many rows at exactly that `at` this list has already shown. The count keeps
|
|
391
|
+
* the keyset exact when several memories share one millisecond: the next read
|
|
392
|
+
* asks past them in id order instead of skipping the tail of a tied group.
|
|
393
|
+
*/
|
|
394
|
+
export function encodeMemoriesCursor({ at, id, ties }) {
|
|
395
|
+
return Buffer.from(JSON.stringify([MEMORIES_CURSOR, at, id, ties]), "utf8").toString("base64url")
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
/** The cursor a previous page answered, or a refusal in plain words. */
|
|
399
|
+
function readMemoriesCursor(value) {
|
|
400
|
+
if (value === undefined || value === null || value === "") return null
|
|
401
|
+
const text = typeof value === "string" ? value.trim() : ""
|
|
402
|
+
const invalid = () => refuse("That is not a cursor this read gave out. Start again without a cursor.")
|
|
403
|
+
if (!text || text.length > CURSOR_CHARS) invalid()
|
|
404
|
+
let parts
|
|
405
|
+
try {
|
|
406
|
+
parts = JSON.parse(Buffer.from(text, "base64url").toString("utf8"))
|
|
407
|
+
} catch {
|
|
408
|
+
invalid()
|
|
409
|
+
}
|
|
410
|
+
if (!Array.isArray(parts) || parts[0] !== MEMORIES_CURSOR) invalid()
|
|
411
|
+
const [tag, at, id, ties] = parts
|
|
412
|
+
if (tag !== MEMORIES_CURSOR || typeof at !== "string" || !at || at.length > 40) invalid()
|
|
413
|
+
if (!idOf(id)) invalid()
|
|
414
|
+
if (!Number.isInteger(ties) || ties < 0 || ties > 1000) invalid()
|
|
415
|
+
if (!Number.isFinite(Date.parse(at))) invalid()
|
|
416
|
+
return { at, id, ties }
|
|
417
|
+
}
|
|
418
|
+
|
|
385
419
|
/**
|
|
386
420
|
* @param {{ runtime: any, personalScopes: () => Promise<Array<{ context: any }>>, available: () => Promise<void>, describe?: (error: unknown) => string }} input
|
|
387
421
|
* `describe` says a failed read in words (a space that could not be read is
|
|
@@ -720,6 +754,73 @@ export function createRecordsReads({ runtime, personalScopes, available, describ
|
|
|
720
754
|
}
|
|
721
755
|
}
|
|
722
756
|
|
|
757
|
+
/**
|
|
758
|
+
* Every memory the caller can read, newest first, paged by this read's own
|
|
759
|
+
* cursor (0.12.1, jelly's #89 ask B7): `memories.recent`'s rows, space by
|
|
760
|
+
* space, without its 100-row ceiling. The cursor is a keyset on the last row
|
|
761
|
+
* shown (`at`, then `id`), so a later page never repeats a row and never
|
|
762
|
+
* skips one, even when several memories share a millisecond. `memories.recent`
|
|
763
|
+
* is unchanged.
|
|
764
|
+
*/
|
|
765
|
+
async function memoriesList(input) {
|
|
766
|
+
await available()
|
|
767
|
+
const key = readMemoriesCursor(input?.cursor)
|
|
768
|
+
const filters = recentFilters(input)
|
|
769
|
+
const { targets, skipped } = await recentTargets(input)
|
|
770
|
+
// Each space answers its newest rows past the cursor: one page, the ties
|
|
771
|
+
// already shown, and one more row, which tells whether a further page exists.
|
|
772
|
+
const limit = Math.min(RECENT_PAGE + 1 + (key ? key.ties : 0), 200)
|
|
773
|
+
const results = await mapLimit(targets, RECENT_CONCURRENCY, async (entry) => {
|
|
774
|
+
const row = { id: entry.space.id, name: entry.space.name }
|
|
775
|
+
try {
|
|
776
|
+
const answer = await records(
|
|
777
|
+
"records.list",
|
|
778
|
+
{
|
|
779
|
+
kind: "memory",
|
|
780
|
+
limit,
|
|
781
|
+
...(key ? { updated_before: new Date(Date.parse(key.at) + 1).toISOString() } : {}),
|
|
782
|
+
...filters.body,
|
|
783
|
+
},
|
|
784
|
+
entry.where,
|
|
785
|
+
)
|
|
786
|
+
const rows = []
|
|
787
|
+
for (const record of answer?.records ?? []) {
|
|
788
|
+
if (!idOf(record?.id)) continue
|
|
789
|
+
const item = memoryRow(record, row)
|
|
790
|
+
if (key && !(item.at < key.at || (item.at === key.at && item.id < key.id))) continue
|
|
791
|
+
rows.push(item)
|
|
792
|
+
}
|
|
793
|
+
return { entry, rows }
|
|
794
|
+
} catch (error) {
|
|
795
|
+
runtime.log?.warn?.(`view memories.list: a read in space ${entry.space.id} failed`, error)
|
|
796
|
+
return { entry, error }
|
|
797
|
+
}
|
|
798
|
+
})
|
|
799
|
+
const failed = results.filter((result) => result.error !== undefined)
|
|
800
|
+
if (!results.some((result) => Array.isArray(result.rows))) throw failed[0].error
|
|
801
|
+
const rows = results.flatMap((result) => result.rows).sort((a, b) => b.at.localeCompare(a.at) || b.id.localeCompare(a.id))
|
|
802
|
+
const items = rows.slice(0, RECENT_PAGE)
|
|
803
|
+
const more = rows.length > RECENT_PAGE
|
|
804
|
+
const out = {
|
|
805
|
+
space: targets[0].space,
|
|
806
|
+
spaces: results.filter((result) => Array.isArray(result.rows)).map((result) => result.entry.space),
|
|
807
|
+
incomplete: [
|
|
808
|
+
...failed.map((result) => ({ id: result.entry.space.id, name: result.entry.space.name, reason: clip(describe(result.error), 200) })),
|
|
809
|
+
...skipped.map((entry) => ({ id: entry.space.id, name: entry.space.name, reason: clip(`Not read: a list covers at most ${RECENT_SPACES_MAX} spaces. Choose this space to see its memories.`, 200) })),
|
|
810
|
+
].slice(0, LIST_MAX),
|
|
811
|
+
items,
|
|
812
|
+
more,
|
|
813
|
+
}
|
|
814
|
+
if (!more) return out
|
|
815
|
+
// The cursor names the last row shown and how many rows at its own `at`
|
|
816
|
+
// this list has shown, so a tied run continues in id order.
|
|
817
|
+
const last = items[items.length - 1]
|
|
818
|
+
const atHere = items.filter((item) => item.at === last.at).length
|
|
819
|
+
const ties = atHere + (key && key.at === last.at ? key.ties : 0)
|
|
820
|
+
out.cursor = encodeMemoriesCursor({ at: last.at, id: last.id, ties })
|
|
821
|
+
return out
|
|
822
|
+
}
|
|
823
|
+
|
|
723
824
|
/** The kinds of term one answer may list, in the order it lists them. */
|
|
724
825
|
const TERM_RANK = Object.freeze(Object.fromEntries(TERM_TYPES.map((type, index) => [type, index])))
|
|
725
826
|
|
|
@@ -1750,6 +1851,7 @@ export function createRecordsReads({ runtime, personalScopes, available, describ
|
|
|
1750
1851
|
reviewWith,
|
|
1751
1852
|
methods: {
|
|
1752
1853
|
"memories.recent": memoriesRecent,
|
|
1854
|
+
"memories.list": memoriesList,
|
|
1753
1855
|
"journal.list": journalList,
|
|
1754
1856
|
"journal.read": journalRead,
|
|
1755
1857
|
"review.list": reviewList,
|
package/src/wiki-ingest.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { errorMessage } from "./util.js"
|
|
2
|
+
import { wikiWriteDigest } from "./wiki-queue.js"
|
|
2
3
|
|
|
3
4
|
/** How long a write waits for the rest of its burst before one ingest starts. */
|
|
4
5
|
export const WIKI_INGEST_DELAY_MS = 3000
|
|
@@ -20,10 +21,13 @@ export function isWikiBusy(error) {
|
|
|
20
21
|
* (`delay`, 3 seconds by default); a burst of writes makes one ingest. Three
|
|
21
22
|
* seconds is longer than the gap between the calls of one writing burst and
|
|
22
23
|
* short next to an ingest, which takes minutes.
|
|
23
|
-
* - **Queued, not refused.** The hub refuses a write while it
|
|
24
|
-
* "wiki is processing"), so such a write is
|
|
25
|
-
*
|
|
26
|
-
*
|
|
24
|
+
* - **Queued, not refused, and durable.** The hub refuses a write while it
|
|
25
|
+
* ingests (409 "wiki is processing"), so such a write is saved to the
|
|
26
|
+
* durable queue (`src/wiki-queue.js`) before the answer says `queued`,
|
|
27
|
+
* written when the ingest finishes, and one ingest follows it. After a
|
|
28
|
+
* restart the saved entries are replayed in order, and one whose body the
|
|
29
|
+
* wiki already holds is skipped (its filename and digest), so a crash
|
|
30
|
+
* between the hub's accept and the entry's removal never writes it twice.
|
|
27
31
|
* - **One ingest per wiki at a time here.** All of a wiki's queue work is
|
|
28
32
|
* serialised (`running`), and the hub's 409 is the lock across engines.
|
|
29
33
|
*
|
|
@@ -33,6 +37,7 @@ export function isWikiBusy(error) {
|
|
|
33
37
|
* backend: any,
|
|
34
38
|
* log?: any,
|
|
35
39
|
* delay?: number,
|
|
40
|
+
* queue: { append: Function, entries: Function, remove: Function },
|
|
36
41
|
* rememberError?: (wikiID: string, message: string) => void,
|
|
37
42
|
* clearError?: (wikiID: string) => void,
|
|
38
43
|
* rememberPending?: (wikiID: string, scope: object) => void,
|
|
@@ -44,25 +49,44 @@ export function createWikiIngest({
|
|
|
44
49
|
backend,
|
|
45
50
|
log = null,
|
|
46
51
|
delay = WIKI_INGEST_DELAY_MS,
|
|
52
|
+
queue: durable,
|
|
47
53
|
rememberError = () => {},
|
|
48
54
|
clearError = () => {},
|
|
49
55
|
rememberPending = () => {},
|
|
50
56
|
clearPending = () => {},
|
|
51
57
|
pending = [],
|
|
52
58
|
} = {}) {
|
|
59
|
+
if (!durable || typeof durable.append !== "function" || typeof durable.entries !== "function") {
|
|
60
|
+
// A held write is never taken without a durable record (jelly #89, 0.11.4).
|
|
61
|
+
throw new Error("the wiki ingest queue needs its durable store")
|
|
62
|
+
}
|
|
63
|
+
|
|
53
64
|
/** `${team}/${space}/${user}/${wiki}` -> one queue, so one ingest at a time. */
|
|
54
65
|
const queues = new Map()
|
|
55
66
|
let stopped = false
|
|
56
67
|
let startTimer = null
|
|
57
68
|
|
|
58
69
|
/**
|
|
59
|
-
*
|
|
60
|
-
*
|
|
61
|
-
* after setup returns, so a start never waits on the state
|
|
70
|
+
* What an earlier engine start left: the writes it queued, replayed in the
|
|
71
|
+
* order they were saved, and the ingests its debounce did not finish. Read
|
|
72
|
+
* after setup returns, so a start never waits on the state files.
|
|
62
73
|
*/
|
|
63
74
|
function recover() {
|
|
64
75
|
startTimer = null
|
|
65
76
|
if (stopped) return
|
|
77
|
+
let saved = []
|
|
78
|
+
try {
|
|
79
|
+
saved = typeof durable.entries === "function" ? durable.entries() : []
|
|
80
|
+
} catch (error) {
|
|
81
|
+
log?.warn?.("the wiki write queue could not be read", { error: errorMessage(error) })
|
|
82
|
+
}
|
|
83
|
+
for (const entry of saved) {
|
|
84
|
+
if (!entry?.wiki || !entry?.file) continue
|
|
85
|
+
const queue = queueFor(entry.scope, entry.wiki)
|
|
86
|
+
queue.writes.push(entry)
|
|
87
|
+
// Replay promptly: a busy hub answers 409, and the retry waits.
|
|
88
|
+
arm(queue, 10)
|
|
89
|
+
}
|
|
66
90
|
const waiting = typeof pending === "function" ? pending() : pending
|
|
67
91
|
for (const entry of waiting ?? []) {
|
|
68
92
|
if (!entry?.wikiID || !entry?.scope) continue
|
|
@@ -102,13 +126,29 @@ export function createWikiIngest({
|
|
|
102
126
|
}
|
|
103
127
|
|
|
104
128
|
/**
|
|
105
|
-
*
|
|
106
|
-
* the ingest finishes, in the order it arrived, and one
|
|
129
|
+
* Save a write the hub refused because it is ingesting, before answering.
|
|
130
|
+
* It is written when the ingest finishes, in the order it arrived, and one
|
|
131
|
+
* ingest follows. A disk that refuses the save throws, so the answer is
|
|
132
|
+
* `refused`, never `queued` (jelly #89, 0.11.4).
|
|
107
133
|
*/
|
|
108
134
|
function hold(queue, filename, content) {
|
|
109
|
-
|
|
135
|
+
const entry = durable.append({ wikiID: queue.wikiID, scope: queue.scope, filename, content })
|
|
136
|
+
queue.writes.push(entry)
|
|
110
137
|
queue.failures += 1
|
|
111
138
|
arm(queue, retryDelay(queue))
|
|
139
|
+
return entry
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/** True when the wiki already holds exactly this body, so a replay skips it. */
|
|
143
|
+
async function alreadyLanded(queue, entry) {
|
|
144
|
+
try {
|
|
145
|
+
const data = await backend.readWikiRaw({ scope: queue.scope, wikiID: queue.wikiID, filenames: [entry.file] })
|
|
146
|
+
const item = (data?.items ?? []).find((row) => row?.filename === entry.file)
|
|
147
|
+
return typeof item?.content === "string" && wikiWriteDigest(item.content) === entry.digest
|
|
148
|
+
} catch {
|
|
149
|
+
// An unreadable file is not proof it landed: attempt the write.
|
|
150
|
+
return false
|
|
151
|
+
}
|
|
112
152
|
}
|
|
113
153
|
|
|
114
154
|
async function flush(queue) {
|
|
@@ -118,8 +158,15 @@ export function createWikiIngest({
|
|
|
118
158
|
try {
|
|
119
159
|
while (queue.writes.length > 0) {
|
|
120
160
|
const next = queue.writes[0]
|
|
161
|
+
if (await alreadyLanded(queue, next)) {
|
|
162
|
+
// The hub accepted it and the engine stopped before the removal:
|
|
163
|
+
// the same body is never written twice.
|
|
164
|
+
queue.writes.shift()
|
|
165
|
+
durable.remove(next.id)
|
|
166
|
+
continue
|
|
167
|
+
}
|
|
121
168
|
try {
|
|
122
|
-
await backend.writeWikiRaw({ scope: queue.scope, wikiID: queue.wikiID, files: [{ filename: next.
|
|
169
|
+
await backend.writeWikiRaw({ scope: queue.scope, wikiID: queue.wikiID, files: [{ filename: next.file, content: next.content }] })
|
|
123
170
|
} catch (error) {
|
|
124
171
|
if (isWikiBusy(error)) {
|
|
125
172
|
queue.dirty = true
|
|
@@ -134,11 +181,13 @@ export function createWikiIngest({
|
|
|
134
181
|
// The same refusal three times is not going to pass: say so, and do
|
|
135
182
|
// not repeat it forever.
|
|
136
183
|
queue.writes.shift()
|
|
137
|
-
|
|
138
|
-
|
|
184
|
+
durable.remove(next.id)
|
|
185
|
+
rememberError(queue.wikiID, `A queued write to ${next.file} failed: ${errorMessage(error)}`)
|
|
186
|
+
log?.warn?.("a queued wiki write was dropped", { wikiID: queue.wikiID, filename: next.file, error: errorMessage(error) })
|
|
139
187
|
continue
|
|
140
188
|
}
|
|
141
189
|
queue.writes.shift()
|
|
190
|
+
durable.remove(next.id)
|
|
142
191
|
queue.dirty = true
|
|
143
192
|
landed = true
|
|
144
193
|
}
|
|
@@ -182,7 +231,13 @@ export function createWikiIngest({
|
|
|
182
231
|
await backend.writeWikiRaw({ scope, wikiID, files: [{ filename, content }] })
|
|
183
232
|
} catch (error) {
|
|
184
233
|
if (!isWikiBusy(error)) throw error
|
|
185
|
-
|
|
234
|
+
try {
|
|
235
|
+
hold(queue, filename, content)
|
|
236
|
+
} catch (queueError) {
|
|
237
|
+
// No durable record, so this is not queued: the answer is refused.
|
|
238
|
+
log?.warn?.("a queued wiki write could not be saved", { wikiID, filename, error: errorMessage(queueError) })
|
|
239
|
+
return { queued: false, refused: true, reason: String(queueError?.code ?? "") }
|
|
240
|
+
}
|
|
186
241
|
return { queued: true }
|
|
187
242
|
}
|
|
188
243
|
queue.dirty = true
|
|
@@ -200,7 +255,7 @@ export function createWikiIngest({
|
|
|
200
255
|
return count
|
|
201
256
|
},
|
|
202
257
|
|
|
203
|
-
/** Stop every timer;
|
|
258
|
+
/** Stop every timer; what waited is on disk, and the next start replays it. */
|
|
204
259
|
stop() {
|
|
205
260
|
stopped = true
|
|
206
261
|
if (startTimer) clearTimeout(startTimer)
|