@joe-broadhead/opencode-mem 0.9.5 → 0.9.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/CHANGELOG.md +1408 -0
  2. package/README.md +120 -41
  3. package/docs/agent-instructions.md +17 -0
  4. package/docs/agent-protocol.md +28 -27
  5. package/docs/permissions.md +150 -0
  6. package/package.json +386 -17
  7. package/skills/mem.pages/SKILL.md +40 -16
  8. package/skills/mem.sharing/SKILL.md +39 -6
  9. package/skills/mem.sources/SKILL.md +41 -10
  10. package/skills/mem.usage/SKILL.md +37 -16
  11. package/src/backend/records.js +8 -0
  12. package/src/backend/v3.js +15 -10
  13. package/src/call-shapes.js +18 -25
  14. package/src/capture-pause.js +136 -0
  15. package/src/capture.js +7 -5
  16. package/src/collector-service.js +212 -35
  17. package/src/collector.js +187 -47
  18. package/src/commands.js +37 -0
  19. package/src/compaction.js +4 -2
  20. package/src/config.js +151 -1
  21. package/src/curation.js +6 -10
  22. package/src/events.js +144 -36
  23. package/src/forget-purge.js +377 -0
  24. package/src/forgotten.js +152 -0
  25. package/src/hooks.js +9 -4
  26. package/src/index.js +5 -3
  27. package/src/permissions.js +116 -38
  28. package/src/protocol.js +32 -11
  29. package/src/recall.js +13 -1
  30. package/src/records.js +4 -1
  31. package/src/runtime.js +202 -15
  32. package/src/shapes.js +63 -0
  33. package/src/sources.js +205 -12
  34. package/src/spaces.js +58 -1
  35. package/src/spool.js +442 -64
  36. package/src/stated.js +334 -0
  37. package/src/storable.js +477 -0
  38. package/src/tools/aliases.js +36 -147
  39. package/src/tools/commitments.js +79 -13
  40. package/src/tools/held-write.js +5 -1
  41. package/src/tools/index.js +25 -43
  42. package/src/tools/ingest.js +33 -128
  43. package/src/tools/memory.js +525 -911
  44. package/src/tools/page-sections.js +147 -0
  45. package/src/tools/page.js +314 -82
  46. package/src/tools/recall-browse.js +173 -0
  47. package/src/tools/recall.js +36 -8
  48. package/src/tools/records-kit.js +2 -1
  49. package/src/tools/share.js +494 -131
  50. package/src/tools/shared.js +97 -19
  51. package/src/tools/spaces.js +194 -101
  52. package/src/tools/status.js +31 -3
  53. package/src/tools/wiki.js +5 -13
  54. package/src/util.js +10 -59
  55. package/src/views/contracts.js +258 -20
  56. package/src/views/overdue.js +1 -1
  57. package/src/views/profile.js +2 -3
  58. package/src/views/receipt.js +81 -0
  59. package/src/views/records-reads.js +545 -25
  60. package/src/views/rpc.js +41 -3
  61. package/views/memory.html +16 -3
  62. package/views/remembered.html +130 -0
  63. package/src/erase.js +0 -36
@@ -1,6 +1,6 @@
1
1
  import fs from "node:fs"
2
2
  import path from "node:path"
3
- import { COLLECT_SESSION, COLLECT_SESSION_RULES, collectableSources, createCollector, hostScheduler, hostSourceLister } from "./collector.js"
3
+ import { DEFAULT_ACCOUNT, cadenceMs, collectSessionId, collectSessionRules, collectableSources, createCollector, hostSourceLister, rulesHash } from "./collector.js"
4
4
  import { createLease } from "./collector-lock.js"
5
5
  import { knownSources, sourceEntry } from "./sources.js"
6
6
  import { errorMessage, isPlainObject, oneLine } from "./util.js"
@@ -16,6 +16,10 @@ export const CATCH_UP_MS = 60_000
16
16
  export const CATCH_UP_RUNS = 10
17
17
  /** The first run after startup comes after a random wait in this range, so processes that start together spread out. */
18
18
  export const STARTUP_JITTER_MS = Object.freeze([10_000, 60_000])
19
+ /** A source is due this long before its cadence is up, so a 30-minute timer does not skip a 30-minute source by a few seconds. */
20
+ export const DUE_SLACK_MS = 60_000
21
+ /** Triggers a person started: they collect every bound source, whatever its cadence. */
22
+ const ASKED = new Set(["tool", "command", "request"])
19
23
 
20
24
  const REAL_CLOCK = Object.freeze({
21
25
  now: () => Date.now(),
@@ -40,7 +44,13 @@ const REAL_CLOCK = Object.freeze({
40
44
  * runs, a first run shortly after startup (which catches up from the
41
45
  * cursors), follow-up runs while a source still has pages waiting, and on
42
46
  * request (`mem_collect`, `/mem-collect`). `collect: false` (or
43
- * `MEMORY_COLLECT=0`) turns the timer off; a request still runs.
47
+ * `MEMORY_COLLECT=0`) turns the timer off; a request still runs. A timer
48
+ * run collects a source only when its own cadence is up (the registry's
49
+ * `everyMinutes`: X every 120 minutes); a request collects every source.
50
+ * - **Bind requests:** each run first answers the person's bind requests
51
+ * (`sources.list` `requests`): only for those sources, it reads the
52
+ * signed-in account and reports it (`sources.report`) for the person to
53
+ * confirm. No other source is ever asked.
44
54
  * - **Who:** one OpenCode process per machine, the holder of `collector.lock`
45
55
  * beside the spool (src/collector-lock.js). Another process that is asked
46
56
  * to collect leaves a request the holder picks up at its next heartbeat.
@@ -70,35 +80,143 @@ export function createCollectorService({ ctx, runtime, toolkit, clock = REAL_CLO
70
80
  const lock = lease ?? createLease({ file: path.join(stateDir, "collector.lock"), now: clock.now, log })
71
81
  const everyMs = Math.max(5, Number(config.collectEveryMinutes) || 30) * 60_000
72
82
 
73
- let sessionReady = null
74
- /** The reserved session the RPC calls run in; `fallback` (the caller's session) when the host cannot make one. */
75
- async function rpcSession(fallback) {
76
- if (typeof ctx?.session?.create !== "function") {
77
- if (fallback) return fallback
78
- throw new Error("this host cannot make the collector's session")
83
+ const sessionsFile = path.join(stateDir, "collector-sessions.json")
84
+ const sessionsReady = new Map()
85
+
86
+ /** The binding sessions this machine made, by id: so an unbound binding's session is found and deleted. */
87
+ function readSessions() {
88
+ try {
89
+ const value = JSON.parse(fs.readFileSync(sessionsFile, "utf8"))
90
+ return isPlainObject(value) ? value : {}
91
+ } catch {
92
+ return {}
79
93
  }
80
- sessionReady ??= (async () => {
81
- const agent = await pickAgent()
82
- await ctx.session.create({
83
- id: COLLECT_SESSION,
84
- title: "Memory collection",
85
- agent,
86
- ...(ctx?.location?.directory ? { location: { directory: ctx.location.directory } } : {}),
87
- metadata: { memory: { role: "collector" } },
88
- permissions: COLLECT_SESSION_RULES.map((rule) => ({ ...rule })),
89
- })
90
- return COLLECT_SESSION
91
- })().catch((error) => {
92
- sessionReady = null
93
- // Creation by id is idempotent; a host that says the session exists is fine.
94
- if (/exist/i.test(errorMessage(error))) return COLLECT_SESSION
95
- throw error
94
+ }
95
+
96
+ function writeSessions(value) {
97
+ try {
98
+ fs.mkdirSync(stateDir, { recursive: true, mode: 0o700 })
99
+ const temp = `${sessionsFile}.${lock.id}.tmp`
100
+ fs.writeFileSync(temp, JSON.stringify(value, null, 2), { mode: 0o600 })
101
+ fs.renameSync(temp, sessionsFile)
102
+ } catch (error) {
103
+ log?.warn?.("collector sessions could not be written", { error: errorMessage(error) })
104
+ }
105
+ }
106
+
107
+ /**
108
+ * Make a reserved session with these rules, or bring an existing one up to
109
+ * date: each carries a hash of its rules in its metadata, and one whose hash
110
+ * differs (an older registry's rules) is updated, else removed and made
111
+ * again. Anything that fails throws: the caller reads nothing through it.
112
+ */
113
+ function ensureSession({ id, title, permissions, extra = {} }) {
114
+ const hash = rulesHash(permissions)
115
+ const cached = sessionsReady.get(id)
116
+ if (cached?.hash === hash) return cached.promise
117
+ const promise = (async () => {
118
+ const session = ctx?.session
119
+ if (typeof session?.create !== "function") throw new Error("this host cannot make the collector's sessions")
120
+ const metadata = { memory: { role: "collector", ...extra, rules: hash } }
121
+ const rules = permissions.map((rule) => ({ ...rule }))
122
+ const make = async () =>
123
+ session.create({
124
+ id,
125
+ title,
126
+ agent: await pickAgent(),
127
+ ...(ctx?.location?.directory ? { location: { directory: ctx.location.directory } } : {}),
128
+ metadata,
129
+ permissions: rules,
130
+ })
131
+ try {
132
+ await make()
133
+ return id
134
+ } catch (error) {
135
+ if (!/exist/i.test(errorMessage(error))) throw error
136
+ }
137
+ // It exists: use it as it is only when its rules are these.
138
+ let current = null
139
+ try {
140
+ const answer = await session.get?.({ sessionID: id })
141
+ current = answer?.data ?? answer
142
+ } catch {
143
+ current = null
144
+ }
145
+ if (current?.metadata?.memory?.rules === hash) return id
146
+ if (typeof session.update === "function") {
147
+ await session.update({ sessionID: id, metadata, permissions: rules })
148
+ return id
149
+ }
150
+ if (typeof session.remove === "function") {
151
+ await session.remove({ sessionID: id })
152
+ await make()
153
+ return id
154
+ }
155
+ throw new Error("its rules are out of date, and this host can neither update nor remove it")
156
+ })()
157
+ sessionsReady.set(id, { hash, promise })
158
+ promise.catch(() => {
159
+ if (sessionsReady.get(id)?.promise === promise) sessionsReady.delete(id)
96
160
  })
161
+ return promise
162
+ }
163
+
164
+ /**
165
+ * The reserved session an RPC call runs in: the binding's own session
166
+ * (`collectSessionId`), whose rules allow only its source's events
167
+ * action, the exact inner reads its events_list asserts, and for gws the
168
+ * Gmail event reads of that account (`collectSessionRules`). A bind's
169
+ * probe runs in the same session. **Fail closed:** without a binding, or when the
170
+ * session cannot be made or brought up to date, this throws, and the caller
171
+ * reads nothing this run. It never falls back to the caller's own session,
172
+ * and since 0.9.7 there is no session for every source at once (audit
173
+ * runtime #8). (`_caller` is kept for the callers' shape and ignored.)
174
+ */
175
+ async function rpcSession(_caller, binding = null) {
176
+ if (!binding?.source) throw new Error("the collector reads a source only in that binding's own session; no source was named, so nothing was read")
177
+ const account = binding.account || DEFAULT_ACCOUNT
178
+ const id = collectSessionId({ source: binding.source, account })
97
179
  try {
98
- return await sessionReady
180
+ await ensureSession({ id, title: `Memory collection: ${binding.source}`, permissions: collectSessionRules({ source: binding.source, account }), extra: { source: binding.source } })
99
181
  } catch (error) {
100
- if (fallback) return fallback
101
- throw new Error(`the collector's session could not be made (${oneLine(errorMessage(error), 200)})`)
182
+ throw new Error(`the collector's session for ${binding.source}:${account} could not be made (${oneLine(errorMessage(error), 200)}); nothing was read`)
183
+ }
184
+ const known = readSessions()
185
+ if (!known[id]) writeSessions({ ...known, [id]: { source: binding.source, account } })
186
+ return id
187
+ }
188
+
189
+ /** Delete a binding's session (unbind, a cancelled request), so no session keeps its allow rules. */
190
+ async function forgetSession(binding) {
191
+ const account = binding.account || DEFAULT_ACCOUNT
192
+ const id = collectSessionId({ source: binding.source, account })
193
+ sessionsReady.delete(id)
194
+ const known = readSessions()
195
+ try {
196
+ if (typeof ctx?.session?.remove === "function") await ctx.session.remove({ sessionID: id })
197
+ } catch (error) {
198
+ // A session the host no longer has is already gone; anything else is kept, to try at the next run.
199
+ if (!/not.?found|no session|does not exist/i.test(errorMessage(error))) {
200
+ log?.warn?.("a collector session could not be deleted", { error: errorMessage(error) })
201
+ return false
202
+ }
203
+ }
204
+ if (known[id]) {
205
+ delete known[id]
206
+ writeSessions(known)
207
+ }
208
+ return true
209
+ }
210
+
211
+ /** Delete the sessions of bindings that are gone (an unbind or a cancel in the console). */
212
+ async function pruneSessions(sources, requests) {
213
+ const wanted = new Set(
214
+ [...(Array.isArray(sources) ? sources : []), ...(Array.isArray(requests) ? requests : [])]
215
+ .filter(isPlainObject)
216
+ .map((row) => collectSessionId({ source: String(row.source ?? ""), account: String(row.account ?? "") || DEFAULT_ACCOUNT })),
217
+ )
218
+ for (const [id, binding] of Object.entries(readSessions())) {
219
+ if (!wanted.has(id) && isPlainObject(binding)) await forgetSession(binding)
102
220
  }
103
221
  }
104
222
 
@@ -113,7 +231,8 @@ export function createCollectorService({ ctx, runtime, toolkit, clock = REAL_CLO
113
231
  }
114
232
  }
115
233
 
116
- const collector = createCollector({ scheduler: hostScheduler(ctx, () => rpcSession()), lister: hostSourceLister(ctx), log })
234
+ // No host schedule (0.9.7): it needed one session for every source; collection runs here, per binding.
235
+ const collector = createCollector({ scheduler: null, lister: hostSourceLister(ctx), log })
117
236
 
118
237
  const timers = { startup: null, interval: null, heartbeat: null, catchUp: null }
119
238
  let active = null
@@ -205,7 +324,7 @@ export function createCollectorService({ ctx, runtime, toolkit, clock = REAL_CLO
205
324
  const started = clock.now()
206
325
  lastStartedAt = started
207
326
  const results = []
208
- active = { started, trigger, only, results, promise: null }
327
+ active = { started, trigger, only, results, probes: [], promise: null }
209
328
  active.promise = collect(trigger, { only, sessionID, results, started })
210
329
  .catch((error) => {
211
330
  log?.warn?.("collection failed", { error: errorMessage(error) })
@@ -224,7 +343,10 @@ export function createCollectorService({ ctx, runtime, toolkit, clock = REAL_CLO
224
343
  }
225
344
 
226
345
  async function collect(trigger, { only, sessionID, results, started }) {
227
- writeStatus({ state: "running", trigger, pid: process.pid, started: new Date(started).toISOString(), sources: [] })
346
+ // The last run's state, read before this run's "running" line replaces it: its cadence times and what was left behind.
347
+ const previous = readStatus()
348
+ const collected = isPlainObject(previous?.collected) ? previous.collected : {}
349
+ writeStatus({ state: "running", trigger, pid: process.pid, started: new Date(started).toISOString(), sources: [], collected })
228
350
  const records = recordsOf(toolkit)
229
351
  let where = { sessionID }
230
352
  if (toolkit.control) {
@@ -232,14 +354,28 @@ export function createCollectorService({ ctx, runtime, toolkit, clock = REAL_CLO
232
354
  if (resolved.invalid || resolved.error) throw new Error(resolved.invalid || resolved.error)
233
355
  where = { space: resolved.context.space_id }
234
356
  }
235
- const sources = (await records.call("sources.list", {}, where))?.sources ?? []
357
+ const listed = await records.call("sources.list", {}, where)
358
+ const sources = listed?.sources ?? []
236
359
  await collector.sync(sources)
237
- const rpc = await rpcSession(sessionID)
360
+ await pruneSessions(sources, listed?.requests)
361
+ // Each binding's calls run in that binding's own session (least privilege); never a broader one.
362
+ const rpc = (binding) => rpcSession(null, binding)
363
+ const probes = active?.probes ?? []
364
+ await collector.probeRequests({ requests: listed?.requests, only, now: clock.now(), sessionID: rpc, probes, report: (body) => records.call("sources.report", body, where) })
365
+ const behind = new Set((Array.isArray(previous?.sources) ? previous.sources : []).filter((row) => row?.more && !row?.error).map((row) => keyOf(row)))
366
+ const rows = new Map((Array.isArray(sources) ? sources : []).filter(isPlainObject).map((row) => [keyOf(row), row]))
238
367
  await collector.runOnce({
239
368
  sources,
240
369
  only,
241
370
  sessionID: rpc,
242
371
  results,
372
+ due: (binding) => {
373
+ if (ASKED.has(trigger) || only) return true
374
+ const key = keyOf(binding)
375
+ if (behind.has(key)) return true
376
+ const last = Date.parse(collected[key] ?? rows.get(key)?.cursor_at ?? "")
377
+ return !Number.isFinite(last) || clock.now() - last >= cadenceMs(binding.source) - DUE_SLACK_MS
378
+ },
243
379
  cursor: (binding) => records.call("events.cursor", binding, where),
244
380
  ingest: async (input) => {
245
381
  try {
@@ -253,11 +389,34 @@ export function createCollectorService({ ctx, runtime, toolkit, clock = REAL_CLO
253
389
  }
254
390
  },
255
391
  })
256
- return finish(trigger, started, results, "", collectableSources(sources, { only }).length === 0)
392
+ await reportCollections(records, where, results)
393
+ return finish(trigger, started, results, "", collectableSources(sources, { only }).length === 0, { probes, collected })
257
394
  }
258
395
 
259
- function finish(trigger, started, results, error, none = false) {
396
+ /**
397
+ * Tell the records service how each binding's collection went, so Sources
398
+ * in the console shows its last collection and its error (0.9.8). Best
399
+ * effort: a service older than 0.9.8 does not know the call, and a failed
400
+ * report never fails the collection.
401
+ */
402
+ async function reportCollections(records, where, results) {
403
+ for (const result of results) {
404
+ if (result.notDue || typeof result.source !== "string") continue
405
+ const body = { source: result.source, account: result.account ?? "default", ok: !result.error, accepted: Number.isInteger(result.accepted) ? result.accepted : undefined }
406
+ if (result.error) body.error = oneLine(result.error, 300)
407
+ try {
408
+ await records.call("sources.collected", body, where)
409
+ } catch (error) {
410
+ log?.debug?.("the collection report was not recorded", { source: result.source, error: errorMessage(error) })
411
+ }
412
+ }
413
+ }
414
+
415
+ function finish(trigger, started, results, error, none = false, { probes = [], collected } = {}) {
260
416
  const ended = clock.now()
417
+ // When each binding was last attempted, for its cadence: kept across runs and processes in collector.json.
418
+ const at = { ...(collected ?? (isPlainObject(readStatus()?.collected) ? readStatus().collected : {})) }
419
+ for (const result of results) if (!result.notDue) at[keyOf(result)] = new Date(started).toISOString()
261
420
  last = {
262
421
  state: error ? "failed" : "done",
263
422
  trigger,
@@ -267,6 +426,8 @@ export function createCollectorService({ ctx, runtime, toolkit, clock = REAL_CLO
267
426
  ...(error ? { error: oneLine(error, 300) } : {}),
268
427
  ...(none ? { none: true } : {}),
269
428
  sources: results.map(({ done: _done, ...result }) => ({ ...result, ...(result.error ? { error: oneLine(result.error, 300) } : {}) })),
429
+ ...(probes.length ? { probes: probes.map((probe) => ({ ...probe })) } : {}),
430
+ collected: at,
270
431
  }
271
432
  writeStatus(last)
272
433
  const behind = results.some((result) => result.more && !result.error)
@@ -395,6 +556,7 @@ export function createCollectorService({ ctx, runtime, toolkit, clock = REAL_CLO
395
556
  const when = status.state === "running" ? `running since ${oneLine(status.started ?? "", 40)}` : `last run ${oneLine(status.ended ?? status.started ?? "", 40)}`
396
557
  lines.push(`collector: ${when} (${oneLine(status.trigger ?? "?", 20)}${status.pid ? `, pid ${status.pid}` : ""})${status.state === "failed" ? ` failed: ${oneLine(status.error ?? "", 200)}` : ""}${status.none ? ": no source is enabled and confirmed" : ""}`)
397
558
  for (const result of Array.isArray(status.sources) ? status.sources : []) lines.push(`- ${describeResult(result)}`)
559
+ for (const probe of Array.isArray(status.probes) ? status.probes : []) lines.push(`- ${describeProbe(probe)}`)
398
560
  return lines.join("\n")
399
561
  }
400
562
 
@@ -412,6 +574,7 @@ export function createCollectorService({ ctx, runtime, toolkit, clock = REAL_CLO
412
574
  runOnce: collector.runOnce,
413
575
  // The service's.
414
576
  rpcSession,
577
+ forgetSession,
415
578
  start,
416
579
  stop,
417
580
  begin,
@@ -433,9 +596,23 @@ export function createCollectorService({ ctx, runtime, toolkit, clock = REAL_CLO
433
596
  }
434
597
  }
435
598
 
599
+ function keyOf(row) {
600
+ return `${row?.source ?? ""}\u0000${row?.account || DEFAULT_ACCOUNT}`
601
+ }
602
+
603
+ /** One bind request's line: the account the source reported, or why it could not. */
604
+ export function describeProbe(probe) {
605
+ const name = `${oneLine(probe.source ?? "?", 40)}:${oneLine(probe.account ?? "?", 80)}`
606
+ if (probe.error && !probe.owner?.id && probe.note === undefined) return `${name}: bind request: not asked this run (${oneLine(probe.error, 240)})`
607
+ const what = probe.owner?.id ? `bind request: signed in as ${oneLine(probe.owner.label ?? probe.owner.id, 120)}; waiting for the person to confirm it` : `bind request: no account yet${probe.note ? ` (${oneLine(probe.note, 240)})` : ""}`
608
+ const note = probe.owner?.id && probe.note ? `; the source says: ${oneLine(probe.note, 240)}` : ""
609
+ return `${name}: ${what}${note}${probe.error ? `; the report failed (${oneLine(probe.error, 160)})` : ""}`
610
+ }
611
+
436
612
  /** One source's line: its counts, and why it stopped. */
437
613
  export function describeResult(result) {
614
+ if (result.notDue) return `${oneLine(result.source ?? "?", 40)}:${oneLine(result.account ?? "?", 80)}: not due (collected every ${Number(result.everyMinutes) || 30} minutes)`
438
615
  const counts = `${Number(result.accepted) || 0} new, ${Number(result.duplicates) || 0} already had, ${Number(result.refused) || 0} refused, ${Number(result.pages) || 0} page(s)`
439
616
  const name = `${oneLine(result.source ?? "?", 40)}:${oneLine(result.account ?? "?", 80)}`
440
- return `${name}: ${result.error ? `stopped (${oneLine(result.error, 240)}); ` : ""}${counts}${result.more ? "; more waits for the next run" : ""}`
617
+ return `${name}: ${result.error ? `stopped (${oneLine(result.error, 240)}); ` : ""}${counts}${result.more ? "; more waits for the next run" : ""}${result.setup ? `; the source says: ${oneLine(result.setup, 240)}` : ""}`
441
618
  }