@joe-broadhead/opencode-mem 0.9.2 → 0.9.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +324 -0
- package/README.md +30 -12
- package/docs/agent-protocol.md +1 -1
- package/package.json +254 -5
- package/skills/mem.usage/SKILL.md +10 -0
- package/src/backend/records.js +405 -50
- package/src/call-shapes.js +1 -1
- package/src/capture.js +4 -3
- package/src/config.js +11 -0
- package/src/connect.js +353 -0
- package/src/copies.js +28 -3
- package/src/credentials.js +35 -7
- package/src/curation.js +27 -4
- package/src/forgotten.js +204 -0
- package/src/index.js +20 -3
- package/src/runtime.js +5 -1
- package/src/tools/held-write.js +9 -1
- package/src/tools/index.js +4 -1
- package/src/tools/memory.js +97 -3
- package/src/tools/recall.js +43 -11
- package/src/tools/spaces.js +22 -4
- package/src/util.js +39 -0
- package/src/views/contracts.js +137 -6
- package/src/views/overdue.js +125 -0
- package/src/views/profile.js +155 -0
- package/src/views/records-reads.js +126 -32
- package/src/views/rpc.js +63 -12
- package/src/views/shape.js +49 -0
- package/views/commitments.html +31 -4
- package/views/home.html +22 -3
- package/views/memory.html +126 -24
package/src/forgotten.js
ADDED
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Recall withholds copies of forgotten text (docs/design/adr-forget-copies.md,
|
|
3
|
+
* "Recall withholds copies after a forget").
|
|
4
|
+
*
|
|
5
|
+
* A forget deletes a record and keeps the turns and atoms that copy it until
|
|
6
|
+
* the person consents to deleting each one. So that no later session answers
|
|
7
|
+
* from those copies, the forget stores a fingerprint in the space's records
|
|
8
|
+
* (`forgotten.put`): the service extracts the text's copy features with the
|
|
9
|
+
* same `copyFeatures` (records/src/copies.js is this repository's
|
|
10
|
+
* src/copies.js, byte for byte), hashes them with the author's own key, and
|
|
11
|
+
* keeps no text. Recall reads the caller's own fingerprints and key
|
|
12
|
+
* (`forgotten.list`) and withholds each of the caller's rows `matchCopy`
|
|
13
|
+
* would call a copy, comparing keyed hashes instead of text.
|
|
14
|
+
*
|
|
15
|
+
* The match is exact, not an approximation: `containsToken` finds an anchor
|
|
16
|
+
* bounded by non-letters-or-digits, so a token anchor (no whitespace, a letter
|
|
17
|
+
* or digit at each end) is a run of `pieces` letter-or-digit runs of the row,
|
|
18
|
+
* and any other anchor (a quoted phrase) is a bounded span of its length.
|
|
19
|
+
* Those spans and the row's stems are hashed and looked up.
|
|
20
|
+
* test/forgotten-parity.test.js holds the two equal on a corpus.
|
|
21
|
+
*
|
|
22
|
+
* The plugin package does not ship the records service, so the keyed hash is
|
|
23
|
+
* computed here as there (HMAC-SHA256, the first 16 bytes, base64url).
|
|
24
|
+
*/
|
|
25
|
+
import { createHmac } from "node:crypto"
|
|
26
|
+
import { MAX_MATCH_CHARS, MIN_STRONG_ANCHORS, PROPORTION, PROPORTIONAL_ABOVE, candidateStems } from "./copies.js"
|
|
27
|
+
|
|
28
|
+
/** The fingerprint ids `forgotten.put` mints. */
|
|
29
|
+
export const FORGOTTEN_ID = /^fp_[0-9A-HJKMNP-TV-Z]{26}$/
|
|
30
|
+
/** The feature rules this plugin matches by (records/src/forgotten.js FEATURE_VERSION). */
|
|
31
|
+
export const FEATURE_VERSION = 1
|
|
32
|
+
const HASH_BYTES = 16
|
|
33
|
+
const PIECE = /[\p{L}\p{N}]+/gu
|
|
34
|
+
const ALNUM = /[\p{L}\p{N}]/u
|
|
35
|
+
const SPACE = /\s/
|
|
36
|
+
|
|
37
|
+
/** The keyed hash of one feature, as the records service computes it. */
|
|
38
|
+
export function hashTerm(key, term) {
|
|
39
|
+
return createHmac("sha256", Buffer.from(String(key), "base64url")).update(String(term), "utf8").digest().subarray(0, HASH_BYTES).toString("base64url")
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** The code point that ends just before `index`, or "". */
|
|
43
|
+
function before(text, index) {
|
|
44
|
+
if (index <= 0) return ""
|
|
45
|
+
const low = text.charCodeAt(index - 1)
|
|
46
|
+
if (index >= 2 && low >= 0xdc00 && low <= 0xdfff) {
|
|
47
|
+
const high = text.charCodeAt(index - 2)
|
|
48
|
+
if (high >= 0xd800 && high <= 0xdbff) return text.slice(index - 2, index)
|
|
49
|
+
}
|
|
50
|
+
return text[index - 1]
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** The code point that starts at `index`, or "". */
|
|
54
|
+
function at(text, index) {
|
|
55
|
+
if (index >= text.length) return ""
|
|
56
|
+
const point = text.codePointAt(index)
|
|
57
|
+
return String.fromCodePoint(point)
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* A matcher over the caller's own fingerprints, as `forgotten.list` answers
|
|
62
|
+
* them. `usable` is false when there are none it can apply; `unknown` counts
|
|
63
|
+
* fingerprints of feature rules this plugin does not know (a newer service),
|
|
64
|
+
* which it cannot apply: recall says so.
|
|
65
|
+
*
|
|
66
|
+
* @param {{ version?: number, key?: string | null, fingerprints?: any[] } | null | undefined} list
|
|
67
|
+
*/
|
|
68
|
+
export function createForgottenMatcher(list) {
|
|
69
|
+
const key = typeof list?.key === "string" && list.key ? list.key : ""
|
|
70
|
+
const all = Array.isArray(list?.fingerprints) ? list.fingerprints : []
|
|
71
|
+
const known = all.filter((fp) => fp && Number(fp.v) === FEATURE_VERSION && typeof fp.id === "string" && Array.isArray(fp.anchors) && Array.isArray(fp.stems))
|
|
72
|
+
const unknown = all.length - known.length
|
|
73
|
+
const prints = known.map((fp) => ({
|
|
74
|
+
id: fp.id,
|
|
75
|
+
anchors: new Set(fp.anchors),
|
|
76
|
+
strong: new Set(Array.isArray(fp.strong) ? fp.strong : []),
|
|
77
|
+
stems: new Set(fp.stems),
|
|
78
|
+
}))
|
|
79
|
+
const numbers = (field) => [...new Set(known.flatMap((fp) => (Array.isArray(fp[field]) ? fp[field] : [])).map(Number).filter((n) => Number.isInteger(n) && n >= 1))]
|
|
80
|
+
const pieces = numbers("pieces")
|
|
81
|
+
const phrases = numbers("phrases")
|
|
82
|
+
const memo = new Map()
|
|
83
|
+
const hashed = (term) => {
|
|
84
|
+
let value = memo.get(term)
|
|
85
|
+
if (value === undefined) {
|
|
86
|
+
value = hashTerm(key, term)
|
|
87
|
+
if (memo.size < 100_000) memo.set(term, value)
|
|
88
|
+
}
|
|
89
|
+
return value
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** The hashes of every span of the row that could equal an anchor. */
|
|
93
|
+
function spanHashes(lower) {
|
|
94
|
+
const spans = new Set()
|
|
95
|
+
if (pieces.length > 0) {
|
|
96
|
+
const runs = [...lower.matchAll(PIECE)].map((found) => ({ start: found.index, end: found.index + found[0].length }))
|
|
97
|
+
for (let i = 0; i < runs.length; i += 1) {
|
|
98
|
+
for (const n of pieces) {
|
|
99
|
+
const last = runs[i + n - 1]
|
|
100
|
+
if (!last) continue
|
|
101
|
+
const span = lower.slice(runs[i].start, last.end)
|
|
102
|
+
if (n > 1 && SPACE.test(span)) continue
|
|
103
|
+
spans.add(hashed(span))
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
if (phrases.length > 0) {
|
|
108
|
+
for (let p = 0; p < lower.length; p += 1) {
|
|
109
|
+
const code = lower.charCodeAt(p)
|
|
110
|
+
if (code >= 0xdc00 && code <= 0xdfff) continue
|
|
111
|
+
if (ALNUM.test(before(lower, p))) continue
|
|
112
|
+
for (const length of phrases) {
|
|
113
|
+
const end = p + length
|
|
114
|
+
if (end > lower.length || ALNUM.test(at(lower, end))) continue
|
|
115
|
+
spans.add(hashed(lower.slice(p, end)))
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
return spans
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** The ids of every fingerprint the text copies, strongest first. */
|
|
123
|
+
function matches(text) {
|
|
124
|
+
if (!key || prints.length === 0) return []
|
|
125
|
+
// As matchCopy reads it: the first MAX_MATCH_CHARS characters, lower case.
|
|
126
|
+
const lower = String(text ?? "").slice(0, MAX_MATCH_CHARS).toLowerCase()
|
|
127
|
+
if (!lower.trim()) return []
|
|
128
|
+
const spans = spanHashes(lower)
|
|
129
|
+
const stems = new Set([...candidateStems(lower)].map(hashed))
|
|
130
|
+
const found = []
|
|
131
|
+
for (const fp of prints) {
|
|
132
|
+
let anchors = 0
|
|
133
|
+
let strong = 0
|
|
134
|
+
for (const hash of fp.anchors) {
|
|
135
|
+
if (!spans.has(hash)) continue
|
|
136
|
+
anchors += 1
|
|
137
|
+
if (fp.strong.has(hash)) strong += 1
|
|
138
|
+
}
|
|
139
|
+
const weak = anchors - strong
|
|
140
|
+
let shared = 0
|
|
141
|
+
for (const hash of fp.stems) if (stems.has(hash)) shared += 1
|
|
142
|
+
const signals = anchors + shared
|
|
143
|
+
let copy
|
|
144
|
+
if (fp.strong.size > 0) copy = (strong >= 1 && signals >= 2) || weak >= 2
|
|
145
|
+
else if (fp.anchors.size > 0) copy = weak >= 1 && signals >= 2
|
|
146
|
+
else copy = fp.stems.size >= 2 && shared >= 2 && shared >= Math.ceil(0.6 * fp.stems.size)
|
|
147
|
+
const size = fp.anchors.size + fp.stems.size
|
|
148
|
+
if (copy && size > PROPORTIONAL_ABOVE && signals < Math.ceil(PROPORTION * size) && strong < MIN_STRONG_ANCHORS) copy = false
|
|
149
|
+
if (copy) found.push({ id: fp.id, strength: anchors * 2 + shared })
|
|
150
|
+
}
|
|
151
|
+
return found.sort((a, b) => b.strength - a.strength).map((entry) => entry.id)
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
return {
|
|
155
|
+
usable: Boolean(key) && prints.length > 0,
|
|
156
|
+
unknown,
|
|
157
|
+
ids: prints.map((fp) => fp.id),
|
|
158
|
+
matches,
|
|
159
|
+
match: (text) => matches(text)[0] ?? null,
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* Split rows into those recall may serve and those that copy text the caller
|
|
165
|
+
* forgot. A fingerprint is its author's, and recall reads only the caller's,
|
|
166
|
+
* so a row another user wrote is never withheld by it.
|
|
167
|
+
*
|
|
168
|
+
* @template T
|
|
169
|
+
* @param {T[]} rows
|
|
170
|
+
* @param {ReturnType<typeof createForgottenMatcher> | null} matcher
|
|
171
|
+
* @param {string} [userID] the caller
|
|
172
|
+
* Matching is bounded in time: `budget.ms` is the matching time left for the
|
|
173
|
+
* whole recall, and only the time spent matching counts against it (never a
|
|
174
|
+
* slow search). A row not checked once it runs out is withheld too, as
|
|
175
|
+
* unchecked, never served.
|
|
176
|
+
*
|
|
177
|
+
* @param {string} [layer] the rows' layer, which keys the withheld ids
|
|
178
|
+
* @param {{ ms: number }} [budget] shared by every call of one recall; spent here
|
|
179
|
+
* @returns {{ served: T[], hidden: string[], unchecked: string[] }} `layer:id` of each withheld row, never shown
|
|
180
|
+
*/
|
|
181
|
+
export function hideForgotten(rows, matcher, userID, layer = "", budget = { ms: Infinity }) {
|
|
182
|
+
if (!matcher?.usable || !Array.isArray(rows) || rows.length === 0) return { served: Array.isArray(rows) ? rows : [], hidden: [], unchecked: [] }
|
|
183
|
+
const served = []
|
|
184
|
+
const hidden = []
|
|
185
|
+
const unchecked = []
|
|
186
|
+
for (const row of rows) {
|
|
187
|
+
const any = /** @type {any} */ (row)
|
|
188
|
+
if (!(budget.ms > 0)) {
|
|
189
|
+
unchecked.push(`${layer}:${any?.id ?? unchecked.length}`)
|
|
190
|
+
continue
|
|
191
|
+
}
|
|
192
|
+
const author = any?.user_id ?? any?.userId
|
|
193
|
+
const own = !userID || author === undefined || author === null || author === "" || String(author) === String(userID)
|
|
194
|
+
const started = performance.now()
|
|
195
|
+
const copy = own && matcher.match(any?.content ?? any?.text ?? "")
|
|
196
|
+
budget.ms -= performance.now() - started
|
|
197
|
+
if (copy) hidden.push(`${layer}:${any?.id ?? hidden.length}`)
|
|
198
|
+
else served.push(row)
|
|
199
|
+
}
|
|
200
|
+
return { served, hidden, unchecked }
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/** Recall's time for matching rows against fingerprints, in milliseconds: matching only, never the searches. */
|
|
204
|
+
export const MATCH_BUDGET_MS = 1_000
|
package/src/index.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { configWarnings, resolveConfig } from "./config.js"
|
|
2
2
|
import { asSession, createCredentials, followOrigin } from "./credentials.js"
|
|
3
|
+
import { createConnectSource, gatewayOrigin, registerConnect } from "./connect.js"
|
|
3
4
|
import { createLogger } from "./log.js"
|
|
4
5
|
import { createBackend } from "./backend/v3.js"
|
|
5
6
|
import { createRuntime } from "./runtime.js"
|
|
@@ -59,7 +60,10 @@ export default {
|
|
|
59
60
|
|
|
60
61
|
// Credentials and, inside Jelly, the gateway origin are read on use: the
|
|
61
62
|
// accessors Jelly's core plugin publishes may appear after this setup.
|
|
62
|
-
|
|
63
|
+
// With a gateway address, a person can also sign in with OpenCode's
|
|
64
|
+
// /connect (src/connect.js); OpenCode keeps and renews that credential.
|
|
65
|
+
const signIn = config.controlURL ? createConnectSource({ integration: ctx?.integration, origin: () => gatewayOrigin(config.controlURL), log }) : null
|
|
66
|
+
const credentials = createCredentials({ token: config.token, log, connect: signIn?.supported ? signIn : null })
|
|
63
67
|
const jelly = !config.controlURL && credentials.inJelly()
|
|
64
68
|
if (jelly) followOrigin(config, () => credentials.origin())
|
|
65
69
|
const gateway = Boolean(config.controlURL) || jelly
|
|
@@ -67,12 +71,24 @@ export default {
|
|
|
67
71
|
const runtime = createRuntime({ config, log, backend, credentials, gateway })
|
|
68
72
|
runtime.pruneClaims()
|
|
69
73
|
|
|
74
|
+
// Register Memory in /connect, then learn whether a sign-in is stored.
|
|
75
|
+
let connectRegistration = null
|
|
76
|
+
if (signIn) {
|
|
77
|
+
try {
|
|
78
|
+
connectRegistration = await registerConnect(ctx, { origin: () => gatewayOrigin(config.controlURL), log })
|
|
79
|
+
} catch (error) {
|
|
80
|
+
log.warn("could not add Memory to /connect; keys still work", errorMessage(error))
|
|
81
|
+
}
|
|
82
|
+
if (signIn.supported && !config.token) await signIn.probe()
|
|
83
|
+
}
|
|
84
|
+
|
|
70
85
|
log.info("setup", { version, directory: ctx?.location?.directory, jelly, credential: credentials.source() || "none" })
|
|
71
86
|
for (const warning of configWarnings(config, { credentialSource: credentials.source(), gateway })) log.warn(warning)
|
|
72
87
|
const unavailable = runtime.unavailable()
|
|
73
88
|
if (unavailable) log.info("memory unavailable at setup; tools stay registered", { reason: unavailable })
|
|
74
89
|
|
|
75
90
|
const disposers = []
|
|
91
|
+
if (typeof connectRegistration?.dispose === "function") disposers.push(() => connectRegistration.dispose())
|
|
76
92
|
let skillState = { ok: true, count: 0, ids: [], spaceIds: [], sourceSpaces: 0, registration: null }
|
|
77
93
|
// Setup and a session bind can race. Coalesce them so control is asked once
|
|
78
94
|
// and the last requested refresh always runs after the current one.
|
|
@@ -90,8 +106,9 @@ export default {
|
|
|
90
106
|
// /mem-sync or bind asked for it.
|
|
91
107
|
skillRefreshPromise = asSession("", async () => {
|
|
92
108
|
try {
|
|
93
|
-
// Nothing to mirror from while memory is unavailable
|
|
94
|
-
|
|
109
|
+
// Nothing to mirror from while memory is unavailable (a /connect
|
|
110
|
+
// sign-in made since setup counts: OpenCode is asked first).
|
|
111
|
+
if (runtime.unavailable() && !(await credentials.probeSignIn?.())) return skillState
|
|
95
112
|
const next = await syncSkills(ctx, runtime, {
|
|
96
113
|
...options,
|
|
97
114
|
dispose: async () => {
|
package/src/runtime.js
CHANGED
|
@@ -131,7 +131,11 @@ export function createRuntime(input) {
|
|
|
131
131
|
*/
|
|
132
132
|
async function availability() {
|
|
133
133
|
const reason = unavailable()
|
|
134
|
-
if (reason
|
|
134
|
+
if (reason) {
|
|
135
|
+
// A /connect sign-in made since the last look (src/connect.js): ask OpenCode once.
|
|
136
|
+
return control && (await credentials?.probeSignIn?.()) ? unavailable() : reason
|
|
137
|
+
}
|
|
138
|
+
if (!control || !credentials?.check) return reason
|
|
135
139
|
await credentials.check()
|
|
136
140
|
return unavailable()
|
|
137
141
|
}
|
package/src/tools/held-write.js
CHANGED
|
@@ -2,12 +2,20 @@ import { createHash } from "node:crypto"
|
|
|
2
2
|
import { asVerifiedView } from "../credentials.js"
|
|
3
3
|
import { oneLine, redactSecrets } from "../util.js"
|
|
4
4
|
import { callRecords, RECORD_ID } from "./records-kit.js"
|
|
5
|
+
import { writeCallRoot } from "../backend/records.js"
|
|
5
6
|
|
|
6
7
|
const hash = (value) => createHash("sha256").update(value).digest("hex")
|
|
7
8
|
const SPACE = /^[A-Za-z0-9_.-]{1,128}$/
|
|
8
9
|
|
|
9
|
-
/**
|
|
10
|
+
/**
|
|
11
|
+
* The held proposal's key: from the tool call's root (`withWriteCall`), so a
|
|
12
|
+
* call sent again after a restart holds the same proposal, and two identical
|
|
13
|
+
* calls in one message, or one Code Mode script, hold two.
|
|
14
|
+
*/
|
|
10
15
|
function writeKey(context, space) {
|
|
16
|
+
const root = writeCallRoot()
|
|
17
|
+
if (root) return `hw_${hash(JSON.stringify([space, root]))}`
|
|
18
|
+
// Outside a tool call (a direct caller): the call's own ids, as before.
|
|
11
19
|
if (![context?.sessionID, context?.messageID, context?.id].every((value) => typeof value === "string" && value)) throw new Error("held write needs a stable session, message and tool call id")
|
|
12
20
|
return `hw_${hash(JSON.stringify([space, context.sessionID, context.messageID, context.id]))}`
|
|
13
21
|
}
|
package/src/tools/index.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { MEM_NAMESPACE_DESCRIPTION, PINNED_TOOLS, WIKI_DOCUMENTS_NOTE } from "../call-shapes.js"
|
|
2
2
|
import { asSession, asVerifiedSession } from "../credentials.js"
|
|
3
|
+
import { withWriteCall } from "../backend/records.js"
|
|
3
4
|
import { createCollectorService } from "../collector-service.js"
|
|
4
5
|
import { createToolkit } from "./shared.js"
|
|
5
6
|
import { registerWiki } from "./wiki.js"
|
|
@@ -130,7 +131,9 @@ function gateEditor(editor, runtime) {
|
|
|
130
131
|
const run = async () => {
|
|
131
132
|
const reason = (runtime.availability ? await runtime.availability() : runtime.unavailable?.()) ?? ""
|
|
132
133
|
if (reason) return { content: reason }
|
|
133
|
-
|
|
134
|
+
// One tool call is one logical call: its records writes derive
|
|
135
|
+
// their idempotency keys from its ids (docs/design/records-idempotency.md).
|
|
136
|
+
return withWriteCall(context, definition.name, input, () => definition.execute(input, context))
|
|
134
137
|
}
|
|
135
138
|
return held ? asVerifiedSession(context?.sessionID, run) : asSession(context?.sessionID, run)
|
|
136
139
|
},
|
package/src/tools/memory.js
CHANGED
|
@@ -17,7 +17,8 @@ import {
|
|
|
17
17
|
} from "../copies.js"
|
|
18
18
|
import { errorMessage, neutralizeFenceClosers, oneLine, redactSecrets } from "../util.js"
|
|
19
19
|
import { isCoded } from "../errors.js"
|
|
20
|
-
import { RecordsError } from "../backend/records.js"
|
|
20
|
+
import { RecordsError, isRecordsUnavailable } from "../backend/records.js"
|
|
21
|
+
import { FORGOTTEN_ID, createForgottenMatcher } from "../forgotten.js"
|
|
21
22
|
import { TASK_INPUT, canWriteShared } from "./shared.js"
|
|
22
23
|
import { holdTeamWrite } from "./held-write.js"
|
|
23
24
|
import { CORE_ID, MEMORY_TYPES, PAGE_PATH, RECORD_ID, TURN_ID, callRecords, changeNote, dateOf, lineageIds, noteRecordSpace, notWritten, recordLine, recordSpace, titleFrom } from "./records-kit.js"
|
|
@@ -283,6 +284,7 @@ export function createCoreForget(toolkit) {
|
|
|
283
284
|
groups.get(key).ids.push(id)
|
|
284
285
|
}
|
|
285
286
|
const removed = { l0: 0, l1: 0 }
|
|
287
|
+
const confirmedIDs = []
|
|
286
288
|
const unconfirmed = []
|
|
287
289
|
const noSession = []
|
|
288
290
|
const errors = []
|
|
@@ -300,6 +302,7 @@ export function createCoreForget(toolkit) {
|
|
|
300
302
|
const count = Math.max(0, Math.min(group.ids.length, Number(data?.deleted_count) || 0))
|
|
301
303
|
removed[group.layer] += count
|
|
302
304
|
if (count < group.ids.length) unconfirmed.push({ ...group, missing: group.ids.length - count })
|
|
305
|
+
else confirmedIDs.push(...group.ids)
|
|
303
306
|
} catch (error) {
|
|
304
307
|
errors.push({ ...group, error: errorMessage(error) })
|
|
305
308
|
}
|
|
@@ -403,7 +406,9 @@ export function createCoreForget(toolkit) {
|
|
|
403
406
|
unscannedAgents: knownAgents(spaceContext).filter((agent) => !copyAgents.includes(agent)),
|
|
404
407
|
})
|
|
405
408
|
: ""
|
|
406
|
-
|
|
409
|
+
// The rows confirmed gone: a forgotten atom among them may need a fingerprint.
|
|
410
|
+
const deletedIDs = [...new Set([...confirmedIDs, ...ownDeleted.map((row) => row.id)])]
|
|
411
|
+
return { removed, total, fatal, lines, report, errors: errors.length, unlocated: unlocated.length, forgottenRows, deletedIDs }
|
|
407
412
|
}
|
|
408
413
|
|
|
409
414
|
/**
|
|
@@ -711,6 +716,65 @@ export function createCoreForget(toolkit) {
|
|
|
711
716
|
|
|
712
717
|
const coreForgets = new WeakMap()
|
|
713
718
|
|
|
719
|
+
/** A records service before 0.9.5 has no `forgotten.*` methods (or no records at all). */
|
|
720
|
+
function noFingerprints(error) {
|
|
721
|
+
return isRecordsUnavailable(error) || error?.code === "records_missing" || error?.status === 404
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
/**
|
|
725
|
+
* Store a fingerprint of a forgotten text (src/forgotten.js), so recall
|
|
726
|
+
* withholds from this person, in every later session, every turn and atom
|
|
727
|
+
* that copies it, including copies written after the forget (its own reply
|
|
728
|
+
* quotes the turns). The service extracts the features and keeps keyed
|
|
729
|
+
* hashes, never the text. It never lifts itself: only the person lifts it,
|
|
730
|
+
* in Memory's console. Answers the line for the forget's report.
|
|
731
|
+
*
|
|
732
|
+
* The words about copies are said only when this forget kept some: with none,
|
|
733
|
+
* the report must not send the agent looking for them.
|
|
734
|
+
*
|
|
735
|
+
* @param {{ text: string, kind: string, kept?: number }} entry
|
|
736
|
+
*/
|
|
737
|
+
async function storeFingerprint(toolkit, entry, spaceContext, scoped) {
|
|
738
|
+
try {
|
|
739
|
+
await callRecords(toolkit, "forgotten.put", { text: String(entry.text).slice(0, 4_000), kind: entry.kind }, spaceContext, scoped)
|
|
740
|
+
return entry.kept > 0
|
|
741
|
+
? "Recall won't show these copies. To delete them, ask your agent; to undo this forget, use Memory's console (Memories › Forgotten)."
|
|
742
|
+
: "Recall won't show a copy of it written later. To undo this forget, use Memory's console (Memories › Forgotten)."
|
|
743
|
+
} catch (error) {
|
|
744
|
+
if (error?.code === "fingerprint_cap") return `Copies of the forgotten text can't be withheld from recall: ${errorMessage(error)} Lift old fingerprints in Memory's console (Memories › Forgotten).`
|
|
745
|
+
if (error?.code === "invalid_fingerprint") return "The forgotten text has nothing a copy can be matched by, so recall cannot withhold its copies; a later mem_recall can show them until they are deleted."
|
|
746
|
+
if (noFingerprints(error)) return "This Memory server can't withhold forgotten text (it is older than 0.9.5): a later mem_recall can show its copies until they are deleted."
|
|
747
|
+
return `Recall was not told to withhold copies of the forgotten text (${errorMessage(error)}): a later mem_recall can show them until they are deleted.`
|
|
748
|
+
}
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
/**
|
|
752
|
+
* After core rows are deleted by id: fingerprint each forgotten atom that
|
|
753
|
+
* copies none of the caller's fingerprints (its turns may stay). Deleting a
|
|
754
|
+
* copy never lifts a fingerprint: a later copy would show the fact again.
|
|
755
|
+
*
|
|
756
|
+
* @param {{ deletedIDs: string[], forgottenRows: any[] }} deleted
|
|
757
|
+
* @returns {Promise<string[]>}
|
|
758
|
+
*/
|
|
759
|
+
async function fingerprintForgottenAtoms(toolkit, deleted, spaceContext, scoped) {
|
|
760
|
+
const gone = new Set(deleted?.deletedIDs ?? [])
|
|
761
|
+
const atoms = (deleted?.forgottenRows ?? []).filter((row) => row.layer === "l1" && gone.has(row.id) && String(row.content ?? "").trim())
|
|
762
|
+
if (atoms.length === 0) return []
|
|
763
|
+
let matcher = null
|
|
764
|
+
try {
|
|
765
|
+
matcher = createForgottenMatcher(await callRecords(toolkit, "forgotten.list", {}, spaceContext, scoped))
|
|
766
|
+
} catch (error) {
|
|
767
|
+
return noFingerprints(error) ? [] : [`The forgotten-copy fingerprints could not be read (${errorMessage(error)}), so this forget added none.`]
|
|
768
|
+
}
|
|
769
|
+
const lines = []
|
|
770
|
+
for (const row of atoms) {
|
|
771
|
+
if (matcher.match(row.content)) continue
|
|
772
|
+
const line = await storeFingerprint(toolkit, { text: row.content, kind: "atom" }, spaceContext, scoped)
|
|
773
|
+
if (line) lines.push(line)
|
|
774
|
+
}
|
|
775
|
+
return lines
|
|
776
|
+
}
|
|
777
|
+
|
|
714
778
|
/** Ids from a string or a list, trimmed and de-duplicated. */
|
|
715
779
|
function idList(value) {
|
|
716
780
|
if (Array.isArray(value)) return [...new Set(value.map((id) => String(id ?? "").trim()).filter(Boolean))]
|
|
@@ -1003,13 +1067,14 @@ export function registerRevise(editor, toolkit) {
|
|
|
1003
1067
|
const ids = idList(input?.id)
|
|
1004
1068
|
if (ids.length === 0) return text(`id is required for ${verb}: the rec_… id mem_remember or mem_recall gave.`)
|
|
1005
1069
|
const id = ids[0]
|
|
1070
|
+
const fingerprint = verb === "restore" && FORGOTTEN_ID.test(id)
|
|
1006
1071
|
// A record id, a turn id, or else the core's atom id (m_… on the
|
|
1007
1072
|
// deployed core; other cores mint other shapes), as supersedes takes it.
|
|
1008
1073
|
const unknown = ids.filter((value) => !RECORD_ID.test(value) && !CORE_ID.test(value))
|
|
1009
1074
|
if (unknown.length > 0) {
|
|
1010
1075
|
return text(`${unknown.map((value) => JSON.stringify(oneLine(value, 80))).join(", ")} ${unknown.length === 1 ? "is not a memory id" : "are not memory ids"}: pass rec_… ids from mem_remember or mem_recall (forget also takes m_… and msg-…). Nothing changed.`)
|
|
1011
1076
|
}
|
|
1012
|
-
if (verb !== "forget" && !RECORD_ID.test(id) && !(verb === "correct" && !TURN_ID.test(id))) {
|
|
1077
|
+
if (verb !== "forget" && !fingerprint && !RECORD_ID.test(id) && !(verb === "correct" && !TURN_ID.test(id))) {
|
|
1013
1078
|
return text(`${verb} takes a record id (rec_…); ${oneLine(id, 80)} is in the core${verb === "correct" ? " as a turn" : ""}. ${TURN_ID.test(id) ? "Forget it, or remember the corrected text." : "Correct it to store the correction as a record, or forget it."} Nothing changed.`)
|
|
1014
1079
|
}
|
|
1015
1080
|
// Every refusal before any request.
|
|
@@ -1048,6 +1113,7 @@ export function registerRevise(editor, toolkit) {
|
|
|
1048
1113
|
return text(renderHistory(id, data))
|
|
1049
1114
|
}
|
|
1050
1115
|
if (verb === "forget") return text(await forget({ ids, context, spaceContext, scoped, label }))
|
|
1116
|
+
if (fingerprint) return text(restoreCopies())
|
|
1051
1117
|
|
|
1052
1118
|
if (shared) {
|
|
1053
1119
|
try {
|
|
@@ -1111,6 +1177,16 @@ export function registerRevise(editor, toolkit) {
|
|
|
1111
1177
|
},
|
|
1112
1178
|
})
|
|
1113
1179
|
|
|
1180
|
+
/**
|
|
1181
|
+
* Showing again the copies a forget withholds undoes the forget in recall.
|
|
1182
|
+
* Only the person does that, in Memory's console, in a browser session: the
|
|
1183
|
+
* records service refuses `forgotten.lift` to every key and token, so the
|
|
1184
|
+
* plugin does not call it, and says where it is done.
|
|
1185
|
+
*/
|
|
1186
|
+
function restoreCopies() {
|
|
1187
|
+
return "not restored: only the person can undo a forget, in a browser: Memory's console (Memories › Forgotten). Nothing changed, and recall keeps withholding the copies."
|
|
1188
|
+
}
|
|
1189
|
+
|
|
1114
1190
|
/** The fact a memory record states: its body before "Why:", else its title. */
|
|
1115
1191
|
function factOf(record) {
|
|
1116
1192
|
const body = String(record?.body ?? "").split(/\n\s*\nWhy:/)[0].trim()
|
|
@@ -1157,6 +1233,7 @@ export function registerRevise(editor, toolkit) {
|
|
|
1157
1233
|
const deleted = await core.deleteByIds({ ids: coreIDs, wanted: ["l1", "l0"], spaceContext, base, explicitAgent: "", approve: (ownIDs) => ask(ownIDs), retryArgs: {} })
|
|
1158
1234
|
if (deleted.denied) return `forgetting denied: ${deleted.denied}. Nothing was deleted.`
|
|
1159
1235
|
parts.push(`deleted ${deleted.total} core item(s) (${deleted.removed.l1} l1 atom(s), ${deleted.removed.l0} l0 turn(s)).`, ...deleted.lines)
|
|
1236
|
+
parts.push(...(await fingerprintForgottenAtoms(toolkit, deleted, spaceContext, scoped)))
|
|
1160
1237
|
report = deleted.report
|
|
1161
1238
|
for (const row of deleted.forgottenRows ?? []) {
|
|
1162
1239
|
if (!row.content) {
|
|
@@ -1222,6 +1299,22 @@ export function registerRevise(editor, toolkit) {
|
|
|
1222
1299
|
if (copies.kept.length > 0) parts.push(`Not deleted: ${copies.kept.map((row) => `${row.layer} ${oneLine(row.id, 80)} (${row.why})`).join("; ")}.`)
|
|
1223
1300
|
report = [report, core.copiesReport({ forgotten: scan.forgotten.length, copies: copies.kept.map((row) => ({ ...row, note: row.why })), windows: scan.windows, base, retryArgs: {}, unscannedAgents: scan.unscannedAgents })].filter(Boolean).join("\n")
|
|
1224
1301
|
|
|
1302
|
+
// The kept copies stay in the core until the person consents, and the
|
|
1303
|
+
// forget's own reply may copy the fact again: a fingerprint makes every
|
|
1304
|
+
// later recall withhold all of them from this person.
|
|
1305
|
+
for (const record of goneRecords) {
|
|
1306
|
+
const fact = factOf(record)
|
|
1307
|
+
if (!fact) continue
|
|
1308
|
+
const kind = MEMORY_TYPES.includes(record.type) ? record.type : "fact"
|
|
1309
|
+
const features = [copyFeatures(fact)]
|
|
1310
|
+
const kept = copies.kept.filter((row) => {
|
|
1311
|
+
const parts = rememberParts(row.content)
|
|
1312
|
+
return matchCopy(features, parts.kind === "wrapper" ? `${parts.body}\n${parts.reason}` : row.content)
|
|
1313
|
+
}).length
|
|
1314
|
+
const line = await storeFingerprint(toolkit, { text: fact, kind, kept }, spaceContext, scoped)
|
|
1315
|
+
if (line) parts.push(line)
|
|
1316
|
+
}
|
|
1317
|
+
|
|
1225
1318
|
// Read back, as recall searches: records, then atoms and turns.
|
|
1226
1319
|
const back = { records: [], core: { left: [], failed: false }, recordsFailed: false }
|
|
1227
1320
|
for (const fact of facts) {
|
|
@@ -1391,6 +1484,7 @@ export function registerForget(editor, toolkit) {
|
|
|
1391
1484
|
parts.push(
|
|
1392
1485
|
`deleted ${deleted.total} item(s) from ${label} (${idLayers.map((name) => `${deleted.removed[name]} ${name}`).join(" and ")}).`,
|
|
1393
1486
|
...deleted.lines,
|
|
1487
|
+
...(await fingerprintForgottenAtoms(toolkit, deleted, spaceContext, scoped)),
|
|
1394
1488
|
)
|
|
1395
1489
|
report = deleted.report
|
|
1396
1490
|
}
|
package/src/tools/recall.js
CHANGED
|
@@ -6,6 +6,7 @@ import { TASK_INPUT } from "./shared.js"
|
|
|
6
6
|
import { MEMORY_TYPES, PAGE_TYPES, callRecords, grantedSpace, noteRecordSpace } from "./records-kit.js"
|
|
7
7
|
import { isRecordsUnavailable } from "../backend/records.js"
|
|
8
8
|
import { recallResultView } from "../views/recall-result.js"
|
|
9
|
+
import { FEATURE_VERSION, MATCH_BUDGET_MS, createForgottenMatcher, hideForgotten } from "../forgotten.js"
|
|
9
10
|
|
|
10
11
|
/** Upper bound on hits per layer; the character budget still applies. */
|
|
11
12
|
const MAX_LIMIT = 25
|
|
@@ -255,6 +256,29 @@ export function registerRecall(editor, toolkit) {
|
|
|
255
256
|
)
|
|
256
257
|
: Promise.resolve({ hits: [], failed: false })
|
|
257
258
|
|
|
259
|
+
// Copies of forgotten text (src/forgotten.js): the caller's own
|
|
260
|
+
// fingerprints, read beside the records search, so no later session
|
|
261
|
+
// serves a turn or atom that copies something the caller forgot. When
|
|
262
|
+
// they cannot be checked (a service before 0.9.5, a 404, a failure,
|
|
263
|
+
// or feature rules this plugin does not know), the answer says so.
|
|
264
|
+
const forgottenRead =
|
|
265
|
+
(layers.atoms || layers.turns) && !resolved.granted
|
|
266
|
+
? callRecords(toolkit, "forgotten.list", {}, spaceContext, scope).then(
|
|
267
|
+
(data) => {
|
|
268
|
+
const matcher = createForgottenMatcher(data)
|
|
269
|
+
return { matcher, unchecked: matcher.unknown > 0 || (data?.version !== undefined && Number(data.version) !== FEATURE_VERSION) }
|
|
270
|
+
},
|
|
271
|
+
(error) => {
|
|
272
|
+
log.debug("forgotten-copy fingerprints unavailable", { error: errorMessage(error) })
|
|
273
|
+
return { matcher: null, unchecked: true }
|
|
274
|
+
},
|
|
275
|
+
)
|
|
276
|
+
: Promise.resolve({ matcher: null, unchecked: false })
|
|
277
|
+
const forgotten = await forgottenRead
|
|
278
|
+
// The time spent matching rows against fingerprints is bounded (the
|
|
279
|
+
// searches are not counted); a row not checked in time is withheld.
|
|
280
|
+
const matchBudget = { ms: MATCH_BUDGET_MS }
|
|
281
|
+
|
|
258
282
|
async function searchIn(searchScope) {
|
|
259
283
|
const failures = []
|
|
260
284
|
const [atomPage, turnPage] = await Promise.all([
|
|
@@ -288,10 +312,12 @@ export function registerRecall(editor, toolkit) {
|
|
|
288
312
|
const label = (row) => (key ? { ...row, _bucket: bucketOf(row) } : row)
|
|
289
313
|
const atomPart = partitionAtoms(atomPage?.items ?? [])
|
|
290
314
|
const turnPart = partitionTurns(turnPage?.messages ?? [], { isCheckpoint: isCheckpointMessage })
|
|
315
|
+
const atomCopies = hideForgotten(atomPart.served, forgotten.matcher, scope.user_id, "l1", matchBudget)
|
|
316
|
+
const turnCopies = hideForgotten(turnPart.served, forgotten.matcher, scope.user_id, "l0", matchBudget)
|
|
291
317
|
return {
|
|
292
|
-
atoms:
|
|
293
|
-
conversations:
|
|
294
|
-
withheld: addWithheld(atomPart.withheld, turnPart.withheld),
|
|
318
|
+
atoms: atomCopies.served.map(label),
|
|
319
|
+
conversations: turnCopies.served.map(label),
|
|
320
|
+
withheld: addWithheld(addWithheld(atomPart.withheld, turnPart.withheld), { forgotten: [...atomCopies.hidden, ...turnCopies.hidden], unchecked: [...atomCopies.unchecked, ...turnCopies.unchecked] }),
|
|
295
321
|
failures,
|
|
296
322
|
}
|
|
297
323
|
}
|
|
@@ -344,10 +370,13 @@ export function registerRecall(editor, toolkit) {
|
|
|
344
370
|
for (const record of records) noteRecordSpace(toolkit, record.id, spaceContext)
|
|
345
371
|
const { atoms, conversations } = result
|
|
346
372
|
const failures = [...result.failures, ...(found.failed || found.unavailable ? [found.missingEndpoint ? "records endpoint (404)" : "records"] : [])]
|
|
347
|
-
const failureDetail =
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
373
|
+
const failureDetail = [
|
|
374
|
+
found.unavailable
|
|
375
|
+
? `${found.unavailable === "build" ? "This plugin build has no records client" : "Records are not enabled on this Memory server yet"}, so remembered memories, pages and commitments were not searched${layers.atoms && layers.turns && result.failures.length === 0 ? "; past turns and atoms were" : ""}.`
|
|
376
|
+
: "",
|
|
377
|
+
forgotten.unchecked ? "This Memory server can't withhold forgotten text; copies may appear." : "",
|
|
378
|
+
].filter(Boolean).join(" ") || undefined
|
|
379
|
+
let withheld = addWithheld(result.withheld, recordPart.withheld)
|
|
351
380
|
if (layers.records && types.length > 0) scopeNote = `Only records of type ${types.join(", ")} were searched; ${layers.atoms ? "legacy atoms were searched without kind filtering; turns were not searched" : "atoms and turns were not"}.`
|
|
352
381
|
if (layers.records && status === "all") scopeNote = `${scopeNote ? `${scopeNote} ` : ""}Superseded and retired records are included (status "all").`
|
|
353
382
|
if (resolved.granted) scopeNote = `${scopeNote ? `${scopeNote} ` : ""}This space is shared with you by page grant: only the pages granted were searched.`
|
|
@@ -431,8 +460,11 @@ export function registerRecall(editor, toolkit) {
|
|
|
431
460
|
// curation guards as the search path.
|
|
432
461
|
noteLocations(spaceContext, "l1", recentAtoms?.items, { agent_id: scope.agent_id })
|
|
433
462
|
noteLocations(spaceContext, "l0", recentTurns?.messages, { agent_id: scope.agent_id })
|
|
434
|
-
const
|
|
435
|
-
const
|
|
463
|
+
const atomCopies = hideForgotten(partitionAtoms(recentAtoms?.items ?? []).served, forgotten.matcher, scope.user_id, "l1", matchBudget)
|
|
464
|
+
const turnCopies = hideForgotten(partitionTurns(recentTurns?.messages ?? [], { isCheckpoint: isCheckpointMessage }).served, forgotten.matcher, scope.user_id, "l0", matchBudget)
|
|
465
|
+
withheld = addWithheld(withheld, { forgotten: [...atomCopies.hidden, ...turnCopies.hidden], unchecked: [...atomCopies.unchecked, ...turnCopies.unchecked] })
|
|
466
|
+
const browseAtoms = atomCopies.served.slice(0, BROWSE_ATOMS)
|
|
467
|
+
const browseConversations = turnCopies.served.slice(0, BROWSE_TURNS)
|
|
436
468
|
const listed = (recentRecords?.records ?? []).filter((record) => recordKinds.includes(record.kind) && (types.length === 0 || types.includes(record.type)))
|
|
437
469
|
const browseRecords = partitionRecords(listed.map((record) => ({ ...record, snippet: record.body }))).served.slice(0, BROWSE_RECORDS)
|
|
438
470
|
if (browseAtoms.length + browseConversations.length + browseRecords.length > 0) {
|
|
@@ -498,8 +530,8 @@ export function registerRecall(editor, toolkit) {
|
|
|
498
530
|
|
|
499
531
|
const label = neutralizeFenceClosers(oneLine(spaceContext ? displayScope(spaceContext, scoped) : `${scoped.team_id}/${scoped.agent_id}`, 200))
|
|
500
532
|
const coverage = `Searched in ${label}: ${searched.length ? searched.join(", ") : "no memory layers"}. ${documentCoverage}`.trim()
|
|
501
|
-
const empty = where === "all" && types.length === 0 && !resolved.granted && census && !census.failed && !withheld.status && !withheld.instruction && !withheld.task && !withheld.redacted && census.records + census.atoms + census.conversations === 0 && documentsEmpty
|
|
502
|
-
const memoryEmpty = where === "all" && types.length === 0 && !resolved.granted && census && !census.failed && !withheld.status && !withheld.instruction && !withheld.task && !withheld.redacted && census.records + census.atoms + census.conversations === 0
|
|
533
|
+
const empty = where === "all" && types.length === 0 && !resolved.granted && census && !census.failed && !withheld.status && !withheld.instruction && !withheld.task && !withheld.redacted && !withheld.forgotten && !withheld.unchecked && census.records + census.atoms + census.conversations === 0 && documentsEmpty
|
|
534
|
+
const memoryEmpty = where === "all" && types.length === 0 && !resolved.granted && census && !census.failed && !withheld.status && !withheld.instruction && !withheld.task && !withheld.redacted && !withheld.forgotten && !withheld.unchecked && census.records + census.atoms + census.conversations === 0
|
|
503
535
|
const next = empty ? "Nothing is recorded in this space. Answer that it is not recorded, or continue the requested work. Do not search again in pages, wiki, spaces, commitments or other tools; a named document can be read directly when requested." : memoryEmpty ? "Nothing is recorded in the searched memory layers of this space. Report any document search limitation above; do not claim unsearched documents are empty. Do not repeat the search in pages, wiki, spaces or commitments." : ""
|
|
504
536
|
const memoryBudget = Math.max(100, config.recallChars - coverage.length - 1 - (documents ? documents.length + 2 : 0) - (next ? next.length + 1 : 0))
|
|
505
537
|
const memory = renderMemory(memoryBudget)
|