@joe-broadhead/opencode-mem 0.0.0-stage → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +323 -2
- package/SECURITY.md +131 -0
- package/docs/agent-protocol.md +190 -0
- package/index.js +4 -0
- package/package.json +174 -4
- package/skills/mem-usage/SKILL.md +427 -0
- package/src/backend/v3.js +562 -0
- package/src/bootstrap.js +85 -0
- package/src/capture.js +143 -0
- package/src/commands.js +151 -0
- package/src/compaction.js +402 -0
- package/src/config.js +193 -0
- package/src/copies.js +322 -0
- package/src/credentials.js +294 -0
- package/src/curation.js +240 -0
- package/src/erase.js +36 -0
- package/src/events.js +394 -0
- package/src/hooks.js +182 -0
- package/src/index.js +189 -0
- package/src/limits.js +90 -0
- package/src/log.js +161 -0
- package/src/protocol.js +48 -0
- package/src/recall.js +368 -0
- package/src/runtime.js +868 -0
- package/src/scope.js +127 -0
- package/src/sessions.js +62 -0
- package/src/skill-contexts.js +168 -0
- package/src/skills.js +252 -0
- package/src/spaces.js +478 -0
- package/src/spool.js +520 -0
- package/src/tools/index.js +51 -0
- package/src/tools/knowledge.js +90 -0
- package/src/tools/memory.js +1239 -0
- package/src/tools/recall.js +273 -0
- package/src/tools/shared.js +339 -0
- package/src/tools/spaces.js +179 -0
- package/src/tools/status.js +64 -0
- package/src/tools/wiki.js +412 -0
- package/src/util.js +176 -0
package/src/config.js
ADDED
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
import os from "node:os"
|
|
2
|
+
import path from "node:path"
|
|
3
|
+
import { isPlainObject, toPositiveInt } from "./util.js"
|
|
4
|
+
|
|
5
|
+
/** The bundled simulator: the core URL of local direct mode. */
|
|
6
|
+
export const LOCAL_BASE_URL = "http://127.0.0.1:8420"
|
|
7
|
+
|
|
8
|
+
const DEFAULTS = Object.freeze({
|
|
9
|
+
/** Memory core base URL. Unset: `<controlURL>/core`, else the local simulator. */
|
|
10
|
+
baseURL: "",
|
|
11
|
+
/**
|
|
12
|
+
* Public gateway origin. When empty (and not inside Jelly), the plugin runs
|
|
13
|
+
* against one fixed local scope.
|
|
14
|
+
*/
|
|
15
|
+
controlURL: "",
|
|
16
|
+
/** Knowledge service base URL (wiki + code graph). Unset: `<controlURL>/knowledge`, else `baseURL`. */
|
|
17
|
+
knowledgeURL: "",
|
|
18
|
+
/** Bearer token for the knowledge service; falls back to `token`. */
|
|
19
|
+
knowledgeToken: "",
|
|
20
|
+
/** Memory space / instance id. */
|
|
21
|
+
space: "default",
|
|
22
|
+
/** Kernel bearer token. */
|
|
23
|
+
token: "",
|
|
24
|
+
/** Identity used for scope isolation. */
|
|
25
|
+
user: "",
|
|
26
|
+
agent: "auto",
|
|
27
|
+
|
|
28
|
+
/** Capture completed turns. `false` also turns the compaction checkpoint off unless that is set. */
|
|
29
|
+
capture: true,
|
|
30
|
+
/** Completed turns are private unless the user explicitly shares them. */
|
|
31
|
+
captureVisibility: "private",
|
|
32
|
+
/**
|
|
33
|
+
* Subagent (child) sessions are not captured or pinned. Their first user
|
|
34
|
+
* message is the parent's brief, not the user's words; the parent turn
|
|
35
|
+
* already records the task call and its result.
|
|
36
|
+
*/
|
|
37
|
+
captureSubagents: false,
|
|
38
|
+
captureScenarios: true,
|
|
39
|
+
bootstrap: true,
|
|
40
|
+
/**
|
|
41
|
+
* Write a pre-compaction checkpoint to L0. Unset, it follows `capture`: a
|
|
42
|
+
* person who turns capture off has no conversation state written for them.
|
|
43
|
+
*/
|
|
44
|
+
compactionCheckpoint: undefined,
|
|
45
|
+
tools: true,
|
|
46
|
+
skills: true,
|
|
47
|
+
commands: true,
|
|
48
|
+
spool: true,
|
|
49
|
+
|
|
50
|
+
bootstrapChars: 4000,
|
|
51
|
+
recallLimit: 8,
|
|
52
|
+
recallChars: 6000,
|
|
53
|
+
timeoutMs: 2000,
|
|
54
|
+
retries: 1,
|
|
55
|
+
|
|
56
|
+
debug: false,
|
|
57
|
+
debugFile: "",
|
|
58
|
+
stateDir: "",
|
|
59
|
+
})
|
|
60
|
+
|
|
61
|
+
const ENV_MAP = {
|
|
62
|
+
baseURL: "MEMORY_BASE_URL",
|
|
63
|
+
controlURL: "MEMORY_CONTROL_URL",
|
|
64
|
+
knowledgeURL: "MEMORY_KNOWLEDGE_URL",
|
|
65
|
+
knowledgeToken: "MEMORY_KNOWLEDGE_TOKEN",
|
|
66
|
+
space: "MEMORY_SPACE",
|
|
67
|
+
token: "MEMORY_TOKEN",
|
|
68
|
+
user: "MEMORY_USER",
|
|
69
|
+
agent: "MEMORY_AGENT",
|
|
70
|
+
captureVisibility: "MEMORY_CAPTURE_VISIBILITY",
|
|
71
|
+
captureSubagents: "MEMORY_CAPTURE_SUBAGENTS",
|
|
72
|
+
stateDir: "MEMORY_STATE_DIR",
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
const BOOLEAN_KEYS = [
|
|
76
|
+
"capture",
|
|
77
|
+
"captureSubagents",
|
|
78
|
+
"captureScenarios",
|
|
79
|
+
"bootstrap",
|
|
80
|
+
"compactionCheckpoint",
|
|
81
|
+
"tools",
|
|
82
|
+
"skills",
|
|
83
|
+
"commands",
|
|
84
|
+
"spool",
|
|
85
|
+
"debug",
|
|
86
|
+
]
|
|
87
|
+
|
|
88
|
+
const NUMBER_KEYS = ["bootstrapChars", "recallLimit", "recallChars", "timeoutMs", "retries"]
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Resolve plugin options into a validated configuration object.
|
|
92
|
+
*
|
|
93
|
+
* Accepts either a flat options object or `{ backend: {...}, ... }` so hosts can
|
|
94
|
+
* group transport settings without changing the plugin's own defaults.
|
|
95
|
+
*
|
|
96
|
+
* @param {Record<string, unknown>} [options]
|
|
97
|
+
* @param {Record<string, string | undefined>} [env]
|
|
98
|
+
*/
|
|
99
|
+
export function resolveConfig(options = {}, env = process.env) {
|
|
100
|
+
const input = isPlainObject(options) ? options : {}
|
|
101
|
+
const backend = isPlainObject(input.backend) ? input.backend : {}
|
|
102
|
+
const merged = { ...DEFAULTS }
|
|
103
|
+
for (const key of Object.keys(DEFAULTS)) {
|
|
104
|
+
const value = input[key] ?? backend[key]
|
|
105
|
+
if (value !== undefined) merged[key] = value
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
for (const [key, envName] of Object.entries(ENV_MAP)) {
|
|
109
|
+
const fromEnv = env[envName]
|
|
110
|
+
const explicit = input[key] ?? backend[key]
|
|
111
|
+
if (explicit === undefined && typeof fromEnv === "string" && fromEnv.length > 0) {
|
|
112
|
+
merged[key] = fromEnv
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
for (const key of BOOLEAN_KEYS) {
|
|
117
|
+
if (merged[key] === undefined) continue
|
|
118
|
+
if (typeof merged[key] === "string") {
|
|
119
|
+
merged[key] = !["0", "false", "no", "off"].includes(merged[key].toLowerCase())
|
|
120
|
+
}
|
|
121
|
+
merged[key] = Boolean(merged[key])
|
|
122
|
+
}
|
|
123
|
+
// Unset, the checkpoint follows capture: `capture: false` writes nothing.
|
|
124
|
+
if (merged.compactionCheckpoint === undefined) merged.compactionCheckpoint = merged.capture
|
|
125
|
+
|
|
126
|
+
for (const key of NUMBER_KEYS) {
|
|
127
|
+
merged[key] = toPositiveInt(merged[key], DEFAULTS[key])
|
|
128
|
+
}
|
|
129
|
+
merged.retries = Math.min(merged.retries, 5)
|
|
130
|
+
|
|
131
|
+
for (const key of ["baseURL", "controlURL", "knowledgeURL"]) {
|
|
132
|
+
if (!merged[key]) continue
|
|
133
|
+
let parsed
|
|
134
|
+
try {
|
|
135
|
+
parsed = new URL(String(merged[key]))
|
|
136
|
+
} catch {
|
|
137
|
+
throw new Error(`mem: invalid ${key} "${merged[key]}"`)
|
|
138
|
+
}
|
|
139
|
+
if (!/^https?:$/.test(parsed.protocol)) {
|
|
140
|
+
throw new Error(`mem: ${key} must be http(s), got "${parsed.protocol}"`)
|
|
141
|
+
}
|
|
142
|
+
merged[key] = String(merged[key]).replace(/\/+$/, "")
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
// Which URLs an option or variable set; the rest default from the gateway
|
|
146
|
+
// origin (here `controlURL`; inside Jelly, see credentials.followOrigin).
|
|
147
|
+
const explicitURLs = Object.freeze({
|
|
148
|
+
baseURL: Boolean(merged.baseURL),
|
|
149
|
+
knowledgeURL: Boolean(merged.knowledgeURL),
|
|
150
|
+
controlURL: Boolean(merged.controlURL),
|
|
151
|
+
})
|
|
152
|
+
if (!merged.baseURL) merged.baseURL = merged.controlURL ? `${merged.controlURL}/core` : LOCAL_BASE_URL
|
|
153
|
+
if (!merged.knowledgeURL && merged.controlURL) merged.knowledgeURL = `${merged.controlURL}/knowledge`
|
|
154
|
+
Object.defineProperty(merged, "explicitURLs", { value: explicitURLs, enumerable: false })
|
|
155
|
+
|
|
156
|
+
merged.space = String(merged.space || "default")
|
|
157
|
+
merged.agent = String(merged.agent || "auto")
|
|
158
|
+
merged.captureVisibility = String(merged.captureVisibility || "private").toLowerCase()
|
|
159
|
+
if (!["private", "team"].includes(merged.captureVisibility)) merged.captureVisibility = "private"
|
|
160
|
+
merged.user = String(merged.user || env.USER || env.USERNAME || "unknown")
|
|
161
|
+
merged.stateDir = String(
|
|
162
|
+
merged.stateDir || path.join(env.XDG_STATE_HOME || path.join(os.homedir(), ".local", "state"), "opencode-mem"),
|
|
163
|
+
)
|
|
164
|
+
merged.debugFile = String(merged.debugFile || path.join(merged.stateDir, "debug.log"))
|
|
165
|
+
|
|
166
|
+
return merged
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* A config is usable for network calls when it has a base URL. Tokens are
|
|
171
|
+
* optional because local deployments often disable auth.
|
|
172
|
+
*
|
|
173
|
+
* @param {ReturnType<typeof resolveConfig>} config
|
|
174
|
+
*/
|
|
175
|
+
export function configWarnings(config, { credentialSource = config.token ? "option" : "", gateway = Boolean(config.controlURL) } = {}) {
|
|
176
|
+
const warnings = []
|
|
177
|
+
if (!credentialSource) warnings.push("no MEMORY_TOKEN set; requests are sent without a bearer token")
|
|
178
|
+
if (!gateway) {
|
|
179
|
+
warnings.push(
|
|
180
|
+
"no MEMORY_CONTROL_URL set; using the fixed local scope. Spaces, grants, and members require a control plane",
|
|
181
|
+
)
|
|
182
|
+
}
|
|
183
|
+
if (config.agent === "auto") warnings.push("agent=auto; scope follows the active OpenCode agent per session")
|
|
184
|
+
if (config.captureVisibility === "team")
|
|
185
|
+
warnings.push(
|
|
186
|
+
"captureVisibility=team; completed turns are written to the session's bound shared space, and kept private when the session has none",
|
|
187
|
+
)
|
|
188
|
+
if (!config.compactionCheckpoint)
|
|
189
|
+
warnings.push("compactionCheckpoint=false; no pre-compaction checkpoint is written to memory")
|
|
190
|
+
if (config.captureSubagents)
|
|
191
|
+
warnings.push("captureSubagents=true; subagent sessions are captured and pinned like primary sessions")
|
|
192
|
+
return warnings
|
|
193
|
+
}
|
package/src/copies.js
ADDED
|
@@ -0,0 +1,322 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copies of forgotten text (docs/design/adr-forget-copies.md).
|
|
3
|
+
*
|
|
4
|
+
* One fact is stored several times: the `mem_remember` request and its echo
|
|
5
|
+
* (L0), the atoms the core extracts from them (L1), a captured turn that
|
|
6
|
+
* repeats it (L0), and the atoms extracted from that turn (L1). Upstream does
|
|
7
|
+
* not return an atom's source rows, and its extractor rewrites wording, so a
|
|
8
|
+
* copy is found by the specific tokens a fact is made of (numbers, dates,
|
|
9
|
+
* identifiers, quoted and capitalised names), which a rewrite keeps, and not
|
|
10
|
+
* by verbatim text, which a rewrite changes.
|
|
11
|
+
*
|
|
12
|
+
* Everything here is a pure function of the text, except `readListing`, which
|
|
13
|
+
* reads one bounded, newest-first listing through a caller-supplied reader.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
/** The request row `mem_remember` writes; recall never shows it. */
|
|
17
|
+
export const REMEMBER_WRAPPER = /^Please remember this (?:fact|decision|constraint|preference|procedure) for later: /
|
|
18
|
+
/** The assistant row `mem_remember` writes after the request. */
|
|
19
|
+
const REMEMBER_ECHO = /^Understood\. /
|
|
20
|
+
/** `mem_remember` appends the reason after this separator. */
|
|
21
|
+
const REASON_SEPARATOR = "\n\nContext: "
|
|
22
|
+
/** `mem_remember` writes under these synthetic sessions; captured turns use the host's session id. */
|
|
23
|
+
const REMEMBER_SESSION = /^(?:priv|team|space)-/
|
|
24
|
+
/** Rows recall hides as remember requests, in either format the plugin has written. */
|
|
25
|
+
const HIDDEN_FROM_RECALL = /^\s*(?:please remember this|\[mem:remember)/i
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Words that never make two texts the same fact: English function words and
|
|
29
|
+
* the boilerplate of memory writes ("the user", "please remember", "context").
|
|
30
|
+
* Generic on purpose: no product, project, or scenario word belongs here.
|
|
31
|
+
*/
|
|
32
|
+
const STOPWORDS = new Set(
|
|
33
|
+
(
|
|
34
|
+
"a an the and or but nor not no yes so if then than that this these those there here it its is are was were be been being " +
|
|
35
|
+
"am do does did done doing have has had having will would shall should can could may might must ought of to in on at by " +
|
|
36
|
+
"for from with without into onto over under about above below after before between during through until upon via per " +
|
|
37
|
+
"as also just only very really still even again ever never always often sometimes now today tomorrow yesterday soon " +
|
|
38
|
+
"i me my mine we us our ours you your yours he him his she her hers they them their theirs who whom whose which what " +
|
|
39
|
+
"when where why how all any some each every both either neither more most less least other others another same such " +
|
|
40
|
+
"own one ones use used uses using make made makes get got gets set sets need needs needed want wants like likes " +
|
|
41
|
+
"user users team teams agent agents project projects someone something thing things note noted notes remember " +
|
|
42
|
+
"remembered please later context understood fact facts decision decisions constraint constraints preference " +
|
|
43
|
+
"preferences procedure procedures stated says said tell told know known think keep kept e.g i.e etc"
|
|
44
|
+
).split(/\s+/),
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
/** Most anchors and stems kept per text, so one long turn cannot make the scan quadratic. */
|
|
48
|
+
const MAX_ANCHORS = 40
|
|
49
|
+
const MAX_STEMS = 60
|
|
50
|
+
/** Text kept per forgotten row; the specific tokens of a fact sit well inside it. */
|
|
51
|
+
export const MAX_FORGOTTEN_CHARS = 2_000
|
|
52
|
+
|
|
53
|
+
const LETTERS_OR_DIGITS = /[\p{L}\p{N}]/u
|
|
54
|
+
const QUOTED = /[`"“”]([^`"“”\n]{2,60})[`"“”]/gu
|
|
55
|
+
|
|
56
|
+
function stem(word) {
|
|
57
|
+
if (word.length >= 7) return word.slice(0, 6)
|
|
58
|
+
if (word.length >= 5 && word.endsWith("s")) return word.slice(0, -1)
|
|
59
|
+
return word
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function trimToken(raw) {
|
|
63
|
+
return raw.replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, "")
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Is this token specific enough to name a fact, and how specific? Strong:
|
|
68
|
+
* numbers and dates, identifiers (`checkout_v3`, `src/app.js`, `eu-west-1`),
|
|
69
|
+
* and mixed case (`PostgreSQL`). Weak: all capitals (`UTC`) and a capitalised
|
|
70
|
+
* word that does not start a sentence (a name). A lowercase hyphenated word
|
|
71
|
+
* ("short-lived") is ordinary English.
|
|
72
|
+
*
|
|
73
|
+
* @returns {"strong" | "weak" | null}
|
|
74
|
+
*/
|
|
75
|
+
function anchorTier(token, sentenceInitial) {
|
|
76
|
+
const lower = token.toLowerCase()
|
|
77
|
+
if (STOPWORDS.has(lower)) return null
|
|
78
|
+
if (/\p{N}/u.test(token)) return token.length >= 2 ? "strong" : null
|
|
79
|
+
if (/[\p{L}\p{N}][_./:@#$][\p{L}\p{N}]/u.test(token)) return "strong"
|
|
80
|
+
if (/\p{Ll}/u.test(token) && /\p{Lu}/u.test(token.slice(1))) return "strong"
|
|
81
|
+
if (/\p{Lu}/u.test(token.slice(1))) return "weak"
|
|
82
|
+
return !sentenceInitial && /^\p{Lu}/u.test(token) ? "weak" : null
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* The part of a stored row that is memory: a remember request without its
|
|
87
|
+
* fixed prefix and reason, an echo without "Understood.", anything else whole.
|
|
88
|
+
*
|
|
89
|
+
* @param {unknown} content
|
|
90
|
+
* @returns {{ kind: "wrapper" | "echo" | null, body: string, reason: string }}
|
|
91
|
+
*/
|
|
92
|
+
export function rememberParts(content) {
|
|
93
|
+
const raw = String(content ?? "")
|
|
94
|
+
const wrapper = raw.match(REMEMBER_WRAPPER)
|
|
95
|
+
if (wrapper) {
|
|
96
|
+
const rest = raw.slice(wrapper[0].length)
|
|
97
|
+
const at = rest.indexOf(REASON_SEPARATOR)
|
|
98
|
+
return at < 0
|
|
99
|
+
? { kind: "wrapper", body: rest.trim(), reason: "" }
|
|
100
|
+
: { kind: "wrapper", body: rest.slice(0, at).trim(), reason: rest.slice(at + REASON_SEPARATOR.length).trim() }
|
|
101
|
+
}
|
|
102
|
+
if (REMEMBER_ECHO.test(raw)) return { kind: "echo", body: raw.replace(REMEMBER_ECHO, "").trim(), reason: "" }
|
|
103
|
+
return { kind: null, body: raw.trim(), reason: "" }
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** True for a row recall drops as a remember request (src/recall.js). */
|
|
107
|
+
export function hiddenFromRecall(content) {
|
|
108
|
+
return HIDDEN_FROM_RECALL.test(String(content ?? ""))
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* The anchors (specific tokens; `strong` lists the strong ones, see
|
|
113
|
+
* `anchorTier`) and stems (other content words) of a text. A word is counted
|
|
114
|
+
* once: an anchor never also counts as a stem. Quoted text is a strong anchor.
|
|
115
|
+
*
|
|
116
|
+
* @param {unknown} text
|
|
117
|
+
* @returns {{ anchors: string[], strong: string[], stems: string[], lower: string }}
|
|
118
|
+
*/
|
|
119
|
+
export function copyFeatures(text) {
|
|
120
|
+
const source = String(text ?? "").slice(0, MAX_FORGOTTEN_CHARS)
|
|
121
|
+
const anchors = new Set()
|
|
122
|
+
const strong = new Set()
|
|
123
|
+
const quotedWords = new Set()
|
|
124
|
+
for (const match of source.matchAll(QUOTED)) {
|
|
125
|
+
const phrase = match[1].trim().toLowerCase().replace(/\s+/g, " ")
|
|
126
|
+
if (phrase.length < 2 || STOPWORDS.has(phrase) || !LETTERS_OR_DIGITS.test(phrase)) continue
|
|
127
|
+
anchors.add(phrase)
|
|
128
|
+
strong.add(phrase)
|
|
129
|
+
for (const word of phrase.split(" ")) quotedWords.add(trimToken(word))
|
|
130
|
+
}
|
|
131
|
+
const stems = new Set()
|
|
132
|
+
let sentenceInitial = true
|
|
133
|
+
for (const raw of source.split(/\s+/)) {
|
|
134
|
+
if (!raw) continue
|
|
135
|
+
const initial = sentenceInitial
|
|
136
|
+
sentenceInitial = /[.!?:]["'”)\]]*$/.test(raw)
|
|
137
|
+
const token = trimToken(raw)
|
|
138
|
+
if (token.length < 2) continue
|
|
139
|
+
const lower = token.toLowerCase()
|
|
140
|
+
if (quotedWords.has(lower)) continue
|
|
141
|
+
const tier = anchorTier(token, initial)
|
|
142
|
+
if (tier) {
|
|
143
|
+
anchors.add(lower)
|
|
144
|
+
if (tier === "strong") strong.add(lower)
|
|
145
|
+
continue
|
|
146
|
+
}
|
|
147
|
+
for (const word of lower.match(/\p{L}{4,}/gu) ?? []) {
|
|
148
|
+
if (!STOPWORDS.has(word)) stems.add(stem(word))
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
const kept = [...anchors].slice(0, MAX_ANCHORS)
|
|
152
|
+
return {
|
|
153
|
+
anchors: kept,
|
|
154
|
+
strong: kept.filter((anchor) => strong.has(anchor)),
|
|
155
|
+
stems: [...stems].slice(0, MAX_STEMS),
|
|
156
|
+
lower: source.toLowerCase(),
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* The features of a forgotten text without the text itself: what a later
|
|
162
|
+
* forget matches a reported copy against (see `matchCopy`).
|
|
163
|
+
*/
|
|
164
|
+
export function featuresOnly(features) {
|
|
165
|
+
return { anchors: features.anchors, strong: features.strong, stems: features.stems, lower: "" }
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
function escapeRegExp(value) {
|
|
169
|
+
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/** True when `lower` contains `anchor` as a whole token (not inside a longer word or number). */
|
|
173
|
+
function containsToken(lower, anchor) {
|
|
174
|
+
return new RegExp(`(?<![\\p{L}\\p{N}])${escapeRegExp(anchor)}(?![\\p{L}\\p{N}])`, "u").test(lower)
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/** The stems of every content word in a candidate row, anchors included. */
|
|
178
|
+
function candidateStems(lower) {
|
|
179
|
+
const stems = new Set()
|
|
180
|
+
for (const word of lower.match(/\p{L}{4,}/gu) ?? []) if (!STOPWORDS.has(word)) stems.add(stem(word))
|
|
181
|
+
return stems
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* Is `text` a copy of one of the forgotten texts?
|
|
186
|
+
*
|
|
187
|
+
* - The forgotten text has strong anchors: one of them and a second shared
|
|
188
|
+
* signal (another anchor or a content word), or two weak anchors (two names).
|
|
189
|
+
* - It has only weak anchors: one of them and a second shared signal.
|
|
190
|
+
* - It has none (no number, identifier, or name): at least two shared content
|
|
191
|
+
* words, and 60% of its content words.
|
|
192
|
+
*
|
|
193
|
+
* A weak anchor and topical words are not enough when the fact has a strong
|
|
194
|
+
* one: "export runs at 04:00 UTC" is not a copy of "export runs at 02:30 UTC".
|
|
195
|
+
*
|
|
196
|
+
* @param {Array<ReturnType<typeof copyFeatures>>} forgotten
|
|
197
|
+
* @param {unknown} text
|
|
198
|
+
* @returns {null | { anchors: string[], stems: string[] }}
|
|
199
|
+
*/
|
|
200
|
+
export function matchCopy(forgotten, text) {
|
|
201
|
+
const lower = String(text ?? "").toLowerCase()
|
|
202
|
+
if (!lower.trim()) return null
|
|
203
|
+
const stems = candidateStems(lower)
|
|
204
|
+
let best = null
|
|
205
|
+
for (const features of forgotten) {
|
|
206
|
+
const anchors = features.anchors.filter((anchor) => containsToken(lower, anchor))
|
|
207
|
+
const strong = anchors.filter((anchor) => features.strong.includes(anchor)).length
|
|
208
|
+
const weak = anchors.length - strong
|
|
209
|
+
const shared = features.stems.filter((value) => stems.has(value))
|
|
210
|
+
const signals = anchors.length + shared.length
|
|
211
|
+
let copy
|
|
212
|
+
if (features.strong.length > 0) copy = (strong >= 1 && signals >= 2) || weak >= 2
|
|
213
|
+
else if (features.anchors.length > 0) copy = weak >= 1 && signals >= 2
|
|
214
|
+
else copy = features.stems.length >= 2 && shared.length >= 2 && shared.length >= Math.ceil(0.6 * features.stems.length)
|
|
215
|
+
if (!copy) continue
|
|
216
|
+
const strength = anchors.length * 2 + shared.length
|
|
217
|
+
if (!best || strength > best.strength) best = { anchors, stems: shared, strength }
|
|
218
|
+
}
|
|
219
|
+
return best ? { anchors: best.anchors, stems: best.stems } : null
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/**
|
|
223
|
+
* Does `body` hold nothing but forgotten text? Every anchor of it appears in a
|
|
224
|
+
* forgotten text; a body without anchors needs 80% of its content words there.
|
|
225
|
+
* This decides whether the plugin's own remember row may be deleted with the
|
|
226
|
+
* fact, so it errs towards "no": a row that also names something else is
|
|
227
|
+
* reported, never deleted.
|
|
228
|
+
*
|
|
229
|
+
* @param {Array<ReturnType<typeof copyFeatures>>} forgotten
|
|
230
|
+
* @param {unknown} body
|
|
231
|
+
*/
|
|
232
|
+
export function holdsNothingElse(forgotten, body) {
|
|
233
|
+
const own = copyFeatures(body)
|
|
234
|
+
if (own.anchors.length === 0 && own.stems.length === 0) return false
|
|
235
|
+
if (own.anchors.length > 0) {
|
|
236
|
+
return own.anchors.every((anchor) =>
|
|
237
|
+
forgotten.some((features) => features.anchors.includes(anchor) || containsToken(features.lower, anchor)),
|
|
238
|
+
)
|
|
239
|
+
}
|
|
240
|
+
const known = new Set(forgotten.flatMap((features) => [...features.stems, ...candidateStems(features.lower)]))
|
|
241
|
+
const covered = own.stems.filter((value) => known.has(value)).length
|
|
242
|
+
return covered >= Math.ceil(0.8 * own.stems.length)
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/** True when the plugin wrote this row as a remember request (prefix and synthetic session). */
|
|
246
|
+
export function isRememberRequest(row) {
|
|
247
|
+
return REMEMBER_WRAPPER.test(String(row?.content ?? "")) && REMEMBER_SESSION.test(String(row?.session_id ?? ""))
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/** True when this row is the echo of a remember request: same synthetic session, same text. */
|
|
251
|
+
export function isRememberEcho(row, requests) {
|
|
252
|
+
const content = String(row?.content ?? "")
|
|
253
|
+
if (!REMEMBER_ECHO.test(content) || !REMEMBER_SESSION.test(String(row?.session_id ?? ""))) return false
|
|
254
|
+
const body = rememberParts(content).body
|
|
255
|
+
return requests.some((request) => request.session_id === row.session_id && rememberParts(request.content).body === body)
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/** A one-line window of `text` around the first matched token. */
|
|
259
|
+
export function copyExcerpt(text, terms = [], max = 120) {
|
|
260
|
+
const clean = String(text ?? "").replace(/\s+/g, " ").trim()
|
|
261
|
+
if (clean.length <= max) return clean
|
|
262
|
+
const lower = clean.toLowerCase()
|
|
263
|
+
let hit = -1
|
|
264
|
+
for (const term of terms) {
|
|
265
|
+
const index = lower.indexOf(String(term).toLowerCase())
|
|
266
|
+
if (index >= 0 && (hit < 0 || index < hit)) hit = index
|
|
267
|
+
}
|
|
268
|
+
const start = hit < 0 ? 0 : Math.max(0, Math.min(hit - Math.floor(max / 3), clean.length - max))
|
|
269
|
+
const end = Math.min(clean.length, start + max)
|
|
270
|
+
return `${start > 0 ? "…" : ""}${clean.slice(start, end).trim()}${end < clean.length ? "…" : ""}`
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* Read one newest-first listing, page by page, reusing pages already read
|
|
275
|
+
* (`pages` is shared by every pass over the same listing). Each page starts
|
|
276
|
+
* where the rows so far end, so a backend that returns fewer rows than asked
|
|
277
|
+
* costs a page, not a gap. The listing is complete at an empty page, or at a
|
|
278
|
+
* short one unless the backend's `total` says more rows exist.
|
|
279
|
+
*
|
|
280
|
+
* It stops early when `stop()` is true after a page, before a page once
|
|
281
|
+
* `deadline` has passed, at the first failed read, or after `maxPages`.
|
|
282
|
+
*
|
|
283
|
+
* @param {{
|
|
284
|
+
* pages: Map<number, { rows: any[], total?: number }>,
|
|
285
|
+
* readPage: (offset: number) => Promise<{ rows: any[], total?: number }>,
|
|
286
|
+
* pageSize: number,
|
|
287
|
+
* maxPages: number,
|
|
288
|
+
* deadline?: number,
|
|
289
|
+
* now?: () => number,
|
|
290
|
+
* onPage?: (rows: any[]) => void,
|
|
291
|
+
* stop?: () => boolean,
|
|
292
|
+
* }} input
|
|
293
|
+
* @returns {Promise<{ rows: any[], complete: boolean, truncated: boolean, timedOut: boolean, stopped: boolean, failed: string | null }>}
|
|
294
|
+
*/
|
|
295
|
+
export async function readListing({ pages, readPage, pageSize, maxPages, deadline = Infinity, now = Date.now, onPage, stop }) {
|
|
296
|
+
const rows = []
|
|
297
|
+
const end = (state) => ({ rows, complete: false, truncated: false, timedOut: false, stopped: false, failed: null, ...state })
|
|
298
|
+
for (let page = 0; page < maxPages; page += 1) {
|
|
299
|
+
let batch = pages.get(page)
|
|
300
|
+
if (!batch) {
|
|
301
|
+
if (now() > deadline) return end({ timedOut: true })
|
|
302
|
+
try {
|
|
303
|
+
const data = await readPage(rows.length)
|
|
304
|
+
batch = { rows: Array.isArray(data?.rows) ? data.rows : [], total: Number.isFinite(Number(data?.total)) ? Number(data.total) : undefined }
|
|
305
|
+
} catch (error) {
|
|
306
|
+
return end({ failed: String(error?.message ?? error) })
|
|
307
|
+
}
|
|
308
|
+
pages.set(page, batch)
|
|
309
|
+
}
|
|
310
|
+
rows.push(...batch.rows)
|
|
311
|
+
onPage?.(batch.rows)
|
|
312
|
+
// A full page never ends the listing, whatever `total` says. A short page
|
|
313
|
+
// does, unless the backend reports more rows than were read (it returned
|
|
314
|
+
// fewer than asked).
|
|
315
|
+
const complete =
|
|
316
|
+
batch.rows.length === 0 ||
|
|
317
|
+
(batch.rows.length < pageSize && (batch.total === undefined || rows.length >= batch.total))
|
|
318
|
+
if (complete) return end({ complete: true })
|
|
319
|
+
if (stop?.()) return end({ stopped: true })
|
|
320
|
+
}
|
|
321
|
+
return end({ truncated: true })
|
|
322
|
+
}
|