@xiangyixie/dsh-superagent 0.9.3 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/memory.js ADDED
@@ -0,0 +1,736 @@
1
+ /**
2
+ * Cross-run memory for dsh-superagent.
3
+ *
4
+ * A run is stateless by design: planner and workers start empty, and a finished
5
+ * run leaves only its state file behind. That is good for isolation and bad for
6
+ * continuity — the next run cannot know which approach was already rejected or
7
+ * which step left a trap. This module keeps a small, curated notebook between
8
+ * runs without turning the state directory into a growing pile of logs:
9
+ *
10
+ * - Exact activity history is *derived* from the run store at planning time
11
+ * (which runs existed, how they ended). Nothing is copied into memory for it.
12
+ * - Durable knowledge (decisions, pitfalls, next steps) is written only when a
13
+ * run's aggregator emits a fenced `memory` block. One short entry per topic.
14
+ * - Reads are bounded: at most `maxEntries` notes, each shortened, inside a
15
+ * fixed character budget. The notebook is a digest, never the full file.
16
+ * - A read can be a search: `recall` ranks notes against the goal's keywords and
17
+ * the requested tags, so an old relevant note beats a recent irrelevant one.
18
+ * The score is word hits (key 2, tags 1.5, text 0.5 per occurrence, capped at
19
+ * 3) normalised over at most five query terms, a tag-match ratio, and
20
+ * exponential time decay with a 14-day half-life. No embeddings, no index
21
+ * file, no dependencies.
22
+ * - A note is refuted, not deleted silently: a newer entry with the same `key`
23
+ * supersedes it, a `retracted` entry drops it, and the `superagent_memory`
24
+ * tool can list, set, or forget a key by hand.
25
+ * - Entries older than `maxAgeDays` are ignored and then compacted away, so a
26
+ * stale conclusion cannot become a permanent bias.
27
+ *
28
+ * Storage is `<stateDir>/memory/<scope>.jsonl`, one JSON object per line. The
29
+ * directory is created with a `.gitignore` containing `*`, so even a state
30
+ * directory that points inside a repository cannot be committed by accident.
31
+ * The package `files` whitelist never ships this directory.
32
+ *
33
+ * Writes append one short line with a single `write` call (atomic for a small
34
+ * buffer on a local filesystem) while holding a lock file, so a concurrent
35
+ * append and a compaction cannot drop each other's lines. Compaction replaces
36
+ * the file from a temp file under the same lock. Two runs finishing at the same
37
+ * time therefore serialise instead of overwriting; the per-key resolution is by
38
+ * timestamp, so order alone never changes a conclusion.
39
+ *
40
+ * @module dsh-superagent/memory
41
+ */
42
+ import { randomBytes } from 'node:crypto'
43
+ import fsp from 'node:fs/promises'
44
+ import path from 'node:path'
45
+ import { acquireFileLock } from './lock.js'
46
+
47
+ /** Kinds a note may declare. Anything else is stored as `note`. */
48
+ export const MEMORY_KINDS = new Set(['dispatch', 'decision', 'pitfall', 'next', 'note'])
49
+
50
+ /** Longest note kept; longer text is cut, not rejected. */
51
+ export const MAX_ENTRY_CHARS = 400
52
+
53
+ /** Longest key; keys are lowercase slugs so a later note can match an older one. */
54
+ export const MAX_KEY_CHARS = 48
55
+
56
+ /** Most tags one note may carry; tags are slugs, so a tag query is an exact match. */
57
+ export const MAX_TAGS = 6
58
+
59
+ /** Days after which a note's recency weight halves. */
60
+ export const RECENCY_HALF_LIFE_DAYS = 14
61
+
62
+ /**
63
+ * Recall weights. Keyword relevance dominates, an explicit tag request is a
64
+ * strong signal, and recency only breaks near-ties: a two-week-old note that
65
+ * names the goal outranks a fresh one that does not.
66
+ */
67
+ export const SCORE_WEIGHTS = Object.freeze({ keyword: 0.55, tag: 0.3, recency: 0.15 })
68
+
69
+ /** Per-term weights inside the keyword score. */
70
+ const KEY_TERM_WEIGHT = 2
71
+ const TAG_TERM_WEIGHT = 1.5
72
+ const TEXT_TERM_WEIGHT = 0.5
73
+ const MAX_TEXT_OCCURRENCES = 3
74
+
75
+ /**
76
+ * How many query terms the keyword score divides by. A goal is a sentence, not
77
+ * a keyword list: without this cap a twenty-word goal would spread its weight so
78
+ * thin that time decay outranked relevance.
79
+ */
80
+ const KEYWORD_TERM_BUDGET = 5
81
+
82
+ /**
83
+ * Words that carry no retrieval signal. Kept deliberately small: an aggressive
84
+ * list would need maintenance per language.
85
+ */
86
+ const STOPWORDS = new Set([
87
+ 'a', 'an', 'and', 'are', 'as', 'at', 'be', 'been', 'but', 'by', 'can', 'could', 'did', 'do', 'does', 'for',
88
+ 'from', 'had', 'has', 'have', 'how', 'if', 'in', 'into', 'is', 'it', 'its', 'just', 'not', 'of', 'on', 'or',
89
+ 'our', 'should', 'so', 'some', 'than', 'that', 'the', 'their', 'then', 'there', 'these', 'they', 'this',
90
+ 'to', 'too', 'use', 'using', 'was', 'were', 'what', 'when', 'which', 'will', 'with', 'would', 'you',
91
+ ])
92
+
93
+ /** Latin words and CJK runs; CJK has no spaces, so a run becomes bigrams. */
94
+ const TOKEN_PATTERN = /[a-z0-9]+|[\u3400-\u4dbf\u4e00-\u9fff]+/g
95
+ const CJK_PATTERN = /^[\u3400-\u4dbf\u4e00-\u9fff]/
96
+
97
+ /** File-level limits; crossing either one, or `maxEntries`, triggers compaction. */
98
+ const MAX_FILE_BYTES = 128 * 1024
99
+ const MAX_FILE_LINES = 400
100
+
101
+ /** How long a writer waits for another process's lock before failing open. */
102
+ const LOCK_RETRIES = 10
103
+ const LOCK_RETRY_DELAY_MS = 50
104
+
105
+ const DEFAULT_MAX_AGE_DAYS = 30
106
+ const DEFAULT_MAX_ENTRIES = 60
107
+ const DEFAULT_DIGEST_ENTRIES = 12
108
+ const DEFAULT_DIGEST_CHARS = 1600
109
+
110
+ const SCOPE_PATTERN = /^[a-z0-9][a-z0-9_-]{0,31}$/
111
+ const MEMORY_FENCE = /```memory[^\n]*\n([\s\S]*?)```/i
112
+
113
+ /**
114
+ * Coerce a configured scope into a safe file name. Anything unrecognised falls
115
+ * back to `default` instead of producing a path from user input.
116
+ * @param {unknown} value - configured scope.
117
+ * @returns {string} a scope that matches `SCOPE_PATTERN`.
118
+ */
119
+ export function normalizeScope(value) {
120
+ const text = typeof value === 'string' ? value.trim().toLowerCase() : ''
121
+ return SCOPE_PATTERN.test(text) ? text : 'default'
122
+ }
123
+
124
+ /**
125
+ * Remove the shapes of secret that a model is most likely to paste into a note:
126
+ * API keys, bearer headers, known token prefixes, long hex/base64 blobs, email
127
+ * addresses, and absolute home paths. Redaction happens before storage, so a
128
+ * file that is copied or backed up later cannot leak the original.
129
+ *
130
+ * @param {unknown} value - text to clean.
131
+ * @returns {string} text with secrets replaced by markers.
132
+ */
133
+ export function redact(value) {
134
+ if (typeof value !== 'string') return ''
135
+ return value
136
+ .replace(/\b(?:sk|pk|rk)-[A-Za-z0-9_-]{8,}/g, '[redacted-key]')
137
+ .replace(/\bBearer\s+[A-Za-z0-9._~+/=-]{8,}/gi, 'Bearer [redacted]')
138
+ .replace(/\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{16,}\b/g, '[redacted-token]')
139
+ .replace(/\bAKIA[0-9A-Z]{16}\b/g, '[redacted-key]')
140
+ .replace(/\b[A-Za-z0-9+/]{40,}={0,2}\b/g, '[redacted-blob]')
141
+ .replace(/\b[A-Fa-f0-9]{32,}\b/g, '[redacted-hex]')
142
+ .replace(/\b[\w.+-]+@[\w-]+\.[\w.-]+\b/g, '[redacted-email]')
143
+ .replace(/\b[A-Za-z]:\\Users\\[^\\\s]+/gi, '%USERPROFILE%')
144
+ .replace(/\/(?:home|Users)\/[^/\s]+/g, '~')
145
+ }
146
+
147
+ /**
148
+ * Turn arbitrary text into one bounded line: redacted, no newlines, trimmed.
149
+ * @param {unknown} value - source text.
150
+ * @param {number} [limit] - maximum characters.
151
+ * @returns {string} a single line.
152
+ */
153
+ export function cleanText(value, limit = MAX_ENTRY_CHARS) {
154
+ const text = redact(String(value ?? ''))
155
+ .replace(/\s+/g, ' ')
156
+ .trim()
157
+ if (!text) return ''
158
+ // Slice by code point so a cut never leaves a lone surrogate behind.
159
+ return [...text].slice(0, limit).join('')
160
+ }
161
+
162
+ /**
163
+ * @param {unknown} value - proposed key.
164
+ * @returns {string} lowercase slug, possibly empty.
165
+ */
166
+ export function slugKey(value) {
167
+ return redact(String(value ?? ''))
168
+ .toLowerCase()
169
+ .replace(/[^a-z0-9._-]+/g, '-')
170
+ .replace(/^[-.]+|[-.]+$/g, '')
171
+ .slice(0, MAX_KEY_CHARS)
172
+ }
173
+
174
+ /**
175
+ * True when `child` is `parent` or inside it. Used to refuse a memory directory
176
+ * that would end up inside the installed package.
177
+ * @param {string} child - absolute path to test.
178
+ * @param {string} parent - absolute path to compare against.
179
+ */
180
+ export function isInside(child, parent) {
181
+ const from = path.resolve(parent)
182
+ const to = path.resolve(child)
183
+ const relative = path.relative(from, to)
184
+ return relative === '' || (!relative.startsWith('..') && !path.isAbsolute(relative))
185
+ }
186
+
187
+ function normalizeEntry(item) {
188
+ if (!item || typeof item !== 'object') return null
189
+ const status = item.status === 'retracted' ? 'retracted' : 'active'
190
+ const kind = MEMORY_KINDS.has(item.kind) ? item.kind : 'note'
191
+ const text = status === 'retracted' ? '' : cleanText(item.text)
192
+ const key = slugKey(item.key) || slugKey(text.slice(0, 48)) || `${kind}-${randomBytes(3).toString('hex')}`
193
+ if (status === 'active' && !text) return null
194
+ return { key, kind, status, tags: normalizeTags(item.tags), text }
195
+ }
196
+
197
+ /**
198
+ * Split text into retrieval terms. Latin words are lowercased and dropped when
199
+ * shorter than two characters or in `STOPWORDS`; a CJK run is expanded into
200
+ * overlapping bigrams (`登录超时` → `登录`, `录超`, `超时`) because those languages
201
+ * do not delimit words. Duplicates are kept so a caller can count occurrences.
202
+ *
203
+ * @param {unknown} value - text to tokenise.
204
+ * @returns {string[]} terms, in order, possibly repeated.
205
+ */
206
+ export function tokenize(value) {
207
+ const text = typeof value === 'string' ? value.toLowerCase() : ''
208
+ const terms = []
209
+ for (const [match] of text.matchAll(TOKEN_PATTERN)) {
210
+ if (CJK_PATTERN.test(match)) {
211
+ const chars = [...match]
212
+ if (chars.length === 1) terms.push(chars[0])
213
+ else for (let index = 0; index < chars.length - 1; index++) terms.push(chars[index] + chars[index + 1])
214
+ continue
215
+ }
216
+ if (match.length < 2 || STOPWORDS.has(match)) continue
217
+ terms.push(match)
218
+ }
219
+ return terms
220
+ }
221
+
222
+ /**
223
+ * Coerce a tag list (array or comma-separated string) into unique slugs.
224
+ * @param {unknown} value - proposed tags or entry tags.
225
+ * @returns {string[]} at most `MAX_TAGS` slugs.
226
+ */
227
+ export function normalizeTags(value) {
228
+ const list = Array.isArray(value) ? value : typeof value === 'string' ? value.split(',') : []
229
+ const tags = []
230
+ for (const item of list) {
231
+ const tag = slugKey(item)
232
+ if (tag && !tags.includes(tag)) tags.push(tag)
233
+ if (tags.length >= MAX_TAGS) break
234
+ }
235
+ return tags
236
+ }
237
+
238
+ /**
239
+ * Build the term indexes a score needs. Doing it once per note keeps a search
240
+ * over a few hundred notes linear and allocation-light.
241
+ */
242
+ function indexOf(record) {
243
+ const key = new Set(tokenize(record.key))
244
+ // `normalizeTags` also tolerates a hand-edited string, which `.join` would not.
245
+ const tags = new Set(tokenize(normalizeTags(record.tags).join(' ')))
246
+ const text = new Map()
247
+ for (const term of tokenize(record.text)) text.set(term, (text.get(term) ?? 0) + 1)
248
+ return { key, tags, text }
249
+ }
250
+
251
+ /**
252
+ * Score one note against a query. The three parts are independent and each is
253
+ * in `[0, 1]`, so a caller can explain a ranking field by field:
254
+ *
255
+ * - `keyword`: per query term, 2 for a hit in the key, 1.5 in a tag, 0.5 per
256
+ * occurrence in the text (capped at 3), divided by `2 × min(terms, 5)` and
257
+ * clamped at 1. A key hit is therefore worth a full term, a term that only
258
+ * appears once in the body is worth a quarter, and a long goal cannot dilute
259
+ * a match below the recency term.
260
+ * - `tag`: share of the explicitly requested tags the note carries (0 when no
261
+ * tag was requested).
262
+ * - `recency`: `0.5 ** (ageDays / halfLifeDays)`, so 14 days halves it.
263
+ *
264
+ * @param {object} record - a note record.
265
+ * @param {{ query?: string, tags?: unknown, now?: number, halfLifeDays?: number }} [options] - query.
266
+ * @returns {{ score: number, keyword: number, tag: number, recency: number, ageDays: number, matchedTerms: string[] }}
267
+ */
268
+ export function scoreEntry(record, { query = '', tags = [], now = Date.now(), halfLifeDays = RECENCY_HALF_LIFE_DAYS } = {}) {
269
+ const index = indexOf(record)
270
+ const terms = [...new Set(tokenize(query))]
271
+ const wanted = normalizeTags(tags)
272
+ let hits = 0
273
+ const matchedTerms = []
274
+ for (const term of terms) {
275
+ const weight = (index.key.has(term) ? KEY_TERM_WEIGHT : 0)
276
+ + (index.tags.has(term) ? TAG_TERM_WEIGHT : 0)
277
+ + Math.min(index.text.get(term) ?? 0, MAX_TEXT_OCCURRENCES) * TEXT_TERM_WEIGHT
278
+ if (weight > 0) {
279
+ hits += weight
280
+ matchedTerms.push(term)
281
+ }
282
+ }
283
+ const keyword = terms.length > 0
284
+ ? Math.min(1, hits / (KEY_TERM_WEIGHT * Math.min(terms.length, KEYWORD_TERM_BUDGET)))
285
+ : 0
286
+ const noteTags = normalizeTags(record.tags)
287
+ const tagHits = wanted.filter(tag => noteTags.includes(tag)).length
288
+ const tag = wanted.length > 0 ? tagHits / wanted.length : 0
289
+ const ageDays = Math.max(0, (now - tsOf(record)) / 86400000)
290
+ const recency = Number.isFinite(ageDays) ? 0.5 ** (ageDays / Math.max(1, halfLifeDays)) : 0
291
+ const score = SCORE_WEIGHTS.keyword * keyword + SCORE_WEIGHTS.tag * tag + SCORE_WEIGHTS.recency * recency
292
+ return { score, keyword, tag, recency, ageDays, matchedTerms }
293
+ }
294
+
295
+ function tsOf(record) {
296
+ const value = Date.parse(record?.ts ?? '')
297
+ return Number.isFinite(value) ? value : 0
298
+ }
299
+
300
+ /**
301
+ * Resolve raw lines into the notes a reader should see.
302
+ *
303
+ * Records are applied oldest-first, so a newer entry wins on the same key and a
304
+ * `retracted` entry removes it. Timestamps are used rather than file order
305
+ * because a shared state directory can be written by two clocks. TTL is applied
306
+ * after resolution: because a tombstone is newer than its target, an expired
307
+ * tombstone cannot resurrect an older active entry.
308
+ *
309
+ * @param {object[]} records - parsed lines.
310
+ * @param {{ now?: number, maxAgeDays?: number }} [options] - clock and TTL.
311
+ * @returns {object[]} active entries, newest first.
312
+ */
313
+ export function effectiveEntries(records, { now = Date.now(), maxAgeDays = DEFAULT_MAX_AGE_DAYS } = {}) {
314
+ const active = new Map()
315
+ const ordered = records
316
+ .map((record, index) => ({ record, index }))
317
+ .filter(({ record }) => record && typeof record === 'object' && typeof record.key === 'string')
318
+ .sort((a, b) => (tsOf(a.record) - tsOf(b.record)) || (a.index - b.index))
319
+ for (const { record, index } of ordered) {
320
+ if (record.status === 'retracted') active.delete(record.key)
321
+ else active.set(record.key, { record, index })
322
+ }
323
+ const cutoff = maxAgeDays > 0 ? now - maxAgeDays * 86400000 : -Infinity
324
+ return [...active.values()]
325
+ .filter(({ record }) => tsOf(record) >= cutoff)
326
+ // Newest first. Notes written in the same millisecond keep reverse line
327
+ // order, so a later line is treated as newer rather than older.
328
+ .sort((a, b) => (tsOf(b.record) - tsOf(a.record)) || (b.index - a.index))
329
+ .map(({ record }) => record)
330
+ }
331
+
332
+ function ageFromDays(days) {
333
+ const whole = Math.max(0, Math.floor(days))
334
+ return whole === 0 ? 'today' : `${whole}d`
335
+ }
336
+
337
+ function capLine(text, limit) {
338
+ const points = [...text]
339
+ return points.length <= limit ? text : `${points.slice(0, Math.max(0, limit - 1)).join('')}…`
340
+ }
341
+
342
+ /**
343
+ * Pull the fenced `memory` block out of an aggregator report.
344
+ *
345
+ * The block is removed from the report whether or not it parses: the user asked
346
+ * for a report, not for the plugin's notebook syntax. A malformed block is
347
+ * reported through `error` so the run record shows why no note was written.
348
+ *
349
+ * @param {unknown} text - final report.
350
+ * @returns {{ report: string, entries: object[], found: boolean, error?: string }}
351
+ */
352
+ export function extractMemoryBlock(text) {
353
+ if (typeof text !== 'string' || !text) return { report: typeof text === 'string' ? text : '', entries: [], found: false }
354
+ const match = MEMORY_FENCE.exec(text)
355
+ if (!match) return { report: text, entries: [], found: false }
356
+ const report = `${text.slice(0, match.index)}${text.slice(match.index + match[0].length)}`
357
+ .replace(/\n{3,}/g, '\n\n')
358
+ .trim()
359
+ let parsed
360
+ try {
361
+ parsed = JSON.parse(match[1].trim())
362
+ } catch {
363
+ return { report, entries: [], found: true, error: 'the memory block was not valid JSON' }
364
+ }
365
+ const raw = Array.isArray(parsed) ? parsed : Array.isArray(parsed?.entries) ? parsed.entries : []
366
+ const entries = []
367
+ for (const item of raw) {
368
+ const entry = normalizeEntry(item)
369
+ if (entry) entries.push(entry)
370
+ }
371
+ if (raw.length > 0 && entries.length === 0) {
372
+ return { report, entries, found: true, error: 'the memory block had no usable entry' }
373
+ }
374
+ return { report, entries, found: true }
375
+ }
376
+
377
+ /**
378
+ * Build the memory notebook bound to one state root.
379
+ *
380
+ * @param {string} root - absolute state directory (`<root>/memory` is used).
381
+ * @param {object} [options] - collaborators.
382
+ * @param {(message: string) => void} [options.warn] - diagnostic sink.
383
+ * @param {() => number} [options.now] - clock, injectable for tests.
384
+ */
385
+ export function createMemory(root, { warn = () => {}, now = () => Date.now() } = {}) {
386
+ const dir = path.join(root, 'memory')
387
+ const fileOf = scope => path.join(dir, `${normalizeScope(scope)}.jsonl`)
388
+ const lockOf = scope => path.join(dir, `${normalizeScope(scope)}.lock`)
389
+ /** Per-scope write chain, so two settles in this process cannot interleave. */
390
+ const chains = new Map()
391
+
392
+ /**
393
+ * Create the directory and make it git-invisible even inside a repository.
394
+ * A `*` in a nested .gitignore ignores every file here, including the
395
+ * notebook, without editing the repository's own .gitignore.
396
+ */
397
+ async function initDir() {
398
+ await fsp.mkdir(dir, { recursive: true })
399
+ const ignore = path.join(dir, '.gitignore')
400
+ try {
401
+ await fsp.access(ignore)
402
+ } catch {
403
+ await fsp.writeFile(ignore, '*\n', 'utf8')
404
+ }
405
+ }
406
+
407
+ function serialize(scope, task) {
408
+ const previous = chains.get(scope) ?? Promise.resolve()
409
+ const next = previous.catch(() => {}).then(task)
410
+ chains.set(scope, next.catch(() => {}))
411
+ return next
412
+ }
413
+
414
+ async function readRecords(file) {
415
+ let text
416
+ try {
417
+ text = await fsp.readFile(file, 'utf8')
418
+ } catch (error) {
419
+ if (error?.code === 'ENOENT') return { records: [], bad: 0, bytes: 0 }
420
+ throw error
421
+ }
422
+ const records = []
423
+ let bad = 0
424
+ for (const line of text.split('\n')) {
425
+ const trimmed = line.trim()
426
+ if (!trimmed) continue
427
+ try {
428
+ const parsed = JSON.parse(trimmed)
429
+ if (parsed && typeof parsed === 'object') records.push(parsed)
430
+ else bad++
431
+ } catch {
432
+ // A process killed mid-append leaves one truncated line; skipping it is
433
+ // the whole reason for the JSONL format.
434
+ bad++
435
+ }
436
+ }
437
+ return { records, bad, bytes: Buffer.byteLength(text, 'utf8') }
438
+ }
439
+
440
+ /**
441
+ * Rewrite the file with the newest notes. The caller already holds the scope
442
+ * lock; acquiring it here as well would deadlock against that same writer.
443
+ */
444
+ async function compactLocked(scope, { maxEntries, maxAgeDays }) {
445
+ try {
446
+ const file = fileOf(scope)
447
+ const { records } = await readRecords(file)
448
+ // Keep the newest notes but write them oldest-first, so the file stays in
449
+ // append order and a same-millisecond tie is still resolved by line order.
450
+ const keep = effectiveEntries(records, { now: now(), maxAgeDays })
451
+ .slice(0, maxEntries)
452
+ .reverse()
453
+ const body = keep.map(record => `${JSON.stringify(record)}\n`).join('')
454
+ const tmp = path.join(dir, `.${normalizeScope(scope)}.${process.pid}.${Date.now()}.tmp`)
455
+ await fsp.writeFile(tmp, body, 'utf8')
456
+ await fsp.rename(tmp, file)
457
+ return true
458
+ } catch (error) {
459
+ warn(`cannot compact memory scope "${scope}": ${error?.message ?? error}`)
460
+ return false
461
+ }
462
+ }
463
+
464
+ async function append(scope, records) {
465
+ if (records.length === 0) return 0
466
+ const file = fileOf(scope)
467
+ const body = records.map(record => `${JSON.stringify(record)}\n`).join('')
468
+ // One write for the whole batch: a concurrent append either lands before or
469
+ // after this buffer, never inside it on a local filesystem.
470
+ const handle = await fsp.open(file, 'a')
471
+ try {
472
+ await handle.write(body)
473
+ } finally {
474
+ await handle.close()
475
+ }
476
+ return records.length
477
+ }
478
+
479
+ return {
480
+ root,
481
+ dir,
482
+ fileOf,
483
+
484
+ /** Create the directory and make it git-invisible even inside a repository. */
485
+ async init() {
486
+ await initDir()
487
+ },
488
+
489
+ /**
490
+ * Read the notebook's current notes.
491
+ * @param {{ scope?: string, maxAgeDays?: number }} [options] - read options.
492
+ */
493
+ async read({ scope = 'default', maxAgeDays = DEFAULT_MAX_AGE_DAYS } = {}) {
494
+ const file = fileOf(scope)
495
+ const { records, bad, bytes } = await readRecords(file)
496
+ return {
497
+ entries: effectiveEntries(records, { now: now(), maxAgeDays }),
498
+ bad,
499
+ bytes,
500
+ path: file,
501
+ }
502
+ },
503
+
504
+ /**
505
+ * Append the notes a finished run produced. Idempotent per run id: a resume
506
+ * settles the same run again and must not duplicate its notes.
507
+ *
508
+ * @param {{ scope?: string, runId: string, entries: object[], maxEntries?: number, maxAgeDays?: number }} options - note batch.
509
+ * @returns {Promise<{ written: number, skipped: number, reason: string | null }>}
510
+ */
511
+ async write({ scope = 'default', runId, entries, maxEntries = DEFAULT_MAX_ENTRIES, maxAgeDays = DEFAULT_MAX_AGE_DAYS } = {}) {
512
+ const normalized = (Array.isArray(entries) ? entries : []).map(normalizeEntry).filter(Boolean)
513
+ if (normalized.length === 0) return { written: 0, skipped: 0, reason: 'no usable entry' }
514
+ const key = normalizeScope(scope)
515
+ return serialize(key, async () => {
516
+ await initDir()
517
+ // One lock for the whole read/dedupe/append/compaction sequence. Without
518
+ // it a compaction in another process could rename the file between this
519
+ // writer's read and append and silently drop the new line.
520
+ const release = await acquireFileLock(lockOf(key), {
521
+ now,
522
+ retries: LOCK_RETRIES,
523
+ retryDelayMs: LOCK_RETRY_DELAY_MS,
524
+ })
525
+ if (!release) {
526
+ warn(`memory scope "${key}" is locked by another process; appended without compaction`)
527
+ }
528
+ try {
529
+ const file = fileOf(key)
530
+ const { records } = await readRecords(file)
531
+ if (runId && records.some(record => record?.runId === runId)) {
532
+ return { written: 0, skipped: normalized.length, reason: 'this run already wrote its notes' }
533
+ }
534
+ const ts = new Date(now()).toISOString()
535
+ const batch = normalized.map(entry => ({
536
+ v: 1,
537
+ id: randomBytes(4).toString('hex'),
538
+ ts,
539
+ runId: runId ?? null,
540
+ ...entry,
541
+ }))
542
+ const written = await append(key, batch)
543
+ let bytes = 0
544
+ try {
545
+ bytes = (await fsp.stat(file)).size
546
+ } catch {
547
+ // A missing stat only means compaction cannot be sized this round.
548
+ }
549
+ // `maxEntries` is a trigger, not just the size kept by an existing
550
+ // compaction: lowering it must bound the notebook on the next write.
551
+ const total = records.length + batch.length
552
+ if (release && (bytes > MAX_FILE_BYTES || total > MAX_FILE_LINES || total > maxEntries)) {
553
+ await compactLocked(key, { maxEntries, maxAgeDays })
554
+ }
555
+ return { written, skipped: normalized.length - written, reason: null }
556
+ } finally {
557
+ await release?.()
558
+ }
559
+ })
560
+ },
561
+
562
+ /**
563
+ * Record one note by hand (the `superagent_memory` tool's `set`).
564
+ * @param {{ scope?: string, key: string, kind?: string, text: string, tags?: unknown, runId?: string }} options - note.
565
+ */
566
+ async set({ scope = 'default', key, kind = 'note', text, tags = [], runId = null } = {}) {
567
+ const entry = normalizeEntry({ key, kind, text, tags })
568
+ if (!entry) return { written: 0, reason: 'a note needs non-empty text' }
569
+ const result = await this.write({ scope, runId: runId ?? `manual-${randomBytes(3).toString('hex')}`, entries: [entry] })
570
+ return { written: result.written, reason: result.reason }
571
+ },
572
+
573
+ /**
574
+ * Retract a note by key. The tombstone is itself a line, so a later
575
+ * compaction that drops the old note also drops the tombstone.
576
+ * @param {{ scope?: string, key: string }} options - key to forget.
577
+ */
578
+ async forget({ scope = 'default', key } = {}) {
579
+ const slug = slugKey(key)
580
+ if (!slug) return { forgotten: false, reason: 'a key is required' }
581
+ const result = await this.write({
582
+ scope,
583
+ runId: `forget-${randomBytes(3).toString('hex')}`,
584
+ entries: [{ key: slug, kind: 'note', status: 'retracted' }],
585
+ })
586
+ return { forgotten: result.written > 0, reason: result.reason }
587
+ },
588
+
589
+ /**
590
+ * Rank the notebook against a query. With no query every note is a hit and
591
+ * the order is recency, which is what a digest without a goal means.
592
+ *
593
+ * Retracted notes and expired ones never come back: `read` resolves
594
+ * tombstones and the TTL before scoring, and one key keeps only its newest
595
+ * value, so a corrected note is recalled once, in its corrected form.
596
+ *
597
+ * @param {{ scope?: string, query?: string, tags?: unknown, maxEntries?: number, maxAgeDays?: number, halfLifeDays?: number }} [options] - query.
598
+ * @returns {Promise<{ query: string, tags: string[], searching: boolean, total: number, matched: number, entries: object[] }>}
599
+ */
600
+ async recall({
601
+ scope = 'default',
602
+ query = '',
603
+ tags = [],
604
+ maxEntries = DEFAULT_DIGEST_ENTRIES,
605
+ maxAgeDays = DEFAULT_MAX_AGE_DAYS,
606
+ halfLifeDays = RECENCY_HALF_LIFE_DAYS,
607
+ } = {}) {
608
+ const at = now()
609
+ const { entries } = await this.read({ scope, maxAgeDays })
610
+ const wanted = normalizeTags(tags)
611
+ const text = typeof query === 'string' ? query : ''
612
+ const searching = tokenize(text).length > 0 || wanted.length > 0
613
+ const scored = entries.map(record => ({
614
+ record,
615
+ ...scoreEntry(record, { query: text, tags: wanted, now: at, halfLifeDays }),
616
+ }))
617
+ // A search hides notes that share no term with the goal; otherwise the
618
+ // digest would spend its budget on fresh but irrelevant entries.
619
+ const hits = searching ? scored.filter(item => item.keyword > 0 || item.tag > 0) : scored
620
+ // `sort` is stable, so equal scores keep `read`'s newest-first order.
621
+ hits.sort((a, b) => b.score - a.score)
622
+ return {
623
+ query: text,
624
+ tags: wanted,
625
+ searching,
626
+ total: entries.length,
627
+ matched: hits.length,
628
+ entries: hits.slice(0, Math.max(1, maxEntries)).map(({ record, score, keyword, tag, recency, ageDays, matchedTerms }) => ({
629
+ key: record.key,
630
+ kind: record.kind ?? 'note',
631
+ text: record.text ?? '',
632
+ tags: normalizeTags(record.tags),
633
+ score,
634
+ keyword,
635
+ tag,
636
+ recency,
637
+ ageDays,
638
+ matchedTerms,
639
+ source: {
640
+ runId: record.runId ?? null,
641
+ ts: record.ts ?? null,
642
+ // `read` already dropped retractions, so a recalled note is live.
643
+ retracted: false,
644
+ },
645
+ })),
646
+ }
647
+ },
648
+
649
+ /**
650
+ * Bounded digest injected into prompts. Never the whole file.
651
+ *
652
+ * With a query the lines are ranked by relevance and each one carries its
653
+ * source, so the planner can judge how much to trust it. Without a query —
654
+ * or when nothing matches — it degrades to the newest notes.
655
+ *
656
+ * @param {{ scope?: string, query?: string, tags?: unknown, maxEntries?: number, maxChars?: number, maxAgeDays?: number }} [options] - digest limits.
657
+ * @returns {Promise<string>} note lines, or an empty string.
658
+ */
659
+ async digest({
660
+ scope = 'default',
661
+ query = '',
662
+ tags = [],
663
+ maxEntries = DEFAULT_DIGEST_ENTRIES,
664
+ maxChars = DEFAULT_DIGEST_CHARS,
665
+ maxAgeDays = DEFAULT_MAX_AGE_DAYS,
666
+ } = {}) {
667
+ const recalled = await this.recall({ scope, query, tags, maxEntries, maxAgeDays })
668
+ let items = recalled.entries
669
+ let sourced = recalled.searching
670
+ if (items.length === 0 && recalled.searching) {
671
+ // An unfamiliar goal must not lose continuity: fall back to the newest
672
+ // notes, which is what this digest did before it could search.
673
+ const { entries } = await this.read({ scope, maxAgeDays })
674
+ items = entries.slice(0, maxEntries).map(record => ({
675
+ key: record.key,
676
+ kind: record.kind ?? 'note',
677
+ text: record.text ?? '',
678
+ ageDays: (now() - tsOf(record)) / 86400000,
679
+ source: { runId: record.runId ?? null, ts: record.ts ?? null, retracted: false },
680
+ }))
681
+ sourced = false
682
+ }
683
+ if (items.length === 0) return ''
684
+ const picked = items.slice(0, maxEntries)
685
+ const share = Math.max(60, Math.floor(maxChars / Math.max(1, picked.length)))
686
+ const lines = []
687
+ let used = 0
688
+ let omitted = 0
689
+ for (const item of picked) {
690
+ const header = sourced
691
+ ? `- [${ageFromDays(item.ageDays)}] ${item.kind} ${item.key} `
692
+ + `(run ${item.source.runId ?? 'manual'}, ${String(item.source.ts ?? '').slice(0, 10)}, `
693
+ + `${item.source.retracted ? 'retracted' : 'active'}): `
694
+ : `- [${ageFromDays(item.ageDays)}] ${item.kind} ${item.key}: `
695
+ const line = header + capLine(item.text, Math.max(40, share - header.length))
696
+ if (used + line.length + 1 > maxChars) {
697
+ omitted++
698
+ continue
699
+ }
700
+ lines.push(line)
701
+ used += line.length + 1
702
+ }
703
+ if (lines.length === 0) return ''
704
+ if (omitted > 0) {
705
+ lines.push(`- (+${omitted} more; call superagent_memory action="${sourced ? 'search' : 'list'}" to read them)`)
706
+ }
707
+ return lines.join('\n')
708
+ },
709
+
710
+ /**
711
+ * Rows for the `superagent_memory` tool.
712
+ * @param {{ scope?: string, limit?: number, maxAgeDays?: number }} [options] - list options.
713
+ */
714
+ async list({ scope = 'default', limit = 20, maxAgeDays = DEFAULT_MAX_AGE_DAYS } = {}) {
715
+ const { entries, bad, bytes, path: file } = await this.read({ scope, maxAgeDays })
716
+ return { entries: entries.slice(0, limit), total: entries.length, bad, bytes, path: file }
717
+ },
718
+
719
+ /**
720
+ * Size and count for `superagent_list`.
721
+ * @param {{ scope?: string, maxAgeDays?: number }} [options] - stats options.
722
+ */
723
+ async stats({ scope = 'default', maxAgeDays = DEFAULT_MAX_AGE_DAYS } = {}) {
724
+ const { entries, bytes, path: file } = await this.read({ scope, maxAgeDays })
725
+ return { entries: entries.length, bytes, path: file }
726
+ },
727
+
728
+ /** Defaults exposed so the orchestrator and tests agree on one set of limits. */
729
+ defaults: {
730
+ maxEntries: DEFAULT_MAX_ENTRIES,
731
+ maxAgeDays: DEFAULT_MAX_AGE_DAYS,
732
+ digestEntries: DEFAULT_DIGEST_ENTRIES,
733
+ digestChars: DEFAULT_DIGEST_CHARS,
734
+ },
735
+ }
736
+ }