@xiangyixie/dsh-superagent 0.11.0 → 0.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +97 -10
- package/README.md +322 -28
- package/bin/archive.js +185 -185
- package/bin/doctor.js +84 -84
- package/bin/sync.js +187 -0
- package/bin/workbench.js +98 -0
- package/docs/policy.example.json +38 -0
- package/lib/audit.js +555 -0
- package/lib/cost.js +267 -267
- package/lib/doctor.js +878 -876
- package/lib/guard.js +841 -0
- package/lib/index.js +2031 -1547
- package/lib/inject.js +138 -0
- package/lib/ledger.js +270 -270
- package/lib/memory.js +736 -736
- package/lib/orchestrator.js +3851 -3312
- package/lib/policy.js +739 -0
- package/lib/report.js +1068 -1068
- package/lib/store.js +873 -873
- package/lib/sync.js +456 -0
- package/lib/templates.js +690 -690
- package/lib/watch.js +810 -810
- package/lib/workbench.js +1119 -0
- package/lib/workspace.js +433 -433
- package/package.json +92 -80
- package/public/workbench.html +836 -0
package/lib/memory.js
CHANGED
|
@@ -1,736 +1,736 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Cross-run memory for dsh-superagent.
|
|
3
|
-
*
|
|
4
|
-
* A run is stateless by design: planner and workers start empty, and a finished
|
|
5
|
-
* run leaves only its state file behind. That is good for isolation and bad for
|
|
6
|
-
* continuity — the next run cannot know which approach was already rejected or
|
|
7
|
-
* which step left a trap. This module keeps a small, curated notebook between
|
|
8
|
-
* runs without turning the state directory into a growing pile of logs:
|
|
9
|
-
*
|
|
10
|
-
* - Exact activity history is *derived* from the run store at planning time
|
|
11
|
-
* (which runs existed, how they ended). Nothing is copied into memory for it.
|
|
12
|
-
* - Durable knowledge (decisions, pitfalls, next steps) is written only when a
|
|
13
|
-
* run's aggregator emits a fenced `memory` block. One short entry per topic.
|
|
14
|
-
* - Reads are bounded: at most `maxEntries` notes, each shortened, inside a
|
|
15
|
-
* fixed character budget. The notebook is a digest, never the full file.
|
|
16
|
-
* - A read can be a search: `recall` ranks notes against the goal's keywords and
|
|
17
|
-
* the requested tags, so an old relevant note beats a recent irrelevant one.
|
|
18
|
-
* The score is word hits (key 2, tags 1.5, text 0.5 per occurrence, capped at
|
|
19
|
-
* 3) normalised over at most five query terms, a tag-match ratio, and
|
|
20
|
-
* exponential time decay with a 14-day half-life. No embeddings, no index
|
|
21
|
-
* file, no dependencies.
|
|
22
|
-
* - A note is refuted, not deleted silently: a newer entry with the same `key`
|
|
23
|
-
* supersedes it, a `retracted` entry drops it, and the `superagent_memory`
|
|
24
|
-
* tool can list, set, or forget a key by hand.
|
|
25
|
-
* - Entries older than `maxAgeDays` are ignored and then compacted away, so a
|
|
26
|
-
* stale conclusion cannot become a permanent bias.
|
|
27
|
-
*
|
|
28
|
-
* Storage is `<stateDir>/memory/<scope>.jsonl`, one JSON object per line. The
|
|
29
|
-
* directory is created with a `.gitignore` containing `*`, so even a state
|
|
30
|
-
* directory that points inside a repository cannot be committed by accident.
|
|
31
|
-
* The package `files` whitelist never ships this directory.
|
|
32
|
-
*
|
|
33
|
-
* Writes append one short line with a single `write` call (atomic for a small
|
|
34
|
-
* buffer on a local filesystem) while holding a lock file, so a concurrent
|
|
35
|
-
* append and a compaction cannot drop each other's lines. Compaction replaces
|
|
36
|
-
* the file from a temp file under the same lock. Two runs finishing at the same
|
|
37
|
-
* time therefore serialise instead of overwriting; the per-key resolution is by
|
|
38
|
-
* timestamp, so order alone never changes a conclusion.
|
|
39
|
-
*
|
|
40
|
-
* @module dsh-superagent/memory
|
|
41
|
-
*/
|
|
42
|
-
import { randomBytes } from 'node:crypto'
|
|
43
|
-
import fsp from 'node:fs/promises'
|
|
44
|
-
import path from 'node:path'
|
|
45
|
-
import { acquireFileLock } from './lock.js'
|
|
46
|
-
|
|
47
|
-
/** Kinds a note may declare. Anything else is stored as `note`. */
|
|
48
|
-
export const MEMORY_KINDS = new Set(['dispatch', 'decision', 'pitfall', 'next', 'note'])
|
|
49
|
-
|
|
50
|
-
/** Longest note kept; longer text is cut, not rejected. */
|
|
51
|
-
export const MAX_ENTRY_CHARS = 400
|
|
52
|
-
|
|
53
|
-
/** Longest key; keys are lowercase slugs so a later note can match an older one. */
|
|
54
|
-
export const MAX_KEY_CHARS = 48
|
|
55
|
-
|
|
56
|
-
/** Most tags one note may carry; tags are slugs, so a tag query is an exact match. */
|
|
57
|
-
export const MAX_TAGS = 6
|
|
58
|
-
|
|
59
|
-
/** Days after which a note's recency weight halves. */
|
|
60
|
-
export const RECENCY_HALF_LIFE_DAYS = 14
|
|
61
|
-
|
|
62
|
-
/**
|
|
63
|
-
* Recall weights. Keyword relevance dominates, an explicit tag request is a
|
|
64
|
-
* strong signal, and recency only breaks near-ties: a two-week-old note that
|
|
65
|
-
* names the goal outranks a fresh one that does not.
|
|
66
|
-
*/
|
|
67
|
-
export const SCORE_WEIGHTS = Object.freeze({ keyword: 0.55, tag: 0.3, recency: 0.15 })
|
|
68
|
-
|
|
69
|
-
/** Per-term weights inside the keyword score. */
|
|
70
|
-
const KEY_TERM_WEIGHT = 2
|
|
71
|
-
const TAG_TERM_WEIGHT = 1.5
|
|
72
|
-
const TEXT_TERM_WEIGHT = 0.5
|
|
73
|
-
const MAX_TEXT_OCCURRENCES = 3
|
|
74
|
-
|
|
75
|
-
/**
|
|
76
|
-
* How many query terms the keyword score divides by. A goal is a sentence, not
|
|
77
|
-
* a keyword list: without this cap a twenty-word goal would spread its weight so
|
|
78
|
-
* thin that time decay outranked relevance.
|
|
79
|
-
*/
|
|
80
|
-
const KEYWORD_TERM_BUDGET = 5
|
|
81
|
-
|
|
82
|
-
/**
|
|
83
|
-
* Words that carry no retrieval signal. Kept deliberately small: an aggressive
|
|
84
|
-
* list would need maintenance per language.
|
|
85
|
-
*/
|
|
86
|
-
const STOPWORDS = new Set([
|
|
87
|
-
'a', 'an', 'and', 'are', 'as', 'at', 'be', 'been', 'but', 'by', 'can', 'could', 'did', 'do', 'does', 'for',
|
|
88
|
-
'from', 'had', 'has', 'have', 'how', 'if', 'in', 'into', 'is', 'it', 'its', 'just', 'not', 'of', 'on', 'or',
|
|
89
|
-
'our', 'should', 'so', 'some', 'than', 'that', 'the', 'their', 'then', 'there', 'these', 'they', 'this',
|
|
90
|
-
'to', 'too', 'use', 'using', 'was', 'were', 'what', 'when', 'which', 'will', 'with', 'would', 'you',
|
|
91
|
-
])
|
|
92
|
-
|
|
93
|
-
/** Latin words and CJK runs; CJK has no spaces, so a run becomes bigrams. */
|
|
94
|
-
const TOKEN_PATTERN = /[a-z0-9]+|[\u3400-\u4dbf\u4e00-\u9fff]+/g
|
|
95
|
-
const CJK_PATTERN = /^[\u3400-\u4dbf\u4e00-\u9fff]/
|
|
96
|
-
|
|
97
|
-
/** File-level limits; crossing either one, or `maxEntries`, triggers compaction. */
|
|
98
|
-
const MAX_FILE_BYTES = 128 * 1024
|
|
99
|
-
const MAX_FILE_LINES = 400
|
|
100
|
-
|
|
101
|
-
/** How long a writer waits for another process's lock before failing open. */
|
|
102
|
-
const LOCK_RETRIES = 10
|
|
103
|
-
const LOCK_RETRY_DELAY_MS = 50
|
|
104
|
-
|
|
105
|
-
const DEFAULT_MAX_AGE_DAYS = 30
|
|
106
|
-
const DEFAULT_MAX_ENTRIES = 60
|
|
107
|
-
const DEFAULT_DIGEST_ENTRIES = 12
|
|
108
|
-
const DEFAULT_DIGEST_CHARS = 1600
|
|
109
|
-
|
|
110
|
-
const SCOPE_PATTERN = /^[a-z0-9][a-z0-9_-]{0,31}$/
|
|
111
|
-
const MEMORY_FENCE = /```memory[^\n]*\n([\s\S]*?)```/i
|
|
112
|
-
|
|
113
|
-
/**
|
|
114
|
-
* Coerce a configured scope into a safe file name. Anything unrecognised falls
|
|
115
|
-
* back to `default` instead of producing a path from user input.
|
|
116
|
-
* @param {unknown} value - configured scope.
|
|
117
|
-
* @returns {string} a scope that matches `SCOPE_PATTERN`.
|
|
118
|
-
*/
|
|
119
|
-
export function normalizeScope(value) {
|
|
120
|
-
const text = typeof value === 'string' ? value.trim().toLowerCase() : ''
|
|
121
|
-
return SCOPE_PATTERN.test(text) ? text : 'default'
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
/**
|
|
125
|
-
* Remove the shapes of secret that a model is most likely to paste into a note:
|
|
126
|
-
* API keys, bearer headers, known token prefixes, long hex/base64 blobs, email
|
|
127
|
-
* addresses, and absolute home paths. Redaction happens before storage, so a
|
|
128
|
-
* file that is copied or backed up later cannot leak the original.
|
|
129
|
-
*
|
|
130
|
-
* @param {unknown} value - text to clean.
|
|
131
|
-
* @returns {string} text with secrets replaced by markers.
|
|
132
|
-
*/
|
|
133
|
-
export function redact(value) {
|
|
134
|
-
if (typeof value !== 'string') return ''
|
|
135
|
-
return value
|
|
136
|
-
.replace(/\b(?:sk|pk|rk)-[A-Za-z0-9_-]{8,}/g, '[redacted-key]')
|
|
137
|
-
.replace(/\bBearer\s+[A-Za-z0-9._~+/=-]{8,}/gi, 'Bearer [redacted]')
|
|
138
|
-
.replace(/\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{16,}\b/g, '[redacted-token]')
|
|
139
|
-
.replace(/\bAKIA[0-9A-Z]{16}\b/g, '[redacted-key]')
|
|
140
|
-
.replace(/\b[A-Za-z0-9+/]{40,}={0,2}\b/g, '[redacted-blob]')
|
|
141
|
-
.replace(/\b[A-Fa-f0-9]{32,}\b/g, '[redacted-hex]')
|
|
142
|
-
.replace(/\b[\w.+-]+@[\w-]+\.[\w.-]+\b/g, '[redacted-email]')
|
|
143
|
-
.replace(/\b[A-Za-z]:\\Users\\[^\\\s]+/gi, '%USERPROFILE%')
|
|
144
|
-
.replace(/\/(?:home|Users)\/[^/\s]+/g, '~')
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
/**
|
|
148
|
-
* Turn arbitrary text into one bounded line: redacted, no newlines, trimmed.
|
|
149
|
-
* @param {unknown} value - source text.
|
|
150
|
-
* @param {number} [limit] - maximum characters.
|
|
151
|
-
* @returns {string} a single line.
|
|
152
|
-
*/
|
|
153
|
-
export function cleanText(value, limit = MAX_ENTRY_CHARS) {
|
|
154
|
-
const text = redact(String(value ?? ''))
|
|
155
|
-
.replace(/\s+/g, ' ')
|
|
156
|
-
.trim()
|
|
157
|
-
if (!text) return ''
|
|
158
|
-
// Slice by code point so a cut never leaves a lone surrogate behind.
|
|
159
|
-
return [...text].slice(0, limit).join('')
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
/**
|
|
163
|
-
* @param {unknown} value - proposed key.
|
|
164
|
-
* @returns {string} lowercase slug, possibly empty.
|
|
165
|
-
*/
|
|
166
|
-
export function slugKey(value) {
|
|
167
|
-
return redact(String(value ?? ''))
|
|
168
|
-
.toLowerCase()
|
|
169
|
-
.replace(/[^a-z0-9._-]+/g, '-')
|
|
170
|
-
.replace(/^[-.]+|[-.]+$/g, '')
|
|
171
|
-
.slice(0, MAX_KEY_CHARS)
|
|
172
|
-
}
|
|
173
|
-
|
|
174
|
-
/**
|
|
175
|
-
* True when `child` is `parent` or inside it. Used to refuse a memory directory
|
|
176
|
-
* that would end up inside the installed package.
|
|
177
|
-
* @param {string} child - absolute path to test.
|
|
178
|
-
* @param {string} parent - absolute path to compare against.
|
|
179
|
-
*/
|
|
180
|
-
export function isInside(child, parent) {
|
|
181
|
-
const from = path.resolve(parent)
|
|
182
|
-
const to = path.resolve(child)
|
|
183
|
-
const relative = path.relative(from, to)
|
|
184
|
-
return relative === '' || (!relative.startsWith('..') && !path.isAbsolute(relative))
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
function normalizeEntry(item) {
|
|
188
|
-
if (!item || typeof item !== 'object') return null
|
|
189
|
-
const status = item.status === 'retracted' ? 'retracted' : 'active'
|
|
190
|
-
const kind = MEMORY_KINDS.has(item.kind) ? item.kind : 'note'
|
|
191
|
-
const text = status === 'retracted' ? '' : cleanText(item.text)
|
|
192
|
-
const key = slugKey(item.key) || slugKey(text.slice(0, 48)) || `${kind}-${randomBytes(3).toString('hex')}`
|
|
193
|
-
if (status === 'active' && !text) return null
|
|
194
|
-
return { key, kind, status, tags: normalizeTags(item.tags), text }
|
|
195
|
-
}
|
|
196
|
-
|
|
197
|
-
/**
|
|
198
|
-
* Split text into retrieval terms. Latin words are lowercased and dropped when
|
|
199
|
-
* shorter than two characters or in `STOPWORDS`; a CJK run is expanded into
|
|
200
|
-
* overlapping bigrams (`登录超时` → `登录`, `录超`, `超时`) because those languages
|
|
201
|
-
* do not delimit words. Duplicates are kept so a caller can count occurrences.
|
|
202
|
-
*
|
|
203
|
-
* @param {unknown} value - text to tokenise.
|
|
204
|
-
* @returns {string[]} terms, in order, possibly repeated.
|
|
205
|
-
*/
|
|
206
|
-
export function tokenize(value) {
|
|
207
|
-
const text = typeof value === 'string' ? value.toLowerCase() : ''
|
|
208
|
-
const terms = []
|
|
209
|
-
for (const [match] of text.matchAll(TOKEN_PATTERN)) {
|
|
210
|
-
if (CJK_PATTERN.test(match)) {
|
|
211
|
-
const chars = [...match]
|
|
212
|
-
if (chars.length === 1) terms.push(chars[0])
|
|
213
|
-
else for (let index = 0; index < chars.length - 1; index++) terms.push(chars[index] + chars[index + 1])
|
|
214
|
-
continue
|
|
215
|
-
}
|
|
216
|
-
if (match.length < 2 || STOPWORDS.has(match)) continue
|
|
217
|
-
terms.push(match)
|
|
218
|
-
}
|
|
219
|
-
return terms
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
/**
|
|
223
|
-
* Coerce a tag list (array or comma-separated string) into unique slugs.
|
|
224
|
-
* @param {unknown} value - proposed tags or entry tags.
|
|
225
|
-
* @returns {string[]} at most `MAX_TAGS` slugs.
|
|
226
|
-
*/
|
|
227
|
-
export function normalizeTags(value) {
|
|
228
|
-
const list = Array.isArray(value) ? value : typeof value === 'string' ? value.split(',') : []
|
|
229
|
-
const tags = []
|
|
230
|
-
for (const item of list) {
|
|
231
|
-
const tag = slugKey(item)
|
|
232
|
-
if (tag && !tags.includes(tag)) tags.push(tag)
|
|
233
|
-
if (tags.length >= MAX_TAGS) break
|
|
234
|
-
}
|
|
235
|
-
return tags
|
|
236
|
-
}
|
|
237
|
-
|
|
238
|
-
/**
|
|
239
|
-
* Build the term indexes a score needs. Doing it once per note keeps a search
|
|
240
|
-
* over a few hundred notes linear and allocation-light.
|
|
241
|
-
*/
|
|
242
|
-
function indexOf(record) {
|
|
243
|
-
const key = new Set(tokenize(record.key))
|
|
244
|
-
// `normalizeTags` also tolerates a hand-edited string, which `.join` would not.
|
|
245
|
-
const tags = new Set(tokenize(normalizeTags(record.tags).join(' ')))
|
|
246
|
-
const text = new Map()
|
|
247
|
-
for (const term of tokenize(record.text)) text.set(term, (text.get(term) ?? 0) + 1)
|
|
248
|
-
return { key, tags, text }
|
|
249
|
-
}
|
|
250
|
-
|
|
251
|
-
/**
|
|
252
|
-
* Score one note against a query. The three parts are independent and each is
|
|
253
|
-
* in `[0, 1]`, so a caller can explain a ranking field by field:
|
|
254
|
-
*
|
|
255
|
-
* - `keyword`: per query term, 2 for a hit in the key, 1.5 in a tag, 0.5 per
|
|
256
|
-
* occurrence in the text (capped at 3), divided by `2 × min(terms, 5)` and
|
|
257
|
-
* clamped at 1. A key hit is therefore worth a full term, a term that only
|
|
258
|
-
* appears once in the body is worth a quarter, and a long goal cannot dilute
|
|
259
|
-
* a match below the recency term.
|
|
260
|
-
* - `tag`: share of the explicitly requested tags the note carries (0 when no
|
|
261
|
-
* tag was requested).
|
|
262
|
-
* - `recency`: `0.5 ** (ageDays / halfLifeDays)`, so 14 days halves it.
|
|
263
|
-
*
|
|
264
|
-
* @param {object} record - a note record.
|
|
265
|
-
* @param {{ query?: string, tags?: unknown, now?: number, halfLifeDays?: number }} [options] - query.
|
|
266
|
-
* @returns {{ score: number, keyword: number, tag: number, recency: number, ageDays: number, matchedTerms: string[] }}
|
|
267
|
-
*/
|
|
268
|
-
export function scoreEntry(record, { query = '', tags = [], now = Date.now(), halfLifeDays = RECENCY_HALF_LIFE_DAYS } = {}) {
|
|
269
|
-
const index = indexOf(record)
|
|
270
|
-
const terms = [...new Set(tokenize(query))]
|
|
271
|
-
const wanted = normalizeTags(tags)
|
|
272
|
-
let hits = 0
|
|
273
|
-
const matchedTerms = []
|
|
274
|
-
for (const term of terms) {
|
|
275
|
-
const weight = (index.key.has(term) ? KEY_TERM_WEIGHT : 0)
|
|
276
|
-
+ (index.tags.has(term) ? TAG_TERM_WEIGHT : 0)
|
|
277
|
-
+ Math.min(index.text.get(term) ?? 0, MAX_TEXT_OCCURRENCES) * TEXT_TERM_WEIGHT
|
|
278
|
-
if (weight > 0) {
|
|
279
|
-
hits += weight
|
|
280
|
-
matchedTerms.push(term)
|
|
281
|
-
}
|
|
282
|
-
}
|
|
283
|
-
const keyword = terms.length > 0
|
|
284
|
-
? Math.min(1, hits / (KEY_TERM_WEIGHT * Math.min(terms.length, KEYWORD_TERM_BUDGET)))
|
|
285
|
-
: 0
|
|
286
|
-
const noteTags = normalizeTags(record.tags)
|
|
287
|
-
const tagHits = wanted.filter(tag => noteTags.includes(tag)).length
|
|
288
|
-
const tag = wanted.length > 0 ? tagHits / wanted.length : 0
|
|
289
|
-
const ageDays = Math.max(0, (now - tsOf(record)) / 86400000)
|
|
290
|
-
const recency = Number.isFinite(ageDays) ? 0.5 ** (ageDays / Math.max(1, halfLifeDays)) : 0
|
|
291
|
-
const score = SCORE_WEIGHTS.keyword * keyword + SCORE_WEIGHTS.tag * tag + SCORE_WEIGHTS.recency * recency
|
|
292
|
-
return { score, keyword, tag, recency, ageDays, matchedTerms }
|
|
293
|
-
}
|
|
294
|
-
|
|
295
|
-
function tsOf(record) {
|
|
296
|
-
const value = Date.parse(record?.ts ?? '')
|
|
297
|
-
return Number.isFinite(value) ? value : 0
|
|
298
|
-
}
|
|
299
|
-
|
|
300
|
-
/**
|
|
301
|
-
* Resolve raw lines into the notes a reader should see.
|
|
302
|
-
*
|
|
303
|
-
* Records are applied oldest-first, so a newer entry wins on the same key and a
|
|
304
|
-
* `retracted` entry removes it. Timestamps are used rather than file order
|
|
305
|
-
* because a shared state directory can be written by two clocks. TTL is applied
|
|
306
|
-
* after resolution: because a tombstone is newer than its target, an expired
|
|
307
|
-
* tombstone cannot resurrect an older active entry.
|
|
308
|
-
*
|
|
309
|
-
* @param {object[]} records - parsed lines.
|
|
310
|
-
* @param {{ now?: number, maxAgeDays?: number }} [options] - clock and TTL.
|
|
311
|
-
* @returns {object[]} active entries, newest first.
|
|
312
|
-
*/
|
|
313
|
-
export function effectiveEntries(records, { now = Date.now(), maxAgeDays = DEFAULT_MAX_AGE_DAYS } = {}) {
|
|
314
|
-
const active = new Map()
|
|
315
|
-
const ordered = records
|
|
316
|
-
.map((record, index) => ({ record, index }))
|
|
317
|
-
.filter(({ record }) => record && typeof record === 'object' && typeof record.key === 'string')
|
|
318
|
-
.sort((a, b) => (tsOf(a.record) - tsOf(b.record)) || (a.index - b.index))
|
|
319
|
-
for (const { record, index } of ordered) {
|
|
320
|
-
if (record.status === 'retracted') active.delete(record.key)
|
|
321
|
-
else active.set(record.key, { record, index })
|
|
322
|
-
}
|
|
323
|
-
const cutoff = maxAgeDays > 0 ? now - maxAgeDays * 86400000 : -Infinity
|
|
324
|
-
return [...active.values()]
|
|
325
|
-
.filter(({ record }) => tsOf(record) >= cutoff)
|
|
326
|
-
// Newest first. Notes written in the same millisecond keep reverse line
|
|
327
|
-
// order, so a later line is treated as newer rather than older.
|
|
328
|
-
.sort((a, b) => (tsOf(b.record) - tsOf(a.record)) || (b.index - a.index))
|
|
329
|
-
.map(({ record }) => record)
|
|
330
|
-
}
|
|
331
|
-
|
|
332
|
-
function ageFromDays(days) {
|
|
333
|
-
const whole = Math.max(0, Math.floor(days))
|
|
334
|
-
return whole === 0 ? 'today' : `${whole}d`
|
|
335
|
-
}
|
|
336
|
-
|
|
337
|
-
function capLine(text, limit) {
|
|
338
|
-
const points = [...text]
|
|
339
|
-
return points.length <= limit ? text : `${points.slice(0, Math.max(0, limit - 1)).join('')}…`
|
|
340
|
-
}
|
|
341
|
-
|
|
342
|
-
/**
|
|
343
|
-
* Pull the fenced `memory` block out of an aggregator report.
|
|
344
|
-
*
|
|
345
|
-
* The block is removed from the report whether or not it parses: the user asked
|
|
346
|
-
* for a report, not for the plugin's notebook syntax. A malformed block is
|
|
347
|
-
* reported through `error` so the run record shows why no note was written.
|
|
348
|
-
*
|
|
349
|
-
* @param {unknown} text - final report.
|
|
350
|
-
* @returns {{ report: string, entries: object[], found: boolean, error?: string }}
|
|
351
|
-
*/
|
|
352
|
-
export function extractMemoryBlock(text) {
|
|
353
|
-
if (typeof text !== 'string' || !text) return { report: typeof text === 'string' ? text : '', entries: [], found: false }
|
|
354
|
-
const match = MEMORY_FENCE.exec(text)
|
|
355
|
-
if (!match) return { report: text, entries: [], found: false }
|
|
356
|
-
const report = `${text.slice(0, match.index)}${text.slice(match.index + match[0].length)}`
|
|
357
|
-
.replace(/\n{3,}/g, '\n\n')
|
|
358
|
-
.trim()
|
|
359
|
-
let parsed
|
|
360
|
-
try {
|
|
361
|
-
parsed = JSON.parse(match[1].trim())
|
|
362
|
-
} catch {
|
|
363
|
-
return { report, entries: [], found: true, error: 'the memory block was not valid JSON' }
|
|
364
|
-
}
|
|
365
|
-
const raw = Array.isArray(parsed) ? parsed : Array.isArray(parsed?.entries) ? parsed.entries : []
|
|
366
|
-
const entries = []
|
|
367
|
-
for (const item of raw) {
|
|
368
|
-
const entry = normalizeEntry(item)
|
|
369
|
-
if (entry) entries.push(entry)
|
|
370
|
-
}
|
|
371
|
-
if (raw.length > 0 && entries.length === 0) {
|
|
372
|
-
return { report, entries, found: true, error: 'the memory block had no usable entry' }
|
|
373
|
-
}
|
|
374
|
-
return { report, entries, found: true }
|
|
375
|
-
}
|
|
376
|
-
|
|
377
|
-
/**
|
|
378
|
-
* Build the memory notebook bound to one state root.
|
|
379
|
-
*
|
|
380
|
-
* @param {string} root - absolute state directory (`<root>/memory` is used).
|
|
381
|
-
* @param {object} [options] - collaborators.
|
|
382
|
-
* @param {(message: string) => void} [options.warn] - diagnostic sink.
|
|
383
|
-
* @param {() => number} [options.now] - clock, injectable for tests.
|
|
384
|
-
*/
|
|
385
|
-
export function createMemory(root, { warn = () => {}, now = () => Date.now() } = {}) {
|
|
386
|
-
const dir = path.join(root, 'memory')
|
|
387
|
-
const fileOf = scope => path.join(dir, `${normalizeScope(scope)}.jsonl`)
|
|
388
|
-
const lockOf = scope => path.join(dir, `${normalizeScope(scope)}.lock`)
|
|
389
|
-
/** Per-scope write chain, so two settles in this process cannot interleave. */
|
|
390
|
-
const chains = new Map()
|
|
391
|
-
|
|
392
|
-
/**
|
|
393
|
-
* Create the directory and make it git-invisible even inside a repository.
|
|
394
|
-
* A `*` in a nested .gitignore ignores every file here, including the
|
|
395
|
-
* notebook, without editing the repository's own .gitignore.
|
|
396
|
-
*/
|
|
397
|
-
async function initDir() {
|
|
398
|
-
await fsp.mkdir(dir, { recursive: true })
|
|
399
|
-
const ignore = path.join(dir, '.gitignore')
|
|
400
|
-
try {
|
|
401
|
-
await fsp.access(ignore)
|
|
402
|
-
} catch {
|
|
403
|
-
await fsp.writeFile(ignore, '*\n', 'utf8')
|
|
404
|
-
}
|
|
405
|
-
}
|
|
406
|
-
|
|
407
|
-
function serialize(scope, task) {
|
|
408
|
-
const previous = chains.get(scope) ?? Promise.resolve()
|
|
409
|
-
const next = previous.catch(() => {}).then(task)
|
|
410
|
-
chains.set(scope, next.catch(() => {}))
|
|
411
|
-
return next
|
|
412
|
-
}
|
|
413
|
-
|
|
414
|
-
async function readRecords(file) {
|
|
415
|
-
let text
|
|
416
|
-
try {
|
|
417
|
-
text = await fsp.readFile(file, 'utf8')
|
|
418
|
-
} catch (error) {
|
|
419
|
-
if (error?.code === 'ENOENT') return { records: [], bad: 0, bytes: 0 }
|
|
420
|
-
throw error
|
|
421
|
-
}
|
|
422
|
-
const records = []
|
|
423
|
-
let bad = 0
|
|
424
|
-
for (const line of text.split('\n')) {
|
|
425
|
-
const trimmed = line.trim()
|
|
426
|
-
if (!trimmed) continue
|
|
427
|
-
try {
|
|
428
|
-
const parsed = JSON.parse(trimmed)
|
|
429
|
-
if (parsed && typeof parsed === 'object') records.push(parsed)
|
|
430
|
-
else bad++
|
|
431
|
-
} catch {
|
|
432
|
-
// A process killed mid-append leaves one truncated line; skipping it is
|
|
433
|
-
// the whole reason for the JSONL format.
|
|
434
|
-
bad++
|
|
435
|
-
}
|
|
436
|
-
}
|
|
437
|
-
return { records, bad, bytes: Buffer.byteLength(text, 'utf8') }
|
|
438
|
-
}
|
|
439
|
-
|
|
440
|
-
/**
|
|
441
|
-
* Rewrite the file with the newest notes. The caller already holds the scope
|
|
442
|
-
* lock; acquiring it here as well would deadlock against that same writer.
|
|
443
|
-
*/
|
|
444
|
-
async function compactLocked(scope, { maxEntries, maxAgeDays }) {
|
|
445
|
-
try {
|
|
446
|
-
const file = fileOf(scope)
|
|
447
|
-
const { records } = await readRecords(file)
|
|
448
|
-
// Keep the newest notes but write them oldest-first, so the file stays in
|
|
449
|
-
// append order and a same-millisecond tie is still resolved by line order.
|
|
450
|
-
const keep = effectiveEntries(records, { now: now(), maxAgeDays })
|
|
451
|
-
.slice(0, maxEntries)
|
|
452
|
-
.reverse()
|
|
453
|
-
const body = keep.map(record => `${JSON.stringify(record)}\n`).join('')
|
|
454
|
-
const tmp = path.join(dir, `.${normalizeScope(scope)}.${process.pid}.${Date.now()}.tmp`)
|
|
455
|
-
await fsp.writeFile(tmp, body, 'utf8')
|
|
456
|
-
await fsp.rename(tmp, file)
|
|
457
|
-
return true
|
|
458
|
-
} catch (error) {
|
|
459
|
-
warn(`cannot compact memory scope "${scope}": ${error?.message ?? error}`)
|
|
460
|
-
return false
|
|
461
|
-
}
|
|
462
|
-
}
|
|
463
|
-
|
|
464
|
-
async function append(scope, records) {
|
|
465
|
-
if (records.length === 0) return 0
|
|
466
|
-
const file = fileOf(scope)
|
|
467
|
-
const body = records.map(record => `${JSON.stringify(record)}\n`).join('')
|
|
468
|
-
// One write for the whole batch: a concurrent append either lands before or
|
|
469
|
-
// after this buffer, never inside it on a local filesystem.
|
|
470
|
-
const handle = await fsp.open(file, 'a')
|
|
471
|
-
try {
|
|
472
|
-
await handle.write(body)
|
|
473
|
-
} finally {
|
|
474
|
-
await handle.close()
|
|
475
|
-
}
|
|
476
|
-
return records.length
|
|
477
|
-
}
|
|
478
|
-
|
|
479
|
-
return {
|
|
480
|
-
root,
|
|
481
|
-
dir,
|
|
482
|
-
fileOf,
|
|
483
|
-
|
|
484
|
-
/** Create the directory and make it git-invisible even inside a repository. */
|
|
485
|
-
async init() {
|
|
486
|
-
await initDir()
|
|
487
|
-
},
|
|
488
|
-
|
|
489
|
-
/**
|
|
490
|
-
* Read the notebook's current notes.
|
|
491
|
-
* @param {{ scope?: string, maxAgeDays?: number }} [options] - read options.
|
|
492
|
-
*/
|
|
493
|
-
async read({ scope = 'default', maxAgeDays = DEFAULT_MAX_AGE_DAYS } = {}) {
|
|
494
|
-
const file = fileOf(scope)
|
|
495
|
-
const { records, bad, bytes } = await readRecords(file)
|
|
496
|
-
return {
|
|
497
|
-
entries: effectiveEntries(records, { now: now(), maxAgeDays }),
|
|
498
|
-
bad,
|
|
499
|
-
bytes,
|
|
500
|
-
path: file,
|
|
501
|
-
}
|
|
502
|
-
},
|
|
503
|
-
|
|
504
|
-
/**
|
|
505
|
-
* Append the notes a finished run produced. Idempotent per run id: a resume
|
|
506
|
-
* settles the same run again and must not duplicate its notes.
|
|
507
|
-
*
|
|
508
|
-
* @param {{ scope?: string, runId: string, entries: object[], maxEntries?: number, maxAgeDays?: number }} options - note batch.
|
|
509
|
-
* @returns {Promise<{ written: number, skipped: number, reason: string | null }>}
|
|
510
|
-
*/
|
|
511
|
-
async write({ scope = 'default', runId, entries, maxEntries = DEFAULT_MAX_ENTRIES, maxAgeDays = DEFAULT_MAX_AGE_DAYS } = {}) {
|
|
512
|
-
const normalized = (Array.isArray(entries) ? entries : []).map(normalizeEntry).filter(Boolean)
|
|
513
|
-
if (normalized.length === 0) return { written: 0, skipped: 0, reason: 'no usable entry' }
|
|
514
|
-
const key = normalizeScope(scope)
|
|
515
|
-
return serialize(key, async () => {
|
|
516
|
-
await initDir()
|
|
517
|
-
// One lock for the whole read/dedupe/append/compaction sequence. Without
|
|
518
|
-
// it a compaction in another process could rename the file between this
|
|
519
|
-
// writer's read and append and silently drop the new line.
|
|
520
|
-
const release = await acquireFileLock(lockOf(key), {
|
|
521
|
-
now,
|
|
522
|
-
retries: LOCK_RETRIES,
|
|
523
|
-
retryDelayMs: LOCK_RETRY_DELAY_MS,
|
|
524
|
-
})
|
|
525
|
-
if (!release) {
|
|
526
|
-
warn(`memory scope "${key}" is locked by another process; appended without compaction`)
|
|
527
|
-
}
|
|
528
|
-
try {
|
|
529
|
-
const file = fileOf(key)
|
|
530
|
-
const { records } = await readRecords(file)
|
|
531
|
-
if (runId && records.some(record => record?.runId === runId)) {
|
|
532
|
-
return { written: 0, skipped: normalized.length, reason: 'this run already wrote its notes' }
|
|
533
|
-
}
|
|
534
|
-
const ts = new Date(now()).toISOString()
|
|
535
|
-
const batch = normalized.map(entry => ({
|
|
536
|
-
v: 1,
|
|
537
|
-
id: randomBytes(4).toString('hex'),
|
|
538
|
-
ts,
|
|
539
|
-
runId: runId ?? null,
|
|
540
|
-
...entry,
|
|
541
|
-
}))
|
|
542
|
-
const written = await append(key, batch)
|
|
543
|
-
let bytes = 0
|
|
544
|
-
try {
|
|
545
|
-
bytes = (await fsp.stat(file)).size
|
|
546
|
-
} catch {
|
|
547
|
-
// A missing stat only means compaction cannot be sized this round.
|
|
548
|
-
}
|
|
549
|
-
// `maxEntries` is a trigger, not just the size kept by an existing
|
|
550
|
-
// compaction: lowering it must bound the notebook on the next write.
|
|
551
|
-
const total = records.length + batch.length
|
|
552
|
-
if (release && (bytes > MAX_FILE_BYTES || total > MAX_FILE_LINES || total > maxEntries)) {
|
|
553
|
-
await compactLocked(key, { maxEntries, maxAgeDays })
|
|
554
|
-
}
|
|
555
|
-
return { written, skipped: normalized.length - written, reason: null }
|
|
556
|
-
} finally {
|
|
557
|
-
await release?.()
|
|
558
|
-
}
|
|
559
|
-
})
|
|
560
|
-
},
|
|
561
|
-
|
|
562
|
-
/**
|
|
563
|
-
* Record one note by hand (the `superagent_memory` tool's `set`).
|
|
564
|
-
* @param {{ scope?: string, key: string, kind?: string, text: string, tags?: unknown, runId?: string }} options - note.
|
|
565
|
-
*/
|
|
566
|
-
async set({ scope = 'default', key, kind = 'note', text, tags = [], runId = null } = {}) {
|
|
567
|
-
const entry = normalizeEntry({ key, kind, text, tags })
|
|
568
|
-
if (!entry) return { written: 0, reason: 'a note needs non-empty text' }
|
|
569
|
-
const result = await this.write({ scope, runId: runId ?? `manual-${randomBytes(3).toString('hex')}`, entries: [entry] })
|
|
570
|
-
return { written: result.written, reason: result.reason }
|
|
571
|
-
},
|
|
572
|
-
|
|
573
|
-
/**
|
|
574
|
-
* Retract a note by key. The tombstone is itself a line, so a later
|
|
575
|
-
* compaction that drops the old note also drops the tombstone.
|
|
576
|
-
* @param {{ scope?: string, key: string }} options - key to forget.
|
|
577
|
-
*/
|
|
578
|
-
async forget({ scope = 'default', key } = {}) {
|
|
579
|
-
const slug = slugKey(key)
|
|
580
|
-
if (!slug) return { forgotten: false, reason: 'a key is required' }
|
|
581
|
-
const result = await this.write({
|
|
582
|
-
scope,
|
|
583
|
-
runId: `forget-${randomBytes(3).toString('hex')}`,
|
|
584
|
-
entries: [{ key: slug, kind: 'note', status: 'retracted' }],
|
|
585
|
-
})
|
|
586
|
-
return { forgotten: result.written > 0, reason: result.reason }
|
|
587
|
-
},
|
|
588
|
-
|
|
589
|
-
/**
|
|
590
|
-
* Rank the notebook against a query. With no query every note is a hit and
|
|
591
|
-
* the order is recency, which is what a digest without a goal means.
|
|
592
|
-
*
|
|
593
|
-
* Retracted notes and expired ones never come back: `read` resolves
|
|
594
|
-
* tombstones and the TTL before scoring, and one key keeps only its newest
|
|
595
|
-
* value, so a corrected note is recalled once, in its corrected form.
|
|
596
|
-
*
|
|
597
|
-
* @param {{ scope?: string, query?: string, tags?: unknown, maxEntries?: number, maxAgeDays?: number, halfLifeDays?: number }} [options] - query.
|
|
598
|
-
* @returns {Promise<{ query: string, tags: string[], searching: boolean, total: number, matched: number, entries: object[] }>}
|
|
599
|
-
*/
|
|
600
|
-
async recall({
|
|
601
|
-
scope = 'default',
|
|
602
|
-
query = '',
|
|
603
|
-
tags = [],
|
|
604
|
-
maxEntries = DEFAULT_DIGEST_ENTRIES,
|
|
605
|
-
maxAgeDays = DEFAULT_MAX_AGE_DAYS,
|
|
606
|
-
halfLifeDays = RECENCY_HALF_LIFE_DAYS,
|
|
607
|
-
} = {}) {
|
|
608
|
-
const at = now()
|
|
609
|
-
const { entries } = await this.read({ scope, maxAgeDays })
|
|
610
|
-
const wanted = normalizeTags(tags)
|
|
611
|
-
const text = typeof query === 'string' ? query : ''
|
|
612
|
-
const searching = tokenize(text).length > 0 || wanted.length > 0
|
|
613
|
-
const scored = entries.map(record => ({
|
|
614
|
-
record,
|
|
615
|
-
...scoreEntry(record, { query: text, tags: wanted, now: at, halfLifeDays }),
|
|
616
|
-
}))
|
|
617
|
-
// A search hides notes that share no term with the goal; otherwise the
|
|
618
|
-
// digest would spend its budget on fresh but irrelevant entries.
|
|
619
|
-
const hits = searching ? scored.filter(item => item.keyword > 0 || item.tag > 0) : scored
|
|
620
|
-
// `sort` is stable, so equal scores keep `read`'s newest-first order.
|
|
621
|
-
hits.sort((a, b) => b.score - a.score)
|
|
622
|
-
return {
|
|
623
|
-
query: text,
|
|
624
|
-
tags: wanted,
|
|
625
|
-
searching,
|
|
626
|
-
total: entries.length,
|
|
627
|
-
matched: hits.length,
|
|
628
|
-
entries: hits.slice(0, Math.max(1, maxEntries)).map(({ record, score, keyword, tag, recency, ageDays, matchedTerms }) => ({
|
|
629
|
-
key: record.key,
|
|
630
|
-
kind: record.kind ?? 'note',
|
|
631
|
-
text: record.text ?? '',
|
|
632
|
-
tags: normalizeTags(record.tags),
|
|
633
|
-
score,
|
|
634
|
-
keyword,
|
|
635
|
-
tag,
|
|
636
|
-
recency,
|
|
637
|
-
ageDays,
|
|
638
|
-
matchedTerms,
|
|
639
|
-
source: {
|
|
640
|
-
runId: record.runId ?? null,
|
|
641
|
-
ts: record.ts ?? null,
|
|
642
|
-
// `read` already dropped retractions, so a recalled note is live.
|
|
643
|
-
retracted: false,
|
|
644
|
-
},
|
|
645
|
-
})),
|
|
646
|
-
}
|
|
647
|
-
},
|
|
648
|
-
|
|
649
|
-
/**
|
|
650
|
-
* Bounded digest injected into prompts. Never the whole file.
|
|
651
|
-
*
|
|
652
|
-
* With a query the lines are ranked by relevance and each one carries its
|
|
653
|
-
* source, so the planner can judge how much to trust it. Without a query —
|
|
654
|
-
* or when nothing matches — it degrades to the newest notes.
|
|
655
|
-
*
|
|
656
|
-
* @param {{ scope?: string, query?: string, tags?: unknown, maxEntries?: number, maxChars?: number, maxAgeDays?: number }} [options] - digest limits.
|
|
657
|
-
* @returns {Promise<string>} note lines, or an empty string.
|
|
658
|
-
*/
|
|
659
|
-
async digest({
|
|
660
|
-
scope = 'default',
|
|
661
|
-
query = '',
|
|
662
|
-
tags = [],
|
|
663
|
-
maxEntries = DEFAULT_DIGEST_ENTRIES,
|
|
664
|
-
maxChars = DEFAULT_DIGEST_CHARS,
|
|
665
|
-
maxAgeDays = DEFAULT_MAX_AGE_DAYS,
|
|
666
|
-
} = {}) {
|
|
667
|
-
const recalled = await this.recall({ scope, query, tags, maxEntries, maxAgeDays })
|
|
668
|
-
let items = recalled.entries
|
|
669
|
-
let sourced = recalled.searching
|
|
670
|
-
if (items.length === 0 && recalled.searching) {
|
|
671
|
-
// An unfamiliar goal must not lose continuity: fall back to the newest
|
|
672
|
-
// notes, which is what this digest did before it could search.
|
|
673
|
-
const { entries } = await this.read({ scope, maxAgeDays })
|
|
674
|
-
items = entries.slice(0, maxEntries).map(record => ({
|
|
675
|
-
key: record.key,
|
|
676
|
-
kind: record.kind ?? 'note',
|
|
677
|
-
text: record.text ?? '',
|
|
678
|
-
ageDays: (now() - tsOf(record)) / 86400000,
|
|
679
|
-
source: { runId: record.runId ?? null, ts: record.ts ?? null, retracted: false },
|
|
680
|
-
}))
|
|
681
|
-
sourced = false
|
|
682
|
-
}
|
|
683
|
-
if (items.length === 0) return ''
|
|
684
|
-
const picked = items.slice(0, maxEntries)
|
|
685
|
-
const share = Math.max(60, Math.floor(maxChars / Math.max(1, picked.length)))
|
|
686
|
-
const lines = []
|
|
687
|
-
let used = 0
|
|
688
|
-
let omitted = 0
|
|
689
|
-
for (const item of picked) {
|
|
690
|
-
const header = sourced
|
|
691
|
-
? `- [${ageFromDays(item.ageDays)}] ${item.kind} ${item.key} `
|
|
692
|
-
+ `(run ${item.source.runId ?? 'manual'}, ${String(item.source.ts ?? '').slice(0, 10)}, `
|
|
693
|
-
+ `${item.source.retracted ? 'retracted' : 'active'}): `
|
|
694
|
-
: `- [${ageFromDays(item.ageDays)}] ${item.kind} ${item.key}: `
|
|
695
|
-
const line = header + capLine(item.text, Math.max(40, share - header.length))
|
|
696
|
-
if (used + line.length + 1 > maxChars) {
|
|
697
|
-
omitted++
|
|
698
|
-
continue
|
|
699
|
-
}
|
|
700
|
-
lines.push(line)
|
|
701
|
-
used += line.length + 1
|
|
702
|
-
}
|
|
703
|
-
if (lines.length === 0) return ''
|
|
704
|
-
if (omitted > 0) {
|
|
705
|
-
lines.push(`- (+${omitted} more; call superagent_memory action="${sourced ? 'search' : 'list'}" to read them)`)
|
|
706
|
-
}
|
|
707
|
-
return lines.join('\n')
|
|
708
|
-
},
|
|
709
|
-
|
|
710
|
-
/**
|
|
711
|
-
* Rows for the `superagent_memory` tool.
|
|
712
|
-
* @param {{ scope?: string, limit?: number, maxAgeDays?: number }} [options] - list options.
|
|
713
|
-
*/
|
|
714
|
-
async list({ scope = 'default', limit = 20, maxAgeDays = DEFAULT_MAX_AGE_DAYS } = {}) {
|
|
715
|
-
const { entries, bad, bytes, path: file } = await this.read({ scope, maxAgeDays })
|
|
716
|
-
return { entries: entries.slice(0, limit), total: entries.length, bad, bytes, path: file }
|
|
717
|
-
},
|
|
718
|
-
|
|
719
|
-
/**
|
|
720
|
-
* Size and count for `superagent_list`.
|
|
721
|
-
* @param {{ scope?: string, maxAgeDays?: number }} [options] - stats options.
|
|
722
|
-
*/
|
|
723
|
-
async stats({ scope = 'default', maxAgeDays = DEFAULT_MAX_AGE_DAYS } = {}) {
|
|
724
|
-
const { entries, bytes, path: file } = await this.read({ scope, maxAgeDays })
|
|
725
|
-
return { entries: entries.length, bytes, path: file }
|
|
726
|
-
},
|
|
727
|
-
|
|
728
|
-
/** Defaults exposed so the orchestrator and tests agree on one set of limits. */
|
|
729
|
-
defaults: {
|
|
730
|
-
maxEntries: DEFAULT_MAX_ENTRIES,
|
|
731
|
-
maxAgeDays: DEFAULT_MAX_AGE_DAYS,
|
|
732
|
-
digestEntries: DEFAULT_DIGEST_ENTRIES,
|
|
733
|
-
digestChars: DEFAULT_DIGEST_CHARS,
|
|
734
|
-
},
|
|
735
|
-
}
|
|
736
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* Cross-run memory for dsh-superagent.
|
|
3
|
+
*
|
|
4
|
+
* A run is stateless by design: planner and workers start empty, and a finished
|
|
5
|
+
* run leaves only its state file behind. That is good for isolation and bad for
|
|
6
|
+
* continuity — the next run cannot know which approach was already rejected or
|
|
7
|
+
* which step left a trap. This module keeps a small, curated notebook between
|
|
8
|
+
* runs without turning the state directory into a growing pile of logs:
|
|
9
|
+
*
|
|
10
|
+
* - Exact activity history is *derived* from the run store at planning time
|
|
11
|
+
* (which runs existed, how they ended). Nothing is copied into memory for it.
|
|
12
|
+
* - Durable knowledge (decisions, pitfalls, next steps) is written only when a
|
|
13
|
+
* run's aggregator emits a fenced `memory` block. One short entry per topic.
|
|
14
|
+
* - Reads are bounded: at most `maxEntries` notes, each shortened, inside a
|
|
15
|
+
* fixed character budget. The notebook is a digest, never the full file.
|
|
16
|
+
* - A read can be a search: `recall` ranks notes against the goal's keywords and
|
|
17
|
+
* the requested tags, so an old relevant note beats a recent irrelevant one.
|
|
18
|
+
* The score is word hits (key 2, tags 1.5, text 0.5 per occurrence, capped at
|
|
19
|
+
* 3) normalised over at most five query terms, a tag-match ratio, and
|
|
20
|
+
* exponential time decay with a 14-day half-life. No embeddings, no index
|
|
21
|
+
* file, no dependencies.
|
|
22
|
+
* - A note is refuted, not deleted silently: a newer entry with the same `key`
|
|
23
|
+
* supersedes it, a `retracted` entry drops it, and the `superagent_memory`
|
|
24
|
+
* tool can list, set, or forget a key by hand.
|
|
25
|
+
* - Entries older than `maxAgeDays` are ignored and then compacted away, so a
|
|
26
|
+
* stale conclusion cannot become a permanent bias.
|
|
27
|
+
*
|
|
28
|
+
* Storage is `<stateDir>/memory/<scope>.jsonl`, one JSON object per line. The
|
|
29
|
+
* directory is created with a `.gitignore` containing `*`, so even a state
|
|
30
|
+
* directory that points inside a repository cannot be committed by accident.
|
|
31
|
+
* The package `files` whitelist never ships this directory.
|
|
32
|
+
*
|
|
33
|
+
* Writes append one short line with a single `write` call (atomic for a small
|
|
34
|
+
* buffer on a local filesystem) while holding a lock file, so a concurrent
|
|
35
|
+
* append and a compaction cannot drop each other's lines. Compaction replaces
|
|
36
|
+
* the file from a temp file under the same lock. Two runs finishing at the same
|
|
37
|
+
* time therefore serialise instead of overwriting; the per-key resolution is by
|
|
38
|
+
* timestamp, so order alone never changes a conclusion.
|
|
39
|
+
*
|
|
40
|
+
* @module dsh-superagent/memory
|
|
41
|
+
*/
|
|
42
|
+
import { randomBytes } from 'node:crypto'
|
|
43
|
+
import fsp from 'node:fs/promises'
|
|
44
|
+
import path from 'node:path'
|
|
45
|
+
import { acquireFileLock } from './lock.js'
|
|
46
|
+
|
|
47
|
+
/** Kinds a note may declare. Anything else is stored as `note`. */
|
|
48
|
+
export const MEMORY_KINDS = new Set(['dispatch', 'decision', 'pitfall', 'next', 'note'])
|
|
49
|
+
|
|
50
|
+
/** Longest note kept; longer text is cut, not rejected. */
|
|
51
|
+
export const MAX_ENTRY_CHARS = 400
|
|
52
|
+
|
|
53
|
+
/** Longest key; keys are lowercase slugs so a later note can match an older one. */
|
|
54
|
+
export const MAX_KEY_CHARS = 48
|
|
55
|
+
|
|
56
|
+
/** Most tags one note may carry; tags are slugs, so a tag query is an exact match. */
|
|
57
|
+
export const MAX_TAGS = 6
|
|
58
|
+
|
|
59
|
+
/** Days after which a note's recency weight halves. */
|
|
60
|
+
export const RECENCY_HALF_LIFE_DAYS = 14
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Recall weights. Keyword relevance dominates, an explicit tag request is a
|
|
64
|
+
* strong signal, and recency only breaks near-ties: a two-week-old note that
|
|
65
|
+
* names the goal outranks a fresh one that does not.
|
|
66
|
+
*/
|
|
67
|
+
export const SCORE_WEIGHTS = Object.freeze({ keyword: 0.55, tag: 0.3, recency: 0.15 })
|
|
68
|
+
|
|
69
|
+
/** Per-term weights inside the keyword score. */
|
|
70
|
+
const KEY_TERM_WEIGHT = 2
|
|
71
|
+
const TAG_TERM_WEIGHT = 1.5
|
|
72
|
+
const TEXT_TERM_WEIGHT = 0.5
|
|
73
|
+
const MAX_TEXT_OCCURRENCES = 3
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* How many query terms the keyword score divides by. A goal is a sentence, not
|
|
77
|
+
* a keyword list: without this cap a twenty-word goal would spread its weight so
|
|
78
|
+
* thin that time decay outranked relevance.
|
|
79
|
+
*/
|
|
80
|
+
const KEYWORD_TERM_BUDGET = 5
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Words that carry no retrieval signal. Kept deliberately small: an aggressive
|
|
84
|
+
* list would need maintenance per language.
|
|
85
|
+
*/
|
|
86
|
+
const STOPWORDS = new Set([
|
|
87
|
+
'a', 'an', 'and', 'are', 'as', 'at', 'be', 'been', 'but', 'by', 'can', 'could', 'did', 'do', 'does', 'for',
|
|
88
|
+
'from', 'had', 'has', 'have', 'how', 'if', 'in', 'into', 'is', 'it', 'its', 'just', 'not', 'of', 'on', 'or',
|
|
89
|
+
'our', 'should', 'so', 'some', 'than', 'that', 'the', 'their', 'then', 'there', 'these', 'they', 'this',
|
|
90
|
+
'to', 'too', 'use', 'using', 'was', 'were', 'what', 'when', 'which', 'will', 'with', 'would', 'you',
|
|
91
|
+
])
|
|
92
|
+
|
|
93
|
+
/** Latin words and CJK runs; CJK has no spaces, so a run becomes bigrams. */
|
|
94
|
+
const TOKEN_PATTERN = /[a-z0-9]+|[\u3400-\u4dbf\u4e00-\u9fff]+/g
|
|
95
|
+
const CJK_PATTERN = /^[\u3400-\u4dbf\u4e00-\u9fff]/
|
|
96
|
+
|
|
97
|
+
/** File-level limits; crossing either one, or `maxEntries`, triggers compaction. */
|
|
98
|
+
const MAX_FILE_BYTES = 128 * 1024
|
|
99
|
+
const MAX_FILE_LINES = 400
|
|
100
|
+
|
|
101
|
+
/** How long a writer waits for another process's lock before failing open. */
|
|
102
|
+
const LOCK_RETRIES = 10
|
|
103
|
+
const LOCK_RETRY_DELAY_MS = 50
|
|
104
|
+
|
|
105
|
+
const DEFAULT_MAX_AGE_DAYS = 30
|
|
106
|
+
const DEFAULT_MAX_ENTRIES = 60
|
|
107
|
+
const DEFAULT_DIGEST_ENTRIES = 12
|
|
108
|
+
const DEFAULT_DIGEST_CHARS = 1600
|
|
109
|
+
|
|
110
|
+
const SCOPE_PATTERN = /^[a-z0-9][a-z0-9_-]{0,31}$/
|
|
111
|
+
const MEMORY_FENCE = /```memory[^\n]*\n([\s\S]*?)```/i
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Coerce a configured scope into a safe file name. Anything unrecognised falls
|
|
115
|
+
* back to `default` instead of producing a path from user input.
|
|
116
|
+
* @param {unknown} value - configured scope.
|
|
117
|
+
* @returns {string} a scope that matches `SCOPE_PATTERN`.
|
|
118
|
+
*/
|
|
119
|
+
export function normalizeScope(value) {
|
|
120
|
+
const text = typeof value === 'string' ? value.trim().toLowerCase() : ''
|
|
121
|
+
return SCOPE_PATTERN.test(text) ? text : 'default'
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* Remove the shapes of secret that a model is most likely to paste into a note:
|
|
126
|
+
* API keys, bearer headers, known token prefixes, long hex/base64 blobs, email
|
|
127
|
+
* addresses, and absolute home paths. Redaction happens before storage, so a
|
|
128
|
+
* file that is copied or backed up later cannot leak the original.
|
|
129
|
+
*
|
|
130
|
+
* @param {unknown} value - text to clean.
|
|
131
|
+
* @returns {string} text with secrets replaced by markers.
|
|
132
|
+
*/
|
|
133
|
+
export function redact(value) {
|
|
134
|
+
if (typeof value !== 'string') return ''
|
|
135
|
+
return value
|
|
136
|
+
.replace(/\b(?:sk|pk|rk)-[A-Za-z0-9_-]{8,}/g, '[redacted-key]')
|
|
137
|
+
.replace(/\bBearer\s+[A-Za-z0-9._~+/=-]{8,}/gi, 'Bearer [redacted]')
|
|
138
|
+
.replace(/\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{16,}\b/g, '[redacted-token]')
|
|
139
|
+
.replace(/\bAKIA[0-9A-Z]{16}\b/g, '[redacted-key]')
|
|
140
|
+
.replace(/\b[A-Za-z0-9+/]{40,}={0,2}\b/g, '[redacted-blob]')
|
|
141
|
+
.replace(/\b[A-Fa-f0-9]{32,}\b/g, '[redacted-hex]')
|
|
142
|
+
.replace(/\b[\w.+-]+@[\w-]+\.[\w.-]+\b/g, '[redacted-email]')
|
|
143
|
+
.replace(/\b[A-Za-z]:\\Users\\[^\\\s]+/gi, '%USERPROFILE%')
|
|
144
|
+
.replace(/\/(?:home|Users)\/[^/\s]+/g, '~')
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Turn arbitrary text into one bounded line: redacted, no newlines, trimmed.
|
|
149
|
+
* @param {unknown} value - source text.
|
|
150
|
+
* @param {number} [limit] - maximum characters.
|
|
151
|
+
* @returns {string} a single line.
|
|
152
|
+
*/
|
|
153
|
+
export function cleanText(value, limit = MAX_ENTRY_CHARS) {
|
|
154
|
+
const text = redact(String(value ?? ''))
|
|
155
|
+
.replace(/\s+/g, ' ')
|
|
156
|
+
.trim()
|
|
157
|
+
if (!text) return ''
|
|
158
|
+
// Slice by code point so a cut never leaves a lone surrogate behind.
|
|
159
|
+
return [...text].slice(0, limit).join('')
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* @param {unknown} value - proposed key.
|
|
164
|
+
* @returns {string} lowercase slug, possibly empty.
|
|
165
|
+
*/
|
|
166
|
+
export function slugKey(value) {
|
|
167
|
+
return redact(String(value ?? ''))
|
|
168
|
+
.toLowerCase()
|
|
169
|
+
.replace(/[^a-z0-9._-]+/g, '-')
|
|
170
|
+
.replace(/^[-.]+|[-.]+$/g, '')
|
|
171
|
+
.slice(0, MAX_KEY_CHARS)
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* True when `child` is `parent` or inside it. Used to refuse a memory directory
|
|
176
|
+
* that would end up inside the installed package.
|
|
177
|
+
* @param {string} child - absolute path to test.
|
|
178
|
+
* @param {string} parent - absolute path to compare against.
|
|
179
|
+
*/
|
|
180
|
+
export function isInside(child, parent) {
|
|
181
|
+
const from = path.resolve(parent)
|
|
182
|
+
const to = path.resolve(child)
|
|
183
|
+
const relative = path.relative(from, to)
|
|
184
|
+
return relative === '' || (!relative.startsWith('..') && !path.isAbsolute(relative))
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
function normalizeEntry(item) {
|
|
188
|
+
if (!item || typeof item !== 'object') return null
|
|
189
|
+
const status = item.status === 'retracted' ? 'retracted' : 'active'
|
|
190
|
+
const kind = MEMORY_KINDS.has(item.kind) ? item.kind : 'note'
|
|
191
|
+
const text = status === 'retracted' ? '' : cleanText(item.text)
|
|
192
|
+
const key = slugKey(item.key) || slugKey(text.slice(0, 48)) || `${kind}-${randomBytes(3).toString('hex')}`
|
|
193
|
+
if (status === 'active' && !text) return null
|
|
194
|
+
return { key, kind, status, tags: normalizeTags(item.tags), text }
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/**
|
|
198
|
+
* Split text into retrieval terms. Latin words are lowercased and dropped when
|
|
199
|
+
* shorter than two characters or in `STOPWORDS`; a CJK run is expanded into
|
|
200
|
+
* overlapping bigrams (`登录超时` → `登录`, `录超`, `超时`) because those languages
|
|
201
|
+
* do not delimit words. Duplicates are kept so a caller can count occurrences.
|
|
202
|
+
*
|
|
203
|
+
* @param {unknown} value - text to tokenise.
|
|
204
|
+
* @returns {string[]} terms, in order, possibly repeated.
|
|
205
|
+
*/
|
|
206
|
+
export function tokenize(value) {
|
|
207
|
+
const text = typeof value === 'string' ? value.toLowerCase() : ''
|
|
208
|
+
const terms = []
|
|
209
|
+
for (const [match] of text.matchAll(TOKEN_PATTERN)) {
|
|
210
|
+
if (CJK_PATTERN.test(match)) {
|
|
211
|
+
const chars = [...match]
|
|
212
|
+
if (chars.length === 1) terms.push(chars[0])
|
|
213
|
+
else for (let index = 0; index < chars.length - 1; index++) terms.push(chars[index] + chars[index + 1])
|
|
214
|
+
continue
|
|
215
|
+
}
|
|
216
|
+
if (match.length < 2 || STOPWORDS.has(match)) continue
|
|
217
|
+
terms.push(match)
|
|
218
|
+
}
|
|
219
|
+
return terms
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/**
|
|
223
|
+
* Coerce a tag list (array or comma-separated string) into unique slugs.
|
|
224
|
+
* @param {unknown} value - proposed tags or entry tags.
|
|
225
|
+
* @returns {string[]} at most `MAX_TAGS` slugs.
|
|
226
|
+
*/
|
|
227
|
+
export function normalizeTags(value) {
|
|
228
|
+
const list = Array.isArray(value) ? value : typeof value === 'string' ? value.split(',') : []
|
|
229
|
+
const tags = []
|
|
230
|
+
for (const item of list) {
|
|
231
|
+
const tag = slugKey(item)
|
|
232
|
+
if (tag && !tags.includes(tag)) tags.push(tag)
|
|
233
|
+
if (tags.length >= MAX_TAGS) break
|
|
234
|
+
}
|
|
235
|
+
return tags
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* Build the term indexes a score needs. Doing it once per note keeps a search
|
|
240
|
+
* over a few hundred notes linear and allocation-light.
|
|
241
|
+
*/
|
|
242
|
+
function indexOf(record) {
|
|
243
|
+
const key = new Set(tokenize(record.key))
|
|
244
|
+
// `normalizeTags` also tolerates a hand-edited string, which `.join` would not.
|
|
245
|
+
const tags = new Set(tokenize(normalizeTags(record.tags).join(' ')))
|
|
246
|
+
const text = new Map()
|
|
247
|
+
for (const term of tokenize(record.text)) text.set(term, (text.get(term) ?? 0) + 1)
|
|
248
|
+
return { key, tags, text }
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* Score one note against a query. The three parts are independent and each is
|
|
253
|
+
* in `[0, 1]`, so a caller can explain a ranking field by field:
|
|
254
|
+
*
|
|
255
|
+
* - `keyword`: per query term, 2 for a hit in the key, 1.5 in a tag, 0.5 per
|
|
256
|
+
* occurrence in the text (capped at 3), divided by `2 × min(terms, 5)` and
|
|
257
|
+
* clamped at 1. A key hit is therefore worth a full term, a term that only
|
|
258
|
+
* appears once in the body is worth a quarter, and a long goal cannot dilute
|
|
259
|
+
* a match below the recency term.
|
|
260
|
+
* - `tag`: share of the explicitly requested tags the note carries (0 when no
|
|
261
|
+
* tag was requested).
|
|
262
|
+
* - `recency`: `0.5 ** (ageDays / halfLifeDays)`, so 14 days halves it.
|
|
263
|
+
*
|
|
264
|
+
* @param {object} record - a note record.
|
|
265
|
+
* @param {{ query?: string, tags?: unknown, now?: number, halfLifeDays?: number }} [options] - query.
|
|
266
|
+
* @returns {{ score: number, keyword: number, tag: number, recency: number, ageDays: number, matchedTerms: string[] }}
|
|
267
|
+
*/
|
|
268
|
+
export function scoreEntry(record, { query = '', tags = [], now = Date.now(), halfLifeDays = RECENCY_HALF_LIFE_DAYS } = {}) {
|
|
269
|
+
const index = indexOf(record)
|
|
270
|
+
const terms = [...new Set(tokenize(query))]
|
|
271
|
+
const wanted = normalizeTags(tags)
|
|
272
|
+
let hits = 0
|
|
273
|
+
const matchedTerms = []
|
|
274
|
+
for (const term of terms) {
|
|
275
|
+
const weight = (index.key.has(term) ? KEY_TERM_WEIGHT : 0)
|
|
276
|
+
+ (index.tags.has(term) ? TAG_TERM_WEIGHT : 0)
|
|
277
|
+
+ Math.min(index.text.get(term) ?? 0, MAX_TEXT_OCCURRENCES) * TEXT_TERM_WEIGHT
|
|
278
|
+
if (weight > 0) {
|
|
279
|
+
hits += weight
|
|
280
|
+
matchedTerms.push(term)
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
const keyword = terms.length > 0
|
|
284
|
+
? Math.min(1, hits / (KEY_TERM_WEIGHT * Math.min(terms.length, KEYWORD_TERM_BUDGET)))
|
|
285
|
+
: 0
|
|
286
|
+
const noteTags = normalizeTags(record.tags)
|
|
287
|
+
const tagHits = wanted.filter(tag => noteTags.includes(tag)).length
|
|
288
|
+
const tag = wanted.length > 0 ? tagHits / wanted.length : 0
|
|
289
|
+
const ageDays = Math.max(0, (now - tsOf(record)) / 86400000)
|
|
290
|
+
const recency = Number.isFinite(ageDays) ? 0.5 ** (ageDays / Math.max(1, halfLifeDays)) : 0
|
|
291
|
+
const score = SCORE_WEIGHTS.keyword * keyword + SCORE_WEIGHTS.tag * tag + SCORE_WEIGHTS.recency * recency
|
|
292
|
+
return { score, keyword, tag, recency, ageDays, matchedTerms }
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
function tsOf(record) {
|
|
296
|
+
const value = Date.parse(record?.ts ?? '')
|
|
297
|
+
return Number.isFinite(value) ? value : 0
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/**
|
|
301
|
+
* Resolve raw lines into the notes a reader should see.
|
|
302
|
+
*
|
|
303
|
+
* Records are applied oldest-first, so a newer entry wins on the same key and a
|
|
304
|
+
* `retracted` entry removes it. Timestamps are used rather than file order
|
|
305
|
+
* because a shared state directory can be written by two clocks. TTL is applied
|
|
306
|
+
* after resolution: because a tombstone is newer than its target, an expired
|
|
307
|
+
* tombstone cannot resurrect an older active entry.
|
|
308
|
+
*
|
|
309
|
+
* @param {object[]} records - parsed lines.
|
|
310
|
+
* @param {{ now?: number, maxAgeDays?: number }} [options] - clock and TTL.
|
|
311
|
+
* @returns {object[]} active entries, newest first.
|
|
312
|
+
*/
|
|
313
|
+
export function effectiveEntries(records, { now = Date.now(), maxAgeDays = DEFAULT_MAX_AGE_DAYS } = {}) {
|
|
314
|
+
const active = new Map()
|
|
315
|
+
const ordered = records
|
|
316
|
+
.map((record, index) => ({ record, index }))
|
|
317
|
+
.filter(({ record }) => record && typeof record === 'object' && typeof record.key === 'string')
|
|
318
|
+
.sort((a, b) => (tsOf(a.record) - tsOf(b.record)) || (a.index - b.index))
|
|
319
|
+
for (const { record, index } of ordered) {
|
|
320
|
+
if (record.status === 'retracted') active.delete(record.key)
|
|
321
|
+
else active.set(record.key, { record, index })
|
|
322
|
+
}
|
|
323
|
+
const cutoff = maxAgeDays > 0 ? now - maxAgeDays * 86400000 : -Infinity
|
|
324
|
+
return [...active.values()]
|
|
325
|
+
.filter(({ record }) => tsOf(record) >= cutoff)
|
|
326
|
+
// Newest first. Notes written in the same millisecond keep reverse line
|
|
327
|
+
// order, so a later line is treated as newer rather than older.
|
|
328
|
+
.sort((a, b) => (tsOf(b.record) - tsOf(a.record)) || (b.index - a.index))
|
|
329
|
+
.map(({ record }) => record)
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
function ageFromDays(days) {
|
|
333
|
+
const whole = Math.max(0, Math.floor(days))
|
|
334
|
+
return whole === 0 ? 'today' : `${whole}d`
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
function capLine(text, limit) {
|
|
338
|
+
const points = [...text]
|
|
339
|
+
return points.length <= limit ? text : `${points.slice(0, Math.max(0, limit - 1)).join('')}…`
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
/**
|
|
343
|
+
* Pull the fenced `memory` block out of an aggregator report.
|
|
344
|
+
*
|
|
345
|
+
* The block is removed from the report whether or not it parses: the user asked
|
|
346
|
+
* for a report, not for the plugin's notebook syntax. A malformed block is
|
|
347
|
+
* reported through `error` so the run record shows why no note was written.
|
|
348
|
+
*
|
|
349
|
+
* @param {unknown} text - final report.
|
|
350
|
+
* @returns {{ report: string, entries: object[], found: boolean, error?: string }}
|
|
351
|
+
*/
|
|
352
|
+
export function extractMemoryBlock(text) {
|
|
353
|
+
if (typeof text !== 'string' || !text) return { report: typeof text === 'string' ? text : '', entries: [], found: false }
|
|
354
|
+
const match = MEMORY_FENCE.exec(text)
|
|
355
|
+
if (!match) return { report: text, entries: [], found: false }
|
|
356
|
+
const report = `${text.slice(0, match.index)}${text.slice(match.index + match[0].length)}`
|
|
357
|
+
.replace(/\n{3,}/g, '\n\n')
|
|
358
|
+
.trim()
|
|
359
|
+
let parsed
|
|
360
|
+
try {
|
|
361
|
+
parsed = JSON.parse(match[1].trim())
|
|
362
|
+
} catch {
|
|
363
|
+
return { report, entries: [], found: true, error: 'the memory block was not valid JSON' }
|
|
364
|
+
}
|
|
365
|
+
const raw = Array.isArray(parsed) ? parsed : Array.isArray(parsed?.entries) ? parsed.entries : []
|
|
366
|
+
const entries = []
|
|
367
|
+
for (const item of raw) {
|
|
368
|
+
const entry = normalizeEntry(item)
|
|
369
|
+
if (entry) entries.push(entry)
|
|
370
|
+
}
|
|
371
|
+
if (raw.length > 0 && entries.length === 0) {
|
|
372
|
+
return { report, entries, found: true, error: 'the memory block had no usable entry' }
|
|
373
|
+
}
|
|
374
|
+
return { report, entries, found: true }
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
/**
|
|
378
|
+
* Build the memory notebook bound to one state root.
|
|
379
|
+
*
|
|
380
|
+
* @param {string} root - absolute state directory (`<root>/memory` is used).
|
|
381
|
+
* @param {object} [options] - collaborators.
|
|
382
|
+
* @param {(message: string) => void} [options.warn] - diagnostic sink.
|
|
383
|
+
* @param {() => number} [options.now] - clock, injectable for tests.
|
|
384
|
+
*/
|
|
385
|
+
export function createMemory(root, { warn = () => {}, now = () => Date.now() } = {}) {
|
|
386
|
+
const dir = path.join(root, 'memory')
|
|
387
|
+
const fileOf = scope => path.join(dir, `${normalizeScope(scope)}.jsonl`)
|
|
388
|
+
const lockOf = scope => path.join(dir, `${normalizeScope(scope)}.lock`)
|
|
389
|
+
/** Per-scope write chain, so two settles in this process cannot interleave. */
|
|
390
|
+
const chains = new Map()
|
|
391
|
+
|
|
392
|
+
/**
|
|
393
|
+
* Create the directory and make it git-invisible even inside a repository.
|
|
394
|
+
* A `*` in a nested .gitignore ignores every file here, including the
|
|
395
|
+
* notebook, without editing the repository's own .gitignore.
|
|
396
|
+
*/
|
|
397
|
+
async function initDir() {
|
|
398
|
+
await fsp.mkdir(dir, { recursive: true })
|
|
399
|
+
const ignore = path.join(dir, '.gitignore')
|
|
400
|
+
try {
|
|
401
|
+
await fsp.access(ignore)
|
|
402
|
+
} catch {
|
|
403
|
+
await fsp.writeFile(ignore, '*\n', 'utf8')
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
function serialize(scope, task) {
|
|
408
|
+
const previous = chains.get(scope) ?? Promise.resolve()
|
|
409
|
+
const next = previous.catch(() => {}).then(task)
|
|
410
|
+
chains.set(scope, next.catch(() => {}))
|
|
411
|
+
return next
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
async function readRecords(file) {
|
|
415
|
+
let text
|
|
416
|
+
try {
|
|
417
|
+
text = await fsp.readFile(file, 'utf8')
|
|
418
|
+
} catch (error) {
|
|
419
|
+
if (error?.code === 'ENOENT') return { records: [], bad: 0, bytes: 0 }
|
|
420
|
+
throw error
|
|
421
|
+
}
|
|
422
|
+
const records = []
|
|
423
|
+
let bad = 0
|
|
424
|
+
for (const line of text.split('\n')) {
|
|
425
|
+
const trimmed = line.trim()
|
|
426
|
+
if (!trimmed) continue
|
|
427
|
+
try {
|
|
428
|
+
const parsed = JSON.parse(trimmed)
|
|
429
|
+
if (parsed && typeof parsed === 'object') records.push(parsed)
|
|
430
|
+
else bad++
|
|
431
|
+
} catch {
|
|
432
|
+
// A process killed mid-append leaves one truncated line; skipping it is
|
|
433
|
+
// the whole reason for the JSONL format.
|
|
434
|
+
bad++
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
return { records, bad, bytes: Buffer.byteLength(text, 'utf8') }
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
/**
|
|
441
|
+
* Rewrite the file with the newest notes. The caller already holds the scope
|
|
442
|
+
* lock; acquiring it here as well would deadlock against that same writer.
|
|
443
|
+
*/
|
|
444
|
+
async function compactLocked(scope, { maxEntries, maxAgeDays }) {
|
|
445
|
+
try {
|
|
446
|
+
const file = fileOf(scope)
|
|
447
|
+
const { records } = await readRecords(file)
|
|
448
|
+
// Keep the newest notes but write them oldest-first, so the file stays in
|
|
449
|
+
// append order and a same-millisecond tie is still resolved by line order.
|
|
450
|
+
const keep = effectiveEntries(records, { now: now(), maxAgeDays })
|
|
451
|
+
.slice(0, maxEntries)
|
|
452
|
+
.reverse()
|
|
453
|
+
const body = keep.map(record => `${JSON.stringify(record)}\n`).join('')
|
|
454
|
+
const tmp = path.join(dir, `.${normalizeScope(scope)}.${process.pid}.${Date.now()}.tmp`)
|
|
455
|
+
await fsp.writeFile(tmp, body, 'utf8')
|
|
456
|
+
await fsp.rename(tmp, file)
|
|
457
|
+
return true
|
|
458
|
+
} catch (error) {
|
|
459
|
+
warn(`cannot compact memory scope "${scope}": ${error?.message ?? error}`)
|
|
460
|
+
return false
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
async function append(scope, records) {
|
|
465
|
+
if (records.length === 0) return 0
|
|
466
|
+
const file = fileOf(scope)
|
|
467
|
+
const body = records.map(record => `${JSON.stringify(record)}\n`).join('')
|
|
468
|
+
// One write for the whole batch: a concurrent append either lands before or
|
|
469
|
+
// after this buffer, never inside it on a local filesystem.
|
|
470
|
+
const handle = await fsp.open(file, 'a')
|
|
471
|
+
try {
|
|
472
|
+
await handle.write(body)
|
|
473
|
+
} finally {
|
|
474
|
+
await handle.close()
|
|
475
|
+
}
|
|
476
|
+
return records.length
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
return {
|
|
480
|
+
root,
|
|
481
|
+
dir,
|
|
482
|
+
fileOf,
|
|
483
|
+
|
|
484
|
+
/** Create the directory and make it git-invisible even inside a repository. */
|
|
485
|
+
async init() {
|
|
486
|
+
await initDir()
|
|
487
|
+
},
|
|
488
|
+
|
|
489
|
+
/**
|
|
490
|
+
* Read the notebook's current notes.
|
|
491
|
+
* @param {{ scope?: string, maxAgeDays?: number }} [options] - read options.
|
|
492
|
+
*/
|
|
493
|
+
async read({ scope = 'default', maxAgeDays = DEFAULT_MAX_AGE_DAYS } = {}) {
|
|
494
|
+
const file = fileOf(scope)
|
|
495
|
+
const { records, bad, bytes } = await readRecords(file)
|
|
496
|
+
return {
|
|
497
|
+
entries: effectiveEntries(records, { now: now(), maxAgeDays }),
|
|
498
|
+
bad,
|
|
499
|
+
bytes,
|
|
500
|
+
path: file,
|
|
501
|
+
}
|
|
502
|
+
},
|
|
503
|
+
|
|
504
|
+
/**
|
|
505
|
+
* Append the notes a finished run produced. Idempotent per run id: a resume
|
|
506
|
+
* settles the same run again and must not duplicate its notes.
|
|
507
|
+
*
|
|
508
|
+
* @param {{ scope?: string, runId: string, entries: object[], maxEntries?: number, maxAgeDays?: number }} options - note batch.
|
|
509
|
+
* @returns {Promise<{ written: number, skipped: number, reason: string | null }>}
|
|
510
|
+
*/
|
|
511
|
+
async write({ scope = 'default', runId, entries, maxEntries = DEFAULT_MAX_ENTRIES, maxAgeDays = DEFAULT_MAX_AGE_DAYS } = {}) {
|
|
512
|
+
const normalized = (Array.isArray(entries) ? entries : []).map(normalizeEntry).filter(Boolean)
|
|
513
|
+
if (normalized.length === 0) return { written: 0, skipped: 0, reason: 'no usable entry' }
|
|
514
|
+
const key = normalizeScope(scope)
|
|
515
|
+
return serialize(key, async () => {
|
|
516
|
+
await initDir()
|
|
517
|
+
// One lock for the whole read/dedupe/append/compaction sequence. Without
|
|
518
|
+
// it a compaction in another process could rename the file between this
|
|
519
|
+
// writer's read and append and silently drop the new line.
|
|
520
|
+
const release = await acquireFileLock(lockOf(key), {
|
|
521
|
+
now,
|
|
522
|
+
retries: LOCK_RETRIES,
|
|
523
|
+
retryDelayMs: LOCK_RETRY_DELAY_MS,
|
|
524
|
+
})
|
|
525
|
+
if (!release) {
|
|
526
|
+
warn(`memory scope "${key}" is locked by another process; appended without compaction`)
|
|
527
|
+
}
|
|
528
|
+
try {
|
|
529
|
+
const file = fileOf(key)
|
|
530
|
+
const { records } = await readRecords(file)
|
|
531
|
+
if (runId && records.some(record => record?.runId === runId)) {
|
|
532
|
+
return { written: 0, skipped: normalized.length, reason: 'this run already wrote its notes' }
|
|
533
|
+
}
|
|
534
|
+
const ts = new Date(now()).toISOString()
|
|
535
|
+
const batch = normalized.map(entry => ({
|
|
536
|
+
v: 1,
|
|
537
|
+
id: randomBytes(4).toString('hex'),
|
|
538
|
+
ts,
|
|
539
|
+
runId: runId ?? null,
|
|
540
|
+
...entry,
|
|
541
|
+
}))
|
|
542
|
+
const written = await append(key, batch)
|
|
543
|
+
let bytes = 0
|
|
544
|
+
try {
|
|
545
|
+
bytes = (await fsp.stat(file)).size
|
|
546
|
+
} catch {
|
|
547
|
+
// A missing stat only means compaction cannot be sized this round.
|
|
548
|
+
}
|
|
549
|
+
// `maxEntries` is a trigger, not just the size kept by an existing
|
|
550
|
+
// compaction: lowering it must bound the notebook on the next write.
|
|
551
|
+
const total = records.length + batch.length
|
|
552
|
+
if (release && (bytes > MAX_FILE_BYTES || total > MAX_FILE_LINES || total > maxEntries)) {
|
|
553
|
+
await compactLocked(key, { maxEntries, maxAgeDays })
|
|
554
|
+
}
|
|
555
|
+
return { written, skipped: normalized.length - written, reason: null }
|
|
556
|
+
} finally {
|
|
557
|
+
await release?.()
|
|
558
|
+
}
|
|
559
|
+
})
|
|
560
|
+
},
|
|
561
|
+
|
|
562
|
+
/**
|
|
563
|
+
* Record one note by hand (the `superagent_memory` tool's `set`).
|
|
564
|
+
* @param {{ scope?: string, key: string, kind?: string, text: string, tags?: unknown, runId?: string }} options - note.
|
|
565
|
+
*/
|
|
566
|
+
async set({ scope = 'default', key, kind = 'note', text, tags = [], runId = null } = {}) {
|
|
567
|
+
const entry = normalizeEntry({ key, kind, text, tags })
|
|
568
|
+
if (!entry) return { written: 0, reason: 'a note needs non-empty text' }
|
|
569
|
+
const result = await this.write({ scope, runId: runId ?? `manual-${randomBytes(3).toString('hex')}`, entries: [entry] })
|
|
570
|
+
return { written: result.written, reason: result.reason }
|
|
571
|
+
},
|
|
572
|
+
|
|
573
|
+
/**
|
|
574
|
+
* Retract a note by key. The tombstone is itself a line, so a later
|
|
575
|
+
* compaction that drops the old note also drops the tombstone.
|
|
576
|
+
* @param {{ scope?: string, key: string }} options - key to forget.
|
|
577
|
+
*/
|
|
578
|
+
async forget({ scope = 'default', key } = {}) {
|
|
579
|
+
const slug = slugKey(key)
|
|
580
|
+
if (!slug) return { forgotten: false, reason: 'a key is required' }
|
|
581
|
+
const result = await this.write({
|
|
582
|
+
scope,
|
|
583
|
+
runId: `forget-${randomBytes(3).toString('hex')}`,
|
|
584
|
+
entries: [{ key: slug, kind: 'note', status: 'retracted' }],
|
|
585
|
+
})
|
|
586
|
+
return { forgotten: result.written > 0, reason: result.reason }
|
|
587
|
+
},
|
|
588
|
+
|
|
589
|
+
/**
|
|
590
|
+
* Rank the notebook against a query. With no query every note is a hit and
|
|
591
|
+
* the order is recency, which is what a digest without a goal means.
|
|
592
|
+
*
|
|
593
|
+
* Retracted notes and expired ones never come back: `read` resolves
|
|
594
|
+
* tombstones and the TTL before scoring, and one key keeps only its newest
|
|
595
|
+
* value, so a corrected note is recalled once, in its corrected form.
|
|
596
|
+
*
|
|
597
|
+
* @param {{ scope?: string, query?: string, tags?: unknown, maxEntries?: number, maxAgeDays?: number, halfLifeDays?: number }} [options] - query.
|
|
598
|
+
* @returns {Promise<{ query: string, tags: string[], searching: boolean, total: number, matched: number, entries: object[] }>}
|
|
599
|
+
*/
|
|
600
|
+
async recall({
|
|
601
|
+
scope = 'default',
|
|
602
|
+
query = '',
|
|
603
|
+
tags = [],
|
|
604
|
+
maxEntries = DEFAULT_DIGEST_ENTRIES,
|
|
605
|
+
maxAgeDays = DEFAULT_MAX_AGE_DAYS,
|
|
606
|
+
halfLifeDays = RECENCY_HALF_LIFE_DAYS,
|
|
607
|
+
} = {}) {
|
|
608
|
+
const at = now()
|
|
609
|
+
const { entries } = await this.read({ scope, maxAgeDays })
|
|
610
|
+
const wanted = normalizeTags(tags)
|
|
611
|
+
const text = typeof query === 'string' ? query : ''
|
|
612
|
+
const searching = tokenize(text).length > 0 || wanted.length > 0
|
|
613
|
+
const scored = entries.map(record => ({
|
|
614
|
+
record,
|
|
615
|
+
...scoreEntry(record, { query: text, tags: wanted, now: at, halfLifeDays }),
|
|
616
|
+
}))
|
|
617
|
+
// A search hides notes that share no term with the goal; otherwise the
|
|
618
|
+
// digest would spend its budget on fresh but irrelevant entries.
|
|
619
|
+
const hits = searching ? scored.filter(item => item.keyword > 0 || item.tag > 0) : scored
|
|
620
|
+
// `sort` is stable, so equal scores keep `read`'s newest-first order.
|
|
621
|
+
hits.sort((a, b) => b.score - a.score)
|
|
622
|
+
return {
|
|
623
|
+
query: text,
|
|
624
|
+
tags: wanted,
|
|
625
|
+
searching,
|
|
626
|
+
total: entries.length,
|
|
627
|
+
matched: hits.length,
|
|
628
|
+
entries: hits.slice(0, Math.max(1, maxEntries)).map(({ record, score, keyword, tag, recency, ageDays, matchedTerms }) => ({
|
|
629
|
+
key: record.key,
|
|
630
|
+
kind: record.kind ?? 'note',
|
|
631
|
+
text: record.text ?? '',
|
|
632
|
+
tags: normalizeTags(record.tags),
|
|
633
|
+
score,
|
|
634
|
+
keyword,
|
|
635
|
+
tag,
|
|
636
|
+
recency,
|
|
637
|
+
ageDays,
|
|
638
|
+
matchedTerms,
|
|
639
|
+
source: {
|
|
640
|
+
runId: record.runId ?? null,
|
|
641
|
+
ts: record.ts ?? null,
|
|
642
|
+
// `read` already dropped retractions, so a recalled note is live.
|
|
643
|
+
retracted: false,
|
|
644
|
+
},
|
|
645
|
+
})),
|
|
646
|
+
}
|
|
647
|
+
},
|
|
648
|
+
|
|
649
|
+
/**
|
|
650
|
+
* Bounded digest injected into prompts. Never the whole file.
|
|
651
|
+
*
|
|
652
|
+
* With a query the lines are ranked by relevance and each one carries its
|
|
653
|
+
* source, so the planner can judge how much to trust it. Without a query —
|
|
654
|
+
* or when nothing matches — it degrades to the newest notes.
|
|
655
|
+
*
|
|
656
|
+
* @param {{ scope?: string, query?: string, tags?: unknown, maxEntries?: number, maxChars?: number, maxAgeDays?: number }} [options] - digest limits.
|
|
657
|
+
* @returns {Promise<string>} note lines, or an empty string.
|
|
658
|
+
*/
|
|
659
|
+
async digest({
|
|
660
|
+
scope = 'default',
|
|
661
|
+
query = '',
|
|
662
|
+
tags = [],
|
|
663
|
+
maxEntries = DEFAULT_DIGEST_ENTRIES,
|
|
664
|
+
maxChars = DEFAULT_DIGEST_CHARS,
|
|
665
|
+
maxAgeDays = DEFAULT_MAX_AGE_DAYS,
|
|
666
|
+
} = {}) {
|
|
667
|
+
const recalled = await this.recall({ scope, query, tags, maxEntries, maxAgeDays })
|
|
668
|
+
let items = recalled.entries
|
|
669
|
+
let sourced = recalled.searching
|
|
670
|
+
if (items.length === 0 && recalled.searching) {
|
|
671
|
+
// An unfamiliar goal must not lose continuity: fall back to the newest
|
|
672
|
+
// notes, which is what this digest did before it could search.
|
|
673
|
+
const { entries } = await this.read({ scope, maxAgeDays })
|
|
674
|
+
items = entries.slice(0, maxEntries).map(record => ({
|
|
675
|
+
key: record.key,
|
|
676
|
+
kind: record.kind ?? 'note',
|
|
677
|
+
text: record.text ?? '',
|
|
678
|
+
ageDays: (now() - tsOf(record)) / 86400000,
|
|
679
|
+
source: { runId: record.runId ?? null, ts: record.ts ?? null, retracted: false },
|
|
680
|
+
}))
|
|
681
|
+
sourced = false
|
|
682
|
+
}
|
|
683
|
+
if (items.length === 0) return ''
|
|
684
|
+
const picked = items.slice(0, maxEntries)
|
|
685
|
+
const share = Math.max(60, Math.floor(maxChars / Math.max(1, picked.length)))
|
|
686
|
+
const lines = []
|
|
687
|
+
let used = 0
|
|
688
|
+
let omitted = 0
|
|
689
|
+
for (const item of picked) {
|
|
690
|
+
const header = sourced
|
|
691
|
+
? `- [${ageFromDays(item.ageDays)}] ${item.kind} ${item.key} `
|
|
692
|
+
+ `(run ${item.source.runId ?? 'manual'}, ${String(item.source.ts ?? '').slice(0, 10)}, `
|
|
693
|
+
+ `${item.source.retracted ? 'retracted' : 'active'}): `
|
|
694
|
+
: `- [${ageFromDays(item.ageDays)}] ${item.kind} ${item.key}: `
|
|
695
|
+
const line = header + capLine(item.text, Math.max(40, share - header.length))
|
|
696
|
+
if (used + line.length + 1 > maxChars) {
|
|
697
|
+
omitted++
|
|
698
|
+
continue
|
|
699
|
+
}
|
|
700
|
+
lines.push(line)
|
|
701
|
+
used += line.length + 1
|
|
702
|
+
}
|
|
703
|
+
if (lines.length === 0) return ''
|
|
704
|
+
if (omitted > 0) {
|
|
705
|
+
lines.push(`- (+${omitted} more; call superagent_memory action="${sourced ? 'search' : 'list'}" to read them)`)
|
|
706
|
+
}
|
|
707
|
+
return lines.join('\n')
|
|
708
|
+
},
|
|
709
|
+
|
|
710
|
+
/**
|
|
711
|
+
* Rows for the `superagent_memory` tool.
|
|
712
|
+
* @param {{ scope?: string, limit?: number, maxAgeDays?: number }} [options] - list options.
|
|
713
|
+
*/
|
|
714
|
+
async list({ scope = 'default', limit = 20, maxAgeDays = DEFAULT_MAX_AGE_DAYS } = {}) {
|
|
715
|
+
const { entries, bad, bytes, path: file } = await this.read({ scope, maxAgeDays })
|
|
716
|
+
return { entries: entries.slice(0, limit), total: entries.length, bad, bytes, path: file }
|
|
717
|
+
},
|
|
718
|
+
|
|
719
|
+
/**
|
|
720
|
+
* Size and count for `superagent_list`.
|
|
721
|
+
* @param {{ scope?: string, maxAgeDays?: number }} [options] - stats options.
|
|
722
|
+
*/
|
|
723
|
+
async stats({ scope = 'default', maxAgeDays = DEFAULT_MAX_AGE_DAYS } = {}) {
|
|
724
|
+
const { entries, bytes, path: file } = await this.read({ scope, maxAgeDays })
|
|
725
|
+
return { entries: entries.length, bytes, path: file }
|
|
726
|
+
},
|
|
727
|
+
|
|
728
|
+
/** Defaults exposed so the orchestrator and tests agree on one set of limits. */
|
|
729
|
+
defaults: {
|
|
730
|
+
maxEntries: DEFAULT_MAX_ENTRIES,
|
|
731
|
+
maxAgeDays: DEFAULT_MAX_AGE_DAYS,
|
|
732
|
+
digestEntries: DEFAULT_DIGEST_ENTRIES,
|
|
733
|
+
digestChars: DEFAULT_DIGEST_CHARS,
|
|
734
|
+
},
|
|
735
|
+
}
|
|
736
|
+
}
|