@xiangyixie/dsh-superagent 0.12.1 → 0.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +192 -9
- package/LICENSE +21 -21
- package/README.en.md +1771 -0
- package/README.md +1167 -1274
- package/README.zh.md +1167 -0
- package/bin/doctor.js +1 -1
- package/cordis.patch.yml +9 -9
- package/docs/DETAIL.md +37 -0
- package/docs/DETAIL.zh.md +37 -0
- package/icon.svg +10 -0
- package/lib/audit.js +8 -1
- package/lib/browser.js +795 -0
- package/lib/console.js +763 -0
- package/lib/cost.js +71 -11
- package/lib/cron.js +387 -0
- package/lib/desktop.js +507 -0
- package/lib/doctor.js +34 -1
- package/lib/goal.js +649 -0
- package/lib/guard.js +15 -0
- package/lib/index.js +1296 -42
- package/lib/ledger.js +26 -12
- package/lib/orchestrator.js +1765 -136
- package/lib/report.js +59 -29
- package/lib/runtime-control.js +1151 -0
- package/lib/schedule.js +1030 -0
- package/lib/store.js +6 -2
- package/lib/user-memory.js +587 -0
- package/lib/watch.js +25 -17
- package/lib/workbench.js +7 -7
- package/lib/workspace.js +88 -27
- package/locale/en.json +7 -0
- package/locale/zh.json +7 -0
- package/package.json +109 -91
- package/public/console.html +1618 -0
package/lib/store.js
CHANGED
|
@@ -17,6 +17,7 @@ import { createHash } from 'node:crypto'
|
|
|
17
17
|
import fsp from 'node:fs/promises'
|
|
18
18
|
import os from 'node:os'
|
|
19
19
|
import path from 'node:path'
|
|
20
|
+
import { formatCurrencyTally } from './cost.js'
|
|
20
21
|
|
|
21
22
|
/** Format marker written into every exported archive. */
|
|
22
23
|
export const ARCHIVE_FORMAT = 'dsh-superagent/run-archive'
|
|
@@ -109,8 +110,11 @@ function renderGuardsMarkdown(guards, at) {
|
|
|
109
110
|
|
|
110
111
|
/** Human-readable cost with the unpriced caveat stated, never guessed. */
|
|
111
112
|
function renderCost(cost) {
|
|
112
|
-
if (!cost
|
|
113
|
-
const
|
|
113
|
+
if (!cost) return 'unpriced (no configured price for these models)'
|
|
114
|
+
const tally = cost.byCurrency && typeof cost.byCurrency === 'object' ? cost.byCurrency : null
|
|
115
|
+
const mixed = tally !== null && Object.keys(tally).length > 1
|
|
116
|
+
if (!mixed && !Number.isFinite(cost.total)) return 'unpriced (no configured price for these models)'
|
|
117
|
+
const amount = mixed ? formatCurrencyTally(tally) : `${cost.currency ?? 'USD'} ${cost.total.toFixed(4)}`
|
|
114
118
|
if (cost.complete === false) {
|
|
115
119
|
return `${amount} (partial; unpriced: ${(cost.unpriced ?? []).join(', ') || 'unknown'})`
|
|
116
120
|
}
|
|
@@ -0,0 +1,587 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* User-level memory for dsh-superagent.
|
|
3
|
+
*
|
|
4
|
+
* `memory.js` answers "what did the last run learn about this task". That is a
|
|
5
|
+
* run notebook: it is ranked against one goal and it is meant to age out. This
|
|
6
|
+
* module answers a different question — "who is this person, what do they want,
|
|
7
|
+
* and why did we choose this before" — and it has to survive across unrelated
|
|
8
|
+
* tasks, so it is stored apart and read apart:
|
|
9
|
+
*
|
|
10
|
+
* - Three categories, three files: `facts` (durable technical context such as
|
|
11
|
+
* the stack or the environment), `preferences` (how the user wants work done)
|
|
12
|
+
* and `decisions` (a choice plus what was rejected, why, and how to undo it).
|
|
13
|
+
* A search opens one category file, so a fact can never be returned as a
|
|
14
|
+
* preference just because the words match.
|
|
15
|
+
* - A decision is only stored when it has a choice and a reason. A conclusion
|
|
16
|
+
* without its rationale teaches the next run nothing, so the reason is part
|
|
17
|
+
* of the record and part of every digest line.
|
|
18
|
+
* - Everything is redacted before it is written, by default: credentials,
|
|
19
|
+
* emails, phone numbers, and usernames inside home paths. The profile is meant
|
|
20
|
+
* to be readable by a future model, not to be a copy of the user's secrets.
|
|
21
|
+
* - Forgetting is a real deletion: the matching lines are removed from the file
|
|
22
|
+
* under a lock and the file is replaced. There is no tombstone to leak the
|
|
23
|
+
* value back through a later search. Three filters are supported — a key, a
|
|
24
|
+
* run id, and a time range — and at least one is required, so a bare "forget"
|
|
25
|
+
* cannot wipe the profile by accident.
|
|
26
|
+
*
|
|
27
|
+
* The module is off by default (`memoryUserProfile: false`): unless the operator
|
|
28
|
+
* enables it, nothing here is constructed, no directory is created, and the
|
|
29
|
+
* existing run notebook behaves exactly as before. It has no dependencies and
|
|
30
|
+
* reuses `memory.js` primitives (redaction, slug keys, tokenising, scoring) so
|
|
31
|
+
* both layers agree on what a key is and how relevance is measured.
|
|
32
|
+
*
|
|
33
|
+
* @module dsh-superagent/user-memory
|
|
34
|
+
*/
|
|
35
|
+
import { randomBytes } from 'node:crypto'
|
|
36
|
+
import fsp from 'node:fs/promises'
|
|
37
|
+
import path from 'node:path'
|
|
38
|
+
import { acquireFileLock } from './lock.js'
|
|
39
|
+
import { cleanText, normalizeTags, scoreEntry, slugKey } from './memory.js'
|
|
40
|
+
|
|
41
|
+
/** The three categories, in the order a digest prints them. */
|
|
42
|
+
export const USER_MEMORY_CATEGORIES = Object.freeze(['fact', 'preference', 'decision'])
|
|
43
|
+
|
|
44
|
+
/** One file per category, so a search can never read across categories. */
|
|
45
|
+
const CATEGORY_FILES = Object.freeze({
|
|
46
|
+
fact: 'facts.jsonl',
|
|
47
|
+
preference: 'preferences.jsonl',
|
|
48
|
+
decision: 'decisions.jsonl',
|
|
49
|
+
})
|
|
50
|
+
|
|
51
|
+
/** Longest value kept per decision field. A decision is a sentence, not a log. */
|
|
52
|
+
const MAX_CHOICE_CHARS = 200
|
|
53
|
+
const MAX_REJECTED_CHARS = 160
|
|
54
|
+
const MAX_REASON_CHARS = 240
|
|
55
|
+
const MAX_ROLLBACK_CHARS = 200
|
|
56
|
+
|
|
57
|
+
/** File-level limits; crossing either one, or `maxEntries`, triggers compaction. */
|
|
58
|
+
const MAX_FILE_BYTES = 128 * 1024
|
|
59
|
+
const MAX_FILE_LINES = 400
|
|
60
|
+
|
|
61
|
+
/** How long a writer waits for another process's lock before failing open. */
|
|
62
|
+
const LOCK_RETRIES = 10
|
|
63
|
+
const LOCK_RETRY_DELAY_MS = 50
|
|
64
|
+
|
|
65
|
+
/** Defaults; a profile is long-lived, so it expires far more slowly than a run note. */
|
|
66
|
+
export const USER_MEMORY_DEFAULTS = Object.freeze({
|
|
67
|
+
maxEntries: 200,
|
|
68
|
+
maxAgeDays: 365,
|
|
69
|
+
digestEntries: 8,
|
|
70
|
+
digestChars: 1200,
|
|
71
|
+
})
|
|
72
|
+
|
|
73
|
+
const USER_MEMORY_FENCE = /```user-memory[^\n]*\n([\s\S]*?)```/i
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Redact the shapes that identify a person but are not secrets in the API-key
|
|
77
|
+
* sense: phone numbers. The credential, email and home-path rules live in
|
|
78
|
+
* `memory.redact`, which `cleanText` applies after this step.
|
|
79
|
+
*
|
|
80
|
+
* @param {unknown} value - text that may contain a phone number.
|
|
81
|
+
* @returns {string} the text with phone numbers replaced by a marker.
|
|
82
|
+
*/
|
|
83
|
+
export function redactPhones(value) {
|
|
84
|
+
if (typeof value !== 'string') return ''
|
|
85
|
+
return value
|
|
86
|
+
// Mainland Chinese mobile numbers, with or without a +86 prefix.
|
|
87
|
+
.replace(/(?<!\d)(?:\+?86[-\s]?)?1[3-9]\d{9}(?!\d)/g, '[redacted-phone]')
|
|
88
|
+
// Grouped international numbers such as +1 415 555 0100.
|
|
89
|
+
.replace(/(?<!\d)\+?\d{1,3}[-\s]\d{3,4}[-\s]\d{4}(?!\d)/g, '[redacted-phone]')
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* One bounded, redacted line. Applied to every field of every user-level
|
|
94
|
+
* record, so a caller cannot bypass redaction by writing a decision instead of
|
|
95
|
+
* a note.
|
|
96
|
+
*
|
|
97
|
+
* @param {unknown} value - source text.
|
|
98
|
+
* @param {number} [limit] - maximum characters.
|
|
99
|
+
* @returns {string} a clean single line.
|
|
100
|
+
*/
|
|
101
|
+
export function sanitizeUserText(value, limit) {
|
|
102
|
+
return cleanText(redactPhones(String(value ?? '')), limit)
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function categoryOf(value) {
|
|
106
|
+
const text = typeof value === 'string' ? value.trim().toLowerCase() : ''
|
|
107
|
+
if (text === 'facts') return 'fact'
|
|
108
|
+
if (text === 'preferences') return 'preference'
|
|
109
|
+
if (text === 'decisions') return 'decision'
|
|
110
|
+
return USER_MEMORY_CATEGORIES.includes(text) ? text : null
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* The searchable projection of a decision. Scoring works on `key`, `tags` and
|
|
115
|
+
* `text`, so a decision has to present its rationale as its text: a search for
|
|
116
|
+
* "sqlite" must find the decision that rejected it, not only the choice.
|
|
117
|
+
*
|
|
118
|
+
* @param {object} record - a decision record.
|
|
119
|
+
* @returns {string} key and rationale joined into one line.
|
|
120
|
+
*/
|
|
121
|
+
function decisionText(record) {
|
|
122
|
+
return [record.choice, record.rejected, record.reason, record.rollback].filter(Boolean).join('; ')
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function normalizeRecord(category, item) {
|
|
126
|
+
if (!item || typeof item !== 'object') return null
|
|
127
|
+
const tags = normalizeTags(item.tags)
|
|
128
|
+
if (category === 'decision') {
|
|
129
|
+
const choice = sanitizeUserText(item.choice, MAX_CHOICE_CHARS)
|
|
130
|
+
const rejected = sanitizeUserText(item.rejected, MAX_REJECTED_CHARS)
|
|
131
|
+
const reason = sanitizeUserText(item.reason, MAX_REASON_CHARS)
|
|
132
|
+
const rollback = sanitizeUserText(item.rollback, MAX_ROLLBACK_CHARS)
|
|
133
|
+
// A decision without its rationale is exactly the note this module exists
|
|
134
|
+
// to avoid, so it is refused rather than stored as a bare conclusion.
|
|
135
|
+
if (!choice || !reason) return null
|
|
136
|
+
const key = slugKey(item.key) || slugKey(choice) || `decision-${randomBytes(3).toString('hex')}`
|
|
137
|
+
return { key, category, choice, rejected, reason, rollback, tags }
|
|
138
|
+
}
|
|
139
|
+
const text = sanitizeUserText(item.text)
|
|
140
|
+
if (!text) return null
|
|
141
|
+
const key = slugKey(item.key) || slugKey(text.slice(0, 48)) || `${category}-${randomBytes(3).toString('hex')}`
|
|
142
|
+
return { key, category, text, tags }
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
function tsOf(record) {
|
|
146
|
+
const value = Date.parse(record?.ts ?? '')
|
|
147
|
+
return Number.isFinite(value) ? value : 0
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function msOf(value) {
|
|
151
|
+
if (value === undefined || value === null || value === '') return null
|
|
152
|
+
if (value instanceof Date) return Number.isFinite(value.getTime()) ? value.getTime() : null
|
|
153
|
+
if (typeof value === 'number') return Number.isFinite(value) ? value : null
|
|
154
|
+
const parsed = Date.parse(String(value))
|
|
155
|
+
return Number.isFinite(parsed) ? parsed : null
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* Resolve raw lines into live records: a newer record with the same key wins,
|
|
160
|
+
* and records older than the TTL are ignored. There is no tombstone here —
|
|
161
|
+
* forgetting rewrites the file — so this is only supersession plus age.
|
|
162
|
+
*
|
|
163
|
+
* @param {object[]} records - parsed lines.
|
|
164
|
+
* @param {{ now?: number, maxAgeDays?: number }} [options] - clock and TTL.
|
|
165
|
+
* @returns {object[]} active records, newest first.
|
|
166
|
+
*/
|
|
167
|
+
export function effectiveUserRecords(records, { now = Date.now(), maxAgeDays = USER_MEMORY_DEFAULTS.maxAgeDays } = {}) {
|
|
168
|
+
const active = new Map()
|
|
169
|
+
const ordered = records
|
|
170
|
+
.map((record, index) => ({ record, index }))
|
|
171
|
+
.filter(({ record }) => record && typeof record === 'object' && typeof record.key === 'string')
|
|
172
|
+
.sort((a, b) => (tsOf(a.record) - tsOf(b.record)) || (a.index - b.index))
|
|
173
|
+
for (const { record, index } of ordered) active.set(record.key, { record, index })
|
|
174
|
+
const cutoff = maxAgeDays > 0 ? now - maxAgeDays * 86400000 : -Infinity
|
|
175
|
+
return [...active.values()]
|
|
176
|
+
.filter(({ record }) => tsOf(record) >= cutoff)
|
|
177
|
+
.sort((a, b) => (tsOf(b.record) - tsOf(a.record)) || (b.index - a.index))
|
|
178
|
+
.map(({ record }) => record)
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* Pull the fenced `user-memory` block out of an aggregator report.
|
|
183
|
+
*
|
|
184
|
+
* The block is removed whether or not it parses, because it is plugin syntax
|
|
185
|
+
* the user did not ask to see. A malformed block is reported through `error`
|
|
186
|
+
* so the run record can say why nothing was stored.
|
|
187
|
+
*
|
|
188
|
+
* @param {unknown} text - final report.
|
|
189
|
+
* @returns {{ report: string, found: boolean, error?: string, block: { facts: object[], preferences: object[], decisions: object[] } }}
|
|
190
|
+
*/
|
|
191
|
+
export function extractUserMemoryBlock(text) {
|
|
192
|
+
const empty = { facts: [], preferences: [], decisions: [] }
|
|
193
|
+
if (typeof text !== 'string' || !text) return { report: typeof text === 'string' ? text : '', found: false, block: empty }
|
|
194
|
+
const match = USER_MEMORY_FENCE.exec(text)
|
|
195
|
+
if (!match) return { report: text, found: false, block: empty }
|
|
196
|
+
const report = `${text.slice(0, match.index)}${text.slice(match.index + match[0].length)}`
|
|
197
|
+
.replace(/\n{3,}/g, '\n\n')
|
|
198
|
+
.trim()
|
|
199
|
+
let parsed
|
|
200
|
+
try {
|
|
201
|
+
parsed = JSON.parse(match[1].trim())
|
|
202
|
+
} catch {
|
|
203
|
+
return { report, found: true, error: 'the user-memory block was not valid JSON', block: empty }
|
|
204
|
+
}
|
|
205
|
+
const list = key => (Array.isArray(parsed?.[key]) ? parsed[key] : [])
|
|
206
|
+
const block = { facts: list('facts'), preferences: list('preferences'), decisions: list('decisions') }
|
|
207
|
+
if (block.facts.length + block.preferences.length + block.decisions.length === 0) {
|
|
208
|
+
return { report, found: true, error: 'the user-memory block had no usable entry', block }
|
|
209
|
+
}
|
|
210
|
+
return { report, found: true, block }
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* Build the user-level profile bound to one state root.
|
|
215
|
+
*
|
|
216
|
+
* @param {string} root - absolute state directory (`<root>/user-memory` is used).
|
|
217
|
+
* @param {object} [options] - collaborators.
|
|
218
|
+
* @param {(message: string) => void} [options.warn] - diagnostic sink.
|
|
219
|
+
* @param {() => number} [options.now] - clock, injectable for tests.
|
|
220
|
+
*/
|
|
221
|
+
export function createUserMemory(root, { warn = () => {}, now = () => Date.now() } = {}) {
|
|
222
|
+
const dir = path.join(root, 'user-memory')
|
|
223
|
+
const fileOf = category => path.join(dir, CATEGORY_FILES[category])
|
|
224
|
+
const lockOf = category => path.join(dir, `${category}.lock`)
|
|
225
|
+
/** Per-category write chain, so two records in this process cannot interleave. */
|
|
226
|
+
const chains = new Map()
|
|
227
|
+
|
|
228
|
+
async function initDir() {
|
|
229
|
+
await fsp.mkdir(dir, { recursive: true })
|
|
230
|
+
const ignore = path.join(dir, '.gitignore')
|
|
231
|
+
try {
|
|
232
|
+
await fsp.access(ignore)
|
|
233
|
+
} catch {
|
|
234
|
+
await fsp.writeFile(ignore, '*\n', 'utf8')
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
function serialize(category, task) {
|
|
239
|
+
const previous = chains.get(category) ?? Promise.resolve()
|
|
240
|
+
const next = previous.catch(() => {}).then(task)
|
|
241
|
+
chains.set(category, next.catch(() => {}))
|
|
242
|
+
return next
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
async function readRecords(category) {
|
|
246
|
+
let text
|
|
247
|
+
try {
|
|
248
|
+
text = await fsp.readFile(fileOf(category), 'utf8')
|
|
249
|
+
} catch (error) {
|
|
250
|
+
if (error?.code === 'ENOENT') return { records: [], bad: 0, bytes: 0 }
|
|
251
|
+
throw error
|
|
252
|
+
}
|
|
253
|
+
const records = []
|
|
254
|
+
let bad = 0
|
|
255
|
+
for (const line of text.split('\n')) {
|
|
256
|
+
const trimmed = line.trim()
|
|
257
|
+
if (!trimmed) continue
|
|
258
|
+
try {
|
|
259
|
+
const parsed = JSON.parse(trimmed)
|
|
260
|
+
if (parsed && typeof parsed === 'object') records.push(parsed)
|
|
261
|
+
else bad++
|
|
262
|
+
} catch {
|
|
263
|
+
bad++
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
return { records, bad, bytes: Buffer.byteLength(text, 'utf8') }
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/** Rewrite one category file with an exact set of records, under its lock. */
|
|
270
|
+
async function rewriteLocked(category, records) {
|
|
271
|
+
const file = fileOf(category)
|
|
272
|
+
const body = records.map(record => `${JSON.stringify(record)}\n`).join('')
|
|
273
|
+
const tmp = path.join(dir, `.${category}.${process.pid}.${Date.now()}.tmp`)
|
|
274
|
+
await fsp.writeFile(tmp, body, 'utf8')
|
|
275
|
+
await fsp.rename(tmp, file)
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
async function compactLocked(category, { maxEntries, maxAgeDays }) {
|
|
279
|
+
try {
|
|
280
|
+
const { records } = await readRecords(category)
|
|
281
|
+
const keep = effectiveUserRecords(records, { now: now(), maxAgeDays })
|
|
282
|
+
.slice(0, maxEntries)
|
|
283
|
+
.reverse()
|
|
284
|
+
await rewriteLocked(category, keep)
|
|
285
|
+
return true
|
|
286
|
+
} catch (error) {
|
|
287
|
+
warn(`cannot compact user memory category "${category}": ${error?.message ?? error}`)
|
|
288
|
+
return false
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
async function appendLocked(category, records) {
|
|
293
|
+
if (records.length === 0) return 0
|
|
294
|
+
const file = fileOf(category)
|
|
295
|
+
await initDir()
|
|
296
|
+
const body = records.map(record => `${JSON.stringify(record)}\n`).join('')
|
|
297
|
+
const handle = await fsp.open(file, 'a')
|
|
298
|
+
try {
|
|
299
|
+
await handle.write(body)
|
|
300
|
+
} finally {
|
|
301
|
+
await handle.close()
|
|
302
|
+
}
|
|
303
|
+
return records.length
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
/**
|
|
307
|
+
* Write a batch into one category. Exposed through the per-category helpers
|
|
308
|
+
* below so a caller cannot pass a category that has no file.
|
|
309
|
+
*/
|
|
310
|
+
async function writeCategory(category, entries, { runId = null, maxEntries = USER_MEMORY_DEFAULTS.maxEntries, maxAgeDays = USER_MEMORY_DEFAULTS.maxAgeDays } = {}) {
|
|
311
|
+
const normalized = (Array.isArray(entries) ? entries : [])
|
|
312
|
+
.map(item => normalizeRecord(category, item))
|
|
313
|
+
.filter(Boolean)
|
|
314
|
+
if (normalized.length === 0) return { written: 0, reason: 'no usable entry' }
|
|
315
|
+
return serialize(category, async () => {
|
|
316
|
+
await initDir()
|
|
317
|
+
const release = await acquireFileLock(lockOf(category), {
|
|
318
|
+
now,
|
|
319
|
+
retries: LOCK_RETRIES,
|
|
320
|
+
retryDelayMs: LOCK_RETRY_DELAY_MS,
|
|
321
|
+
})
|
|
322
|
+
if (!release) warn(`user memory category "${category}" is locked by another process; appended without compaction`)
|
|
323
|
+
try {
|
|
324
|
+
const ts = new Date(now()).toISOString()
|
|
325
|
+
const batch = normalized.map(entry => ({
|
|
326
|
+
v: 1,
|
|
327
|
+
id: randomBytes(4).toString('hex'),
|
|
328
|
+
ts,
|
|
329
|
+
runId: runId ?? null,
|
|
330
|
+
...entry,
|
|
331
|
+
}))
|
|
332
|
+
const written = await appendLocked(category, batch)
|
|
333
|
+
let bytes = 0
|
|
334
|
+
try {
|
|
335
|
+
bytes = (await fsp.stat(fileOf(category))).size
|
|
336
|
+
} catch {
|
|
337
|
+
// A missing stat only means compaction cannot be sized this round.
|
|
338
|
+
}
|
|
339
|
+
const { records } = await readRecords(category)
|
|
340
|
+
const total = records.length + batch.length
|
|
341
|
+
if (release && (bytes > MAX_FILE_BYTES || total > MAX_FILE_LINES || total > maxEntries)) {
|
|
342
|
+
await compactLocked(category, { maxEntries, maxAgeDays })
|
|
343
|
+
}
|
|
344
|
+
return { written, reason: null }
|
|
345
|
+
} finally {
|
|
346
|
+
await release?.()
|
|
347
|
+
}
|
|
348
|
+
})
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
/** Search one category. Never reads another category's file. */
|
|
352
|
+
async function recallCategory(category, {
|
|
353
|
+
query = '',
|
|
354
|
+
tags = [],
|
|
355
|
+
maxEntries = USER_MEMORY_DEFAULTS.digestEntries,
|
|
356
|
+
maxAgeDays = USER_MEMORY_DEFAULTS.maxAgeDays,
|
|
357
|
+
} = {}) {
|
|
358
|
+
const at = now()
|
|
359
|
+
const { records } = await readRecords(category)
|
|
360
|
+
const entries = effectiveUserRecords(records, { now: at, maxAgeDays })
|
|
361
|
+
const wanted = normalizeTags(tags)
|
|
362
|
+
const text = typeof query === 'string' ? query : ''
|
|
363
|
+
const searching = text.trim().length > 0 || wanted.length > 0
|
|
364
|
+
const scored = entries.map(record => {
|
|
365
|
+
const view = category === 'decision'
|
|
366
|
+
? { key: record.key, tags: record.tags, ts: record.ts, text: decisionText(record) }
|
|
367
|
+
: record
|
|
368
|
+
return { record, ...scoreEntry(view, { query: text, tags: wanted, now: at }) }
|
|
369
|
+
})
|
|
370
|
+
const hits = searching ? scored.filter(item => item.keyword > 0 || item.tag > 0) : scored
|
|
371
|
+
hits.sort((a, b) => b.score - a.score)
|
|
372
|
+
return {
|
|
373
|
+
category,
|
|
374
|
+
searching,
|
|
375
|
+
total: entries.length,
|
|
376
|
+
matched: hits.length,
|
|
377
|
+
entries: hits.slice(0, Math.max(1, maxEntries)).map(({ record, score, keyword, tag, ageDays, matchedTerms }) => ({
|
|
378
|
+
key: record.key,
|
|
379
|
+
category,
|
|
380
|
+
...(category === 'decision'
|
|
381
|
+
? {
|
|
382
|
+
choice: record.choice ?? '',
|
|
383
|
+
rejected: record.rejected ?? '',
|
|
384
|
+
reason: record.reason ?? '',
|
|
385
|
+
rollback: record.rollback ?? '',
|
|
386
|
+
}
|
|
387
|
+
: { text: record.text ?? '' }),
|
|
388
|
+
tags: normalizeTags(record.tags),
|
|
389
|
+
score,
|
|
390
|
+
keyword,
|
|
391
|
+
tag,
|
|
392
|
+
ageDays,
|
|
393
|
+
matchedTerms,
|
|
394
|
+
source: { runId: record.runId ?? null, ts: record.ts ?? null },
|
|
395
|
+
})),
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
return {
|
|
400
|
+
root,
|
|
401
|
+
dir,
|
|
402
|
+
fileOf,
|
|
403
|
+
|
|
404
|
+
/** Create the directory and make it git-invisible even inside a repository. */
|
|
405
|
+
async init() {
|
|
406
|
+
await initDir()
|
|
407
|
+
},
|
|
408
|
+
|
|
409
|
+
/**
|
|
410
|
+
* Record one fact, preference or decision.
|
|
411
|
+
* @param {object} options - `category` plus the fields that category needs.
|
|
412
|
+
*/
|
|
413
|
+
async record({ category, entry, runId = null, maxEntries, maxAgeDays } = {}) {
|
|
414
|
+
const key = categoryOf(category)
|
|
415
|
+
if (!key) return { written: 0, reason: `category must be one of ${USER_MEMORY_CATEGORIES.join(', ')}` }
|
|
416
|
+
return writeCategory(key, [entry], { runId, maxEntries, maxAgeDays })
|
|
417
|
+
},
|
|
418
|
+
|
|
419
|
+
/**
|
|
420
|
+
* Store the three lists of a parsed `user-memory` block.
|
|
421
|
+
* @param {{ runId: string, block: { facts?: object[], preferences?: object[], decisions?: object[] }, maxEntries?: number, maxAgeDays?: number }} options - block.
|
|
422
|
+
* @returns {Promise<{ fact: number, preference: number, decision: number, skipped: number }>}
|
|
423
|
+
*/
|
|
424
|
+
async writeBlock({ runId = null, block = {}, maxEntries, maxAgeDays } = {}) {
|
|
425
|
+
const result = { fact: 0, preference: 0, decision: 0, skipped: 0 }
|
|
426
|
+
const jobs = [
|
|
427
|
+
['fact', block.facts],
|
|
428
|
+
['preference', block.preferences],
|
|
429
|
+
['decision', block.decisions],
|
|
430
|
+
]
|
|
431
|
+
for (const [category, items] of jobs) {
|
|
432
|
+
if (!Array.isArray(items) || items.length === 0) continue
|
|
433
|
+
const written = await writeCategory(category, items, { runId, maxEntries, maxAgeDays })
|
|
434
|
+
result[category] += written.written
|
|
435
|
+
result.skipped += items.length - written.written
|
|
436
|
+
}
|
|
437
|
+
return result
|
|
438
|
+
},
|
|
439
|
+
|
|
440
|
+
/**
|
|
441
|
+
* Search one category.
|
|
442
|
+
* @param {{ category: string, query?: string, tags?: unknown, maxEntries?: number, maxAgeDays?: number }} options - query.
|
|
443
|
+
*/
|
|
444
|
+
async recall({ category, query = '', tags = [], maxEntries, maxAgeDays } = {}) {
|
|
445
|
+
const key = categoryOf(category)
|
|
446
|
+
if (!key) return { category: null, searching: false, total: 0, matched: 0, entries: [], reason: 'unknown category' }
|
|
447
|
+
return recallCategory(key, { query, tags, maxEntries, maxAgeDays })
|
|
448
|
+
},
|
|
449
|
+
|
|
450
|
+
/**
|
|
451
|
+
* Bounded, category-labelled digest for prompts. Facts, preferences and
|
|
452
|
+
* decisions are printed under separate headings, and a decision line always
|
|
453
|
+
* carries what was rejected, why, and how to roll back.
|
|
454
|
+
*
|
|
455
|
+
* @param {{ query?: string, maxEntries?: number, maxChars?: number, maxAgeDays?: number, categories?: string[] }} [options] - digest.
|
|
456
|
+
* @returns {Promise<string>} digest text, or an empty string.
|
|
457
|
+
*/
|
|
458
|
+
async digest({
|
|
459
|
+
query = '',
|
|
460
|
+
maxEntries = USER_MEMORY_DEFAULTS.digestEntries,
|
|
461
|
+
maxChars = USER_MEMORY_DEFAULTS.digestChars,
|
|
462
|
+
maxAgeDays = USER_MEMORY_DEFAULTS.maxAgeDays,
|
|
463
|
+
categories = USER_MEMORY_CATEGORIES,
|
|
464
|
+
} = {}) {
|
|
465
|
+
const wanted = categories.map(categoryOf).filter(Boolean)
|
|
466
|
+
const headings = { fact: 'Facts', preference: 'Preferences', decision: 'Decisions (with rationale)' }
|
|
467
|
+
const sections = []
|
|
468
|
+
const share = Math.max(120, Math.floor(maxChars / Math.max(1, wanted.length)))
|
|
469
|
+
for (const category of wanted) {
|
|
470
|
+
const recalled = await recallCategory(category, { query, maxEntries, maxAgeDays })
|
|
471
|
+
let items = recalled.entries
|
|
472
|
+
if (items.length === 0 && recalled.searching) {
|
|
473
|
+
// An unfamiliar goal must not hide who the user is: fall back to the
|
|
474
|
+
// newest entries, exactly as the run notebook does.
|
|
475
|
+
const { records } = await readRecords(category)
|
|
476
|
+
items = effectiveUserRecords(records, { now: now(), maxAgeDays })
|
|
477
|
+
.slice(0, maxEntries)
|
|
478
|
+
.map(record => ({
|
|
479
|
+
...record,
|
|
480
|
+
ageDays: (now() - tsOf(record)) / 86400000,
|
|
481
|
+
}))
|
|
482
|
+
}
|
|
483
|
+
if (items.length === 0) continue
|
|
484
|
+
const lines = []
|
|
485
|
+
let used = 0
|
|
486
|
+
for (const item of items.slice(0, maxEntries)) {
|
|
487
|
+
const age = Number.isFinite(item.ageDays) ? `${Math.max(0, Math.floor(item.ageDays))}d` : '?'
|
|
488
|
+
const body = category === 'decision'
|
|
489
|
+
? `chose ${item.choice}; rejected ${item.rejected || 'nothing recorded'}; `
|
|
490
|
+
+ `why ${item.reason}; rollback ${item.rollback || 'not recorded'}`
|
|
491
|
+
: item.text
|
|
492
|
+
const line = `- [${age}] ${category} ${item.key} `
|
|
493
|
+
+ `(run ${item.source?.runId ?? item.runId ?? 'manual'}, `
|
|
494
|
+
+ `${String(item.source?.ts ?? item.ts ?? '').slice(0, 10)}): ${body}`
|
|
495
|
+
if (used + line.length + 1 > share) break
|
|
496
|
+
lines.push(line)
|
|
497
|
+
used += line.length + 1
|
|
498
|
+
}
|
|
499
|
+
if (lines.length > 0) sections.push(`${headings[category]}:\n${lines.join('\n')}`)
|
|
500
|
+
}
|
|
501
|
+
const text = sections.join('\n\n')
|
|
502
|
+
return text.length > maxChars ? `${[...text].slice(0, Math.max(0, maxChars - 1)).join('')}…` : text
|
|
503
|
+
},
|
|
504
|
+
|
|
505
|
+
/**
|
|
506
|
+
* Physical deletion. At least one filter is required, and only matching
|
|
507
|
+
* lines are rewritten away — there is no tombstone left behind, so a later
|
|
508
|
+
* search cannot observe the deleted value.
|
|
509
|
+
*
|
|
510
|
+
* @param {{ key?: string, runId?: string, before?: string | number | Date, after?: string | number | Date, category?: string }} [options] - filters.
|
|
511
|
+
* @returns {Promise<{ deleted: number, categories: string[], reason: string | null }>}
|
|
512
|
+
*/
|
|
513
|
+
async forget({ key, runId, before, after, category } = {}) {
|
|
514
|
+
const slug = key === undefined || key === null || key === '' ? null : slugKey(key)
|
|
515
|
+
const run = typeof runId === 'string' && runId.trim() ? runId.trim() : null
|
|
516
|
+
const from = msOf(after)
|
|
517
|
+
const to = msOf(before)
|
|
518
|
+
if (!slug && !run && from === null && to === null) {
|
|
519
|
+
return { deleted: 0, categories: [], reason: 'a key, a run id, or a time range is required' }
|
|
520
|
+
}
|
|
521
|
+
if ((after !== undefined && from === null) || (before !== undefined && to === null)) {
|
|
522
|
+
return { deleted: 0, categories: [], reason: 'before/after must be a date or a timestamp' }
|
|
523
|
+
}
|
|
524
|
+
const targets = category ? [categoryOf(category)].filter(Boolean) : [...USER_MEMORY_CATEGORIES]
|
|
525
|
+
if (targets.length === 0) return { deleted: 0, categories: [], reason: 'unknown category' }
|
|
526
|
+
const match = record => {
|
|
527
|
+
if (slug && record.key !== slug) return false
|
|
528
|
+
if (run && record.runId !== run) return false
|
|
529
|
+
const ts = tsOf(record)
|
|
530
|
+
if (from !== null && ts < from) return false
|
|
531
|
+
if (to !== null && ts > to) return false
|
|
532
|
+
return true
|
|
533
|
+
}
|
|
534
|
+
let deleted = 0
|
|
535
|
+
const touched = []
|
|
536
|
+
for (const target of targets) {
|
|
537
|
+
const count = await serialize(target, async () => {
|
|
538
|
+
await initDir()
|
|
539
|
+
const release = await acquireFileLock(lockOf(target), {
|
|
540
|
+
now,
|
|
541
|
+
retries: LOCK_RETRIES,
|
|
542
|
+
retryDelayMs: LOCK_RETRY_DELAY_MS,
|
|
543
|
+
})
|
|
544
|
+
if (!release) warn(`user memory category "${target}" is locked by another process; deleting without hands-off compaction`)
|
|
545
|
+
try {
|
|
546
|
+
const { records } = await readRecords(target)
|
|
547
|
+
const keep = records.filter(record => !match(record))
|
|
548
|
+
const removed = records.length - keep.length
|
|
549
|
+
if (removed > 0) await rewriteLocked(target, keep)
|
|
550
|
+
return removed
|
|
551
|
+
} finally {
|
|
552
|
+
await release?.()
|
|
553
|
+
}
|
|
554
|
+
})
|
|
555
|
+
if (count > 0) {
|
|
556
|
+
deleted += count
|
|
557
|
+
touched.push(target)
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
return { deleted, categories: touched, reason: null }
|
|
561
|
+
},
|
|
562
|
+
|
|
563
|
+
/**
|
|
564
|
+
* Every live record in one category, newest first.
|
|
565
|
+
* @param {{ category: string, limit?: number, maxAgeDays?: number }} options - read.
|
|
566
|
+
*/
|
|
567
|
+
async list({ category, limit = 50, maxAgeDays = USER_MEMORY_DEFAULTS.maxAgeDays } = {}) {
|
|
568
|
+
const key = categoryOf(category)
|
|
569
|
+
if (!key) return { category: null, entries: [], total: 0, path: dir }
|
|
570
|
+
const { records } = await readRecords(key)
|
|
571
|
+
const entries = effectiveUserRecords(records, { now: now(), maxAgeDays })
|
|
572
|
+
return { category: key, entries: entries.slice(0, limit), total: entries.length, path: fileOf(key) }
|
|
573
|
+
},
|
|
574
|
+
|
|
575
|
+
/** Active counts per category, for status lines and tests. */
|
|
576
|
+
async stats({ maxAgeDays = USER_MEMORY_DEFAULTS.maxAgeDays } = {}) {
|
|
577
|
+
const counts = {}
|
|
578
|
+
for (const category of USER_MEMORY_CATEGORIES) {
|
|
579
|
+
const { records } = await readRecords(category)
|
|
580
|
+
counts[category] = effectiveUserRecords(records, { now: now(), maxAgeDays }).length
|
|
581
|
+
}
|
|
582
|
+
return { ...counts, total: counts.fact + counts.preference + counts.decision, path: dir }
|
|
583
|
+
},
|
|
584
|
+
|
|
585
|
+
defaults: USER_MEMORY_DEFAULTS,
|
|
586
|
+
}
|
|
587
|
+
}
|
package/lib/watch.js
CHANGED
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
*
|
|
18
18
|
* @module dsh-superagent/watch
|
|
19
19
|
*/
|
|
20
|
-
import {
|
|
20
|
+
import { addCurrencyMicros, currencyTotals, formatCurrencyTally } from './cost.js'
|
|
21
21
|
import { ACTIVE_STATUSES, createStore, resolveStateRoot } from './store.js'
|
|
22
22
|
|
|
23
23
|
/** Refresh cadence. A run file is small, so one read per second is cheap. */
|
|
@@ -369,10 +369,9 @@ export function orderRuns(runs) {
|
|
|
369
369
|
export function totalsOf(runs) {
|
|
370
370
|
let calls = 0
|
|
371
371
|
let tokens = 0
|
|
372
|
-
// Integer micro-units keep a long-running panel's total exactly additive
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
const currencies = new Set()
|
|
372
|
+
// Integer micro-units keep a long-running panel's total exactly additive —
|
|
373
|
+
// within one currency. Different currencies are tallied separately.
|
|
374
|
+
const byCurrency = {}
|
|
376
375
|
let unpriced = false
|
|
377
376
|
let active = 0
|
|
378
377
|
let failed = 0
|
|
@@ -380,12 +379,16 @@ export function totalsOf(runs) {
|
|
|
380
379
|
calls += callsOf(run)
|
|
381
380
|
tokens += run?.usage?.totalTokens ?? 0
|
|
382
381
|
if (run?.cost) {
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
382
|
+
if (run.cost.byCurrency && typeof run.cost.byCurrency === 'object'
|
|
383
|
+
&& Object.keys(run.cost.byCurrency).length > 0) {
|
|
384
|
+
for (const [code, micros] of Object.entries(run.cost.byCurrency)) {
|
|
385
|
+
addCurrencyMicros(byCurrency, code, Number(micros))
|
|
386
|
+
}
|
|
387
|
+
} else if (Number.isFinite(run.cost.totalMicros) || Number.isFinite(run.cost.total)) {
|
|
388
|
+
addCurrencyMicros(byCurrency, run.cost.currency, Number.isFinite(run.cost.totalMicros)
|
|
389
|
+
? run.cost.totalMicros
|
|
390
|
+
: Math.round(run.cost.total * 1e6))
|
|
391
|
+
}
|
|
389
392
|
if (run.cost.complete === false) unpriced = true
|
|
390
393
|
} else if ((run?.usage?.modelCalls ?? 0) > 0) {
|
|
391
394
|
unpriced = true
|
|
@@ -393,13 +396,15 @@ export function totalsOf(runs) {
|
|
|
393
396
|
if (ACTIVE_STATUSES.has(run?.status)) active += 1
|
|
394
397
|
if (run?.status === 'failed') failed += 1
|
|
395
398
|
}
|
|
399
|
+
const totals = currencyTotals(byCurrency)
|
|
396
400
|
return {
|
|
397
401
|
calls,
|
|
398
402
|
tokens,
|
|
399
|
-
cost:
|
|
400
|
-
costMicros,
|
|
401
|
-
|
|
402
|
-
|
|
403
|
+
cost: totals.total,
|
|
404
|
+
costMicros: totals.totalMicros,
|
|
405
|
+
byCurrency,
|
|
406
|
+
currency: totals.currency,
|
|
407
|
+
mixedCurrency: totals.mixedCurrency,
|
|
403
408
|
unpriced,
|
|
404
409
|
active,
|
|
405
410
|
failed,
|
|
@@ -638,9 +643,12 @@ export function renderFrame(runs, options = {}) {
|
|
|
638
643
|
|
|
639
644
|
// An unpriced panel says so. Printing 0.0000 there would look like "this run
|
|
640
645
|
// was free", when the truth is "no price is configured for its model".
|
|
641
|
-
const
|
|
646
|
+
const hasSpend = Object.values(totals.byCurrency).some(value => value > 0)
|
|
647
|
+
const shown = Object.keys(totals.byCurrency).length > 0
|
|
648
|
+
? formatCurrencyTally(totals.byCurrency)
|
|
649
|
+
: formatCurrencyTally({ USD: 0 })
|
|
642
650
|
const money = totals.unpriced
|
|
643
|
-
? (
|
|
651
|
+
? (hasSpend ? `${shown}+` : 'unpriced')
|
|
644
652
|
: shown
|
|
645
653
|
const summary = [
|
|
646
654
|
`calls ${totals.calls}`,
|