@ilaomianyang/dsh-memory-graph 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +259 -0
- package/LICENSE +21 -0
- package/README.en.md +466 -0
- package/README.md +459 -0
- package/client.js +1953 -0
- package/cordis.patch.yml +13 -0
- package/docs/auto-settings.png +0 -0
- package/docs/distill-button.png +0 -0
- package/docs/graph-dark.png +0 -0
- package/docs/graph-light.png +0 -0
- package/docs/session-tab-cards.png +0 -0
- package/docs/session-tab-open.png +0 -0
- package/docs/session-tab.png +0 -0
- package/docs/vault-panel.png +0 -0
- package/eval/REPORT.md +437 -0
- package/eval/questions.json +277 -0
- package/eval/run.mjs +460 -0
- package/eval/seed.mjs +489 -0
- package/icon.svg +26 -0
- package/index.js +243 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +77 -0
- package/src/auto.js +194 -0
- package/src/capture.js +237 -0
- package/src/config.js +280 -0
- package/src/distillapi.js +30 -0
- package/src/distillrun.js +106 -0
- package/src/extract.js +660 -0
- package/src/graph.js +1582 -0
- package/src/graphdata.js +475 -0
- package/src/http.js +32 -0
- package/src/pageapi.js +40 -0
- package/src/privacy.js +78 -0
- package/src/schema.js +323 -0
- package/src/search.js +317 -0
- package/src/tools.js +534 -0
- package/src/vault.js +888 -0
- package/test/auto.mjs +295 -0
- package/test/capture.mjs +264 -0
- package/test/client.mjs +1035 -0
- package/test/distillapi.mjs +212 -0
- package/test/extract.mjs +660 -0
- package/test/graphdata.mjs +295 -0
- package/test/live-supersede.mjs +78 -0
- package/test/live.mjs +156 -0
- package/test/page.mjs +116 -0
- package/test/schema.mjs +256 -0
- package/test/smoke.mjs +428 -0
- package/test/tools.mjs +238 -0
package/src/capture.js
ADDED
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Capture: turn the session log into raw episodes.
|
|
3
|
+
*
|
|
4
|
+
* Deliberately model-free. Every turn is appended as it closes, so the "never
|
|
5
|
+
* loses anything" guarantee holds even while the extractor is unconfigured,
|
|
6
|
+
* offline, or out of quota. Distillation happens later and separately.
|
|
7
|
+
*/
|
|
8
|
+
import { nowIso } from './vault.js'
|
|
9
|
+
|
|
10
|
+
const FLUSH_AFTER_MS = 90_000
|
|
11
|
+
const MAX_ASSISTANT_PARTS = 6
|
|
12
|
+
|
|
13
|
+
function collectText(node, depth = 0, seen = new Set(), out = []) {
|
|
14
|
+
if (node === null || node === undefined || depth > 4) return out
|
|
15
|
+
if (typeof node === 'string') return out
|
|
16
|
+
if (Array.isArray(node)) {
|
|
17
|
+
for (const item of node) collectText(item, depth + 1, seen, out)
|
|
18
|
+
return out
|
|
19
|
+
}
|
|
20
|
+
if (typeof node !== 'object') return out
|
|
21
|
+
if (seen.has(node)) return out
|
|
22
|
+
seen.add(node)
|
|
23
|
+
|
|
24
|
+
if (node.type === 'text' && typeof node.text === 'string') {
|
|
25
|
+
out.push(node.text)
|
|
26
|
+
return out
|
|
27
|
+
}
|
|
28
|
+
for (const key of ['content', 'message', 'text', 'parts', 'blocks']) {
|
|
29
|
+
if (key in node) collectText(node[key], depth + 1, seen, out)
|
|
30
|
+
}
|
|
31
|
+
return out
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function eventText(event) {
|
|
35
|
+
const parts = collectText(event?.data)
|
|
36
|
+
return parts.join('\n').replace(/\n{3,}/g, '\n\n').trim()
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function projectOf(session, fallback) {
|
|
40
|
+
const cwd = session?.header?.cwd ?? session?.meta?.cwd ?? session?.cwd
|
|
41
|
+
if (typeof cwd === 'string' && cwd.trim() !== '') {
|
|
42
|
+
// Both separators, whatever platform this is running on: the path comes from
|
|
43
|
+
// the session header, which may well have been written on the other OS.
|
|
44
|
+
const parts = cwd.replace(/[\\/]+$/, '').split(/[\\/]/)
|
|
45
|
+
const base = parts[parts.length - 1] ?? ''
|
|
46
|
+
if (base !== '') return base
|
|
47
|
+
}
|
|
48
|
+
return fallback
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function truncate(text, max) {
|
|
52
|
+
const value = String(text ?? '').trim()
|
|
53
|
+
if (value.length <= max) return value
|
|
54
|
+
return `${value.slice(0, max)}…`
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export function registerCapture(ctx, { cfg, vault, redactor }) {
|
|
58
|
+
const buffers = new Map()
|
|
59
|
+
const disposers = []
|
|
60
|
+
const timers = new Map()
|
|
61
|
+
|
|
62
|
+
// Both the switch and the skip list are editable while the plugin runs (they
|
|
63
|
+
// are `.volatile()` in the schema), so neither is baked in here: the switch is
|
|
64
|
+
// read per event, and the compiled rules are rebuilt only when the list's text
|
|
65
|
+
// actually changes — a `new RegExp` on every token would be pure waste.
|
|
66
|
+
let skipKey
|
|
67
|
+
let skipRules = []
|
|
68
|
+
|
|
69
|
+
function compiledSkipRules() {
|
|
70
|
+
const list = cfg.capture.skipPatterns ?? []
|
|
71
|
+
const key = list.join('\u0000')
|
|
72
|
+
if (key !== skipKey) {
|
|
73
|
+
skipKey = key
|
|
74
|
+
skipRules = list.flatMap((pattern) => {
|
|
75
|
+
try {
|
|
76
|
+
return [new RegExp(pattern, 'i')]
|
|
77
|
+
} catch {
|
|
78
|
+
return []
|
|
79
|
+
}
|
|
80
|
+
})
|
|
81
|
+
}
|
|
82
|
+
return skipRules
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** True when the "user" side is really a harness injection, not a person typing. */
|
|
86
|
+
const injected = (text) => text !== '' && compiledSkipRules().some((re) => re.test(text))
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Delimited harness blocks come off before the patterns are consulted.
|
|
90
|
+
*
|
|
91
|
+
* They are the one kind of injection whose extent is knowable, so removing them
|
|
92
|
+
* keeps the person's own words when a real message happens to carry one — the
|
|
93
|
+
* alternative (dropping the whole message) loses the part that was worth
|
|
94
|
+
* keeping. Undelimited blocks (a runtime snapshot) are handled by the patterns.
|
|
95
|
+
*/
|
|
96
|
+
function stripInjectedBlocks(text) {
|
|
97
|
+
return String(text ?? '')
|
|
98
|
+
.replace(/<system-reminder>[\s\S]*?<\/system-reminder>/gi, '')
|
|
99
|
+
.replace(/[ \t]+\n/g, '\n')
|
|
100
|
+
.replace(/\n{3,}/g, '\n\n')
|
|
101
|
+
.trim()
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
const idOf = (session) => String(session?.id ?? session?.header?.id ?? 'unknown')
|
|
105
|
+
|
|
106
|
+
function bufferOf(session) {
|
|
107
|
+
const id = idOf(session)
|
|
108
|
+
let buffer = buffers.get(id)
|
|
109
|
+
if (!buffer) {
|
|
110
|
+
buffer = { project: projectOf(session, cfg.defaultProject), user: '', assistant: [], tools: 0, turn: null }
|
|
111
|
+
buffers.set(id, buffer)
|
|
112
|
+
}
|
|
113
|
+
return { id, buffer }
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function drop(id) {
|
|
117
|
+
buffers.delete(id)
|
|
118
|
+
const timer = timers.get(id)
|
|
119
|
+
if (timer) {
|
|
120
|
+
clearTimeout(timer)
|
|
121
|
+
timers.delete(id)
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
async function flush(id) {
|
|
126
|
+
const buffer = buffers.get(id)
|
|
127
|
+
if (!buffer) return
|
|
128
|
+
const toolCount = buffer.tools
|
|
129
|
+
const user = truncate(buffer.user, cfg.capture.userChars)
|
|
130
|
+
const assistant = truncate(buffer.assistant.join('\n'), cfg.capture.assistantChars)
|
|
131
|
+
|
|
132
|
+
// Every episode is anchored to something the user actually said. A turn with
|
|
133
|
+
// no user side is context noise, and one too short to mean anything — such as
|
|
134
|
+
// "ok" — would only fill the vault with lines the extractor cannot use.
|
|
135
|
+
if (user === '' || user.length + assistant.length < cfg.capture.minChars) {
|
|
136
|
+
drop(id)
|
|
137
|
+
return
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
drop(id)
|
|
141
|
+
|
|
142
|
+
const redactedUser = redactor.redact(user).text
|
|
143
|
+
const redactedAssistant = redactor.redact(assistant).text
|
|
144
|
+
await vault.appendEpisode({
|
|
145
|
+
t: nowIso(),
|
|
146
|
+
project: buffer.project,
|
|
147
|
+
session: id,
|
|
148
|
+
turn: buffer.turn,
|
|
149
|
+
user: redactedUser,
|
|
150
|
+
assistant: redactedAssistant,
|
|
151
|
+
...(cfg.capture.includeToolOutput && toolCount > 0 ? { tools: toolCount } : {}),
|
|
152
|
+
})
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
function schedule(id) {
|
|
156
|
+
const existing = timers.get(id)
|
|
157
|
+
if (existing) clearTimeout(existing)
|
|
158
|
+
const timer = setTimeout(() => {
|
|
159
|
+
flush(id).catch(() => {})
|
|
160
|
+
}, FLUSH_AFTER_MS)
|
|
161
|
+
timer.unref?.()
|
|
162
|
+
timers.set(id, timer)
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
disposers.push(
|
|
166
|
+
ctx.on('session/event', (session, event) => {
|
|
167
|
+
// The switch is read here, not at registration: turning capture off in
|
|
168
|
+
// 设置 must stop writes immediately without unregistering the listener.
|
|
169
|
+
if (!cfg.capture.enabled) return
|
|
170
|
+
|
|
171
|
+
const type = event?.type
|
|
172
|
+
|
|
173
|
+
// A turn is the natural write unit: flush it as soon as it closes, so an
|
|
174
|
+
// episode exists even if the next turn never comes.
|
|
175
|
+
if (type === 'turn/end') {
|
|
176
|
+
const id = idOf(session)
|
|
177
|
+
const buffer = buffers.get(id)
|
|
178
|
+
if (buffer) {
|
|
179
|
+
if (typeof event?.data?.turn === 'number') buffer.turn = event.data.turn
|
|
180
|
+
flush(id).catch(() => {})
|
|
181
|
+
}
|
|
182
|
+
return
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
if (type !== 'user/message' && type !== 'assistant/message' && type !== 'tool/result') return
|
|
186
|
+
|
|
187
|
+
if (type === 'user/message') {
|
|
188
|
+
// The harness delivers runtime snapshots, skill catalogs, sampled time,
|
|
189
|
+
// approval notices and compaction checkpoints through the same channel as
|
|
190
|
+
// the person typing — and not always with a telling `source.kind`: a real
|
|
191
|
+
// runtime snapshot has arrived here tagged `user` (it was captured as if
|
|
192
|
+
// the person had typed it). So provenance can only *add* a skip, never
|
|
193
|
+
// veto the patterns; both are consulted, and delimited blocks come off
|
|
194
|
+
// first so a message that merely carried a reminder keeps its own words.
|
|
195
|
+
const kind = event?.data?.source?.kind
|
|
196
|
+
const text = stripInjectedBlocks(eventText(event))
|
|
197
|
+
const synthetic = (typeof kind === 'string' && kind !== 'user') || text === '' || injected(text)
|
|
198
|
+
if (synthetic) return
|
|
199
|
+
|
|
200
|
+
const id = idOf(session)
|
|
201
|
+
const open = buffers.get(id)
|
|
202
|
+
if (open && (open.user !== '' || open.assistant.length > 0)) {
|
|
203
|
+
// A turn that never closed (interrupted, or an older harness): keep it.
|
|
204
|
+
flush(id).catch(() => {})
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
const { buffer } = bufferOf(session)
|
|
208
|
+
buffer.user = text
|
|
209
|
+
if (typeof event?.data?.turn === 'number') buffer.turn = event.data.turn
|
|
210
|
+
schedule(id)
|
|
211
|
+
return
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
const { id, buffer } = bufferOf(session)
|
|
215
|
+
|
|
216
|
+
if (type === 'assistant/message') {
|
|
217
|
+
const text = eventText(event)
|
|
218
|
+
if (text !== '' && buffer.assistant.length < MAX_ASSISTANT_PARTS) buffer.assistant.push(text)
|
|
219
|
+
if (typeof event?.data?.turn === 'number') buffer.turn = event.data.turn
|
|
220
|
+
schedule(id)
|
|
221
|
+
return
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
buffer.tools += 1
|
|
225
|
+
schedule(id)
|
|
226
|
+
}),
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
return [
|
|
230
|
+
...disposers,
|
|
231
|
+
() => {
|
|
232
|
+
for (const timer of timers.values()) clearTimeout(timer)
|
|
233
|
+
timers.clear()
|
|
234
|
+
for (const id of [...buffers.keys()]) flush(id).catch(() => {})
|
|
235
|
+
},
|
|
236
|
+
]
|
|
237
|
+
}
|
package/src/config.js
ADDED
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Config defaults and normalisation.
|
|
3
|
+
*
|
|
4
|
+
* The Settings form is drawn from the schema in `src/schema.js`, but that form
|
|
5
|
+
* is a convenience, not the contract: every field is normalised here again, so a
|
|
6
|
+
* config written by hand in `cordis.patch.yml`, or one carrying a stale key from
|
|
7
|
+
* an older version, still resolves to a usable configuration.
|
|
8
|
+
*
|
|
9
|
+
* This is also the seam between the two config worlds. A field declared
|
|
10
|
+
* `.volatile()` arrives here as a live reference rather than a value; this module
|
|
11
|
+
* unwraps it, so everything downstream sees plain values, and re-resolving on
|
|
12
|
+
* `loader/volatile-update` turns a Settings write into a new snapshot.
|
|
13
|
+
*/
|
|
14
|
+
import os from 'node:os'
|
|
15
|
+
import path from 'node:path'
|
|
16
|
+
|
|
17
|
+
/** Credential shapes that must never reach a vault file. User rules are added to these. */
|
|
18
|
+
export const DEFAULT_DENY_PATTERNS = [
|
|
19
|
+
'sk-ant-[A-Za-z0-9_-]{20,}',
|
|
20
|
+
'sk-[A-Za-z0-9_-]{24,}',
|
|
21
|
+
'ghp_[A-Za-z0-9]{20,}',
|
|
22
|
+
'github_pat_[A-Za-z0-9_]{20,}',
|
|
23
|
+
'glpat-[A-Za-z0-9_-]{16,}',
|
|
24
|
+
'xox[baprs]-[A-Za-z0-9-]{10,}',
|
|
25
|
+
'AKIA[0-9A-Z]{16}',
|
|
26
|
+
'AIza[0-9A-Za-z_-]{30,}',
|
|
27
|
+
'-----BEGIN [A-Z ]*PRIVATE KEY-----[\\s\\S]*?-----END [A-Z ]*PRIVATE KEY-----',
|
|
28
|
+
'(?i)bearer\\s+[A-Za-z0-9._~+/-]{20,}={0,2}',
|
|
29
|
+
'(?i)(password|passwd|secret|api[_-]?key|access[_-]?token)\\s*[:=]\\s*["\']?\\S{6,}',
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Text the harness injects into the conversation as a user message: compaction
|
|
34
|
+
* checkpoints, runtime-context snapshots, skill-catalog reminders. They are not
|
|
35
|
+
* user intent, and distilling them would manufacture memories out of nothing.
|
|
36
|
+
*/
|
|
37
|
+
export const DEFAULT_SKIP_PATTERNS = [
|
|
38
|
+
'^This is an automatically generated checkpoint',
|
|
39
|
+
'^Current runtime context\\.',
|
|
40
|
+
'<system-reminder>',
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
const DEFAULTS = {
|
|
44
|
+
enabled: true,
|
|
45
|
+
vaultDir: '',
|
|
46
|
+
/** Project tag used when a session has no usable cwd. */
|
|
47
|
+
defaultProject: 'shared',
|
|
48
|
+
/** Link new cards into project-level entity hubs so cross-project recall works. */
|
|
49
|
+
crossProject: true,
|
|
50
|
+
capture: {
|
|
51
|
+
enabled: true,
|
|
52
|
+
userChars: 1500,
|
|
53
|
+
assistantChars: 1500,
|
|
54
|
+
includeToolOutput: false,
|
|
55
|
+
minChars: 12,
|
|
56
|
+
skipPatterns: [...DEFAULT_SKIP_PATTERNS],
|
|
57
|
+
},
|
|
58
|
+
privacy: {
|
|
59
|
+
useDefaultPatterns: true,
|
|
60
|
+
denyPatterns: [],
|
|
61
|
+
redactEmails: true,
|
|
62
|
+
redactPaths: true,
|
|
63
|
+
},
|
|
64
|
+
extractor: {
|
|
65
|
+
enabled: false,
|
|
66
|
+
/**
|
|
67
|
+
* `harness` borrows a route the Settings → Models page already configured
|
|
68
|
+
* (provider, key, base URL, thinking level), so the extractor needs no
|
|
69
|
+
* credentials of its own. `http` posts to an OpenAI-compatible endpoint with
|
|
70
|
+
* the key below, for machines where no model is configured in Settings.
|
|
71
|
+
*/
|
|
72
|
+
transport: 'harness',
|
|
73
|
+
/** harness: follow the model currently selected in Settings instead of provider/model. */
|
|
74
|
+
useCurrentModel: true,
|
|
75
|
+
/** harness: provider route id, e.g. `deepseek`, `wb`, `agy-local`. */
|
|
76
|
+
provider: '',
|
|
77
|
+
/** harness: thinking level for this call, when the route offers one. '' keeps the route default. */
|
|
78
|
+
reasoningEffort: '',
|
|
79
|
+
baseUrl: 'https://api.deepseek.com/v1',
|
|
80
|
+
path: '/chat/completions',
|
|
81
|
+
/**
|
|
82
|
+
* Empty by default: in harness mode the model comes from Settings, and an
|
|
83
|
+
* id left here would only be used with `useCurrentModel` off. http mode
|
|
84
|
+
* fills in `deepseek-chat` at normalisation time.
|
|
85
|
+
*/
|
|
86
|
+
model: '',
|
|
87
|
+
apiKeyEnv: 'MEMORY_LLM_API_KEY',
|
|
88
|
+
apiKey: '',
|
|
89
|
+
maxOutputTokens: 3000,
|
|
90
|
+
temperature: 0.1,
|
|
91
|
+
timeoutMs: 90000,
|
|
92
|
+
maxEpisodesPerRun: 40,
|
|
93
|
+
},
|
|
94
|
+
retrieval: {
|
|
95
|
+
limit: 8,
|
|
96
|
+
hops: 1,
|
|
97
|
+
snippetChars: 220,
|
|
98
|
+
timelineLimit: 30,
|
|
99
|
+
},
|
|
100
|
+
/**
|
|
101
|
+
* The automatic distiller: two independent triggers that share one tick and
|
|
102
|
+
* one lock. Both default to off — a background model call is the user's
|
|
103
|
+
* decision, never the plugin's.
|
|
104
|
+
*/
|
|
105
|
+
auto: {
|
|
106
|
+
enabled: false,
|
|
107
|
+
/** Undistilled-episode threshold; 0 turns the threshold rule off. */
|
|
108
|
+
minEpisodes: 0,
|
|
109
|
+
/** Minutes between runs; 0 turns the schedule rule off. */
|
|
110
|
+
everyMinutes: 0,
|
|
111
|
+
/** Never two automatic runs closer than this, whichever rule fired. */
|
|
112
|
+
minGapMinutes: 30,
|
|
113
|
+
/** `23:00-08:00` style window in which nothing runs. Empty means always. */
|
|
114
|
+
quietHours: '',
|
|
115
|
+
/** Project tag for automatic cards; empty uses the newest episode's project. */
|
|
116
|
+
project: '',
|
|
117
|
+
},
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function isPlainObject(value) {
|
|
121
|
+
return value !== null && typeof value === 'object' && !Array.isArray(value)
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
const VOLATILE_WRITE = Symbol.for('cosmokit.volatile.write')
|
|
125
|
+
|
|
126
|
+
/** True for the frozen reference a `.volatile()` leaf validates to, not for a value. */
|
|
127
|
+
function isVolatileRef(value) {
|
|
128
|
+
return value !== null && typeof value === 'object' && VOLATILE_WRITE in value
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* Replace every volatile reference with the value it holds right now.
|
|
133
|
+
*
|
|
134
|
+
* A field marked `.volatile()` in `src/schema.js` does not validate to its value:
|
|
135
|
+
* it validates to a frozen `{ get() }` handle, and the settings page writes
|
|
136
|
+
* through that handle. Everything below this function — the defaults merge, the
|
|
137
|
+
* normalisation, and the caller's `cfg` — deals in plain values, so the refs are
|
|
138
|
+
* unwrapped at the one place config enters the plugin. `resolveConfig` runs again
|
|
139
|
+
* on every `loader/volatile-update`, which is how a Settings write becomes a
|
|
140
|
+
* fresh snapshot.
|
|
141
|
+
*
|
|
142
|
+
* This is also what makes `{ ...DEFAULTS, ...config }` safe: a ref is a plain
|
|
143
|
+
* object whose only own key is `get`, so merging without unwrapping would keep
|
|
144
|
+
* the defaults and drop the user's value without any error.
|
|
145
|
+
*/
|
|
146
|
+
export function unwrapConfig(value) {
|
|
147
|
+
if (isVolatileRef(value)) return unwrapConfig(value.get())
|
|
148
|
+
if (Array.isArray(value)) return value.map(unwrapConfig)
|
|
149
|
+
if (isPlainObject(value)) {
|
|
150
|
+
const out = {}
|
|
151
|
+
for (const [key, child] of Object.entries(value)) out[key] = unwrapConfig(child)
|
|
152
|
+
return out
|
|
153
|
+
}
|
|
154
|
+
return value
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function merge(base, override) {
|
|
158
|
+
if (!isPlainObject(override)) return { ...base }
|
|
159
|
+
const out = { ...base }
|
|
160
|
+
for (const [key, value] of Object.entries(override)) {
|
|
161
|
+
if (value === undefined) continue
|
|
162
|
+
out[key] = isPlainObject(value) && isPlainObject(base[key]) ? merge(base[key], value) : value
|
|
163
|
+
}
|
|
164
|
+
return out
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
function expandHome(input) {
|
|
168
|
+
if (typeof input !== 'string' || input.trim() === '') return ''
|
|
169
|
+
const trimmed = input.trim()
|
|
170
|
+
if (trimmed === '~') return os.homedir()
|
|
171
|
+
if (trimmed.startsWith('~/') || trimmed.startsWith('~\\')) {
|
|
172
|
+
return path.join(os.homedir(), trimmed.slice(2))
|
|
173
|
+
}
|
|
174
|
+
return trimmed
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
function int(value, fallback, min, max) {
|
|
178
|
+
const n = Number(value)
|
|
179
|
+
if (!Number.isFinite(n)) return fallback
|
|
180
|
+
return Math.min(max, Math.max(min, Math.round(n)))
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
export function resolveConfig(raw) {
|
|
184
|
+
const cfg = merge(DEFAULTS, unwrapConfig(raw ?? {}))
|
|
185
|
+
|
|
186
|
+
const dir = expandHome(cfg.vaultDir)
|
|
187
|
+
cfg.vaultDir = dir === '' ? path.join(os.homedir(), 'dsh-memory') : path.resolve(dir)
|
|
188
|
+
|
|
189
|
+
cfg.capture.userChars = int(cfg.capture.userChars, 1500, 200, 20000)
|
|
190
|
+
cfg.capture.assistantChars = int(cfg.capture.assistantChars, 1500, 200, 20000)
|
|
191
|
+
cfg.capture.minChars = int(cfg.capture.minChars, 12, 0, 500)
|
|
192
|
+
|
|
193
|
+
cfg.retrieval.limit = int(cfg.retrieval.limit, 8, 1, 40)
|
|
194
|
+
cfg.retrieval.hops = int(cfg.retrieval.hops, 1, 0, 2)
|
|
195
|
+
cfg.retrieval.snippetChars = int(cfg.retrieval.snippetChars, 220, 60, 2000)
|
|
196
|
+
cfg.retrieval.timelineLimit = int(cfg.retrieval.timelineLimit, 30, 1, 200)
|
|
197
|
+
|
|
198
|
+
cfg.extractor.maxOutputTokens = int(cfg.extractor.maxOutputTokens, 3000, 256, 32000)
|
|
199
|
+
cfg.extractor.timeoutMs = int(cfg.extractor.timeoutMs, 90000, 5000, 600000)
|
|
200
|
+
cfg.extractor.maxEpisodesPerRun = int(cfg.extractor.maxEpisodesPerRun, 40, 1, 400)
|
|
201
|
+
|
|
202
|
+
cfg.privacy.denyPatterns = Array.isArray(cfg.privacy.denyPatterns)
|
|
203
|
+
? cfg.privacy.denyPatterns.filter((p) => typeof p === 'string' && p.trim() !== '')
|
|
204
|
+
: []
|
|
205
|
+
|
|
206
|
+
// A user list replaces the defaults; drop anything that is not a usable regex
|
|
207
|
+
// rather than letting one typo disable capture for the whole session.
|
|
208
|
+
cfg.capture.skipPatterns = Array.isArray(cfg.capture.skipPatterns)
|
|
209
|
+
? cfg.capture.skipPatterns.filter((p) => {
|
|
210
|
+
if (typeof p !== 'string' || p.trim() === '') return false
|
|
211
|
+
try {
|
|
212
|
+
new RegExp(p, 'i')
|
|
213
|
+
return true
|
|
214
|
+
} catch {
|
|
215
|
+
return false
|
|
216
|
+
}
|
|
217
|
+
})
|
|
218
|
+
: []
|
|
219
|
+
|
|
220
|
+
// Anything that is not the explicit escape hatch means the borrowed route:
|
|
221
|
+
// a stale or misspelled transport must not silently become an HTTP call that
|
|
222
|
+
// fails later for a missing key.
|
|
223
|
+
cfg.extractor.transport = cfg.extractor.transport === 'http' ? 'http' : 'harness'
|
|
224
|
+
cfg.extractor.useCurrentModel = cfg.extractor.useCurrentModel !== false
|
|
225
|
+
cfg.extractor.provider = String(cfg.extractor.provider || '').trim()
|
|
226
|
+
cfg.extractor.reasoningEffort = String(cfg.extractor.reasoningEffort || '').trim()
|
|
227
|
+
cfg.extractor.baseUrl = String(cfg.extractor.baseUrl || '').replace(/\/+$/, '')
|
|
228
|
+
cfg.extractor.path = String(cfg.extractor.path || '/chat/completions')
|
|
229
|
+
// In http mode `model` is an OpenAI model name with a sensible default; in
|
|
230
|
+
// harness mode it is an id that must be taken literally, so a cleared field
|
|
231
|
+
// stays empty and `resolveRoute` can say what is missing.
|
|
232
|
+
cfg.extractor.model =
|
|
233
|
+
cfg.extractor.transport === 'http' ? String(cfg.extractor.model || 'deepseek-chat') : String(cfg.extractor.model || '')
|
|
234
|
+
cfg.extractor.apiKeyEnv = String(cfg.extractor.apiKeyEnv || 'MEMORY_LLM_API_KEY')
|
|
235
|
+
|
|
236
|
+
cfg.auto.enabled = cfg.auto.enabled === true
|
|
237
|
+
cfg.auto.minEpisodes = int(cfg.auto.minEpisodes, 0, 0, 400)
|
|
238
|
+
cfg.auto.everyMinutes = int(cfg.auto.everyMinutes, 0, 0, 10080)
|
|
239
|
+
cfg.auto.minGapMinutes = int(cfg.auto.minGapMinutes, 30, 0, 1440)
|
|
240
|
+
cfg.auto.project = String(cfg.auto.project || '').trim()
|
|
241
|
+
// A malformed window must not disable the rule or, worse, silence it forever:
|
|
242
|
+
// parse what is parseable and fall back to "no quiet hours".
|
|
243
|
+
cfg.auto.quietHours = parseQuietHours(cfg.auto.quietHours) === null ? '' : String(cfg.auto.quietHours).trim()
|
|
244
|
+
|
|
245
|
+
return cfg
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* `23:00-08:00` → minute bounds for comparison; `null` when the text is unusable.
|
|
250
|
+
*
|
|
251
|
+
* Exported because the auto distiller and the Settings hint must agree on what
|
|
252
|
+
* counts as a valid window, and a window crossing midnight (start > end) is the
|
|
253
|
+
* common case, not an edge case.
|
|
254
|
+
*/
|
|
255
|
+
export function parseQuietHours(text) {
|
|
256
|
+
const raw = String(text ?? '').trim()
|
|
257
|
+
if (raw === '') return null
|
|
258
|
+
const match = /^(\d{1,2}):(\d{2})\s*-\s*(\d{1,2}):(\d{2})$/u.exec(raw)
|
|
259
|
+
if (!match) return null
|
|
260
|
+
const [fromH, fromM, toH, toM] = match.slice(1).map(Number)
|
|
261
|
+
if (fromH > 23 || toH > 23 || fromM > 59 || toM > 59) return null
|
|
262
|
+
return { from: fromH * 60 + fromM, to: toH * 60 + toM }
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
export function paths(vaultDir) {
|
|
266
|
+
return {
|
|
267
|
+
root: vaultDir,
|
|
268
|
+
gitignore: path.join(vaultDir, '.gitignore'),
|
|
269
|
+
vaultJson: path.join(vaultDir, 'vault.json'),
|
|
270
|
+
readme: path.join(vaultDir, 'README.md'),
|
|
271
|
+
graph: path.join(vaultDir, 'graph.html'),
|
|
272
|
+
projects: path.join(vaultDir, 'projects.json'),
|
|
273
|
+
episodes: path.join(vaultDir, 'episodes'),
|
|
274
|
+
cards: path.join(vaultDir, 'cards'),
|
|
275
|
+
entities: path.join(vaultDir, 'entities'),
|
|
276
|
+
audit: path.join(vaultDir, 'audit'),
|
|
277
|
+
distillState: path.join(vaultDir, '.distill-state.json'),
|
|
278
|
+
autoState: path.join(vaultDir, '.auto-state.json'),
|
|
279
|
+
}
|
|
280
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The "distill now" door: one exact Fetch route that runs the same distillation
|
|
3
|
+
* `memory_distill` runs, so the panel's button and the model's tool cannot drift
|
|
4
|
+
* apart. The run itself — and the one-run-at-a-time lock — live in
|
|
5
|
+
* `src/distillrun.js`, because the automatic distiller fires through the same
|
|
6
|
+
* door.
|
|
7
|
+
*
|
|
8
|
+
* This module only translates: a report becomes a 200 (including the reports that
|
|
9
|
+
* say `ok: false`, because "the extractor is off" is an answer the panel prints,
|
|
10
|
+
* not a failure it has to explain), and an unexpected throw becomes a 500.
|
|
11
|
+
*/
|
|
12
|
+
import { json } from './http.js'
|
|
13
|
+
|
|
14
|
+
export const DISTILL_PATH = '/api/memory-graph.distill'
|
|
15
|
+
|
|
16
|
+
/** 200 for an answer (including a refusal), 500 only for a crash. */
|
|
17
|
+
function respond(report) {
|
|
18
|
+
const { failed, ...rest } = report
|
|
19
|
+
return json(rest, failed === true ? 500 : 200)
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function createDistillHandler({ getRunner }) {
|
|
23
|
+
return async function handleDistill() {
|
|
24
|
+
const runner = getRunner()
|
|
25
|
+
if (typeof runner?.run !== 'function') {
|
|
26
|
+
return json({ ok: false, reason: 'unavailable', message: 'the distiller is not available' }, 500)
|
|
27
|
+
}
|
|
28
|
+
return respond(await runner.run())
|
|
29
|
+
}
|
|
30
|
+
}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One distillation, shared by the three ways to ask for it: the tool, the
|
|
3
|
+
* panel's button, and the automatic distiller.
|
|
4
|
+
*
|
|
5
|
+
* The lock lives here and nowhere else. Two of those three can fire without
|
|
6
|
+
* anyone watching — a scheduled tick and a pressed button can land in the same
|
|
7
|
+
* second — and a distillation spends real model budget while advancing one
|
|
8
|
+
* cursor in the vault. So the second caller is told the first is running; it is
|
|
9
|
+
* never queued and never runs concurrently.
|
|
10
|
+
*/
|
|
11
|
+
import { distill as runDistill } from './extract.js'
|
|
12
|
+
|
|
13
|
+
/** How many written cards a caller gets to list before it just shows a count. */
|
|
14
|
+
const CARD_LIMIT = 40
|
|
15
|
+
|
|
16
|
+
export function createDistillRunner(options) {
|
|
17
|
+
const {
|
|
18
|
+
getVault,
|
|
19
|
+
getConfig,
|
|
20
|
+
getRedactor,
|
|
21
|
+
getServices = () => ({}),
|
|
22
|
+
// Injected so the tests can drive the success/failure paths without a model.
|
|
23
|
+
distill = runDistill,
|
|
24
|
+
// Rendering the vault's own views (graph.html, README) is best effort: the
|
|
25
|
+
// cards are already on disk, and the next write renders them again.
|
|
26
|
+
refreshViews = () => {},
|
|
27
|
+
log = () => {},
|
|
28
|
+
} = options ?? {}
|
|
29
|
+
|
|
30
|
+
let inflight = null
|
|
31
|
+
|
|
32
|
+
/** True while a run is in flight — the auto ticker asks before counting. */
|
|
33
|
+
function busy() {
|
|
34
|
+
return inflight !== null
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Run one distillation. Always resolves with a report, never throws: an
|
|
39
|
+
* extractor that is switched off, a vault with nothing new and a model call
|
|
40
|
+
* that blew up are all answers a caller has to print, not exceptions it has to
|
|
41
|
+
* catch three different ways.
|
|
42
|
+
*/
|
|
43
|
+
function run({ project, dryRun = false, signal } = {}) {
|
|
44
|
+
if (inflight) {
|
|
45
|
+
return Promise.resolve({ ok: false, busy: true, reason: 'A distillation is already running.' })
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
const task = (async () => {
|
|
49
|
+
try {
|
|
50
|
+
const report = await distill({
|
|
51
|
+
cfg: getConfig(),
|
|
52
|
+
vault: getVault(),
|
|
53
|
+
redactor: getRedactor(),
|
|
54
|
+
project,
|
|
55
|
+
dryRun,
|
|
56
|
+
signal,
|
|
57
|
+
...getServices(),
|
|
58
|
+
})
|
|
59
|
+
const cards = (report.cards ?? report.proposed ?? []).slice(0, CARD_LIMIT).map((card) => ({
|
|
60
|
+
id: card.id ?? null,
|
|
61
|
+
kind: String(card.kind ?? 'note'),
|
|
62
|
+
title: String(card.title ?? ''),
|
|
63
|
+
deduped: Boolean(card.deduped),
|
|
64
|
+
}))
|
|
65
|
+
if (report.ok && (report.written ?? 0) > 0) {
|
|
66
|
+
try {
|
|
67
|
+
await refreshViews()
|
|
68
|
+
} catch (error) {
|
|
69
|
+
log('warn', `distilled, but a vault view did not render: ${error?.message ?? error}`)
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return {
|
|
73
|
+
ok: Boolean(report.ok),
|
|
74
|
+
busy: false,
|
|
75
|
+
dryRun: Boolean(report.dryRun ?? dryRun),
|
|
76
|
+
episodes: report.episodes ?? 0,
|
|
77
|
+
written: report.written ?? 0,
|
|
78
|
+
remaining: report.remaining ?? 0,
|
|
79
|
+
cards,
|
|
80
|
+
batches: report.batches ?? null,
|
|
81
|
+
model: report.model ?? null,
|
|
82
|
+
keySource: report.keySource ?? null,
|
|
83
|
+
usage: report.usage ?? null,
|
|
84
|
+
skipped: report.skipped ?? null,
|
|
85
|
+
reason: report.reason ?? null,
|
|
86
|
+
}
|
|
87
|
+
} catch (error) {
|
|
88
|
+
log('warn', `distillation failed: ${error?.message ?? error}`)
|
|
89
|
+
return {
|
|
90
|
+
ok: false,
|
|
91
|
+
busy: false,
|
|
92
|
+
failed: true,
|
|
93
|
+
reason: 'unavailable',
|
|
94
|
+
message: String(error?.message ?? error),
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
})()
|
|
98
|
+
|
|
99
|
+
inflight = task
|
|
100
|
+
return task.finally(() => {
|
|
101
|
+
if (inflight === task) inflight = null
|
|
102
|
+
})
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
return { run, busy }
|
|
106
|
+
}
|