@try-works/dsh-memory 0.1.19 → 0.1.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/LICENSE +21 -21
  2. package/README.md +585 -585
  3. package/STATUS.md +294 -287
  4. package/cordis.patch.yml +69 -69
  5. package/package.json +1 -1
  6. package/scripts/check-package.mjs +76 -76
  7. package/scripts/inspect-log.ts +40 -0
  8. package/scripts/repair-ignorable.ts +130 -0
  9. package/scripts/scan-sessions.ts +43 -0
  10. package/scripts/verify-log.ts +26 -0
  11. package/src/SYSTEM.template.md +311 -311
  12. package/src/capture.js +242 -242
  13. package/src/client/index.js +886 -886
  14. package/src/config.js +57 -57
  15. package/src/consolidate-run.js +132 -132
  16. package/src/consolidate.js +330 -330
  17. package/src/context.js +88 -88
  18. package/src/decisions.js +106 -106
  19. package/src/defaults.js +93 -93
  20. package/src/digest.js +255 -255
  21. package/src/entry.js +179 -179
  22. package/src/eval.js +72 -72
  23. package/src/files.js +60 -60
  24. package/src/fixture.js +56 -56
  25. package/src/forget.js +71 -71
  26. package/src/freshness.js +65 -65
  27. package/src/fts.js +166 -166
  28. package/src/git.js +53 -53
  29. package/src/history.js +226 -226
  30. package/src/host.js +884 -866
  31. package/src/index.js +115 -94
  32. package/src/inject.js +108 -108
  33. package/src/layers.js +178 -178
  34. package/src/ledger.js +19 -19
  35. package/src/map.js +184 -184
  36. package/src/models.js +136 -136
  37. package/src/queue.js +602 -602
  38. package/src/rank.js +31 -31
  39. package/src/recover.js +91 -91
  40. package/src/refresh.js +80 -80
  41. package/src/remember.js +58 -58
  42. package/src/rerank.js +90 -90
  43. package/src/search.js +75 -75
  44. package/src/select.js +57 -57
  45. package/src/session.js +60 -60
  46. package/src/state.js +240 -240
  47. package/src/store.js +128 -128
  48. package/src/summarize.js +195 -195
  49. package/src/system-doc.js +171 -171
  50. package/src/tasks.js +617 -617
  51. package/src/telemetry.js +122 -122
  52. package/src/tools.js +828 -818
  53. package/src/working.js +288 -288
  54. package/src/workspace-key.js +68 -68
  55. package/src/workspace.js +87 -87
package/src/host.js CHANGED
@@ -1,866 +1,884 @@
1
- // DSH host wiring: deep hooks into session lifecycle, agent pre-step, LLM, and
2
- // subprocess. This is the only module that touches DSH services directly.
3
- import { existsSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'
4
- import { join } from 'node:path'
5
- import { BlockAssembler, ReasoningEffortId, createUserMessage } from '@deepseek-ai/dsh-llm'
6
- import { captureSession, pendingDigests, pruneDigests } from './capture.js'
7
- import { runConsolidation } from './consolidate-run.js'
8
- import { runRefresh } from './refresh.js'
9
- import { buildInjectionText } from './inject.js'
10
- import { parseModelRef } from './defaults.js'
11
- import { ingestHistory } from './history.js'
12
- import { rerankHits } from './rerank.js'
13
- import { isSubagentSession } from './session.js'
14
- import { buildModelLadder, createModelMemory, runLadder } from './models.js'
15
- import { due, problems, readQueue, reapStalled, REASONS, STATES, summarize as summarizeQueue, transition } from './queue.js'
16
- import { planRecovery, recoverOne } from './recover.js'
17
- import { logRun, noteActivity, readMetrics, record } from './telemetry.js'
18
- import { GitRunner, SyncScheduler, classifyGitResult, commitMessage } from './git.js'
19
- import { buildMap } from './map.js'
20
- import { backupTasks, defaultSchedulePath, planRestore, readScheduleStore } from './tasks.js'
21
- import { autoAssociateSession } from './session.js'
22
- import { readAssociations } from './workspace.js'
23
-
24
- const PLUGIN = 'dsh-memory'
25
-
26
- /** How often the idle ticker checks whether a cadence is due (the cadences self-throttle). */
27
- const MAP_PUBLISH_MS = 60 * 1000
28
- const TASKS_BACKUP_MS = 30 * 60 * 1000
29
- const IDLE_TICK_MS = 60_000
30
-
31
- function makeGitRunner(ctx) {
32
- const subprocess = ctx.get('subprocess')
33
- if (!subprocess || typeof subprocess.resolveExecutable !== 'function' || typeof subprocess.spawn !== 'function') return null
34
- let executablePromise
35
- const run = async (argv, opts = {}) => {
36
- if (!executablePromise) executablePromise = subprocess.resolveExecutable('git', undefined, undefined).catch(() => null)
37
- const executable = await executablePromise
38
- if (!executable) return { exitCode: 127, stdout: '', stderr: 'git unavailable' }
39
- try {
40
- const handle = subprocess.spawn({
41
- argv: [executable, ...argv],
42
- cwd: opts.cwd,
43
- stdio: { stdin: 'ignore', stdout: { maxBytes: 4 * 1024 * 1024 }, stderr: { maxBytes: 16 * 1024 } },
44
- graceMs: 2000,
45
- env: { GIT_CONFIG_COUNT: '0', GIT_TERMINAL_PROMPT: '0', GIT_OPTIONAL_LOCKS: '0', LC_ALL: 'C' },
46
- })
47
- const outcome = await handle.done
48
- const stdout = handle.collected?.stdout?.readFrom?.(0)?.text ?? ''
49
- const stderr = handle.collected?.stderr?.readFrom?.(0)?.text ?? ''
50
- return { exitCode: outcome?.exitCode ?? null, stdout, stderr }
51
- } catch (error) {
52
- return { exitCode: 1, stdout: '', stderr: String(error) }
53
- }
54
- }
55
- return new GitRunner(run)
56
- }
57
-
58
- function resolveTarget(agent) {
59
- const h = agent?.session?.requestHeader?.()?.config
60
- if (h && h.provider && h.model) return { provider: h.provider, model: h.model }
61
- const o = agent?.options
62
- if (o && o.provider && o.model) return { provider: o.provider, model: o.model }
63
- return undefined
64
- }
65
-
66
- export function wireHost(ctx, store, cfg) {
67
- const keyBySession = new Map()
68
- const scheduler = new SyncScheduler({ commitDebounceMs: cfg.commitDebounceMs, syncIntervalMs: cfg.autoSyncIntervalMs })
69
- const captureThrottle = new Map()
70
- const injectionThrottle = new Map()
71
- let lastConsolidateAttempt = 0
72
- let lastRefreshAttempt = 0
73
- let lastTasksBackupAttempt = 0
74
- let restoreAttempted = false
75
- const lastMapPublish = new Map()
76
- const lastMapSignature = new Map()
77
- let lastHistoryAttempt = 0
78
- let syncInFlight = false
79
-
80
- const git = makeGitRunner(ctx)
81
- const llmService = ctx.get('llm')
82
-
83
- /** Which model last worked, and which recently returned nothing. Learned, not configured. */
84
- const modelMemory = createModelMemory()
85
- let memoryPreferred
86
- let availableCache
87
-
88
- /**
89
- * Every model this deployment can actually route to right now.
90
- *
91
- * Read from the live registry rather than a constant: providers appear and disappear, a
92
- * subscription lapses, and role-model routing changes what is reachable. A short cache keeps
93
- * a burst of memory writes from re-enumerating providers on every call, and a failure to
94
- * enumerate is not fatal - the ladder still has the configured and session routes.
95
- */
96
- async function listAvailableModels({ maxAgeMs = 60000 } = {}) {
97
- const now = Date.now()
98
- if (availableCache && now - availableCache.at < maxAgeMs) return availableCache.models
99
- const models = []
100
- try {
101
- const providers = llmService.listProviders()
102
- for (const provider of providers ?? []) {
103
- try {
104
- const list = await llmService.listModels(provider.id)
105
- for (const model of list ?? []) models.push({ provider: provider.id, model: model.id })
106
- } catch { /* one unroutable provider must not empty the ladder */ }
107
- }
108
- } catch { /* fall through with whatever was collected */ }
109
- availableCache = { at: now, models }
110
- return models
111
- }
112
-
113
-
114
- const runLlm = async (target, prompt, purpose = 'memory-consolidation', maxTokens = 4000) => {
115
- if (!llmService) throw new Error('llm service unavailable')
116
-
117
- /** One attempt against ONE model, with the reasoning-effort recovery. */
118
- const attemptOn = async (model, controls, called) => {
119
- called.push({ provider: model.provider, model: model.model })
120
- record(store.root, 'model-call', { modelCalls: 1, purpose, detail: { provider: model.provider, model: model.model } })
121
- const assembler = new BlockAssembler()
122
- const messages = [{ role: 'user', content: [{ type: 'text', text: prompt }] }]
123
- for await (const chunk of llmService.stream({
124
- provider: model.provider, model: model.model, messages, maxTokens, purpose, ...controls,
125
- })) assembler.push(chunk)
126
- // Text blocks only. A reasoning model streams its thinking as `reasoning` blocks;
127
- // folding those into the reply prepends prose to the answer, which makes any
128
- // "parse the JSON reply" step match the wrong braces and fail. Worse, a model that
129
- // spends its whole budget reasoning returns NO text block at all - which is exactly
130
- // the failure the ladder exists to route around.
131
- return assembler.blocks().filter((b) => b && b.type === 'text').map((b) => b.text || '').join('\n').trim()
132
- }
133
-
134
- /**
135
- * One model, tried at the memory reasoning effort and then with reasoning off.
136
- * Memory work is extraction: it must not run at the session's reasoning budget, and a
137
- * model that rejects the effort outright is retried without the control rather than
138
- * failing every memory write.
139
- */
140
- const attemptModel = async (model) => {
141
- const called = []
142
- const effort = cfg.memoryReasoningEffort
143
- if (effort) {
144
- let text = ''
145
- try {
146
- text = await attemptOn(model, { reasoningEffort: ReasoningEffortId(effort) }, called)
147
- } catch (error) {
148
- const message = String((error && error.message) || error)
149
- if (!/UNSUPPORTED_REASONING_EFFORT|does not support reasoning effort/i.test(message)) {
150
- modelCalls.push(...called)
151
- throw error
152
- }
153
- ctx.logger.debug('dsh-memory: ' + model.model + ' does not support reasoning effort "' + effort + '"; retrying without it')
154
- const plain = await attemptOn(model, {}, called)
155
- modelCalls.push(...called)
156
- return plain
157
- }
158
- if (text !== '') { modelCalls.push(...called); return text }
159
- ctx.logger.debug('dsh-memory: ' + model.model + ' returned no text at effort "' + effort + '"; retrying with reasoning off')
160
- const off = await attemptOn(model, { reasoningEffort: ReasoningEffortId('off') }, called)
161
- modelCalls.push(...called)
162
- return off
163
- }
164
- const plain = await attemptOn(model, {}, called)
165
- modelCalls.push(...called)
166
- return plain
167
- }
168
-
169
- // Resolve the ladder from the LIVE registry: the configured memory model, then the
170
- // session's route, then everything else routable. The model list is deployment state,
171
- // not a constant, so it is read per call and the winner is remembered.
172
- const available = await listAvailableModels()
173
- const ladder = buildModelLadder({
174
- preferred: memoryPreferred ?? parseModelRef(cfg.summaryModel),
175
- session: target,
176
- available,
177
- })
178
- if (ladder.length === 0) throw new Error('dsh-memory: no routable model for memory work')
179
- const modelCalls = []
180
- const outcome = await runLadder({
181
- ladder,
182
- memory: modelMemory,
183
- attempt: attemptModel,
184
- onAttempt: (model, text, error) => {
185
- if (error) record(store.root, 'model-failed', { detail: { provider: model.provider, model: model.model, error: String(error.message || error).slice(0, 120) } })
186
- else if (text === '') record(store.root, 'model-empty', { detail: { provider: model.provider, model: model.model } })
187
- },
188
- })
189
- if (outcome.text !== '') {
190
- if (outcome.tried.length > 1) {
191
- logRun(store.root, 'model-fallback', {
192
- purpose,
193
- used: outcome.target.provider + '/' + outcome.target.model,
194
- skipped: outcome.tried.slice(0, -1).map((x) => x.target.provider + '/' + x.target.model).join(','),
195
- })
196
- }
197
- memoryPreferred = outcome.target
198
- return outcome.text
199
- }
200
- // Every model returned nothing. That is a deployment problem, not a verdict about the
201
- // material, and it is reported as an error so the session is retried rather than settled.
202
- throw new Error('dsh-memory: no model returned text (' + outcome.tried.map((x) => x.target.model + (x.error ? ':' + x.error : ':empty')).join(', ') + ')')
203
- }
204
- /** Summarizer bound to the session's own model; undefined when the session is unrouted. */
205
- /** Model for memory writes: the configured memory model, else the agent's route. */
206
- function memoryTarget(agent) {
207
- return parseModelRef(cfg.summaryModel) ?? resolveTarget(agent)
208
- }
209
-
210
- /** The configured memory model wins; otherwise the session's own model. */
211
- function summaryLlm(session) {
212
- const configured = parseModelRef(cfg.summaryModel)
213
- if (configured) return (prompt) => runLlm(configured, prompt, 'memory-summary', 1200)
214
- const header = session?.requestHeader?.()?.config
215
- if (!header || !header.provider || !header.model) return undefined
216
- return (prompt) => runLlm({ provider: header.provider, model: header.model }, prompt, 'memory-summary', 1200)
217
- }
218
-
219
- async function resolveKey(session) {
220
- const cached = keyBySession.get(session.id)
221
- if (cached) return cached
222
- const key = await autoAssociateSession(store, ctx, session).catch(() => 'global')
223
- keyBySession.set(session.id, key)
224
- return key
225
- }
226
-
227
- /**
228
- * Record the repository commit history for an agent's workspace. The workspace key comes
229
- * from the session; the repository path comes from the device-local association, so the
230
- * plugin never has to guess where the checkout lives.
231
- */
232
- async function historyOf(agent, { limit } = {}) {
233
- const workspace = await workspaceOf(agent)
234
- if (!workspace) return { ok: false, reason: 'no workspace for this session', added: 0 }
235
- const path = readAssociations(store.root)[workspace]?.path
236
- if (!path) return { ok: false, reason: 'no local folder associated with ' + workspace, added: 0 }
237
- return ingestHistory(store.root, workspace, path, (argv, opts) => git.run(argv, opts), { limit })
238
- }
239
-
240
- /** The workspace key for an agent's session, or undefined when it has none. */
241
- async function workspaceOf(agent) {
242
- const session = agent && agent.session
243
- if (!session) return undefined
244
- const key = await resolveKey(session).catch(() => 'global')
245
- return key === 'global' ? undefined : key
246
- }
247
-
248
- function maybeCapture(session) {
249
- // Memory belongs to the MAIN agent. A subagent's work is already described in the parent
250
- // transcript that delegated it, so capturing the child duplicates that work and buries the
251
- // parent's account of it.
252
- if (isSubagentSession(session)) return
253
- const id = session.id
254
- const now = Date.now()
255
- if (now - (captureThrottle.get(id) ?? 0) < cfg.autoCaptureIntervalMs) return
256
- captureThrottle.set(id, now)
257
- const sessionQuery = ctx.get('sessionQuery')
258
- if (!sessionQuery) return
259
- const options = {
260
- maxUserTurns: cfg.digestMaxUserTurns,
261
- maxAssistantGists: cfg.digestMaxAssistantGists,
262
- maxMessageChars: cfg.digestMaxMessageChars,
263
- maxBytes: cfg.digestMaxBytes,
264
- maxBullets: cfg.summaryMaxBullets,
265
- }
266
- void captureSession(store, { sessionQuery, llm: summaryLlm(session) }, session, options)
267
- .then((r) => {
268
- if (r && r.changed) scheduler.markDirty()
269
- if (r && r.summarySource === 'thin') record(store.root, 'summary-skipped', { saved: 'thin', detail: { sessionId: r.sessionId } })
270
- if (r && r.summarySource === 'skipped') record(store.root, 'summary-skipped', { saved: 'trivial', detail: { sessionId: r.sessionId } })
271
- if (r && r.summarySource) logRun(store.root, 'capture', { sessionId: r.sessionId, summarySource: r.summarySource, fresh: r.fresh, bullets: r.bullets })
272
- })
273
- .catch((error) => ctx.logger.warn('dsh-memory: capture failed: ' + (error && error.message)))
274
- }
275
-
276
- /**
277
- * What the consolidation cadence should act on now.
278
- *
279
- * The watchdog runs FIRST, because an entry stranded mid-stage by a crash is invisible to every
280
- * other query: it is neither pending nor failed, so nothing would ever move it.
281
- *
282
- * Failures whose backoff has expired come back here automatically, which is what makes 'we
283
- * cannot let things stay failed' true in the running system rather than only in the model.
284
- * Entries that have spent their budget are abandoned explicitly, so the queue drains instead of
285
- * accumulating work that can never succeed.
286
- */
287
- /**
288
- * Recovery, exposed to the tool and to the cadence's parking decision.
289
- *
290
- * The cadence can PARK a large recovery but can never PERFORM one: approval.request() throws
291
- * outside an open turn, and the idle ticker has no turn. That is a hard boundary rather than a
292
- * limitation to route around, so the cadence's job is to make the work visible and a live tool
293
- * call is the only path.
294
- */
295
- const recoverNow = {
296
- plan() {
297
- const p = planRecovery(store.root, { thresholdBytes: cfg.recoverApprovalThresholdBytes })
298
- return { total: p.total, recoverable: p.recoverable, oversized: p.oversized, missing: p.missing, thresholdBytes: p.threshold }
299
- },
300
- async run(args, exec) {
301
- const p = planRecovery(store.root, { thresholdBytes: cfg.recoverApprovalThresholdBytes })
302
- const candidates = args.id
303
- ? p.entries.filter((e) => e.id === args.id)
304
- : p.entries.filter((e) => !e.needsApproval).sort((a, b) => (a.bytes ?? 0) - (b.bytes ?? 0)).slice(0, args.limit ?? 1)
305
- const recovered = []
306
- const declined = []
307
- for (const target of candidates) {
308
- if (target.needsApproval) {
309
- const approval = ctx.get('approval')
310
- if (!approval) { declined.push({ id: target.id, why: 'no approval service' }); continue }
311
- let outcome
312
- try {
313
- outcome = await approval.request({
314
- agent: exec?.agent,
315
- toolName: 'memory_recover',
316
- reason: 'Rebuild a ' + (target.bytes ? Math.round(target.bytes / 1048576) + 'MB' : 'large') + ' session digest',
317
- })
318
- } catch (error) {
319
- // Outside an open turn. Refusing is the whole point: an unapproved large recovery must
320
- // not proceed on the grounds that asking was impossible.
321
- declined.push({ id: target.id, why: 'cannot ask outside a turn' })
322
- continue
323
- }
324
- // ONLY an explicit once-grant proceeds. 'unavailable' FAILS CLOSED, by design.
325
- if (outcome !== 'allowed-once') { declined.push({ id: target.id, why: String(outcome) }); continue }
326
- }
327
- try {
328
- await recoverOne(store, services, target.id, { cfg, now: () => Date.now() })
329
- recovered.push({ id: target.id, bytes: target.bytes })
330
- logRun(store.root, 'recover', { count: 1, detail: { id: target.id } })
331
- } catch (error) {
332
- declined.push({ id: target.id, why: 'recovery failed', error: String((error && error.message) || error).slice(0, 140) })
333
- }
334
- }
335
- return { mode: 'run', ...this.plan(), recovered, declined }
336
- },
337
- }
338
-
339
- function dueWork(now = Date.now()) {
340
- const recovered = reapStalled(store.root, now)
341
- if (recovered.length) logRun(store.root, 'queue-reaped', { count: recovered.length, detail: { ids: recovered.slice(0, 10).join(',') } })
342
- // An orphan too large to rebuild unasked is PARKED, not left looking like ordinary waiting
343
- // work. Parked is visible and costs nothing; the tool is the only thing that can proceed, and
344
- // it has to be able to find them.
345
- for (const target of planRecovery(store.root, { thresholdBytes: cfg.recoverApprovalThresholdBytes }).entries) {
346
- if (!target.needsApproval) continue
347
- const parked = readQueue(store.root).get(target.id)
348
- if (!parked || parked.state !== STATES.ORPHANED) continue
349
- transition(store.root, target.id, STATES.HELD, { reason: REASONS.NEEDS_APPROVAL, detail: { sourceBytes: target.bytes ?? null } })
350
- }
351
- const work = due(store.root, now)
352
- const abandon = []
353
- const digests = []
354
- for (const item of work) {
355
- if (item.action === 'abandon') { abandon.push(item.entry); continue }
356
- // A recovery is not a digest: consolidating it would spend a call on material that is known
357
- // to be missing. The parking loop above has already handled the ones that need approval.
358
- if (item.action === 'recover') continue
359
- digests.push({ id: item.entry.id, tier: item.entry.detail?.tier })
360
- }
361
- for (const entry of abandon) {
362
- // Out of the retry loop, but never hidden: problems() reports it and status counts it.
363
- transition(store.root, entry.id, STATES.ABANDONED, { reason: entry.reason ?? 'attempts-exhausted' })
364
- }
365
- if (abandon.length) logRun(store.root, 'queue-abandoned', { count: abandon.length })
366
- return digests
367
- }
368
-
369
- function maybeConsolidate(agent) {
370
- const now = Date.now()
371
- if (now - lastConsolidateAttempt < cfg.autoConsolidateIntervalMs) return
372
- lastConsolidateAttempt = now
373
- // The queue decides what is due: waiting work, plus failures whose backoff expired. A count of
374
- // pending digests would miss exactly the work this phase exists to bring back.
375
- const work = dueWork(now)
376
- if (work.length < cfg.autoConsolidateMinPending) return
377
- const target = memoryTarget(agent)
378
- if (!target) return
379
- const llm = (prompt) => runLlm(target, prompt, 'memory-consolidation', 8000)
380
- void runConsolidation(store, {
381
- llm, digests: work, batchSize: cfg.autoConsolidateBatchSize,
382
- maxPromptBytes: cfg.consolidateMaxPromptBytes, maxDigestBytes: cfg.consolidateMaxDigestBytes,
383
- })
384
- .then((r) => { scheduler.markDirty(); logRun(store.root, 'consolidate', r ?? {}) })
385
- .catch((error) => { logRun(store.root, 'consolidate-error', { error: String(error) }); ctx.logger.warn('dsh-memory: consolidate failed: ' + (error && error.message)) })
386
- }
387
-
388
- /** On-demand freshness pass: the same work the daily cadence does, triggered explicitly. */
389
- async function refreshNow(agent, { limit } = {}) {
390
- const target = memoryTarget(agent)
391
- if (!target) throw new Error('memory_refresh needs a routed provider/model: send a message first or set summaryModel')
392
- const llm = (prompt) => runLlm(target, prompt, 'memory-consolidation', 8000)
393
- const result = await runRefresh(store.root === undefined ? store : store, { llm, limit: limit ?? 3 })
394
- if (result.updated || result.superseded) scheduler.markDirty()
395
- return result
396
- }
397
-
398
- /**
399
- * Drop captured digests that have already been consolidated and are beyond the newest N.
400
- * Digests are raw session material: gitignored, but one file per session accumulates forever.
401
- * A pending digest is never touched - it is the only copy of work not yet in the store.
402
- */
403
- function maybePrune() {
404
- try {
405
- const result = pruneDigests(store, { keep: cfg.digestKeepCount })
406
- if (result.removed > 0) { record(store.root, 'digest-prune', { count: result.removed }); ctx.logger.debug('dsh-memory: pruned ' + result.removed + ' consolidated digest(s)') }
407
- } catch (error) {
408
- ctx.logger.debug('dsh-memory: digest prune skipped: ' + (error && error.message))
409
- }
410
- }
411
-
412
- /**
413
- * A snapshot for the agent: what the store holds and what needs attention.
414
- *
415
- * The queue's states are reported individually rather than as one 'pending' number, because the
416
- * states mean different things and only some of them are work. 'abandoned' and 'orphaned' in
417
- * particular are the answers to 'did anything quietly give up', which nothing could answer before.
418
- */
419
- function statusNow(agent) {
420
- const base = store.status()
421
- const states = summarizeQueue(store.root)
422
- const counts = {}
423
- for (const [state, n] of Object.entries(states)) if (n > 0) counts[state] = n
424
- // Parked work is separated by WHY it is parked: waiting for a person is a different situation
425
- // from waiting for material to be rebuilt, and they need different actions.
426
- const parked = {}
427
- for (const entry of readQueue(store.root).values()) {
428
- if (entry.state !== STATES.HELD) continue
429
- const key = entry.reason ?? 'unspecified'
430
- parked[key] = (parked[key] ?? 0) + 1
431
- }
432
- // Grouped by what is wrong, not listed per entry. The first version listed every problem and
433
- // produced 79 rows, 78 of them identical 'orphaned/digest-missing' - which is noise wearing the
434
- // costume of visibility. A reader needs the SHAPE of the problem first (one class of 78, one of
435
- // 1), then a sample of ids to act on. The full list is one query away for anyone who wants it.
436
- const attention = groupProblems(problems(store.root))
437
- const work = due(store.root)
438
- return {
439
- root: base.root,
440
- pendingDigests: pendingDigests(store).length,
441
- queue: {
442
- states: counts,
443
- ...(Object.keys(parked).length ? { parked } : {}),
444
- waiting: work.filter((w) => w.action === 'work').length,
445
- retrying: work.filter((w) => w.action === 'retry').length,
446
- recoverable: work.filter((w) => w.action === 'recover').length,
447
- // Named rather than counted: a count of problems tells a reader something is wrong, and a
448
- // list tells them WHAT, which is the difference between a warning and something actionable.
449
- ...(attention.length ? { attention } : {}),
450
- },
451
- conflicted: Boolean(base.sync && base.sync.conflicted),
452
- lastSync: base.sync && base.sync.last ? base.sync.last : null,
453
- metrics: readMetrics(store.root),
454
- ...(agent ? {} : {}),
455
- }
456
- }
457
-
458
- /**
459
- * Collapse problems into classes a reader can act on: one row per kind, with a sample of ids.
460
- * Sorted by how much of the problem each class accounts for, so the biggest comes first.
461
- */
462
- function groupProblems(entries) {
463
- const groups = new Map()
464
- for (const entry of entries) {
465
- const key = entry.state + ':' + (entry.reason ?? 'unspecified')
466
- const g = groups.get(key) ?? { state: entry.state, reason: entry.reason ?? null, count: 0, oldestHours: 0, sample: [] }
467
- g.count += 1
468
- const ms = Date.now() - Date.parse(entry.at)
469
- const hours = Number.isFinite(ms) ? Math.round(ms / 3600000) : 0
470
- if (hours > g.oldestHours) g.oldestHours = hours
471
- if (g.sample.length < 3) g.sample.push(entry.id)
472
- if (entry.attempts !== undefined && g.attempts === undefined) g.attempts = entry.attempts
473
- groups.set(key, g)
474
- }
475
- return [...groups.values()].sort((a, b) => b.count - a.count)
476
- }
477
- /** On-demand consolidation: the same pass the cadence runs, triggered explicitly. */
478
- async function consolidateNow(agent, { batchSize } = {}) {
479
- const target = memoryTarget(agent)
480
- if (!target) throw new Error('memory_consolidate needs a routed provider/model: send a message first or set the agent model')
481
- const work = dueWork()
482
- if (work.length === 0) return { consolidated: 0, written: 0, pending: 0 }
483
- const llm = (prompt) => runLlm(target, prompt, 'memory-consolidation', 8000)
484
- const result = await runConsolidation(store, {
485
- llm, digests: work, batchSize: batchSize ?? cfg.autoConsolidateBatchSize,
486
- maxPromptBytes: cfg.consolidateMaxPromptBytes, maxDigestBytes: cfg.consolidateMaxDigestBytes,
487
- })
488
- if (result.consolidated > 0) scheduler.markDirty()
489
- return {
490
- consolidated: result.consolidated ?? 0,
491
- written: result.written ?? 0,
492
- pending: due(store.root).length,
493
- ...(result.parseFailed ? { parseFailed: true } : {}),
494
- }
495
- }
496
-
497
- /**
498
- * Record new commits for the agent's workspace. Silent by design: a repository that is
499
- * absent, not a repo, or has no new commits must never interrupt a turn.
500
- */
501
- function maybeHistory(agent) {
502
- const now = Date.now()
503
- if (now - lastHistoryAttempt < cfg.autoHistoryIntervalMs) return
504
- lastHistoryAttempt = now
505
- void historyOf(agent, { limit: cfg.autoHistoryLimit })
506
- .then((r) => {
507
- // Record the ATTEMPT, not only a success. Logging solely on added > 0 makes an
508
- // unexercised hook indistinguishable from a satisfied one.
509
- record(store.root, 'history-check', { count: r && r.added ? r.added : 0, detail: { added: (r && r.added) || 0, ok: Boolean(r && r.ok), reason: (r && r.reason) || '' } })
510
- if (r && r.added > 0) { scheduler.markDirty(); record(store.root, 'history-ingest', { count: r.added, detail: { workspace: r.repoPath } }); ctx.logger.debug('dsh-memory: recorded ' + r.added + ' commit(s) for ' + r.repoPath) }
511
- })
512
- .catch((error) => {
513
- // The catch path left no trace, so a history attempt that THREW was indistinguishable
514
- // from one that never ran - the same blind spot the success-only logging had.
515
- record(store.root, 'history-check', { detail: { ok: false, error: String((error && error.message) || error).slice(0, 120) } })
516
- ctx.logger.debug('dsh-memory: history ingest skipped: ' + (error && error.message))
517
- })
518
- }
519
-
520
- function maybeRefresh(agent) {
521
- const now = Date.now()
522
- if (now - lastRefreshAttempt < cfg.autoRefreshIntervalMs) return
523
- lastRefreshAttempt = now
524
- const target = memoryTarget(agent)
525
- if (!target) return
526
- const llm = (prompt) => runLlm(target, prompt, 'memory-consolidation', 8000)
527
- void runRefresh(store, { llm, limit: 3 })
528
- .then((r) => { scheduler.markDirty(); logRun(store.root, 'refresh', r ?? {}) })
529
- .catch((error) => { logRun(store.root, 'refresh-error', { error: String(error) }); ctx.logger.warn('dsh-memory: refresh failed: ' + (error && error.message)) })
530
- }
531
-
532
- /** Persist the last sync outcome so status() can report it after the fact. */
533
- function recordSync(value) {
534
- try {
535
- mkdirSync(join(store.root, 'state'), { recursive: true })
536
- writeFileSync(join(store.root, 'state', 'sync.json'), JSON.stringify(value, null, 2) + '\n')
537
- } catch {}
538
- }
539
-
540
- /** Record a failed sync: a visible marker file, a log line, and a status record. */
541
- function failSync(stage, verdict, committed) {
542
- const reason = verdict.reason || 'unknown git failure'
543
- ctx.logger.warn('dsh-memory: git ' + stage + ' failed: ' + reason)
544
- try {
545
- mkdirSync(join(store.root, 'state'), { recursive: true })
546
- writeFileSync(join(store.root, 'state', 'CONFLICT.md'), [
547
- '# dsh-memory: git ' + stage + ' failed',
548
- '',
549
- 'When: ' + new Date().toISOString(),
550
- 'Store: ' + store.root,
551
- 'Remote: ' + (cfg.gitRemote || '(none)'),
552
- 'Conflicted: ' + String(Boolean(verdict.conflicted)),
553
- '',
554
- 'Reason:',
555
- reason,
556
- '',
557
- 'Resolve the repository by hand (git status / git rebase --continue), then run memory_sync.',
558
- 'This marker is cleared automatically by the next successful sync.',
559
- '',
560
- ].join('\n'))
561
- } catch {}
562
- recordSync({ at: new Date().toISOString(), ok: false, stage, error: reason, conflicted: Boolean(verdict.conflicted), remote: cfg.gitRemote })
563
- return { initialized: false, committed, synced: false, conflict: true, stage, error: reason }
564
- }
565
-
566
- /** Clear the conflict marker after a clean sync. */
567
- function clearSyncProblem() {
568
- try { rmSync(join(store.root, 'state', 'CONFLICT.md'), { force: true }) } catch {}
569
- }
570
-
571
- async function runSync(force = false) {
572
- const branch = cfg.gitBranch || 'main'
573
- const hasGit = existsSync(join(store.root, '.git'))
574
- if (!hasGit) {
575
- await git.init(store.root)
576
- if (cfg.gitRemote) {
577
- await git.remoteAdd(store.root, cfg.gitRemote).catch(() => {})
578
- await git.push(store.root, branch).catch(() => {})
579
- } else {
580
- await git.commit(store.root, commitMessage('store', 'init')).catch(() => {})
581
- }
582
- scheduler.synced()
583
- return { initialized: true, committed: false, synced: Boolean(cfg.gitRemote) }
584
- }
585
- let committed = false
586
- let synced = false
587
- if (force || scheduler.shouldCommit()) {
588
- try { await git.addAll(store.root); await git.commit(store.root, commitMessage('store', 'update')); scheduler.committed(); committed = true }
589
- catch (error) { ctx.logger.warn('dsh-memory: commit failed: ' + (error && error.message)) }
590
- }
591
- if (cfg.gitRemote && (force || scheduler.shouldSync())) {
592
- const pulled = await git.pull(store.root, branch).catch((error) => ({ code: -1, stderr: String((error && error.message) || error) }))
593
- const pullVerdict = classifyGitResult(pulled)
594
- if (!pullVerdict.ok) return failSync('pull', pullVerdict, committed)
595
- const pushed = await git.push(store.root, branch).catch((error) => ({ code: -1, stderr: String((error && error.message) || error) }))
596
- const pushVerdict = classifyGitResult(pushed)
597
- if (!pushVerdict.ok) return failSync('push', pushVerdict, committed)
598
- clearSyncProblem()
599
- scheduler.synced()
600
- synced = true
601
- recordSync({ at: new Date().toISOString(), ok: true, remote: cfg.gitRemote })
602
- }
603
- return { initialized: false, committed, synced }
604
- }
605
-
606
- /**
607
- * Back up the scheduled tasks on the sync cadence.
608
- *
609
- * This rides a cadence rather than an event because the thing it protects against is not an
610
- * event: the host DELETES a task when its session stops (schedule index.ts:421), so a task can
611
- * vanish between two turns with nothing to observe. Only a periodic pass notices.
612
- *
613
- * Throttled like the other cadences, and it reports through the run log the same way: a backup
614
- * that silently stopped working would be indistinguishable from a machine with no tasks.
615
- */
616
- function maybeBackupTasks() {
617
- const now = Date.now()
618
- if (now - lastTasksBackupAttempt < TASKS_BACKUP_MS) return
619
- lastTasksBackupAttempt = now
620
- try {
621
- // `||`, not `??`: the config default is an empty string, and `??` passes that straight
622
- // through, so the pass would read a path of '' , find no store, and report a clean run that
623
- // backed up nothing. A silent no-op is the failure this whole layer exists to prevent.
624
- const result = backupTasks(store.root, { schedulePath: cfg.schedulePath || defaultSchedulePath() })
625
- logRun(store.root, 'tasks-backup', {
626
- ok: result.error === null,
627
- written: result.written,
628
- refreshed: result.refreshed,
629
- stopped: result.stopped,
630
- skipped: result.skipped.length,
631
- error: result.error ?? '',
632
- })
633
- } catch (error) {
634
- logRun(store.root, 'tasks-backup', { ok: false, error: String((error && error.message) || error).slice(0, 160) })
635
- }
636
- }
637
-
638
- /**
639
- * Restore backed-up tasks into the live schedule store, once per process.
640
- *
641
- * WHY IT WAITS FOR A SESSION. The schedule runtime resolves an agent from the task's sessionId
642
- * before every delivery and throws when it cannot (schedule runtime.ts:101). At boot there is no
643
- * session, so a task restored then would be armed against an owner that does not exist and would
644
- * fail on every due scan. The first real session is therefore the earliest moment a restore can
645
- * produce a task that actually fires.
646
- *
647
- * The OWNER IS CHOSEN, NEVER ASKED FOR. That session becomes the task's owner, and the report
648
- * says so: the host deletes a task when its owning session stops (schedule index.ts:421), so a
649
- * reader needs to know which session holds them.
650
- *
651
- * ADDITIVE. It only ever creates; nothing here deletes or updates. A task already in the live
652
- * store is left exactly as it is, matched by CONTENT because create() mints a fresh id
653
- * (schedule index.ts:256) and ids therefore cannot identify anything across a restore.
654
- */
655
- async function maybeRestoreTasks(agent, { force = false } = {}) {
656
- // EVERY early return REPORTS WHY. These used to return silently, and the tool that calls this
657
- // reported success regardless - so a manual restore after the boot pass returned immediately at
658
- // the once-per-process guard while telling the caller it had restored. An action that does
659
- // nothing and says it worked is worse than one that fails.
660
- if (restoreAttempted && !force) return { ran: false, reason: 'already restored in this process; pass force to run it again' }
661
- if (!agent || isSubagentSession(agent.session)) return { ran: false, reason: 'a restored task needs a main session to own it' }
662
- const sessionId = agent.session && agent.session.id
663
- if (!sessionId) return { ran: false, reason: 'the session has no id to bind the task to' }
664
- const schedule = ctx.get('schedule')
665
- if (!schedule || typeof schedule.create !== 'function') return { ran: false, reason: 'the schedule service is unavailable' }
666
- restoreAttempted = true
667
- try {
668
- const path = cfg.schedulePath || defaultSchedulePath()
669
- const live = readScheduleStore(path)
670
- if (live.error !== null) {
671
- logRun(store.root, 'tasks-restore', { ok: false, error: live.error })
672
- return { ran: true, owner: sessionId, error: live.error }
673
- }
674
- const plan = planRestore(store.root, { liveTasks: live.tasks })
675
- const added = []
676
- const failed = []
677
- for (const entry of plan.add) {
678
- try {
679
- const record = await schedule.create(sessionId, entry.request)
680
- added.push({ id: record && record.id, title: entry.request.title })
681
- } catch (error) {
682
- failed.push({ title: entry.request.title, error: String((error && error.message) || error).slice(0, 120) })
683
- }
684
- }
685
- logRun(store.root, 'tasks-restore', {
686
- ok: failed.length === 0,
687
- owner: sessionId,
688
- added: added.length,
689
- present: plan.present.length,
690
- paused: plan.paused.length,
691
- refused: plan.refused.length,
692
- failed: failed.length,
693
- detail: failed.length ? failed : undefined,
694
- })
695
- return { ran: true, owner: sessionId, added: added.length, present: plan.present.length, paused: plan.paused.length, refused: plan.refused.length, failed: failed.length }
696
- } catch (error) {
697
- logRun(store.root, 'tasks-restore', { ok: false, error: String((error && error.message) || error).slice(0, 160) })
698
- return { ran: true, owner: sessionId, error: String((error && error.message) || error).slice(0, 160) }
699
- }
700
- }
701
-
702
- /**
703
- * Publish the memory map into the session log, so a UI can draw it without anyone calling a tool.
704
- *
705
- * WHY AN EVENT AND NOT A TOOL CALL. The Memory tab used to read the newest memory_map TOOL RESULT
706
- * out of the session, which meant it stayed empty until an agent happened to call the tool - the
707
- * reader had to be shown the data before the tab could show it back. That is backwards for a view
708
- * whose whole job is to report state nobody asked about. Session.append is the harness's own
709
- * answer: SessionEventMap is documented as "the merge-extensible, append-only source of truth",
710
- * plugins add their own event types (session/title does exactly this), and a log-only event never
711
- * reaches the model surface or the derived history. So the map is published BY the host, INTO the
712
- * log, and the client reads it from the same event window it already subscribes to.
713
- *
714
- * TWO GUARDS, both about not making the log grow without limit:
715
- * - a time throttle, because building the map walks the ledger;
716
- * - a signature comparison that ignores generatedAt, so an unchanged store appends nothing.
717
- * The signature is the point. Without it every turn would add an identical snapshot to a log that
718
- * is the durable record of the conversation.
719
- */
720
- function publishMemoryMap(session, { agent, force = false } = {}) {
721
- try {
722
- if (!session || typeof session.append !== 'function' || !session.id) return
723
- if (isSubagentSession(session)) return
724
- const now = Date.now()
725
- if (!force && now - (lastMapPublish.get(session.id) ?? 0) < MAP_PUBLISH_MS) return
726
- lastMapPublish.set(session.id, now)
727
- const key = keyBySession.get(session.id) ?? 'global'
728
- const scope = key === 'global' ? 'global' : 'workspace'
729
- // goals.get() takes an AGENT, not a session (goal/src/index.ts:277), so the goal is only
730
- // available on the pre-step path. Elsewhere the map is published without it, which is honest:
731
- // it describes what it can see.
732
- const goal = agent ? ctx.get('goals')?.get(agent)?.objective : undefined
733
- const map = buildMap(store, { scope: 'all', sessionId: session.id, memoryScope: scope, key, goal, now, maxChars: cfg.injectionMaxChars })
734
- // JSON round-trip is not decoration: append() REJECTS data containing undefined, and the
735
- // round-trip is what guarantees we hand it lossless JSON.
736
- const payload = JSON.parse(JSON.stringify(map))
737
- const signature = JSON.stringify({ ...payload, generatedAt: '' })
738
- if (!force && lastMapSignature.get(session.id) === signature) return
739
- lastMapSignature.set(session.id, signature)
740
- session.append('memory/map', payload)
741
- logRun(store.root, 'map-publish', { ok: true, scope, key: key ?? '', bytes: signature.length })
742
- } catch (error) {
743
- // Never let a view break the conversation it is describing.
744
- logRun(store.root, 'map-publish', { ok: false, error: String((error && error.message) || error).slice(0, 160) })
745
- }
746
- }
747
-
748
- function maybeSync() {
749
- if (syncInFlight || !git) return
750
- if (!scheduler.shouldCommit() && !scheduler.shouldSync()) return
751
- syncInFlight = true
752
- // Sync wrote its outcome only to state/sync.json, so the log could not answer whether the
753
- // hook ran. Both outcomes are recorded now.
754
- void runSync()
755
- .then((r) => logRun(store.root, 'sync', { ok: (r && r.ok) !== false, committed: (r && r.committed) || 0, synced: Boolean(r && r.synced) }))
756
- .catch((error) => logRun(store.root, 'sync', { ok: false, error: String((error && error.message) || error).slice(0, 160) }))
757
- .finally(() => { syncInFlight = false })
758
- }
759
-
760
- // session lifecycle: associate workspace + capture on turn end
761
- ctx.on('session/event', (session, event) => {
762
- // DEFERRED, and it has to be. This handler runs INSIDE an append - the harness sets a reentrancy
763
- // flag for the duration of the session/event emission - and Session.append refuses to reenter:
764
- // "session append cannot reenter while another append is being published". Measured in the live
765
- // log, exactly half the publishes failed with that message while the pre-step ones succeeded.
766
- // A macrotask lands after the append has settled, so the guard is satisfied.
767
- if (event.type === 'turn/start') {
768
- void resolveKey(session).then(() => setTimeout(() => publishMemoryMap(session), 0))
769
- } else if (event.type === 'turn/end') {
770
- maybeCapture(session)
771
- setTimeout(() => publishMemoryMap(session), 0)
772
- }
773
- })
774
-
775
- // Disposal is the LAST chance to capture a session, so it bypasses the throttle.
776
- //
777
- // The throttle exists to bound cost, and on a long session it works: turn ends arrive every
778
- // few minutes and each one that clears the window captures the new material. But a SHORT
779
- // session inverts it - the first turn consumes the window, and every later turn is skipped.
780
- // A one-shot subagent that does its whole job in its second turn was therefore captured from
781
- // its trivial first turn and never again, which is how 300+ sessions ended up with no memory
782
- // while their logs held 11k-35k characters of real material each.
783
- ctx.on('session/disposed', (session) => {
784
- if (!session || !session.id) return
785
- captureThrottle.delete(session.id)
786
- maybeCapture(session)
787
- })
788
-
789
- // agent pre-step: cadence + session-start injection (+ mid-session refresh interval)
790
- ctx.on('agent/pre-step', async ({ agent, turn, step, signal }, next) => {
791
- const decision = await next()
792
- if (decision.kind === 'reject' || signal.aborted) return decision
793
- maybeConsolidate(agent)
794
- maybeRefresh(agent)
795
- maybeHistory(agent)
796
- maybeSync()
797
- maybeBackupTasks()
798
- // Only a real session can own a restored task, so the restore waits for this hook.
799
- void maybeRestoreTasks(agent)
800
- const now = Date.now()
801
- if (step !== 1 && now - (injectionThrottle.get(agent.id) ?? 0) < cfg.injectionRefreshMs) return decision
802
- injectionThrottle.set(agent.id, now)
803
- const cwd = agent.session?.header?.cwd
804
- if (cwd && cwd === store.root) return decision // never inject into the memory store itself
805
- const key = await resolveKey(agent.session)
806
- const goal = ctx.get('goals')?.get(agent)?.objective
807
- const subagent = isSubagentSession(agent.session)
808
- const scope = key === 'global' ? 'global' : 'workspace'
809
- const text = buildInjectionText(store, { scope, key, goal, maxChars: cfg.injectionMaxChars, subagent })
810
- // Injection writes nothing, so without this it is invisible: the one hook whose effect is
811
- // entirely in the conversation could break and leave no trace anywhere on disk.
812
- record(store.root, 'inject', { detail: { scope, key: key ?? '', subagent, step, chars: text.length } })
813
- // Published here as well as on turn boundaries, because this is the only place the AGENT (and
814
- // so the session's goal) is in reach - and the received panel is supposed to describe what was
815
- // actually injected, goal included.
816
- publishMemoryMap(agent.session, { agent })
817
- return {
818
- ...decision,
819
- messages: [...decision.messages, createUserMessage({ content: [{ type: 'text', text }], source: { kind: PLUGIN, form: 'memory-context', version: 1 } })],
820
- }
821
- })
822
-
823
- /**
824
- * Idle ticker. Every cadence used to hang off `agent/pre-step`, so a host where
825
- * nobody was chatting never consolidated, refreshed, or synced anything. The
826
- * ticker is a cheap no-op unless a cadence is actually due, and it needs no agent:
827
- * with `summaryModel` configured the model resolves from config.
828
- */
829
- ctx.effect(() => {
830
- const timer = setInterval(() => {
831
- try {
832
- maybeConsolidate(null)
833
- maybeRefresh(null)
834
- maybePrune()
835
- // The ticker has no agent, so history ingest waits for the next step in a session.
836
- maybeSync()
837
- maybeBackupTasks()
838
- } catch (error) {
839
- ctx.logger.warn('dsh-memory: idle tick failed: ' + (error && error.message))
840
- }
841
- }, IDLE_TICK_MS)
842
- if (typeof timer.unref === 'function') timer.unref()
843
- return () => clearInterval(timer)
844
- }, 'dsh-memory: idle ticker')
845
-
846
- /**
847
- * Rerank search hits with the session model. Advisory by contract: rerankHits returns the
848
- * original order whenever anything is missing or unparseable, so search never depends on it.
849
- */
850
- async function rerankSearch(agent, query, hits, options = {}) {
851
- const enabled = options.enabled ?? cfg.rerankSearch
852
- if (!enabled) return { hits, reranked: false, reason: 'disabled' }
853
- const target = memoryTarget(agent)
854
- return rerankHits(query, hits, {
855
- ...options, enabled: true,
856
- llm: (prompt) => runLlm(target, prompt, 'memory-rerank', 500),
857
- })
858
- }
859
-
860
- // injectionMaxChars travels with the host so memory_map can describe the card that was
861
- // ACTUALLY built: the cap is configuration, and a snapshot built with a different cap would
862
- // report a size the session never saw.
863
- // Resolved once, here, where the config lives: tools.js has no access to cfg and a tool that
864
- // reached for it would throw at call time (which is how memory_tasks first failed).
865
- return { scheduler, schedulePath: cfg.schedulePath || defaultSchedulePath(), tasksNow: maybeBackupTasks, tasksRestore: maybeRestoreTasks, syncNow: () => runSync(true), consolidate: consolidateNow, refresh: refreshNow, status: statusNow, recover: recoverNow, rerankSearch, workspaceOf, historyOf, injectionMaxChars: cfg.injectionMaxChars }
866
- }
1
+ // DSH host wiring: deep hooks into session lifecycle, agent pre-step, LLM, and
2
+ // subprocess. This is the only module that touches DSH services directly.
3
+ import { existsSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'
4
+ import { join } from 'node:path'
5
+ import { BlockAssembler, ReasoningEffortId, createUserMessage } from '@deepseek-ai/dsh-llm'
6
+ import { captureSession, pendingDigests, pruneDigests } from './capture.js'
7
+ import { runConsolidation } from './consolidate-run.js'
8
+ import { runRefresh } from './refresh.js'
9
+ import { buildInjectionText } from './inject.js'
10
+ import { parseModelRef } from './defaults.js'
11
+ import { ingestHistory } from './history.js'
12
+ import { rerankHits } from './rerank.js'
13
+ import { isSubagentSession } from './session.js'
14
+ import { buildModelLadder, createModelMemory, runLadder } from './models.js'
15
+ import { due, problems, readQueue, reapStalled, REASONS, STATES, summarize as summarizeQueue, transition } from './queue.js'
16
+ import { planRecovery, recoverOne } from './recover.js'
17
+ import { logRun, noteActivity, readMetrics, record } from './telemetry.js'
18
+ import { GitRunner, SyncScheduler, classifyGitResult, commitMessage } from './git.js'
19
+ import { buildMap } from './map.js'
20
+ import { backupTasks, defaultSchedulePath, planRestore, readScheduleStore } from './tasks.js'
21
+ import { autoAssociateSession } from './session.js'
22
+ import { readAssociations } from './workspace.js'
23
+
24
+ const PLUGIN = 'dsh-memory'
25
+
26
+ /** How often the idle ticker checks whether a cadence is due (the cadences self-throttle). */
27
+ const MAP_PUBLISH_MS = 60 * 1000
28
+ const TASKS_BACKUP_MS = 30 * 60 * 1000
29
+ const IDLE_TICK_MS = 60_000
30
+
31
+ function makeGitRunner(ctx) {
32
+ const subprocess = ctx.get('subprocess')
33
+ if (!subprocess || typeof subprocess.resolveExecutable !== 'function' || typeof subprocess.spawn !== 'function') return null
34
+ let executablePromise
35
+ const run = async (argv, opts = {}) => {
36
+ if (!executablePromise) executablePromise = subprocess.resolveExecutable('git', undefined, undefined).catch(() => null)
37
+ const executable = await executablePromise
38
+ if (!executable) return { exitCode: 127, stdout: '', stderr: 'git unavailable' }
39
+ try {
40
+ const handle = subprocess.spawn({
41
+ argv: [executable, ...argv],
42
+ cwd: opts.cwd,
43
+ stdio: { stdin: 'ignore', stdout: { maxBytes: 4 * 1024 * 1024 }, stderr: { maxBytes: 16 * 1024 } },
44
+ graceMs: 2000,
45
+ env: { GIT_CONFIG_COUNT: '0', GIT_TERMINAL_PROMPT: '0', GIT_OPTIONAL_LOCKS: '0', LC_ALL: 'C' },
46
+ })
47
+ const outcome = await handle.done
48
+ const stdout = handle.collected?.stdout?.readFrom?.(0)?.text ?? ''
49
+ const stderr = handle.collected?.stderr?.readFrom?.(0)?.text ?? ''
50
+ return { exitCode: outcome?.exitCode ?? null, stdout, stderr }
51
+ } catch (error) {
52
+ return { exitCode: 1, stdout: '', stderr: String(error) }
53
+ }
54
+ }
55
+ return new GitRunner(run)
56
+ }
57
+
58
+ function resolveTarget(agent) {
59
+ const h = agent?.session?.requestHeader?.()?.config
60
+ if (h && h.provider && h.model) return { provider: h.provider, model: h.model }
61
+ const o = agent?.options
62
+ if (o && o.provider && o.model) return { provider: o.provider, model: o.model }
63
+ return undefined
64
+ }
65
+
66
+ export function wireHost(ctx, store, cfg) {
67
+ const keyBySession = new Map()
68
+ const scheduler = new SyncScheduler({ commitDebounceMs: cfg.commitDebounceMs, syncIntervalMs: cfg.autoSyncIntervalMs })
69
+ const captureThrottle = new Map()
70
+ const injectionThrottle = new Map()
71
+ let lastConsolidateAttempt = 0
72
+ let lastRefreshAttempt = 0
73
+ let lastTasksBackupAttempt = 0
74
+ let restoreAttempted = false
75
+ const lastMapPublish = new Map()
76
+ const lastMapSignature = new Map()
77
+ const mapPublishSuppressed = new Set()
78
+ let lastHistoryAttempt = 0
79
+ let syncInFlight = false
80
+
81
+ const git = makeGitRunner(ctx)
82
+ const llmService = ctx.get('llm')
83
+
84
+ /** Which model last worked, and which recently returned nothing. Learned, not configured. */
85
+ const modelMemory = createModelMemory()
86
+ let memoryPreferred
87
+ let availableCache
88
+
89
+ /**
90
+ * Every model this deployment can actually route to right now.
91
+ *
92
+ * Read from the live registry rather than a constant: providers appear and disappear, a
93
+ * subscription lapses, and role-model routing changes what is reachable. A short cache keeps
94
+ * a burst of memory writes from re-enumerating providers on every call, and a failure to
95
+ * enumerate is not fatal - the ladder still has the configured and session routes.
96
+ */
97
+ async function listAvailableModels({ maxAgeMs = 60000 } = {}) {
98
+ const now = Date.now()
99
+ if (availableCache && now - availableCache.at < maxAgeMs) return availableCache.models
100
+ const models = []
101
+ try {
102
+ const providers = llmService.listProviders()
103
+ for (const provider of providers ?? []) {
104
+ try {
105
+ const list = await llmService.listModels(provider.id)
106
+ for (const model of list ?? []) models.push({ provider: provider.id, model: model.id })
107
+ } catch { /* one unroutable provider must not empty the ladder */ }
108
+ }
109
+ } catch { /* fall through with whatever was collected */ }
110
+ availableCache = { at: now, models }
111
+ return models
112
+ }
113
+
114
+
115
+ const runLlm = async (target, prompt, purpose = 'memory-consolidation', maxTokens = 4000) => {
116
+ if (!llmService) throw new Error('llm service unavailable')
117
+
118
+ /** One attempt against ONE model, with the reasoning-effort recovery. */
119
+ const attemptOn = async (model, controls, called) => {
120
+ called.push({ provider: model.provider, model: model.model })
121
+ record(store.root, 'model-call', { modelCalls: 1, purpose, detail: { provider: model.provider, model: model.model } })
122
+ const assembler = new BlockAssembler()
123
+ const messages = [{ role: 'user', content: [{ type: 'text', text: prompt }] }]
124
+ for await (const chunk of llmService.stream({
125
+ provider: model.provider, model: model.model, messages, maxTokens, purpose, ...controls,
126
+ })) assembler.push(chunk)
127
+ // Text blocks only. A reasoning model streams its thinking as `reasoning` blocks;
128
+ // folding those into the reply prepends prose to the answer, which makes any
129
+ // "parse the JSON reply" step match the wrong braces and fail. Worse, a model that
130
+ // spends its whole budget reasoning returns NO text block at all - which is exactly
131
+ // the failure the ladder exists to route around.
132
+ return assembler.blocks().filter((b) => b && b.type === 'text').map((b) => b.text || '').join('\n').trim()
133
+ }
134
+
135
+ /**
136
+ * One model, tried at the memory reasoning effort and then with reasoning off.
137
+ * Memory work is extraction: it must not run at the session's reasoning budget, and a
138
+ * model that rejects the effort outright is retried without the control rather than
139
+ * failing every memory write.
140
+ */
141
+ const attemptModel = async (model) => {
142
+ const called = []
143
+ const effort = cfg.memoryReasoningEffort
144
+ if (effort) {
145
+ let text = ''
146
+ try {
147
+ text = await attemptOn(model, { reasoningEffort: ReasoningEffortId(effort) }, called)
148
+ } catch (error) {
149
+ const message = String((error && error.message) || error)
150
+ if (!/UNSUPPORTED_REASONING_EFFORT|does not support reasoning effort/i.test(message)) {
151
+ modelCalls.push(...called)
152
+ throw error
153
+ }
154
+ ctx.logger.debug('dsh-memory: ' + model.model + ' does not support reasoning effort "' + effort + '"; retrying without it')
155
+ const plain = await attemptOn(model, {}, called)
156
+ modelCalls.push(...called)
157
+ return plain
158
+ }
159
+ if (text !== '') { modelCalls.push(...called); return text }
160
+ ctx.logger.debug('dsh-memory: ' + model.model + ' returned no text at effort "' + effort + '"; retrying with reasoning off')
161
+ const off = await attemptOn(model, { reasoningEffort: ReasoningEffortId('off') }, called)
162
+ modelCalls.push(...called)
163
+ return off
164
+ }
165
+ const plain = await attemptOn(model, {}, called)
166
+ modelCalls.push(...called)
167
+ return plain
168
+ }
169
+
170
+ // Resolve the ladder from the LIVE registry: the configured memory model, then the
171
+ // session's route, then everything else routable. The model list is deployment state,
172
+ // not a constant, so it is read per call and the winner is remembered.
173
+ const available = await listAvailableModels()
174
+ const ladder = buildModelLadder({
175
+ preferred: memoryPreferred ?? parseModelRef(cfg.summaryModel),
176
+ session: target,
177
+ available,
178
+ })
179
+ if (ladder.length === 0) throw new Error('dsh-memory: no routable model for memory work')
180
+ const modelCalls = []
181
+ const outcome = await runLadder({
182
+ ladder,
183
+ memory: modelMemory,
184
+ attempt: attemptModel,
185
+ onAttempt: (model, text, error) => {
186
+ if (error) record(store.root, 'model-failed', { detail: { provider: model.provider, model: model.model, error: String(error.message || error).slice(0, 120) } })
187
+ else if (text === '') record(store.root, 'model-empty', { detail: { provider: model.provider, model: model.model } })
188
+ },
189
+ })
190
+ if (outcome.text !== '') {
191
+ if (outcome.tried.length > 1) {
192
+ logRun(store.root, 'model-fallback', {
193
+ purpose,
194
+ used: outcome.target.provider + '/' + outcome.target.model,
195
+ skipped: outcome.tried.slice(0, -1).map((x) => x.target.provider + '/' + x.target.model).join(','),
196
+ })
197
+ }
198
+ memoryPreferred = outcome.target
199
+ return outcome.text
200
+ }
201
+ // Every model returned nothing. That is a deployment problem, not a verdict about the
202
+ // material, and it is reported as an error so the session is retried rather than settled.
203
+ throw new Error('dsh-memory: no model returned text (' + outcome.tried.map((x) => x.target.model + (x.error ? ':' + x.error : ':empty')).join(', ') + ')')
204
+ }
205
+ /** Summarizer bound to the session's own model; undefined when the session is unrouted. */
206
+ /** Model for memory writes: the configured memory model, else the agent's route. */
207
+ function memoryTarget(agent) {
208
+ return parseModelRef(cfg.summaryModel) ?? resolveTarget(agent)
209
+ }
210
+
211
+ /** The configured memory model wins; otherwise the session's own model. */
212
+ function summaryLlm(session) {
213
+ const configured = parseModelRef(cfg.summaryModel)
214
+ if (configured) return (prompt) => runLlm(configured, prompt, 'memory-summary', 1200)
215
+ const header = session?.requestHeader?.()?.config
216
+ if (!header || !header.provider || !header.model) return undefined
217
+ return (prompt) => runLlm({ provider: header.provider, model: header.model }, prompt, 'memory-summary', 1200)
218
+ }
219
+
220
+ async function resolveKey(session) {
221
+ const cached = keyBySession.get(session.id)
222
+ if (cached) return cached
223
+ const key = await autoAssociateSession(store, ctx, session).catch(() => 'global')
224
+ keyBySession.set(session.id, key)
225
+ return key
226
+ }
227
+
228
+ /**
229
+ * Record the repository commit history for an agent's workspace. The workspace key comes
230
+ * from the session; the repository path comes from the device-local association, so the
231
+ * plugin never has to guess where the checkout lives.
232
+ */
233
+ async function historyOf(agent, { limit } = {}) {
234
+ const workspace = await workspaceOf(agent)
235
+ if (!workspace) return { ok: false, reason: 'no workspace for this session', added: 0 }
236
+ const path = readAssociations(store.root)[workspace]?.path
237
+ if (!path) return { ok: false, reason: 'no local folder associated with ' + workspace, added: 0 }
238
+ return ingestHistory(store.root, workspace, path, (argv, opts) => git.run(argv, opts), { limit })
239
+ }
240
+
241
+ /** The workspace key for an agent's session, or undefined when it has none. */
242
+ async function workspaceOf(agent) {
243
+ const session = agent && agent.session
244
+ if (!session) return undefined
245
+ const key = await resolveKey(session).catch(() => 'global')
246
+ return key === 'global' ? undefined : key
247
+ }
248
+
249
+ function maybeCapture(session) {
250
+ // Memory belongs to the MAIN agent. A subagent's work is already described in the parent
251
+ // transcript that delegated it, so capturing the child duplicates that work and buries the
252
+ // parent's account of it.
253
+ if (isSubagentSession(session)) return
254
+ const id = session.id
255
+ const now = Date.now()
256
+ if (now - (captureThrottle.get(id) ?? 0) < cfg.autoCaptureIntervalMs) return
257
+ captureThrottle.set(id, now)
258
+ const sessionQuery = ctx.get('sessionQuery')
259
+ if (!sessionQuery) return
260
+ const options = {
261
+ maxUserTurns: cfg.digestMaxUserTurns,
262
+ maxAssistantGists: cfg.digestMaxAssistantGists,
263
+ maxMessageChars: cfg.digestMaxMessageChars,
264
+ maxBytes: cfg.digestMaxBytes,
265
+ maxBullets: cfg.summaryMaxBullets,
266
+ }
267
+ void captureSession(store, { sessionQuery, llm: summaryLlm(session) }, session, options)
268
+ .then((r) => {
269
+ if (r && r.changed) scheduler.markDirty()
270
+ if (r && r.summarySource === 'thin') record(store.root, 'summary-skipped', { saved: 'thin', detail: { sessionId: r.sessionId } })
271
+ if (r && r.summarySource === 'skipped') record(store.root, 'summary-skipped', { saved: 'trivial', detail: { sessionId: r.sessionId } })
272
+ if (r && r.summarySource) logRun(store.root, 'capture', { sessionId: r.sessionId, summarySource: r.summarySource, fresh: r.fresh, bullets: r.bullets })
273
+ })
274
+ .catch((error) => ctx.logger.warn('dsh-memory: capture failed: ' + (error && error.message)))
275
+ }
276
+
277
+ /**
278
+ * What the consolidation cadence should act on now.
279
+ *
280
+ * The watchdog runs FIRST, because an entry stranded mid-stage by a crash is invisible to every
281
+ * other query: it is neither pending nor failed, so nothing would ever move it.
282
+ *
283
+ * Failures whose backoff has expired come back here automatically, which is what makes 'we
284
+ * cannot let things stay failed' true in the running system rather than only in the model.
285
+ * Entries that have spent their budget are abandoned explicitly, so the queue drains instead of
286
+ * accumulating work that can never succeed.
287
+ */
288
+ /**
289
+ * Recovery, exposed to the tool and to the cadence's parking decision.
290
+ *
291
+ * The cadence can PARK a large recovery but can never PERFORM one: approval.request() throws
292
+ * outside an open turn, and the idle ticker has no turn. That is a hard boundary rather than a
293
+ * limitation to route around, so the cadence's job is to make the work visible and a live tool
294
+ * call is the only path.
295
+ */
296
+ const recoverNow = {
297
+ plan() {
298
+ const p = planRecovery(store.root, { thresholdBytes: cfg.recoverApprovalThresholdBytes })
299
+ return { total: p.total, recoverable: p.recoverable, oversized: p.oversized, missing: p.missing, thresholdBytes: p.threshold }
300
+ },
301
+ async run(args, exec) {
302
+ const p = planRecovery(store.root, { thresholdBytes: cfg.recoverApprovalThresholdBytes })
303
+ const candidates = args.id
304
+ ? p.entries.filter((e) => e.id === args.id)
305
+ : p.entries.filter((e) => !e.needsApproval).sort((a, b) => (a.bytes ?? 0) - (b.bytes ?? 0)).slice(0, args.limit ?? 1)
306
+ const recovered = []
307
+ const declined = []
308
+ for (const target of candidates) {
309
+ if (target.needsApproval) {
310
+ const approval = ctx.get('approval')
311
+ if (!approval) { declined.push({ id: target.id, why: 'no approval service' }); continue }
312
+ let outcome
313
+ try {
314
+ outcome = await approval.request({
315
+ agent: exec?.agent,
316
+ toolName: 'memory_recover',
317
+ reason: 'Rebuild a ' + (target.bytes ? Math.round(target.bytes / 1048576) + 'MB' : 'large') + ' session digest',
318
+ })
319
+ } catch (error) {
320
+ // Outside an open turn. Refusing is the whole point: an unapproved large recovery must
321
+ // not proceed on the grounds that asking was impossible.
322
+ declined.push({ id: target.id, why: 'cannot ask outside a turn' })
323
+ continue
324
+ }
325
+ // ONLY an explicit once-grant proceeds. 'unavailable' FAILS CLOSED, by design.
326
+ if (outcome !== 'allowed-once') { declined.push({ id: target.id, why: String(outcome) }); continue }
327
+ }
328
+ try {
329
+ await recoverOne(store, services, target.id, { cfg, now: () => Date.now() })
330
+ recovered.push({ id: target.id, bytes: target.bytes })
331
+ logRun(store.root, 'recover', { count: 1, detail: { id: target.id } })
332
+ } catch (error) {
333
+ declined.push({ id: target.id, why: 'recovery failed', error: String((error && error.message) || error).slice(0, 140) })
334
+ }
335
+ }
336
+ return { mode: 'run', ...this.plan(), recovered, declined }
337
+ },
338
+ }
339
+
340
+ function dueWork(now = Date.now()) {
341
+ const recovered = reapStalled(store.root, now)
342
+ if (recovered.length) logRun(store.root, 'queue-reaped', { count: recovered.length, detail: { ids: recovered.slice(0, 10).join(',') } })
343
+ // An orphan too large to rebuild unasked is PARKED, not left looking like ordinary waiting
344
+ // work. Parked is visible and costs nothing; the tool is the only thing that can proceed, and
345
+ // it has to be able to find them.
346
+ for (const target of planRecovery(store.root, { thresholdBytes: cfg.recoverApprovalThresholdBytes }).entries) {
347
+ if (!target.needsApproval) continue
348
+ const parked = readQueue(store.root).get(target.id)
349
+ if (!parked || parked.state !== STATES.ORPHANED) continue
350
+ transition(store.root, target.id, STATES.HELD, { reason: REASONS.NEEDS_APPROVAL, detail: { sourceBytes: target.bytes ?? null } })
351
+ }
352
+ const work = due(store.root, now)
353
+ const abandon = []
354
+ const digests = []
355
+ for (const item of work) {
356
+ if (item.action === 'abandon') { abandon.push(item.entry); continue }
357
+ // A recovery is not a digest: consolidating it would spend a call on material that is known
358
+ // to be missing. The parking loop above has already handled the ones that need approval.
359
+ if (item.action === 'recover') continue
360
+ digests.push({ id: item.entry.id, tier: item.entry.detail?.tier })
361
+ }
362
+ for (const entry of abandon) {
363
+ // Out of the retry loop, but never hidden: problems() reports it and status counts it.
364
+ transition(store.root, entry.id, STATES.ABANDONED, { reason: entry.reason ?? 'attempts-exhausted' })
365
+ }
366
+ if (abandon.length) logRun(store.root, 'queue-abandoned', { count: abandon.length })
367
+ return digests
368
+ }
369
+
370
+ function maybeConsolidate(agent) {
371
+ const now = Date.now()
372
+ if (now - lastConsolidateAttempt < cfg.autoConsolidateIntervalMs) return
373
+ lastConsolidateAttempt = now
374
+ // The queue decides what is due: waiting work, plus failures whose backoff expired. A count of
375
+ // pending digests would miss exactly the work this phase exists to bring back.
376
+ const work = dueWork(now)
377
+ if (work.length < cfg.autoConsolidateMinPending) return
378
+ const target = memoryTarget(agent)
379
+ if (!target) return
380
+ const llm = (prompt) => runLlm(target, prompt, 'memory-consolidation', 8000)
381
+ void runConsolidation(store, {
382
+ llm, digests: work, batchSize: cfg.autoConsolidateBatchSize,
383
+ maxPromptBytes: cfg.consolidateMaxPromptBytes, maxDigestBytes: cfg.consolidateMaxDigestBytes,
384
+ })
385
+ .then((r) => { scheduler.markDirty(); logRun(store.root, 'consolidate', r ?? {}) })
386
+ .catch((error) => { logRun(store.root, 'consolidate-error', { error: String(error) }); ctx.logger.warn('dsh-memory: consolidate failed: ' + (error && error.message)) })
387
+ }
388
+
389
+ /** On-demand freshness pass: the same work the daily cadence does, triggered explicitly. */
390
+ async function refreshNow(agent, { limit } = {}) {
391
+ const target = memoryTarget(agent)
392
+ if (!target) throw new Error('memory_refresh needs a routed provider/model: send a message first or set summaryModel')
393
+ const llm = (prompt) => runLlm(target, prompt, 'memory-consolidation', 8000)
394
+ const result = await runRefresh(store.root === undefined ? store : store, { llm, limit: limit ?? 3 })
395
+ if (result.updated || result.superseded) scheduler.markDirty()
396
+ return result
397
+ }
398
+
399
+ /**
400
+ * Drop captured digests that have already been consolidated and are beyond the newest N.
401
+ * Digests are raw session material: gitignored, but one file per session accumulates forever.
402
+ * A pending digest is never touched - it is the only copy of work not yet in the store.
403
+ */
404
+ function maybePrune() {
405
+ try {
406
+ const result = pruneDigests(store, { keep: cfg.digestKeepCount })
407
+ if (result.removed > 0) { record(store.root, 'digest-prune', { count: result.removed }); ctx.logger.debug('dsh-memory: pruned ' + result.removed + ' consolidated digest(s)') }
408
+ } catch (error) {
409
+ ctx.logger.debug('dsh-memory: digest prune skipped: ' + (error && error.message))
410
+ }
411
+ }
412
+
413
+ /**
414
+ * A snapshot for the agent: what the store holds and what needs attention.
415
+ *
416
+ * The queue's states are reported individually rather than as one 'pending' number, because the
417
+ * states mean different things and only some of them are work. 'abandoned' and 'orphaned' in
418
+ * particular are the answers to 'did anything quietly give up', which nothing could answer before.
419
+ */
420
+ function statusNow(agent) {
421
+ const base = store.status()
422
+ const states = summarizeQueue(store.root)
423
+ const counts = {}
424
+ for (const [state, n] of Object.entries(states)) if (n > 0) counts[state] = n
425
+ // Parked work is separated by WHY it is parked: waiting for a person is a different situation
426
+ // from waiting for material to be rebuilt, and they need different actions.
427
+ const parked = {}
428
+ for (const entry of readQueue(store.root).values()) {
429
+ if (entry.state !== STATES.HELD) continue
430
+ const key = entry.reason ?? 'unspecified'
431
+ parked[key] = (parked[key] ?? 0) + 1
432
+ }
433
+ // Grouped by what is wrong, not listed per entry. The first version listed every problem and
434
+ // produced 79 rows, 78 of them identical 'orphaned/digest-missing' - which is noise wearing the
435
+ // costume of visibility. A reader needs the SHAPE of the problem first (one class of 78, one of
436
+ // 1), then a sample of ids to act on. The full list is one query away for anyone who wants it.
437
+ const attention = groupProblems(problems(store.root))
438
+ const work = due(store.root)
439
+ return {
440
+ root: base.root,
441
+ pendingDigests: pendingDigests(store).length,
442
+ queue: {
443
+ states: counts,
444
+ ...(Object.keys(parked).length ? { parked } : {}),
445
+ waiting: work.filter((w) => w.action === 'work').length,
446
+ retrying: work.filter((w) => w.action === 'retry').length,
447
+ recoverable: work.filter((w) => w.action === 'recover').length,
448
+ // Named rather than counted: a count of problems tells a reader something is wrong, and a
449
+ // list tells them WHAT, which is the difference between a warning and something actionable.
450
+ ...(attention.length ? { attention } : {}),
451
+ },
452
+ conflicted: Boolean(base.sync && base.sync.conflicted),
453
+ lastSync: base.sync && base.sync.last ? base.sync.last : null,
454
+ metrics: readMetrics(store.root),
455
+ ...(agent ? {} : {}),
456
+ }
457
+ }
458
+
459
+ /**
460
+ * Collapse problems into classes a reader can act on: one row per kind, with a sample of ids.
461
+ * Sorted by how much of the problem each class accounts for, so the biggest comes first.
462
+ */
463
+ function groupProblems(entries) {
464
+ const groups = new Map()
465
+ for (const entry of entries) {
466
+ const key = entry.state + ':' + (entry.reason ?? 'unspecified')
467
+ const g = groups.get(key) ?? { state: entry.state, reason: entry.reason ?? null, count: 0, oldestHours: 0, sample: [] }
468
+ g.count += 1
469
+ const ms = Date.now() - Date.parse(entry.at)
470
+ const hours = Number.isFinite(ms) ? Math.round(ms / 3600000) : 0
471
+ if (hours > g.oldestHours) g.oldestHours = hours
472
+ if (g.sample.length < 3) g.sample.push(entry.id)
473
+ if (entry.attempts !== undefined && g.attempts === undefined) g.attempts = entry.attempts
474
+ groups.set(key, g)
475
+ }
476
+ return [...groups.values()].sort((a, b) => b.count - a.count)
477
+ }
478
+ /** On-demand consolidation: the same pass the cadence runs, triggered explicitly. */
479
+ async function consolidateNow(agent, { batchSize } = {}) {
480
+ const target = memoryTarget(agent)
481
+ if (!target) throw new Error('memory_consolidate needs a routed provider/model: send a message first or set the agent model')
482
+ const work = dueWork()
483
+ if (work.length === 0) return { consolidated: 0, written: 0, pending: 0 }
484
+ const llm = (prompt) => runLlm(target, prompt, 'memory-consolidation', 8000)
485
+ const result = await runConsolidation(store, {
486
+ llm, digests: work, batchSize: batchSize ?? cfg.autoConsolidateBatchSize,
487
+ maxPromptBytes: cfg.consolidateMaxPromptBytes, maxDigestBytes: cfg.consolidateMaxDigestBytes,
488
+ })
489
+ if (result.consolidated > 0) scheduler.markDirty()
490
+ return {
491
+ consolidated: result.consolidated ?? 0,
492
+ written: result.written ?? 0,
493
+ pending: due(store.root).length,
494
+ ...(result.parseFailed ? { parseFailed: true } : {}),
495
+ }
496
+ }
497
+
498
+ /**
499
+ * Record new commits for the agent's workspace. Silent by design: a repository that is
500
+ * absent, not a repo, or has no new commits must never interrupt a turn.
501
+ */
502
+ function maybeHistory(agent) {
503
+ const now = Date.now()
504
+ if (now - lastHistoryAttempt < cfg.autoHistoryIntervalMs) return
505
+ lastHistoryAttempt = now
506
+ void historyOf(agent, { limit: cfg.autoHistoryLimit })
507
+ .then((r) => {
508
+ // Record the ATTEMPT, not only a success. Logging solely on added > 0 makes an
509
+ // unexercised hook indistinguishable from a satisfied one.
510
+ record(store.root, 'history-check', { count: r && r.added ? r.added : 0, detail: { added: (r && r.added) || 0, ok: Boolean(r && r.ok), reason: (r && r.reason) || '' } })
511
+ if (r && r.added > 0) { scheduler.markDirty(); record(store.root, 'history-ingest', { count: r.added, detail: { workspace: r.repoPath } }); ctx.logger.debug('dsh-memory: recorded ' + r.added + ' commit(s) for ' + r.repoPath) }
512
+ })
513
+ .catch((error) => {
514
+ // The catch path left no trace, so a history attempt that THREW was indistinguishable
515
+ // from one that never ran - the same blind spot the success-only logging had.
516
+ record(store.root, 'history-check', { detail: { ok: false, error: String((error && error.message) || error).slice(0, 120) } })
517
+ ctx.logger.debug('dsh-memory: history ingest skipped: ' + (error && error.message))
518
+ })
519
+ }
520
+
521
+ function maybeRefresh(agent) {
522
+ const now = Date.now()
523
+ if (now - lastRefreshAttempt < cfg.autoRefreshIntervalMs) return
524
+ lastRefreshAttempt = now
525
+ const target = memoryTarget(agent)
526
+ if (!target) return
527
+ const llm = (prompt) => runLlm(target, prompt, 'memory-consolidation', 8000)
528
+ void runRefresh(store, { llm, limit: 3 })
529
+ .then((r) => { scheduler.markDirty(); logRun(store.root, 'refresh', r ?? {}) })
530
+ .catch((error) => { logRun(store.root, 'refresh-error', { error: String(error) }); ctx.logger.warn('dsh-memory: refresh failed: ' + (error && error.message)) })
531
+ }
532
+
533
+ /** Persist the last sync outcome so status() can report it after the fact. */
534
+ function recordSync(value) {
535
+ try {
536
+ mkdirSync(join(store.root, 'state'), { recursive: true })
537
+ writeFileSync(join(store.root, 'state', 'sync.json'), JSON.stringify(value, null, 2) + '\n')
538
+ } catch {}
539
+ }
540
+
541
+ /** Record a failed sync: a visible marker file, a log line, and a status record. */
542
+ function failSync(stage, verdict, committed) {
543
+ const reason = verdict.reason || 'unknown git failure'
544
+ ctx.logger.warn('dsh-memory: git ' + stage + ' failed: ' + reason)
545
+ try {
546
+ mkdirSync(join(store.root, 'state'), { recursive: true })
547
+ writeFileSync(join(store.root, 'state', 'CONFLICT.md'), [
548
+ '# dsh-memory: git ' + stage + ' failed',
549
+ '',
550
+ 'When: ' + new Date().toISOString(),
551
+ 'Store: ' + store.root,
552
+ 'Remote: ' + (cfg.gitRemote || '(none)'),
553
+ 'Conflicted: ' + String(Boolean(verdict.conflicted)),
554
+ '',
555
+ 'Reason:',
556
+ reason,
557
+ '',
558
+ 'Resolve the repository by hand (git status / git rebase --continue), then run memory_sync.',
559
+ 'This marker is cleared automatically by the next successful sync.',
560
+ '',
561
+ ].join('\n'))
562
+ } catch {}
563
+ recordSync({ at: new Date().toISOString(), ok: false, stage, error: reason, conflicted: Boolean(verdict.conflicted), remote: cfg.gitRemote })
564
+ return { initialized: false, committed, synced: false, conflict: true, stage, error: reason }
565
+ }
566
+
567
+ /** Clear the conflict marker after a clean sync. */
568
+ function clearSyncProblem() {
569
+ try { rmSync(join(store.root, 'state', 'CONFLICT.md'), { force: true }) } catch {}
570
+ }
571
+
572
+ async function runSync(force = false) {
573
+ const branch = cfg.gitBranch || 'main'
574
+ const hasGit = existsSync(join(store.root, '.git'))
575
+ if (!hasGit) {
576
+ await git.init(store.root)
577
+ if (cfg.gitRemote) {
578
+ await git.remoteAdd(store.root, cfg.gitRemote).catch(() => {})
579
+ await git.push(store.root, branch).catch(() => {})
580
+ } else {
581
+ await git.commit(store.root, commitMessage('store', 'init')).catch(() => {})
582
+ }
583
+ scheduler.synced()
584
+ return { initialized: true, committed: false, synced: Boolean(cfg.gitRemote) }
585
+ }
586
+ let committed = false
587
+ let synced = false
588
+ if (force || scheduler.shouldCommit()) {
589
+ try { await git.addAll(store.root); await git.commit(store.root, commitMessage('store', 'update')); scheduler.committed(); committed = true }
590
+ catch (error) { ctx.logger.warn('dsh-memory: commit failed: ' + (error && error.message)) }
591
+ }
592
+ if (cfg.gitRemote && (force || scheduler.shouldSync())) {
593
+ const pulled = await git.pull(store.root, branch).catch((error) => ({ code: -1, stderr: String((error && error.message) || error) }))
594
+ const pullVerdict = classifyGitResult(pulled)
595
+ if (!pullVerdict.ok) return failSync('pull', pullVerdict, committed)
596
+ const pushed = await git.push(store.root, branch).catch((error) => ({ code: -1, stderr: String((error && error.message) || error) }))
597
+ const pushVerdict = classifyGitResult(pushed)
598
+ if (!pushVerdict.ok) return failSync('push', pushVerdict, committed)
599
+ clearSyncProblem()
600
+ scheduler.synced()
601
+ synced = true
602
+ recordSync({ at: new Date().toISOString(), ok: true, remote: cfg.gitRemote })
603
+ }
604
+ return { initialized: false, committed, synced }
605
+ }
606
+
607
+ /**
608
+ * Back up the scheduled tasks on the sync cadence.
609
+ *
610
+ * This rides a cadence rather than an event because the thing it protects against is not an
611
+ * event: the host DELETES a task when its session stops (schedule index.ts:421), so a task can
612
+ * vanish between two turns with nothing to observe. Only a periodic pass notices.
613
+ *
614
+ * Throttled like the other cadences, and it reports through the run log the same way: a backup
615
+ * that silently stopped working would be indistinguishable from a machine with no tasks.
616
+ */
617
+ function maybeBackupTasks() {
618
+ const now = Date.now()
619
+ if (now - lastTasksBackupAttempt < TASKS_BACKUP_MS) return
620
+ lastTasksBackupAttempt = now
621
+ try {
622
+ // `||`, not `??`: the config default is an empty string, and `??` passes that straight
623
+ // through, so the pass would read a path of '' , find no store, and report a clean run that
624
+ // backed up nothing. A silent no-op is the failure this whole layer exists to prevent.
625
+ const result = backupTasks(store.root, { schedulePath: cfg.schedulePath || defaultSchedulePath() })
626
+ logRun(store.root, 'tasks-backup', {
627
+ ok: result.error === null,
628
+ written: result.written,
629
+ refreshed: result.refreshed,
630
+ stopped: result.stopped,
631
+ skipped: result.skipped.length,
632
+ error: result.error ?? '',
633
+ })
634
+ } catch (error) {
635
+ logRun(store.root, 'tasks-backup', { ok: false, error: String((error && error.message) || error).slice(0, 160) })
636
+ }
637
+ }
638
+
639
+ /**
640
+ * Restore backed-up tasks into the live schedule store, once per process.
641
+ *
642
+ * WHY IT WAITS FOR A SESSION. The schedule runtime resolves an agent from the task's sessionId
643
+ * before every delivery and throws when it cannot (schedule runtime.ts:101). At boot there is no
644
+ * session, so a task restored then would be armed against an owner that does not exist and would
645
+ * fail on every due scan. The first real session is therefore the earliest moment a restore can
646
+ * produce a task that actually fires.
647
+ *
648
+ * The OWNER IS CHOSEN, NEVER ASKED FOR. That session becomes the task's owner, and the report
649
+ * says so: the host deletes a task when its owning session stops (schedule index.ts:421), so a
650
+ * reader needs to know which session holds them.
651
+ *
652
+ * ADDITIVE. It only ever creates; nothing here deletes or updates. A task already in the live
653
+ * store is left exactly as it is, matched by CONTENT because create() mints a fresh id
654
+ * (schedule index.ts:256) and ids therefore cannot identify anything across a restore.
655
+ */
656
+ async function maybeRestoreTasks(agent, { force = false } = {}) {
657
+ // EVERY early return REPORTS WHY. These used to return silently, and the tool that calls this
658
+ // reported success regardless - so a manual restore after the boot pass returned immediately at
659
+ // the once-per-process guard while telling the caller it had restored. An action that does
660
+ // nothing and says it worked is worse than one that fails.
661
+ if (restoreAttempted && !force) return { ran: false, reason: 'already restored in this process; pass force to run it again' }
662
+ if (!agent || isSubagentSession(agent.session)) return { ran: false, reason: 'a restored task needs a main session to own it' }
663
+ const sessionId = agent.session && agent.session.id
664
+ if (!sessionId) return { ran: false, reason: 'the session has no id to bind the task to' }
665
+ const schedule = ctx.get('schedule')
666
+ if (!schedule || typeof schedule.create !== 'function') return { ran: false, reason: 'the schedule service is unavailable' }
667
+ restoreAttempted = true
668
+ try {
669
+ const path = cfg.schedulePath || defaultSchedulePath()
670
+ const live = readScheduleStore(path)
671
+ if (live.error !== null) {
672
+ logRun(store.root, 'tasks-restore', { ok: false, error: live.error })
673
+ return { ran: true, owner: sessionId, error: live.error }
674
+ }
675
+ const plan = planRestore(store.root, { liveTasks: live.tasks })
676
+ const added = []
677
+ const failed = []
678
+ for (const entry of plan.add) {
679
+ try {
680
+ const record = await schedule.create(sessionId, entry.request)
681
+ added.push({ id: record && record.id, title: entry.request.title })
682
+ } catch (error) {
683
+ failed.push({ title: entry.request.title, error: String((error && error.message) || error).slice(0, 120) })
684
+ }
685
+ }
686
+ logRun(store.root, 'tasks-restore', {
687
+ ok: failed.length === 0,
688
+ owner: sessionId,
689
+ added: added.length,
690
+ present: plan.present.length,
691
+ paused: plan.paused.length,
692
+ refused: plan.refused.length,
693
+ failed: failed.length,
694
+ detail: failed.length ? failed : undefined,
695
+ })
696
+ return { ran: true, owner: sessionId, added: added.length, present: plan.present.length, paused: plan.paused.length, refused: plan.refused.length, failed: failed.length }
697
+ } catch (error) {
698
+ logRun(store.root, 'tasks-restore', { ok: false, error: String((error && error.message) || error).slice(0, 160) })
699
+ return { ran: true, owner: sessionId, error: String((error && error.message) || error).slice(0, 160) }
700
+ }
701
+ }
702
+
703
+ /**
704
+ * Publish the memory map into the session log, so a UI can draw it without anyone calling a tool.
705
+ *
706
+ * WHY AN EVENT AND NOT A TOOL CALL. The Memory tab used to read the newest memory_map TOOL RESULT
707
+ * out of the session, which meant it stayed empty until an agent happened to call the tool - the
708
+ * reader had to be shown the data before the tab could show it back. That is backwards for a view
709
+ * whose whole job is to report state nobody asked about. Session.append is the harness's own
710
+ * answer: SessionEventMap is documented as "the merge-extensible, append-only source of truth",
711
+ * plugins add their own event types (session/title does exactly this), and a log-only event never
712
+ * reaches the model surface or the derived history. So the map is published BY the host, INTO the
713
+ * log, and the client reads it from the same event window it already subscribes to.
714
+ *
715
+ * TWO GUARDS, both about not making the log grow without limit:
716
+ * - a time throttle, because building the map walks the ledger;
717
+ * - a signature comparison that ignores generatedAt, so an unchanged store appends nothing.
718
+ * The signature is the point. Without it every turn would add an identical snapshot to a log that
719
+ * is the durable record of the conversation.
720
+ */
721
+ function publishMemoryMap(session, { agent, force = false } = {}) {
722
+ try {
723
+ if (!session || typeof session.append !== 'function' || !session.id) return
724
+ if (isSubagentSession(session)) return
725
+ // A session whose harness did not honor ignorable is poisoned by every further append;
726
+ // the decision to stop is made once, below, and honored on every later attempt.
727
+ if (mapPublishSuppressed.has(session.id)) return
728
+ const now = Date.now()
729
+ if (!force && now - (lastMapPublish.get(session.id) ?? 0) < MAP_PUBLISH_MS) return
730
+ lastMapPublish.set(session.id, now)
731
+ const key = keyBySession.get(session.id) ?? 'global'
732
+ const scope = key === 'global' ? 'global' : 'workspace'
733
+ // goals.get() takes an AGENT, not a session (goal/src/index.ts:277), so the goal is only
734
+ // available on the pre-step path. Elsewhere the map is published without it, which is honest:
735
+ // it describes what it can see.
736
+ const goal = agent ? ctx.get('goals')?.get(agent)?.objective : undefined
737
+ const map = buildMap(store, { scope: 'all', sessionId: session.id, memoryScope: scope, key, goal, now, maxChars: cfg.injectionMaxChars })
738
+ // JSON round-trip is not decoration: append() REJECTS data containing undefined, and the
739
+ // round-trip is what guarantees we hand it lossless JSON.
740
+ const payload = JSON.parse(JSON.stringify(map))
741
+ const signature = JSON.stringify({ ...payload, generatedAt: '' })
742
+ if (!force && lastMapSignature.get(session.id) === signature) return
743
+ lastMapSignature.set(session.id, signature)
744
+ // Out-of-repo event types are absent from KNOWN_SESSION_EVENT_TYPES; without
745
+ // ignorable:true the persistence read path refuses the whole session log.
746
+ // The append returns the logged event: if the harness did NOT stamp
747
+ // ignorable onto it (a harness older than the option), the very next load
748
+ // of this session would fail with "unknown to this harness and not marked
749
+ // ignorable". Detecting that from the return value lets us stop before a
750
+ // log is poisoned, instead of discovering it when history fails to load.
751
+ const logged = session.append('memory/map', payload, { ignorable: true })
752
+ if (logged?.ignorable !== true) {
753
+ if (!mapPublishSuppressed.has(session.id)) {
754
+ mapPublishSuppressed.add(session.id)
755
+ logRun(store.root, 'map-publish', { ok: false, error: 'harness did not stamp ignorable; publishing suppressed for this session' })
756
+ }
757
+ return
758
+ }
759
+ logRun(store.root, 'map-publish', { ok: true, scope, key: key ?? '', bytes: signature.length })
760
+ } catch (error) {
761
+ // Never let a view break the conversation it is describing.
762
+ logRun(store.root, 'map-publish', { ok: false, error: String((error && error.message) || error).slice(0, 160) })
763
+ }
764
+ }
765
+
766
+ function maybeSync() {
767
+ if (syncInFlight || !git) return
768
+ if (!scheduler.shouldCommit() && !scheduler.shouldSync()) return
769
+ syncInFlight = true
770
+ // Sync wrote its outcome only to state/sync.json, so the log could not answer whether the
771
+ // hook ran. Both outcomes are recorded now.
772
+ void runSync()
773
+ .then((r) => logRun(store.root, 'sync', { ok: (r && r.ok) !== false, committed: (r && r.committed) || 0, synced: Boolean(r && r.synced) }))
774
+ .catch((error) => logRun(store.root, 'sync', { ok: false, error: String((error && error.message) || error).slice(0, 160) }))
775
+ .finally(() => { syncInFlight = false })
776
+ }
777
+
778
+ // session lifecycle: associate workspace + capture on turn end
779
+ ctx.on('session/event', (session, event) => {
780
+ // DEFERRED, and it has to be. This handler runs INSIDE an append - the harness sets a reentrancy
781
+ // flag for the duration of the session/event emission - and Session.append refuses to reenter:
782
+ // "session append cannot reenter while another append is being published". Measured in the live
783
+ // log, exactly half the publishes failed with that message while the pre-step ones succeeded.
784
+ // A macrotask lands after the append has settled, so the guard is satisfied.
785
+ if (event.type === 'turn/start') {
786
+ void resolveKey(session).then(() => setTimeout(() => publishMemoryMap(session), 0))
787
+ } else if (event.type === 'turn/end') {
788
+ maybeCapture(session)
789
+ setTimeout(() => publishMemoryMap(session), 0)
790
+ }
791
+ })
792
+
793
+ // Disposal is the LAST chance to capture a session, so it bypasses the throttle.
794
+ //
795
+ // The throttle exists to bound cost, and on a long session it works: turn ends arrive every
796
+ // few minutes and each one that clears the window captures the new material. But a SHORT
797
+ // session inverts it - the first turn consumes the window, and every later turn is skipped.
798
+ // A one-shot subagent that does its whole job in its second turn was therefore captured from
799
+ // its trivial first turn and never again, which is how 300+ sessions ended up with no memory
800
+ // while their logs held 11k-35k characters of real material each.
801
+ ctx.on('session/disposed', (session) => {
802
+ if (!session || !session.id) return
803
+ captureThrottle.delete(session.id)
804
+ maybeCapture(session)
805
+ })
806
+
807
+ // agent pre-step: cadence + session-start injection (+ mid-session refresh interval)
808
+ ctx.on('agent/pre-step', async ({ agent, turn, step, signal }, next) => {
809
+ const decision = await next()
810
+ if (decision.kind === 'reject' || signal.aborted) return decision
811
+ maybeConsolidate(agent)
812
+ maybeRefresh(agent)
813
+ maybeHistory(agent)
814
+ maybeSync()
815
+ maybeBackupTasks()
816
+ // Only a real session can own a restored task, so the restore waits for this hook.
817
+ void maybeRestoreTasks(agent)
818
+ const now = Date.now()
819
+ if (step !== 1 && now - (injectionThrottle.get(agent.id) ?? 0) < cfg.injectionRefreshMs) return decision
820
+ injectionThrottle.set(agent.id, now)
821
+ const cwd = agent.session?.header?.cwd
822
+ if (cwd && cwd === store.root) return decision // never inject into the memory store itself
823
+ const key = await resolveKey(agent.session)
824
+ const goal = ctx.get('goals')?.get(agent)?.objective
825
+ const subagent = isSubagentSession(agent.session)
826
+ const scope = key === 'global' ? 'global' : 'workspace'
827
+ const text = buildInjectionText(store, { scope, key, goal, maxChars: cfg.injectionMaxChars, subagent })
828
+ // Injection writes nothing, so without this it is invisible: the one hook whose effect is
829
+ // entirely in the conversation could break and leave no trace anywhere on disk.
830
+ record(store.root, 'inject', { detail: { scope, key: key ?? '', subagent, step, chars: text.length } })
831
+ // Published here as well as on turn boundaries, because this is the only place the AGENT (and
832
+ // so the session's goal) is in reach - and the received panel is supposed to describe what was
833
+ // actually injected, goal included.
834
+ publishMemoryMap(agent.session, { agent })
835
+ return {
836
+ ...decision,
837
+ messages: [...decision.messages, createUserMessage({ content: [{ type: 'text', text }], source: { kind: PLUGIN, form: 'memory-context', version: 1 } })],
838
+ }
839
+ })
840
+
841
+ /**
842
+ * Idle ticker. Every cadence used to hang off `agent/pre-step`, so a host where
843
+ * nobody was chatting never consolidated, refreshed, or synced anything. The
844
+ * ticker is a cheap no-op unless a cadence is actually due, and it needs no agent:
845
+ * with `summaryModel` configured the model resolves from config.
846
+ */
847
+ ctx.effect(() => {
848
+ const timer = setInterval(() => {
849
+ try {
850
+ maybeConsolidate(null)
851
+ maybeRefresh(null)
852
+ maybePrune()
853
+ // The ticker has no agent, so history ingest waits for the next step in a session.
854
+ maybeSync()
855
+ maybeBackupTasks()
856
+ } catch (error) {
857
+ ctx.logger.warn('dsh-memory: idle tick failed: ' + (error && error.message))
858
+ }
859
+ }, IDLE_TICK_MS)
860
+ if (typeof timer.unref === 'function') timer.unref()
861
+ return () => clearInterval(timer)
862
+ }, 'dsh-memory: idle ticker')
863
+
864
+ /**
865
+ * Rerank search hits with the session model. Advisory by contract: rerankHits returns the
866
+ * original order whenever anything is missing or unparseable, so search never depends on it.
867
+ */
868
+ async function rerankSearch(agent, query, hits, options = {}) {
869
+ const enabled = options.enabled ?? cfg.rerankSearch
870
+ if (!enabled) return { hits, reranked: false, reason: 'disabled' }
871
+ const target = memoryTarget(agent)
872
+ return rerankHits(query, hits, {
873
+ ...options, enabled: true,
874
+ llm: (prompt) => runLlm(target, prompt, 'memory-rerank', 500),
875
+ })
876
+ }
877
+
878
+ // injectionMaxChars travels with the host so memory_map can describe the card that was
879
+ // ACTUALLY built: the cap is configuration, and a snapshot built with a different cap would
880
+ // report a size the session never saw.
881
+ // Resolved once, here, where the config lives: tools.js has no access to cfg and a tool that
882
+ // reached for it would throw at call time (which is how memory_tasks first failed).
883
+ return { scheduler, schedulePath: cfg.schedulePath || defaultSchedulePath(), tasksNow: maybeBackupTasks, tasksRestore: maybeRestoreTasks, syncNow: () => runSync(true), consolidate: consolidateNow, refresh: refreshNow, status: statusNow, recover: recoverNow, rerankSearch, workspaceOf, historyOf, injectionMaxChars: cfg.injectionMaxChars }
884
+ }