@try-works/dsh-memory 0.1.19 → 0.1.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +585 -585
- package/STATUS.md +294 -287
- package/cordis.patch.yml +69 -69
- package/package.json +1 -1
- package/scripts/check-package.mjs +76 -76
- package/scripts/inspect-log.ts +40 -0
- package/scripts/repair-ignorable.ts +130 -0
- package/scripts/scan-sessions.ts +43 -0
- package/scripts/verify-log.ts +26 -0
- package/src/SYSTEM.template.md +311 -311
- package/src/capture.js +242 -242
- package/src/client/index.js +886 -886
- package/src/config.js +57 -57
- package/src/consolidate-run.js +132 -132
- package/src/consolidate.js +330 -330
- package/src/context.js +88 -88
- package/src/decisions.js +106 -106
- package/src/defaults.js +93 -93
- package/src/digest.js +255 -255
- package/src/entry.js +179 -179
- package/src/eval.js +72 -72
- package/src/files.js +60 -60
- package/src/fixture.js +56 -56
- package/src/forget.js +71 -71
- package/src/freshness.js +65 -65
- package/src/fts.js +166 -166
- package/src/git.js +53 -53
- package/src/history.js +226 -226
- package/src/host.js +884 -866
- package/src/index.js +115 -94
- package/src/inject.js +108 -108
- package/src/layers.js +178 -178
- package/src/ledger.js +19 -19
- package/src/map.js +184 -184
- package/src/models.js +136 -136
- package/src/queue.js +602 -602
- package/src/rank.js +31 -31
- package/src/recover.js +91 -91
- package/src/refresh.js +80 -80
- package/src/remember.js +58 -58
- package/src/rerank.js +90 -90
- package/src/search.js +75 -75
- package/src/select.js +57 -57
- package/src/session.js +60 -60
- package/src/state.js +240 -240
- package/src/store.js +128 -128
- package/src/summarize.js +195 -195
- package/src/system-doc.js +171 -171
- package/src/tasks.js +617 -617
- package/src/telemetry.js +122 -122
- package/src/tools.js +828 -818
- package/src/working.js +288 -288
- package/src/workspace-key.js +68 -68
- package/src/workspace.js +87 -87
package/src/capture.js
CHANGED
|
@@ -1,242 +1,242 @@
|
|
|
1
|
-
// Incremental, summarized capture: sessionQuery events -> dated summary sections.
|
|
2
|
-
//
|
|
3
|
-
// Capture is O(new events), not O(session): a lastSeq watermark in state/sessions.json
|
|
4
|
-
// means each pass summarizes only what arrived since the previous one. Raw message
|
|
5
|
-
// bodies never reach the digest file - a digest holds a bounded metadata header plus
|
|
6
|
-
// the summarizer's durable bullets (or a deterministic fallback when no model is
|
|
7
|
-
// available). Tool mechanics are reduced to a count.
|
|
8
|
-
import { createHash } from 'node:crypto'
|
|
9
|
-
import { appendFileSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'
|
|
10
|
-
import { join } from 'node:path'
|
|
11
|
-
import { projectDigest, classifyJunk, extractCandidates, renderDigestHeader, totalToolCalls, dateOf } from './digest.js'
|
|
12
|
-
import { summarizeSession, renderSummarySection } from './summarize.js'
|
|
13
|
-
import { advance, fail, isSettled, readQueue, REASONS, seed, STATES } from './queue.js'
|
|
14
|
-
|
|
15
|
-
/** Digest file format. A stored state with a different value forces a full re-capture. */
|
|
16
|
-
export const DIGEST_FORMAT = 2
|
|
17
|
-
|
|
18
|
-
export function digestPath(root, id) { return join(root, 'digests', id + '.md') }
|
|
19
|
-
|
|
20
|
-
export function readDigestText(root, id) {
|
|
21
|
-
try { return readFileSync(digestPath(root, id), 'utf8') } catch { return '' }
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
/** Log position of an event: its own seq when present, else its index in the log. */
|
|
25
|
-
function seqOf(event, index) {
|
|
26
|
-
return typeof event?.seq === 'number' ? event.seq : index
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
export function digestHash(digest) {
|
|
30
|
-
return createHash('sha256')
|
|
31
|
-
.update(digest.sessionId + '|' + digest.eventCount + '|' + digest.userTexts.join('\n') + '|' + digest.assistantTexts.join('\n'))
|
|
32
|
-
.digest('hex')
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
/**
|
|
36
|
-
* Capture one session incrementally.
|
|
37
|
-
* @param store - memory store.
|
|
38
|
-
* @param services - { sessionQuery, llm? }; llm(prompt) summarizes, omit for deterministic notes.
|
|
39
|
-
* @param session - the live session (id + header).
|
|
40
|
-
* @param options - capture budgets (see defaults.js).
|
|
41
|
-
* @returns {Promise<{sessionId: string, tier: string, changed: boolean, fresh: number, summarySource: string, bullets: number}>}
|
|
42
|
-
*/
|
|
43
|
-
/** Does the queue already track this candidate? */
|
|
44
|
-
function queueHas(root, id) { return readQueue(root).has(id) }
|
|
45
|
-
|
|
46
|
-
export async function captureSession(store, services, session, options = {}) {
|
|
47
|
-
const cfg = {
|
|
48
|
-
maxUserTurns: 60, maxAssistantGists: 40, maxMessageChars: 600,
|
|
49
|
-
maxGoalChars: 200, maxCheckpoints: 4, maxBytes: 40000,
|
|
50
|
-
maxBullets: 16, maxBulletChars: 2000, summaryMinSignalChars: 400,
|
|
51
|
-
now: () => new Date().toISOString(),
|
|
52
|
-
...options,
|
|
53
|
-
}
|
|
54
|
-
let events = []
|
|
55
|
-
try {
|
|
56
|
-
const snapshot = await services?.sessionQuery?.readSession(session.id)
|
|
57
|
-
events = snapshot?.events ?? []
|
|
58
|
-
} catch { events = [] }
|
|
59
|
-
|
|
60
|
-
// Read the queue entry directly. The previous record shape is a projection over the queue, so
|
|
61
|
-
// asking for it here meant projecting every candidate to answer a question about one.
|
|
62
|
-
const prevEntry = readQueue(store.root).get(session.id)
|
|
63
|
-
const prev = {
|
|
64
|
-
digestFormat: prevEntry?.detail?.digestFormat,
|
|
65
|
-
lastSeq: prevEntry?.detail?.lastSeq,
|
|
66
|
-
summaryAttempts: prevEntry?.attempts,
|
|
67
|
-
}
|
|
68
|
-
const formatChanged = prev.digestFormat !== DIGEST_FORMAT
|
|
69
|
-
const watermark = (!formatChanged && typeof prev.lastSeq === 'number') ? prev.lastSeq : -1
|
|
70
|
-
const fresh = events.filter((event, index) => seqOf(event, index) > watermark)
|
|
71
|
-
const logEnd = events.length === 0 ? watermark : seqOf(events[events.length - 1], events.length - 1)
|
|
72
|
-
// The watermark advances only over material that was actually CONSUMED. A trivial ping is
|
|
73
|
-
// dropped outright, but a thin delta is provisional: it is below the signal floor now and may
|
|
74
|
-
// not be later, so it stays in the window and accumulates with the next delta. Advancing past
|
|
75
|
-
// it would skip that text forever, which is how content quietly disappeared.
|
|
76
|
-
// Material is CONSUMED only when it was actually dealt with: summarized, or deliberately
|
|
77
|
-
// concluded to hold nothing durable. A thin delta accumulates, and a FAILED summary keeps its
|
|
78
|
-
// material so the retry has something to retry - consuming it would leave the retry with no
|
|
79
|
-
// input, which is a silent no-op dressed up as a retry. After the cap the failure is accepted
|
|
80
|
-
// and the window advances, so a permanently broken session cannot block progress forever.
|
|
81
|
-
// A thin delta is a DELIBERATE skip, so it counts as dealt with; only a FAILURE is re-presented.
|
|
82
|
-
const consumes = (failed, givingUp) => !failed || givingUp
|
|
83
|
-
|
|
84
|
-
// Tier is decided once, on the first capture, from the whole session; later passes
|
|
85
|
-
// reuse it so a long session is never re-projected wholesale.
|
|
86
|
-
const tier = prev.tier ?? classifyJunk(projectDigest({
|
|
87
|
-
sessionId: session.id, createdAt: session.header?.createdAt, cwd: session.header?.cwd,
|
|
88
|
-
}, events))
|
|
89
|
-
|
|
90
|
-
if (fresh.length === 0) {
|
|
91
|
-
return { sessionId: session.id, tier, changed: false, fresh: 0, summarySource: 'none', bullets: 0 }
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
const freshDigest = projectDigest({
|
|
95
|
-
sessionId: session.id, createdAt: session.header?.createdAt, cwd: session.header?.cwd,
|
|
96
|
-
}, fresh)
|
|
97
|
-
const toolCallCount = (prev.toolCallCount ?? 0) + totalToolCalls(freshDigest.toolTally)
|
|
98
|
-
const meta = {
|
|
99
|
-
sessionId: session.id,
|
|
100
|
-
title: freshDigest.title || prev.title || session.header?.title || '',
|
|
101
|
-
createdAt: session.header?.createdAt,
|
|
102
|
-
cwd: session.header?.cwd,
|
|
103
|
-
date: dateOf(session.header?.createdAt),
|
|
104
|
-
eventCount: events.length,
|
|
105
|
-
toolCallCount,
|
|
106
|
-
capturedAt: cfg.now(),
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
mkdirSync(join(store.root, 'digests'), { recursive: true })
|
|
110
|
-
const path = digestPath(store.root, session.id)
|
|
111
|
-
if (!existsSync(path) || formatChanged) writeFileSync(path, renderDigestHeader(meta) + '\n', 'utf8')
|
|
112
|
-
|
|
113
|
-
// Signal check: how much NEW material would a summary actually see? Counting the FILTERED
|
|
114
|
-
// candidates, not raw events, so a burst of tool noise cannot masquerade as content.
|
|
115
|
-
const candidateSet = tier === "trivial" ? undefined : extractCandidates(freshDigest, cfg)
|
|
116
|
-
const signalChars = candidateSet
|
|
117
|
-
? candidateSet.userTurns.reduce((n, text) => n + text.length, 0)
|
|
118
|
-
+ candidateSet.assistantGists.reduce((n, text) => n + text.length, 0)
|
|
119
|
-
: 0
|
|
120
|
-
const enoughSignal = signalChars >= (cfg.summaryMinSignalChars ?? 0)
|
|
121
|
-
|
|
122
|
-
// A ping has nothing durable to find, and a thin delta is not worth a model call either.
|
|
123
|
-
const summary = tier === "trivial"
|
|
124
|
-
? { bullets: [], source: "skipped" }
|
|
125
|
-
: (enoughSignal
|
|
126
|
-
? await summarizeSession(services?.llm, candidateSet, { date: meta.date }, cfg)
|
|
127
|
-
: { bullets: [], source: "thin" })
|
|
128
|
-
if (summary.bullets.length > 0) {
|
|
129
|
-
appendFileSync(path, '\n' + renderSummarySection(summary.bullets, { stamp: meta.capturedAt, source: summary.source }), 'utf8')
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
// SETTLED versus FAILED. Nothing durable written is only a conclusion when the model SAID
|
|
133
|
-
// so (NO_DURABLE_CONTENT), or when the session was never worth a call at all. An empty or
|
|
134
|
-
// unparseable reply is a FAILURE: it stays pending so the next pass retries it. A failed
|
|
135
|
-
// summary is retried at most summaryMaxAttempts times before it is laid to rest as failed
|
|
136
|
-
// instead of being retried forever.
|
|
137
|
-
// SETTLED versus FAILED. Nothing durable written is only a conclusion when the model SAID so
|
|
138
|
-
// (NO_DURABLE_CONTENT), or when the session was never worth a call at all. An empty or
|
|
139
|
-
// unparseable reply is a FAILURE, and a failure is retried rather than settled.
|
|
140
|
-
//
|
|
141
|
-
// This writes TRANSITIONS now, not flags. The state IS the answer to what happened, so no
|
|
142
|
-
// reader has to derive it: a deliberate skip, a defect and a finished job are three states,
|
|
143
|
-
// not three combinations of a timestamp, a source string and a counter.
|
|
144
|
-
const summaryFailed = summary.bullets.length === 0
|
|
145
|
-
&& (summary.reason === 'empty-reply' || summary.reason === 'unparseable' || summary.reason === 'error')
|
|
146
|
-
const giveUp = summaryFailed && (Number(prev.summaryAttempts ?? 0) + 1) >= (cfg.summaryMaxAttempts ?? 2)
|
|
147
|
-
// The watermark advances only over material that was DEALT WITH. A failure keeps its material
|
|
148
|
-
// so the retry has something to retry; a deliberate skip consumes it.
|
|
149
|
-
const consumed = !summaryFailed || giveUp
|
|
150
|
-
const detail = {
|
|
151
|
-
tier,
|
|
152
|
-
title: meta.title,
|
|
153
|
-
capturedAt: meta.capturedAt,
|
|
154
|
-
digestFormat: DIGEST_FORMAT,
|
|
155
|
-
lastSeq: consumed ? logEnd : watermark,
|
|
156
|
-
digestHash: digestHash(freshDigest),
|
|
157
|
-
eventCount: events.length,
|
|
158
|
-
toolCallCount,
|
|
159
|
-
};
|
|
160
|
-
// Which state this pass produced, and why. A stopping state always carries a reason, because
|
|
161
|
-
// a decision and a defect must be told apart by a reader who was not here.
|
|
162
|
-
let outcome;
|
|
163
|
-
if (tier !== 'content') outcome = { state: STATES.SKIPPED, reason: REASONS.TRIVIAL };
|
|
164
|
-
else if (summary.source === 'thin') outcome = { state: STATES.SKIPPED, reason: REASONS.THIN };
|
|
165
|
-
else if (summary.bullets.length > 0) outcome = { state: STATES.SUMMARIZED };
|
|
166
|
-
else if (summary.reason === REASONS.NO_DURABLE) outcome = { state: STATES.SKIPPED, reason: REASONS.NO_DURABLE };
|
|
167
|
-
else if (summaryFailed) {
|
|
168
|
-
outcome = {
|
|
169
|
-
state: STATES.FAILED,
|
|
170
|
-
reason: summary.reason === 'error' ? REASONS.CALL_FAILED : summary.reason,
|
|
171
|
-
error: summary.error,
|
|
172
|
-
};
|
|
173
|
-
} else outcome = { state: STATES.SKIPPED, reason: REASONS.UNCLASSIFIED };
|
|
174
|
-
|
|
175
|
-
// A new candidate is seeded PENDING and then walks the real path. Recording the intermediate
|
|
176
|
-
// steps is not ceremony: the summarization genuinely ran, and a failure that appears straight
|
|
177
|
-
// from pending would describe something that never happened.
|
|
178
|
-
if (!queueHas(store.root, session.id)) seed(store.root, session.id, STATES.PENDING, { detail });
|
|
179
|
-
advance(store.root, session.id, STATES.SUMMARIZING, { detail });
|
|
180
|
-
if (outcome.state === STATES.FAILED) {
|
|
181
|
-
// fail() schedules the next attempt and enforces the cap itself, so a caller cannot leave
|
|
182
|
-
// a candidate failed forever by forgetting to check it.
|
|
183
|
-
fail(store.root, session.id, { reason: outcome.reason, error: outcome.error });
|
|
184
|
-
} else {
|
|
185
|
-
advance(store.root, session.id, outcome.state, { reason: outcome.reason, attempts: 0, detail });
|
|
186
|
-
}
|
|
187
|
-
// A successful summary clears the attempt budget: the failure is behind it.
|
|
188
|
-
if (outcome.state === STATES.SUMMARIZED) advance(store.root, session.id, STATES.SUMMARIZED, { attempts: 0, detail });
|
|
189
|
-
return {
|
|
190
|
-
sessionId: session.id, tier, changed: true, fresh: fresh.length,
|
|
191
|
-
summarySource: summary.source, bullets: summary.bullets.length,
|
|
192
|
-
}
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
/**
|
|
196
|
-
* Prune captured digests. Digests are raw session material and gitignored, but one file per
|
|
197
|
-
* session accumulates forever. Only digests that have already been consolidated AND are beyond
|
|
198
|
-
* the newest `keep` are removed: a pending digest is the only copy of work not yet in the store.
|
|
199
|
-
* @returns {{removed: number, kept: number, pending: number}}
|
|
200
|
-
*/
|
|
201
|
-
export function pruneDigests(store, options = {}) {
|
|
202
|
-
const keep = options.keep ?? 50
|
|
203
|
-
// A digest may only be removed when the work is DEALT WITH: while a candidate is waiting, its
|
|
204
|
-
// digest is the only copy of material that is not in the store yet.
|
|
205
|
-
const entries = [...readQueue(store.root).values()]
|
|
206
|
-
.map((entry) => ({
|
|
207
|
-
id: entry.id,
|
|
208
|
-
settled: isSettled(entry),
|
|
209
|
-
mtime: statSafe(digestPath(store.root, entry.id)),
|
|
210
|
-
capturedAt: Date.parse(String(entry.detail?.capturedAt ?? '')) || 0,
|
|
211
|
-
}))
|
|
212
|
-
// "Newest" must not depend on file mtime alone: several digests written in the same tick
|
|
213
|
-
// are otherwise indistinguishable, and a same-instant test (or a fast machine) would prune
|
|
214
|
-
// the wrong one. capturedAt is the record of when the session was captured; mtime breaks
|
|
215
|
-
// ties when records are missing it.
|
|
216
|
-
.sort((a, b) => (b.capturedAt - a.capturedAt) || ((b.mtime ?? 0) - (a.mtime ?? 0)))
|
|
217
|
-
let removed = 0
|
|
218
|
-
let pending = 0
|
|
219
|
-
for (const [index, item] of entries.entries()) {
|
|
220
|
-
if (!item.settled) { pending += 1; continue }
|
|
221
|
-
if (index < keep) continue
|
|
222
|
-
try {
|
|
223
|
-
rmSync(digestPath(store.root, item.id))
|
|
224
|
-
removed += 1
|
|
225
|
-
} catch {}
|
|
226
|
-
}
|
|
227
|
-
return { removed, kept: Math.max(0, entries.length - removed - pending), pending }
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
/** Modification time of a file, or undefined when it is absent. */
|
|
231
|
-
function statSafe(file) {
|
|
232
|
-
try { return statSync(file).mtimeMs } catch { return undefined }
|
|
233
|
-
}
|
|
234
|
-
/** Sessions captured but not yet consolidated (oldest-first is handled by the caller). */
|
|
235
|
-
export function pendingDigests(store) {
|
|
236
|
-
// Waiting work, which is exactly what the queue calls PENDING or SUMMARIZED - the same set the
|
|
237
|
-
// cadence treats as work. Neither a failure on its backoff nor an orphan belongs here: those are
|
|
238
|
-
// different actions, and folding them in is what made 78 orphans look like 78 retries.
|
|
239
|
-
return [...readQueue(store.root).values()]
|
|
240
|
-
.filter((entry) => entry.state === STATES.PENDING || entry.state === STATES.SUMMARIZED)
|
|
241
|
-
.map((entry) => ({ id: entry.id, tier: entry.detail?.tier, digestHash: entry.detail?.digestHash }))
|
|
242
|
-
}
|
|
1
|
+
// Incremental, summarized capture: sessionQuery events -> dated summary sections.
|
|
2
|
+
//
|
|
3
|
+
// Capture is O(new events), not O(session): a lastSeq watermark in state/sessions.json
|
|
4
|
+
// means each pass summarizes only what arrived since the previous one. Raw message
|
|
5
|
+
// bodies never reach the digest file - a digest holds a bounded metadata header plus
|
|
6
|
+
// the summarizer's durable bullets (or a deterministic fallback when no model is
|
|
7
|
+
// available). Tool mechanics are reduced to a count.
|
|
8
|
+
import { createHash } from 'node:crypto'
|
|
9
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'
|
|
10
|
+
import { join } from 'node:path'
|
|
11
|
+
import { projectDigest, classifyJunk, extractCandidates, renderDigestHeader, totalToolCalls, dateOf } from './digest.js'
|
|
12
|
+
import { summarizeSession, renderSummarySection } from './summarize.js'
|
|
13
|
+
import { advance, fail, isSettled, readQueue, REASONS, seed, STATES } from './queue.js'
|
|
14
|
+
|
|
15
|
+
/** Digest file format. A stored state with a different value forces a full re-capture. */
|
|
16
|
+
export const DIGEST_FORMAT = 2
|
|
17
|
+
|
|
18
|
+
export function digestPath(root, id) { return join(root, 'digests', id + '.md') }
|
|
19
|
+
|
|
20
|
+
export function readDigestText(root, id) {
|
|
21
|
+
try { return readFileSync(digestPath(root, id), 'utf8') } catch { return '' }
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** Log position of an event: its own seq when present, else its index in the log. */
|
|
25
|
+
function seqOf(event, index) {
|
|
26
|
+
return typeof event?.seq === 'number' ? event.seq : index
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export function digestHash(digest) {
|
|
30
|
+
return createHash('sha256')
|
|
31
|
+
.update(digest.sessionId + '|' + digest.eventCount + '|' + digest.userTexts.join('\n') + '|' + digest.assistantTexts.join('\n'))
|
|
32
|
+
.digest('hex')
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Capture one session incrementally.
|
|
37
|
+
* @param store - memory store.
|
|
38
|
+
* @param services - { sessionQuery, llm? }; llm(prompt) summarizes, omit for deterministic notes.
|
|
39
|
+
* @param session - the live session (id + header).
|
|
40
|
+
* @param options - capture budgets (see defaults.js).
|
|
41
|
+
* @returns {Promise<{sessionId: string, tier: string, changed: boolean, fresh: number, summarySource: string, bullets: number}>}
|
|
42
|
+
*/
|
|
43
|
+
/** Does the queue already track this candidate? */
|
|
44
|
+
function queueHas(root, id) { return readQueue(root).has(id) }
|
|
45
|
+
|
|
46
|
+
export async function captureSession(store, services, session, options = {}) {
|
|
47
|
+
const cfg = {
|
|
48
|
+
maxUserTurns: 60, maxAssistantGists: 40, maxMessageChars: 600,
|
|
49
|
+
maxGoalChars: 200, maxCheckpoints: 4, maxBytes: 40000,
|
|
50
|
+
maxBullets: 16, maxBulletChars: 2000, summaryMinSignalChars: 400,
|
|
51
|
+
now: () => new Date().toISOString(),
|
|
52
|
+
...options,
|
|
53
|
+
}
|
|
54
|
+
let events = []
|
|
55
|
+
try {
|
|
56
|
+
const snapshot = await services?.sessionQuery?.readSession(session.id)
|
|
57
|
+
events = snapshot?.events ?? []
|
|
58
|
+
} catch { events = [] }
|
|
59
|
+
|
|
60
|
+
// Read the queue entry directly. The previous record shape is a projection over the queue, so
|
|
61
|
+
// asking for it here meant projecting every candidate to answer a question about one.
|
|
62
|
+
const prevEntry = readQueue(store.root).get(session.id)
|
|
63
|
+
const prev = {
|
|
64
|
+
digestFormat: prevEntry?.detail?.digestFormat,
|
|
65
|
+
lastSeq: prevEntry?.detail?.lastSeq,
|
|
66
|
+
summaryAttempts: prevEntry?.attempts,
|
|
67
|
+
}
|
|
68
|
+
const formatChanged = prev.digestFormat !== DIGEST_FORMAT
|
|
69
|
+
const watermark = (!formatChanged && typeof prev.lastSeq === 'number') ? prev.lastSeq : -1
|
|
70
|
+
const fresh = events.filter((event, index) => seqOf(event, index) > watermark)
|
|
71
|
+
const logEnd = events.length === 0 ? watermark : seqOf(events[events.length - 1], events.length - 1)
|
|
72
|
+
// The watermark advances only over material that was actually CONSUMED. A trivial ping is
|
|
73
|
+
// dropped outright, but a thin delta is provisional: it is below the signal floor now and may
|
|
74
|
+
// not be later, so it stays in the window and accumulates with the next delta. Advancing past
|
|
75
|
+
// it would skip that text forever, which is how content quietly disappeared.
|
|
76
|
+
// Material is CONSUMED only when it was actually dealt with: summarized, or deliberately
|
|
77
|
+
// concluded to hold nothing durable. A thin delta accumulates, and a FAILED summary keeps its
|
|
78
|
+
// material so the retry has something to retry - consuming it would leave the retry with no
|
|
79
|
+
// input, which is a silent no-op dressed up as a retry. After the cap the failure is accepted
|
|
80
|
+
// and the window advances, so a permanently broken session cannot block progress forever.
|
|
81
|
+
// A thin delta is a DELIBERATE skip, so it counts as dealt with; only a FAILURE is re-presented.
|
|
82
|
+
const consumes = (failed, givingUp) => !failed || givingUp
|
|
83
|
+
|
|
84
|
+
// Tier is decided once, on the first capture, from the whole session; later passes
|
|
85
|
+
// reuse it so a long session is never re-projected wholesale.
|
|
86
|
+
const tier = prev.tier ?? classifyJunk(projectDigest({
|
|
87
|
+
sessionId: session.id, createdAt: session.header?.createdAt, cwd: session.header?.cwd,
|
|
88
|
+
}, events))
|
|
89
|
+
|
|
90
|
+
if (fresh.length === 0) {
|
|
91
|
+
return { sessionId: session.id, tier, changed: false, fresh: 0, summarySource: 'none', bullets: 0 }
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const freshDigest = projectDigest({
|
|
95
|
+
sessionId: session.id, createdAt: session.header?.createdAt, cwd: session.header?.cwd,
|
|
96
|
+
}, fresh)
|
|
97
|
+
const toolCallCount = (prev.toolCallCount ?? 0) + totalToolCalls(freshDigest.toolTally)
|
|
98
|
+
const meta = {
|
|
99
|
+
sessionId: session.id,
|
|
100
|
+
title: freshDigest.title || prev.title || session.header?.title || '',
|
|
101
|
+
createdAt: session.header?.createdAt,
|
|
102
|
+
cwd: session.header?.cwd,
|
|
103
|
+
date: dateOf(session.header?.createdAt),
|
|
104
|
+
eventCount: events.length,
|
|
105
|
+
toolCallCount,
|
|
106
|
+
capturedAt: cfg.now(),
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
mkdirSync(join(store.root, 'digests'), { recursive: true })
|
|
110
|
+
const path = digestPath(store.root, session.id)
|
|
111
|
+
if (!existsSync(path) || formatChanged) writeFileSync(path, renderDigestHeader(meta) + '\n', 'utf8')
|
|
112
|
+
|
|
113
|
+
// Signal check: how much NEW material would a summary actually see? Counting the FILTERED
|
|
114
|
+
// candidates, not raw events, so a burst of tool noise cannot masquerade as content.
|
|
115
|
+
const candidateSet = tier === "trivial" ? undefined : extractCandidates(freshDigest, cfg)
|
|
116
|
+
const signalChars = candidateSet
|
|
117
|
+
? candidateSet.userTurns.reduce((n, text) => n + text.length, 0)
|
|
118
|
+
+ candidateSet.assistantGists.reduce((n, text) => n + text.length, 0)
|
|
119
|
+
: 0
|
|
120
|
+
const enoughSignal = signalChars >= (cfg.summaryMinSignalChars ?? 0)
|
|
121
|
+
|
|
122
|
+
// A ping has nothing durable to find, and a thin delta is not worth a model call either.
|
|
123
|
+
const summary = tier === "trivial"
|
|
124
|
+
? { bullets: [], source: "skipped" }
|
|
125
|
+
: (enoughSignal
|
|
126
|
+
? await summarizeSession(services?.llm, candidateSet, { date: meta.date }, cfg)
|
|
127
|
+
: { bullets: [], source: "thin" })
|
|
128
|
+
if (summary.bullets.length > 0) {
|
|
129
|
+
appendFileSync(path, '\n' + renderSummarySection(summary.bullets, { stamp: meta.capturedAt, source: summary.source }), 'utf8')
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// SETTLED versus FAILED. Nothing durable written is only a conclusion when the model SAID
|
|
133
|
+
// so (NO_DURABLE_CONTENT), or when the session was never worth a call at all. An empty or
|
|
134
|
+
// unparseable reply is a FAILURE: it stays pending so the next pass retries it. A failed
|
|
135
|
+
// summary is retried at most summaryMaxAttempts times before it is laid to rest as failed
|
|
136
|
+
// instead of being retried forever.
|
|
137
|
+
// SETTLED versus FAILED. Nothing durable written is only a conclusion when the model SAID so
|
|
138
|
+
// (NO_DURABLE_CONTENT), or when the session was never worth a call at all. An empty or
|
|
139
|
+
// unparseable reply is a FAILURE, and a failure is retried rather than settled.
|
|
140
|
+
//
|
|
141
|
+
// This writes TRANSITIONS now, not flags. The state IS the answer to what happened, so no
|
|
142
|
+
// reader has to derive it: a deliberate skip, a defect and a finished job are three states,
|
|
143
|
+
// not three combinations of a timestamp, a source string and a counter.
|
|
144
|
+
const summaryFailed = summary.bullets.length === 0
|
|
145
|
+
&& (summary.reason === 'empty-reply' || summary.reason === 'unparseable' || summary.reason === 'error')
|
|
146
|
+
const giveUp = summaryFailed && (Number(prev.summaryAttempts ?? 0) + 1) >= (cfg.summaryMaxAttempts ?? 2)
|
|
147
|
+
// The watermark advances only over material that was DEALT WITH. A failure keeps its material
|
|
148
|
+
// so the retry has something to retry; a deliberate skip consumes it.
|
|
149
|
+
const consumed = !summaryFailed || giveUp
|
|
150
|
+
const detail = {
|
|
151
|
+
tier,
|
|
152
|
+
title: meta.title,
|
|
153
|
+
capturedAt: meta.capturedAt,
|
|
154
|
+
digestFormat: DIGEST_FORMAT,
|
|
155
|
+
lastSeq: consumed ? logEnd : watermark,
|
|
156
|
+
digestHash: digestHash(freshDigest),
|
|
157
|
+
eventCount: events.length,
|
|
158
|
+
toolCallCount,
|
|
159
|
+
};
|
|
160
|
+
// Which state this pass produced, and why. A stopping state always carries a reason, because
|
|
161
|
+
// a decision and a defect must be told apart by a reader who was not here.
|
|
162
|
+
let outcome;
|
|
163
|
+
if (tier !== 'content') outcome = { state: STATES.SKIPPED, reason: REASONS.TRIVIAL };
|
|
164
|
+
else if (summary.source === 'thin') outcome = { state: STATES.SKIPPED, reason: REASONS.THIN };
|
|
165
|
+
else if (summary.bullets.length > 0) outcome = { state: STATES.SUMMARIZED };
|
|
166
|
+
else if (summary.reason === REASONS.NO_DURABLE) outcome = { state: STATES.SKIPPED, reason: REASONS.NO_DURABLE };
|
|
167
|
+
else if (summaryFailed) {
|
|
168
|
+
outcome = {
|
|
169
|
+
state: STATES.FAILED,
|
|
170
|
+
reason: summary.reason === 'error' ? REASONS.CALL_FAILED : summary.reason,
|
|
171
|
+
error: summary.error,
|
|
172
|
+
};
|
|
173
|
+
} else outcome = { state: STATES.SKIPPED, reason: REASONS.UNCLASSIFIED };
|
|
174
|
+
|
|
175
|
+
// A new candidate is seeded PENDING and then walks the real path. Recording the intermediate
|
|
176
|
+
// steps is not ceremony: the summarization genuinely ran, and a failure that appears straight
|
|
177
|
+
// from pending would describe something that never happened.
|
|
178
|
+
if (!queueHas(store.root, session.id)) seed(store.root, session.id, STATES.PENDING, { detail });
|
|
179
|
+
advance(store.root, session.id, STATES.SUMMARIZING, { detail });
|
|
180
|
+
if (outcome.state === STATES.FAILED) {
|
|
181
|
+
// fail() schedules the next attempt and enforces the cap itself, so a caller cannot leave
|
|
182
|
+
// a candidate failed forever by forgetting to check it.
|
|
183
|
+
fail(store.root, session.id, { reason: outcome.reason, error: outcome.error });
|
|
184
|
+
} else {
|
|
185
|
+
advance(store.root, session.id, outcome.state, { reason: outcome.reason, attempts: 0, detail });
|
|
186
|
+
}
|
|
187
|
+
// A successful summary clears the attempt budget: the failure is behind it.
|
|
188
|
+
if (outcome.state === STATES.SUMMARIZED) advance(store.root, session.id, STATES.SUMMARIZED, { attempts: 0, detail });
|
|
189
|
+
return {
|
|
190
|
+
sessionId: session.id, tier, changed: true, fresh: fresh.length,
|
|
191
|
+
summarySource: summary.source, bullets: summary.bullets.length,
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/**
|
|
196
|
+
* Prune captured digests. Digests are raw session material and gitignored, but one file per
|
|
197
|
+
* session accumulates forever. Only digests that have already been consolidated AND are beyond
|
|
198
|
+
* the newest `keep` are removed: a pending digest is the only copy of work not yet in the store.
|
|
199
|
+
* @returns {{removed: number, kept: number, pending: number}}
|
|
200
|
+
*/
|
|
201
|
+
export function pruneDigests(store, options = {}) {
|
|
202
|
+
const keep = options.keep ?? 50
|
|
203
|
+
// A digest may only be removed when the work is DEALT WITH: while a candidate is waiting, its
|
|
204
|
+
// digest is the only copy of material that is not in the store yet.
|
|
205
|
+
const entries = [...readQueue(store.root).values()]
|
|
206
|
+
.map((entry) => ({
|
|
207
|
+
id: entry.id,
|
|
208
|
+
settled: isSettled(entry),
|
|
209
|
+
mtime: statSafe(digestPath(store.root, entry.id)),
|
|
210
|
+
capturedAt: Date.parse(String(entry.detail?.capturedAt ?? '')) || 0,
|
|
211
|
+
}))
|
|
212
|
+
// "Newest" must not depend on file mtime alone: several digests written in the same tick
|
|
213
|
+
// are otherwise indistinguishable, and a same-instant test (or a fast machine) would prune
|
|
214
|
+
// the wrong one. capturedAt is the record of when the session was captured; mtime breaks
|
|
215
|
+
// ties when records are missing it.
|
|
216
|
+
.sort((a, b) => (b.capturedAt - a.capturedAt) || ((b.mtime ?? 0) - (a.mtime ?? 0)))
|
|
217
|
+
let removed = 0
|
|
218
|
+
let pending = 0
|
|
219
|
+
for (const [index, item] of entries.entries()) {
|
|
220
|
+
if (!item.settled) { pending += 1; continue }
|
|
221
|
+
if (index < keep) continue
|
|
222
|
+
try {
|
|
223
|
+
rmSync(digestPath(store.root, item.id))
|
|
224
|
+
removed += 1
|
|
225
|
+
} catch {}
|
|
226
|
+
}
|
|
227
|
+
return { removed, kept: Math.max(0, entries.length - removed - pending), pending }
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/** Modification time of a file, or undefined when it is absent. */
|
|
231
|
+
function statSafe(file) {
|
|
232
|
+
try { return statSync(file).mtimeMs } catch { return undefined }
|
|
233
|
+
}
|
|
234
|
+
/** Sessions captured but not yet consolidated (oldest-first is handled by the caller). */
|
|
235
|
+
export function pendingDigests(store) {
|
|
236
|
+
// Waiting work, which is exactly what the queue calls PENDING or SUMMARIZED - the same set the
|
|
237
|
+
// cadence treats as work. Neither a failure on its backoff nor an orphan belongs here: those are
|
|
238
|
+
// different actions, and folding them in is what made 78 orphans look like 78 retries.
|
|
239
|
+
return [...readQueue(store.root).values()]
|
|
240
|
+
.filter((entry) => entry.state === STATES.PENDING || entry.state === STATES.SUMMARIZED)
|
|
241
|
+
.map((entry) => ({ id: entry.id, tier: entry.detail?.tier, digestHash: entry.detail?.digestHash }))
|
|
242
|
+
}
|