@aexol/spectral 0.9.16 → 0.9.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/memory/compaction.d.ts +1 -1
- package/dist/memory/compaction.js +3 -3
- package/dist/memory/deterministic-pruner.d.ts +65 -0
- package/dist/memory/deterministic-pruner.d.ts.map +1 -0
- package/dist/memory/deterministic-pruner.js +333 -0
- package/dist/memory/hooks/compaction-hook.d.ts.map +1 -1
- package/dist/memory/hooks/compaction-hook.js +158 -132
- package/dist/memory/prompts.d.ts.map +1 -1
- package/dist/memory/prompts.js +9 -0
- package/dist/memory/unified-compaction.d.ts +59 -0
- package/dist/memory/unified-compaction.d.ts.map +1 -0
- package/dist/memory/unified-compaction.js +332 -0
- package/package.json +1 -1
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { agentLoop } from "../sdk/agent-core/index.js";
|
|
2
2
|
import { type Model } from "../sdk/ai/index.js";
|
|
3
3
|
import type { MemoryReflection, ObservationRecord } from "./types.js";
|
|
4
|
-
export declare const REFLECTOR_MAX_PASSES =
|
|
4
|
+
export declare const REFLECTOR_MAX_PASSES = 1;
|
|
5
5
|
export declare const PRUNER_MAX_PASSES = 2;
|
|
6
6
|
export declare function observationPoolTokens(observations: ObservationRecord[]): number;
|
|
7
7
|
interface LlmArgs {
|
|
@@ -8,8 +8,8 @@ import { buildPrunerPassGuidance, buildReflectorPassGuidance, CONTEXT_USAGE_INST
|
|
|
8
8
|
import { truncateRecordContent } from "./serialize.js";
|
|
9
9
|
import { estimateStringTokens } from "./tokens.js";
|
|
10
10
|
import { reflectionContent, reflectionToPromptLine } from "./types.js";
|
|
11
|
-
export const REFLECTOR_MAX_PASSES =
|
|
12
|
-
export const PRUNER_MAX_PASSES = 2;
|
|
11
|
+
export const REFLECTOR_MAX_PASSES = 1;
|
|
12
|
+
export const PRUNER_MAX_PASSES = 2; // Kept for compatibility; deterministic pruner replaces this
|
|
13
13
|
const PRUNER_TARGET_RATIO = 0.8;
|
|
14
14
|
export function observationPoolTokens(observations) {
|
|
15
15
|
return estimateStringTokens(observationsToPromptLines(observations).join("\n"));
|
|
@@ -293,7 +293,7 @@ function reflectorPassContext(pass) {
|
|
|
293
293
|
return {
|
|
294
294
|
pass,
|
|
295
295
|
maxPasses: REFLECTOR_MAX_PASSES,
|
|
296
|
-
minSupportingObservationIds: pass === 1 ? 2 : 1,
|
|
296
|
+
minSupportingObservationIds: REFLECTOR_MAX_PASSES === 1 ? 1 : (pass === 1 ? 2 : 1),
|
|
297
297
|
};
|
|
298
298
|
}
|
|
299
299
|
function reflectionContentKey(reflection) {
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic Pruner — replaces the 2-pass LLM-based pruner with a
|
|
3
|
+
* sorting algorithm that drops observations based on coverage, relevance,
|
|
4
|
+
* and age.
|
|
5
|
+
*
|
|
6
|
+
* The algorithm mirrors the LLM pruner's strategy:
|
|
7
|
+
* 1. Coverage-tagged observations (reinforced by reflections) drop first
|
|
8
|
+
* 2. Within same coverage: low > medium > high relevance
|
|
9
|
+
* 3. Within same relevance: oldest first (age-gradient rule)
|
|
10
|
+
* 4. Protected observations NEVER drop (critical, user assertions, completions)
|
|
11
|
+
*
|
|
12
|
+
* This eliminates ~2 LLM calls per compaction while producing equivalent results.
|
|
13
|
+
*/
|
|
14
|
+
import type { MemoryReflection, ObservationRecord } from "./types.js";
|
|
15
|
+
export type CoverageTag = "reinforced" | "cited" | "uncited";
|
|
16
|
+
export interface TaggedObservation {
|
|
17
|
+
observation: ObservationRecord;
|
|
18
|
+
coverage: CoverageTag;
|
|
19
|
+
/** Number of reflections that cite this observation */
|
|
20
|
+
citationCount: number;
|
|
21
|
+
/** True if this observation should never be dropped */
|
|
22
|
+
protected: boolean;
|
|
23
|
+
/** Reason for protection (for debugging) */
|
|
24
|
+
protectedReason?: string;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Compute coverage tags for all observations based on reflection citations.
|
|
28
|
+
* - reinforced: ≥4 reflections cite this observation
|
|
29
|
+
* - cited: 1-3 reflections cite this observation
|
|
30
|
+
* - uncited: no reflection cites this observation
|
|
31
|
+
*/
|
|
32
|
+
export declare function computeCoverageTags(observations: ObservationRecord[], reflections: MemoryReflection[]): Map<string, CoverageTag>;
|
|
33
|
+
/**
|
|
34
|
+
* Tag observations with protection status.
|
|
35
|
+
*/
|
|
36
|
+
export declare function tagProtectedObservations(observations: ObservationRecord[]): Map<string, {
|
|
37
|
+
protected: boolean;
|
|
38
|
+
reason?: string;
|
|
39
|
+
}>;
|
|
40
|
+
export interface PrunerResult {
|
|
41
|
+
/** Kept observations */
|
|
42
|
+
observations: ObservationRecord[];
|
|
43
|
+
/** IDs of dropped observations */
|
|
44
|
+
droppedIds: string[];
|
|
45
|
+
/** Whether any protected observations were at risk of being dropped */
|
|
46
|
+
fellBack: boolean;
|
|
47
|
+
/** Tokens before pruning */
|
|
48
|
+
tokensBefore: number;
|
|
49
|
+
/** Tokens after pruning */
|
|
50
|
+
tokensAfter: number;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Deterministically prune an observation pool to fit within a token budget.
|
|
54
|
+
*
|
|
55
|
+
* Algorithm:
|
|
56
|
+
* 1. Compute coverage tags from reflection citations
|
|
57
|
+
* 2. Detect protected observations (critical, assertions, completions, errors, unique IDs)
|
|
58
|
+
* 3. Sort by droppability (reinforced+low+oldest first)
|
|
59
|
+
* 4. Drop from the sorted list until pool ≤ targetTokens
|
|
60
|
+
* 5. Never drop protected observations
|
|
61
|
+
*
|
|
62
|
+
* Target is 80% of the budget to leave breathing room for the summary text.
|
|
63
|
+
*/
|
|
64
|
+
export declare function deterministicPrune(observations: ObservationRecord[], reflections: MemoryReflection[], budgetTokens: number): PrunerResult;
|
|
65
|
+
//# sourceMappingURL=deterministic-pruner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"deterministic-pruner.d.ts","sourceRoot":"","sources":["../../src/memory/deterministic-pruner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAStE,MAAM,MAAM,WAAW,GAAG,YAAY,GAAG,OAAO,GAAG,SAAS,CAAC;AAE7D,MAAM,WAAW,iBAAiB;IACjC,WAAW,EAAE,iBAAiB,CAAC;IAC/B,QAAQ,EAAE,WAAW,CAAC;IACtB,uDAAuD;IACvD,aAAa,EAAE,MAAM,CAAC;IACtB,uDAAuD;IACvD,SAAS,EAAE,OAAO,CAAC;IACnB,4CAA4C;IAC5C,eAAe,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAClC,YAAY,EAAE,iBAAiB,EAAE,EACjC,WAAW,EAAE,gBAAgB,EAAE,GAC7B,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,CA2B1B;AAuHD;;GAEG;AACH,wBAAgB,wBAAwB,CACvC,YAAY,EAAE,iBAAiB,EAAE,GAC/B,GAAG,CAAC,MAAM,EAAE;IAAE,SAAS,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CA0CtD;AAoDD,MAAM,WAAW,YAAY;IAC5B,wBAAwB;IACxB,YAAY,EAAE,iBAAiB,EAAE,CAAC;IAClC,kCAAkC;IAClC,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,uEAAuE;IACvE,QAAQ,EAAE,OAAO,CAAC;IAClB,4BAA4B;IAC5B,YAAY,EAAE,MAAM,CAAC;IACrB,2BAA2B;IAC3B,WAAW,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,kBAAkB,CACjC,YAAY,EAAE,iBAAiB,EAAE,EACjC,WAAW,EAAE,gBAAgB,EAAE,EAC/B,YAAY,EAAE,MAAM,GAClB,YAAY,CA8Fd"}
|
|
@@ -0,0 +1,333 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic Pruner — replaces the 2-pass LLM-based pruner with a
|
|
3
|
+
* sorting algorithm that drops observations based on coverage, relevance,
|
|
4
|
+
* and age.
|
|
5
|
+
*
|
|
6
|
+
* The algorithm mirrors the LLM pruner's strategy:
|
|
7
|
+
* 1. Coverage-tagged observations (reinforced by reflections) drop first
|
|
8
|
+
* 2. Within same coverage: low > medium > high relevance
|
|
9
|
+
* 3. Within same relevance: oldest first (age-gradient rule)
|
|
10
|
+
* 4. Protected observations NEVER drop (critical, user assertions, completions)
|
|
11
|
+
*
|
|
12
|
+
* This eliminates ~2 LLM calls per compaction while producing equivalent results.
|
|
13
|
+
*/
|
|
14
|
+
import { estimateStringTokens } from "./tokens.js";
|
|
15
|
+
import { debugLog, isDebugLogEnabled } from "./debug-log.js";
|
|
16
|
+
/**
|
|
17
|
+
* Compute coverage tags for all observations based on reflection citations.
|
|
18
|
+
* - reinforced: ≥4 reflections cite this observation
|
|
19
|
+
* - cited: 1-3 reflections cite this observation
|
|
20
|
+
* - uncited: no reflection cites this observation
|
|
21
|
+
*/
|
|
22
|
+
export function computeCoverageTags(observations, reflections) {
|
|
23
|
+
const citationCounts = new Map();
|
|
24
|
+
// Initialize counts
|
|
25
|
+
for (const obs of observations) {
|
|
26
|
+
citationCounts.set(obs.id, 0);
|
|
27
|
+
}
|
|
28
|
+
// Count citations from reflections
|
|
29
|
+
for (const reflection of reflections) {
|
|
30
|
+
if (typeof reflection === "string" || reflection.legacy === true)
|
|
31
|
+
continue;
|
|
32
|
+
for (const obsId of reflection.supportingObservationIds) {
|
|
33
|
+
const count = citationCounts.get(obsId);
|
|
34
|
+
if (count !== undefined) {
|
|
35
|
+
citationCounts.set(obsId, count + 1);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
// Compute tags
|
|
40
|
+
const tags = new Map();
|
|
41
|
+
for (const obs of observations) {
|
|
42
|
+
const count = citationCounts.get(obs.id) ?? 0;
|
|
43
|
+
tags.set(obs.id, count >= 4 ? "reinforced" : count >= 1 ? "cited" : "uncited");
|
|
44
|
+
}
|
|
45
|
+
return tags;
|
|
46
|
+
}
|
|
47
|
+
// ============================================================================
|
|
48
|
+
// Protection Detection
|
|
49
|
+
// ============================================================================
|
|
50
|
+
/** Patterns indicating a user assertion (identity, preference, constraint). */
|
|
51
|
+
const USER_ASSERTION_PATTERNS = [
|
|
52
|
+
/\buser\s+(stated|said|mentioned|confirmed|reported|asserted|told|specified)\b/i,
|
|
53
|
+
/\buser\s+(prefers|prefer|wants|needs|requires|insists|demands)\b/i,
|
|
54
|
+
/\buser\s+(is|has|was|will|can|does)\b/i,
|
|
55
|
+
/\buser\s+(decided|chose|picked|selected|opted)\b/i,
|
|
56
|
+
];
|
|
57
|
+
/** Patterns indicating a concrete completion that should not be redone. */
|
|
58
|
+
const COMPLETION_PATTERNS = [
|
|
59
|
+
/\bcompleted[:\s]/i,
|
|
60
|
+
/\bresolved[:\s]/i,
|
|
61
|
+
/\bconfirmed working\b/i,
|
|
62
|
+
/\bfinished[:\s]/i,
|
|
63
|
+
/\bimplemented[:\s]/i,
|
|
64
|
+
/\bmerged[:\s]/i,
|
|
65
|
+
/\bdeployed[:\s]/i,
|
|
66
|
+
/\bfix(ed)?[:\s]/i,
|
|
67
|
+
];
|
|
68
|
+
/** Patterns indicating verbatim error messages. */
|
|
69
|
+
const ERROR_PATTERNS = [
|
|
70
|
+
/\bTS\d{4,5}\b/, // TypeScript error codes
|
|
71
|
+
/\bE[A-Z]+\d+\b/, // Generic error codes
|
|
72
|
+
/\bError:\s/i,
|
|
73
|
+
/\bfailed[:\s]/i,
|
|
74
|
+
/\bpanic[:\s]/i,
|
|
75
|
+
/\bstack trace\b/i,
|
|
76
|
+
/\bat\s+\S+:\d+:\d+\b/, // Stack trace lines
|
|
77
|
+
/\bCannot\s+(find|read|write|open|access|parse|resolve)\b/i,
|
|
78
|
+
/\bnot\s+(found|supported|allowed|permitted|valid)\b/i,
|
|
79
|
+
];
|
|
80
|
+
/**
|
|
81
|
+
* Detect if an observation content contains a user assertion.
|
|
82
|
+
*/
|
|
83
|
+
function isUserAssertion(content) {
|
|
84
|
+
return USER_ASSERTION_PATTERNS.some((p) => p.test(content));
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Detect if an observation content marks a concrete completion.
|
|
88
|
+
*/
|
|
89
|
+
function isCompletion(content) {
|
|
90
|
+
return COMPLETION_PATTERNS.some((p) => p.test(content));
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Detect if an observation content contains a verbatim error.
|
|
94
|
+
*/
|
|
95
|
+
function isVerbatimError(content) {
|
|
96
|
+
return ERROR_PATTERNS.some((p) => p.test(content));
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Detect if an observation contains unique identifiers that appear nowhere else.
|
|
100
|
+
* Checks for file paths, package names, commit SHAs, ticket IDs, error codes.
|
|
101
|
+
*/
|
|
102
|
+
function hasUniqueIdentifiers(observation, allObservations) {
|
|
103
|
+
// Extract potential identifiers
|
|
104
|
+
const identifiers = extractIdentifiers(observation.content);
|
|
105
|
+
if (identifiers.length === 0)
|
|
106
|
+
return false;
|
|
107
|
+
// Check if ANY of these identifiers appear in other observations
|
|
108
|
+
for (const id of identifiers) {
|
|
109
|
+
let foundElsewhere = false;
|
|
110
|
+
for (const other of allObservations) {
|
|
111
|
+
if (other.id === observation.id)
|
|
112
|
+
continue;
|
|
113
|
+
if (other.content.includes(id)) {
|
|
114
|
+
foundElsewhere = true;
|
|
115
|
+
break;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
if (!foundElsewhere)
|
|
119
|
+
return true; // This identifier is unique to this observation
|
|
120
|
+
}
|
|
121
|
+
return false;
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Extract potential identifiers from observation content.
|
|
125
|
+
* Looks for: file paths, package names, commit SHAs, ticket IDs, error codes.
|
|
126
|
+
*/
|
|
127
|
+
function extractIdentifiers(content) {
|
|
128
|
+
const ids = [];
|
|
129
|
+
// File paths: /path/to/file.ext or relative/path/file.ext or src/file.ts
|
|
130
|
+
const pathMatches = content.match(/\b[\w./-]+\.[\w]{1,10}\b/g);
|
|
131
|
+
if (pathMatches)
|
|
132
|
+
ids.push(...pathMatches);
|
|
133
|
+
// Package names: @scope/name or package-name
|
|
134
|
+
const pkgMatches = content.match(/@[\w-]+\/[\w-]+|\b[\w-]+-[\w-]+(?:\/[\w-]+)*\b/g);
|
|
135
|
+
if (pkgMatches)
|
|
136
|
+
ids.push(...pkgMatches);
|
|
137
|
+
// Commit SHAs: 7-40 hex chars
|
|
138
|
+
const shaMatches = content.match(/\b[a-f0-9]{7,40}\b/gi);
|
|
139
|
+
if (shaMatches)
|
|
140
|
+
ids.push(...shaMatches);
|
|
141
|
+
// Ticket IDs: PROJ-123 or #1234
|
|
142
|
+
const ticketMatches = content.match(/\b[A-Z]{2,}-\d+\b|#\d{3,}\b/g);
|
|
143
|
+
if (ticketMatches)
|
|
144
|
+
ids.push(...ticketMatches);
|
|
145
|
+
// Error codes: TS2322, ENOENT, etc.
|
|
146
|
+
const errorCodeMatches = content.match(/\b(?:TS|E[A-Z])\d{2,}\b/g);
|
|
147
|
+
if (errorCodeMatches)
|
|
148
|
+
ids.push(...errorCodeMatches);
|
|
149
|
+
// Deduplicate and filter very short/generic identifiers
|
|
150
|
+
return [...new Set(ids)].filter((id) => id.length > 4 && !/^[\d.]+$/.test(id));
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* Tag observations with protection status.
|
|
154
|
+
*/
|
|
155
|
+
export function tagProtectedObservations(observations) {
|
|
156
|
+
const protected_ = new Map();
|
|
157
|
+
for (const obs of observations) {
|
|
158
|
+
// Critical relevance is always protected
|
|
159
|
+
if (obs.relevance === "critical") {
|
|
160
|
+
protected_.set(obs.id, { protected: true, reason: "critical relevance" });
|
|
161
|
+
continue;
|
|
162
|
+
}
|
|
163
|
+
// User assertions about identity/preferences/constraints
|
|
164
|
+
if (isUserAssertion(obs.content)) {
|
|
165
|
+
protected_.set(obs.id, { protected: true, reason: "user assertion" });
|
|
166
|
+
continue;
|
|
167
|
+
}
|
|
168
|
+
// Concrete completions
|
|
169
|
+
if (isCompletion(obs.content)) {
|
|
170
|
+
protected_.set(obs.id, { protected: true, reason: "completion marker" });
|
|
171
|
+
continue;
|
|
172
|
+
}
|
|
173
|
+
// Verbatim errors
|
|
174
|
+
if (isVerbatimError(obs.content)) {
|
|
175
|
+
protected_.set(obs.id, { protected: true, reason: "verbatim error" });
|
|
176
|
+
continue;
|
|
177
|
+
}
|
|
178
|
+
// Unique identifiers (will be checked below)
|
|
179
|
+
protected_.set(obs.id, { protected: false });
|
|
180
|
+
}
|
|
181
|
+
// Second pass: check for unique identifiers across all observations
|
|
182
|
+
for (const obs of observations) {
|
|
183
|
+
const status = protected_.get(obs.id);
|
|
184
|
+
if (status?.protected)
|
|
185
|
+
continue; // Already protected
|
|
186
|
+
if (hasUniqueIdentifiers(obs, observations)) {
|
|
187
|
+
protected_.set(obs.id, { protected: true, reason: "unique identifiers" });
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
return protected_;
|
|
191
|
+
}
|
|
192
|
+
// ============================================================================
|
|
193
|
+
// Sorting & Pruning
|
|
194
|
+
// ============================================================================
|
|
195
|
+
const COVERAGE_PRIORITY = {
|
|
196
|
+
reinforced: 0, // Drop first
|
|
197
|
+
cited: 1,
|
|
198
|
+
uncited: 2, // Drop last (among non-protected)
|
|
199
|
+
};
|
|
200
|
+
const RELEVANCE_PRIORITY = {
|
|
201
|
+
low: 0,
|
|
202
|
+
medium: 1,
|
|
203
|
+
high: 2,
|
|
204
|
+
critical: 3,
|
|
205
|
+
};
|
|
206
|
+
/**
|
|
207
|
+
* Sort observations for pruning: most droppable first, protected last.
|
|
208
|
+
*
|
|
209
|
+
* Priority order (ascending = drop first):
|
|
210
|
+
* 1. Coverage: reinforced → cited → uncited
|
|
211
|
+
* 2. Relevance: low → medium → high → critical
|
|
212
|
+
* 3. Age: oldest → newest
|
|
213
|
+
* 4. Protected: always last
|
|
214
|
+
*/
|
|
215
|
+
function sortForPruning(tagged) {
|
|
216
|
+
return [...tagged].sort((a, b) => {
|
|
217
|
+
// Protected always last
|
|
218
|
+
if (a.protected !== b.protected) {
|
|
219
|
+
return a.protected ? 1 : -1;
|
|
220
|
+
}
|
|
221
|
+
// Coverage priority
|
|
222
|
+
const covA = COVERAGE_PRIORITY[a.coverage];
|
|
223
|
+
const covB = COVERAGE_PRIORITY[b.coverage];
|
|
224
|
+
if (covA !== covB)
|
|
225
|
+
return covA - covB;
|
|
226
|
+
// Relevance priority
|
|
227
|
+
const relA = RELEVANCE_PRIORITY[a.observation.relevance] ?? 1;
|
|
228
|
+
const relB = RELEVANCE_PRIORITY[b.observation.relevance] ?? 1;
|
|
229
|
+
if (relA !== relB)
|
|
230
|
+
return relA - relB;
|
|
231
|
+
// Age: oldest first (lexicographic timestamp comparison)
|
|
232
|
+
const timeA = a.observation.timestamp ?? "";
|
|
233
|
+
const timeB = b.observation.timestamp ?? "";
|
|
234
|
+
return timeA.localeCompare(timeB);
|
|
235
|
+
});
|
|
236
|
+
}
|
|
237
|
+
/**
|
|
238
|
+
* Deterministically prune an observation pool to fit within a token budget.
|
|
239
|
+
*
|
|
240
|
+
* Algorithm:
|
|
241
|
+
* 1. Compute coverage tags from reflection citations
|
|
242
|
+
* 2. Detect protected observations (critical, assertions, completions, errors, unique IDs)
|
|
243
|
+
* 3. Sort by droppability (reinforced+low+oldest first)
|
|
244
|
+
* 4. Drop from the sorted list until pool ≤ targetTokens
|
|
245
|
+
* 5. Never drop protected observations
|
|
246
|
+
*
|
|
247
|
+
* Target is 80% of the budget to leave breathing room for the summary text.
|
|
248
|
+
*/
|
|
249
|
+
export function deterministicPrune(observations, reflections, budgetTokens) {
|
|
250
|
+
if (observations.length === 0) {
|
|
251
|
+
return {
|
|
252
|
+
observations: [],
|
|
253
|
+
droppedIds: [],
|
|
254
|
+
fellBack: false,
|
|
255
|
+
tokensBefore: 0,
|
|
256
|
+
tokensAfter: 0,
|
|
257
|
+
};
|
|
258
|
+
}
|
|
259
|
+
const targetTokens = Math.max(1, Math.floor(budgetTokens * 0.8));
|
|
260
|
+
const coverageTags = computeCoverageTags(observations, reflections);
|
|
261
|
+
const protectionStatus = tagProtectedObservations(observations);
|
|
262
|
+
// Build tagged observations
|
|
263
|
+
const tagged = observations.map((obs) => {
|
|
264
|
+
const coverage = coverageTags.get(obs.id) ?? "uncited";
|
|
265
|
+
const citationCount = [...reflections]
|
|
266
|
+
.filter((r) => typeof r !== "string" && !r.legacy)
|
|
267
|
+
.reduce((sum, r) => {
|
|
268
|
+
if (typeof r === "string")
|
|
269
|
+
return sum;
|
|
270
|
+
return sum + (r.supportingObservationIds.includes(obs.id) ? 1 : 0);
|
|
271
|
+
}, 0);
|
|
272
|
+
const prot = protectionStatus.get(obs.id);
|
|
273
|
+
return {
|
|
274
|
+
observation: obs,
|
|
275
|
+
coverage,
|
|
276
|
+
citationCount,
|
|
277
|
+
protected: prot?.protected ?? false,
|
|
278
|
+
protectedReason: prot?.reason,
|
|
279
|
+
};
|
|
280
|
+
});
|
|
281
|
+
const tokensBefore = observations.reduce((sum, o) => sum + estimateStringTokens(o.content), 0);
|
|
282
|
+
// If already under budget, no pruning needed
|
|
283
|
+
if (tokensBefore <= targetTokens) {
|
|
284
|
+
debugLog("deterministic_pruner.under_target", {
|
|
285
|
+
tokensBefore,
|
|
286
|
+
targetTokens,
|
|
287
|
+
observationCount: observations.length,
|
|
288
|
+
});
|
|
289
|
+
return {
|
|
290
|
+
observations,
|
|
291
|
+
droppedIds: [],
|
|
292
|
+
fellBack: false,
|
|
293
|
+
tokensBefore,
|
|
294
|
+
tokensAfter: tokensBefore,
|
|
295
|
+
};
|
|
296
|
+
}
|
|
297
|
+
// Sort for pruning
|
|
298
|
+
const sorted = sortForPruning(tagged);
|
|
299
|
+
// Drop from the front until budget is met
|
|
300
|
+
const dropped = new Set();
|
|
301
|
+
let currentTokens = tokensBefore;
|
|
302
|
+
for (const item of sorted) {
|
|
303
|
+
// Never drop protected observations
|
|
304
|
+
if (item.protected)
|
|
305
|
+
continue;
|
|
306
|
+
// Stop if we're under budget
|
|
307
|
+
if (currentTokens <= targetTokens)
|
|
308
|
+
break;
|
|
309
|
+
dropped.add(item.observation.id);
|
|
310
|
+
currentTokens -= estimateStringTokens(item.observation.content);
|
|
311
|
+
}
|
|
312
|
+
// Check if we couldn't reach the budget because too many observations are protected
|
|
313
|
+
const fellBack = currentTokens > targetTokens && dropped.size < sorted.filter((s) => !s.protected).length;
|
|
314
|
+
const kept = observations.filter((o) => !dropped.has(o.id));
|
|
315
|
+
const droppedIds = Array.from(dropped);
|
|
316
|
+
debugLog("deterministic_pruner.result", {
|
|
317
|
+
tokensBefore,
|
|
318
|
+
tokensAfter: currentTokens,
|
|
319
|
+
targetTokens,
|
|
320
|
+
dropped: droppedIds.length,
|
|
321
|
+
remaining: kept.length,
|
|
322
|
+
protectedCount: sorted.filter((s) => s.protected).length,
|
|
323
|
+
fellBack,
|
|
324
|
+
droppedIds: isDebugLogEnabled() ? droppedIds : undefined,
|
|
325
|
+
});
|
|
326
|
+
return {
|
|
327
|
+
observations: kept,
|
|
328
|
+
droppedIds,
|
|
329
|
+
fellBack,
|
|
330
|
+
tokensBefore,
|
|
331
|
+
tokensAfter: currentTokens,
|
|
332
|
+
};
|
|
333
|
+
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"compaction-hook.d.ts","sourceRoot":"","sources":["../../../src/memory/hooks/compaction-hook.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAA+C,MAAM,iCAAiC,CAAC;
|
|
1
|
+
{"version":3,"file":"compaction-hook.d.ts","sourceRoot":"","sources":["../../../src/memory/hooks/compaction-hook.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAA+C,MAAM,iCAAiC,CAAC;AAoBjH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAgE7C,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,YAAY,EAAE,OAAO,EAAE,OAAO,GAAG,IAAI,CA0bhF"}
|
|
@@ -3,11 +3,12 @@ import { resolveTurnLimits } from "../config.js";
|
|
|
3
3
|
import { getProjectObsStore } from "../project-observations-store.js";
|
|
4
4
|
import { relayProjectObservations } from "../inter-agent-relay.js";
|
|
5
5
|
import { collectObservationsByCoverage, findLastCompactionIndex, gapRawEntries, getMemoryState, } from "../branch.js";
|
|
6
|
-
import { coverageTagCounts, migrateLegacyReflections, observationPoolTokens, renderSummary,
|
|
7
|
-
import { observationsToPromptLines, runObserver } from "../observer.js";
|
|
6
|
+
import { coverageTagCounts, migrateLegacyReflections, observationPoolTokens, renderSummary, runReflector, } from "../compaction.js";
|
|
8
7
|
import { serializeSourceAddressedBranchEntries } from "../serialize.js";
|
|
9
8
|
import { estimateStringTokens } from "../tokens.js";
|
|
10
|
-
import { OBSERVATION_CUSTOM_TYPE,
|
|
9
|
+
import { OBSERVATION_CUSTOM_TYPE, } from "../types.js";
|
|
10
|
+
import { deterministicPrune } from "../deterministic-pruner.js";
|
|
11
|
+
import { generateUnifiedCompaction } from "../unified-compaction.js";
|
|
11
12
|
function plural(count, singular, pluralForm = `${singular}s`) {
|
|
12
13
|
return `${count.toLocaleString()} ${count === 1 ? singular : pluralForm}`;
|
|
13
14
|
}
|
|
@@ -18,9 +19,6 @@ function formatReflectorStats(stats) {
|
|
|
18
19
|
const failed = stats.failedPass === undefined ? "" : `, failed pass ${stats.failedPass}`;
|
|
19
20
|
return `reflector ${plural(stats.toolCalls, "tool call")}, +${stats.added.toLocaleString()} added, ${stats.merged.toLocaleString()} merged, ${stats.promoted.toLocaleString()} promoted, ${stats.duplicates.toLocaleString()} duplicate/no-op, ${stats.unsupported.toLocaleString()} unsupported${failed}`;
|
|
20
21
|
}
|
|
21
|
-
function formatPrunerStats(result) {
|
|
22
|
-
return `pruner dropped ${plural(result.droppedIds.length, "observation")} in ${plural(result.passes.length, "pass", "passes")}, stop: ${result.stopReason}`;
|
|
23
|
-
}
|
|
24
22
|
function emitProgressLine(emitProgress, line) {
|
|
25
23
|
emitProgress?.(`${line}\n`);
|
|
26
24
|
}
|
|
@@ -61,8 +59,6 @@ function emitAgentLoopProgress(emitProgress, phase, event) {
|
|
|
61
59
|
export function registerCompactionHook(ext, runtime) {
|
|
62
60
|
ext.on("session_before_compact", async (event, ctx) => {
|
|
63
61
|
if (runtime.compactHookInFlight) {
|
|
64
|
-
// Expected race: another compaction process reached the hook first.
|
|
65
|
-
// Let the in-flight compaction continue; this is not a user-facing error.
|
|
66
62
|
debugLog("compaction.skip_duplicate", { reason: "hook already in flight" });
|
|
67
63
|
return { cancel: true };
|
|
68
64
|
}
|
|
@@ -72,11 +68,9 @@ export function registerCompactionHook(ext, runtime) {
|
|
|
72
68
|
const runId = `compaction-${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 8)}`;
|
|
73
69
|
return await withDebugLogContext({ enabled: runtime.config.debugLog === true, cwd: ctx.cwd, runId }, async () => {
|
|
74
70
|
const { preparation, branchEntries, signal } = event;
|
|
75
|
-
const { firstKeptEntryId, tokensBefore } = preparation;
|
|
71
|
+
const { firstKeptEntryId, tokensBefore, messagesToSummarize, settings: compactSettings } = preparation;
|
|
76
72
|
const emitProgress = event.emitProgress;
|
|
77
73
|
emitProgressLine(emitProgress, `Observational memory compaction started on ~${tokensBefore.toLocaleString()} tokens.`);
|
|
78
|
-
// Capture ctx properties synchronously — after multiple awaits below,
|
|
79
|
-
// the extension ctx may become stale (e.g. after session replacement/reload).
|
|
80
74
|
const hasUI = ctx.hasUI;
|
|
81
75
|
const ui = ctx.ui;
|
|
82
76
|
const turnLimits = resolveTurnLimits(runtime.config);
|
|
@@ -86,7 +80,7 @@ export function registerCompactionHook(ext, runtime) {
|
|
|
86
80
|
branchEntryCount: branchEntries.length,
|
|
87
81
|
reflectionThresholdTokens: runtime.config.reflectionThresholdTokens,
|
|
88
82
|
turnLimits,
|
|
89
|
-
|
|
83
|
+
messagesToSummarize: messagesToSummarize?.length ?? 0,
|
|
90
84
|
});
|
|
91
85
|
emitProgressLine(emitProgress, "Resolving memory model…");
|
|
92
86
|
const resolved = await runtime.resolveModel(ctx);
|
|
@@ -105,9 +99,7 @@ export function registerCompactionHook(ext, runtime) {
|
|
|
105
99
|
try {
|
|
106
100
|
await runtime.observerPromise;
|
|
107
101
|
}
|
|
108
|
-
catch { /* already notified
|
|
109
|
-
// In-flight observer may have appended a new observation entry during the await;
|
|
110
|
-
// refresh from sessionManager so gap computation and coverage collection see it
|
|
102
|
+
catch { /* already notified */ }
|
|
111
103
|
entries = ctx.sessionManager.getBranch();
|
|
112
104
|
}
|
|
113
105
|
const memoryState = getMemoryState(entries);
|
|
@@ -117,88 +109,84 @@ export function registerCompactionHook(ext, runtime) {
|
|
|
117
109
|
reflections: memoryState.reflections.length,
|
|
118
110
|
});
|
|
119
111
|
emitProgressLine(emitProgress, `Loaded memory state: ${memoryState.committedObs.length} committed observations, ${memoryState.pendingObs.length} pending observations, ${memoryState.reflections.length} reflections.`);
|
|
112
|
+
// ================================================================
|
|
113
|
+
// UNIFIED COMPACTION: replaces core generateSummary() + sync
|
|
114
|
+
// catch-up observer with a single LLM call.
|
|
115
|
+
// ================================================================
|
|
120
116
|
let gapObservationData = null;
|
|
117
|
+
let unifiedNarrativeSummary;
|
|
118
|
+
let unifiedObservations = [];
|
|
121
119
|
const gap = gapRawEntries(entries, firstKeptEntryId);
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
120
|
+
const gapChunk = gap.length > 0
|
|
121
|
+
? serializeSourceAddressedBranchEntries(gap)
|
|
122
|
+
: { text: "", sourceEntryIds: [] };
|
|
123
|
+
const hasGapContent = gapChunk.text.trim() && gapChunk.sourceEntryIds.length > 0;
|
|
124
|
+
// Always run unified compaction — it replaces both the core narrative
|
|
125
|
+
// summary and, when gap exists, the sync catch-up observer.
|
|
126
|
+
emitProgressLine(emitProgress, "Running unified compaction (narrative + observations)…");
|
|
127
|
+
debugLog("compaction.unified.start", {
|
|
128
|
+
messagesToSummarize: messagesToSummarize.length,
|
|
129
|
+
hasGapContent,
|
|
130
|
+
gapEntryCount: gap.length,
|
|
131
|
+
});
|
|
132
|
+
try {
|
|
133
|
+
const unifiedResult = await generateUnifiedCompaction({
|
|
134
|
+
messagesToSummarize,
|
|
135
|
+
gapChunk: hasGapContent ? gapChunk.text : undefined,
|
|
136
|
+
gapSourceEntryIds: hasGapContent ? gapChunk.sourceEntryIds : undefined,
|
|
137
|
+
model: resolved.model,
|
|
138
|
+
apiKey: resolved.apiKey,
|
|
139
|
+
headers: resolved.headers,
|
|
140
|
+
signal,
|
|
141
|
+
previousSummary: preparation.previousSummary,
|
|
142
|
+
customInstructions: event.customInstructions,
|
|
143
|
+
thinkingLevel: undefined, // Use model default
|
|
144
|
+
reserveTokens: compactSettings.reserveTokens,
|
|
145
|
+
});
|
|
146
|
+
unifiedNarrativeSummary = unifiedResult.narrativeSummary;
|
|
147
|
+
unifiedObservations = unifiedResult.observations;
|
|
148
|
+
// If gap entries produced observations, record them as an
|
|
149
|
+
// observation entry so recall can resolve source provenance.
|
|
150
|
+
if (hasGapContent && unifiedObservations.length > 0) {
|
|
125
151
|
const gapFromId = gap[0].id;
|
|
126
152
|
const gapUpToId = gap[gap.length - 1].id;
|
|
127
|
-
const
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
gapUpToId,
|
|
137
|
-
tokenEstimate: gapTokenEstimate,
|
|
138
|
-
});
|
|
139
|
-
if (hasUI)
|
|
140
|
-
ui?.notify(`Observational memory: sync catch-up observer running on ~${gapTokenEstimate.toLocaleString()}-token gap`, "info");
|
|
141
|
-
runtime.observerInFlight = true;
|
|
142
|
-
const gapCall = runObserver({
|
|
143
|
-
model: resolved.model,
|
|
144
|
-
apiKey: resolved.apiKey,
|
|
145
|
-
headers: resolved.headers,
|
|
146
|
-
priorReflections: memoryState.reflections.map(reflectionToPromptLine),
|
|
147
|
-
priorObservations: priorObservationLines,
|
|
148
|
-
chunk: gapChunk,
|
|
149
|
-
allowedSourceEntryIds: sourceEntryIds,
|
|
150
|
-
signal,
|
|
151
|
-
onEvent: (agentEvent) => emitAgentLoopProgress(emitProgress, "observer", agentEvent),
|
|
152
|
-
maxTurns: turnLimits.observerMaxTurnsPerRun,
|
|
153
|
-
});
|
|
154
|
-
const gapPromise = gapCall.then(() => undefined, () => undefined);
|
|
155
|
-
runtime.observerPromise = gapPromise;
|
|
156
|
-
try {
|
|
157
|
-
const records = await gapCall;
|
|
158
|
-
if (records && records.length > 0) {
|
|
159
|
-
const observationTokens = records.reduce((sum, r) => sum + estimateStringTokens(r.content), 0);
|
|
160
|
-
gapObservationData = {
|
|
161
|
-
records,
|
|
162
|
-
coversFromId: gapFromId,
|
|
163
|
-
coversUpToId: gapUpToId,
|
|
164
|
-
tokenCount: observationTokens,
|
|
165
|
-
};
|
|
166
|
-
debugLog("compaction.sync_catchup.records", {
|
|
167
|
-
count: records.length,
|
|
168
|
-
observationTokens,
|
|
169
|
-
coversFromId: gapFromId,
|
|
170
|
-
coversUpToId: gapUpToId,
|
|
171
|
-
records,
|
|
172
|
-
});
|
|
173
|
-
ext.appendEntry(OBSERVATION_CUSTOM_TYPE, gapObservationData);
|
|
174
|
-
emitProgressLine(emitProgress, `Sync catch-up recorded ${records.length} observation${records.length === 1 ? "" : "s"} (~${observationTokens.toLocaleString()} tokens).`);
|
|
175
|
-
if (hasUI && ui)
|
|
176
|
-
ui.notify(`Observational memory: sync catch-up recorded ${records.length} observation${records.length === 1 ? "" : "s"} (~${observationTokens.toLocaleString()} tokens)`, "info");
|
|
177
|
-
}
|
|
178
|
-
else if (hasUI && ui) {
|
|
179
|
-
debugLog("compaction.sync_catchup.empty", { gapEntryCount: gap.length });
|
|
180
|
-
ui.notify("Observational memory: sync catch-up observer returned empty — proceeding with compaction", "warning");
|
|
181
|
-
}
|
|
182
|
-
}
|
|
183
|
-
catch (error) {
|
|
184
|
-
const msg = error instanceof Error ? error.message : String(error);
|
|
185
|
-
debugLog("compaction.sync_catchup.error", { gapEntryCount: gap.length, errorMessage: msg });
|
|
186
|
-
if (hasUI && ui)
|
|
187
|
-
ui.notify(`Observational memory: sync catch-up observer failed: ${msg}. Cancelling compaction — ${gap.length} unobserved raw entries would be pruned without coverage. Try /compact again.`, "warning");
|
|
188
|
-
return { cancel: true };
|
|
189
|
-
}
|
|
190
|
-
finally {
|
|
191
|
-
runtime.observerInFlight = false;
|
|
192
|
-
if (runtime.observerPromise === gapPromise)
|
|
193
|
-
runtime.observerPromise = null;
|
|
194
|
-
}
|
|
153
|
+
const observationTokens = unifiedObservations.reduce((sum, r) => sum + estimateStringTokens(r.content), 0);
|
|
154
|
+
gapObservationData = {
|
|
155
|
+
records: unifiedObservations,
|
|
156
|
+
coversFromId: gapFromId,
|
|
157
|
+
coversUpToId: gapUpToId,
|
|
158
|
+
tokenCount: observationTokens,
|
|
159
|
+
};
|
|
160
|
+
ext.appendEntry(OBSERVATION_CUSTOM_TYPE, gapObservationData);
|
|
161
|
+
emitProgressLine(emitProgress, `Unified compaction recorded ${unifiedObservations.length} observation${unifiedObservations.length === 1 ? "" : "s"} from gap (~${observationTokens.toLocaleString()} tokens).`);
|
|
195
162
|
}
|
|
163
|
+
debugLog("compaction.unified.result", {
|
|
164
|
+
narrativeLength: unifiedNarrativeSummary?.length ?? 0,
|
|
165
|
+
observationCount: unifiedObservations.length,
|
|
166
|
+
});
|
|
167
|
+
if (hasUI && ui) {
|
|
168
|
+
ui.notify(`Observational memory: unified compaction produced ${unifiedObservations.length} observation${unifiedObservations.length === 1 ? "" : "s"} + narrative summary`, "info");
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
catch (error) {
|
|
172
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
173
|
+
debugLog("compaction.unified.error", { errorMessage: msg });
|
|
174
|
+
if (hasUI && ui)
|
|
175
|
+
ui.notify(`Observational memory: unified compaction failed: ${msg}. Cancelling compaction.`, "warning");
|
|
176
|
+
return { cancel: true };
|
|
196
177
|
}
|
|
178
|
+
// ================================================================
|
|
179
|
+
// DELTA OBSERVATIONS: collect observer-trigger observations that
|
|
180
|
+
// fall within this compaction window (from prior FKI to new FKI).
|
|
181
|
+
// ================================================================
|
|
197
182
|
const priorCompactionIdx = findLastCompactionIndex(entries);
|
|
198
183
|
const priorFirstKeptEntryId = priorCompactionIdx >= 0 ? entries[priorCompactionIdx].firstKeptEntryId : undefined;
|
|
199
184
|
const deltaObservationData = collectObservationsByCoverage(entries, priorFirstKeptEntryId, firstKeptEntryId);
|
|
200
|
-
|
|
201
|
-
|
|
185
|
+
// Gap observations were already appended via ext.appendEntry above,
|
|
186
|
+
// so collectObservationsByCoverage will pick them up. But to avoid
|
|
187
|
+
// double-counting (the unified observations are also in our local set),
|
|
188
|
+
// we build the working set from committed + delta, then merge unified
|
|
189
|
+
// observations (deduplicated by content hash).
|
|
202
190
|
const deltaObservationRecords = deltaObservationData.reduce((sum, data) => sum + data.records.length, 0);
|
|
203
191
|
debugLog("compaction.delta", {
|
|
204
192
|
priorFirstKeptEntryId,
|
|
@@ -208,10 +196,10 @@ export function registerCompactionHook(ext, runtime) {
|
|
|
208
196
|
gapObservationRecords: gapObservationData?.records.length ?? 0,
|
|
209
197
|
});
|
|
210
198
|
emitProgressLine(emitProgress, `Compaction delta contains ${deltaObservationRecords} observation${deltaObservationRecords === 1 ? "" : "s"}.`);
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
199
|
+
// ================================================================
|
|
200
|
+
// NO-DELTA CASE: carry forward existing memory.
|
|
201
|
+
// ================================================================
|
|
202
|
+
if (deltaObservationData.length === 0 && unifiedObservations.length === 0) {
|
|
215
203
|
if (memoryState.committedObs.length === 0 && memoryState.reflections.length === 0) {
|
|
216
204
|
debugLog("compaction.no_delta_cancel", {
|
|
217
205
|
committedObservations: memoryState.committedObs.length,
|
|
@@ -223,7 +211,6 @@ export function registerCompactionHook(ext, runtime) {
|
|
|
223
211
|
}
|
|
224
212
|
return { cancel: true };
|
|
225
213
|
}
|
|
226
|
-
// Carry forward existing memory without running reflector/pruner
|
|
227
214
|
emitProgressLine(emitProgress, "No new observations; carrying forward existing memory.");
|
|
228
215
|
const workingReflections = migrateLegacyReflections(memoryState.reflections);
|
|
229
216
|
debugLog("compaction.no_delta_carry_forward", {
|
|
@@ -239,7 +226,6 @@ export function registerCompactionHook(ext, runtime) {
|
|
|
239
226
|
};
|
|
240
227
|
if (hasUI)
|
|
241
228
|
ui?.notify(`Observational memory: no new observations — carrying forward ${memoryState.committedObs.length} observation${memoryState.committedObs.length === 1 ? "" : "s"}, ${workingReflections.length} reflection${workingReflections.length === 1 ? "" : "s"}`, "info");
|
|
242
|
-
// Promote reflections as project observations (cross-session durable memory).
|
|
243
229
|
persistProjectObservations(ctx, workingReflections);
|
|
244
230
|
return {
|
|
245
231
|
compaction: {
|
|
@@ -250,32 +236,61 @@ export function registerCompactionHook(ext, runtime) {
|
|
|
250
236
|
},
|
|
251
237
|
};
|
|
252
238
|
}
|
|
239
|
+
// ================================================================
|
|
240
|
+
// BUILD WORKING SET: committed + delta (from observer triggers) +
|
|
241
|
+
// unified observations (from this compaction).
|
|
242
|
+
// Deduplicate by content hash.
|
|
243
|
+
// ================================================================
|
|
244
|
+
const seenObsIds = new Set();
|
|
245
|
+
const workingObservations = [];
|
|
246
|
+
// Add committed observations first (most stable)
|
|
247
|
+
for (const obs of memoryState.committedObs) {
|
|
248
|
+
if (!seenObsIds.has(obs.id)) {
|
|
249
|
+
seenObsIds.add(obs.id);
|
|
250
|
+
workingObservations.push(obs);
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
// Add delta observations (from observer triggers between compactions)
|
|
254
|
+
for (const data of deltaObservationData) {
|
|
255
|
+
for (const obs of data.records) {
|
|
256
|
+
if (!seenObsIds.has(obs.id)) {
|
|
257
|
+
seenObsIds.add(obs.id);
|
|
258
|
+
workingObservations.push(obs);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
// Add unified observations (from this compaction's LLM call)
|
|
263
|
+
for (const obs of unifiedObservations) {
|
|
264
|
+
if (!seenObsIds.has(obs.id)) {
|
|
265
|
+
seenObsIds.add(obs.id);
|
|
266
|
+
workingObservations.push(obs);
|
|
267
|
+
}
|
|
268
|
+
}
|
|
253
269
|
const workingReflections = migrateLegacyReflections(memoryState.reflections);
|
|
254
|
-
const workingObservations = [
|
|
255
|
-
...memoryState.committedObs,
|
|
256
|
-
...deltaObservationData.flatMap((d) => d.records),
|
|
257
|
-
];
|
|
258
270
|
const observationTokens = observationPoolTokens(workingObservations);
|
|
259
271
|
emitProgressLine(emitProgress, `Working set: ${workingObservations.length} observations, ${workingReflections.length} reflections, ~${observationTokens.toLocaleString()} observation tokens.`);
|
|
260
|
-
debugLog("compaction.
|
|
272
|
+
debugLog("compaction.working_set", {
|
|
273
|
+
observations: workingObservations.length,
|
|
274
|
+
reflections: workingReflections.length,
|
|
261
275
|
observationTokens,
|
|
262
|
-
reflectionThresholdTokens: runtime.config.reflectionThresholdTokens,
|
|
263
|
-
willRun: observationTokens >= runtime.config.reflectionThresholdTokens,
|
|
264
|
-
workingObservations: workingObservations.length,
|
|
265
|
-
workingReflections: workingReflections.length,
|
|
266
276
|
});
|
|
277
|
+
// ================================================================
|
|
278
|
+
// REFLECTOR (1 pass): crystallize durable reflections from
|
|
279
|
+
// the observation pool. Preserves quality-critical iteration
|
|
280
|
+
// via agentLoop with record_reflections tool.
|
|
281
|
+
// ================================================================
|
|
267
282
|
let finalReflections = workingReflections;
|
|
268
283
|
let finalObservations = workingObservations;
|
|
269
284
|
if (observationTokens >= runtime.config.reflectionThresholdTokens) {
|
|
270
285
|
try {
|
|
271
|
-
debugLog("compaction.
|
|
286
|
+
debugLog("compaction.reflect.start", {
|
|
272
287
|
workingObservations: workingObservations.length,
|
|
273
288
|
workingReflections: workingReflections.length,
|
|
274
289
|
observationTokens,
|
|
275
290
|
});
|
|
276
291
|
if (hasUI)
|
|
277
|
-
ui?.notify("Observational memory: running reflector
|
|
278
|
-
emitProgressLine(emitProgress, "Running reflector
|
|
292
|
+
ui?.notify("Observational memory: running reflector (1 pass)...", "info");
|
|
293
|
+
emitProgressLine(emitProgress, "Running reflector (1 pass)…");
|
|
279
294
|
const coverageBefore = coverageTagCounts(workingReflections, workingObservations);
|
|
280
295
|
const reflectorResult = await runReflector({
|
|
281
296
|
model: resolved.model,
|
|
@@ -294,39 +309,49 @@ export function registerCompactionHook(ext, runtime) {
|
|
|
294
309
|
beforeReflections: workingReflections.length,
|
|
295
310
|
afterReflections: finalReflections.length,
|
|
296
311
|
});
|
|
297
|
-
const prunerResult = await runPruner({
|
|
298
|
-
model: resolved.model,
|
|
299
|
-
apiKey: resolved.apiKey,
|
|
300
|
-
headers: resolved.headers,
|
|
301
|
-
signal,
|
|
302
|
-
onEvent: (agentEvent) => emitAgentLoopProgress(emitProgress, "pruner", agentEvent),
|
|
303
|
-
maxTurns: turnLimits.prunerMaxTurnsPerPass,
|
|
304
|
-
}, finalReflections, workingObservations, runtime.config.reflectionThresholdTokens, (pass, maxPasses) => emitProgressLine(emitProgress, `Pruner pass ${pass}/${maxPasses} started.`));
|
|
305
|
-
finalObservations = prunerResult.observations;
|
|
306
|
-
debugLog("compaction.pruner.result", {
|
|
307
|
-
stopReason: prunerResult.stopReason,
|
|
308
|
-
fellBack: prunerResult.fellBack,
|
|
309
|
-
droppedIds: prunerResult.droppedIds,
|
|
310
|
-
passes: prunerResult.passes,
|
|
311
|
-
beforeObservations: workingObservations.length,
|
|
312
|
-
afterObservations: finalObservations.length,
|
|
313
|
-
});
|
|
314
312
|
if (hasUI) {
|
|
315
|
-
ui?.notify(`Observational memory:
|
|
316
|
-
}
|
|
317
|
-
if (prunerResult.fellBack && hasUI) {
|
|
318
|
-
ui?.notify("Observational memory: pruner run failed; kept observation set unchanged", "warning");
|
|
313
|
+
ui?.notify(`Observational memory: reflector — ${formatReflectorStats(reflectorResult.stats)}; coverage ${formatCoverageCounts(coverageBefore)} → ${formatCoverageCounts(coverageAfter)}`, "info");
|
|
319
314
|
}
|
|
320
315
|
}
|
|
321
316
|
catch (error) {
|
|
322
317
|
const msg = error instanceof Error ? error.message : String(error);
|
|
323
|
-
debugLog("compaction.
|
|
318
|
+
debugLog("compaction.reflector.error", { errorMessage: msg });
|
|
324
319
|
if (hasUI)
|
|
325
|
-
ui?.notify(`Observational memory:
|
|
320
|
+
ui?.notify(`Observational memory: reflector failed: ${msg}`, "warning");
|
|
326
321
|
}
|
|
327
322
|
}
|
|
323
|
+
// ================================================================
|
|
324
|
+
// DETERMINISTIC PRUNER: sort-based dropping instead of LLM agent.
|
|
325
|
+
// Same logic as the LLM pruner (coverage → relevance → age)
|
|
326
|
+
// but zero-cost and deterministic.
|
|
327
|
+
// ================================================================
|
|
328
|
+
const prunerResult = deterministicPrune(finalObservations, finalReflections, runtime.config.reflectionThresholdTokens);
|
|
329
|
+
finalObservations = prunerResult.observations;
|
|
330
|
+
debugLog("compaction.pruner.result", {
|
|
331
|
+
fellBack: prunerResult.fellBack,
|
|
332
|
+
droppedIds: prunerResult.droppedIds,
|
|
333
|
+
tokensBefore: prunerResult.tokensBefore,
|
|
334
|
+
tokensAfter: prunerResult.tokensAfter,
|
|
335
|
+
beforeObservations: workingObservations.length,
|
|
336
|
+
afterObservations: finalObservations.length,
|
|
337
|
+
});
|
|
338
|
+
emitProgressLine(emitProgress, `Deterministic pruner dropped ${prunerResult.droppedIds.length} observation${prunerResult.droppedIds.length === 1 ? "" : "s"} (~${prunerResult.tokensBefore.toLocaleString()} → ~${prunerResult.tokensAfter.toLocaleString()} tokens).`);
|
|
339
|
+
if (prunerResult.fellBack && hasUI) {
|
|
340
|
+
ui?.notify("Observational memory: pruner could not reach budget (too many protected observations)", "warning");
|
|
341
|
+
}
|
|
342
|
+
// ================================================================
|
|
343
|
+
// BUILD FINAL SUMMARY: narrative (from unified call) +
|
|
344
|
+
// observations/reflections markdown (from renderSummary).
|
|
345
|
+
// This gives the LLM BOTH narrative context AND factual memory.
|
|
346
|
+
// ================================================================
|
|
328
347
|
emitProgressLine(emitProgress, `Rendering final memory summary from ${finalObservations.length} observations and ${finalReflections.length} reflections…`);
|
|
329
|
-
const
|
|
348
|
+
const memoryMarkdown = renderSummary(finalReflections, finalObservations);
|
|
349
|
+
// Combine: narrative summary first (what happened), then memory
|
|
350
|
+
// (what was learned). Previous compaction's narrative is already
|
|
351
|
+
// merged into the unified call's output via previousSummary.
|
|
352
|
+
const summary = unifiedNarrativeSummary
|
|
353
|
+
? `${unifiedNarrativeSummary}\n\n---\n\n${memoryMarkdown}`
|
|
354
|
+
: memoryMarkdown;
|
|
330
355
|
if (finalObservations.length === 0) {
|
|
331
356
|
throw new Error("invariant violated: finalObservations empty after delta guard");
|
|
332
357
|
}
|
|
@@ -341,6 +366,7 @@ export function registerCompactionHook(ext, runtime) {
|
|
|
341
366
|
finalReflections: finalReflections.length,
|
|
342
367
|
firstKeptEntryId,
|
|
343
368
|
tokensBefore,
|
|
369
|
+
hasNarrative: !!unifiedNarrativeSummary,
|
|
344
370
|
});
|
|
345
371
|
emitProgressLine(emitProgress, `Compaction assembled — ${finalObservations.length} observation${finalObservations.length === 1 ? "" : "s"}, ${finalReflections.length} reflection${finalReflections.length === 1 ? "" : "s"}.`);
|
|
346
372
|
if (hasUI)
|
|
@@ -364,10 +390,10 @@ export function registerCompactionHook(ext, runtime) {
|
|
|
364
390
|
function persistProjectObservations(ctx, reflections) {
|
|
365
391
|
const store = getProjectObsStore();
|
|
366
392
|
if (!store)
|
|
367
|
-
return;
|
|
393
|
+
return;
|
|
368
394
|
const projectId = store.getProjectByCwd(ctx.cwd);
|
|
369
395
|
if (!projectId)
|
|
370
|
-
return;
|
|
396
|
+
return;
|
|
371
397
|
const sessionId = ctx.sessionManager.getSessionId?.();
|
|
372
398
|
if (!sessionId)
|
|
373
399
|
return;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"prompts.d.ts","sourceRoot":"","sources":["../../src/memory/prompts.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,aAAa,+QAA+Q,CAAC;AAE1S,eAAO,MAAM,yBAAyB,6oHAmD+F,CAAC;AAEtI,eAAO,MAAM,0BAA0B,67CAkB+F,CAAC;AAEvI,eAAO,MAAM,gBAAgB,8+CAasI,CAAC;AAEpK,eAAO,MAAM,eAAe,knTAoC2G,CAAC;AAExI,eAAO,MAAM,gBAAgB,69UA0DwH,CAAC;AAEtJ,eAAO,MAAM,aAAa,u2cAgFuP,CAAC;
|
|
1
|
+
{"version":3,"file":"prompts.d.ts","sourceRoot":"","sources":["../../src/memory/prompts.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,aAAa,+QAA+Q,CAAC;AAE1S,eAAO,MAAM,yBAAyB,6oHAmD+F,CAAC;AAEtI,eAAO,MAAM,0BAA0B,67CAkB+F,CAAC;AAEvI,eAAO,MAAM,gBAAgB,8+CAasI,CAAC;AAEpK,eAAO,MAAM,eAAe,knTAoC2G,CAAC;AAExI,eAAO,MAAM,gBAAgB,69UA0DwH,CAAC;AAEtJ,eAAO,MAAM,aAAa,u2cAgFuP,CAAC;AAiBlR,wBAAgB,0BAA0B,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAIlF;AASD,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAG/E;AAED,eAAO,MAAM,0BAA0B,07BAOoS,CAAC"}
|
package/dist/memory/prompts.js
CHANGED
|
@@ -261,11 +261,20 @@ What you CANNOT do:
|
|
|
261
261
|
It is valid to end a pass with zero drops if the pool genuinely has nothing more to cut — a follow-up pass will be skipped when a run returns zero drops. On late pressure passes, first re-check old [coverage: reinforced] and [coverage: cited] observations as active-memory redundancies before deciding there are no sound drops. Do not force drops you don't believe in.
|
|
262
262
|
|
|
263
263
|
Remember: pruning is active-memory management, not source deletion. A drop that looks reasonable at "low" still becomes a mistake if the content was a user correction with a mis-labeled relevance and no reflection captures it with equivalent fidelity. Read before you cut.`;
|
|
264
|
+
const REFLECTOR_UNIFIED_PASS = `Pass strategy — combined broad synthesis + atomic facts in a single pass.
|
|
265
|
+
|
|
266
|
+
Phase 1 — Multi-observation synthesis: First crystallize broad durable patterns, repeated preferences, recurring constraints, stable work style, and project-level themes supported by multiple observations. Every reflection in this phase must cite at least 2 distinct supportingObservationIds whose durable meaning is captured by the reflection. If an existing reflection already captures the pattern, repeat the exact same content when adding support ids materially improves active-memory coverage.
|
|
267
|
+
|
|
268
|
+
Phase 2 — Atomic durable facts + safety review: Then capture important durable facts that may be supported by a single authoritative observation: explicit user preferences, hard constraints, corrections, decisions, completed milestones, project facts, release or rollback caveats, important technical context, and other load-bearing facts future agents must not forget. Review high and critical observations against reflections created in phase 1, and catch durable information still missing. Strengthen existing or newly-created reflections by repeating exact reflection content with additional supportingObservationIds for high, critical, and old medium observations whose durable meaning is already captured by that reflection.
|
|
269
|
+
|
|
270
|
+
Do not duplicate earlier reflections; repeat exact existing content only to add missing support or promote no-provenance legacy memory. Create new reflections only when durable meaning is still missing. Do not create low-quality reflections just for coverage, and do not attach observations whose unique exact detail or current task state is not captured with equivalent fidelity.`;
|
|
264
271
|
const REFLECTOR_PASS_STRATEGIES = {
|
|
265
272
|
1: `Pass strategy — multi-observation synthesis. Find broad durable patterns, repeated preferences, recurring constraints, stable work style, and project-level themes supported by multiple observations. Every reflection recorded in this pass must cite at least 2 distinct supportingObservationIds whose durable meaning is captured by the reflection. Do not create one-off event summaries; leave important single-observation facts, safety review, and coverage strengthening for the final reflector pass. If an existing reflection already captures the pattern, repeat the exact same content when adding support ids materially improves active-memory coverage.`,
|
|
266
273
|
2: `Pass strategy — final atomic durable facts, safety review, and coverage strengthening. Capture important durable facts that may be supported by a single authoritative observation: explicit user preferences, hard constraints, corrections, decisions, completed milestones, project facts, release or rollback caveats, important technical context, and other load-bearing facts future agents must not forget. Review high and critical observations against current reflections, including reflections created in pass 1, and catch durable information still missing. Strengthen existing or newly-created reflections by repeating exact reflection content with additional supportingObservationIds for high, critical, and old medium observations whose durable meaning is already captured by that reflection. Do not duplicate earlier reflections; repeat exact existing content only to add missing support or promote no-provenance legacy memory. Create new reflections only when durable meaning is still missing. Do not create low-quality reflections just for coverage, and do not attach observations whose unique exact detail or current task state is not captured with equivalent fidelity.`,
|
|
267
274
|
};
|
|
268
275
|
export function buildReflectorPassGuidance(pass, maxPasses) {
|
|
276
|
+
if (maxPasses === 1)
|
|
277
|
+
return REFLECTOR_UNIFIED_PASS;
|
|
269
278
|
const tier = Math.min(2, Math.max(1, pass));
|
|
270
279
|
return `Pass ${pass} of up to ${maxPasses}. ${REFLECTOR_PASS_STRATEGIES[tier]}`;
|
|
271
280
|
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Unified Compaction — single LLM call replacing both:
|
|
3
|
+
* 1. Core spectral generateSummary() (narrative conversation summary)
|
|
4
|
+
* 2. Sync catch-up observer (observation extraction from gap entries)
|
|
5
|
+
*
|
|
6
|
+
* Uses a text-based delimited output format (compatible with any model
|
|
7
|
+
* without requiring native structured-output support):
|
|
8
|
+
*
|
|
9
|
+
* ===NARRATIVE===
|
|
10
|
+
* ## Goal
|
|
11
|
+
* ...
|
|
12
|
+
*
|
|
13
|
+
* ===OBSERVATIONS===
|
|
14
|
+
* [YYYY-MM-DD HH:MM] [relevance] Content line
|
|
15
|
+
* ...
|
|
16
|
+
*/
|
|
17
|
+
import type { AgentMessage, ThinkingLevel } from "../sdk/agent-core/index.js";
|
|
18
|
+
import type { Model } from "../sdk/ai/index.js";
|
|
19
|
+
import type { ObservationRecord } from "./types.js";
|
|
20
|
+
export interface UnifiedCompactionInput {
|
|
21
|
+
/** Messages to summarize (from core compaction preparation) */
|
|
22
|
+
messagesToSummarize: AgentMessage[];
|
|
23
|
+
/** Serialized gap entries as text with source entry labels */
|
|
24
|
+
gapChunk?: string;
|
|
25
|
+
/** Source entry IDs from gap entries */
|
|
26
|
+
gapSourceEntryIds?: string[];
|
|
27
|
+
/** Model to use */
|
|
28
|
+
model: Model<any>;
|
|
29
|
+
/** API key */
|
|
30
|
+
apiKey: string;
|
|
31
|
+
/** Request headers */
|
|
32
|
+
headers?: Record<string, string>;
|
|
33
|
+
/** Abort signal */
|
|
34
|
+
signal?: AbortSignal;
|
|
35
|
+
/** Previous compaction summary for iterative update */
|
|
36
|
+
previousSummary?: string;
|
|
37
|
+
/** Custom instructions for summarization focus */
|
|
38
|
+
customInstructions?: string;
|
|
39
|
+
/** Thinking level for reasoning models */
|
|
40
|
+
thinkingLevel?: ThinkingLevel;
|
|
41
|
+
/** Reserve tokens for prompt + output */
|
|
42
|
+
reserveTokens: number;
|
|
43
|
+
}
|
|
44
|
+
export interface UnifiedCompactionOutput {
|
|
45
|
+
/** Markdown narrative summary (for CompactionEntry.summary) */
|
|
46
|
+
narrativeSummary: string;
|
|
47
|
+
/** Observations extracted from the conversation */
|
|
48
|
+
observations: ObservationRecord[];
|
|
49
|
+
/** Files read */
|
|
50
|
+
readFiles: string[];
|
|
51
|
+
/** Files modified */
|
|
52
|
+
modifiedFiles: string[];
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Generate a unified compaction result: narrative summary + observations
|
|
56
|
+
* in a single LLM call. Uses delimited text output for broad model compatibility.
|
|
57
|
+
*/
|
|
58
|
+
export declare function generateUnifiedCompaction(input: UnifiedCompactionInput): Promise<UnifiedCompactionOutput>;
|
|
59
|
+
//# sourceMappingURL=unified-compaction.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"unified-compaction.d.ts","sourceRoot":"","sources":["../../src/memory/unified-compaction.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC9E,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAUhD,OAAO,KAAK,EAAE,iBAAiB,EAAa,MAAM,YAAY,CAAC;AAkO/D,MAAM,WAAW,sBAAsB;IACtC,+DAA+D;IAC/D,mBAAmB,EAAE,YAAY,EAAE,CAAC;IACpC,8DAA8D;IAC9D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,wCAAwC;IACxC,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,mBAAmB;IACnB,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;IAClB,cAAc;IACd,MAAM,EAAE,MAAM,CAAC;IACf,sBAAsB;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,mBAAmB;IACnB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,uDAAuD;IACvD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,kDAAkD;IAClD,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,0CAA0C;IAC1C,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,yCAAyC;IACzC,aAAa,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,uBAAuB;IACvC,+DAA+D;IAC/D,gBAAgB,EAAE,MAAM,CAAC;IACzB,mDAAmD;IACnD,YAAY,EAAE,iBAAiB,EAAE,CAAC;IAClC,iBAAiB;IACjB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,qBAAqB;IACrB,aAAa,EAAE,MAAM,EAAE,CAAC;CACxB;AAED;;;GAGG;AACH,wBAAsB,yBAAyB,CAC9C,KAAK,EAAE,sBAAsB,GAC3B,OAAO,CAAC,uBAAuB,CAAC,CA0IlC"}
|
|
@@ -0,0 +1,332 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Unified Compaction — single LLM call replacing both:
|
|
3
|
+
* 1. Core spectral generateSummary() (narrative conversation summary)
|
|
4
|
+
* 2. Sync catch-up observer (observation extraction from gap entries)
|
|
5
|
+
*
|
|
6
|
+
* Uses a text-based delimited output format (compatible with any model
|
|
7
|
+
* without requiring native structured-output support):
|
|
8
|
+
*
|
|
9
|
+
* ===NARRATIVE===
|
|
10
|
+
* ## Goal
|
|
11
|
+
* ...
|
|
12
|
+
*
|
|
13
|
+
* ===OBSERVATIONS===
|
|
14
|
+
* [YYYY-MM-DD HH:MM] [relevance] Content line
|
|
15
|
+
* ...
|
|
16
|
+
*/
|
|
17
|
+
import { completeSimple } from "../sdk/ai/index.js";
|
|
18
|
+
import { extractFileOpsFromMessage, computeFileLists, createFileOps, formatFileOperations, } from "../sdk/coding-agent/core/compaction/utils.js";
|
|
19
|
+
import { hashId } from "./ids.js";
|
|
20
|
+
import { nowTimestamp, truncateRecordContent } from "./serialize.js";
|
|
21
|
+
import { debugLog } from "./debug-log.js";
|
|
22
|
+
// ============================================================================
|
|
23
|
+
// Section Delimiters
|
|
24
|
+
// ============================================================================
|
|
25
|
+
const NARRATIVE_SECTION = "===NARRATIVE===";
|
|
26
|
+
const OBSERVATIONS_SECTION = "===OBSERVATIONS===";
|
|
27
|
+
// ============================================================================
|
|
28
|
+
// Prompt Templates
|
|
29
|
+
// ============================================================================
|
|
30
|
+
const UNIFIED_SYSTEM_PROMPT = `You are a context compression agent. Your task has TWO complementary outputs that MUST be produced together in a single response.
|
|
31
|
+
|
|
32
|
+
TASK A — NARRATIVE SUMMARY: Create a structured checkpoint of the conversation progress. This is what another LLM reads to understand where the work stands.
|
|
33
|
+
|
|
34
|
+
TASK B — FACTUAL MEMORY: Extract timestamped observations (single-line facts with relevance tags). These records are the assistant's ONLY memory of this conversation after the raw messages fall out of context.
|
|
35
|
+
|
|
36
|
+
Do NOT continue the conversation. Do NOT respond to any questions in the conversation. ONLY output the structured result using the EXACT format specified.`;
|
|
37
|
+
function buildUnifiedPrompt(conversationText, previousSummary, customInstructions, currentTime) {
|
|
38
|
+
let prompt = `<conversation>\n${conversationText}\n</conversation>\n\n`;
|
|
39
|
+
if (previousSummary) {
|
|
40
|
+
prompt += `<previous-summary>\n${previousSummary}\n</previous-summary>\n\n`;
|
|
41
|
+
}
|
|
42
|
+
prompt += `Current local time: ${currentTime}\n\n`;
|
|
43
|
+
prompt += `Output your response in TWO sections delimited by the exact markers shown below.
|
|
44
|
+
|
|
45
|
+
SECTION A — ${NARRATIVE_SECTION}
|
|
46
|
+
|
|
47
|
+
Produce a structured context checkpoint summary using this EXACT format:
|
|
48
|
+
|
|
49
|
+
## Goal
|
|
50
|
+
[What is the user trying to accomplish? Can be multiple items. If updating a previous summary, preserve existing goals and add new ones.]
|
|
51
|
+
|
|
52
|
+
## Constraints & Preferences
|
|
53
|
+
- [Any constraints, preferences, or requirements mentioned by user]
|
|
54
|
+
- [Or "(none)" if none were mentioned]
|
|
55
|
+
[When updating: preserve existing constraints, add new ones.]
|
|
56
|
+
|
|
57
|
+
## Progress
|
|
58
|
+
### Done
|
|
59
|
+
- [x] [Completed tasks/changes. Preserve previously completed items, add newly completed ones.]
|
|
60
|
+
|
|
61
|
+
### In Progress
|
|
62
|
+
- [ ] [Current work. Update based on progress.]
|
|
63
|
+
|
|
64
|
+
### Blocked
|
|
65
|
+
- [Issues preventing progress, if any. Remove blockers that have been resolved.]
|
|
66
|
+
|
|
67
|
+
## Key Decisions
|
|
68
|
+
- **[Decision]**: [Brief rationale]
|
|
69
|
+
[Preserve all previous decisions, add new ones.]
|
|
70
|
+
|
|
71
|
+
## Next Steps
|
|
72
|
+
1. [Ordered list of what should happen next. Update based on current state.]
|
|
73
|
+
|
|
74
|
+
## Critical Context
|
|
75
|
+
- [Any data, examples, or references needed to continue]
|
|
76
|
+
- [Or "(none)" if not applicable]
|
|
77
|
+
|
|
78
|
+
Keep each section concise. Preserve exact file paths, function names, and error messages.
|
|
79
|
+
${customInstructions ? `\nAdditional focus for the narrative: ${customInstructions}` : ""}
|
|
80
|
+
|
|
81
|
+
SECTION B — ${OBSERVATIONS_SECTION}
|
|
82
|
+
|
|
83
|
+
Extract NEW timestamped facts from the conversation. Each observation must be on its own line in this exact format:
|
|
84
|
+
|
|
85
|
+
[YYYY-MM-DD HH:MM] [relevance] Single-line plain prose fact. No markdown, no bullets.
|
|
86
|
+
|
|
87
|
+
Do NOT restate facts already captured in the <previous-summary>. Skip routine, low-information events (tool-call acks, status updates).
|
|
88
|
+
|
|
89
|
+
OBSERVATION CONTENT RULES:
|
|
90
|
+
- Single line of plain prose. No markdown, no bullets, no code fences, no XML/HTML tags, no emojis.
|
|
91
|
+
- Preserve user assertions exactly. When the user STATES something about themselves, their project, or their environment, capture it as an assertion. When the user ASKS something, capture it as a question.
|
|
92
|
+
- Use precise action verbs (completed, resolved, implemented, chose, configured etc.).
|
|
93
|
+
- Mark concrete completions explicitly: "completed: implemented X at path/to/file.ts; user confirmed tests pass."
|
|
94
|
+
- Split compound statements into separate observations (one fact per line).
|
|
95
|
+
- Group repeated similar tool calls into a single observation.
|
|
96
|
+
- Preserve exact file paths, function names, line numbers, error messages (verbatim), package names, identifiers.
|
|
97
|
+
- Use the timestamp from the closest conversation message. Fall back to the current local time if no message timestamp applies.
|
|
98
|
+
|
|
99
|
+
RELEVANCE LEVELS (pick one per observation):
|
|
100
|
+
- critical: user identity, role, persistent preferences, explicit corrections, concrete completions that future runs MUST NOT redo.
|
|
101
|
+
- high: non-trivial technical decisions, architectural direction, unresolved blockers, key constraints.
|
|
102
|
+
- medium: task-level context that helps within the current work. Default when unsure between medium and high.
|
|
103
|
+
- low: routine tool-call acks, repetitive status updates, trivially re-derivable content. Skip these unless they carry unique detail.
|
|
104
|
+
|
|
105
|
+
Do NOT default everything to "high" or "critical". Most observations should be "medium". Use "low" only for truly routine items.
|
|
106
|
+
|
|
107
|
+
Focus areas for observations:
|
|
108
|
+
- User identity, preferences, constraints, corrections.
|
|
109
|
+
- Project goals, architectural decisions, rationale.
|
|
110
|
+
- File paths and line numbers of changes.
|
|
111
|
+
- Error messages (verbatim).
|
|
112
|
+
- Completed work that should not be redone.
|
|
113
|
+
- Named identifiers (package names, function/variable names, ticket IDs, commit SHAs).`;
|
|
114
|
+
return prompt;
|
|
115
|
+
}
|
|
116
|
+
// ============================================================================
|
|
117
|
+
// Conversation Serialization
|
|
118
|
+
// ============================================================================
|
|
119
|
+
const TOOL_RESULT_MAX_CHARS = 2000;
|
|
120
|
+
const VISION_DESCRIPTION_PREFIX = "[Image description from ";
|
|
121
|
+
function stripVisionBlocks(text) {
|
|
122
|
+
if (!text.includes(VISION_DESCRIPTION_PREFIX))
|
|
123
|
+
return text;
|
|
124
|
+
const lines = text.split("\n");
|
|
125
|
+
const kept = [];
|
|
126
|
+
let skipping = false;
|
|
127
|
+
for (const line of lines) {
|
|
128
|
+
if (!skipping && line.startsWith(VISION_DESCRIPTION_PREFIX)) {
|
|
129
|
+
skipping = true;
|
|
130
|
+
continue;
|
|
131
|
+
}
|
|
132
|
+
if (skipping) {
|
|
133
|
+
if (line === "]") {
|
|
134
|
+
skipping = false;
|
|
135
|
+
}
|
|
136
|
+
continue;
|
|
137
|
+
}
|
|
138
|
+
kept.push(line);
|
|
139
|
+
}
|
|
140
|
+
return kept.join("\n").replace(/\n{3,}/g, "\n\n").trim();
|
|
141
|
+
}
|
|
142
|
+
function serializeMessagesForUnified(messages) {
|
|
143
|
+
const parts = [];
|
|
144
|
+
for (const msg of messages) {
|
|
145
|
+
if (msg.role === "user") {
|
|
146
|
+
const content = typeof msg.content === "string"
|
|
147
|
+
? msg.content
|
|
148
|
+
: Array.isArray(msg.content)
|
|
149
|
+
? msg.content.filter((c) => c.type === "text").map(c => c.text).join("")
|
|
150
|
+
: "";
|
|
151
|
+
const cleaned = stripVisionBlocks(content);
|
|
152
|
+
if (cleaned)
|
|
153
|
+
parts.push(`[User]: ${cleaned}`);
|
|
154
|
+
}
|
|
155
|
+
else if (msg.role === "assistant" && "content" in msg && Array.isArray(msg.content)) {
|
|
156
|
+
const textParts = [];
|
|
157
|
+
const toolParts = [];
|
|
158
|
+
for (const block of msg.content) {
|
|
159
|
+
if (block.type === "text" && "text" in block) {
|
|
160
|
+
textParts.push(block.text);
|
|
161
|
+
}
|
|
162
|
+
else if (block.type === "toolCall" && "name" in block) {
|
|
163
|
+
const args = block.arguments;
|
|
164
|
+
const argsStr = Object.keys(args).sort().map(k => `${k}=${JSON.stringify(args[k])}`).join(", ");
|
|
165
|
+
toolParts.push(`${block.name}(${argsStr})`);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
const combined = [...textParts, ...(toolParts.length ? [`[Tools: ${toolParts.join("; ")}]`] : [])];
|
|
169
|
+
if (combined.length)
|
|
170
|
+
parts.push(`[Assistant]: ${combined.join("\n")}`);
|
|
171
|
+
}
|
|
172
|
+
else if (msg.role === "toolResult") {
|
|
173
|
+
const text = Array.isArray(msg.content)
|
|
174
|
+
? msg.content.filter((c) => c.type === "text").map(c => c.text).join("")
|
|
175
|
+
: typeof msg.content === "string" ? msg.content : "";
|
|
176
|
+
if (text) {
|
|
177
|
+
const truncated = text.length > TOOL_RESULT_MAX_CHARS
|
|
178
|
+
? `${text.slice(0, TOOL_RESULT_MAX_CHARS)}\n[...truncated]`
|
|
179
|
+
: text;
|
|
180
|
+
parts.push(`[Tool result]: ${truncated}`);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
else if (msg.role === "compactionSummary" && "summary" in msg) {
|
|
184
|
+
parts.push(`[Previous compaction summary]: ${msg.summary}`);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
return parts.join("\n\n");
|
|
188
|
+
}
|
|
189
|
+
// ============================================================================
|
|
190
|
+
// Output Parsing
|
|
191
|
+
// ============================================================================
|
|
192
|
+
const OBSERVATION_LINE_RE = /^\[(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2})\]\s*\[(low|medium|high|critical)\]\s*(.+)$/i;
|
|
193
|
+
function parseObservations(text) {
|
|
194
|
+
const results = [];
|
|
195
|
+
const lines = text.split("\n");
|
|
196
|
+
for (const line of lines) {
|
|
197
|
+
const trimmed = line.trim();
|
|
198
|
+
if (!trimmed || trimmed.startsWith("#") || trimmed === OBSERVATIONS_SECTION)
|
|
199
|
+
continue;
|
|
200
|
+
const match = trimmed.match(OBSERVATION_LINE_RE);
|
|
201
|
+
if (match) {
|
|
202
|
+
results.push({
|
|
203
|
+
timestamp: match[1],
|
|
204
|
+
relevance: match[2].toLowerCase(),
|
|
205
|
+
content: match[3].trim(),
|
|
206
|
+
});
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
return results;
|
|
210
|
+
}
|
|
211
|
+
function normalizeObservation(raw) {
|
|
212
|
+
const content = truncateRecordContent(raw.content);
|
|
213
|
+
if (!content)
|
|
214
|
+
return undefined;
|
|
215
|
+
const validRelevance = ["low", "medium", "high", "critical"];
|
|
216
|
+
const relevance = validRelevance.includes(raw.relevance)
|
|
217
|
+
? raw.relevance
|
|
218
|
+
: "medium";
|
|
219
|
+
return {
|
|
220
|
+
id: hashId(content),
|
|
221
|
+
content,
|
|
222
|
+
timestamp: raw.timestamp,
|
|
223
|
+
relevance,
|
|
224
|
+
};
|
|
225
|
+
}
|
|
226
|
+
/**
|
|
227
|
+
* Generate a unified compaction result: narrative summary + observations
|
|
228
|
+
* in a single LLM call. Uses delimited text output for broad model compatibility.
|
|
229
|
+
*/
|
|
230
|
+
export async function generateUnifiedCompaction(input) {
|
|
231
|
+
const { messagesToSummarize, gapChunk, model, apiKey, headers, signal, previousSummary, customInstructions, thinkingLevel, reserveTokens, } = input;
|
|
232
|
+
const maxTokens = Math.min(Math.floor(0.8 * reserveTokens), model.maxTokens > 0 ? model.maxTokens : Number.POSITIVE_INFINITY);
|
|
233
|
+
// Build conversation text
|
|
234
|
+
const parts = [];
|
|
235
|
+
const coreText = serializeMessagesForUnified(messagesToSummarize);
|
|
236
|
+
if (coreText)
|
|
237
|
+
parts.push(coreText);
|
|
238
|
+
if (gapChunk?.trim()) {
|
|
239
|
+
parts.push(`\n--- Gap entries (unobserved raw conversation) ---\n${gapChunk}`);
|
|
240
|
+
}
|
|
241
|
+
const conversationText = parts.join("\n\n");
|
|
242
|
+
const currentTime = nowTimestamp();
|
|
243
|
+
const userPrompt = buildUnifiedPrompt(conversationText, previousSummary, customInstructions, currentTime);
|
|
244
|
+
const promptMessages = [
|
|
245
|
+
{
|
|
246
|
+
role: "user",
|
|
247
|
+
content: [{ type: "text", text: userPrompt }],
|
|
248
|
+
timestamp: Date.now(),
|
|
249
|
+
},
|
|
250
|
+
];
|
|
251
|
+
const options = { maxTokens, signal, apiKey, headers };
|
|
252
|
+
if (model.reasoning && thinkingLevel && thinkingLevel !== "off") {
|
|
253
|
+
options.reasoning = thinkingLevel;
|
|
254
|
+
}
|
|
255
|
+
debugLog("unified_compaction.start", {
|
|
256
|
+
messageCount: messagesToSummarize.length,
|
|
257
|
+
hasGap: !!gapChunk?.trim(),
|
|
258
|
+
conversationLength: conversationText.length,
|
|
259
|
+
});
|
|
260
|
+
const response = await completeSimple(model, { systemPrompt: UNIFIED_SYSTEM_PROMPT, messages: promptMessages }, options);
|
|
261
|
+
if (response.stopReason === "error") {
|
|
262
|
+
throw new Error(`Unified compaction failed: ${response.errorMessage || "Unknown error"}`);
|
|
263
|
+
}
|
|
264
|
+
if (response.stopReason === "aborted") {
|
|
265
|
+
throw new Error("Unified compaction aborted");
|
|
266
|
+
}
|
|
267
|
+
const fullText = response.content
|
|
268
|
+
.filter((c) => c.type === "text")
|
|
269
|
+
.map((c) => c.text)
|
|
270
|
+
.join("\n");
|
|
271
|
+
// Parse the two sections
|
|
272
|
+
const narrativeIdx = fullText.indexOf(NARRATIVE_SECTION);
|
|
273
|
+
const obsIdx = fullText.indexOf(OBSERVATIONS_SECTION);
|
|
274
|
+
let narrativeSummary;
|
|
275
|
+
let rawObservationsText;
|
|
276
|
+
if (narrativeIdx >= 0 && obsIdx >= 0) {
|
|
277
|
+
// Both sections present
|
|
278
|
+
if (narrativeIdx < obsIdx) {
|
|
279
|
+
narrativeSummary = fullText.slice(narrativeIdx + NARRATIVE_SECTION.length, obsIdx).trim();
|
|
280
|
+
}
|
|
281
|
+
else {
|
|
282
|
+
narrativeSummary = fullText.slice(narrativeIdx + NARRATIVE_SECTION.length).trim();
|
|
283
|
+
}
|
|
284
|
+
rawObservationsText = fullText.slice(obsIdx + OBSERVATIONS_SECTION.length).trim();
|
|
285
|
+
}
|
|
286
|
+
else if (narrativeIdx >= 0) {
|
|
287
|
+
// Only narrative
|
|
288
|
+
narrativeSummary = fullText.slice(narrativeIdx + NARRATIVE_SECTION.length).trim();
|
|
289
|
+
rawObservationsText = "";
|
|
290
|
+
}
|
|
291
|
+
else if (obsIdx >= 0) {
|
|
292
|
+
// Only observations (unusual but handle gracefully)
|
|
293
|
+
narrativeSummary = fullText.slice(0, obsIdx).trim();
|
|
294
|
+
rawObservationsText = fullText.slice(obsIdx + OBSERVATIONS_SECTION.length).trim();
|
|
295
|
+
}
|
|
296
|
+
else {
|
|
297
|
+
// No delimiters found — treat entire output as narrative
|
|
298
|
+
debugLog("unified_compaction.no_delimiters", { textLength: fullText.length });
|
|
299
|
+
narrativeSummary = fullText;
|
|
300
|
+
rawObservationsText = "";
|
|
301
|
+
}
|
|
302
|
+
// Parse observations
|
|
303
|
+
const rawObservations = parseObservations(rawObservationsText);
|
|
304
|
+
const observations = [];
|
|
305
|
+
const seen = new Set();
|
|
306
|
+
for (const raw of rawObservations) {
|
|
307
|
+
const normalized = normalizeObservation(raw);
|
|
308
|
+
if (normalized && !seen.has(normalized.id)) {
|
|
309
|
+
seen.add(normalized.id);
|
|
310
|
+
observations.push(normalized);
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
// Compute file operations
|
|
314
|
+
const fileOps = createFileOps();
|
|
315
|
+
for (const msg of messagesToSummarize) {
|
|
316
|
+
extractFileOpsFromMessage(msg, fileOps);
|
|
317
|
+
}
|
|
318
|
+
const { readFiles, modifiedFiles } = computeFileLists(fileOps);
|
|
319
|
+
// Append file operations to narrative
|
|
320
|
+
const fullNarrativeSummary = narrativeSummary + formatFileOperations(readFiles, modifiedFiles);
|
|
321
|
+
debugLog("unified_compaction.result", {
|
|
322
|
+
narrativeLength: fullNarrativeSummary.length,
|
|
323
|
+
observationCount: observations.length,
|
|
324
|
+
rawObservationLines: rawObservations.length,
|
|
325
|
+
});
|
|
326
|
+
return {
|
|
327
|
+
narrativeSummary: fullNarrativeSummary,
|
|
328
|
+
observations,
|
|
329
|
+
readFiles,
|
|
330
|
+
modifiedFiles,
|
|
331
|
+
};
|
|
332
|
+
}
|