@aexol/spectral 0.9.16 → 0.9.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/memory/compaction.d.ts +1 -1
- package/dist/memory/compaction.js +3 -3
- package/dist/memory/deterministic-pruner.d.ts +65 -0
- package/dist/memory/deterministic-pruner.d.ts.map +1 -0
- package/dist/memory/deterministic-pruner.js +333 -0
- package/dist/memory/hooks/compaction-hook.d.ts.map +1 -1
- package/dist/memory/hooks/compaction-hook.js +158 -132
- package/dist/memory/prompts.d.ts.map +1 -1
- package/dist/memory/prompts.js +9 -0
- package/dist/memory/unified-compaction.d.ts +59 -0
- package/dist/memory/unified-compaction.d.ts.map +1 -0
- package/dist/memory/unified-compaction.js +332 -0
- package/dist/relay/dispatcher.d.ts +1 -1
- package/dist/relay/dispatcher.d.ts.map +1 -1
- package/dist/relay/dispatcher.js +13 -1
- package/dist/server/handlers/projects.d.ts +8 -0
- package/dist/server/handlers/projects.d.ts.map +1 -1
- package/dist/server/handlers/projects.js +6 -0
- package/dist/server/storage.d.ts +11 -0
- package/dist/server/storage.d.ts.map +1 -1
- package/dist/server/storage.js +20 -0
- package/package.json +1 -1
|
@@ -3,11 +3,12 @@ import { resolveTurnLimits } from "../config.js";
|
|
|
3
3
|
import { getProjectObsStore } from "../project-observations-store.js";
|
|
4
4
|
import { relayProjectObservations } from "../inter-agent-relay.js";
|
|
5
5
|
import { collectObservationsByCoverage, findLastCompactionIndex, gapRawEntries, getMemoryState, } from "../branch.js";
|
|
6
|
-
import { coverageTagCounts, migrateLegacyReflections, observationPoolTokens, renderSummary,
|
|
7
|
-
import { observationsToPromptLines, runObserver } from "../observer.js";
|
|
6
|
+
import { coverageTagCounts, migrateLegacyReflections, observationPoolTokens, renderSummary, runReflector, } from "../compaction.js";
|
|
8
7
|
import { serializeSourceAddressedBranchEntries } from "../serialize.js";
|
|
9
8
|
import { estimateStringTokens } from "../tokens.js";
|
|
10
|
-
import { OBSERVATION_CUSTOM_TYPE,
|
|
9
|
+
import { OBSERVATION_CUSTOM_TYPE, } from "../types.js";
|
|
10
|
+
import { deterministicPrune } from "../deterministic-pruner.js";
|
|
11
|
+
import { generateUnifiedCompaction } from "../unified-compaction.js";
|
|
11
12
|
function plural(count, singular, pluralForm = `${singular}s`) {
|
|
12
13
|
return `${count.toLocaleString()} ${count === 1 ? singular : pluralForm}`;
|
|
13
14
|
}
|
|
@@ -18,9 +19,6 @@ function formatReflectorStats(stats) {
|
|
|
18
19
|
const failed = stats.failedPass === undefined ? "" : `, failed pass ${stats.failedPass}`;
|
|
19
20
|
return `reflector ${plural(stats.toolCalls, "tool call")}, +${stats.added.toLocaleString()} added, ${stats.merged.toLocaleString()} merged, ${stats.promoted.toLocaleString()} promoted, ${stats.duplicates.toLocaleString()} duplicate/no-op, ${stats.unsupported.toLocaleString()} unsupported${failed}`;
|
|
20
21
|
}
|
|
21
|
-
function formatPrunerStats(result) {
|
|
22
|
-
return `pruner dropped ${plural(result.droppedIds.length, "observation")} in ${plural(result.passes.length, "pass", "passes")}, stop: ${result.stopReason}`;
|
|
23
|
-
}
|
|
24
22
|
function emitProgressLine(emitProgress, line) {
|
|
25
23
|
emitProgress?.(`${line}\n`);
|
|
26
24
|
}
|
|
@@ -61,8 +59,6 @@ function emitAgentLoopProgress(emitProgress, phase, event) {
|
|
|
61
59
|
export function registerCompactionHook(ext, runtime) {
|
|
62
60
|
ext.on("session_before_compact", async (event, ctx) => {
|
|
63
61
|
if (runtime.compactHookInFlight) {
|
|
64
|
-
// Expected race: another compaction process reached the hook first.
|
|
65
|
-
// Let the in-flight compaction continue; this is not a user-facing error.
|
|
66
62
|
debugLog("compaction.skip_duplicate", { reason: "hook already in flight" });
|
|
67
63
|
return { cancel: true };
|
|
68
64
|
}
|
|
@@ -72,11 +68,9 @@ export function registerCompactionHook(ext, runtime) {
|
|
|
72
68
|
const runId = `compaction-${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 8)}`;
|
|
73
69
|
return await withDebugLogContext({ enabled: runtime.config.debugLog === true, cwd: ctx.cwd, runId }, async () => {
|
|
74
70
|
const { preparation, branchEntries, signal } = event;
|
|
75
|
-
const { firstKeptEntryId, tokensBefore } = preparation;
|
|
71
|
+
const { firstKeptEntryId, tokensBefore, messagesToSummarize, settings: compactSettings } = preparation;
|
|
76
72
|
const emitProgress = event.emitProgress;
|
|
77
73
|
emitProgressLine(emitProgress, `Observational memory compaction started on ~${tokensBefore.toLocaleString()} tokens.`);
|
|
78
|
-
// Capture ctx properties synchronously — after multiple awaits below,
|
|
79
|
-
// the extension ctx may become stale (e.g. after session replacement/reload).
|
|
80
74
|
const hasUI = ctx.hasUI;
|
|
81
75
|
const ui = ctx.ui;
|
|
82
76
|
const turnLimits = resolveTurnLimits(runtime.config);
|
|
@@ -86,7 +80,7 @@ export function registerCompactionHook(ext, runtime) {
|
|
|
86
80
|
branchEntryCount: branchEntries.length,
|
|
87
81
|
reflectionThresholdTokens: runtime.config.reflectionThresholdTokens,
|
|
88
82
|
turnLimits,
|
|
89
|
-
|
|
83
|
+
messagesToSummarize: messagesToSummarize?.length ?? 0,
|
|
90
84
|
});
|
|
91
85
|
emitProgressLine(emitProgress, "Resolving memory model…");
|
|
92
86
|
const resolved = await runtime.resolveModel(ctx);
|
|
@@ -105,9 +99,7 @@ export function registerCompactionHook(ext, runtime) {
|
|
|
105
99
|
try {
|
|
106
100
|
await runtime.observerPromise;
|
|
107
101
|
}
|
|
108
|
-
catch { /* already notified
|
|
109
|
-
// In-flight observer may have appended a new observation entry during the await;
|
|
110
|
-
// refresh from sessionManager so gap computation and coverage collection see it
|
|
102
|
+
catch { /* already notified */ }
|
|
111
103
|
entries = ctx.sessionManager.getBranch();
|
|
112
104
|
}
|
|
113
105
|
const memoryState = getMemoryState(entries);
|
|
@@ -117,88 +109,84 @@ export function registerCompactionHook(ext, runtime) {
|
|
|
117
109
|
reflections: memoryState.reflections.length,
|
|
118
110
|
});
|
|
119
111
|
emitProgressLine(emitProgress, `Loaded memory state: ${memoryState.committedObs.length} committed observations, ${memoryState.pendingObs.length} pending observations, ${memoryState.reflections.length} reflections.`);
|
|
112
|
+
// ================================================================
|
|
113
|
+
// UNIFIED COMPACTION: replaces core generateSummary() + sync
|
|
114
|
+
// catch-up observer with a single LLM call.
|
|
115
|
+
// ================================================================
|
|
120
116
|
let gapObservationData = null;
|
|
117
|
+
let unifiedNarrativeSummary;
|
|
118
|
+
let unifiedObservations = [];
|
|
121
119
|
const gap = gapRawEntries(entries, firstKeptEntryId);
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
120
|
+
const gapChunk = gap.length > 0
|
|
121
|
+
? serializeSourceAddressedBranchEntries(gap)
|
|
122
|
+
: { text: "", sourceEntryIds: [] };
|
|
123
|
+
const hasGapContent = gapChunk.text.trim() && gapChunk.sourceEntryIds.length > 0;
|
|
124
|
+
// Always run unified compaction — it replaces both the core narrative
|
|
125
|
+
// summary and, when gap exists, the sync catch-up observer.
|
|
126
|
+
emitProgressLine(emitProgress, "Running unified compaction (narrative + observations)…");
|
|
127
|
+
debugLog("compaction.unified.start", {
|
|
128
|
+
messagesToSummarize: messagesToSummarize.length,
|
|
129
|
+
hasGapContent,
|
|
130
|
+
gapEntryCount: gap.length,
|
|
131
|
+
});
|
|
132
|
+
try {
|
|
133
|
+
const unifiedResult = await generateUnifiedCompaction({
|
|
134
|
+
messagesToSummarize,
|
|
135
|
+
gapChunk: hasGapContent ? gapChunk.text : undefined,
|
|
136
|
+
gapSourceEntryIds: hasGapContent ? gapChunk.sourceEntryIds : undefined,
|
|
137
|
+
model: resolved.model,
|
|
138
|
+
apiKey: resolved.apiKey,
|
|
139
|
+
headers: resolved.headers,
|
|
140
|
+
signal,
|
|
141
|
+
previousSummary: preparation.previousSummary,
|
|
142
|
+
customInstructions: event.customInstructions,
|
|
143
|
+
thinkingLevel: undefined, // Use model default
|
|
144
|
+
reserveTokens: compactSettings.reserveTokens,
|
|
145
|
+
});
|
|
146
|
+
unifiedNarrativeSummary = unifiedResult.narrativeSummary;
|
|
147
|
+
unifiedObservations = unifiedResult.observations;
|
|
148
|
+
// If gap entries produced observations, record them as an
|
|
149
|
+
// observation entry so recall can resolve source provenance.
|
|
150
|
+
if (hasGapContent && unifiedObservations.length > 0) {
|
|
125
151
|
const gapFromId = gap[0].id;
|
|
126
152
|
const gapUpToId = gap[gap.length - 1].id;
|
|
127
|
-
const
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
gapUpToId,
|
|
137
|
-
tokenEstimate: gapTokenEstimate,
|
|
138
|
-
});
|
|
139
|
-
if (hasUI)
|
|
140
|
-
ui?.notify(`Observational memory: sync catch-up observer running on ~${gapTokenEstimate.toLocaleString()}-token gap`, "info");
|
|
141
|
-
runtime.observerInFlight = true;
|
|
142
|
-
const gapCall = runObserver({
|
|
143
|
-
model: resolved.model,
|
|
144
|
-
apiKey: resolved.apiKey,
|
|
145
|
-
headers: resolved.headers,
|
|
146
|
-
priorReflections: memoryState.reflections.map(reflectionToPromptLine),
|
|
147
|
-
priorObservations: priorObservationLines,
|
|
148
|
-
chunk: gapChunk,
|
|
149
|
-
allowedSourceEntryIds: sourceEntryIds,
|
|
150
|
-
signal,
|
|
151
|
-
onEvent: (agentEvent) => emitAgentLoopProgress(emitProgress, "observer", agentEvent),
|
|
152
|
-
maxTurns: turnLimits.observerMaxTurnsPerRun,
|
|
153
|
-
});
|
|
154
|
-
const gapPromise = gapCall.then(() => undefined, () => undefined);
|
|
155
|
-
runtime.observerPromise = gapPromise;
|
|
156
|
-
try {
|
|
157
|
-
const records = await gapCall;
|
|
158
|
-
if (records && records.length > 0) {
|
|
159
|
-
const observationTokens = records.reduce((sum, r) => sum + estimateStringTokens(r.content), 0);
|
|
160
|
-
gapObservationData = {
|
|
161
|
-
records,
|
|
162
|
-
coversFromId: gapFromId,
|
|
163
|
-
coversUpToId: gapUpToId,
|
|
164
|
-
tokenCount: observationTokens,
|
|
165
|
-
};
|
|
166
|
-
debugLog("compaction.sync_catchup.records", {
|
|
167
|
-
count: records.length,
|
|
168
|
-
observationTokens,
|
|
169
|
-
coversFromId: gapFromId,
|
|
170
|
-
coversUpToId: gapUpToId,
|
|
171
|
-
records,
|
|
172
|
-
});
|
|
173
|
-
ext.appendEntry(OBSERVATION_CUSTOM_TYPE, gapObservationData);
|
|
174
|
-
emitProgressLine(emitProgress, `Sync catch-up recorded ${records.length} observation${records.length === 1 ? "" : "s"} (~${observationTokens.toLocaleString()} tokens).`);
|
|
175
|
-
if (hasUI && ui)
|
|
176
|
-
ui.notify(`Observational memory: sync catch-up recorded ${records.length} observation${records.length === 1 ? "" : "s"} (~${observationTokens.toLocaleString()} tokens)`, "info");
|
|
177
|
-
}
|
|
178
|
-
else if (hasUI && ui) {
|
|
179
|
-
debugLog("compaction.sync_catchup.empty", { gapEntryCount: gap.length });
|
|
180
|
-
ui.notify("Observational memory: sync catch-up observer returned empty — proceeding with compaction", "warning");
|
|
181
|
-
}
|
|
182
|
-
}
|
|
183
|
-
catch (error) {
|
|
184
|
-
const msg = error instanceof Error ? error.message : String(error);
|
|
185
|
-
debugLog("compaction.sync_catchup.error", { gapEntryCount: gap.length, errorMessage: msg });
|
|
186
|
-
if (hasUI && ui)
|
|
187
|
-
ui.notify(`Observational memory: sync catch-up observer failed: ${msg}. Cancelling compaction — ${gap.length} unobserved raw entries would be pruned without coverage. Try /compact again.`, "warning");
|
|
188
|
-
return { cancel: true };
|
|
189
|
-
}
|
|
190
|
-
finally {
|
|
191
|
-
runtime.observerInFlight = false;
|
|
192
|
-
if (runtime.observerPromise === gapPromise)
|
|
193
|
-
runtime.observerPromise = null;
|
|
194
|
-
}
|
|
153
|
+
const observationTokens = unifiedObservations.reduce((sum, r) => sum + estimateStringTokens(r.content), 0);
|
|
154
|
+
gapObservationData = {
|
|
155
|
+
records: unifiedObservations,
|
|
156
|
+
coversFromId: gapFromId,
|
|
157
|
+
coversUpToId: gapUpToId,
|
|
158
|
+
tokenCount: observationTokens,
|
|
159
|
+
};
|
|
160
|
+
ext.appendEntry(OBSERVATION_CUSTOM_TYPE, gapObservationData);
|
|
161
|
+
emitProgressLine(emitProgress, `Unified compaction recorded ${unifiedObservations.length} observation${unifiedObservations.length === 1 ? "" : "s"} from gap (~${observationTokens.toLocaleString()} tokens).`);
|
|
195
162
|
}
|
|
163
|
+
debugLog("compaction.unified.result", {
|
|
164
|
+
narrativeLength: unifiedNarrativeSummary?.length ?? 0,
|
|
165
|
+
observationCount: unifiedObservations.length,
|
|
166
|
+
});
|
|
167
|
+
if (hasUI && ui) {
|
|
168
|
+
ui.notify(`Observational memory: unified compaction produced ${unifiedObservations.length} observation${unifiedObservations.length === 1 ? "" : "s"} + narrative summary`, "info");
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
catch (error) {
|
|
172
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
173
|
+
debugLog("compaction.unified.error", { errorMessage: msg });
|
|
174
|
+
if (hasUI && ui)
|
|
175
|
+
ui.notify(`Observational memory: unified compaction failed: ${msg}. Cancelling compaction.`, "warning");
|
|
176
|
+
return { cancel: true };
|
|
196
177
|
}
|
|
178
|
+
// ================================================================
|
|
179
|
+
// DELTA OBSERVATIONS: collect observer-trigger observations that
|
|
180
|
+
// fall within this compaction window (from prior FKI to new FKI).
|
|
181
|
+
// ================================================================
|
|
197
182
|
const priorCompactionIdx = findLastCompactionIndex(entries);
|
|
198
183
|
const priorFirstKeptEntryId = priorCompactionIdx >= 0 ? entries[priorCompactionIdx].firstKeptEntryId : undefined;
|
|
199
184
|
const deltaObservationData = collectObservationsByCoverage(entries, priorFirstKeptEntryId, firstKeptEntryId);
|
|
200
|
-
|
|
201
|
-
|
|
185
|
+
// Gap observations were already appended via ext.appendEntry above,
|
|
186
|
+
// so collectObservationsByCoverage will pick them up. But to avoid
|
|
187
|
+
// double-counting (the unified observations are also in our local set),
|
|
188
|
+
// we build the working set from committed + delta, then merge unified
|
|
189
|
+
// observations (deduplicated by content hash).
|
|
202
190
|
const deltaObservationRecords = deltaObservationData.reduce((sum, data) => sum + data.records.length, 0);
|
|
203
191
|
debugLog("compaction.delta", {
|
|
204
192
|
priorFirstKeptEntryId,
|
|
@@ -208,10 +196,10 @@ export function registerCompactionHook(ext, runtime) {
|
|
|
208
196
|
gapObservationRecords: gapObservationData?.records.length ?? 0,
|
|
209
197
|
});
|
|
210
198
|
emitProgressLine(emitProgress, `Compaction delta contains ${deltaObservationRecords} observation${deltaObservationRecords === 1 ? "" : "s"}.`);
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
199
|
+
// ================================================================
|
|
200
|
+
// NO-DELTA CASE: carry forward existing memory.
|
|
201
|
+
// ================================================================
|
|
202
|
+
if (deltaObservationData.length === 0 && unifiedObservations.length === 0) {
|
|
215
203
|
if (memoryState.committedObs.length === 0 && memoryState.reflections.length === 0) {
|
|
216
204
|
debugLog("compaction.no_delta_cancel", {
|
|
217
205
|
committedObservations: memoryState.committedObs.length,
|
|
@@ -223,7 +211,6 @@ export function registerCompactionHook(ext, runtime) {
|
|
|
223
211
|
}
|
|
224
212
|
return { cancel: true };
|
|
225
213
|
}
|
|
226
|
-
// Carry forward existing memory without running reflector/pruner
|
|
227
214
|
emitProgressLine(emitProgress, "No new observations; carrying forward existing memory.");
|
|
228
215
|
const workingReflections = migrateLegacyReflections(memoryState.reflections);
|
|
229
216
|
debugLog("compaction.no_delta_carry_forward", {
|
|
@@ -239,7 +226,6 @@ export function registerCompactionHook(ext, runtime) {
|
|
|
239
226
|
};
|
|
240
227
|
if (hasUI)
|
|
241
228
|
ui?.notify(`Observational memory: no new observations — carrying forward ${memoryState.committedObs.length} observation${memoryState.committedObs.length === 1 ? "" : "s"}, ${workingReflections.length} reflection${workingReflections.length === 1 ? "" : "s"}`, "info");
|
|
242
|
-
// Promote reflections as project observations (cross-session durable memory).
|
|
243
229
|
persistProjectObservations(ctx, workingReflections);
|
|
244
230
|
return {
|
|
245
231
|
compaction: {
|
|
@@ -250,32 +236,61 @@ export function registerCompactionHook(ext, runtime) {
|
|
|
250
236
|
},
|
|
251
237
|
};
|
|
252
238
|
}
|
|
239
|
+
// ================================================================
|
|
240
|
+
// BUILD WORKING SET: committed + delta (from observer triggers) +
|
|
241
|
+
// unified observations (from this compaction).
|
|
242
|
+
// Deduplicate by content hash.
|
|
243
|
+
// ================================================================
|
|
244
|
+
const seenObsIds = new Set();
|
|
245
|
+
const workingObservations = [];
|
|
246
|
+
// Add committed observations first (most stable)
|
|
247
|
+
for (const obs of memoryState.committedObs) {
|
|
248
|
+
if (!seenObsIds.has(obs.id)) {
|
|
249
|
+
seenObsIds.add(obs.id);
|
|
250
|
+
workingObservations.push(obs);
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
// Add delta observations (from observer triggers between compactions)
|
|
254
|
+
for (const data of deltaObservationData) {
|
|
255
|
+
for (const obs of data.records) {
|
|
256
|
+
if (!seenObsIds.has(obs.id)) {
|
|
257
|
+
seenObsIds.add(obs.id);
|
|
258
|
+
workingObservations.push(obs);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
// Add unified observations (from this compaction's LLM call)
|
|
263
|
+
for (const obs of unifiedObservations) {
|
|
264
|
+
if (!seenObsIds.has(obs.id)) {
|
|
265
|
+
seenObsIds.add(obs.id);
|
|
266
|
+
workingObservations.push(obs);
|
|
267
|
+
}
|
|
268
|
+
}
|
|
253
269
|
const workingReflections = migrateLegacyReflections(memoryState.reflections);
|
|
254
|
-
const workingObservations = [
|
|
255
|
-
...memoryState.committedObs,
|
|
256
|
-
...deltaObservationData.flatMap((d) => d.records),
|
|
257
|
-
];
|
|
258
270
|
const observationTokens = observationPoolTokens(workingObservations);
|
|
259
271
|
emitProgressLine(emitProgress, `Working set: ${workingObservations.length} observations, ${workingReflections.length} reflections, ~${observationTokens.toLocaleString()} observation tokens.`);
|
|
260
|
-
debugLog("compaction.
|
|
272
|
+
debugLog("compaction.working_set", {
|
|
273
|
+
observations: workingObservations.length,
|
|
274
|
+
reflections: workingReflections.length,
|
|
261
275
|
observationTokens,
|
|
262
|
-
reflectionThresholdTokens: runtime.config.reflectionThresholdTokens,
|
|
263
|
-
willRun: observationTokens >= runtime.config.reflectionThresholdTokens,
|
|
264
|
-
workingObservations: workingObservations.length,
|
|
265
|
-
workingReflections: workingReflections.length,
|
|
266
276
|
});
|
|
277
|
+
// ================================================================
|
|
278
|
+
// REFLECTOR (1 pass): crystallize durable reflections from
|
|
279
|
+
// the observation pool. Preserves quality-critical iteration
|
|
280
|
+
// via agentLoop with record_reflections tool.
|
|
281
|
+
// ================================================================
|
|
267
282
|
let finalReflections = workingReflections;
|
|
268
283
|
let finalObservations = workingObservations;
|
|
269
284
|
if (observationTokens >= runtime.config.reflectionThresholdTokens) {
|
|
270
285
|
try {
|
|
271
|
-
debugLog("compaction.
|
|
286
|
+
debugLog("compaction.reflect.start", {
|
|
272
287
|
workingObservations: workingObservations.length,
|
|
273
288
|
workingReflections: workingReflections.length,
|
|
274
289
|
observationTokens,
|
|
275
290
|
});
|
|
276
291
|
if (hasUI)
|
|
277
|
-
ui?.notify("Observational memory: running reflector
|
|
278
|
-
emitProgressLine(emitProgress, "Running reflector
|
|
292
|
+
ui?.notify("Observational memory: running reflector (1 pass)...", "info");
|
|
293
|
+
emitProgressLine(emitProgress, "Running reflector (1 pass)…");
|
|
279
294
|
const coverageBefore = coverageTagCounts(workingReflections, workingObservations);
|
|
280
295
|
const reflectorResult = await runReflector({
|
|
281
296
|
model: resolved.model,
|
|
@@ -294,39 +309,49 @@ export function registerCompactionHook(ext, runtime) {
|
|
|
294
309
|
beforeReflections: workingReflections.length,
|
|
295
310
|
afterReflections: finalReflections.length,
|
|
296
311
|
});
|
|
297
|
-
const prunerResult = await runPruner({
|
|
298
|
-
model: resolved.model,
|
|
299
|
-
apiKey: resolved.apiKey,
|
|
300
|
-
headers: resolved.headers,
|
|
301
|
-
signal,
|
|
302
|
-
onEvent: (agentEvent) => emitAgentLoopProgress(emitProgress, "pruner", agentEvent),
|
|
303
|
-
maxTurns: turnLimits.prunerMaxTurnsPerPass,
|
|
304
|
-
}, finalReflections, workingObservations, runtime.config.reflectionThresholdTokens, (pass, maxPasses) => emitProgressLine(emitProgress, `Pruner pass ${pass}/${maxPasses} started.`));
|
|
305
|
-
finalObservations = prunerResult.observations;
|
|
306
|
-
debugLog("compaction.pruner.result", {
|
|
307
|
-
stopReason: prunerResult.stopReason,
|
|
308
|
-
fellBack: prunerResult.fellBack,
|
|
309
|
-
droppedIds: prunerResult.droppedIds,
|
|
310
|
-
passes: prunerResult.passes,
|
|
311
|
-
beforeObservations: workingObservations.length,
|
|
312
|
-
afterObservations: finalObservations.length,
|
|
313
|
-
});
|
|
314
312
|
if (hasUI) {
|
|
315
|
-
ui?.notify(`Observational memory:
|
|
316
|
-
}
|
|
317
|
-
if (prunerResult.fellBack && hasUI) {
|
|
318
|
-
ui?.notify("Observational memory: pruner run failed; kept observation set unchanged", "warning");
|
|
313
|
+
ui?.notify(`Observational memory: reflector — ${formatReflectorStats(reflectorResult.stats)}; coverage ${formatCoverageCounts(coverageBefore)} → ${formatCoverageCounts(coverageAfter)}`, "info");
|
|
319
314
|
}
|
|
320
315
|
}
|
|
321
316
|
catch (error) {
|
|
322
317
|
const msg = error instanceof Error ? error.message : String(error);
|
|
323
|
-
debugLog("compaction.
|
|
318
|
+
debugLog("compaction.reflector.error", { errorMessage: msg });
|
|
324
319
|
if (hasUI)
|
|
325
|
-
ui?.notify(`Observational memory:
|
|
320
|
+
ui?.notify(`Observational memory: reflector failed: ${msg}`, "warning");
|
|
326
321
|
}
|
|
327
322
|
}
|
|
323
|
+
// ================================================================
|
|
324
|
+
// DETERMINISTIC PRUNER: sort-based dropping instead of LLM agent.
|
|
325
|
+
// Same logic as the LLM pruner (coverage → relevance → age)
|
|
326
|
+
// but zero-cost and deterministic.
|
|
327
|
+
// ================================================================
|
|
328
|
+
const prunerResult = deterministicPrune(finalObservations, finalReflections, runtime.config.reflectionThresholdTokens);
|
|
329
|
+
finalObservations = prunerResult.observations;
|
|
330
|
+
debugLog("compaction.pruner.result", {
|
|
331
|
+
fellBack: prunerResult.fellBack,
|
|
332
|
+
droppedIds: prunerResult.droppedIds,
|
|
333
|
+
tokensBefore: prunerResult.tokensBefore,
|
|
334
|
+
tokensAfter: prunerResult.tokensAfter,
|
|
335
|
+
beforeObservations: workingObservations.length,
|
|
336
|
+
afterObservations: finalObservations.length,
|
|
337
|
+
});
|
|
338
|
+
emitProgressLine(emitProgress, `Deterministic pruner dropped ${prunerResult.droppedIds.length} observation${prunerResult.droppedIds.length === 1 ? "" : "s"} (~${prunerResult.tokensBefore.toLocaleString()} → ~${prunerResult.tokensAfter.toLocaleString()} tokens).`);
|
|
339
|
+
if (prunerResult.fellBack && hasUI) {
|
|
340
|
+
ui?.notify("Observational memory: pruner could not reach budget (too many protected observations)", "warning");
|
|
341
|
+
}
|
|
342
|
+
// ================================================================
|
|
343
|
+
// BUILD FINAL SUMMARY: narrative (from unified call) +
|
|
344
|
+
// observations/reflections markdown (from renderSummary).
|
|
345
|
+
// This gives the LLM BOTH narrative context AND factual memory.
|
|
346
|
+
// ================================================================
|
|
328
347
|
emitProgressLine(emitProgress, `Rendering final memory summary from ${finalObservations.length} observations and ${finalReflections.length} reflections…`);
|
|
329
|
-
const
|
|
348
|
+
const memoryMarkdown = renderSummary(finalReflections, finalObservations);
|
|
349
|
+
// Combine: narrative summary first (what happened), then memory
|
|
350
|
+
// (what was learned). Previous compaction's narrative is already
|
|
351
|
+
// merged into the unified call's output via previousSummary.
|
|
352
|
+
const summary = unifiedNarrativeSummary
|
|
353
|
+
? `${unifiedNarrativeSummary}\n\n---\n\n${memoryMarkdown}`
|
|
354
|
+
: memoryMarkdown;
|
|
330
355
|
if (finalObservations.length === 0) {
|
|
331
356
|
throw new Error("invariant violated: finalObservations empty after delta guard");
|
|
332
357
|
}
|
|
@@ -341,6 +366,7 @@ export function registerCompactionHook(ext, runtime) {
|
|
|
341
366
|
finalReflections: finalReflections.length,
|
|
342
367
|
firstKeptEntryId,
|
|
343
368
|
tokensBefore,
|
|
369
|
+
hasNarrative: !!unifiedNarrativeSummary,
|
|
344
370
|
});
|
|
345
371
|
emitProgressLine(emitProgress, `Compaction assembled — ${finalObservations.length} observation${finalObservations.length === 1 ? "" : "s"}, ${finalReflections.length} reflection${finalReflections.length === 1 ? "" : "s"}.`);
|
|
346
372
|
if (hasUI)
|
|
@@ -364,10 +390,10 @@ export function registerCompactionHook(ext, runtime) {
|
|
|
364
390
|
function persistProjectObservations(ctx, reflections) {
|
|
365
391
|
const store = getProjectObsStore();
|
|
366
392
|
if (!store)
|
|
367
|
-
return;
|
|
393
|
+
return;
|
|
368
394
|
const projectId = store.getProjectByCwd(ctx.cwd);
|
|
369
395
|
if (!projectId)
|
|
370
|
-
return;
|
|
396
|
+
return;
|
|
371
397
|
const sessionId = ctx.sessionManager.getSessionId?.();
|
|
372
398
|
if (!sessionId)
|
|
373
399
|
return;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"prompts.d.ts","sourceRoot":"","sources":["../../src/memory/prompts.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,aAAa,+QAA+Q,CAAC;AAE1S,eAAO,MAAM,yBAAyB,6oHAmD+F,CAAC;AAEtI,eAAO,MAAM,0BAA0B,67CAkB+F,CAAC;AAEvI,eAAO,MAAM,gBAAgB,8+CAasI,CAAC;AAEpK,eAAO,MAAM,eAAe,knTAoC2G,CAAC;AAExI,eAAO,MAAM,gBAAgB,69UA0DwH,CAAC;AAEtJ,eAAO,MAAM,aAAa,u2cAgFuP,CAAC;
|
|
1
|
+
{"version":3,"file":"prompts.d.ts","sourceRoot":"","sources":["../../src/memory/prompts.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,aAAa,+QAA+Q,CAAC;AAE1S,eAAO,MAAM,yBAAyB,6oHAmD+F,CAAC;AAEtI,eAAO,MAAM,0BAA0B,67CAkB+F,CAAC;AAEvI,eAAO,MAAM,gBAAgB,8+CAasI,CAAC;AAEpK,eAAO,MAAM,eAAe,knTAoC2G,CAAC;AAExI,eAAO,MAAM,gBAAgB,69UA0DwH,CAAC;AAEtJ,eAAO,MAAM,aAAa,u2cAgFuP,CAAC;AAiBlR,wBAAgB,0BAA0B,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAIlF;AASD,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAG/E;AAED,eAAO,MAAM,0BAA0B,07BAOoS,CAAC"}
|
package/dist/memory/prompts.js
CHANGED
|
@@ -261,11 +261,20 @@ What you CANNOT do:
|
|
|
261
261
|
It is valid to end a pass with zero drops if the pool genuinely has nothing more to cut — a follow-up pass will be skipped when a run returns zero drops. On late pressure passes, first re-check old [coverage: reinforced] and [coverage: cited] observations as active-memory redundancies before deciding there are no sound drops. Do not force drops you don't believe in.
|
|
262
262
|
|
|
263
263
|
Remember: pruning is active-memory management, not source deletion. A drop that looks reasonable at "low" still becomes a mistake if the content was a user correction with a mis-labeled relevance and no reflection captures it with equivalent fidelity. Read before you cut.`;
|
|
264
|
+
const REFLECTOR_UNIFIED_PASS = `Pass strategy — combined broad synthesis + atomic facts in a single pass.
|
|
265
|
+
|
|
266
|
+
Phase 1 — Multi-observation synthesis: First crystallize broad durable patterns, repeated preferences, recurring constraints, stable work style, and project-level themes supported by multiple observations. Every reflection in this phase must cite at least 2 distinct supportingObservationIds whose durable meaning is captured by the reflection. If an existing reflection already captures the pattern, repeat the exact same content when adding support ids materially improves active-memory coverage.
|
|
267
|
+
|
|
268
|
+
Phase 2 — Atomic durable facts + safety review: Then capture important durable facts that may be supported by a single authoritative observation: explicit user preferences, hard constraints, corrections, decisions, completed milestones, project facts, release or rollback caveats, important technical context, and other load-bearing facts future agents must not forget. Review high and critical observations against reflections created in phase 1, and catch durable information still missing. Strengthen existing or newly-created reflections by repeating exact reflection content with additional supportingObservationIds for high, critical, and old medium observations whose durable meaning is already captured by that reflection.
|
|
269
|
+
|
|
270
|
+
Do not duplicate earlier reflections; repeat exact existing content only to add missing support or promote no-provenance legacy memory. Create new reflections only when durable meaning is still missing. Do not create low-quality reflections just for coverage, and do not attach observations whose unique exact detail or current task state is not captured with equivalent fidelity.`;
|
|
264
271
|
const REFLECTOR_PASS_STRATEGIES = {
|
|
265
272
|
1: `Pass strategy — multi-observation synthesis. Find broad durable patterns, repeated preferences, recurring constraints, stable work style, and project-level themes supported by multiple observations. Every reflection recorded in this pass must cite at least 2 distinct supportingObservationIds whose durable meaning is captured by the reflection. Do not create one-off event summaries; leave important single-observation facts, safety review, and coverage strengthening for the final reflector pass. If an existing reflection already captures the pattern, repeat the exact same content when adding support ids materially improves active-memory coverage.`,
|
|
266
273
|
2: `Pass strategy — final atomic durable facts, safety review, and coverage strengthening. Capture important durable facts that may be supported by a single authoritative observation: explicit user preferences, hard constraints, corrections, decisions, completed milestones, project facts, release or rollback caveats, important technical context, and other load-bearing facts future agents must not forget. Review high and critical observations against current reflections, including reflections created in pass 1, and catch durable information still missing. Strengthen existing or newly-created reflections by repeating exact reflection content with additional supportingObservationIds for high, critical, and old medium observations whose durable meaning is already captured by that reflection. Do not duplicate earlier reflections; repeat exact existing content only to add missing support or promote no-provenance legacy memory. Create new reflections only when durable meaning is still missing. Do not create low-quality reflections just for coverage, and do not attach observations whose unique exact detail or current task state is not captured with equivalent fidelity.`,
|
|
267
274
|
};
|
|
268
275
|
export function buildReflectorPassGuidance(pass, maxPasses) {
|
|
276
|
+
if (maxPasses === 1)
|
|
277
|
+
return REFLECTOR_UNIFIED_PASS;
|
|
269
278
|
const tier = Math.min(2, Math.max(1, pass));
|
|
270
279
|
return `Pass ${pass} of up to ${maxPasses}. ${REFLECTOR_PASS_STRATEGIES[tier]}`;
|
|
271
280
|
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Unified Compaction — single LLM call replacing both:
|
|
3
|
+
* 1. Core spectral generateSummary() (narrative conversation summary)
|
|
4
|
+
* 2. Sync catch-up observer (observation extraction from gap entries)
|
|
5
|
+
*
|
|
6
|
+
* Uses a text-based delimited output format (compatible with any model
|
|
7
|
+
* without requiring native structured-output support):
|
|
8
|
+
*
|
|
9
|
+
* ===NARRATIVE===
|
|
10
|
+
* ## Goal
|
|
11
|
+
* ...
|
|
12
|
+
*
|
|
13
|
+
* ===OBSERVATIONS===
|
|
14
|
+
* [YYYY-MM-DD HH:MM] [relevance] Content line
|
|
15
|
+
* ...
|
|
16
|
+
*/
|
|
17
|
+
import type { AgentMessage, ThinkingLevel } from "../sdk/agent-core/index.js";
|
|
18
|
+
import type { Model } from "../sdk/ai/index.js";
|
|
19
|
+
import type { ObservationRecord } from "./types.js";
|
|
20
|
+
export interface UnifiedCompactionInput {
|
|
21
|
+
/** Messages to summarize (from core compaction preparation) */
|
|
22
|
+
messagesToSummarize: AgentMessage[];
|
|
23
|
+
/** Serialized gap entries as text with source entry labels */
|
|
24
|
+
gapChunk?: string;
|
|
25
|
+
/** Source entry IDs from gap entries */
|
|
26
|
+
gapSourceEntryIds?: string[];
|
|
27
|
+
/** Model to use */
|
|
28
|
+
model: Model<any>;
|
|
29
|
+
/** API key */
|
|
30
|
+
apiKey: string;
|
|
31
|
+
/** Request headers */
|
|
32
|
+
headers?: Record<string, string>;
|
|
33
|
+
/** Abort signal */
|
|
34
|
+
signal?: AbortSignal;
|
|
35
|
+
/** Previous compaction summary for iterative update */
|
|
36
|
+
previousSummary?: string;
|
|
37
|
+
/** Custom instructions for summarization focus */
|
|
38
|
+
customInstructions?: string;
|
|
39
|
+
/** Thinking level for reasoning models */
|
|
40
|
+
thinkingLevel?: ThinkingLevel;
|
|
41
|
+
/** Reserve tokens for prompt + output */
|
|
42
|
+
reserveTokens: number;
|
|
43
|
+
}
|
|
44
|
+
export interface UnifiedCompactionOutput {
|
|
45
|
+
/** Markdown narrative summary (for CompactionEntry.summary) */
|
|
46
|
+
narrativeSummary: string;
|
|
47
|
+
/** Observations extracted from the conversation */
|
|
48
|
+
observations: ObservationRecord[];
|
|
49
|
+
/** Files read */
|
|
50
|
+
readFiles: string[];
|
|
51
|
+
/** Files modified */
|
|
52
|
+
modifiedFiles: string[];
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Generate a unified compaction result: narrative summary + observations
|
|
56
|
+
* in a single LLM call. Uses delimited text output for broad model compatibility.
|
|
57
|
+
*/
|
|
58
|
+
export declare function generateUnifiedCompaction(input: UnifiedCompactionInput): Promise<UnifiedCompactionOutput>;
|
|
59
|
+
//# sourceMappingURL=unified-compaction.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"unified-compaction.d.ts","sourceRoot":"","sources":["../../src/memory/unified-compaction.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC9E,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAUhD,OAAO,KAAK,EAAE,iBAAiB,EAAa,MAAM,YAAY,CAAC;AAkO/D,MAAM,WAAW,sBAAsB;IACtC,+DAA+D;IAC/D,mBAAmB,EAAE,YAAY,EAAE,CAAC;IACpC,8DAA8D;IAC9D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,wCAAwC;IACxC,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,mBAAmB;IACnB,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;IAClB,cAAc;IACd,MAAM,EAAE,MAAM,CAAC;IACf,sBAAsB;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,mBAAmB;IACnB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,uDAAuD;IACvD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,kDAAkD;IAClD,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,0CAA0C;IAC1C,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,yCAAyC;IACzC,aAAa,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,uBAAuB;IACvC,+DAA+D;IAC/D,gBAAgB,EAAE,MAAM,CAAC;IACzB,mDAAmD;IACnD,YAAY,EAAE,iBAAiB,EAAE,CAAC;IAClC,iBAAiB;IACjB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,qBAAqB;IACrB,aAAa,EAAE,MAAM,EAAE,CAAC;CACxB;AAED;;;GAGG;AACH,wBAAsB,yBAAyB,CAC9C,KAAK,EAAE,sBAAsB,GAC3B,OAAO,CAAC,uBAAuB,CAAC,CA0IlC"}
|