@matthewfl/pi-contemplator 0.0.9 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -11
- package/package.json +8 -6
- package/src/agents/contemplator/agent.ts +325 -91
- package/src/agents/contemplator/prompts.ts +6 -6
- package/src/agents/observer/agent.ts +14 -6
- package/src/agents/observer/prompts.ts +16 -7
- package/src/agents/reviewer/agent.ts +24 -4
- package/src/agents/reviewer/prompts.ts +1 -1
- package/src/agents/reviewer/tools.ts +24 -9
- package/src/agents/stream-errors.ts +1 -1
- package/src/agents/summarizer/agent.ts +597 -0
- package/src/agents/summarizer/prompts.ts +46 -0
- package/src/agents/summarizer/sampling.ts +80 -0
- package/src/commands/contemplator-view.ts +22 -1
- package/src/commands/settings.ts +73 -69
- package/src/commands/status.ts +60 -36
- package/src/commands/summarizer-view.ts +58 -0
- package/src/commands/view.ts +22 -10
- package/src/config.ts +25 -32
- package/src/hooks/compaction-hook.ts +36 -19
- package/src/hooks/compaction-resume.ts +4 -4
- package/src/hooks/compaction-trigger.ts +96 -56
- package/src/hooks/consolidation-trigger.ts +213 -196
- package/src/memory-citations.ts +37 -0
- package/src/required-tool-choice.ts +28 -0
- package/src/runtime.ts +116 -33
- package/src/session-ledger/fold.ts +82 -53
- package/src/session-ledger/index.ts +1 -0
- package/src/session-ledger/pools.ts +77 -0
- package/src/session-ledger/progress.ts +7 -18
- package/src/session-ledger/projection.ts +45 -177
- package/src/session-ledger/recall.ts +129 -127
- package/src/session-ledger/render-summary.ts +20 -19
- package/src/session-ledger/search.ts +99 -115
- package/src/session-ledger/types.ts +102 -75
- package/src/tools/compact-context.ts +1 -1
- package/src/tools/recall-observation.ts +99 -459
- package/src/tools/search-memories.ts +31 -72
- package/src/agents/dropper/agent.ts +0 -291
- package/src/agents/dropper/coverage.ts +0 -128
- package/src/agents/dropper/pool.ts +0 -67
- package/src/agents/dropper/prompts.ts +0 -48
- package/src/agents/reflector/agent.ts +0 -213
- package/src/agents/reflector/prompts.ts +0 -81
package/src/runtime.ts
CHANGED
|
@@ -6,7 +6,7 @@ export type ResolveResult =
|
|
|
6
6
|
|
|
7
7
|
type NotifyLevel = "warning" | "info" | "error";
|
|
8
8
|
type Notify = (message: string, type?: NotifyLevel) => void;
|
|
9
|
-
export type ConsolidationPhase = "observer"
|
|
9
|
+
export type ConsolidationPhase = "observer";
|
|
10
10
|
|
|
11
11
|
export const OM_SETTINGS = "om.settings";
|
|
12
12
|
|
|
@@ -16,27 +16,18 @@ function isConfiguredModel(value: unknown): value is ConfiguredModel {
|
|
|
16
16
|
return typeof model.provider === "string" && model.provider.length > 0 && typeof model.id === "string" && model.id.length > 0;
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
-
function normalizeSessionSettings(settings: SessionSettings,
|
|
20
|
-
|
|
21
|
-
if (normalized.observationsPoolMaxTokens !== undefined && normalized.observationsPoolMaxTokens < 2) {
|
|
22
|
-
delete normalized.observationsPoolMaxTokens;
|
|
23
|
-
}
|
|
24
|
-
const maxTokens = normalized.observationsPoolMaxTokens ?? baseConfig.observationsPoolMaxTokens;
|
|
25
|
-
const targetTokens = normalized.observationsPoolTargetTokens ?? baseConfig.observationsPoolTargetTokens;
|
|
26
|
-
if (normalized.observationsPoolMaxTokens !== undefined && targetTokens >= maxTokens) {
|
|
27
|
-
normalized.observationsPoolTargetTokens = Math.floor(maxTokens / 2);
|
|
28
|
-
} else if (normalized.observationsPoolTargetTokens !== undefined && normalized.observationsPoolTargetTokens >= maxTokens) {
|
|
29
|
-
delete normalized.observationsPoolTargetTokens;
|
|
30
|
-
}
|
|
31
|
-
return normalized;
|
|
19
|
+
function normalizeSessionSettings(settings: SessionSettings, _baseConfig: Config): SessionSettings {
|
|
20
|
+
return { ...settings };
|
|
32
21
|
}
|
|
33
22
|
|
|
34
23
|
export type SessionSettings = Partial<Pick<Config,
|
|
35
|
-
| "observeAfterTokens" | "
|
|
24
|
+
| "observeAfterTokens" | "observerChunkMaxTokens" | "compactAfterTokens"
|
|
36
25
|
| "compactAfterTokensMode" | "compactAfterTokensRatio"
|
|
37
|
-
| "
|
|
26
|
+
| "newMemoryPoolMaxTokens" | "oldMemoryPoolTargetTokens" | "agentMaxTurns"
|
|
38
27
|
| "showWorkerNotifications" | "passive" | "compactionObserverEnabled" | "contemplatorEnabled" | "showContemplatorMessages" | "reviewerEnabled"
|
|
39
|
-
| "contemplatorMinNewObservations" | "
|
|
28
|
+
| "contemplatorMinNewObservations" | "contemplatorMinNewSummaries" | "contemplatorMinTurns"
|
|
29
|
+
| "summarizerEnabled" | "summarizerRetriggerTokens" | "summarizerSamplingThresholdTokens"
|
|
30
|
+
| "debugLog"
|
|
40
31
|
>> & {
|
|
41
32
|
/** null explicitly means use the configured/session model. */
|
|
42
33
|
model?: ConfiguredModel | null;
|
|
@@ -63,6 +54,30 @@ export interface MemoryUpdateCtx extends LaunchCtx {
|
|
|
63
54
|
sessionManager: { getBranch(): readonly unknown[] };
|
|
64
55
|
}
|
|
65
56
|
|
|
57
|
+
export type SettingsUpdate = Partial<SessionSettings>;
|
|
58
|
+
|
|
59
|
+
export interface SummarizerRunView {
|
|
60
|
+
startedAt: number;
|
|
61
|
+
completedAt?: number;
|
|
62
|
+
status: "running" | "completed" | "incomplete" | "failed";
|
|
63
|
+
messages: readonly unknown[];
|
|
64
|
+
summary?: string;
|
|
65
|
+
error?: string;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export interface ContemplatorRunState {
|
|
69
|
+
running: boolean;
|
|
70
|
+
pendingObservations: number;
|
|
71
|
+
pendingSummaries: number;
|
|
72
|
+
pendingReviews: number;
|
|
73
|
+
/** Completed primary-model responses since the previous contemplator run. */
|
|
74
|
+
responsesSinceRun: number;
|
|
75
|
+
waitingFor: "disabled" | "passive" | "memories" | "responses" | "ready" | "running" | "idle";
|
|
76
|
+
lastStartedAt?: number;
|
|
77
|
+
lastCompletedAt?: number;
|
|
78
|
+
lastError?: string;
|
|
79
|
+
}
|
|
80
|
+
|
|
66
81
|
export interface LlmUsageTotals {
|
|
67
82
|
input: number;
|
|
68
83
|
output: number;
|
|
@@ -108,12 +123,13 @@ export function computeSessionSettings(entries: readonly unknown[]): SessionSett
|
|
|
108
123
|
if (!source || typeof source !== "object") return;
|
|
109
124
|
const data = source as Record<string, unknown>;
|
|
110
125
|
const booleanKeys = [
|
|
111
|
-
"showWorkerNotifications", "passive", "compactionObserverEnabled", "contemplatorEnabled", "showContemplatorMessages", "reviewerEnabled", "debugLog",
|
|
126
|
+
"showWorkerNotifications", "passive", "compactionObserverEnabled", "contemplatorEnabled", "showContemplatorMessages", "reviewerEnabled", "summarizerEnabled", "debugLog",
|
|
112
127
|
] as const;
|
|
113
128
|
const numberKeys = [
|
|
114
|
-
"observeAfterTokens", "
|
|
115
|
-
"
|
|
116
|
-
"contemplatorMinNewObservations", "
|
|
129
|
+
"observeAfterTokens", "observerChunkMaxTokens", "compactAfterTokens",
|
|
130
|
+
"newMemoryPoolMaxTokens", "oldMemoryPoolTargetTokens", "agentMaxTurns",
|
|
131
|
+
"contemplatorMinNewObservations", "contemplatorMinNewSummaries", "contemplatorMinTurns",
|
|
132
|
+
"summarizerRetriggerTokens", "summarizerSamplingThresholdTokens",
|
|
117
133
|
] as const;
|
|
118
134
|
for (const key of booleanKeys) if (typeof data[key] === "boolean") restored[key] = data[key];
|
|
119
135
|
for (const key of numberKeys) if (typeof data[key] === "number" && Number.isInteger(data[key]) && data[key] > 0) restored[key] = data[key];
|
|
@@ -140,25 +156,51 @@ export class Runtime {
|
|
|
140
156
|
consolidationPromise: Promise<void> | null = null;
|
|
141
157
|
reviewInFlight = false;
|
|
142
158
|
reviewPromise: Promise<void> | null = null;
|
|
159
|
+
/**
|
|
160
|
+
* Process-local single-flight lock for this session runtime. Every launch path
|
|
161
|
+
* must go through launchSummarizerTask; a second summarizer cannot start until
|
|
162
|
+
* the tracked promise's finally handler releases this lock.
|
|
163
|
+
*/
|
|
164
|
+
summarizerInFlight = false;
|
|
165
|
+
summarizerPromise: Promise<void> | null = null;
|
|
166
|
+
/** Old-pool token threshold for the next pass; undefined means configured target. */
|
|
167
|
+
summarizerNextTriggerTokens: number | undefined;
|
|
143
168
|
private memoryUpdateListener: ((ctx: MemoryUpdateCtx) => void) | undefined;
|
|
169
|
+
private agentActivityListener: ((ctx: MemoryUpdateCtx) => void) | undefined;
|
|
170
|
+
private settingsUpdateListener: ((ctx: MemoryUpdateCtx, settings: SettingsUpdate) => void) | undefined;
|
|
144
171
|
private contextGeneration = 0;
|
|
145
172
|
consolidationPhase: ConsolidationPhase | undefined;
|
|
146
173
|
compactInFlight = false;
|
|
147
174
|
compactRequested = false;
|
|
148
175
|
/** Agent-authored instructions to deliver after an explicit compact_context request. */
|
|
149
176
|
compactContinuationPrompt: string | undefined;
|
|
150
|
-
compactOrigin: "proactive" | "agent-requested" | undefined;
|
|
177
|
+
compactOrigin: "proactive" | "agent-requested" | "length-stop" | undefined;
|
|
151
178
|
compactHookInFlight = false;
|
|
152
179
|
compactionResumePending = false;
|
|
153
180
|
compactionResumeGeneration = 0;
|
|
154
181
|
compactionResumeTimer: ReturnType<typeof setTimeout> | undefined;
|
|
155
182
|
resolveFailureNotified = false;
|
|
156
183
|
lastObserverError: string | undefined;
|
|
157
|
-
|
|
158
|
-
|
|
184
|
+
lastSummarizerError: string | undefined;
|
|
185
|
+
/** Wall-clock worker boundaries for launch-local status diagnostics. */
|
|
186
|
+
lastObserverStartedAt: number | undefined;
|
|
187
|
+
lastObserverCompletedAt: number | undefined;
|
|
188
|
+
lastSummarizerStartedAt: number | undefined;
|
|
189
|
+
lastSummarizerCompletedAt: number | undefined;
|
|
190
|
+
/** Most recent summarizer transcript in this extension launch/session context. */
|
|
191
|
+
lastSummarizerRun: SummarizerRunView | undefined;
|
|
192
|
+
/** Launch-local liveness and trigger diagnostics published by the contemplator. */
|
|
193
|
+
contemplatorState: ContemplatorRunState = {
|
|
194
|
+
running: false,
|
|
195
|
+
pendingObservations: 0,
|
|
196
|
+
pendingSummaries: 0,
|
|
197
|
+
pendingReviews: 0,
|
|
198
|
+
responsesSinceRun: 0,
|
|
199
|
+
waitingFor: "idle",
|
|
200
|
+
};
|
|
159
201
|
agentUsage: LlmUsageTotals = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, cost: 0, runs: 0 };
|
|
160
202
|
|
|
161
|
-
/** Accumulate usage from one background LLM call
|
|
203
|
+
/** Accumulate usage from one background LLM call. */
|
|
162
204
|
recordAgentUsage(usage: LlmUsageInput): void {
|
|
163
205
|
const totals = this.agentUsage;
|
|
164
206
|
totals.input += usage.input ?? 0;
|
|
@@ -220,6 +262,20 @@ export class Runtime {
|
|
|
220
262
|
this.compactionResumeGeneration += 1;
|
|
221
263
|
if (this.compactionResumeTimer !== undefined) clearTimeout(this.compactionResumeTimer);
|
|
222
264
|
this.compactionResumeTimer = undefined;
|
|
265
|
+
this.summarizerNextTriggerTokens = undefined;
|
|
266
|
+
this.lastObserverStartedAt = undefined;
|
|
267
|
+
this.lastObserverCompletedAt = undefined;
|
|
268
|
+
this.lastSummarizerStartedAt = undefined;
|
|
269
|
+
this.lastSummarizerCompletedAt = undefined;
|
|
270
|
+
this.lastSummarizerRun = undefined;
|
|
271
|
+
this.contemplatorState = {
|
|
272
|
+
running: false,
|
|
273
|
+
pendingObservations: 0,
|
|
274
|
+
pendingSummaries: 0,
|
|
275
|
+
pendingReviews: 0,
|
|
276
|
+
responsesSinceRun: 0,
|
|
277
|
+
waitingFor: "idle",
|
|
278
|
+
};
|
|
223
279
|
}
|
|
224
280
|
|
|
225
281
|
getContextGeneration(): number {
|
|
@@ -237,7 +293,7 @@ export class Runtime {
|
|
|
237
293
|
model = configured;
|
|
238
294
|
} else if (ctx.hasUI && ctx.ui) {
|
|
239
295
|
ctx.ui.notify(
|
|
240
|
-
`
|
|
296
|
+
`pi-contemplator: configured model ${configuredModel.provider}/${configuredModel.id} not found, using session model`,
|
|
241
297
|
"warning",
|
|
242
298
|
);
|
|
243
299
|
}
|
|
@@ -259,12 +315,26 @@ export class Runtime {
|
|
|
259
315
|
this.memoryUpdateListener?.(ctx);
|
|
260
316
|
}
|
|
261
317
|
|
|
318
|
+
setAgentActivityListener(listener: (ctx: MemoryUpdateCtx) => void): void {
|
|
319
|
+
this.agentActivityListener = listener;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
notifyAgentActivity(ctx: MemoryUpdateCtx): void {
|
|
323
|
+
this.agentActivityListener?.(ctx);
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
setSettingsUpdateListener(listener: (ctx: MemoryUpdateCtx, settings: SettingsUpdate) => void): void {
|
|
327
|
+
this.settingsUpdateListener = listener;
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
notifySettingsUpdate(ctx: MemoryUpdateCtx, settings: SettingsUpdate): void {
|
|
331
|
+
this.settingsUpdateListener?.(ctx, settings);
|
|
332
|
+
}
|
|
333
|
+
|
|
262
334
|
launchConsolidationTask(ctx: LaunchCtx, work: () => Promise<void>): Promise<void> {
|
|
263
335
|
this.consolidationInFlight = true;
|
|
264
336
|
this.consolidationPhase = undefined;
|
|
265
337
|
this.lastObserverError = undefined;
|
|
266
|
-
this.lastReflectorError = undefined;
|
|
267
|
-
this.lastDropperError = undefined;
|
|
268
338
|
const promise = this.launchTrackedTask(ctx, "consolidation", work, () => {
|
|
269
339
|
this.consolidationInFlight = false;
|
|
270
340
|
this.consolidationPhase = undefined;
|
|
@@ -274,6 +344,21 @@ export class Runtime {
|
|
|
274
344
|
return promise;
|
|
275
345
|
}
|
|
276
346
|
|
|
347
|
+
launchSummarizerTask(ctx: LaunchCtx, work: () => Promise<void>): Promise<void> | undefined {
|
|
348
|
+
// This is the authoritative single-flight gate, not merely a UI flag.
|
|
349
|
+
// Keep it here even though callers also avoid redundant launch attempts.
|
|
350
|
+
if (this.summarizerInFlight) return undefined;
|
|
351
|
+
this.summarizerInFlight = true;
|
|
352
|
+
this.lastSummarizerError = undefined;
|
|
353
|
+
const promise = this.launchTrackedTask(ctx, "summarizer", work, (error) => {
|
|
354
|
+
this.summarizerInFlight = false;
|
|
355
|
+
this.lastSummarizerError = error;
|
|
356
|
+
if (this.summarizerPromise === promise) this.summarizerPromise = null;
|
|
357
|
+
});
|
|
358
|
+
this.summarizerPromise = promise;
|
|
359
|
+
return promise;
|
|
360
|
+
}
|
|
361
|
+
|
|
277
362
|
launchReviewTask(ctx: LaunchCtx, work: () => Promise<void>): Promise<void> | undefined {
|
|
278
363
|
// Structural reviews are intentionally serialized. Pending requests are
|
|
279
364
|
// persisted in the session ledger and resumed after the active task exits.
|
|
@@ -289,10 +374,8 @@ export class Runtime {
|
|
|
289
374
|
|
|
290
375
|
recordConsolidationStageError(ctx: LaunchCtx, phase: ConsolidationPhase, error: unknown): string {
|
|
291
376
|
const message = error instanceof Error ? error.message : String(error);
|
|
292
|
-
|
|
293
|
-
if (
|
|
294
|
-
if (phase === "dropper") this.lastDropperError = message;
|
|
295
|
-
if (ctx.hasUI && ctx.ui) ctx.ui.notify(`Observational memory: ${phase} failed: ${message}`, "warning");
|
|
377
|
+
this.lastObserverError = message;
|
|
378
|
+
if (ctx.hasUI && ctx.ui) ctx.ui.notify(`pi-contemplator: ${phase} failed: ${message}`, "warning");
|
|
296
379
|
return message;
|
|
297
380
|
}
|
|
298
381
|
|
|
@@ -310,7 +393,7 @@ export class Runtime {
|
|
|
310
393
|
await work();
|
|
311
394
|
} catch (error) {
|
|
312
395
|
errorMessage = error instanceof Error ? error.message : String(error);
|
|
313
|
-
if (hasUI && ui) ui.notify(`
|
|
396
|
+
if (hasUI && ui) ui.notify(`pi-contemplator: ${label} failed: ${errorMessage}`, "warning");
|
|
314
397
|
} finally {
|
|
315
398
|
onFinally(errorMessage);
|
|
316
399
|
}
|
|
@@ -1,16 +1,15 @@
|
|
|
1
1
|
import {
|
|
2
|
-
|
|
2
|
+
isMemoryDetails,
|
|
3
3
|
isObservationsRecordedData,
|
|
4
|
-
|
|
5
|
-
|
|
4
|
+
isReviewResultEntry,
|
|
5
|
+
isSummarizerCommitData,
|
|
6
6
|
OM_OBSERVATIONS_RECORDED,
|
|
7
|
-
OM_REFLECTIONS_RECORDED,
|
|
8
7
|
OM_REVIEW_RESULT,
|
|
9
|
-
|
|
8
|
+
OM_SUMMARIZER_COMMIT,
|
|
10
9
|
type Entry,
|
|
11
10
|
type Observation,
|
|
12
|
-
type Reflection,
|
|
13
11
|
type ReviewResult,
|
|
12
|
+
type Summary,
|
|
14
13
|
} from "./types.js";
|
|
15
14
|
|
|
16
15
|
export type FoldLedgerOptions = {
|
|
@@ -19,19 +18,18 @@ export type FoldLedgerOptions = {
|
|
|
19
18
|
};
|
|
20
19
|
|
|
21
20
|
export type FoldedLedger = {
|
|
22
|
-
/** All first-valid observation records
|
|
21
|
+
/** All first-valid durable observation records through the fold boundary. */
|
|
23
22
|
observations: Observation[];
|
|
24
|
-
/** Observation records not tombstoned by a folded drop entry. */
|
|
25
23
|
activeObservations: Observation[];
|
|
26
|
-
/**
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
reflections: Reflection[];
|
|
30
|
-
/** All first-valid observation records by id, including dropped observations. */
|
|
24
|
+
/** All first-valid durable summary records through the fold boundary. */
|
|
25
|
+
summaries: Summary[];
|
|
26
|
+
activeSummaries: Summary[];
|
|
31
27
|
observationsById: Map<string, Observation>;
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
28
|
+
summariesById: Map<string, Summary>;
|
|
29
|
+
/** Source -> every summary that cites it, including non-consuming citations. */
|
|
30
|
+
citedBySummaryIds: Map<string, string[]>;
|
|
31
|
+
/** Source -> the first summary that removed it from automatic visibility. */
|
|
32
|
+
consumedBySummaryId: Map<string, string>;
|
|
35
33
|
reviews: ReviewResult[];
|
|
36
34
|
reviewsById: Map<string, ReviewResult>;
|
|
37
35
|
};
|
|
@@ -46,70 +44,101 @@ function isCustomEntry(entry: Entry, customType: string): boolean {
|
|
|
46
44
|
return entry.type === "custom" && entry.customType === customType;
|
|
47
45
|
}
|
|
48
46
|
|
|
47
|
+
function appendUnique(map: Map<string, string[]>, key: string, value: string): void {
|
|
48
|
+
const existing = map.get(key);
|
|
49
|
+
if (!existing) {
|
|
50
|
+
map.set(key, [value]);
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
if (!existing.includes(value)) existing.push(value);
|
|
54
|
+
}
|
|
55
|
+
|
|
49
56
|
/**
|
|
50
|
-
* Fold
|
|
57
|
+
* Fold the append-only memory graph through a branch boundary.
|
|
51
58
|
*
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
*
|
|
59
|
+
* Summary bodies occur once. Visibility and forward pointers are derived from
|
|
60
|
+
* their source/consumption edges. Compaction archives seed the same graph when
|
|
61
|
+
* older custom records are no longer present on the current branch.
|
|
55
62
|
*/
|
|
56
63
|
export function foldLedger(entries: Entry[], options: FoldLedgerOptions = {}): FoldedLedger {
|
|
57
64
|
const observationsById = new Map<string, Observation>();
|
|
58
|
-
const
|
|
65
|
+
const summariesById = new Map<string, Summary>();
|
|
59
66
|
const reviewsById = new Map<string, ReviewResult>();
|
|
60
|
-
const
|
|
67
|
+
const citedBySummaryIds = new Map<string, string[]>();
|
|
68
|
+
const consumedBySummaryId = new Map<string, string>();
|
|
61
69
|
const endIdx = foldEndIndex(entries, options.upToEntryId);
|
|
62
70
|
|
|
71
|
+
const registerObservation = (observation: Observation): void => {
|
|
72
|
+
if (!observationsById.has(observation.id)) observationsById.set(observation.id, observation);
|
|
73
|
+
};
|
|
74
|
+
const registerReview = (review: ReviewResult): void => {
|
|
75
|
+
if (!reviewsById.has(review.id)) reviewsById.set(review.id, review);
|
|
76
|
+
};
|
|
77
|
+
const registerSummaryNodes = (summaries: readonly Summary[]): Summary[] => {
|
|
78
|
+
const newlyRegistered: Summary[] = [];
|
|
79
|
+
for (const summary of summaries) {
|
|
80
|
+
if (summariesById.has(summary.id)) continue;
|
|
81
|
+
summariesById.set(summary.id, summary);
|
|
82
|
+
newlyRegistered.push(summary);
|
|
83
|
+
}
|
|
84
|
+
return newlyRegistered;
|
|
85
|
+
};
|
|
86
|
+
const registerSummaryEdges = (summaries: readonly Summary[]): void => {
|
|
87
|
+
for (const summary of summaries) {
|
|
88
|
+
for (const sourceId of summary.sourceMemoryIds) appendUnique(citedBySummaryIds, sourceId, summary.id);
|
|
89
|
+
for (const sourceId of summary.consumedMemoryIds) {
|
|
90
|
+
// Reviews are deliberately non-consumable. Unknown/corrupt edges also
|
|
91
|
+
// cannot hide a node. The first valid consumer wins.
|
|
92
|
+
if (!observationsById.has(sourceId) && !summariesById.has(sourceId)) continue;
|
|
93
|
+
if (!consumedBySummaryId.has(sourceId)) consumedBySummaryId.set(sourceId, summary.id);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
};
|
|
97
|
+
|
|
63
98
|
for (let i = 0; i <= endIdx; i++) {
|
|
64
99
|
const entry = entries[i];
|
|
65
100
|
if (!entry) continue;
|
|
66
101
|
|
|
67
|
-
if (
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
102
|
+
if (entry.type === "compaction" && isMemoryDetails(entry.details)) {
|
|
103
|
+
const archivedObservations = entry.details.archive?.observations ?? entry.details.observations;
|
|
104
|
+
const archivedSummaries = entry.details.archive?.summaries ?? entry.details.summaries;
|
|
105
|
+
for (const observation of archivedObservations) registerObservation(observation);
|
|
106
|
+
for (const review of entry.details.reviews ?? []) registerReview(review);
|
|
107
|
+
const registered = registerSummaryNodes(archivedSummaries);
|
|
108
|
+
registerSummaryEdges(registered);
|
|
74
109
|
continue;
|
|
75
110
|
}
|
|
76
111
|
|
|
77
|
-
if (isCustomEntry(entry,
|
|
78
|
-
if (!
|
|
79
|
-
for (const
|
|
80
|
-
if (!reflectionsById.has(reflection.id)) {
|
|
81
|
-
reflectionsById.set(reflection.id, reflection);
|
|
82
|
-
}
|
|
83
|
-
}
|
|
112
|
+
if (isCustomEntry(entry, OM_OBSERVATIONS_RECORDED)) {
|
|
113
|
+
if (!isObservationsRecordedData(entry.data)) continue;
|
|
114
|
+
for (const observation of entry.data.observations) registerObservation(observation);
|
|
84
115
|
continue;
|
|
85
116
|
}
|
|
86
117
|
|
|
87
|
-
if (isCustomEntry(entry,
|
|
88
|
-
if (!
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
118
|
+
if (isCustomEntry(entry, OM_SUMMARIZER_COMMIT)) {
|
|
119
|
+
if (!isSummarizerCommitData(entry.data)) continue;
|
|
120
|
+
// Register every node before edges so same-commit citation targets are
|
|
121
|
+
// addressable. The summarizer itself prevents consuming same-run nodes.
|
|
122
|
+
const registered = registerSummaryNodes(entry.data.summaries);
|
|
123
|
+
registerSummaryEdges(registered);
|
|
92
124
|
continue;
|
|
93
125
|
}
|
|
94
126
|
|
|
95
|
-
if (entry.customType === OM_REVIEW_RESULT && isReviewResultEntry(entry)
|
|
96
|
-
reviewsById.set(entry.data.result.id, entry.data.result);
|
|
97
|
-
}
|
|
127
|
+
if (entry.customType === OM_REVIEW_RESULT && isReviewResultEntry(entry)) registerReview(entry.data.result);
|
|
98
128
|
}
|
|
99
129
|
|
|
100
130
|
const observations = Array.from(observationsById.values());
|
|
101
|
-
const
|
|
102
|
-
const reflections = Array.from(reflectionsById.values());
|
|
103
|
-
const reviews = Array.from(reviewsById.values());
|
|
104
|
-
|
|
131
|
+
const summaries = Array.from(summariesById.values());
|
|
105
132
|
return {
|
|
106
133
|
observations,
|
|
107
|
-
activeObservations,
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
reviews,
|
|
134
|
+
activeObservations: observations.filter((memory) => !consumedBySummaryId.has(memory.id)),
|
|
135
|
+
summaries,
|
|
136
|
+
activeSummaries: summaries.filter((memory) => !consumedBySummaryId.has(memory.id)),
|
|
111
137
|
observationsById,
|
|
112
|
-
|
|
138
|
+
summariesById,
|
|
139
|
+
citedBySummaryIds,
|
|
140
|
+
consumedBySummaryId,
|
|
141
|
+
reviews: Array.from(reviewsById.values()),
|
|
113
142
|
reviewsById,
|
|
114
143
|
};
|
|
115
144
|
}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import type { Observation, Summary } from "./types.js";
|
|
2
|
+
|
|
3
|
+
export type ActiveMemory =
|
|
4
|
+
| { kind: "observation"; memory: Observation }
|
|
5
|
+
| { kind: "summary"; memory: Summary };
|
|
6
|
+
|
|
7
|
+
export type MemoryPools = {
|
|
8
|
+
/** Older active memories eligible for summarization. */
|
|
9
|
+
old: ActiveMemory[];
|
|
10
|
+
/** Newest contiguous active-memory suffix protected from summarization. */
|
|
11
|
+
new: ActiveMemory[];
|
|
12
|
+
oldTokens: number;
|
|
13
|
+
newTokens: number;
|
|
14
|
+
totalTokens: number;
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
function parsedTimestamp(value: string): number {
|
|
18
|
+
const parsed = Date.parse(value);
|
|
19
|
+
return Number.isFinite(parsed) ? parsed : Number.NEGATIVE_INFINITY;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** Sort observations and summaries together by their effective memory timestamp. */
|
|
23
|
+
export function chronologicalMemories(observations: readonly Observation[], summaries: readonly Summary[]): ActiveMemory[] {
|
|
24
|
+
return [
|
|
25
|
+
...observations.map((memory) => ({ kind: "observation" as const, memory })),
|
|
26
|
+
...summaries.map((memory) => ({ kind: "summary" as const, memory })),
|
|
27
|
+
].sort((a, b) => parsedTimestamp(a.memory.timestamp) - parsedTimestamp(b.memory.timestamp) || a.memory.id.localeCompare(b.memory.id));
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Derive accounting-only pools from active memory. Pool membership is not
|
|
32
|
+
* persisted: the newest whole-memory suffix fitting the configured token cap
|
|
33
|
+
* is protected, and every older record is summarizer-eligible. Because records
|
|
34
|
+
* are indivisible, the newest record is always protected even when it alone
|
|
35
|
+
* exceeds the cap.
|
|
36
|
+
*/
|
|
37
|
+
export function partitionMemoryPools(
|
|
38
|
+
observations: readonly Observation[],
|
|
39
|
+
summaries: readonly Summary[],
|
|
40
|
+
newPoolMaxTokens: number,
|
|
41
|
+
): MemoryPools {
|
|
42
|
+
const memories = chronologicalMemories(observations, summaries);
|
|
43
|
+
const cap = Math.max(0, Math.floor(newPoolMaxTokens));
|
|
44
|
+
let boundary = memories.length;
|
|
45
|
+
let newTokens = 0;
|
|
46
|
+
for (let index = memories.length - 1; index >= 0; index--) {
|
|
47
|
+
const tokens = memories[index].memory.tokenCount;
|
|
48
|
+
const isNewest = index === memories.length - 1;
|
|
49
|
+
if (!isNewest && newTokens + tokens > cap) break;
|
|
50
|
+
newTokens += tokens;
|
|
51
|
+
boundary = index;
|
|
52
|
+
}
|
|
53
|
+
const old = memories.slice(0, boundary);
|
|
54
|
+
const recent = memories.slice(boundary);
|
|
55
|
+
const oldTokens = old.reduce((sum, item) => sum + item.memory.tokenCount, 0);
|
|
56
|
+
return {
|
|
57
|
+
old,
|
|
58
|
+
new: recent,
|
|
59
|
+
oldTokens,
|
|
60
|
+
newTokens,
|
|
61
|
+
totalTokens: oldTokens + newTokens,
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** Effective timestamp for a new summary: newest timestamp among its sources. */
|
|
66
|
+
export function latestMemoryTimestamp(memories: readonly ActiveMemory[]): string | undefined {
|
|
67
|
+
let latest: string | undefined;
|
|
68
|
+
let latestMs = Number.NEGATIVE_INFINITY;
|
|
69
|
+
for (const item of memories) {
|
|
70
|
+
const ms = parsedTimestamp(item.memory.timestamp);
|
|
71
|
+
if (latest === undefined || ms > latestMs || (ms === latestMs && item.memory.timestamp > latest)) {
|
|
72
|
+
latest = item.memory.timestamp;
|
|
73
|
+
latestMs = ms;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
return latest;
|
|
77
|
+
}
|
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
import { estimateEntryTokens } from "../tokens.js";
|
|
2
2
|
import {
|
|
3
3
|
OM_AGENT_ACTIVITY,
|
|
4
|
-
OM_OBSERVATIONS_DROPPED,
|
|
5
4
|
OM_OBSERVATIONS_RECORDED,
|
|
6
|
-
|
|
5
|
+
OM_SUMMARIZER_COMMIT,
|
|
7
6
|
type Entry,
|
|
8
|
-
type
|
|
7
|
+
type MemoryCoverageCustomType,
|
|
9
8
|
} from "./types.js";
|
|
10
9
|
|
|
11
10
|
const SOURCE_ENTRY_TYPES = new Set(["message", "custom_message", "branch_summary"]);
|
|
@@ -72,16 +71,14 @@ function isNonEmptyArray(value: unknown): value is unknown[] {
|
|
|
72
71
|
return Array.isArray(value) && value.length > 0;
|
|
73
72
|
}
|
|
74
73
|
|
|
75
|
-
function isValidCoverageEntry(entry: Entry, customType:
|
|
74
|
+
function isValidCoverageEntry(entry: Entry, customType: MemoryCoverageCustomType): entry is Entry & { data: { coversUpToId: string } } {
|
|
76
75
|
if (entry.type !== "custom" || entry.customType !== customType) return false;
|
|
77
76
|
if (!isObject(entry.data) || typeof entry.data.coversUpToId !== "string") return false;
|
|
78
|
-
|
|
79
77
|
if (customType === OM_OBSERVATIONS_RECORDED) return isNonEmptyArray(entry.data.observations);
|
|
80
|
-
|
|
81
|
-
return isNonEmptyArray(entry.data.observationIds);
|
|
78
|
+
return customType === OM_SUMMARIZER_COMMIT && isNonEmptyArray(entry.data.summaries);
|
|
82
79
|
}
|
|
83
80
|
|
|
84
|
-
export function latestCoverageIndex(entries: Entry[], customType:
|
|
81
|
+
export function latestCoverageIndex(entries: Entry[], customType: MemoryCoverageCustomType): number {
|
|
85
82
|
const idToIndex = entryIndexById(entries);
|
|
86
83
|
let latest = -1;
|
|
87
84
|
|
|
@@ -95,7 +92,7 @@ export function latestCoverageIndex(entries: Entry[], customType: V3MemoryCustom
|
|
|
95
92
|
return latest;
|
|
96
93
|
}
|
|
97
94
|
|
|
98
|
-
export function latestCoverageMarkerId(entries: Entry[], customType:
|
|
95
|
+
export function latestCoverageMarkerId(entries: Entry[], customType: MemoryCoverageCustomType): string | undefined {
|
|
99
96
|
const idToIndex = entryIndexById(entries);
|
|
100
97
|
let latestIndex = -1;
|
|
101
98
|
let latestMarkerId: string | undefined;
|
|
@@ -133,7 +130,7 @@ export function rawTokensAfterIndex(entries: Entry[], index: number): number {
|
|
|
133
130
|
return total;
|
|
134
131
|
}
|
|
135
132
|
|
|
136
|
-
export function rawTokensSinceCoverage(entries: Entry[], customType:
|
|
133
|
+
export function rawTokensSinceCoverage(entries: Entry[], customType: MemoryCoverageCustomType): number {
|
|
137
134
|
return rawTokensAfterIndex(entries, latestCoverageIndex(entries, customType));
|
|
138
135
|
}
|
|
139
136
|
|
|
@@ -141,14 +138,6 @@ export function rawTokensSinceObservationCoverage(entries: Entry[]): number {
|
|
|
141
138
|
return rawTokensSinceCoverage(entries, OM_OBSERVATIONS_RECORDED);
|
|
142
139
|
}
|
|
143
140
|
|
|
144
|
-
export function rawTokensSinceReflectionCoverage(entries: Entry[]): number {
|
|
145
|
-
return rawTokensSinceCoverage(entries, OM_REFLECTIONS_RECORDED);
|
|
146
|
-
}
|
|
147
|
-
|
|
148
|
-
export function rawTokensSinceDropCoverage(entries: Entry[]): number {
|
|
149
|
-
return rawTokensSinceCoverage(entries, OM_OBSERVATIONS_DROPPED);
|
|
150
|
-
}
|
|
151
|
-
|
|
152
141
|
export function findLastCompactionIndex(entries: Entry[]): number {
|
|
153
142
|
for (let i = entries.length - 1; i >= 0; i--) {
|
|
154
143
|
if (entries[i].type === "compaction") return i;
|