@matthewfl/pi-contemplator 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +120 -0
- package/package.json +60 -0
- package/src/agents/contemplator/agent.ts +718 -0
- package/src/agents/contemplator/prompts.ts +212 -0
- package/src/agents/dropper/agent.ts +291 -0
- package/src/agents/dropper/coverage.ts +128 -0
- package/src/agents/dropper/pool.ts +67 -0
- package/src/agents/dropper/prompts.ts +48 -0
- package/src/agents/observer/agent.ts +207 -0
- package/src/agents/observer/prompts.ts +119 -0
- package/src/agents/reflector/agent.ts +213 -0
- package/src/agents/reflector/prompts.ts +81 -0
- package/src/agents/reviewer/agent.ts +187 -0
- package/src/agents/reviewer/history-tools.ts +337 -0
- package/src/agents/reviewer/prompts.ts +135 -0
- package/src/agents/reviewer/tools.ts +84 -0
- package/src/agents/stream-errors.ts +22 -0
- package/src/clipboard.ts +63 -0
- package/src/commands/contemplator-view.ts +128 -0
- package/src/commands/reviewer-view.ts +89 -0
- package/src/commands/settings.ts +257 -0
- package/src/commands/status.ts +176 -0
- package/src/commands/view.ts +171 -0
- package/src/config.ts +284 -0
- package/src/debug-log.ts +72 -0
- package/src/hooks/compaction-hook.ts +99 -0
- package/src/hooks/compaction-resume.ts +124 -0
- package/src/hooks/compaction-trigger.ts +122 -0
- package/src/hooks/consolidation-trigger.ts +488 -0
- package/src/ids.ts +5 -0
- package/src/index.ts +32 -0
- package/src/model-budget.ts +16 -0
- package/src/runtime.ts +316 -0
- package/src/serialize.ts +274 -0
- package/src/session-ledger/fold.ts +115 -0
- package/src/session-ledger/index.ts +7 -0
- package/src/session-ledger/progress.ts +156 -0
- package/src/session-ledger/projection.ts +243 -0
- package/src/session-ledger/recall.ts +258 -0
- package/src/session-ledger/render-summary.ts +31 -0
- package/src/session-ledger/search.ts +184 -0
- package/src/session-ledger/types.ts +329 -0
- package/src/tokens.ts +27 -0
- package/src/tools/compact-context.ts +54 -0
- package/src/tools/recall-observation.ts +532 -0
- package/src/tools/search-memories.ts +131 -0
package/src/config.ts
ADDED
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import type { ModelThinkingLevel } from "@earendil-works/pi-ai";
|
|
4
|
+
import { getAgentDir } from "@earendil-works/pi-coding-agent";
|
|
5
|
+
|
|
6
|
+
export interface ConfiguredModel {
|
|
7
|
+
provider: string;
|
|
8
|
+
id: string;
|
|
9
|
+
thinking?: ModelThinkingLevel;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* How `compactAfterTokens` is interpreted.
|
|
14
|
+
*
|
|
15
|
+
* - `"calibrated"` (default): use the static `compactAfterTokens` value directly.
|
|
16
|
+
* Backwards-compatible with all existing V3 configs.
|
|
17
|
+
*
|
|
18
|
+
* - `"ratio"`: compute the effective threshold as
|
|
19
|
+
* `floor(model.contextWindow * compactAfterTokensRatio)`. This auto-scales the
|
|
20
|
+
* proactive compaction trigger to the active model's context window, so a 1M
|
|
21
|
+
* context model is not preempted at the same 81K threshold as a 128K model.
|
|
22
|
+
*
|
|
23
|
+
* Some models advertise a large context window but lose attention at long
|
|
24
|
+
* range; users can lower `compactAfterTokensRatio` to compact earlier on such
|
|
25
|
+
* models without giving up the window on models that stay sharp.
|
|
26
|
+
*
|
|
27
|
+
* When the active model's `contextWindow` is unavailable (undefined, 0, or
|
|
28
|
+
* negative), ratio mode falls back to the calibrated `compactAfterTokens`
|
|
29
|
+
* value so compaction still triggers safely.
|
|
30
|
+
*/
|
|
31
|
+
export type CompactAfterTokensMode = "calibrated" | "ratio";
|
|
32
|
+
|
|
33
|
+
export interface Config {
|
|
34
|
+
observeAfterTokens: number;
|
|
35
|
+
reflectAfterTokens: number;
|
|
36
|
+
/**
|
|
37
|
+
* Maximum estimated source tokens serialized into a single observer chunk.
|
|
38
|
+
* Unset (default) derives the cap from the resolved memory model's context
|
|
39
|
+
* window; see {@link resolveObserverChunkMaxTokens}.
|
|
40
|
+
*/
|
|
41
|
+
observerChunkMaxTokens?: number;
|
|
42
|
+
compactAfterTokens: number;
|
|
43
|
+
compactAfterTokensMode: CompactAfterTokensMode;
|
|
44
|
+
compactAfterTokensRatio: number;
|
|
45
|
+
observationsPoolMaxTokens: number;
|
|
46
|
+
observationsPoolTargetTokens: number;
|
|
47
|
+
agentMaxTurns: number;
|
|
48
|
+
model?: ConfiguredModel;
|
|
49
|
+
showWorkerNotifications: boolean;
|
|
50
|
+
passive: boolean;
|
|
51
|
+
/** Run the asynchronous observer when a compaction begins. */
|
|
52
|
+
compactionObserverEnabled: boolean;
|
|
53
|
+
contemplatorEnabled: boolean;
|
|
54
|
+
contemplatorModel?: ConfiguredModel;
|
|
55
|
+
/** Allow the contemplator to commission scoped structural reviewers. */
|
|
56
|
+
reviewerEnabled: boolean;
|
|
57
|
+
/** Optional model override used only by short-lived structural reviewers. */
|
|
58
|
+
reviewerModel?: ConfiguredModel;
|
|
59
|
+
contemplatorMinNewObservations: number;
|
|
60
|
+
contemplatorMinNewReflections: number;
|
|
61
|
+
contemplatorMinTurns: number;
|
|
62
|
+
debugLog: boolean;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export const DEFAULTS: Config = {
|
|
66
|
+
observeAfterTokens: 10_000,
|
|
67
|
+
reflectAfterTokens: 20_000,
|
|
68
|
+
compactAfterTokens: 81_000,
|
|
69
|
+
compactAfterTokensMode: "calibrated",
|
|
70
|
+
compactAfterTokensRatio: 0.68,
|
|
71
|
+
observationsPoolMaxTokens: 20_000,
|
|
72
|
+
observationsPoolTargetTokens: 10_000,
|
|
73
|
+
agentMaxTurns: 16,
|
|
74
|
+
showWorkerNotifications: true,
|
|
75
|
+
passive: false,
|
|
76
|
+
compactionObserverEnabled: true,
|
|
77
|
+
contemplatorEnabled: true,
|
|
78
|
+
reviewerEnabled: true,
|
|
79
|
+
contemplatorMinNewObservations: 8,
|
|
80
|
+
contemplatorMinNewReflections: 1,
|
|
81
|
+
contemplatorMinTurns: 10,
|
|
82
|
+
debugLog: false,
|
|
83
|
+
};
|
|
84
|
+
|
|
85
|
+
export const COMPACT_AFTER_TOKENS_MODE_VALUES: readonly CompactAfterTokensMode[] = ["calibrated", "ratio"] as const;
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Resolve the effective proactive-compaction token threshold for the given
|
|
89
|
+
* config and active model context window.
|
|
90
|
+
*
|
|
91
|
+
* In `"calibrated"` mode this is always `config.compactAfterTokens`.
|
|
92
|
+
*
|
|
93
|
+
* In `"ratio"` mode this is `floor(contextWindow * compactAfterTokensRatio)`
|
|
94
|
+
* (clamped to a minimum of 1) when `contextWindow` is a positive number, and
|
|
95
|
+
* falls back to `config.compactAfterTokens` otherwise.
|
|
96
|
+
*/
|
|
97
|
+
export function resolveCompactAfterTokens(config: Config, contextWindow: number | undefined): number {
|
|
98
|
+
if (config.compactAfterTokensMode === "ratio" && typeof contextWindow === "number" && contextWindow > 0) {
|
|
99
|
+
return Math.max(1, Math.floor(contextWindow * config.compactAfterTokensRatio));
|
|
100
|
+
}
|
|
101
|
+
return config.compactAfterTokens;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
export const THINKING_LEVEL_VALUES: readonly ModelThinkingLevel[] = ["off", "minimal", "low", "medium", "high", "xhigh"] as const;
|
|
105
|
+
|
|
106
|
+
/** Observer chunk cap used when no config is set and the model's context window is unknown. */
|
|
107
|
+
export const OBSERVER_CHUNK_FALLBACK_MAX_TOKENS = 60_000;
|
|
108
|
+
|
|
109
|
+
/** Smallest useful observer chunk: enough for labels, omission markers, and source context. */
|
|
110
|
+
export const OBSERVER_CHUNK_MIN_TOKENS = 256;
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Fraction of the memory model's context window used for the derived observer
|
|
114
|
+
* chunk cap. Chunk sizes are estimated at ~4 chars/token, which can undercount
|
|
115
|
+
* real tokens by up to ~4x on non-ASCII content, so 0.2 keeps even the worst
|
|
116
|
+
* case at ~80% of the window with room left for the system prompt, prior
|
|
117
|
+
* memory, and the response.
|
|
118
|
+
*/
|
|
119
|
+
export const OBSERVER_CHUNK_CONTEXT_RATIO = 0.2;
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Resolve the maximum estimated tokens the observer serializes into one chunk.
|
|
123
|
+
*
|
|
124
|
+
* An explicit `observerChunkMaxTokens` config value always wins. Otherwise the
|
|
125
|
+
* cap is `floor(contextWindow * OBSERVER_CHUNK_CONTEXT_RATIO)` for the resolved
|
|
126
|
+
* memory model, falling back to {@link OBSERVER_CHUNK_FALLBACK_MAX_TOKENS} when
|
|
127
|
+
* the context window is unavailable.
|
|
128
|
+
*
|
|
129
|
+
* Without a cap, a backlog that outgrows the model's context window (e.g.
|
|
130
|
+
* after repeated observer failures, or when the extension is enabled mid-way
|
|
131
|
+
* into a long session) makes every observer call fail, so coverage never
|
|
132
|
+
* advances and the session can never recover. With the cap, oversized backlogs
|
|
133
|
+
* are drained oldest-first across successive runs.
|
|
134
|
+
*/
|
|
135
|
+
export function resolveObserverChunkMaxTokens(config: Config, contextWindow: number | undefined): number {
|
|
136
|
+
if (config.observerChunkMaxTokens !== undefined && config.observerChunkMaxTokens > 0) {
|
|
137
|
+
return Math.max(OBSERVER_CHUNK_MIN_TOKENS, config.observerChunkMaxTokens);
|
|
138
|
+
}
|
|
139
|
+
if (typeof contextWindow === "number" && Number.isFinite(contextWindow) && contextWindow > 0) {
|
|
140
|
+
return Math.max(
|
|
141
|
+
OBSERVER_CHUNK_MIN_TOKENS,
|
|
142
|
+
Math.floor(contextWindow * OBSERVER_CHUNK_CONTEXT_RATIO),
|
|
143
|
+
);
|
|
144
|
+
}
|
|
145
|
+
return OBSERVER_CHUNK_FALLBACK_MAX_TOKENS;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
const SETTINGS_KEY = "observational-memory";
|
|
149
|
+
const PASSIVE_ENV = "PI_OBSERVATIONAL_MEMORY_PASSIVE";
|
|
150
|
+
|
|
151
|
+
function positiveIntegerOrUndefined(value: unknown): number | undefined {
|
|
152
|
+
return Number.isInteger(value) && typeof value === "number" && value > 0 ? value : undefined;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
function validTargetOrUndefined(value: unknown, maxTokens: number): number | undefined {
|
|
156
|
+
const target = positiveIntegerOrUndefined(value);
|
|
157
|
+
return target !== undefined && target < maxTokens ? target : undefined;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
function derivedObservationPoolTarget(maxTokens: number): number {
|
|
161
|
+
return Math.floor(maxTokens / 2);
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function isThinkingLevel(value: unknown): value is ModelThinkingLevel {
|
|
165
|
+
return typeof value === "string" && (THINKING_LEVEL_VALUES as readonly string[]).includes(value);
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
function isCompactAfterTokensMode(value: unknown): value is CompactAfterTokensMode {
|
|
169
|
+
return typeof value === "string" && (COMPACT_AFTER_TOKENS_MODE_VALUES as readonly string[]).includes(value);
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* A valid ratio is a finite number strictly between 0 and 1.
|
|
174
|
+
* 0 would never trigger; >= 1 would compact at/after the full window with no
|
|
175
|
+
* room left for the response.
|
|
176
|
+
*/
|
|
177
|
+
function validRatioOrUndefined(value: unknown): number | undefined {
|
|
178
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0 && value < 1 ? value : undefined;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
182
|
+
return typeof value === "object" && value !== null;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
function nonEmptyString(value: unknown): string | undefined {
|
|
186
|
+
return typeof value === "string" && value.length > 0 ? value : undefined;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
function normalizeModel(value: unknown): ConfiguredModel | undefined {
|
|
190
|
+
if (!isRecord(value)) return undefined;
|
|
191
|
+
const provider = nonEmptyString(value.provider);
|
|
192
|
+
const id = nonEmptyString(value.id);
|
|
193
|
+
if (!provider || !id) return undefined;
|
|
194
|
+
const model: ConfiguredModel = { provider, id };
|
|
195
|
+
if (isThinkingLevel(value.thinking)) model.thinking = value.thinking;
|
|
196
|
+
return model;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
function normalizeSettingsConfig(value: Record<string, unknown>): Partial<Config> {
|
|
200
|
+
const normalized: Partial<Config> = {};
|
|
201
|
+
const numberKeys = [
|
|
202
|
+
"observeAfterTokens",
|
|
203
|
+
"reflectAfterTokens",
|
|
204
|
+
"observerChunkMaxTokens",
|
|
205
|
+
"compactAfterTokens",
|
|
206
|
+
"observationsPoolMaxTokens",
|
|
207
|
+
"observationsPoolTargetTokens",
|
|
208
|
+
"agentMaxTurns",
|
|
209
|
+
"contemplatorMinNewObservations",
|
|
210
|
+
"contemplatorMinNewReflections",
|
|
211
|
+
"contemplatorMinTurns",
|
|
212
|
+
] as const;
|
|
213
|
+
for (const key of numberKeys) {
|
|
214
|
+
const normalizedValue = positiveIntegerOrUndefined(value[key]);
|
|
215
|
+
if (normalizedValue !== undefined) normalized[key] = normalizedValue;
|
|
216
|
+
}
|
|
217
|
+
if (isCompactAfterTokensMode(value.compactAfterTokensMode)) {
|
|
218
|
+
normalized.compactAfterTokensMode = value.compactAfterTokensMode;
|
|
219
|
+
}
|
|
220
|
+
const ratio = validRatioOrUndefined(value.compactAfterTokensRatio);
|
|
221
|
+
if (ratio !== undefined) normalized.compactAfterTokensRatio = ratio;
|
|
222
|
+
if (typeof value.showWorkerNotifications === "boolean") normalized.showWorkerNotifications = value.showWorkerNotifications;
|
|
223
|
+
if (typeof value.passive === "boolean") normalized.passive = value.passive;
|
|
224
|
+
if (typeof value.compactionObserverEnabled === "boolean") normalized.compactionObserverEnabled = value.compactionObserverEnabled;
|
|
225
|
+
if (typeof value.contemplatorEnabled === "boolean") normalized.contemplatorEnabled = value.contemplatorEnabled;
|
|
226
|
+
if (typeof value.reviewerEnabled === "boolean") normalized.reviewerEnabled = value.reviewerEnabled;
|
|
227
|
+
if (typeof value.debugLog === "boolean") normalized.debugLog = value.debugLog;
|
|
228
|
+
const model = normalizeModel(value.model);
|
|
229
|
+
if (model) normalized.model = model;
|
|
230
|
+
const contemplatorModel = normalizeModel(value.contemplatorModel);
|
|
231
|
+
if (contemplatorModel) normalized.contemplatorModel = contemplatorModel;
|
|
232
|
+
const reviewerModel = normalizeModel(value.reviewerModel);
|
|
233
|
+
if (reviewerModel) normalized.reviewerModel = reviewerModel;
|
|
234
|
+
return normalized;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
export function readEnvConfig(env: NodeJS.ProcessEnv = process.env): Partial<Config> {
|
|
238
|
+
const result: Partial<Config> = {};
|
|
239
|
+
const rawPassive = env[PASSIVE_ENV];
|
|
240
|
+
if (rawPassive !== undefined) {
|
|
241
|
+
const passive = rawPassive.trim().toLowerCase();
|
|
242
|
+
if (["1", "true", "yes", "on"].includes(passive)) result.passive = true;
|
|
243
|
+
if (["0", "false", "no", "off"].includes(passive)) result.passive = false;
|
|
244
|
+
}
|
|
245
|
+
const compactionObserver = env.PI_OBSERVATIONAL_MEMORY_COMPACTION_OBSERVER?.trim().toLowerCase();
|
|
246
|
+
if (["1", "true", "yes", "on"].includes(compactionObserver ?? "")) result.compactionObserverEnabled = true;
|
|
247
|
+
if (["0", "false", "no", "off"].includes(compactionObserver ?? "")) result.compactionObserverEnabled = false;
|
|
248
|
+
return result;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
function readNamespacedConfig(path: string): Partial<Config> {
|
|
252
|
+
if (!existsSync(path)) return {};
|
|
253
|
+
try {
|
|
254
|
+
const raw = JSON.parse(readFileSync(path, "utf-8")) as Record<string, unknown>;
|
|
255
|
+
const nested = raw[SETTINGS_KEY];
|
|
256
|
+
return isRecord(nested) ? normalizeSettingsConfig(nested) : {};
|
|
257
|
+
} catch {
|
|
258
|
+
return {};
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
export function loadConfig(cwd: string, env: NodeJS.ProcessEnv = process.env): Config {
|
|
263
|
+
const globalPath = join(getAgentDir(), "settings.json");
|
|
264
|
+
const projectPath = join(cwd, ".pi", "settings.json");
|
|
265
|
+
const globalConfig = readNamespacedConfig(globalPath);
|
|
266
|
+
const projectConfig = readNamespacedConfig(projectPath);
|
|
267
|
+
const envConfig = readEnvConfig(env);
|
|
268
|
+
const merged = {
|
|
269
|
+
...DEFAULTS,
|
|
270
|
+
observationsPoolTargetTokens: undefined,
|
|
271
|
+
...globalConfig,
|
|
272
|
+
...projectConfig,
|
|
273
|
+
...envConfig,
|
|
274
|
+
};
|
|
275
|
+
const target = validTargetOrUndefined(
|
|
276
|
+
merged.observationsPoolTargetTokens,
|
|
277
|
+
merged.observationsPoolMaxTokens,
|
|
278
|
+
) ?? derivedObservationPoolTarget(merged.observationsPoolMaxTokens);
|
|
279
|
+
|
|
280
|
+
return {
|
|
281
|
+
...merged,
|
|
282
|
+
observationsPoolTargetTokens: target,
|
|
283
|
+
};
|
|
284
|
+
}
|
package/src/debug-log.ts
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
2
|
+
import { existsSync, mkdirSync, renameSync, statSync, unlinkSync, appendFileSync } from "node:fs";
|
|
3
|
+
import { dirname, join } from "node:path";
|
|
4
|
+
import { getAgentDir } from "@earendil-works/pi-coding-agent";
|
|
5
|
+
|
|
6
|
+
export const DEBUG_LOG_MAX_BYTES = 10 * 1024 * 1024;
|
|
7
|
+
export const DEBUG_LOG_RELATIVE_PATH = join("observational-memory", "debug.ndjson");
|
|
8
|
+
export const DEBUG_LOG_SESSION_DIR_RELATIVE_PATH = join("observational-memory", "debug");
|
|
9
|
+
|
|
10
|
+
export interface DebugLogContext {
|
|
11
|
+
enabled: boolean;
|
|
12
|
+
cwd?: string;
|
|
13
|
+
sessionId?: string;
|
|
14
|
+
sessionFile?: string;
|
|
15
|
+
runId?: string;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
const storage = new AsyncLocalStorage<DebugLogContext>();
|
|
19
|
+
|
|
20
|
+
export function withDebugLogContext<T>(context: DebugLogContext, fn: () => T): T {
|
|
21
|
+
const parent = storage.getStore();
|
|
22
|
+
return storage.run({ ...parent, ...context }, fn);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function safeDebugLogSessionId(sessionId: string | undefined): string | undefined {
|
|
26
|
+
const trimmed = sessionId?.trim();
|
|
27
|
+
if (!trimmed) return undefined;
|
|
28
|
+
const sanitized = trimmed
|
|
29
|
+
.replace(/[^A-Za-z0-9._-]+/g, "_")
|
|
30
|
+
.replace(/^_+|_+$/g, "")
|
|
31
|
+
.slice(0, 128);
|
|
32
|
+
if (!/[A-Za-z0-9]/.test(sanitized)) return undefined;
|
|
33
|
+
return sanitized;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function debugLogRelativePath(context: Pick<DebugLogContext, "sessionId">): string {
|
|
37
|
+
const safeSessionId = safeDebugLogSessionId(context.sessionId);
|
|
38
|
+
return safeSessionId
|
|
39
|
+
? join(DEBUG_LOG_SESSION_DIR_RELATIVE_PATH, `${safeSessionId}.ndjson`)
|
|
40
|
+
: DEBUG_LOG_RELATIVE_PATH;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export function debugLog(event: string, data: Record<string, unknown> = {}): void {
|
|
44
|
+
const context = storage.getStore();
|
|
45
|
+
if (context?.enabled !== true) return;
|
|
46
|
+
|
|
47
|
+
try {
|
|
48
|
+
const path = join(getAgentDir(), debugLogRelativePath(context));
|
|
49
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
50
|
+
rotateIfNeeded(path);
|
|
51
|
+
const payload = {
|
|
52
|
+
ts: new Date().toISOString(),
|
|
53
|
+
event,
|
|
54
|
+
cwd: context.cwd,
|
|
55
|
+
sessionId: context.sessionId,
|
|
56
|
+
sessionFile: context.sessionFile,
|
|
57
|
+
runId: context.runId,
|
|
58
|
+
data,
|
|
59
|
+
};
|
|
60
|
+
appendFileSync(path, `${JSON.stringify(payload)}\n`, "utf-8");
|
|
61
|
+
} catch {
|
|
62
|
+
// Debug logging must never affect memory behavior.
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function rotateIfNeeded(path: string): void {
|
|
67
|
+
if (!existsSync(path)) return;
|
|
68
|
+
if (statSync(path).size < DEBUG_LOG_MAX_BYTES) return;
|
|
69
|
+
const backupPath = `${path}.1`;
|
|
70
|
+
if (existsSync(backupPath)) unlinkSync(backupPath);
|
|
71
|
+
renameSync(path, backupPath);
|
|
72
|
+
}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
2
|
+
|
|
3
|
+
import { computeSessionSettings, type Runtime } from "../runtime.js";
|
|
4
|
+
import { launchCompactionObserver, type ConsolidationCtx } from "./consolidation-trigger.js";
|
|
5
|
+
import { buildCompactionProjection, renderSummary, type Entry } from "../session-ledger/index.js";
|
|
6
|
+
import { watchForNativeCompactionResume } from "./compaction-resume.js";
|
|
7
|
+
|
|
8
|
+
const DEFAULT_OBSERVATIONS_POOL_MAX_TOKENS = 20_000;
|
|
9
|
+
const COMPACTION_STATUS_KEY = "observational-memory-compaction";
|
|
10
|
+
|
|
11
|
+
function observationsPoolMaxTokens(runtime: Runtime): number {
|
|
12
|
+
const value = (runtime.config as { observationsPoolMaxTokens?: unknown }).observationsPoolMaxTokens;
|
|
13
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0
|
|
14
|
+
? value
|
|
15
|
+
: DEFAULT_OBSERVATIONS_POOL_MAX_TOKENS;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function registerCompactionHook(pi: ExtensionAPI, runtime: Runtime): void {
|
|
19
|
+
pi.on("session_before_compact", async (event: any, ctx: any) => {
|
|
20
|
+
if (runtime.compactHookInFlight) {
|
|
21
|
+
if (ctx.hasUI) {
|
|
22
|
+
ctx.ui.notify(
|
|
23
|
+
"Observational memory: another compaction is already in progress; cancelling duplicate",
|
|
24
|
+
"warning",
|
|
25
|
+
);
|
|
26
|
+
}
|
|
27
|
+
return { cancel: true };
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const initiatedByOm = runtime.compactInFlight && event.reason === "manual";
|
|
31
|
+
const reason = initiatedByOm ? (runtime.compactOrigin ?? "proactive") : event.reason;
|
|
32
|
+
if (ctx.hasUI) {
|
|
33
|
+
let pending = "";
|
|
34
|
+
if (event.willRetry) pending = ", retry pending";
|
|
35
|
+
else if (initiatedByOm) pending = ", resume pending";
|
|
36
|
+
ctx.ui.setStatus?.(COMPACTION_STATUS_KEY, `OM compaction: running (${reason}${pending})`);
|
|
37
|
+
if (!initiatedByOm) {
|
|
38
|
+
const continuation = event.willRetry ? "; the interrupted agent run will resume automatically" : "";
|
|
39
|
+
ctx.ui.notify(`Observational memory: compaction started (${reason})${continuation}`, "info");
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
event.signal?.addEventListener?.("abort", () => {
|
|
43
|
+
if (ctx.hasUI) ctx.ui.setStatus?.(COMPACTION_STATUS_KEY, undefined);
|
|
44
|
+
}, { once: true });
|
|
45
|
+
|
|
46
|
+
runtime.compactHookInFlight = true;
|
|
47
|
+
try {
|
|
48
|
+
runtime.ensureConfig(ctx.cwd);
|
|
49
|
+
const { preparation, branchEntries } = event;
|
|
50
|
+
const branch = branchEntries as Entry[];
|
|
51
|
+
// Start memory capture without delaying compaction. This is configurable so
|
|
52
|
+
// users can compare native compaction with and without the observer sidecar.
|
|
53
|
+
if (runtime.config.compactionObserverEnabled !== false) {
|
|
54
|
+
launchCompactionObserver(pi, runtime, ctx as ConsolidationCtx, branch);
|
|
55
|
+
}
|
|
56
|
+
const { firstKeptEntryId, tokensBefore } = preparation;
|
|
57
|
+
const projection = buildCompactionProjection(
|
|
58
|
+
branch,
|
|
59
|
+
firstKeptEntryId,
|
|
60
|
+
{ observationsPoolMaxTokens: observationsPoolMaxTokens(runtime) },
|
|
61
|
+
);
|
|
62
|
+
const summary = renderSummary(projection.reflections, projection.observations);
|
|
63
|
+
// Compaction removes older custom entries from the active branch. Keep
|
|
64
|
+
// session-scoped overrides in the compaction details so they can be
|
|
65
|
+
// restored after a reload from the surviving branch. Bake the merged
|
|
66
|
+
// branch intent (live om.settings entries winning over earlier snapshots)
|
|
67
|
+
// rather than the raw in-memory overlay, which can lag out-of-band
|
|
68
|
+
// om.settings appends and would otherwise silently override newer entries
|
|
69
|
+
// at the next restore.
|
|
70
|
+
const details = {
|
|
71
|
+
...projection.details,
|
|
72
|
+
sessionSettings: computeSessionSettings(branch),
|
|
73
|
+
};
|
|
74
|
+
|
|
75
|
+
return {
|
|
76
|
+
compaction: {
|
|
77
|
+
summary,
|
|
78
|
+
firstKeptEntryId,
|
|
79
|
+
tokensBefore,
|
|
80
|
+
details,
|
|
81
|
+
},
|
|
82
|
+
};
|
|
83
|
+
} finally {
|
|
84
|
+
runtime.compactHookInFlight = false;
|
|
85
|
+
}
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
pi.on("session_compact", (event: any, ctx: any) => {
|
|
89
|
+
const initiatedByOm = runtime.compactInFlight && event.reason === "manual";
|
|
90
|
+
const reason = initiatedByOm ? (runtime.compactOrigin ?? "proactive") : event.reason;
|
|
91
|
+
if (event.willRetry) watchForNativeCompactionResume(pi, runtime, ctx);
|
|
92
|
+
if (!ctx.hasUI) return;
|
|
93
|
+
ctx.ui.setStatus?.(COMPACTION_STATUS_KEY, undefined);
|
|
94
|
+
let continuation = "";
|
|
95
|
+
if (event.willRetry) continuation = "; resuming the interrupted agent run";
|
|
96
|
+
else if (initiatedByOm) continuation = "; resuming the agent run";
|
|
97
|
+
ctx.ui.notify(`Observational memory: compaction complete (${reason})${continuation}`, "info");
|
|
98
|
+
});
|
|
99
|
+
}
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
2
|
+
import type { Runtime } from "../runtime.js";
|
|
3
|
+
|
|
4
|
+
const NATIVE_RESUME_GRACE_MS = 5_000;
|
|
5
|
+
const RESUME_RETRY_DELAYS_MS = [250, 1_000] as const;
|
|
6
|
+
|
|
7
|
+
interface ResumeCtx {
|
|
8
|
+
hasUI: boolean;
|
|
9
|
+
ui?: {
|
|
10
|
+
notify?: (message: string, level?: "info" | "warning" | "error") => void;
|
|
11
|
+
};
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function clearResumeWatch(runtime: Runtime): void {
|
|
15
|
+
if (runtime.compactionResumeTimer !== undefined) {
|
|
16
|
+
clearTimeout(runtime.compactionResumeTimer);
|
|
17
|
+
runtime.compactionResumeTimer = undefined;
|
|
18
|
+
}
|
|
19
|
+
runtime.compactionResumePending = false;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function beginResumeWatch(runtime: Runtime): number {
|
|
23
|
+
clearResumeWatch(runtime);
|
|
24
|
+
runtime.compactionResumePending = true;
|
|
25
|
+
runtime.compactionResumeGeneration += 1;
|
|
26
|
+
return runtime.compactionResumeGeneration;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function isCurrentWatch(runtime: Runtime, generation: number): boolean {
|
|
30
|
+
return runtime.compactionResumePending && runtime.compactionResumeGeneration === generation;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function sendResumeMessage(pi: ExtensionAPI, ctx: ResumeCtx, afterFailure: boolean): void {
|
|
34
|
+
try {
|
|
35
|
+
pi.sendMessage({
|
|
36
|
+
customType: "om.compaction.resume",
|
|
37
|
+
content: afterFailure
|
|
38
|
+
? "Context compaction failed. Continue the current task without waiting for another user message."
|
|
39
|
+
: "Continue the current task from the compacted context without waiting for another user message.",
|
|
40
|
+
display: false,
|
|
41
|
+
}, {
|
|
42
|
+
deliverAs: "followUp",
|
|
43
|
+
triggerTurn: true,
|
|
44
|
+
});
|
|
45
|
+
} catch (error) {
|
|
46
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
47
|
+
ctx.ui?.notify?.(`Observational memory: failed to request continuation: ${message}`, "error");
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function scheduleResumeRetries(
|
|
52
|
+
pi: ExtensionAPI,
|
|
53
|
+
runtime: Runtime,
|
|
54
|
+
ctx: ResumeCtx,
|
|
55
|
+
generation: number,
|
|
56
|
+
afterFailure: boolean,
|
|
57
|
+
retryIndex = 0,
|
|
58
|
+
): void {
|
|
59
|
+
if (!isCurrentWatch(runtime, generation)) return;
|
|
60
|
+
if (retryIndex >= RESUME_RETRY_DELAYS_MS.length) {
|
|
61
|
+
runtime.compactionResumeTimer = setTimeout(() => {
|
|
62
|
+
if (!isCurrentWatch(runtime, generation)) return;
|
|
63
|
+
clearResumeWatch(runtime);
|
|
64
|
+
ctx.ui?.notify?.(
|
|
65
|
+
"Observational memory: the agent did not acknowledge continuation after compaction",
|
|
66
|
+
"error",
|
|
67
|
+
);
|
|
68
|
+
}, RESUME_RETRY_DELAYS_MS.at(-1));
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
runtime.compactionResumeTimer = setTimeout(() => {
|
|
73
|
+
if (!isCurrentWatch(runtime, generation)) return;
|
|
74
|
+
ctx.ui?.notify?.(
|
|
75
|
+
`Observational memory: continuation did not start; retrying (${retryIndex + 1}/${RESUME_RETRY_DELAYS_MS.length})`,
|
|
76
|
+
"warning",
|
|
77
|
+
);
|
|
78
|
+
sendResumeMessage(pi, ctx, afterFailure);
|
|
79
|
+
scheduleResumeRetries(pi, runtime, ctx, generation, afterFailure, retryIndex + 1);
|
|
80
|
+
}, RESUME_RETRY_DELAYS_MS[retryIndex]);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** Send OM's continuation immediately, then retry unless a new agent run acknowledges it. */
|
|
84
|
+
export function resumeAfterCompaction(
|
|
85
|
+
pi: ExtensionAPI,
|
|
86
|
+
runtime: Runtime,
|
|
87
|
+
ctx: ResumeCtx,
|
|
88
|
+
afterFailure = false,
|
|
89
|
+
): void {
|
|
90
|
+
const generation = beginResumeWatch(runtime);
|
|
91
|
+
sendResumeMessage(pi, ctx, afterFailure);
|
|
92
|
+
scheduleResumeRetries(pi, runtime, ctx, generation, afterFailure);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Pi normally retries native overflow compaction itself. If no new agent_start
|
|
97
|
+
* arrives after the compaction event settles, send OM's hidden continuation as
|
|
98
|
+
* a fallback. This also covers length-stop retries that cannot continue from an
|
|
99
|
+
* assistant message.
|
|
100
|
+
*/
|
|
101
|
+
export function watchForNativeCompactionResume(
|
|
102
|
+
pi: ExtensionAPI,
|
|
103
|
+
runtime: Runtime,
|
|
104
|
+
ctx: ResumeCtx,
|
|
105
|
+
): void {
|
|
106
|
+
const generation = beginResumeWatch(runtime);
|
|
107
|
+
runtime.compactionResumeTimer = setTimeout(() => {
|
|
108
|
+
if (!isCurrentWatch(runtime, generation)) return;
|
|
109
|
+
ctx.ui?.notify?.(
|
|
110
|
+
"Observational memory: native compaction did not resume the agent; sending fallback continuation",
|
|
111
|
+
"warning",
|
|
112
|
+
);
|
|
113
|
+
sendResumeMessage(pi, ctx, false);
|
|
114
|
+
scheduleResumeRetries(pi, runtime, ctx, generation, false);
|
|
115
|
+
}, NATIVE_RESUME_GRACE_MS);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** Any new agent run proves that the post-compaction continuation started. */
|
|
119
|
+
export function registerCompactionResumeAcknowledgement(pi: ExtensionAPI, runtime: Runtime): void {
|
|
120
|
+
pi.on("agent_start", () => {
|
|
121
|
+
if (!runtime.compactionResumePending) return;
|
|
122
|
+
clearResumeWatch(runtime);
|
|
123
|
+
});
|
|
124
|
+
}
|