@hicaru/pi-rlm 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +237 -0
- package/README.ru.md +200 -0
- package/README.zh-CN.md +224 -0
- package/package.json +54 -0
- package/src/bridge/fallback-todo.ts +137 -0
- package/src/bridge/interactive.ts +65 -0
- package/src/bridge/llm-query.ts +124 -0
- package/src/bridge/model.ts +97 -0
- package/src/bridge/pi-interactive.ts +86 -0
- package/src/bridge/rlm-query.ts +78 -0
- package/src/commands/rlm-config.ts +42 -0
- package/src/commands/rlm.ts +165 -0
- package/src/config/defaults.ts +38 -0
- package/src/config/settings.ts +185 -0
- package/src/context/repomix-context.ts +253 -0
- package/src/core/answer.ts +97 -0
- package/src/core/compaction.ts +64 -0
- package/src/core/engine.ts +408 -0
- package/src/core/history.ts +13 -0
- package/src/core/iteration.ts +45 -0
- package/src/core/limits.ts +90 -0
- package/src/core/pipeline.ts +100 -0
- package/src/core/resource-limits.ts +14 -0
- package/src/core/types.ts +131 -0
- package/src/index.ts +165 -0
- package/src/mode/input-router.ts +23 -0
- package/src/mode/rlm-mode.ts +149 -0
- package/src/patch/apply.ts +148 -0
- package/src/patch/index.ts +37 -0
- package/src/prompts/system.ts +278 -0
- package/src/prompts/user.ts +21 -0
- package/src/sandbox/protocol.ts +191 -0
- package/src/sandbox/sandbox-manager.ts +143 -0
- package/src/sandbox/sandbox.ts +362 -0
- package/src/sandbox/worker.py +457 -0
- package/src/state/events.ts +22 -0
- package/src/state/index.ts +23 -0
- package/src/state/internal.ts +46 -0
- package/src/state/paths.ts +42 -0
- package/src/state/reads.ts +96 -0
- package/src/state/resume.ts +154 -0
- package/src/state/rows.ts +117 -0
- package/src/state/writes.ts +56 -0
- package/src/telemetry/dispatcher.ts +116 -0
- package/src/telemetry/index.ts +14 -0
- package/src/telemetry/mlflow-config.ts +15 -0
- package/src/telemetry/mlflow-sink.ts +136 -0
- package/src/telemetry/mlflow.ts +99 -0
- package/src/telemetry/sink.ts +8 -0
- package/src/text/edits.ts +16 -0
- package/src/text/parsing.ts +35 -0
- package/src/text/preview.ts +18 -0
- package/src/text/tokens.ts +64 -0
- package/src/tool/apply-diff-tool.ts +125 -0
- package/src/tool/emitter-listener.ts +24 -0
- package/src/tool/repl-details.ts +23 -0
- package/src/tool/repl-tool.ts +528 -0
- package/src/tool/rlm-aggregator.ts +115 -0
- package/src/tool/rlm-details.ts +53 -0
- package/src/tool/rlm-events.ts +215 -0
- package/src/tool/rlm-tool.ts +199 -0
- package/src/tool/subcall-render.ts +129 -0
- package/src/tool/subcall-store.ts +90 -0
- package/src/tool/tool-utils.ts +73 -0
- package/src/ui/config-panel.ts +92 -0
- package/src/ui/intro.ts +23 -0
- package/src/ui/model-picker.ts +139 -0
- package/src/ui/status.ts +26 -0
- package/src/ui/theme.ts +47 -0
- package/src/util/concurrency.ts +15 -0
- package/src/util/errors.ts +27 -0
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
/** Persist RLM settings (tunable config + chosen worker model id). */
|
|
2
|
+
|
|
3
|
+
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
4
|
+
import { dirname, join } from "node:path";
|
|
5
|
+
import { getAgentDir, type ModelRegistry } from "@earendil-works/pi-coding-agent";
|
|
6
|
+
import type { Api, Model, ThinkingLevel } from "@earendil-works/pi-ai";
|
|
7
|
+
import type { RlmConfig, RunLogConfig, TelemetryConfig } from "../core/types.ts";
|
|
8
|
+
import { DEFAULT_CONFIG } from "./defaults.ts";
|
|
9
|
+
|
|
10
|
+
export interface PersistedSettings {
|
|
11
|
+
readonly config: Partial<RlmConfig>;
|
|
12
|
+
readonly worker?: string;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
type MutablePartialRlmConfig = { -readonly [K in keyof RlmConfig]?: RlmConfig[K] };
|
|
16
|
+
|
|
17
|
+
function settingsPath(): string {
|
|
18
|
+
return join(getAgentDir(), "rlm.json");
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function validateNumber(v: unknown, min: number): number | undefined {
|
|
22
|
+
return typeof v === "number" && Number.isFinite(v) && v >= min ? v : undefined;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function validateBoolean(v: unknown): boolean | undefined {
|
|
26
|
+
return typeof v === "boolean" ? v : undefined;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function validateString(v: unknown): string | undefined {
|
|
30
|
+
return typeof v === "string" && v.trim() ? v : undefined;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function validateTelemetry(raw: unknown): TelemetryConfig | undefined {
|
|
34
|
+
if (typeof raw !== "object" || raw === null) return undefined;
|
|
35
|
+
const r = raw as Record<string, unknown>;
|
|
36
|
+
const out: {
|
|
37
|
+
enabled?: boolean;
|
|
38
|
+
trackingUri?: string;
|
|
39
|
+
experimentId?: string;
|
|
40
|
+
maxQueueSize?: number;
|
|
41
|
+
} = {};
|
|
42
|
+
const enabled = validateBoolean(r.enabled);
|
|
43
|
+
if (enabled !== undefined) out.enabled = enabled;
|
|
44
|
+
const trackingUri = validateString(r.trackingUri);
|
|
45
|
+
if (trackingUri !== undefined) out.trackingUri = trackingUri;
|
|
46
|
+
const experimentId = validateString(r.experimentId);
|
|
47
|
+
if (experimentId !== undefined) out.experimentId = experimentId;
|
|
48
|
+
const maxQueueSize = validateNumber(r.maxQueueSize, 1);
|
|
49
|
+
if (maxQueueSize !== undefined) out.maxQueueSize = maxQueueSize;
|
|
50
|
+
return Object.freeze(out);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function validateRunLog(raw: unknown): Partial<RunLogConfig> | undefined {
|
|
54
|
+
if (typeof raw !== "object" || raw === null) return undefined;
|
|
55
|
+
const r = raw as Record<string, unknown>;
|
|
56
|
+
const out: { enabled?: boolean; dir?: string; snapshot?: boolean; maxRuns?: number } = {};
|
|
57
|
+
const enabled = validateBoolean(r.enabled);
|
|
58
|
+
if (enabled !== undefined) out.enabled = enabled;
|
|
59
|
+
const dir = validateString(r.dir);
|
|
60
|
+
if (dir !== undefined) out.dir = dir;
|
|
61
|
+
const snapshot = validateBoolean(r.snapshot);
|
|
62
|
+
if (snapshot !== undefined) out.snapshot = snapshot;
|
|
63
|
+
const maxRuns = validateNumber(r.maxRuns, 1);
|
|
64
|
+
if (maxRuns !== undefined) out.maxRuns = maxRuns;
|
|
65
|
+
return Object.keys(out).length > 0 ? Object.freeze(out) : undefined;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function validateConfig(raw: unknown): Partial<RlmConfig> {
|
|
69
|
+
if (typeof raw !== "object" || raw === null) return {};
|
|
70
|
+
const r = raw as Record<string, unknown>;
|
|
71
|
+
const out: MutablePartialRlmConfig = {};
|
|
72
|
+
const enabled = validateBoolean(r.enabled);
|
|
73
|
+
if (enabled !== undefined) out.enabled = enabled;
|
|
74
|
+
const maxDepth = validateNumber(r.maxDepth, 1);
|
|
75
|
+
if (maxDepth !== undefined) out.maxDepth = maxDepth;
|
|
76
|
+
const maxIterations = validateNumber(r.maxIterations, 1);
|
|
77
|
+
if (maxIterations !== undefined) out.maxIterations = maxIterations;
|
|
78
|
+
const execTimeoutS = validateNumber(r.execTimeoutS, 1);
|
|
79
|
+
if (execTimeoutS !== undefined) out.execTimeoutS = execTimeoutS;
|
|
80
|
+
const requestTimeoutMs = validateNumber(r.requestTimeoutMs, 1000);
|
|
81
|
+
if (requestTimeoutMs !== undefined) out.requestTimeoutMs = requestTimeoutMs;
|
|
82
|
+
const maxConcurrentSubcalls = validateNumber(r.maxConcurrentSubcalls, 1);
|
|
83
|
+
if (maxConcurrentSubcalls !== undefined) out.maxConcurrentSubcalls = maxConcurrentSubcalls;
|
|
84
|
+
const maxPromptChars = validateNumber(r.maxPromptChars, 1000);
|
|
85
|
+
if (maxPromptChars !== undefined) out.maxPromptChars = maxPromptChars;
|
|
86
|
+
const maxBudgetUsd = validateNumber(r.maxBudgetUsd, 0.01);
|
|
87
|
+
if (maxBudgetUsd !== undefined) out.maxBudgetUsd = maxBudgetUsd;
|
|
88
|
+
const maxTimeoutMs = validateNumber(r.maxTimeoutMs, 1000);
|
|
89
|
+
if (maxTimeoutMs !== undefined) out.maxTimeoutMs = maxTimeoutMs;
|
|
90
|
+
const maxTokens = validateNumber(r.maxTokens, 1);
|
|
91
|
+
if (maxTokens !== undefined) out.maxTokens = maxTokens;
|
|
92
|
+
const maxErrors = validateNumber(r.maxErrors, 1);
|
|
93
|
+
if (maxErrors !== undefined) out.maxErrors = maxErrors;
|
|
94
|
+
const orchestrator = validateBoolean(r.orchestrator);
|
|
95
|
+
if (orchestrator !== undefined) out.orchestrator = orchestrator;
|
|
96
|
+
const compaction = validateBoolean(r.compaction);
|
|
97
|
+
if (compaction !== undefined) out.compaction = compaction;
|
|
98
|
+
const compactionThresholdPct = validateNumber(r.compactionThresholdPct, 0);
|
|
99
|
+
if (compactionThresholdPct !== undefined && compactionThresholdPct <= 1) out.compactionThresholdPct = compactionThresholdPct;
|
|
100
|
+
const python = validateString(r.python);
|
|
101
|
+
if (python !== undefined) out.python = python;
|
|
102
|
+
if (typeof r.smartReasoning === "string") out.smartReasoning = r.smartReasoning as ThinkingLevel;
|
|
103
|
+
const subSystemPrompt = validateString(r.subSystemPrompt);
|
|
104
|
+
if (subSystemPrompt !== undefined) out.subSystemPrompt = subSystemPrompt;
|
|
105
|
+
const telemetry = validateTelemetry(r.telemetry);
|
|
106
|
+
if (telemetry) out.telemetry = telemetry;
|
|
107
|
+
const runLog = validateRunLog(r.runLog);
|
|
108
|
+
if (runLog) out.runLog = runLog;
|
|
109
|
+
const sandboxInitTimeoutMs = validateNumber(r.sandboxInitTimeoutMs, 100);
|
|
110
|
+
if (sandboxInitTimeoutMs !== undefined) out.sandboxInitTimeoutMs = sandboxInitTimeoutMs;
|
|
111
|
+
const askUserQuestion = validateBoolean(r.askUserQuestion);
|
|
112
|
+
if (askUserQuestion !== undefined) out.askUserQuestion = askUserQuestion;
|
|
113
|
+
const todo = validateBoolean(r.todo);
|
|
114
|
+
if (todo !== undefined) out.todo = todo;
|
|
115
|
+
if (typeof r.subSampling === "object" && r.subSampling !== null) {
|
|
116
|
+
const ss = r.subSampling as Record<string, unknown>;
|
|
117
|
+
const sampling: { maxTokens?: number; temperature?: number; reasoning?: ThinkingLevel } = {};
|
|
118
|
+
const maxTokensValue = validateNumber(ss.maxTokens, 1);
|
|
119
|
+
if (maxTokensValue !== undefined) sampling.maxTokens = maxTokensValue;
|
|
120
|
+
const temperature = validateNumber(ss.temperature, 0);
|
|
121
|
+
if (temperature !== undefined) sampling.temperature = temperature;
|
|
122
|
+
if (typeof ss.reasoning === "string") sampling.reasoning = ss.reasoning as ThinkingLevel;
|
|
123
|
+
out.subSampling = sampling;
|
|
124
|
+
}
|
|
125
|
+
if (typeof r.rootSampling === "object" && r.rootSampling !== null) {
|
|
126
|
+
const rs = r.rootSampling as Record<string, unknown>;
|
|
127
|
+
const rootSampling: { maxTokens?: number; temperature?: number; reasoning?: ThinkingLevel } = {};
|
|
128
|
+
const rsMaxTokens = validateNumber(rs.maxTokens, 1);
|
|
129
|
+
if (rsMaxTokens !== undefined) rootSampling.maxTokens = rsMaxTokens;
|
|
130
|
+
const rsTemperature = validateNumber(rs.temperature, 0);
|
|
131
|
+
if (rsTemperature !== undefined) rootSampling.temperature = rsTemperature;
|
|
132
|
+
if (typeof rs.reasoning === "string") rootSampling.reasoning = rs.reasoning as ThinkingLevel;
|
|
133
|
+
out.rootSampling = Object.freeze(rootSampling);
|
|
134
|
+
}
|
|
135
|
+
return out;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
export async function loadSettings(): Promise<PersistedSettings> {
|
|
139
|
+
try {
|
|
140
|
+
const raw = JSON.parse(await readFile(settingsPath(), "utf8")) as unknown;
|
|
141
|
+
if (typeof raw !== "object" || raw === null) return { config: {} };
|
|
142
|
+
const r = raw as Record<string, unknown>;
|
|
143
|
+
return {
|
|
144
|
+
config: validateConfig(r.config),
|
|
145
|
+
worker: typeof r.worker === "string" ? r.worker : undefined,
|
|
146
|
+
};
|
|
147
|
+
} catch {
|
|
148
|
+
return { config: {} };
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
export async function saveSettings(s: PersistedSettings): Promise<boolean> {
|
|
153
|
+
try {
|
|
154
|
+
const p = settingsPath();
|
|
155
|
+
await mkdir(dirname(p), { recursive: true });
|
|
156
|
+
await writeFile(p, `${JSON.stringify(s, null, 2)}\n`);
|
|
157
|
+
return true;
|
|
158
|
+
} catch {
|
|
159
|
+
return false;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/** Merge persisted tunables over the defaults. */
|
|
164
|
+
export function mergeConfig(partial: Partial<RlmConfig>): RlmConfig {
|
|
165
|
+
return {
|
|
166
|
+
...DEFAULT_CONFIG,
|
|
167
|
+
...partial,
|
|
168
|
+
subSampling: { ...DEFAULT_CONFIG.subSampling, ...partial.subSampling },
|
|
169
|
+
rootSampling: Object.freeze({ ...DEFAULT_CONFIG.rootSampling, ...partial.rootSampling }),
|
|
170
|
+
...(partial.telemetry ? { telemetry: Object.freeze({ ...partial.telemetry }) } : {}),
|
|
171
|
+
...(partial.runLog ? { runLog: Object.freeze({ ...DEFAULT_CONFIG.runLog, ...partial.runLog }) } : {}),
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/** Resolve a "provider/id" string against the registry. */
|
|
176
|
+
export function resolveModelId(registry: ModelRegistry, ref?: string): Model<Api> | undefined {
|
|
177
|
+
if (!ref) return undefined;
|
|
178
|
+
const slash = ref.indexOf("/");
|
|
179
|
+
if (slash < 0) return undefined;
|
|
180
|
+
return registry.find(ref.slice(0, slash), ref.slice(slash + 1));
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
export function modelRef(model: Model<Api> | undefined): string | undefined {
|
|
184
|
+
return model ? `${model.provider}/${model.id}` : undefined;
|
|
185
|
+
}
|
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* repomix-context — pre-packs the entire codebase into a structured JSON array
|
|
3
|
+
* for the RLM sandbox. Replaces the legacy filesystem-tool context
|
|
4
|
+
* (buildProjectManifest / listProjectFiles / gitLsFiles).
|
|
5
|
+
*
|
|
6
|
+
* Uses repomix internally (worker-thread pool, built-in gitignore support)
|
|
7
|
+
* and caches results in a module-level Map with TTL to avoid re-packing on
|
|
8
|
+
* every run within the same process. `patchContextAfterEdits` updates the
|
|
9
|
+
* cached bundle in-memory after file edits are applied to disk — no re-packing.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { pack } from "repomix";
|
|
13
|
+
import type { PackResult as RepomixPackResult } from "repomix";
|
|
14
|
+
import { resolve } from "node:path";
|
|
15
|
+
import { tmpdir } from "node:os";
|
|
16
|
+
import { errorMessage } from "../util/errors.ts";
|
|
17
|
+
|
|
18
|
+
// ── Public types ──
|
|
19
|
+
|
|
20
|
+
export interface ContextFile {
|
|
21
|
+
readonly path: string;
|
|
22
|
+
readonly content: string;
|
|
23
|
+
readonly tokens: number;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export interface ContextBundle {
|
|
27
|
+
readonly files: readonly ContextFile[];
|
|
28
|
+
readonly totalFiles: number;
|
|
29
|
+
readonly totalTokens: number;
|
|
30
|
+
readonly totalChars: number;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export interface PackSuccess {
|
|
34
|
+
readonly ok: true;
|
|
35
|
+
readonly value: ContextBundle;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export interface PackFailure {
|
|
39
|
+
readonly ok: false;
|
|
40
|
+
readonly error: string;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export type PackResult = PackSuccess | PackFailure;
|
|
44
|
+
|
|
45
|
+
// ── Module-level cache ──
|
|
46
|
+
|
|
47
|
+
interface CacheEntry {
|
|
48
|
+
readonly bundle: ContextBundle;
|
|
49
|
+
readonly ts: number;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
const cache = new Map<string, CacheEntry>();
|
|
53
|
+
const DEFAULT_CACHE_TTL_MS = 30_000;
|
|
54
|
+
|
|
55
|
+
/** Exported for tests — empties the module-level cache. */
|
|
56
|
+
export function clearCache(): void {
|
|
57
|
+
cache.clear();
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function cacheKey(cwd: string): string {
|
|
61
|
+
return resolve(cwd);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function cacheGet(key: string, ttlMs: number): ContextBundle | undefined {
|
|
65
|
+
const entry = cache.get(key);
|
|
66
|
+
if (!entry) return undefined;
|
|
67
|
+
if (Date.now() - entry.ts > ttlMs) {
|
|
68
|
+
cache.delete(key);
|
|
69
|
+
return undefined;
|
|
70
|
+
}
|
|
71
|
+
return entry.bundle;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function cacheSet(key: string, bundle: ContextBundle): void {
|
|
75
|
+
cache.set(key, { bundle, ts: Date.now() });
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// ── Core functions ──
|
|
79
|
+
|
|
80
|
+
const ESTIMATED_CHARS_PER_TOKEN = 4;
|
|
81
|
+
|
|
82
|
+
export async function packRepository(
|
|
83
|
+
cwd: string,
|
|
84
|
+
signal?: AbortSignal,
|
|
85
|
+
ttlMs: number = DEFAULT_CACHE_TTL_MS,
|
|
86
|
+
): Promise<PackResult> {
|
|
87
|
+
if (signal?.aborted) return { ok: false, error: "aborted" };
|
|
88
|
+
|
|
89
|
+
const key = cacheKey(cwd);
|
|
90
|
+
const cached = cacheGet(key, ttlMs);
|
|
91
|
+
if (cached) return { ok: true, value: cached };
|
|
92
|
+
|
|
93
|
+
try {
|
|
94
|
+
const result: RepomixPackResult = await Promise.race([
|
|
95
|
+
pack([cwd], {
|
|
96
|
+
input: { maxFileSize: 1048576 },
|
|
97
|
+
cwd,
|
|
98
|
+
output: {
|
|
99
|
+
filePath: `${tmpdir()}/repomix-out-${Date.now()}.txt`,
|
|
100
|
+
style: "plain",
|
|
101
|
+
parsableStyle: false,
|
|
102
|
+
headerText: undefined,
|
|
103
|
+
instructionFilePath: undefined,
|
|
104
|
+
fileSummary: false,
|
|
105
|
+
directoryStructure: false,
|
|
106
|
+
files: true,
|
|
107
|
+
removeComments: false,
|
|
108
|
+
removeEmptyLines: false,
|
|
109
|
+
compress: false,
|
|
110
|
+
topFilesLength: 5,
|
|
111
|
+
showLineNumbers: false,
|
|
112
|
+
truncateBase64: false,
|
|
113
|
+
copyToClipboard: false,
|
|
114
|
+
includeEmptyDirectories: undefined,
|
|
115
|
+
includeFullDirectoryStructure: false,
|
|
116
|
+
splitOutput: undefined,
|
|
117
|
+
tokenCountTree: false,
|
|
118
|
+
tokenBudget: undefined,
|
|
119
|
+
git: {
|
|
120
|
+
sortByChanges: true,
|
|
121
|
+
sortByChangesMaxCommits: 100,
|
|
122
|
+
includeDiffs: false,
|
|
123
|
+
includeLogs: false,
|
|
124
|
+
includeLogsCount: 50,
|
|
125
|
+
},
|
|
126
|
+
},
|
|
127
|
+
include: [],
|
|
128
|
+
ignore: {
|
|
129
|
+
useGitignore: true,
|
|
130
|
+
useDotIgnore: true,
|
|
131
|
+
useDefaultPatterns: true,
|
|
132
|
+
customPatterns: [],
|
|
133
|
+
},
|
|
134
|
+
security: { enableSecurityCheck: false },
|
|
135
|
+
tokenCount: { encoding: "o200k_base" as const },
|
|
136
|
+
} as Parameters<typeof pack>[1]),
|
|
137
|
+
new Promise<never>((_, reject) => {
|
|
138
|
+
signal?.addEventListener("abort", () => reject(new Error("aborted")), { once: true });
|
|
139
|
+
}),
|
|
140
|
+
]);
|
|
141
|
+
|
|
142
|
+
const processedFiles = result.processedFiles;
|
|
143
|
+
const files = new Array<ContextFile>(processedFiles.length);
|
|
144
|
+
const tokenCounts = result.fileTokenCounts;
|
|
145
|
+
let totalTokens = 0;
|
|
146
|
+
let totalChars = 0;
|
|
147
|
+
|
|
148
|
+
for (let i = 0; i < processedFiles.length; i++) {
|
|
149
|
+
const file = processedFiles[i];
|
|
150
|
+
const tokens = tokenCounts[file.path]
|
|
151
|
+
?? Math.ceil(file.content.length / ESTIMATED_CHARS_PER_TOKEN);
|
|
152
|
+
files[i] = { path: file.path, content: file.content, tokens };
|
|
153
|
+
totalTokens += tokens;
|
|
154
|
+
totalChars += file.content.length;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
const bundle: ContextBundle = {
|
|
158
|
+
files,
|
|
159
|
+
totalFiles: files.length,
|
|
160
|
+
totalTokens,
|
|
161
|
+
totalChars,
|
|
162
|
+
};
|
|
163
|
+
cacheSet(key, bundle);
|
|
164
|
+
return { ok: true, value: bundle };
|
|
165
|
+
} catch (err: unknown) {
|
|
166
|
+
if (signal?.aborted) return { ok: false, error: "aborted" };
|
|
167
|
+
return {
|
|
168
|
+
ok: false,
|
|
169
|
+
error: errorMessage(err),
|
|
170
|
+
};
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
export function patchContextAfterEdits(
|
|
175
|
+
cached: ContextBundle,
|
|
176
|
+
edits: readonly { readonly path: string; readonly newContent: string }[],
|
|
177
|
+
): ContextBundle {
|
|
178
|
+
const editMap = new Map<string, string>();
|
|
179
|
+
for (const edit of edits) {
|
|
180
|
+
editMap.set(edit.path, edit.newContent);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
const files = new Array<ContextFile>(cached.files.length);
|
|
184
|
+
let totalTokens = 0;
|
|
185
|
+
let totalChars = 0;
|
|
186
|
+
|
|
187
|
+
for (let i = 0; i < cached.files.length; i++) {
|
|
188
|
+
const file = cached.files[i];
|
|
189
|
+
const newContent = editMap.get(file.path);
|
|
190
|
+
if (newContent !== undefined) {
|
|
191
|
+
const tokens = Math.ceil(newContent.length / ESTIMATED_CHARS_PER_TOKEN);
|
|
192
|
+
files[i] = { path: file.path, content: newContent, tokens };
|
|
193
|
+
totalTokens += tokens;
|
|
194
|
+
totalChars += newContent.length;
|
|
195
|
+
} else {
|
|
196
|
+
files[i] = file;
|
|
197
|
+
totalTokens += file.tokens;
|
|
198
|
+
totalChars += file.content.length;
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
return {
|
|
203
|
+
files,
|
|
204
|
+
totalFiles: files.length,
|
|
205
|
+
totalTokens,
|
|
206
|
+
totalChars,
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
export function serializeForSandbox(
|
|
211
|
+
bundle: ContextBundle,
|
|
212
|
+
): readonly ContextFile[] {
|
|
213
|
+
return bundle.files;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/** Maximum files shown in the compact LLM listing before truncation. */
|
|
217
|
+
const MAX_LLM_LISTING_FILES = 200;
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* Produces a compact human-readable text block for the parent LLM's context window.
|
|
221
|
+
* Shows file paths and token estimates — NOT full file contents (those are too large
|
|
222
|
+
* for the context window). The LLM uses its file-reading tools to inspect specific files.
|
|
223
|
+
*/
|
|
224
|
+
export function formatForLLM(bundle: ContextBundle): string {
|
|
225
|
+
const files = bundle.files.slice(0, MAX_LLM_LISTING_FILES);
|
|
226
|
+
const truncated = bundle.totalFiles > MAX_LLM_LISTING_FILES
|
|
227
|
+
? `... and ${bundle.totalFiles - MAX_LLM_LISTING_FILES} more files (truncated)`
|
|
228
|
+
: "";
|
|
229
|
+
|
|
230
|
+
const listing = files.map((f) =>
|
|
231
|
+
`${f.path} (${f.tokens.toLocaleString()} tok, ${f.content.length.toLocaleString()} chars)`,
|
|
232
|
+
).join("\n");
|
|
233
|
+
|
|
234
|
+
return [
|
|
235
|
+
`Repository context: ${bundle.totalFiles.toLocaleString()} files, ${bundle.totalTokens.toLocaleString()} estimated tokens, ${bundle.totalChars.toLocaleString()} total characters.`,
|
|
236
|
+
"",
|
|
237
|
+
listing,
|
|
238
|
+
truncated,
|
|
239
|
+
"",
|
|
240
|
+
"To read a file, use the file-reading tools with the exact path.",
|
|
241
|
+
].join("\n");
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
export function patchCachedContext(
|
|
245
|
+
cwd: string,
|
|
246
|
+
edits: readonly { readonly path: string; readonly newContent: string }[],
|
|
247
|
+
): void {
|
|
248
|
+
const key = cacheKey(cwd);
|
|
249
|
+
const entry = cache.get(key);
|
|
250
|
+
if (!entry) return;
|
|
251
|
+
const patched = patchContextAfterEdits(entry.bundle, edits);
|
|
252
|
+
cache.set(key, { bundle: patched, ts: entry.ts });
|
|
253
|
+
}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
/** Helpers for detecting and formatting the RLM final answer from a turn's REPL results. */
|
|
2
|
+
|
|
3
|
+
import type { ProposedDiffEdit, ProposedEdit, ReplResult } from "../sandbox/protocol.ts";
|
|
4
|
+
import { truncateOutput } from "../text/parsing.ts";
|
|
5
|
+
|
|
6
|
+
/** First non-null final answer across a turn's executed blocks, or null. */
|
|
7
|
+
export function finalAnswerOf(results: readonly ReplResult[]): string | null {
|
|
8
|
+
for (const r of results) if (r.finalAnswer != null) return r.finalAnswer;
|
|
9
|
+
return null;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
/** Last non-empty answer content set by the REPL, even if answer.ready was not flipped. */
|
|
13
|
+
export function latestAnswerContentOf(results: readonly ReplResult[]): string | null {
|
|
14
|
+
for (let i = results.length - 1; i >= 0; i--) {
|
|
15
|
+
const content = results[i]?.answerContent.trim();
|
|
16
|
+
if (content) return content;
|
|
17
|
+
}
|
|
18
|
+
return null;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** Last cumulative legacy anchor proposed-edit set reported by a turn. */
|
|
22
|
+
export function collectEdits(results: readonly ReplResult[]): ProposedEdit[] {
|
|
23
|
+
for (let i = results.length - 1; i >= 0; i--) {
|
|
24
|
+
const edits = results[i]?.edits;
|
|
25
|
+
if (edits && edits.length > 0) return [...edits];
|
|
26
|
+
}
|
|
27
|
+
return [];
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** Last cumulative diff proposal set reported by a turn. */
|
|
31
|
+
export function collectDiffs(results: readonly ReplResult[]): ProposedDiffEdit[] {
|
|
32
|
+
for (let i = results.length - 1; i >= 0; i--) {
|
|
33
|
+
const diffs = results[i]?.diffs;
|
|
34
|
+
if (diffs && diffs.length > 0) return [...diffs];
|
|
35
|
+
}
|
|
36
|
+
return [];
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** Parses a ```diff fence from answer text; returns [] if none found. */
|
|
40
|
+
export function tryExtractDiff(answer: string): ProposedDiffEdit[] {
|
|
41
|
+
const match = /```diff\n([\s\S]*?)```/.exec(answer);
|
|
42
|
+
if (!match) return [];
|
|
43
|
+
return [{ diff: match[1].trim() }];
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** True if any block in the turn raised an exception. Plain stderr does not count. */
|
|
47
|
+
export function turnHadError(results: readonly ReplResult[]): boolean {
|
|
48
|
+
return results.some((r) => r.raised);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** Max stdout kept verbatim in history. Larger outputs collapse to a small preview + elision note —
|
|
52
|
+
* the full content persists in REPL variables, never in the root model's history (Algorithm 1:
|
|
53
|
+
* hist ← hist ∥ code ∥ Metadata(stdout)). */
|
|
54
|
+
const SMALL_STDOUT_LIMIT = 800;
|
|
55
|
+
const STDOUT_PREVIEW_LIMIT = 200;
|
|
56
|
+
const STDERR_LIMIT = 8_000;
|
|
57
|
+
|
|
58
|
+
/** The REPL output fed back to the model as the next user message. */
|
|
59
|
+
export function formatReplOutputs(results: readonly ReplResult[]): string {
|
|
60
|
+
if (results.length === 0) {
|
|
61
|
+
return "No ```repl``` block found in your response. Write one to interact with the REPL.";
|
|
62
|
+
}
|
|
63
|
+
const multi = results.length > 1;
|
|
64
|
+
const parts: string[] = [];
|
|
65
|
+
let hadElision = false;
|
|
66
|
+
for (const [i, r] of results.entries()) {
|
|
67
|
+
const head = multi ? `[block ${i + 1}]\n` : "";
|
|
68
|
+
const { text, elided } = formatStdout(r);
|
|
69
|
+
hadElision ||= elided;
|
|
70
|
+
parts.push(`${head}${text}${formatStderr(r)}`);
|
|
71
|
+
}
|
|
72
|
+
const body = parts.join("\n\n");
|
|
73
|
+
// Orientation hint only when the model lost output to elision — otherwise it sees everything.
|
|
74
|
+
if (!hadElision) return body;
|
|
75
|
+
// The REPL namespace is persistent across blocks in a turn, so the last block's varNames reflect
|
|
76
|
+
// every variable created in any earlier block too.
|
|
77
|
+
const varNames = results.at(-1)?.varNames ?? [];
|
|
78
|
+
const hint = varNames.length > 0
|
|
79
|
+
? `REPL vars: ${varNames.join(", ")}`
|
|
80
|
+
: `No REPL vars yet — assign results to variables before printing large outputs.`;
|
|
81
|
+
return `${body}\n\n${hint}`;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** Stdout ≤ SMALL_STDOUT_LIMIT flows through verbatim; larger output keeps a short head + a note
|
|
85
|
+
* telling the model how to inspect it in slices. Returns whether elision occurred (drives the var-list). */
|
|
86
|
+
function formatStdout(r: ReplResult): { text: string; elided: boolean } {
|
|
87
|
+
const out = r.stdout.trim();
|
|
88
|
+
if (!out) return { text: "(no stdout)", elided: false };
|
|
89
|
+
if (out.length <= SMALL_STDOUT_LIMIT) return { text: out, elided: false };
|
|
90
|
+
const note = `[+${out.length - STDOUT_PREVIEW_LIMIT} chars elided — use slices to inspect: print(result[:500])]`;
|
|
91
|
+
return { text: `${out.slice(0, STDOUT_PREVIEW_LIMIT)}\n${note}`, elided: true };
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function formatStderr(r: ReplResult): string {
|
|
95
|
+
const err = r.stderr.trim();
|
|
96
|
+
return err ? `\n[stderr]\n${truncateOutput(err, STDERR_LIMIT)}` : "";
|
|
97
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Trajectory compaction (port of rlm/core/rlm.py `_compact_history`).
|
|
3
|
+
*
|
|
4
|
+
* When the root history grows past a fraction of the model's context window, replace the middle
|
|
5
|
+
* of the conversation with a single running summary — a bounded-memory recap (the linear-space
|
|
6
|
+
* idea from DP sequence alignment). Keeps the system message + a fresh "continue" instruction.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import type { Api, Model, Usage } from "@earendil-works/pi-ai";
|
|
10
|
+
import type { ModelRegistry } from "@earendil-works/pi-coding-agent";
|
|
11
|
+
import { type ChatMsg, modelComplete } from "../bridge/model.ts";
|
|
12
|
+
import { estimateMessageTokens } from "../text/tokens.ts";
|
|
13
|
+
|
|
14
|
+
const DEFAULT_CONTEXT_WINDOW = 128_000;
|
|
15
|
+
|
|
16
|
+
const SUMMARY_REQUEST =
|
|
17
|
+
"Summarize your progress so far. Include: (1) which sub-tasks are done and which remain; " +
|
|
18
|
+
"(2) any concrete intermediate results — numbers, values, variable names — preserved exactly; " +
|
|
19
|
+
"(3) your next action. Be concise (1–3 paragraphs) but preserve all key results.";
|
|
20
|
+
|
|
21
|
+
export interface CompactionDeps {
|
|
22
|
+
readonly model: Model<Api>;
|
|
23
|
+
readonly registry: ModelRegistry;
|
|
24
|
+
readonly contextWindow?: number;
|
|
25
|
+
readonly thresholdPct?: number;
|
|
26
|
+
readonly signal?: AbortSignal;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** True if the history is at/over the compaction threshold. */
|
|
30
|
+
export function shouldCompact(history: ChatMsg[], deps: CompactionDeps): boolean {
|
|
31
|
+
const contextWindow = deps.contextWindow && deps.contextWindow > 0 ? deps.contextWindow : DEFAULT_CONTEXT_WINDOW;
|
|
32
|
+
const threshold = (deps.thresholdPct ?? 0.85) * contextWindow;
|
|
33
|
+
return estimateMessageTokens(history) >= threshold;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Summarize the trajectory and return a compacted history: [system, summary(assistant),
|
|
38
|
+
* continue(user)]. The caller continues appending turns from there.
|
|
39
|
+
*/
|
|
40
|
+
export async function compactHistory(
|
|
41
|
+
history: ChatMsg[],
|
|
42
|
+
deps: CompactionDeps,
|
|
43
|
+
count = 1,
|
|
44
|
+
onUsage?: (u: Usage) => void,
|
|
45
|
+
): Promise<ChatMsg[]> {
|
|
46
|
+
const { text: summary, usage } = await modelComplete([...history, { role: "user", content: SUMMARY_REQUEST }], {
|
|
47
|
+
model: deps.model,
|
|
48
|
+
registry: deps.registry,
|
|
49
|
+
signal: deps.signal,
|
|
50
|
+
});
|
|
51
|
+
onUsage?.(usage);
|
|
52
|
+
const system = history.find((m) => m.role === "system");
|
|
53
|
+
const head: ChatMsg[] = system ? [system] : [];
|
|
54
|
+
return [
|
|
55
|
+
...head,
|
|
56
|
+
{ role: "assistant", content: summary },
|
|
57
|
+
{
|
|
58
|
+
role: "user",
|
|
59
|
+
content:
|
|
60
|
+
`Your conversation has been compacted ${count} time(s). Continue from the summary above. ` +
|
|
61
|
+
"Do NOT repeat completed work. Use SHOW_VARS() to see existing REPL variables. Your next action:",
|
|
62
|
+
},
|
|
63
|
+
];
|
|
64
|
+
}
|