@hicaru/pi-rlm 0.1.8 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -19
- package/package.json +2 -1
- package/src/bridge/library.ts +93 -15
- package/src/bridge/llm-query.ts +60 -36
- package/src/bridge/rlm-query.ts +63 -79
- package/src/commands/rlm-config.ts +8 -8
- package/src/commands/rlm.ts +48 -12
- package/src/config/settings.ts +33 -3
- package/src/context/library-context.ts +209 -22
- package/src/context/repomix-context.ts +7 -58
- package/src/core/answer.ts +5 -13
- package/src/core/artifacts.ts +4 -3
- package/src/core/critique.ts +92 -0
- package/src/core/engine.ts +94 -299
- package/src/core/gates.ts +33 -4
- package/src/core/limits.ts +19 -1
- package/src/core/pipeline-handlers.ts +319 -0
- package/src/core/pipeline.ts +40 -15
- package/src/core/types.ts +26 -30
- package/src/index.ts +36 -26
- package/src/mode/native-guards.ts +2 -2
- package/src/mode/rlm-mode.ts +8 -11
- package/src/prompts/phases.ts +18 -39
- package/src/prompts/system.ts +167 -64
- package/src/prompts/user.ts +1 -5
- package/src/sandbox/protocol.ts +5 -17
- package/src/sandbox/sandbox-manager.ts +5 -5
- package/src/sandbox/sandbox.ts +67 -27
- package/src/sandbox/worker.py +534 -48
- package/src/state/paths.ts +1 -1
- package/src/state/reads.ts +12 -4
- package/src/state/resume.ts +26 -25
- package/src/state/rows.ts +2 -2
- package/src/text/parsing.ts +0 -6
- package/src/text/tokens.ts +7 -1
- package/src/tool/repl-details.ts +2 -3
- package/src/tool/repl-tool.ts +132 -337
- package/src/tool/rlm-aggregator.ts +7 -7
- package/src/tool/rlm-details.ts +6 -13
- package/src/tool/rlm-events.ts +14 -11
- package/src/tool/rlm-tool.ts +20 -38
- package/src/tool/subcall-render.ts +61 -9
- package/src/tool/subcall-store.ts +4 -2
- package/src/ui/config-panel.ts +43 -23
- package/src/ui/intro.ts +2 -1
- package/src/ui/status.ts +8 -5
- package/src/ui/theme-adapter.ts +36 -0
- package/src/ui/theme.ts +0 -25
- package/src/mode/input-router.ts +0 -23
- package/src/registry/edit-registry.ts +0 -22
- package/src/text/edits.ts +0 -164
- package/src/tool/apply-edits-tool.ts +0 -295
package/src/commands/rlm.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/** `/rlm` — toggle persistent Recursive Language Model mode. */
|
|
2
2
|
|
|
3
|
-
import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
3
|
+
import type { ExtensionAPI, ExtensionContext, Theme } from "@earendil-works/pi-coding-agent";
|
|
4
|
+
import { Container, Text, type Component } from "@earendil-works/pi-tui";
|
|
4
5
|
import { createPiInteractiveDeps } from "../bridge/pi-interactive.ts";
|
|
5
6
|
import type { RlmController, RunHandle } from "../mode/rlm-mode.ts";
|
|
6
7
|
import { postRlmGuide } from "../ui/intro.ts";
|
|
@@ -13,13 +14,19 @@ import type { RunHeader } from "../state/rows.ts";
|
|
|
13
14
|
import { buildRlmSystemPrompt } from "../prompts/system.ts";
|
|
14
15
|
import { RlmEmitter } from "../tool/rlm-events.ts";
|
|
15
16
|
import { RlmEventAggregator } from "../tool/rlm-aggregator.ts";
|
|
17
|
+
import type { RlmDetails } from "../tool/rlm-details.ts";
|
|
18
|
+
import { cardHeader, cardStatsLine, renderCollapsedSubcallTree } from "../tool/subcall-render.ts";
|
|
19
|
+
import { errorMessage } from "../util/errors.ts";
|
|
20
|
+
|
|
21
|
+
/** Run ids offered for `/rlm-resume <TAB>`. */
|
|
22
|
+
const MAX_COMPLETIONS = 20;
|
|
16
23
|
|
|
17
24
|
export function registerRlmCommand(pi: ExtensionAPI, controller: RlmController): void {
|
|
18
25
|
pi.registerCommand("rlm", {
|
|
19
26
|
description: "Toggle persistent RLM mode (route plain prompts through the RLM engine).",
|
|
20
27
|
handler: async (_args, ctx) => {
|
|
21
28
|
const enabled = controller.toggle();
|
|
22
|
-
setRlmModeStatus(ctx.ui, controller);
|
|
29
|
+
setRlmModeStatus(ctx.ui, controller, ctx.getContextUsage());
|
|
23
30
|
ctx.ui.notify(`RLM mode ${enabled ? "ON" : "OFF"}`, "info");
|
|
24
31
|
},
|
|
25
32
|
});
|
|
@@ -45,6 +52,15 @@ export function registerRlmCommand(pi: ExtensionAPI, controller: RlmController):
|
|
|
45
52
|
|
|
46
53
|
pi.registerCommand("rlm-resume", {
|
|
47
54
|
description: "Resume an interrupted RLM run (default @latest).",
|
|
55
|
+
getArgumentCompletions: async (prefix) => {
|
|
56
|
+
const dir = controller.config.runLog?.dir ?? DEFAULT_RUN_DIR;
|
|
57
|
+
const ids = await listRunIds(process.cwd(), dir);
|
|
58
|
+
const candidates = ["@latest", ...ids];
|
|
59
|
+
return candidates
|
|
60
|
+
.filter((value) => value.startsWith(prefix))
|
|
61
|
+
.slice(0, MAX_COMPLETIONS)
|
|
62
|
+
.map((value) => ({ value, label: value }));
|
|
63
|
+
},
|
|
48
64
|
handler: async (args, ctx) => {
|
|
49
65
|
if (controller.isBusy()) {
|
|
50
66
|
ctx.ui.notify("RLM is busy (use /rlm-stop to cancel).", "warning");
|
|
@@ -69,7 +85,7 @@ export function registerRlmCommand(pi: ExtensionAPI, controller: RlmController):
|
|
|
69
85
|
let recon: ReconstructResult;
|
|
70
86
|
try { recon = await reconstructRlmState(cwd, dir, runId, systemPrompt); }
|
|
71
87
|
catch (e) {
|
|
72
|
-
ctx.ui.notify(`RLM resume failed: corrupt run state — ${
|
|
88
|
+
ctx.ui.notify(`RLM resume failed: corrupt run state — ${errorMessage(e)}`, "error");
|
|
73
89
|
return;
|
|
74
90
|
}
|
|
75
91
|
if (!recon.ok) { ctx.ui.notify(`Cannot resume ${runId}: ${recon.reason}.`, "error"); return; }
|
|
@@ -94,12 +110,29 @@ export function registerRlmCommand(pi: ExtensionAPI, controller: RlmController):
|
|
|
94
110
|
description: "Toggle RLM mode (off also stops a running query)",
|
|
95
111
|
handler: async (ctx) => {
|
|
96
112
|
const enabled = controller.toggle();
|
|
97
|
-
setRlmModeStatus(ctx.ui, controller);
|
|
113
|
+
setRlmModeStatus(ctx.ui, controller, ctx.getContextUsage());
|
|
98
114
|
ctx.ui.notify(`RLM mode ${enabled ? "ON" : "OFF"}`, "info");
|
|
99
115
|
},
|
|
100
116
|
});
|
|
101
117
|
}
|
|
102
118
|
|
|
119
|
+
/** Above-editor progress card for a `/rlm-resume` run: header + the live sub-call tree. */
|
|
120
|
+
function renderResumeWidget(details: RlmDetails | undefined, theme: Theme): Component {
|
|
121
|
+
const container = new Container();
|
|
122
|
+
if (!details) return container;
|
|
123
|
+
const turns = details.turns;
|
|
124
|
+
const stats = cardStatsLine(
|
|
125
|
+
details.totals,
|
|
126
|
+
theme,
|
|
127
|
+
turns.max > 0 ? `turn ${turns.current}/${turns.max}` : undefined,
|
|
128
|
+
);
|
|
129
|
+
container.addChild(new Text(cardHeader("RLM resume", details.status, stats, theme), 0, 0));
|
|
130
|
+
if (details.subcalls.length > 0) {
|
|
131
|
+
container.addChild(new Text(renderCollapsedSubcallTree(details.subcalls, theme), 0, 0));
|
|
132
|
+
}
|
|
133
|
+
return container;
|
|
134
|
+
}
|
|
135
|
+
|
|
103
136
|
async function executeRlmRunWithResume(
|
|
104
137
|
pi: ExtensionAPI,
|
|
105
138
|
controller: RlmController,
|
|
@@ -113,13 +146,16 @@ async function executeRlmRunWithResume(
|
|
|
113
146
|
let aggregator: RlmEventAggregator | undefined;
|
|
114
147
|
try {
|
|
115
148
|
emitter = new RlmEmitter();
|
|
149
|
+
// Component factory rather than the string[] form: the array form is hard-capped at 10
|
|
150
|
+
// lines by pi, which the live sub-call tree exceeds as soon as a run fans out. The factory
|
|
151
|
+
// also receives the live theme, so the widget follows /theme switches.
|
|
152
|
+
let latest: RlmDetails | undefined;
|
|
116
153
|
aggregator = new RlmEventAggregator(emitter, (partial) => {
|
|
117
|
-
|
|
118
|
-
if (!
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
ctx.ui.setWidget?.("rlm-status", [`${glyph} RLM resume${turn}${cost}`], { placement: "aboveEditor" });
|
|
154
|
+
latest = partial.details;
|
|
155
|
+
if (!latest) return;
|
|
156
|
+
ctx.ui.setWidget?.("rlm-status", (_tui, theme) => renderResumeWidget(latest, theme), {
|
|
157
|
+
placement: "aboveEditor",
|
|
158
|
+
});
|
|
123
159
|
});
|
|
124
160
|
emitter.emitRootPrompt(header.rootPrompt);
|
|
125
161
|
const interactive = createPiInteractiveDeps(ctx);
|
|
@@ -131,7 +167,7 @@ async function executeRlmRunWithResume(
|
|
|
131
167
|
onTodo: controller.config.todo ? interactive.onTodo : undefined,
|
|
132
168
|
});
|
|
133
169
|
} catch (e) {
|
|
134
|
-
ctx.ui.notify(`RLM resume failed: ${
|
|
170
|
+
ctx.ui.notify(`RLM resume failed: ${errorMessage(e)}`, "error");
|
|
135
171
|
return;
|
|
136
172
|
}
|
|
137
173
|
pi.sendMessage({ customType: "rlm-question", content: `[resume] ${header.rootPrompt}`, display: true });
|
|
@@ -140,7 +176,7 @@ async function executeRlmRunWithResume(
|
|
|
140
176
|
const result = await done;
|
|
141
177
|
pi.sendMessage({ customType: "rlm-answer", content: result.answer, display: true });
|
|
142
178
|
} catch (e) {
|
|
143
|
-
ctx.ui.notify(`RLM resume failed: ${
|
|
179
|
+
ctx.ui.notify(`RLM resume failed: ${errorMessage(e)}`, "error");
|
|
144
180
|
} finally {
|
|
145
181
|
clearRlmStatus(ctx.ui);
|
|
146
182
|
ctx.ui.setWidget?.("rlm-status", undefined);
|
package/src/config/settings.ts
CHANGED
|
@@ -30,6 +30,19 @@ function validateString(v: unknown): string | undefined {
|
|
|
30
30
|
return typeof v === "string" && v.trim() ? v : undefined;
|
|
31
31
|
}
|
|
32
32
|
|
|
33
|
+
/**
|
|
34
|
+
* Every value pi-ai accepts for `reasoning`. Keyed by the union so a new level added upstream
|
|
35
|
+
* is a compile error here rather than a silently-rejected setting. Note `off` and `max` are
|
|
36
|
+
* NOT ThinkingLevels — a hand-edited rlm.json carrying one is dropped, not forwarded.
|
|
37
|
+
*/
|
|
38
|
+
const THINKING_LEVELS: Readonly<Record<ThinkingLevel, true>> = Object.freeze({
|
|
39
|
+
minimal: true, low: true, medium: true, high: true, xhigh: true,
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
function validateThinkingLevel(v: unknown): ThinkingLevel | undefined {
|
|
43
|
+
return typeof v === "string" && Object.hasOwn(THINKING_LEVELS, v) ? (v as ThinkingLevel) : undefined;
|
|
44
|
+
}
|
|
45
|
+
|
|
33
46
|
function validateRunLog(raw: unknown): Partial<RunLogConfig> | undefined {
|
|
34
47
|
if (typeof raw !== "object" || raw === null) return undefined;
|
|
35
48
|
const r = raw as Record<string, unknown>;
|
|
@@ -83,7 +96,8 @@ function validateConfig(raw: unknown): Partial<RlmConfig> {
|
|
|
83
96
|
if (compactionThresholdPct !== undefined && compactionThresholdPct <= 1) out.compactionThresholdPct = compactionThresholdPct;
|
|
84
97
|
const python = validateString(r.python);
|
|
85
98
|
if (python !== undefined) out.python = python;
|
|
86
|
-
|
|
99
|
+
const smartReasoning = validateThinkingLevel(r.smartReasoning);
|
|
100
|
+
if (smartReasoning !== undefined) out.smartReasoning = smartReasoning;
|
|
87
101
|
const subSystemPrompt = validateString(r.subSystemPrompt);
|
|
88
102
|
if (subSystemPrompt !== undefined) out.subSystemPrompt = subSystemPrompt;
|
|
89
103
|
const runLog = validateRunLog(r.runLog);
|
|
@@ -103,7 +117,8 @@ function validateConfig(raw: unknown): Partial<RlmConfig> {
|
|
|
103
117
|
if (maxTokensValue !== undefined) sampling.maxTokens = maxTokensValue;
|
|
104
118
|
const temperature = validateNumber(ss.temperature, 0);
|
|
105
119
|
if (temperature !== undefined) sampling.temperature = temperature;
|
|
106
|
-
|
|
120
|
+
const ssReasoning = validateThinkingLevel(ss.reasoning);
|
|
121
|
+
if (ssReasoning !== undefined) sampling.reasoning = ssReasoning;
|
|
107
122
|
out.subSampling = sampling;
|
|
108
123
|
}
|
|
109
124
|
if (typeof r.rootSampling === "object" && r.rootSampling !== null) {
|
|
@@ -113,7 +128,8 @@ function validateConfig(raw: unknown): Partial<RlmConfig> {
|
|
|
113
128
|
if (rsMaxTokens !== undefined) rootSampling.maxTokens = rsMaxTokens;
|
|
114
129
|
const rsTemperature = validateNumber(rs.temperature, 0);
|
|
115
130
|
if (rsTemperature !== undefined) rootSampling.temperature = rsTemperature;
|
|
116
|
-
|
|
131
|
+
const rsReasoning = validateThinkingLevel(rs.reasoning);
|
|
132
|
+
if (rsReasoning !== undefined) rootSampling.reasoning = rsReasoning;
|
|
117
133
|
out.rootSampling = Object.freeze(rootSampling);
|
|
118
134
|
}
|
|
119
135
|
return out;
|
|
@@ -166,3 +182,17 @@ export function resolveModelId(registry: ModelRegistry, ref?: string): Model<Api
|
|
|
166
182
|
export function modelRef(model: Model<Api> | undefined): string | undefined {
|
|
167
183
|
return model ? `${model.provider}/${model.id}` : undefined;
|
|
168
184
|
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* Human-readable "provider/id" for a sub-call node: the resolved override when one was
|
|
188
|
+
* supplied and resolves, otherwise the fallback model. Shared by the llm and rlm bridges
|
|
189
|
+
* so sub-call trees label their nodes identically.
|
|
190
|
+
*/
|
|
191
|
+
export function displayModelRef(
|
|
192
|
+
registry: ModelRegistry,
|
|
193
|
+
override: string | null,
|
|
194
|
+
fallback: Model<Api>,
|
|
195
|
+
): string {
|
|
196
|
+
const resolved = override ? (resolveModelId(registry, override) ?? fallback) : fallback;
|
|
197
|
+
return modelRef(resolved) ?? fallback.id;
|
|
198
|
+
}
|
|
@@ -1,30 +1,211 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Resolve load_library(source) into a sandbox-ready payload.
|
|
3
3
|
*
|
|
4
|
-
* Sources: local directory (repomix-packed), single file (utf-8
|
|
5
|
-
*
|
|
4
|
+
* Sources: local directory (repomix-packed), single file (utf-8), or remote git URL
|
|
5
|
+
* (shallow clone then pack). Host-side only — never runs in the sandbox.
|
|
6
|
+
*
|
|
7
|
+
* Every successful payload is a namespaced list of ContextFile under
|
|
8
|
+
* `lib/<source_id>/…` so the worker can append into the single `context` variable.
|
|
9
|
+
* Source ids include a short content fingerprint so two libraries that share a
|
|
10
|
+
* basename never collide.
|
|
6
11
|
*/
|
|
7
12
|
|
|
13
|
+
import { createHash } from "node:crypto";
|
|
8
14
|
import { execFile } from "node:child_process";
|
|
9
15
|
import { mkdtemp, readFile, rm, stat } from "node:fs/promises";
|
|
16
|
+
import { basename, isAbsolute, join, resolve } from "node:path";
|
|
10
17
|
import { tmpdir } from "node:os";
|
|
11
|
-
import { isAbsolute, join, resolve } from "node:path";
|
|
12
18
|
import { promisify } from "node:util";
|
|
13
|
-
import {
|
|
19
|
+
import {
|
|
20
|
+
packRepository,
|
|
21
|
+
serializeForSandbox,
|
|
22
|
+
type ContextBundle,
|
|
23
|
+
type ContextFile,
|
|
24
|
+
} from "./repomix-context.ts";
|
|
25
|
+
import { estimateTokens } from "../text/tokens.ts";
|
|
14
26
|
import type { Result } from "../util/errors.ts";
|
|
15
27
|
import { errorMessage } from "../util/errors.ts";
|
|
16
28
|
|
|
17
29
|
const execFileP = promisify(execFile);
|
|
18
30
|
|
|
31
|
+
/** Single-file sources above this must use open() + llm_query_chunked in the REPL. */
|
|
32
|
+
export const MAX_LIBRARY_FILE_BYTES = 8 * 1024 * 1024;
|
|
33
|
+
|
|
34
|
+
/** Legacy catch-all prefix for pre-namespace string sidecars — never an identity key. */
|
|
35
|
+
const LEGACY_UNKNOWN_PREFIX = "lib/unknown/";
|
|
36
|
+
|
|
19
37
|
export interface LibrarySource {
|
|
20
|
-
|
|
21
|
-
readonly
|
|
38
|
+
/** Always a namespaced file list (dirs, single files, and git clones). */
|
|
39
|
+
readonly payload: readonly ContextFile[];
|
|
40
|
+
readonly files: number;
|
|
41
|
+
/** Sum of raw content lengths — what the model should size batches against. */
|
|
22
42
|
readonly chars: number;
|
|
43
|
+
readonly sourceId: string;
|
|
44
|
+
readonly pathPrefix: string;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export interface LibraryNamespace {
|
|
48
|
+
readonly sourceId: string;
|
|
49
|
+
readonly pathPrefix: string;
|
|
23
50
|
}
|
|
24
51
|
|
|
25
52
|
/** https://host/… or git@host:… — option-injection safe (never starts with "-"). */
|
|
26
53
|
const GIT_URL = /^(https:\/\/|git@)[\w.-]+[:/]\S+$/;
|
|
27
54
|
|
|
55
|
+
/** Short, stable discriminator so two sources never share a namespace. */
|
|
56
|
+
function sourceFingerprint(canonical: string): string {
|
|
57
|
+
return createHash("sha256").update(canonical).digest("hex").slice(0, 8);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Sanitize a path/url basename into a stable, filesystem-safe source id.
|
|
62
|
+
* `resolvedPath` (absolute path) or the git URL is fingerprinted into the id so
|
|
63
|
+
* distinct sources sharing a basename get distinct namespaces
|
|
64
|
+
* (`lib/utils-3f9a1c02/` vs `lib/utils-a1b2c3d4/`).
|
|
65
|
+
*/
|
|
66
|
+
export function librarySourceId(source: string, resolvedPath?: string): string {
|
|
67
|
+
const trimmed = source.trim();
|
|
68
|
+
const isGit = GIT_URL.test(trimmed);
|
|
69
|
+
let raw: string;
|
|
70
|
+
if (isGit) {
|
|
71
|
+
const m = trimmed.match(/(?:\/|:)([\w.-]+?)(?:\.git)?\/?\s*$/);
|
|
72
|
+
raw = m?.[1] ?? "repo";
|
|
73
|
+
} else {
|
|
74
|
+
raw = basename(resolvedPath ?? trimmed);
|
|
75
|
+
}
|
|
76
|
+
const cleaned = raw
|
|
77
|
+
.replace(/\.git$/i, "")
|
|
78
|
+
.replace(/[^\w.-]+/g, "-")
|
|
79
|
+
.replace(/^-+|-+$/g, "")
|
|
80
|
+
.slice(0, 80);
|
|
81
|
+
const canonical = isGit ? trimmed : (resolvedPath ?? trimmed);
|
|
82
|
+
return `${cleaned.length > 0 ? cleaned : "lib"}-${sourceFingerprint(canonical)}`;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export function pathPrefixFor(sourceId: string): string {
|
|
86
|
+
return `lib/${sourceId}/`;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Derive the namespace for a source string without packing (host-side idempotency).
|
|
91
|
+
* Returns both sourceId and pathPrefix so callers never un-parse the prefix.
|
|
92
|
+
*/
|
|
93
|
+
export function libraryNamespace(source: string, cwd: string): LibraryNamespace {
|
|
94
|
+
const trimmed = source.trim();
|
|
95
|
+
const sourceId = GIT_URL.test(trimmed)
|
|
96
|
+
? librarySourceId(trimmed)
|
|
97
|
+
: librarySourceId(trimmed, isAbsolute(trimmed) ? trimmed : resolve(cwd, trimmed));
|
|
98
|
+
return Object.freeze({ sourceId, pathPrefix: pathPrefixFor(sourceId) });
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** Derive the path prefix for a source string without packing. */
|
|
102
|
+
export function libraryPathPrefix(source: string, cwd: string): string {
|
|
103
|
+
return libraryNamespace(source, cwd).pathPrefix;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** Namespaced files plus summed content chars (one pass). */
|
|
107
|
+
export function namespaceLibraryFilesWithChars(
|
|
108
|
+
payload: unknown,
|
|
109
|
+
sourceId: string,
|
|
110
|
+
): { readonly files: readonly ContextFile[]; readonly chars: number } {
|
|
111
|
+
const prefix = pathPrefixFor(sourceId);
|
|
112
|
+
if (typeof payload === "string") {
|
|
113
|
+
return Object.freeze({
|
|
114
|
+
files: Object.freeze([
|
|
115
|
+
Object.freeze({
|
|
116
|
+
path: `${prefix}content`,
|
|
117
|
+
content: payload,
|
|
118
|
+
tokens: Math.max(1, estimateTokens(payload.length)),
|
|
119
|
+
}),
|
|
120
|
+
]),
|
|
121
|
+
chars: payload.length,
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
if (!Array.isArray(payload)) return Object.freeze({ files: Object.freeze([]), chars: 0 });
|
|
125
|
+
const out = new Array<ContextFile>(payload.length);
|
|
126
|
+
let n = 0;
|
|
127
|
+
let chars = 0;
|
|
128
|
+
for (const item of payload) {
|
|
129
|
+
if (item === null || typeof item !== "object") continue;
|
|
130
|
+
const rec = item as Record<string, unknown>;
|
|
131
|
+
const content = typeof rec.content === "string" ? rec.content : String(rec.content ?? "");
|
|
132
|
+
let path = typeof rec.path === "string" ? rec.path : "unknown";
|
|
133
|
+
path = path.replace(/^\/+/, "");
|
|
134
|
+
if (!path.startsWith(prefix)) path = `${prefix}${path}`;
|
|
135
|
+
const tokens = typeof rec.tokens === "number" && Number.isFinite(rec.tokens)
|
|
136
|
+
? Math.max(0, Math.floor(rec.tokens))
|
|
137
|
+
: Math.max(1, estimateTokens(content.length));
|
|
138
|
+
out[n++] = Object.freeze({ path, content, tokens });
|
|
139
|
+
chars += content.length;
|
|
140
|
+
}
|
|
141
|
+
out.length = n;
|
|
142
|
+
return Object.freeze({ files: Object.freeze(out), chars });
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** Namespace file entries under `lib/<sourceId>/…` (single shared implementation). */
|
|
146
|
+
export function namespaceLibraryFiles(
|
|
147
|
+
payload: unknown,
|
|
148
|
+
sourceId: string,
|
|
149
|
+
): readonly ContextFile[] {
|
|
150
|
+
return namespaceLibraryFilesWithChars(payload, sourceId).files;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/** First `lib/<id>/` prefix found in the payload, or undefined. Skips `lib/unknown/`. */
|
|
154
|
+
export function payloadPrefix(payload: readonly unknown[]): string | undefined {
|
|
155
|
+
for (let i = 0; i < payload.length; i++) {
|
|
156
|
+
const item = payload[i];
|
|
157
|
+
if (item === null || typeof item !== "object") continue;
|
|
158
|
+
const path = (item as { path?: unknown }).path;
|
|
159
|
+
if (typeof path !== "string") continue;
|
|
160
|
+
const m = /^(lib\/[^/]+\/)/.exec(path);
|
|
161
|
+
// `lib/unknown/` is the legacy catch-all: never treat it as an identity.
|
|
162
|
+
if (m?.[1] !== undefined && m[1] !== LEGACY_UNKNOWN_PREFIX) return m[1];
|
|
163
|
+
}
|
|
164
|
+
return undefined;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Append a library payload into an existing list context (host-side resume merge).
|
|
169
|
+
* Skips a payload whose path prefix is already present (resume-safe dedup).
|
|
170
|
+
*/
|
|
171
|
+
export function mergeLibraryIntoContext(base: unknown, libraryPayload: unknown): unknown {
|
|
172
|
+
if (!Array.isArray(base)) return base;
|
|
173
|
+
if (Array.isArray(libraryPayload)) {
|
|
174
|
+
if (libraryPayload.length === 0) return base;
|
|
175
|
+
const prefix = payloadPrefix(libraryPayload);
|
|
176
|
+
if (prefix !== undefined) {
|
|
177
|
+
for (let i = 0; i < base.length; i++) {
|
|
178
|
+
const item = base[i];
|
|
179
|
+
if (item !== null && typeof item === "object"
|
|
180
|
+
&& typeof (item as { path?: unknown }).path === "string"
|
|
181
|
+
&& (item as { path: string }).path.startsWith(prefix)) {
|
|
182
|
+
return base; // already present
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
const merged = new Array<unknown>(base.length + libraryPayload.length);
|
|
187
|
+
for (let i = 0; i < base.length; i++) merged[i] = base[i];
|
|
188
|
+
for (let i = 0; i < libraryPayload.length; i++) merged[base.length + i] = libraryPayload[i];
|
|
189
|
+
return merged;
|
|
190
|
+
}
|
|
191
|
+
if (typeof libraryPayload === "string") {
|
|
192
|
+
// Legacy string sidecars: wrap once under an unknown prefix.
|
|
193
|
+
return mergeLibraryIntoContext(base, namespaceLibraryFiles(libraryPayload, "unknown"));
|
|
194
|
+
}
|
|
195
|
+
return base;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
function toLibrarySource(payload: unknown, sourceId: string): LibrarySource {
|
|
199
|
+
const { files, chars } = namespaceLibraryFilesWithChars(payload, sourceId);
|
|
200
|
+
return {
|
|
201
|
+
payload: files,
|
|
202
|
+
files: files.length,
|
|
203
|
+
chars,
|
|
204
|
+
sourceId,
|
|
205
|
+
pathPrefix: pathPrefixFor(sourceId),
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
|
|
28
209
|
export async function resolveLibrarySource(
|
|
29
210
|
source: string,
|
|
30
211
|
cwd: string,
|
|
@@ -43,34 +224,40 @@ export async function resolveLibrarySource(
|
|
|
43
224
|
} catch {
|
|
44
225
|
return { ok: false, error: `load_library: path not found: ${path}` };
|
|
45
226
|
}
|
|
46
|
-
|
|
227
|
+
const sourceId = librarySourceId(trimmed, path);
|
|
228
|
+
if (s.isDirectory()) return await packDir(path, sourceId, signal);
|
|
229
|
+
if (s.size > MAX_LIBRARY_FILE_BYTES) {
|
|
230
|
+
return {
|
|
231
|
+
ok: false,
|
|
232
|
+
error: `load_library: ${path} is ${s.size.toLocaleString()} bytes `
|
|
233
|
+
+ `(limit ${MAX_LIBRARY_FILE_BYTES.toLocaleString()}) — `
|
|
234
|
+
+ "open() it in the REPL and delegate with llm_query_chunked instead",
|
|
235
|
+
};
|
|
236
|
+
}
|
|
47
237
|
const text = await readFile(path, "utf-8");
|
|
48
|
-
return { ok: true, value:
|
|
238
|
+
return { ok: true, value: toLibrarySource(text, sourceId) };
|
|
49
239
|
}
|
|
50
240
|
|
|
51
|
-
async function packDir(
|
|
52
|
-
|
|
241
|
+
async function packDir(
|
|
242
|
+
dir: string,
|
|
243
|
+
sourceId: string,
|
|
244
|
+
signal?: AbortSignal,
|
|
245
|
+
): Promise<Result<LibrarySource, string>> {
|
|
246
|
+
const packed = await packRepository(dir, signal);
|
|
53
247
|
if (!packed.ok) return { ok: false, error: `pack failed for ${dir} — ${packed.error}` };
|
|
54
|
-
return { ok: true, value: bundleToSource(packed.value) };
|
|
248
|
+
return { ok: true, value: bundleToSource(packed.value, sourceId) };
|
|
55
249
|
}
|
|
56
250
|
|
|
57
|
-
function bundleToSource(bundle: ContextBundle): LibrarySource {
|
|
58
|
-
|
|
59
|
-
// Match slot-0 contextLength for object/array payloads: JSON-serialized size,
|
|
60
|
-
// not sum-of-content-lengths (bundle.totalChars), so the model sees one ruler.
|
|
61
|
-
return {
|
|
62
|
-
payload,
|
|
63
|
-
files: bundle.totalFiles,
|
|
64
|
-
chars: JSON.stringify(payload).length,
|
|
65
|
-
};
|
|
251
|
+
function bundleToSource(bundle: ContextBundle, sourceId: string): LibrarySource {
|
|
252
|
+
return toLibrarySource(serializeForSandbox(bundle), sourceId);
|
|
66
253
|
}
|
|
67
254
|
|
|
68
255
|
async function cloneAndPack(url: string, signal?: AbortSignal): Promise<Result<LibrarySource, string>> {
|
|
69
256
|
const dir = await mkdtemp(join(tmpdir(), "rlm-lib-"));
|
|
257
|
+
const sourceId = librarySourceId(url);
|
|
70
258
|
try {
|
|
71
|
-
// "--" terminates options; URL regex already forbids a leading dash.
|
|
72
259
|
await execFileP("git", ["clone", "--depth", "1", "--", url, dir], { signal, timeout: 120_000 });
|
|
73
|
-
return await packDir(dir, signal);
|
|
260
|
+
return await packDir(dir, sourceId, signal);
|
|
74
261
|
} catch (err: unknown) {
|
|
75
262
|
return { ok: false, error: `git clone failed for ${url} — ${errorMessage(err)}` };
|
|
76
263
|
} finally {
|
|
@@ -5,15 +5,17 @@
|
|
|
5
5
|
*
|
|
6
6
|
* Uses repomix internally (worker-thread pool, built-in gitignore support)
|
|
7
7
|
* and caches results in a module-level Map with TTL to avoid re-packing on
|
|
8
|
-
* every run within the same process.
|
|
9
|
-
* cached bundle in-memory after file edits are applied to disk — no re-packing.
|
|
8
|
+
* every run within the same process.
|
|
10
9
|
*/
|
|
11
10
|
|
|
12
11
|
import { pack } from "repomix";
|
|
13
12
|
import type { PackResult as RepomixPackResult } from "repomix";
|
|
13
|
+
/** repomix's own config parameter type — `satisfies` keeps the literal checked against it. */
|
|
14
|
+
type PackConfig = NonNullable<Parameters<typeof pack>[1]>;
|
|
14
15
|
import { resolve } from "node:path";
|
|
15
16
|
import { tmpdir } from "node:os";
|
|
16
17
|
import { errorMessage } from "../util/errors.ts";
|
|
18
|
+
import { estimateTokens } from "../text/tokens.ts";
|
|
17
19
|
|
|
18
20
|
// ── Public types ──
|
|
19
21
|
|
|
@@ -52,11 +54,6 @@ interface CacheEntry {
|
|
|
52
54
|
const cache = new Map<string, CacheEntry>();
|
|
53
55
|
const DEFAULT_CACHE_TTL_MS = 30_000;
|
|
54
56
|
|
|
55
|
-
/** Exported for tests — empties the module-level cache. */
|
|
56
|
-
export function clearCache(): void {
|
|
57
|
-
cache.clear();
|
|
58
|
-
}
|
|
59
|
-
|
|
60
57
|
function cacheKey(cwd: string): string {
|
|
61
58
|
return resolve(cwd);
|
|
62
59
|
}
|
|
@@ -77,8 +74,6 @@ function cacheSet(key: string, bundle: ContextBundle): void {
|
|
|
77
74
|
|
|
78
75
|
// ── Core functions ──
|
|
79
76
|
|
|
80
|
-
const ESTIMATED_CHARS_PER_TOKEN = 4;
|
|
81
|
-
|
|
82
77
|
export async function packRepository(
|
|
83
78
|
cwd: string,
|
|
84
79
|
signal?: AbortSignal,
|
|
@@ -134,7 +129,7 @@ export async function packRepository(
|
|
|
134
129
|
},
|
|
135
130
|
security: { enableSecurityCheck: false },
|
|
136
131
|
tokenCount: { encoding: "o200k_base" as const },
|
|
137
|
-
}
|
|
132
|
+
} satisfies PackConfig),
|
|
138
133
|
new Promise<never>((_, reject) => {
|
|
139
134
|
signal?.addEventListener("abort", () => reject(new Error("aborted")), { once: true });
|
|
140
135
|
}),
|
|
@@ -148,8 +143,7 @@ export async function packRepository(
|
|
|
148
143
|
|
|
149
144
|
for (let i = 0; i < processedFiles.length; i++) {
|
|
150
145
|
const file = processedFiles[i];
|
|
151
|
-
const tokens = tokenCounts[file.path]
|
|
152
|
-
?? Math.ceil(file.content.length / ESTIMATED_CHARS_PER_TOKEN);
|
|
146
|
+
const tokens = tokenCounts[file.path] ?? estimateTokens(file.content.length);
|
|
153
147
|
files[i] = { path: file.path, content: file.content, tokens };
|
|
154
148
|
totalTokens += tokens;
|
|
155
149
|
totalChars += file.content.length;
|
|
@@ -172,42 +166,6 @@ export async function packRepository(
|
|
|
172
166
|
}
|
|
173
167
|
}
|
|
174
168
|
|
|
175
|
-
export function patchContextAfterEdits(
|
|
176
|
-
cached: ContextBundle,
|
|
177
|
-
edits: readonly { readonly path: string; readonly newContent: string }[],
|
|
178
|
-
): ContextBundle {
|
|
179
|
-
const editMap = new Map<string, string>();
|
|
180
|
-
for (const edit of edits) {
|
|
181
|
-
editMap.set(edit.path, edit.newContent);
|
|
182
|
-
}
|
|
183
|
-
|
|
184
|
-
const files = new Array<ContextFile>(cached.files.length);
|
|
185
|
-
let totalTokens = 0;
|
|
186
|
-
let totalChars = 0;
|
|
187
|
-
|
|
188
|
-
for (let i = 0; i < cached.files.length; i++) {
|
|
189
|
-
const file = cached.files[i];
|
|
190
|
-
const newContent = editMap.get(file.path);
|
|
191
|
-
if (newContent !== undefined) {
|
|
192
|
-
const tokens = Math.ceil(newContent.length / ESTIMATED_CHARS_PER_TOKEN);
|
|
193
|
-
files[i] = { path: file.path, content: newContent, tokens };
|
|
194
|
-
totalTokens += tokens;
|
|
195
|
-
totalChars += newContent.length;
|
|
196
|
-
} else {
|
|
197
|
-
files[i] = file;
|
|
198
|
-
totalTokens += file.tokens;
|
|
199
|
-
totalChars += file.content.length;
|
|
200
|
-
}
|
|
201
|
-
}
|
|
202
|
-
|
|
203
|
-
return {
|
|
204
|
-
files,
|
|
205
|
-
totalFiles: files.length,
|
|
206
|
-
totalTokens,
|
|
207
|
-
totalChars,
|
|
208
|
-
};
|
|
209
|
-
}
|
|
210
|
-
|
|
211
169
|
export function serializeForSandbox(
|
|
212
170
|
bundle: ContextBundle,
|
|
213
171
|
): readonly ContextFile[] {
|
|
@@ -243,13 +201,4 @@ export function formatForLLM(bundle: ContextBundle): string {
|
|
|
243
201
|
].join("\n");
|
|
244
202
|
}
|
|
245
203
|
|
|
246
|
-
|
|
247
|
-
cwd: string,
|
|
248
|
-
edits: readonly { readonly path: string; readonly newContent: string }[],
|
|
249
|
-
): void {
|
|
250
|
-
const key = cacheKey(cwd);
|
|
251
|
-
const entry = cache.get(key);
|
|
252
|
-
if (!entry) return;
|
|
253
|
-
const patched = patchContextAfterEdits(entry.bundle, edits);
|
|
254
|
-
cache.set(key, { bundle: patched, ts: entry.ts });
|
|
255
|
-
}
|
|
204
|
+
|
package/src/core/answer.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/** Helpers for detecting and formatting the RLM final answer from a turn's REPL results. */
|
|
2
2
|
|
|
3
|
-
import type {
|
|
3
|
+
import type { ReplResult } from "../sandbox/protocol.ts";
|
|
4
4
|
import { truncateOutput } from "../text/parsing.ts";
|
|
5
5
|
|
|
6
6
|
/** First non-null final answer across a turn's executed blocks, or null. */
|
|
@@ -18,15 +18,6 @@ export function latestAnswerContentOf(results: readonly ReplResult[]): string |
|
|
|
18
18
|
return null;
|
|
19
19
|
}
|
|
20
20
|
|
|
21
|
-
/** Last cumulative legacy anchor proposed-edit set reported by a turn. */
|
|
22
|
-
export function collectEdits(results: readonly ReplResult[]): ProposedEdit[] {
|
|
23
|
-
for (let i = results.length - 1; i >= 0; i--) {
|
|
24
|
-
const edits = results[i]?.edits;
|
|
25
|
-
if (edits && edits.length > 0) return [...edits];
|
|
26
|
-
}
|
|
27
|
-
return [];
|
|
28
|
-
}
|
|
29
|
-
|
|
30
21
|
/** True if any block in the turn raised an exception. Plain stderr does not count. */
|
|
31
22
|
export function turnHadError(results: readonly ReplResult[]): boolean {
|
|
32
23
|
return results.some((r) => r.raised);
|
|
@@ -46,13 +37,14 @@ export function formatReplOutputs(results: readonly ReplResult[], skippedBlocks
|
|
|
46
37
|
return "No ```repl``` block found in your response. Write one to interact with the REPL.";
|
|
47
38
|
}
|
|
48
39
|
const multi = results.length > 1;
|
|
49
|
-
const parts
|
|
40
|
+
const parts = new Array<string>(results.length);
|
|
50
41
|
let hadElision = false;
|
|
51
|
-
for (
|
|
42
|
+
for (let i = 0; i < results.length; i++) {
|
|
43
|
+
const r = results[i];
|
|
52
44
|
const head = multi ? `[block ${i + 1}]\n` : "";
|
|
53
45
|
const { text, elided } = formatStdout(r);
|
|
54
46
|
hadElision ||= elided;
|
|
55
|
-
parts
|
|
47
|
+
parts[i] = `${head}${text}${formatStderr(r)}`;
|
|
56
48
|
}
|
|
57
49
|
const body = parts.join("\n\n");
|
|
58
50
|
const skipNote = skippedBlocks > 0
|
package/src/core/artifacts.ts
CHANGED
|
@@ -6,6 +6,7 @@ import { execFileSync } from "node:child_process";
|
|
|
6
6
|
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
7
7
|
import { join } from "node:path";
|
|
8
8
|
import type { Result } from "../util/errors.ts";
|
|
9
|
+
import { errorMessage } from "../util/errors.ts";
|
|
9
10
|
|
|
10
11
|
export const ARTIFACTS_DIR = ".rlm/artifacts";
|
|
11
12
|
|
|
@@ -59,7 +60,7 @@ export function captureGoal(cwd: string, brief: string): GoalCaptureResult {
|
|
|
59
60
|
writeFileSync(join(cwd, baselinePath), JSON.stringify({ paths }, null, 2), "utf-8");
|
|
60
61
|
return { ok: true, value: { goalPath, baselinePath } };
|
|
61
62
|
} catch (err) {
|
|
62
|
-
const message =
|
|
63
|
+
const message = errorMessage(err);
|
|
63
64
|
return { ok: false, error: message };
|
|
64
65
|
}
|
|
65
66
|
}
|
|
@@ -72,7 +73,7 @@ export function saveArtifact(cwd: string, dir: string, slug: string, content: st
|
|
|
72
73
|
writeFileSync(join(cwd, rel), content, "utf-8");
|
|
73
74
|
return { ok: true, path: rel };
|
|
74
75
|
} catch (err) {
|
|
75
|
-
const message =
|
|
76
|
+
const message = errorMessage(err);
|
|
76
77
|
return { ok: false, error: message };
|
|
77
78
|
}
|
|
78
79
|
}
|
|
@@ -82,7 +83,7 @@ export function readArtifact(cwd: string, relPath: string): Result<string, strin
|
|
|
82
83
|
try {
|
|
83
84
|
return { ok: true, value: readFileSync(join(cwd, relPath), "utf-8") };
|
|
84
85
|
} catch (err) {
|
|
85
|
-
const message =
|
|
86
|
+
const message = errorMessage(err);
|
|
86
87
|
return { ok: false, error: `could not read artifact ${relPath}: ${message}` };
|
|
87
88
|
}
|
|
88
89
|
}
|