@hicaru/pi-rlm 0.1.8 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -19
- package/package.json +2 -1
- package/src/bridge/library.ts +93 -15
- package/src/bridge/llm-query.ts +1 -0
- package/src/bridge/rlm-query.ts +2 -4
- package/src/context/library-context.ts +209 -22
- package/src/context/repomix-context.ts +2 -48
- package/src/core/answer.ts +1 -10
- package/src/core/critique.ts +92 -0
- package/src/core/engine.ts +130 -134
- package/src/core/gates.ts +30 -1
- package/src/core/pipeline.ts +38 -13
- package/src/core/types.ts +1 -3
- package/src/index.ts +1 -9
- package/src/mode/native-guards.ts +2 -2
- package/src/prompts/phases.ts +18 -39
- package/src/prompts/system.ts +31 -19
- package/src/sandbox/protocol.ts +5 -10
- package/src/sandbox/sandbox.ts +43 -9
- package/src/sandbox/worker.py +180 -48
- package/src/state/resume.ts +21 -14
- package/src/state/rows.ts +2 -2
- package/src/text/tokens.ts +7 -1
- package/src/tool/repl-details.ts +2 -3
- package/src/tool/repl-tool.ts +29 -55
- package/src/tool/rlm-aggregator.ts +7 -7
- package/src/tool/rlm-details.ts +6 -3
- package/src/tool/rlm-events.ts +14 -11
- package/src/tool/rlm-tool.ts +2 -7
- package/src/tool/subcall-store.ts +2 -0
- package/src/ui/config-panel.ts +2 -2
- package/src/registry/edit-registry.ts +0 -22
- package/src/text/edits.ts +0 -164
- package/src/tool/apply-edits-tool.ts +0 -295
package/README.md
CHANGED
|
@@ -100,8 +100,8 @@ sub-LLM calls, hence the name.
|
|
|
100
100
|
|
|
101
101
|
While a run is active, a **live tree** shows the root orchestrator and every sub-LLM /
|
|
102
102
|
recursive child with status, model, cost, tokens, and duration. The final answer is posted
|
|
103
|
-
to the chat as markdown
|
|
104
|
-
|
|
103
|
+
to the chat as markdown. File changes use Pi's native `edit` / `write` tools (with their
|
|
104
|
+
built-in diff preview).
|
|
105
105
|
|
|
106
106
|
## Sandbox API
|
|
107
107
|
|
|
@@ -117,10 +117,9 @@ These functions are injected into the model's Python namespace inside the REPL:
|
|
|
117
117
|
| `rlm_query_batched` | `(prompts, model=None) -> list[str]` | Concurrent recursive child RLMs |
|
|
118
118
|
| `todo` | `(action, **kwargs) -> str` | Task list: `create`/`update`/`list`/`get`/`delete`/`clear` |
|
|
119
119
|
| `ask_user_question` | `(questions) -> list[dict]` | Ask the user structured questions (depth 0 only) |
|
|
120
|
-
| `load_library` | `(source) -> dict \| str` |
|
|
121
|
-
| `
|
|
122
|
-
| `
|
|
123
|
-
| `advance_phase` | `(phase, summary=None) -> str` | Advance one step in order `clarify → research → blueprint → implement → validate` (clarify skipped when `askUserQuestion` is off). **Engine-gated** on the latest artifact + interview rounds. Rejected transitions return the gate error. |
|
|
120
|
+
| `load_library` | `(source) -> dict \| str` | Append an external dir, file, or git URL into `context` under `lib/<id>/` |
|
|
121
|
+
| `save_artifact` | `(kind, content) -> str` | Persist a stage artifact (`clarification` / `research` / `plan` / `validation`) under `.rlm/artifacts/` (root depth only). Returns preflight gate critique. |
|
|
122
|
+
| `advance_phase` | `(phase, summary=None) -> str` | Advance one step in order `clarify → research → blueprint → validate` (clarify skipped when `askUserQuestion` is off). **Engine-gated** on the latest artifact + interview rounds. Rejected transitions return the gate error. |
|
|
124
123
|
| `SHOW_VARS` | `() -> str` | List currently defined variables & their types |
|
|
125
124
|
| `answer` | `dict` | Set `answer["content"]=...; answer["ready"]=True` to finalize |
|
|
126
125
|
|
|
@@ -130,27 +129,31 @@ When the task needs an **external library, another source tree, or standalone do
|
|
|
130
129
|
not in the packed repo `context`, the model calls `load_library(source)` mid-run:
|
|
131
130
|
|
|
132
131
|
```python
|
|
133
|
-
info = load_library("../some-lib")
|
|
134
|
-
info = load_library("docs/api.md")
|
|
135
|
-
info = load_library("https://github.com/x/y.git")
|
|
136
|
-
#
|
|
137
|
-
#
|
|
132
|
+
info = load_library("../some-lib") # local directory → packed + appended
|
|
133
|
+
info = load_library("docs/api.md") # single file → one entry in context
|
|
134
|
+
info = load_library("https://github.com/x/y.git") # shallow clone, then pack + append
|
|
135
|
+
# Files land in the SAME `context` list under lib/<source_id>/…
|
|
136
|
+
# info == {"source_id", "path_prefix", "files", "chars", "context_len", "already_loaded", …}
|
|
137
|
+
lib = [f for f in context if f["path"].startswith(info["path_prefix"])]
|
|
138
138
|
```
|
|
139
139
|
|
|
140
|
-
|
|
141
|
-
`/rlm-config` → **Library loader** (`libraryLoader`,
|
|
142
|
-
persistence, each
|
|
140
|
+
There is no `context_1` / `context_2` — only `context`. Paths are namespaced so multiple
|
|
141
|
+
libraries do not collide. Toggle via `/rlm-config` → **Library loader** (`libraryLoader`,
|
|
142
|
+
default on). On headless runs with persistence, each load writes a resume sidecar
|
|
143
|
+
(`context.<N>.json`) that is **merged back into `context`** on resume.
|
|
143
144
|
|
|
144
145
|
### Artifact-gated pipeline (opt-in via `pipeline: true`)
|
|
145
146
|
|
|
146
147
|
When enabled at root depth:
|
|
147
148
|
|
|
148
149
|
1. **Goal capture** — the brief is written verbatim to `.rlm/artifacts/goal/goal-<ts>.md` with a pre-run dirty-tree baseline.
|
|
149
|
-
2. **Stages** — `clarify → research → blueprint →
|
|
150
|
+
2. **Stages** — `clarify → research → blueprint → validate` (**read-only** — produces a validated plan; does not write code). Each produces a durable markdown artifact with frontmatter contracts; chat history is **reset** at every phase boundary (artifacts are the only channel; REPL vars persist).
|
|
150
151
|
3. **Clarify (intake)** — interviews the user via `ask_user_question` (intent first, then evidence-confirmed decisions). Writes `.rlm/artifacts/clarifications/*` with `decisions_count` / `open_questions_count`. Engine gate: **≥1 serviced ask round** + artifact contract. When **`askUserQuestion` is off**, clarify is skipped and the run starts at research.
|
|
151
|
-
4. **Gates (TypeScript, never LLM judgment)** — `status: ready`; clarify structure; plan `phases:` ≡ fence-aware `## Phase N:` headings; every `file:line` citation resolves; validate carries `blockers_count` + `verdict`.
|
|
152
|
-
5. **
|
|
153
|
-
6. **Corrective loop** — `blockers_count > 0` re-enters blueprint, bounded by `maxBackwardJumps` (default 2).
|
|
152
|
+
4. **Gates (TypeScript, never LLM judgment)** — `status: ready`; clarify structure; plan `phases:` ≡ fence-aware `## Phase N:` headings; every `file:line` citation resolves; validate carries `blockers_count` + `verdict`. Preflight critique runs on every `save_artifact`.
|
|
153
|
+
5. **Validate** — adversarial plan review against the tree (not a post-implementation diff check). Final answer is the validated plan.
|
|
154
|
+
6. **Corrective loop** — `blockers_count > 0` re-enters blueprint (superseded plan kept for context), bounded by `maxBackwardJumps` (default 2).
|
|
155
|
+
|
|
156
|
+
Native RLM mode authors file changes with Pi's native `edit` / `write` tools. Sub-LLMs extract and locate; they never ship code.
|
|
154
157
|
|
|
155
158
|
## Settings (`/rlm-config`)
|
|
156
159
|
|
|
@@ -167,7 +170,7 @@ When enabled at root depth:
|
|
|
167
170
|
| Token ceiling | none | total input+output token cap for the whole recursive tree |
|
|
168
171
|
| Max consecutive errors | `5` | stop after N consecutive failing turns (none = off) |
|
|
169
172
|
| Orchestrator addendum | on | divide-and-conquer guidance in the root system prompt |
|
|
170
|
-
| Phase pipeline | off | artifact-gated clarify→research→blueprint→
|
|
173
|
+
| Phase pipeline | off | artifact-gated clarify→research→blueprint→validate (read-only plan pipeline) |
|
|
171
174
|
| Max validate→blueprint loops | `2` | bounded corrective re-entries when validation reports blockers |
|
|
172
175
|
| Ask user question | on | when pipeline is on, enables clarify intake; when off, pipeline starts at research |
|
|
173
176
|
| Trajectory compaction | on (0.65) | summarize old turns when history nears the context window |
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hicaru/pi-rlm",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.9",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Save 99% tokens, Recursive Language Model (RLM) for the Pi",
|
|
6
6
|
"license": "MIT",
|
|
@@ -27,6 +27,7 @@
|
|
|
27
27
|
],
|
|
28
28
|
"scripts": {
|
|
29
29
|
"check": "tsc --noEmit",
|
|
30
|
+
"test": "bun run test/smoke.ts",
|
|
30
31
|
"prepublishOnly": "npm run check"
|
|
31
32
|
},
|
|
32
33
|
"pi": {
|
package/src/bridge/library.ts
CHANGED
|
@@ -1,8 +1,13 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Shared load_library handler for headless engine and native repl() mode.
|
|
3
3
|
*
|
|
4
|
-
* Host
|
|
5
|
-
*
|
|
4
|
+
* Host packs the source via resolveLibrarySource (namespaced under lib/<id>/),
|
|
5
|
+
* assigns a resume-sidecar index, and returns the payload for the worker to
|
|
6
|
+
* append into the single `context` list.
|
|
7
|
+
*
|
|
8
|
+
* Idempotency is host-side: re-loading a source that was already packed does
|
|
9
|
+
* not consume an index, write a sidecar, or re-clone/pack. That keeps resume
|
|
10
|
+
* trails free of duplicate library slots.
|
|
6
11
|
*
|
|
7
12
|
* Late-bound deps (getCwd / getEmitter) keep a single handler closure correct
|
|
8
13
|
* across native repl() calls — getOrCreate only installs handlers at spawn.
|
|
@@ -10,7 +15,10 @@
|
|
|
10
15
|
|
|
11
16
|
import type { RlmEmitter } from "../tool/rlm-events.ts";
|
|
12
17
|
import type { SubLlmHandlers } from "../sandbox/sandbox.ts";
|
|
13
|
-
import {
|
|
18
|
+
import {
|
|
19
|
+
libraryNamespace,
|
|
20
|
+
resolveLibrarySource,
|
|
21
|
+
} from "../context/library-context.ts";
|
|
14
22
|
import { previewText } from "../text/preview.ts";
|
|
15
23
|
|
|
16
24
|
export interface LibraryBridgeOpts {
|
|
@@ -23,22 +31,35 @@ export interface LibraryBridgeOpts {
|
|
|
23
31
|
readonly getEmitter?: () => RlmEmitter | null | undefined;
|
|
24
32
|
readonly parentId?: string;
|
|
25
33
|
readonly signal?: AbortSignal;
|
|
26
|
-
/** First
|
|
34
|
+
/** First resume-sidecar index (slot 0 = repo). Resume passes 1 + max restored. */
|
|
27
35
|
readonly startIndex: number;
|
|
36
|
+
/**
|
|
37
|
+
* Prefixes already present in context (e.g. restored from sidecars).
|
|
38
|
+
* Seeded so re-load after resume is still a no-op without re-packing.
|
|
39
|
+
*/
|
|
40
|
+
readonly loadedPrefixes?: readonly string[];
|
|
28
41
|
/** Post-load hook — the engine writes the resume sidecar here; native mode omits it. */
|
|
29
42
|
readonly onLoaded?: (index: number, payload: unknown) => void | Promise<void>;
|
|
30
43
|
}
|
|
31
44
|
|
|
32
45
|
export interface LibraryHandlerBundle {
|
|
33
46
|
readonly handlers: Pick<SubLlmHandlers, "loadLibrary">;
|
|
34
|
-
/** Reset the
|
|
47
|
+
/** Reset the sidecar index counter (call when the sandbox is discarded and will re-spawn). */
|
|
35
48
|
readonly reset: () => void;
|
|
49
|
+
/** Prefixes loaded in this sandbox lifetime (for tests). */
|
|
50
|
+
readonly loadedPrefixes: () => ReadonlySet<string>;
|
|
36
51
|
}
|
|
37
52
|
|
|
38
53
|
export function buildLibraryHandler(opts: LibraryBridgeOpts): LibraryHandlerBundle {
|
|
39
54
|
let nextIndex = opts.startIndex;
|
|
55
|
+
/** Prefixes already loaded in this sandbox — mirrors the worker's context state. */
|
|
56
|
+
const loaded = new Set<string>(opts.loadedPrefixes ?? []);
|
|
40
57
|
return {
|
|
41
|
-
reset: () => {
|
|
58
|
+
reset: () => {
|
|
59
|
+
nextIndex = opts.startIndex;
|
|
60
|
+
loaded.clear();
|
|
61
|
+
},
|
|
62
|
+
loadedPrefixes: () => loaded,
|
|
42
63
|
handlers: {
|
|
43
64
|
async loadLibrary(source, depth) {
|
|
44
65
|
const emitter = opts.getEmitter?.() ?? opts.emitter;
|
|
@@ -53,19 +74,76 @@ export function buildLibraryHandler(opts: LibraryBridgeOpts): LibraryHandlerBund
|
|
|
53
74
|
depth,
|
|
54
75
|
});
|
|
55
76
|
try {
|
|
77
|
+
// Cheap pre-check BEFORE cloning/packing: same namespace ⇒ nothing to do.
|
|
78
|
+
const { sourceId: preId, pathPrefix: prefix } = libraryNamespace(source, cwd);
|
|
79
|
+
if (loaded.has(prefix)) {
|
|
80
|
+
if (id) {
|
|
81
|
+
emitter?.emitSubcallUpdated({
|
|
82
|
+
id,
|
|
83
|
+
status: "done",
|
|
84
|
+
resultPreview: `already loaded (${prefix}*)`,
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
// No index consumed, no sidecar written — resume stays consistent.
|
|
88
|
+
return {
|
|
89
|
+
payload: Object.freeze([]),
|
|
90
|
+
index: -1,
|
|
91
|
+
files: 0,
|
|
92
|
+
chars: 0,
|
|
93
|
+
sourceId: preId,
|
|
94
|
+
pathPrefix: prefix,
|
|
95
|
+
alreadyLoaded: true,
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
|
|
56
99
|
const resolved = await resolveLibrarySource(source, cwd, opts.signal);
|
|
57
100
|
if (!resolved.ok) throw new Error(resolved.error);
|
|
58
|
-
const
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
101
|
+
const { payload, files, chars, sourceId, pathPrefix } = resolved.value;
|
|
102
|
+
|
|
103
|
+
// Race: another concurrent load of the same prefix finished while we packed.
|
|
104
|
+
if (loaded.has(pathPrefix)) {
|
|
105
|
+
if (id) {
|
|
106
|
+
emitter?.emitSubcallUpdated({
|
|
107
|
+
id,
|
|
108
|
+
status: "done",
|
|
109
|
+
resultPreview: `already loaded (${pathPrefix}*)`,
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
return {
|
|
113
|
+
payload: Object.freeze([]),
|
|
114
|
+
index: -1,
|
|
115
|
+
files: 0,
|
|
116
|
+
chars: 0,
|
|
117
|
+
sourceId,
|
|
118
|
+
pathPrefix,
|
|
119
|
+
alreadyLoaded: true,
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// Increment only after a successful sidecar write (or when no hook is set).
|
|
124
|
+
const index = nextIndex;
|
|
125
|
+
if (opts.onLoaded) {
|
|
126
|
+
await opts.onLoaded(index, payload);
|
|
127
|
+
}
|
|
128
|
+
nextIndex = index + 1;
|
|
129
|
+
loaded.add(pathPrefix);
|
|
130
|
+
|
|
131
|
+
if (id) {
|
|
132
|
+
emitter?.emitSubcallUpdated({
|
|
133
|
+
id,
|
|
134
|
+
status: "done",
|
|
135
|
+
resultPreview:
|
|
136
|
+
`+${files} file(s) → context (${pathPrefix}*, ${chars.toLocaleString()} chars)`,
|
|
137
|
+
});
|
|
138
|
+
}
|
|
64
139
|
return {
|
|
65
|
-
payload
|
|
140
|
+
payload,
|
|
66
141
|
index,
|
|
67
|
-
files
|
|
68
|
-
chars
|
|
142
|
+
files,
|
|
143
|
+
chars,
|
|
144
|
+
sourceId,
|
|
145
|
+
pathPrefix,
|
|
146
|
+
alreadyLoaded: false,
|
|
69
147
|
};
|
|
70
148
|
} catch (err) {
|
|
71
149
|
if (id) emitter?.emitSubcallUpdated({ id, status: "error", detail: String(err) });
|
package/src/bridge/llm-query.ts
CHANGED
|
@@ -125,6 +125,7 @@ export function createLlmBridge(opts: LlmBridgeOptions): LlmBridge {
|
|
|
125
125
|
if (emitter && id !== undefined) emitter.emitSubcallUpdated({ id,
|
|
126
126
|
status: error ? "error" : "done", costUsd: cost, tokens,
|
|
127
127
|
resultPreview, detail: error,
|
|
128
|
+
failedCount: failed, totalCount: out.length,
|
|
128
129
|
});
|
|
129
130
|
return out;
|
|
130
131
|
},
|
package/src/bridge/rlm-query.ts
CHANGED
|
@@ -5,8 +5,7 @@
|
|
|
5
5
|
* depth cap it degrades to a plain `llm_query` (ported from rlm/core/rlm.py `_subcall`). The
|
|
6
6
|
* concurrency pool bounds parallel children for `rlm_query_batched`.
|
|
7
7
|
*
|
|
8
|
-
* `childRun` is the single spawn path
|
|
9
|
-
* needs `.edits` — both share this implementation (DRY).
|
|
8
|
+
* `childRun` is the single spawn path shared by `rlmQuery` / `rlmQueryBatched`.
|
|
10
9
|
*/
|
|
11
10
|
|
|
12
11
|
import type { RlmResult, RunRlm } from "../core/types.ts";
|
|
@@ -27,7 +26,7 @@ export interface ChildRunInput {
|
|
|
27
26
|
export interface RlmHandlers {
|
|
28
27
|
rlmQuery(prompt: string, model: string | null, depth: number): Promise<string>;
|
|
29
28
|
rlmQueryBatched(prompts: string[], model: string | null, depth: number): Promise<string[]>;
|
|
30
|
-
/** Full child-run result (answer +
|
|
29
|
+
/** Full child-run result (answer + usage) for recursive spawns. */
|
|
31
30
|
childRun(input: ChildRunInput): Promise<RlmResult>;
|
|
32
31
|
}
|
|
33
32
|
|
|
@@ -49,7 +48,6 @@ export interface RlmBridgeOptions {
|
|
|
49
48
|
function emptyResult(answer: string): RlmResult {
|
|
50
49
|
return {
|
|
51
50
|
answer,
|
|
52
|
-
edits: [],
|
|
53
51
|
iterations: 0,
|
|
54
52
|
costUsd: 0,
|
|
55
53
|
inputTokens: 0,
|
|
@@ -1,30 +1,211 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Resolve load_library(source) into a sandbox-ready payload.
|
|
3
3
|
*
|
|
4
|
-
* Sources: local directory (repomix-packed), single file (utf-8
|
|
5
|
-
*
|
|
4
|
+
* Sources: local directory (repomix-packed), single file (utf-8), or remote git URL
|
|
5
|
+
* (shallow clone then pack). Host-side only — never runs in the sandbox.
|
|
6
|
+
*
|
|
7
|
+
* Every successful payload is a namespaced list of ContextFile under
|
|
8
|
+
* `lib/<source_id>/…` so the worker can append into the single `context` variable.
|
|
9
|
+
* Source ids include a short content fingerprint so two libraries that share a
|
|
10
|
+
* basename never collide.
|
|
6
11
|
*/
|
|
7
12
|
|
|
13
|
+
import { createHash } from "node:crypto";
|
|
8
14
|
import { execFile } from "node:child_process";
|
|
9
15
|
import { mkdtemp, readFile, rm, stat } from "node:fs/promises";
|
|
16
|
+
import { basename, isAbsolute, join, resolve } from "node:path";
|
|
10
17
|
import { tmpdir } from "node:os";
|
|
11
|
-
import { isAbsolute, join, resolve } from "node:path";
|
|
12
18
|
import { promisify } from "node:util";
|
|
13
|
-
import {
|
|
19
|
+
import {
|
|
20
|
+
packRepository,
|
|
21
|
+
serializeForSandbox,
|
|
22
|
+
type ContextBundle,
|
|
23
|
+
type ContextFile,
|
|
24
|
+
} from "./repomix-context.ts";
|
|
25
|
+
import { estimateTokens } from "../text/tokens.ts";
|
|
14
26
|
import type { Result } from "../util/errors.ts";
|
|
15
27
|
import { errorMessage } from "../util/errors.ts";
|
|
16
28
|
|
|
17
29
|
const execFileP = promisify(execFile);
|
|
18
30
|
|
|
31
|
+
/** Single-file sources above this must use open() + llm_query_chunked in the REPL. */
|
|
32
|
+
export const MAX_LIBRARY_FILE_BYTES = 8 * 1024 * 1024;
|
|
33
|
+
|
|
34
|
+
/** Legacy catch-all prefix for pre-namespace string sidecars — never an identity key. */
|
|
35
|
+
const LEGACY_UNKNOWN_PREFIX = "lib/unknown/";
|
|
36
|
+
|
|
19
37
|
export interface LibrarySource {
|
|
20
|
-
|
|
21
|
-
readonly
|
|
38
|
+
/** Always a namespaced file list (dirs, single files, and git clones). */
|
|
39
|
+
readonly payload: readonly ContextFile[];
|
|
40
|
+
readonly files: number;
|
|
41
|
+
/** Sum of raw content lengths — what the model should size batches against. */
|
|
22
42
|
readonly chars: number;
|
|
43
|
+
readonly sourceId: string;
|
|
44
|
+
readonly pathPrefix: string;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export interface LibraryNamespace {
|
|
48
|
+
readonly sourceId: string;
|
|
49
|
+
readonly pathPrefix: string;
|
|
23
50
|
}
|
|
24
51
|
|
|
25
52
|
/** https://host/… or git@host:… — option-injection safe (never starts with "-"). */
|
|
26
53
|
const GIT_URL = /^(https:\/\/|git@)[\w.-]+[:/]\S+$/;
|
|
27
54
|
|
|
55
|
+
/** Short, stable discriminator so two sources never share a namespace. */
|
|
56
|
+
function sourceFingerprint(canonical: string): string {
|
|
57
|
+
return createHash("sha256").update(canonical).digest("hex").slice(0, 8);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Sanitize a path/url basename into a stable, filesystem-safe source id.
|
|
62
|
+
* `resolvedPath` (absolute path) or the git URL is fingerprinted into the id so
|
|
63
|
+
* distinct sources sharing a basename get distinct namespaces
|
|
64
|
+
* (`lib/utils-3f9a1c02/` vs `lib/utils-a1b2c3d4/`).
|
|
65
|
+
*/
|
|
66
|
+
export function librarySourceId(source: string, resolvedPath?: string): string {
|
|
67
|
+
const trimmed = source.trim();
|
|
68
|
+
const isGit = GIT_URL.test(trimmed);
|
|
69
|
+
let raw: string;
|
|
70
|
+
if (isGit) {
|
|
71
|
+
const m = trimmed.match(/(?:\/|:)([\w.-]+?)(?:\.git)?\/?\s*$/);
|
|
72
|
+
raw = m?.[1] ?? "repo";
|
|
73
|
+
} else {
|
|
74
|
+
raw = basename(resolvedPath ?? trimmed);
|
|
75
|
+
}
|
|
76
|
+
const cleaned = raw
|
|
77
|
+
.replace(/\.git$/i, "")
|
|
78
|
+
.replace(/[^\w.-]+/g, "-")
|
|
79
|
+
.replace(/^-+|-+$/g, "")
|
|
80
|
+
.slice(0, 80);
|
|
81
|
+
const canonical = isGit ? trimmed : (resolvedPath ?? trimmed);
|
|
82
|
+
return `${cleaned.length > 0 ? cleaned : "lib"}-${sourceFingerprint(canonical)}`;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export function pathPrefixFor(sourceId: string): string {
|
|
86
|
+
return `lib/${sourceId}/`;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Derive the namespace for a source string without packing (host-side idempotency).
|
|
91
|
+
* Returns both sourceId and pathPrefix so callers never un-parse the prefix.
|
|
92
|
+
*/
|
|
93
|
+
export function libraryNamespace(source: string, cwd: string): LibraryNamespace {
|
|
94
|
+
const trimmed = source.trim();
|
|
95
|
+
const sourceId = GIT_URL.test(trimmed)
|
|
96
|
+
? librarySourceId(trimmed)
|
|
97
|
+
: librarySourceId(trimmed, isAbsolute(trimmed) ? trimmed : resolve(cwd, trimmed));
|
|
98
|
+
return Object.freeze({ sourceId, pathPrefix: pathPrefixFor(sourceId) });
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** Derive the path prefix for a source string without packing. */
|
|
102
|
+
export function libraryPathPrefix(source: string, cwd: string): string {
|
|
103
|
+
return libraryNamespace(source, cwd).pathPrefix;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** Namespaced files plus summed content chars (one pass). */
|
|
107
|
+
export function namespaceLibraryFilesWithChars(
|
|
108
|
+
payload: unknown,
|
|
109
|
+
sourceId: string,
|
|
110
|
+
): { readonly files: readonly ContextFile[]; readonly chars: number } {
|
|
111
|
+
const prefix = pathPrefixFor(sourceId);
|
|
112
|
+
if (typeof payload === "string") {
|
|
113
|
+
return Object.freeze({
|
|
114
|
+
files: Object.freeze([
|
|
115
|
+
Object.freeze({
|
|
116
|
+
path: `${prefix}content`,
|
|
117
|
+
content: payload,
|
|
118
|
+
tokens: Math.max(1, estimateTokens(payload.length)),
|
|
119
|
+
}),
|
|
120
|
+
]),
|
|
121
|
+
chars: payload.length,
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
if (!Array.isArray(payload)) return Object.freeze({ files: Object.freeze([]), chars: 0 });
|
|
125
|
+
const out = new Array<ContextFile>(payload.length);
|
|
126
|
+
let n = 0;
|
|
127
|
+
let chars = 0;
|
|
128
|
+
for (const item of payload) {
|
|
129
|
+
if (item === null || typeof item !== "object") continue;
|
|
130
|
+
const rec = item as Record<string, unknown>;
|
|
131
|
+
const content = typeof rec.content === "string" ? rec.content : String(rec.content ?? "");
|
|
132
|
+
let path = typeof rec.path === "string" ? rec.path : "unknown";
|
|
133
|
+
path = path.replace(/^\/+/, "");
|
|
134
|
+
if (!path.startsWith(prefix)) path = `${prefix}${path}`;
|
|
135
|
+
const tokens = typeof rec.tokens === "number" && Number.isFinite(rec.tokens)
|
|
136
|
+
? Math.max(0, Math.floor(rec.tokens))
|
|
137
|
+
: Math.max(1, estimateTokens(content.length));
|
|
138
|
+
out[n++] = Object.freeze({ path, content, tokens });
|
|
139
|
+
chars += content.length;
|
|
140
|
+
}
|
|
141
|
+
out.length = n;
|
|
142
|
+
return Object.freeze({ files: Object.freeze(out), chars });
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** Namespace file entries under `lib/<sourceId>/…` (single shared implementation). */
|
|
146
|
+
export function namespaceLibraryFiles(
|
|
147
|
+
payload: unknown,
|
|
148
|
+
sourceId: string,
|
|
149
|
+
): readonly ContextFile[] {
|
|
150
|
+
return namespaceLibraryFilesWithChars(payload, sourceId).files;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/** First `lib/<id>/` prefix found in the payload, or undefined. Skips `lib/unknown/`. */
|
|
154
|
+
export function payloadPrefix(payload: readonly unknown[]): string | undefined {
|
|
155
|
+
for (let i = 0; i < payload.length; i++) {
|
|
156
|
+
const item = payload[i];
|
|
157
|
+
if (item === null || typeof item !== "object") continue;
|
|
158
|
+
const path = (item as { path?: unknown }).path;
|
|
159
|
+
if (typeof path !== "string") continue;
|
|
160
|
+
const m = /^(lib\/[^/]+\/)/.exec(path);
|
|
161
|
+
// `lib/unknown/` is the legacy catch-all: never treat it as an identity.
|
|
162
|
+
if (m?.[1] !== undefined && m[1] !== LEGACY_UNKNOWN_PREFIX) return m[1];
|
|
163
|
+
}
|
|
164
|
+
return undefined;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Append a library payload into an existing list context (host-side resume merge).
|
|
169
|
+
* Skips a payload whose path prefix is already present (resume-safe dedup).
|
|
170
|
+
*/
|
|
171
|
+
export function mergeLibraryIntoContext(base: unknown, libraryPayload: unknown): unknown {
|
|
172
|
+
if (!Array.isArray(base)) return base;
|
|
173
|
+
if (Array.isArray(libraryPayload)) {
|
|
174
|
+
if (libraryPayload.length === 0) return base;
|
|
175
|
+
const prefix = payloadPrefix(libraryPayload);
|
|
176
|
+
if (prefix !== undefined) {
|
|
177
|
+
for (let i = 0; i < base.length; i++) {
|
|
178
|
+
const item = base[i];
|
|
179
|
+
if (item !== null && typeof item === "object"
|
|
180
|
+
&& typeof (item as { path?: unknown }).path === "string"
|
|
181
|
+
&& (item as { path: string }).path.startsWith(prefix)) {
|
|
182
|
+
return base; // already present
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
const merged = new Array<unknown>(base.length + libraryPayload.length);
|
|
187
|
+
for (let i = 0; i < base.length; i++) merged[i] = base[i];
|
|
188
|
+
for (let i = 0; i < libraryPayload.length; i++) merged[base.length + i] = libraryPayload[i];
|
|
189
|
+
return merged;
|
|
190
|
+
}
|
|
191
|
+
if (typeof libraryPayload === "string") {
|
|
192
|
+
// Legacy string sidecars: wrap once under an unknown prefix.
|
|
193
|
+
return mergeLibraryIntoContext(base, namespaceLibraryFiles(libraryPayload, "unknown"));
|
|
194
|
+
}
|
|
195
|
+
return base;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
function toLibrarySource(payload: unknown, sourceId: string): LibrarySource {
|
|
199
|
+
const { files, chars } = namespaceLibraryFilesWithChars(payload, sourceId);
|
|
200
|
+
return {
|
|
201
|
+
payload: files,
|
|
202
|
+
files: files.length,
|
|
203
|
+
chars,
|
|
204
|
+
sourceId,
|
|
205
|
+
pathPrefix: pathPrefixFor(sourceId),
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
|
|
28
209
|
export async function resolveLibrarySource(
|
|
29
210
|
source: string,
|
|
30
211
|
cwd: string,
|
|
@@ -43,34 +224,40 @@ export async function resolveLibrarySource(
|
|
|
43
224
|
} catch {
|
|
44
225
|
return { ok: false, error: `load_library: path not found: ${path}` };
|
|
45
226
|
}
|
|
46
|
-
|
|
227
|
+
const sourceId = librarySourceId(trimmed, path);
|
|
228
|
+
if (s.isDirectory()) return await packDir(path, sourceId, signal);
|
|
229
|
+
if (s.size > MAX_LIBRARY_FILE_BYTES) {
|
|
230
|
+
return {
|
|
231
|
+
ok: false,
|
|
232
|
+
error: `load_library: ${path} is ${s.size.toLocaleString()} bytes `
|
|
233
|
+
+ `(limit ${MAX_LIBRARY_FILE_BYTES.toLocaleString()}) — `
|
|
234
|
+
+ "open() it in the REPL and delegate with llm_query_chunked instead",
|
|
235
|
+
};
|
|
236
|
+
}
|
|
47
237
|
const text = await readFile(path, "utf-8");
|
|
48
|
-
return { ok: true, value:
|
|
238
|
+
return { ok: true, value: toLibrarySource(text, sourceId) };
|
|
49
239
|
}
|
|
50
240
|
|
|
51
|
-
async function packDir(
|
|
52
|
-
|
|
241
|
+
async function packDir(
|
|
242
|
+
dir: string,
|
|
243
|
+
sourceId: string,
|
|
244
|
+
signal?: AbortSignal,
|
|
245
|
+
): Promise<Result<LibrarySource, string>> {
|
|
246
|
+
const packed = await packRepository(dir, signal);
|
|
53
247
|
if (!packed.ok) return { ok: false, error: `pack failed for ${dir} — ${packed.error}` };
|
|
54
|
-
return { ok: true, value: bundleToSource(packed.value) };
|
|
248
|
+
return { ok: true, value: bundleToSource(packed.value, sourceId) };
|
|
55
249
|
}
|
|
56
250
|
|
|
57
|
-
function bundleToSource(bundle: ContextBundle): LibrarySource {
|
|
58
|
-
|
|
59
|
-
// Match slot-0 contextLength for object/array payloads: JSON-serialized size,
|
|
60
|
-
// not sum-of-content-lengths (bundle.totalChars), so the model sees one ruler.
|
|
61
|
-
return {
|
|
62
|
-
payload,
|
|
63
|
-
files: bundle.totalFiles,
|
|
64
|
-
chars: JSON.stringify(payload).length,
|
|
65
|
-
};
|
|
251
|
+
function bundleToSource(bundle: ContextBundle, sourceId: string): LibrarySource {
|
|
252
|
+
return toLibrarySource(serializeForSandbox(bundle), sourceId);
|
|
66
253
|
}
|
|
67
254
|
|
|
68
255
|
async function cloneAndPack(url: string, signal?: AbortSignal): Promise<Result<LibrarySource, string>> {
|
|
69
256
|
const dir = await mkdtemp(join(tmpdir(), "rlm-lib-"));
|
|
257
|
+
const sourceId = librarySourceId(url);
|
|
70
258
|
try {
|
|
71
|
-
// "--" terminates options; URL regex already forbids a leading dash.
|
|
72
259
|
await execFileP("git", ["clone", "--depth", "1", "--", url, dir], { signal, timeout: 120_000 });
|
|
73
|
-
return await packDir(dir, signal);
|
|
260
|
+
return await packDir(dir, sourceId, signal);
|
|
74
261
|
} catch (err: unknown) {
|
|
75
262
|
return { ok: false, error: `git clone failed for ${url} — ${errorMessage(err)}` };
|
|
76
263
|
} finally {
|
|
@@ -5,8 +5,7 @@
|
|
|
5
5
|
*
|
|
6
6
|
* Uses repomix internally (worker-thread pool, built-in gitignore support)
|
|
7
7
|
* and caches results in a module-level Map with TTL to avoid re-packing on
|
|
8
|
-
* every run within the same process.
|
|
9
|
-
* cached bundle in-memory after file edits are applied to disk — no re-packing.
|
|
8
|
+
* every run within the same process.
|
|
10
9
|
*/
|
|
11
10
|
|
|
12
11
|
import { pack } from "repomix";
|
|
@@ -172,42 +171,6 @@ export async function packRepository(
|
|
|
172
171
|
}
|
|
173
172
|
}
|
|
174
173
|
|
|
175
|
-
export function patchContextAfterEdits(
|
|
176
|
-
cached: ContextBundle,
|
|
177
|
-
edits: readonly { readonly path: string; readonly newContent: string }[],
|
|
178
|
-
): ContextBundle {
|
|
179
|
-
const editMap = new Map<string, string>();
|
|
180
|
-
for (const edit of edits) {
|
|
181
|
-
editMap.set(edit.path, edit.newContent);
|
|
182
|
-
}
|
|
183
|
-
|
|
184
|
-
const files = new Array<ContextFile>(cached.files.length);
|
|
185
|
-
let totalTokens = 0;
|
|
186
|
-
let totalChars = 0;
|
|
187
|
-
|
|
188
|
-
for (let i = 0; i < cached.files.length; i++) {
|
|
189
|
-
const file = cached.files[i];
|
|
190
|
-
const newContent = editMap.get(file.path);
|
|
191
|
-
if (newContent !== undefined) {
|
|
192
|
-
const tokens = Math.ceil(newContent.length / ESTIMATED_CHARS_PER_TOKEN);
|
|
193
|
-
files[i] = { path: file.path, content: newContent, tokens };
|
|
194
|
-
totalTokens += tokens;
|
|
195
|
-
totalChars += newContent.length;
|
|
196
|
-
} else {
|
|
197
|
-
files[i] = file;
|
|
198
|
-
totalTokens += file.tokens;
|
|
199
|
-
totalChars += file.content.length;
|
|
200
|
-
}
|
|
201
|
-
}
|
|
202
|
-
|
|
203
|
-
return {
|
|
204
|
-
files,
|
|
205
|
-
totalFiles: files.length,
|
|
206
|
-
totalTokens,
|
|
207
|
-
totalChars,
|
|
208
|
-
};
|
|
209
|
-
}
|
|
210
|
-
|
|
211
174
|
export function serializeForSandbox(
|
|
212
175
|
bundle: ContextBundle,
|
|
213
176
|
): readonly ContextFile[] {
|
|
@@ -243,13 +206,4 @@ export function formatForLLM(bundle: ContextBundle): string {
|
|
|
243
206
|
].join("\n");
|
|
244
207
|
}
|
|
245
208
|
|
|
246
|
-
|
|
247
|
-
cwd: string,
|
|
248
|
-
edits: readonly { readonly path: string; readonly newContent: string }[],
|
|
249
|
-
): void {
|
|
250
|
-
const key = cacheKey(cwd);
|
|
251
|
-
const entry = cache.get(key);
|
|
252
|
-
if (!entry) return;
|
|
253
|
-
const patched = patchContextAfterEdits(entry.bundle, edits);
|
|
254
|
-
cache.set(key, { bundle: patched, ts: entry.ts });
|
|
255
|
-
}
|
|
209
|
+
|
package/src/core/answer.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/** Helpers for detecting and formatting the RLM final answer from a turn's REPL results. */
|
|
2
2
|
|
|
3
|
-
import type {
|
|
3
|
+
import type { ReplResult } from "../sandbox/protocol.ts";
|
|
4
4
|
import { truncateOutput } from "../text/parsing.ts";
|
|
5
5
|
|
|
6
6
|
/** First non-null final answer across a turn's executed blocks, or null. */
|
|
@@ -18,15 +18,6 @@ export function latestAnswerContentOf(results: readonly ReplResult[]): string |
|
|
|
18
18
|
return null;
|
|
19
19
|
}
|
|
20
20
|
|
|
21
|
-
/** Last cumulative legacy anchor proposed-edit set reported by a turn. */
|
|
22
|
-
export function collectEdits(results: readonly ReplResult[]): ProposedEdit[] {
|
|
23
|
-
for (let i = results.length - 1; i >= 0; i--) {
|
|
24
|
-
const edits = results[i]?.edits;
|
|
25
|
-
if (edits && edits.length > 0) return [...edits];
|
|
26
|
-
}
|
|
27
|
-
return [];
|
|
28
|
-
}
|
|
29
|
-
|
|
30
21
|
/** True if any block in the turn raised an exception. Plain stderr does not count. */
|
|
31
22
|
export function turnHadError(results: readonly ReplResult[]): boolean {
|
|
32
23
|
return results.some((r) => r.raised);
|