@hicaru/pi-rlm 0.2.2 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/README.md +38 -16
  2. package/README.ru.md +2 -2
  3. package/README.zh-CN.md +2 -2
  4. package/package.json +22 -19
  5. package/src/bridge/add-context.ts +322 -0
  6. package/src/bridge/subcall-handlers.ts +1 -1
  7. package/src/config/defaults.ts +2 -1
  8. package/src/config/settings.ts +5 -2
  9. package/src/context/anydoc.ts +67 -0
  10. package/src/context/listing.ts +70 -0
  11. package/src/context/md-cache.ts +112 -0
  12. package/src/context/merge.ts +97 -0
  13. package/src/context/namespace.ts +180 -0
  14. package/src/context/resolve.ts +122 -0
  15. package/src/context/source-dir.ts +166 -0
  16. package/src/context/source-doc.ts +71 -0
  17. package/src/context/source-git.ts +51 -0
  18. package/src/context/source-text.ts +45 -0
  19. package/src/context/types.ts +88 -0
  20. package/src/context/walk.ts +250 -0
  21. package/src/core/engine.ts +15 -19
  22. package/src/core/types.ts +7 -2
  23. package/src/index.ts +119 -47
  24. package/src/mode/rlm-mode.ts +5 -4
  25. package/src/mode/subagent.ts +68 -0
  26. package/src/prompts/glossary.ts +31 -28
  27. package/src/prompts/native.ts +4 -4
  28. package/src/prompts/system.ts +2 -2
  29. package/src/sandbox/context-file.ts +4 -4
  30. package/src/sandbox/interrupts.ts +25 -10
  31. package/src/sandbox/protocol.ts +13 -7
  32. package/src/sandbox/py/__pycache__/guards.cpython-314.pyc +0 -0
  33. package/src/sandbox/py/__pycache__/hostio.cpython-314.pyc +0 -0
  34. package/src/sandbox/py/__pycache__/retrieval.cpython-314.pyc +0 -0
  35. package/src/sandbox/py/__pycache__/tasks.cpython-314.pyc +0 -0
  36. package/src/sandbox/py/guards.py +8 -1
  37. package/src/sandbox/py/hostio.py +57 -0
  38. package/src/sandbox/py/retrieval.py +1 -1
  39. package/src/sandbox/py/tasks.py +17 -4
  40. package/src/sandbox/py/worker.py +71 -52
  41. package/src/sandbox/sandbox-manager.ts +18 -16
  42. package/src/sandbox/sandbox.ts +9 -2
  43. package/src/text/tokens.ts +3 -3
  44. package/src/tool/repl-details.ts +1 -1
  45. package/src/tool/repl-tool.ts +31 -19
  46. package/src/tool/rlm-tool.ts +1 -1
  47. package/src/ui/config-panel.ts +8 -4
  48. package/src/bridge/library.ts +0 -190
  49. package/src/context/library-context.ts +0 -339
  50. package/src/context/repomix-context.ts +0 -204
package/README.md CHANGED
@@ -59,9 +59,9 @@ sub-LLM calls, hence the name.
59
59
  - A **root orchestrator** model drives a **persistent Python REPL** turn-by-turn.
60
60
  - Long-context work is **delegated** to cheap worker models via `llm_query` / `llm_query_batched`.
61
61
  - Hard sub-problems **recurse** into child RLMs via `rlm_query` (depth-capped). A child inherits
62
- its parent's `context` — the repository plus every library loaded with `load_library()` — so it
63
- runs the same retrieval primitives over the same paths. Inheritance costs no extra tokens: the
64
- content lives in the sandbox, and only a size line reaches the model.
62
+ its parent's `context` — every file loaded so far, including sources added with `add_context()` —
63
+ so it runs the same retrieval primitives over the same paths. Inheritance costs no extra tokens:
64
+ the content lives in the sandbox, and only a size line reaches the model.
65
65
  - Everything runs **in-process** — the only external process is one local `python3` worker.
66
66
 
67
67
  > This is a Pi-plugin reimplementation of the RLM method (see the [RLM paper](https://arxiv.org/abs/2512.24601)).
@@ -109,33 +109,36 @@ These functions are injected into the model's Python namespace inside the REPL:
109
109
 
110
110
  | Function | Signature | Description |
111
111
  |---|---|---|
112
- | `context` | `list[dict]` | Repository packed as `[{"path","content","tokens"}, ...]` — the full codebase |
112
+ | `context` | `list[dict]` | Loaded files as `[{"path","content","tokens"}, ]` — starts empty; cwd seeds on first `repl()` |
113
113
  | `llm_query` | `(prompt, model=None) -> str` | One-shot sub-LLM call (worker model) |
114
114
  | `llm_query_batched` | `(prompts, model=None) -> list[str]` | Concurrent sub-LLM calls (pool-bounded) |
115
115
  | `llm_query_chunked` | `(text, prompt, model=None) -> list[str]` | Split large text into cap-sized chunks and fan out via sub-LLMs |
116
116
  | `rlm_query` | `(prompt, model=None, paths=None) -> str` | Recursive child RLM with its own sandbox (depth-capped). Inherits your `context`; `paths` narrows it by prefix |
117
117
  | `rlm_query_batched` | `(prompts, model=None, paths=None) -> list[str]` | Concurrent recursive child RLMs, sharing one `paths` slice |
118
- | `load_library` | `(source) -> dict \| str` | Append an external dir, file, or git URL into `context` under `lib/<id>/` |
118
+ | `add_context` | `(source) -> dict \| str` | Append a dir, file, document, or git URL into `context` under `ctx/<id>/` |
119
119
  | `SHOW_VARS` | `() -> str` | List currently defined variables & their types |
120
120
  | `answer` | `dict` | Set `answer["content"]=...; answer["ready"]=True` to finalize |
121
121
 
122
- ### Loading external libraries
122
+ ### Adding context
123
123
 
124
- When the task needs an **external library, another source tree, or standalone docs** that are
125
- not in the packed repo `context`, the model calls `load_library(source)` mid-run:
124
+ `context` starts empty. The working directory seeds automatically on the first `repl()` call
125
+ (un-prefixed paths so `search()` hits remain real paths for `edit`/`write`). For an **external
126
+ tree, document, or git URL**, call `add_context(source)`:
126
127
 
127
128
  ```python
128
- info = load_library("../some-lib") # local directory → packed + appended
129
- info = load_library("docs/api.md") # single file → one entry in context
130
- info = load_library("https://github.com/x/y.git") # shallow clone, then pack + append
131
- # Files land in the SAME `context` list under lib/<source_id>/…
132
- # info == {"source_id", "path_prefix", "files", "chars", "context_len", "already_loaded", …}
129
+ info = add_context("../some-lib") # local directory → packed + appended
130
+ info = add_context("docs/api.md") # single file → one entry in context
131
+ info = add_context("report.pdf") # document → Markdown, then appended
132
+ info = add_context("https://github.com/x/y.git") # shallow clone, then pack + append
133
+ # Files land in the SAME `context` list under ctx/<source_id>/…
134
+ # info == {"source_id", "path_prefix", "files", "chars", "context_len", "already_loaded", "converted", "skipped", …}
133
135
  lib = [f for f in context if f["path"].startswith(info["path_prefix"])]
134
136
  ```
135
137
 
136
138
  There is no `context_1` / `context_2` — only `context`. Paths are namespaced so multiple
137
- libraries do not collide. Toggle via `/rlm-config` → **Library loader** (`libraryLoader`,
138
- default on). A library loaded at any point is inherited by every child spawned afterwards.
139
+ sources do not collide. Toggle via `/rlm-config` → **Context loader** (`contextLoader`,
140
+ default on) and **Auto-seed cwd** (`autoSeedCwd`, default on). A source loaded at any point is
141
+ inherited by every child spawned afterwards.
139
142
 
140
143
  ## Settings (`/rlm-config`)
141
144
 
@@ -155,13 +158,32 @@ default on). A library loaded at any point is inherited by every child spawned a
155
158
  | Trajectory compaction | on (0.65) | summarize old turns when history nears the context window |
156
159
  | Root model output cap (tok) | `16384` | max output tokens per root-model turn |
157
160
  | Sandbox init timeout | `30000` ms | how long to wait for the Python worker to start |
158
- | Library loader | on | expose `load_library()` for external dirs/files/git repos |
161
+ | Context loader | on | expose `add_context()` for external dirs/files/documents/git repos |
162
+ | Auto-seed cwd | on | seed the working directory into `context` on the first `repl()` |
159
163
 
160
164
  > **Concurrency note:** each `rlm_query` child spawns its own `python3` worker (~50–150 ms
161
165
  > cold start). Children are bounded separately (`maxConcurrentChildren`, default 6) because
162
166
  > each holds a full Python process and its own copy of the inherited context. Error and
163
167
  > wall-clock caps (above) still bound a runaway tree.
164
168
 
169
+ ## Subagents and environment
170
+
171
+ RLM never confiscates native file tools (`read` / `grep` / bash readers) unless `repl` is in
172
+ the **active** tool set — the paper's trade is all-or-nothing. Process-boundary subagents that
173
+ spawn pi with a `--tools` allowlist without `repl` therefore keep ordinary file access.
174
+
175
+ Optional env conventions (for packages that want an explicit full bypass):
176
+
177
+ | Env | Meaning |
178
+ |---|---|
179
+ | `PI_SUBAGENT_CHILD=1` | Full RLM bypass in this process (no tools / hooks / flags). |
180
+ | `PI_RLM_FORCE_IN_SUBAGENT=1` | Experimental: opt a child back into RLM. **Consumed on activate** (not inherited after). Refused when `PI_RLM_DEPTH >= maxDepth`. |
181
+ | `PI_RLM_DEPTH` | Cross-process depth counter (default `0`). Bumped when force-in activates. |
182
+
183
+ In-process recursion (`rlm_query`) still uses `maxDepth` from `/rlm-config` and is unrelated to
184
+ these env vars. Set `RLM_TRACE_FILE` to a path for JSONL traces of bypass / force / block-skip
185
+ decisions.
186
+
165
187
  ## Security
166
188
 
167
189
  - **Key isolation**: provider keys live only in TypeScript (`AuthStorage`); the sandbox
package/README.ru.md CHANGED
@@ -35,7 +35,7 @@
35
35
  - Работа с длинным контекстом **делегируется** дешевым worker-моделям через `llm_query` / `llm_query_batched`.
36
36
  - Сложные подзадачи **рекурсивно** передаются в дочерние RLM через `rlm_query` (с ограничением глубины).
37
37
  Дочерний RLM наследует `context` родителя — репозиторий и все библиотеки, загруженные через
38
- `load_library()`, — и работает с теми же путями. Наследование не стоит дополнительных токенов:
38
+ `add_context()`, — и работает с теми же путями. Наследование не стоит дополнительных токенов:
39
39
  содержимое живёт в песочнице, модель видит только строку с размером.
40
40
  - Все работает **in-process** — единственным внешним процессом является локальный worker `python3`.
41
41
 
@@ -154,7 +154,7 @@ src/
154
154
  text/ parsing (repl blocks) · tokens · preview
155
155
  tool/ repl-tool · repl-result · repl-render · rlm-tool · rlm-events · rlm-aggregator · subcall-store · background-tasks
156
156
  config/ defaults · settings (rlm.json persistence + validation)
157
- context/ repomix repository packing + library context merge
157
+ context/ native walker + anydoc document conversion + add_context
158
158
  ui/ status · model-picker · config-panel · intro · theme
159
159
  commands/ rlm · rlm-config
160
160
  mode/ rlm-mode (controller) · worker-model (cheapest pick) · native-guards
package/README.zh-CN.md CHANGED
@@ -34,7 +34,7 @@
34
34
  - **根编排器**模型逐轮驱动一个**持久化的 Python REPL**。
35
35
  - 长上下文工作通过 `llm_query` / `llm_query_batched` **委派**给廉价的工作模型。
36
36
  - 困难的子问题通过 `rlm_query` **递归**到子 RLM 中(设有深度限制)。子 RLM 继承父级的 `context`
37
- ——仓库以及通过 `load_library()` 加载的所有库——因此可以在相同的路径上使用相同的检索原语。
37
+ ——已加载的文件以及通过 `add_context()` 追加的来源——因此可以在相同的路径上使用相同的检索原语。
38
38
  继承不消耗额外的 token:内容存放在沙箱中,模型只看到一行大小信息。
39
39
  - 所有内容均**在进程内**运行 —— 唯一的外部进程是一个本地的 `python3` worker。
40
40
 
@@ -168,7 +168,7 @@ src/
168
168
  text/ parsing (repl blocks) · tokens · preview
169
169
  tool/ repl-tool · repl-result · repl-render · rlm-tool · rlm-events · rlm-aggregator · subcall-store · background-tasks
170
170
  config/ defaults · settings (rlm.json persistence + validation)
171
- context/ repomix repository packing + library context merge
171
+ context/ native walker + anydoc document conversion + add_context
172
172
  ui/ status · model-picker · config-panel · intro · theme
173
173
  commands/ rlm · rlm-config
174
174
  mode/ rlm-mode (controller) · worker-model (cheapest pick) · native-guards
package/package.json CHANGED
@@ -1,23 +1,30 @@
1
1
  {
2
2
  "name": "@hicaru/pi-rlm",
3
- "version": "0.2.2",
4
- "type": "module",
5
- "description": "Save 99% tokens, Recursive Language Model (RLM) for the Pi",
6
- "license": "MIT",
3
+ "version": "0.3.1",
7
4
  "author": "hicaru",
8
5
  "repository": {
9
6
  "type": "git",
10
7
  "url": "git+https://github.com/openzebra/rlm.pi.git"
11
8
  },
12
- "homepage": "https://github.com/openzebra/rlm.pi",
9
+ "devDependencies": {
10
+ "typescript": "^5.0.0"
11
+ },
12
+ "peerDependencies": {
13
+ "@earendil-works/pi-ai": "*",
14
+ "@earendil-works/pi-coding-agent": "*",
15
+ "@earendil-works/pi-tui": "*",
16
+ "typebox": "*"
17
+ },
13
18
  "bugs": {
14
19
  "url": "https://github.com/openzebra/rlm.pi/issues"
15
20
  },
21
+ "description": "Save 99% tokens, Recursive Language Model (RLM) for the Pi",
16
22
  "files": [
17
23
  "src/",
18
24
  "README.md",
19
25
  "LICENSE"
20
26
  ],
27
+ "homepage": "https://github.com/openzebra/rlm.pi",
21
28
  "keywords": [
22
29
  "pi-package",
23
30
  "pi-extension",
@@ -25,11 +32,7 @@
25
32
  "recursive",
26
33
  "ai-agent"
27
34
  ],
28
- "scripts": {
29
- "check": "tsc --noEmit",
30
- "test": "bun run test/smoke.ts",
31
- "prepublishOnly": "npm run check"
32
- },
35
+ "license": "MIT",
33
36
  "pi": {
34
37
  "extensions": [
35
38
  "./src/index.ts"
@@ -38,16 +41,16 @@
38
41
  "publishConfig": {
39
42
  "access": "public"
40
43
  },
41
- "peerDependencies": {
42
- "@earendil-works/pi-ai": "*",
43
- "@earendil-works/pi-coding-agent": "*",
44
- "@earendil-works/pi-tui": "*",
45
- "typebox": "*"
44
+ "scripts": {
45
+ "check": "tsc --noEmit",
46
+ "test": "bun run test/smoke.ts",
47
+ "prepublishOnly": "npm run check"
46
48
  },
47
- "dependencies": {
48
- "repomix": "^1.15.0"
49
+ "type": "module",
50
+ "engines": {
51
+ "node": ">=20"
49
52
  },
50
- "devDependencies": {
51
- "typescript": "^5.0.0"
53
+ "dependencies": {
54
+ "@firecrawl/anydoc": "^0.1.7"
52
55
  }
53
56
  }
@@ -0,0 +1,322 @@
1
+ /**
2
+ * Shared add_context handler for headless engine and native repl() mode.
3
+ *
4
+ * Host packs the source via resolveSource (namespaced under ctx/<id>/) and returns the
5
+ * payload for the worker to append into the single `context` list.
6
+ *
7
+ * Idempotency is host-side:
8
+ * - prefix set (ctx/<id>/) for external sources
9
+ * - cwd seed: markSeededCwd + exact-path short-circuit for add_context(".")
10
+ * - subpath of seed: short-circuit only when the live context already holds un-prefixed
11
+ * entries under that relative path (gitignored subtrees still pack for real)
12
+ *
13
+ * Late-bound deps (getCwd / getEmitter) keep a single handler closure correct
14
+ * across native repl() calls — getOrCreate only installs handlers at spawn.
15
+ */
16
+
17
+ import { isAbsolute, relative, resolve, sep } from "node:path";
18
+ import type { RlmEmitter } from "../tool/rlm-events.ts";
19
+ import type { SubLlmHandlers } from "../sandbox/sandbox.ts";
20
+ import {
21
+ contextNamespace,
22
+ isContextFile,
23
+ } from "../context/namespace.ts";
24
+ import { resolveSource } from "../context/resolve.ts";
25
+ import { previewText } from "../text/preview.ts";
26
+
27
+ export interface AddContextBridgeOpts {
28
+ /** Fixed cwd (headless). Prefer getCwd when the sandbox outlives a single invocation. */
29
+ readonly cwd?: string;
30
+ /** Late-bound cwd (native mode — sandbox handlers outlive a single repl()). */
31
+ readonly getCwd?: () => string;
32
+ readonly emitter?: RlmEmitter;
33
+ /** Native mode: read the live emitter each call. */
34
+ readonly getEmitter?: () => RlmEmitter | null | undefined;
35
+ readonly parentId?: string;
36
+ readonly signal?: AbortSignal;
37
+ /** Prefixes already present in context — seeds host-side idempotency after a sandbox restart. */
38
+ readonly loadedPrefixes?: readonly string[];
39
+ /**
40
+ * The live context this sandbox holds. Read to refuse pre-flight exactly what the worker's
41
+ * `_append_context` would reject, before any prefix is committed.
42
+ */
43
+ readonly getContext?: () => unknown;
44
+ /**
45
+ * Post-load hook. The engine grows its live context here; native mode grows
46
+ * SandboxManager.contextPayload.
47
+ */
48
+ readonly onLoaded?: (payload: unknown) => void | Promise<void>;
49
+ }
50
+
51
+ export interface AddContextHandlerBundle {
52
+ readonly handlers: Pick<SubLlmHandlers, "addContext">;
53
+ /**
54
+ * Reset the loaded-prefix cache (call when the sandbox is
55
+ * discarded and will re-spawn).
56
+ *
57
+ * `keep` re-seeds the cache from the payload that will be replayed into the fresh worker.
58
+ * `loaded` is a CACHE of `contextPrefixesIn(context)` plus the cwd sentinel `""`, never
59
+ * independent state, so it may only be cleared by re-deriving it — clearing it outright
60
+ * would make the host re-clone a source the recreated worker already holds.
61
+ */
62
+ readonly reset: (keep?: readonly string[]) => void;
63
+ /**
64
+ * Register a prefix as already loaded without packing. Used by the cwd seed to plant the
65
+ * `""` sentinel so add_context of the same tree is a no-op.
66
+ */
67
+ readonly markLoaded: (prefix: string) => void;
68
+ /**
69
+ * Record the absolute path of the cwd seed. add_context(".") resolves to a ctx/<id>/
70
+ * namespace, not "", so the absolute-path check is the only reliable short-circuit.
71
+ */
72
+ readonly markSeededCwd: (absPath: string) => void;
73
+ /** Prefixes loaded in this sandbox lifetime (for tests). */
74
+ readonly loadedPrefixes: () => ReadonlySet<string>;
75
+ /** Absolute seeded cwd, if any (for tests). */
76
+ readonly seededCwd: () => string | undefined;
77
+ }
78
+
79
+ /**
80
+ * JS runtime kind → the Python type name worker.py reports, so both sides emit exactly one
81
+ * message for the same refusal. Covers every shape a context payload can take after JSON
82
+ * transport; anything else is a plain object, which `json.load` materializes as a dict.
83
+ */
84
+ const PY_TYPE_NAME: Readonly<Record<string, string>> = Object.freeze({
85
+ string: "str", boolean: "bool", number: "int", bigint: "int", undefined: "None",
86
+ });
87
+
88
+ function pythonKindOf(value: unknown): string {
89
+ if (value === null) return "None"; // matches worker.py's `if ctx is not None else "None"`
90
+ return PY_TYPE_NAME[typeof value] ?? "dict";
91
+ }
92
+
93
+ /**
94
+ * Refusal messages shared with worker.py `_append_context`. The worker is the backstop; the host
95
+ * pre-flights the same two conditions so it never commits a prefix for an append that
96
+ * will be rejected. Keep the wording identical — a comment in worker.py points back here.
97
+ */
98
+ const LIST_CONTEXT_REQUIRED = (kind: string): string =>
99
+ `add_context requires list context (file bundle); got ${kind}`;
100
+ const NO_FILES_PRODUCED = "add_context produced no files";
101
+
102
+ /** Absolute path with no trailing slash (except root). */
103
+ function absKey(path: string): string {
104
+ const resolved = resolve(path);
105
+ return resolved.length > 1 && (resolved.endsWith("/") || resolved.endsWith("\\"))
106
+ ? resolved.slice(0, -1)
107
+ : resolved;
108
+ }
109
+
110
+ /**
111
+ * True when the live context already holds un-prefixed (cwd-seed) entries under `relPrefix`.
112
+ * Used so add_context("./src/context") does not double-load a subpath that the seed already
113
+ * has — without blocking genuinely-absent (gitignored) subtrees.
114
+ */
115
+ function contextHasUnprefixedUnder(context: unknown, relPrefix: string): boolean {
116
+ if (!Array.isArray(context) || relPrefix === "") return false;
117
+ const clean = relPrefix.replace(/\\/g, "/").replace(/^\/+|\/+$/g, "");
118
+ if (clean === "" || clean.startsWith("..")) return false;
119
+ const withSlash = `${clean}/`;
120
+ for (let i = 0; i < context.length; i++) {
121
+ const entry: unknown = context[i];
122
+ if (!isContextFile(entry)) continue;
123
+ // Only cwd-seed paths are un-prefixed; ctx/<id>/… is a different source.
124
+ if (entry.path.startsWith("ctx/")) continue;
125
+ if (entry.path === clean || entry.path.startsWith(withSlash)) return true;
126
+ }
127
+ return false;
128
+ }
129
+
130
+ function alreadyLoadedResult(
131
+ sourceId: string,
132
+ pathPrefix: string,
133
+ ): Awaited<ReturnType<SubLlmHandlers["addContext"]>> {
134
+ return {
135
+ payload: Object.freeze([]),
136
+ files: 0,
137
+ chars: 0,
138
+ sourceId,
139
+ pathPrefix,
140
+ alreadyLoaded: true,
141
+ documents: 0,
142
+ converted: 0,
143
+ skipped: Object.freeze([]),
144
+ };
145
+ }
146
+
147
+ export function buildAddContextHandler(opts: AddContextBridgeOpts): AddContextHandlerBundle {
148
+ /** Prefixes already loaded in this sandbox — mirrors the worker's context state. */
149
+ const loaded = new Set<string>(opts.loadedPrefixes ?? []);
150
+ /** Absolute path of the cwd seed, if autoSeedCwd planted one successfully. */
151
+ let seededCwdAbs: string | undefined;
152
+ return {
153
+ reset: (keep) => {
154
+ const seed = keep ?? opts.loadedPrefixes ?? [];
155
+ loaded.clear();
156
+ for (const prefix of seed) loaded.add(prefix);
157
+ // Do NOT clear seededCwdAbs — the payload is still on disk in the manager and will be
158
+ // replayed; the absolute-path short-circuit must keep working after a death-recreate.
159
+ },
160
+ markLoaded: (prefix) => { loaded.add(prefix); },
161
+ markSeededCwd: (absPath) => {
162
+ seededCwdAbs = absKey(absPath);
163
+ loaded.add(""); // cwd sentinel — payloadPrefix never sees un-prefixed files
164
+ },
165
+ loadedPrefixes: () => loaded,
166
+ seededCwd: () => seededCwdAbs,
167
+ handlers: {
168
+ async addContext(source, depth) {
169
+ const emitter = opts.getEmitter?.() ?? opts.emitter;
170
+ const cwd = opts.getCwd?.() ?? opts.cwd;
171
+ if (cwd === undefined || cwd === "") {
172
+ throw new Error("add_context: no cwd configured");
173
+ }
174
+ const id = emitter?.emitSubcallCreated({
175
+ kind: "tool", parentId: opts.parentId,
176
+ label: "add_context",
177
+ args: previewText(source, 80),
178
+ depth,
179
+ });
180
+ try {
181
+ // Pre-flight the worker's own refusal: a non-list context cannot be appended to, and
182
+ // committing a prefix for it would make the NEXT load lie with already_loaded.
183
+ const current = opts.getContext?.();
184
+ if (current !== undefined && !Array.isArray(current)) {
185
+ throw new Error(LIST_CONTEXT_REQUIRED(pythonKindOf(current)));
186
+ }
187
+
188
+ const trimmed = source.trim();
189
+ const isLocal = trimmed !== "" && !/^(https:\/\/|git@)/.test(trimmed)
190
+ && !/^[a-z][a-z0-9+.-]*:\/\//i.test(trimmed);
191
+ const candidate = isLocal
192
+ ? absKey(isAbsolute(trimmed) ? trimmed : resolve(cwd, trimmed))
193
+ : undefined;
194
+ const cwdAbs = absKey(cwd);
195
+
196
+ // ── Cwd seed short-circuit / recovery ──
197
+ // Exact cwd: if already seeded → no-op; if seed failed (sticky, no markSeededCwd)
198
+ // pack un-prefixed so paths stay edit/write-friendly.
199
+ if (candidate !== undefined && candidate === cwdAbs) {
200
+ if (seededCwdAbs !== undefined) {
201
+ if (id) {
202
+ emitter?.emitSubcallUpdated({
203
+ id, status: "done", resultPreview: "already loaded (cwd seed)",
204
+ });
205
+ }
206
+ return alreadyLoadedResult("cwd", "");
207
+ }
208
+ // Recovery after failed seed (or autoSeedCwd off): pack as primary, un-prefixed.
209
+ const recovered = await resolveSource(source, {
210
+ cwd, pathPrefix: "", signal: opts.signal,
211
+ });
212
+ if (!recovered.ok) throw new Error(recovered.error);
213
+ const r = recovered.value;
214
+ if (r.payload.length === 0) throw new Error(NO_FILES_PRODUCED);
215
+ if (opts.onLoaded) await opts.onLoaded(r.payload);
216
+ seededCwdAbs = cwdAbs;
217
+ loaded.add("");
218
+ if (id) {
219
+ emitter?.emitSubcallUpdated({
220
+ id, status: "done",
221
+ resultPreview: `+${r.files} file(s) → context (cwd seed recovery, ${r.chars.toLocaleString()} chars)`,
222
+ });
223
+ }
224
+ return {
225
+ payload: r.payload,
226
+ files: r.files,
227
+ chars: r.chars,
228
+ sourceId: r.sourceId,
229
+ pathPrefix: "",
230
+ alreadyLoaded: false,
231
+ documents: r.documents,
232
+ converted: r.converted,
233
+ skipped: r.skipped,
234
+ };
235
+ }
236
+
237
+ // Subpath of the seeded cwd: only short-circuit when those files are already in
238
+ // context (un-prefixed). A gitignored subtree that the seed never had still packs.
239
+ if (candidate !== undefined && seededCwdAbs !== undefined
240
+ && candidate !== seededCwdAbs
241
+ && (candidate.startsWith(seededCwdAbs + sep) || candidate.startsWith(seededCwdAbs + "/"))) {
242
+ const rel = relative(seededCwdAbs, candidate).split(sep).join("/");
243
+ if (rel !== "" && !rel.startsWith("..") && contextHasUnprefixedUnder(current, rel)) {
244
+ if (id) {
245
+ emitter?.emitSubcallUpdated({
246
+ id, status: "done",
247
+ resultPreview:
248
+ `already in cwd seed under '${rel}/' — filter context by path prefix`,
249
+ });
250
+ }
251
+ return alreadyLoadedResult("cwd", "");
252
+ }
253
+ }
254
+
255
+ // Cheap pre-check BEFORE cloning/packing: same namespace ⇒ nothing to do.
256
+ const { sourceId: preId, pathPrefix: prefix } = contextNamespace(source, cwd);
257
+ if (loaded.has(prefix)) {
258
+ if (id) {
259
+ emitter?.emitSubcallUpdated({
260
+ id,
261
+ status: "done",
262
+ resultPreview: `already loaded (${prefix}*)`,
263
+ });
264
+ }
265
+ return alreadyLoadedResult(preId, prefix);
266
+ }
267
+
268
+ const resolved = await resolveSource(source, { cwd, signal: opts.signal });
269
+ if (!resolved.ok) throw new Error(resolved.error);
270
+ const { payload, files, chars, sourceId, pathPrefix, documents, converted, skipped } =
271
+ resolved.value;
272
+ // The worker's other refusal, pre-flighted for the same reason.
273
+ if (payload.length === 0) throw new Error(NO_FILES_PRODUCED);
274
+
275
+ // Race: another concurrent load of the same prefix finished while we packed.
276
+ if (loaded.has(pathPrefix)) {
277
+ if (id) {
278
+ emitter?.emitSubcallUpdated({
279
+ id,
280
+ status: "done",
281
+ resultPreview: `already loaded (${pathPrefix}*)`,
282
+ });
283
+ }
284
+ return alreadyLoadedResult(sourceId, pathPrefix);
285
+ }
286
+
287
+ // Mark loaded only after the host has grown its own copy of the context.
288
+ if (opts.onLoaded) {
289
+ await opts.onLoaded(payload);
290
+ }
291
+ loaded.add(pathPrefix);
292
+
293
+ if (id) {
294
+ const docNote = documents > 0
295
+ ? `, ${documents} doc(s)${converted > 0 ? ` (${converted} fresh)` : " (cached)"}`
296
+ : "";
297
+ emitter?.emitSubcallUpdated({
298
+ id,
299
+ status: "done",
300
+ resultPreview:
301
+ `+${files} file(s) → context (${pathPrefix}*, ${chars.toLocaleString()} chars${docNote})`,
302
+ });
303
+ }
304
+ return {
305
+ payload,
306
+ files,
307
+ chars,
308
+ sourceId,
309
+ pathPrefix,
310
+ alreadyLoaded: false,
311
+ documents,
312
+ converted,
313
+ skipped,
314
+ };
315
+ } catch (err) {
316
+ if (id) emitter?.emitSubcallUpdated({ id, status: "error", detail: String(err) });
317
+ throw err; // serviceInterrupt catch → {error} reply → "Error: …" in the REPL
318
+ }
319
+ },
320
+ },
321
+ };
322
+ }
@@ -20,7 +20,7 @@ import { displayModelRef, modelRef, resolveModelId } from "../config/settings.ts
20
20
  import { type ChatMsg, modelComplete } from "./model.ts";
21
21
  import { previewText } from "../text/preview.ts";
22
22
  import { checkResourceLimits } from "../core/resource-limits.ts";
23
- import { filterContextByPaths } from "../context/library-context.ts";
23
+ import { filterContextByPaths } from "../context/merge.ts";
24
24
  import type { RlmInput, RlmResult, Sampling } from "../core/types.ts";
25
25
  import type { SubcallGates } from "../util/concurrency.ts";
26
26
  import type { SubcallOpts, SubLlmHandlers } from "../sandbox/sandbox.ts";
@@ -26,7 +26,8 @@ export const DEFAULT_CONFIG: Readonly<RlmConfig> = Object.freeze({
26
26
  compactionThresholdPct: 0.65,
27
27
  python: "python3",
28
28
  sandboxInitTimeoutMs: 30_000,
29
- libraryLoader: true,
29
+ contextLoader: true,
30
+ autoSeedCwd: true,
30
31
  rootSampling: Object.freeze({ maxTokens: 16_384 }),
31
32
  subSystemPrompt: DEFAULT_SUB_SYSTEM_PROMPT,
32
33
  subSampling: Object.freeze({ maxTokens: 8192 }),
@@ -84,8 +84,11 @@ function validateConfig(raw: unknown): Partial<RlmConfig> {
84
84
  if (subSystemPrompt !== undefined) out.subSystemPrompt = subSystemPrompt;
85
85
  const sandboxInitTimeoutMs = validateNumber(r.sandboxInitTimeoutMs, 100);
86
86
  if (sandboxInitTimeoutMs !== undefined) out.sandboxInitTimeoutMs = sandboxInitTimeoutMs;
87
- const libraryLoader = validateBoolean(r.libraryLoader);
88
- if (libraryLoader !== undefined) out.libraryLoader = libraryLoader;
87
+ // `libraryLoader` is the pre-rename key — still read so an existing rlm.json survives the upgrade.
88
+ const contextLoader = validateBoolean(r.contextLoader) ?? validateBoolean(r.libraryLoader);
89
+ if (contextLoader !== undefined) out.contextLoader = contextLoader;
90
+ const autoSeedCwd = validateBoolean(r.autoSeedCwd);
91
+ if (autoSeedCwd !== undefined) out.autoSeedCwd = autoSeedCwd;
89
92
  if (typeof r.subSampling === "object" && r.subSampling !== null) {
90
93
  const ss = r.subSampling as Record<string, unknown>;
91
94
  const sampling: { maxTokens?: number; temperature?: number; reasoning?: ThinkingLevel } = {};
@@ -0,0 +1,67 @@
1
+ /**
2
+ * Lazy NAPI handle for @firecrawl/anydoc.
3
+ *
4
+ * anydoc is a native addon with per-platform optionalDependencies (darwin x64/arm64,
5
+ * linux x64/arm64 gnu+musl, win32 x64 — NO win32-arm64). A static `import` on an uncovered
6
+ * platform is a plugin-load crash. So: import() once, memoised, catch → null. Absence is a
7
+ * value; documents degrade to skipped: "no-converter".
8
+ */
9
+
10
+ /** Minimal surface we consume — never re-export the full anydoc package. */
11
+ export interface AnydocHandle {
12
+ /** Detect document format from a path's extension; null for plain text / unknown. */
13
+ readonly formatFromPath: (path: string) => string | null;
14
+ /** Convert a document file to Markdown. Rejects with Error.code = ConvertErrorCode. */
15
+ readonly toMarkdown: (path: string) => Promise<string>;
16
+ }
17
+
18
+ /**
19
+ * Document container extensions anydoc knows about, including Office macro/variants.
20
+ * Used when the native addon is absent so we still route these to "no-converter"
21
+ * instead of reading them as broken binary text.
22
+ */
23
+ const DOCUMENT_EXTENSIONS: ReadonlySet<string> = Object.freeze(new Set([
24
+ "doc", "docx", "docm", "odt", "rtf", "epub", "pdf",
25
+ "ppt", "pptx", "pptm", "ppsx", "odp",
26
+ "xls", "xlsx", "xlsm", "xlsb", "ods", "csv",
27
+ ]));
28
+
29
+ /**
30
+ * Detect a document container from a path when the anydoc handle is unavailable.
31
+ * Returns a non-null token for known extensions so the router can skip as "no-converter".
32
+ */
33
+ export function documentExtFromPath(path: string): string | null {
34
+ const base = path.includes("/") ? path.slice(path.lastIndexOf("/") + 1) : path;
35
+ const dot = base.lastIndexOf(".");
36
+ if (dot < 0) return null;
37
+ const ext = base.slice(dot + 1).toLowerCase();
38
+ return DOCUMENT_EXTENSIONS.has(ext) ? ext : null;
39
+ }
40
+
41
+ let cached: Promise<AnydocHandle | null> | undefined;
42
+
43
+ /**
44
+ * Resolve the anydoc handle once per process. Returns null when the native addon is
45
+ * missing or fails to load — callers treat documents as skipped: "no-converter".
46
+ */
47
+ export function getAnydoc(): Promise<AnydocHandle | null> {
48
+ cached ??= import("@firecrawl/anydoc")
49
+ .then((mod): AnydocHandle => Object.freeze({
50
+ formatFromPath: (p: string) => mod.formatFromPath(p),
51
+ toMarkdown: (p: string) => mod.toMarkdown(p),
52
+ }))
53
+ .catch(() => null);
54
+ return cached;
55
+ }
56
+
57
+ /**
58
+ * Test seam: force the next getAnydoc() call to return a fixed handle (or null).
59
+ * Pass `undefined` to restore the real lazy loader.
60
+ */
61
+ export function setAnydocForTest(handle: AnydocHandle | null | undefined): void {
62
+ if (handle === undefined) {
63
+ cached = undefined;
64
+ return;
65
+ }
66
+ cached = Promise.resolve(handle);
67
+ }