@hicaru/pi-rlm 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -16
- package/README.ru.md +2 -2
- package/README.zh-CN.md +2 -2
- package/package.json +22 -19
- package/src/bridge/add-context.ts +322 -0
- package/src/bridge/subcall-handlers.ts +1 -1
- package/src/config/defaults.ts +2 -1
- package/src/config/settings.ts +5 -2
- package/src/context/anydoc.ts +67 -0
- package/src/context/listing.ts +70 -0
- package/src/context/md-cache.ts +112 -0
- package/src/context/merge.ts +97 -0
- package/src/context/namespace.ts +180 -0
- package/src/context/resolve.ts +122 -0
- package/src/context/source-dir.ts +166 -0
- package/src/context/source-doc.ts +71 -0
- package/src/context/source-git.ts +51 -0
- package/src/context/source-text.ts +45 -0
- package/src/context/types.ts +88 -0
- package/src/context/walk.ts +250 -0
- package/src/core/engine.ts +15 -19
- package/src/core/types.ts +7 -2
- package/src/index.ts +69 -42
- package/src/mode/rlm-mode.ts +5 -4
- package/src/prompts/glossary.ts +31 -28
- package/src/prompts/native.ts +4 -4
- package/src/prompts/system.ts +2 -2
- package/src/sandbox/context-file.ts +4 -4
- package/src/sandbox/interrupts.ts +25 -10
- package/src/sandbox/protocol.ts +13 -7
- package/src/sandbox/py/__pycache__/guards.cpython-314.pyc +0 -0
- package/src/sandbox/py/__pycache__/retrieval.cpython-314.pyc +0 -0
- package/src/sandbox/py/__pycache__/tasks.cpython-314.pyc +0 -0
- package/src/sandbox/py/guards.py +1 -1
- package/src/sandbox/py/retrieval.py +1 -1
- package/src/sandbox/py/tasks.py +17 -4
- package/src/sandbox/py/worker.py +68 -48
- package/src/sandbox/sandbox-manager.ts +18 -16
- package/src/sandbox/sandbox.ts +1 -1
- package/src/text/tokens.ts +3 -3
- package/src/tool/repl-details.ts +1 -1
- package/src/tool/repl-tool.ts +31 -19
- package/src/tool/rlm-tool.ts +1 -1
- package/src/ui/config-panel.ts +8 -4
- package/src/bridge/library.ts +0 -190
- package/src/context/library-context.ts +0 -339
- package/src/context/repomix-context.ts +0 -204
package/src/sandbox/protocol.ts
CHANGED
|
@@ -18,15 +18,21 @@ export interface LlmReply {
|
|
|
18
18
|
readonly rid: string;
|
|
19
19
|
readonly response?: string;
|
|
20
20
|
readonly responses?: readonly string[];
|
|
21
|
-
/**
|
|
21
|
+
/** add_context reply: temp file with the packed payload (+ namespace metadata). */
|
|
22
22
|
readonly path?: string;
|
|
23
23
|
readonly json?: boolean;
|
|
24
24
|
readonly files?: number;
|
|
25
25
|
readonly chars?: number;
|
|
26
26
|
readonly source_id?: string;
|
|
27
27
|
readonly path_prefix?: string;
|
|
28
|
-
/** Host-side idempotency:
|
|
28
|
+
/** Host-side idempotency: source already loaded — no path payload. */
|
|
29
29
|
readonly already_loaded?: boolean;
|
|
30
|
+
/** Document-type files in the payload (fresh + cache hits). */
|
|
31
|
+
readonly documents?: number;
|
|
32
|
+
/** Documents freshly converted this call (cache hits excluded). */
|
|
33
|
+
readonly converted?: number;
|
|
34
|
+
/** Paths skipped during packing (binary, no-converter, …). */
|
|
35
|
+
readonly skipped?: readonly { readonly path: string; readonly reason: string }[];
|
|
30
36
|
readonly error?: string;
|
|
31
37
|
}
|
|
32
38
|
|
|
@@ -56,7 +62,7 @@ export type InterruptKind =
|
|
|
56
62
|
| "llm_query_batched"
|
|
57
63
|
| "rlm_query"
|
|
58
64
|
| "rlm_query_batched"
|
|
59
|
-
| "
|
|
65
|
+
| "add_context";
|
|
60
66
|
|
|
61
67
|
interface InterruptBase {
|
|
62
68
|
readonly rid: string;
|
|
@@ -88,8 +94,8 @@ interface BatchedPromptInterrupt extends InterruptBase {
|
|
|
88
94
|
readonly paths?: readonly string[];
|
|
89
95
|
}
|
|
90
96
|
|
|
91
|
-
export interface
|
|
92
|
-
readonly type: "
|
|
97
|
+
export interface AddContextInterrupt extends InterruptBase {
|
|
98
|
+
readonly type: "add_context";
|
|
93
99
|
readonly source?: string;
|
|
94
100
|
}
|
|
95
101
|
|
|
@@ -97,7 +103,7 @@ export interface LoadLibraryInterrupt extends InterruptBase {
|
|
|
97
103
|
export type WorkerInterrupt =
|
|
98
104
|
| PromptInterrupt
|
|
99
105
|
| BatchedPromptInterrupt
|
|
100
|
-
|
|
|
106
|
+
| AddContextInterrupt;
|
|
101
107
|
|
|
102
108
|
export type WorkerMessage = WorkerResponse | WorkerInterrupt;
|
|
103
109
|
|
|
@@ -106,7 +112,7 @@ export const INTERRUPT_KINDS = Object.freeze(new Set<InterruptKind>([
|
|
|
106
112
|
"llm_query_batched",
|
|
107
113
|
"rlm_query",
|
|
108
114
|
"rlm_query_batched",
|
|
109
|
-
"
|
|
115
|
+
"add_context",
|
|
110
116
|
]));
|
|
111
117
|
|
|
112
118
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
package/src/sandbox/py/guards.py
CHANGED
|
@@ -46,7 +46,7 @@ def _chunk_text(text: str, chunk_chars: int) -> list[str]:
|
|
|
46
46
|
|
|
47
47
|
|
|
48
48
|
_INDEX_WINDOW_LINES = 40 # a window is the retrieval unit: big enough to carry meaning
|
|
49
|
-
_INDEX_MAX_WINDOWS = 20_000 # ceiling so a huge
|
|
49
|
+
_INDEX_MAX_WINDOWS = 20_000 # ceiling so a huge add_context() cannot exhaust worker memory
|
|
50
50
|
_SNIPPET_CHARS = 400
|
|
51
51
|
_GREP_HARD_CAP = 200 # absolute ceiling on returned grep hits, whatever k asks for
|
|
52
52
|
_BM25_K1 = 1.2
|
package/src/sandbox/py/tasks.py
CHANGED
|
@@ -41,14 +41,27 @@ def _reduce_batch(n: int):
|
|
|
41
41
|
return reduce
|
|
42
42
|
|
|
43
43
|
|
|
44
|
-
def _reduce_chunked(sizes: list[int]):
|
|
45
|
-
"""Concatenate several llm_query_batched replies back into one flat chunk list.
|
|
44
|
+
def _reduce_chunked(sizes: list[int], drop_empty: bool = True):
|
|
45
|
+
"""Concatenate several llm_query_batched replies back into one flat chunk list.
|
|
46
|
+
|
|
47
|
+
drop_empty=True (llm_query_chunked): filter "" replies so results never degrade
|
|
48
|
+
to blank entries; the flattened list may then be shorter than the chunk count,
|
|
49
|
+
so callers must consume answers in order and never index-match a chunk to a slot.
|
|
50
|
+
drop_empty=False (map_files): keep "" placeholders because _reduce_map_files
|
|
51
|
+
regroups by position — dropping an empty reply there would shift every later
|
|
52
|
+
slice and merge file contents into the wrong paths.
|
|
53
|
+
"""
|
|
46
54
|
per = [_reduce_batch(n) for n in sizes]
|
|
47
55
|
|
|
48
56
|
def reduce(replies: list[dict[str, Any]]) -> list[str]:
|
|
57
|
+
if len(replies) != len(per):
|
|
58
|
+
return [f"Error: chunk reply count mismatch ({len(replies)} != {len(per)})"] * sum(sizes)
|
|
49
59
|
out: list[str] = []
|
|
50
60
|
for red, rep in zip(per, replies):
|
|
51
|
-
|
|
61
|
+
if drop_empty:
|
|
62
|
+
out.extend(x for x in red([rep]) if x)
|
|
63
|
+
else:
|
|
64
|
+
out.extend(red([rep]))
|
|
52
65
|
return out
|
|
53
66
|
return reduce
|
|
54
67
|
|
|
@@ -59,7 +72,7 @@ def _reduce_map_files(sizes: list[int], spans: list[tuple[str, int]]):
|
|
|
59
72
|
A file larger than the per-prompt budget contributed several requests; its answers rejoin
|
|
60
73
|
in order, which is what makes map_files a {path: answer} dict rather than a flat list.
|
|
61
74
|
"""
|
|
62
|
-
flatten = _reduce_chunked(sizes)
|
|
75
|
+
flatten = _reduce_chunked(sizes, drop_empty=False)
|
|
63
76
|
|
|
64
77
|
def reduce(replies: list[dict[str, Any]]) -> dict[str, str]:
|
|
65
78
|
responses = flatten(replies)
|
package/src/sandbox/py/worker.py
CHANGED
|
@@ -7,9 +7,9 @@ This is NOT a security sandbox: __import__ and open are available, so code can i
|
|
|
7
7
|
Protocol (parent -> worker): {"id","type":"exec"|"load_context"|"shutdown", ...}
|
|
8
8
|
Protocol (worker -> parent): {"id","ok",...result} # response to a request
|
|
9
9
|
{"type":"llm_query"|"llm_query_batched"|"rlm_query"|
|
|
10
|
-
"rlm_query_batched"|"
|
|
10
|
+
"rlm_query_batched"|"add_context","rid",...}
|
|
11
11
|
# mid-exec helper request
|
|
12
|
-
When sandbox code calls llm_query/rlm_query/
|
|
12
|
+
When sandbox code calls llm_query/rlm_query/add_context, the worker writes a
|
|
13
13
|
request line and BLOCKS reading stdin until the matching {"type":"llm_reply","rid",...} arrives.
|
|
14
14
|
The parent services the request in-process (it holds API keys).
|
|
15
15
|
|
|
@@ -113,7 +113,7 @@ class Worker:
|
|
|
113
113
|
builtins = _SAFE_BUILTINS.copy()
|
|
114
114
|
builtins["open"] = open
|
|
115
115
|
self.ns = {"__builtins__": builtins, "__name__": "__main__"}
|
|
116
|
-
self._context_payload: Any
|
|
116
|
+
self._context_payload: Any = [] # empty list — the only starting value that needs no bootstrap branch
|
|
117
117
|
self._nudged: set[str] = set()
|
|
118
118
|
self._index: _Bm25Index | None = None
|
|
119
119
|
self._index_stamp: tuple[int, int] | None = None # (id(context), len(context))
|
|
@@ -144,7 +144,7 @@ class Worker:
|
|
|
144
144
|
ns["answers"] = {}
|
|
145
145
|
if not isinstance(ns.get("plan"), dict):
|
|
146
146
|
ns["plan"] = {}
|
|
147
|
-
ns["
|
|
147
|
+
ns["add_context"] = self._add_context
|
|
148
148
|
ns["SHOW_VARS"] = self._show_vars
|
|
149
149
|
if not isinstance(ns.get("answer"), _AnswerDict):
|
|
150
150
|
cur = ns.get("answer")
|
|
@@ -157,9 +157,8 @@ class Worker:
|
|
|
157
157
|
ns["answer"] = ans
|
|
158
158
|
# Single context variable (RLM paper: the context lives in the environment and
|
|
159
159
|
# the model may transform it in place). Re-inject only if the model deleted the
|
|
160
|
-
# name entirely; mutations and re-binds persist within the run.
|
|
161
|
-
|
|
162
|
-
ns.setdefault("context", self._context_payload)
|
|
160
|
+
# name entirely; mutations and re-binds persist within the run. Always a list.
|
|
161
|
+
ns.setdefault("context", self._context_payload)
|
|
163
162
|
# Scrub any legacy context_N names so the model never sees multi-slot APIs.
|
|
164
163
|
for k in list(ns.keys()):
|
|
165
164
|
if k != "context" and _CONTEXT_NAME.match(k):
|
|
@@ -412,7 +411,7 @@ class Worker:
|
|
|
412
411
|
"""Build the BM25 index on first use; rebuild when `context` was replaced or resized.
|
|
413
412
|
|
|
414
413
|
Identity+length is a cheap stamp that catches the two ways context actually changes:
|
|
415
|
-
|
|
414
|
+
add_context() extending the list, and the model re-binding the name. In-place edits
|
|
416
415
|
that preserve length are not detected — documented, and rare in practice.
|
|
417
416
|
"""
|
|
418
417
|
ctx = self.ns.get("context")
|
|
@@ -556,23 +555,24 @@ class Worker:
|
|
|
556
555
|
def _rlm_query(self, prompt: str, model: str | None = None, paths=None) -> str:
|
|
557
556
|
return self._await_task(self._start_rlm_query(prompt, model, paths))
|
|
558
557
|
|
|
559
|
-
def
|
|
558
|
+
def _add_context(self, source: str) -> dict[str, Any] | str:
|
|
560
559
|
"""Pack an external dir/file/git-URL on the host and append it into `context`.
|
|
561
560
|
|
|
562
|
-
Paths are namespaced under
|
|
561
|
+
Paths are namespaced under ctx/<source_id>/ (host). Content is always in the
|
|
563
562
|
single `context` list — never a new context_N variable.
|
|
564
563
|
Host-side idempotency may return already_loaded without a payload path.
|
|
564
|
+
Documents (PDF/DOCX/…) are converted to Markdown on the host.
|
|
565
565
|
"""
|
|
566
|
-
r = self._rpc("
|
|
566
|
+
r = self._rpc("add_context", {"source": str(source)})
|
|
567
567
|
if r.get("error"):
|
|
568
568
|
return f"Error: {r['error']}"
|
|
569
569
|
if r.get("already_loaded"):
|
|
570
|
-
source_id = r.get("source_id") if isinstance(r.get("source_id"), str) else "
|
|
571
|
-
path_prefix = r.get("path_prefix") if isinstance(r.get("path_prefix"), str) else f"
|
|
570
|
+
source_id = r.get("source_id") if isinstance(r.get("source_id"), str) else "ctx"
|
|
571
|
+
path_prefix = r.get("path_prefix") if isinstance(r.get("path_prefix"), str) else f"ctx/{source_id}/"
|
|
572
572
|
ctx = self.ns.get("context")
|
|
573
573
|
ctx_len = len(ctx) if isinstance(ctx, list) else 0
|
|
574
574
|
print(
|
|
575
|
-
f"[rlm]
|
|
575
|
+
f"[rlm] add_context: already loaded {source_id} "
|
|
576
576
|
f"(paths under {path_prefix}, context len={ctx_len})"
|
|
577
577
|
)
|
|
578
578
|
return {
|
|
@@ -583,10 +583,13 @@ class Worker:
|
|
|
583
583
|
"chars": r.get("chars"),
|
|
584
584
|
"context_len": ctx_len,
|
|
585
585
|
"already_loaded": True,
|
|
586
|
+
"documents": 0,
|
|
587
|
+
"converted": 0,
|
|
588
|
+
"skipped": [],
|
|
586
589
|
}
|
|
587
590
|
path = r.get("path")
|
|
588
591
|
if not isinstance(path, str):
|
|
589
|
-
return "Error: malformed
|
|
592
|
+
return "Error: malformed add_context reply (no path)"
|
|
590
593
|
try:
|
|
591
594
|
with io.open(path, "r") as f:
|
|
592
595
|
payload = json.load(f) if r.get("json") else f.read()
|
|
@@ -595,57 +598,71 @@ class Worker:
|
|
|
595
598
|
os.remove(path) # worker owns temp-file cleanup (host does NOT unlink)
|
|
596
599
|
except OSError:
|
|
597
600
|
pass
|
|
598
|
-
return self.
|
|
601
|
+
return self._append_context(str(source), payload, r)
|
|
599
602
|
|
|
600
|
-
def
|
|
601
|
-
"""Append host-packed
|
|
603
|
+
def _append_context(self, source: str, payload: Any, meta: dict[str, Any]) -> dict[str, Any] | str:
|
|
604
|
+
"""Append host-packed files into `context` (idempotent by path prefix).
|
|
602
605
|
|
|
603
606
|
The two refusals below are pre-flighted host-side by LIST_CONTEXT_REQUIRED /
|
|
604
|
-
NO_FILES_PRODUCED in src/bridge/
|
|
607
|
+
NO_FILES_PRODUCED in src/bridge/add-context.ts, so the host never commits a
|
|
605
608
|
loaded-prefix for an append that fails here. Reaching either one means host and worker
|
|
606
609
|
disagree about `context`; keep the wording identical to its twin.
|
|
607
610
|
"""
|
|
608
611
|
ctx = self.ns.get("context")
|
|
609
612
|
if not isinstance(ctx, list):
|
|
610
613
|
kind = type(ctx).__name__ if ctx is not None else "None"
|
|
611
|
-
return f"Error:
|
|
614
|
+
return f"Error: add_context requires list context (file bundle); got {kind}"
|
|
612
615
|
|
|
613
616
|
source_id = meta.get("source_id")
|
|
614
617
|
if not isinstance(source_id, str) or not source_id:
|
|
615
|
-
source_id = "
|
|
618
|
+
source_id = "ctx"
|
|
616
619
|
path_prefix = meta.get("path_prefix")
|
|
617
|
-
if not isinstance(path_prefix, str)
|
|
618
|
-
path_prefix = f"
|
|
619
|
-
|
|
620
|
-
#
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
)
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
620
|
+
if not isinstance(path_prefix, str):
|
|
621
|
+
path_prefix = f"ctx/{source_id}/"
|
|
622
|
+
# Empty path_prefix is valid (cwd seed) but add_context always sends a non-empty ctx/ prefix.
|
|
623
|
+
# Guard startsWith on empty prefix: "anything".startswith("") is always True.
|
|
624
|
+
check_prefix = path_prefix if path_prefix != "" else None
|
|
625
|
+
|
|
626
|
+
# Idempotent: already present if any path uses this prefix.
|
|
627
|
+
if check_prefix is not None:
|
|
628
|
+
for item in ctx:
|
|
629
|
+
if isinstance(item, dict) and str(item.get("path", "")).startswith(check_prefix):
|
|
630
|
+
print(
|
|
631
|
+
f"[rlm] add_context: already loaded {source_id} "
|
|
632
|
+
f"(paths under {path_prefix}, context len={len(ctx)})"
|
|
633
|
+
)
|
|
634
|
+
return {
|
|
635
|
+
"source": source,
|
|
636
|
+
"source_id": source_id,
|
|
637
|
+
"path_prefix": path_prefix,
|
|
638
|
+
"files": 0,
|
|
639
|
+
"chars": meta.get("chars"),
|
|
640
|
+
"context_len": len(ctx),
|
|
641
|
+
"already_loaded": True,
|
|
642
|
+
"documents": 0,
|
|
643
|
+
"converted": 0,
|
|
644
|
+
"skipped": [],
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
files = self._context_file_entries(payload, path_prefix)
|
|
638
648
|
if not files:
|
|
639
|
-
return "Error:
|
|
649
|
+
return "Error: add_context produced no files"
|
|
640
650
|
|
|
641
651
|
ctx.extend(files)
|
|
642
652
|
# Keep restore payload in sync with the live list.
|
|
643
653
|
self._context_payload = ctx
|
|
644
654
|
self.ns["context"] = ctx
|
|
645
655
|
|
|
656
|
+
documents = meta.get("documents") if isinstance(meta.get("documents"), int) else 0
|
|
657
|
+
converted = meta.get("converted") if isinstance(meta.get("converted"), int) else 0
|
|
658
|
+
skipped = meta.get("skipped") if isinstance(meta.get("skipped"), list) else []
|
|
659
|
+
skip_n = len(skipped)
|
|
660
|
+
extra = ""
|
|
661
|
+
if documents or converted or skip_n:
|
|
662
|
+
extra = f"; documents={documents}, converted={converted}, skipped={skip_n}"
|
|
646
663
|
print(
|
|
647
|
-
f"[rlm]
|
|
648
|
-
f"(len={len(ctx)}); paths under {path_prefix}"
|
|
664
|
+
f"[rlm] add_context: +{len(files)} files into context "
|
|
665
|
+
f"(len={len(ctx)}); paths under {path_prefix}{extra}"
|
|
649
666
|
)
|
|
650
667
|
return {
|
|
651
668
|
"source": source,
|
|
@@ -655,14 +672,17 @@ class Worker:
|
|
|
655
672
|
"chars": meta.get("chars"),
|
|
656
673
|
"context_len": len(ctx),
|
|
657
674
|
"already_loaded": False,
|
|
675
|
+
"documents": documents,
|
|
676
|
+
"converted": converted,
|
|
677
|
+
"skipped": skipped,
|
|
658
678
|
}
|
|
659
679
|
|
|
660
680
|
@staticmethod
|
|
661
|
-
def
|
|
681
|
+
def _context_file_entries(payload: Any, path_prefix: str) -> list[dict[str, Any]]:
|
|
662
682
|
"""Normalize host payload to list[dict]. Host already namespaces; string is fallback."""
|
|
663
683
|
if isinstance(payload, str):
|
|
664
684
|
return [{
|
|
665
|
-
"path": f"{path_prefix}content",
|
|
685
|
+
"path": f"{path_prefix}content" if path_prefix else "content",
|
|
666
686
|
"content": payload,
|
|
667
687
|
"tokens": max(1, (len(payload) + 3) // 4),
|
|
668
688
|
}]
|
|
@@ -684,7 +704,7 @@ class Worker:
|
|
|
684
704
|
"""Load the packed world into the single REPL variable `context`.
|
|
685
705
|
|
|
686
706
|
`index` is accepted for protocol compatibility but ignored — there is only
|
|
687
|
-
one context slot.
|
|
707
|
+
one context slot. Sources are merged on the host (or via add_context).
|
|
688
708
|
"""
|
|
689
709
|
with open(path, "r") as f:
|
|
690
710
|
payload = json.load(f) if is_json else f.read()
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
|
|
7
7
|
import { PythonSandbox, type SubLlmHandlers } from "./sandbox.ts";
|
|
8
8
|
import type { ReplResult } from "./protocol.ts";
|
|
9
|
-
import {
|
|
9
|
+
import { mergeIntoContext } from "../context/merge.ts";
|
|
10
10
|
|
|
11
11
|
/** Static configuration for sandbox creation — set once, reused across getOrCreate calls. */
|
|
12
12
|
export interface SandboxManagerConfig {
|
|
@@ -28,23 +28,26 @@ export class SandboxManager {
|
|
|
28
28
|
/** Serialized execution queue — concurrent repl() calls wait for predecessor. */
|
|
29
29
|
private execQueue: Promise<void> = Promise.resolve();
|
|
30
30
|
private pendingExecCount = 0;
|
|
31
|
-
/**
|
|
32
|
-
|
|
31
|
+
/**
|
|
32
|
+
* Context payload to load on first sandbox creation. Starts as an empty list — context is
|
|
33
|
+
* empty by default; the first repl() seeds the cwd when autoSeedCwd is on.
|
|
34
|
+
*/
|
|
35
|
+
contextPayload: unknown = [];
|
|
33
36
|
/** True once contextPayload has been loaded into the sandbox (prevents reload + race fix). */
|
|
34
37
|
private contextLoaded = false;
|
|
35
38
|
|
|
36
39
|
constructor(private readonly config: SandboxManagerConfig) {}
|
|
37
40
|
|
|
38
41
|
/**
|
|
39
|
-
* Append a
|
|
42
|
+
* Append a source payload to the context this manager replays on death-recreate.
|
|
40
43
|
*
|
|
41
|
-
* The live worker has ALREADY appended it in-process (worker.py `
|
|
44
|
+
* The live worker has ALREADY appended it in-process (worker.py `_append_context`), so this
|
|
42
45
|
* deliberately does not reload — it only keeps the host's replay copy truthful. Without it a
|
|
43
|
-
* recreate silently rolls the sandbox back to a
|
|
44
|
-
* this payload would never see the
|
|
46
|
+
* recreate silently rolls the sandbox back to a pre-append context, and any child inheriting
|
|
47
|
+
* this payload would never see the source. Dedups by `ctx/<id>/` prefix.
|
|
45
48
|
*/
|
|
46
|
-
|
|
47
|
-
this.contextPayload =
|
|
49
|
+
appendContext(payload: unknown): void {
|
|
50
|
+
this.contextPayload = mergeIntoContext(this.contextPayload, payload);
|
|
48
51
|
}
|
|
49
52
|
|
|
50
53
|
/**
|
|
@@ -52,15 +55,14 @@ export class SandboxManager {
|
|
|
52
55
|
* given handlers. Subsequent calls return the existing sandbox immediately.
|
|
53
56
|
* Deduplicates concurrent calls via initPromise.
|
|
54
57
|
*
|
|
55
|
-
* If contextPayload is
|
|
58
|
+
* If contextPayload is defined, it is loaded before the sandbox is returned.
|
|
56
59
|
*/
|
|
57
60
|
async getOrCreate(handlers: Partial<SubLlmHandlers>): Promise<PythonSandbox> {
|
|
58
61
|
if (this.disposed) throw new Error("SandboxManager disposed");
|
|
59
62
|
if (this.sandbox) {
|
|
60
|
-
// RACE FIX: contextPayload may arrive after the sandbox was created (
|
|
61
|
-
//
|
|
62
|
-
|
|
63
|
-
if (this.contextPayload !== null && !this.contextLoaded) {
|
|
63
|
+
// RACE FIX: contextPayload may arrive after the sandbox was created (lazy seed
|
|
64
|
+
// resolves after the first getOrCreate). Load it into the live sandbox now if pending.
|
|
65
|
+
if (this.contextPayload !== undefined && !this.contextLoaded) {
|
|
64
66
|
await this.sandbox.loadContext(this.contextPayload);
|
|
65
67
|
this.contextLoaded = true;
|
|
66
68
|
}
|
|
@@ -79,8 +81,8 @@ export class SandboxManager {
|
|
|
79
81
|
awaitTimeoutS: this.config.awaitTimeoutS,
|
|
80
82
|
handlers,
|
|
81
83
|
}).then(async (s) => {
|
|
82
|
-
// Load context on first creation if available.
|
|
83
|
-
if (this.contextPayload !==
|
|
84
|
+
// Load context on first creation if available (empty list is a valid starting value).
|
|
85
|
+
if (this.contextPayload !== undefined) {
|
|
84
86
|
await s.loadContext(this.contextPayload);
|
|
85
87
|
this.contextLoaded = true;
|
|
86
88
|
}
|
package/src/sandbox/sandbox.ts
CHANGED
|
@@ -24,7 +24,7 @@ import { pinContext, type PinnedContext } from "./context-file.ts";
|
|
|
24
24
|
import { REJECT, serviceInterrupt, type ReplyBody, type SubLlmHandlers } from "./interrupts.ts";
|
|
25
25
|
import { trace, traceEnabled } from "../util/trace.ts";
|
|
26
26
|
|
|
27
|
-
export type {
|
|
27
|
+
export type { AddContextResult, SubcallOpts, SubLlmHandlers } from "./interrupts.ts";
|
|
28
28
|
|
|
29
29
|
export interface SandboxOptions {
|
|
30
30
|
/** Sandbox recursion depth label (passed to the worker, used in interrupt routing). */
|
package/src/text/tokens.ts
CHANGED
|
@@ -25,7 +25,7 @@ export function estimateMessageTokens(messages: { content: string }[]): number {
|
|
|
25
25
|
* Character length of one context entry. `ContextFile`-shaped entries report their content
|
|
26
26
|
* length; anything else falls back to its serialized form.
|
|
27
27
|
*
|
|
28
|
-
* Deliberately does NOT import `isContextFile` from context/
|
|
28
|
+
* Deliberately does NOT import `isContextFile` from context/namespace.ts: that module
|
|
29
29
|
* imports `estimateTokens` from here, so the reverse import would be a cycle. `in`-narrowing
|
|
30
30
|
* needs no type guard and no cast.
|
|
31
31
|
*/
|
|
@@ -74,8 +74,8 @@ const isTokenizedEntry = (v: unknown): v is { readonly tokens: number } =>
|
|
|
74
74
|
typeof v === "object" && v !== null && typeof (v as { readonly tokens?: unknown }).tokens === "number";
|
|
75
75
|
|
|
76
76
|
/** Per-file token distribution for a context payload; `undefined` for plain strings or empty arrays.
|
|
77
|
-
* Handles both
|
|
78
|
-
*
|
|
77
|
+
* Handles both a flat ContextFile[] and a raw bundle object ({ files: [...] }) so callers
|
|
78
|
+
* don't need to know which form they received. */
|
|
79
79
|
export function contextSizeStats(context: unknown): ContextSizeStats | undefined {
|
|
80
80
|
// Normalise to a flat entry list: accept either a direct array or an object with a .files array.
|
|
81
81
|
const entries: readonly unknown[] = Array.isArray(context)
|
package/src/tool/repl-details.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* ReplDetails — structured payload for the repl() tool's AgentToolResult<T>.
|
|
3
3
|
*
|
|
4
4
|
* Mirrors RlmDetails but scoped to a single code execution. Sub-calls (llm_query,
|
|
5
|
-
* rlm_query,
|
|
5
|
+
* rlm_query, add_context) triggered during sandbox execution are
|
|
6
6
|
* accumulated into the subcalls array for tree rendering.
|
|
7
7
|
*/
|
|
8
8
|
|
package/src/tool/repl-tool.ts
CHANGED
|
@@ -20,8 +20,8 @@ import type { ToolDefinition } from "@earendil-works/pi-coding-agent";
|
|
|
20
20
|
import { Text } from "@earendil-works/pi-tui";
|
|
21
21
|
import type { Model, Usage, Api } from "@earendil-works/pi-ai";
|
|
22
22
|
import type { ModelRegistry } from "@earendil-works/pi-coding-agent";
|
|
23
|
-
import {
|
|
24
|
-
import {
|
|
23
|
+
import { buildAddContextHandler, type AddContextHandlerBundle } from "../bridge/add-context.ts";
|
|
24
|
+
import { contextPrefixesIn } from "../context/namespace.ts";
|
|
25
25
|
import type { SubcallGates } from "../util/concurrency.ts";
|
|
26
26
|
import { LimitGuard, limitsFromConfig } from "../core/limits.ts";
|
|
27
27
|
import type { RlmConfig, RlmInput, RlmResult } from "../core/types.ts";
|
|
@@ -115,8 +115,13 @@ export interface ReplToolDeps {
|
|
|
115
115
|
readonly signal?: AbortSignal;
|
|
116
116
|
readonly onUsage?: (usage: Usage, role: "sub") => void;
|
|
117
117
|
readonly ensureContext?: () => Promise<void>;
|
|
118
|
-
/** Register a reset hook for sandbox death/dispose (e.g.
|
|
118
|
+
/** Register a reset hook for sandbox death/dispose (e.g. add_context prefix cache). */
|
|
119
119
|
readonly registerDiscardHook?: (reset: () => void) => void;
|
|
120
|
+
/**
|
|
121
|
+
* Hands the live add_context bundle to the extension so the cwd seed can
|
|
122
|
+
* markLoaded("") / markSeededCwd(abs) — without this, add_context(".") doubles the tree.
|
|
123
|
+
*/
|
|
124
|
+
readonly registerContextBundle?: (bundle: AddContextHandlerBundle) => void;
|
|
120
125
|
}
|
|
121
126
|
|
|
122
127
|
export function createReplTool(deps: ReplToolDeps): ToolDefinition<typeof ReplToolParams, ReplDetails> {
|
|
@@ -159,15 +164,15 @@ export function createReplTool(deps: ReplToolDeps): ToolDefinition<typeof ReplTo
|
|
|
159
164
|
signal,
|
|
160
165
|
onUsage,
|
|
161
166
|
runChild,
|
|
162
|
-
// The session sandbox's context is the child's world. Read lazily so
|
|
167
|
+
// The session sandbox's context is the child's world. Read lazily so an add_context from an
|
|
163
168
|
// earlier repl() reaches a child spawned in a later one. Populated before any interrupt can
|
|
164
169
|
// fire: execute() awaits ensureContext() before getOrCreate().
|
|
165
170
|
getChildContext: () => sandboxManager.contextPayload ?? undefined,
|
|
166
171
|
trackDetached: (task) => background.track(task),
|
|
167
172
|
});
|
|
168
173
|
|
|
169
|
-
const
|
|
170
|
-
?
|
|
174
|
+
const contextBundle = getConfig().contextLoader
|
|
175
|
+
? buildAddContextHandler({
|
|
171
176
|
getCwd: () => sessionCwd,
|
|
172
177
|
getEmitter: () => bridgeState.currentEmitter,
|
|
173
178
|
// Refuse pre-flight whatever the worker would reject, so host idempotency is never
|
|
@@ -177,14 +182,20 @@ export function createReplTool(deps: ReplToolDeps): ToolDefinition<typeof ReplTo
|
|
|
177
182
|
signal,
|
|
178
183
|
// Keep the manager's replay copy in step with the worker's live `context`, and with it
|
|
179
184
|
// whatever a child spawned after this load will inherit.
|
|
180
|
-
onLoaded: (payload) => { sandboxManager.
|
|
185
|
+
onLoaded: (payload) => { sandboxManager.appendContext(payload); },
|
|
181
186
|
})
|
|
182
187
|
: undefined;
|
|
183
|
-
if (
|
|
188
|
+
if (contextBundle) {
|
|
184
189
|
// Re-derive the loaded-prefix cache from the payload that will actually be replayed —
|
|
185
|
-
// clearing it outright would make the host re-clone a
|
|
186
|
-
|
|
187
|
-
|
|
190
|
+
// clearing it outright would make the host re-clone a source the recreated worker already has.
|
|
191
|
+
// Re-plant the cwd sentinel if the seed is still in the payload (un-prefixed files).
|
|
192
|
+
const bundle = contextBundle;
|
|
193
|
+
deps.registerDiscardHook?.(() => {
|
|
194
|
+
const prefixes = contextPrefixesIn(sandboxManager.contextPayload);
|
|
195
|
+
bundle.reset(prefixes);
|
|
196
|
+
if (bundle.seededCwd() !== undefined) bundle.markLoaded("");
|
|
197
|
+
});
|
|
198
|
+
deps.registerContextBundle?.(contextBundle);
|
|
188
199
|
}
|
|
189
200
|
|
|
190
201
|
return {
|
|
@@ -192,14 +203,15 @@ export function createReplTool(deps: ReplToolDeps): ToolDefinition<typeof ReplTo
|
|
|
192
203
|
label: "REPL",
|
|
193
204
|
description:
|
|
194
205
|
"PRIMARY tool for ALL repository reading and analysis (read/grep are disabled in RLM mode). " +
|
|
195
|
-
"Persistent Python sandbox with
|
|
196
|
-
"free primitives search(query) / grep_context(pattern) /
|
|
197
|
-
"semantic reading to map_files / llm_query /
|
|
198
|
-
"(rlm_query for iterative sub-tasks) — stdout
|
|
199
|
-
"so printing file bodies is useless.
|
|
200
|
-
"persist across calls. Also supports
|
|
206
|
+
"Persistent Python sandbox with loaded files in `context` (starts empty; cwd seeds on first " +
|
|
207
|
+
"call). Locate first with the free primitives search(query) / grep_context(pattern) / " +
|
|
208
|
+
"outline(path), then delegate the semantic reading to map_files / llm_query / " +
|
|
209
|
+
"llm_query_batched / llm_query_chunked (rlm_query for iterative sub-tasks) — stdout " +
|
|
210
|
+
"returned to you is hard-capped at 4K chars, so printing file bodies is useless. " +
|
|
211
|
+
"Variables, imports, and the `answers`/`plan` memo persist across calls. Also supports " +
|
|
212
|
+
"add_context for external dirs/files/git URLs and document conversion.",
|
|
201
213
|
promptSnippet:
|
|
202
|
-
"repl: run Python in a persistent sandbox holding
|
|
214
|
+
"repl: run Python in a persistent sandbox holding loaded files in `context`; " +
|
|
203
215
|
"search/grep_context/outline to locate, map_files/llm_query* to read.",
|
|
204
216
|
promptGuidelines: [
|
|
205
217
|
"In RLM mode, read the repository through `repl` only — `read`/`grep` and bash readers are blocked.",
|
|
@@ -271,7 +283,7 @@ export function createReplTool(deps: ReplToolDeps): ToolDefinition<typeof ReplTo
|
|
|
271
283
|
await deps.ensureContext?.();
|
|
272
284
|
await sandboxManager.getOrCreate({
|
|
273
285
|
...subcallHandlers,
|
|
274
|
-
...(
|
|
286
|
+
...(contextBundle?.handlers ?? {}),
|
|
275
287
|
});
|
|
276
288
|
|
|
277
289
|
// Detect queue contention AFTER sandbox init (initPromise settled, isExecuting now accurate)
|
package/src/tool/rlm-tool.ts
CHANGED
|
@@ -31,7 +31,7 @@ const CALL_PREVIEW_CHARS = 80;
|
|
|
31
31
|
|
|
32
32
|
export const RlmToolParams = Object.freeze(Type.Object({
|
|
33
33
|
prompt: Type.String({ description: "The task or question for the RLM engine" }),
|
|
34
|
-
context: Type.Optional(Type.String({ description: "Optional context. If omitted,
|
|
34
|
+
context: Type.Optional(Type.String({ description: "Optional context. If omitted, the working directory is packed into context." })),
|
|
35
35
|
}));
|
|
36
36
|
|
|
37
37
|
// ── Rendering helpers ──
|
package/src/ui/config-panel.ts
CHANGED
|
@@ -20,7 +20,8 @@ const CHOICES = Object.freeze({
|
|
|
20
20
|
rootSamplingMaxTokens: Object.freeze(["4096", "8192", "16384", "32768"]),
|
|
21
21
|
sandboxInitTimeoutMs: Object.freeze(["10000", "30000", "60000", "120000"]),
|
|
22
22
|
requestTimeoutMs: Object.freeze(["2", "5", "10", "20"]),
|
|
23
|
-
|
|
23
|
+
contextLoader: Object.freeze(["on", "off"]),
|
|
24
|
+
autoSeedCwd: Object.freeze(["on", "off"]),
|
|
24
25
|
});
|
|
25
26
|
|
|
26
27
|
function item(id: string, label: string, currentValue: string, values: readonly string[], description: string): SettingItem {
|
|
@@ -49,8 +50,10 @@ export async function showConfigPanel(ctx: ExtensionContext, config: RlmConfig):
|
|
|
49
50
|
item("rootSamplingMaxTokens", "Root model output cap (tok)", String(config.rootSampling?.maxTokens ?? 16384), CHOICES.rootSamplingMaxTokens, "Max output tokens per root-model turn. Lower values keep each turn lean."),
|
|
50
51
|
item("sandboxInitTimeoutMs", "Sandbox init timeout", String(config.sandboxInitTimeoutMs), CHOICES.sandboxInitTimeoutMs, "How long to wait for the Python worker to start."),
|
|
51
52
|
item("requestTimeoutMs", "Sandbox request timeout (min)", String(Math.round(config.requestTimeoutMs / 60_000)), CHOICES.requestTimeoutMs, "Parent-side watchdog per sandbox request; on breach the Python worker is killed."),
|
|
52
|
-
item("
|
|
53
|
-
"Allow
|
|
53
|
+
item("contextLoader", "Context loader", config.contextLoader ? "on" : "off", CHOICES.contextLoader,
|
|
54
|
+
"Allow add_context() to pull an external dir, file, document, or git repo into context."),
|
|
55
|
+
item("autoSeedCwd", "Auto-seed cwd", config.autoSeedCwd ? "on" : "off", CHOICES.autoSeedCwd,
|
|
56
|
+
"Seed the working directory into context on the first repl() call (otherwise starts empty)."),
|
|
54
57
|
item("__save__", "Save & close", "↵", ["↵"], "Save these settings and close (Esc also saves)."),
|
|
55
58
|
];
|
|
56
59
|
|
|
@@ -104,7 +107,8 @@ export function applySetting(config: RlmConfig, id: string, value: string): RlmC
|
|
|
104
107
|
return Object.freeze({ ...config, rootSampling: Object.freeze({ ...config.rootSampling, maxTokens: Number(value) }) });
|
|
105
108
|
case "sandboxInitTimeoutMs": return Object.freeze({ ...config, sandboxInitTimeoutMs: Number(value) });
|
|
106
109
|
case "requestTimeoutMs": return Object.freeze({ ...config, requestTimeoutMs: Number(value) * 60_000 });
|
|
107
|
-
case "
|
|
110
|
+
case "contextLoader": return Object.freeze({ ...config, contextLoader: value === "on" });
|
|
111
|
+
case "autoSeedCwd": return Object.freeze({ ...config, autoSeedCwd: value === "on" });
|
|
108
112
|
default: return config;
|
|
109
113
|
}
|
|
110
114
|
}
|