@hicaru/pi-rlm 0.3.16 → 0.3.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -4
- package/README.ru.md +56 -66
- package/README.zh-CN.md +61 -65
- package/package.json +5 -5
- package/src/bridge/add-context.ts +1 -1
- package/src/bridge/handlers/await.ts +13 -22
- package/src/bridge/handlers/completion.ts +27 -5
- package/src/bridge/handlers/emitting.ts +2 -2
- package/src/bridge/handlers/llm-query.ts +46 -68
- package/src/bridge/handlers/rlm-query.ts +14 -84
- package/src/bridge/handlers/task-registry.ts +22 -17
- package/src/bridge/handlers/types.ts +8 -6
- package/src/bridge/model.ts +6 -3
- package/src/commands/rlm-llm.ts +1 -10
- package/src/commands/rlm-rlm.ts +1 -8
- package/src/config/defaults.ts +32 -12
- package/src/config/settings.ts +53 -31
- package/src/config/skillstate.ts +465 -0
- package/src/context/md-cache.ts +1 -1
- package/src/context/merge.ts +1 -1
- package/src/context/namespace.ts +2 -2
- package/src/context/refresh.ts +1 -1
- package/src/context/source-dir.ts +21 -11
- package/src/context/source-doc.ts +1 -1
- package/src/context/source-git.ts +3 -15
- package/src/context/source-text.ts +1 -1
- package/src/context/walk.ts +6 -14
- package/src/core/budget.ts +107 -21
- package/src/core/compaction.ts +44 -1
- package/src/core/engine.ts +141 -84
- package/src/core/iteration.ts +1 -1
- package/src/core/ledger.ts +10 -13
- package/src/core/limits.ts +1 -1
- package/src/core/model-registry.ts +1 -1
- package/src/core/resource-limits.ts +1 -1
- package/src/core/root-context.ts +184 -0
- package/src/core/root-digest.ts +213 -0
- package/src/core/root-state.ts +310 -0
- package/src/core/run-state.ts +587 -0
- package/src/core/types.ts +51 -12
- package/src/index.ts +220 -39
- package/src/mode/llm-model.ts +13 -1
- package/src/mode/native-guards.ts +0 -6
- package/src/mode/rlm-mode.ts +34 -11
- package/src/mode/subagent.ts +5 -5
- package/src/prompts/glossary.ts +46 -25
- package/src/prompts/native.ts +27 -5
- package/src/prompts/system.ts +12 -4
- package/src/sandbox/context-file.ts +1 -1
- package/src/sandbox/interrupts.ts +25 -31
- package/src/sandbox/protocol.ts +14 -20
- package/src/sandbox/py/__pycache__/guards.cpython-314.pyc +0 -0
- package/src/sandbox/py/__pycache__/scaffold.cpython-314.pyc +0 -0
- package/src/sandbox/py/__pycache__/worker.cpython-314.pyc +0 -0
- package/src/sandbox/py/guards.py +1 -1
- package/src/sandbox/py/scaffold.py +24 -31
- package/src/sandbox/py/worker.py +3 -1
- package/src/sandbox/sandbox-manager.ts +2 -2
- package/src/sandbox/sandbox.ts +21 -4
- package/src/text/agent-text.ts +58 -0
- package/src/text/parsing.ts +35 -3
- package/src/text/preview.ts +3 -0
- package/src/text/repl-output.ts +1 -1
- package/src/tool/background-tasks.ts +1 -1
- package/src/tool/repl-render.ts +1 -1
- package/src/tool/repl-result.ts +1 -1
- package/src/tool/repl-tool.ts +50 -26
- package/src/tool/rlm-tool.ts +4 -5
- package/src/tool/subcall-render.ts +1 -1
- package/src/tool/subcall-store.ts +2 -2
- package/src/tool/tool-utils.ts +5 -5
- package/src/ui/config-panel.ts +12 -0
- package/src/ui/intro.ts +1 -1
- package/src/ui/modal/timeline-store.ts +1 -1
- package/src/ui/model-picker/drilldown.ts +1 -1
- package/src/ui/model-picker/levels.ts +1 -1
- package/src/ui/panel/run-registry.ts +1 -1
- package/src/ui/status.ts +39 -4
- package/src/ui/tree/tree-rows.ts +1 -1
- package/src/ui/tree/tree-widget.ts +1 -1
- package/src/util/bm25.ts +97 -0
- package/src/util/concurrency.ts +1 -1
- package/src/util/errors.ts +1 -1
- package/src/util/retry.ts +22 -7
- package/src/util/state-merge.ts +34 -0
- package/src/util/throttle.ts +1 -1
- package/src/util/type-guards.ts +6 -0
- package/src/core/memory.ts +0 -589
package/README.md
CHANGED
|
@@ -43,10 +43,6 @@ models, recursively. Same Pi session, same tools, same keys: `/rlm` and go. Read
|
|
|
43
43
|
|
|
44
44
|
## Benchmarks
|
|
45
45
|
|
|
46
|
-
<p align="center">
|
|
47
|
-
<img src="https://github.com/openzebra/rlm.pi/blob/master/assets/hero.png?raw=true" width="100%" alt="OOLONG benchmark — latest results">
|
|
48
|
-
</p>
|
|
49
|
-
|
|
50
46
|
**OOLONG (oolong-synth)** — paper-tier long-context suite; latest journal per model,
|
|
51
47
|
cost per task from real `costUsd` (older journals estimated at OpenRouter list prices):
|
|
52
48
|
|
package/README.ru.md
CHANGED
|
@@ -1,11 +1,5 @@
|
|
|
1
1
|
<div align="center">
|
|
2
2
|
|
|
3
|
-
<img src="https://github.com/openzebra/rlm.pi/blob/master/assets/hero.png?raw=true" alt="pi-rlm">
|
|
4
|
-
|
|
5
|
-
</div>
|
|
6
|
-
|
|
7
|
-
<div align="center">
|
|
8
|
-
|
|
9
3
|
<sub>
|
|
10
4
|
<a href="README.md">English</a> · <a href="README.zh-CN.md">中文</a> · **Русский**
|
|
11
5
|
</sub>
|
|
@@ -14,33 +8,68 @@
|
|
|
14
8
|
|
|
15
9
|
---
|
|
16
10
|
|
|
17
|
-
# pi
|
|
11
|
+
# rlm.pi PI plugin
|
|
18
12
|
|
|
19
|
-
|
|
13
|
+
> pi-rlm — большие контексты на дешёвых моделях: рекурсивная языковая модель (RLM) для Pi
|
|
20
14
|
|
|
21
|
-
|
|
22
|
-
ПОЛНОСТЬЮ ЛОКАЛЬНО.
|
|
15
|
+
## Установка
|
|
23
16
|
|
|
24
|
-
|
|
17
|
+
```bash
|
|
18
|
+
pi install npm:@hicaru/pi-rlm
|
|
19
|
+
```
|
|
25
20
|
|
|
26
|
-
|
|
21
|
+
Чтобы удалить позже:
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
pi uninstall npm:@hicaru/pi-rlm
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
Затем выполните `/reload` или перезапустите Pi — `/rlm`, `/rlm-config` и `/rlm-stop` появятся в разделе **[Extensions]**. Переключение — `Ctrl+Shift+R` или `/rlm`.
|
|
28
|
+
|
|
29
|
+
<p align="center">
|
|
30
|
+
<img src="https://github.com/openzebra/rlm.pi/blob/master/assets/hero.png?raw=true" width="100%" alt="rlm.pi — результаты OOLONG">
|
|
31
|
+
</p>
|
|
27
32
|
|
|
28
|
-
|
|
33
|
+
## Что такое pi-rlm?
|
|
29
34
|
|
|
30
|
-
|
|
35
|
+
Плагин, который превращает сессию Pi в **рекурсивную языковую модель (RLM)**: вместо того чтобы заталкивать огромный документ в промпт, контекст живёт в Python REPL, а ваша лучшая модель им управляет — ищет, декомпозирует и делегирует чтения дешёвым worker-моделям, рекурсивно. Та же сессия Pi, те же инструменты, те же ключи — включите `/rlm` и вперёд. По методу [RLM paper](https://arxiv.org/abs/2512.24601); подробности — в разделе **Как это работает**.
|
|
31
36
|
|
|
32
|
-
|
|
37
|
+
## Бенчмарки
|
|
38
|
+
|
|
39
|
+
**OOLONG (oolong-synth)** — paper-tier сюит длинного контекста; последний журнал каждой модели, цена за задачу из реального `costUsd` (старые журналы оценены по прайс-листу OpenRouter):
|
|
40
|
+
|
|
41
|
+
| Модель | Счёт | Цена/задача |
|
|
42
|
+
|--------|------|-------------|
|
|
43
|
+
| `qwen/qwen3.8-27b` | **100%** | $0.0127 |
|
|
44
|
+
| `google/gemma-3-27b-it` | 83.3% | $0.0013 |
|
|
45
|
+
| `qwen/qwen3-30b-a3b-instruct-2507` | 66.7% | $0.0009 |
|
|
46
|
+
| `mistralai/mistral-small-3.2-24b-instruct` | 66.7% | $0.0025 |
|
|
47
|
+
|
|
48
|
+
Lite-сьют — `needle` (поиск игл в куче), `codeqa` (вопросы по коду), `coding` (задача-фикс; 7 задач × 2 прохода на модель, детерминированные грейдеры, без LLM-судьи):
|
|
33
49
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
50
|
+
| Модель | Счёт | Точность |
|
|
51
|
+
|--------|------|----------|
|
|
52
|
+
| `qwen/qwen3-30b-a3b-instruct-2507` | **14/14** | **100%** |
|
|
53
|
+
| `google/gemma-3-27b-it` | 12/14 | 86% |
|
|
54
|
+
| `mistralai/mistral-small-3.2-24b-instruct` | 12/14 | 86% |
|
|
55
|
+
|
|
56
|
+
Построчные результаты (correct, recall, latency, токены, цена) — в
|
|
57
|
+
`bench/runs/*.jsonl`, одна JSONL-строка на задачу.
|
|
58
|
+
|
|
59
|
+
### Как запустить
|
|
41
60
|
|
|
42
|
-
|
|
43
|
-
|
|
61
|
+
```bash
|
|
62
|
+
export OPENROUTER_API_KEY=sk-or-... # обязателен — ключи ходят только через env
|
|
63
|
+
|
|
64
|
+
bun run bench # lite-сьют: needle + codeqa + coding
|
|
65
|
+
bun run bench --suite needle --limit 1 # один сьют, первая задача
|
|
66
|
+
bun run bench --model openrouter/qwen/qwen3-30b-a3b-instruct-2507
|
|
67
|
+
bun run bench --list # показать задачи, без движка и ключа
|
|
68
|
+
bun run bench --suite paper # paper-сьют: s_niah, oolong, browsecomp, codeqa_lb (скачивает датасеты)
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Сьюты: `all` (lite, по умолчанию) · `needle` · `codeqa` · `coding` · `paper` · `s_niah` ·
|
|
72
|
+
`oolong` · `browsecomp` · `codeqa_lb`.
|
|
44
73
|
|
|
45
74
|
## Как это работает
|
|
46
75
|
|
|
@@ -60,24 +89,17 @@ pi process (TypeScript)
|
|
|
60
89
|
`rlm_query_batched`, `SHOW_VARS()`, `ask_user_question()` и словарь `answer`.
|
|
61
90
|
Модель отправляет окончательный результат, устанавливая `answer["ready"] = True`.
|
|
62
91
|
|
|
63
|
-
## Установка
|
|
92
|
+
## Установка из исходников (для разработки)
|
|
64
93
|
|
|
65
94
|
`pi-rlm` — это пакет Pi. Pi предоставляет peer-зависимости `@earendil-works/pi-*` и `typebox`; **не** устанавливайте их отдельную копию в этот пакет. Требуется `python3` в `PATH` (только стандартная библиотека).
|
|
66
95
|
|
|
67
|
-
|
|
96
|
+
Локальная установка при разработке:
|
|
68
97
|
|
|
69
98
|
```bash
|
|
70
99
|
pi install /path/to/this-repo/pi-plugin/rlm
|
|
71
100
|
```
|
|
72
101
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
```bash
|
|
76
|
-
npm publish # например, как @<you>/pi-rlm
|
|
77
|
-
pi install npm:@<you>/pi-rlm
|
|
78
|
-
```
|
|
79
|
-
|
|
80
|
-
> **Установка через Git** требует, чтобы манифест пакета находился в корне устанавливаемого репозитория. Для поддиректорий монорепозитория, таких как эта, предпочтительнее использовать локальный путь или npm, как указано выше.
|
|
102
|
+
> **Установка через Git** требует, чтобы манифест пакета находился в корне устанавливаемого репозитория. Для поддиректорий монорепозитория, таких как эта, предпочтительнее использовать локальный путь, как указано выше.
|
|
81
103
|
|
|
82
104
|
Если вы ранее копировали папку расширения напрямую, удалите ее, чтобы она не перекрывала пакет:
|
|
83
105
|
|
|
@@ -172,38 +194,6 @@ rm -rf ~/.pi/agent/extensions/rlm
|
|
|
172
194
|
## Логи запусков
|
|
173
195
|
|
|
174
196
|
|
|
175
|
-
## Бенчмарки
|
|
176
|
-
|
|
177
|
-
E2E-прогоны реального движка против моделей OpenRouter — lite-сьют (`needle` поиск игл в
|
|
178
|
-
куче, `codeqa` вопросы по коду, `coding` задача-фикс; 7 задач × 2 прохода на модель).
|
|
179
|
-
Детерминированные грейдеры (recall / вхождение эталона / regex), без LLM-судьи.
|
|
180
|
-
|
|
181
|
-
Актуальные результаты — маленькие модели (≤32B параметров, платный тариф):
|
|
182
|
-
|
|
183
|
-
| Модель | Параметры | Счёт | Точность | Latency/задача |
|
|
184
|
-
|--------|-----------|------|----------|----------------|
|
|
185
|
-
| `qwen/qwen3-30b-a3b-instruct-2507` | MoE 30B / 3B активных | **14/14** | **100%** | ~15s |
|
|
186
|
-
| `google/gemma-3-27b-it` | dense 27B | 12/14 | 86% | ~26s |
|
|
187
|
-
| `mistralai/mistral-small-3.2-24b-instruct` | dense 24B | 12/14 | 86% | ~28s |
|
|
188
|
-
|
|
189
|
-
Построчные результаты (correct, recall, latency, токены, цена) — в
|
|
190
|
-
`bench/runs/bench-<ts>.jsonl`, одна JSONL-строка на задачу.
|
|
191
|
-
|
|
192
|
-
### Как запустить
|
|
193
|
-
|
|
194
|
-
```bash
|
|
195
|
-
export OPENROUTER_API_KEY=sk-or-... # обязателен — ключи ходят только через env
|
|
196
|
-
|
|
197
|
-
bun run bench # lite-сьют: needle + codeqa + coding
|
|
198
|
-
bun run bench --suite needle --limit 1 # один сьют, первая задача
|
|
199
|
-
bun run bench --model openrouter/qwen/qwen3-30b-a3b-instruct-2507
|
|
200
|
-
bun run bench --list # показать задачи, без движка и ключа
|
|
201
|
-
bun run bench --suite paper # paper-сьют: s_niah, oolong, browsecomp, codeqa_lb (скачивает датасеты)
|
|
202
|
-
```
|
|
203
|
-
|
|
204
|
-
Сьюты: `all` (lite, по умолчанию) · `needle` · `codeqa` · `coding` · `paper` · `s_niah` ·
|
|
205
|
-
`oolong` · `browsecomp` · `codeqa_lb`.
|
|
206
|
-
|
|
207
197
|
## Безопасность
|
|
208
198
|
|
|
209
199
|
- **Изоляция ключей**: ключи провайдеров хранятся только в TypeScript (`AuthStorage`); песочница получает промпты и возвращает текст, но никогда не получает ключи.
|
package/README.zh-CN.md
CHANGED
|
@@ -1,11 +1,5 @@
|
|
|
1
1
|
<div align="center">
|
|
2
2
|
|
|
3
|
-
<img src="https://github.com/openzebra/rlm.pi/blob/master/assets/hero.png?raw=true" alt="pi-rlm">
|
|
4
|
-
|
|
5
|
-
</div>
|
|
6
|
-
|
|
7
|
-
<div align="center">
|
|
8
|
-
|
|
9
3
|
<sub>
|
|
10
4
|
<a href="README.md">English</a> · **中文** · <a href="README.ru.md">Русский</a>
|
|
11
5
|
</sub>
|
|
@@ -14,32 +8,73 @@
|
|
|
14
8
|
|
|
15
9
|
---
|
|
16
10
|
|
|
17
|
-
#
|
|
11
|
+
# rlm.pi PI plugin
|
|
18
12
|
|
|
19
|
-
|
|
13
|
+
> pi-rlm — 大上下文,廉价模型:为 Pi 提供的递归语言模型 (RLM)
|
|
20
14
|
|
|
21
|
-
|
|
22
|
-
完全本地。
|
|
15
|
+
## 安装
|
|
23
16
|
|
|
24
|
-
|
|
17
|
+
```bash
|
|
18
|
+
pi install npm:@hicaru/pi-rlm
|
|
19
|
+
```
|
|
25
20
|
|
|
26
|
-
|
|
21
|
+
以后要移除:
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
pi uninstall npm:@hicaru/pi-rlm
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
然后在 Pi 中运行 `/reload` —— `/rlm`、`/rlm-config` 和 `/rlm-stop` 会出现在 **[Extensions]** 下。使用 `Ctrl+Shift+R` 或 `/rlm` 切换。
|
|
28
|
+
|
|
29
|
+
<p align="center">
|
|
30
|
+
<img src="https://github.com/openzebra/rlm.pi/blob/master/assets/hero.png?raw=true" width="100%" alt="rlm.pi — OOLONG 基准测试结果">
|
|
31
|
+
</p>
|
|
27
32
|
|
|
28
|
-
|
|
33
|
+
## 什么是 pi-rlm?
|
|
29
34
|
|
|
30
|
-
|
|
35
|
+
一个把 Pi 会话变成**递归语言模型 (RLM)** 的插件:不必把庞大的文档塞进提示词,上下文
|
|
36
|
+
驻留在 Python REPL 中,由你最好的模型编排 —— 检索、分解,并把叶子读取递归地委派给
|
|
37
|
+
廉价的工作模型。同一个 Pi 会话、同样的工具、同样的密钥 —— 打开 `/rlm` 即可。
|
|
38
|
+
方法基于 [RLM 论文](https://arxiv.org/abs/2512.24601);详见下方的**工作原理**。
|
|
31
39
|
|
|
32
|
-
|
|
40
|
+
## 基准测试
|
|
41
|
+
|
|
42
|
+
**OOLONG (oolong-synth)** —— paper 级长上下文套件;取每个模型最新的日志,每任务成本
|
|
43
|
+
来自真实 `costUsd`(旧日志按 OpenRouter 牌价估算):
|
|
44
|
+
|
|
45
|
+
| 模型 | 得分 | 每任务成本 |
|
|
46
|
+
|------|------|------------|
|
|
47
|
+
| `qwen/qwen3.8-27b` | **100%** | $0.0127 |
|
|
48
|
+
| `google/gemma-3-27b-it` | 83.3% | $0.0013 |
|
|
49
|
+
| `qwen/qwen3-30b-a3b-instruct-2507` | 66.7% | $0.0009 |
|
|
50
|
+
| `mistralai/mistral-small-3.2-24b-instruct` | 66.7% | $0.0025 |
|
|
51
|
+
|
|
52
|
+
精简套件 —— `needle`(多针召回)、`codeqa`(代码库问答)、`coding`(修复任务;
|
|
53
|
+
每模型 7 个任务 × 2 轮,确定性评分,无 LLM 评审):
|
|
33
54
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
55
|
+
| 模型 | 得分 | 准确率 |
|
|
56
|
+
|------|------|--------|
|
|
57
|
+
| `qwen/qwen3-30b-a3b-instruct-2507` | **14/14** | **100%** |
|
|
58
|
+
| `google/gemma-3-27b-it` | 12/14 | 86% |
|
|
59
|
+
| `mistralai/mistral-small-3.2-24b-instruct` | 12/14 | 86% |
|
|
60
|
+
|
|
61
|
+
逐任务原始数据(正确性、召回率、延迟、token、成本)位于 `bench/runs/*.jsonl`
|
|
62
|
+
—— 每个任务一行 JSONL,作为历史记录提交。
|
|
63
|
+
|
|
64
|
+
### 运行基准测试
|
|
40
65
|
|
|
41
|
-
|
|
42
|
-
|
|
66
|
+
```bash
|
|
67
|
+
export OPENROUTER_API_KEY=sk-or-... # 必需 —— 密钥仅通过环境变量传递
|
|
68
|
+
|
|
69
|
+
bun run bench # 精简套件:needle + codeqa + coding
|
|
70
|
+
bun run bench --suite needle --limit 1 # 单个套件,仅第一个任务
|
|
71
|
+
bun run bench --model openrouter/qwen/qwen3-30b-a3b-instruct-2507
|
|
72
|
+
bun run bench --list # 仅列出任务,无需引擎和密钥
|
|
73
|
+
bun run bench --suite paper # paper 套件:s_niah, oolong, browsecomp, codeqa_lb(需下载数据集)
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
套件:`all`(精简版,默认) · `needle` · `codeqa` · `coding` · `paper` · `s_niah` ·
|
|
77
|
+
`oolong` · `browsecomp` · `codeqa_lb`。
|
|
43
78
|
|
|
44
79
|
## 工作原理
|
|
45
80
|
|
|
@@ -61,26 +96,19 @@ pi 进程 (TypeScript)
|
|
|
61
96
|
`rlm_query_batched`, `SHOW_VARS()`, `ask_user_question()` 以及一个 `answer` 字典。
|
|
62
97
|
模型通过设置 `answer["ready"] = True` 来提交最终结果。
|
|
63
98
|
|
|
64
|
-
##
|
|
99
|
+
## 从源码安装(开发)
|
|
65
100
|
|
|
66
101
|
`pi-rlm` 是一个 Pi 包。Pi 提供了 `@earendil-works/pi-*` 和 `typebox` peer
|
|
67
102
|
依赖;请**不要**在该包中安装它们的独立副本。要求 `PATH` 中有 `python3` (仅限标准库)。
|
|
68
103
|
|
|
69
|
-
|
|
104
|
+
开发时的本地安装方式:
|
|
70
105
|
|
|
71
106
|
```bash
|
|
72
107
|
pi install /path/to/this-repo/pi-plugin/rlm
|
|
73
108
|
```
|
|
74
109
|
|
|
75
|
-
已发布的 npm 包安装方式:
|
|
76
|
-
|
|
77
|
-
```bash
|
|
78
|
-
npm publish # 例如 as @<you>/pi-rlm
|
|
79
|
-
pi install npm:@<you>/pi-rlm
|
|
80
|
-
```
|
|
81
|
-
|
|
82
110
|
> **Git 安装**要求包清单位于安装的仓库根目录下。
|
|
83
|
-
> 对于像这样一个 monorepo
|
|
111
|
+
> 对于像这样一个 monorepo 子目录,请优先使用上述的本地路径流程。
|
|
84
112
|
|
|
85
113
|
如果您之前直接复制了扩展文件夹,请将其删除,以免遮蔽 (shadow) 该包:
|
|
86
114
|
|
|
@@ -191,38 +219,6 @@ r3 基准测试表明:最大的能力杠杆不是模型,而是采样。仅 t
|
|
|
191
219
|
连续错误上限。
|
|
192
220
|
- **信任**:本地安装需要 Pi 项目信任。
|
|
193
221
|
|
|
194
|
-
## 基准测试
|
|
195
|
-
|
|
196
|
-
针对 OpenRouter 聊天模型对真实引擎进行端到端测试 —— 精简套件(`needle` 多针召回、
|
|
197
|
-
`codeqa` 代码库问答、`coding` 修复任务;每个模型 7 个任务 × 2 轮)。确定性评分
|
|
198
|
-
(召回率 / 标准答案包含 / 正则),无 LLM 评审。
|
|
199
|
-
|
|
200
|
-
最新结果 —— 小型模型(≤32B 参数,付费额度):
|
|
201
|
-
|
|
202
|
-
| 模型 | 参数 | 得分 | 准确率 | 每任务延迟 |
|
|
203
|
-
|------|------|------|--------|------------|
|
|
204
|
-
| `qwen/qwen3-30b-a3b-instruct-2507` | MoE 30B / 3B 激活 | **14/14** | **100%** | ~15s |
|
|
205
|
-
| `google/gemma-3-27b-it` | dense 27B | 12/14 | 86% | ~26s |
|
|
206
|
-
| `mistralai/mistral-small-3.2-24b-instruct` | dense 24B | 12/14 | 86% | ~28s |
|
|
207
|
-
|
|
208
|
-
逐任务原始数据(正确性、召回率、延迟、token、成本)位于 `bench/runs/bench-<ts>.jsonl`
|
|
209
|
-
—— 每个任务一行 JSONL,作为历史记录提交。
|
|
210
|
-
|
|
211
|
-
### 运行基准测试
|
|
212
|
-
|
|
213
|
-
```bash
|
|
214
|
-
export OPENROUTER_API_KEY=sk-or-... # 必需 —— 密钥仅通过环境变量传递
|
|
215
|
-
|
|
216
|
-
bun run bench # 精简套件:needle + codeqa + coding
|
|
217
|
-
bun run bench --suite needle --limit 1 # 单个套件,仅第一个任务
|
|
218
|
-
bun run bench --model openrouter/qwen/qwen3-30b-a3b-instruct-2507
|
|
219
|
-
bun run bench --list # 仅列出任务,无需引擎和密钥
|
|
220
|
-
bun run bench --suite paper # paper 套件:s_niah, oolong, browsecomp, codeqa_lb(需下载数据集)
|
|
221
|
-
```
|
|
222
|
-
|
|
223
|
-
套件:`all`(精简版,默认) · `needle` · `codeqa` · `coding` · `paper` · `s_niah` ·
|
|
224
|
-
`oolong` · `browsecomp` · `codeqa_lb`。
|
|
225
|
-
|
|
226
222
|
## 项目布局
|
|
227
223
|
|
|
228
224
|
```
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hicaru/pi-rlm",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.18",
|
|
4
4
|
"author": "hicaru",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -10,9 +10,9 @@
|
|
|
10
10
|
"typescript": "^5.0.0"
|
|
11
11
|
},
|
|
12
12
|
"peerDependencies": {
|
|
13
|
-
"@earendil-works/pi-ai": "^0.
|
|
14
|
-
"@earendil-works/pi-coding-agent": "^0.
|
|
15
|
-
"@earendil-works/pi-tui": "^0.
|
|
13
|
+
"@earendil-works/pi-ai": "^0.85.1",
|
|
14
|
+
"@earendil-works/pi-coding-agent": "^0.85.1",
|
|
15
|
+
"@earendil-works/pi-tui": "^0.85.1",
|
|
16
16
|
"typebox": "*"
|
|
17
17
|
},
|
|
18
18
|
"bugs": {
|
|
@@ -52,7 +52,7 @@
|
|
|
52
52
|
"node": ">=20"
|
|
53
53
|
},
|
|
54
54
|
"dependencies": {
|
|
55
|
-
"@earendil-works/pi-agent-core": "^0.
|
|
55
|
+
"@earendil-works/pi-agent-core": "^0.85.1",
|
|
56
56
|
"@firecrawl/anydoc": "^0.2.4"
|
|
57
57
|
}
|
|
58
58
|
}
|
|
@@ -24,7 +24,7 @@ import {
|
|
|
24
24
|
import { resolveSource } from "../context/resolve.ts";
|
|
25
25
|
import { previewText } from "../text/preview.ts";
|
|
26
26
|
|
|
27
|
-
|
|
27
|
+
interface AddContextBridgeOpts {
|
|
28
28
|
/** Fixed cwd (headless). Prefer getCwd when the sandbox outlives a single invocation. */
|
|
29
29
|
readonly cwd?: string;
|
|
30
30
|
/** Late-bound cwd (native mode — sandbox handlers outlive a single repl()). */
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
import { errorMessage, formatError } from "../../util/errors.ts";
|
|
10
10
|
import type { AwaitResult, SubcallHandlerDeps, TaskEntry } from "./types.ts";
|
|
11
11
|
import type { SubcallOpts } from "../../sandbox/interrupts.ts";
|
|
12
|
-
import type
|
|
12
|
+
import { entryToAwaitResult, type AwaitDeps } from "./task-registry.ts";
|
|
13
13
|
|
|
14
14
|
export function createAwaitHandler(_deps: SubcallHandlerDeps, ad: AwaitDeps) {
|
|
15
15
|
return async (
|
|
@@ -37,7 +37,7 @@ export function createAwaitHandler(_deps: SubcallHandlerDeps, ad: AwaitDeps) {
|
|
|
37
37
|
if (entry.status === "pending") {
|
|
38
38
|
try {
|
|
39
39
|
const resolved = await ad.wait(taskId, timeoutMs);
|
|
40
|
-
return
|
|
40
|
+
return entryToAwaitResult(resolved);
|
|
41
41
|
} catch (err: unknown) {
|
|
42
42
|
return {
|
|
43
43
|
ok: false,
|
|
@@ -49,7 +49,7 @@ export function createAwaitHandler(_deps: SubcallHandlerDeps, ad: AwaitDeps) {
|
|
|
49
49
|
}
|
|
50
50
|
}
|
|
51
51
|
|
|
52
|
-
return
|
|
52
|
+
return entryToAwaitResult(entry);
|
|
53
53
|
}
|
|
54
54
|
|
|
55
55
|
// Multiple tasks
|
|
@@ -91,7 +91,7 @@ export function createAwaitHandler(_deps: SubcallHandlerDeps, ad: AwaitDeps) {
|
|
|
91
91
|
}),
|
|
92
92
|
);
|
|
93
93
|
|
|
94
|
-
const awaited = resolved.map(
|
|
94
|
+
const awaited = resolved.map(entryToAwaitResult);
|
|
95
95
|
const allDone = awaited.every((a) => a.status === "done");
|
|
96
96
|
const hasResults = awaited.some((a) => a.results !== undefined);
|
|
97
97
|
const firstError = awaited.find((a) => a.error)?.error;
|
|
@@ -99,12 +99,18 @@ export function createAwaitHandler(_deps: SubcallHandlerDeps, ad: AwaitDeps) {
|
|
|
99
99
|
const kind = first?.kind ?? "unknown";
|
|
100
100
|
|
|
101
101
|
if (hasResults) {
|
|
102
|
-
|
|
102
|
+
// Pre-allocated (rule: no .push() growth when the size is computable) — count, then fill.
|
|
103
|
+
const total = awaited.reduce(
|
|
104
|
+
(n, a) => n + (a.results !== undefined ? a.results.length : a.result !== undefined ? 1 : 0),
|
|
105
|
+
0,
|
|
106
|
+
);
|
|
107
|
+
const allResults = new Array<string>(total);
|
|
108
|
+
let n = 0;
|
|
103
109
|
for (const a of awaited) {
|
|
104
110
|
if (a.results !== undefined) {
|
|
105
|
-
for (const r of a.results) allResults
|
|
111
|
+
for (const r of a.results) allResults[n++] = r;
|
|
106
112
|
} else if (a.result !== undefined) {
|
|
107
|
-
allResults
|
|
113
|
+
allResults[n++] = a.result;
|
|
108
114
|
}
|
|
109
115
|
}
|
|
110
116
|
return {
|
|
@@ -131,18 +137,3 @@ export function createAwaitHandler(_deps: SubcallHandlerDeps, ad: AwaitDeps) {
|
|
|
131
137
|
};
|
|
132
138
|
};
|
|
133
139
|
}
|
|
134
|
-
|
|
135
|
-
function toAwaitResult(entry: TaskEntry): AwaitResult {
|
|
136
|
-
const status = entry.status === "pending" ? "error" : entry.status;
|
|
137
|
-
return {
|
|
138
|
-
ok: entry.status === "done",
|
|
139
|
-
task_id: entry.taskId,
|
|
140
|
-
kind: entry.kind,
|
|
141
|
-
status,
|
|
142
|
-
result: entry.result,
|
|
143
|
-
results: entry.results,
|
|
144
|
-
error:
|
|
145
|
-
entry.error ??
|
|
146
|
-
(entry.status === "pending" ? "Task still pending" : undefined),
|
|
147
|
-
};
|
|
148
|
-
}
|
|
@@ -14,15 +14,33 @@ import { checkResourceLimits } from "../../core/resource-limits.ts";
|
|
|
14
14
|
import { errorMessage, formatError } from "../../util/errors.ts";
|
|
15
15
|
import { retryPolicy } from "../../util/retry.ts";
|
|
16
16
|
import type { Semaphore } from "../../util/concurrency.ts";
|
|
17
|
-
import type { Invocation, SubcallConfig } from "./types.ts";
|
|
17
|
+
import type { Invocation, SubcallConfig, SubcallHandlerDeps } from "./types.ts";
|
|
18
18
|
|
|
19
|
-
|
|
19
|
+
interface Complete1Deps {
|
|
20
20
|
readonly leafGate: Semaphore;
|
|
21
21
|
readonly registry: ModelRegistry;
|
|
22
22
|
readonly getLlmModel: () => Model<Api>;
|
|
23
23
|
readonly getConfig: () => SubcallConfig;
|
|
24
24
|
readonly signal?: AbortSignal;
|
|
25
25
|
readonly onUsage?: (usage: Usage, role: "sub") => void;
|
|
26
|
+
/** SKILL.state (Workstream D): the one grounding seam, projected verbatim from deps. */
|
|
27
|
+
readonly groundLeaf?: (prompt: string) => string;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* AGENTS.md DRY: the ONE projection of SubcallHandlerDeps for complete1 — shared verbatim by
|
|
32
|
+
* the llm-query and rlm-query handlers. Never inline a second copy.
|
|
33
|
+
*/
|
|
34
|
+
export function completeDeps(deps: SubcallHandlerDeps): Complete1Deps {
|
|
35
|
+
return {
|
|
36
|
+
leafGate: deps.gates.leaf,
|
|
37
|
+
registry: deps.registry,
|
|
38
|
+
getLlmModel: deps.getLlmModel,
|
|
39
|
+
getConfig: deps.getConfig,
|
|
40
|
+
signal: deps.signal,
|
|
41
|
+
onUsage: deps.onUsage,
|
|
42
|
+
groundLeaf: deps.groundLeaf,
|
|
43
|
+
};
|
|
26
44
|
}
|
|
27
45
|
|
|
28
46
|
/**
|
|
@@ -41,14 +59,18 @@ export async function complete1(
|
|
|
41
59
|
timeoutMs: inv.limits.remainingTimeoutMs(),
|
|
42
60
|
});
|
|
43
61
|
if (limitError !== undefined) return limitError;
|
|
44
|
-
|
|
62
|
+
// Workstream D (DRY #1): the ONE grounding application — llm_query, batch items, rlm→llm
|
|
63
|
+
// demotion and depth-cap degrade all pass through here, so all of them get Ξ facts or none
|
|
64
|
+
// does. Below-threshold prompts come back byte-identical from the groundLeaf impl.
|
|
65
|
+
const grounded = deps.groundLeaf !== undefined ? deps.groundLeaf(prompt) : prompt;
|
|
66
|
+
if (grounded.length > config.maxPromptChars) {
|
|
45
67
|
return formatError(
|
|
46
|
-
`sub-LLM prompt exceeded the size limit (${
|
|
68
|
+
`sub-LLM prompt exceeded the size limit (${grounded.length.toLocaleString()} chars > ` +
|
|
47
69
|
`${config.maxPromptChars.toLocaleString()}). Shorten or chunk the prompt before calling llm_query.`,
|
|
48
70
|
);
|
|
49
71
|
}
|
|
50
72
|
try {
|
|
51
|
-
const messages: ChatMsg[] = [{ role: "user", content:
|
|
73
|
+
const messages: ChatMsg[] = [{ role: "user", content: grounded }];
|
|
52
74
|
const res = await deps.leafGate.run(() =>
|
|
53
75
|
modelComplete(messages, {
|
|
54
76
|
model: deps.getLlmModel(),
|
|
@@ -11,14 +11,14 @@ import { previewText } from "../../text/preview.ts";
|
|
|
11
11
|
import type { SubcallPhase } from "../../tool/rlm-details.ts";
|
|
12
12
|
import type { Invocation } from "./types.ts";
|
|
13
13
|
|
|
14
|
-
|
|
14
|
+
interface EmitOpts {
|
|
15
15
|
readonly kind: "llm" | "batch";
|
|
16
16
|
readonly label: string;
|
|
17
17
|
readonly args: string;
|
|
18
18
|
readonly model?: string;
|
|
19
19
|
}
|
|
20
20
|
|
|
21
|
-
|
|
21
|
+
interface EmitSummary {
|
|
22
22
|
readonly preview: string;
|
|
23
23
|
readonly error?: string;
|
|
24
24
|
readonly failed?: number;
|