dsh-awesome-optimizer 1.8.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -15
- package/README.zh.md +33 -15
- package/package.json +6 -2
- package/scripts/benchmark.mjs +53 -2
- package/src/custom-prompt.js +6 -3
- package/src/index.d.ts +14 -0
- package/src/index.js +18 -1
- package/src/persistent-eval/daemon-pool.js +120 -0
- package/src/persistent-eval/index.d.ts +56 -0
- package/src/persistent-eval/index.js +192 -0
- package/src/persistent-eval/stdio-runner.js +304 -0
- package/src/persistent-eval/workers/node_worker.mjs +247 -0
- package/src/persistent-eval/workers/py_worker.py +199 -0
package/README.md
CHANGED
|
@@ -5,22 +5,40 @@ English | [中文](README.zh.md)
|
|
|
5
5
|
[](https://www.npmjs.com/package/dsh-awesome-optimizer)
|
|
6
6
|
[](LICENSE)
|
|
7
7
|
[](https://github.com/Jaylor-Wang/dsh-awesome-optimizer/releases)
|
|
8
|
-
[](test/)
|
|
9
9
|
[](src/)
|
|
10
10
|
|
|
11
|
-
High-performance runtime policy guards, PTC-backed composite batch tooling, standalone refined system prompt engine, and objective A/B turn telemetry plugin for the official [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (`@deepseek-ai/dsh-agent-loop`).
|
|
11
|
+
High-performance runtime policy guards, persistent Python & Node evaluation daemons, PTC-backed composite batch tooling, standalone refined system prompt engine, and objective A/B turn telemetry plugin for the official [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (`@deepseek-ai/dsh-agent-loop`).
|
|
12
12
|
|
|
13
13
|
---
|
|
14
14
|
|
|
15
|
-
## Highlights in
|
|
15
|
+
## Highlights in v2.1.0
|
|
16
16
|
|
|
17
|
-
- **
|
|
18
|
-
|
|
19
|
-
- **
|
|
20
|
-
|
|
21
|
-
- **
|
|
22
|
-
-
|
|
23
|
-
- **
|
|
17
|
+
- **Pre-warming Daemon**:
|
|
18
|
+
Hooks into `turn/start` lifecycle events to pre-spawn and handshake worker processes during model first-token inference, completely eliminating the 80ms cold-start spike down to **0.52ms (98.8% speedup)**.
|
|
19
|
+
- **Eager Scope Ingestion**:
|
|
20
|
+
Standard algorithm libraries (`math`, `re`, `json`, `time`, `collections`, `itertools`) are eagerly pre-imported into global Python scope, eliminating repetitive `import` boilerplate and saving token generation time.
|
|
21
|
+
- **Zero-Token-Waste Formatting**:
|
|
22
|
+
Ultra-compact single-line return format for pure expressions (`=> 42`), trimmed trailing newlines, and intelligent smoothing of IEEE-754 binary floating-point noise (`0.1 + 0.2` -> `=> 0.3`).
|
|
23
|
+
- **Dynamic Ref/Unref & Cordis Lifecycle Cascading**:
|
|
24
|
+
Implements dynamic stream reference counting (`_ref` / `_unref`) so idle daemons never hold the Node.js event loop hostage while ensuring reliable teardown via Cordis `ctx.effect`.
|
|
25
|
+
|
|
26
|
+
---
|
|
27
|
+
|
|
28
|
+
## Highlights in v2.0.0
|
|
29
|
+
|
|
30
|
+
- **Persistent Python & Node Evaluation Daemons (`eval_python` / `eval_node`)**:
|
|
31
|
+
Zero-dependency stateful execution kernels over line-delimited stdio JSON frames. Retains global scopes, functions, imports, and variables across turns (with Node top-level `await`). Slashes repetitive calculation latency from 37ms to **0.61ms (98.5% speedup)**, eliminating temporary file pollution.
|
|
32
|
+
- **Three Extreme Performance Dimensions**:
|
|
33
|
+
1. **Pre-warming Daemon**: Listens to session `turn/start` events to pre-spawn and handshake worker processes during model first-token generation, completely eliminating the 80ms cold-start spike.
|
|
34
|
+
2. **Eager Scope Ingestion**: Standard algorithm libraries (`math`, `re`, `json`, `time`, `collections`, `itertools`) are eagerly pre-imported into global Python scope, saving LLM token generation overhead.
|
|
35
|
+
3. **Zero-Token-Waste Formatting**: Ultra-compact single line return format for pure expressions (`=> 42`), trimmed trailing newlines, and intelligent smoothing of IEEE-754 binary floating-point noise (`0.30000000000000004` -> `0.3`) to accelerate LLM Prefill.
|
|
36
|
+
- **Watchdog Timeout & Self-Healing Auto-Restart**:
|
|
37
|
+
Built-in execution watchdog forces termination of infinite loops (`while True:`) or blocked input across platforms without freezing the host process, automatically spinning up fresh instances for seamless queued recovery.
|
|
38
|
+
- **Anti-Dumping Output Guard**:
|
|
39
|
+
Enforces configurable payload truncation (default 8KB) with disk-redirect guidance, directly reinforcing the `NEVER dump raw data into context` directive.
|
|
40
|
+
- **Evaluation vs Engineering Code Separation**:
|
|
41
|
+
Formalizes prompt directives ensuring intermediate calculations leverage persistent in-memory kernels while deliverable production files are committed to disk via native filesystem operations.
|
|
24
42
|
|
|
25
43
|
---
|
|
26
44
|
|
|
@@ -136,7 +154,7 @@ You are a master autonomous software engineer powered by DeepSeek Harness.
|
|
|
136
154
|
|
|
137
155
|
- Prioritize direct action over speculation: investigate via concurrent tools (fs, bash, skills) without orchestration overhead.
|
|
138
156
|
- NEVER emit pre-action narration. Deliver surgical edits and high-signal conclusions.
|
|
139
|
-
- Irreversible operations (npm publish): STRICTLY conservative and non-autonomous. NEVER bump versions or mutate artifacts to bypass failures; registry validation causes transient 404 or stale views—NEVER treat as failure or retry publish. MUST verify remote state with read-only commands before declaring status.
|
|
157
|
+
- Irreversible operations (`npm publish`): STRICTLY conservative and non-autonomous. NEVER bump versions or mutate artifacts to bypass failures; registry validation causes transient 404 or stale views—NEVER treat as failure or retry publish. MUST verify remote state with read-only commands before declaring status.
|
|
140
158
|
|
|
141
159
|
## Defaults
|
|
142
160
|
|
|
@@ -146,14 +164,14 @@ You are a master autonomous software engineer powered by DeepSeek Harness.
|
|
|
146
164
|
- Ask first (via `ask_user_question`) ONLY when ambiguity materially affects scope, architecture, or data safety; otherwise execute autonomously.
|
|
147
165
|
- Shell: ONLY `bash`. Launch `pwsh.exe` from `bash` when needed.
|
|
148
166
|
- Web: Research/fetch STRICTLY via skills (`agent-reach`, `gh`, `opencli-*`, `smart-search`) or bash CLIs.
|
|
149
|
-
- Tool boundaries: External tools (playwright-cli, gh, ast-grep) are standalone CLIs via bash
|
|
167
|
+
- Tool boundaries: External tools (playwright-cli, gh, ast-grep) are standalone CLIs via `bash`, NEVER require/import as node modules.
|
|
150
168
|
|
|
151
169
|
## Efficiency
|
|
152
170
|
|
|
153
171
|
### Hard Constraints
|
|
154
172
|
|
|
155
173
|
- Retry ONLY when the error or new evidence supports a specific correction; after two failures without progress, switch diagnostic approaches.
|
|
156
|
-
- Windows arguments & paths: use forward slashes in paths to prevent escape corruption. NEVER embed
|
|
174
|
+
- Windows arguments & paths: use forward slashes in paths to prevent escape corruption. NEVER embed scripts or multiline code in shell arguments (`node -e`, `python -c`, `pwsh -Command`). Quote single-line patterns carefully (`ast-grep` patterns STRICTLY in single quotes `'...'` to prevent `$$` expansion). On quoting failures, switch input method rather than re-escaping.
|
|
157
175
|
- Byte-fidelity: default new text files to UTF-8. Preserve existing file encodings, BOMs, line endings, `.gitattributes`, and hook policies unless explicitly requested.
|
|
158
176
|
- Policy denials (`file access denied`, network blocked, approval disabled/rejected) are not command bugs: report; do not bypass. Request wider access via `sandbox_permissions` ONLY when escalation is explicitly available; otherwise do not retry or escalate. Diagnose ordinary lock, permission, or connectivity errors normally.
|
|
159
177
|
- Tool exclusivity: STRICTLY native `glob`/`grep`/`read_batch`/`read`/`edit_batch`/`edit`/`write`. NEVER use shell commands or redirects for workspace search or reading/writing files (`grep`, `rg`, `find`, `ls`, `dir`, `cat`, `type`, `head`, `tail`, `sed`, `awk`, `tee`, `>`/`>>`). Filtering piped command output (`cmd | grep`) and batch removing temporary artifacts (`rm -rf`) are permitted.
|
|
@@ -172,7 +190,7 @@ You are a master autonomous software engineer powered by DeepSeek Harness.
|
|
|
172
190
|
- Modification & execution:
|
|
173
191
|
- File ops: MUST use `edit_batch` for 2+ edits (whether in one file or across multiple files); use `edit` for a single edit, and `write` for new files or ad-hoc Node/Python scripts.
|
|
174
192
|
- Staged refactor: preview, then apply — symbol renames via `lsp rename` (dry-run → `--apply`); structural rewrites via `ast-grep run -p ... -r ...` (then `-U`).
|
|
175
|
-
- Structured data & calculations:
|
|
193
|
+
- Structured data & calculations: Prioritize persistent kernels (`eval_python`, `eval_node`) for stateful computation, inverse kinematics, or data exploration without disk pollution. For deliverable scripts or build pipelines, MUST write to disk first via native `write` and execute by file path; NEVER embed scripts, calculations, or processing in `python -c` or `node -e`, and NEVER dump raw data into context.
|
|
176
194
|
- Shell execution: run heavy tasks (`npm test`, `npm run build`, `vitest`, `npm publish`) standalone to prevent Windows deadlocks; chain lightweight commands via `&&` (`git add ... && git commit ... && git push`) to eliminate round trips.
|
|
177
195
|
- Reporting: diffs and evidence first; zero filler. Routine status ≤ 300 Chinese characters. Answer/explain/review/diagnose/plan: no length cap. Compact locators, lists, and deltas only.
|
|
178
196
|
|
|
@@ -186,7 +204,7 @@ Applies ONLY to change/build/fix.
|
|
|
186
204
|
- Behavior changes: check static diagnostics via primary tools (e.g. `lsp diag <file|dir> -e`, skip if language unsupported; no redundant CLI linters), then run the smallest non-destructive targeted build/test covering affected behavior and internal interfaces.
|
|
187
205
|
- Shared API, schema, or migration changes: run the full related suite plus project-wide type checking where applicable (e.g. `tsc --noEmit`).
|
|
188
206
|
- Report checks, results, and verification gaps; if primary diagnostics are unavailable, use an existing equivalent check when possible.
|
|
189
|
-
- Browser checks: STRICTLY `playwright-cli` post-write. Playwright CLI blocks
|
|
207
|
+
- Browser checks: STRICTLY `playwright-cli` post-write. Playwright CLI blocks `file:` URLs—serve static HTML via loopback HTTP (`http://127.0.0.1:<port>`). `screenshot` takes selector target ONLY (no output filename; auto-saved to .playwright-cli/). NEVER pre-probe (`--help`, `--version`, `node -v`), create dummy pages, or switch tools. Reuse runtime Web GUI or active app URL (loopback for static); stabilize animations before capture. Batch-cleanup sessions/servers/artifacts without redundant listings. On failure, report root cause.
|
|
190
208
|
- Persistent failure: report failed command(s), 1–2 sentence root cause, blast radius, next steps.
|
|
191
209
|
```
|
|
192
210
|
</details>
|
package/README.zh.md
CHANGED
|
@@ -5,22 +5,40 @@
|
|
|
5
5
|
[](https://www.npmjs.com/package/dsh-awesome-optimizer)
|
|
6
6
|
[](LICENSE)
|
|
7
7
|
[](https://github.com/Jaylor-Wang/dsh-awesome-optimizer/releases)
|
|
8
|
-
[](test/)
|
|
9
9
|
[](src/)
|
|
10
10
|
|
|
11
|
-
围绕官方 [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (`@deepseek-ai/dsh-agent-loop`)
|
|
11
|
+
围绕官方 [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (`@deepseek-ai/dsh-agent-loop`) 打造的高性能运行时策略防护、持久化 Python/Node 执行内核、PTC 复合批处理工具、全新独立系统提示词接管与客观 A/B 轮次度量插件。
|
|
12
12
|
|
|
13
13
|
---
|
|
14
14
|
|
|
15
|
-
##
|
|
15
|
+
## v2.1.0 版本重大更新
|
|
16
16
|
|
|
17
|
-
-
|
|
18
|
-
|
|
19
|
-
-
|
|
20
|
-
|
|
21
|
-
-
|
|
22
|
-
|
|
23
|
-
-
|
|
17
|
+
- **会话起手后台静默预热(Pre-warming Daemon)**:
|
|
18
|
+
监听 `turn/start` 生命周期事件,在模型思考与首字生成的窗口期并行完成子进程握手,消灭首次 80ms 冷启毛刺,单次执行延迟降低至 **0.52ms(相比冷启提速 98.8%)**。
|
|
19
|
+
- **常用数学/算法标准库预载入(Eager Scope Ingestion)**:
|
|
20
|
+
静默向全局 Python 作用域预注入 `math, re, json, time, collections, itertools`,省去繁复 `import` 语句生成开销。
|
|
21
|
+
- **极致紧凑输出(Zero-Token-Waste Formatting)**:
|
|
22
|
+
纯表达式极简单行输出(`=> 42`),剔除多余首尾换行,平滑 IEEE-754 浮点二进制噪声(`0.1 + 0.2` -> `=> 0.3`),杜绝无效 Token 占用 Prefill 窗口。
|
|
23
|
+
- **动态引用计数与 Cordis 级联释放(Ref/Unref & Lifecycle Cascading)**:
|
|
24
|
+
基于 Node.js libuv 管道流的动态引用计数(`_ref` / `_unref`),确保在任务执行期间稳定守卫,任务完成后释放事件循环强引用,杜绝孤儿进程与退出超时挂起。
|
|
25
|
+
|
|
26
|
+
---
|
|
27
|
+
|
|
28
|
+
## v2.0.0 版本重大里程碑更新
|
|
29
|
+
|
|
30
|
+
- **持久化 Python 与 Node 执行内核 (`eval_python` / `eval_node`)**:
|
|
31
|
+
引入基于轻量级 stdio 行定界 JSON 协议帧的双语常驻守护引擎,**零外部依赖**(无需 `ipykernel`、`pyzmq` 等重型外部包)。在会话生命周期内跨调用保留全局变量、导入模块、类与函数,支持 Node 顶层 `await`。将算法、逆运动学与几何试算单次延迟从 37ms 直接压降至 **0.61ms(提速 98.5%)**,彻底终结多轮试算频繁向磁盘落盘临时脚本的损耗与文件污染。
|
|
32
|
+
- **三大极致性能提速升级**:
|
|
33
|
+
1. **后台静默预热(Pre-warming Daemon)**:监听 `turn/start` 会话事件,在模型生成首个 Token 的思考间隙预先握手拉起子进程,消除首次 80ms 冷启尖刺;
|
|
34
|
+
2. **核心算法库预注入(Eager Scope Ingestion)**:默认在 Python 全局命名空间预载 `math, re, json, time, collections, itertools`,省去繁冗 `import` 代码,显著降低 LLM Token 消耗;
|
|
35
|
+
3. **输出极致紧凑压缩(Zero-Token-Waste Formatting)**:纯表达式求值单行紧凑返回(`=> 42`),剔除尾部多余换行,智能抹平 IEEE-754 二进制浮点噪声(如 `0.30000000000000004` -> `0.3`),为 LLM Prefill 阶段省出宝贵上下文。
|
|
36
|
+
- **看门狗超时强杀与自愈热重启 (Watchdog & Self-Healing)**:
|
|
37
|
+
内置细粒度硬执行看门狗。当代码陷入死循环(如 `while True:`)或阻塞输入超时(默认 15s)后,主进程永不卡死挂起,跨平台强杀卡死子进程树并自动拉起新生干净实例,后续排队任务零感平滑恢复。
|
|
38
|
+
- **上下文防冲爆守卫 (Anti-Dumping Interceptor)**:
|
|
39
|
+
在工具管道层对输出实施 8KB 阈值截断并追加引导落盘说明,与 `NEVER dump raw data into context` 规则深度协同,彻底杜绝超大输出冲爆 LLM 上下文。
|
|
40
|
+
- **心智分流规约升维**:
|
|
41
|
+
在 `SYSTEM.md` 与 Custom Prompt 中确立“探索性解算/数据筛选优先走持久内核,工程交付物必须通过 native `write` 落盘”的最高指令分流契约。
|
|
24
42
|
|
|
25
43
|
---
|
|
26
44
|
|
|
@@ -136,7 +154,7 @@ You are a master autonomous software engineer powered by DeepSeek Harness.
|
|
|
136
154
|
|
|
137
155
|
- Prioritize direct action over speculation: investigate via concurrent tools (fs, bash, skills) without orchestration overhead.
|
|
138
156
|
- NEVER emit pre-action narration. Deliver surgical edits and high-signal conclusions.
|
|
139
|
-
- Irreversible operations (npm publish): STRICTLY conservative and non-autonomous. NEVER bump versions or mutate artifacts to bypass failures; registry validation causes transient 404 or stale views—NEVER treat as failure or retry publish. MUST verify remote state with read-only commands before declaring status.
|
|
157
|
+
- Irreversible operations (`npm publish`): STRICTLY conservative and non-autonomous. NEVER bump versions or mutate artifacts to bypass failures; registry validation causes transient 404 or stale views—NEVER treat as failure or retry publish. MUST verify remote state with read-only commands before declaring status.
|
|
140
158
|
|
|
141
159
|
## Defaults
|
|
142
160
|
|
|
@@ -146,14 +164,14 @@ You are a master autonomous software engineer powered by DeepSeek Harness.
|
|
|
146
164
|
- Ask first (via `ask_user_question`) ONLY when ambiguity materially affects scope, architecture, or data safety; otherwise execute autonomously.
|
|
147
165
|
- Shell: ONLY `bash`. Launch `pwsh.exe` from `bash` when needed.
|
|
148
166
|
- Web: Research/fetch STRICTLY via skills (`agent-reach`, `gh`, `opencli-*`, `smart-search`) or bash CLIs.
|
|
149
|
-
- Tool boundaries: External tools (playwright-cli, gh, ast-grep) are standalone CLIs via bash
|
|
167
|
+
- Tool boundaries: External tools (playwright-cli, gh, ast-grep) are standalone CLIs via `bash`, NEVER require/import as node modules.
|
|
150
168
|
|
|
151
169
|
## Efficiency
|
|
152
170
|
|
|
153
171
|
### Hard Constraints
|
|
154
172
|
|
|
155
173
|
- Retry ONLY when the error or new evidence supports a specific correction; after two failures without progress, switch diagnostic approaches.
|
|
156
|
-
- Windows arguments & paths: use forward slashes in paths to prevent escape corruption. NEVER embed
|
|
174
|
+
- Windows arguments & paths: use forward slashes in paths to prevent escape corruption. NEVER embed scripts or multiline code in shell arguments (`node -e`, `python -c`, `pwsh -Command`). Quote single-line patterns carefully (`ast-grep` patterns STRICTLY in single quotes `'...'` to prevent `$$` expansion). On quoting failures, switch input method rather than re-escaping.
|
|
157
175
|
- Byte-fidelity: default new text files to UTF-8. Preserve existing file encodings, BOMs, line endings, `.gitattributes`, and hook policies unless explicitly requested.
|
|
158
176
|
- Policy denials (`file access denied`, network blocked, approval disabled/rejected) are not command bugs: report; do not bypass. Request wider access via `sandbox_permissions` ONLY when escalation is explicitly available; otherwise do not retry or escalate. Diagnose ordinary lock, permission, or connectivity errors normally.
|
|
159
177
|
- Tool exclusivity: STRICTLY native `glob`/`grep`/`read_batch`/`read`/`edit_batch`/`edit`/`write`. NEVER use shell commands or redirects for workspace search or reading/writing files (`grep`, `rg`, `find`, `ls`, `dir`, `cat`, `type`, `head`, `tail`, `sed`, `awk`, `tee`, `>`/`>>`). Filtering piped command output (`cmd | grep`) and batch removing temporary artifacts (`rm -rf`) are permitted.
|
|
@@ -172,7 +190,7 @@ You are a master autonomous software engineer powered by DeepSeek Harness.
|
|
|
172
190
|
- Modification & execution:
|
|
173
191
|
- File ops: MUST use `edit_batch` for 2+ edits (whether in one file or across multiple files); use `edit` for a single edit, and `write` for new files or ad-hoc Node/Python scripts.
|
|
174
192
|
- Staged refactor: preview, then apply — symbol renames via `lsp rename` (dry-run → `--apply`); structural rewrites via `ast-grep run -p ... -r ...` (then `-U`).
|
|
175
|
-
- Structured data & calculations:
|
|
193
|
+
- Structured data & calculations: Prioritize persistent kernels (`eval_python`, `eval_node`) for stateful computation, inverse kinematics, or data exploration without disk pollution. For deliverable scripts or build pipelines, MUST write to disk first via native `write` and execute by file path; NEVER embed scripts, calculations, or processing in `python -c` or `node -e`, and NEVER dump raw data into context.
|
|
176
194
|
- Shell execution: run heavy tasks (`npm test`, `npm run build`, `vitest`, `npm publish`) standalone to prevent Windows deadlocks; chain lightweight commands via `&&` (`git add ... && git commit ... && git push`) to eliminate round trips.
|
|
177
195
|
- Reporting: diffs and evidence first; zero filler. Routine status ≤ 300 Chinese characters. Answer/explain/review/diagnose/plan: no length cap. Compact locators, lists, and deltas only.
|
|
178
196
|
|
|
@@ -186,7 +204,7 @@ Applies ONLY to change/build/fix.
|
|
|
186
204
|
- Behavior changes: check static diagnostics via primary tools (e.g. `lsp diag <file|dir> -e`, skip if language unsupported; no redundant CLI linters), then run the smallest non-destructive targeted build/test covering affected behavior and internal interfaces.
|
|
187
205
|
- Shared API, schema, or migration changes: run the full related suite plus project-wide type checking where applicable (e.g. `tsc --noEmit`).
|
|
188
206
|
- Report checks, results, and verification gaps; if primary diagnostics are unavailable, use an existing equivalent check when possible.
|
|
189
|
-
- Browser checks: STRICTLY `playwright-cli` post-write. Playwright CLI blocks
|
|
207
|
+
- Browser checks: STRICTLY `playwright-cli` post-write. Playwright CLI blocks `file:` URLs—serve static HTML via loopback HTTP (`http://127.0.0.1:<port>`). `screenshot` takes selector target ONLY (no output filename; auto-saved to .playwright-cli/). NEVER pre-probe (`--help`, `--version`, `node -v`), create dummy pages, or switch tools. Reuse runtime Web GUI or active app URL (loopback for static); stabilize animations before capture. Batch-cleanup sessions/servers/artifacts without redundant listings. On failure, report root cause.
|
|
190
208
|
- Persistent failure: report failed command(s), 1–2 sentence root cause, blast radius, next steps.
|
|
191
209
|
```
|
|
192
210
|
</details>
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-awesome-optimizer",
|
|
3
|
-
"version": "1.
|
|
4
|
-
"description": "Runtime policy guards, composite batch tooling, soft convergence guidance, reactive custom prompt engine, and A/B telemetry plugin for the official DeepSeek Harness (dsh) agent loop.",
|
|
3
|
+
"version": "2.1.0",
|
|
4
|
+
"description": "Runtime policy guards, composite batch tooling, persistent Python & Node eval daemons, soft convergence guidance, reactive custom prompt engine, and A/B telemetry plugin for the official DeepSeek Harness (dsh) agent loop.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.js",
|
|
7
7
|
"types": "src/index.d.ts",
|
|
@@ -26,6 +26,10 @@
|
|
|
26
26
|
"types": "./src/composite-tools.d.ts",
|
|
27
27
|
"default": "./src/composite-tools.js"
|
|
28
28
|
},
|
|
29
|
+
"./persistent-eval": {
|
|
30
|
+
"types": "./src/persistent-eval/index.d.ts",
|
|
31
|
+
"default": "./src/persistent-eval/index.js"
|
|
32
|
+
},
|
|
29
33
|
"./convergence": {
|
|
30
34
|
"types": "./src/convergence.d.ts",
|
|
31
35
|
"default": "./src/convergence.js"
|
package/scripts/benchmark.mjs
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import assert from 'node:assert/strict';
|
|
2
|
+
import { spawnSync } from 'node:child_process';
|
|
2
3
|
import { createTestRuntime, call } from '../test/support/runtime.js';
|
|
3
|
-
import { compareTurnSummaries, PAIRING_ERROR } from '../src/index.js';
|
|
4
|
+
import { compareTurnSummaries, PAIRING_ERROR, StdioWorkerRunner } from '../src/index.js';
|
|
5
|
+
import { PY_WORKER_SCRIPT } from '../src/persistent-eval/daemon-pool.js';
|
|
4
6
|
|
|
5
7
|
// Run deterministic A/B verification scenarios over the unchanged official @deepseek-ai/dsh-agent-loop.
|
|
6
8
|
async function runBatchReadScenario() {
|
|
@@ -101,13 +103,62 @@ async function runWriteRecoveryScenario() {
|
|
|
101
103
|
}
|
|
102
104
|
}
|
|
103
105
|
|
|
106
|
+
async function runPersistentEvalBenchmarkScenario() {
|
|
107
|
+
const code = 'import math\nr = math.sqrt(144) + 10\nprint(f"r={r}")';
|
|
108
|
+
|
|
109
|
+
// 1. Baseline: 3 consecutive cold process launches via python
|
|
110
|
+
const coldDurations = [];
|
|
111
|
+
for (let i = 0; i < 3; i++) {
|
|
112
|
+
const t0 = performance.now();
|
|
113
|
+
const res = spawnSync('python', ['-c', code], { encoding: 'utf-8' });
|
|
114
|
+
const t1 = performance.now();
|
|
115
|
+
assert.equal(res.status, 0);
|
|
116
|
+
coldDurations.push(t1 - t0);
|
|
117
|
+
}
|
|
118
|
+
const avgColdMs = Number((coldDurations.reduce((a, b) => a + b, 0) / coldDurations.length).toFixed(2));
|
|
119
|
+
|
|
120
|
+
// 2. Candidate: persistent eval worker (hot stateful execution)
|
|
121
|
+
const runner = new StdioWorkerRunner({
|
|
122
|
+
command: 'python',
|
|
123
|
+
args: ['-u', PY_WORKER_SCRIPT]
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
await runner.ensureStarted();
|
|
127
|
+
const hotDurations = [];
|
|
128
|
+
try {
|
|
129
|
+
for (let i = 0; i < 3; i++) {
|
|
130
|
+
const t0 = performance.now();
|
|
131
|
+
const res = await runner.execute(code);
|
|
132
|
+
const t1 = performance.now();
|
|
133
|
+
assert.equal(res.success, true);
|
|
134
|
+
hotDurations.push(t1 - t0);
|
|
135
|
+
}
|
|
136
|
+
} finally {
|
|
137
|
+
runner.destroy();
|
|
138
|
+
}
|
|
139
|
+
const avgHotMs = Number((hotDurations.reduce((a, b) => a + b, 0) / hotDurations.length).toFixed(2));
|
|
140
|
+
const speedupRatio = Number(((avgColdMs - avgHotMs) / avgColdMs * 100).toFixed(1));
|
|
141
|
+
|
|
142
|
+
assert.ok(avgHotMs <= 15, `Expected hot execution <= 15ms but got ${avgHotMs}ms`);
|
|
143
|
+
assert.ok(speedupRatio >= 85, `Expected speedup >= 85% but got ${speedupRatio}%`);
|
|
144
|
+
|
|
145
|
+
return {
|
|
146
|
+
scenario: 'persistent-kernel-vs-cold-spawn',
|
|
147
|
+
baselineAvgColdMs: avgColdMs,
|
|
148
|
+
candidateAvgHotMs: avgHotMs,
|
|
149
|
+
speedupPercentage: `${speedupRatio}%`,
|
|
150
|
+
targetMet: true
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
|
|
104
154
|
const batchScenario = await runBatchReadScenario();
|
|
105
155
|
const recoveryScenario = await runWriteRecoveryScenario();
|
|
156
|
+
const persistentEvalScenario = await runPersistentEvalBenchmarkScenario();
|
|
106
157
|
|
|
107
158
|
console.log(JSON.stringify({
|
|
108
159
|
mode: 'official-loop-ab-verification',
|
|
109
160
|
officialLoopReplaced: false,
|
|
110
161
|
networkRequests: 0,
|
|
111
162
|
note: 'Offline scripted verification proves mechanism and telemetry fidelity. Run identical live tasks to evaluate model-specific round-trip reductions.',
|
|
112
|
-
scenarios: [batchScenario, recoveryScenario]
|
|
163
|
+
scenarios: [batchScenario, recoveryScenario, persistentEvalScenario]
|
|
113
164
|
}, null, 2));
|
package/src/custom-prompt.js
CHANGED
|
@@ -66,6 +66,7 @@ export function formatEnvironmentFacts(processRef = process, pythonDetector = de
|
|
|
66
66
|
'- Standalone CLIs: playwright-cli, gh, ast-grep, and opencli are standalone binaries via bash; NEVER require/import as node modules.',
|
|
67
67
|
'- Browser automation: playwright-cli blocks "file:" URLs (serve via loopback HTTP); \'screenshot\' takes selector target ONLY (no output filename; auto-saved to .playwright-cli/). NEVER probe via \'--help\' or \'--version\'.',
|
|
68
68
|
'- Zero startup probing: workspace context and host facts are already hydrated. NEVER run exploratory probing (pwd, ls, node -v, which).',
|
|
69
|
+
'- Persistent evaluation: `eval_python` and `eval_node` maintain stateful in-memory kernels across turns without disk pollution (math, re, json, time, collections, itertools pre-imported); prioritize for intermediate math, kinematic solving, and data filtering.',
|
|
69
70
|
'- Ad-hoc scripts & cleanup: MUST write ad-hoc Node/Python scripts to disk via native write first and execute by file path; remove temporary artifacts directly via bash (rm -rf).'
|
|
70
71
|
].join('\n');
|
|
71
72
|
}
|
|
@@ -351,9 +352,11 @@ export function applyCustomPrompt(ctx, rawConfig = {}) {
|
|
|
351
352
|
return assembly;
|
|
352
353
|
});
|
|
353
354
|
|
|
354
|
-
ctx.
|
|
355
|
-
|
|
356
|
-
|
|
355
|
+
if (typeof ctx.effect === 'function') {
|
|
356
|
+
ctx.effect(() => () => {
|
|
357
|
+
manager.dispose();
|
|
358
|
+
});
|
|
359
|
+
}
|
|
357
360
|
|
|
358
361
|
return manager;
|
|
359
362
|
}
|
package/src/index.d.ts
CHANGED
|
@@ -3,6 +3,7 @@ import type z from '@deepseek-ai/schemastery';
|
|
|
3
3
|
import type { RuntimePolicyOptions } from './runtime-policy.d.ts';
|
|
4
4
|
import type { FileMutationPolicyOptions } from './file-mutation-policy.d.ts';
|
|
5
5
|
import type { CompositeToolsOptions } from './composite-tools.d.ts';
|
|
6
|
+
import type { PersistentEvalOptions, PersistentEvalService } from './persistent-eval/index.d.ts';
|
|
6
7
|
import type { ConvergenceOptions } from './convergence.d.ts';
|
|
7
8
|
import type { CustomPromptOptions } from './custom-prompt.d.ts';
|
|
8
9
|
import type { TelemetryOptions } from './telemetry.d.ts';
|
|
@@ -15,6 +16,7 @@ export interface AgentOptimizerConfig {
|
|
|
15
16
|
mutationPolicy?: FileMutationPolicyOptions;
|
|
16
17
|
recoverFailedWrites?: boolean;
|
|
17
18
|
compositeTools?: CompositeToolsOptions;
|
|
19
|
+
persistentEval?: PersistentEvalOptions;
|
|
18
20
|
convergence?: ConvergenceOptions;
|
|
19
21
|
customPrompt?: CustomPromptOptions;
|
|
20
22
|
telemetry?: TelemetryOptions;
|
|
@@ -88,6 +90,18 @@ export {
|
|
|
88
90
|
type TurnTelemetrySummary,
|
|
89
91
|
type TurnComparisonDelta
|
|
90
92
|
} from './telemetry.d.ts';
|
|
93
|
+
export {
|
|
94
|
+
EVAL_PYTHON_TOOL,
|
|
95
|
+
EVAL_NODE_TOOL,
|
|
96
|
+
formatEvalOutput,
|
|
97
|
+
applyPersistentEval,
|
|
98
|
+
PersistentEvalConfig,
|
|
99
|
+
type PersistentEvalOptions,
|
|
100
|
+
type PersistentEvalResponse,
|
|
101
|
+
type PersistentEvalService,
|
|
102
|
+
StdioWorkerRunner,
|
|
103
|
+
DaemonPool
|
|
104
|
+
} from './persistent-eval/index.d.ts';
|
|
91
105
|
export {
|
|
92
106
|
lintPrompt,
|
|
93
107
|
fixPrompt,
|
package/src/index.js
CHANGED
|
@@ -3,6 +3,7 @@ import { applyRuntimePolicy, RuntimePolicyConfig, RuntimePolicyManager, POLICY_S
|
|
|
3
3
|
import { applyFileMutationPolicy, FileMutationPolicyConfig, FileMutationPolicyManager, MUTATION_SOURCE } from './file-mutation-policy.js';
|
|
4
4
|
import { applyFailureRecovery, RecoveryRecord, RECOVERY_SOURCE, RETRY_TOOL, PAIRING_ERROR } from './failure-recovery.js';
|
|
5
5
|
import { applyCompositeTools, CompositeToolsConfig, READ_BATCH_TOOL, EDIT_BATCH_TOOL } from './composite-tools.js';
|
|
6
|
+
import { applyPersistentEval, PersistentEvalConfig, EVAL_PYTHON_TOOL, EVAL_NODE_TOOL, formatEvalOutput, StdioWorkerRunner, DaemonPool } from './persistent-eval/index.js';
|
|
6
7
|
import { applyConvergence, ConvergenceConfig, SOURCE as CONVERGENCE_SOURCE, GUIDANCE, REMINDER } from './convergence.js';
|
|
7
8
|
import { applyCustomPrompt, CustomPromptConfig, CustomPromptManager, SECTION_NAME as CUSTOM_PROMPT_SECTION, ENVIRONMENT_FACTS_SECTION, formatEnvironmentFacts, detectPythonEnv, resetPythonCacheForTesting } from './custom-prompt.js';
|
|
8
9
|
import { applyTelemetry, TelemetryConfig, AgentTelemetryManager, compareTurnSummaries } from './telemetry.js';
|
|
@@ -20,6 +21,7 @@ export const Config = z.object({
|
|
|
20
21
|
mutationPolicy: FileMutationPolicyConfig.default({}),
|
|
21
22
|
recoverFailedWrites: z.boolean().default(true),
|
|
22
23
|
compositeTools: CompositeToolsConfig.default({}),
|
|
24
|
+
persistentEval: PersistentEvalConfig.default({}),
|
|
23
25
|
convergence: ConvergenceConfig.default({}),
|
|
24
26
|
customPrompt: CustomPromptConfig.default({}),
|
|
25
27
|
telemetry: TelemetryConfig.default({})
|
|
@@ -27,7 +29,8 @@ export const Config = z.object({
|
|
|
27
29
|
|
|
28
30
|
/**
|
|
29
31
|
* Mount runtime policy guards, write recovery, composite batch tools,
|
|
30
|
-
* soft convergence guidance, and objective turn telemetry
|
|
32
|
+
* persistent evaluation kernels, soft convergence guidance, and objective turn telemetry
|
|
33
|
+
* around the official DSH agent loop.
|
|
31
34
|
* Never disables @deepseek-ai/dsh-agent-loop or mutates agent/request parameters.
|
|
32
35
|
*/
|
|
33
36
|
export function apply(ctx, raw = {}) {
|
|
@@ -54,6 +57,13 @@ export function apply(ctx, raw = {}) {
|
|
|
54
57
|
const compositeService = applyCompositeTools(ctx, compositeConfig);
|
|
55
58
|
if (compositeService) ctx.provide('compositeTools', compositeService);
|
|
56
59
|
|
|
60
|
+
const evalConfig = raw.persistentEval ?? config.persistentEval ?? {};
|
|
61
|
+
const evalService = applyPersistentEval(ctx, evalConfig);
|
|
62
|
+
if (evalService) {
|
|
63
|
+
ctx.provide('persistentEval');
|
|
64
|
+
ctx.persistentEval = evalService;
|
|
65
|
+
}
|
|
66
|
+
|
|
57
67
|
const convergenceConfig = raw.convergence ?? config.convergence ?? {};
|
|
58
68
|
const convergenceService = applyConvergence(ctx, convergenceConfig);
|
|
59
69
|
if (convergenceService) ctx.provide('convergenceGuidance', convergenceService);
|
|
@@ -75,6 +85,13 @@ export {
|
|
|
75
85
|
compareTurnSummaries,
|
|
76
86
|
READ_BATCH_TOOL,
|
|
77
87
|
EDIT_BATCH_TOOL,
|
|
88
|
+
EVAL_PYTHON_TOOL,
|
|
89
|
+
EVAL_NODE_TOOL,
|
|
90
|
+
formatEvalOutput,
|
|
91
|
+
applyPersistentEval,
|
|
92
|
+
PersistentEvalConfig,
|
|
93
|
+
StdioWorkerRunner,
|
|
94
|
+
DaemonPool,
|
|
78
95
|
RETRY_TOOL,
|
|
79
96
|
PAIRING_ERROR,
|
|
80
97
|
CONVERGENCE_SOURCE,
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
import { fileURLToPath } from 'node:url';
|
|
3
|
+
import { StdioWorkerRunner } from './stdio-runner.js';
|
|
4
|
+
|
|
5
|
+
const __filename = fileURLToPath(import.meta.url);
|
|
6
|
+
const __dirname = path.dirname(__filename);
|
|
7
|
+
|
|
8
|
+
export const PY_WORKER_SCRIPT = path.resolve(__dirname, 'workers/py_worker.py');
|
|
9
|
+
export const NODE_WORKER_SCRIPT = path.resolve(__dirname, 'workers/node_worker.mjs');
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Session-scoped daemon pool managing persistent Python and Node worker runners.
|
|
13
|
+
*/
|
|
14
|
+
export class DaemonPool {
|
|
15
|
+
/**
|
|
16
|
+
* @param {object} config
|
|
17
|
+
* @param {number} [config.timeoutMs=15000]
|
|
18
|
+
* @param {number} [config.maxOutputBytes=8192]
|
|
19
|
+
*/
|
|
20
|
+
constructor(config = {}) {
|
|
21
|
+
this.config = config;
|
|
22
|
+
this.runnersBySession = new Map();
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
_getKey(session) {
|
|
26
|
+
if (!session) return '__global__';
|
|
27
|
+
return session.id || session;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
getPythonRunner(session) {
|
|
31
|
+
const key = this._getKey(session);
|
|
32
|
+
let sessionMap = this.runnersBySession.get(key);
|
|
33
|
+
if (!sessionMap) {
|
|
34
|
+
sessionMap = {};
|
|
35
|
+
this.runnersBySession.set(key, sessionMap);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
if (!sessionMap.python || sessionMap.python.isDestroyed) {
|
|
39
|
+
sessionMap.python = new StdioWorkerRunner({
|
|
40
|
+
command: 'python',
|
|
41
|
+
args: ['-u', PY_WORKER_SCRIPT],
|
|
42
|
+
timeoutMs: this.config.timeoutMs,
|
|
43
|
+
maxOutputBytes: this.config.maxOutputBytes
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
return sessionMap.python;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
getNodeRunner(session) {
|
|
51
|
+
const key = this._getKey(session);
|
|
52
|
+
let sessionMap = this.runnersBySession.get(key);
|
|
53
|
+
if (!sessionMap) {
|
|
54
|
+
sessionMap = {};
|
|
55
|
+
this.runnersBySession.set(key, sessionMap);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
if (!sessionMap.node || sessionMap.node.isDestroyed) {
|
|
59
|
+
sessionMap.node = new StdioWorkerRunner({
|
|
60
|
+
command: 'node',
|
|
61
|
+
args: [NODE_WORKER_SCRIPT],
|
|
62
|
+
timeoutMs: this.config.timeoutMs,
|
|
63
|
+
maxOutputBytes: this.config.maxOutputBytes
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
return sessionMap.node;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Pre-warm worker processes in background so they are ready before tool execution.
|
|
72
|
+
* Silently swallows startup failures (e.g. missing runtime) so it never interrupts session flow.
|
|
73
|
+
* @param {any} [session]
|
|
74
|
+
* @param {object} [options]
|
|
75
|
+
* @param {boolean} [options.python=true]
|
|
76
|
+
* @param {boolean} [options.node=true]
|
|
77
|
+
*/
|
|
78
|
+
async prewarm(session, { python = true, node = true } = {}) {
|
|
79
|
+
const promises = [];
|
|
80
|
+
if (python) {
|
|
81
|
+
try {
|
|
82
|
+
const py = this.getPythonRunner(session);
|
|
83
|
+
promises.push(py.ensureStarted().catch(() => {}));
|
|
84
|
+
} catch {
|
|
85
|
+
// Ignore prewarm failure
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
if (node) {
|
|
89
|
+
try {
|
|
90
|
+
const nd = this.getNodeRunner(session);
|
|
91
|
+
promises.push(nd.ensureStarted().catch(() => {}));
|
|
92
|
+
} catch {
|
|
93
|
+
// Ignore prewarm failure
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
await Promise.all(promises);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
destroySession(session) {
|
|
100
|
+
const key = this._getKey(session);
|
|
101
|
+
const sessionMap = this.runnersBySession.get(key);
|
|
102
|
+
if (!sessionMap) return;
|
|
103
|
+
|
|
104
|
+
if (sessionMap.python) {
|
|
105
|
+
sessionMap.python.destroy();
|
|
106
|
+
}
|
|
107
|
+
if (sessionMap.node) {
|
|
108
|
+
sessionMap.node.destroy();
|
|
109
|
+
}
|
|
110
|
+
this.runnersBySession.delete(key);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
destroyAll() {
|
|
114
|
+
for (const [key, sessionMap] of this.runnersBySession.entries()) {
|
|
115
|
+
if (sessionMap.python) sessionMap.python.destroy();
|
|
116
|
+
if (sessionMap.node) sessionMap.node.destroy();
|
|
117
|
+
}
|
|
118
|
+
this.runnersBySession.clear();
|
|
119
|
+
}
|
|
120
|
+
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { Context } from '@deepseek-ai/cordis';
|
|
2
|
+
import z from '@deepseek-ai/schemastery';
|
|
3
|
+
|
|
4
|
+
export const EVAL_PYTHON_TOOL: 'eval_python';
|
|
5
|
+
export const EVAL_NODE_TOOL: 'eval_node';
|
|
6
|
+
|
|
7
|
+
export interface PersistentEvalOptions {
|
|
8
|
+
enabled?: boolean;
|
|
9
|
+
python?: boolean;
|
|
10
|
+
node?: boolean;
|
|
11
|
+
timeoutMs?: number;
|
|
12
|
+
maxOutputBytes?: number;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export declare const PersistentEvalConfig: z<PersistentEvalOptions>;
|
|
16
|
+
|
|
17
|
+
export interface PersistentEvalResponse {
|
|
18
|
+
id?: string;
|
|
19
|
+
success: boolean;
|
|
20
|
+
stdout?: string;
|
|
21
|
+
stderr?: string;
|
|
22
|
+
result?: any;
|
|
23
|
+
resultRepr?: string;
|
|
24
|
+
error?: string;
|
|
25
|
+
executionTimeMs?: number;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export declare function formatEvalOutput(resp: PersistentEvalResponse | null | undefined): string;
|
|
29
|
+
|
|
30
|
+
export interface PersistentEvalService {
|
|
31
|
+
pool: any;
|
|
32
|
+
config: PersistentEvalOptions;
|
|
33
|
+
prewarm(session?: unknown): Promise<void>;
|
|
34
|
+
destroy(): void;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export declare class StdioWorkerRunner {
|
|
38
|
+
constructor(options: any);
|
|
39
|
+
ensureStarted(): Promise<void>;
|
|
40
|
+
execute(code: string, options?: any): Promise<PersistentEvalResponse>;
|
|
41
|
+
destroy(): void;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export declare class DaemonPool {
|
|
45
|
+
constructor(config?: any);
|
|
46
|
+
prewarm(session?: any, options?: { python?: boolean; node?: boolean }): Promise<void>;
|
|
47
|
+
getPythonRunner(session?: any): StdioWorkerRunner;
|
|
48
|
+
getNodeRunner(session?: any): StdioWorkerRunner;
|
|
49
|
+
destroySession(session?: any): void;
|
|
50
|
+
destroyAll(): void;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export declare function applyPersistentEval(
|
|
54
|
+
ctx: Context,
|
|
55
|
+
rawConfig?: PersistentEvalOptions
|
|
56
|
+
): PersistentEvalService | null;
|