dsh-awesome-optimizer 1.8.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -15
- package/README.zh.md +16 -15
- package/package.json +6 -2
- package/scripts/benchmark.mjs +53 -2
- package/src/custom-prompt.js +1 -0
- package/src/index.d.ts +14 -0
- package/src/index.js +15 -1
- package/src/persistent-eval/daemon-pool.js +91 -0
- package/src/persistent-eval/index.d.ts +54 -0
- package/src/persistent-eval/index.js +161 -0
- package/src/persistent-eval/stdio-runner.js +270 -0
- package/src/persistent-eval/workers/node_worker.mjs +238 -0
- package/src/persistent-eval/workers/py_worker.py +182 -0
package/README.md
CHANGED
|
@@ -5,22 +5,23 @@ English | [中文](README.zh.md)
|
|
|
5
5
|
[](https://www.npmjs.com/package/dsh-awesome-optimizer)
|
|
6
6
|
[](LICENSE)
|
|
7
7
|
[](https://github.com/Jaylor-Wang/dsh-awesome-optimizer/releases)
|
|
8
|
-
[](test/)
|
|
9
9
|
[](src/)
|
|
10
10
|
|
|
11
|
-
High-performance runtime policy guards, PTC-backed composite batch tooling, standalone refined system prompt engine, and objective A/B turn telemetry plugin for the official [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (`@deepseek-ai/dsh-agent-loop`).
|
|
11
|
+
High-performance runtime policy guards, persistent Python & Node evaluation daemons, PTC-backed composite batch tooling, standalone refined system prompt engine, and objective A/B turn telemetry plugin for the official [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (`@deepseek-ai/dsh-agent-loop`).
|
|
12
12
|
|
|
13
13
|
---
|
|
14
14
|
|
|
15
|
-
## Highlights in
|
|
15
|
+
## Highlights in v2.0.0
|
|
16
16
|
|
|
17
|
-
- **
|
|
18
|
-
-
|
|
19
|
-
- **
|
|
20
|
-
-
|
|
21
|
-
- **
|
|
22
|
-
|
|
23
|
-
- **
|
|
17
|
+
- **Persistent Python & Node Evaluation Daemons (`eval_python` / `eval_node`)**:
|
|
18
|
+
Zero-dependency stateful execution kernels over line-delimited stdio JSON frames. Retains global scopes, functions, imports, and variables across turns (with Node top-level `await`). Slashes repetitive calculation latency from 37ms to **0.65ms (98.3% speedup)**, eliminating temporary file pollution.
|
|
19
|
+
- **Watchdog Timeout & Self-Healing Auto-Restart**:
|
|
20
|
+
Built-in execution watchdog forces termination of infinite loops (`while True:`) or blocked input across platforms without freezing the host process, automatically spinning up fresh instances for seamless queued recovery.
|
|
21
|
+
- **Anti-Dumping Output Guard**:
|
|
22
|
+
Enforces configurable payload truncation (default 8KB) with disk-redirect guidance, directly reinforcing the `NEVER dump raw data into context` directive.
|
|
23
|
+
- **Evaluation vs Engineering Code Separation**:
|
|
24
|
+
Formalizes prompt directives ensuring intermediate calculations leverage persistent in-memory kernels while deliverable production files are committed to disk via native filesystem operations.
|
|
24
25
|
|
|
25
26
|
---
|
|
26
27
|
|
|
@@ -136,7 +137,7 @@ You are a master autonomous software engineer powered by DeepSeek Harness.
|
|
|
136
137
|
|
|
137
138
|
- Prioritize direct action over speculation: investigate via concurrent tools (fs, bash, skills) without orchestration overhead.
|
|
138
139
|
- NEVER emit pre-action narration. Deliver surgical edits and high-signal conclusions.
|
|
139
|
-
- Irreversible operations (npm publish): STRICTLY conservative and non-autonomous. NEVER bump versions or mutate artifacts to bypass failures; registry validation causes transient 404 or stale views—NEVER treat as failure or retry publish. MUST verify remote state with read-only commands before declaring status.
|
|
140
|
+
- Irreversible operations (`npm publish`): STRICTLY conservative and non-autonomous. NEVER bump versions or mutate artifacts to bypass failures; registry validation causes transient 404 or stale views—NEVER treat as failure or retry publish. MUST verify remote state with read-only commands before declaring status.
|
|
140
141
|
|
|
141
142
|
## Defaults
|
|
142
143
|
|
|
@@ -146,14 +147,14 @@ You are a master autonomous software engineer powered by DeepSeek Harness.
|
|
|
146
147
|
- Ask first (via `ask_user_question`) ONLY when ambiguity materially affects scope, architecture, or data safety; otherwise execute autonomously.
|
|
147
148
|
- Shell: ONLY `bash`. Launch `pwsh.exe` from `bash` when needed.
|
|
148
149
|
- Web: Research/fetch STRICTLY via skills (`agent-reach`, `gh`, `opencli-*`, `smart-search`) or bash CLIs.
|
|
149
|
-
- Tool boundaries: External tools (playwright-cli, gh, ast-grep) are standalone CLIs via bash
|
|
150
|
+
- Tool boundaries: External tools (playwright-cli, gh, ast-grep) are standalone CLIs via `bash`, NEVER require/import as node modules.
|
|
150
151
|
|
|
151
152
|
## Efficiency
|
|
152
153
|
|
|
153
154
|
### Hard Constraints
|
|
154
155
|
|
|
155
156
|
- Retry ONLY when the error or new evidence supports a specific correction; after two failures without progress, switch diagnostic approaches.
|
|
156
|
-
- Windows arguments & paths: use forward slashes in paths to prevent escape corruption. NEVER embed
|
|
157
|
+
- Windows arguments & paths: use forward slashes in paths to prevent escape corruption. NEVER embed scripts or multiline code in shell arguments (`node -e`, `python -c`, `pwsh -Command`). Quote single-line patterns carefully (`ast-grep` patterns STRICTLY in single quotes `'...'` to prevent `$$` expansion). On quoting failures, switch input method rather than re-escaping.
|
|
157
158
|
- Byte-fidelity: default new text files to UTF-8. Preserve existing file encodings, BOMs, line endings, `.gitattributes`, and hook policies unless explicitly requested.
|
|
158
159
|
- Policy denials (`file access denied`, network blocked, approval disabled/rejected) are not command bugs: report; do not bypass. Request wider access via `sandbox_permissions` ONLY when escalation is explicitly available; otherwise do not retry or escalate. Diagnose ordinary lock, permission, or connectivity errors normally.
|
|
159
160
|
- Tool exclusivity: STRICTLY native `glob`/`grep`/`read_batch`/`read`/`edit_batch`/`edit`/`write`. NEVER use shell commands or redirects for workspace search or reading/writing files (`grep`, `rg`, `find`, `ls`, `dir`, `cat`, `type`, `head`, `tail`, `sed`, `awk`, `tee`, `>`/`>>`). Filtering piped command output (`cmd | grep`) and batch removing temporary artifacts (`rm -rf`) are permitted.
|
|
@@ -172,7 +173,7 @@ You are a master autonomous software engineer powered by DeepSeek Harness.
|
|
|
172
173
|
- Modification & execution:
|
|
173
174
|
- File ops: MUST use `edit_batch` for 2+ edits (whether in one file or across multiple files); use `edit` for a single edit, and `write` for new files or ad-hoc Node/Python scripts.
|
|
174
175
|
- Staged refactor: preview, then apply — symbol renames via `lsp rename` (dry-run → `--apply`); structural rewrites via `ast-grep run -p ... -r ...` (then `-U`).
|
|
175
|
-
- Structured data & calculations:
|
|
176
|
+
- Structured data & calculations: Prioritize persistent kernels (`eval_python`, `eval_node`) for stateful computation, inverse kinematics, or data exploration without disk pollution. For deliverable scripts or build pipelines, MUST write to disk first via native `write` and execute by file path; NEVER embed scripts, calculations, or processing in `python -c` or `node -e`, and NEVER dump raw data into context.
|
|
176
177
|
- Shell execution: run heavy tasks (`npm test`, `npm run build`, `vitest`, `npm publish`) standalone to prevent Windows deadlocks; chain lightweight commands via `&&` (`git add ... && git commit ... && git push`) to eliminate round trips.
|
|
177
178
|
- Reporting: diffs and evidence first; zero filler. Routine status ≤ 300 Chinese characters. Answer/explain/review/diagnose/plan: no length cap. Compact locators, lists, and deltas only.
|
|
178
179
|
|
|
@@ -186,7 +187,7 @@ Applies ONLY to change/build/fix.
|
|
|
186
187
|
- Behavior changes: check static diagnostics via primary tools (e.g. `lsp diag <file|dir> -e`, skip if language unsupported; no redundant CLI linters), then run the smallest non-destructive targeted build/test covering affected behavior and internal interfaces.
|
|
187
188
|
- Shared API, schema, or migration changes: run the full related suite plus project-wide type checking where applicable (e.g. `tsc --noEmit`).
|
|
188
189
|
- Report checks, results, and verification gaps; if primary diagnostics are unavailable, use an existing equivalent check when possible.
|
|
189
|
-
- Browser checks: STRICTLY `playwright-cli` post-write. Playwright CLI blocks
|
|
190
|
+
- Browser checks: STRICTLY `playwright-cli` post-write. Playwright CLI blocks `file:` URLs—serve static HTML via loopback HTTP (`http://127.0.0.1:<port>`). `screenshot` takes selector target ONLY (no output filename; auto-saved to .playwright-cli/). NEVER pre-probe (`--help`, `--version`, `node -v`), create dummy pages, or switch tools. Reuse runtime Web GUI or active app URL (loopback for static); stabilize animations before capture. Batch-cleanup sessions/servers/artifacts without redundant listings. On failure, report root cause.
|
|
190
191
|
- Persistent failure: report failed command(s), 1–2 sentence root cause, blast radius, next steps.
|
|
191
192
|
```
|
|
192
193
|
</details>
|
package/README.zh.md
CHANGED
|
@@ -5,22 +5,23 @@
|
|
|
5
5
|
[](https://www.npmjs.com/package/dsh-awesome-optimizer)
|
|
6
6
|
[](LICENSE)
|
|
7
7
|
[](https://github.com/Jaylor-Wang/dsh-awesome-optimizer/releases)
|
|
8
|
-
[](test/)
|
|
9
9
|
[](src/)
|
|
10
10
|
|
|
11
|
-
围绕官方 [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (`@deepseek-ai/dsh-agent-loop`)
|
|
11
|
+
围绕官方 [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (`@deepseek-ai/dsh-agent-loop`) 打造的高性能运行时策略防护、持久化 Python/Node 执行内核、PTC 复合批处理工具、全新独立系统提示词接管与客观 A/B 轮次度量插件。
|
|
12
12
|
|
|
13
13
|
---
|
|
14
14
|
|
|
15
|
-
##
|
|
15
|
+
## v2.0.0 版本重大里程碑更新
|
|
16
16
|
|
|
17
|
-
-
|
|
18
|
-
|
|
19
|
-
-
|
|
20
|
-
|
|
21
|
-
-
|
|
22
|
-
|
|
23
|
-
-
|
|
17
|
+
- **持久化 Python 与 Node 执行内核 (`eval_python` / `eval_node`)**:
|
|
18
|
+
引入基于轻量级 stdio 行定界 JSON 协议帧的双语常驻守护引擎,**零外部依赖**(无需 `ipykernel`、`pyzmq` 等重型外部包)。在会话生命周期内跨调用保留全局变量、导入模块、类与函数,支持 Node 顶层 `await`。将算法、逆运动学与几何试算单次延迟从 37ms 直接压降至 **0.65ms(提速 98.3%)**,彻底终结多轮试算频繁向磁盘落盘临时脚本的损耗与文件污染。
|
|
19
|
+
- **看门狗超时强杀与自愈热重启 (Watchdog & Self-Healing)**:
|
|
20
|
+
内置细粒度硬执行看门狗。当代码陷入死循环(如 `while True:`)或阻塞输入超时(默认 15s)后,主进程永不卡死挂起,跨平台强杀卡死子进程树并自动拉起新生干净实例,后续排队任务零感平滑恢复。
|
|
21
|
+
- **上下文防冲爆守卫 (Anti-Dumping Interceptor)**:
|
|
22
|
+
在工具管道层对输出实施 8KB 阈值截断并追加引导落盘说明,与 `NEVER dump raw data into context` 规则深度协同,彻底杜绝超大输出冲爆 LLM 上下文。
|
|
23
|
+
- **心智分流规约升维**:
|
|
24
|
+
在 `SYSTEM.md` 与 Custom Prompt 中确立“探索性解算/数据筛选优先走持久内核,工程交付物必须通过 native `write` 落盘”的最高指令分流契约。
|
|
24
25
|
|
|
25
26
|
---
|
|
26
27
|
|
|
@@ -136,7 +137,7 @@ You are a master autonomous software engineer powered by DeepSeek Harness.
|
|
|
136
137
|
|
|
137
138
|
- Prioritize direct action over speculation: investigate via concurrent tools (fs, bash, skills) without orchestration overhead.
|
|
138
139
|
- NEVER emit pre-action narration. Deliver surgical edits and high-signal conclusions.
|
|
139
|
-
- Irreversible operations (npm publish): STRICTLY conservative and non-autonomous. NEVER bump versions or mutate artifacts to bypass failures; registry validation causes transient 404 or stale views—NEVER treat as failure or retry publish. MUST verify remote state with read-only commands before declaring status.
|
|
140
|
+
- Irreversible operations (`npm publish`): STRICTLY conservative and non-autonomous. NEVER bump versions or mutate artifacts to bypass failures; registry validation causes transient 404 or stale views—NEVER treat as failure or retry publish. MUST verify remote state with read-only commands before declaring status.
|
|
140
141
|
|
|
141
142
|
## Defaults
|
|
142
143
|
|
|
@@ -146,14 +147,14 @@ You are a master autonomous software engineer powered by DeepSeek Harness.
|
|
|
146
147
|
- Ask first (via `ask_user_question`) ONLY when ambiguity materially affects scope, architecture, or data safety; otherwise execute autonomously.
|
|
147
148
|
- Shell: ONLY `bash`. Launch `pwsh.exe` from `bash` when needed.
|
|
148
149
|
- Web: Research/fetch STRICTLY via skills (`agent-reach`, `gh`, `opencli-*`, `smart-search`) or bash CLIs.
|
|
149
|
-
- Tool boundaries: External tools (playwright-cli, gh, ast-grep) are standalone CLIs via bash
|
|
150
|
+
- Tool boundaries: External tools (playwright-cli, gh, ast-grep) are standalone CLIs via `bash`, NEVER require/import as node modules.
|
|
150
151
|
|
|
151
152
|
## Efficiency
|
|
152
153
|
|
|
153
154
|
### Hard Constraints
|
|
154
155
|
|
|
155
156
|
- Retry ONLY when the error or new evidence supports a specific correction; after two failures without progress, switch diagnostic approaches.
|
|
156
|
-
- Windows arguments & paths: use forward slashes in paths to prevent escape corruption. NEVER embed
|
|
157
|
+
- Windows arguments & paths: use forward slashes in paths to prevent escape corruption. NEVER embed scripts or multiline code in shell arguments (`node -e`, `python -c`, `pwsh -Command`). Quote single-line patterns carefully (`ast-grep` patterns STRICTLY in single quotes `'...'` to prevent `$$` expansion). On quoting failures, switch input method rather than re-escaping.
|
|
157
158
|
- Byte-fidelity: default new text files to UTF-8. Preserve existing file encodings, BOMs, line endings, `.gitattributes`, and hook policies unless explicitly requested.
|
|
158
159
|
- Policy denials (`file access denied`, network blocked, approval disabled/rejected) are not command bugs: report; do not bypass. Request wider access via `sandbox_permissions` ONLY when escalation is explicitly available; otherwise do not retry or escalate. Diagnose ordinary lock, permission, or connectivity errors normally.
|
|
159
160
|
- Tool exclusivity: STRICTLY native `glob`/`grep`/`read_batch`/`read`/`edit_batch`/`edit`/`write`. NEVER use shell commands or redirects for workspace search or reading/writing files (`grep`, `rg`, `find`, `ls`, `dir`, `cat`, `type`, `head`, `tail`, `sed`, `awk`, `tee`, `>`/`>>`). Filtering piped command output (`cmd | grep`) and batch removing temporary artifacts (`rm -rf`) are permitted.
|
|
@@ -172,7 +173,7 @@ You are a master autonomous software engineer powered by DeepSeek Harness.
|
|
|
172
173
|
- Modification & execution:
|
|
173
174
|
- File ops: MUST use `edit_batch` for 2+ edits (whether in one file or across multiple files); use `edit` for a single edit, and `write` for new files or ad-hoc Node/Python scripts.
|
|
174
175
|
- Staged refactor: preview, then apply — symbol renames via `lsp rename` (dry-run → `--apply`); structural rewrites via `ast-grep run -p ... -r ...` (then `-U`).
|
|
175
|
-
- Structured data & calculations:
|
|
176
|
+
- Structured data & calculations: Prioritize persistent kernels (`eval_python`, `eval_node`) for stateful computation, inverse kinematics, or data exploration without disk pollution. For deliverable scripts or build pipelines, MUST write to disk first via native `write` and execute by file path; NEVER embed scripts, calculations, or processing in `python -c` or `node -e`, and NEVER dump raw data into context.
|
|
176
177
|
- Shell execution: run heavy tasks (`npm test`, `npm run build`, `vitest`, `npm publish`) standalone to prevent Windows deadlocks; chain lightweight commands via `&&` (`git add ... && git commit ... && git push`) to eliminate round trips.
|
|
177
178
|
- Reporting: diffs and evidence first; zero filler. Routine status ≤ 300 Chinese characters. Answer/explain/review/diagnose/plan: no length cap. Compact locators, lists, and deltas only.
|
|
178
179
|
|
|
@@ -186,7 +187,7 @@ Applies ONLY to change/build/fix.
|
|
|
186
187
|
- Behavior changes: check static diagnostics via primary tools (e.g. `lsp diag <file|dir> -e`, skip if language unsupported; no redundant CLI linters), then run the smallest non-destructive targeted build/test covering affected behavior and internal interfaces.
|
|
187
188
|
- Shared API, schema, or migration changes: run the full related suite plus project-wide type checking where applicable (e.g. `tsc --noEmit`).
|
|
188
189
|
- Report checks, results, and verification gaps; if primary diagnostics are unavailable, use an existing equivalent check when possible.
|
|
189
|
-
- Browser checks: STRICTLY `playwright-cli` post-write. Playwright CLI blocks
|
|
190
|
+
- Browser checks: STRICTLY `playwright-cli` post-write. Playwright CLI blocks `file:` URLs—serve static HTML via loopback HTTP (`http://127.0.0.1:<port>`). `screenshot` takes selector target ONLY (no output filename; auto-saved to .playwright-cli/). NEVER pre-probe (`--help`, `--version`, `node -v`), create dummy pages, or switch tools. Reuse runtime Web GUI or active app URL (loopback for static); stabilize animations before capture. Batch-cleanup sessions/servers/artifacts without redundant listings. On failure, report root cause.
|
|
190
191
|
- Persistent failure: report failed command(s), 1–2 sentence root cause, blast radius, next steps.
|
|
191
192
|
```
|
|
192
193
|
</details>
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-awesome-optimizer",
|
|
3
|
-
"version": "
|
|
4
|
-
"description": "Runtime policy guards, composite batch tooling, soft convergence guidance, reactive custom prompt engine, and A/B telemetry plugin for the official DeepSeek Harness (dsh) agent loop.",
|
|
3
|
+
"version": "2.0.0",
|
|
4
|
+
"description": "Runtime policy guards, composite batch tooling, persistent Python & Node eval daemons, soft convergence guidance, reactive custom prompt engine, and A/B telemetry plugin for the official DeepSeek Harness (dsh) agent loop.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.js",
|
|
7
7
|
"types": "src/index.d.ts",
|
|
@@ -26,6 +26,10 @@
|
|
|
26
26
|
"types": "./src/composite-tools.d.ts",
|
|
27
27
|
"default": "./src/composite-tools.js"
|
|
28
28
|
},
|
|
29
|
+
"./persistent-eval": {
|
|
30
|
+
"types": "./src/persistent-eval/index.d.ts",
|
|
31
|
+
"default": "./src/persistent-eval/index.js"
|
|
32
|
+
},
|
|
29
33
|
"./convergence": {
|
|
30
34
|
"types": "./src/convergence.d.ts",
|
|
31
35
|
"default": "./src/convergence.js"
|
package/scripts/benchmark.mjs
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import assert from 'node:assert/strict';
|
|
2
|
+
import { spawnSync } from 'node:child_process';
|
|
2
3
|
import { createTestRuntime, call } from '../test/support/runtime.js';
|
|
3
|
-
import { compareTurnSummaries, PAIRING_ERROR } from '../src/index.js';
|
|
4
|
+
import { compareTurnSummaries, PAIRING_ERROR, StdioWorkerRunner } from '../src/index.js';
|
|
5
|
+
import { PY_WORKER_SCRIPT } from '../src/persistent-eval/daemon-pool.js';
|
|
4
6
|
|
|
5
7
|
// Run deterministic A/B verification scenarios over the unchanged official @deepseek-ai/dsh-agent-loop.
|
|
6
8
|
async function runBatchReadScenario() {
|
|
@@ -101,13 +103,62 @@ async function runWriteRecoveryScenario() {
|
|
|
101
103
|
}
|
|
102
104
|
}
|
|
103
105
|
|
|
106
|
+
async function runPersistentEvalBenchmarkScenario() {
|
|
107
|
+
const code = 'import math\nr = math.sqrt(144) + 10\nprint(f"r={r}")';
|
|
108
|
+
|
|
109
|
+
// 1. Baseline: 3 consecutive cold process launches via python
|
|
110
|
+
const coldDurations = [];
|
|
111
|
+
for (let i = 0; i < 3; i++) {
|
|
112
|
+
const t0 = performance.now();
|
|
113
|
+
const res = spawnSync('python', ['-c', code], { encoding: 'utf-8' });
|
|
114
|
+
const t1 = performance.now();
|
|
115
|
+
assert.equal(res.status, 0);
|
|
116
|
+
coldDurations.push(t1 - t0);
|
|
117
|
+
}
|
|
118
|
+
const avgColdMs = Number((coldDurations.reduce((a, b) => a + b, 0) / coldDurations.length).toFixed(2));
|
|
119
|
+
|
|
120
|
+
// 2. Candidate: persistent eval worker (hot stateful execution)
|
|
121
|
+
const runner = new StdioWorkerRunner({
|
|
122
|
+
command: 'python',
|
|
123
|
+
args: ['-u', PY_WORKER_SCRIPT]
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
await runner.ensureStarted();
|
|
127
|
+
const hotDurations = [];
|
|
128
|
+
try {
|
|
129
|
+
for (let i = 0; i < 3; i++) {
|
|
130
|
+
const t0 = performance.now();
|
|
131
|
+
const res = await runner.execute(code);
|
|
132
|
+
const t1 = performance.now();
|
|
133
|
+
assert.equal(res.success, true);
|
|
134
|
+
hotDurations.push(t1 - t0);
|
|
135
|
+
}
|
|
136
|
+
} finally {
|
|
137
|
+
runner.destroy();
|
|
138
|
+
}
|
|
139
|
+
const avgHotMs = Number((hotDurations.reduce((a, b) => a + b, 0) / hotDurations.length).toFixed(2));
|
|
140
|
+
const speedupRatio = Number(((avgColdMs - avgHotMs) / avgColdMs * 100).toFixed(1));
|
|
141
|
+
|
|
142
|
+
assert.ok(avgHotMs <= 15, `Expected hot execution <= 15ms but got ${avgHotMs}ms`);
|
|
143
|
+
assert.ok(speedupRatio >= 85, `Expected speedup >= 85% but got ${speedupRatio}%`);
|
|
144
|
+
|
|
145
|
+
return {
|
|
146
|
+
scenario: 'persistent-kernel-vs-cold-spawn',
|
|
147
|
+
baselineAvgColdMs: avgColdMs,
|
|
148
|
+
candidateAvgHotMs: avgHotMs,
|
|
149
|
+
speedupPercentage: `${speedupRatio}%`,
|
|
150
|
+
targetMet: true
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
|
|
104
154
|
const batchScenario = await runBatchReadScenario();
|
|
105
155
|
const recoveryScenario = await runWriteRecoveryScenario();
|
|
156
|
+
const persistentEvalScenario = await runPersistentEvalBenchmarkScenario();
|
|
106
157
|
|
|
107
158
|
console.log(JSON.stringify({
|
|
108
159
|
mode: 'official-loop-ab-verification',
|
|
109
160
|
officialLoopReplaced: false,
|
|
110
161
|
networkRequests: 0,
|
|
111
162
|
note: 'Offline scripted verification proves mechanism and telemetry fidelity. Run identical live tasks to evaluate model-specific round-trip reductions.',
|
|
112
|
-
scenarios: [batchScenario, recoveryScenario]
|
|
163
|
+
scenarios: [batchScenario, recoveryScenario, persistentEvalScenario]
|
|
113
164
|
}, null, 2));
|
package/src/custom-prompt.js
CHANGED
|
@@ -66,6 +66,7 @@ export function formatEnvironmentFacts(processRef = process, pythonDetector = de
|
|
|
66
66
|
'- Standalone CLIs: playwright-cli, gh, ast-grep, and opencli are standalone binaries via bash; NEVER require/import as node modules.',
|
|
67
67
|
'- Browser automation: playwright-cli blocks "file:" URLs (serve via loopback HTTP); \'screenshot\' takes selector target ONLY (no output filename; auto-saved to .playwright-cli/). NEVER probe via \'--help\' or \'--version\'.',
|
|
68
68
|
'- Zero startup probing: workspace context and host facts are already hydrated. NEVER run exploratory probing (pwd, ls, node -v, which).',
|
|
69
|
+
'- Persistent evaluation: `eval_python` and `eval_node` maintain stateful in-memory kernels across turns without disk pollution; prioritize for intermediate math, kinematic solving, and data filtering.',
|
|
69
70
|
'- Ad-hoc scripts & cleanup: MUST write ad-hoc Node/Python scripts to disk via native write first and execute by file path; remove temporary artifacts directly via bash (rm -rf).'
|
|
70
71
|
].join('\n');
|
|
71
72
|
}
|
package/src/index.d.ts
CHANGED
|
@@ -3,6 +3,7 @@ import type z from '@deepseek-ai/schemastery';
|
|
|
3
3
|
import type { RuntimePolicyOptions } from './runtime-policy.d.ts';
|
|
4
4
|
import type { FileMutationPolicyOptions } from './file-mutation-policy.d.ts';
|
|
5
5
|
import type { CompositeToolsOptions } from './composite-tools.d.ts';
|
|
6
|
+
import type { PersistentEvalOptions, PersistentEvalService } from './persistent-eval/index.d.ts';
|
|
6
7
|
import type { ConvergenceOptions } from './convergence.d.ts';
|
|
7
8
|
import type { CustomPromptOptions } from './custom-prompt.d.ts';
|
|
8
9
|
import type { TelemetryOptions } from './telemetry.d.ts';
|
|
@@ -15,6 +16,7 @@ export interface AgentOptimizerConfig {
|
|
|
15
16
|
mutationPolicy?: FileMutationPolicyOptions;
|
|
16
17
|
recoverFailedWrites?: boolean;
|
|
17
18
|
compositeTools?: CompositeToolsOptions;
|
|
19
|
+
persistentEval?: PersistentEvalOptions;
|
|
18
20
|
convergence?: ConvergenceOptions;
|
|
19
21
|
customPrompt?: CustomPromptOptions;
|
|
20
22
|
telemetry?: TelemetryOptions;
|
|
@@ -88,6 +90,18 @@ export {
|
|
|
88
90
|
type TurnTelemetrySummary,
|
|
89
91
|
type TurnComparisonDelta
|
|
90
92
|
} from './telemetry.d.ts';
|
|
93
|
+
export {
|
|
94
|
+
EVAL_PYTHON_TOOL,
|
|
95
|
+
EVAL_NODE_TOOL,
|
|
96
|
+
formatEvalOutput,
|
|
97
|
+
applyPersistentEval,
|
|
98
|
+
PersistentEvalConfig,
|
|
99
|
+
type PersistentEvalOptions,
|
|
100
|
+
type PersistentEvalResponse,
|
|
101
|
+
type PersistentEvalService,
|
|
102
|
+
StdioWorkerRunner,
|
|
103
|
+
DaemonPool
|
|
104
|
+
} from './persistent-eval/index.d.ts';
|
|
91
105
|
export {
|
|
92
106
|
lintPrompt,
|
|
93
107
|
fixPrompt,
|
package/src/index.js
CHANGED
|
@@ -3,6 +3,7 @@ import { applyRuntimePolicy, RuntimePolicyConfig, RuntimePolicyManager, POLICY_S
|
|
|
3
3
|
import { applyFileMutationPolicy, FileMutationPolicyConfig, FileMutationPolicyManager, MUTATION_SOURCE } from './file-mutation-policy.js';
|
|
4
4
|
import { applyFailureRecovery, RecoveryRecord, RECOVERY_SOURCE, RETRY_TOOL, PAIRING_ERROR } from './failure-recovery.js';
|
|
5
5
|
import { applyCompositeTools, CompositeToolsConfig, READ_BATCH_TOOL, EDIT_BATCH_TOOL } from './composite-tools.js';
|
|
6
|
+
import { applyPersistentEval, PersistentEvalConfig, EVAL_PYTHON_TOOL, EVAL_NODE_TOOL, formatEvalOutput, StdioWorkerRunner, DaemonPool } from './persistent-eval/index.js';
|
|
6
7
|
import { applyConvergence, ConvergenceConfig, SOURCE as CONVERGENCE_SOURCE, GUIDANCE, REMINDER } from './convergence.js';
|
|
7
8
|
import { applyCustomPrompt, CustomPromptConfig, CustomPromptManager, SECTION_NAME as CUSTOM_PROMPT_SECTION, ENVIRONMENT_FACTS_SECTION, formatEnvironmentFacts, detectPythonEnv, resetPythonCacheForTesting } from './custom-prompt.js';
|
|
8
9
|
import { applyTelemetry, TelemetryConfig, AgentTelemetryManager, compareTurnSummaries } from './telemetry.js';
|
|
@@ -20,6 +21,7 @@ export const Config = z.object({
|
|
|
20
21
|
mutationPolicy: FileMutationPolicyConfig.default({}),
|
|
21
22
|
recoverFailedWrites: z.boolean().default(true),
|
|
22
23
|
compositeTools: CompositeToolsConfig.default({}),
|
|
24
|
+
persistentEval: PersistentEvalConfig.default({}),
|
|
23
25
|
convergence: ConvergenceConfig.default({}),
|
|
24
26
|
customPrompt: CustomPromptConfig.default({}),
|
|
25
27
|
telemetry: TelemetryConfig.default({})
|
|
@@ -27,7 +29,8 @@ export const Config = z.object({
|
|
|
27
29
|
|
|
28
30
|
/**
|
|
29
31
|
* Mount runtime policy guards, write recovery, composite batch tools,
|
|
30
|
-
* soft convergence guidance, and objective turn telemetry
|
|
32
|
+
* persistent evaluation kernels, soft convergence guidance, and objective turn telemetry
|
|
33
|
+
* around the official DSH agent loop.
|
|
31
34
|
* Never disables @deepseek-ai/dsh-agent-loop or mutates agent/request parameters.
|
|
32
35
|
*/
|
|
33
36
|
export function apply(ctx, raw = {}) {
|
|
@@ -54,6 +57,10 @@ export function apply(ctx, raw = {}) {
|
|
|
54
57
|
const compositeService = applyCompositeTools(ctx, compositeConfig);
|
|
55
58
|
if (compositeService) ctx.provide('compositeTools', compositeService);
|
|
56
59
|
|
|
60
|
+
const evalConfig = raw.persistentEval ?? config.persistentEval ?? {};
|
|
61
|
+
const evalService = applyPersistentEval(ctx, evalConfig);
|
|
62
|
+
if (evalService) ctx.provide('persistentEval', evalService);
|
|
63
|
+
|
|
57
64
|
const convergenceConfig = raw.convergence ?? config.convergence ?? {};
|
|
58
65
|
const convergenceService = applyConvergence(ctx, convergenceConfig);
|
|
59
66
|
if (convergenceService) ctx.provide('convergenceGuidance', convergenceService);
|
|
@@ -75,6 +82,13 @@ export {
|
|
|
75
82
|
compareTurnSummaries,
|
|
76
83
|
READ_BATCH_TOOL,
|
|
77
84
|
EDIT_BATCH_TOOL,
|
|
85
|
+
EVAL_PYTHON_TOOL,
|
|
86
|
+
EVAL_NODE_TOOL,
|
|
87
|
+
formatEvalOutput,
|
|
88
|
+
applyPersistentEval,
|
|
89
|
+
PersistentEvalConfig,
|
|
90
|
+
StdioWorkerRunner,
|
|
91
|
+
DaemonPool,
|
|
78
92
|
RETRY_TOOL,
|
|
79
93
|
PAIRING_ERROR,
|
|
80
94
|
CONVERGENCE_SOURCE,
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
import { fileURLToPath } from 'node:url';
|
|
3
|
+
import { StdioWorkerRunner } from './stdio-runner.js';
|
|
4
|
+
|
|
5
|
+
const __filename = fileURLToPath(import.meta.url);
|
|
6
|
+
const __dirname = path.dirname(__filename);
|
|
7
|
+
|
|
8
|
+
export const PY_WORKER_SCRIPT = path.resolve(__dirname, 'workers/py_worker.py');
|
|
9
|
+
export const NODE_WORKER_SCRIPT = path.resolve(__dirname, 'workers/node_worker.mjs');
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Session-scoped daemon pool managing persistent Python and Node worker runners.
|
|
13
|
+
*/
|
|
14
|
+
export class DaemonPool {
|
|
15
|
+
/**
|
|
16
|
+
* @param {object} config
|
|
17
|
+
* @param {number} [config.timeoutMs=15000]
|
|
18
|
+
* @param {number} [config.maxOutputBytes=8192]
|
|
19
|
+
*/
|
|
20
|
+
constructor(config = {}) {
|
|
21
|
+
this.config = config;
|
|
22
|
+
this.runnersBySession = new Map();
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
_getKey(session) {
|
|
26
|
+
if (!session) return '__global__';
|
|
27
|
+
return session.id || session;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
getPythonRunner(session) {
|
|
31
|
+
const key = this._getKey(session);
|
|
32
|
+
let sessionMap = this.runnersBySession.get(key);
|
|
33
|
+
if (!sessionMap) {
|
|
34
|
+
sessionMap = {};
|
|
35
|
+
this.runnersBySession.set(key, sessionMap);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
if (!sessionMap.python || sessionMap.python.isDestroyed) {
|
|
39
|
+
sessionMap.python = new StdioWorkerRunner({
|
|
40
|
+
command: 'python',
|
|
41
|
+
args: ['-u', PY_WORKER_SCRIPT],
|
|
42
|
+
timeoutMs: this.config.timeoutMs,
|
|
43
|
+
maxOutputBytes: this.config.maxOutputBytes
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
return sessionMap.python;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
getNodeRunner(session) {
|
|
51
|
+
const key = this._getKey(session);
|
|
52
|
+
let sessionMap = this.runnersBySession.get(key);
|
|
53
|
+
if (!sessionMap) {
|
|
54
|
+
sessionMap = {};
|
|
55
|
+
this.runnersBySession.set(key, sessionMap);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
if (!sessionMap.node || sessionMap.node.isDestroyed) {
|
|
59
|
+
sessionMap.node = new StdioWorkerRunner({
|
|
60
|
+
command: 'node',
|
|
61
|
+
args: [NODE_WORKER_SCRIPT],
|
|
62
|
+
timeoutMs: this.config.timeoutMs,
|
|
63
|
+
maxOutputBytes: this.config.maxOutputBytes
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
return sessionMap.node;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
destroySession(session) {
|
|
71
|
+
const key = this._getKey(session);
|
|
72
|
+
const sessionMap = this.runnersBySession.get(key);
|
|
73
|
+
if (!sessionMap) return;
|
|
74
|
+
|
|
75
|
+
if (sessionMap.python) {
|
|
76
|
+
sessionMap.python.destroy();
|
|
77
|
+
}
|
|
78
|
+
if (sessionMap.node) {
|
|
79
|
+
sessionMap.node.destroy();
|
|
80
|
+
}
|
|
81
|
+
this.runnersBySession.delete(key);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
destroyAll() {
|
|
85
|
+
for (const [key, sessionMap] of this.runnersBySession.entries()) {
|
|
86
|
+
if (sessionMap.python) sessionMap.python.destroy();
|
|
87
|
+
if (sessionMap.node) sessionMap.node.destroy();
|
|
88
|
+
}
|
|
89
|
+
this.runnersBySession.clear();
|
|
90
|
+
}
|
|
91
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { Context } from '@deepseek-ai/cordis';
|
|
2
|
+
import z from '@deepseek-ai/schemastery';
|
|
3
|
+
|
|
4
|
+
export const EVAL_PYTHON_TOOL: 'eval_python';
|
|
5
|
+
export const EVAL_NODE_TOOL: 'eval_node';
|
|
6
|
+
|
|
7
|
+
export interface PersistentEvalOptions {
|
|
8
|
+
enabled?: boolean;
|
|
9
|
+
python?: boolean;
|
|
10
|
+
node?: boolean;
|
|
11
|
+
timeoutMs?: number;
|
|
12
|
+
maxOutputBytes?: number;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export declare const PersistentEvalConfig: z<PersistentEvalOptions>;
|
|
16
|
+
|
|
17
|
+
export interface PersistentEvalResponse {
|
|
18
|
+
id?: string;
|
|
19
|
+
success: boolean;
|
|
20
|
+
stdout?: string;
|
|
21
|
+
stderr?: string;
|
|
22
|
+
result?: any;
|
|
23
|
+
resultRepr?: string;
|
|
24
|
+
error?: string;
|
|
25
|
+
executionTimeMs?: number;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export declare function formatEvalOutput(resp: PersistentEvalResponse | null | undefined): string;
|
|
29
|
+
|
|
30
|
+
export interface PersistentEvalService {
|
|
31
|
+
pool: any;
|
|
32
|
+
config: PersistentEvalOptions;
|
|
33
|
+
destroy(): void;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export declare class StdioWorkerRunner {
|
|
37
|
+
constructor(options: any);
|
|
38
|
+
ensureStarted(): Promise<void>;
|
|
39
|
+
execute(code: string, options?: any): Promise<PersistentEvalResponse>;
|
|
40
|
+
destroy(): void;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export declare class DaemonPool {
|
|
44
|
+
constructor(config?: any);
|
|
45
|
+
getPythonRunner(session?: any): StdioWorkerRunner;
|
|
46
|
+
getNodeRunner(session?: any): StdioWorkerRunner;
|
|
47
|
+
destroySession(session?: any): void;
|
|
48
|
+
destroyAll(): void;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export declare function applyPersistentEval(
|
|
52
|
+
ctx: Context,
|
|
53
|
+
rawConfig?: PersistentEvalOptions
|
|
54
|
+
): PersistentEvalService | null;
|