dsh-auto-review 0.5.4 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/README.es.md +87 -1
- package/README.hi.md +87 -1
- package/README.md +89 -3
- package/README.pt.md +87 -1
- package/README.zh.md +87 -1
- package/bin/dsh-auto-review-mcp.mjs +19 -0
- package/lib/client.js +11 -5
- package/lib/client.js.map +1 -1
- package/lib/eval-cli.js +7 -1
- package/lib/eval.js +2 -2
- package/lib/index.js +246 -5
- package/lib/mcp.js +720 -0
- package/lib/{report-DERBXGXW.js → report-NzSUyIog.js} +435 -17
- package/lib/types/cache.d.ts +90 -0
- package/lib/types/cache.d.ts.map +1 -0
- package/lib/types/cache.js +183 -0
- package/lib/types/cache.js.map +1 -0
- package/lib/types/client/ReviewPanel.d.ts +5 -5
- package/lib/types/client/ReviewPanel.d.ts.map +1 -1
- package/lib/types/client/ReviewPanel.js +6 -6
- package/lib/types/client/ReviewPanel.js.map +1 -1
- package/lib/types/client/locales.d.ts +1 -0
- package/lib/types/client/locales.d.ts.map +1 -1
- package/lib/types/client/locales.js +2 -0
- package/lib/types/client/locales.js.map +1 -1
- package/lib/types/config.d.ts +10 -0
- package/lib/types/config.d.ts.map +1 -1
- package/lib/types/config.js +10 -0
- package/lib/types/config.js.map +1 -1
- package/lib/types/eval/assert.d.ts +3 -1
- package/lib/types/eval/assert.d.ts.map +1 -1
- package/lib/types/eval/assert.js +159 -1
- package/lib/types/eval/assert.js.map +1 -1
- package/lib/types/eval/diff.d.ts +41 -0
- package/lib/types/eval/diff.d.ts.map +1 -0
- package/lib/types/eval/diff.js +136 -0
- package/lib/types/eval/diff.js.map +1 -0
- package/lib/types/eval/dsl.d.ts +57 -0
- package/lib/types/eval/dsl.d.ts.map +1 -1
- package/lib/types/eval/dsl.js +40 -0
- package/lib/types/eval/dsl.js.map +1 -1
- package/lib/types/eval/index.d.ts +2 -0
- package/lib/types/eval/index.d.ts.map +1 -1
- package/lib/types/eval/index.js +2 -0
- package/lib/types/eval/index.js.map +1 -1
- package/lib/types/eval/report.d.ts.map +1 -1
- package/lib/types/eval/report.js +5 -0
- package/lib/types/eval/report.js.map +1 -1
- package/lib/types/eval/runner.d.ts +11 -0
- package/lib/types/eval/runner.d.ts.map +1 -1
- package/lib/types/eval/runner.js +40 -4
- package/lib/types/eval/runner.js.map +1 -1
- package/lib/types/eval/stress.d.ts +45 -0
- package/lib/types/eval/stress.d.ts.map +1 -0
- package/lib/types/eval/stress.js +68 -0
- package/lib/types/eval/stress.js.map +1 -0
- package/lib/types/eval/trace.d.ts +19 -9
- package/lib/types/eval/trace.d.ts.map +1 -1
- package/lib/types/eval/trace.js +43 -9
- package/lib/types/eval/trace.js.map +1 -1
- package/lib/types/events.d.ts +8 -0
- package/lib/types/events.d.ts.map +1 -1
- package/lib/types/events.js.map +1 -1
- package/lib/types/index.d.ts +1 -0
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/index.js +1 -0
- package/lib/types/index.js.map +1 -1
- package/lib/types/mcp/index.d.ts +11 -0
- package/lib/types/mcp/index.d.ts.map +1 -0
- package/lib/types/mcp/index.js +9 -0
- package/lib/types/mcp/index.js.map +1 -0
- package/lib/types/mcp/server.d.ts +44 -0
- package/lib/types/mcp/server.d.ts.map +1 -0
- package/lib/types/mcp/server.js +146 -0
- package/lib/types/mcp/server.js.map +1 -0
- package/lib/types/mcp/standalone.d.ts +72 -0
- package/lib/types/mcp/standalone.d.ts.map +1 -0
- package/lib/types/mcp/standalone.js +162 -0
- package/lib/types/mcp/standalone.js.map +1 -0
- package/lib/types/projection-types.d.ts +3 -0
- package/lib/types/projection-types.d.ts.map +1 -1
- package/lib/types/projection.d.ts +2 -0
- package/lib/types/projection.d.ts.map +1 -1
- package/lib/types/projection.js +8 -3
- package/lib/types/projection.js.map +1 -1
- package/lib/types/runtime.d.ts +14 -0
- package/lib/types/runtime.d.ts.map +1 -1
- package/lib/types/runtime.js +50 -2
- package/lib/types/runtime.js.map +1 -1
- package/package.json +17 -12
- package/src/cache.ts +199 -0
- package/src/client/ReviewPanel.tsx +9 -5
- package/src/client/locales.ts +2 -0
- package/src/config.ts +20 -0
- package/src/eval/assert.ts +158 -1
- package/src/eval/diff.ts +151 -0
- package/src/eval/dsl.ts +61 -0
- package/src/eval/index.ts +2 -0
- package/src/eval/report.ts +5 -0
- package/src/eval/runner.ts +39 -3
- package/src/eval/stress.ts +89 -0
- package/src/eval/trace.ts +73 -0
- package/src/events.ts +8 -0
- package/src/index.ts +1 -0
- package/src/mcp/index.ts +23 -0
- package/src/mcp/server.ts +165 -0
- package/src/mcp/standalone.ts +195 -0
- package/src/projection-types.ts +3 -0
- package/src/projection.ts +8 -3
- package/src/runtime.ts +50 -3
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,32 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to `dsh-auto-review` are documented here. The repo is pre-release; versions follow the DeepSeek Harness `0.1.0-rc.x` target runtime and bump on every behavior change.
|
|
4
4
|
|
|
5
|
+
## [0.7.0] — 2026-08-26
|
|
6
|
+
|
|
7
|
+
### Added
|
|
8
|
+
|
|
9
|
+
- **stdio MCP server export** (`dsh-auto-review/mcp` + the `dsh-auto-review-mcp` bin): a standalone stdio JSON-RPC 2.0 / MCP server (newline-delimited JSON, one object per line) exposing two tools — `review_action` (deterministic deny on a matched never-rule or fail-closed, otherwise replay a cached verdict for an identical fingerprint) and `cache_stats` (cache counters + TTL). The new `src/mcp/` modules own only protocol dispatch; every decision lives in the standalone reviewer.
|
|
10
|
+
- **Same-fingerprint verdict cache with TTL**: a repeated, identical tool action reuses its recent reviewer verdict instead of paying another second-model round-trip. The fingerprint is a SHA-256 digest of the tool name plus canonicalized call arguments (volatile keys stripped, keys sorted); only the digest is stored, so plaintext arguments never enter the cache. New config knobs control TTL and cacheability, `cache_stats` reports hits/stores/live size, and the web panel shows the cache state.
|
|
11
|
+
|
|
12
|
+
### Changed
|
|
13
|
+
|
|
14
|
+
- Renovate is enabled via the shared `dsh-plugin-kit` preset, and the five-language READMEs credit community contributors. No plugin behavior changed.
|
|
15
|
+
|
|
16
|
+
### Fixed
|
|
17
|
+
|
|
18
|
+
- Packed smoke install now resolves the next-tagged rc line: the `@deepseek-ai/dsh-*` peer floor is raised to `0.1.1-rc.2` (the peer ranges already cover it), so `pnpm pack` plus the smoke install no longer fails to resolve peers.
|
|
19
|
+
|
|
20
|
+
## [0.6.0] — 2026-08-23
|
|
21
|
+
|
|
22
|
+
### Added
|
|
23
|
+
|
|
24
|
+
- **dsh-eval assertion engine gains three structured assertion families** (completing the evaluation-platform direction):
|
|
25
|
+
- **Prompt regression** (`expect.prompt`): the rendered system prompt must match a committed `baseline` (inline or via `baselineFrom` file); any drift is reported as a **side-by-side diff** in the report, with `allowedChanges` regexes to whitelist intended edits. The baseline file is resolved before any agent runs, and a missing/unreadable file fails the suite loudly.
|
|
26
|
+
- **Stress metrics** (`expect.stress`): gates P99 step latency (`maxP99Ms`), worst time-to-first-token (`maxTtftMs`), and aggregate token generation speed (`minTokensPerSecond`), folded from per-step timing records (`step/start` → first/last `assistant/chunk` → `step/end`).
|
|
27
|
+
- **Bias radar** (`expect.bias`): per-category regex counts over the final output (`categories`), hard `forbid` patterns, and `maxHits`/`maxCategoryHits` caps — the fairness (Bias-Radar) assertion.
|
|
28
|
+
- Per-step timing is now collected into `trace.steps`, and the assertion engine reports optional multi-line `detail` (the prompt diff, the bias radar) which the Markdown report renders as a code block.
|
|
29
|
+
- Five-language READMEs document the assertion families with a combined `prompt`/`stress`/`bias` example, a GitHub Action CI snippet, and the contrast with `openai/codex-research`.
|
|
30
|
+
|
|
5
31
|
## [0.5.4] — 2026-08-22
|
|
6
32
|
|
|
7
33
|
### Changed
|
package/README.es.md
CHANGED
|
@@ -95,6 +95,8 @@ Todas las opciones son campos Schemastery `Config` (modificables desde cordis.ym
|
|
|
95
95
|
| `riskPolicy` | `{maxAutoAllow: high, onHighRisk: delegate}` | Los veredictos `allow` por encima de `maxAutoAllow` delegan o deniegan |
|
|
96
96
|
| `circuitBreaker` | `{consecutiveDenies: 3, windowDenies: 6, windowSize: 10, action: delegate}` | Disyuntor de rechazos |
|
|
97
97
|
| `overrideTtlMs` | `300000` | Cuánto dura una anulación de `/auto-review approve` |
|
|
98
|
+
| `verdictCacheTtlMs` | `60000` | Reutiliza un veredicto reciente para una huella `herramienta + argumentos` idéntica; `0` desactiva la caché |
|
|
99
|
+
| `verdictCacheMaxEntries` | `256` | Máximo de huellas en caché antes de desalojar la más antigua |
|
|
98
100
|
| `language` | `en` | Idioma de la UI de la salida del comando `/auto-review` (`en` \| `zh`) |
|
|
99
101
|
| `allowUnmarkedAudit` | `false` | Fuerza la auditoría del registro de sesión en hosts que descartan el marcador `ignorable` (peligroso: los eventos sin marcar hacen las sesiones irrecuperables en otros hosts); por defecto se detecta y se degrada |
|
|
100
102
|
|
|
@@ -210,7 +212,89 @@ Ejecútalo (una clave de API de DeepSeek debe estar en el entorno):
|
|
|
210
212
|
dsh-eval eval/cases --model deepseek-v4-flash --timeout-ms 240000 --out .eval-reports
|
|
211
213
|
```
|
|
212
214
|
|
|
213
|
-
|
|
215
|
+
### Familias de aserciones
|
|
216
|
+
|
|
217
|
+
El bloque `expect` admite seis familias de aserciones; cada aserción se evalúa de forma independiente e informa su propio aprobado/fallo con valores esperados/reales, de modo que un caso que falla se explica a sí mismo sin volver a ejecutarlo.
|
|
218
|
+
|
|
219
|
+
| Familia | Claves DSL | Qué verifica |
|
|
220
|
+
|---|---|---|
|
|
221
|
+
| Rastro de herramientas | `toolCalls`, `toolCallsExact`, `noToolCalls`, `results` | secuencia ordenada de llamadas (subsecuencia con saltos), secuencia exacta de nombres, resultado por herramienta (`isError`/`contains`/`regex`) |
|
|
222
|
+
| Salida y presupuesto | `output`, `turnEnds`, `maxTokens` | subcadena/regex de la salida final, resultado del turno, presupuesto de tokens |
|
|
223
|
+
| Regresión de prompt | `prompt` | el prompt de sistema renderizado debe coincidir con un `baseline` comprometido (o un archivo `baselineFrom`); cualquier deriva se informa como un **diff lado a lado**, con regex `allowedChanges` para permitir ediciones intencionales |
|
|
224
|
+
| Métricas de estrés | `stress` | latencia de paso P99 (`maxP99Ms`), peor tiempo al primer token (`maxTtftMs`), velocidad agregada de generación de tokens (`minTokensPerSecond`) |
|
|
225
|
+
| Equidad | `bias` | radar de sesgo sobre la salida final: conteos por categoría con regex (`categories`), patrones `forbid` duros, topes `maxHits`/`maxCategoryHits` |
|
|
226
|
+
| Revisión de segundo modelo | `review` | un veredicto aprobado/fallo complementario del subagente revisor (capa separada, la misma costura que el revisor de aprobación) |
|
|
227
|
+
|
|
228
|
+
```yaml
|
|
229
|
+
- id: regression-gate
|
|
230
|
+
input: Answer in one sentence.
|
|
231
|
+
expect:
|
|
232
|
+
prompt:
|
|
233
|
+
baseline: "You are a helpful software engineer assistant."
|
|
234
|
+
allowedChanges: ["copyright-year"]
|
|
235
|
+
stress:
|
|
236
|
+
maxP99Ms: 8000
|
|
237
|
+
maxTtftMs: 3000
|
|
238
|
+
minTokensPerSecond: 20
|
|
239
|
+
bias:
|
|
240
|
+
categories: { gender: ["[Hh]e is (un)?stable"] }
|
|
241
|
+
forbid: ["[Ss]crew that"]
|
|
242
|
+
maxCategoryHits: 0
|
|
243
|
+
```
|
|
244
|
+
|
|
245
|
+
Puerta de CI: el proceso sale con 0 solo cuando todos los casos de todas las suites han pasado — las evaluaciones que fallen hacen fallar la compilación. Cada caso deja un JSONL de sesión reproducible y un JSON de rastro junto a `report.md`/`report.json`; los resultados de las aserciones (incluido el diff lado a lado del prompt), el uso de tokens, las métricas de estrés/equidad y el veredicto de revisión se escriben todos en los archivos de informe.
|
|
246
|
+
|
|
247
|
+
```yaml
|
|
248
|
+
- name: dsh-eval
|
|
249
|
+
run: npx dsh-eval eval/cases --model deepseek-v4-flash --timeout-ms 240000 --out .eval-reports
|
|
250
|
+
env:
|
|
251
|
+
DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
|
|
252
|
+
```
|
|
253
|
+
|
|
254
|
+
`dsh-eval` difiere de [openai/codex-research](https://github.com/openai/codex-research): codex-research puntúa trayectorias de agentes para comparación de investigación; `dsh-eval` es un arnés declarativo de regresión aprobado/fallo — casos YAML, aserciones estructuradas de rastro/prompt/estrés/equidad, una revisión opcional de segundo modelo y un código de salida de CI — para gatear cualquier agente DSH, no para ranking de investigación.
|
|
255
|
+
|
|
256
|
+
## Servidor MCP (independiente)
|
|
257
|
+
|
|
258
|
+
`dsh-auto-review` también incluye un **servidor MCP** stdio (`dsh-auto-review-mcp`) para que clientes MCP externos (Claude, Codex, …) consuman una ruta de revisión determinista sin el harness. Habla JSON-RPC 2.0 sobre JSON delimitado por saltos de línea (NDJSON): un objeto JSON por línea, sin tramado `Content-Length`.
|
|
259
|
+
|
|
260
|
+
**Límite.** El revisor completo necesita el seam de subagentes del harness y un segundo modelo, que un proceso stdio separado no puede alcanzar. Por lo tanto, el servidor independiente es **reglas deterministas + caché, sin revisión por modelo**:
|
|
261
|
+
|
|
262
|
+
- `review_action` reutiliza la caché de veredictos por huella (`src/cache.ts`) y la resolución de reglas de riesgo / políticas de herramientas (`src/config.ts`): una regla `never` → `deny`; un acierto de caché sobre una huella idéntica de `tool + arguments` reproduce ese veredicto; todo lo demás (`ai` necesita modelo, `human` necesita a un humano) → `deny` fail-closed con `reason: "standalone path, no model"`. Nunca permite una acción que un modelo no haya permitido ya.
|
|
263
|
+
- `cache_stats` informa los contadores de aciertos/almacenamiento y el estado del TTL.
|
|
264
|
+
|
|
265
|
+
| Herramienta | Propósito |
|
|
266
|
+
|---|---|
|
|
267
|
+
| `review_action` | `{tool, args?, reason?}` → `{decision, reason, riskLevel}` — denegación determinista / reproducción de caché |
|
|
268
|
+
| `cache_stats` | `{}` → `{hits, stores, size, ttlMs, enabled}` |
|
|
269
|
+
|
|
270
|
+
Ejecución directa:
|
|
271
|
+
|
|
272
|
+
```sh
|
|
273
|
+
# las reglas de riesgo vienen de variables de entorno
|
|
274
|
+
export DSH_AUTO_REVIEW_RISK_RULES='[{"pattern":"rm -rf","policy":"never","field":"arguments"}]'
|
|
275
|
+
node bin/dsh-auto-review-mcp.mjs
|
|
276
|
+
# o, tras npm install: npx dsh-auto-review-mcp
|
|
277
|
+
```
|
|
278
|
+
|
|
279
|
+
Configuración por entorno: `DSH_AUTO_REVIEW_RISK_RULES` (array JSON de `{pattern, policy, field?}`), `DSH_AUTO_REVIEW_TOOLS_POLICY` (JSON `{default?, overrides?}`), `DSH_AUTO_REVIEW_CACHE_TTL_MS`, `DSH_AUTO_REVIEW_CACHE_MAX_ENTRIES`.
|
|
280
|
+
|
|
281
|
+
Ejemplo para Claude Desktop (`claude_desktop_config.json`):
|
|
282
|
+
|
|
283
|
+
```json
|
|
284
|
+
{
|
|
285
|
+
"mcpServers": {
|
|
286
|
+
"dsh-auto-review": {
|
|
287
|
+
"command": "npx",
|
|
288
|
+
"args": ["-y", "dsh-auto-review-mcp"],
|
|
289
|
+
"env": {
|
|
290
|
+
"DSH_AUTO_REVIEW_RISK_RULES": "[{\"pattern\":\"rm -rf\",\"policy\":\"never\",\"field\":\"arguments\"}]"
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
```
|
|
296
|
+
|
|
297
|
+
El servidor es de solo lectura y determinista: sin red, sin modelo, sin escrituras.
|
|
214
298
|
|
|
215
299
|
## Permisos y datos
|
|
216
300
|
|
|
@@ -265,6 +349,8 @@ Estructura del repositorio: `src/index.ts` (contrato del plugin) · `src/config.
|
|
|
265
349
|
## Contribuidores
|
|
266
350
|
|
|
267
351
|
- [@PerryLink](https://github.com/PerryLink) — creador y mantenedor: el answerer de aprobación, el subagente revisor, la política de riesgo y el disyuntor, el panel de revisión por proyección de sesión, el compañero invariant, dsh-eval y la documentación en cinco idiomas.
|
|
352
|
+
- [@weipeng1999](https://github.com/weipeng1999) — propuso el enrutamiento independiente de proveedor/modelo del revisor ([#11](https://github.com/PerryLink/dsh-auto-review/issues/11), [discusión #12](https://github.com/PerryLink/dsh-auto-review/discussions/12)), que se publicó como `reviewerProvider` / `reviewerModel`.
|
|
353
|
+
- [@alexchenzl](https://github.com/alexchenzl) — incluyó el plugin en el directorio de plugins de DSH ([#10](https://github.com/PerryLink/dsh-auto-review/issues/10)).
|
|
268
354
|
|
|
269
355
|
## Familia de plugins DSH de PerryLink
|
|
270
356
|
|
package/README.hi.md
CHANGED
|
@@ -95,6 +95,8 @@ dsh --profile web --dump-config | grep -A4 'id: auto-review'
|
|
|
95
95
|
| `riskPolicy` | `{maxAutoAllow: high, onHighRisk: delegate}` | `maxAutoAllow` से ऊपर के `allow` निर्णय सौंपते या अस्वीकार करते हैं |
|
|
96
96
|
| `circuitBreaker` | `{consecutiveDenies: 3, windowDenies: 6, windowSize: 10, action: delegate}` | अस्वीकृति सर्किट ब्रेकर |
|
|
97
97
|
| `overrideTtlMs` | `300000` | `/auto-review approve` ओवरराइड कितनी देर उपयोगी रहता है |
|
|
98
|
+
| `verdictCacheTtlMs` | `60000` | समान `टूल + तर्क` फ़िंगरप्रिंट के लिए हालिया निर्णय पुनः उपयोग करें; `0` कैश बंद करता है |
|
|
99
|
+
| `verdictCacheMaxEntries` | `256` | सबसे पुराने को निकालने से पहले कैश किए गए फ़िंगरप्रिंट की अधिकतम संख्या |
|
|
98
100
|
| `language` | `en` | `/auto-review` कमांड आउटपुट की UI भाषा (`en` \| `zh`) |
|
|
99
101
|
| `allowUnmarkedAudit` | `false` | `ignorable` मार्कर छोड़ने वाले होस्ट पर सत्र-लॉग ऑडिट बलपूर्वक चालू करें (खतरनाक: बिना मार्कर वाली घटनाएँ अन्य होस्ट पर सत्र अप्राप्य बनाती हैं); डिफ़ॉल्ट पहचान-और-डिग्रेड है |
|
|
100
102
|
|
|
@@ -210,7 +212,89 @@ suite:
|
|
|
210
212
|
dsh-eval eval/cases --model deepseek-v4-flash --timeout-ms 240000 --out .eval-reports
|
|
211
213
|
```
|
|
212
214
|
|
|
213
|
-
|
|
215
|
+
### दावा परिवार
|
|
216
|
+
|
|
217
|
+
`expect` ब्लॉक छह दावा परिवारों का समर्थन करता है; हर दावा स्वतंत्र रूप से मूल्यांकित होता है और अपने पास/फेल को अपेक्षित/वास्तविक मानों के साथ रिपोर्ट करता है, इसलिए विफल केस बिना दोबारा चलाए खुद को समझा देता है।
|
|
218
|
+
|
|
219
|
+
| परिवार | DSL कुंजियाँ | यह क्या जाँचता है |
|
|
220
|
+
|---|---|---|
|
|
221
|
+
| टूल ट्रेस | `toolCalls`, `toolCallsExact`, `noToolCalls`, `results` | क्रमबद्ध टूल-कॉल अनुक्रम (छूट के साथ उप-अनुक्रम), सटीक नाम अनुक्रम, प्रति-टूल परिणाम (`isError`/`contains`/`regex`) |
|
|
222
|
+
| आउटपुट और बजट | `output`, `turnEnds`, `maxTokens` | अंतिम आउटपुट का सबस्ट्रिंग/रेगेक्स, टर्न परिणाम, टोकन बजट |
|
|
223
|
+
| प्रॉम्प्ट रिग्रेशन | `prompt` | रेंडर किया गया सिस्टम प्रॉम्प्ट एक प्रतिबद्ध `baseline` (या `baselineFrom` फ़ाइल) से मेल खाना चाहिए; कोई भी विचलन **side-by-side diff** के रूप में रिपोर्ट होता है, जिसमें इच्छित संपादनों की अनुमति के लिए `allowedChanges` रेगेक्स होते हैं |
|
|
224
|
+
| स्ट्रेस मेट्रिक्स | `stress` | P99 स्टेप लेटेंसी (`maxP99Ms`), सबसे खराब पहला-टोकन समय (`maxTtftMs`), कुल टोकन निर्माण गति (`minTokensPerSecond`) |
|
|
225
|
+
| निष्पक्षता | `bias` | अंतिम आउटपुट पर पूर्वाग्रह रडार: प्रति-श्रेणी रेगेक्स गणना (`categories`), कठोर `forbid` पैटर्न, `maxHits`/`maxCategoryHits` सीमाएँ |
|
|
226
|
+
| द्वितीय-मॉडल समीक्षा | `review` | समीक्षक उप-एजेंट से एक पूरक पास/फेल निर्णय (अलग परत, अनुमोदन समीक्षक जैसी ही सीम) |
|
|
227
|
+
|
|
228
|
+
```yaml
|
|
229
|
+
- id: regression-gate
|
|
230
|
+
input: Answer in one sentence.
|
|
231
|
+
expect:
|
|
232
|
+
prompt:
|
|
233
|
+
baseline: "You are a helpful software engineer assistant."
|
|
234
|
+
allowedChanges: ["copyright-year"]
|
|
235
|
+
stress:
|
|
236
|
+
maxP99Ms: 8000
|
|
237
|
+
maxTtftMs: 3000
|
|
238
|
+
minTokensPerSecond: 20
|
|
239
|
+
bias:
|
|
240
|
+
categories: { gender: ["[Hh]e is (un)?stable"] }
|
|
241
|
+
forbid: ["[Ss]crew that"]
|
|
242
|
+
maxCategoryHits: 0
|
|
243
|
+
```
|
|
244
|
+
|
|
245
|
+
CI द्वार: प्रक्रिया केवल तभी 0 से निकलती है जब हर सुइट का हर केस पास हो — असफल मूल्यांकन बिल्ड को असफल कर देते हैं। हर केस `report.md`/`report.json` के पास एक पुनः-चलाने योग्य सत्र JSONL और एक ट्रेस JSON छोड़ता है; दावा परिणाम (प्रॉम्प्ट side-by-side diff सहित), टोकन उपयोग, स्ट्रेस/निष्पक्षता मेट्रिक्स और समीक्षा निर्णय सभी रिपोर्ट फ़ाइलों में लिखे जाते हैं।
|
|
246
|
+
|
|
247
|
+
```yaml
|
|
248
|
+
- name: dsh-eval
|
|
249
|
+
run: npx dsh-eval eval/cases --model deepseek-v4-flash --timeout-ms 240000 --out .eval-reports
|
|
250
|
+
env:
|
|
251
|
+
DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
|
|
252
|
+
```
|
|
253
|
+
|
|
254
|
+
`dsh-eval` [openai/codex-research](https://github.com/openai/codex-research) से भिन्न है: codex-research अनुसंधान तुलना के लिए एजेंट ट्रैजेक्टरी को स्कोर करता है; `dsh-eval` एक घोषणात्मक पास/फेल रिग्रेशन हार्नेस है — YAML केस, संरचित ट्रेस/प्रॉम्प्ट/स्ट्रेस/निष्पक्षता दावे, एक वैकल्पिक द्वितीय-मॉडल समीक्षा और एक CI एग्जिट कोड — किसी भी DSH एजेंट को गेट करने के लिए, अनुसंधान रैंकिंग के लिए नहीं।
|
|
255
|
+
|
|
256
|
+
## MCP सर्वर (स्टैंडअलोन)
|
|
257
|
+
|
|
258
|
+
`dsh-auto-review` एक stdio **MCP सर्वर** (`dsh-auto-review-mcp`) भी देता है ताकि बाहरी MCP क्लाइंट (Claude, Codex, …) बिना harness के एक निर्धारक समीक्षा पथ का उपयोग कर सकें। यह newline-delimited JSON (NDJSON) पर JSON-RPC 2.0 बोलता है — प्रति पंक्ति एक JSON ऑब्जेक्ट, कोई `Content-Length` फ़्रेमिंग नहीं।
|
|
259
|
+
|
|
260
|
+
**सीमा।** पूर्ण reviewer को harness के subagent seam और दूसरे मॉडल की आवश्यकता होती है, जिसे एक अलग stdio प्रक्रिया नहीं पहुँच सकती। इसलिए स्टैंडअलोन सर्वर **निर्धारक नियम + कैश, कोई मॉडल समीक्षा नहीं** है:
|
|
261
|
+
|
|
262
|
+
- `review_action` समान-फ़िंगरप्रिंट वर्डिक्ट कैश (`src/cache.ts`) और जोखिम-नियम / टूल-नीति समाधान (`src/config.ts`) का पुनः उपयोग करता है: एक `never` नियम → `deny`; समान `tool + arguments` फ़िंगरप्रिंट पर कैश हिट उस वर्डिक्ट को दोहराता है; बाकी सब (`ai` को मॉडल चाहिए, `human` को मानव चाहिए) → fail-closed `deny` जिसका reason `"standalone path, no model"` है। यह कभी भी ऐसी कार्रवाई की अनुमति नहीं देता जिसे किसी मॉडल ने पहले से अनुमति न दी हो।
|
|
263
|
+
- `cache_stats` हिट/स्टोर गणना और TTL स्थिति की सूचना देता है।
|
|
264
|
+
|
|
265
|
+
| टूल | उद्देश्य |
|
|
266
|
+
|---|---|
|
|
267
|
+
| `review_action` | `{tool, args?, reason?}` → `{decision, reason, riskLevel}` — निर्धारक अस्वीकृति / कैश पुनरावृत्ति |
|
|
268
|
+
| `cache_stats` | `{}` → `{hits, stores, size, ttlMs, enabled}` |
|
|
269
|
+
|
|
270
|
+
सीधे चलाएँ:
|
|
271
|
+
|
|
272
|
+
```sh
|
|
273
|
+
# जोखिम नियम पर्यावरण चरों से आते हैं
|
|
274
|
+
export DSH_AUTO_REVIEW_RISK_RULES='[{"pattern":"rm -rf","policy":"never","field":"arguments"}]'
|
|
275
|
+
node bin/dsh-auto-review-mcp.mjs
|
|
276
|
+
# या, npm install के बाद: npx dsh-auto-review-mcp
|
|
277
|
+
```
|
|
278
|
+
|
|
279
|
+
पर्यावरण विन्यास: `DSH_AUTO_REVIEW_RISK_RULES` (`{pattern, policy, field?}` की JSON सरणी), `DSH_AUTO_REVIEW_TOOLS_POLICY` (JSON `{default?, overrides?}`), `DSH_AUTO_REVIEW_CACHE_TTL_MS`, `DSH_AUTO_REVIEW_CACHE_MAX_ENTRIES`।
|
|
280
|
+
|
|
281
|
+
Claude Desktop (`claude_desktop_config.json`) उदाहरण:
|
|
282
|
+
|
|
283
|
+
```json
|
|
284
|
+
{
|
|
285
|
+
"mcpServers": {
|
|
286
|
+
"dsh-auto-review": {
|
|
287
|
+
"command": "npx",
|
|
288
|
+
"args": ["-y", "dsh-auto-review-mcp"],
|
|
289
|
+
"env": {
|
|
290
|
+
"DSH_AUTO_REVIEW_RISK_RULES": "[{\"pattern\":\"rm -rf\",\"policy\":\"never\",\"field\":\"arguments\"}]"
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
```
|
|
296
|
+
|
|
297
|
+
सर्वर केवल-पठन और निर्धारक है: कोई नेटवर्क नहीं, कोई मॉडल नहीं, कोई लेखन नहीं।
|
|
214
298
|
|
|
215
299
|
## अनुमतियाँ और डेटा
|
|
216
300
|
|
|
@@ -265,6 +349,8 @@ pnpm pack # प्रकाशित tarball
|
|
|
265
349
|
## योगदानकर्ता
|
|
266
350
|
|
|
267
351
|
- [@PerryLink](https://github.com/PerryLink) — निर्माता और अनुरक्षक: अनुमोदन answerer, समीक्षक उप-एजेंट, जोखिम नीति और सर्किट ब्रेकर, सत्र-प्रोजेक्शन समीक्षा पैनल, invariant साथी, dsh-eval, और पाँच-भाषा दस्तावेज़।
|
|
352
|
+
- [@weipeng1999](https://github.com/weipeng1999) — समीक्षक के स्वतंत्र प्रदाता/मॉडल रूटिंग का प्रस्ताव रखा ([#11](https://github.com/PerryLink/dsh-auto-review/issues/11), [चर्चा #12](https://github.com/PerryLink/dsh-auto-review/discussions/12)), जो `reviewerProvider` / `reviewerModel` के रूप में जारी हुआ।
|
|
353
|
+
- [@alexchenzl](https://github.com/alexchenzl) — DSH प्लगइन निर्देशिका में प्लगइन को सूचीबद्ध किया ([#10](https://github.com/PerryLink/dsh-auto-review/issues/10))।
|
|
268
354
|
|
|
269
355
|
## PerryLink DSH प्लगइन परिवार
|
|
270
356
|
|
package/README.md
CHANGED
|
@@ -96,6 +96,8 @@ All tunables are Schemastery `Config` fields (changeable from cordis.yml). An id
|
|
|
96
96
|
| `riskPolicy` | `{maxAutoAllow: high, onHighRisk: delegate}` | `allow` verdicts above `maxAutoAllow` delegate or deny |
|
|
97
97
|
| `circuitBreaker` | `{consecutiveDenies: 3, windowDenies: 6, windowSize: 10, action: delegate}` | Rejection circuit breaker |
|
|
98
98
|
| `overrideTtlMs` | `300000` | How long a `/auto-review approve` override stays usable |
|
|
99
|
+
| `verdictCacheTtlMs` | `60000` | Reuse a recent verdict for an identical `tool + arguments` fingerprint; `0` disables the cache |
|
|
100
|
+
| `verdictCacheMaxEntries` | `256` | Maximum cached fingerprints before oldest-eviction |
|
|
99
101
|
| `language` | `en` | UI language of the `/auto-review` command output (`en` \| `zh`) |
|
|
100
102
|
| `allowUnmarkedAudit` | `false` | Force session-log audit on hosts that drop the `ignorable` marker (dangerous: unmarked events make sessions unresumable elsewhere); default is detect-and-degrade |
|
|
101
103
|
|
|
@@ -142,7 +144,7 @@ Example (annotated full form: `fixtures/config/config-full.yaml`):
|
|
|
142
144
|
|
|
143
145
|
## Web review panel
|
|
144
146
|
|
|
145
|
-
In the Web GUI (web profile), the package contributes a session-header action (**AI Review**) that opens a panel with the session's auto-review state: the switch with on/off buttons (they execute `/auto-review on|off`), both per-turn budgets, cumulative statistics (including hard-disable rejections), the circuit trip, the recent verdicts, and one-shot **approve** buttons for recent denials (they execute `/auto-review approve [n]`).
|
|
147
|
+
In the Web GUI (web profile), the package contributes a session-header action (**AI Review**) that opens a panel with the session's auto-review state: the switch with on/off buttons (they execute `/auto-review on|off`), both per-turn budgets, cumulative statistics (including hard-disable rejections and cache hits), the circuit trip, the recent verdicts, and one-shot **approve** buttons for recent denials (they execute `/auto-review approve [n]`).
|
|
146
148
|
|
|
147
149
|
How it is wired:
|
|
148
150
|
|
|
@@ -211,7 +213,89 @@ Run it (a DeepSeek API key must be in the environment):
|
|
|
211
213
|
dsh-eval eval/cases --model deepseek-v4-flash --timeout-ms 240000 --out .eval-reports
|
|
212
214
|
```
|
|
213
215
|
|
|
214
|
-
|
|
216
|
+
### Assertion families
|
|
217
|
+
|
|
218
|
+
The `expect` block supports six assertion families; each assertion is evaluated independently and reports its own pass/fail with expected/actual values, so a failing case explains itself without a rerun.
|
|
219
|
+
|
|
220
|
+
| Family | DSL keys | What it gates |
|
|
221
|
+
|---|---|---|
|
|
222
|
+
| Tool trace | `toolCalls`, `toolCallsExact`, `noToolCalls`, `results` | ordered tool-call sequence (subsequence with skips), exact name sequence, per-tool result (`isError`/`contains`/`regex`) |
|
|
223
|
+
| Output & budget | `output`, `turnEnds`, `maxTokens` | final-output substring/regex, turn outcome, token budget |
|
|
224
|
+
| Prompt regression | `prompt` | the rendered system prompt must match a committed `baseline` (or a `baselineFrom` file); any drift is reported as a **side-by-side diff**, with `allowedChanges` regexes to whitelist intended edits |
|
|
225
|
+
| Stress metrics | `stress` | P99 step latency (`maxP99Ms`), worst time-to-first-token (`maxTtftMs`), aggregate token generation speed (`minTokensPerSecond`) |
|
|
226
|
+
| Fairness | `bias` | bias radar over the final output: per-category regex counts (`categories`), hard `forbid` patterns, `maxHits`/`maxCategoryHits` caps |
|
|
227
|
+
| Second-model review | `review` | a supplementary pass/fail verdict from the reviewer subagent (a separate layer, same seam as the approval reviewer) |
|
|
228
|
+
|
|
229
|
+
```yaml
|
|
230
|
+
- id: regression-gate
|
|
231
|
+
input: Answer in one sentence.
|
|
232
|
+
expect:
|
|
233
|
+
prompt:
|
|
234
|
+
baseline: "You are a helpful software engineer assistant."
|
|
235
|
+
allowedChanges: ["copyright-year"]
|
|
236
|
+
stress:
|
|
237
|
+
maxP99Ms: 8000
|
|
238
|
+
maxTtftMs: 3000
|
|
239
|
+
minTokensPerSecond: 20
|
|
240
|
+
bias:
|
|
241
|
+
categories: { gender: ["[Hh]e is (un)?stable"] }
|
|
242
|
+
forbid: ["[Ss]crew that"]
|
|
243
|
+
maxCategoryHits: 0
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
CI gate: the process exits 0 only when every case of every suite passed — failing evaluations fail the build. Each case leaves a replayable session JSONL and a trace JSON beside `report.md`/`report.json`; assertion results (including the prompt side-by-side diff), token usage, stress/bias metrics, and the review verdict are all written into the report files.
|
|
247
|
+
|
|
248
|
+
```yaml
|
|
249
|
+
- name: dsh-eval
|
|
250
|
+
run: npx dsh-eval eval/cases --model deepseek-v4-flash --timeout-ms 240000 --out .eval-reports
|
|
251
|
+
env:
|
|
252
|
+
DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
|
|
253
|
+
```
|
|
254
|
+
|
|
255
|
+
`dsh-eval` differs from [openai/codex-research](https://github.com/openai/codex-research): codex-research scores agent trajectories for research comparison; `dsh-eval` is a declarative pass/fail regression harness — YAML cases, structured trace/prompt/stress/bias assertions, an optional second-model review, and a CI exit code — for gating any DSH agent, not research ranking.
|
|
256
|
+
|
|
257
|
+
## MCP server (standalone)
|
|
258
|
+
|
|
259
|
+
`dsh-auto-review` also ships a stdio **MCP server** (`dsh-auto-review-mcp`) so external MCP clients (Claude, Codex, …) can consume a deterministic review path without a harness. It speaks JSON-RPC 2.0 over newline-delimited JSON (NDJSON) — one JSON object per line, no `Content-Length` framing.
|
|
260
|
+
|
|
261
|
+
**Boundary.** The full reviewer needs the harness subagent seam and a second model, which a separate stdio process cannot reach. The standalone server is therefore **deterministic rules + cache, no model review**:
|
|
262
|
+
|
|
263
|
+
- `review_action` reuses the same-fingerprint verdict cache (`src/cache.ts`) and the risk-rule / tool-policy resolution (`src/config.ts`): a `never` rule → `deny`; a cache hit on an identical `tool + arguments` fingerprint replays that verdict; anything else (`ai` needs a model, `human` needs a human) → fail-closed `deny` with `reason: "standalone path, no model"`. It never allows an action a model did not already allow.
|
|
264
|
+
- `cache_stats` reports hit/store counts and the TTL status.
|
|
265
|
+
|
|
266
|
+
| Tool | Purpose |
|
|
267
|
+
|---|---|
|
|
268
|
+
| `review_action` | `{tool, args?, reason?}` → `{decision, reason, riskLevel}` — deterministic deny / cache replay |
|
|
269
|
+
| `cache_stats` | `{}` → `{hits, stores, size, ttlMs, enabled}` |
|
|
270
|
+
|
|
271
|
+
Run it directly:
|
|
272
|
+
|
|
273
|
+
```sh
|
|
274
|
+
# risk rules come from environment variables
|
|
275
|
+
export DSH_AUTO_REVIEW_RISK_RULES='[{"pattern":"rm -rf","policy":"never","field":"arguments"}]'
|
|
276
|
+
node bin/dsh-auto-review-mcp.mjs
|
|
277
|
+
# or, after npm install: npx dsh-auto-review-mcp
|
|
278
|
+
```
|
|
279
|
+
|
|
280
|
+
Environment config: `DSH_AUTO_REVIEW_RISK_RULES` (JSON array of `{pattern, policy, field?}`), `DSH_AUTO_REVIEW_TOOLS_POLICY` (JSON `{default?, overrides?}`), `DSH_AUTO_REVIEW_CACHE_TTL_MS`, `DSH_AUTO_REVIEW_CACHE_MAX_ENTRIES`.
|
|
281
|
+
|
|
282
|
+
Claude Desktop (`claude_desktop_config.json`) example:
|
|
283
|
+
|
|
284
|
+
```json
|
|
285
|
+
{
|
|
286
|
+
"mcpServers": {
|
|
287
|
+
"dsh-auto-review": {
|
|
288
|
+
"command": "npx",
|
|
289
|
+
"args": ["-y", "dsh-auto-review-mcp"],
|
|
290
|
+
"env": {
|
|
291
|
+
"DSH_AUTO_REVIEW_RISK_RULES": "[{\"pattern\":\"rm -rf\",\"policy\":\"never\",\"field\":\"arguments\"}]"
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
```
|
|
297
|
+
|
|
298
|
+
The server is read-only and deterministic: no network, no model, no writes.
|
|
215
299
|
|
|
216
300
|
## Permissions & data
|
|
217
301
|
|
|
@@ -251,7 +335,7 @@ CI gate: the process exits 0 only when every case of every suite passed — drop
|
|
|
251
335
|
```sh
|
|
252
336
|
pnpm install # node ^22.19 || >=24
|
|
253
337
|
pnpm run typecheck # tsc: src + tests against the local harness checkout
|
|
254
|
-
pnpm test # vitest:
|
|
338
|
+
pnpm test # vitest: 233 tests, 20 files
|
|
255
339
|
pnpm run build # tsc declarations + tsdown bundles (lib/, incl. the client bundle)
|
|
256
340
|
pnpm run verify:self-contained
|
|
257
341
|
pnpm pack # the published tarball
|
|
@@ -266,6 +350,8 @@ Repository layout: `src/index.ts` (plugin contract) · `src/config.ts` (Schemast
|
|
|
266
350
|
## Contributors
|
|
267
351
|
|
|
268
352
|
- [@PerryLink](https://github.com/PerryLink) — creator and maintainer: the approval answerer, the reviewer subagent, risk policy and circuit breaker, the session-projection review panel, the invariant companion, dsh-eval, and the five-language docs.
|
|
353
|
+
- [@weipeng1999](https://github.com/weipeng1999) — proposed independent reviewer provider/model routing ([#11](https://github.com/PerryLink/dsh-auto-review/issues/11), [discussion #12](https://github.com/PerryLink/dsh-auto-review/discussions/12)), which shipped as `reviewerProvider` / `reviewerModel`.
|
|
354
|
+
- [@alexchenzl](https://github.com/alexchenzl) — listed the plugin on the DSH plugin directory ([#10](https://github.com/PerryLink/dsh-auto-review/issues/10)).
|
|
269
355
|
|
|
270
356
|
## PerryLink DSH Plugin Family
|
|
271
357
|
|
package/README.pt.md
CHANGED
|
@@ -95,6 +95,8 @@ Todos os ajustes são campos Schemastery `Config` (alteráveis no cordis.yml). U
|
|
|
95
95
|
| `riskPolicy` | `{maxAutoAllow: high, onHighRisk: delegate}` | Vereditos `allow` acima de `maxAutoAllow` delegam ou negam |
|
|
96
96
|
| `circuitBreaker` | `{consecutiveDenies: 3, windowDenies: 6, windowSize: 10, action: delegate}` | Disjuntor de rejeições |
|
|
97
97
|
| `overrideTtlMs` | `300000` | Quanto tempo dura uma anulação de `/auto-review approve` |
|
|
98
|
+
| `verdictCacheTtlMs` | `60000` | Reutiliza um veredito recente para uma impressão `ferramenta + argumentos` idêntica; `0` desativa o cache |
|
|
99
|
+
| `verdictCacheMaxEntries` | `256` | Máximo de impressões em cache antes de despejar a mais antiga |
|
|
98
100
|
| `language` | `en` | Idioma da UI da saída do comando `/auto-review` (`en` \| `zh`) |
|
|
99
101
|
| `allowUnmarkedAudit` | `false` | Força a auditoria do registro de sessão em hosts que descartam o marcador `ignorable` (perigoso: eventos sem marcador tornam sessões irrecuperáveis em outros hosts); o padrão é detectar e degradar |
|
|
100
102
|
|
|
@@ -210,7 +212,89 @@ Execute-o (uma chave de API da DeepSeek deve estar no ambiente):
|
|
|
210
212
|
dsh-eval eval/cases --model deepseek-v4-flash --timeout-ms 240000 --out .eval-reports
|
|
211
213
|
```
|
|
212
214
|
|
|
213
|
-
|
|
215
|
+
### Famílias de asserção
|
|
216
|
+
|
|
217
|
+
O bloco `expect` suporta seis famílias de asserção; cada asserção é avaliada de forma independente e informa seu próprio aprovação/falha com valores esperados/reais, de modo que um caso que falha se explica sozinho sem nova execução.
|
|
218
|
+
|
|
219
|
+
| Família | Chaves DSL | O que valida |
|
|
220
|
+
|---|---|---|
|
|
221
|
+
| Rastro de ferramentas | `toolCalls`, `toolCallsExact`, `noToolCalls`, `results` | sequência ordenada de chamadas (subsequência com saltos), sequência exata de nomes, resultado por ferramenta (`isError`/`contains`/`regex`) |
|
|
222
|
+
| Saída e orçamento | `output`, `turnEnds`, `maxTokens` | substring/regex da saída final, resultado do turno, orçamento de tokens |
|
|
223
|
+
| Regressão de prompt | `prompt` | o prompt de sistema renderizado deve corresponder a um `baseline` confirmado (ou um arquivo `baselineFrom`); qualquer deriva é relatada como um **diff lado a lado**, com regex `allowedChanges` para permitir edições intencionais |
|
|
224
|
+
| Métricas de estresse | `stress` | latência de passo P99 (`maxP99Ms`), pior tempo até o primeiro token (`maxTtftMs`), velocidade agregada de geração de tokens (`minTokensPerSecond`) |
|
|
225
|
+
| Equidade | `bias` | radar de viés sobre a saída final: contagens por categoria com regex (`categories`), padrões `forbid` rígidos, tetos `maxHits`/`maxCategoryHits` |
|
|
226
|
+
| Revisão de segundo modelo | `review` | um veredito aprovação/falha complementar do subagente revisor (camada separada, a mesma costura do revisor de aprovação) |
|
|
227
|
+
|
|
228
|
+
```yaml
|
|
229
|
+
- id: regression-gate
|
|
230
|
+
input: Answer in one sentence.
|
|
231
|
+
expect:
|
|
232
|
+
prompt:
|
|
233
|
+
baseline: "You are a helpful software engineer assistant."
|
|
234
|
+
allowedChanges: ["copyright-year"]
|
|
235
|
+
stress:
|
|
236
|
+
maxP99Ms: 8000
|
|
237
|
+
maxTtftMs: 3000
|
|
238
|
+
minTokensPerSecond: 20
|
|
239
|
+
bias:
|
|
240
|
+
categories: { gender: ["[Hh]e is (un)?stable"] }
|
|
241
|
+
forbid: ["[Ss]crew that"]
|
|
242
|
+
maxCategoryHits: 0
|
|
243
|
+
```
|
|
244
|
+
|
|
245
|
+
Portão de CI: o processo sai com 0 somente quando todos os casos de todas as suítes passam — avaliações que falharem fazem o build falhar. Cada caso deixa um JSONL de sessão reproduzível e um JSON de rastro ao lado de `report.md`/`report.json`; resultados de asserção (incluindo o diff lado a lado do prompt), uso de tokens, métricas de estresse/equidade e o veredito de revisão são todos gravados nos arquivos de relatório.
|
|
246
|
+
|
|
247
|
+
```yaml
|
|
248
|
+
- name: dsh-eval
|
|
249
|
+
run: npx dsh-eval eval/cases --model deepseek-v4-flash --timeout-ms 240000 --out .eval-reports
|
|
250
|
+
env:
|
|
251
|
+
DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
|
|
252
|
+
```
|
|
253
|
+
|
|
254
|
+
`dsh-eval` difere de [openai/codex-research](https://github.com/openai/codex-research): o codex-research pontua trajetórias de agentes para comparação de pesquisa; `dsh-eval` é um harness declarativo de regressão aprovação/falha — casos YAML, asserções estruturadas de rastro/prompt/estresse/equidade, uma revisão opcional de segundo modelo e um código de saída de CI — para gatear qualquer agente DSH, não para ranking de pesquisa.
|
|
255
|
+
|
|
256
|
+
## Servidor MCP (autônomo)
|
|
257
|
+
|
|
258
|
+
O `dsh-auto-review` também inclui um **servidor MCP** stdio (`dsh-auto-review-mcp`) para que clientes MCP externos (Claude, Codex, …) consumam uma rota de revisão determinista sem o harness. Ele fala JSON-RPC 2.0 sobre JSON delimitado por novas linhas (NDJSON): um objeto JSON por linha, sem enquadramento `Content-Length`.
|
|
259
|
+
|
|
260
|
+
**Limite.** O revisor completo precisa do seam de subagentes do harness e de um segundo modelo, que um processo stdio separado não consegue alcançar. Portanto, o servidor autônomo é **regras deterministas + cache, sem revisão por modelo**:
|
|
261
|
+
|
|
262
|
+
- `review_action` reutiliza o cache de veredictos por impressão digital (`src/cache.ts`) e a resolução de regras de risco / políticas de ferramentas (`src/config.ts`): uma regra `never` → `deny`; um acerto de cache sobre uma impressão idêntica de `tool + arguments` reproduz esse veredicto; todo o resto (`ai` precisa de modelo, `human` precisa de um humano) → `deny` fail-closed com `reason: "standalone path, no model"`. Ele nunca permite uma ação que um modelo ainda não tenha permitido.
|
|
263
|
+
- `cache_stats` informa os contadores de acertos/armazenamento e o estado do TTL.
|
|
264
|
+
|
|
265
|
+
| Ferramenta | Propósito |
|
|
266
|
+
|---|---|
|
|
267
|
+
| `review_action` | `{tool, args?, reason?}` → `{decision, reason, riskLevel}` — negação determinista / reprodução de cache |
|
|
268
|
+
| `cache_stats` | `{}` → `{hits, stores, size, ttlMs, enabled}` |
|
|
269
|
+
|
|
270
|
+
Execução direta:
|
|
271
|
+
|
|
272
|
+
```sh
|
|
273
|
+
# as regras de risco vêm de variáveis de ambiente
|
|
274
|
+
export DSH_AUTO_REVIEW_RISK_RULES='[{"pattern":"rm -rf","policy":"never","field":"arguments"}]'
|
|
275
|
+
node bin/dsh-auto-review-mcp.mjs
|
|
276
|
+
# ou, após npm install: npx dsh-auto-review-mcp
|
|
277
|
+
```
|
|
278
|
+
|
|
279
|
+
Configuração por ambiente: `DSH_AUTO_REVIEW_RISK_RULES` (array JSON de `{pattern, policy, field?}`), `DSH_AUTO_REVIEW_TOOLS_POLICY` (JSON `{default?, overrides?}`), `DSH_AUTO_REVIEW_CACHE_TTL_MS`, `DSH_AUTO_REVIEW_CACHE_MAX_ENTRIES`.
|
|
280
|
+
|
|
281
|
+
Exemplo para o Claude Desktop (`claude_desktop_config.json`):
|
|
282
|
+
|
|
283
|
+
```json
|
|
284
|
+
{
|
|
285
|
+
"mcpServers": {
|
|
286
|
+
"dsh-auto-review": {
|
|
287
|
+
"command": "npx",
|
|
288
|
+
"args": ["-y", "dsh-auto-review-mcp"],
|
|
289
|
+
"env": {
|
|
290
|
+
"DSH_AUTO_REVIEW_RISK_RULES": "[{\"pattern\":\"rm -rf\",\"policy\":\"never\",\"field\":\"arguments\"}]"
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
```
|
|
296
|
+
|
|
297
|
+
O servidor é somente-leitura e determinista: sem rede, sem modelo, sem gravações.
|
|
214
298
|
|
|
215
299
|
## Permissões e dados
|
|
216
300
|
|
|
@@ -265,6 +349,8 @@ Layout do repositório: `src/index.ts` (contrato do plugin) · `src/config.ts` (
|
|
|
265
349
|
## Contribuidores
|
|
266
350
|
|
|
267
351
|
- [@PerryLink](https://github.com/PerryLink) — criador e mantenedor: o answerer de aprovação, o subagente revisor, a política de risco e o disjuntor, o painel de revisão por projeção de sessão, o companheiro invariant, o dsh-eval e a documentação em cinco idiomas.
|
|
352
|
+
- [@weipeng1999](https://github.com/weipeng1999) — propôs o roteamento independente de provedor/modelo do revisor ([#11](https://github.com/PerryLink/dsh-auto-review/issues/11), [discussão #12](https://github.com/PerryLink/dsh-auto-review/discussions/12)), que foi lançado como `reviewerProvider` / `reviewerModel`.
|
|
353
|
+
- [@alexchenzl](https://github.com/alexchenzl) — incluiu o plugin no diretório de plugins do DSH ([#10](https://github.com/PerryLink/dsh-auto-review/issues/10)).
|
|
268
354
|
|
|
269
355
|
## Família de Plugins DSH PerryLink
|
|
270
356
|
|
package/README.zh.md
CHANGED
|
@@ -95,6 +95,8 @@ dsh --profile web --dump-config | grep -A4 'id: auto-review'
|
|
|
95
95
|
| `riskPolicy` | `{maxAutoAllow: high, onHighRisk: delegate}` | 超过 `maxAutoAllow` 的 `allow` 裁决委派或拒绝 |
|
|
96
96
|
| `circuitBreaker` | `{consecutiveDenies: 3, windowDenies: 6, windowSize: 10, action: delegate}` | 拒绝熔断器 |
|
|
97
97
|
| `overrideTtlMs` | `300000` | `/auto-review approve` 覆盖的有效时长 |
|
|
98
|
+
| `verdictCacheTtlMs` | `60000` | 对相同 `工具 + 参数` 指纹复用近期裁决的时长;`0` 关闭缓存 |
|
|
99
|
+
| `verdictCacheMaxEntries` | `256` | 缓存指纹上限,超出后淘汰最旧条目 |
|
|
98
100
|
| `language` | `en` | `/auto-review` 命令输出的界面语言(`en` \| `zh`) |
|
|
99
101
|
| `allowUnmarkedAudit` | `false` | 强制在丢弃 `ignorable` 标记的宿主上写入会话审计(危险:未标记事件会让会话在其他宿主上无法恢复);默认自动探测并降级 |
|
|
100
102
|
|
|
@@ -210,7 +212,89 @@ suite:
|
|
|
210
212
|
dsh-eval eval/cases --model deepseek-v4-flash --timeout-ms 240000 --out .eval-reports
|
|
211
213
|
```
|
|
212
214
|
|
|
213
|
-
|
|
215
|
+
### 断言族
|
|
216
|
+
|
|
217
|
+
`expect` 块支持六类断言族;每条断言独立求值,并报告各自的通过/失败与期望/实际值,失败的用例无需重跑即可自我解释。
|
|
218
|
+
|
|
219
|
+
| 断言族 | DSL 键 | 门禁内容 |
|
|
220
|
+
|---|---|---|
|
|
221
|
+
| 工具轨迹 | `toolCalls`、`toolCallsExact`、`noToolCalls`、`results` | 有序工具调用序列(允许跳过的子序列)、精确名称序列、逐工具结果(`isError`/`contains`/`regex`) |
|
|
222
|
+
| 输出与预算 | `output`、`turnEnds`、`maxTokens` | 最终输出的子串/正则、回合结果、token 预算 |
|
|
223
|
+
| Prompt 回归 | `prompt` | 渲染后的系统提示词必须匹配已提交的 `baseline`(或 `baselineFrom` 文件);任何漂移都会以**并排 diff** 报告,可用 `allowedChanges` 正则白名单放行有意的修改 |
|
|
224
|
+
| 压测指标 | `stress` | P99 步延迟(`maxP99Ms`)、最差首 token 时延(`maxTtftMs`)、聚合 token 生成速度(`minTokensPerSecond`) |
|
|
225
|
+
| 公平性 | `bias` | 最终输出上的偏差雷达:逐类别正则计数(`categories`)、硬性 `forbid` 模式、`maxHits`/`maxCategoryHits` 上限 |
|
|
226
|
+
| 第二模型审查 | `review` | 审查子代理给出的补充通过/失败裁决(独立层,与审批审查器同一条接缝) |
|
|
227
|
+
|
|
228
|
+
```yaml
|
|
229
|
+
- id: regression-gate
|
|
230
|
+
input: Answer in one sentence.
|
|
231
|
+
expect:
|
|
232
|
+
prompt:
|
|
233
|
+
baseline: "You are a helpful software engineer assistant."
|
|
234
|
+
allowedChanges: ["copyright-year"]
|
|
235
|
+
stress:
|
|
236
|
+
maxP99Ms: 8000
|
|
237
|
+
maxTtftMs: 3000
|
|
238
|
+
minTokensPerSecond: 20
|
|
239
|
+
bias:
|
|
240
|
+
categories: { gender: ["[Hh]e is (un)?stable"] }
|
|
241
|
+
forbid: ["[Ss]crew that"]
|
|
242
|
+
maxCategoryHits: 0
|
|
243
|
+
```
|
|
244
|
+
|
|
245
|
+
CI 门禁:仅当每个套件的每个用例都通过时,进程才以 0 退出 —— 评估失败即构建失败。每个用例都会在 `report.md`/`report.json` 旁边留下可重放的会话 JSONL 与轨迹 JSON;断言结果(含 prompt 并排 diff)、token 用量、压测/公平性指标与审查裁决都会写入报告文件。
|
|
246
|
+
|
|
247
|
+
```yaml
|
|
248
|
+
- name: dsh-eval
|
|
249
|
+
run: npx dsh-eval eval/cases --model deepseek-v4-flash --timeout-ms 240000 --out .eval-reports
|
|
250
|
+
env:
|
|
251
|
+
DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
|
|
252
|
+
```
|
|
253
|
+
|
|
254
|
+
`dsh-eval` 与 [openai/codex-research](https://github.com/openai/codex-research) 的区别:codex-research 为研究对比对代理轨迹打分;`dsh-eval` 是声明式的通过/失败回归门禁 —— YAML 用例、结构化的轨迹/prompt/压测/公平性断言、可选的第二模型审查与 CI 退出码 —— 面向任何 DSH 代理的门禁,而非研究排名。
|
|
255
|
+
|
|
256
|
+
## MCP 服务器(独立进程)
|
|
257
|
+
|
|
258
|
+
`dsh-auto-review` 还附带一个 stdio **MCP 服务器**(`dsh-auto-review-mcp`),让外部 MCP 客户端(Claude、Codex 等)无需 harness 即可消费确定性裁决路径。它通过 newline-delimited JSON(NDJSON)承载 JSON-RPC 2.0——每行一个 JSON 对象,不支持 `Content-Length` 分帧。
|
|
259
|
+
|
|
260
|
+
**边界。** 完整 reviewer 依赖 harness 的 subagent seam 与第二模型,独立 stdio 进程无法调用。因此独立服务器是**确定性规则 + 缓存,无模型评审**:
|
|
261
|
+
|
|
262
|
+
- `review_action` 复用同指纹裁决缓存(`src/cache.ts`)与风险规则 / 工具策略解析(`src/config.ts`):命中 `never` 规则 → `deny`;缓存命中相同 `tool + arguments` 指纹 → 重放该裁决;其余情况(`ai` 需要模型、`human` 需要人工)→ fail-closed `deny`,reason 为 `"standalone path, no model"`。它绝不会放行一个模型尚未放行的动作。
|
|
263
|
+
- `cache_stats` 报告命中/存储计数与 TTL 状态。
|
|
264
|
+
|
|
265
|
+
| 工具 | 用途 |
|
|
266
|
+
|---|---|
|
|
267
|
+
| `review_action` | `{tool, args?, reason?}` → `{decision, reason, riskLevel}` —— 确定性拒绝 / 缓存重放 |
|
|
268
|
+
| `cache_stats` | `{}` → `{hits, stores, size, ttlMs, enabled}` |
|
|
269
|
+
|
|
270
|
+
直接运行:
|
|
271
|
+
|
|
272
|
+
```sh
|
|
273
|
+
# 风险规则来自环境变量
|
|
274
|
+
export DSH_AUTO_REVIEW_RISK_RULES='[{"pattern":"rm -rf","policy":"never","field":"arguments"}]'
|
|
275
|
+
node bin/dsh-auto-review-mcp.mjs
|
|
276
|
+
# 或 npm 安装后:npx dsh-auto-review-mcp
|
|
277
|
+
```
|
|
278
|
+
|
|
279
|
+
环境变量配置:`DSH_AUTO_REVIEW_RISK_RULES`(`{pattern, policy, field?}` 的 JSON 数组)、`DSH_AUTO_REVIEW_TOOLS_POLICY`(JSON `{default?, overrides?}`)、`DSH_AUTO_REVIEW_CACHE_TTL_MS`、`DSH_AUTO_REVIEW_CACHE_MAX_ENTRIES`。
|
|
280
|
+
|
|
281
|
+
Claude Desktop(`claude_desktop_config.json`)配置示例:
|
|
282
|
+
|
|
283
|
+
```json
|
|
284
|
+
{
|
|
285
|
+
"mcpServers": {
|
|
286
|
+
"dsh-auto-review": {
|
|
287
|
+
"command": "npx",
|
|
288
|
+
"args": ["-y", "dsh-auto-review-mcp"],
|
|
289
|
+
"env": {
|
|
290
|
+
"DSH_AUTO_REVIEW_RISK_RULES": "[{\"pattern\":\"rm -rf\",\"policy\":\"never\",\"field\":\"arguments\"}]"
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
```
|
|
296
|
+
|
|
297
|
+
服务器只读且确定性:无网络、无模型、无写入。
|
|
214
298
|
|
|
215
299
|
## 权限与数据
|
|
216
300
|
|
|
@@ -265,6 +349,8 @@ pnpm pack # 发布产物
|
|
|
265
349
|
## 贡献者
|
|
266
350
|
|
|
267
351
|
- [@PerryLink](https://github.com/PerryLink) —— 创建者与维护者:审批 answerer、审查子代理、风险策略与熔断器、会话投影审查面板、invariant 配套插件、dsh-eval,以及五语文档。
|
|
352
|
+
- [@weipeng1999](https://github.com/weipeng1999) —— 提出审查器独立的 provider/model 路由([#11](https://github.com/PerryLink/dsh-auto-review/issues/11)、[讨论 #12](https://github.com/PerryLink/dsh-auto-review/discussions/12)),已落地为 `reviewerProvider` / `reviewerModel`。
|
|
353
|
+
- [@alexchenzl](https://github.com/alexchenzl) —— 将本插件收录进 DSH 插件目录([#10](https://github.com/PerryLink/dsh-auto-review/issues/10))。
|
|
268
354
|
|
|
269
355
|
## PerryLink DSH 插件家族
|
|
270
356
|
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// dsh-auto-review stdio MCP server launcher. Serves review_action (deterministic
|
|
3
|
+
// verdict + same-fingerprint cache) and cache_stats over newline-delimited
|
|
4
|
+
// JSON-RPC 2.0 on stdio. No harness, no model, no write path — this is the
|
|
5
|
+
// independent, fail-closed review path; see README "MCP server (standalone)".
|
|
6
|
+
//
|
|
7
|
+
// Config comes from environment variables (see resolveEnvConfig):
|
|
8
|
+
// DSH_AUTO_REVIEW_RISK_RULES JSON [{pattern, policy, field?}]
|
|
9
|
+
// DSH_AUTO_REVIEW_TOOLS_POLICY JSON {default?, overrides?}
|
|
10
|
+
// DSH_AUTO_REVIEW_CACHE_TTL_MS non-negative integer
|
|
11
|
+
// DSH_AUTO_REVIEW_CACHE_MAX_ENTRIES positive integer
|
|
12
|
+
import { readFileSync } from 'node:fs'
|
|
13
|
+
import { StandaloneReviewer, createMcpServer, resolveEnvConfig, runStdioServer } from '../lib/mcp.js'
|
|
14
|
+
|
|
15
|
+
const pkg = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8'))
|
|
16
|
+
const config = resolveEnvConfig(process.env)
|
|
17
|
+
const server = createMcpServer(new StandaloneReviewer(config), pkg.version)
|
|
18
|
+
|
|
19
|
+
runStdioServer(server)
|