dsh-model-jury 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/COMPATIBILITY.md +17 -0
- package/CONTRIBUTING.md +31 -0
- package/DEMO.md +64 -0
- package/LICENSE +21 -0
- package/PRE_FLIGHT.md +39 -0
- package/README.md +196 -0
- package/REAL_DEMO.md +78 -0
- package/SECURITY.md +9 -0
- package/cordis.patch.yml +30 -0
- package/lib/command.d.ts +13 -0
- package/lib/command.js +45 -0
- package/lib/config.d.ts +35 -0
- package/lib/config.js +39 -0
- package/lib/council-service.d.ts +33 -0
- package/lib/council-service.js +86 -0
- package/lib/index.d.ts +13 -0
- package/lib/index.js +12 -0
- package/lib/persistence/run-store.d.ts +15 -0
- package/lib/persistence/run-store.js +55 -0
- package/lib/protocol/aggregate.d.ts +40 -0
- package/lib/protocol/aggregate.js +87 -0
- package/lib/protocol/anonymize.d.ts +20 -0
- package/lib/protocol/anonymize.js +60 -0
- package/lib/protocol/prompts.d.ts +12 -0
- package/lib/protocol/prompts.js +102 -0
- package/lib/protocol/report.d.ts +4 -0
- package/lib/protocol/report.js +101 -0
- package/lib/protocol/schemas.d.ts +66 -0
- package/lib/protocol/schemas.js +48 -0
- package/lib/protocol/state-machine.d.ts +41 -0
- package/lib/protocol/state-machine.js +109 -0
- package/lib/seats/codex-seat.d.ts +17 -0
- package/lib/seats/codex-seat.js +81 -0
- package/lib/seats/llm-seat.d.ts +17 -0
- package/lib/seats/llm-seat.js +96 -0
- package/lib/seats/structured.d.ts +22 -0
- package/lib/seats/structured.js +126 -0
- package/lib/seats/types.d.ts +58 -0
- package/lib/seats/types.js +16 -0
- package/lib/seats/unavailable-seat.d.ts +13 -0
- package/lib/seats/unavailable-seat.js +40 -0
- package/package.json +80 -0
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project will be documented here.
|
|
4
|
+
|
|
5
|
+
## [0.1.0] - 2026-08-31
|
|
6
|
+
|
|
7
|
+
### Added
|
|
8
|
+
|
|
9
|
+
- Blind concurrent first-round answers from GPT through native Codex, GLM, and DeepSeek.
|
|
10
|
+
- Random P1/P2/P3 position mapping and anonymous cross-critique.
|
|
11
|
+
- Structured revision, deterministic quorum and vote aggregation, explicit dissent, and critical-risk handling.
|
|
12
|
+
- Provider health checks through `/jury doctor`.
|
|
13
|
+
- Owner-only redacted run artifacts, deterministic fake integration tests, and a real three-provider validation transcript.
|
|
14
|
+
- Prebuilt DSH bundle packaging and compatibility documentation for DSH `0.1.2-alpha.2` at `0a53fb55bea101816fa226bb964ae2bed71c343b`.
|
|
15
|
+
|
|
16
|
+
### Release validation
|
|
17
|
+
|
|
18
|
+
- 3 providers, 3 rounds, and 9 successful real model calls from the installed package.
|
|
19
|
+
- Native Codex/ChatGPT subscription authentication passed with `OPENAI_API_KEY` absent.
|
|
20
|
+
- Round 1 ran concurrently with a maximum 6 ms dispatch skew in the final validation.
|
|
21
|
+
- Anonymous competitor packets produced zero provider-identity leaks.
|
|
22
|
+
- Degraded 2/3 quorum, failed 0–1/3 quorum, dissent preservation, and critical-risk policy passed automated tests.
|
package/COMPATIBILITY.md
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# Compatibility
|
|
2
|
+
|
|
3
|
+
| Component | Tested value |
|
|
4
|
+
|---|---|
|
|
5
|
+
| DeepSeek Harness | `0.1.2-alpha.2`, commit `0a53fb55bea101816fa226bb964ae2bed71c343b` |
|
|
6
|
+
| DSH Codex provider baseline | `@openai/codex` `0.149.1` through `@deepseek-ai/dsh-subagent-codex` |
|
|
7
|
+
| Host Codex CLI observed | `codex-cli 0.146.0` |
|
|
8
|
+
| Node.js | `v25.2.1` |
|
|
9
|
+
| pnpm | `11.7.0` for the pinned DSH workspace; local default observed as `10.28.2` |
|
|
10
|
+
| Operating system | macOS 26.4.1, Apple Silicon |
|
|
11
|
+
| GPT route | `ctx.subagents` provider `model-jury-codex` backed by first-party Codex app-server |
|
|
12
|
+
| GLM route tested | `ctx.llm` provider `glm`, model `glm-5.3`, through `llm-pi-ai` OpenAI-compatible configuration |
|
|
13
|
+
| DeepSeek route tested | `ctx.llm` provider `deepseek-official`, model `deepseek-v4-flash` |
|
|
14
|
+
|
|
15
|
+
The bundle imports only public package entry points. It does not import DSH source subpaths or patch the host.
|
|
16
|
+
|
|
17
|
+
The package-local Codex binary actually selected by the first-party provider was `@openai/codex` `0.149.1`. The host-wide `codex --version` value is recorded only for comparison and was not the backend selected by Model Jury.
|
package/CONTRIBUTING.md
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# Contributing to dsh-model-jury
|
|
2
|
+
|
|
3
|
+
Thank you for improving structured cross-model peer review for DeepSeek Harness.
|
|
4
|
+
|
|
5
|
+
## Before opening a change
|
|
6
|
+
|
|
7
|
+
- Keep provider credentials outside the repository, fixtures, logs, and artifacts.
|
|
8
|
+
- Preserve Round 1 blindness and Round 2 anonymity.
|
|
9
|
+
- Keep aggregation deterministic. A model must not become a permanent chairman or silently replace quorum, vote, dissent, or critical-risk policy.
|
|
10
|
+
- Keep Model Jury deliberation-only. Do not add automatic code changes, commits, pushes, deployments, or mutation tools to the protocol.
|
|
11
|
+
- Use public DSH APIs and provider-neutral routes where available.
|
|
12
|
+
|
|
13
|
+
## Development
|
|
14
|
+
|
|
15
|
+
Use Node.js `^22.19.0` or `>=24` and pnpm `10.28.2` or a compatible release:
|
|
16
|
+
|
|
17
|
+
```sh
|
|
18
|
+
pnpm install --frozen-lockfile
|
|
19
|
+
pnpm test
|
|
20
|
+
pnpm typecheck
|
|
21
|
+
pnpm build
|
|
22
|
+
pnpm pack
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
Tests must use fake seats unless a change specifically requires an opt-in real-provider validation. Never put API keys into test fixtures. Real-provider transcripts must contain only visible structured responses and safe metadata.
|
|
26
|
+
|
|
27
|
+
## Pull requests
|
|
28
|
+
|
|
29
|
+
Explain the user-visible problem, the resulting behavior, and how you verified it. Keep protocol changes separate from provider or documentation changes when practical. Add a regression test when a bug could confuse anonymous labels, leak provider identity, change quorum behavior, drop dissent, or alter critical-risk handling.
|
|
30
|
+
|
|
31
|
+
By contributing, you agree that your contribution is licensed under the MIT License.
|
package/DEMO.md
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# Complete deterministic demo run
|
|
2
|
+
|
|
3
|
+
This transcript is produced by the same `CouncilStateMachine`, Zod schemas, and
|
|
4
|
+
aggregator exercised in `tests/state-machine.test.ts`. The seats are deterministic
|
|
5
|
+
test doubles so the protocol can be demonstrated without provider credentials.
|
|
6
|
+
|
|
7
|
+
## Input
|
|
8
|
+
|
|
9
|
+
- Question: Should the service use strategy A or strategy B?
|
|
10
|
+
- Style: balanced
|
|
11
|
+
- Anonymous mapping for this run: GPT → P2, GLM → P3, DeepSeek → P1
|
|
12
|
+
|
|
13
|
+
## Round 1 — blind independent positions
|
|
14
|
+
|
|
15
|
+
All three seats receive the same prompt and none receives another seat's answer.
|
|
16
|
+
|
|
17
|
+
| Seat | Thesis / recommendation | Confidence |
|
|
18
|
+
|---|---|---:|
|
|
19
|
+
| GPT | Choose A | 70 |
|
|
20
|
+
| GLM | Choose B | 70 |
|
|
21
|
+
| DeepSeek | Choose A | 70 |
|
|
22
|
+
|
|
23
|
+
Each position also returns the validated fields `reasoning`, `assumptions`,
|
|
24
|
+
`risks`, and `evidence_needed`. In this fixture those are, respectively,
|
|
25
|
+
`Reason for <choice>`, `Stable requirements`, `Execution risk`, and `A benchmark`.
|
|
26
|
+
|
|
27
|
+
## Round 2 — anonymous cross-critique
|
|
28
|
+
|
|
29
|
+
The positions are redistributed only as P1, P2, and P3. These participant labels
|
|
30
|
+
are separate from the strategy A/B names in the question.
|
|
31
|
+
Provider and model identity terms are scrubbed before redistribution.
|
|
32
|
+
|
|
33
|
+
| Reviewing seat | Preferred anonymous position | Critique | Confidence |
|
|
34
|
+
|---|---:|---|---:|
|
|
35
|
+
| GPT | P1 | Clear objective; limited evidence; benchmark data is missing | 72 |
|
|
36
|
+
| GLM | P2 | Clear objective; limited evidence; benchmark data is missing | 72 |
|
|
37
|
+
| DeepSeek | P1 | Clear objective; limited evidence; benchmark data is missing | 72 |
|
|
38
|
+
|
|
39
|
+
## Round 3 — final revision
|
|
40
|
+
|
|
41
|
+
| Seat | Final vote | Final recommendation | Changed mind | Confidence |
|
|
42
|
+
|---|---:|---|---:|---:|
|
|
43
|
+
| GPT | P1 | Choose A | no | 78 |
|
|
44
|
+
| GLM | P2 | Choose B | no | 78 |
|
|
45
|
+
| DeepSeek | P1 | Choose A | no | 78 |
|
|
46
|
+
|
|
47
|
+
All seats agree that outcomes should be measured. Their remaining disagreement is
|
|
48
|
+
the implementation order. No critical-risk flag is active.
|
|
49
|
+
|
|
50
|
+
## Deterministic verdict
|
|
51
|
+
|
|
52
|
+
- Valid: yes
|
|
53
|
+
- Result: majority
|
|
54
|
+
- Quorum: 3/3
|
|
55
|
+
- Vote distribution: P1 = 2, P2 = 1
|
|
56
|
+
- Recommendation: Choose A
|
|
57
|
+
- Consensus: Measure outcomes
|
|
58
|
+
- Minority view: GLM — Choose B
|
|
59
|
+
- Evidence still needed: A benchmark
|
|
60
|
+
- Changed-mind count: 0
|
|
61
|
+
- Critical review required: no
|
|
62
|
+
|
|
63
|
+
The test suite separately verifies unanimous, split, degraded 2/3, failed 0–1/3,
|
|
64
|
+
hybrid, and matching critical-risk-category outcomes.
|
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 dsh-model-jury contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/PRE_FLIGHT.md
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# Model Jury preflight
|
|
2
|
+
|
|
3
|
+
Verified on 2026-08-31 against `deepseek-ai/deepseek-harness` commit `0a53fb55bea101816fa226bb964ae2bed71c343b` (`0.1.2-alpha.2`). The source checkout remained unmodified.
|
|
4
|
+
|
|
5
|
+
## Public APIs
|
|
6
|
+
|
|
7
|
+
- `ctx.commands.register()` accepts a lowercase slash-command definition and an abortable handler. Command results are rendered by the UI without entering model history.
|
|
8
|
+
- `ctx.llm.stream()` accepts a hand-built provider-neutral request with `provider`, `model`, messages, optional output controls, and an `AbortSignal`. `BlockAssembler` is the public stream assembly helper. `ctx.llm.listProviders()` exposes the active route set.
|
|
9
|
+
- `ctx.subagents.start(provider, request)` starts a named one-shot provider and returns a disposable run. The first-party `@deepseek-ai/dsh-subagent-codex` package registers a Codex provider backed by the official `codex app-server --stdio` protocol.
|
|
10
|
+
- The Codex provider exposes provider name, model, environment overlay, permission mode, and process shutdown grace as public configuration. It preserves native Codex configuration and authentication.
|
|
11
|
+
- `@deepseek-ai/dsh-llm-pi-ai` supports named custom routes using `openai-completions` or `openai-responses`, a configurable base URL, credential reference, and model list. This is sufficient for GLM without a Model Jury-owned adapter.
|
|
12
|
+
- `@deepseek-ai/dsh-llm-deepseek` owns the first-party `deepseek-official` route and accepts configurable model IDs at call time.
|
|
13
|
+
|
|
14
|
+
## Native Codex authentication proof
|
|
15
|
+
|
|
16
|
+
The preflight launched a real child through `ctx.subagents.start('codex', ...)` with the first-party DSH Codex provider and the official package-local app-server. The process environment was launched with `OPENAI_API_KEY` unset. The child returned exactly `CODEX_SUBSCRIPTION_OK` and settled with `stopReason: completed`.
|
|
17
|
+
|
|
18
|
+
The preflight did not write or replace `~/.codex/config.toml`, did not set `OPENAI_API_KEY`, and did not configure an OpenAI API fallback. Native Codex account state remained authoritative.
|
|
19
|
+
|
|
20
|
+
The final built `CodexSeat` was then checked independently through the bundle's
|
|
21
|
+
`model-jury-codex` provider with `OPENAI_API_KEY` explicitly removed. Its real
|
|
22
|
+
app-server child returned `ok: true` with the detail `native Codex authentication
|
|
23
|
+
completed a real app-server child`.
|
|
24
|
+
|
|
25
|
+
## Bundle validation
|
|
26
|
+
|
|
27
|
+
The final package was packed with `npm pack`, installed through `dsh plugin` into
|
|
28
|
+
a fresh isolated Web profile, found in the composed configuration, and booted by
|
|
29
|
+
the pinned DSH Web host. `/jury doctor` returned READY through the real Web
|
|
30
|
+
command surface. Full real GPT + GLM + DeepSeek runs completed Blind Round 1,
|
|
31
|
+
anonymous Round 2, Round 3 revision, and deterministic aggregation.
|
|
32
|
+
|
|
33
|
+
## Risks and deviations
|
|
34
|
+
|
|
35
|
+
- The public Codex provider has no strict read-only permission mode. Its safest exposed mode is `never`, which disables approval prompts but leaves the sandbox to native Codex configuration. Model Jury prompts prohibit tools, project modifications, installs, commits, pushes, deployments, and network mutations. This is prompt enforcement rather than a hard capability boundary.
|
|
36
|
+
- The Codex provider does not accept an output schema. The seat requests JSON and performs one bounded repair call when validation fails.
|
|
37
|
+
- GLM and DeepSeek credentials were discovered in the user's existing API file and injected only into the isolated DSH launch process. Both real provider probes and full Model Jury calls passed. No credential value was copied into this package, its docs, tests, configuration, logs, or artifacts.
|
|
38
|
+
- Model Jury artifacts are intentionally written beneath `.dsh-model-jury/`; ordinary project files are not changed by runs.
|
|
39
|
+
- Round 2 and Round 3 include all three anonymous labels, including the reviewing seat's own position, because the required `P1|P2|P3|hybrid|undecided` vote schema has no `own` value. The original answer is also supplied separately, so this duplicates content but preserves stable aggregation without explicitly exposing the provider mapping.
|
package/README.md
ADDED
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
# dsh-model-jury
|
|
2
|
+
|
|
3
|
+
> **Structured cross-model peer review for DeepSeek Harness.**
|
|
4
|
+
|
|
5
|
+
**Models deliberate. Code counts. Dissent survives.**
|
|
6
|
+
|
|
7
|
+
Blind independent answers → anonymous peer critique → revision → deterministic verdict.
|
|
8
|
+
|
|
9
|
+
**No permanent chairman. The judge is code.**
|
|
10
|
+
|
|
11
|
+
> Compatibility: verified on DeepSeek Harness `0.1.2-alpha.2` at commit `0a53fb55bea101816fa226bb964ae2bed71c343b`. DSH is in developer preview and may introduce breaking changes. See [COMPATIBILITY.md](COMPATIBILITY.md).
|
|
12
|
+
|
|
13
|
+
```text
|
|
14
|
+
One Question
|
|
15
|
+
│
|
|
16
|
+
┌─────────────────┼─────────────────┐
|
|
17
|
+
▼ ▼ ▼
|
|
18
|
+
GPT via Codex GLM DeepSeek
|
|
19
|
+
ChatGPT subscription API API
|
|
20
|
+
│ │ │
|
|
21
|
+
└──────── Blind Round ─────────────┘
|
|
22
|
+
│
|
|
23
|
+
P1 / P2 / P3
|
|
24
|
+
│
|
|
25
|
+
Anonymous Peer Review
|
|
26
|
+
│
|
|
27
|
+
Revision
|
|
28
|
+
│
|
|
29
|
+
Deterministic Aggregation
|
|
30
|
+
┌─────────┼─────────┐
|
|
31
|
+
▼ ▼ ▼
|
|
32
|
+
Majority Dissent Risks
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
Release validation: **3 providers · 3 rounds · 9 real model calls · 0 identity leaks · 6 ms maximum Round-1 dispatch skew**.
|
|
36
|
+
|
|
37
|
+
The full redacted provider transcript, vote, dissent, risk, and timing record is in [REAL_DEMO.md](REAL_DEMO.md).
|
|
38
|
+
|
|
39
|
+
## Install in 30 seconds
|
|
40
|
+
|
|
41
|
+
Install the prebuilt `v0.1.0` tarball into the DSH Web profile, inspect the composed layer, and boot DSH:
|
|
42
|
+
|
|
43
|
+
```sh
|
|
44
|
+
dsh plugin --profile web add ./dsh-model-jury-0.1.0.tgz
|
|
45
|
+
dsh --profile web --dump-config
|
|
46
|
+
dsh web
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
After an npm release, the first command can instead be:
|
|
50
|
+
|
|
51
|
+
```sh
|
|
52
|
+
dsh plugin --profile web add dsh-model-jury
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
The tarball and npm package contain compiled `lib/` output. Installation does not require permission to run a package build script.
|
|
56
|
+
|
|
57
|
+
## Setup
|
|
58
|
+
|
|
59
|
+
Model Jury uses DSH's provider-neutral public services. It does not implement private HTTP clients or proxy provider traffic.
|
|
60
|
+
|
|
61
|
+
### GPT through native Codex authentication
|
|
62
|
+
|
|
63
|
+
The GPT seat uses DSH's first-party Codex app-server provider and the user's native ChatGPT/Codex subscription authentication. `OPENAI_API_KEY` is not required. The bundle registers an isolated `model-jury-codex` route so GLM and DeepSeek provider configuration cannot replace Codex authentication.
|
|
64
|
+
|
|
65
|
+
### GLM
|
|
66
|
+
|
|
67
|
+
Configure a GLM route through DSH's `@deepseek-ai/dsh-llm-pi-ai` provider. Keep the credential in DSH settings or the DSH launch environment:
|
|
68
|
+
|
|
69
|
+
```yaml
|
|
70
|
+
- id: llm-pi-ai
|
|
71
|
+
config:
|
|
72
|
+
providers:
|
|
73
|
+
glm:
|
|
74
|
+
displayName: GLM
|
|
75
|
+
api: openai-completions
|
|
76
|
+
baseURL: '<GLM OpenAI-compatible base URL>'
|
|
77
|
+
apiKeyEnv: GLM_API_KEY
|
|
78
|
+
models:
|
|
79
|
+
- id: '<your GLM model id>'
|
|
80
|
+
name: GLM jury model
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
### DeepSeek and jury models
|
|
84
|
+
|
|
85
|
+
The DSH base profile supplies the first-party `deepseek-official` route. Configure `DEEPSEEK_API_KEY` through DSH settings or the launch environment, then override the complete `model-jury` row in the profile's `cordis.patch.yml` with the model IDs available to your account:
|
|
86
|
+
|
|
87
|
+
```yaml
|
|
88
|
+
- id: model-jury
|
|
89
|
+
config:
|
|
90
|
+
storageDir: .dsh-model-jury
|
|
91
|
+
maxQuestionChars: 12000
|
|
92
|
+
maxFieldChars: 6000
|
|
93
|
+
codex:
|
|
94
|
+
provider: model-jury-codex
|
|
95
|
+
timeoutMs: 180000
|
|
96
|
+
maxRetries: 1
|
|
97
|
+
glm:
|
|
98
|
+
provider: glm
|
|
99
|
+
model: '<your GLM model id>'
|
|
100
|
+
timeoutMs: 120000
|
|
101
|
+
maxTokens: 16384
|
|
102
|
+
maxRetries: 1
|
|
103
|
+
deepseek:
|
|
104
|
+
provider: deepseek-official
|
|
105
|
+
model: '<your DeepSeek model id>'
|
|
106
|
+
timeoutMs: 120000
|
|
107
|
+
maxTokens: 16384
|
|
108
|
+
maxRetries: 1
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
Credentials never belong in this package, patch, test fixtures, or run artifacts.
|
|
112
|
+
|
|
113
|
+
## Doctor
|
|
114
|
+
|
|
115
|
+
```text
|
|
116
|
+
/jury doctor
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
Doctor performs one minimal real request per seat, so it can consume provider quota. It reports provider, model, status, duration, safe diagnostics, and native Codex authentication without printing credential values.
|
|
120
|
+
|
|
121
|
+
## Run
|
|
122
|
+
|
|
123
|
+
```text
|
|
124
|
+
/jury Should this inference runtime prioritize iOS support or ARM SIMD optimization?
|
|
125
|
+
/jury --style adversarial Should this inference runtime prioritize iOS support or ARM SIMD optimization?
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
`balanced` is the default. `adversarial` strengthens the critique instructions without changing provider topology or aggregation.
|
|
129
|
+
|
|
130
|
+
## Why Model Jury
|
|
131
|
+
|
|
132
|
+
Most model councils ask several models, then ask another model to judge them. Model Jury takes a different approach:
|
|
133
|
+
|
|
134
|
+
1. Models answer independently.
|
|
135
|
+
2. Competing answers are anonymized.
|
|
136
|
+
3. Models critique one another.
|
|
137
|
+
4. Models get a chance to revise.
|
|
138
|
+
5. Code computes quorum and vote state.
|
|
139
|
+
6. Dissent and critical risks remain visible.
|
|
140
|
+
|
|
141
|
+
The current reference deployment uses GPT/Codex, GLM, and DeepSeek. The protocol itself is provider-neutral: `CouncilSeat` implementations can route other compatible models through the same blind, anonymous, structured state machine.
|
|
142
|
+
|
|
143
|
+
## Protocol
|
|
144
|
+
|
|
145
|
+
Round 1 sends the same question, instructions, and JSON schema to all three seats concurrently. No seat sees another response. A random per-run mapping then assigns the responses to P1, P2, and P3. These labels identify participant positions only; answer choices such as Strategy A/B remain ordinary question content.
|
|
146
|
+
|
|
147
|
+
Round 2 gives each model its own first answer plus the anonymized positions. Provider, model, and common product identity terms are scrubbed from redistributed fields. The prompt requests strongest points, weakest points, missing evidence, actual disagreement, and calibrated severity without rewarding consensus.
|
|
148
|
+
|
|
149
|
+
Round 3 allows each surviving model to revise, merge, remain undecided, or preserve dissent. The deterministic aggregator accepts only `P1`, `P2`, `P3`, `hybrid`, or `undecided`; computes vote state and quorum; and keeps named final dissent visible to the user.
|
|
150
|
+
|
|
151
|
+
Three final seats produce a normal verdict. Two produce an explicit `DEGRADED` verdict. Zero or one produces a failed report. A single structured critical-risk flag is surfaced prominently but has no veto. Two compatible normalized critical-risk categories trigger `CRITICAL REVIEW REQUIRED`.
|
|
152
|
+
|
|
153
|
+
## Artifacts
|
|
154
|
+
|
|
155
|
+
Every run writes owner-only trace files beneath:
|
|
156
|
+
|
|
157
|
+
```text
|
|
158
|
+
.dsh-model-jury/runs/<run-id>/
|
|
159
|
+
request.json
|
|
160
|
+
config.json
|
|
161
|
+
state.json
|
|
162
|
+
prompts/round1|round2|round3/<seat>.txt
|
|
163
|
+
round1|round2|round3/<seat>.json
|
|
164
|
+
redistribution.json
|
|
165
|
+
calls.json
|
|
166
|
+
verdict.json
|
|
167
|
+
report.md
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
Artifacts contain visible structured responses, safe call metadata, prompts, aggregation data, and the final report. They exclude environment snapshots, request headers, API keys, OAuth state, provider request bodies, and hidden chain-of-thought. Credential-shaped fields are recursively redacted.
|
|
171
|
+
|
|
172
|
+
## Safety
|
|
173
|
+
|
|
174
|
+
Model Jury is deliberation-only. It never applies a recommendation, edits project files, installs dependencies, commits, pushes, deploys, or performs deliberate network mutations. GLM and DeepSeek receive no tools. Codex uses `permissionMode: never` plus mutation-prohibiting prompts; DSH's tested public Codex provider does not expose a strict read-only flag, so this remains a documented limitation. Read [SECURITY.md](SECURITY.md) before using sensitive workspace content.
|
|
175
|
+
|
|
176
|
+
## Develop and package
|
|
177
|
+
|
|
178
|
+
```sh
|
|
179
|
+
pnpm install --frozen-lockfile
|
|
180
|
+
pnpm test
|
|
181
|
+
pnpm typecheck
|
|
182
|
+
pnpm build
|
|
183
|
+
pnpm pack
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
The deterministic fake transcript is in [DEMO.md](DEMO.md). Provider and host validation is recorded in [PRE_FLIGHT.md](PRE_FLIGHT.md). Contributions are welcome under [CONTRIBUTING.md](CONTRIBUTING.md).
|
|
187
|
+
|
|
188
|
+
## Known limitations
|
|
189
|
+
|
|
190
|
+
- GLM and DeepSeek require user-supplied provider credentials and model IDs.
|
|
191
|
+
- The tested public DSH Codex provider cannot enforce a hard read-only sandbox through its plugin configuration.
|
|
192
|
+
- Vote consensus uses explicit structured labels, not semantic clustering of textual `hybrid` proposals.
|
|
193
|
+
- Each Codex call starts an ephemeral app-server thread.
|
|
194
|
+
- There is no automatic implementation, research, chairman model, deployment, or five/seven-seat mode.
|
|
195
|
+
|
|
196
|
+
MIT licensed. See [LICENSE](LICENSE).
|
package/REAL_DEMO.md
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# Real three-provider Model Jury demo
|
|
2
|
+
|
|
3
|
+
This is a visible-output transcript summary from a real installed-package run on 2026-08-31. It contains no credentials, request headers, environment snapshot, or hidden reasoning. Public route names are shown after the nonfunctional `dsh-model-jury` branding migration; the provider backends and protocol are unchanged.
|
|
4
|
+
|
|
5
|
+
## Question
|
|
6
|
+
|
|
7
|
+
> For a small engineering team building a local-first developer tool, should the core orchestration engine use a deterministic workflow or allow the LLM to dynamically select the next agent? Compare reliability, debuggability, flexibility, cost, failure recovery, and long-term maintainability.
|
|
8
|
+
|
|
9
|
+
## Providers
|
|
10
|
+
|
|
11
|
+
| Seat | Provider | Model/backend | Health |
|
|
12
|
+
|---|---|---|---|
|
|
13
|
+
| GPT | `model-jury-codex` | native Codex app-server `0.149.1` | PASS with `OPENAI_API_KEY` absent |
|
|
14
|
+
| GLM | `glm` | `glm-5.3` | PASS |
|
|
15
|
+
| DeepSeek | `deepseek-official` | `deepseek-v4-flash` | PASS |
|
|
16
|
+
|
|
17
|
+
Anonymous mapping for this run:
|
|
18
|
+
|
|
19
|
+
| Seat | Position |
|
|
20
|
+
|---|---|
|
|
21
|
+
| GPT | P2 |
|
|
22
|
+
| GLM | P1 |
|
|
23
|
+
| DeepSeek | P3 |
|
|
24
|
+
|
|
25
|
+
## Round 1 — blind independent positions
|
|
26
|
+
|
|
27
|
+
- **GPT:** deterministic, versioned state-machine control should be the core, with narrowly allowlisted LLM routing only where explicit rules are inadequate. Confidence 92.
|
|
28
|
+
- **GLM:** deterministic workflow control with schema-validated, bounded LLM discretion; avoid fully dynamic agent selection. Confidence 78.
|
|
29
|
+
- **DeepSeek:** deterministic state machine or DAG, with LLM variability confined to predefined recoverable nodes. Confidence 82.
|
|
30
|
+
|
|
31
|
+
The three prompts were byte-identical and contained no other seat output. Their calls started within 80 ms of one another.
|
|
32
|
+
|
|
33
|
+
## Round 2 — anonymous cross-critique
|
|
34
|
+
|
|
35
|
+
Only P1/P2/P3 identified competing positions. No GPT, OpenAI, Codex, GLM, Zhipu, DeepSeek, provider id, or model id appeared in any competitor packet.
|
|
36
|
+
|
|
37
|
+
- **GPT** preferred P2 and argued it best combined deterministic reliability with bounded routing; it criticized P1 for overclaiming exact replay and P3 for understating flexibility costs. Confidence 94.
|
|
38
|
+
- **GLM** preferred P2, emphasizing its operational detail while challenging unsupported assumptions about bounded routing and future retrofit cost. Confidence 72.
|
|
39
|
+
- **DeepSeek** preferred P3, arguing that even allowlisted next-step routing can reintroduce nondeterministic control flow and harder replay. Confidence 84.
|
|
40
|
+
|
|
41
|
+
Every response named the strongest point, weakest point, possible error, and missing evidence for both competing positions. The critiques contained substantive disagreement and were not consensus-only summaries.
|
|
42
|
+
|
|
43
|
+
## Round 3 — revision and rebuttal
|
|
44
|
+
|
|
45
|
+
- **GPT:** P2, unchanged. Deterministic state machine with guarded, recorded, allowlisted routing checkpoints. Confidence 86.
|
|
46
|
+
- **GLM:** changed to `hybrid`. It retained deterministic graph control, narrowed routing, and added an explicit concern that the recommendation depends on task decomposability. Confidence 82.
|
|
47
|
+
- **DeepSeek:** changed to P2 after accepting guarded next-step routing with deterministic fallbacks and persisted decisions. Confidence 80.
|
|
48
|
+
|
|
49
|
+
Both `changed_mind: true` and `changed_mind: false` were accepted in the same real run.
|
|
50
|
+
|
|
51
|
+
## Deterministic verdict
|
|
52
|
+
|
|
53
|
+
```text
|
|
54
|
+
Result: MAJORITY
|
|
55
|
+
Quorum: 3/3
|
|
56
|
+
Votes: P2 = 2, hybrid = 1
|
|
57
|
+
Winning choice: P2
|
|
58
|
+
Changed minds: 2
|
|
59
|
+
Critical review required: no
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Recommendation: use a deterministic, versioned state machine or DAG as the core. Permit LLM routing only at selected checkpoints over a small allowlist, with schema validation, recorded decision context, retry limits, checkpoints, and an explicit safe fallback.
|
|
63
|
+
|
|
64
|
+
Minority / dissent: GLM retained a distinct hybrid recommendation. It warned that a stable workflow graph and testable bounded routing are assumptions; a product centered on novel, unstructured autonomy could reverse the recommendation. This single structured critical risk was surfaced but did not veto the majority.
|
|
65
|
+
|
|
66
|
+
## Timing
|
|
67
|
+
|
|
68
|
+
| Round | GPT | GLM | DeepSeek |
|
|
69
|
+
|---|---:|---:|---:|
|
|
70
|
+
| Round 1 | 33.279 s | 35.889 s | 10.866 s |
|
|
71
|
+
| Round 2 | 26.759 s | 103.273 s | 45.349 s |
|
|
72
|
+
| Round 3 | 27.786 s | 70.860 s | 35.887 s |
|
|
73
|
+
|
|
74
|
+
Run id: `20260831T084039975Z-e819e49e`.
|
|
75
|
+
|
|
76
|
+
## Deliberation stress run
|
|
77
|
+
|
|
78
|
+
A second real run compared Apple/iOS support with ARM64/SIMD optimization. DeepSeek initially chose ARM64/SIMD while GPT and GLM chose Apple support. Anonymous critique changed both GLM and DeepSeek to guarded hybrid proposals. The final deterministic vote was `hybrid = 2, P3 = 1`, with GPT preserved as the named minority. All three submitted structured critical risks, demonstrating that critical-risk reporting remains independent of the vote.
|
package/SECURITY.md
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# Security
|
|
2
|
+
|
|
3
|
+
Model Jury is decision support. It never applies a recommendation, edits project files, installs software, commits, pushes, deploys, or sends deliberate network mutations. Provider calls are the only intended network activity. Run artifacts are written with owner-only file permissions beneath the configured Model Jury directory.
|
|
4
|
+
|
|
5
|
+
Prompts sent to every seat prohibit tool use and mutations. GLM and DeepSeek seats call `ctx.llm` without tools. The Codex seat uses the first-party one-shot app-server provider with `permissionMode: never`; the current public provider API cannot force `sandbox: read-only`, so Codex confinement also depends on the user's native Codex sandbox configuration. This limitation is reported by `/jury doctor` and in `PRE_FLIGHT.md`.
|
|
6
|
+
|
|
7
|
+
Credentials stay in DSH credential and provider services. The plugin configuration contains credential references only indirectly through the selected provider route. Persistence recursively redacts fields whose names indicate keys, tokens, secrets, passwords, authorization, cookies, or credentials. It never serializes environment snapshots, headers, OAuth state, or provider request bodies.
|
|
8
|
+
|
|
9
|
+
The plugin stores only final structured model responses, safe call metadata, protocol prompts, aggregation data, and the user-visible report. It does not request or store hidden chain-of-thought.
|
package/cordis.patch.yml
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# Register the first-party Codex provider under a Model Jury-owned name so this
|
|
2
|
+
# bundle does not collide with a user's ordinary `codex` provider instance.
|
|
3
|
+
- insert:
|
|
4
|
+
- id: model-jury-codex-provider
|
|
5
|
+
name: '@deepseek-ai/dsh-subagent-codex'
|
|
6
|
+
config:
|
|
7
|
+
providerName: model-jury-codex
|
|
8
|
+
permissionMode: never
|
|
9
|
+
disposeGraceMs: 3000
|
|
10
|
+
|
|
11
|
+
- id: model-jury
|
|
12
|
+
name: 'dsh-model-jury'
|
|
13
|
+
config:
|
|
14
|
+
storageDir: .dsh-model-jury
|
|
15
|
+
maxQuestionChars: 12000
|
|
16
|
+
maxFieldChars: 6000
|
|
17
|
+
codex:
|
|
18
|
+
provider: model-jury-codex
|
|
19
|
+
timeoutMs: 180000
|
|
20
|
+
maxRetries: 1
|
|
21
|
+
glm:
|
|
22
|
+
provider: glm
|
|
23
|
+
timeoutMs: 120000
|
|
24
|
+
maxTokens: 16384
|
|
25
|
+
maxRetries: 1
|
|
26
|
+
deepseek:
|
|
27
|
+
provider: deepseek-official
|
|
28
|
+
timeoutMs: 120000
|
|
29
|
+
maxTokens: 16384
|
|
30
|
+
maxRetries: 1
|
package/lib/command.d.ts
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { Context } from '@deepseek-ai/cordis';
|
|
2
|
+
import type { CouncilService } from './council-service.js';
|
|
3
|
+
import type { CouncilStyle } from './protocol/prompts.js';
|
|
4
|
+
interface ParsedCouncilInput {
|
|
5
|
+
readonly style: CouncilStyle;
|
|
6
|
+
readonly question: string;
|
|
7
|
+
}
|
|
8
|
+
/** Parse the intentionally small `/jury` option surface. */
|
|
9
|
+
export declare function parseCouncilInput(rawInput: string): ParsedCouncilInput;
|
|
10
|
+
/** Register `/jury` and `/jury doctor` on the documented command seam. */
|
|
11
|
+
export declare function registerCouncilCommand(ctx: Context, service: CouncilService): void;
|
|
12
|
+
export {};
|
|
13
|
+
//# sourceMappingURL=command.d.ts.map
|
package/lib/command.js
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { renderDoctor } from './council-service.js';
|
|
2
|
+
/** Parse the intentionally small `/jury` option surface. */
|
|
3
|
+
export function parseCouncilInput(rawInput) {
|
|
4
|
+
let remaining = rawInput.trim();
|
|
5
|
+
let style = 'balanced';
|
|
6
|
+
if (remaining.startsWith('--style')) {
|
|
7
|
+
const match = /^--style(?:=|\s+)(\S+)(?:\s+|$)/u.exec(remaining);
|
|
8
|
+
if (match === null)
|
|
9
|
+
throw new Error('usage: /jury [--style balanced|adversarial] <question>');
|
|
10
|
+
const selected = match[1];
|
|
11
|
+
if (selected !== 'balanced' && selected !== 'adversarial') {
|
|
12
|
+
throw new Error(`unsupported Council style "${selected}"; use balanced or adversarial`);
|
|
13
|
+
}
|
|
14
|
+
style = selected;
|
|
15
|
+
remaining = remaining.slice(match[0].length).trim();
|
|
16
|
+
}
|
|
17
|
+
if (remaining.length === 0)
|
|
18
|
+
throw new Error('usage: /jury [--style balanced|adversarial] <question>');
|
|
19
|
+
return { style, question: remaining };
|
|
20
|
+
}
|
|
21
|
+
/** Register `/jury` and `/jury doctor` on the documented command seam. */
|
|
22
|
+
export function registerCouncilCommand(ctx, service) {
|
|
23
|
+
ctx.commands.register({
|
|
24
|
+
name: 'jury',
|
|
25
|
+
description: 'Run structured cross-model peer review',
|
|
26
|
+
input: { hint: '[--style balanced|adversarial] <question> | doctor' },
|
|
27
|
+
async handler({ agent, rawInput, signal }) {
|
|
28
|
+
try {
|
|
29
|
+
if (rawInput.trim() === 'doctor') {
|
|
30
|
+
return { kind: 'success', text: renderDoctor(await service.doctor(agent, signal)) };
|
|
31
|
+
}
|
|
32
|
+
const input = parseCouncilInput(rawInput);
|
|
33
|
+
const run = await service.run(input.question, input.style, agent, signal);
|
|
34
|
+
const suffix = `\n\nArtifacts: ${run.directory}`;
|
|
35
|
+
return run.result.verdict.valid
|
|
36
|
+
? { kind: 'success', text: `${run.report}${suffix}` }
|
|
37
|
+
: { kind: 'error', text: `${run.report}${suffix}` };
|
|
38
|
+
}
|
|
39
|
+
catch (error) {
|
|
40
|
+
return { kind: 'error', text: error instanceof Error ? error.message : String(error) };
|
|
41
|
+
}
|
|
42
|
+
},
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
//# sourceMappingURL=command.js.map
|
package/lib/config.d.ts
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import Schema from '@deepseek-ai/schemastery';
|
|
2
|
+
/** Shared retry and timeout policy for one Council seat. */
|
|
3
|
+
export interface SeatConnectionConfig {
|
|
4
|
+
/** DSH provider route or subagent provider name. */
|
|
5
|
+
provider: string;
|
|
6
|
+
/** Maximum wall-clock duration for one structured invocation. */
|
|
7
|
+
timeoutMs: number;
|
|
8
|
+
/** Maximum transient transport retries after the first attempt. */
|
|
9
|
+
maxRetries: number;
|
|
10
|
+
}
|
|
11
|
+
/** Provider/model policy for a `ctx.llm` seat. */
|
|
12
|
+
export interface LlmSeatConnectionConfig extends SeatConnectionConfig {
|
|
13
|
+
/** Model id passed to the configured DSH route; omission leaves the seat unavailable. */
|
|
14
|
+
model: string | null;
|
|
15
|
+
/** Maximum response tokens for each structured model call. */
|
|
16
|
+
maxTokens: number;
|
|
17
|
+
}
|
|
18
|
+
/** Public Council plugin configuration. */
|
|
19
|
+
export interface Config {
|
|
20
|
+
/** Artifact directory, resolved from the receiving session workspace. */
|
|
21
|
+
storageDir: string;
|
|
22
|
+
/** Maximum accepted user-question length. */
|
|
23
|
+
maxQuestionChars: number;
|
|
24
|
+
/** Maximum length of one string before it is redistributed to another seat. */
|
|
25
|
+
maxFieldChars: number;
|
|
26
|
+
/** Native Codex subagent route used for the GPT seat. */
|
|
27
|
+
codex: SeatConnectionConfig;
|
|
28
|
+
/** Configurable DSH LLM route used for the GLM seat. */
|
|
29
|
+
glm: LlmSeatConnectionConfig;
|
|
30
|
+
/** First-party or configured DSH LLM route used for the DeepSeek seat. */
|
|
31
|
+
deepseek: LlmSeatConnectionConfig;
|
|
32
|
+
}
|
|
33
|
+
/** Schemastery validation and defaults for the public plugin configuration. */
|
|
34
|
+
export declare const Config: Schema<Config>;
|
|
35
|
+
//# sourceMappingURL=config.d.ts.map
|
package/lib/config.js
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import Schema from '@deepseek-ai/schemastery';
|
|
2
|
+
const seatConnection = Schema.object({
|
|
3
|
+
provider: Schema.string().min(1).required(),
|
|
4
|
+
timeoutMs: Schema.number().min(1).required(),
|
|
5
|
+
maxRetries: Schema.number().min(0).max(1).required(),
|
|
6
|
+
});
|
|
7
|
+
const llmSeatConnection = Schema.object({
|
|
8
|
+
provider: Schema.string().min(1).required(),
|
|
9
|
+
model: Schema.union([Schema.string().min(1), Schema.const(null)]).default(null),
|
|
10
|
+
timeoutMs: Schema.number().min(1).required(),
|
|
11
|
+
maxRetries: Schema.number().min(0).max(1).required(),
|
|
12
|
+
maxTokens: Schema.number().min(1).required(),
|
|
13
|
+
});
|
|
14
|
+
/** Schemastery validation and defaults for the public plugin configuration. */
|
|
15
|
+
export const Config = Schema.object({
|
|
16
|
+
storageDir: Schema.string().min(1).default('.dsh-model-jury'),
|
|
17
|
+
maxQuestionChars: Schema.number().min(1).default(12_000),
|
|
18
|
+
maxFieldChars: Schema.number().min(256).default(6_000),
|
|
19
|
+
codex: seatConnection.default({
|
|
20
|
+
provider: 'model-jury-codex',
|
|
21
|
+
timeoutMs: 180_000,
|
|
22
|
+
maxRetries: 1,
|
|
23
|
+
}),
|
|
24
|
+
glm: llmSeatConnection.default({
|
|
25
|
+
provider: 'glm',
|
|
26
|
+
model: null,
|
|
27
|
+
timeoutMs: 120_000,
|
|
28
|
+
maxTokens: 16_384,
|
|
29
|
+
maxRetries: 1,
|
|
30
|
+
}),
|
|
31
|
+
deepseek: llmSeatConnection.default({
|
|
32
|
+
provider: 'deepseek-official',
|
|
33
|
+
model: null,
|
|
34
|
+
timeoutMs: 120_000,
|
|
35
|
+
maxTokens: 16_384,
|
|
36
|
+
maxRetries: 1,
|
|
37
|
+
}),
|
|
38
|
+
});
|
|
39
|
+
//# sourceMappingURL=config.js.map
|