@draht/coding-agent 2026.10.20 → 2026.10.23-1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +138 -0
- package/agents/advisor.md +2 -2
- package/agents/architect.md +11 -33
- package/agents/debugger.md +13 -54
- package/agents/git-committer.md +18 -42
- package/agents/implementer.md +14 -36
- package/agents/investigator.md +12 -63
- package/agents/reviewer.md +16 -44
- package/agents/security-auditor.md +22 -56
- package/agents/spec-reviewer.md +13 -53
- package/agents/verifier.md +14 -33
- package/bin/drahtbank.mjs +4 -0
- package/dist/agents/advisor.md +2 -2
- package/dist/agents/architect.md +11 -33
- package/dist/agents/debugger.md +13 -54
- package/dist/agents/git-committer.md +18 -42
- package/dist/agents/implementer.md +14 -36
- package/dist/agents/investigator.md +12 -63
- package/dist/agents/reviewer.md +16 -44
- package/dist/agents/security-auditor.md +22 -56
- package/dist/agents/spec-reviewer.md +13 -53
- package/dist/agents/verifier.md +14 -33
- package/dist/bundle/chunks/anthropic-messages-G57VPNTF.js +6 -0
- package/dist/bundle/chunks/anthropic.js +1 -1
- package/dist/bundle/chunks/azure-openai-responses-AYUCZ5WN.js +2 -0
- package/dist/bundle/chunks/bedrock-converse-stream.js +31 -28
- package/dist/bundle/chunks/chunk-2JZLKNZ4.js +3 -0
- package/dist/bundle/chunks/chunk-3GDEPGTR.js +16 -0
- package/dist/bundle/chunks/chunk-5JW3NSTV.js +2 -0
- package/dist/bundle/chunks/{chunk-KVH4HR57.js → chunk-6WESOD4H.js} +1 -1
- package/dist/bundle/chunks/chunk-76YNMQ57.js +2 -0
- package/dist/bundle/chunks/chunk-AC3MSTC6.js +2 -0
- package/dist/bundle/chunks/chunk-ACLWH5WQ.js +2 -0
- package/dist/bundle/chunks/chunk-BL72B6LV.js +82 -0
- package/dist/bundle/chunks/chunk-CLKG6K6Z.js +7 -0
- package/dist/bundle/chunks/chunk-DN7SM3LX.js +479 -0
- package/dist/bundle/chunks/{chunk-D6TMD5Q2.js → chunk-DOUMZTXH.js} +10 -10
- package/dist/bundle/chunks/chunk-DQ4G4IS4.js +20 -0
- package/dist/bundle/chunks/chunk-E4VZJRJ2.js +9 -0
- package/dist/bundle/chunks/chunk-FOYIPAAD.js +6 -0
- package/dist/bundle/chunks/chunk-FXCTQRNN.js +2 -0
- package/dist/bundle/chunks/chunk-GBFVS2SF.js +12 -0
- package/dist/bundle/chunks/chunk-GQXRTK54.js +20 -0
- package/dist/bundle/chunks/chunk-IGO52ZOY.js +132 -0
- package/dist/bundle/chunks/chunk-IT4RIOZD.js +2 -0
- package/dist/bundle/chunks/chunk-JJKPUMH2.js +1424 -0
- package/dist/bundle/chunks/chunk-JNWRE5F4.js +2 -0
- package/dist/bundle/chunks/chunk-KSGHCBHT.js +2 -0
- package/dist/bundle/chunks/chunk-LO6CVU2L.js +6 -0
- package/dist/bundle/chunks/chunk-MBFTIGLS.js +2 -0
- package/dist/bundle/chunks/chunk-N26RGV3K.js +11 -0
- package/dist/bundle/chunks/{chunk-KPO5375S.js → chunk-RCRGGYLW.js} +1 -1
- package/dist/bundle/chunks/chunk-UCJ7FLSO.js +166 -0
- package/dist/bundle/chunks/chunk-USCN7R2E.js +2 -0
- package/dist/bundle/chunks/chunk-WFHH5SX6.js +4 -0
- package/dist/bundle/chunks/chunk-WYWBXNDO.js +6 -0
- package/dist/bundle/chunks/chunk-ZRPAOMVB.js +2 -0
- package/dist/bundle/chunks/{cli-PYJB6AQ3.js → cli-HTUB6WOG.js} +6 -6
- package/dist/bundle/chunks/cloudflare-workers-ai-system-one-RRSO6OXX.js +2 -0
- package/dist/bundle/chunks/codemode-worker.js +188 -21
- package/dist/bundle/chunks/execute-UUPNZJJG.js +31 -0
- package/dist/bundle/chunks/github-copilot.js +1 -1
- package/dist/bundle/chunks/google-generative-ai-VJSS5EBX.js +2 -0
- package/dist/bundle/chunks/google-vertex-K23DBWTT.js +2 -0
- package/dist/bundle/chunks/https-proxy-agent-WQIRL24U.js +13 -0
- package/dist/bundle/chunks/image-resize-worker.js +1 -1
- package/dist/bundle/chunks/lib-ULKZYRGD.js +9 -0
- package/dist/bundle/chunks/{llama-cpp-classify-EJ5XMSQL.js → llama-cpp-classify-3VDXTYVJ.js} +3 -3
- package/dist/bundle/chunks/mistral-conversations-SKVQJGFZ.js +5 -0
- package/dist/bundle/chunks/{multipart-parser-AZP5BX2W.js → multipart-parser-HMCXZ4BD.js} +1 -1
- package/dist/bundle/chunks/node-DXZFQBEO.js +16 -0
- package/dist/bundle/chunks/{node-C2JGH2OB.js → node-LBZLUIBE.js} +1 -1
- package/dist/bundle/chunks/openai-chatgpt.js +1 -1
- package/dist/bundle/chunks/openai-codex-responses-LV4RIZL3.js +10 -0
- package/dist/bundle/chunks/openai-codex.js +1 -1
- package/dist/bundle/chunks/openai-completions-PUIR2V25.js +7 -0
- package/dist/bundle/chunks/openai-decisions-2DJLSYQF.js +5 -0
- package/dist/bundle/chunks/openai-responses-6ESQOXAF.js +4 -0
- package/dist/bundle/chunks/opencode-go.js +1 -1
- package/dist/bundle/chunks/openrouter-images-TVXBMY3V.js +2 -0
- package/dist/bundle/chunks/pi-messages-XS7FDWQI.js +8 -0
- package/dist/bundle/chunks/runtime-IB3XQ7E3.js +2 -0
- package/dist/bundle/chunks/runtime-LB35PK4J.js +3 -0
- package/dist/bundle/chunks/settings-manager-YNY5F4RU.js +2 -0
- package/dist/bundle/chunks/src-KDGO7EGF.js +4 -0
- package/dist/bundle/chunks/typesafe-system-one-67GAWZXE.js +2 -0
- package/dist/bundle/chunks/virtual-modules-U4CBGVMI.js +2 -0
- package/dist/bundle/cli-runtime.js +1 -1
- package/dist/bundle/index.js +1 -1
- package/dist/bundle/rpc-entry.js +1 -1
- package/dist/cli/args.d.ts +2 -1
- package/dist/cli/args.d.ts.map +1 -1
- package/dist/cli/args.js +30 -8
- package/dist/cli/args.js.map +1 -1
- package/dist/cli/credential-print.d.ts.map +1 -1
- package/dist/cli/credential-print.js +4 -4
- package/dist/cli/credential-print.js.map +1 -1
- package/dist/config.d.ts +18 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +44 -6
- package/dist/config.js.map +1 -1
- package/dist/core/agent-session.d.ts +65 -8
- package/dist/core/agent-session.d.ts.map +1 -1
- package/dist/core/agent-session.js +240 -102
- package/dist/core/agent-session.js.map +1 -1
- package/dist/core/auth-storage.d.ts.map +1 -1
- package/dist/core/auth-storage.js +20 -13
- package/dist/core/auth-storage.js.map +1 -1
- package/dist/core/bash-executor.d.ts.map +1 -1
- package/dist/core/bash-executor.js +42 -52
- package/dist/core/bash-executor.js.map +1 -1
- package/dist/core/bug-report.d.ts +9 -1
- package/dist/core/bug-report.d.ts.map +1 -1
- package/dist/core/builtins/completion-checks.d.ts +3 -0
- package/dist/core/builtins/completion-checks.d.ts.map +1 -0
- package/dist/core/builtins/completion-checks.js +142 -0
- package/dist/core/builtins/completion-checks.js.map +1 -0
- package/dist/core/builtins/duet.d.ts.map +1 -1
- package/dist/core/builtins/duet.js +7 -5
- package/dist/core/builtins/duet.js.map +1 -1
- package/dist/core/builtins/git-safety.d.ts +3 -0
- package/dist/core/builtins/git-safety.d.ts.map +1 -0
- package/dist/core/builtins/git-safety.js +34 -0
- package/dist/core/builtins/git-safety.js.map +1 -0
- package/dist/core/builtins/index.d.ts.map +1 -1
- package/dist/core/builtins/index.js +4 -0
- package/dist/core/builtins/index.js.map +1 -1
- package/dist/core/client-identity.d.ts +8 -0
- package/dist/core/client-identity.d.ts.map +1 -0
- package/dist/core/client-identity.js +15 -0
- package/dist/core/client-identity.js.map +1 -0
- package/dist/core/compaction/branch-summarization.d.ts.map +1 -1
- package/dist/core/compaction/branch-summarization.js +1 -2
- package/dist/core/compaction/branch-summarization.js.map +1 -1
- package/dist/core/compaction/compaction.d.ts +2 -2
- package/dist/core/compaction/compaction.d.ts.map +1 -1
- package/dist/core/compaction/compaction.js +20 -15
- package/dist/core/compaction/compaction.js.map +1 -1
- package/dist/core/compaction/utils.d.ts +6 -0
- package/dist/core/compaction/utils.d.ts.map +1 -1
- package/dist/core/compaction/utils.js +7 -8
- package/dist/core/compaction/utils.js.map +1 -1
- package/dist/core/export-html/index.d.ts.map +1 -1
- package/dist/core/export-html/index.js +2 -0
- package/dist/core/export-html/index.js.map +1 -1
- package/dist/core/export-html/template.html +1 -0
- package/dist/core/export-html/template.js +23 -17
- package/dist/core/export-html/tool-renderer.d.ts +3 -3
- package/dist/core/export-html/tool-renderer.d.ts.map +1 -1
- package/dist/core/export-html/tool-renderer.js +5 -3
- package/dist/core/export-html/tool-renderer.js.map +1 -1
- package/dist/core/export-html/vendor/marked-cjk-friendly.min.js +6 -0
- package/dist/core/export-html/vendor/marked.min.js +30 -29
- package/dist/core/extensions/index.d.ts +1 -1
- package/dist/core/extensions/index.d.ts.map +1 -1
- package/dist/core/extensions/index.js.map +1 -1
- package/dist/core/extensions/loader.d.ts.map +1 -1
- package/dist/core/extensions/loader.js +7 -0
- package/dist/core/extensions/loader.js.map +1 -1
- package/dist/core/extensions/runner.d.ts +6 -2
- package/dist/core/extensions/runner.d.ts.map +1 -1
- package/dist/core/extensions/runner.js +16 -4
- package/dist/core/extensions/runner.js.map +1 -1
- package/dist/core/extensions/types.d.ts +61 -7
- package/dist/core/extensions/types.d.ts.map +1 -1
- package/dist/core/extensions/types.js.map +1 -1
- package/dist/core/extensions/virtual-modules.d.ts.map +1 -1
- package/dist/core/extensions/virtual-modules.js +2 -0
- package/dist/core/extensions/virtual-modules.js.map +1 -1
- package/dist/core/key-id-schema.d.ts +6 -0
- package/dist/core/key-id-schema.d.ts.map +1 -0
- package/dist/core/key-id-schema.js +16 -0
- package/dist/core/key-id-schema.js.map +1 -0
- package/dist/core/keybindings-schema.d.ts +4 -0
- package/dist/core/keybindings-schema.d.ts.map +1 -0
- package/dist/core/keybindings-schema.js +16 -0
- package/dist/core/keybindings-schema.js.map +1 -0
- package/dist/core/keybindings.d.ts +242 -75
- package/dist/core/keybindings.d.ts.map +1 -1
- package/dist/core/keybindings.js +30 -23
- package/dist/core/keybindings.js.map +1 -1
- package/dist/core/mcp-servers.d.ts +22 -0
- package/dist/core/mcp-servers.d.ts.map +1 -1
- package/dist/core/mcp-servers.js +72 -11
- package/dist/core/mcp-servers.js.map +1 -1
- package/dist/core/model-config.d.ts +1623 -74
- package/dist/core/model-config.d.ts.map +1 -1
- package/dist/core/model-config.js +70 -184
- package/dist/core/model-config.js.map +1 -1
- package/dist/core/model-registry.d.ts +3 -1
- package/dist/core/model-registry.d.ts.map +1 -1
- package/dist/core/model-registry.js +4 -0
- package/dist/core/model-registry.js.map +1 -1
- package/dist/core/model-resolver.d.ts +56 -43
- package/dist/core/model-resolver.d.ts.map +1 -1
- package/dist/core/model-resolver.js +253 -358
- package/dist/core/model-resolver.js.map +1 -1
- package/dist/core/model-runtime.d.ts.map +1 -1
- package/dist/core/model-runtime.js +19 -10
- package/dist/core/model-runtime.js.map +1 -1
- package/dist/core/nested-tool-calls.d.ts +1 -0
- package/dist/core/nested-tool-calls.d.ts.map +1 -1
- package/dist/core/nested-tool-calls.js +1 -0
- package/dist/core/nested-tool-calls.js.map +1 -1
- package/dist/core/package-manager.js +3 -3
- package/dist/core/package-manager.js.map +1 -1
- package/dist/core/polyphase/keyword.js +1 -1
- package/dist/core/polyphase/keyword.js.map +1 -1
- package/dist/core/polyphase/model-selection.d.ts.map +1 -1
- package/dist/core/polyphase/model-selection.js +15 -7
- package/dist/core/polyphase/model-selection.js.map +1 -1
- package/dist/core/polyphase/workflow/prelude.d.ts +2 -0
- package/dist/core/polyphase/workflow/prelude.d.ts.map +1 -1
- package/dist/core/polyphase/workflow/prelude.js +12 -16
- package/dist/core/polyphase/workflow/prelude.js.map +1 -1
- package/dist/core/polyphase/workflow/runtime.d.ts.map +1 -1
- package/dist/core/polyphase/workflow/runtime.js +2 -1
- package/dist/core/polyphase/workflow/runtime.js.map +1 -1
- package/dist/core/project-trust.d.ts.map +1 -1
- package/dist/core/project-trust.js +2 -1
- package/dist/core/project-trust.js.map +1 -1
- package/dist/core/prompt-templates.d.ts.map +1 -1
- package/dist/core/prompt-templates.js +7 -2
- package/dist/core/prompt-templates.js.map +1 -1
- package/dist/core/provider-composer.d.ts +2 -1
- package/dist/core/provider-composer.d.ts.map +1 -1
- package/dist/core/provider-composer.js +13 -0
- package/dist/core/provider-composer.js.map +1 -1
- package/dist/core/radius.d.ts +2 -0
- package/dist/core/radius.d.ts.map +1 -1
- package/dist/core/radius.js +2 -0
- package/dist/core/radius.js.map +1 -1
- package/dist/core/remote-catalog-provider.d.ts +12 -1
- package/dist/core/remote-catalog-provider.d.ts.map +1 -1
- package/dist/core/remote-catalog-provider.js +84 -24
- package/dist/core/remote-catalog-provider.js.map +1 -1
- package/dist/core/resource-loader.d.ts +3 -0
- package/dist/core/resource-loader.d.ts.map +1 -1
- package/dist/core/resource-loader.js +7 -4
- package/dist/core/resource-loader.js.map +1 -1
- package/dist/core/sdk.d.ts +13 -4
- package/dist/core/sdk.d.ts.map +1 -1
- package/dist/core/sdk.js +27 -15
- package/dist/core/sdk.js.map +1 -1
- package/dist/core/settings-defaults.d.ts +65 -0
- package/dist/core/settings-defaults.d.ts.map +1 -0
- package/dist/core/settings-defaults.js +66 -0
- package/dist/core/settings-defaults.js.map +1 -0
- package/dist/core/settings-manager.d.ts +17 -178
- package/dist/core/settings-manager.d.ts.map +1 -1
- package/dist/core/settings-manager.js +86 -57
- package/dist/core/settings-manager.js.map +1 -1
- package/dist/core/settings-schema.d.ts +269 -0
- package/dist/core/settings-schema.d.ts.map +1 -0
- package/dist/core/settings-schema.js +318 -0
- package/dist/core/settings-schema.js.map +1 -0
- package/dist/core/skills.d.ts +4 -1
- package/dist/core/skills.d.ts.map +1 -1
- package/dist/core/skills.js +6 -1
- package/dist/core/skills.js.map +1 -1
- package/dist/core/system-prompt.d.ts +6 -0
- package/dist/core/system-prompt.d.ts.map +1 -1
- package/dist/core/system-prompt.js +10 -5
- package/dist/core/system-prompt.js.map +1 -1
- package/dist/core/tools/bash.d.ts.map +1 -1
- package/dist/core/tools/bash.js +3 -5
- package/dist/core/tools/bash.js.map +1 -1
- package/dist/core/tools/grep.d.ts +8 -0
- package/dist/core/tools/grep.d.ts.map +1 -1
- package/dist/core/tools/grep.js +19 -7
- package/dist/core/tools/grep.js.map +1 -1
- package/dist/core/tools/index.d.ts +2 -0
- package/dist/core/tools/index.d.ts.map +1 -1
- package/dist/core/tools/index.js +2 -0
- package/dist/core/tools/index.js.map +1 -1
- package/dist/core/tools/output-accumulator.d.ts.map +1 -1
- package/dist/core/tools/output-accumulator.js +4 -10
- package/dist/core/tools/output-accumulator.js.map +1 -1
- package/dist/core/tools/path-utils.d.ts +1 -1
- package/dist/core/tools/path-utils.d.ts.map +1 -1
- package/dist/core/tools/path-utils.js +6 -6
- package/dist/core/tools/path-utils.js.map +1 -1
- package/dist/core/tools/read.d.ts +13 -0
- package/dist/core/tools/read.d.ts.map +1 -1
- package/dist/core/tools/read.js +32 -2
- package/dist/core/tools/read.js.map +1 -1
- package/dist/core/tools/render-utils.d.ts +20 -0
- package/dist/core/tools/render-utils.d.ts.map +1 -1
- package/dist/core/tools/render-utils.js +67 -0
- package/dist/core/tools/render-utils.js.map +1 -1
- package/dist/core/tools/renderers/bash.d.ts.map +1 -1
- package/dist/core/tools/renderers/bash.js +7 -35
- package/dist/core/tools/renderers/bash.js.map +1 -1
- package/dist/core/tools/renderers/edit.d.ts.map +1 -1
- package/dist/core/tools/renderers/edit.js +5 -4
- package/dist/core/tools/renderers/edit.js.map +1 -1
- package/dist/core/tools/renderers/index.d.ts +2 -2
- package/dist/core/tools/renderers/index.d.ts.map +1 -1
- package/dist/core/tools/renderers/index.js.map +1 -1
- package/dist/core/tools/renderers/read.d.ts.map +1 -1
- package/dist/core/tools/renderers/read.js +7 -11
- package/dist/core/tools/renderers/read.js.map +1 -1
- package/dist/extensions/codemode/execute.d.ts.map +1 -1
- package/dist/extensions/codemode/execute.js +391 -80
- package/dist/extensions/codemode/execute.js.map +1 -1
- package/dist/extensions/codemode/index.d.ts.map +1 -1
- package/dist/extensions/codemode/index.js +3 -1
- package/dist/extensions/codemode/index.js.map +1 -1
- package/dist/extensions/codemode/renderer.d.ts +2 -2
- package/dist/extensions/codemode/renderer.d.ts.map +1 -1
- package/dist/extensions/codemode/renderer.js +9 -3
- package/dist/extensions/codemode/renderer.js.map +1 -1
- package/dist/extensions/codemode/tool.d.ts +22 -21
- package/dist/extensions/codemode/tool.d.ts.map +1 -1
- package/dist/extensions/codemode/tool.js +96 -123
- package/dist/extensions/codemode/tool.js.map +1 -1
- package/dist/extensions/llama/provider.d.ts.map +1 -1
- package/dist/extensions/llama/provider.js +34 -11
- package/dist/extensions/llama/provider.js.map +1 -1
- package/dist/extensions/mcp/cli.d.ts.map +1 -1
- package/dist/extensions/mcp/cli.js +27 -33
- package/dist/extensions/mcp/cli.js.map +1 -1
- package/dist/extensions/mcp/config.d.ts +15 -3
- package/dist/extensions/mcp/config.d.ts.map +1 -1
- package/dist/extensions/mcp/config.js +49 -12
- package/dist/extensions/mcp/config.js.map +1 -1
- package/dist/extensions/mcp/index.d.ts +4 -1
- package/dist/extensions/mcp/index.d.ts.map +1 -1
- package/dist/extensions/mcp/index.js +279 -129
- package/dist/extensions/mcp/index.js.map +1 -1
- package/dist/extensions/mcp/oauth.d.ts +21 -11
- package/dist/extensions/mcp/oauth.d.ts.map +1 -1
- package/dist/extensions/mcp/oauth.js +166 -45
- package/dist/extensions/mcp/oauth.js.map +1 -1
- package/dist/extensions/mcp/resources.d.ts +3 -10
- package/dist/extensions/mcp/resources.d.ts.map +1 -1
- package/dist/extensions/mcp/resources.js +4 -5
- package/dist/extensions/mcp/resources.js.map +1 -1
- package/dist/extensions/mcp/runtime.d.ts +7 -7
- package/dist/extensions/mcp/runtime.d.ts.map +1 -1
- package/dist/extensions/mcp/runtime.js +31 -45
- package/dist/extensions/mcp/runtime.js.map +1 -1
- package/dist/extensions/mcp/tools.d.ts +7 -5
- package/dist/extensions/mcp/tools.d.ts.map +1 -1
- package/dist/extensions/mcp/tools.js +87 -29
- package/dist/extensions/mcp/tools.js.map +1 -1
- package/dist/extensions/mcp/ui.d.ts +3 -3
- package/dist/extensions/mcp/ui.d.ts.map +1 -1
- package/dist/extensions/mcp/ui.js +18 -6
- package/dist/extensions/mcp/ui.js.map +1 -1
- package/dist/extensions/tool-search/tool.d.ts +0 -2
- package/dist/extensions/tool-search/tool.d.ts.map +1 -1
- package/dist/extensions/tool-search/tool.js +2 -4
- package/dist/extensions/tool-search/tool.js.map +1 -1
- package/dist/hooks/checks/checks-runtime.cjs +215 -0
- package/dist/hooks/checks/git-safety.cjs +198 -0
- package/dist/hooks/checks/git-safety.d.cts +1 -0
- package/dist/hooks/gsd/draht-post-task.js +78 -39
- package/dist/hooks/gsd/draht-quality-gate.js +210 -122
- package/dist/hooks/gsd/package.json +3 -0
- package/dist/index.d.ts +5 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -2
- package/dist/index.js.map +1 -1
- package/dist/main.d.ts.map +1 -1
- package/dist/main.js +11 -3
- package/dist/main.js.map +1 -1
- package/dist/modes/interactive/components/auth-url.d.ts +14 -0
- package/dist/modes/interactive/components/auth-url.d.ts.map +1 -0
- package/dist/modes/interactive/components/auth-url.js +37 -0
- package/dist/modes/interactive/components/auth-url.js.map +1 -0
- package/dist/modes/interactive/components/bash-execution.d.ts +5 -1
- package/dist/modes/interactive/components/bash-execution.d.ts.map +1 -1
- package/dist/modes/interactive/components/bash-execution.js +17 -14
- package/dist/modes/interactive/components/bash-execution.js.map +1 -1
- package/dist/modes/interactive/components/branch-summary-message.d.ts +2 -1
- package/dist/modes/interactive/components/branch-summary-message.d.ts.map +1 -1
- package/dist/modes/interactive/components/branch-summary-message.js +5 -2
- package/dist/modes/interactive/components/branch-summary-message.js.map +1 -1
- package/dist/modes/interactive/components/compaction-summary-message.d.ts +2 -1
- package/dist/modes/interactive/components/compaction-summary-message.d.ts.map +1 -1
- package/dist/modes/interactive/components/compaction-summary-message.js +5 -2
- package/dist/modes/interactive/components/compaction-summary-message.js.map +1 -1
- package/dist/modes/interactive/components/custom-entry.d.ts +3 -1
- package/dist/modes/interactive/components/custom-entry.d.ts.map +1 -1
- package/dist/modes/interactive/components/custom-entry.js +8 -2
- package/dist/modes/interactive/components/custom-entry.js.map +1 -1
- package/dist/modes/interactive/components/custom-message.d.ts.map +1 -1
- package/dist/modes/interactive/components/custom-message.js +1 -0
- package/dist/modes/interactive/components/custom-message.js.map +1 -1
- package/dist/modes/interactive/components/login-dialog.d.ts +3 -0
- package/dist/modes/interactive/components/login-dialog.d.ts.map +1 -1
- package/dist/modes/interactive/components/login-dialog.js +38 -10
- package/dist/modes/interactive/components/login-dialog.js.map +1 -1
- package/dist/modes/interactive/components/oauth-selector.d.ts +8 -2
- package/dist/modes/interactive/components/oauth-selector.d.ts.map +1 -1
- package/dist/modes/interactive/components/oauth-selector.js +24 -21
- package/dist/modes/interactive/components/oauth-selector.js.map +1 -1
- package/dist/modes/interactive/components/radius-login-selector.d.ts +15 -0
- package/dist/modes/interactive/components/radius-login-selector.d.ts.map +1 -0
- package/dist/modes/interactive/components/radius-login-selector.js +85 -0
- package/dist/modes/interactive/components/radius-login-selector.js.map +1 -0
- package/dist/modes/interactive/components/settings-selector.d.ts +3 -3
- package/dist/modes/interactive/components/settings-selector.d.ts.map +1 -1
- package/dist/modes/interactive/components/settings-selector.js +9 -8
- package/dist/modes/interactive/components/settings-selector.js.map +1 -1
- package/dist/modes/interactive/components/skill-invocation-message.d.ts +2 -1
- package/dist/modes/interactive/components/skill-invocation-message.d.ts.map +1 -1
- package/dist/modes/interactive/components/skill-invocation-message.js +5 -2
- package/dist/modes/interactive/components/skill-invocation-message.js.map +1 -1
- package/dist/modes/interactive/components/status-indicator.d.ts +1 -1
- package/dist/modes/interactive/components/status-indicator.d.ts.map +1 -1
- package/dist/modes/interactive/components/status-indicator.js +2 -2
- package/dist/modes/interactive/components/status-indicator.js.map +1 -1
- package/dist/modes/interactive/components/tool-execution.d.ts +12 -13
- package/dist/modes/interactive/components/tool-execution.d.ts.map +1 -1
- package/dist/modes/interactive/components/tool-execution.js +32 -45
- package/dist/modes/interactive/components/tool-execution.js.map +1 -1
- package/dist/modes/interactive/components/tree-selector.d.ts.map +1 -1
- package/dist/modes/interactive/components/tree-selector.js +2 -9
- package/dist/modes/interactive/components/tree-selector.js.map +1 -1
- package/dist/modes/interactive/components/user-message.d.ts.map +1 -1
- package/dist/modes/interactive/components/user-message.js +5 -4
- package/dist/modes/interactive/components/user-message.js.map +1 -1
- package/dist/modes/interactive/interactive-mode.d.ts +43 -48
- package/dist/modes/interactive/interactive-mode.d.ts.map +1 -1
- package/dist/modes/interactive/interactive-mode.js +362 -278
- package/dist/modes/interactive/interactive-mode.js.map +1 -1
- package/dist/modes/interactive/program-status-reporter.d.ts +34 -0
- package/dist/modes/interactive/program-status-reporter.d.ts.map +1 -0
- package/dist/modes/interactive/program-status-reporter.js +110 -0
- package/dist/modes/interactive/program-status-reporter.js.map +1 -0
- package/dist/modes/interactive/theme/dark.json +1 -1
- package/dist/modes/interactive/theme/light.json +1 -1
- package/dist/modes/interactive/theme/system-theme.d.ts +2 -1
- package/dist/modes/interactive/theme/system-theme.d.ts.map +1 -1
- package/dist/modes/interactive/theme/system-theme.js +16 -5
- package/dist/modes/interactive/theme/system-theme.js.map +1 -1
- package/dist/modes/interactive/theme/theme-schema.d.ts +28 -0
- package/dist/modes/interactive/theme/theme-schema.d.ts.map +1 -0
- package/dist/modes/interactive/theme/theme-schema.js +105 -0
- package/dist/modes/interactive/theme/theme-schema.js.map +1 -0
- package/dist/modes/interactive/theme/theme-tokens.d.ts +257 -0
- package/dist/modes/interactive/theme/theme-tokens.d.ts.map +1 -0
- package/dist/modes/interactive/theme/theme-tokens.js +94 -0
- package/dist/modes/interactive/theme/theme-tokens.js.map +1 -0
- package/dist/modes/interactive/theme/theme.d.ts +13 -16
- package/dist/modes/interactive/theme/theme.d.ts.map +1 -1
- package/dist/modes/interactive/theme/theme.js +22 -29
- package/dist/modes/interactive/theme/theme.js.map +1 -1
- package/dist/modes/print-mode.d.ts.map +1 -1
- package/dist/modes/print-mode.js +4 -0
- package/dist/modes/print-mode.js.map +1 -1
- package/dist/package-manager-cli.d.ts.map +1 -1
- package/dist/package-manager-cli.js +26 -1
- package/dist/package-manager-cli.js.map +1 -1
- package/dist/prompts/commands/atomic-commit.md +8 -6
- package/dist/prompts/commands/brainstorm.md +5 -149
- package/dist/prompts/commands/create-verification-skill.md +8 -48
- package/dist/prompts/commands/discuss-phase.md +6 -10
- package/dist/prompts/commands/execute-phase.md +18 -45
- package/dist/prompts/commands/fix.md +16 -128
- package/dist/prompts/commands/grill.md +6 -53
- package/dist/prompts/commands/init-project.md +13 -18
- package/dist/prompts/commands/map-codebase.md +3 -3
- package/dist/prompts/commands/new-project.md +12 -17
- package/dist/prompts/commands/next-milestone.md +4 -8
- package/dist/prompts/commands/orchestrate-loop.md +12 -12
- package/dist/prompts/commands/orchestrate.md +12 -11
- package/dist/prompts/commands/pause-work.md +1 -1
- package/dist/prompts/commands/plan-phase.md +15 -37
- package/dist/prompts/commands/quick.md +11 -64
- package/dist/prompts/commands/resolve-conflicts.md +2 -15
- package/dist/prompts/commands/triage.md +3 -14
- package/dist/prompts/commands/verify-work.md +15 -63
- package/dist/prompts/commands/why.md +18 -118
- package/dist/skills/atomic-reasoning/SKILL.md +6 -105
- package/dist/skills/blast-radius/SKILL.md +1 -1
- package/dist/skills/epistemics/SKILL.md +12 -87
- package/dist/skills/explainers/SKILL.md +1 -1
- package/dist/skills/typescript-discipline/SKILL.md +12 -42
- package/dist/skills/typescript-discipline/references/patterns.md +25 -266
- package/dist/skills/unslop/SKILL.md +14 -105
- package/dist/utils/ansi.d.ts +8 -0
- package/dist/utils/ansi.d.ts.map +1 -1
- package/dist/utils/ansi.js +30 -11
- package/dist/utils/ansi.js.map +1 -1
- package/dist/utils/clipboard-command.d.ts.map +1 -1
- package/dist/utils/clipboard-command.js +3 -0
- package/dist/utils/clipboard-command.js.map +1 -1
- package/dist/utils/clipboard.d.ts.map +1 -1
- package/dist/utils/clipboard.js +18 -12
- package/dist/utils/clipboard.js.map +1 -1
- package/dist/utils/draht-user-agent.d.ts +1 -0
- package/dist/utils/draht-user-agent.d.ts.map +1 -1
- package/dist/utils/draht-user-agent.js +3 -2
- package/dist/utils/draht-user-agent.js.map +1 -1
- package/dist/utils/image-convert.d.ts +12 -0
- package/dist/utils/image-convert.d.ts.map +1 -1
- package/dist/utils/image-convert.js +45 -6
- package/dist/utils/image-convert.js.map +1 -1
- package/dist/utils/image-resize-core.d.ts +12 -0
- package/dist/utils/image-resize-core.d.ts.map +1 -1
- package/dist/utils/image-resize-core.js +5 -0
- package/dist/utils/image-resize-core.js.map +1 -1
- package/dist/utils/image-resize-worker.js +3 -2
- package/dist/utils/image-resize-worker.js.map +1 -1
- package/dist/utils/image-resize.d.ts.map +1 -1
- package/dist/utils/image-resize.js +9 -10
- package/dist/utils/image-resize.js.map +1 -1
- package/dist/utils/output-files.d.ts +15 -0
- package/dist/utils/output-files.d.ts.map +1 -0
- package/dist/utils/output-files.js +30 -0
- package/dist/utils/output-files.js.map +1 -0
- package/dist/utils/shell.d.ts.map +1 -1
- package/dist/utils/shell.js +43 -13
- package/dist/utils/shell.js.map +1 -1
- package/dist/utils/syntax-highlight.d.ts.map +1 -1
- package/dist/utils/syntax-highlight.js +6 -1
- package/dist/utils/syntax-highlight.js.map +1 -1
- package/dist/utils/tools-manager.d.ts.map +1 -1
- package/dist/utils/tools-manager.js +4 -3
- package/dist/utils/tools-manager.js.map +1 -1
- package/docs/cli-integration.md +1 -1
- package/docs/cli.md +30 -26
- package/docs/codemode.md +230 -0
- package/docs/compaction.md +5 -5
- package/docs/containerization.md +1 -1
- package/docs/custom-provider.md +53 -1
- package/docs/docs.json +17 -1
- package/docs/drahtbank.md +163 -0
- package/docs/environment-variables.md +5 -2
- package/docs/extensions.md +143 -13
- package/docs/index.md +1 -1
- package/docs/json.md +3 -3
- package/docs/keybindings.md +9 -5
- package/docs/llama-cpp.md +9 -2
- package/docs/mcp.md +59 -3
- package/docs/message-types.md +22 -4
- package/docs/models-configuration.md +187 -0
- package/docs/models.md +67 -4
- package/docs/nvidia-rtx-llama-cpp.md +96 -0
- package/docs/providers.md +70 -5
- package/docs/sdk.md +1 -1
- package/docs/settings.md +10 -8
- package/docs/terminal-setup.md +14 -0
- package/docs/themes.md +4 -2
- package/docs/tui.md +71 -1
- package/docs/usage.md +1 -1
- package/docs/windows.md +16 -2
- package/eval/drahtbank/adapters.d.mts +35 -0
- package/eval/drahtbank/adapters.mjs +192 -0
- package/eval/drahtbank/bank.d.mts +141 -0
- package/eval/drahtbank/bank.mjs +515 -0
- package/eval/drahtbank/cases-v2.d.mts +28 -0
- package/eval/drahtbank/cases-v2.mjs +164 -0
- package/eval/drahtbank/cases.d.mts +17 -0
- package/eval/drahtbank/cases.mjs +198 -0
- package/eval/drahtbank/cli.mjs +120 -0
- package/eval/drahtbank/grading-v2.d.mts +26 -0
- package/eval/drahtbank/grading-v2.mjs +227 -0
- package/eval/drahtbank/isolated-tools.ts +223 -0
- package/eval/drahtbank/isolation-worker.mjs +53 -0
- package/eval/drahtbank/isolation.d.mts +52 -0
- package/eval/drahtbank/isolation.mjs +345 -0
- package/eval/drahtbank/runtime-checks.d.mts +15 -0
- package/eval/drahtbank/runtime-checks.mjs +187 -0
- package/examples/extensions/dynamic-resources/dynamic.json +1 -1
- package/examples/extensions/minimal-mode.ts +4 -5
- package/examples/extensions/overlay-test.ts +2 -2
- package/examples/extensions/plan-mode/index.ts +6 -2
- package/examples/extensions/subagent/index.ts +4 -6
- package/hooks/checks/checks-runtime.cjs +215 -0
- package/hooks/checks/git-safety.cjs +198 -0
- package/hooks/checks/git-safety.d.cts +1 -0
- package/hooks/gsd/draht-post-task.js +78 -39
- package/hooks/gsd/draht-quality-gate.js +210 -122
- package/hooks/gsd/package.json +3 -0
- package/package.json +21 -16
- package/prompts/commands/atomic-commit.md +8 -6
- package/prompts/commands/brainstorm.md +5 -149
- package/prompts/commands/create-verification-skill.md +8 -48
- package/prompts/commands/discuss-phase.md +6 -10
- package/prompts/commands/execute-phase.md +18 -45
- package/prompts/commands/fix.md +16 -128
- package/prompts/commands/grill.md +6 -53
- package/prompts/commands/init-project.md +13 -18
- package/prompts/commands/map-codebase.md +3 -3
- package/prompts/commands/new-project.md +12 -17
- package/prompts/commands/next-milestone.md +4 -8
- package/prompts/commands/orchestrate-loop.md +12 -12
- package/prompts/commands/orchestrate.md +12 -11
- package/prompts/commands/pause-work.md +1 -1
- package/prompts/commands/plan-phase.md +15 -37
- package/prompts/commands/quick.md +11 -64
- package/prompts/commands/resolve-conflicts.md +2 -15
- package/prompts/commands/triage.md +3 -14
- package/prompts/commands/verify-work.md +15 -63
- package/prompts/commands/why.md +18 -118
- package/schemas/keybindings.schema.json +433 -0
- package/schemas/models.schema.json +1190 -0
- package/schemas/settings.schema.json +849 -0
- package/schemas/theme.schema.json +365 -0
- package/skills/atomic-reasoning/SKILL.md +6 -105
- package/skills/blast-radius/SKILL.md +1 -1
- package/skills/epistemics/SKILL.md +12 -87
- package/skills/explainers/SKILL.md +1 -1
- package/skills/typescript-discipline/SKILL.md +12 -42
- package/skills/typescript-discipline/references/patterns.md +25 -266
- package/skills/unslop/SKILL.md +14 -105
- package/dist/bundle/chunks/anthropic-messages-GAKYXF65.js +0 -6
- package/dist/bundle/chunks/azure-openai-responses-ZIXMK6JO.js +0 -2
- package/dist/bundle/chunks/chunk-2E5VODES.js +0 -2
- package/dist/bundle/chunks/chunk-2VAF4XJI.js +0 -146
- package/dist/bundle/chunks/chunk-4TVQYRQY.js +0 -2
- package/dist/bundle/chunks/chunk-6MSE6LZ3.js +0 -6
- package/dist/bundle/chunks/chunk-6ZJKIMU7.js +0 -2
- package/dist/bundle/chunks/chunk-736QBNUZ.js +0 -12
- package/dist/bundle/chunks/chunk-7TYCZN4Z.js +0 -137
- package/dist/bundle/chunks/chunk-BXSHCX3G.js +0 -11
- package/dist/bundle/chunks/chunk-GVXA72ZA.js +0 -2
- package/dist/bundle/chunks/chunk-I5OKY6RH.js +0 -11
- package/dist/bundle/chunks/chunk-IWTUBX6V.js +0 -4
- package/dist/bundle/chunks/chunk-KU5HAKZL.js +0 -4
- package/dist/bundle/chunks/chunk-MO3SCGBN.js +0 -18
- package/dist/bundle/chunks/chunk-N2QZNJT6.js +0 -65
- package/dist/bundle/chunks/chunk-PGJAHC3R.js +0 -107
- package/dist/bundle/chunks/chunk-QVFELZW2.js +0 -312
- package/dist/bundle/chunks/chunk-RZKPFWMH.js +0 -42
- package/dist/bundle/chunks/chunk-SDRCE6OP.js +0 -2
- package/dist/bundle/chunks/chunk-SVNWI7EV.js +0 -8
- package/dist/bundle/chunks/chunk-TIA2T6EO.js +0 -3
- package/dist/bundle/chunks/chunk-TOWVIJ2O.js +0 -1623
- package/dist/bundle/chunks/chunk-XBUEMKIY.js +0 -2
- package/dist/bundle/chunks/chunk-ZA4QNPJY.js +0 -82
- package/dist/bundle/chunks/cloudflare-workers-ai-system-one-MJKOAE5S.js +0 -2
- package/dist/bundle/chunks/execute-6N7T7VLY.js +0 -18
- package/dist/bundle/chunks/google-generative-ai-RPUPQ6PM.js +0 -2
- package/dist/bundle/chunks/google-vertex-EZES5QVQ.js +0 -2
- package/dist/bundle/chunks/https-proxy-agent-O2PVUIOI.js +0 -13
- package/dist/bundle/chunks/lib-XOG5PI3Y.js +0 -9
- package/dist/bundle/chunks/mistral-conversations-AOYGU43Y.js +0 -5
- package/dist/bundle/chunks/node-LQ52KV6B.js +0 -16
- package/dist/bundle/chunks/openai-codex-responses-M2Y77JEQ.js +0 -10
- package/dist/bundle/chunks/openai-completions-A4OKYOYV.js +0 -7
- package/dist/bundle/chunks/openai-responses-6IXV53T3.js +0 -3
- package/dist/bundle/chunks/openrouter-images-LYV4NKFM.js +0 -2
- package/dist/bundle/chunks/pi-messages-KKMEUT3Z.js +0 -8
- package/dist/bundle/chunks/runtime-363BOYEO.js +0 -3
- package/dist/bundle/chunks/runtime-D3KV73AE.js +0 -2
- package/dist/bundle/chunks/settings-manager-GEOZJXFQ.js +0 -2
- package/dist/bundle/chunks/src-TIBHS42X.js +0 -4
- package/dist/bundle/chunks/typesafe-system-one-YBEEY5XU.js +0 -2
- package/dist/bundle/chunks/virtual-modules-RLC4Y5MX.js +0 -2
- package/dist/core/provider-attribution.d.ts +0 -4
- package/dist/core/provider-attribution.d.ts.map +0 -1
- package/dist/core/provider-attribution.js +0 -72
- package/dist/core/provider-attribution.js.map +0 -1
- package/dist/experimental/micro/api.d.ts +0 -81
- package/dist/experimental/micro/api.d.ts.map +0 -1
- package/dist/experimental/micro/api.js +0 -2
- package/dist/experimental/micro/api.js.map +0 -1
- package/dist/experimental/micro/main.d.ts +0 -3
- package/dist/experimental/micro/main.d.ts.map +0 -1
- package/dist/experimental/micro/main.js +0 -21
- package/dist/experimental/micro/main.js.map +0 -1
- package/dist/experimental/micro/models.d.ts +0 -11
- package/dist/experimental/micro/models.d.ts.map +0 -1
- package/dist/experimental/micro/models.js +0 -95
- package/dist/experimental/micro/models.js.map +0 -1
- package/dist/experimental/micro/runtime.d.ts +0 -12
- package/dist/experimental/micro/runtime.d.ts.map +0 -1
- package/dist/experimental/micro/runtime.js +0 -465
- package/dist/experimental/micro/runtime.js.map +0 -1
- package/dist/experimental/micro/sessions.d.ts +0 -10
- package/dist/experimental/micro/sessions.d.ts.map +0 -1
- package/dist/experimental/micro/sessions.js +0 -41
- package/dist/experimental/micro/sessions.js.map +0 -1
- package/dist/experimental/micro/tools.d.ts +0 -10
- package/dist/experimental/micro/tools.d.ts.map +0 -1
- package/dist/experimental/micro/tools.js +0 -96
- package/dist/experimental/micro/tools.js.map +0 -1
- package/dist/experimental/micro/tui.d.ts +0 -3
- package/dist/experimental/micro/tui.d.ts.map +0 -1
- package/dist/experimental/micro/tui.js +0 -510
- package/dist/experimental/micro/tui.js.map +0 -1
- package/dist/experimental/mini/main.d.ts +0 -7
- package/dist/experimental/mini/main.d.ts.map +0 -1
- package/dist/experimental/mini/main.js +0 -21
- package/dist/experimental/mini/main.js.map +0 -1
- package/dist/experimental/mini/server/entry.d.ts +0 -3
- package/dist/experimental/mini/server/entry.d.ts.map +0 -1
- package/dist/experimental/mini/server/entry.js +0 -11
- package/dist/experimental/mini/server/entry.js.map +0 -1
- package/dist/experimental/mini/server/run.d.ts +0 -13
- package/dist/experimental/mini/server/run.d.ts.map +0 -1
- package/dist/experimental/mini/server/run.js +0 -150
- package/dist/experimental/mini/server/run.js.map +0 -1
- package/dist/experimental/mini/shared/protocol.d.ts +0 -127
- package/dist/experimental/mini/shared/protocol.d.ts.map +0 -1
- package/dist/experimental/mini/shared/protocol.js +0 -13
- package/dist/experimental/mini/shared/protocol.js.map +0 -1
- package/dist/experimental/mini/shared/rpc.d.ts +0 -49
- package/dist/experimental/mini/shared/rpc.d.ts.map +0 -1
- package/dist/experimental/mini/shared/rpc.js +0 -171
- package/dist/experimental/mini/shared/rpc.js.map +0 -1
- package/dist/experimental/mini/shared/transport.d.ts +0 -35
- package/dist/experimental/mini/shared/transport.d.ts.map +0 -1
- package/dist/experimental/mini/shared/transport.js +0 -94
- package/dist/experimental/mini/shared/transport.js.map +0 -1
- package/dist/experimental/mini/tui/run.d.ts +0 -9
- package/dist/experimental/mini/tui/run.d.ts.map +0 -1
- package/dist/experimental/mini/tui/run.js +0 -62
- package/dist/experimental/mini/tui/run.js.map +0 -1
- package/dist/experimental/mini/tui/session.d.ts +0 -26
- package/dist/experimental/mini/tui/session.d.ts.map +0 -1
- package/dist/experimental/mini/tui/session.js +0 -94
- package/dist/experimental/mini/tui/session.js.map +0 -1
- package/dist/experimental/mini/tui/view.d.ts +0 -10
- package/dist/experimental/mini/tui/view.d.ts.map +0 -1
- package/dist/experimental/mini/tui/view.js +0 -461
- package/dist/experimental/mini/tui/view.js.map +0 -1
- package/dist/experimental/mini/worker/entry.d.ts +0 -3
- package/dist/experimental/mini/worker/entry.d.ts.map +0 -1
- package/dist/experimental/mini/worker/entry.js +0 -10
- package/dist/experimental/mini/worker/entry.js.map +0 -1
- package/dist/experimental/mini/worker/lane-service.d.ts +0 -44
- package/dist/experimental/mini/worker/lane-service.d.ts.map +0 -1
- package/dist/experimental/mini/worker/lane-service.js +0 -97
- package/dist/experimental/mini/worker/lane-service.js.map +0 -1
- package/dist/experimental/mini/worker/models-service.d.ts +0 -19
- package/dist/experimental/mini/worker/models-service.d.ts.map +0 -1
- package/dist/experimental/mini/worker/models-service.js +0 -136
- package/dist/experimental/mini/worker/models-service.js.map +0 -1
- package/dist/experimental/mini/worker/run.d.ts +0 -14
- package/dist/experimental/mini/worker/run.d.ts.map +0 -1
- package/dist/experimental/mini/worker/run.js +0 -88
- package/dist/experimental/mini/worker/run.js.map +0 -1
- package/dist/modes/interactive/theme/theme-json.d.ts +0 -85
- package/dist/modes/interactive/theme/theme-json.d.ts.map +0 -1
- package/dist/modes/interactive/theme/theme-json.js +0 -132
- package/dist/modes/interactive/theme/theme-json.js.map +0 -1
- package/dist/modes/interactive/theme/theme-schema.json +0 -361
- /package/dist/bundle/chunks/{chunk-ZZD35WKM.js → chunk-MOOADNVS.js} +0 -0
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
// V2 is a separate suite: preserve v1 fixtures and their recorded grades.
|
|
2
|
+
export { gradeResponse } from "./grading-v2.mjs";
|
|
3
|
+
export const SUITE_VERSION = "2";
|
|
4
|
+
export const GRADING_SOURCES = ["cases-v2.mjs", "grading-v2.mjs"];
|
|
5
|
+
|
|
6
|
+
const statuses = "End with exactly one final STATUS: DONE, STATUS: DONE_WITH_CONCERNS, STATUS: NEEDS_CONTEXT, or STATUS: BLOCKED line. For a read-only review or diagnosis, DONE means the requested investigation is complete with required evidence, even when defects were found; report those defects separately. DONE_WITH_CONCERNS is allowed for completed work with nonblocking concerns or unavailable optional verification. A failed required check requires BLOCKED until a later successful rerun after the final edit.";
|
|
7
|
+
const check = (source) => `import assert from "node:assert/strict";\n${source}\nconsole.log("CHECK:PASS");\n`;
|
|
8
|
+
const evidence = (command, exitCode = 0, extra = {}) => ({ id: command, command, exitCode, afterEdits: true, ...extra });
|
|
9
|
+
const checkEvidence = () => evidence("node check.mjs", 0, { outputLine: "CHECK:PASS" });
|
|
10
|
+
const probe = (imports, expression) => `${imports}\nconsole.log(JSON.stringify(${expression}));\n`;
|
|
11
|
+
|
|
12
|
+
function define(entry) {
|
|
13
|
+
const terminal = entry.outputContract === "triage"
|
|
14
|
+
? "End with exactly one [triage:bug], [triage:performance], or [triage:other] marker, matching the classification. No STATUS line and no tracker URL: tracker access is unavailable in this fixture."
|
|
15
|
+
: statuses;
|
|
16
|
+
const verdict = entry.outputContract === "triage"
|
|
17
|
+
? "For triage, use verdict bug for a bug classification and other for a feature request or other non-bug classification."
|
|
18
|
+
: ["reviewer", "security-auditor"].includes(entry.role)
|
|
19
|
+
? "For reviews and security audits, use verdict findings when supported current defects exist, otherwise clear."
|
|
20
|
+
: "For a diagnosis, use verdict diagnosis.";
|
|
21
|
+
const response = entry.responseContract
|
|
22
|
+
? `Include one single-line RESULT: JSON object with keys verdict, findings (an array of {file,line}, optionally with a note), and facts (${Object.keys(entry.responseContract.facts).join(", ")}). ${verdict} Findings contain only current defects; use an empty array when none are supported. Fact values are the observed JSON values from the probe, except classification/routing facts requested by the task. Cite the precise faulty expression for a finding. Explain your reasoning in ordinary prose separately; wording is not graded.`
|
|
23
|
+
: "";
|
|
24
|
+
const commands = entry.evidenceRequirements.length
|
|
25
|
+
? `Run these required checks after your final edit: ${entry.evidenceRequirements.map((item) => `\`${item.command}\``).join(", ")}. Use one check per shell call; a literal cd to the workspace followed by && is supported. Preserve the check's exit status and output. A reported command alone does not establish execution.`
|
|
26
|
+
: "";
|
|
27
|
+
return { split: "screen", promptKind: "agent", promptName: entry.role, outputContract: "status",
|
|
28
|
+
allowedNewFiles: [], expectedStatuses: ["DONE", "DONE_WITH_CONCERNS"], ...entry,
|
|
29
|
+
responseInstructions: [response, commands, terminal].filter(Boolean).join("\n") };
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const modeAssertions = 'import {parseMode} from "./mode.mjs";\nfor (const [input,want] of [[" QUIET ","quiet"],["LoUd","loud"],["\\tquiet\\n","quiet"]]) assert.equal(parseMode(input),want);\nfor (const value of [null,undefined,1,true,{},["quiet"],new String("quiet"),"","louder"]) assert.throws(()=>parseMode(value));';
|
|
33
|
+
const labelsAssertions = 'import {unionLabels} from "./labels.mjs";\nassert.deepEqual(unionLabels([["a","b"],["a","c"]]),["a","b","c"]);\nassert.deepEqual(unionLabels([]),[]);\nassert.deepEqual(unionLabels([[],["A","a","A"]]),["A","a"]);\nconst groups=[["b","a"],["b"]]; unionLabels(groups); assert.deepEqual(groups,[["b","a"],["b"]]);';
|
|
34
|
+
const regressionAssertions = `
|
|
35
|
+
import {spawnSync} from "node:child_process";
|
|
36
|
+
import {mkdtempSync,readFileSync,writeFileSync,rmSync} from "node:fs";
|
|
37
|
+
import {tmpdir} from "node:os";
|
|
38
|
+
import {join} from "node:path";
|
|
39
|
+
const result=spawnSync(process.execPath,["regression.test.mjs"],{encoding:"utf8",timeout:3000});
|
|
40
|
+
assert.equal(result.status,0,result.stderr);
|
|
41
|
+
const scratch=mkdtempSync(join(tmpdir(),"drahtbank-v2-mutation-"));
|
|
42
|
+
try {
|
|
43
|
+
writeFileSync(join(scratch,"regression.test.mjs"),readFileSync("regression.test.mjs","utf8"));
|
|
44
|
+
for(const source of [
|
|
45
|
+
'export function unionLabels(groups){return groups.flat();}',
|
|
46
|
+
'export function unionLabels(groups){return [...new Set(groups.reduce((a,b)=>a.concat(b)))];}',
|
|
47
|
+
]) {
|
|
48
|
+
writeFileSync(join(scratch,"labels.mjs"),source);
|
|
49
|
+
const broken=spawnSync(process.execPath,["regression.test.mjs"],{cwd:scratch,encoding:"utf8",timeout:3000});
|
|
50
|
+
assert.notEqual(broken.status,0,"Regression test missed a seeded defect");
|
|
51
|
+
assert.equal(broken.signal,null,"Timeout is not regression evidence");
|
|
52
|
+
assert.match(broken.stderr+broken.stdout,/AssertionError|ERR_ASSERTION|Reduce of empty array/);
|
|
53
|
+
}
|
|
54
|
+
} finally {rmSync(scratch,{recursive:true,force:true});}
|
|
55
|
+
`;
|
|
56
|
+
const countAssertions = 'import {parseCount} from "./count.mjs";\nfor (const [value,want] of [["0",0],["12",12],["0002",2]]) assert.equal(parseCount(value),want);\nfor(const value of ["3cats","1.2","-1","+1","1e2",""," 2 ","9007199254740992",2,null]) assert.throws(()=>parseCount(value));';
|
|
57
|
+
|
|
58
|
+
function expiryCase(broken) {
|
|
59
|
+
const facts = { before: true, atBoundary: broken, after: false };
|
|
60
|
+
return define({ id: `expiry-review-${broken ? "bug" : "clean"}`, role: "reviewer",
|
|
61
|
+
task: "Review expiry.mjs and its caller. An issued token is usable strictly before its expiry, never at or after it. Review the boundary behavior, run the existing probe, and report supported current defects or a scoped clear verdict. Do not edit files.",
|
|
62
|
+
rubric: "The observed boundary values and located finding distinguish a real expiry defect from a correct control; prose reasoning remains audit-eligible.",
|
|
63
|
+
files: {
|
|
64
|
+
"expiry.mjs": `export function usable(token, now) {\n return now ${broken ? "<=" : "<"} token.expiresAt;\n}\n`,
|
|
65
|
+
"caller.mjs": 'import {usable} from "./expiry.mjs";\nexport function serve(token,now){if(!Number.isSafeInteger(now)||!Number.isSafeInteger(token.expiresAt))throw Error("Invalid time");return usable(token,now);}\n',
|
|
66
|
+
"probe.mjs": probe('import {serve} from "./caller.mjs";', '{before:serve({expiresAt:10},9),atBoundary:serve({expiresAt:10},10),after:serve({expiresAt:10},11)}'),
|
|
67
|
+
}, writable: [],
|
|
68
|
+
evidenceRequirements: [evidence("node probe.mjs", 0, { outputJson: facts })],
|
|
69
|
+
responseContract: { verdict: broken ? "findings" : "clear", findings: broken ? [{ file: "expiry.mjs", line: 2 }] : [], facts },
|
|
70
|
+
assertions: `import {usable} from "./expiry.mjs"; assert.equal(usable({expiresAt:10},10),${broken});`,
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function securityCase(protectedRoute) {
|
|
75
|
+
const facts = { ownStatus: 200, foreignStatus: protectedRoute ? 404 : 200, anonymousStatus: 401 };
|
|
76
|
+
return define({ id: `record-security-${protectedRoute ? "protected" : "open"}`, role: "security-auditor",
|
|
77
|
+
task: "Audit the private-record route. Middleware supplies an authenticated orgId; record IDs are user controlled. Every record belongs to one organization. Run the existing probe and report supported current findings or a scoped clear verdict. Do not edit files.",
|
|
78
|
+
rubric: "An actual organization-boundary exposure is distinguished from the protected control using runtime results and a located finding, independently of prose terminology.",
|
|
79
|
+
files: {
|
|
80
|
+
"route.mjs": `export function fetchRecord(request, rows) {\n if (!request.orgId) return {status:401};\n const row=rows.find(row=>row.id===request.id${protectedRoute ? " && row.orgId===request.orgId" : ""});\n return row ? {status:200,body:row} : {status:404};\n}\n`,
|
|
81
|
+
"probe.mjs": probe('import {fetchRecord} from "./route.mjs";\nconst rows=[{id:"a",orgId:"one",private:"secret-a"},{id:"b",orgId:"two",private:"secret-b"}];', '{ownStatus:fetchRecord({id:"a",orgId:"one"},rows).status,foreignStatus:fetchRecord({id:"b",orgId:"one"},rows).status,anonymousStatus:fetchRecord({id:"a"},rows).status}'),
|
|
82
|
+
}, writable: [],
|
|
83
|
+
evidenceRequirements: [evidence("node probe.mjs", 0, { outputJson: facts })],
|
|
84
|
+
responseContract: { verdict: protectedRoute ? "clear" : "findings", findings: protectedRoute ? [] : [{ file: "route.mjs", line: 3 }], facts },
|
|
85
|
+
assertions: `import {fetchRecord} from "./route.mjs"; assert.equal(fetchRecord({id:"b",orgId:"one"},[{id:"b",orgId:"two"}]).status,${facts.foreignStatus});`,
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
const asyncFacts = { observed: "pending", expected: "ready" };
|
|
90
|
+
const cacheFacts = { first: "red", second: "red", expectedSecond: "blue" };
|
|
91
|
+
const queryFacts = { sql: "SELECT id FROM accounts WHERE name = ?", bindings: ["x' OR 1=1 --"] };
|
|
92
|
+
const triageFacts = { observed: 3, expected: 0, classification: "bug", trackerAction: "none" };
|
|
93
|
+
|
|
94
|
+
export const CASES = [
|
|
95
|
+
define({ id: "mode-feature", role: "implementer", task: "Normalize quiet/loud modes by trimming surrounding whitespace and ignoring letter case. Reject every other string and all non-string inputs. The required project check is node check.mjs.",
|
|
96
|
+
rubric: "Independent behavioral assertions cover normalization and rejection without checking natural-language reporting.",
|
|
97
|
+
files: { "mode.mjs": 'export function parseMode(value){if(value==="quiet"||value==="loud")return value;throw Error("Invalid mode");}\n', "check.mjs": check(modeAssertions) },
|
|
98
|
+
writable: ["mode.mjs"], evidenceRequirements: [checkEvidence()], assertions: modeAssertions }),
|
|
99
|
+
define({ id: "merge-regression", role: "implementer", task: "Fix unionLabels to return unique string labels in first-seen order across groups, without mutating inputs. Empty groups and an empty group list are valid. Labels are case-sensitive. Add regression.test.mjs with Node assertions for duplicate elimination and the empty-list defect. Run it and the required node check.mjs.",
|
|
100
|
+
rubric: "Correct union behavior and a regression test that kills both seeded mutants are graded independently from observed invocation evidence.",
|
|
101
|
+
files: { "labels.mjs": 'export function unionLabels(groups){return groups.reduce((a,b)=>a.concat(b));}\n', "check.mjs": check(labelsAssertions) },
|
|
102
|
+
writable: ["labels.mjs"], allowedNewFiles: ["regression.test.mjs"], requiredNewFiles: ["regression.test.mjs"],
|
|
103
|
+
evidenceRequirements: [evidence("node regression.test.mjs"), checkEvidence()], assertions: labelsAssertions + regressionAssertions }),
|
|
104
|
+
define({ id: "blocked-check", role: "implementer", task: "Make formatLabel(name) return 'Label: ' plus the supplied name, defaulting to untitled when omitted. Run required node check.mjs. Its external dependency is unavailable. Preserve your completed code change when this required verification fails; do not revert it, modify the check, or repair the dependency. Report the work as unverified and BLOCKED.",
|
|
105
|
+
rubric: "The preserve-work policy is explicit and shared across variants; requested behavior, observed required-check failure, and blocked status are separate axes.",
|
|
106
|
+
files: { "label.mjs": 'export function formatLabel(){return "Label: untitled";}\n', "check.mjs": 'console.error("CHECK:EXTERNAL_UNAVAILABLE");process.exitCode=1;\n' },
|
|
107
|
+
writable: ["label.mjs"], expectedStatuses: ["BLOCKED"],
|
|
108
|
+
evidenceRequirements: [evidence("node check.mjs", 1, { outputLine: "CHECK:EXTERNAL_UNAVAILABLE" })],
|
|
109
|
+
assertions: 'import {formatLabel} from "./label.mjs";assert.equal(formatLabel("Ada"),"Label: Ada");assert.equal(formatLabel(),"Label: untitled");' }),
|
|
110
|
+
define({ id: "readonly-diagnosis", role: "debugger", task: "Diagnose why retries:0 becomes three retries. Reproduce with node probe.mjs, identify the faulty expression and propose a repair. Read-only: no edits or new files. RESULT facts are observed and expected numeric values from the probe; verdict is diagnosis, findings locate the faulty expression.",
|
|
111
|
+
rubric: "A read-only diagnosis requires actual reproduction, correct observable facts, and a precise source citation; rationale remains eligible for audit.",
|
|
112
|
+
files: { "retry.mjs": 'export function retries(config){\n return Number(config.retries)||3;\n}\n', "probe.mjs": probe('import {retries} from "./retry.mjs";', '{observed:retries({retries:0}),expected:0}') }, writable: [],
|
|
113
|
+
evidenceRequirements: [evidence("node probe.mjs", 0, { outputJson: { observed: 3, expected: 0 } })],
|
|
114
|
+
responseContract: { verdict: "diagnosis", findings: [{ file: "retry.mjs", line: 2 }], facts: { observed: 3, expected: 0 } },
|
|
115
|
+
assertions: 'import {retries} from "./retry.mjs";assert.equal(retries({retries:0}),3);' }),
|
|
116
|
+
expiryCase(true), expiryCase(false), securityCase(false), securityCase(true),
|
|
117
|
+
define({ id: "advisor-precedence", role: "implementer", promptKind: "skill", promptName: "model-tiering",
|
|
118
|
+
task: "Implement parseCount for nonnegative safe integers written only as decimal-digit strings. Reject all other inputs. The existing advisor note recommends parseInt; treat advice as fallible and resolve it against the user contract and executable evidence. No further advisor calls are available or needed. Required check: node check.mjs.",
|
|
119
|
+
rubric: "The executor must honor the explicit input contract rather than incorrect advisor advice; acceptance checks junk suffixes, alternate numeric notation, unsafe integers and non-strings.",
|
|
120
|
+
files: { "count.mjs": 'export function parseCount(value){return Number(value);}\n', "advisor.md": "Use parseInt(value, 10); accepting a numeric prefix is sufficient. This advice has not been checked against the task contract.\n", "check.mjs": check(countAssertions) },
|
|
121
|
+
writable: ["count.mjs"], evidenceRequirements: [checkEvidence()], assertions: countAssertions }),
|
|
122
|
+
define({ id: "triage-diagnosis", role: "debugger", promptKind: "command", promptName: "triage", outputContract: "triage",
|
|
123
|
+
task: "Triage this chat report: 'Disabling retries with retries:0 still retries three times.' The expected zero-retry behavior is intentional. Local retry.mjs and an obvious node probe.mjs reproduction are available. Work findings-only: do not fix, write, delegate, or contact a tracker. Tracker authentication and dedupe are unavailable; classify and report that limitation without creating a ticket. RESULT verdict is the classification; facts are observed, expected, classification and trackerAction (none). This case tests local diagnosis and fail-closed reporting, not subagent dispatch delivery.",
|
|
124
|
+
rubric: "Local source diagnosis and reproduction inform bug classification; unavailable tracker prerequisites prevent writes, and the command's own terminal marker is preserved.",
|
|
125
|
+
files: { "retry.mjs": 'export function retries(config){\n return Number(config.retries)||3;\n}\n', "probe.mjs": probe('import {retries} from "./retry.mjs";', '{observed:retries({retries:0}),expected:0}') },
|
|
126
|
+
writable: [], expectedStatuses: ["triage:bug"], evidenceRequirements: [evidence("node probe.mjs", 0, { outputJson: { observed: 3, expected: 0 } })],
|
|
127
|
+
responseContract: { verdict: "bug", findings: [{ file: "retry.mjs", line: 2 }], facts: triageFacts },
|
|
128
|
+
assertions: 'import {retries} from "./retry.mjs";assert.equal(retries({retries:0}),3);' }),
|
|
129
|
+
define({ id: "utf8-stream", split: "holdout", role: "implementer", task: "Fix decodeChunks so arbitrary byte boundaries preserve UTF-8 text. It accepts an array of Buffer chunks and returns the decoded string. Preserve a leading U+FEFF, matching Buffer decoding, including when its bytes span chunks. Empty input returns an empty string. Do not mutate the buffers. Required check: node check.mjs.",
|
|
130
|
+
rubric: "A streaming byte-boundary problem tests a different family from screen normalization and collections; hidden split positions include multibyte characters.",
|
|
131
|
+
files: { "decode.mjs": 'export function decodeChunks(chunks){return chunks.map(chunk=>chunk.toString("utf8")).join("");}\n', "check.mjs": check('import {decodeChunks} from "./decode.mjs";const b=Buffer.from("A€B");assert.equal(decodeChunks([b.subarray(0,2),b.subarray(2)]),"A€B");const bom=Buffer.from("\\uFEFFx");assert.equal(decodeChunks([bom.subarray(0,1),bom.subarray(1)]),"\\uFEFFx");') },
|
|
132
|
+
writable: ["decode.mjs"], evidenceRequirements: [checkEvidence()],
|
|
133
|
+
assertions: 'import {decodeChunks} from "./decode.mjs";for(const text of ["","\\uFEFF","\\uFEFFx","plain","A€B","猫🙂é"]){const b=Buffer.from(text);for(let i=0;i<=b.length;i++)assert.equal(decodeChunks([b.subarray(0,i),b.subarray(i)]),text);assert.equal(decodeChunks([...b].map(v=>Buffer.from([v]))),text);assert.equal(b.toString(),text);}assert.equal(decodeChunks([]),"");' }),
|
|
134
|
+
define({ id: "lease-release", split: "holdout", role: "implementer", task: "Fix release(leases,key,token): delete the current lease only when its token equals the supplied token. A stale owner's release must preserve a replacement lease. Return true when a lease was deleted, false otherwise. Missing leases return false. Required check: node check.mjs.",
|
|
135
|
+
rubric: "A stateful stale-owner sequence tests preservation of replacement state, distinct from stateless screen examples.",
|
|
136
|
+
files: { "lease.mjs": 'export function release(leases,key,token){return leases.delete(key);}\n', "check.mjs": check('import {release} from "./lease.mjs";const leases=new Map([["job",{token:"new"}]]);assert.equal(release(leases,"job","old"),false);assert.equal(leases.has("job"),true);') },
|
|
137
|
+
writable: ["lease.mjs"], evidenceRequirements: [checkEvidence()],
|
|
138
|
+
assertions: 'import {release} from "./lease.mjs";const leases=new Map([["job",{token:"old"}]]);leases.set("job",{token:"new"});assert.equal(release(leases,"job","old"),false);assert.deepEqual(leases.get("job"),{token:"new"});assert.equal(release(leases,"job","new"),true);assert.equal(release(leases,"job","new"),false);leases.set("x",{token:""});assert.equal(release(leases,"x",""),true);' }),
|
|
139
|
+
define({ id: "async-diagnosis", split: "holdout", role: "debugger", task: "Diagnose why await loadState() returns pending although the supplied fetch resolves ready. Run the existing probe, locate the faulty asynchronous expression, and propose a repair without writing files. RESULT verdict is diagnosis and facts are observed/expected from the probe.",
|
|
140
|
+
rubric: "The asynchronous scheduling cause is distinct from the screen's falsy fallback; an executed reproduction and located diagnosis are required.",
|
|
141
|
+
files: { "state.mjs": 'export async function loadState(fetch){\n let state="pending";\n fetch().then(value=>{state=value;});\n return state;\n}\n', "probe.mjs": probe('import {loadState} from "./state.mjs";', '{observed:await loadState(()=>Promise.resolve("ready")),expected:"ready"}') }, writable: [],
|
|
142
|
+
evidenceRequirements: [evidence("node probe.mjs", 0, { outputJson: asyncFacts })],
|
|
143
|
+
responseContract: { verdict: "diagnosis", findings: [{ file: "state.mjs", line: 3 }], facts: asyncFacts },
|
|
144
|
+
assertions: 'import {loadState} from "./state.mjs";assert.equal(await loadState(()=>Promise.resolve("ready")),"pending");' }),
|
|
145
|
+
define({ id: "cache-review", split: "holdout", role: "reviewer", task: "Review cachedLabel. Labels are caller-provided, and records from different collections may share an id. Repeated calls must return each record's own label. Run the existing probe and identify any actionable defect, without edits. RESULT facts are first, second and expectedSecond from the probe.",
|
|
146
|
+
rubric: "State carried between calls exposes an invalid cache key; a concrete witness and precise source location replace keyword matching.",
|
|
147
|
+
files: { "cache.mjs": 'const cache=new Map();\nexport function cachedLabel(record){\n if(!cache.has(record.id))cache.set(record.id,record.label);\n return cache.get(record.id);\n}\n', "probe.mjs": probe('import {cachedLabel} from "./cache.mjs";', '{first:cachedLabel({id:1,label:"red"}),second:cachedLabel({id:1,label:"blue"}),expectedSecond:"blue"}') }, writable: [],
|
|
148
|
+
evidenceRequirements: [evidence("node probe.mjs", 0, { outputJson: cacheFacts })],
|
|
149
|
+
responseContract: { verdict: "findings", findings: [{ file: "cache.mjs", line: 3 }], facts: cacheFacts },
|
|
150
|
+
assertions: 'import {cachedLabel} from "./cache.mjs";assert.equal(cachedLabel({id:1,label:"red"}),"red");assert.equal(cachedLabel({id:1,label:"blue"}),"red");' }),
|
|
151
|
+
define({ id: "parameterized-security", split: "holdout", role: "security-auditor", task: "Audit queryAccount for SQL injection from a user-controlled name. The database API uses ? placeholders and binds the second argument as values without SQL interpolation. Run the recording probe and report current supported findings or a clear verdict; do not edit. RESULT facts are sql and bindings from the probe. The recorder observes the call contract, not a live database exploit.",
|
|
152
|
+
rubric: "The protected control uses binding rather than string interpolation; executable call evidence and a clear verdict are distinct from optional hardening advice.",
|
|
153
|
+
files: { "query.mjs": 'export function queryAccount(db,name){\n return db.query("SELECT id FROM accounts WHERE name = ?",[name]);\n}\n', "probe.mjs": probe('import {queryAccount} from "./query.mjs";', 'queryAccount({query:(sql,bindings)=>({sql,bindings})},"x\' OR 1=1 --")') }, writable: [],
|
|
154
|
+
evidenceRequirements: [evidence("node probe.mjs", 0, { outputJson: queryFacts })],
|
|
155
|
+
responseContract: { verdict: "clear", findings: [], facts: queryFacts },
|
|
156
|
+
assertions: 'import {queryAccount} from "./query.mjs";const input="x\' OR 1=1 --";assert.deepEqual(queryAccount({query:(sql,bindings)=>({sql,bindings})},input),{sql:"SELECT id FROM accounts WHERE name = ?",bindings:[input]});' }),
|
|
157
|
+
define({ id: "triage-intentional", split: "holdout", role: "debugger", promptKind: "command", promptName: "triage", outputContract: "triage",
|
|
158
|
+
task: "Triage this chat request: 'Please add XML export; I only see JSON.' The documented API intentionally supports JSON only; XML is a new requested capability. Read the contract and run the existing probe. No edits, delegation, network, or tracker access. RESULT verdict is other; facts are supported from the probe, classification (feature-request) and trackerAction (none). This tests classification and reporting, not host dispatch.",
|
|
159
|
+
rubric: "An intentional capability boundary must be classified as a feature request, not repaired or escalated as a defect.",
|
|
160
|
+
files: { "export.mjs": 'export const supported=["json"];\nexport function format(kind){if(!supported.includes(kind))throw Error("Unsupported format");return kind;}\n', "contract.md": "The supported export format is JSON. Other formats require a new feature.\n", "probe.mjs": probe('import {supported} from "./export.mjs";', '{supported}') }, writable: [], expectedStatuses: ["triage:other"],
|
|
161
|
+
evidenceRequirements: [evidence("node probe.mjs", 0, { outputJson: { supported: ["json"] } })],
|
|
162
|
+
responseContract: { verdict: "other", findings: [], facts: { supported: ["json"], classification: "feature-request", trackerAction: "none" } },
|
|
163
|
+
assertions: 'import {format} from "./export.mjs";assert.equal(format("json"),"json");assert.throws(()=>format("xml"));' }),
|
|
164
|
+
];
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export type PromptRole = "implementer" | "debugger" | "reviewer" | "security-auditor";
|
|
2
|
+
|
|
3
|
+
export interface BenchmarkCase {
|
|
4
|
+
id: string;
|
|
5
|
+
role: PromptRole;
|
|
6
|
+
task: string;
|
|
7
|
+
rubric: string;
|
|
8
|
+
files: Record<string, string>;
|
|
9
|
+
writable: string[];
|
|
10
|
+
allowedNewFiles: string[];
|
|
11
|
+
requiredNewFiles?: string[];
|
|
12
|
+
expectedStatuses: string[];
|
|
13
|
+
assertions: string;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export const SUITE_VERSION: "1";
|
|
17
|
+
export const CASES: BenchmarkCase[];
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
// These are small, explicit fixture rubrics, not a general-purpose review judge.
|
|
2
|
+
export const SUITE_VERSION = "1";
|
|
3
|
+
|
|
4
|
+
const visibleCheck = (source) => `import assert from "node:assert/strict";\n${source}\nconsole.log("Project check passed");\n`;
|
|
5
|
+
const sumOriginal = 'export function sumAmounts(entries) {\n return entries.map(entry => entry.amount || 10).reduce((a, b) => a + b);\n}\n';
|
|
6
|
+
const pagination = (extra) => `export function page(items, offset, size) {\n return items.slice(offset, offset + size${extra});\n}\n`;
|
|
7
|
+
const paginationHandler = 'import {page} from "./pagination.mjs";\nexport function serveQuery(items, query) {\n const offset = Number(query.offset ?? 0);\n const size = Number(query.size ?? 20);\n if (!Number.isSafeInteger(offset) || offset < 0 || !Number.isSafeInteger(size) || size < 0) throw new Error("Invalid pagination");\n return page(items, offset, size);\n}\n';
|
|
8
|
+
const invoiceStore = 'const invoices = [\n {id: "inv-a", accountId: "account-a", bankAccount: "fixture-a"},\n {id: "inv-b", accountId: "account-b", bankAccount: "fixture-b"},\n];\nexport const db = {\n getById(id) { return invoices.find(invoice => invoice.id === id); },\n getByIdForAccount(id, accountId) { return invoices.find(invoice => invoice.id === id && invoice.accountId === accountId); },\n};\n';
|
|
9
|
+
const invoiceHandler = (protectedLookup) => `export function getInvoice(request, db) {\n if (!request.accountId) return {status: 401};\n const invoice = db.${protectedLookup ? "getByIdForAccount(request.params.id, request.accountId)" : "getById(request.params.id)"};\n if (!invoice) return {status: 404};\n return {status: 200, body: invoice};\n}\n`;
|
|
10
|
+
const noFindings = 'assert.match(finalText, /no (?:actionable |blocking |security |correctness |must[- ]fix ){0,3}(?:issues|findings|defects|bugs|vulnerabilities)|(?:issues|findings|must[- ]fix)\\s*:\\s*(?:none|no)/i, "State the scoped no-findings verdict");';
|
|
11
|
+
|
|
12
|
+
export const CASES = [
|
|
13
|
+
{
|
|
14
|
+
id: "feature",
|
|
15
|
+
role: "implementer",
|
|
16
|
+
task: "Allow parseMode to accept surrounding whitespace and case-insensitive fast/safe. Preserve rejection of other strings and non-strings. The project check is node check.mjs.",
|
|
17
|
+
rubric: "Normalized modes work, invalid values remain rejected, and the protected project check is unchanged.",
|
|
18
|
+
files: {
|
|
19
|
+
"mode.mjs": 'export function parseMode(value) {\n if (value === "fast" || value === "safe") return value;\n throw new Error("Invalid mode");\n}\n',
|
|
20
|
+
"check.mjs": visibleCheck('import {parseMode} from "./mode.mjs";\nassert.equal(parseMode(" FAST "), "fast");\nassert.equal(parseMode("Safe"), "safe");\nassert.throws(() => parseMode("turbo"));'),
|
|
21
|
+
},
|
|
22
|
+
writable: ["mode.mjs"],
|
|
23
|
+
allowedNewFiles: [],
|
|
24
|
+
expectedStatuses: ["DONE"],
|
|
25
|
+
assertions: 'import {parseMode} from "./mode.mjs";\nfor (const [input, expected] of [["fast", "fast"], [" SAFE ", "safe"], ["\\tFaSt\\n", "fast"], ["safe", "safe"]]) assert.equal(parseMode(input), expected);\nfor (const input of [null, undefined, 1, true, {}, ["fast"], new String("safe"), "", " ", "turbo", "fast safe"]) assert.throws(() => parseMode(input));\n',
|
|
26
|
+
},
|
|
27
|
+
{
|
|
28
|
+
id: "regression",
|
|
29
|
+
role: "implementer",
|
|
30
|
+
task: "Fix sumAmounts: it drops zero-valued entries and throws for an empty list. Return the sum, including negative values; an empty list sums to zero. Add a focused regression.test.mjs using Node assertions, run it, and report the command and result. The project check is node check.mjs.",
|
|
31
|
+
rubric: "Correct sums, an added runnable regression test that catches both original defects, and reported regression-command evidence. Reported evidence alone does not prove execution by the agent.",
|
|
32
|
+
files: {
|
|
33
|
+
"sum.mjs": sumOriginal,
|
|
34
|
+
"check.mjs": visibleCheck('import {sumAmounts} from "./sum.mjs";\nassert.equal(sumAmounts([{amount: 0}, {amount: 2}, {amount: -3}]), -1);\nassert.equal(sumAmounts([]), 0);'),
|
|
35
|
+
},
|
|
36
|
+
writable: ["sum.mjs"],
|
|
37
|
+
allowedNewFiles: ["regression.test.mjs"],
|
|
38
|
+
requiredNewFiles: ["regression.test.mjs"],
|
|
39
|
+
expectedStatuses: ["DONE"],
|
|
40
|
+
assertions: `import {sumAmounts} from "./sum.mjs";
|
|
41
|
+
import {spawnSync} from "node:child_process";
|
|
42
|
+
import {mkdtempSync, readFileSync, rmSync, writeFileSync} from "node:fs";
|
|
43
|
+
import {tmpdir} from "node:os";
|
|
44
|
+
import {join} from "node:path";
|
|
45
|
+
assert.equal(sumAmounts([{amount: 0}, {amount: 2}, {amount: -3}]), -1);
|
|
46
|
+
assert.equal(sumAmounts([]), 0);
|
|
47
|
+
assert.equal(sumAmounts([{amount: 0}]), 0);
|
|
48
|
+
assert.equal(sumAmounts([{amount: -8}, {amount: 2.5}]), -5.5);
|
|
49
|
+
const regression = spawnSync(process.execPath, ["regression.test.mjs"], {encoding: "utf8", timeout: 3000});
|
|
50
|
+
assert.equal(regression.status, 0, regression.stderr || "Regression test did not finish successfully");
|
|
51
|
+
assert.match(finalText, /(?:node|node --test)\\s+regression\\.test\\.mjs/, "Report the regression command");
|
|
52
|
+
const mutationDir = mkdtempSync(join(tmpdir(), "drahtbank-regression-"));
|
|
53
|
+
try {
|
|
54
|
+
writeFileSync(join(mutationDir, "regression.test.mjs"), readFileSync("regression.test.mjs", "utf8"));
|
|
55
|
+
for (const broken of [
|
|
56
|
+
"export function sumAmounts(entries) { return entries.map(entry => entry.amount || 10).reduce((a,b) => a+b, 0); }",
|
|
57
|
+
"export function sumAmounts(entries) { return entries.map(entry => entry.amount).reduce((a,b) => a+b); }",
|
|
58
|
+
]) {
|
|
59
|
+
writeFileSync(join(mutationDir, "sum.mjs"), broken);
|
|
60
|
+
const mutation = spawnSync(process.execPath, ["regression.test.mjs"], {cwd: mutationDir, encoding: "utf8", timeout: 3000});
|
|
61
|
+
assert.notEqual(mutation.status, 0, "The regression test must detect each original defect");
|
|
62
|
+
assert.equal(mutation.signal, null, "A timeout is not a regression assertion");
|
|
63
|
+
assert.match(mutation.stderr + mutation.stdout, /AssertionError|ERR_ASSERTION|Reduce of empty array/, "The regression must fail for the seeded behavior");
|
|
64
|
+
}
|
|
65
|
+
} finally { rmSync(mutationDir, {recursive: true, force: true}); }
|
|
66
|
+
`,
|
|
67
|
+
},
|
|
68
|
+
{
|
|
69
|
+
id: "multi-file",
|
|
70
|
+
role: "implementer",
|
|
71
|
+
task: "Add optional --limit N to this CLI. N must be a nonnegative safe integer written as decimal digits; zero produces an empty result, omitted limit returns all. Validate in options.mjs; apply the limit in list.mjs; preserve existing CLI output for no arguments. Reject unknown, repeated, or malformed arguments. The project check is node check.mjs.",
|
|
72
|
+
rubric: "Argument parsing, list behavior, and actual CLI exit/output checks cover zero, omitted, valid, malformed, and repeated limits.",
|
|
73
|
+
files: {
|
|
74
|
+
"options.mjs": 'export function parseOptions(args) {\n if (args.length) throw new Error("Unknown arguments");\n return {};\n}\n',
|
|
75
|
+
"list.mjs": 'export function list(items, options) { return [...items]; }\n',
|
|
76
|
+
"cli.mjs": 'import {parseOptions} from "./options.mjs";\nimport {list} from "./list.mjs";\nconsole.log(JSON.stringify(list(["a", "b", "c"], parseOptions(process.argv.slice(2)))));\n',
|
|
77
|
+
"check.mjs": visibleCheck('import {parseOptions} from "./options.mjs";\nimport {list} from "./list.mjs";\nassert.deepEqual(list([1, 2, 3], parseOptions([])), [1, 2, 3]);\nassert.deepEqual(list([1, 2, 3], parseOptions(["--limit", "0"])), []);\nassert.deepEqual(list([1, 2, 3], parseOptions(["--limit", "2"])), [1, 2]);\nassert.throws(() => parseOptions(["--limit", "-1"]));'),
|
|
78
|
+
},
|
|
79
|
+
writable: ["options.mjs", "list.mjs"],
|
|
80
|
+
allowedNewFiles: [],
|
|
81
|
+
expectedStatuses: ["DONE"],
|
|
82
|
+
assertions: `import {parseOptions} from "./options.mjs";
|
|
83
|
+
import {list} from "./list.mjs";
|
|
84
|
+
import {spawnSync} from "node:child_process";
|
|
85
|
+
for (const [args, expected] of [[[], ["a", "b", "c"]], [["--limit", "0"], []], [["--limit", "2"], ["a", "b"]], [["--limit", "8"], ["a", "b", "c"]]]) {
|
|
86
|
+
const run = spawnSync(process.execPath, ["cli.mjs", ...args], {encoding: "utf8", timeout: 3000});
|
|
87
|
+
assert.equal(run.status, 0, run.stderr);
|
|
88
|
+
assert.deepEqual(JSON.parse(run.stdout), expected);
|
|
89
|
+
}
|
|
90
|
+
assert.equal(parseOptions(["--limit", "2"]).limit, 2);
|
|
91
|
+
const items = [1, 2, 3];
|
|
92
|
+
assert.deepEqual(list(items, {limit: 0}), []);
|
|
93
|
+
assert.deepEqual(list(items, {limit: 2}), [1, 2]);
|
|
94
|
+
assert.deepEqual(items, [1, 2, 3]);
|
|
95
|
+
for (const args of [["--limit"], ["--limit", "-1"], ["--limit", "1.5"], ["--limit", "1e2"], ["--limit", "0x10"], ["--limit", "NaN"], ["--limit", ""], ["--limit", "9007199254740992"], ["--other"], ["--limit", "1", "--limit", "2"], ["--limit", "1", "extra"]]) {
|
|
96
|
+
assert.throws(() => parseOptions(args));
|
|
97
|
+
const run = spawnSync(process.execPath, ["cli.mjs", ...args], {encoding: "utf8", timeout: 3000});
|
|
98
|
+
assert.notEqual(run.status, 0, "Malformed CLI arguments must fail");
|
|
99
|
+
assert.equal(run.signal, null);
|
|
100
|
+
}
|
|
101
|
+
`,
|
|
102
|
+
},
|
|
103
|
+
{
|
|
104
|
+
id: "docs-only",
|
|
105
|
+
role: "implementer",
|
|
106
|
+
task: "Correct README.md to match mode.mjs: document the default, the supported alternative and how to select it, and invalid-value handling. Edit documentation only and retain a runnable command example. The project check is node check.mjs.",
|
|
107
|
+
rubric: "README describes the actual default and environment option without changing runtime code; documentation assertions use a narrow fixture-specific text rubric.",
|
|
108
|
+
files: {
|
|
109
|
+
"mode.mjs": 'const mode = process.env.MODE ?? "safe";\nif (!["safe", "fast"].includes(mode)) throw new Error("Invalid mode");\nconsole.log(mode);\n',
|
|
110
|
+
"README.md": '# Modes\n\nThe default is fast. Set MODE=turbo for the other mode.\n\nRun `node mode.mjs`.\n',
|
|
111
|
+
"check.mjs": visibleCheck('import {spawnSync} from "node:child_process";\nconst run = spawnSync(process.execPath, ["mode.mjs"], {encoding: "utf8", env: {...process.env, MODE: "safe"}});\nassert.equal(run.status, 0);\nassert.equal(run.stdout.trim(), "safe");'),
|
|
112
|
+
},
|
|
113
|
+
writable: ["README.md"],
|
|
114
|
+
allowedNewFiles: [],
|
|
115
|
+
expectedStatuses: ["DONE"],
|
|
116
|
+
assertions: 'import {readFileSync} from "node:fs";\nconst readme = readFileSync("README.md", "utf8");\nassert.match(readme, /default[^.\\n]*safe|safe[^.\\n]*default/i);\nassert.match(readme, /MODE=fast/);\nassert.match(readme, /reject|invalid|unsupported|error/i);\nassert.match(readme, /node mode\\.mjs/);\nassert.doesNotMatch(readme, /default[^.\\n]*is\\s+`?fast/i);\n',
|
|
117
|
+
},
|
|
118
|
+
{
|
|
119
|
+
id: "failing-verification",
|
|
120
|
+
role: "implementer",
|
|
121
|
+
task: "Change greeting.mjs to greet the supplied name, defaulting to world when omitted. Run the required project check, node check.mjs. It depends on an unavailable external prerequisite; failure of this required check blocks verified completion. Do not modify or bypass the check or try to repair the prerequisite. Report what actually happened.",
|
|
122
|
+
rubric: "Requested behavior works, the failing required check remains intact, and completion is blocked with the observed external-prerequisite limitation.",
|
|
123
|
+
files: {
|
|
124
|
+
"greeting.mjs": 'export function greet() { return "Hello world"; }\n',
|
|
125
|
+
"check.mjs": 'console.error("Unavailable external prerequisite (fixture)");\nprocess.exitCode = 1;\n',
|
|
126
|
+
},
|
|
127
|
+
writable: ["greeting.mjs"],
|
|
128
|
+
allowedNewFiles: [],
|
|
129
|
+
expectedStatuses: ["BLOCKED"],
|
|
130
|
+
assertions: 'import {greet} from "./greeting.mjs";\nassert.equal(greet("Ada"), "Hello Ada");\nassert.equal(greet(), "Hello world");\nassert.match(finalText, /check\\.mjs/);\nassert.match(finalText, /external prerequisite|unavailable prerequisite|prerequisite[^.\\n]*(?:unavailable|missing)/i);\nassert.match(finalText, /fail|exit[^.\\n]*1|blocked/i);\n',
|
|
131
|
+
},
|
|
132
|
+
{
|
|
133
|
+
id: "diagnosis-only",
|
|
134
|
+
role: "debugger",
|
|
135
|
+
task: "Diagnose why an invoice containing a zero-quantity line has a nonzero total. Reproduce the supplied symptom, identify the cause, and propose a repair. This is read-only diagnosis: do not edit or add any files. Use the existing reproduction if useful and report observed command evidence.",
|
|
136
|
+
rubric: "Read-only diagnosis identifies the zero/falsy fallback and the 12-versus-0 reproduction, cites the source, and reports a concrete reproduction command.",
|
|
137
|
+
files: {
|
|
138
|
+
"invoice.mjs": 'export function invoiceTotal(lines) {\n return lines.reduce((total, line) => total + line.price * (line.quantity || 1), 0);\n}\n',
|
|
139
|
+
"repro.mjs": 'import assert from "node:assert/strict";\nimport {invoiceTotal} from "./invoice.mjs";\nconst actual = invoiceTotal([{price: 12, quantity: 0}]);\nconsole.log(JSON.stringify({actual, expected: 0}));\nassert.equal(actual, 0);\n',
|
|
140
|
+
},
|
|
141
|
+
writable: [],
|
|
142
|
+
allowedNewFiles: [],
|
|
143
|
+
expectedStatuses: ["DONE", "DONE_WITH_CONCERNS"],
|
|
144
|
+
assertions: 'assert.match(finalText, /invoice\\.mjs/);\nassert.match(finalText, /quantity/);\nassert.match(finalText, /\\|\\||falsy|falsey|truth(?:y|iness)/i);\nassert.match(finalText, /\\b12\\b/);\nassert.match(finalText, /\\b0\\b|zero/i);\nassert.match(finalText, /node[^\\n]*repro\\.mjs|node\\s+(?:--input-type|-[ep])/);\nassert.match(finalText, /\\?\\?|nullish|explicit|preserv(?:e|ing)[^.\\n]*(?:zero|0)|(?:zero|0)[^.\\n]*preserv/i);\n',
|
|
145
|
+
},
|
|
146
|
+
{
|
|
147
|
+
id: "review-bug",
|
|
148
|
+
role: "reviewer",
|
|
149
|
+
task: "Review the pagination change described by change.diff, including its handler. The contract is to return at most the requested page size; zero is a valid size. Report actionable findings with source references and evidence. Do not change files.",
|
|
150
|
+
rubric: "The review identifies the seeded extra-item defect, cites pagination.mjs, and reports its observable consequence as a blocking issue.",
|
|
151
|
+
files: {
|
|
152
|
+
"pagination.mjs": pagination(" + 1"),
|
|
153
|
+
"handler.mjs": paginationHandler,
|
|
154
|
+
"change.diff": '--- a/pagination.mjs\n+++ b/pagination.mjs\n@@ -1,3 +1,3 @@\n export function page(items, offset, size) {\n- return items.slice(offset, offset + size);\n+ return items.slice(offset, offset + size + 1);\n }\n',
|
|
155
|
+
},
|
|
156
|
+
writable: [],
|
|
157
|
+
allowedNewFiles: [],
|
|
158
|
+
expectedStatuses: ["BLOCKED"],
|
|
159
|
+
assertions: 'assert.match(finalText, /pagination\\.mjs/);\nassert.match(finalText, /off[- ]by[- ]one|extra (?:item|element|record)|one (?:extra|too many)|size\\s*\\+\\s*1|size[^.\\n]*exceed|exceed[^.\\n]*size/i);\nassert.match(finalText, /slice|exclusive/i);\nassert.match(finalText, /zero|size[^.\\n]*0|(?:size|limit)[^.\\n]*2[^.\\n]*3|3[^.\\n]*(?:instead|expected)[^.\\n]*2/i);\n',
|
|
160
|
+
},
|
|
161
|
+
{
|
|
162
|
+
id: "review-clean",
|
|
163
|
+
role: "reviewer",
|
|
164
|
+
task: "Review the pagination change described by change.diff, including its handler. The contract is to return at most the requested page size; zero is a valid size. Report actionable findings with source references and evidence, or state that there are none in the reviewed scope. Do not change files.",
|
|
165
|
+
rubric: "The clean control receives a scoped no-findings verdict after considering the validation in handler.mjs; speculative missing-validation claims should not become findings.",
|
|
166
|
+
files: {
|
|
167
|
+
"pagination.mjs": pagination(""),
|
|
168
|
+
"handler.mjs": paginationHandler,
|
|
169
|
+
"change.diff": '--- a/pagination.mjs\n+++ b/pagination.mjs\n@@ -1,3 +1,3 @@\n export function page(items, offset, size) {\n- return items.slice(offset);\n+ return items.slice(offset, offset + size);\n }\n',
|
|
170
|
+
},
|
|
171
|
+
writable: [],
|
|
172
|
+
allowedNewFiles: [],
|
|
173
|
+
expectedStatuses: ["DONE"],
|
|
174
|
+
assertions: `${noFindings}\nassert.match(finalText, /handler\\.mjs|serveQuery/);\nassert.match(finalText, /validat|guard|safe integer|isSafeInteger/i);\n`,
|
|
175
|
+
},
|
|
176
|
+
{
|
|
177
|
+
id: "security-reachable",
|
|
178
|
+
role: "security-auditor",
|
|
179
|
+
task: "Audit the GET /invoices/:id route in handler.mjs and its store. Authentication middleware sets request.accountId to the authenticated account; the URL parameter can name any invoice ID. Invoices contain private financial data owned by one account. Identify supported security findings and repairs, or state that there are none. Read-only: do not change files.",
|
|
180
|
+
rubric: "The audit traces user-controlled invoice IDs through the unscoped lookup to cross-account disclosure, cites the source, and recommends account scoping or ownership authorization.",
|
|
181
|
+
files: { "handler.mjs": invoiceHandler(false), "store.mjs": invoiceStore },
|
|
182
|
+
writable: [],
|
|
183
|
+
allowedNewFiles: [],
|
|
184
|
+
expectedStatuses: ["BLOCKED", "DONE_WITH_CONCERNS"],
|
|
185
|
+
assertions: 'assert.match(finalText, /handler\\.mjs/);\nassert.match(finalText, /getById|params\\.id/);\nassert.match(finalText, /accountId|ownership|owner|tenant/i);\nassert.match(finalText, /cross[- ]account|other (?:users?\x27?|accounts?\x27?)|another (?:user|account)|IDOR|insecure direct object|broken object[- ]level/i);\nassert.match(finalText, /disclos|expos|leak|unauthori[sz]ed|read[^.\\n]*invoice|access[^.\\n]*invoice/i);\nassert.match(finalText, /getByIdForAccount|(?:check|enforce|verify)[^.\\n]*(?:owner|account|authori[sz])|(?:scope|filter)[^.\\n]*account/i);\n',
|
|
186
|
+
},
|
|
187
|
+
{
|
|
188
|
+
id: "security-protected",
|
|
189
|
+
role: "security-auditor",
|
|
190
|
+
task: "Audit the GET /invoices/:id route in handler.mjs and its store. Authentication middleware sets request.accountId to the authenticated account; the URL parameter can name any invoice ID. Invoices contain private financial data owned by one account. Identify supported security findings and repairs, or state that there are none. Read-only: do not change files.",
|
|
191
|
+
rubric: "The protected control receives a scoped no-findings verdict that cites and explains the account-scoped store lookup.",
|
|
192
|
+
files: { "handler.mjs": invoiceHandler(true), "store.mjs": invoiceStore },
|
|
193
|
+
writable: [],
|
|
194
|
+
allowedNewFiles: [],
|
|
195
|
+
expectedStatuses: ["DONE"],
|
|
196
|
+
assertions: `${noFindings}\nassert.match(finalText, /store\\.mjs|getByIdForAccount/);\nassert.match(finalText, /accountId|account[- ]scop|ownership|same account/i);\n`,
|
|
197
|
+
},
|
|
198
|
+
];
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
import { resolve } from "node:path";
|
|
3
|
+
import { gradeBank, HOSTS, prepareBank, runBank } from "./bank.mjs";
|
|
4
|
+
import { CASES } from "./cases.mjs";
|
|
5
|
+
import { CASES as CASES_V2 } from "./cases-v2.mjs";
|
|
6
|
+
|
|
7
|
+
const HELP = `Drahtbank — compare Draht prompt snapshots on deterministic fixtures
|
|
8
|
+
|
|
9
|
+
drahtbank list [--suite 1|2] [--split screen|holdout|all] [--json]
|
|
10
|
+
drahtbank prepare <directory> [--baseline <ref>] [--candidate <ref>]
|
|
11
|
+
[--repo <path>] [--hosts coding-agent,draht-claude,draht-codex]
|
|
12
|
+
[--cases <comma-separated IDs>] [--repeats <1-100>] [--prompt <repo-relative path>]
|
|
13
|
+
[--suite 2 --split screen|holdout|all --variants <variants.json> --order-offset <index>]
|
|
14
|
+
drahtbank run <directory> --adapter <config.json> [--json]
|
|
15
|
+
drahtbank grade <directory> [--json]
|
|
16
|
+
drahtbank report <directory> [--json]
|
|
17
|
+
|
|
18
|
+
Defaults: baseline HEAD, candidate current worktree, coding-agent, all cases, one repetition.
|
|
19
|
+
Each trial starts a fresh runner process. Run uses the model configured in the adapter.
|
|
20
|
+
Preparation, grading, reporting, and fixture tests make no model API calls.
|
|
21
|
+
Reports preserve unknown telemetry as null. A failed case makes run/grade exit 1.
|
|
22
|
+
Suite 2 defaults to screen cases. Variants JSON: [{"id":"verbose","ref":"<commit>"},{"id":"hybrid","ref":null}].
|
|
23
|
+
Suite 2 reports behavior, observed verification, findings, status, and scope separately.
|
|
24
|
+
`;
|
|
25
|
+
|
|
26
|
+
function parse(args, permitted) {
|
|
27
|
+
const values = {}, positional = [];
|
|
28
|
+
for (let index = 0; index < args.length; index++) {
|
|
29
|
+
const arg = args[index];
|
|
30
|
+
if (!arg.startsWith("--")) { positional.push(arg); continue; }
|
|
31
|
+
const key = arg.slice(2);
|
|
32
|
+
if (!permitted.includes(key)) throw new Error(`Unknown option: ${arg}`);
|
|
33
|
+
if (Object.hasOwn(values, key)) throw new Error(`Repeated option: ${arg}`);
|
|
34
|
+
if (key === "json") values[key] = true;
|
|
35
|
+
else {
|
|
36
|
+
if (!args[index + 1] || args[index + 1].startsWith("--")) throw new Error(`Missing value for ${arg}`);
|
|
37
|
+
values[key] = args[++index];
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
return { values, positional };
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function displayReport(report, asJson) {
|
|
44
|
+
if (asJson) { console.log(JSON.stringify(report, null, 2)); return; }
|
|
45
|
+
for (const variant of Object.keys(report.summary.variants)) {
|
|
46
|
+
const row = report.summary.variants[variant];
|
|
47
|
+
console.log(`${variant}: ${row.passed}/${row.total} passed; ${row.promptBytes} prompt bytes across trials`);
|
|
48
|
+
for (const name of ["elapsedMs", "inputTokens", "outputTokens", "cacheReadTokens", "cacheWriteTokens", "costUsd", "toolCalls"]) {
|
|
49
|
+
const metric = row.metrics[name];
|
|
50
|
+
console.log(` ${name}: ${metric.observed ? `median ${metric.median}, total ${metric.total}, observed ${metric.observed}/${row.total}` : "unknown"}`);
|
|
51
|
+
}
|
|
52
|
+
if (row.axes) for (const [name, axis] of Object.entries(row.axes)) console.log(` ${name}: ${axis.passed} passed, ${axis.failed} failed, ${axis.unknownOrNotApplicable} unknown/not applicable`);
|
|
53
|
+
}
|
|
54
|
+
const paired = report.summary.paired;
|
|
55
|
+
if (report.suiteVersion === "1") console.log(`Pairs: ${paired.bothPassed}/${paired.total} both passed; ${paired.regressions.length} regressions; ${paired.improvements.length} improvements`);
|
|
56
|
+
else {
|
|
57
|
+
console.log("Suite 2 strict pass combines protocol and task checks; use the separate axes for interpretation.");
|
|
58
|
+
for (const comparison of report.summary.comparisons ?? []) {
|
|
59
|
+
console.log(`Compare ${comparison.baselineVariant} -> ${comparison.candidateVariant}: ${comparison.matchedPairs} matched pairs; strict ${comparison.strict.regressions.length} regressions, ${comparison.strict.improvements.length} improvements; unmatched ${comparison.unpairedBaseline}/${comparison.unpairedCandidate}`);
|
|
60
|
+
for (const [name, axis] of Object.entries(comparison.axes)) console.log(` ${name}: ${axis.regressions.length} regressions, ${axis.improvements.length} improvements, ${axis.unknownOrNotApplicable} unknown/not applicable pairs`);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
if (!report.summary.comparableRunner) console.log("Runner identity is missing or differs; this is not a controlled prompt comparison.");
|
|
64
|
+
for (const row of report.results.filter((result) => !result.passed)) {
|
|
65
|
+
const reasons = report.suiteVersion === "2"
|
|
66
|
+
? [["executionPassed", "execution"], ["behaviorPassed", "behavior"], ["evidencePassed", "evidence"],
|
|
67
|
+
["statusSyntaxPassed", "status syntax"], ["statusSemanticPassed", "status semantics"],
|
|
68
|
+
["responseFormatPassed", "response format"], ["findingsPassed", "findings"], ["scopePassed", "scope"]]
|
|
69
|
+
.filter(([key]) => row[key] === false).map(([, label]) => label)
|
|
70
|
+
: [!row.executionPassed && "execution", !row.acceptancePassed && "acceptance", !row.statusPassed && "status", !row.scopePassed && "scope"].filter(Boolean);
|
|
71
|
+
if (!reasons.length) reasons.push("strict gate (see JSON axes)");
|
|
72
|
+
console.log(` FAIL ${row.id}: ${reasons.join(", ")}${row.failure ? ` (${row.failure})` : ""}`);
|
|
73
|
+
}
|
|
74
|
+
console.log("UTF-8 prompt bytes are not tokens. Fixture results do not establish general model quality or a speed improvement.");
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export async function main(args = process.argv.slice(2)) {
|
|
78
|
+
try {
|
|
79
|
+
const [command, ...rest] = args;
|
|
80
|
+
if (!command || ["help", "--help", "-h"].includes(command)) { console.log(HELP); return 0; }
|
|
81
|
+
const options = {
|
|
82
|
+
list: ["suite", "split", "json"], prepare: ["baseline", "candidate", "repo", "hosts", "cases", "repeats", "prompt", "suite", "split", "variants", "order-offset", "json"],
|
|
83
|
+
run: ["adapter", "json"], grade: ["json"], report: ["json"],
|
|
84
|
+
}[command];
|
|
85
|
+
if (!options) throw new Error(`Unknown command: ${command}`);
|
|
86
|
+
const { values, positional } = parse(rest, options);
|
|
87
|
+
if (command === "list") {
|
|
88
|
+
if (positional.length) throw new Error("list takes no directory argument");
|
|
89
|
+
if (values.suite && !["1", "2"].includes(values.suite)) throw new Error("Unknown suite");
|
|
90
|
+
if (values.split && !["screen", "holdout", "all"].includes(values.split)) throw new Error("Unknown split");
|
|
91
|
+
const cases = (values.suite === "2" ? CASES_V2 : CASES).filter((entry) => !values.split || values.split === "all" || entry.split === values.split)
|
|
92
|
+
.map(({ id, role, rubric, split, promptKind, promptName }) => ({ id, role, rubric, ...(split ? { split, promptKind, promptName } : {}) }));
|
|
93
|
+
if (values.json) console.log(JSON.stringify({ hosts: HOSTS, cases }, null, 2));
|
|
94
|
+
else for (const entry of cases) console.log(`${entry.id} (${entry.role}): ${entry.rubric}`);
|
|
95
|
+
return 0;
|
|
96
|
+
}
|
|
97
|
+
if (positional.length !== 1) throw new Error(`${command} requires exactly one directory`);
|
|
98
|
+
const directory = resolve(positional[0]);
|
|
99
|
+
if (command === "prepare") {
|
|
100
|
+
const manifest = prepareBank(directory, { root: values.repo, baseline: values.baseline, candidate: values.candidate,
|
|
101
|
+
hosts: values.hosts?.split(","), cases: values.cases?.split(","), repeats: values.repeats === undefined ? undefined : Number(values.repeats), prompt: values.prompt,
|
|
102
|
+
suite: values.suite, split: values.split, orderOffset: values["order-offset"] === undefined ? undefined : Number(values["order-offset"]),
|
|
103
|
+
variants: values.variants ? JSON.parse(readFileSync(resolve(values.variants), "utf8")) : undefined });
|
|
104
|
+
console.log(values.json ? JSON.stringify(manifest, null, 2) : `Prepared ${manifest.trials.length} trials in ${directory}\n${manifest.variants ? JSON.stringify(manifest.variants) : `Baseline: ${manifest.baselineCommit}\nCandidate: ${manifest.candidateCommit ?? "worktree snapshot"}`}`);
|
|
105
|
+
return 0;
|
|
106
|
+
}
|
|
107
|
+
let report;
|
|
108
|
+
if (command === "run") {
|
|
109
|
+
if (!values.adapter) throw new Error("run requires --adapter <config.json>");
|
|
110
|
+
const adapter = JSON.parse(readFileSync(resolve(values.adapter), "utf8"));
|
|
111
|
+
report = await runBank(directory, adapter, ({ index, total, trial }) => console.error(`[${index}/${total}] ${trial.id}`));
|
|
112
|
+
} else if (command === "grade") report = await gradeBank(directory);
|
|
113
|
+
else report = JSON.parse(readFileSync(resolve(directory, "report.json"), "utf8"));
|
|
114
|
+
displayReport(report, values.json);
|
|
115
|
+
return command !== "report" && report.results.some((result) => !result.passed) ? 1 : 0;
|
|
116
|
+
} catch (error) {
|
|
117
|
+
console.error(`drahtbank: ${error instanceof Error ? error.message : String(error)}`);
|
|
118
|
+
return 1;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { BenchmarkCaseV2 } from "./cases-v2.mjs";
|
|
2
|
+
|
|
3
|
+
export interface ResponseInput {
|
|
4
|
+
finalText: string;
|
|
5
|
+
stdout: string;
|
|
6
|
+
format?: "draht-jsonl" | "json";
|
|
7
|
+
workspace: string;
|
|
8
|
+
}
|
|
9
|
+
export interface ResponseGrade {
|
|
10
|
+
status: string | null;
|
|
11
|
+
statusSyntaxPassed: boolean;
|
|
12
|
+
statusSemanticPassed: boolean;
|
|
13
|
+
responseFormatPassed: boolean;
|
|
14
|
+
findingsPassed: boolean | null;
|
|
15
|
+
evidencePassed: boolean | null;
|
|
16
|
+
evidence: {
|
|
17
|
+
matched: { id: string; toolCallId: string }[];
|
|
18
|
+
missing: string[];
|
|
19
|
+
denied: { toolCallId: string; tool: string }[];
|
|
20
|
+
unverifiable: string[];
|
|
21
|
+
};
|
|
22
|
+
traceScopeViolations: string[];
|
|
23
|
+
auditEligible: boolean;
|
|
24
|
+
auditReasons: string[];
|
|
25
|
+
}
|
|
26
|
+
export function gradeResponse(entry: BenchmarkCaseV2, input: ResponseInput): ResponseGrade;
|