@draht/coding-agent 2026.10.20-1 → 2026.10.23-1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +138 -0
- package/agents/advisor.md +2 -2
- package/agents/architect.md +11 -33
- package/agents/debugger.md +13 -54
- package/agents/git-committer.md +18 -42
- package/agents/implementer.md +14 -36
- package/agents/investigator.md +12 -63
- package/agents/reviewer.md +16 -44
- package/agents/security-auditor.md +22 -56
- package/agents/spec-reviewer.md +13 -53
- package/agents/verifier.md +14 -33
- package/bin/drahtbank.mjs +4 -0
- package/dist/agents/advisor.md +2 -2
- package/dist/agents/architect.md +11 -33
- package/dist/agents/debugger.md +13 -54
- package/dist/agents/git-committer.md +18 -42
- package/dist/agents/implementer.md +14 -36
- package/dist/agents/investigator.md +12 -63
- package/dist/agents/reviewer.md +16 -44
- package/dist/agents/security-auditor.md +22 -56
- package/dist/agents/spec-reviewer.md +13 -53
- package/dist/agents/verifier.md +14 -33
- package/dist/bundle/chunks/anthropic-messages-G57VPNTF.js +6 -0
- package/dist/bundle/chunks/anthropic.js +1 -1
- package/dist/bundle/chunks/azure-openai-responses-AYUCZ5WN.js +2 -0
- package/dist/bundle/chunks/bedrock-converse-stream.js +23 -20
- package/dist/bundle/chunks/chunk-3GDEPGTR.js +16 -0
- package/dist/bundle/chunks/chunk-5JW3NSTV.js +2 -0
- package/dist/bundle/chunks/{chunk-FHX2S5IG.js → chunk-6WESOD4H.js} +1 -1
- package/dist/bundle/chunks/chunk-76YNMQ57.js +2 -0
- package/dist/bundle/chunks/chunk-AC3MSTC6.js +2 -0
- package/dist/bundle/chunks/chunk-ACLWH5WQ.js +2 -0
- package/dist/bundle/chunks/chunk-BL72B6LV.js +82 -0
- package/dist/bundle/chunks/{chunk-5QJFWTKA.js → chunk-CLKG6K6Z.js} +5 -2
- package/dist/bundle/chunks/chunk-DN7SM3LX.js +479 -0
- package/dist/bundle/chunks/{chunk-UTRX2VHB.js → chunk-DOUMZTXH.js} +3 -3
- package/dist/bundle/chunks/chunk-DQ4G4IS4.js +20 -0
- package/dist/bundle/chunks/chunk-E4VZJRJ2.js +9 -0
- package/dist/bundle/chunks/chunk-FOYIPAAD.js +6 -0
- package/dist/bundle/chunks/chunk-FXCTQRNN.js +2 -0
- package/dist/bundle/chunks/chunk-GQXRTK54.js +20 -0
- package/dist/bundle/chunks/chunk-IGO52ZOY.js +132 -0
- package/dist/bundle/chunks/{chunk-VXUJMEMO.js → chunk-IT4RIOZD.js} +1 -1
- package/dist/bundle/chunks/chunk-JJKPUMH2.js +1424 -0
- package/dist/bundle/chunks/chunk-JNWRE5F4.js +2 -0
- package/dist/bundle/chunks/chunk-KSGHCBHT.js +2 -0
- package/dist/bundle/chunks/chunk-MBFTIGLS.js +2 -0
- package/dist/bundle/chunks/chunk-N26RGV3K.js +11 -0
- package/dist/bundle/chunks/{chunk-KPO5375S.js → chunk-RCRGGYLW.js} +1 -1
- package/dist/bundle/chunks/chunk-UCJ7FLSO.js +166 -0
- package/dist/bundle/chunks/chunk-USCN7R2E.js +2 -0
- package/dist/bundle/chunks/chunk-WFHH5SX6.js +4 -0
- package/dist/bundle/chunks/chunk-WYWBXNDO.js +6 -0
- package/dist/bundle/chunks/chunk-ZRPAOMVB.js +2 -0
- package/dist/bundle/chunks/{cli-VCHSHNKT.js → cli-HTUB6WOG.js} +6 -6
- package/dist/bundle/chunks/cloudflare-workers-ai-system-one-RRSO6OXX.js +2 -0
- package/dist/bundle/chunks/codemode-worker.js +188 -21
- package/dist/bundle/chunks/execute-UUPNZJJG.js +31 -0
- package/dist/bundle/chunks/github-copilot.js +1 -1
- package/dist/bundle/chunks/google-generative-ai-VJSS5EBX.js +2 -0
- package/dist/bundle/chunks/google-vertex-K23DBWTT.js +2 -0
- package/dist/bundle/chunks/image-resize-worker.js +1 -1
- package/dist/bundle/chunks/{llama-cpp-classify-EJ5XMSQL.js → llama-cpp-classify-3VDXTYVJ.js} +3 -3
- package/dist/bundle/chunks/mistral-conversations-SKVQJGFZ.js +5 -0
- package/dist/bundle/chunks/node-DXZFQBEO.js +16 -0
- package/dist/bundle/chunks/{node-6IIDYB3K.js → node-LBZLUIBE.js} +1 -1
- package/dist/bundle/chunks/openai-chatgpt.js +1 -1
- package/dist/bundle/chunks/openai-codex-responses-LV4RIZL3.js +10 -0
- package/dist/bundle/chunks/openai-codex.js +1 -1
- package/dist/bundle/chunks/openai-completions-PUIR2V25.js +7 -0
- package/dist/bundle/chunks/openai-decisions-2DJLSYQF.js +5 -0
- package/dist/bundle/chunks/openai-responses-6ESQOXAF.js +4 -0
- package/dist/bundle/chunks/opencode-go.js +1 -1
- package/dist/bundle/chunks/openrouter-images-TVXBMY3V.js +2 -0
- package/dist/bundle/chunks/pi-messages-XS7FDWQI.js +8 -0
- package/dist/bundle/chunks/runtime-IB3XQ7E3.js +2 -0
- package/dist/bundle/chunks/runtime-LB35PK4J.js +3 -0
- package/dist/bundle/chunks/settings-manager-YNY5F4RU.js +2 -0
- package/dist/bundle/chunks/typesafe-system-one-67GAWZXE.js +2 -0
- package/dist/bundle/chunks/virtual-modules-U4CBGVMI.js +2 -0
- package/dist/bundle/cli-runtime.js +1 -1
- package/dist/bundle/index.js +1 -1
- package/dist/bundle/rpc-entry.js +1 -1
- package/dist/cli/args.d.ts +2 -1
- package/dist/cli/args.d.ts.map +1 -1
- package/dist/cli/args.js +30 -8
- package/dist/cli/args.js.map +1 -1
- package/dist/cli/credential-print.d.ts.map +1 -1
- package/dist/cli/credential-print.js +4 -4
- package/dist/cli/credential-print.js.map +1 -1
- package/dist/config.d.ts +18 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +44 -6
- package/dist/config.js.map +1 -1
- package/dist/core/agent-session.d.ts +65 -8
- package/dist/core/agent-session.d.ts.map +1 -1
- package/dist/core/agent-session.js +240 -102
- package/dist/core/agent-session.js.map +1 -1
- package/dist/core/auth-storage.d.ts.map +1 -1
- package/dist/core/auth-storage.js +20 -13
- package/dist/core/auth-storage.js.map +1 -1
- package/dist/core/bash-executor.d.ts.map +1 -1
- package/dist/core/bash-executor.js +42 -52
- package/dist/core/bash-executor.js.map +1 -1
- package/dist/core/bug-report.d.ts +9 -1
- package/dist/core/bug-report.d.ts.map +1 -1
- package/dist/core/builtins/completion-checks.d.ts +3 -0
- package/dist/core/builtins/completion-checks.d.ts.map +1 -0
- package/dist/core/builtins/completion-checks.js +142 -0
- package/dist/core/builtins/completion-checks.js.map +1 -0
- package/dist/core/builtins/duet.d.ts.map +1 -1
- package/dist/core/builtins/duet.js +7 -5
- package/dist/core/builtins/duet.js.map +1 -1
- package/dist/core/builtins/git-safety.d.ts +3 -0
- package/dist/core/builtins/git-safety.d.ts.map +1 -0
- package/dist/core/builtins/git-safety.js +34 -0
- package/dist/core/builtins/git-safety.js.map +1 -0
- package/dist/core/builtins/index.d.ts.map +1 -1
- package/dist/core/builtins/index.js +4 -0
- package/dist/core/builtins/index.js.map +1 -1
- package/dist/core/client-identity.d.ts +8 -0
- package/dist/core/client-identity.d.ts.map +1 -0
- package/dist/core/client-identity.js +15 -0
- package/dist/core/client-identity.js.map +1 -0
- package/dist/core/compaction/branch-summarization.d.ts.map +1 -1
- package/dist/core/compaction/branch-summarization.js +1 -2
- package/dist/core/compaction/branch-summarization.js.map +1 -1
- package/dist/core/compaction/compaction.d.ts +2 -2
- package/dist/core/compaction/compaction.d.ts.map +1 -1
- package/dist/core/compaction/compaction.js +20 -15
- package/dist/core/compaction/compaction.js.map +1 -1
- package/dist/core/compaction/utils.d.ts +6 -0
- package/dist/core/compaction/utils.d.ts.map +1 -1
- package/dist/core/compaction/utils.js +7 -8
- package/dist/core/compaction/utils.js.map +1 -1
- package/dist/core/export-html/index.d.ts.map +1 -1
- package/dist/core/export-html/index.js +2 -0
- package/dist/core/export-html/index.js.map +1 -1
- package/dist/core/export-html/template.html +1 -0
- package/dist/core/export-html/template.js +23 -17
- package/dist/core/export-html/tool-renderer.d.ts +3 -3
- package/dist/core/export-html/tool-renderer.d.ts.map +1 -1
- package/dist/core/export-html/tool-renderer.js +5 -3
- package/dist/core/export-html/tool-renderer.js.map +1 -1
- package/dist/core/export-html/vendor/marked-cjk-friendly.min.js +6 -0
- package/dist/core/export-html/vendor/marked.min.js +30 -29
- package/dist/core/extensions/index.d.ts +1 -1
- package/dist/core/extensions/index.d.ts.map +1 -1
- package/dist/core/extensions/index.js.map +1 -1
- package/dist/core/extensions/loader.d.ts.map +1 -1
- package/dist/core/extensions/loader.js +7 -0
- package/dist/core/extensions/loader.js.map +1 -1
- package/dist/core/extensions/runner.d.ts +6 -2
- package/dist/core/extensions/runner.d.ts.map +1 -1
- package/dist/core/extensions/runner.js +16 -4
- package/dist/core/extensions/runner.js.map +1 -1
- package/dist/core/extensions/types.d.ts +61 -7
- package/dist/core/extensions/types.d.ts.map +1 -1
- package/dist/core/extensions/types.js.map +1 -1
- package/dist/core/extensions/virtual-modules.d.ts.map +1 -1
- package/dist/core/extensions/virtual-modules.js +2 -0
- package/dist/core/extensions/virtual-modules.js.map +1 -1
- package/dist/core/key-id-schema.d.ts +6 -0
- package/dist/core/key-id-schema.d.ts.map +1 -0
- package/dist/core/key-id-schema.js +16 -0
- package/dist/core/key-id-schema.js.map +1 -0
- package/dist/core/keybindings-schema.d.ts +4 -0
- package/dist/core/keybindings-schema.d.ts.map +1 -0
- package/dist/core/keybindings-schema.js +16 -0
- package/dist/core/keybindings-schema.js.map +1 -0
- package/dist/core/keybindings.d.ts +242 -75
- package/dist/core/keybindings.d.ts.map +1 -1
- package/dist/core/keybindings.js +30 -23
- package/dist/core/keybindings.js.map +1 -1
- package/dist/core/mcp-servers.d.ts +22 -0
- package/dist/core/mcp-servers.d.ts.map +1 -1
- package/dist/core/mcp-servers.js +72 -11
- package/dist/core/mcp-servers.js.map +1 -1
- package/dist/core/model-config.d.ts +1623 -74
- package/dist/core/model-config.d.ts.map +1 -1
- package/dist/core/model-config.js +70 -184
- package/dist/core/model-config.js.map +1 -1
- package/dist/core/model-registry.d.ts +3 -1
- package/dist/core/model-registry.d.ts.map +1 -1
- package/dist/core/model-registry.js +4 -0
- package/dist/core/model-registry.js.map +1 -1
- package/dist/core/model-resolver.d.ts +56 -43
- package/dist/core/model-resolver.d.ts.map +1 -1
- package/dist/core/model-resolver.js +253 -358
- package/dist/core/model-resolver.js.map +1 -1
- package/dist/core/model-runtime.d.ts.map +1 -1
- package/dist/core/model-runtime.js +19 -10
- package/dist/core/model-runtime.js.map +1 -1
- package/dist/core/nested-tool-calls.d.ts +1 -0
- package/dist/core/nested-tool-calls.d.ts.map +1 -1
- package/dist/core/nested-tool-calls.js +1 -0
- package/dist/core/nested-tool-calls.js.map +1 -1
- package/dist/core/package-manager.js +3 -3
- package/dist/core/package-manager.js.map +1 -1
- package/dist/core/polyphase/keyword.js +1 -1
- package/dist/core/polyphase/keyword.js.map +1 -1
- package/dist/core/polyphase/model-selection.d.ts.map +1 -1
- package/dist/core/polyphase/model-selection.js +15 -7
- package/dist/core/polyphase/model-selection.js.map +1 -1
- package/dist/core/polyphase/workflow/prelude.d.ts +2 -0
- package/dist/core/polyphase/workflow/prelude.d.ts.map +1 -1
- package/dist/core/polyphase/workflow/prelude.js +12 -16
- package/dist/core/polyphase/workflow/prelude.js.map +1 -1
- package/dist/core/polyphase/workflow/runtime.d.ts.map +1 -1
- package/dist/core/polyphase/workflow/runtime.js +2 -1
- package/dist/core/polyphase/workflow/runtime.js.map +1 -1
- package/dist/core/project-trust.d.ts.map +1 -1
- package/dist/core/project-trust.js +2 -1
- package/dist/core/project-trust.js.map +1 -1
- package/dist/core/prompt-templates.d.ts.map +1 -1
- package/dist/core/prompt-templates.js +7 -2
- package/dist/core/prompt-templates.js.map +1 -1
- package/dist/core/provider-composer.d.ts +2 -1
- package/dist/core/provider-composer.d.ts.map +1 -1
- package/dist/core/provider-composer.js +13 -0
- package/dist/core/provider-composer.js.map +1 -1
- package/dist/core/radius.d.ts +2 -0
- package/dist/core/radius.d.ts.map +1 -1
- package/dist/core/radius.js +2 -0
- package/dist/core/radius.js.map +1 -1
- package/dist/core/remote-catalog-provider.d.ts +12 -1
- package/dist/core/remote-catalog-provider.d.ts.map +1 -1
- package/dist/core/remote-catalog-provider.js +84 -24
- package/dist/core/remote-catalog-provider.js.map +1 -1
- package/dist/core/resource-loader.d.ts +3 -0
- package/dist/core/resource-loader.d.ts.map +1 -1
- package/dist/core/resource-loader.js +7 -4
- package/dist/core/resource-loader.js.map +1 -1
- package/dist/core/sdk.d.ts +13 -4
- package/dist/core/sdk.d.ts.map +1 -1
- package/dist/core/sdk.js +27 -15
- package/dist/core/sdk.js.map +1 -1
- package/dist/core/settings-defaults.d.ts +65 -0
- package/dist/core/settings-defaults.d.ts.map +1 -0
- package/dist/core/settings-defaults.js +66 -0
- package/dist/core/settings-defaults.js.map +1 -0
- package/dist/core/settings-manager.d.ts +17 -178
- package/dist/core/settings-manager.d.ts.map +1 -1
- package/dist/core/settings-manager.js +86 -57
- package/dist/core/settings-manager.js.map +1 -1
- package/dist/core/settings-schema.d.ts +269 -0
- package/dist/core/settings-schema.d.ts.map +1 -0
- package/dist/core/settings-schema.js +318 -0
- package/dist/core/settings-schema.js.map +1 -0
- package/dist/core/skills.d.ts +4 -1
- package/dist/core/skills.d.ts.map +1 -1
- package/dist/core/skills.js +6 -1
- package/dist/core/skills.js.map +1 -1
- package/dist/core/system-prompt.d.ts +6 -0
- package/dist/core/system-prompt.d.ts.map +1 -1
- package/dist/core/system-prompt.js +10 -5
- package/dist/core/system-prompt.js.map +1 -1
- package/dist/core/tools/bash.d.ts.map +1 -1
- package/dist/core/tools/bash.js +3 -5
- package/dist/core/tools/bash.js.map +1 -1
- package/dist/core/tools/grep.d.ts +8 -0
- package/dist/core/tools/grep.d.ts.map +1 -1
- package/dist/core/tools/grep.js +19 -7
- package/dist/core/tools/grep.js.map +1 -1
- package/dist/core/tools/index.d.ts +2 -0
- package/dist/core/tools/index.d.ts.map +1 -1
- package/dist/core/tools/index.js +2 -0
- package/dist/core/tools/index.js.map +1 -1
- package/dist/core/tools/output-accumulator.d.ts.map +1 -1
- package/dist/core/tools/output-accumulator.js +4 -10
- package/dist/core/tools/output-accumulator.js.map +1 -1
- package/dist/core/tools/path-utils.d.ts +1 -1
- package/dist/core/tools/path-utils.d.ts.map +1 -1
- package/dist/core/tools/path-utils.js +6 -6
- package/dist/core/tools/path-utils.js.map +1 -1
- package/dist/core/tools/read.d.ts +13 -0
- package/dist/core/tools/read.d.ts.map +1 -1
- package/dist/core/tools/read.js +32 -2
- package/dist/core/tools/read.js.map +1 -1
- package/dist/core/tools/render-utils.d.ts +20 -0
- package/dist/core/tools/render-utils.d.ts.map +1 -1
- package/dist/core/tools/render-utils.js +67 -0
- package/dist/core/tools/render-utils.js.map +1 -1
- package/dist/core/tools/renderers/bash.d.ts.map +1 -1
- package/dist/core/tools/renderers/bash.js +7 -35
- package/dist/core/tools/renderers/bash.js.map +1 -1
- package/dist/core/tools/renderers/edit.d.ts.map +1 -1
- package/dist/core/tools/renderers/edit.js +5 -4
- package/dist/core/tools/renderers/edit.js.map +1 -1
- package/dist/core/tools/renderers/index.d.ts +2 -2
- package/dist/core/tools/renderers/index.d.ts.map +1 -1
- package/dist/core/tools/renderers/index.js.map +1 -1
- package/dist/core/tools/renderers/read.d.ts.map +1 -1
- package/dist/core/tools/renderers/read.js +7 -11
- package/dist/core/tools/renderers/read.js.map +1 -1
- package/dist/extensions/codemode/execute.d.ts.map +1 -1
- package/dist/extensions/codemode/execute.js +391 -80
- package/dist/extensions/codemode/execute.js.map +1 -1
- package/dist/extensions/codemode/index.d.ts.map +1 -1
- package/dist/extensions/codemode/index.js +3 -1
- package/dist/extensions/codemode/index.js.map +1 -1
- package/dist/extensions/codemode/renderer.d.ts +2 -2
- package/dist/extensions/codemode/renderer.d.ts.map +1 -1
- package/dist/extensions/codemode/renderer.js +9 -3
- package/dist/extensions/codemode/renderer.js.map +1 -1
- package/dist/extensions/codemode/tool.d.ts +22 -21
- package/dist/extensions/codemode/tool.d.ts.map +1 -1
- package/dist/extensions/codemode/tool.js +96 -123
- package/dist/extensions/codemode/tool.js.map +1 -1
- package/dist/extensions/llama/provider.d.ts.map +1 -1
- package/dist/extensions/llama/provider.js +34 -11
- package/dist/extensions/llama/provider.js.map +1 -1
- package/dist/extensions/mcp/cli.d.ts.map +1 -1
- package/dist/extensions/mcp/cli.js +27 -33
- package/dist/extensions/mcp/cli.js.map +1 -1
- package/dist/extensions/mcp/config.d.ts +15 -3
- package/dist/extensions/mcp/config.d.ts.map +1 -1
- package/dist/extensions/mcp/config.js +49 -12
- package/dist/extensions/mcp/config.js.map +1 -1
- package/dist/extensions/mcp/index.d.ts +4 -1
- package/dist/extensions/mcp/index.d.ts.map +1 -1
- package/dist/extensions/mcp/index.js +279 -129
- package/dist/extensions/mcp/index.js.map +1 -1
- package/dist/extensions/mcp/oauth.d.ts +21 -11
- package/dist/extensions/mcp/oauth.d.ts.map +1 -1
- package/dist/extensions/mcp/oauth.js +166 -45
- package/dist/extensions/mcp/oauth.js.map +1 -1
- package/dist/extensions/mcp/resources.d.ts +3 -10
- package/dist/extensions/mcp/resources.d.ts.map +1 -1
- package/dist/extensions/mcp/resources.js +4 -5
- package/dist/extensions/mcp/resources.js.map +1 -1
- package/dist/extensions/mcp/runtime.d.ts +7 -7
- package/dist/extensions/mcp/runtime.d.ts.map +1 -1
- package/dist/extensions/mcp/runtime.js +31 -45
- package/dist/extensions/mcp/runtime.js.map +1 -1
- package/dist/extensions/mcp/tools.d.ts +7 -5
- package/dist/extensions/mcp/tools.d.ts.map +1 -1
- package/dist/extensions/mcp/tools.js +87 -29
- package/dist/extensions/mcp/tools.js.map +1 -1
- package/dist/extensions/mcp/ui.d.ts +3 -3
- package/dist/extensions/mcp/ui.d.ts.map +1 -1
- package/dist/extensions/mcp/ui.js +18 -6
- package/dist/extensions/mcp/ui.js.map +1 -1
- package/dist/extensions/tool-search/tool.d.ts +0 -2
- package/dist/extensions/tool-search/tool.d.ts.map +1 -1
- package/dist/extensions/tool-search/tool.js +2 -4
- package/dist/extensions/tool-search/tool.js.map +1 -1
- package/dist/hooks/checks/checks-runtime.cjs +215 -0
- package/dist/hooks/checks/git-safety.cjs +198 -0
- package/dist/hooks/checks/git-safety.d.cts +1 -0
- package/dist/hooks/gsd/draht-post-task.js +78 -39
- package/dist/hooks/gsd/draht-quality-gate.js +210 -122
- package/dist/hooks/gsd/package.json +3 -0
- package/dist/index.d.ts +5 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -2
- package/dist/index.js.map +1 -1
- package/dist/main.d.ts.map +1 -1
- package/dist/main.js +11 -3
- package/dist/main.js.map +1 -1
- package/dist/modes/interactive/components/auth-url.d.ts +14 -0
- package/dist/modes/interactive/components/auth-url.d.ts.map +1 -0
- package/dist/modes/interactive/components/auth-url.js +37 -0
- package/dist/modes/interactive/components/auth-url.js.map +1 -0
- package/dist/modes/interactive/components/bash-execution.d.ts +5 -1
- package/dist/modes/interactive/components/bash-execution.d.ts.map +1 -1
- package/dist/modes/interactive/components/bash-execution.js +17 -14
- package/dist/modes/interactive/components/bash-execution.js.map +1 -1
- package/dist/modes/interactive/components/branch-summary-message.d.ts +2 -1
- package/dist/modes/interactive/components/branch-summary-message.d.ts.map +1 -1
- package/dist/modes/interactive/components/branch-summary-message.js +5 -2
- package/dist/modes/interactive/components/branch-summary-message.js.map +1 -1
- package/dist/modes/interactive/components/compaction-summary-message.d.ts +2 -1
- package/dist/modes/interactive/components/compaction-summary-message.d.ts.map +1 -1
- package/dist/modes/interactive/components/compaction-summary-message.js +5 -2
- package/dist/modes/interactive/components/compaction-summary-message.js.map +1 -1
- package/dist/modes/interactive/components/custom-entry.d.ts +3 -1
- package/dist/modes/interactive/components/custom-entry.d.ts.map +1 -1
- package/dist/modes/interactive/components/custom-entry.js +8 -2
- package/dist/modes/interactive/components/custom-entry.js.map +1 -1
- package/dist/modes/interactive/components/custom-message.d.ts.map +1 -1
- package/dist/modes/interactive/components/custom-message.js +1 -0
- package/dist/modes/interactive/components/custom-message.js.map +1 -1
- package/dist/modes/interactive/components/login-dialog.d.ts +3 -0
- package/dist/modes/interactive/components/login-dialog.d.ts.map +1 -1
- package/dist/modes/interactive/components/login-dialog.js +38 -10
- package/dist/modes/interactive/components/login-dialog.js.map +1 -1
- package/dist/modes/interactive/components/oauth-selector.d.ts +8 -2
- package/dist/modes/interactive/components/oauth-selector.d.ts.map +1 -1
- package/dist/modes/interactive/components/oauth-selector.js +24 -21
- package/dist/modes/interactive/components/oauth-selector.js.map +1 -1
- package/dist/modes/interactive/components/radius-login-selector.d.ts +15 -0
- package/dist/modes/interactive/components/radius-login-selector.d.ts.map +1 -0
- package/dist/modes/interactive/components/radius-login-selector.js +85 -0
- package/dist/modes/interactive/components/radius-login-selector.js.map +1 -0
- package/dist/modes/interactive/components/settings-selector.d.ts +3 -3
- package/dist/modes/interactive/components/settings-selector.d.ts.map +1 -1
- package/dist/modes/interactive/components/settings-selector.js +9 -8
- package/dist/modes/interactive/components/settings-selector.js.map +1 -1
- package/dist/modes/interactive/components/skill-invocation-message.d.ts +2 -1
- package/dist/modes/interactive/components/skill-invocation-message.d.ts.map +1 -1
- package/dist/modes/interactive/components/skill-invocation-message.js +5 -2
- package/dist/modes/interactive/components/skill-invocation-message.js.map +1 -1
- package/dist/modes/interactive/components/status-indicator.d.ts +1 -1
- package/dist/modes/interactive/components/status-indicator.d.ts.map +1 -1
- package/dist/modes/interactive/components/status-indicator.js +2 -2
- package/dist/modes/interactive/components/status-indicator.js.map +1 -1
- package/dist/modes/interactive/components/tool-execution.d.ts +12 -13
- package/dist/modes/interactive/components/tool-execution.d.ts.map +1 -1
- package/dist/modes/interactive/components/tool-execution.js +32 -45
- package/dist/modes/interactive/components/tool-execution.js.map +1 -1
- package/dist/modes/interactive/components/tree-selector.d.ts.map +1 -1
- package/dist/modes/interactive/components/tree-selector.js +2 -9
- package/dist/modes/interactive/components/tree-selector.js.map +1 -1
- package/dist/modes/interactive/components/user-message.d.ts.map +1 -1
- package/dist/modes/interactive/components/user-message.js +5 -4
- package/dist/modes/interactive/components/user-message.js.map +1 -1
- package/dist/modes/interactive/interactive-mode.d.ts +43 -48
- package/dist/modes/interactive/interactive-mode.d.ts.map +1 -1
- package/dist/modes/interactive/interactive-mode.js +362 -278
- package/dist/modes/interactive/interactive-mode.js.map +1 -1
- package/dist/modes/interactive/program-status-reporter.d.ts +34 -0
- package/dist/modes/interactive/program-status-reporter.d.ts.map +1 -0
- package/dist/modes/interactive/program-status-reporter.js +110 -0
- package/dist/modes/interactive/program-status-reporter.js.map +1 -0
- package/dist/modes/interactive/theme/dark.json +1 -1
- package/dist/modes/interactive/theme/light.json +1 -1
- package/dist/modes/interactive/theme/system-theme.d.ts +2 -1
- package/dist/modes/interactive/theme/system-theme.d.ts.map +1 -1
- package/dist/modes/interactive/theme/system-theme.js +16 -5
- package/dist/modes/interactive/theme/system-theme.js.map +1 -1
- package/dist/modes/interactive/theme/theme-schema.d.ts +28 -0
- package/dist/modes/interactive/theme/theme-schema.d.ts.map +1 -0
- package/dist/modes/interactive/theme/theme-schema.js +105 -0
- package/dist/modes/interactive/theme/theme-schema.js.map +1 -0
- package/dist/modes/interactive/theme/theme-tokens.d.ts +257 -0
- package/dist/modes/interactive/theme/theme-tokens.d.ts.map +1 -0
- package/dist/modes/interactive/theme/theme-tokens.js +94 -0
- package/dist/modes/interactive/theme/theme-tokens.js.map +1 -0
- package/dist/modes/interactive/theme/theme.d.ts +13 -16
- package/dist/modes/interactive/theme/theme.d.ts.map +1 -1
- package/dist/modes/interactive/theme/theme.js +22 -29
- package/dist/modes/interactive/theme/theme.js.map +1 -1
- package/dist/modes/print-mode.d.ts.map +1 -1
- package/dist/modes/print-mode.js +4 -0
- package/dist/modes/print-mode.js.map +1 -1
- package/dist/package-manager-cli.d.ts.map +1 -1
- package/dist/package-manager-cli.js +26 -1
- package/dist/package-manager-cli.js.map +1 -1
- package/dist/prompts/commands/atomic-commit.md +8 -6
- package/dist/prompts/commands/brainstorm.md +5 -149
- package/dist/prompts/commands/create-verification-skill.md +8 -48
- package/dist/prompts/commands/discuss-phase.md +6 -10
- package/dist/prompts/commands/execute-phase.md +18 -45
- package/dist/prompts/commands/fix.md +16 -128
- package/dist/prompts/commands/grill.md +6 -53
- package/dist/prompts/commands/init-project.md +13 -18
- package/dist/prompts/commands/map-codebase.md +3 -3
- package/dist/prompts/commands/new-project.md +12 -17
- package/dist/prompts/commands/next-milestone.md +4 -8
- package/dist/prompts/commands/orchestrate-loop.md +12 -12
- package/dist/prompts/commands/orchestrate.md +12 -11
- package/dist/prompts/commands/pause-work.md +1 -1
- package/dist/prompts/commands/plan-phase.md +15 -37
- package/dist/prompts/commands/quick.md +11 -64
- package/dist/prompts/commands/resolve-conflicts.md +2 -15
- package/dist/prompts/commands/triage.md +3 -14
- package/dist/prompts/commands/verify-work.md +15 -63
- package/dist/prompts/commands/why.md +18 -118
- package/dist/skills/atomic-reasoning/SKILL.md +6 -105
- package/dist/skills/blast-radius/SKILL.md +1 -1
- package/dist/skills/epistemics/SKILL.md +12 -87
- package/dist/skills/explainers/SKILL.md +1 -1
- package/dist/skills/typescript-discipline/SKILL.md +12 -42
- package/dist/skills/typescript-discipline/references/patterns.md +25 -266
- package/dist/skills/unslop/SKILL.md +14 -105
- package/dist/utils/ansi.d.ts +8 -0
- package/dist/utils/ansi.d.ts.map +1 -1
- package/dist/utils/ansi.js +30 -11
- package/dist/utils/ansi.js.map +1 -1
- package/dist/utils/clipboard-command.d.ts.map +1 -1
- package/dist/utils/clipboard-command.js +3 -0
- package/dist/utils/clipboard-command.js.map +1 -1
- package/dist/utils/clipboard.d.ts.map +1 -1
- package/dist/utils/clipboard.js +18 -12
- package/dist/utils/clipboard.js.map +1 -1
- package/dist/utils/draht-user-agent.d.ts +1 -0
- package/dist/utils/draht-user-agent.d.ts.map +1 -1
- package/dist/utils/draht-user-agent.js +3 -2
- package/dist/utils/draht-user-agent.js.map +1 -1
- package/dist/utils/image-convert.d.ts +12 -0
- package/dist/utils/image-convert.d.ts.map +1 -1
- package/dist/utils/image-convert.js +45 -6
- package/dist/utils/image-convert.js.map +1 -1
- package/dist/utils/image-resize-core.d.ts +12 -0
- package/dist/utils/image-resize-core.d.ts.map +1 -1
- package/dist/utils/image-resize-core.js +5 -0
- package/dist/utils/image-resize-core.js.map +1 -1
- package/dist/utils/image-resize-worker.js +3 -2
- package/dist/utils/image-resize-worker.js.map +1 -1
- package/dist/utils/image-resize.d.ts.map +1 -1
- package/dist/utils/image-resize.js +9 -10
- package/dist/utils/image-resize.js.map +1 -1
- package/dist/utils/output-files.d.ts +15 -0
- package/dist/utils/output-files.d.ts.map +1 -0
- package/dist/utils/output-files.js +30 -0
- package/dist/utils/output-files.js.map +1 -0
- package/dist/utils/shell.d.ts.map +1 -1
- package/dist/utils/shell.js +43 -13
- package/dist/utils/shell.js.map +1 -1
- package/dist/utils/syntax-highlight.d.ts.map +1 -1
- package/dist/utils/syntax-highlight.js +6 -1
- package/dist/utils/syntax-highlight.js.map +1 -1
- package/dist/utils/tools-manager.d.ts.map +1 -1
- package/dist/utils/tools-manager.js +4 -3
- package/dist/utils/tools-manager.js.map +1 -1
- package/docs/cli-integration.md +1 -1
- package/docs/cli.md +30 -26
- package/docs/codemode.md +230 -0
- package/docs/compaction.md +5 -5
- package/docs/containerization.md +1 -1
- package/docs/custom-provider.md +53 -1
- package/docs/docs.json +17 -1
- package/docs/drahtbank.md +163 -0
- package/docs/environment-variables.md +5 -2
- package/docs/extensions.md +143 -13
- package/docs/index.md +1 -1
- package/docs/json.md +3 -3
- package/docs/keybindings.md +9 -5
- package/docs/llama-cpp.md +9 -2
- package/docs/mcp.md +59 -3
- package/docs/message-types.md +22 -4
- package/docs/models-configuration.md +187 -0
- package/docs/models.md +67 -4
- package/docs/nvidia-rtx-llama-cpp.md +96 -0
- package/docs/providers.md +70 -5
- package/docs/sdk.md +1 -1
- package/docs/settings.md +10 -8
- package/docs/terminal-setup.md +14 -0
- package/docs/themes.md +4 -2
- package/docs/tui.md +71 -1
- package/docs/usage.md +1 -1
- package/docs/windows.md +16 -2
- package/eval/drahtbank/adapters.d.mts +35 -0
- package/eval/drahtbank/adapters.mjs +192 -0
- package/eval/drahtbank/bank.d.mts +141 -0
- package/eval/drahtbank/bank.mjs +515 -0
- package/eval/drahtbank/cases-v2.d.mts +28 -0
- package/eval/drahtbank/cases-v2.mjs +164 -0
- package/eval/drahtbank/cases.d.mts +17 -0
- package/eval/drahtbank/cases.mjs +198 -0
- package/eval/drahtbank/cli.mjs +120 -0
- package/eval/drahtbank/grading-v2.d.mts +26 -0
- package/eval/drahtbank/grading-v2.mjs +227 -0
- package/eval/drahtbank/isolated-tools.ts +223 -0
- package/eval/drahtbank/isolation-worker.mjs +53 -0
- package/eval/drahtbank/isolation.d.mts +52 -0
- package/eval/drahtbank/isolation.mjs +345 -0
- package/eval/drahtbank/runtime-checks.d.mts +15 -0
- package/eval/drahtbank/runtime-checks.mjs +187 -0
- package/examples/extensions/dynamic-resources/dynamic.json +1 -1
- package/examples/extensions/minimal-mode.ts +4 -5
- package/examples/extensions/overlay-test.ts +2 -2
- package/examples/extensions/plan-mode/index.ts +6 -2
- package/examples/extensions/subagent/index.ts +4 -6
- package/hooks/checks/checks-runtime.cjs +215 -0
- package/hooks/checks/git-safety.cjs +198 -0
- package/hooks/checks/git-safety.d.cts +1 -0
- package/hooks/gsd/draht-post-task.js +78 -39
- package/hooks/gsd/draht-quality-gate.js +210 -122
- package/hooks/gsd/package.json +3 -0
- package/package.json +21 -16
- package/prompts/commands/atomic-commit.md +8 -6
- package/prompts/commands/brainstorm.md +5 -149
- package/prompts/commands/create-verification-skill.md +8 -48
- package/prompts/commands/discuss-phase.md +6 -10
- package/prompts/commands/execute-phase.md +18 -45
- package/prompts/commands/fix.md +16 -128
- package/prompts/commands/grill.md +6 -53
- package/prompts/commands/init-project.md +13 -18
- package/prompts/commands/map-codebase.md +3 -3
- package/prompts/commands/new-project.md +12 -17
- package/prompts/commands/next-milestone.md +4 -8
- package/prompts/commands/orchestrate-loop.md +12 -12
- package/prompts/commands/orchestrate.md +12 -11
- package/prompts/commands/pause-work.md +1 -1
- package/prompts/commands/plan-phase.md +15 -37
- package/prompts/commands/quick.md +11 -64
- package/prompts/commands/resolve-conflicts.md +2 -15
- package/prompts/commands/triage.md +3 -14
- package/prompts/commands/verify-work.md +15 -63
- package/prompts/commands/why.md +18 -118
- package/schemas/keybindings.schema.json +433 -0
- package/schemas/models.schema.json +1190 -0
- package/schemas/settings.schema.json +849 -0
- package/schemas/theme.schema.json +365 -0
- package/skills/atomic-reasoning/SKILL.md +6 -105
- package/skills/blast-radius/SKILL.md +1 -1
- package/skills/epistemics/SKILL.md +12 -87
- package/skills/explainers/SKILL.md +1 -1
- package/skills/typescript-discipline/SKILL.md +12 -42
- package/skills/typescript-discipline/references/patterns.md +25 -266
- package/skills/unslop/SKILL.md +14 -105
- package/dist/bundle/chunks/anthropic-messages-E2P6C3P7.js +0 -6
- package/dist/bundle/chunks/azure-openai-responses-JZMSTDIM.js +0 -2
- package/dist/bundle/chunks/chunk-2E5VODES.js +0 -2
- package/dist/bundle/chunks/chunk-3DQNSI5W.js +0 -137
- package/dist/bundle/chunks/chunk-4COXSQRQ.js +0 -18
- package/dist/bundle/chunks/chunk-D3HO453P.js +0 -42
- package/dist/bundle/chunks/chunk-GVXA72ZA.js +0 -2
- package/dist/bundle/chunks/chunk-I2SOJ4JN.js +0 -65
- package/dist/bundle/chunks/chunk-I5OKY6RH.js +0 -11
- package/dist/bundle/chunks/chunk-ILSPVHVH.js +0 -1623
- package/dist/bundle/chunks/chunk-KY2BG6BC.js +0 -11
- package/dist/bundle/chunks/chunk-PGJAHC3R.js +0 -107
- package/dist/bundle/chunks/chunk-QVFELZW2.js +0 -312
- package/dist/bundle/chunks/chunk-QVTLBSTH.js +0 -4
- package/dist/bundle/chunks/chunk-SDRCE6OP.js +0 -2
- package/dist/bundle/chunks/chunk-U6LDZZGM.js +0 -146
- package/dist/bundle/chunks/chunk-X3YFUSCK.js +0 -82
- package/dist/bundle/chunks/chunk-XBUEMKIY.js +0 -2
- package/dist/bundle/chunks/chunk-XHCO4PPH.js +0 -8
- package/dist/bundle/chunks/chunk-YH7HNI7D.js +0 -2
- package/dist/bundle/chunks/cloudflare-workers-ai-system-one-MJKOAE5S.js +0 -2
- package/dist/bundle/chunks/execute-AGEWJLIO.js +0 -18
- package/dist/bundle/chunks/google-generative-ai-HPNDIAZD.js +0 -2
- package/dist/bundle/chunks/google-vertex-KPAWJ4SP.js +0 -2
- package/dist/bundle/chunks/mistral-conversations-FMSPWQ7A.js +0 -5
- package/dist/bundle/chunks/node-4JNG6XQO.js +0 -16
- package/dist/bundle/chunks/openai-codex-responses-TU7NGPXE.js +0 -10
- package/dist/bundle/chunks/openai-completions-GANCQQO5.js +0 -7
- package/dist/bundle/chunks/openai-responses-ASDPAX3E.js +0 -3
- package/dist/bundle/chunks/openrouter-images-NROSQXMZ.js +0 -2
- package/dist/bundle/chunks/pi-messages-2QEJYOAO.js +0 -8
- package/dist/bundle/chunks/runtime-2B56SZM6.js +0 -2
- package/dist/bundle/chunks/runtime-TJ3EAMJK.js +0 -3
- package/dist/bundle/chunks/settings-manager-HC5RR36T.js +0 -2
- package/dist/bundle/chunks/typesafe-system-one-YBEEY5XU.js +0 -2
- package/dist/bundle/chunks/virtual-modules-XKLDADFR.js +0 -2
- package/dist/core/provider-attribution.d.ts +0 -4
- package/dist/core/provider-attribution.d.ts.map +0 -1
- package/dist/core/provider-attribution.js +0 -72
- package/dist/core/provider-attribution.js.map +0 -1
- package/dist/experimental/micro/api.d.ts +0 -81
- package/dist/experimental/micro/api.d.ts.map +0 -1
- package/dist/experimental/micro/api.js +0 -2
- package/dist/experimental/micro/api.js.map +0 -1
- package/dist/experimental/micro/main.d.ts +0 -3
- package/dist/experimental/micro/main.d.ts.map +0 -1
- package/dist/experimental/micro/main.js +0 -21
- package/dist/experimental/micro/main.js.map +0 -1
- package/dist/experimental/micro/models.d.ts +0 -11
- package/dist/experimental/micro/models.d.ts.map +0 -1
- package/dist/experimental/micro/models.js +0 -95
- package/dist/experimental/micro/models.js.map +0 -1
- package/dist/experimental/micro/runtime.d.ts +0 -12
- package/dist/experimental/micro/runtime.d.ts.map +0 -1
- package/dist/experimental/micro/runtime.js +0 -465
- package/dist/experimental/micro/runtime.js.map +0 -1
- package/dist/experimental/micro/sessions.d.ts +0 -10
- package/dist/experimental/micro/sessions.d.ts.map +0 -1
- package/dist/experimental/micro/sessions.js +0 -41
- package/dist/experimental/micro/sessions.js.map +0 -1
- package/dist/experimental/micro/tools.d.ts +0 -10
- package/dist/experimental/micro/tools.d.ts.map +0 -1
- package/dist/experimental/micro/tools.js +0 -96
- package/dist/experimental/micro/tools.js.map +0 -1
- package/dist/experimental/micro/tui.d.ts +0 -3
- package/dist/experimental/micro/tui.d.ts.map +0 -1
- package/dist/experimental/micro/tui.js +0 -510
- package/dist/experimental/micro/tui.js.map +0 -1
- package/dist/experimental/mini/main.d.ts +0 -7
- package/dist/experimental/mini/main.d.ts.map +0 -1
- package/dist/experimental/mini/main.js +0 -21
- package/dist/experimental/mini/main.js.map +0 -1
- package/dist/experimental/mini/server/entry.d.ts +0 -3
- package/dist/experimental/mini/server/entry.d.ts.map +0 -1
- package/dist/experimental/mini/server/entry.js +0 -11
- package/dist/experimental/mini/server/entry.js.map +0 -1
- package/dist/experimental/mini/server/run.d.ts +0 -13
- package/dist/experimental/mini/server/run.d.ts.map +0 -1
- package/dist/experimental/mini/server/run.js +0 -150
- package/dist/experimental/mini/server/run.js.map +0 -1
- package/dist/experimental/mini/shared/protocol.d.ts +0 -127
- package/dist/experimental/mini/shared/protocol.d.ts.map +0 -1
- package/dist/experimental/mini/shared/protocol.js +0 -13
- package/dist/experimental/mini/shared/protocol.js.map +0 -1
- package/dist/experimental/mini/shared/rpc.d.ts +0 -49
- package/dist/experimental/mini/shared/rpc.d.ts.map +0 -1
- package/dist/experimental/mini/shared/rpc.js +0 -171
- package/dist/experimental/mini/shared/rpc.js.map +0 -1
- package/dist/experimental/mini/shared/transport.d.ts +0 -35
- package/dist/experimental/mini/shared/transport.d.ts.map +0 -1
- package/dist/experimental/mini/shared/transport.js +0 -94
- package/dist/experimental/mini/shared/transport.js.map +0 -1
- package/dist/experimental/mini/tui/run.d.ts +0 -9
- package/dist/experimental/mini/tui/run.d.ts.map +0 -1
- package/dist/experimental/mini/tui/run.js +0 -62
- package/dist/experimental/mini/tui/run.js.map +0 -1
- package/dist/experimental/mini/tui/session.d.ts +0 -26
- package/dist/experimental/mini/tui/session.d.ts.map +0 -1
- package/dist/experimental/mini/tui/session.js +0 -94
- package/dist/experimental/mini/tui/session.js.map +0 -1
- package/dist/experimental/mini/tui/view.d.ts +0 -10
- package/dist/experimental/mini/tui/view.d.ts.map +0 -1
- package/dist/experimental/mini/tui/view.js +0 -461
- package/dist/experimental/mini/tui/view.js.map +0 -1
- package/dist/experimental/mini/worker/entry.d.ts +0 -3
- package/dist/experimental/mini/worker/entry.d.ts.map +0 -1
- package/dist/experimental/mini/worker/entry.js +0 -10
- package/dist/experimental/mini/worker/entry.js.map +0 -1
- package/dist/experimental/mini/worker/lane-service.d.ts +0 -44
- package/dist/experimental/mini/worker/lane-service.d.ts.map +0 -1
- package/dist/experimental/mini/worker/lane-service.js +0 -97
- package/dist/experimental/mini/worker/lane-service.js.map +0 -1
- package/dist/experimental/mini/worker/models-service.d.ts +0 -19
- package/dist/experimental/mini/worker/models-service.d.ts.map +0 -1
- package/dist/experimental/mini/worker/models-service.js +0 -136
- package/dist/experimental/mini/worker/models-service.js.map +0 -1
- package/dist/experimental/mini/worker/run.d.ts +0 -14
- package/dist/experimental/mini/worker/run.d.ts.map +0 -1
- package/dist/experimental/mini/worker/run.js +0 -88
- package/dist/experimental/mini/worker/run.js.map +0 -1
- package/dist/modes/interactive/theme/theme-json.d.ts +0 -85
- package/dist/modes/interactive/theme/theme-json.d.ts.map +0 -1
- package/dist/modes/interactive/theme/theme-json.js +0 -132
- package/dist/modes/interactive/theme/theme-json.js.map +0 -1
- package/dist/modes/interactive/theme/theme-schema.json +0 -361
- /package/dist/bundle/chunks/{chunk-FBU742QF.js → chunk-MOOADNVS.js} +0 -0
|
@@ -0,0 +1,515 @@
|
|
|
1
|
+
import { execFileSync, spawn } from "node:child_process";
|
|
2
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
3
|
+
import { existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, realpathSync, writeFileSync } from "node:fs";
|
|
4
|
+
import { delimiter, dirname, join, resolve } from "node:path";
|
|
5
|
+
import { buildInvocation, parseOutput } from "./adapters.mjs";
|
|
6
|
+
import { CASES, SUITE_VERSION } from "./cases.mjs";
|
|
7
|
+
import { CASES as CASES_V2, GRADING_SOURCES, gradeResponse } from "./cases-v2.mjs";
|
|
8
|
+
import { createIsolation, ISOLATION_VERSION } from "./isolation.mjs";
|
|
9
|
+
|
|
10
|
+
export const HOSTS = ["coding-agent", "draht-claude", "draht-codex"];
|
|
11
|
+
const hash = (value) => createHash("sha256").update(value).digest("hex");
|
|
12
|
+
const suiteCases = (version = SUITE_VERSION) => version === "1" ? CASES : version === "2" ? CASES_V2 : fail(`Unknown suite: ${version}`);
|
|
13
|
+
const suiteHash = (version = SUITE_VERSION) => hash(JSON.stringify({ version, cases: suiteCases(version),
|
|
14
|
+
...(version === "2" ? { grading: [...GRADING_SOURCES, "bank.mjs", "adapters.mjs", "isolation.mjs", "isolation-worker.mjs", "isolated-tools.ts"].map((name) => [name, hash(readFileSync(new URL(name, import.meta.url)))]) } : {}) }));
|
|
15
|
+
const json = (path, value) => writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`);
|
|
16
|
+
const readJson = (path) => JSON.parse(readFileSync(path, "utf8"));
|
|
17
|
+
const fail = (message) => { throw new Error(message); };
|
|
18
|
+
const caseById = (id, version) => suiteCases(version).find((entry) => entry.id === id) ?? fail(`Unknown case: ${id}`);
|
|
19
|
+
const relativePath = (value) => typeof value === "string" && value.length > 0 && !value.includes("\\") &&
|
|
20
|
+
!value.startsWith("/") && !value.split("/").some((part) => !part || part === "." || part === "..");
|
|
21
|
+
const GIT_REPOSITORY_VARIABLES = new Set([
|
|
22
|
+
"GIT_DIR", "GIT_WORK_TREE", "GIT_COMMON_DIR", "GIT_IMPLICIT_WORK_TREE", "GIT_INDEX_FILE",
|
|
23
|
+
"GIT_OBJECT_DIRECTORY", "GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_GRAFT_FILE", "GIT_SHALLOW_FILE",
|
|
24
|
+
"GIT_NAMESPACE", "GIT_NO_REPLACE_OBJECTS", "GIT_REPLACE_REF_BASE", "GIT_PREFIX", "GIT_INTERNAL_SUPER_PREFIX",
|
|
25
|
+
"GIT_SUPER_PREFIX", "GIT_CEILING_DIRECTORIES", "GIT_DISCOVERY_ACROSS_FILESYSTEM", "GIT_CONFIG",
|
|
26
|
+
"GIT_CONFIG_PARAMETERS", "GIT_CONFIG_COUNT",
|
|
27
|
+
]);
|
|
28
|
+
|
|
29
|
+
function fixtureEnvironment(cwd, overrides) {
|
|
30
|
+
const env = { ...process.env, ...overrides };
|
|
31
|
+
for (const key of Object.keys(env)) {
|
|
32
|
+
const name = key.toUpperCase();
|
|
33
|
+
if (GIT_REPOSITORY_VARIABLES.has(name) || /^GIT_CONFIG_(?:KEY|VALUE)_\d+$/.test(name)) delete env[key];
|
|
34
|
+
}
|
|
35
|
+
let physicalDirectory;
|
|
36
|
+
try { physicalDirectory = realpathSync(cwd); } catch { physicalDirectory = resolve(cwd); }
|
|
37
|
+
// Git ignores a ceiling equal to its initial cwd; stop before entering its parent.
|
|
38
|
+
const ceilingDirectory = dirname(physicalDirectory);
|
|
39
|
+
if (ceilingDirectory.includes(delimiter)) fail("Fixture path contains the Git ceiling path-list delimiter");
|
|
40
|
+
env.GIT_CEILING_DIRECTORIES = ceilingDirectory;
|
|
41
|
+
return { env, gitEnvironment: { policy: "fixture-ceiling-v1", ceilingDirectory } };
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function matrix(hosts, caseIds, repeats, variantIds = ["baseline", "candidate"], orderOffset = 0) {
|
|
45
|
+
const trials = [];
|
|
46
|
+
for (let repetition = 1; repetition <= repeats; repetition++) {
|
|
47
|
+
for (const host of hosts) {
|
|
48
|
+
for (const [index, caseId] of caseIds.entries()) {
|
|
49
|
+
const offset = (repetition - 1 + index + orderOffset) % variantIds.length;
|
|
50
|
+
const variants = variantIds.join(",") === "baseline,candidate" && orderOffset === 0
|
|
51
|
+
? ((repetition + index) % 2 ? ["baseline", "candidate"] : ["candidate", "baseline"])
|
|
52
|
+
: [...variantIds.slice(offset), ...variantIds.slice(0, offset)];
|
|
53
|
+
for (const variant of variants) trials.push({
|
|
54
|
+
id: `${String(repetition).padStart(3, "0")}-${host}-${caseId}-${variant}`,
|
|
55
|
+
host, caseId, repetition, variant,
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
return trials;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function paths(output, trial) {
|
|
64
|
+
const control = join(output, "control", trial.id);
|
|
65
|
+
return { control, workspace: join(output, "workspaces", trial.id),
|
|
66
|
+
promptFile: join(control, "prompt.md"), taskFile: join(control, "task.md") };
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function validateSelection(hosts, caseIds, repeats, version) {
|
|
70
|
+
if (!Array.isArray(hosts) || !hosts.length || hosts.some((host) => !HOSTS.includes(host)) || new Set(hosts).size !== hosts.length) fail("Choose unique known hosts");
|
|
71
|
+
if (!Array.isArray(caseIds) || !caseIds.length || new Set(caseIds).size !== caseIds.length) fail("Choose unique cases");
|
|
72
|
+
for (const id of caseIds) caseById(id, version);
|
|
73
|
+
if (!Number.isInteger(repeats) || repeats < 1 || repeats > 100) fail("repeats must be an integer from 1 to 100");
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function validateVariants(variants) {
|
|
77
|
+
if (!Array.isArray(variants) || variants.length < 2 || variants.length > 8) fail("Choose two to eight prompt variants");
|
|
78
|
+
if (new Set(variants.map((variant) => variant?.id)).size !== variants.length) fail("Variant IDs must be unique");
|
|
79
|
+
for (const variant of variants) {
|
|
80
|
+
if (!variant || typeof variant.id !== "string" || !/^[a-z][a-z0-9-]{0,39}$/.test(variant.id) ||
|
|
81
|
+
!(variant.ref === null || (typeof variant.ref === "string" && variant.ref.trim() && !variant.ref.includes("\0"))) ||
|
|
82
|
+
Object.keys(variant).some((key) => !["id", "ref"].includes(key))) fail("Invalid prompt variant; expected {id, ref: commit reference or null}");
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function promptPathFor(host, entry) {
|
|
87
|
+
if (!entry.promptKind || entry.promptKind === "agent") return `packages/${host}/agents/${entry.promptName ?? entry.role}.md`;
|
|
88
|
+
if (entry.promptKind === "skill") return host === "coding-agent" ? `skills/${entry.promptName}/SKILL.md` : `packages/${host}/skills/${entry.promptName}/SKILL.md`;
|
|
89
|
+
if (entry.promptKind === "command") return host === "coding-agent" ? `packages/${host}/prompts/commands/${entry.promptName}.md`
|
|
90
|
+
: host === "draht-claude" ? `packages/${host}/commands/${entry.promptName}.md` : `packages/${host}/skills/${entry.promptName}/command.md`;
|
|
91
|
+
fail(`Unknown prompt kind: ${entry.promptKind}`);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export function prepareBank(output, options = {}) {
|
|
95
|
+
output = resolve(output);
|
|
96
|
+
const root = execFileSync("git", ["rev-parse", "--show-toplevel"], { cwd: options.root ?? process.cwd(), encoding: "utf8" }).trim();
|
|
97
|
+
const hosts = options.hosts ?? ["coding-agent"];
|
|
98
|
+
const version = options.suite ?? "1";
|
|
99
|
+
const split = options.split ?? (version === "2" ? "screen" : "all");
|
|
100
|
+
if (!["screen", "holdout", "all"].includes(split)) fail("split must be screen, holdout, or all");
|
|
101
|
+
if (version === "1" && (options.variants || options.split || options.orderOffset !== undefined)) fail("Variants, split, and order offset require suite 2");
|
|
102
|
+
const caseIds = options.cases ?? suiteCases(version).filter((entry) => split === "all" || entry.split === split).map(({ id }) => id);
|
|
103
|
+
const repeats = options.repeats ?? 1;
|
|
104
|
+
validateSelection(hosts, caseIds, repeats, version);
|
|
105
|
+
if (version === "2" && split !== "all" && caseIds.some((id) => caseById(id, version).split !== split)) fail("Selected cases do not belong to the requested split");
|
|
106
|
+
if (options.prompt && !relativePath(options.prompt)) fail("prompt must be a repository-relative path without parent segments");
|
|
107
|
+
if (existsSync(output) && readdirSync(output).length) fail("Choose an empty output directory; existing benchmark work will not be overwritten");
|
|
108
|
+
const revision = (ref) => execFileSync("git", ["rev-parse", "--verify", "--end-of-options", `${ref}^{commit}`], { cwd: root, encoding: "utf8" }).trim();
|
|
109
|
+
const baselineCommit = revision(options.baseline ?? "HEAD");
|
|
110
|
+
const candidateCommit = options.candidate ? revision(options.candidate) : null;
|
|
111
|
+
if (options.variants && (options.baseline || options.candidate)) fail("Use variants or baseline/candidate, not both");
|
|
112
|
+
const requestedVariants = options.variants ?? [{ id: "baseline", ref: baselineCommit }, { id: "candidate", ref: candidateCommit }];
|
|
113
|
+
validateVariants(requestedVariants);
|
|
114
|
+
const variants = requestedVariants.map(({ id, ref }) => ({ id, ref: ref === null ? null : revision(ref) }));
|
|
115
|
+
const orderOffset = options.orderOffset ?? 0;
|
|
116
|
+
if (!Number.isInteger(orderOffset) || orderOffset < 0 || orderOffset >= variants.length) fail("orderOffset must select a variant index");
|
|
117
|
+
const trials = matrix(hosts, caseIds, repeats, variants.map(({ id }) => id), orderOffset).map((trial) => {
|
|
118
|
+
const entry = caseById(trial.caseId, version);
|
|
119
|
+
const promptPath = options.prompt ?? promptPathFor(trial.host, entry);
|
|
120
|
+
const commit = variants.find(({ id }) => id === trial.variant).ref;
|
|
121
|
+
const prompt = commit ? execFileSync("git", ["show", `${commit}:${promptPath}`], { cwd: root, encoding: "utf8" }) : readFileSync(join(root, promptPath), "utf8");
|
|
122
|
+
return { ...trial, ...(version === "2" ? { promptCommit: commit, split: entry.split, promptSurface: promptPath.startsWith("skills/") ? "canonical-skill" : "package-asset" } : {}), promptPath, promptSha256: hash(prompt), promptBytes: Buffer.byteLength(prompt), prompt };
|
|
123
|
+
});
|
|
124
|
+
mkdirSync(output, { recursive: true });
|
|
125
|
+
for (const trial of trials) {
|
|
126
|
+
const locations = paths(output, trial);
|
|
127
|
+
mkdirSync(locations.control, { recursive: true });
|
|
128
|
+
mkdirSync(locations.workspace, { recursive: true });
|
|
129
|
+
const entry = caseById(trial.caseId, version);
|
|
130
|
+
for (const [name, contents] of Object.entries(entry.files)) {
|
|
131
|
+
if (!relativePath(name)) fail(`Unsafe fixture path: ${name}`);
|
|
132
|
+
const file = join(locations.workspace, name);
|
|
133
|
+
mkdirSync(dirname(file), { recursive: true });
|
|
134
|
+
writeFileSync(file, contents);
|
|
135
|
+
}
|
|
136
|
+
const responseInstructions = version === "2" ? entry.responseInstructions : "End your response with exactly one status line using the appropriate outcome: STATUS: DONE, STATUS: DONE_WITH_CONCERNS, STATUS: NEEDS_CONTEXT, or STATUS: BLOCKED.";
|
|
137
|
+
const task = `Work in ${locations.workspace}. This is an isolated Drahtbank fixture. Do not use the network, delegate, commit, or modify files outside this workspace.\nAllowed edits: ${entry.writable.join(", ") || "none (read-only)"}. Allowed new files: ${entry.allowedNewFiles.join(", ") || "none"}. Preserve all other files.\n${responseInstructions}\n\n${entry.task}\n`;
|
|
138
|
+
writeFileSync(locations.promptFile, trial.prompt);
|
|
139
|
+
writeFileSync(locations.taskFile, task);
|
|
140
|
+
trial.taskSha256 = hash(task);
|
|
141
|
+
trial.initialWorkspaceSha256 = workspaceDigest(locations.workspace);
|
|
142
|
+
delete trial.prompt;
|
|
143
|
+
}
|
|
144
|
+
const manifest = { schemaVersion: version === "2" ? 2 : 1, suiteVersion: version, suiteSha256: suiteHash(version), createdAt: new Date().toISOString(),
|
|
145
|
+
root, baselineCommit, candidateCommit, hosts, caseIds, repeats, trials,
|
|
146
|
+
...(version === "2" ? { variants, orderOffset, split } : {}) };
|
|
147
|
+
json(join(output, "manifest.json"), manifest);
|
|
148
|
+
return manifest;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
export function readManifest(output) {
|
|
152
|
+
const manifest = readJson(join(output, "manifest.json"));
|
|
153
|
+
if (!((manifest.schemaVersion === 1 && manifest.suiteVersion === "1") || (manifest.schemaVersion === 2 && manifest.suiteVersion === "2")) || manifest.suiteSha256 !== suiteHash(manifest.suiteVersion)) fail("Unsupported or changed benchmark suite; prepare a fresh bank with this version");
|
|
154
|
+
validateSelection(manifest.hosts, manifest.caseIds, manifest.repeats, manifest.suiteVersion);
|
|
155
|
+
if (manifest.schemaVersion === 2) {
|
|
156
|
+
validateVariants(manifest.variants);
|
|
157
|
+
if (manifest.variants.some(({ ref }) => ref !== null && !/^[a-f0-9]{40,64}$/.test(ref))) fail("Variant references must be resolved commits");
|
|
158
|
+
if (!Number.isInteger(manifest.orderOffset) || manifest.orderOffset < 0 || manifest.orderOffset >= manifest.variants.length) fail("Invalid variant order offset");
|
|
159
|
+
if (!["screen", "holdout", "all"].includes(manifest.split) || (manifest.split !== "all" && manifest.caseIds.some((id) => caseById(id, "2").split !== manifest.split))) fail("Invalid benchmark split");
|
|
160
|
+
}
|
|
161
|
+
const expected = matrix(manifest.hosts, manifest.caseIds, manifest.repeats, manifest.variants?.map(({ id }) => id), manifest.orderOffset);
|
|
162
|
+
if (!Array.isArray(manifest.trials) || manifest.trials.length !== expected.length) fail("Incomplete benchmark matrix");
|
|
163
|
+
for (const [index, trial] of manifest.trials.entries()) {
|
|
164
|
+
for (const key of ["id", "host", "caseId", "repetition", "variant"]) if (trial[key] !== expected[index][key]) fail("Invalid benchmark matrix");
|
|
165
|
+
for (const key of ["promptSha256", "taskSha256", "initialWorkspaceSha256"]) if (!/^[a-f0-9]{64}$/.test(trial[key])) fail(`Invalid ${key}`);
|
|
166
|
+
if (!relativePath(trial.promptPath) || !Number.isSafeInteger(trial.promptBytes) || trial.promptBytes < 1) fail("Invalid prompt metadata");
|
|
167
|
+
if (manifest.schemaVersion === 2 && (trial.promptCommit !== manifest.variants.find(({ id }) => id === trial.variant).ref || trial.split !== caseById(trial.caseId, "2").split ||
|
|
168
|
+
trial.promptSurface !== (trial.promptPath.startsWith("skills/") ? "canonical-skill" : "package-asset"))) fail("Invalid variant source metadata");
|
|
169
|
+
}
|
|
170
|
+
return manifest;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
export function validateAdapter(config) {
|
|
174
|
+
if (!config || !["json", "draht-jsonl"].includes(config.format)) fail("Adapter format must be json or draht-jsonl");
|
|
175
|
+
for (const key of Object.keys(config)) if (!["format", "command", "provider", "model", "settings", "timeoutMs"].includes(key)) fail(`Unknown adapter configuration field: ${key}`);
|
|
176
|
+
if (!Array.isArray(config.command) || !config.command.length || config.command.some((arg) => typeof arg !== "string" || !arg || arg.includes("\0"))) fail("Adapter command must be a nonempty argv array");
|
|
177
|
+
for (const key of ["provider", "model"]) if (typeof config[key] !== "string" || !config[key].trim()) fail(`Adapter ${key} is required`);
|
|
178
|
+
if (config.settings !== undefined && (!config.settings || typeof config.settings !== "object" || Array.isArray(config.settings) || Object.values(config.settings).some((value) => !["string", "boolean", "number"].includes(typeof value) || (typeof value === "number" && !Number.isFinite(value))))) fail("Adapter settings must contain primitive values");
|
|
179
|
+
const timeoutMs = config.timeoutMs ?? 300_000;
|
|
180
|
+
if (!Number.isInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 3_600_000) fail("Adapter timeoutMs must be from 1 to 3600000");
|
|
181
|
+
return { ...config, settings: config.settings ?? {}, timeoutMs };
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
export function runProcess(command, options) {
|
|
185
|
+
return new Promise((finish) => {
|
|
186
|
+
const started = performance.now();
|
|
187
|
+
let stdout = "", stderr = "", error = null, timedOut = false, bytes = 0, settled = false;
|
|
188
|
+
const { env, gitEnvironment } = fixtureEnvironment(options.cwd, options.env);
|
|
189
|
+
const child = spawn(command[0], command.slice(1), { cwd: options.cwd, env,
|
|
190
|
+
stdio: ["pipe", "pipe", "pipe"], detached: process.platform !== "win32", shell: false });
|
|
191
|
+
const kill = () => {
|
|
192
|
+
try { if (child.pid && process.platform !== "win32") process.kill(-child.pid, "SIGKILL"); else child.kill("SIGKILL"); } catch {}
|
|
193
|
+
};
|
|
194
|
+
const settle = (exitCode, signal) => {
|
|
195
|
+
if (settled) return;
|
|
196
|
+
settled = true;
|
|
197
|
+
clearTimeout(timer);
|
|
198
|
+
finish({ exitCode, signal, error, timedOut, elapsedMs: Math.round(performance.now() - started), stdout, stderr, gitEnvironment });
|
|
199
|
+
};
|
|
200
|
+
const stop = () => {
|
|
201
|
+
kill();
|
|
202
|
+
child.stdin.destroy();
|
|
203
|
+
child.stdout.destroy();
|
|
204
|
+
child.stderr.destroy();
|
|
205
|
+
child.unref();
|
|
206
|
+
settle(null, "SIGKILL");
|
|
207
|
+
};
|
|
208
|
+
const timer = setTimeout(() => { timedOut = true; error = "Process timed out"; stop(); }, options.timeoutMs);
|
|
209
|
+
for (const [stream, key] of [[child.stdout, "stdout"], [child.stderr, "stderr"]]) {
|
|
210
|
+
stream.setEncoding("utf8");
|
|
211
|
+
stream.on("data", (chunk) => {
|
|
212
|
+
if (settled) return;
|
|
213
|
+
bytes += Buffer.byteLength(chunk);
|
|
214
|
+
if (bytes > 10 * 1024 * 1024) { error = "Process output exceeded 10 MiB"; stop(); return; }
|
|
215
|
+
if (key === "stdout") stdout += chunk; else stderr += chunk;
|
|
216
|
+
});
|
|
217
|
+
}
|
|
218
|
+
child.on("error", (cause) => { error = cause.message; });
|
|
219
|
+
child.stdin.on("error", () => {});
|
|
220
|
+
child.stdin.end(options.stdin ?? "");
|
|
221
|
+
child.on("close", settle);
|
|
222
|
+
});
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function controlMatches(output, trial, manifestHash) {
|
|
226
|
+
const locations = paths(output, trial);
|
|
227
|
+
try {
|
|
228
|
+
return hash(readFileSync(join(output, "manifest.json"))) === manifestHash &&
|
|
229
|
+
hash(readFileSync(locations.promptFile)) === trial.promptSha256 &&
|
|
230
|
+
hash(readFileSync(locations.taskFile)) === trial.taskSha256;
|
|
231
|
+
} catch { return false; }
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
export async function runBank(output, config, onProgress = () => {}) {
|
|
235
|
+
output = resolve(output);
|
|
236
|
+
const manifest = readManifest(output);
|
|
237
|
+
const adapter = validateAdapter(config);
|
|
238
|
+
const manifestHash = hash(readFileSync(join(output, "manifest.json")));
|
|
239
|
+
for (const trial of manifest.trials) {
|
|
240
|
+
if (existsSync(join(paths(output, trial).control, "execution.json"))) fail("This bank has execution records; prepare a fresh bank rather than overwriting trials");
|
|
241
|
+
if (!controlMatches(output, trial, manifestHash)) fail(`Changed control artifacts: ${trial.id}`);
|
|
242
|
+
assertFreshWorkspace(output, trial, manifest.suiteVersion);
|
|
243
|
+
}
|
|
244
|
+
// Validate adapter arguments before starting any run.
|
|
245
|
+
const first = paths(output, manifest.trials[0]);
|
|
246
|
+
buildInvocation(adapter, { ...first, prompt: readFileSync(first.promptFile, "utf8"), task: readFileSync(first.taskFile, "utf8") });
|
|
247
|
+
// Reject unsupported hosts before spending provider tokens. Each native tool worker
|
|
248
|
+
// and each acceptance process also establishes confinement independently.
|
|
249
|
+
const preflight = await createIsolation(first.workspace);
|
|
250
|
+
preflight.dispose();
|
|
251
|
+
for (const [index, trial] of manifest.trials.entries()) {
|
|
252
|
+
const locations = paths(output, trial);
|
|
253
|
+
onProgress({ index: index + 1, total: manifest.trials.length, trial });
|
|
254
|
+
if (!controlMatches(output, trial, manifestHash)) fail(`Changed control artifacts: ${trial.id}`);
|
|
255
|
+
assertFreshWorkspace(output, trial, manifest.suiteVersion);
|
|
256
|
+
const invocation = buildInvocation(adapter, { ...locations, prompt: readFileSync(locations.promptFile, "utf8"), task: readFileSync(locations.taskFile, "utf8") });
|
|
257
|
+
json(join(locations.control, "execution.json"), { state: "running", adapter, manifestSha256: manifestHash,
|
|
258
|
+
isolationPreflight: preflight.evidence });
|
|
259
|
+
const processResult = await runProcess(invocation.command, { cwd: locations.workspace, timeoutMs: adapter.timeoutMs, stdin: invocation.stdin });
|
|
260
|
+
writeFileSync(join(locations.control, "stdout.txt"), processResult.stdout);
|
|
261
|
+
writeFileSync(join(locations.control, "stderr.txt"), processResult.stderr);
|
|
262
|
+
let response;
|
|
263
|
+
try { response = parseOutput(adapter.format, processResult.stdout); }
|
|
264
|
+
catch (cause) { response = { finalText: "", usage: null, toolCalls: null, telemetrySource: adapter.format === "json" ? "adapter-reported" : "draht-events", error: cause.message }; }
|
|
265
|
+
let workspaceSha256 = null, workspaceSnapshotError = null;
|
|
266
|
+
try { workspaceSha256 = workspaceDigest(locations.workspace); } catch (cause) { workspaceSnapshotError = cause.message; }
|
|
267
|
+
const { stdout: _stdout, stderr: _stderr, ...processEvidence } = processResult;
|
|
268
|
+
json(join(locations.control, "execution.json"), { state: "finished", adapter, manifestSha256: manifestHash,
|
|
269
|
+
isolationPreflight: preflight.evidence,
|
|
270
|
+
command: invocation.command, process: processEvidence, response,
|
|
271
|
+
workspaceSha256, workspaceSnapshotError, stdoutSha256: hash(processResult.stdout), stderrSha256: hash(processResult.stderr),
|
|
272
|
+
controlIntact: controlMatches(output, trial, manifestHash) });
|
|
273
|
+
}
|
|
274
|
+
return gradeBank(output);
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
function scanWorkspace(workspace) {
|
|
278
|
+
const actual = new Map();
|
|
279
|
+
const modes = new Map();
|
|
280
|
+
const violations = [];
|
|
281
|
+
let bytes = 0, visited = 0;
|
|
282
|
+
const rootStat = lstatSync(workspace);
|
|
283
|
+
if (!rootStat.isDirectory() || rootStat.isSymbolicLink()) fail("Workspace must be a regular directory");
|
|
284
|
+
const walk = (dir, prefix = "", depth = 0) => {
|
|
285
|
+
if (depth > 20) fail("Fixture exceeds grading depth limit");
|
|
286
|
+
for (const name of readdirSync(dir)) {
|
|
287
|
+
if (++visited > 1000) fail("Fixture exceeds grading file limit");
|
|
288
|
+
const file = join(dir, name), relative = `${prefix}${name}`, stat = lstatSync(file);
|
|
289
|
+
if (stat.isSymbolicLink() || (!stat.isDirectory() && !stat.isFile())) { violations.push(`Unsupported file type: ${relative}`); continue; }
|
|
290
|
+
if (stat.isDirectory()) { walk(file, `${relative}/`, depth + 1); continue; }
|
|
291
|
+
bytes += stat.size;
|
|
292
|
+
if (bytes > 20 * 1024 * 1024 || actual.size >= 1000) fail("Fixture exceeds grading size limit");
|
|
293
|
+
actual.set(relative, readFileSync(file));
|
|
294
|
+
modes.set(relative, stat.mode & 0o777);
|
|
295
|
+
}
|
|
296
|
+
};
|
|
297
|
+
walk(workspace);
|
|
298
|
+
return { actual, modes, violations };
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
function workspaceDigest(workspace) {
|
|
302
|
+
const { actual, modes, violations } = scanWorkspace(workspace);
|
|
303
|
+
if (violations.length) fail(violations.join("; "));
|
|
304
|
+
return hash(JSON.stringify([...actual.keys()].sort().map((path) => [path, modes.get(path), hash(actual.get(path))])));
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
function inspectWorkspace(workspace, entry) {
|
|
308
|
+
const { actual, violations } = scanWorkspace(workspace);
|
|
309
|
+
for (const [name, original] of Object.entries(entry.files)) {
|
|
310
|
+
if (!actual.has(name)) violations.push(`Missing original file: ${name}`);
|
|
311
|
+
else if (!entry.writable.includes(name) && !actual.get(name).equals(Buffer.from(original))) violations.push(`Changed protected file: ${name}`);
|
|
312
|
+
}
|
|
313
|
+
for (const name of actual.keys()) if (!(name in entry.files) && !entry.allowedNewFiles.includes(name)) violations.push(`Out-of-scope file: ${name}`);
|
|
314
|
+
for (const name of entry.requiredNewFiles ?? []) if (!actual.has(name)) violations.push(`Missing required file: ${name}`);
|
|
315
|
+
return violations;
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
function assertFreshWorkspace(output, trial, version) {
|
|
319
|
+
const entry = caseById(trial.caseId, version);
|
|
320
|
+
const violations = inspectWorkspace(paths(output, trial).workspace, { ...entry, writable: [], allowedNewFiles: [], requiredNewFiles: [] });
|
|
321
|
+
if (violations.length) fail(`Workspace differs from prepared fixture: ${trial.id}: ${violations.join("; ")}`);
|
|
322
|
+
if (workspaceDigest(paths(output, trial).workspace) !== trial.initialWorkspaceSha256) fail(`Workspace differs from prepared fixture: ${trial.id}`);
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
async function gradeTrial(output, trial, manifestHash, version) {
|
|
326
|
+
const locations = paths(output, trial), entry = caseById(trial.caseId, version);
|
|
327
|
+
let execution = null;
|
|
328
|
+
try { execution = readJson(join(locations.control, "execution.json")); } catch {}
|
|
329
|
+
let evidenceIntact = false;
|
|
330
|
+
let stdout = "";
|
|
331
|
+
try {
|
|
332
|
+
stdout = readFileSync(join(locations.control, "stdout.txt"), "utf8");
|
|
333
|
+
evidenceIntact = hash(stdout) === execution.stdoutSha256 &&
|
|
334
|
+
hash(readFileSync(join(locations.control, "stderr.txt"))) === execution.stderrSha256 &&
|
|
335
|
+
JSON.stringify(parseOutput(execution.adapter.format, stdout)) === JSON.stringify(execution.response) &&
|
|
336
|
+
workspaceDigest(locations.workspace) === execution.workspaceSha256;
|
|
337
|
+
} catch {}
|
|
338
|
+
const identity = version === "2" ? inspectIdentity(stdout, execution?.adapter) : null;
|
|
339
|
+
const isolation = inspectIsolation(stdout, execution?.adapter, locations.workspace);
|
|
340
|
+
const executionPassed = execution?.state === "finished" && execution?.process?.exitCode === 0 &&
|
|
341
|
+
!execution.process.error && !execution.process.signal && !execution.process.timedOut &&
|
|
342
|
+
!execution.response?.error && typeof execution.response?.finalText === "string" &&
|
|
343
|
+
execution.controlIntact === true && execution.manifestSha256 === manifestHash && controlMatches(output, trial, manifestHash) && evidenceIntact && identity?.passed !== false && isolation.passed !== false;
|
|
344
|
+
const finalText = typeof execution?.response?.finalText === "string" ? execution.response.finalText : "";
|
|
345
|
+
const statuses = [...finalText.matchAll(/^STATUS:\s*(DONE_WITH_CONCERNS|DONE|NEEDS_CONTEXT|BLOCKED)\s*$/gm)];
|
|
346
|
+
const status = statuses.length === 1 && finalText.trimEnd().endsWith(statuses[0][0].trimEnd()) ? statuses[0][1] : null;
|
|
347
|
+
let violations = [];
|
|
348
|
+
let acceptance = null;
|
|
349
|
+
let acceptancePassed = false;
|
|
350
|
+
let acceptanceError = null;
|
|
351
|
+
try {
|
|
352
|
+
violations = inspectWorkspace(locations.workspace, entry);
|
|
353
|
+
} catch (cause) { violations.push(cause.message); }
|
|
354
|
+
if (!violations.length) {
|
|
355
|
+
try {
|
|
356
|
+
const marker = `drahtbank:${randomUUID()}`;
|
|
357
|
+
const source = `import assert from "node:assert/strict";\nconst finalText = ${JSON.stringify(finalText)};\n${entry.assertions}\nconsole.log(${JSON.stringify(marker)});\n`;
|
|
358
|
+
const confined = await createIsolation(locations.workspace);
|
|
359
|
+
try {
|
|
360
|
+
acceptance = { ...await confined.run([confined.nodePath, "--input-type=module", "-e", source], { timeoutMs: 10_000 }),
|
|
361
|
+
gitEnvironment: fixtureEnvironment(locations.workspace).gitEnvironment, isolation: confined.evidence };
|
|
362
|
+
} finally { confined.dispose(); }
|
|
363
|
+
acceptancePassed = acceptance.exitCode === 0 && !acceptance.signal && !acceptance.error && acceptance.stdout.split(/\r?\n/).includes(marker);
|
|
364
|
+
} catch (cause) { acceptanceError = `Acceptance isolation failed: ${cause.message}`; }
|
|
365
|
+
try {
|
|
366
|
+
// A grader must not leave protected files changed either.
|
|
367
|
+
violations.push(...inspectWorkspace(locations.workspace, entry));
|
|
368
|
+
if (execution?.workspaceSha256 && workspaceDigest(locations.workspace) !== execution.workspaceSha256) violations.push("Workspace changed after runner completion");
|
|
369
|
+
} catch (cause) { violations.push(cause.message); }
|
|
370
|
+
}
|
|
371
|
+
const responseGrade = version === "2" ? gradeResponse(entry, { finalText, stdout, format: execution?.adapter?.format, workspace: locations.workspace }) : null;
|
|
372
|
+
if (responseGrade) violations.push(...responseGrade.traceScopeViolations);
|
|
373
|
+
const statusPassed = responseGrade ? responseGrade.statusSyntaxPassed && responseGrade.statusSemanticPassed : entry.expectedStatuses.includes(status);
|
|
374
|
+
const requiredEvidencePassed = !responseGrade || !entry.evidenceRequirements?.length || responseGrade.evidencePassed === true;
|
|
375
|
+
return { ...trial, passed: executionPassed && acceptancePassed && statusPassed && !violations.length &&
|
|
376
|
+
(!responseGrade || (responseGrade.responseFormatPassed && responseGrade.findingsPassed !== false && requiredEvidencePassed)),
|
|
377
|
+
executionPassed, evidenceIntact, acceptancePassed, statusPassed, status, scopePassed: !violations.length, scopeViolations: violations, isolation,
|
|
378
|
+
failure: execution?.process?.error ?? execution?.response?.error ?? (execution?.state !== "finished" ? "No completed execution" :
|
|
379
|
+
execution?.process?.signal ? `Runner terminated by ${execution.process.signal}` : execution?.process?.exitCode !== 0 ? `Runner exited ${execution?.process?.exitCode}` : !evidenceIntact ? "Execution artifacts changed or incomplete" : identity?.passed === false ? "Native model identity mismatch or unavailable" : isolation.passed === false ? "Native isolation readiness missing or invalid" : acceptanceError),
|
|
380
|
+
acceptance, elapsedMs: execution?.process?.elapsedMs ?? null,
|
|
381
|
+
usage: execution?.response?.usage ?? null, toolCalls: execution?.response?.toolCalls ?? null,
|
|
382
|
+
telemetrySource: execution?.response?.telemetrySource ?? null, adapter: execution?.adapter ?? null,
|
|
383
|
+
...(responseGrade ? { ...responseGrade, behaviorPassed: acceptance === null ? null : acceptancePassed, identity } : {}) };
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
function inspectIsolation(stdout, adapter, workspace) {
|
|
387
|
+
if (adapter?.format !== "draht-jsonl") return { passed: null, note: "Generic runner tool isolation is unverified" };
|
|
388
|
+
let receipt = null;
|
|
389
|
+
try {
|
|
390
|
+
for (const line of stdout.split(/\r?\n/).filter((line) => line.trim())) {
|
|
391
|
+
const event = JSON.parse(line);
|
|
392
|
+
if (event.type === "drahtbank_isolation") {
|
|
393
|
+
const evidence = event.evidence;
|
|
394
|
+
if (receipt || event.schemaVersion !== 1 || event.policy !== "fixture-root-v1" ||
|
|
395
|
+
event.toolOperationsVersion !== "fixture-operations-v1" ||
|
|
396
|
+
event.workspace !== realpathSync(workspace) || evidence?.policy !== ISOLATION_VERSION ||
|
|
397
|
+
evidence.workspace !== event.workspace || evidence.hostProc !== false || evidence.inheritedEnvironment !== false ||
|
|
398
|
+
evidence.network !== "private namespace" ||
|
|
399
|
+
evidence.readonlyArtifactsPath !== "/.drahtbank/artifacts" ||
|
|
400
|
+
!Array.isArray(evidence.runtimeRoots) || evidence.runtimeRoots.length === 0 ||
|
|
401
|
+
!evidence.runtimeRoots.every((root) => typeof root === "string" && root.startsWith("/") && root !== "/") ||
|
|
402
|
+
evidence.workerSha256 !== hash(readFileSync(new URL("isolation-worker.mjs", import.meta.url)))) throw new Error("Invalid isolation receipt");
|
|
403
|
+
receipt = event;
|
|
404
|
+
}
|
|
405
|
+
if (["agent_start", "tool_execution_start", "message_start", "message_end"].includes(event.type) && !receipt) {
|
|
406
|
+
throw new Error("Missing isolation receipt before agent execution");
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
return { passed: receipt !== null, receipt, note: "Trusted native extension readiness, not independent runtime attestation" };
|
|
410
|
+
} catch (error) {
|
|
411
|
+
return { passed: false, note: error.message };
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
function inspectIdentity(stdout, adapter) {
|
|
416
|
+
if (adapter?.format !== "draht-jsonl") return { passed: null, observed: 0, mismatches: [], note: "Generic adapter identity is unverified" };
|
|
417
|
+
let observed = 0;
|
|
418
|
+
const mismatches = [];
|
|
419
|
+
try {
|
|
420
|
+
for (const line of stdout.split(/\r?\n/).filter((line) => line.trim())) {
|
|
421
|
+
const event = JSON.parse(line);
|
|
422
|
+
if (event.type !== "message_end" || event.message?.role !== "assistant") continue;
|
|
423
|
+
observed++;
|
|
424
|
+
if (event.message.provider !== adapter.provider || event.message.model !== adapter.model) mismatches.push({ provider: event.message.provider ?? null, model: event.message.model ?? null });
|
|
425
|
+
}
|
|
426
|
+
} catch { mismatches.push({ error: "Malformed stream" }); }
|
|
427
|
+
return { passed: observed > 0 && !mismatches.length, observed, mismatches, note: "Observed CLI metadata, not provider attestation" };
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
const median = (values) => {
|
|
431
|
+
if (!values.length) return null;
|
|
432
|
+
const sorted = [...values].sort((a, b) => a - b), middle = Math.floor(sorted.length / 2);
|
|
433
|
+
return sorted.length % 2 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2;
|
|
434
|
+
};
|
|
435
|
+
|
|
436
|
+
export function summarize(results, variantIds = ["baseline", "candidate"]) {
|
|
437
|
+
const variants = {};
|
|
438
|
+
const axisKeys = ["executionPassed", "behaviorPassed", "evidencePassed", "statusSyntaxPassed", "statusSemanticPassed", "responseFormatPassed", "findingsPassed", "scopePassed"];
|
|
439
|
+
for (const variant of variantIds) {
|
|
440
|
+
const rows = results.filter((row) => row.variant === variant);
|
|
441
|
+
const metrics = {};
|
|
442
|
+
for (const key of ["inputTokens", "outputTokens", "cacheReadTokens", "cacheWriteTokens", "costUsd", "toolCalls", "elapsedMs"]) {
|
|
443
|
+
const values = rows.map((row) => key === "toolCalls" || key === "elapsedMs" ? row[key] : row.usage?.[key]).filter((value) => typeof value === "number" && Number.isFinite(value));
|
|
444
|
+
metrics[key] = { observed: values.length, total: values.length ? values.reduce((a, b) => a + b, 0) : null, median: median(values) };
|
|
445
|
+
}
|
|
446
|
+
variants[variant] = { passed: rows.filter((row) => row.passed).length, total: rows.length,
|
|
447
|
+
promptBytes: rows.reduce((sum, row) => sum + row.promptBytes, 0), metrics };
|
|
448
|
+
if (rows.some((row) => Object.hasOwn(row, "behaviorPassed"))) variants[variant].axes = Object.fromEntries(
|
|
449
|
+
axisKeys.map((key) => [key, {
|
|
450
|
+
passed: rows.filter((row) => row[key] === true).length, failed: rows.filter((row) => row[key] === false).length,
|
|
451
|
+
unknownOrNotApplicable: rows.filter((row) => row[key] !== true && row[key] !== false).length,
|
|
452
|
+
}]),
|
|
453
|
+
);
|
|
454
|
+
}
|
|
455
|
+
const pairKey = (row) => JSON.stringify([row.host, row.caseId, row.repetition]);
|
|
456
|
+
const pairResult = (baseline, candidate, key) => ({
|
|
457
|
+
host: baseline.host, caseId: baseline.caseId, repetition: baseline.repetition,
|
|
458
|
+
baselinePassed: baseline[key], candidatePassed: candidate[key],
|
|
459
|
+
elapsedDeltaMs: baseline[key] === true && candidate[key] === true &&
|
|
460
|
+
Number.isFinite(baseline.elapsedMs) && Number.isFinite(candidate.elapsedMs) ? candidate.elapsedMs - baseline.elapsedMs : null,
|
|
461
|
+
});
|
|
462
|
+
const pairSummary = (pairs) => ({
|
|
463
|
+
total: pairs.length, bothPassed: pairs.filter((pair) => pair.baselinePassed && pair.candidatePassed).length,
|
|
464
|
+
regressions: pairs.filter((pair) => pair.baselinePassed && !pair.candidatePassed),
|
|
465
|
+
improvements: pairs.filter((pair) => !pair.baselinePassed && pair.candidatePassed),
|
|
466
|
+
medianElapsedDeltaMsOnPairedPasses: median(pairs.map((pair) => pair.elapsedDeltaMs).filter((value) => value !== null)),
|
|
467
|
+
});
|
|
468
|
+
const compare = (baselineVariant, candidateVariant) => {
|
|
469
|
+
const baselineRows = results.filter((row) => row.variant === baselineVariant);
|
|
470
|
+
const candidateRows = results.filter((row) => row.variant === candidateVariant);
|
|
471
|
+
const candidates = new Map(candidateRows.map((row) => [pairKey(row), row]));
|
|
472
|
+
const matches = baselineRows.flatMap((baseline) => {
|
|
473
|
+
const candidate = candidates.get(pairKey(baseline));
|
|
474
|
+
return candidate ? [{ baseline, candidate }] : [];
|
|
475
|
+
});
|
|
476
|
+
const axes = {};
|
|
477
|
+
if (matches.some(({ baseline, candidate }) => Object.hasOwn(baseline, "behaviorPassed") || Object.hasOwn(candidate, "behaviorPassed"))) {
|
|
478
|
+
for (const key of axisKeys) {
|
|
479
|
+
const pairs = matches.filter(({ baseline, candidate }) => typeof baseline[key] === "boolean" && typeof candidate[key] === "boolean")
|
|
480
|
+
.map(({ baseline, candidate }) => pairResult(baseline, candidate, key));
|
|
481
|
+
axes[key] = { ...pairSummary(pairs), bothFailed: pairs.filter((pair) => !pair.baselinePassed && !pair.candidatePassed).length,
|
|
482
|
+
unknownOrNotApplicable: matches.length - pairs.length };
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
return { baselineVariant, candidateVariant, matchedPairs: matches.length,
|
|
486
|
+
unpairedBaseline: baselineRows.length - matches.length, unpairedCandidate: candidateRows.length - matches.length,
|
|
487
|
+
strict: pairSummary(matches.map(({ baseline, candidate }) => pairResult(baseline, candidate, "passed"))), axes };
|
|
488
|
+
};
|
|
489
|
+
const comparisons = variantIds.slice(1).map((variant) => compare(variantIds[0], variant));
|
|
490
|
+
const paired = variantIds.includes("baseline") && variantIds.includes("candidate")
|
|
491
|
+
? compare("baseline", "candidate").strict : pairSummary([]);
|
|
492
|
+
const identities = new Set(results.filter((row) => row.adapter).map((row) => JSON.stringify(row.adapter)));
|
|
493
|
+
return { variants, paired, comparisons,
|
|
494
|
+
comparableRunner: identities.size === 1 && results.every((row) => row.adapter),
|
|
495
|
+
limits: ["Fixture scores are not general model quality; text rubrics are narrow checks, not semantic judges.",
|
|
496
|
+
"Prompt bytes are UTF-8 asset sizes, not model tokens. Missing usage remains unknown.",
|
|
497
|
+
"Generic adapter telemetry is adapter-reported. Elapsed time is runner-measured and includes startup.",
|
|
498
|
+
"Prompt text is tested on the selected runner; this does not test installed host skill routing or hooks.",
|
|
499
|
+
"Native tools and acceptance processes require fixture-root confinement; generic runner tools remain adapter-controlled.",
|
|
500
|
+
"Filesystem confinement is not a CPU or memory resource quota. Cache state and external model nondeterminism are uncontrolled."] };
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
export async function gradeBank(output) {
|
|
504
|
+
output = resolve(output);
|
|
505
|
+
const manifest = readManifest(output), manifestHash = hash(readFileSync(join(output, "manifest.json")));
|
|
506
|
+
const results = [];
|
|
507
|
+
for (const trial of manifest.trials) results.push(await gradeTrial(output, trial, manifestHash, manifest.suiteVersion));
|
|
508
|
+
const report = { schemaVersion: manifest.schemaVersion, suiteVersion: manifest.suiteVersion, suiteSha256: manifest.suiteSha256,
|
|
509
|
+
baselineCommit: manifest.baselineCommit, candidateCommit: manifest.candidateCommit, gradedAt: new Date().toISOString(),
|
|
510
|
+
gradingEnvironment: { node: process.version, platform: process.platform, arch: process.arch },
|
|
511
|
+
...(manifest.schemaVersion === 2 ? { variants: manifest.variants, split: manifest.split } : {}),
|
|
512
|
+
results, summary: summarize(results, manifest.variants?.map(({ id }) => id)) };
|
|
513
|
+
json(join(output, "report.json"), report);
|
|
514
|
+
return report;
|
|
515
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { BenchmarkCase } from "./cases.mjs";
|
|
2
|
+
export { gradeResponse } from "./grading-v2.mjs";
|
|
3
|
+
export type { ResponseGrade, ResponseInput } from "./grading-v2.mjs";
|
|
4
|
+
|
|
5
|
+
export interface EvidenceRequirement {
|
|
6
|
+
id: string;
|
|
7
|
+
command: string;
|
|
8
|
+
exitCode: number;
|
|
9
|
+
afterEdits: boolean;
|
|
10
|
+
outputLine?: string;
|
|
11
|
+
outputJson?: Record<string, unknown>;
|
|
12
|
+
}
|
|
13
|
+
export interface BenchmarkCaseV2 extends BenchmarkCase {
|
|
14
|
+
split: "screen" | "holdout";
|
|
15
|
+
promptKind: "agent" | "skill" | "command";
|
|
16
|
+
promptName: string;
|
|
17
|
+
outputContract: "status" | "triage";
|
|
18
|
+
responseInstructions: string;
|
|
19
|
+
evidenceRequirements: EvidenceRequirement[];
|
|
20
|
+
responseContract?: {
|
|
21
|
+
verdict: string;
|
|
22
|
+
findings: { file: string; line: number }[];
|
|
23
|
+
facts: Record<string, unknown>;
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
export const SUITE_VERSION: "2";
|
|
27
|
+
export const GRADING_SOURCES: string[];
|
|
28
|
+
export const CASES: BenchmarkCaseV2[];
|