@bjoernrochel/sevro 0.1.0-rc.8 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +52 -6
- package/README.md +12 -7
- package/dist/examples/basic/graded.json +13 -0
- package/dist/examples/basic/host.js +9 -0
- package/dist/examples/basic/host.js.map +1 -0
- package/dist/examples/basic/prompt-only.json +7 -0
- package/dist/examples/recipes/candidate.js +23 -0
- package/dist/examples/recipes/candidate.js.map +1 -0
- package/dist/examples/recipes/cases/coding-change.json +21 -0
- package/dist/examples/recipes/cases/host-tools.json +21 -0
- package/dist/examples/recipes/cases/mise.json +21 -0
- package/dist/examples/recipes/cases/node.json +21 -0
- package/dist/examples/recipes/cases/python-uv.json +21 -0
- package/dist/examples/recipes/cases/semantic.json +17 -0
- package/dist/examples/recipes/coding-change/check.js +16 -0
- package/dist/examples/recipes/coding-change/check.js.map +1 -0
- package/dist/examples/recipes/executable.js +20 -0
- package/dist/examples/recipes/executable.js.map +1 -0
- package/dist/examples/recipes/extension-artifacts.js +56 -0
- package/dist/examples/recipes/extension-artifacts.js.map +1 -0
- package/dist/examples/recipes/extension-methods.js +167 -0
- package/dist/examples/recipes/extension-methods.js.map +1 -0
- package/dist/examples/recipes/extension.js +28 -0
- package/dist/examples/recipes/extension.js.map +1 -0
- package/dist/examples/recipes/modules.js +9 -0
- package/dist/examples/recipes/modules.js.map +1 -0
- package/dist/examples/recipes/repository-skill/.agents/skills/startup-guide/SKILL.md +7 -0
- package/dist/examples/recipes/repository-skill/.claude/skills/startup-guide/SKILL.md +7 -0
- package/dist/examples/recipes/repository-skill/README.md +3 -0
- package/dist/examples/recipes/run-inputs.js +144 -0
- package/dist/examples/recipes/run-inputs.js.map +1 -0
- package/dist/examples/recipes/run-options.js +103 -0
- package/dist/examples/recipes/run-options.js.map +1 -0
- package/dist/examples/recipes/run.js +62 -0
- package/dist/examples/recipes/run.js.map +1 -0
- package/dist/examples/recipes/runtime/empty.json +1 -0
- package/dist/examples/recipes/runtime/goals.json +1 -0
- package/dist/examples/recipes/runtime/mise.json +6 -0
- package/dist/examples/recipes/runtime/path.json +1 -0
- package/dist/examples/recipes/runtime/plugins.json +4 -0
- package/dist/examples/recipes/runtime/python-uv.json +10 -0
- package/dist/examples/recipes/semantic-judge.js +27 -0
- package/dist/examples/recipes/semantic-judge.js.map +1 -0
- package/dist/examples/templates/coding/adapter.js +14 -0
- package/dist/examples/templates/coding/adapter.js.map +1 -0
- package/dist/examples/templates/coding/case.json +15 -0
- package/dist/examples/templates/repository-skill/.agents/skills/startup-guide/SKILL.md +6 -0
- package/dist/examples/templates/repository-skill/extension-data.js +61 -0
- package/dist/examples/templates/repository-skill/extension-data.js.map +1 -0
- package/dist/examples/templates/repository-skill/extension.js +29 -0
- package/dist/examples/templates/repository-skill/extension.js.map +1 -0
- package/dist/examples/templates/response/adapter.js +13 -0
- package/dist/examples/templates/response/adapter.js.map +1 -0
- package/dist/examples/templates/response/case.json +13 -0
- package/dist/src/advisory.js +86 -0
- package/dist/src/advisory.js.map +1 -0
- package/dist/src/candidate-artifacts.js +139 -0
- package/dist/src/candidate-artifacts.js.map +1 -0
- package/dist/src/candidate-codex-state.js +110 -0
- package/dist/src/candidate-codex-state.js.map +1 -0
- package/dist/src/case-validation.js +92 -0
- package/dist/src/case-validation.js.map +1 -0
- package/dist/src/cli/cancellation.js +21 -0
- package/dist/src/cli/cancellation.js.map +1 -0
- package/dist/src/cli/command.js +60 -0
- package/dist/src/cli/command.js.map +1 -0
- package/dist/src/cli/completion.js +14 -0
- package/dist/src/cli/completion.js.map +1 -0
- package/dist/src/cli/demo.js +45 -0
- package/dist/src/cli/demo.js.map +1 -0
- package/dist/src/cli/evaluation.js +40 -0
- package/dist/src/cli/evaluation.js.map +1 -0
- package/dist/src/cli/extension.js +54 -0
- package/dist/src/cli/extension.js.map +1 -0
- package/dist/src/cli/help.js +190 -0
- package/dist/src/cli/help.js.map +1 -0
- package/dist/src/cli/hosts.js +33 -0
- package/dist/src/cli/hosts.js.map +1 -0
- package/dist/src/cli/init-settings.js +29 -0
- package/dist/src/cli/init-settings.js.map +1 -0
- package/dist/src/cli/init.js +82 -0
- package/dist/src/cli/init.js.map +1 -0
- package/dist/src/cli/invocation.js +431 -0
- package/dist/src/cli/invocation.js.map +1 -0
- package/dist/src/cli/loading.js +110 -0
- package/dist/src/cli/loading.js.map +1 -0
- package/dist/src/cli/preparation.js +88 -0
- package/dist/src/cli/preparation.js.map +1 -0
- package/dist/src/cli/presentation.js +61 -0
- package/dist/src/cli/presentation.js.map +1 -0
- package/dist/src/cli/progress.js +5 -0
- package/dist/src/cli/progress.js.map +1 -0
- package/dist/src/cli/run-manifest.js +108 -0
- package/dist/src/cli/run-manifest.js.map +1 -0
- package/dist/src/cli/run-options.js +367 -0
- package/dist/src/cli/run-options.js.map +1 -0
- package/dist/src/cli/run-overrides.js +51 -0
- package/dist/src/cli/run-overrides.js.map +1 -0
- package/dist/src/cli/settings.js +127 -0
- package/dist/src/cli/settings.js.map +1 -0
- package/dist/src/cli/summary.js +28 -0
- package/dist/src/cli/summary.js.map +1 -0
- package/dist/src/cli.js +20 -0
- package/dist/src/cli.js.map +1 -0
- package/dist/src/contract-types.js +2 -0
- package/dist/src/contract-types.js.map +1 -0
- package/dist/src/engine.js +4 -0
- package/dist/src/engine.js.map +1 -0
- package/dist/src/evaluation/advisory.js +85 -0
- package/dist/src/evaluation/advisory.js.map +1 -0
- package/dist/src/evaluation/check-plan.js +292 -0
- package/dist/src/evaluation/check-plan.js.map +1 -0
- package/dist/src/evaluation/collection.js +61 -0
- package/dist/src/evaluation/collection.js.map +1 -0
- package/dist/src/evaluation/context.js +2 -0
- package/dist/src/evaluation/context.js.map +1 -0
- package/dist/src/evaluation/grading-host.js +28 -0
- package/dist/src/evaluation/grading-host.js.map +1 -0
- package/dist/src/evaluation/grading.js +154 -0
- package/dist/src/evaluation/grading.js.map +1 -0
- package/dist/src/evaluation/invocation-plan.js +312 -0
- package/dist/src/evaluation/invocation-plan.js.map +1 -0
- package/dist/src/evaluation/plan.js +376 -0
- package/dist/src/evaluation/plan.js.map +1 -0
- package/dist/src/evaluation/progress.js +14 -0
- package/dist/src/evaluation/progress.js.map +1 -0
- package/dist/src/evaluation/resources.js +159 -0
- package/dist/src/evaluation/resources.js.map +1 -0
- package/dist/src/evaluation/run.js +148 -0
- package/dist/src/evaluation/run.js.map +1 -0
- package/dist/src/evaluation/scheduler.js +27 -0
- package/dist/src/evaluation/scheduler.js.map +1 -0
- package/dist/src/evaluation/semantic.js +183 -0
- package/dist/src/evaluation/semantic.js.map +1 -0
- package/dist/src/evaluation/trial.js +593 -0
- package/dist/src/evaluation/trial.js.map +1 -0
- package/dist/src/evaluation/types.js +20 -0
- package/dist/src/evaluation/types.js.map +1 -0
- package/dist/src/evidence/host-results.js +67 -0
- package/dist/src/evidence/host-results.js.map +1 -0
- package/dist/src/evidence/retained-artifacts.js +17 -0
- package/dist/src/evidence/retained-artifacts.js.map +1 -0
- package/dist/src/evidence/trial-writer.js +131 -0
- package/dist/src/evidence/trial-writer.js.map +1 -0
- package/dist/src/evidence/usage.js +14 -0
- package/dist/src/evidence/usage.js.map +1 -0
- package/dist/src/executable.js +20 -0
- package/dist/src/executable.js.map +1 -0
- package/dist/src/extension-client.js +212 -0
- package/dist/src/extension-client.js.map +1 -0
- package/dist/src/extension-session.js +236 -0
- package/dist/src/extension-session.js.map +1 -0
- package/dist/src/fixtures/advisory.js +206 -0
- package/dist/src/fixtures/advisory.js.map +1 -0
- package/dist/src/fixtures/bin.js +41 -0
- package/dist/src/fixtures/bin.js.map +1 -0
- package/dist/src/fixtures/cleanup.js +43 -0
- package/dist/src/fixtures/cleanup.js.map +1 -0
- package/dist/src/fixtures/create.js +94 -0
- package/dist/src/fixtures/create.js.map +1 -0
- package/dist/src/fixtures/generated.js +181 -0
- package/dist/src/fixtures/generated.js.map +1 -0
- package/dist/src/fixtures/git-hooks.js +16 -0
- package/dist/src/fixtures/git-hooks.js.map +1 -0
- package/dist/src/fixtures/preparation.js +201 -0
- package/dist/src/fixtures/preparation.js.map +1 -0
- package/dist/src/fixtures/repository.js +255 -0
- package/dist/src/fixtures/repository.js.map +1 -0
- package/dist/src/fixtures/setup.js +175 -0
- package/dist/src/fixtures/setup.js.map +1 -0
- package/dist/src/fixtures/tools.js +57 -0
- package/dist/src/fixtures/tools.js.map +1 -0
- package/dist/src/fixtures/workspace-state.js +155 -0
- package/dist/src/fixtures/workspace-state.js.map +1 -0
- package/dist/src/generated/cli-result.cjs +2 -0
- package/dist/src/generated/extension.cjs +2 -0
- package/dist/src/generated/report.cjs +2 -0
- package/dist/src/generated/run-evidence.cjs +2 -0
- package/dist/src/generated/runtime.cjs +2 -0
- package/dist/src/git.js +72 -0
- package/dist/src/git.js.map +1 -0
- package/dist/src/graders/artifact.js +68 -0
- package/dist/src/graders/artifact.js.map +1 -0
- package/dist/src/graders/git-head.js +112 -0
- package/dist/src/graders/git-head.js.map +1 -0
- package/dist/src/graders/output.js +345 -0
- package/dist/src/graders/output.js.map +1 -0
- package/dist/src/graders/semantic.js +132 -0
- package/dist/src/graders/semantic.js.map +1 -0
- package/dist/src/graders/shell.js +321 -0
- package/dist/src/graders/shell.js.map +1 -0
- package/dist/src/host-diagnostic-redaction.js +52 -0
- package/dist/src/host-diagnostic-redaction.js.map +1 -0
- package/dist/src/host-diagnostics.js +248 -0
- package/dist/src/host-diagnostics.js.map +1 -0
- package/dist/src/hosts/claude-continuation.js +89 -0
- package/dist/src/hosts/claude-continuation.js.map +1 -0
- package/dist/src/hosts/claude-credential.js +121 -0
- package/dist/src/hosts/claude-credential.js.map +1 -0
- package/dist/src/hosts/claude-events.js +136 -0
- package/dist/src/hosts/claude-events.js.map +1 -0
- package/dist/src/hosts/claude-keychain.js +100 -0
- package/dist/src/hosts/claude-keychain.js.map +1 -0
- package/dist/src/hosts/claude-login-recovery.js +67 -0
- package/dist/src/hosts/claude-login-recovery.js.map +1 -0
- package/dist/src/hosts/claude-login-refresh.js +146 -0
- package/dist/src/hosts/claude-login-refresh.js.map +1 -0
- package/dist/src/hosts/claude-login-store.js +124 -0
- package/dist/src/hosts/claude-login-store.js.map +1 -0
- package/dist/src/hosts/claude-login.js +74 -0
- package/dist/src/hosts/claude-login.js.map +1 -0
- package/dist/src/hosts/claude-native-goal.js +138 -0
- package/dist/src/hosts/claude-native-goal.js.map +1 -0
- package/dist/src/hosts/claude-process.js +10 -0
- package/dist/src/hosts/claude-process.js.map +1 -0
- package/dist/src/hosts/claude-refresh-lock.js +76 -0
- package/dist/src/hosts/claude-refresh-lock.js.map +1 -0
- package/dist/src/hosts/claude-repository-invocation.js +229 -0
- package/dist/src/hosts/claude-repository-invocation.js.map +1 -0
- package/dist/src/hosts/claude-settings.js +156 -0
- package/dist/src/hosts/claude-settings.js.map +1 -0
- package/dist/src/hosts/claude-skill-activations.js +131 -0
- package/dist/src/hosts/claude-skill-activations.js.map +1 -0
- package/dist/src/hosts/claude-state.js +237 -0
- package/dist/src/hosts/claude-state.js.map +1 -0
- package/dist/src/hosts/claude-tool-calls.js +150 -0
- package/dist/src/hosts/claude-tool-calls.js.map +1 -0
- package/dist/src/hosts/claude.js +373 -0
- package/dist/src/hosts/claude.js.map +1 -0
- package/dist/src/hosts/codex-app-server.js +764 -0
- package/dist/src/hosts/codex-app-server.js.map +1 -0
- package/dist/src/hosts/codex-configuration.js +91 -0
- package/dist/src/hosts/codex-configuration.js.map +1 -0
- package/dist/src/hosts/codex-events.js +104 -0
- package/dist/src/hosts/codex-events.js.map +1 -0
- package/dist/src/hosts/codex-exec-controls.js +434 -0
- package/dist/src/hosts/codex-exec-controls.js.map +1 -0
- package/dist/src/hosts/codex-invocation.js +54 -0
- package/dist/src/hosts/codex-invocation.js.map +1 -0
- package/dist/src/hosts/codex-marketplace.js +185 -0
- package/dist/src/hosts/codex-marketplace.js.map +1 -0
- package/dist/src/hosts/codex-native-calls.js +516 -0
- package/dist/src/hosts/codex-native-calls.js.map +1 -0
- package/dist/src/hosts/codex-process.js +10 -0
- package/dist/src/hosts/codex-process.js.map +1 -0
- package/dist/src/hosts/codex-profile.js +206 -0
- package/dist/src/hosts/codex-profile.js.map +1 -0
- package/dist/src/hosts/codex-session-files.js +89 -0
- package/dist/src/hosts/codex-session-files.js.map +1 -0
- package/dist/src/hosts/codex-state.js +310 -0
- package/dist/src/hosts/codex-state.js.map +1 -0
- package/dist/src/hosts/codex.js +527 -0
- package/dist/src/hosts/codex.js.map +1 -0
- package/dist/src/hosts/isolation-common.js +43 -0
- package/dist/src/hosts/isolation-common.js.map +1 -0
- package/dist/src/hosts/isolation-roots.js +64 -0
- package/dist/src/hosts/isolation-roots.js.map +1 -0
- package/dist/src/hosts/isolation.js +12 -0
- package/dist/src/hosts/isolation.js.map +1 -0
- package/dist/src/hosts/linux-sandbox.js +127 -0
- package/dist/src/hosts/linux-sandbox.js.map +1 -0
- package/dist/src/hosts/mac-sandbox.js +137 -0
- package/dist/src/hosts/mac-sandbox.js.map +1 -0
- package/dist/src/hosts/native-controls.js +151 -0
- package/dist/src/hosts/native-controls.js.map +1 -0
- package/dist/src/hosts/native-request.js +33 -0
- package/dist/src/hosts/native-request.js.map +1 -0
- package/dist/src/hosts/process-lifecycle.js +167 -0
- package/dist/src/hosts/process-lifecycle.js.map +1 -0
- package/dist/src/hosts/protected-worktrees.js +81 -0
- package/dist/src/hosts/protected-worktrees.js.map +1 -0
- package/dist/src/hosts/skill-hook-catalog.js +100 -0
- package/dist/src/hosts/skill-hook-catalog.js.map +1 -0
- package/dist/src/hosts/skill-hook-handler.js +228 -0
- package/dist/src/hosts/skill-hook-handler.js.map +1 -0
- package/dist/src/hosts/skill-hooks.js +423 -0
- package/dist/src/hosts/skill-hooks.js.map +1 -0
- package/dist/src/hosts/workspace-fingerprint.js +72 -0
- package/dist/src/hosts/workspace-fingerprint.js.map +1 -0
- package/dist/src/identity.js +167 -0
- package/dist/src/identity.js.map +1 -0
- package/dist/src/instrumentation.js +83 -0
- package/dist/src/instrumentation.js.map +1 -0
- package/dist/src/native-transcript-state.js +85 -0
- package/dist/src/native-transcript-state.js.map +1 -0
- package/dist/src/native-transcript-view.js +65 -0
- package/dist/src/native-transcript-view.js.map +1 -0
- package/dist/src/native-transcripts.js +208 -0
- package/dist/src/native-transcripts.js.map +1 -0
- package/dist/src/paths.js +13 -0
- package/dist/src/paths.js.map +1 -0
- package/dist/src/process-lifecycle.js +101 -0
- package/dist/src/process-lifecycle.js.map +1 -0
- package/dist/src/provenance.js +263 -0
- package/dist/src/provenance.js.map +1 -0
- package/dist/src/prune.js +228 -0
- package/dist/src/prune.js.map +1 -0
- package/dist/src/report.js +276 -0
- package/dist/src/report.js.map +1 -0
- package/dist/src/resolved-case.js +29 -0
- package/dist/src/resolved-case.js.map +1 -0
- package/dist/src/results-lock.js +40 -0
- package/dist/src/results-lock.js.map +1 -0
- package/dist/src/results.js +127 -0
- package/dist/src/results.js.map +1 -0
- package/dist/src/run-owner.js +283 -0
- package/dist/src/run-owner.js.map +1 -0
- package/dist/src/runner-paths.js +11 -0
- package/dist/src/runner-paths.js.map +1 -0
- package/dist/src/runtime/config.js +323 -0
- package/dist/src/runtime/config.js.map +1 -0
- package/dist/src/runtime/discovery-apple.js +65 -0
- package/dist/src/runtime/discovery-apple.js.map +1 -0
- package/dist/src/runtime/discovery-providers.js +37 -0
- package/dist/src/runtime/discovery-providers.js.map +1 -0
- package/dist/src/runtime/discovery.js +40 -0
- package/dist/src/runtime/discovery.js.map +1 -0
- package/dist/src/runtime/hook-processes.js +81 -0
- package/dist/src/runtime/hook-processes.js.map +1 -0
- package/dist/src/runtime/hooks.js +304 -0
- package/dist/src/runtime/hooks.js.map +1 -0
- package/dist/src/runtime/paths.js +244 -0
- package/dist/src/runtime/paths.js.map +1 -0
- package/dist/src/runtime/query.js +76 -0
- package/dist/src/runtime/query.js.map +1 -0
- package/dist/src/runtime/seeds.js +98 -0
- package/dist/src/runtime/seeds.js.map +1 -0
- package/dist/src/runtime/shared-cache.js +81 -0
- package/dist/src/runtime/shared-cache.js.map +1 -0
- package/dist/src/runtime/state.js +106 -0
- package/dist/src/runtime/state.js.map +1 -0
- package/dist/src/runtime-process.js +148 -0
- package/dist/src/runtime-process.js.map +1 -0
- package/dist/src/schema-types.js +2 -0
- package/dist/src/schema-types.js.map +1 -0
- package/dist/src/schema.js +16 -0
- package/dist/src/schema.js.map +1 -0
- package/dist/src/shell.js +5 -0
- package/dist/src/shell.js.map +1 -0
- package/dist/src/sqlite-lock.js +18 -0
- package/dist/src/sqlite-lock.js.map +1 -0
- package/dist/src/storage.js +21 -0
- package/dist/src/storage.js.map +1 -0
- package/dist/src/transcript-retention.js +109 -0
- package/dist/src/transcript-retention.js.map +1 -0
- package/dist/src/value-guards.js +26 -0
- package/dist/src/value-guards.js.map +1 -0
- package/docs/README.md +19 -15
- package/docs/architecture.md +141 -17
- package/docs/assets/README.md +5 -4
- package/docs/builtin-graders-v1.md +9 -2
- package/docs/codex-host.md +155 -57
- package/docs/contracts.md +2 -1
- package/docs/creating-cases.md +10 -4
- package/docs/development-cli.md +197 -24
- package/docs/documentation-quality.md +16 -5
- package/docs/evaluation-workflow.md +110 -0
- package/docs/extension-protocol-v1.md +26 -3
- package/docs/getting-started.md +29 -7
- package/docs/guide-evaluation.md +34 -2
- package/docs/identity-v1.md +29 -3
- package/docs/installing.md +35 -17
- package/docs/licensing.md +2 -2
- package/docs/native-controls.md +76 -5
- package/docs/native-hosts.md +197 -8
- package/docs/reading-results.md +102 -0
- package/docs/recipes.md +490 -0
- package/docs/releases.md +48 -13
- package/docs/report-v1.md +29 -1
- package/docs/results-v1.md +122 -4
- package/docs/run-settings-v1.md +147 -0
- package/docs/runtime-v1.md +255 -22
- package/docs/specs/repository-guide.md +1 -1
- package/docs/troubleshooting.md +12 -12
- package/docs/typescript-quality.md +106 -26
- package/examples/recipes/candidate.ts +26 -0
- package/examples/recipes/cases/coding-change.json +21 -0
- package/examples/recipes/cases/host-tools.json +21 -0
- package/examples/recipes/cases/mise.json +21 -0
- package/examples/recipes/cases/node.json +21 -0
- package/examples/recipes/cases/python-uv.json +21 -0
- package/examples/recipes/cases/semantic.json +17 -0
- package/examples/recipes/coding-change/check.ts +23 -0
- package/examples/recipes/executable.ts +19 -0
- package/examples/recipes/extension-artifacts.ts +65 -0
- package/examples/recipes/extension-methods.ts +174 -0
- package/examples/recipes/extension.ts +38 -0
- package/examples/recipes/modules.ts +9 -0
- package/examples/recipes/repository-skill/.agents/skills/startup-guide/SKILL.md +7 -0
- package/examples/recipes/repository-skill/.claude/skills/startup-guide/SKILL.md +7 -0
- package/examples/recipes/repository-skill/README.md +3 -0
- package/examples/recipes/run-inputs.ts +183 -0
- package/examples/recipes/run-options.ts +132 -0
- package/examples/recipes/run.ts +77 -0
- package/examples/recipes/runtime/empty.json +1 -0
- package/examples/recipes/runtime/goals.json +1 -0
- package/examples/recipes/runtime/mise.json +6 -0
- package/examples/recipes/runtime/path.json +1 -0
- package/examples/recipes/runtime/plugins.json +4 -0
- package/examples/recipes/runtime/python-uv.json +10 -0
- package/examples/recipes/semantic-judge.ts +29 -0
- package/examples/templates/coding/adapter.ts +17 -0
- package/examples/templates/coding/case.json +15 -0
- package/examples/templates/repository-skill/.agents/skills/startup-guide/SKILL.md +6 -0
- package/examples/templates/repository-skill/extension-data.ts +64 -0
- package/examples/templates/repository-skill/extension.ts +32 -0
- package/examples/templates/response/adapter.ts +12 -0
- package/examples/templates/response/case.json +13 -0
- package/package.json +23 -9
- package/schemas/report-v1.schema.json +9 -0
- package/schemas/run-evidence-v1.schema.json +2 -1
- package/schemas/runtime-v1.schema.json +43 -7
- package/src/advisory.ts +2 -2
- package/src/candidate-artifacts.ts +42 -11
- package/src/candidate-codex-state.ts +185 -0
- package/src/case-validation.ts +73 -50
- package/src/cli/cancellation.ts +20 -0
- package/src/cli/command.ts +87 -0
- package/src/cli/completion.ts +15 -0
- package/src/cli/demo.ts +61 -0
- package/src/cli/evaluation.ts +55 -0
- package/src/cli/extension.ts +79 -0
- package/src/cli/help.ts +199 -0
- package/src/cli/hosts.ts +55 -0
- package/src/cli/init-settings.ts +31 -0
- package/src/cli/init.ts +112 -0
- package/src/{cli-invocation.ts → cli/invocation.ts} +215 -218
- package/src/cli/loading.ts +151 -0
- package/src/cli/preparation.ts +154 -0
- package/src/cli/presentation.ts +93 -0
- package/src/cli/progress.ts +6 -0
- package/src/cli/run-manifest.ts +128 -0
- package/src/cli/run-options.ts +403 -0
- package/src/cli/run-overrides.ts +69 -0
- package/src/cli/settings.ts +154 -0
- package/src/cli/summary.ts +43 -0
- package/src/cli.ts +14 -592
- package/src/contract-types.ts +94 -0
- package/src/engine.ts +9 -5
- package/src/evaluation/advisory.ts +150 -0
- package/src/{evaluation-check-plan.ts → evaluation/check-plan.ts} +103 -67
- package/src/evaluation/collection.ts +88 -0
- package/src/evaluation/context.ts +10 -0
- package/src/{evaluation-grading-host.ts → evaluation/grading-host.ts} +5 -5
- package/src/evaluation/grading.ts +235 -0
- package/src/{evaluation-invocation-plan.ts → evaluation/invocation-plan.ts} +147 -58
- package/src/{evaluation-plan.ts → evaluation/plan.ts} +208 -141
- package/src/evaluation/progress.ts +19 -0
- package/src/evaluation/resources.ts +241 -0
- package/src/evaluation/run.ts +206 -0
- package/src/{evaluation-semantic.ts → evaluation/semantic.ts} +77 -87
- package/src/evaluation/trial.ts +800 -0
- package/src/{evaluation-types.ts → evaluation/types.ts} +79 -29
- package/src/evidence/host-results.ts +85 -0
- package/src/evidence/retained-artifacts.ts +18 -0
- package/src/evidence/trial-writer.ts +203 -0
- package/src/evidence/usage.ts +17 -0
- package/src/executable.ts +19 -0
- package/src/extension-client.ts +40 -108
- package/src/extension-session.ts +24 -48
- package/src/{advisory-fixture.ts → fixtures/advisory.ts} +12 -72
- package/src/{fixture-bin.ts → fixtures/bin.ts} +1 -1
- package/src/{fixture-cleanup.ts → fixtures/cleanup.ts} +13 -7
- package/src/fixtures/create.ts +183 -0
- package/src/{generated-fixture.ts → fixtures/generated.ts} +10 -53
- package/src/{git-hooks.ts → fixtures/git-hooks.ts} +1 -1
- package/src/{preparation.ts → fixtures/preparation.ts} +6 -16
- package/src/{repository-fixture.ts → fixtures/repository.ts} +30 -64
- package/src/{fixture-setup.ts → fixtures/setup.ts} +36 -53
- package/src/{fixture-tools.ts → fixtures/tools.ts} +1 -1
- package/src/{workspace-state.ts → fixtures/workspace-state.ts} +5 -5
- package/src/generated/cli-result.d.cts +2 -1
- package/src/generated/extension.d.cts +2 -1
- package/src/generated/report.cjs +1 -1
- package/src/generated/report.d.cts +2 -1
- package/src/generated/run-evidence.cjs +1 -1
- package/src/generated/run-evidence.d.cts +2 -1
- package/src/generated/runtime.cjs +1 -1
- package/src/generated/runtime.d.cts +2 -1
- package/src/git.ts +94 -0
- package/src/graders/git-head.ts +31 -85
- package/src/graders/shell.ts +54 -71
- package/src/host-diagnostic-redaction.ts +52 -0
- package/src/host-diagnostics.ts +331 -0
- package/src/hosts/claude-continuation.ts +57 -40
- package/src/hosts/claude-credential.ts +160 -63
- package/src/hosts/claude-events.ts +34 -18
- package/src/hosts/claude-keychain.ts +79 -21
- package/src/hosts/claude-login-recovery.ts +85 -0
- package/src/hosts/claude-login-refresh.ts +179 -0
- package/src/hosts/claude-login-store.ts +162 -0
- package/src/hosts/claude-login.ts +107 -0
- package/src/hosts/claude-native-goal.ts +15 -12
- package/src/hosts/claude-process.ts +10 -50
- package/src/hosts/claude-refresh-lock.ts +82 -0
- package/src/hosts/claude-repository-invocation.ts +9 -34
- package/src/hosts/claude-settings.ts +46 -9
- package/src/hosts/claude-skill-activations.ts +169 -0
- package/src/hosts/claude-state.ts +28 -15
- package/src/hosts/claude-tool-calls.ts +13 -10
- package/src/hosts/claude.ts +121 -81
- package/src/hosts/codex-app-server.ts +78 -52
- package/src/hosts/codex-configuration.ts +5 -10
- package/src/hosts/codex-exec-controls.ts +516 -0
- package/src/hosts/codex-marketplace.ts +12 -10
- package/src/hosts/codex-native-calls.ts +15 -122
- package/src/hosts/codex-process.ts +10 -183
- package/src/hosts/codex-profile.ts +36 -21
- package/src/hosts/codex-session-files.ts +1 -9
- package/src/hosts/codex-state.ts +134 -18
- package/src/hosts/codex.ts +210 -183
- package/src/hosts/isolation-common.ts +8 -11
- package/src/hosts/isolation-roots.ts +5 -4
- package/src/hosts/linux-sandbox.ts +15 -2
- package/src/hosts/mac-sandbox.ts +87 -18
- package/src/hosts/native-controls.ts +72 -22
- package/src/hosts/native-request.ts +55 -0
- package/src/hosts/process-lifecycle.ts +215 -49
- package/src/hosts/protected-worktrees.ts +3 -2
- package/src/hosts/skill-hook-catalog.ts +131 -0
- package/src/hosts/skill-hook-handler.ts +374 -0
- package/src/hosts/skill-hooks.ts +585 -0
- package/src/instrumentation.ts +7 -8
- package/src/native-transcript-state.ts +22 -3
- package/src/native-transcript-view.ts +52 -13
- package/src/native-transcripts.ts +26 -11
- package/src/paths.ts +22 -0
- package/src/process-lifecycle.ts +150 -0
- package/src/provenance.ts +34 -13
- package/src/prune.ts +300 -0
- package/src/report.ts +63 -36
- package/src/resolved-case.ts +1 -1
- package/src/results-lock.ts +42 -0
- package/src/results.ts +14 -18
- package/src/run-owner.ts +15 -9
- package/src/runner-paths.ts +21 -0
- package/src/{runtime-config.ts → runtime/config.ts} +160 -46
- package/src/{runtime-discovery-apple.ts → runtime/discovery-apple.ts} +3 -3
- package/src/{runtime-discovery-providers.ts → runtime/discovery-providers.ts} +3 -9
- package/src/{runtime-discovery.ts → runtime/discovery.ts} +4 -4
- package/src/{runtime-hook-processes.ts → runtime/hook-processes.ts} +3 -2
- package/src/{runtime-hooks.ts → runtime/hooks.ts} +40 -28
- package/src/{runtime-paths.ts → runtime/paths.ts} +5 -19
- package/src/{runtime-query.ts → runtime/query.ts} +5 -5
- package/src/{runtime-seeds.ts → runtime/seeds.ts} +3 -3
- package/src/runtime/shared-cache.ts +130 -0
- package/src/{runtime-state.ts → runtime/state.ts} +28 -11
- package/src/runtime-process.ts +194 -0
- package/src/schema-types.ts +73 -161
- package/src/shell.ts +4 -0
- package/src/sqlite-lock.ts +25 -0
- package/src/transcript-retention.ts +146 -0
- package/src/value-guards.ts +22 -0
- package/docs/assets/sevro-logo.png +0 -0
- package/docs/assets/sevro-symbol.svg +0 -1
- package/docs/assets/sevro-wordmark.svg +0 -1
- package/docs/cache-ownership-validation.md +0 -49
- package/docs/claude-continuation-validation.md +0 -47
- package/docs/concurrency-validation.md +0 -86
- package/docs/configuration-root-validation.md +0 -70
- package/docs/engine-slice.md +0 -114
- package/docs/evidence/runtime-startup-30/README.md +0 -185
- package/docs/evidence/runtime-startup-30/after.json +0 -319
- package/docs/evidence/runtime-startup-30/baseline-timeout.jsonl +0 -2
- package/docs/evidence/runtime-startup-30/before.json +0 -403
- package/docs/evidence/runtime-startup-30/consumer-smoke.json +0 -63
- package/docs/evidence/runtime-startup-30/full-policy-diagnostic.jsonl +0 -3
- package/docs/evidence/runtime-startup-30/minimal-policy-diagnostic.jsonl +0 -3
- package/docs/evidence/runtime-startup-30/plugin-template-assessment.json +0 -20
- package/docs/evidence/runtime-startup-30/seedless-timeout.jsonl +0 -2
- package/docs/evidence.md +0 -17
- package/docs/fixture-cleanup-validation.md +0 -73
- package/docs/guide-verification.json +0 -239
- package/src/evaluation-context.ts +0 -259
- package/src/evaluation-fixture.ts +0 -302
- package/src/evaluation-run.ts +0 -200
- package/src/evaluation-trial.ts +0 -1016
- package/src/hosts/claude-nested-skills.ts +0 -272
- package/src/hosts/codex-command-output.ts +0 -510
- package/src/hosts/codex-executor-command.ts +0 -291
- package/src/hosts/codex-skill-reads.ts +0 -783
- /package/src/{trial-scheduler.ts → evaluation/scheduler.ts} +0 -0
package/src/hosts/claude.ts
CHANGED
|
@@ -1,41 +1,46 @@
|
|
|
1
1
|
import { existsSync } from "node:fs";
|
|
2
|
-
import {
|
|
3
|
-
requireCandidateRuntime,
|
|
4
|
-
requireCandidateArtifactRoots,
|
|
5
|
-
} from "../candidate-artifacts";
|
|
6
2
|
import { readFile, realpath, rm, stat } from "node:fs/promises";
|
|
7
3
|
import { dirname, isAbsolute, join } from "node:path";
|
|
8
4
|
import type { HostAdapter, HostResult } from "../engine";
|
|
9
|
-
import {
|
|
10
|
-
allocateNativeState,
|
|
11
|
-
requireNativeTranscriptView,
|
|
12
|
-
} from "../native-transcript-state";
|
|
5
|
+
import { allocateNativeState } from "../native-transcript-state";
|
|
13
6
|
import {
|
|
14
7
|
failedNativeTranscripts,
|
|
15
8
|
retainEnabledNativeTranscripts,
|
|
16
9
|
} from "../native-transcripts";
|
|
17
|
-
import { fixtureParts } from "../preparation";
|
|
10
|
+
import { fixtureParts } from "../fixtures/preparation";
|
|
18
11
|
import {
|
|
19
12
|
prepareHookMounts,
|
|
20
13
|
releaseRuntimeHooks,
|
|
21
14
|
runtimeHookObservation,
|
|
22
15
|
runtimeHooksEnabled,
|
|
23
16
|
type HookMounts,
|
|
24
|
-
} from "../runtime
|
|
25
|
-
import { insideRuntimeRoot as inside } from "../runtime
|
|
26
|
-
import { runtimeObservations } from "../runtime
|
|
27
|
-
import {
|
|
17
|
+
} from "../runtime/hooks";
|
|
18
|
+
import { insideRuntimeRoot as inside } from "../runtime/paths";
|
|
19
|
+
import { runtimeObservations } from "../runtime/state";
|
|
20
|
+
import { isRecord } from "../value-guards";
|
|
28
21
|
import { runClaudeTurns, type ClaudeSession } from "./claude-continuation";
|
|
22
|
+
import { validateNativeHostRequest, validHostTimeout } from "./native-request";
|
|
29
23
|
import { summarizeClaudeEvents } from "./claude-events";
|
|
30
24
|
import { observeClaudeNativeGoal } from "./claude-native-goal";
|
|
31
|
-
import { claudeNestedSkillsObservation } from "./claude-nested-skills";
|
|
32
25
|
import { runClaudeProcess } from "./claude-process";
|
|
33
26
|
import {
|
|
34
27
|
claudeRepositoryInvocationObservation,
|
|
35
28
|
verifyClaudeRepositoryInvocation,
|
|
36
29
|
} from "./claude-repository-invocation";
|
|
37
30
|
import { prepareClaudeState, type ClaudeHostOptions } from "./claude-state";
|
|
31
|
+
import {
|
|
32
|
+
cleanupNativeState,
|
|
33
|
+
createHostDiagnostics,
|
|
34
|
+
diagnoseHostRequest,
|
|
35
|
+
diagnosticConfiguration,
|
|
36
|
+
type HostDiagnostics,
|
|
37
|
+
} from "../host-diagnostics";
|
|
38
|
+
import {
|
|
39
|
+
CLAUDE_AUTH_VARIABLES,
|
|
40
|
+
CLAUDE_LOGIN_MARGIN_MS,
|
|
41
|
+
} from "./claude-credential";
|
|
38
42
|
import { claudeToolCallsObservation } from "./claude-tool-calls";
|
|
43
|
+
import { claudeSkillActivationsObservation } from "./claude-skill-activations";
|
|
39
44
|
import { claudeNativeControls } from "./native-controls";
|
|
40
45
|
export type { ClaudeHostOptions } from "./claude-state";
|
|
41
46
|
|
|
@@ -221,12 +226,7 @@ function optionalClaudePaths(options: ClaudeHostOptions): boolean {
|
|
|
221
226
|
].every((path) => path === undefined || isAbsolute(path));
|
|
222
227
|
}
|
|
223
228
|
function validClaudeEffortTimeout(effort: string, timeoutMs: number): boolean {
|
|
224
|
-
return (
|
|
225
|
-
!!effort &&
|
|
226
|
-
Number.isSafeInteger(timeoutMs) &&
|
|
227
|
-
timeoutMs >= 1 &&
|
|
228
|
-
timeoutMs <= 30 * 60_000
|
|
229
|
-
);
|
|
229
|
+
return !!effort && validHostTimeout(timeoutMs);
|
|
230
230
|
}
|
|
231
231
|
|
|
232
232
|
/** Run Claude with explicit plugins and its native tool sandbox. */
|
|
@@ -238,11 +238,17 @@ export function createClaudeHost(options: ClaudeHostOptions): HostAdapter {
|
|
|
238
238
|
id: "sevro.host.claude",
|
|
239
239
|
model: options.model,
|
|
240
240
|
effort: options.effort,
|
|
241
|
+
...(options.retainHostDiagnostics
|
|
242
|
+
? {
|
|
243
|
+
configuration: diagnosticConfiguration(options.retainHostDiagnostics),
|
|
244
|
+
}
|
|
245
|
+
: {}),
|
|
241
246
|
hostCapabilities: [
|
|
242
247
|
"sevro.host.native-goal",
|
|
243
248
|
"sevro.host.runtime",
|
|
244
249
|
"sevro.host.native-transcripts",
|
|
245
250
|
"sevro.host.hooks",
|
|
251
|
+
"sevro.host.skill-activations",
|
|
246
252
|
"sevro.claude.plugin-dirs",
|
|
247
253
|
"sevro.claude.explicit-invocation",
|
|
248
254
|
...(options.projectSettings
|
|
@@ -250,7 +256,6 @@ export function createClaudeHost(options: ClaudeHostOptions): HostAdapter {
|
|
|
250
256
|
: []),
|
|
251
257
|
"sevro.claude.tool-calls",
|
|
252
258
|
"sevro.host.native-controls",
|
|
253
|
-
"sevro.claude.nested-skills",
|
|
254
259
|
"sevro.host.continuation",
|
|
255
260
|
"sevro.claude.continuation",
|
|
256
261
|
],
|
|
@@ -260,24 +265,6 @@ export function createClaudeHost(options: ClaudeHostOptions): HostAdapter {
|
|
|
260
265
|
};
|
|
261
266
|
}
|
|
262
267
|
|
|
263
|
-
function requireClaudeFollowUp(request: Request): void {
|
|
264
|
-
if (!isOptionalNonblankString(request.followUpPrompt))
|
|
265
|
-
throw new Error("Claude follow-up prompt must be nonempty");
|
|
266
|
-
}
|
|
267
|
-
function requirePassiveClaudeRequest(request: Request): void {
|
|
268
|
-
if (request.instrumentation?.length || request.condition !== "passive")
|
|
269
|
-
throw new Error("Claude enforcement instrumentation is unavailable");
|
|
270
|
-
requireClaudeFixtureBin(request);
|
|
271
|
-
}
|
|
272
|
-
|
|
273
|
-
function requireClaudeFixtureBin(request: Request): void {
|
|
274
|
-
if (
|
|
275
|
-
request.fixtureBinDir !== undefined &&
|
|
276
|
-
request.fixtureBinDir !== join(request.workspace, ".git", "fixture-bin")
|
|
277
|
-
)
|
|
278
|
-
throw new Error("fixture binary path is outside the workspace");
|
|
279
|
-
}
|
|
280
|
-
|
|
281
268
|
async function repositoryMount(
|
|
282
269
|
request: Request,
|
|
283
270
|
options: ClaudeHostOptions,
|
|
@@ -295,47 +282,58 @@ async function runClaudeRequest(
|
|
|
295
282
|
timeoutMs: number,
|
|
296
283
|
request: Request,
|
|
297
284
|
): Promise<HostResult> {
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
request
|
|
302
|
-
request.runtimeRole,
|
|
285
|
+
const diagnostics = createHostDiagnostics(
|
|
286
|
+
"claude",
|
|
287
|
+
options.retainHostDiagnostics,
|
|
288
|
+
request,
|
|
303
289
|
);
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
290
|
+
return diagnoseHostRequest(diagnostics, async () => {
|
|
291
|
+
diagnostics.stage = "request-validation";
|
|
292
|
+
await diagnostics.readCredentials(options.credentialFile);
|
|
293
|
+
return executeClaudeRequest(options, timeoutMs, request, diagnostics);
|
|
294
|
+
});
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
/** Every turn shares one access token, so it must outlive all of them. */
|
|
298
|
+
function claudeLoginLifetime(timeoutMs: number, request: Request): number {
|
|
299
|
+
const turns = request.followUpPrompt === undefined ? 1 : 2;
|
|
300
|
+
return timeoutMs * turns + CLAUDE_LOGIN_MARGIN_MS;
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
async function executeClaudeRequest(
|
|
304
|
+
options: ClaudeHostOptions,
|
|
305
|
+
timeoutMs: number,
|
|
306
|
+
request: Request,
|
|
307
|
+
diagnostics: HostDiagnostics,
|
|
308
|
+
): Promise<HostResult> {
|
|
309
|
+
const { pluginDirs, repositoryInvocation } = await validateClaudeRequest(
|
|
310
|
+
options,
|
|
311
|
+
request,
|
|
312
|
+
);
|
|
313
|
+
diagnostics.stage = "state-allocation";
|
|
312
314
|
const stateRoot = await allocateNativeState(
|
|
313
315
|
"claude-",
|
|
314
316
|
request.runtimePolicy ? request.workspace : undefined,
|
|
315
317
|
);
|
|
316
318
|
let hooks: HookMounts | undefined;
|
|
317
319
|
try {
|
|
320
|
+
diagnostics.stage = "state-preparation";
|
|
318
321
|
const state = await prepareClaudeState(
|
|
319
322
|
stateRoot,
|
|
320
323
|
options,
|
|
321
324
|
request,
|
|
322
325
|
pluginDirs,
|
|
326
|
+
claudeLoginLifetime(timeoutMs, request),
|
|
323
327
|
);
|
|
328
|
+
diagnostics.registerCredentials(state.env, CLAUDE_AUTH_VARIABLES);
|
|
324
329
|
hooks = state.hooks;
|
|
325
|
-
const run = (
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
state.pluginDirs,
|
|
333
|
-
),
|
|
334
|
-
cwd: request.workspace,
|
|
335
|
-
env: state.env,
|
|
336
|
-
timeoutMs,
|
|
337
|
-
signal: request.signal,
|
|
338
|
-
});
|
|
330
|
+
const run = claudeProcessRunner(
|
|
331
|
+
options,
|
|
332
|
+
timeoutMs,
|
|
333
|
+
request,
|
|
334
|
+
state,
|
|
335
|
+
diagnostics,
|
|
336
|
+
);
|
|
339
337
|
const execution = await runClaudeTurns({
|
|
340
338
|
prompt: request.prompt,
|
|
341
339
|
followUpPrompt: request.followUpPrompt,
|
|
@@ -348,6 +346,10 @@ async function runClaudeRequest(
|
|
|
348
346
|
request,
|
|
349
347
|
repositoryInvocation,
|
|
350
348
|
);
|
|
349
|
+
await diagnostics.retainFailure(
|
|
350
|
+
result,
|
|
351
|
+
"Claude host execution did not complete",
|
|
352
|
+
);
|
|
351
353
|
await addHookObservation(result, state.hooks);
|
|
352
354
|
await retainEnabledNativeTranscripts(
|
|
353
355
|
result,
|
|
@@ -364,13 +366,56 @@ async function runClaudeRequest(
|
|
|
364
366
|
join(stateRoot, "native-home", "projects"),
|
|
365
367
|
"claude",
|
|
366
368
|
request.runtimeRole,
|
|
369
|
+
diagnostics,
|
|
367
370
|
);
|
|
368
371
|
} finally {
|
|
369
|
-
await
|
|
370
|
-
|
|
372
|
+
await cleanupNativeState(diagnostics, async () => {
|
|
373
|
+
await releaseRuntimeHooks(hooks);
|
|
374
|
+
await rm(stateRoot, { recursive: true, force: true });
|
|
375
|
+
});
|
|
371
376
|
}
|
|
372
377
|
}
|
|
373
378
|
|
|
379
|
+
function claudeProcessRunner(
|
|
380
|
+
options: ClaudeHostOptions,
|
|
381
|
+
timeoutMs: number,
|
|
382
|
+
request: Request,
|
|
383
|
+
state: Awaited<ReturnType<typeof prepareClaudeState>>,
|
|
384
|
+
diagnostics: HostDiagnostics,
|
|
385
|
+
) {
|
|
386
|
+
return (prompt: string, session?: ClaudeSession) => {
|
|
387
|
+
diagnostics.stage = "claude-execution";
|
|
388
|
+
return runClaudeProcess({
|
|
389
|
+
diagnostics,
|
|
390
|
+
argv: claudeTurnArgs(
|
|
391
|
+
options,
|
|
392
|
+
prompt,
|
|
393
|
+
session,
|
|
394
|
+
state.settingsPath,
|
|
395
|
+
state.pluginDirs,
|
|
396
|
+
),
|
|
397
|
+
cwd: request.workspace,
|
|
398
|
+
env: state.env,
|
|
399
|
+
timeoutMs,
|
|
400
|
+
signal: request.signal,
|
|
401
|
+
});
|
|
402
|
+
};
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
async function validateClaudeRequest(
|
|
406
|
+
options: ClaudeHostOptions,
|
|
407
|
+
request: Request,
|
|
408
|
+
) {
|
|
409
|
+
await validateNativeHostRequest(request, "Claude");
|
|
410
|
+
const pluginDirs = request.claudePluginDirs
|
|
411
|
+
? await pluginDirectories(request.workspace, request.claudePluginDirs)
|
|
412
|
+
: [];
|
|
413
|
+
await verifyInvocation(request, pluginDirs);
|
|
414
|
+
requireIsolatedHookSources(request, options);
|
|
415
|
+
const repositoryInvocation = await repositoryMount(request, options);
|
|
416
|
+
return { pluginDirs, repositoryInvocation };
|
|
417
|
+
}
|
|
418
|
+
|
|
374
419
|
function requireIsolatedHookSources(
|
|
375
420
|
request: Request,
|
|
376
421
|
options: ClaudeHostOptions,
|
|
@@ -442,11 +487,11 @@ async function repositoryObservation(
|
|
|
442
487
|
if (!invocation) return null;
|
|
443
488
|
return claudeRepositoryInvocationObservation({
|
|
444
489
|
invocation,
|
|
445
|
-
stream: execution.
|
|
490
|
+
stream: execution.initialEvents ?? execution.events,
|
|
446
491
|
configRoot: configRoot,
|
|
447
492
|
workspace: request.workspace,
|
|
448
|
-
tools: execution.
|
|
449
|
-
? claudeToolCallsObservation(execution.
|
|
493
|
+
tools: execution.initialEvents
|
|
494
|
+
? claudeToolCallsObservation(execution.initialEvents, 0)
|
|
450
495
|
: tools,
|
|
451
496
|
});
|
|
452
497
|
}
|
|
@@ -457,12 +502,7 @@ async function claudeObservations(
|
|
|
457
502
|
request: Request,
|
|
458
503
|
invocation: RepositoryInvocation | null,
|
|
459
504
|
) {
|
|
460
|
-
const
|
|
461
|
-
execution.out,
|
|
462
|
-
configRoot,
|
|
463
|
-
request.workspace,
|
|
464
|
-
);
|
|
465
|
-
const tools = claudeToolCallsObservation(execution.out, execution.code);
|
|
505
|
+
const tools = claudeToolCallsObservation(execution.events, execution.code);
|
|
466
506
|
const repository = await repositoryObservation(
|
|
467
507
|
execution,
|
|
468
508
|
configRoot,
|
|
@@ -475,12 +515,12 @@ async function claudeObservations(
|
|
|
475
515
|
...runtimeObservations(request.runtimePolicy, request.runtimeRole),
|
|
476
516
|
await observeClaudeNativeGoal(
|
|
477
517
|
configRoot,
|
|
478
|
-
execution.
|
|
518
|
+
execution.followUpEvents ?? execution.events,
|
|
479
519
|
),
|
|
480
|
-
|
|
481
|
-
claudeNativeControls(execution.out, execution.code),
|
|
520
|
+
claudeNativeControls(execution.events, execution.code),
|
|
482
521
|
...(repository ? [repository] : []),
|
|
483
522
|
...(execution.continuation ? [execution.continuation] : []),
|
|
523
|
+
claudeSkillActivationsObservation(execution.events, execution.code),
|
|
484
524
|
];
|
|
485
525
|
}
|
|
486
526
|
|
|
@@ -535,7 +575,7 @@ async function claudeResult(
|
|
|
535
575
|
request: Request,
|
|
536
576
|
invocation: RepositoryInvocation | null,
|
|
537
577
|
): Promise<HostResult> {
|
|
538
|
-
const summary = summarizeClaudeEvents(execution.
|
|
578
|
+
const summary = summarizeClaudeEvents(execution.events, execution.code);
|
|
539
579
|
const observations = await claudeObservations(
|
|
540
580
|
execution,
|
|
541
581
|
configRoot,
|
|
@@ -1,8 +1,11 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import { join } from "node:path";
|
|
1
|
+
import { redactDiagnosticText } from "../host-diagnostic-redaction";
|
|
2
|
+
import { isRecord, requireRecord } from "../value-guards";
|
|
4
3
|
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
|
|
5
4
|
import { createInterface } from "node:readline";
|
|
5
|
+
import {
|
|
6
|
+
HostProcessDiagnostics,
|
|
7
|
+
type HostDiagnostics,
|
|
8
|
+
} from "../host-diagnostics";
|
|
6
9
|
interface HarnessRunRequest {
|
|
7
10
|
repoDir: string;
|
|
8
11
|
prompt: string;
|
|
@@ -14,9 +17,7 @@ interface HarnessRunRequest {
|
|
|
14
17
|
|
|
15
18
|
type RecordValue = Record<string, unknown>;
|
|
16
19
|
function record(value: unknown): RecordValue {
|
|
17
|
-
|
|
18
|
-
throw new Error("Malformed app-server object");
|
|
19
|
-
return value as RecordValue;
|
|
20
|
+
return requireRecord(value, "Malformed app-server object");
|
|
20
21
|
}
|
|
21
22
|
function identifier(value: unknown): string {
|
|
22
23
|
if (typeof value !== "string" || !value)
|
|
@@ -70,15 +71,7 @@ function publicErrorMessage(value: unknown): {
|
|
|
70
71
|
messageTruncated?: boolean;
|
|
71
72
|
} {
|
|
72
73
|
if (typeof value !== "string") return { message: null };
|
|
73
|
-
const redacted = value
|
|
74
|
-
.slice(0, 20000)
|
|
75
|
-
.replace(/(authorization\s*:\s*)[^\r\n]+/gi, "$1[redacted]")
|
|
76
|
-
.replace(/\bBearer\s+\S+/gi, "Bearer [redacted]")
|
|
77
|
-
.replace(/\bsk-[A-Za-z0-9_-]+/g, "[redacted]")
|
|
78
|
-
.replace(
|
|
79
|
-
/((?:api[_-]?key|access[_-]?token|refresh[_-]?token)\s*[=:]\s*)\S+/gi,
|
|
80
|
-
"$1[redacted]",
|
|
81
|
-
);
|
|
74
|
+
const redacted = redactDiagnosticText(value.slice(0, 20000), []);
|
|
82
75
|
return {
|
|
83
76
|
message: redacted.slice(0, 2000),
|
|
84
77
|
...(value.length > 20000 || redacted.length > 2000
|
|
@@ -145,7 +138,7 @@ class AppServerRpc {
|
|
|
145
138
|
{
|
|
146
139
|
resolve: (value: RecordValue) => void;
|
|
147
140
|
reject: (error: Error) => void;
|
|
148
|
-
|
|
141
|
+
method: string;
|
|
149
142
|
}
|
|
150
143
|
>();
|
|
151
144
|
private wake?: () => void;
|
|
@@ -155,8 +148,15 @@ class AppServerRpc {
|
|
|
155
148
|
private reads: Promise<void>;
|
|
156
149
|
private errors: Promise<void>;
|
|
157
150
|
stderr = "";
|
|
151
|
+
readonly capture: HostProcessDiagnostics;
|
|
158
152
|
|
|
159
|
-
constructor(
|
|
153
|
+
constructor(
|
|
154
|
+
argv: string[],
|
|
155
|
+
cwd: string,
|
|
156
|
+
env: Record<string, string>,
|
|
157
|
+
diagnostics: HostDiagnostics,
|
|
158
|
+
) {
|
|
159
|
+
this.capture = new HostProcessDiagnostics(diagnostics);
|
|
160
160
|
this.process = spawn(identifier(argv[0]), argv.slice(1), {
|
|
161
161
|
detached: process.platform !== "win32",
|
|
162
162
|
cwd,
|
|
@@ -166,10 +166,14 @@ class AppServerRpc {
|
|
|
166
166
|
this.exited = new Promise((resolve) => {
|
|
167
167
|
this.process.once("exit", resolve);
|
|
168
168
|
this.process.once("error", (error) => {
|
|
169
|
+
if (this.process.pid === undefined) this.capture.recordUnstarted();
|
|
169
170
|
this.fail(error);
|
|
170
171
|
resolve(null);
|
|
171
172
|
});
|
|
172
173
|
});
|
|
174
|
+
this.process.stdout.on("data", (chunk: Buffer) => {
|
|
175
|
+
this.capture.stdout.add(chunk);
|
|
176
|
+
});
|
|
173
177
|
this.reads = this.readLines().catch((error: unknown) => {
|
|
174
178
|
this.fail(error);
|
|
175
179
|
});
|
|
@@ -177,7 +181,8 @@ class AppServerRpc {
|
|
|
177
181
|
this.fail(error);
|
|
178
182
|
});
|
|
179
183
|
this.errors = new Promise((resolve) => {
|
|
180
|
-
this.process.stderr.on("data", (chunk) => {
|
|
184
|
+
this.process.stderr.on("data", (chunk: Buffer) => {
|
|
185
|
+
this.capture.stderr.add(chunk);
|
|
181
186
|
this.stderr = (this.stderr + String(chunk)).slice(-16000);
|
|
182
187
|
});
|
|
183
188
|
this.process.stderr.on("end", resolve);
|
|
@@ -195,10 +200,7 @@ class AppServerRpc {
|
|
|
195
200
|
|
|
196
201
|
fail(error: unknown): void {
|
|
197
202
|
this.failure ??= error instanceof Error ? error : new Error(String(error));
|
|
198
|
-
for (const pending of this.pending.values())
|
|
199
|
-
clearTimeout(pending.timer);
|
|
200
|
-
pending.reject(this.failure);
|
|
201
|
-
}
|
|
203
|
+
for (const pending of this.pending.values()) pending.reject(this.failure);
|
|
202
204
|
this.pending.clear();
|
|
203
205
|
this.wake?.();
|
|
204
206
|
}
|
|
@@ -263,7 +265,6 @@ class AppServerRpc {
|
|
|
263
265
|
// Validate before removing the waiter so malformed responses reject it.
|
|
264
266
|
const value = record(message.error ?? message.result);
|
|
265
267
|
this.pending.delete(id);
|
|
266
|
-
clearTimeout(pending.timer);
|
|
267
268
|
if (message.error)
|
|
268
269
|
pending.reject(
|
|
269
270
|
new Error(`App-server RPC failed (${String(value.code)})`),
|
|
@@ -293,14 +294,12 @@ class AppServerRpc {
|
|
|
293
294
|
this.wake?.();
|
|
294
295
|
}
|
|
295
296
|
|
|
297
|
+
/** The session's configured time bound is the only deadline, including startup. */
|
|
296
298
|
async call(method: string, params: RecordValue): Promise<RecordValue> {
|
|
297
299
|
if (this.failure) throw this.failure;
|
|
298
300
|
const id = this.nextId++;
|
|
299
301
|
return new Promise((resolve, reject) => {
|
|
300
|
-
|
|
301
|
-
this.fail(new Error(`App-server RPC timeout: ${method}`));
|
|
302
|
-
}, 45000);
|
|
303
|
-
this.pending.set(id, { resolve, reject, timer });
|
|
302
|
+
this.pending.set(id, { resolve, reject, method });
|
|
304
303
|
try {
|
|
305
304
|
this.send({ id, method, params });
|
|
306
305
|
} catch (error) {
|
|
@@ -309,6 +308,10 @@ class AppServerRpc {
|
|
|
309
308
|
});
|
|
310
309
|
}
|
|
311
310
|
|
|
311
|
+
pendingMethods(): string[] {
|
|
312
|
+
return Array.from(this.pending.values(), (pending) => pending.method);
|
|
313
|
+
}
|
|
314
|
+
|
|
312
315
|
async initialize(): Promise<void> {
|
|
313
316
|
await this.call("initialize", {
|
|
314
317
|
clientInfo: { name: "sevro_eval", version: "0.1.0" },
|
|
@@ -332,11 +335,13 @@ class AppServerRpc {
|
|
|
332
335
|
async close(): Promise<void> {
|
|
333
336
|
this.closing = true;
|
|
334
337
|
this.process.stdin.end();
|
|
338
|
+
let shutdownError: NodeJS.ErrnoException | undefined;
|
|
335
339
|
const terminate = (signal: NodeJS.Signals) => {
|
|
336
340
|
try {
|
|
337
341
|
if (this.process.pid) process.kill(-this.process.pid, signal);
|
|
338
342
|
} catch (error) {
|
|
339
|
-
if ((error as NodeJS.ErrnoException).code !== "ESRCH")
|
|
343
|
+
if ((error as NodeJS.ErrnoException).code !== "ESRCH")
|
|
344
|
+
shutdownError ??= error as NodeJS.ErrnoException;
|
|
340
345
|
}
|
|
341
346
|
};
|
|
342
347
|
const term = setTimeout(() => {
|
|
@@ -353,6 +358,7 @@ class AppServerRpc {
|
|
|
353
358
|
clearTimeout(term);
|
|
354
359
|
clearTimeout(kill);
|
|
355
360
|
}
|
|
361
|
+
if (shutdownError !== undefined) throw shutdownError;
|
|
356
362
|
}
|
|
357
363
|
}
|
|
358
364
|
|
|
@@ -471,6 +477,7 @@ function requireGoalObjective(objective: unknown): void {
|
|
|
471
477
|
}
|
|
472
478
|
|
|
473
479
|
export interface AppServerRunOptions {
|
|
480
|
+
diagnostics: HostDiagnostics;
|
|
474
481
|
vettedHooks?: boolean;
|
|
475
482
|
request: HarnessRunRequest;
|
|
476
483
|
argv: string[];
|
|
@@ -492,9 +499,9 @@ class AppServerSession {
|
|
|
492
499
|
goals: [],
|
|
493
500
|
};
|
|
494
501
|
readonly out: string[] = [];
|
|
502
|
+
finalMessage: string | null = null;
|
|
495
503
|
private threadId = "";
|
|
496
504
|
private goalSeen = false;
|
|
497
|
-
private usage: unknown;
|
|
498
505
|
private emittedItems = new Set<string>();
|
|
499
506
|
private followUp?: string;
|
|
500
507
|
|
|
@@ -621,14 +628,6 @@ class AppServerSession {
|
|
|
621
628
|
if (params.willRetry !== true)
|
|
622
629
|
throw new Error("App-server reported a fatal turn error");
|
|
623
630
|
break;
|
|
624
|
-
case "thread/tokenUsage/updated": {
|
|
625
|
-
const total = record(record(params.tokenUsage).total);
|
|
626
|
-
this.usage = {
|
|
627
|
-
input_tokens: total.inputTokens,
|
|
628
|
-
output_tokens: total.outputTokens,
|
|
629
|
-
};
|
|
630
|
-
break;
|
|
631
|
-
}
|
|
632
631
|
case "item/completed":
|
|
633
632
|
this.emitItem(params.item);
|
|
634
633
|
break;
|
|
@@ -651,13 +650,10 @@ class AppServerSession {
|
|
|
651
650
|
const turnId = this.observeTurn(turn);
|
|
652
651
|
this.requireCompletedTurn(turn, turnId);
|
|
653
652
|
const status = await this.completedGoal(turnId);
|
|
654
|
-
this.emit({
|
|
655
|
-
type: "turn.completed",
|
|
656
|
-
...(this.usage ? { usage: this.usage } : {}),
|
|
657
|
-
});
|
|
653
|
+
this.emit({ type: "turn.completed" });
|
|
658
654
|
if (this.followUp !== undefined)
|
|
659
655
|
return this.deliverFeedback(turnId, status, this.followUp);
|
|
660
|
-
return status !== "active" && (await this.
|
|
656
|
+
return status !== "active" && (await this.captureFinal(turnId));
|
|
661
657
|
}
|
|
662
658
|
|
|
663
659
|
private requireCompletedTurn(turn: RecordValue, turnId: string): void {
|
|
@@ -686,7 +682,7 @@ class AppServerSession {
|
|
|
686
682
|
): Promise<boolean> {
|
|
687
683
|
// User input does not require a terminal goal. Preserve the completed
|
|
688
684
|
// response and actual boundary before delivering the declared follow-up.
|
|
689
|
-
if (!(await this.
|
|
685
|
+
if (!(await this.captureFinal(turnId, true))) return false;
|
|
690
686
|
const boundary = protocolMessage(
|
|
691
687
|
await this.options.followUpBoundary(this.threadId, {
|
|
692
688
|
nativeGoalObserved: this.goalSeen,
|
|
@@ -704,7 +700,7 @@ class AppServerSession {
|
|
|
704
700
|
return false;
|
|
705
701
|
}
|
|
706
702
|
|
|
707
|
-
private async
|
|
703
|
+
private async captureFinal(
|
|
708
704
|
turnId: string,
|
|
709
705
|
forFeedback = false,
|
|
710
706
|
): Promise<boolean> {
|
|
@@ -718,10 +714,7 @@ class AppServerSession {
|
|
|
718
714
|
if (!latestTurnReady(latest, turnId)) return false;
|
|
719
715
|
const final = appServerFinal(thread, turnId);
|
|
720
716
|
for (const value of turns) this.backfillItems(record(value));
|
|
721
|
-
|
|
722
|
-
join(this.options.request.repoDir, ".git", "last-message.md"),
|
|
723
|
-
final,
|
|
724
|
-
);
|
|
717
|
+
this.finalMessage = final;
|
|
725
718
|
this.evidence.finalTurnId = turnId;
|
|
726
719
|
this.emit({
|
|
727
720
|
type: "item.completed",
|
|
@@ -861,10 +854,16 @@ function requireAppServerRun(request: HarnessRunRequest, bound: number): void {
|
|
|
861
854
|
throw new Error("Invalid app-server time bound");
|
|
862
855
|
}
|
|
863
856
|
function sessionFailure(session: AppServerSession, error: unknown): void {
|
|
857
|
+
session.options.diagnostics.failure = error;
|
|
864
858
|
session.evidence.failure =
|
|
865
859
|
error instanceof Error ? error.message : String(error);
|
|
866
860
|
session.emit({ type: "turn.failed" });
|
|
867
861
|
}
|
|
862
|
+
/** Name an unanswered request so a stalled startup stays distinguishable. */
|
|
863
|
+
function timeBoundFailure(pending: string[]): string {
|
|
864
|
+
const reached = "App-server evaluation time bound reached";
|
|
865
|
+
return pending.length ? `${reached} awaiting ${pending.join(", ")}` : reached;
|
|
866
|
+
}
|
|
868
867
|
async function completeSession(
|
|
869
868
|
session: AppServerSession,
|
|
870
869
|
rpc: AppServerRpc,
|
|
@@ -876,19 +875,32 @@ async function completeSession(
|
|
|
876
875
|
};
|
|
877
876
|
request.signal?.addEventListener("abort", cancel, { once: true });
|
|
878
877
|
const deadline = setTimeout(() => {
|
|
879
|
-
rpc.fail(new Error(
|
|
878
|
+
rpc.fail(new Error(timeBoundFailure(rpc.pendingMethods())));
|
|
880
879
|
}, bound);
|
|
880
|
+
let code = 0;
|
|
881
881
|
try {
|
|
882
882
|
await session.run();
|
|
883
883
|
session.assertSettled();
|
|
884
|
-
return 0;
|
|
885
884
|
} catch (error) {
|
|
886
885
|
sessionFailure(session, error);
|
|
887
|
-
|
|
886
|
+
code = 1;
|
|
888
887
|
} finally {
|
|
889
888
|
clearTimeout(deadline);
|
|
890
889
|
request.signal?.removeEventListener("abort", cancel);
|
|
890
|
+
}
|
|
891
|
+
return closeSession(session, rpc, code);
|
|
892
|
+
}
|
|
893
|
+
async function closeSession(
|
|
894
|
+
session: AppServerSession,
|
|
895
|
+
rpc: AppServerRpc,
|
|
896
|
+
code: number,
|
|
897
|
+
): Promise<number> {
|
|
898
|
+
try {
|
|
891
899
|
await rpc.close();
|
|
900
|
+
return code;
|
|
901
|
+
} catch (error) {
|
|
902
|
+
if (code === 0) sessionFailure(session, error);
|
|
903
|
+
return 1;
|
|
892
904
|
}
|
|
893
905
|
}
|
|
894
906
|
function settlementCode(session: AppServerSession, code: number): number {
|
|
@@ -906,20 +918,34 @@ export async function runCodexAppServer(options: AppServerRunOptions): Promise<{
|
|
|
906
918
|
out: string;
|
|
907
919
|
err: string;
|
|
908
920
|
code: number;
|
|
921
|
+
finalMessage: string | null;
|
|
909
922
|
evidence: AppServerEvidence;
|
|
910
923
|
}> {
|
|
911
924
|
const { request, argv, env } = options;
|
|
912
925
|
const bound = request.control?.appServerTimeoutMs ?? 3600000;
|
|
913
926
|
requireAppServerRun(request, bound);
|
|
914
|
-
const rpc = new AppServerRpc(argv, request.repoDir, env);
|
|
927
|
+
const rpc = new AppServerRpc(argv, request.repoDir, env, options.diagnostics);
|
|
915
928
|
const session = new AppServerSession(rpc, options);
|
|
916
929
|
const code = await completeSession(session, rpc, request, bound);
|
|
917
930
|
// Flush stdout before accepting success, including trailing data and terminal errors.
|
|
918
931
|
const finalCode = settlementCode(session, code);
|
|
932
|
+
if (rpc.process.pid !== undefined)
|
|
933
|
+
await rpc.capture.record(
|
|
934
|
+
{ exited: rpc.exited, signalCode: rpc.process.signalCode },
|
|
935
|
+
finalCode === 0 ? "exit" : "error",
|
|
936
|
+
);
|
|
919
937
|
return {
|
|
920
938
|
out: session.out.join("\n"),
|
|
921
939
|
err: rpc.stderr,
|
|
922
940
|
code: finalCode,
|
|
941
|
+
finalMessage: completedMessage(session, finalCode),
|
|
923
942
|
evidence: session.evidence,
|
|
924
943
|
};
|
|
925
944
|
}
|
|
945
|
+
|
|
946
|
+
function completedMessage(
|
|
947
|
+
session: AppServerSession,
|
|
948
|
+
code: number,
|
|
949
|
+
): string | null {
|
|
950
|
+
return code === 0 ? session.finalMessage : null;
|
|
951
|
+
}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import type { Stats } from "node:fs";
|
|
2
2
|
import { isRecord } from "../value-guards";
|
|
3
3
|
import { lstat, readFile, realpath, stat } from "node:fs/promises";
|
|
4
|
-
import { dirname, isAbsolute, join
|
|
4
|
+
import { dirname, isAbsolute, join } from "node:path";
|
|
5
|
+
import { insideRoot } from "../paths";
|
|
5
6
|
import { parse } from "smol-toml";
|
|
6
7
|
|
|
7
8
|
export async function configurationRoot(root: string): Promise<string> {
|
|
@@ -52,7 +53,7 @@ async function readConfiguration(
|
|
|
52
53
|
const info = await lstat(configPath);
|
|
53
54
|
declared = true;
|
|
54
55
|
const canonical = await realpath(configPath);
|
|
55
|
-
validateConfigurationFile(info,
|
|
56
|
+
validateConfigurationFile(info, insideRoot(root, canonical));
|
|
56
57
|
source = await readFile(configPath, "utf8");
|
|
57
58
|
} catch (error) {
|
|
58
59
|
if (!declared && (error as NodeJS.ErrnoException).code === "ENOENT")
|
|
@@ -97,15 +98,9 @@ function concurrencyLimit(limit: unknown, configPath: string): number | null {
|
|
|
97
98
|
return Number(limit);
|
|
98
99
|
}
|
|
99
100
|
|
|
100
|
-
function validateConfigurationFile(info: Stats,
|
|
101
|
-
if (!info.isFile() || info.size > 64 * 1024 ||
|
|
101
|
+
function validateConfigurationFile(info: Stats, contained: boolean): void {
|
|
102
|
+
if (!info.isFile() || info.size > 64 * 1024 || !contained)
|
|
102
103
|
throw new Error(
|
|
103
104
|
"configuration must be a contained regular file up to 64 KiB",
|
|
104
105
|
);
|
|
105
106
|
}
|
|
106
|
-
|
|
107
|
-
function escapedRoot(location: string): boolean {
|
|
108
|
-
return (
|
|
109
|
-
location === ".." || location.startsWith(`..${sep}`) || isAbsolute(location)
|
|
110
|
-
);
|
|
111
|
-
}
|