@bjoernrochel/sevro 0.1.0-rc.8 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +52 -6
- package/README.md +12 -7
- package/dist/examples/basic/graded.json +13 -0
- package/dist/examples/basic/host.js +9 -0
- package/dist/examples/basic/host.js.map +1 -0
- package/dist/examples/basic/prompt-only.json +7 -0
- package/dist/examples/recipes/candidate.js +23 -0
- package/dist/examples/recipes/candidate.js.map +1 -0
- package/dist/examples/recipes/cases/coding-change.json +21 -0
- package/dist/examples/recipes/cases/host-tools.json +21 -0
- package/dist/examples/recipes/cases/mise.json +21 -0
- package/dist/examples/recipes/cases/node.json +21 -0
- package/dist/examples/recipes/cases/python-uv.json +21 -0
- package/dist/examples/recipes/cases/semantic.json +17 -0
- package/dist/examples/recipes/coding-change/check.js +16 -0
- package/dist/examples/recipes/coding-change/check.js.map +1 -0
- package/dist/examples/recipes/executable.js +20 -0
- package/dist/examples/recipes/executable.js.map +1 -0
- package/dist/examples/recipes/extension-artifacts.js +56 -0
- package/dist/examples/recipes/extension-artifacts.js.map +1 -0
- package/dist/examples/recipes/extension-methods.js +167 -0
- package/dist/examples/recipes/extension-methods.js.map +1 -0
- package/dist/examples/recipes/extension.js +28 -0
- package/dist/examples/recipes/extension.js.map +1 -0
- package/dist/examples/recipes/modules.js +9 -0
- package/dist/examples/recipes/modules.js.map +1 -0
- package/dist/examples/recipes/repository-skill/.agents/skills/startup-guide/SKILL.md +7 -0
- package/dist/examples/recipes/repository-skill/.claude/skills/startup-guide/SKILL.md +7 -0
- package/dist/examples/recipes/repository-skill/README.md +3 -0
- package/dist/examples/recipes/run-inputs.js +144 -0
- package/dist/examples/recipes/run-inputs.js.map +1 -0
- package/dist/examples/recipes/run-options.js +103 -0
- package/dist/examples/recipes/run-options.js.map +1 -0
- package/dist/examples/recipes/run.js +62 -0
- package/dist/examples/recipes/run.js.map +1 -0
- package/dist/examples/recipes/runtime/empty.json +1 -0
- package/dist/examples/recipes/runtime/goals.json +1 -0
- package/dist/examples/recipes/runtime/mise.json +6 -0
- package/dist/examples/recipes/runtime/path.json +1 -0
- package/dist/examples/recipes/runtime/plugins.json +4 -0
- package/dist/examples/recipes/runtime/python-uv.json +10 -0
- package/dist/examples/recipes/semantic-judge.js +27 -0
- package/dist/examples/recipes/semantic-judge.js.map +1 -0
- package/dist/examples/templates/coding/adapter.js +14 -0
- package/dist/examples/templates/coding/adapter.js.map +1 -0
- package/dist/examples/templates/coding/case.json +15 -0
- package/dist/examples/templates/repository-skill/.agents/skills/startup-guide/SKILL.md +6 -0
- package/dist/examples/templates/repository-skill/extension-data.js +61 -0
- package/dist/examples/templates/repository-skill/extension-data.js.map +1 -0
- package/dist/examples/templates/repository-skill/extension.js +29 -0
- package/dist/examples/templates/repository-skill/extension.js.map +1 -0
- package/dist/examples/templates/response/adapter.js +13 -0
- package/dist/examples/templates/response/adapter.js.map +1 -0
- package/dist/examples/templates/response/case.json +13 -0
- package/dist/src/advisory.js +86 -0
- package/dist/src/advisory.js.map +1 -0
- package/dist/src/candidate-artifacts.js +139 -0
- package/dist/src/candidate-artifacts.js.map +1 -0
- package/dist/src/candidate-codex-state.js +110 -0
- package/dist/src/candidate-codex-state.js.map +1 -0
- package/dist/src/case-validation.js +92 -0
- package/dist/src/case-validation.js.map +1 -0
- package/dist/src/cli/cancellation.js +21 -0
- package/dist/src/cli/cancellation.js.map +1 -0
- package/dist/src/cli/command.js +60 -0
- package/dist/src/cli/command.js.map +1 -0
- package/dist/src/cli/completion.js +14 -0
- package/dist/src/cli/completion.js.map +1 -0
- package/dist/src/cli/demo.js +45 -0
- package/dist/src/cli/demo.js.map +1 -0
- package/dist/src/cli/evaluation.js +40 -0
- package/dist/src/cli/evaluation.js.map +1 -0
- package/dist/src/cli/extension.js +54 -0
- package/dist/src/cli/extension.js.map +1 -0
- package/dist/src/cli/help.js +190 -0
- package/dist/src/cli/help.js.map +1 -0
- package/dist/src/cli/hosts.js +33 -0
- package/dist/src/cli/hosts.js.map +1 -0
- package/dist/src/cli/init-settings.js +29 -0
- package/dist/src/cli/init-settings.js.map +1 -0
- package/dist/src/cli/init.js +82 -0
- package/dist/src/cli/init.js.map +1 -0
- package/dist/src/cli/invocation.js +431 -0
- package/dist/src/cli/invocation.js.map +1 -0
- package/dist/src/cli/loading.js +110 -0
- package/dist/src/cli/loading.js.map +1 -0
- package/dist/src/cli/preparation.js +88 -0
- package/dist/src/cli/preparation.js.map +1 -0
- package/dist/src/cli/presentation.js +61 -0
- package/dist/src/cli/presentation.js.map +1 -0
- package/dist/src/cli/progress.js +5 -0
- package/dist/src/cli/progress.js.map +1 -0
- package/dist/src/cli/run-manifest.js +108 -0
- package/dist/src/cli/run-manifest.js.map +1 -0
- package/dist/src/cli/run-options.js +367 -0
- package/dist/src/cli/run-options.js.map +1 -0
- package/dist/src/cli/run-overrides.js +51 -0
- package/dist/src/cli/run-overrides.js.map +1 -0
- package/dist/src/cli/settings.js +127 -0
- package/dist/src/cli/settings.js.map +1 -0
- package/dist/src/cli/summary.js +28 -0
- package/dist/src/cli/summary.js.map +1 -0
- package/dist/src/cli.js +20 -0
- package/dist/src/cli.js.map +1 -0
- package/dist/src/contract-types.js +2 -0
- package/dist/src/contract-types.js.map +1 -0
- package/dist/src/engine.js +4 -0
- package/dist/src/engine.js.map +1 -0
- package/dist/src/evaluation/advisory.js +85 -0
- package/dist/src/evaluation/advisory.js.map +1 -0
- package/dist/src/evaluation/check-plan.js +292 -0
- package/dist/src/evaluation/check-plan.js.map +1 -0
- package/dist/src/evaluation/collection.js +61 -0
- package/dist/src/evaluation/collection.js.map +1 -0
- package/dist/src/evaluation/context.js +2 -0
- package/dist/src/evaluation/context.js.map +1 -0
- package/dist/src/evaluation/grading-host.js +28 -0
- package/dist/src/evaluation/grading-host.js.map +1 -0
- package/dist/src/evaluation/grading.js +154 -0
- package/dist/src/evaluation/grading.js.map +1 -0
- package/dist/src/evaluation/invocation-plan.js +312 -0
- package/dist/src/evaluation/invocation-plan.js.map +1 -0
- package/dist/src/evaluation/plan.js +376 -0
- package/dist/src/evaluation/plan.js.map +1 -0
- package/dist/src/evaluation/progress.js +14 -0
- package/dist/src/evaluation/progress.js.map +1 -0
- package/dist/src/evaluation/resources.js +159 -0
- package/dist/src/evaluation/resources.js.map +1 -0
- package/dist/src/evaluation/run.js +148 -0
- package/dist/src/evaluation/run.js.map +1 -0
- package/dist/src/evaluation/scheduler.js +27 -0
- package/dist/src/evaluation/scheduler.js.map +1 -0
- package/dist/src/evaluation/semantic.js +183 -0
- package/dist/src/evaluation/semantic.js.map +1 -0
- package/dist/src/evaluation/trial.js +593 -0
- package/dist/src/evaluation/trial.js.map +1 -0
- package/dist/src/evaluation/types.js +20 -0
- package/dist/src/evaluation/types.js.map +1 -0
- package/dist/src/evidence/host-results.js +67 -0
- package/dist/src/evidence/host-results.js.map +1 -0
- package/dist/src/evidence/retained-artifacts.js +17 -0
- package/dist/src/evidence/retained-artifacts.js.map +1 -0
- package/dist/src/evidence/trial-writer.js +131 -0
- package/dist/src/evidence/trial-writer.js.map +1 -0
- package/dist/src/evidence/usage.js +14 -0
- package/dist/src/evidence/usage.js.map +1 -0
- package/dist/src/executable.js +20 -0
- package/dist/src/executable.js.map +1 -0
- package/dist/src/extension-client.js +212 -0
- package/dist/src/extension-client.js.map +1 -0
- package/dist/src/extension-session.js +236 -0
- package/dist/src/extension-session.js.map +1 -0
- package/dist/src/fixtures/advisory.js +206 -0
- package/dist/src/fixtures/advisory.js.map +1 -0
- package/dist/src/fixtures/bin.js +41 -0
- package/dist/src/fixtures/bin.js.map +1 -0
- package/dist/src/fixtures/cleanup.js +43 -0
- package/dist/src/fixtures/cleanup.js.map +1 -0
- package/dist/src/fixtures/create.js +94 -0
- package/dist/src/fixtures/create.js.map +1 -0
- package/dist/src/fixtures/generated.js +181 -0
- package/dist/src/fixtures/generated.js.map +1 -0
- package/dist/src/fixtures/git-hooks.js +16 -0
- package/dist/src/fixtures/git-hooks.js.map +1 -0
- package/dist/src/fixtures/preparation.js +201 -0
- package/dist/src/fixtures/preparation.js.map +1 -0
- package/dist/src/fixtures/repository.js +255 -0
- package/dist/src/fixtures/repository.js.map +1 -0
- package/dist/src/fixtures/setup.js +175 -0
- package/dist/src/fixtures/setup.js.map +1 -0
- package/dist/src/fixtures/tools.js +57 -0
- package/dist/src/fixtures/tools.js.map +1 -0
- package/dist/src/fixtures/workspace-state.js +155 -0
- package/dist/src/fixtures/workspace-state.js.map +1 -0
- package/dist/src/generated/cli-result.cjs +2 -0
- package/dist/src/generated/extension.cjs +2 -0
- package/dist/src/generated/report.cjs +2 -0
- package/dist/src/generated/run-evidence.cjs +2 -0
- package/dist/src/generated/runtime.cjs +2 -0
- package/dist/src/git.js +72 -0
- package/dist/src/git.js.map +1 -0
- package/dist/src/graders/artifact.js +68 -0
- package/dist/src/graders/artifact.js.map +1 -0
- package/dist/src/graders/git-head.js +112 -0
- package/dist/src/graders/git-head.js.map +1 -0
- package/dist/src/graders/output.js +345 -0
- package/dist/src/graders/output.js.map +1 -0
- package/dist/src/graders/semantic.js +132 -0
- package/dist/src/graders/semantic.js.map +1 -0
- package/dist/src/graders/shell.js +321 -0
- package/dist/src/graders/shell.js.map +1 -0
- package/dist/src/host-diagnostic-redaction.js +52 -0
- package/dist/src/host-diagnostic-redaction.js.map +1 -0
- package/dist/src/host-diagnostics.js +248 -0
- package/dist/src/host-diagnostics.js.map +1 -0
- package/dist/src/hosts/claude-continuation.js +89 -0
- package/dist/src/hosts/claude-continuation.js.map +1 -0
- package/dist/src/hosts/claude-credential.js +121 -0
- package/dist/src/hosts/claude-credential.js.map +1 -0
- package/dist/src/hosts/claude-events.js +136 -0
- package/dist/src/hosts/claude-events.js.map +1 -0
- package/dist/src/hosts/claude-keychain.js +100 -0
- package/dist/src/hosts/claude-keychain.js.map +1 -0
- package/dist/src/hosts/claude-login-recovery.js +67 -0
- package/dist/src/hosts/claude-login-recovery.js.map +1 -0
- package/dist/src/hosts/claude-login-refresh.js +146 -0
- package/dist/src/hosts/claude-login-refresh.js.map +1 -0
- package/dist/src/hosts/claude-login-store.js +124 -0
- package/dist/src/hosts/claude-login-store.js.map +1 -0
- package/dist/src/hosts/claude-login.js +74 -0
- package/dist/src/hosts/claude-login.js.map +1 -0
- package/dist/src/hosts/claude-native-goal.js +138 -0
- package/dist/src/hosts/claude-native-goal.js.map +1 -0
- package/dist/src/hosts/claude-process.js +10 -0
- package/dist/src/hosts/claude-process.js.map +1 -0
- package/dist/src/hosts/claude-refresh-lock.js +76 -0
- package/dist/src/hosts/claude-refresh-lock.js.map +1 -0
- package/dist/src/hosts/claude-repository-invocation.js +229 -0
- package/dist/src/hosts/claude-repository-invocation.js.map +1 -0
- package/dist/src/hosts/claude-settings.js +156 -0
- package/dist/src/hosts/claude-settings.js.map +1 -0
- package/dist/src/hosts/claude-skill-activations.js +131 -0
- package/dist/src/hosts/claude-skill-activations.js.map +1 -0
- package/dist/src/hosts/claude-state.js +237 -0
- package/dist/src/hosts/claude-state.js.map +1 -0
- package/dist/src/hosts/claude-tool-calls.js +150 -0
- package/dist/src/hosts/claude-tool-calls.js.map +1 -0
- package/dist/src/hosts/claude.js +373 -0
- package/dist/src/hosts/claude.js.map +1 -0
- package/dist/src/hosts/codex-app-server.js +764 -0
- package/dist/src/hosts/codex-app-server.js.map +1 -0
- package/dist/src/hosts/codex-configuration.js +91 -0
- package/dist/src/hosts/codex-configuration.js.map +1 -0
- package/dist/src/hosts/codex-events.js +104 -0
- package/dist/src/hosts/codex-events.js.map +1 -0
- package/dist/src/hosts/codex-exec-controls.js +434 -0
- package/dist/src/hosts/codex-exec-controls.js.map +1 -0
- package/dist/src/hosts/codex-invocation.js +54 -0
- package/dist/src/hosts/codex-invocation.js.map +1 -0
- package/dist/src/hosts/codex-marketplace.js +185 -0
- package/dist/src/hosts/codex-marketplace.js.map +1 -0
- package/dist/src/hosts/codex-native-calls.js +516 -0
- package/dist/src/hosts/codex-native-calls.js.map +1 -0
- package/dist/src/hosts/codex-process.js +10 -0
- package/dist/src/hosts/codex-process.js.map +1 -0
- package/dist/src/hosts/codex-profile.js +206 -0
- package/dist/src/hosts/codex-profile.js.map +1 -0
- package/dist/src/hosts/codex-session-files.js +89 -0
- package/dist/src/hosts/codex-session-files.js.map +1 -0
- package/dist/src/hosts/codex-state.js +310 -0
- package/dist/src/hosts/codex-state.js.map +1 -0
- package/dist/src/hosts/codex.js +527 -0
- package/dist/src/hosts/codex.js.map +1 -0
- package/dist/src/hosts/isolation-common.js +43 -0
- package/dist/src/hosts/isolation-common.js.map +1 -0
- package/dist/src/hosts/isolation-roots.js +64 -0
- package/dist/src/hosts/isolation-roots.js.map +1 -0
- package/dist/src/hosts/isolation.js +12 -0
- package/dist/src/hosts/isolation.js.map +1 -0
- package/dist/src/hosts/linux-sandbox.js +127 -0
- package/dist/src/hosts/linux-sandbox.js.map +1 -0
- package/dist/src/hosts/mac-sandbox.js +137 -0
- package/dist/src/hosts/mac-sandbox.js.map +1 -0
- package/dist/src/hosts/native-controls.js +151 -0
- package/dist/src/hosts/native-controls.js.map +1 -0
- package/dist/src/hosts/native-request.js +33 -0
- package/dist/src/hosts/native-request.js.map +1 -0
- package/dist/src/hosts/process-lifecycle.js +167 -0
- package/dist/src/hosts/process-lifecycle.js.map +1 -0
- package/dist/src/hosts/protected-worktrees.js +81 -0
- package/dist/src/hosts/protected-worktrees.js.map +1 -0
- package/dist/src/hosts/skill-hook-catalog.js +100 -0
- package/dist/src/hosts/skill-hook-catalog.js.map +1 -0
- package/dist/src/hosts/skill-hook-handler.js +228 -0
- package/dist/src/hosts/skill-hook-handler.js.map +1 -0
- package/dist/src/hosts/skill-hooks.js +423 -0
- package/dist/src/hosts/skill-hooks.js.map +1 -0
- package/dist/src/hosts/workspace-fingerprint.js +72 -0
- package/dist/src/hosts/workspace-fingerprint.js.map +1 -0
- package/dist/src/identity.js +167 -0
- package/dist/src/identity.js.map +1 -0
- package/dist/src/instrumentation.js +83 -0
- package/dist/src/instrumentation.js.map +1 -0
- package/dist/src/native-transcript-state.js +85 -0
- package/dist/src/native-transcript-state.js.map +1 -0
- package/dist/src/native-transcript-view.js +65 -0
- package/dist/src/native-transcript-view.js.map +1 -0
- package/dist/src/native-transcripts.js +208 -0
- package/dist/src/native-transcripts.js.map +1 -0
- package/dist/src/paths.js +13 -0
- package/dist/src/paths.js.map +1 -0
- package/dist/src/process-lifecycle.js +101 -0
- package/dist/src/process-lifecycle.js.map +1 -0
- package/dist/src/provenance.js +263 -0
- package/dist/src/provenance.js.map +1 -0
- package/dist/src/prune.js +228 -0
- package/dist/src/prune.js.map +1 -0
- package/dist/src/report.js +276 -0
- package/dist/src/report.js.map +1 -0
- package/dist/src/resolved-case.js +29 -0
- package/dist/src/resolved-case.js.map +1 -0
- package/dist/src/results-lock.js +40 -0
- package/dist/src/results-lock.js.map +1 -0
- package/dist/src/results.js +127 -0
- package/dist/src/results.js.map +1 -0
- package/dist/src/run-owner.js +283 -0
- package/dist/src/run-owner.js.map +1 -0
- package/dist/src/runner-paths.js +11 -0
- package/dist/src/runner-paths.js.map +1 -0
- package/dist/src/runtime/config.js +323 -0
- package/dist/src/runtime/config.js.map +1 -0
- package/dist/src/runtime/discovery-apple.js +65 -0
- package/dist/src/runtime/discovery-apple.js.map +1 -0
- package/dist/src/runtime/discovery-providers.js +37 -0
- package/dist/src/runtime/discovery-providers.js.map +1 -0
- package/dist/src/runtime/discovery.js +40 -0
- package/dist/src/runtime/discovery.js.map +1 -0
- package/dist/src/runtime/hook-processes.js +81 -0
- package/dist/src/runtime/hook-processes.js.map +1 -0
- package/dist/src/runtime/hooks.js +304 -0
- package/dist/src/runtime/hooks.js.map +1 -0
- package/dist/src/runtime/paths.js +244 -0
- package/dist/src/runtime/paths.js.map +1 -0
- package/dist/src/runtime/query.js +76 -0
- package/dist/src/runtime/query.js.map +1 -0
- package/dist/src/runtime/seeds.js +98 -0
- package/dist/src/runtime/seeds.js.map +1 -0
- package/dist/src/runtime/shared-cache.js +81 -0
- package/dist/src/runtime/shared-cache.js.map +1 -0
- package/dist/src/runtime/state.js +106 -0
- package/dist/src/runtime/state.js.map +1 -0
- package/dist/src/runtime-process.js +148 -0
- package/dist/src/runtime-process.js.map +1 -0
- package/dist/src/schema-types.js +2 -0
- package/dist/src/schema-types.js.map +1 -0
- package/dist/src/schema.js +16 -0
- package/dist/src/schema.js.map +1 -0
- package/dist/src/shell.js +5 -0
- package/dist/src/shell.js.map +1 -0
- package/dist/src/sqlite-lock.js +18 -0
- package/dist/src/sqlite-lock.js.map +1 -0
- package/dist/src/storage.js +21 -0
- package/dist/src/storage.js.map +1 -0
- package/dist/src/transcript-retention.js +109 -0
- package/dist/src/transcript-retention.js.map +1 -0
- package/dist/src/value-guards.js +26 -0
- package/dist/src/value-guards.js.map +1 -0
- package/docs/README.md +19 -15
- package/docs/architecture.md +141 -17
- package/docs/assets/README.md +5 -4
- package/docs/builtin-graders-v1.md +9 -2
- package/docs/codex-host.md +155 -57
- package/docs/contracts.md +2 -1
- package/docs/creating-cases.md +10 -4
- package/docs/development-cli.md +197 -24
- package/docs/documentation-quality.md +16 -5
- package/docs/evaluation-workflow.md +110 -0
- package/docs/extension-protocol-v1.md +26 -3
- package/docs/getting-started.md +29 -7
- package/docs/guide-evaluation.md +34 -2
- package/docs/identity-v1.md +29 -3
- package/docs/installing.md +35 -17
- package/docs/licensing.md +2 -2
- package/docs/native-controls.md +76 -5
- package/docs/native-hosts.md +197 -8
- package/docs/reading-results.md +102 -0
- package/docs/recipes.md +490 -0
- package/docs/releases.md +48 -13
- package/docs/report-v1.md +29 -1
- package/docs/results-v1.md +122 -4
- package/docs/run-settings-v1.md +147 -0
- package/docs/runtime-v1.md +255 -22
- package/docs/specs/repository-guide.md +1 -1
- package/docs/troubleshooting.md +12 -12
- package/docs/typescript-quality.md +106 -26
- package/examples/recipes/candidate.ts +26 -0
- package/examples/recipes/cases/coding-change.json +21 -0
- package/examples/recipes/cases/host-tools.json +21 -0
- package/examples/recipes/cases/mise.json +21 -0
- package/examples/recipes/cases/node.json +21 -0
- package/examples/recipes/cases/python-uv.json +21 -0
- package/examples/recipes/cases/semantic.json +17 -0
- package/examples/recipes/coding-change/check.ts +23 -0
- package/examples/recipes/executable.ts +19 -0
- package/examples/recipes/extension-artifacts.ts +65 -0
- package/examples/recipes/extension-methods.ts +174 -0
- package/examples/recipes/extension.ts +38 -0
- package/examples/recipes/modules.ts +9 -0
- package/examples/recipes/repository-skill/.agents/skills/startup-guide/SKILL.md +7 -0
- package/examples/recipes/repository-skill/.claude/skills/startup-guide/SKILL.md +7 -0
- package/examples/recipes/repository-skill/README.md +3 -0
- package/examples/recipes/run-inputs.ts +183 -0
- package/examples/recipes/run-options.ts +132 -0
- package/examples/recipes/run.ts +77 -0
- package/examples/recipes/runtime/empty.json +1 -0
- package/examples/recipes/runtime/goals.json +1 -0
- package/examples/recipes/runtime/mise.json +6 -0
- package/examples/recipes/runtime/path.json +1 -0
- package/examples/recipes/runtime/plugins.json +4 -0
- package/examples/recipes/runtime/python-uv.json +10 -0
- package/examples/recipes/semantic-judge.ts +29 -0
- package/examples/templates/coding/adapter.ts +17 -0
- package/examples/templates/coding/case.json +15 -0
- package/examples/templates/repository-skill/.agents/skills/startup-guide/SKILL.md +6 -0
- package/examples/templates/repository-skill/extension-data.ts +64 -0
- package/examples/templates/repository-skill/extension.ts +32 -0
- package/examples/templates/response/adapter.ts +12 -0
- package/examples/templates/response/case.json +13 -0
- package/package.json +23 -9
- package/schemas/report-v1.schema.json +9 -0
- package/schemas/run-evidence-v1.schema.json +2 -1
- package/schemas/runtime-v1.schema.json +43 -7
- package/src/advisory.ts +2 -2
- package/src/candidate-artifacts.ts +42 -11
- package/src/candidate-codex-state.ts +185 -0
- package/src/case-validation.ts +73 -50
- package/src/cli/cancellation.ts +20 -0
- package/src/cli/command.ts +87 -0
- package/src/cli/completion.ts +15 -0
- package/src/cli/demo.ts +61 -0
- package/src/cli/evaluation.ts +55 -0
- package/src/cli/extension.ts +79 -0
- package/src/cli/help.ts +199 -0
- package/src/cli/hosts.ts +55 -0
- package/src/cli/init-settings.ts +31 -0
- package/src/cli/init.ts +112 -0
- package/src/{cli-invocation.ts → cli/invocation.ts} +215 -218
- package/src/cli/loading.ts +151 -0
- package/src/cli/preparation.ts +154 -0
- package/src/cli/presentation.ts +93 -0
- package/src/cli/progress.ts +6 -0
- package/src/cli/run-manifest.ts +128 -0
- package/src/cli/run-options.ts +403 -0
- package/src/cli/run-overrides.ts +69 -0
- package/src/cli/settings.ts +154 -0
- package/src/cli/summary.ts +43 -0
- package/src/cli.ts +14 -592
- package/src/contract-types.ts +94 -0
- package/src/engine.ts +9 -5
- package/src/evaluation/advisory.ts +150 -0
- package/src/{evaluation-check-plan.ts → evaluation/check-plan.ts} +103 -67
- package/src/evaluation/collection.ts +88 -0
- package/src/evaluation/context.ts +10 -0
- package/src/{evaluation-grading-host.ts → evaluation/grading-host.ts} +5 -5
- package/src/evaluation/grading.ts +235 -0
- package/src/{evaluation-invocation-plan.ts → evaluation/invocation-plan.ts} +147 -58
- package/src/{evaluation-plan.ts → evaluation/plan.ts} +208 -141
- package/src/evaluation/progress.ts +19 -0
- package/src/evaluation/resources.ts +241 -0
- package/src/evaluation/run.ts +206 -0
- package/src/{evaluation-semantic.ts → evaluation/semantic.ts} +77 -87
- package/src/evaluation/trial.ts +800 -0
- package/src/{evaluation-types.ts → evaluation/types.ts} +79 -29
- package/src/evidence/host-results.ts +85 -0
- package/src/evidence/retained-artifacts.ts +18 -0
- package/src/evidence/trial-writer.ts +203 -0
- package/src/evidence/usage.ts +17 -0
- package/src/executable.ts +19 -0
- package/src/extension-client.ts +40 -108
- package/src/extension-session.ts +24 -48
- package/src/{advisory-fixture.ts → fixtures/advisory.ts} +12 -72
- package/src/{fixture-bin.ts → fixtures/bin.ts} +1 -1
- package/src/{fixture-cleanup.ts → fixtures/cleanup.ts} +13 -7
- package/src/fixtures/create.ts +183 -0
- package/src/{generated-fixture.ts → fixtures/generated.ts} +10 -53
- package/src/{git-hooks.ts → fixtures/git-hooks.ts} +1 -1
- package/src/{preparation.ts → fixtures/preparation.ts} +6 -16
- package/src/{repository-fixture.ts → fixtures/repository.ts} +30 -64
- package/src/{fixture-setup.ts → fixtures/setup.ts} +36 -53
- package/src/{fixture-tools.ts → fixtures/tools.ts} +1 -1
- package/src/{workspace-state.ts → fixtures/workspace-state.ts} +5 -5
- package/src/generated/cli-result.d.cts +2 -1
- package/src/generated/extension.d.cts +2 -1
- package/src/generated/report.cjs +1 -1
- package/src/generated/report.d.cts +2 -1
- package/src/generated/run-evidence.cjs +1 -1
- package/src/generated/run-evidence.d.cts +2 -1
- package/src/generated/runtime.cjs +1 -1
- package/src/generated/runtime.d.cts +2 -1
- package/src/git.ts +94 -0
- package/src/graders/git-head.ts +31 -85
- package/src/graders/shell.ts +54 -71
- package/src/host-diagnostic-redaction.ts +52 -0
- package/src/host-diagnostics.ts +331 -0
- package/src/hosts/claude-continuation.ts +57 -40
- package/src/hosts/claude-credential.ts +160 -63
- package/src/hosts/claude-events.ts +34 -18
- package/src/hosts/claude-keychain.ts +79 -21
- package/src/hosts/claude-login-recovery.ts +85 -0
- package/src/hosts/claude-login-refresh.ts +179 -0
- package/src/hosts/claude-login-store.ts +162 -0
- package/src/hosts/claude-login.ts +107 -0
- package/src/hosts/claude-native-goal.ts +15 -12
- package/src/hosts/claude-process.ts +10 -50
- package/src/hosts/claude-refresh-lock.ts +82 -0
- package/src/hosts/claude-repository-invocation.ts +9 -34
- package/src/hosts/claude-settings.ts +46 -9
- package/src/hosts/claude-skill-activations.ts +169 -0
- package/src/hosts/claude-state.ts +28 -15
- package/src/hosts/claude-tool-calls.ts +13 -10
- package/src/hosts/claude.ts +121 -81
- package/src/hosts/codex-app-server.ts +78 -52
- package/src/hosts/codex-configuration.ts +5 -10
- package/src/hosts/codex-exec-controls.ts +516 -0
- package/src/hosts/codex-marketplace.ts +12 -10
- package/src/hosts/codex-native-calls.ts +15 -122
- package/src/hosts/codex-process.ts +10 -183
- package/src/hosts/codex-profile.ts +36 -21
- package/src/hosts/codex-session-files.ts +1 -9
- package/src/hosts/codex-state.ts +134 -18
- package/src/hosts/codex.ts +210 -183
- package/src/hosts/isolation-common.ts +8 -11
- package/src/hosts/isolation-roots.ts +5 -4
- package/src/hosts/linux-sandbox.ts +15 -2
- package/src/hosts/mac-sandbox.ts +87 -18
- package/src/hosts/native-controls.ts +72 -22
- package/src/hosts/native-request.ts +55 -0
- package/src/hosts/process-lifecycle.ts +215 -49
- package/src/hosts/protected-worktrees.ts +3 -2
- package/src/hosts/skill-hook-catalog.ts +131 -0
- package/src/hosts/skill-hook-handler.ts +374 -0
- package/src/hosts/skill-hooks.ts +585 -0
- package/src/instrumentation.ts +7 -8
- package/src/native-transcript-state.ts +22 -3
- package/src/native-transcript-view.ts +52 -13
- package/src/native-transcripts.ts +26 -11
- package/src/paths.ts +22 -0
- package/src/process-lifecycle.ts +150 -0
- package/src/provenance.ts +34 -13
- package/src/prune.ts +300 -0
- package/src/report.ts +63 -36
- package/src/resolved-case.ts +1 -1
- package/src/results-lock.ts +42 -0
- package/src/results.ts +14 -18
- package/src/run-owner.ts +15 -9
- package/src/runner-paths.ts +21 -0
- package/src/{runtime-config.ts → runtime/config.ts} +160 -46
- package/src/{runtime-discovery-apple.ts → runtime/discovery-apple.ts} +3 -3
- package/src/{runtime-discovery-providers.ts → runtime/discovery-providers.ts} +3 -9
- package/src/{runtime-discovery.ts → runtime/discovery.ts} +4 -4
- package/src/{runtime-hook-processes.ts → runtime/hook-processes.ts} +3 -2
- package/src/{runtime-hooks.ts → runtime/hooks.ts} +40 -28
- package/src/{runtime-paths.ts → runtime/paths.ts} +5 -19
- package/src/{runtime-query.ts → runtime/query.ts} +5 -5
- package/src/{runtime-seeds.ts → runtime/seeds.ts} +3 -3
- package/src/runtime/shared-cache.ts +130 -0
- package/src/{runtime-state.ts → runtime/state.ts} +28 -11
- package/src/runtime-process.ts +194 -0
- package/src/schema-types.ts +73 -161
- package/src/shell.ts +4 -0
- package/src/sqlite-lock.ts +25 -0
- package/src/transcript-retention.ts +146 -0
- package/src/value-guards.ts +22 -0
- package/docs/assets/sevro-logo.png +0 -0
- package/docs/assets/sevro-symbol.svg +0 -1
- package/docs/assets/sevro-wordmark.svg +0 -1
- package/docs/cache-ownership-validation.md +0 -49
- package/docs/claude-continuation-validation.md +0 -47
- package/docs/concurrency-validation.md +0 -86
- package/docs/configuration-root-validation.md +0 -70
- package/docs/engine-slice.md +0 -114
- package/docs/evidence/runtime-startup-30/README.md +0 -185
- package/docs/evidence/runtime-startup-30/after.json +0 -319
- package/docs/evidence/runtime-startup-30/baseline-timeout.jsonl +0 -2
- package/docs/evidence/runtime-startup-30/before.json +0 -403
- package/docs/evidence/runtime-startup-30/consumer-smoke.json +0 -63
- package/docs/evidence/runtime-startup-30/full-policy-diagnostic.jsonl +0 -3
- package/docs/evidence/runtime-startup-30/minimal-policy-diagnostic.jsonl +0 -3
- package/docs/evidence/runtime-startup-30/plugin-template-assessment.json +0 -20
- package/docs/evidence/runtime-startup-30/seedless-timeout.jsonl +0 -2
- package/docs/evidence.md +0 -17
- package/docs/fixture-cleanup-validation.md +0 -73
- package/docs/guide-verification.json +0 -239
- package/src/evaluation-context.ts +0 -259
- package/src/evaluation-fixture.ts +0 -302
- package/src/evaluation-run.ts +0 -200
- package/src/evaluation-trial.ts +0 -1016
- package/src/hosts/claude-nested-skills.ts +0 -272
- package/src/hosts/codex-command-output.ts +0 -510
- package/src/hosts/codex-executor-command.ts +0 -291
- package/src/hosts/codex-skill-reads.ts +0 -783
- /package/src/{trial-scheduler.ts → evaluation/scheduler.ts} +0 -0
|
@@ -12,12 +12,19 @@ It validates the guide inventory's source/static destinations and case coverage.
|
|
|
12
12
|
It ignores code contents as links and excludes inactive fixture snapshots.
|
|
13
13
|
Local checks need no network. Run `bun run test:docs-examples` for the exact
|
|
14
14
|
marked tutorial commands in fresh consumer/contributor directories, including
|
|
15
|
-
expected verdicts and saved evidence. Candidate
|
|
16
|
-
|
|
15
|
+
expected verdicts and saved evidence. Candidate commands run against compiled JavaScript. The released-package
|
|
16
|
+
check uses the tutorial bundled in that published version; the two results
|
|
17
|
+
are reported separately. These commands create and clean temporary directories.
|
|
18
|
+
The [recipe verifier](../scripts/documentation-recipes.ts) runs the new packaged
|
|
19
|
+
projects separately from the current published version. It executes actual
|
|
20
|
+
isolated tool/check positive and negative controls and both native hosts' dry
|
|
21
|
+
preparation; authenticated model, plugin-hook and goal behavior remain
|
|
22
|
+
unexecuted by that check. Its [prerequisites](recipes.md#prepare-a-recipe-project)
|
|
23
|
+
are mandatory, including UV and mise's declared Node installation.
|
|
17
24
|
|
|
18
25
|
`package.json.version` is the authoritative current version. `bun run docs:sync`
|
|
19
26
|
updates only the marked [Current release](installing.md#current-release) section
|
|
20
|
-
with its exact installation
|
|
27
|
+
with its exact npm and Bun installation commands. Keep its markers and heading in place;
|
|
21
28
|
the command refuses missing, duplicate, or reversed markers and invalid versions.
|
|
22
29
|
The rest of the page stays ordinary Markdown. `check:docs` compares this section
|
|
23
30
|
without writing; release preparation and package-install validation also reject
|
|
@@ -25,6 +32,10 @@ stale output. Run synchronization after a version bump and include the resulting
|
|
|
25
32
|
documentation change. Link installation and update instructions to this section.
|
|
26
33
|
Keep historical versions, licensing transitions, and minimum supported versions
|
|
27
34
|
fixed and explain their purpose beside each reference.
|
|
35
|
+
Link to the current-release section instead of repeating the current Sevro
|
|
36
|
+
version elsewhere. Describe features for the latest version, using the installed
|
|
37
|
+
`sevro` command in user workflows. Keep source-checkout instructions in contributor
|
|
38
|
+
guidance and omit notes about feature unavailability in older releases.
|
|
28
39
|
|
|
29
40
|
## External links
|
|
30
41
|
|
|
@@ -38,8 +49,8 @@ make a network job green. External success does not establish a claim's truth.
|
|
|
38
49
|
|
|
39
50
|
- Can a new user reach installation, first success, results, troubleshooting,
|
|
40
51
|
architecture, and contributing without the guide?
|
|
41
|
-
- Are supported environments, credentials, side effects, and
|
|
42
|
-
|
|
52
|
+
- Are supported environments, credentials, side effects, and prerequisites
|
|
53
|
+
explicit beside the relevant commands?
|
|
43
54
|
- Do descriptive links, meaningful headings, source order, readable tables,
|
|
44
55
|
image alternatives, and words rather than color support accessibility?
|
|
45
56
|
- Are current contracts, accepted choices, research, and past evidence distinct?
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
# From a demo to your next evaluation
|
|
2
|
+
|
|
3
|
+
Create an evaluation, prepare it, run its assertions, and inspect the latest
|
|
4
|
+
result with the installed `sevro` command.
|
|
5
|
+
|
|
6
|
+
## Try the demo and create a project
|
|
7
|
+
|
|
8
|
+
Use Node.js 24.13.0 or newer, or Bun 1.3.13. [Install Sevro](installing.md) and
|
|
9
|
+
[make its command available on `PATH`](installing.md#make-the-command-available),
|
|
10
|
+
then run:
|
|
11
|
+
|
|
12
|
+
```sh
|
|
13
|
+
sevro demo
|
|
14
|
+
sevro init --template response --directory ./response-evaluation
|
|
15
|
+
cd response-evaluation
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
The demo returns an assessed `passed` task and evidence under `.sevro/results`,
|
|
19
|
+
using its own temporary project without credentials. The initializer refuses
|
|
20
|
+
existing destinations, including directories and symbolic links. It writes
|
|
21
|
+
real assertions, a deterministic adapter, `run.json` and template instructions.
|
|
22
|
+
[Saved settings](run-settings-v1.md) use their manifest's directory for paths;
|
|
23
|
+
`sevro.json` retains its separate runtime scope. Keep private `.sevro/` out of Git.
|
|
24
|
+
|
|
25
|
+
## Prepare before execution
|
|
26
|
+
|
|
27
|
+
```sh
|
|
28
|
+
sevro run --run-manifest-file run.json --dry
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
Stderr shows the whitelisted resolved settings before candidate execution:
|
|
32
|
+
case/source selection, routes/model/effort, trials, threshold, concurrency,
|
|
33
|
+
runtime selection and canonical storage paths. Invalid input names its option,
|
|
34
|
+
file or field; missing executables name installation or explicit override paths.
|
|
35
|
+
Correct the named prerequisite and repeat preparation.
|
|
36
|
+
|
|
37
|
+
Dry preparation follows normal fixture, extension and check preparation without
|
|
38
|
+
calling the candidate or graders. Expect
|
|
39
|
+
`execution=not_run grading=not_requested task=not_assessed`. It does not establish
|
|
40
|
+
authentication, provider model availability, native skill dispatch or model success.
|
|
41
|
+
See [native prerequisites](native-hosts.md) and [troubleshooting](troubleshooting.md).
|
|
42
|
+
|
|
43
|
+
## Run, read and change one thing
|
|
44
|
+
|
|
45
|
+
```sh
|
|
46
|
+
sevro run --run-manifest-file run.json
|
|
47
|
+
sevro report --latest
|
|
48
|
+
SEVRO_TEMPLATE_FAIL=1 sevro run --run-manifest-file run.json
|
|
49
|
+
sevro report --latest
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
The first run's adapter returns exactly `ready`; its regex assertion must pass.
|
|
53
|
+
The negative control returns `wrong`: expect exit `1`, completed execution and
|
|
54
|
+
grading, task `failed`, and the named `ready` check's reason. Then edit `case.json`
|
|
55
|
+
and `adapter.js` (`adapter.ts` when initialized from contributor sources), prepare again and rerun. Explicit flags override saved settings;
|
|
56
|
+
`--trials 3` repeats the task without rewriting `run.json`.
|
|
57
|
+
|
|
58
|
+
Progress uses stderr at actual preparation, candidate, grading and evidence
|
|
59
|
+
retention boundaries. Trial numbers distinguish concurrent work, which may
|
|
60
|
+
interleave. Stage `completed` means the attempt finished; the result states whether
|
|
61
|
+
execution/grading succeeded. Dry runs explicitly skip candidate and grading;
|
|
62
|
+
failed candidates skip grading. Failed preparation never prints candidate start.
|
|
63
|
+
Evidence completion follows persistence. Optional synchronous engine observers
|
|
64
|
+
cannot change the result through a thrown exception.
|
|
65
|
+
|
|
66
|
+
Human completion shows per-trial states, failed/unavailable check reasons,
|
|
67
|
+
measured candidate time, known/unknown usage/cost and `result=`/`evidence=` paths.
|
|
68
|
+
Candidate time measures the host call, not total setup time. Absent metrics remain
|
|
69
|
+
`unknown`; dry duration is unknown. Prompt-only completion stays `not_assessed`.
|
|
70
|
+
Add `--json` to emit one `sevro.cli-result.v1` stdout document; progress remains on
|
|
71
|
+
stderr. See [reading results](reading-results.md).
|
|
72
|
+
|
|
73
|
+
Each retained CLI evaluation saves `result.json` beside its run evidence and
|
|
74
|
+
atomically publishes `<results-root>/latest.json`. Latest is the last retained CLI
|
|
75
|
+
result, including dry runs and failed assessments. Pre-run failures do not replace
|
|
76
|
+
it. Concurrent runs publish in completion order. Output files stay under excluded
|
|
77
|
+
results storage, preserving project identity. A latest-write failure cannot turn
|
|
78
|
+
an earlier failed assessment into success.
|
|
79
|
+
|
|
80
|
+
`report --latest` reads `<cwd>/.sevro/results/latest.json`. Use
|
|
81
|
+
`report --latest --results-root ./other-results` for another output root or working
|
|
82
|
+
directory. `--result-file ./path/to/result.json` selects older runs and accepts
|
|
83
|
+
relative paths. Reports validate referenced evidence; missing/moved evidence
|
|
84
|
+
produces a diagnostic. To reclaim native bundle space while keeping reportable
|
|
85
|
+
results and hashes, use [`sevro prune --native-transcripts`](reading-results.md#remove-native-transcript-bundles).
|
|
86
|
+
Reports label bundles intentionally removed by that command as pruned.
|
|
87
|
+
[Report v1](report-v1.md) defines fields and limits.
|
|
88
|
+
|
|
89
|
+
## Choose another starter
|
|
90
|
+
|
|
91
|
+
| Template | Real assertions | Prerequisites and interpretation |
|
|
92
|
+
| ------------------ | ------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------ |
|
|
93
|
+
| `response` | Exact `ready` response; negative control returns `wrong` | Node or Bun; no model credentials |
|
|
94
|
+
| `coding` | Edit `add.sh`; isolated checks test positive, negative and zero addition | Node or Bun plus `sandbox-exec` on macOS or `bubblewrap`/`socat` on Linux; negative control leaves subtraction bug |
|
|
95
|
+
| `repository-skill` | Actual `.agents/skills/startup-guide/SKILL.md` mount; explicit invocation evidence; correct start command and port | Node or Bun, Git, Codex, native isolation, explicit model/effort and readable auth file; dry is unassessed |
|
|
96
|
+
|
|
97
|
+
From a new parent directory, use `sevro init --template coding --directory ./coding-eval`
|
|
98
|
+
or `sevro init --template repository-skill --directory ./skill-eval`. Read its generated
|
|
99
|
+
README for controls and dry/live commands. Repository-skill `command.json` records
|
|
100
|
+
the selected runtime executable and absolute extension path; regenerate it after moving the project.
|
|
101
|
+
|
|
102
|
+
For model evaluations, replace a saved adapter route with explicit `--host`,
|
|
103
|
+
`--model`, `--effort` and host authentication options. Candidate, semantic and
|
|
104
|
+
advisory model choices remain independent. Deterministic adapters measure wiring;
|
|
105
|
+
they do not measure model ability. Native runs may incur provider charges.
|
|
106
|
+
Automated package checks run response/coding pass/fail and native skill preparation;
|
|
107
|
+
authenticated native behavior and human usability remain unmeasured. The
|
|
108
|
+
[study protocol](https://github.com/BjRo/sevro/blob/main/docs/usability-study.md) records deferred human measurements.
|
|
109
|
+
[Runnable recipes](recipes.md) cover Claude, plugins/hooks, goals, semantic grading
|
|
110
|
+
and host tools.
|
|
@@ -106,8 +106,16 @@ trial in the Git fixture after source history and working-tree files are built,
|
|
|
106
106
|
before extension artifacts are mounted or the host starts. The command is never
|
|
107
107
|
interpreted through a shell by Sevro; an extension may explicitly select a
|
|
108
108
|
shell executable. `{{sevro.project}}` and `{{sevro.workspace}}` in environment
|
|
109
|
-
values resolve to the declared roots at execution time.
|
|
110
|
-
|
|
109
|
+
values resolve to the declared roots at execution time. Setup's `HOME` is the
|
|
110
|
+
candidate commands' private trial home, outside the Git fixture. Ordinary
|
|
111
|
+
`$HOME` paths can initialize directories and seed prior state before the candidate
|
|
112
|
+
starts, including without a runtime configuration file. This home is separate
|
|
113
|
+
from host authentication and native configuration, other trials, and grader homes.
|
|
114
|
+
Setup's `TMPDIR` remains disposable and is removed when setup finishes; the shared
|
|
115
|
+
home survives through candidate execution and grading, then is removed even on
|
|
116
|
+
setup failure, cancellation, or dry runs. Trusted injected hosts must use the
|
|
117
|
+
supplied `candidateRuntimeRoot` and runtime policy to preserve this boundary.
|
|
118
|
+
The symbolic values and command enter fixture and comparison identity, while retained configuration
|
|
111
119
|
records only their digest. Setup has a two-minute limit, inherits no credential
|
|
112
120
|
environment variables, and its output is discarded. Failure or interruption
|
|
113
121
|
stops the run before host execution; it cannot become a passing assessment.
|
|
@@ -171,7 +179,7 @@ other invocation declarations and enters retained configuration and identity.
|
|
|
171
179
|
The host validates the bounded mount without following symbolic links. Its
|
|
172
180
|
`sevro.claude.repository-invocation` observation binds one native command, exact
|
|
173
181
|
arguments, and the complete mounted body before the first assistant entry in
|
|
174
|
-
the completed session. It retains only acceptance and
|
|
182
|
+
the completed session. It retains only acceptance and the dispatched skill name.
|
|
175
183
|
Missing or ambiguous session evidence yields a partial observation; later Skill
|
|
176
184
|
calls cannot repair an unverified explicit repository dispatch.
|
|
177
185
|
|
|
@@ -224,6 +232,21 @@ grader error, or unavailable required evidence.
|
|
|
224
232
|
Host adapters can supply additional namespaced observations. The engine bounds
|
|
225
233
|
and retains those records, then passes them to `evaluate`. Required observation
|
|
226
234
|
IDs must be present with `complete` evidence before a task can pass.
|
|
235
|
+
The bundled adapters expose [skill activation evidence](native-hosts.md#skill-activation-evidence)
|
|
236
|
+
as `sevro.host.skill-activations`, with separate methods for Codex tool-result
|
|
237
|
+
body exposure and successful Claude `Skill` calls.
|
|
238
|
+
This replaces the retired Codex skill-read and Claude nested-skill observations.
|
|
239
|
+
Explicit invocation receipts remain dispatch evidence and do not contribute to
|
|
240
|
+
`observedSkills`; extensions must migrate activation checks to the common ID.
|
|
241
|
+
Codex additionally supplies per-skill `skillExposures` and independent
|
|
242
|
+
`collectionStatus` in that observation. Its optional
|
|
243
|
+
`sevro.host.skill-exposure-claims` capability provides complete evidence only
|
|
244
|
+
for listed positive full or partial body exposures, even when collection is
|
|
245
|
+
partial. Extensions must check the requested skill and acceptable exposure
|
|
246
|
+
status explicitly; this evidence cannot establish absence or primary selection.
|
|
247
|
+
See the [exposure contract](native-hosts.md#skill-activation-evidence) for scope,
|
|
248
|
+
unknown states and host limitations. Required aggregate activation evidence
|
|
249
|
+
still needs a `complete` observation as before.
|
|
227
250
|
Host adapters can also supply bounded evidence artifacts. The engine persists
|
|
228
251
|
their bytes outside the candidate workspace before `evaluate`, then passes
|
|
229
252
|
file URLs and digests through the existing artifact references. A required host
|
package/docs/getting-started.md
CHANGED
|
@@ -1,10 +1,31 @@
|
|
|
1
1
|
# Your first evaluation
|
|
2
2
|
|
|
3
|
-
Run a deterministic case and inspect its evidence.
|
|
3
|
+
Run a deterministic case and inspect its evidence. Use Node.js 24.13.0 or newer, or Bun 1.3.13; no
|
|
4
4
|
Darrow checkout, Codex/Claude login, or model credentials are required. The
|
|
5
5
|
adapter returns `ready`; Sevro grades that response against `^ready$`.
|
|
6
6
|
This teaches the runner lifecycle, not the quality of an actual model.
|
|
7
7
|
|
|
8
|
+
<a id="built-in-demo-in-this-checkout"></a>
|
|
9
|
+
|
|
10
|
+
## Built-in demo
|
|
11
|
+
|
|
12
|
+
With [Sevro installed and on `PATH`](installing.md#make-the-command-available), run:
|
|
13
|
+
|
|
14
|
+
```sh
|
|
15
|
+
sevro demo
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
It prints `execution=completed grading=completed task=passed` and an absolute
|
|
19
|
+
`evidence=` path under `.sevro/results` in your current directory. The demo owns
|
|
20
|
+
its packaged case and adapter, uses a separate temporary project, and ignores
|
|
21
|
+
your project's `sevro.json` and native configuration. No credentials are needed.
|
|
22
|
+
Effective run settings appear on stderr before evaluation. Add `--json` for one
|
|
23
|
+
versioned result on stdout.
|
|
24
|
+
|
|
25
|
+
Continue with [the initialized evaluation workflow](evaluation-workflow.md) to
|
|
26
|
+
create response, coding or repository-skill assertions, prepare them, inspect
|
|
27
|
+
progress and access the latest result without copying an evidence path.
|
|
28
|
+
|
|
8
29
|
## From an installed package
|
|
9
30
|
|
|
10
31
|
Prepare a new directory:
|
|
@@ -12,15 +33,16 @@ Prepare a new directory:
|
|
|
12
33
|
```sh
|
|
13
34
|
mkdir sevro-demo
|
|
14
35
|
cd sevro-demo
|
|
15
|
-
|
|
36
|
+
npm init -y
|
|
16
37
|
```
|
|
17
38
|
|
|
18
|
-
Install the [current release](installing.md#current-release)
|
|
19
|
-
here to run the example.
|
|
20
|
-
|
|
39
|
+
Install the [current release](installing.md#current-release)
|
|
40
|
+
there, then return here to run the example. See [licensing](licensing.md) for the current terms and
|
|
41
|
+
the licenses retained by historical releases.
|
|
21
42
|
|
|
22
43
|
Make the local installation available on `PATH` for this shell, then call
|
|
23
|
-
`sevro` directly
|
|
44
|
+
`sevro` directly with Node on `PATH`. Bun users can use the
|
|
45
|
+
[explicit Bun invocation](installing.md#make-the-command-available).
|
|
24
46
|
|
|
25
47
|
<!-- sevro-example:consumer -->
|
|
26
48
|
|
|
@@ -29,7 +51,7 @@ export PATH="$PWD/node_modules/.bin:$PATH"
|
|
|
29
51
|
mkdir -p project
|
|
30
52
|
sevro run --json \
|
|
31
53
|
--case-file "$PWD/node_modules/@bjoernrochel/sevro/examples/basic/graded.json" \
|
|
32
|
-
--adapter-module "$PWD/node_modules/@bjoernrochel/sevro/examples/basic/host.
|
|
54
|
+
--adapter-module "$PWD/node_modules/@bjoernrochel/sevro/dist/examples/basic/host.js" \
|
|
33
55
|
--project-root "$PWD/project" --results-root "$PWD/sevro-results" \
|
|
34
56
|
--condition passive --trials 1 --threshold 1 > sevro-result.json
|
|
35
57
|
```
|
package/docs/guide-evaluation.md
CHANGED
|
@@ -5,7 +5,7 @@ and a small extension for guide-specific evidence and fixtures. The inventory an
|
|
|
5
5
|
cases remain under `.agents/skills/sevro-guide/evals/`. The
|
|
6
6
|
[extension modules](../scripts/guide/README.md) do not implement a second runner.
|
|
7
7
|
|
|
8
|
-
From
|
|
8
|
+
From the Sevro repository root, validate fixture preparation without launching a model:
|
|
9
9
|
|
|
10
10
|
```sh
|
|
11
11
|
bun run eval:guide --case-id orientation --host codex \
|
|
@@ -54,6 +54,34 @@ documentation and both skill mounts; the eval cases and grader code are withheld
|
|
|
54
54
|
The case matrix covers orientation, explicit invocation, unrelated requests,
|
|
55
55
|
missing/conflicting/stale evidence, follow-ups, pressure for effects,
|
|
56
56
|
extension boundaries, contributions, and licensing.
|
|
57
|
+
|
|
58
|
+
The deterministic CLI integration suite runs all eleven positive cases on each
|
|
59
|
+
host. Eight negative cases also run on both hosts: attempted effects, missing
|
|
60
|
+
events, incomplete native turns, missing/wrong invocation receipts, stale/missing
|
|
61
|
+
follow-up reads, and missing continuation evidence. These exercise distinct host
|
|
62
|
+
event formats or receipt identifiers. Ten host-independent negatives run on
|
|
63
|
+
Codex once: changed files or skill mounts, extra directories, host failure,
|
|
64
|
+
missing/partial common selection observations, and four answer contradictions.
|
|
65
|
+
This keeps every failure scenario while reducing the matrix from 58 to 48 CLI
|
|
66
|
+
runs. Two additional Codex positive controls remain, for 50 guide integration
|
|
67
|
+
tests in total. The docs phase still dry-prepares all eleven cases.
|
|
68
|
+
|
|
69
|
+
On 2026-10-24, an instrumented local comparison on macOS arm64 with Bun 1.3.13
|
|
70
|
+
measured the negative cases at 262.55 seconds for 36 passing tests on `adf04a1`,
|
|
71
|
+
then 181.50 seconds for 26 passing tests with #145: 81.05 seconds less (30.9%).
|
|
72
|
+
These are sums of individual case durations in `tests.log`, not a CI speedup
|
|
73
|
+
forecast. Claude's negative cases alone changed from 130.23 to 57.50 seconds;
|
|
74
|
+
that ratio rescales their shard weight. Reproduce the targeted measurement with:
|
|
75
|
+
|
|
76
|
+
```sh
|
|
77
|
+
bun run check:typescript --coverage ./tests/guide/codex-negative.test.ts \
|
|
78
|
+
./tests/guide/claude-negative.test.ts
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
These sparse timing runs passed their tests and report validation, but failed
|
|
82
|
+
the unchanged 95% full-inventory coverage thresholds as expected. Full acceptance
|
|
83
|
+
still requires every CI phase and the combined coverage gate on the candidate.
|
|
84
|
+
|
|
57
85
|
Missing event artifacts, incomplete turns, missing dispatch receipts, or absent
|
|
58
86
|
fresh follow-up reads cannot pass. Built-in `sevro.regex` checks assess answer
|
|
59
87
|
assertions and contradictions. The `sevro.shell` check verifies Git status,
|
|
@@ -73,9 +101,13 @@ Automatic checks remain signals, not a complete truth judgment. Acceptance
|
|
|
73
101
|
requires inspecting material claims against the sources actually read.
|
|
74
102
|
File listings do not count as source inspection; follow-ups require fresh
|
|
75
103
|
content. Explicit invocation requires Sevro's complete host dispatch receipt.
|
|
104
|
+
Automatic selection requires the complete `sevro.host.skill-activations`
|
|
105
|
+
observation on both hosts. Codex uses passive tool-result hooks; Claude uses
|
|
106
|
+
successful native `Skill` calls. The evaluator does not fall back to command
|
|
107
|
+
patterns, direct Claude file reads, or retired skill-read observations.
|
|
76
108
|
See the [guide contract](specs/repository-guide.md).
|
|
77
109
|
|
|
78
|
-
The [earlier verification record](guide-verification.json) describes historical
|
|
110
|
+
The [earlier verification record](https://github.com/BjRo/sevro/blob/main/docs/guide-verification.json) describes historical
|
|
79
111
|
trials of the previous harness. Its outputs and recheck observations are preserved
|
|
80
112
|
as history; they do not validate this conversion. The old `--recheck` command
|
|
81
113
|
and custom evidence format have been removed. Rerun changed cases through Sevro
|
package/docs/identity-v1.md
CHANGED
|
@@ -30,9 +30,15 @@ SHA-256(UTF-8("sevro.identity.v1\n") || UTF-8(JCS(dimensions)))
|
|
|
30
30
|
|
|
31
31
|
The CLI derives `runnerBuildDigest` from sorted packaged file paths and their
|
|
32
32
|
SHA-256 bytes under `src/`, `schemas/`, `docs/`, and `examples/`, plus `README.md` and
|
|
33
|
-
`package.json`.
|
|
33
|
+
`package.json`. Installed JavaScript execution also hashes `dist/`, including
|
|
34
|
+
the compiled runtime, examples and source maps. This binds evidence to executed
|
|
35
|
+
output; raw checkout execution excludes generated builds. The same installed
|
|
36
|
+
artifact has the same build digest under Node and Bun. For a Git project, `projectDigest` binds the revision and dirty
|
|
34
37
|
patch digest without the checkout path. Without a revision it hashes a bounded
|
|
35
|
-
file snapshot, excluding `.git`, results, and run-state storage.
|
|
38
|
+
file snapshot, excluding `.git`, results, and run-state storage. Results and
|
|
39
|
+
run-state storage are also excluded from untracked Git identity, so repeated
|
|
40
|
+
project-local output does not change the project digest. Tracked source changes
|
|
41
|
+
remain included, even under a selected storage directory. A caller may
|
|
36
42
|
provide explicit 64-character digests for controlled development runs.
|
|
37
43
|
|
|
38
44
|
The prefix separates this digest from component digests and future identity
|
|
@@ -48,6 +54,17 @@ The effective positive trial job limit is retained as `configuration.redacted.jo
|
|
|
48
54
|
and enters `configurationDigest`. Changing parallelism changes evaluation identity,
|
|
49
55
|
even when the condition, host route, trial count, and assessment threshold match.
|
|
50
56
|
|
|
57
|
+
CLI [saved run settings](run-settings-v1.md) resolve before evaluation. Equivalent
|
|
58
|
+
manifest and explicit selections produce the same semantic inputs. CLI evidence
|
|
59
|
+
also retains `configuration.redacted.runSettings` for shell isolation, explicit
|
|
60
|
+
protected roots, deprecated runtime selections, selected native timeouts and
|
|
61
|
+
Claude project-settings authority. These values enter the
|
|
62
|
+
configuration digest. Optional `configuration.resolved` stores the whitelisted
|
|
63
|
+
operator preview outside that digest, including operational paths and the
|
|
64
|
+
selected manifest location. Relocating output storage alone does not alter
|
|
65
|
+
identity. A tracked manifest remains project content under the normal project
|
|
66
|
+
identity rules.
|
|
67
|
+
|
|
51
68
|
Extension runs retain the operator-supplied redacted extension configuration
|
|
52
69
|
under `configuration.redacted.extensionConfiguration`, alongside
|
|
53
70
|
`extensionConfigurationDigest`. The retained copy is the same snapshot used
|
|
@@ -64,10 +81,19 @@ for the host default. Configuration paths and unrelated TOML bytes are not
|
|
|
64
81
|
imported settings; a relocated configuration with the same effective values
|
|
65
82
|
keeps this component unchanged.
|
|
66
83
|
|
|
84
|
+
Opted-in bundled native roles also record `sevro.host.retain-diagnostics: true`.
|
|
85
|
+
The [diagnostic retention switch](development-cli.md#retain-native-host-diagnostics)
|
|
86
|
+
therefore changes `configurationDigest`. Omission leaves that host setting
|
|
87
|
+
absent and preserves the previous configuration shape. Retained diagnostic
|
|
88
|
+
contents and temporary native state paths do not enter identity.
|
|
89
|
+
|
|
67
90
|
Selected [runtime configuration](runtime-v1.md) is retained under
|
|
68
91
|
`configuration.redacted.runtimePolicy` and enters `configurationDigest`.
|
|
69
92
|
It includes declared non-secret environment values, canonical read roots,
|
|
70
|
-
private seed digests and optional role selections,
|
|
93
|
+
private seed digests and optional role selections, shared-cache sources/slots/roles,
|
|
94
|
+
and hook authority. Mutable shared-cache contents are excluded; equal identities
|
|
95
|
+
do not establish equal cache contents or trial independence in shared mode.
|
|
96
|
+
Per-role output paths are resolved
|
|
71
97
|
only during execution; runtime placeholders remain symbolic in identity.
|
|
72
98
|
Runtime roots also retain lexical aliases, metadata discovery sources,
|
|
73
99
|
and skipped optional directories. Moving or
|
package/docs/installing.md
CHANGED
|
@@ -1,15 +1,21 @@
|
|
|
1
1
|
# Install, update, or remove Sevro
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
3
|
+
<a id="node-support-preview"></a>
|
|
4
|
+
|
|
5
|
+
## Runtime requirements
|
|
6
|
+
|
|
7
|
+
The installed package contains compiled JavaScript and runs with Node.js
|
|
8
|
+
24.13.0 or newer, or Bun 1.3.13. Node users do not need Bun for installation
|
|
9
|
+
or execution. Development and release builds continue to use Bun. Native
|
|
10
|
+
isolation targets macOS and Ubuntu 24.04 x64. See the [bounded platform evidence](native-hosts.md#validated-platform-scope);
|
|
11
|
+
Windows remains unverified.
|
|
6
12
|
|
|
7
13
|
## Install locally
|
|
8
14
|
|
|
9
15
|
```sh
|
|
10
16
|
mkdir sevro-demo
|
|
11
17
|
cd sevro-demo
|
|
12
|
-
|
|
18
|
+
npm init -y
|
|
13
19
|
```
|
|
14
20
|
|
|
15
21
|
Then install the [current release](#current-release) in that directory.
|
|
@@ -22,10 +28,16 @@ Local installation records the selected version in the project lockfile.
|
|
|
22
28
|
|
|
23
29
|
## Current release
|
|
24
30
|
|
|
25
|
-
The current release is `
|
|
31
|
+
The current release is `1.1.0`. Install this exact version with npm:
|
|
32
|
+
|
|
33
|
+
```sh
|
|
34
|
+
npm install --save-exact @bjoernrochel/sevro@1.1.0
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
Or with Bun:
|
|
26
38
|
|
|
27
39
|
```sh
|
|
28
|
-
bun add --exact @bjoernrochel/sevro@
|
|
40
|
+
bun add --exact @bjoernrochel/sevro@1.1.0
|
|
29
41
|
```
|
|
30
42
|
|
|
31
43
|
<!-- sevro-current-release:end -->
|
|
@@ -41,22 +53,25 @@ export PATH="$PWD/node_modules/.bin:$PATH"
|
|
|
41
53
|
|
|
42
54
|
This selects the locally installed version for the current shell, including
|
|
43
55
|
after changing directories. If your installed `sevro` is already on `PATH`,
|
|
44
|
-
no adjustment is needed.
|
|
45
|
-
|
|
56
|
+
no adjustment is needed. The executable uses `#!/usr/bin/env node`, so Node
|
|
57
|
+
must be on `PATH` for a direct `sevro` invocation. For Bun execution, use
|
|
58
|
+
`bun run --bun sevro` from the installation directory, or run
|
|
59
|
+
`bun ./node_modules/@bjoernrochel/sevro/dist/src/cli.js` with the same arguments.
|
|
60
|
+
Bun-only users can initialize with `bun init -y` and install with `bun add`.
|
|
46
61
|
|
|
47
62
|
## Published package and checkout
|
|
48
63
|
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
[source-available licensing terms](licensing.md). Earlier `rc.1` and `rc.2`
|
|
52
|
-
packages retain BUSL-1.1. The license transition began with `0.1.0-rc.3`.
|
|
53
|
-
Source checkouts can include changes beyond their
|
|
54
|
-
package version; compare the exact release tag when establishing provenance.
|
|
64
|
+
See [licensing](licensing.md) for Sevro's source-available terms and historical
|
|
65
|
+
licensing transitions.
|
|
55
66
|
|
|
56
|
-
The
|
|
67
|
+
The npm package contains compiled runtime and example modules under `dist/`,
|
|
68
|
+
alongside TypeScript sources, schemas, examples, documentation,
|
|
57
69
|
LICENSE, and contribution guidance. It excludes tests, developer scripts, Git
|
|
58
70
|
metadata, and repository guide mounts. Use a source checkout for the guide.
|
|
59
71
|
|
|
72
|
+
Use the exact release tag and package integrity when establishing provenance;
|
|
73
|
+
see the [release workflow](releases.md).
|
|
74
|
+
|
|
60
75
|
## Work from source
|
|
61
76
|
|
|
62
77
|
```sh
|
|
@@ -65,7 +80,8 @@ cd sevro
|
|
|
65
80
|
bun install --frozen-lockfile
|
|
66
81
|
```
|
|
67
82
|
|
|
68
|
-
Run the [
|
|
83
|
+
Run the [contributor tutorial](getting-started.md#from-a-contributor-checkout).
|
|
84
|
+
To build and install a development archive, follow [Contributing](../CONTRIBUTING.md#install-a-development-archive).
|
|
69
85
|
`bun run test:package-install` packs and installs a local tarball in temporary
|
|
70
86
|
directories; it does not publish. [Contributing](../CONTRIBUTING.md) lists checks.
|
|
71
87
|
|
|
@@ -81,8 +97,10 @@ Rollback uses the prior exact version and another verification run.
|
|
|
81
97
|
## Remove
|
|
82
98
|
|
|
83
99
|
```sh
|
|
84
|
-
|
|
100
|
+
npm uninstall @bjoernrochel/sevro
|
|
85
101
|
```
|
|
86
102
|
|
|
103
|
+
With Bun, use `bun remove @bjoernrochel/sevro`.
|
|
104
|
+
|
|
87
105
|
This updates the dependency and lockfile, not saved cases, results, or model
|
|
88
106
|
logins. Remove those separately only when wanted.
|
package/docs/licensing.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Sevro licensing
|
|
2
2
|
|
|
3
|
-
[LICENSE](../LICENSE)
|
|
3
|
+
[LICENSE](../LICENSE) defines Sevro's licensing terms. Sevro Source Available
|
|
4
4
|
License 1.0 allows free use in all contexts, modification, and free redistribution
|
|
5
5
|
under the same terms. Commercial work, paid education, research, CI, and
|
|
6
6
|
consulting are permitted.
|
|
@@ -23,7 +23,7 @@ it does not promise that alternative licenses will never be offered.
|
|
|
23
23
|
|
|
24
24
|
Published `0.1.0-rc.1` and `0.1.0-rc.2` retain BUSL-1.1, their original
|
|
25
25
|
production grant, and their per-version change date. Each changes to MPL-2.0
|
|
26
|
-
two years after its first public availability.
|
|
26
|
+
two years after its first public availability. The current license does not revoke
|
|
27
27
|
or rewrite earlier grants. The [retained rc.2 license](licenses/BUSL-1.1-0.1.0-rc.2.txt)
|
|
28
28
|
records those terms.
|
|
29
29
|
|
package/docs/native-controls.md
CHANGED
|
@@ -5,7 +5,8 @@ retain an observation with that ID. It exposes host facts for extension policy:
|
|
|
5
5
|
|
|
6
6
|
- `method: native_control_calls`;
|
|
7
7
|
- ordered `calls` containing only `ordinal`, `namespace`, and `name`;
|
|
8
|
-
- `acceptedAgentCount`, a verified Codex acceptance count
|
|
8
|
+
- `acceptedAgentCount`, a verified direct native Codex acceptance count when
|
|
9
|
+
overall coverage is complete, otherwise `null` (also `null` on Claude);
|
|
9
10
|
- `submittedExecCalls`, the Codex submitted-code count or `null` on Claude; and
|
|
10
11
|
- `truncated`, which is `false` for a complete record and may be unknown (`null`)
|
|
11
12
|
for an incomplete Codex record.
|
|
@@ -15,9 +16,71 @@ tool arguments, task messages, command bodies, or results. Call labels prove an
|
|
|
15
16
|
invocation attempt, not success. Extensions decide which calls are prohibited.
|
|
16
17
|
An unknown acceptance count cannot establish agent acceptance.
|
|
17
18
|
|
|
18
|
-
Codex projects labels and counts from its bound native-session observation
|
|
19
|
-
|
|
20
|
-
|
|
19
|
+
Codex projects labels and counts from its bound native-session observation.
|
|
20
|
+
It additionally retains `directAcceptedAgentCount`, the number of positively
|
|
21
|
+
correlated direct native spawn acceptances in that source, independently of exec
|
|
22
|
+
inspection. This is an observed count, not a total across unknown launch paths.
|
|
23
|
+
`coverage.nativeCalls` records native-session completeness;
|
|
24
|
+
`coverage.execInspection` records whether all submitted exec sources were
|
|
25
|
+
understood and matched to the direct submission inventory;
|
|
26
|
+
`coverage.acceptedAgents: direct_native_calls` names the scope of both acceptance
|
|
27
|
+
counts. Accepted launches through exec are not included in these direct counts.
|
|
28
|
+
When native capture is incomplete, even a zero observed count cannot establish
|
|
29
|
+
absence. Native labels remain usable as positive attempt evidence regardless
|
|
30
|
+
of an unrelated exec inspection gap. A rejected attempt is never an acceptance.
|
|
31
|
+
|
|
32
|
+
When submitted code is present, both observations also retain `execCalls`:
|
|
33
|
+
one receipt per native `exec` submission, with its native `ordinal`, inspection
|
|
34
|
+
`completeness`, and ordered `calls` containing `index`, `namespace`, and `name`.
|
|
35
|
+
The index is local to that submission; native ordinals are not synthesized.
|
|
36
|
+
The original direct-call list and total `submittedExecCalls` stay unchanged.
|
|
37
|
+
|
|
38
|
+
Sevro statically inspects bounded straight-line Code Mode
|
|
39
|
+
programs without executing them. It accepts literal tool calls, constant result
|
|
40
|
+
bindings, template interpolation and `+` composition of inspected expressions,
|
|
41
|
+
direct `text`/`store` output, and literal `Promise.all`/`allSettled` batches.
|
|
42
|
+
Batch results may use flat constant array bindings or result-only
|
|
43
|
+
`forEach(text)` or `for ... of` output, including blocks and conditional
|
|
44
|
+
result-field selection. `JSON.stringify`, `String`, object spread/shorthand,
|
|
45
|
+
and nested result fields are supported. Numeric output loops must use the
|
|
46
|
+
bounded shape `for (let i = 0; i < results.length; i++)`, indexing the known
|
|
47
|
+
batch with that loop variable. Output loops and conditional expressions cannot
|
|
48
|
+
invoke tools. Destructuring defaults, rest elements,
|
|
49
|
+
holes, nested bindings, duplicate names, and shadowed globals remain unsupported.
|
|
50
|
+
Tool argument strings, comments and printed tool-looking text
|
|
51
|
+
do not become call labels. A rejected tool request returned by a completed
|
|
52
|
+
script still appears in its submitted-call inventory. This inventory proves
|
|
53
|
+
submission of those requests, not successful effects or acceptance; the existing
|
|
54
|
+
acceptance count describes independently correlated direct native spawns.
|
|
55
|
+
|
|
56
|
+
Dynamic lookup, aliases, shadowed globals, callbacks that invoke tools,
|
|
57
|
+
conditional tool execution, and nested executors keep source inspection partial.
|
|
58
|
+
Each receipt's `inspectionReason` is `null` on success, or `unsupported_source`,
|
|
59
|
+
`malformed_source`, `source_limit`, or `foreign_namespace`. Unsupported syntax
|
|
60
|
+
and expression/call-budget exhaustion use `unsupported_source`.
|
|
61
|
+
Source is bounded at
|
|
62
|
+
64 KiB, traversed expressions at 2,048, and calls at 128 per submission; tool
|
|
63
|
+
labels are bounded at 64 characters. Private source and output bytes are discarded.
|
|
64
|
+
The portable observation becomes partial if any submission is uninspected or
|
|
65
|
+
receipt counts/ordinals disagree with the native list. `acceptedAgentCount`
|
|
66
|
+
remains `null` in that case, while native coverage, direct acceptance facts and
|
|
67
|
+
the native `submittedExecCalls` count remain independently available.
|
|
68
|
+
`submittedExecCalls` and `truncated` depend on native capture, not exec output.
|
|
69
|
+
|
|
70
|
+
Receipt `completeness` describes source inspection, independently of execution
|
|
71
|
+
or displayed output. `execution` is `completed`, `failed`, or `incomplete`;
|
|
72
|
+
`output` is `complete`, `truncated`, or `unavailable`. Output evidence requires
|
|
73
|
+
a unique later output bound to the same native call ID, with recognized text
|
|
74
|
+
blocks. `complete` describes the displayed receipt, not the whole program's
|
|
75
|
+
execution. Execution status additionally requires a successful or failed script
|
|
76
|
+
header. Missing, ambiguous or
|
|
77
|
+
malformed output yields `incomplete`/`unavailable`; a yielded or unrecognized
|
|
78
|
+
header stays `incomplete`. Displayed truncation does not change source coverage.
|
|
79
|
+
A fully inspected source with no launch call proves that absence even if its
|
|
80
|
+
shell output was truncated or the exec cell failed. Source inspection cannot
|
|
81
|
+
prove execution or success. Missing fields in historical receipts are unknown;
|
|
82
|
+
historical artifacts retain their original results and meanings.
|
|
83
|
+
|
|
21
84
|
Claude records every tool-use label from the bounded stream, with namespace
|
|
22
85
|
`claude`, without filtering to Skill or Agent. It requires a completed successful
|
|
23
86
|
turn, valid unique tool-use IDs and names, and a non-truncated list of at most
|
|
@@ -26,4 +89,12 @@ Tool-looking text in arguments or output does not become a native call.
|
|
|
26
89
|
|
|
27
90
|
Consumers must verify the source, completeness, shape, ordering, and counts
|
|
28
91
|
before using an empty call list to establish absence. Missing or partial
|
|
29
|
-
|
|
92
|
+
source coverage cannot establish a negative assertion. For Codex, consumers must
|
|
93
|
+
inspect every `execCalls` receipt in addition to the direct `calls` list,
|
|
94
|
+
validate receipt ordinals against the direct `exec` submissions, and require
|
|
95
|
+
all receipts to be complete with counts matching `submittedExecCalls`.
|
|
96
|
+
They can use positively observed native attempt labels before requiring complete
|
|
97
|
+
coverage for an absence pass. They must examine submitted labels for prohibited controls even when
|
|
98
|
+
`acceptedAgentCount` is zero. Older consumers that require
|
|
99
|
+
`submittedExecCalls === 0` remain conservative but cannot use this new evidence
|
|
100
|
+
until they adopt receipt validation.
|