@xoxoai/checkmate 0.4.28 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +259 -224
- package/bin/checkmate.js +1 -1
- package/dist/ai/client.d.ts +45 -30
- package/dist/ai/client.d.ts.map +1 -1
- package/dist/ai/client.js +198 -129
- package/dist/ai/client.js.map +1 -1
- package/dist/ai/message-handler.d.ts +5 -7
- package/dist/ai/message-handler.d.ts.map +1 -1
- package/dist/ai/message-handler.js +38 -21
- package/dist/ai/message-handler.js.map +1 -1
- package/dist/ai/message-history.d.ts +4 -6
- package/dist/ai/message-history.d.ts.map +1 -1
- package/dist/ai/message-history.js +18 -21
- package/dist/ai/message-history.js.map +1 -1
- package/dist/ai/rate-limit-policy.d.ts +5 -3
- package/dist/ai/rate-limit-policy.d.ts.map +1 -1
- package/dist/ai/rate-limit-policy.js +24 -7
- package/dist/ai/rate-limit-policy.js.map +1 -1
- package/dist/ai/tool-response-handler.d.ts +11 -16
- package/dist/ai/tool-response-handler.d.ts.map +1 -1
- package/dist/ai/tool-response-handler.js +59 -28
- package/dist/ai/tool-response-handler.js.map +1 -1
- package/dist/ai/turn-processor.d.ts +34 -0
- package/dist/ai/turn-processor.d.ts.map +1 -0
- package/dist/ai/turn-processor.js +107 -0
- package/dist/ai/turn-processor.js.map +1 -0
- package/dist/api/allocate-run-identity.d.ts +8 -0
- package/dist/api/allocate-run-identity.d.ts.map +1 -0
- package/dist/api/allocate-run-identity.js +10 -0
- package/dist/api/allocate-run-identity.js.map +1 -0
- package/dist/api/execute-prepared-run.d.ts +22 -0
- package/dist/api/execute-prepared-run.d.ts.map +1 -0
- package/dist/api/execute-prepared-run.js +160 -0
- package/dist/api/execute-prepared-run.js.map +1 -0
- package/dist/api/index.d.ts +25 -0
- package/dist/api/index.d.ts.map +1 -0
- package/dist/api/index.js +69 -0
- package/dist/api/index.js.map +1 -0
- package/dist/api/prepare-run.d.ts +44 -0
- package/dist/api/prepare-run.d.ts.map +1 -0
- package/dist/api/prepare-run.js +261 -0
- package/dist/api/prepare-run.js.map +1 -0
- package/dist/cli/diagnostic-relay.d.ts +6 -0
- package/dist/cli/diagnostic-relay.d.ts.map +1 -0
- package/dist/cli/diagnostic-relay.js +6 -0
- package/dist/cli/diagnostic-relay.js.map +1 -0
- package/dist/cli/main.d.ts +21 -0
- package/dist/cli/main.d.ts.map +1 -0
- package/dist/cli/main.js +166 -0
- package/dist/cli/main.js.map +1 -0
- package/dist/cli/protocol.d.ts +57 -0
- package/dist/cli/protocol.d.ts.map +1 -0
- package/dist/cli/protocol.js +116 -0
- package/dist/cli/protocol.js.map +1 -0
- package/dist/cli/reconcile-result.d.ts +13 -0
- package/dist/cli/reconcile-result.d.ts.map +1 -0
- package/dist/cli/reconcile-result.js +300 -0
- package/dist/cli/reconcile-result.js.map +1 -0
- package/dist/cli/run-parent.d.ts +37 -0
- package/dist/cli/run-parent.d.ts.map +1 -0
- package/dist/cli/run-parent.js +491 -0
- package/dist/cli/run-parent.js.map +1 -0
- package/dist/cli/signals.d.ts +36 -0
- package/dist/cli/signals.d.ts.map +1 -0
- package/dist/cli/signals.js +85 -0
- package/dist/cli/signals.js.map +1 -0
- package/dist/cli/static-commands.d.ts +11 -0
- package/dist/cli/static-commands.d.ts.map +1 -0
- package/dist/cli/static-commands.js +83 -0
- package/dist/cli/static-commands.js.map +1 -0
- package/dist/cli/worker-stream-guard.d.ts +20 -0
- package/dist/cli/worker-stream-guard.d.ts.map +1 -0
- package/dist/cli/worker-stream-guard.js +41 -0
- package/dist/cli/worker-stream-guard.js.map +1 -0
- package/dist/cli/worker.d.ts +3 -0
- package/dist/cli/worker.d.ts.map +1 -0
- package/dist/cli/worker.js +111 -0
- package/dist/cli/worker.js.map +1 -0
- package/dist/config/ingestion.d.ts +58 -0
- package/dist/config/ingestion.d.ts.map +1 -0
- package/dist/config/ingestion.js +283 -0
- package/dist/config/ingestion.js.map +1 -0
- package/dist/config/manifest.d.ts +20 -0
- package/dist/config/manifest.d.ts.map +1 -0
- package/dist/config/manifest.js +91 -0
- package/dist/config/manifest.js.map +1 -0
- package/dist/config/model-egress.d.ts +10 -0
- package/dist/config/model-egress.d.ts.map +1 -0
- package/dist/config/model-egress.js +62 -0
- package/dist/config/model-egress.js.map +1 -0
- package/dist/config/package-resolution.d.ts +4 -0
- package/dist/config/package-resolution.d.ts.map +1 -0
- package/dist/config/package-resolution.js +24 -0
- package/dist/config/package-resolution.js.map +1 -0
- package/dist/config/policy.d.ts +44 -0
- package/dist/config/policy.d.ts.map +1 -0
- package/dist/config/policy.js +141 -0
- package/dist/config/policy.js.map +1 -0
- package/dist/config/record.d.ts +4 -0
- package/dist/config/record.d.ts.map +1 -0
- package/dist/config/record.js +10 -0
- package/dist/config/record.js.map +1 -0
- package/dist/config/secrets.d.ts +11 -0
- package/dist/config/secrets.d.ts.map +1 -0
- package/dist/config/secrets.js +42 -0
- package/dist/config/secrets.js.map +1 -0
- package/dist/contracts/diagnostics.d.ts +4 -0
- package/dist/contracts/diagnostics.d.ts.map +1 -0
- package/dist/contracts/diagnostics.js +31 -0
- package/dist/contracts/diagnostics.js.map +1 -0
- package/dist/contracts/serialize.d.ts +2 -0
- package/dist/contracts/serialize.d.ts.map +1 -0
- package/dist/contracts/serialize.js +90 -0
- package/dist/contracts/serialize.js.map +1 -0
- package/dist/contracts/types.d.ts +316 -0
- package/dist/contracts/types.d.ts.map +1 -0
- package/dist/contracts/types.js +74 -0
- package/dist/contracts/types.js.map +1 -0
- package/dist/contracts/validator.d.ts +8 -0
- package/dist/contracts/validator.d.ts.map +1 -0
- package/dist/contracts/validator.js +71 -0
- package/dist/contracts/validator.js.map +1 -0
- package/dist/driver.d.ts +97 -0
- package/dist/driver.d.ts.map +1 -0
- package/dist/driver.js +24 -0
- package/dist/driver.js.map +1 -0
- package/dist/drivers/descriptor.d.ts +15 -0
- package/dist/drivers/descriptor.d.ts.map +1 -0
- package/dist/drivers/descriptor.js +122 -0
- package/dist/drivers/descriptor.js.map +1 -0
- package/dist/drivers/loader.d.ts +18 -0
- package/dist/drivers/loader.d.ts.map +1 -0
- package/dist/drivers/loader.js +97 -0
- package/dist/drivers/loader.js.map +1 -0
- package/dist/drivers/web/checkmate-driver.json +44 -0
- package/dist/drivers/web/index.d.ts +6 -0
- package/dist/drivers/web/index.d.ts.map +1 -0
- package/dist/drivers/web/index.js +27 -0
- package/dist/drivers/web/index.js.map +1 -0
- package/dist/drivers/web/session.d.ts +22 -0
- package/dist/drivers/web/session.d.ts.map +1 -0
- package/dist/drivers/web/session.js +130 -0
- package/dist/drivers/web/session.js.map +1 -0
- package/dist/drivers/web/tools/network-request-recorder.d.ts +46 -0
- package/dist/drivers/web/tools/network-request-recorder.d.ts.map +1 -0
- package/dist/drivers/web/tools/network-request-recorder.js +164 -0
- package/dist/drivers/web/tools/network-request-recorder.js.map +1 -0
- package/dist/{tools/browser → drivers/web/tools}/screenshot-service.d.ts +1 -1
- package/dist/drivers/web/tools/screenshot-service.d.ts.map +1 -0
- package/dist/drivers/web/tools/screenshot-service.js.map +1 -0
- package/dist/drivers/web/tools/snapshot-filter/index.d.ts.map +1 -0
- package/dist/drivers/web/tools/snapshot-filter/index.js.map +1 -0
- package/dist/drivers/web/tools/snapshot-filter/semantic-scorer.d.ts.map +1 -0
- package/dist/drivers/web/tools/snapshot-filter/semantic-scorer.js.map +1 -0
- package/dist/drivers/web/tools/snapshot-filter/snapshot-filter.d.ts +5 -0
- package/dist/drivers/web/tools/snapshot-filter/snapshot-filter.d.ts.map +1 -0
- package/dist/drivers/web/tools/snapshot-filter/snapshot-filter.js +56 -0
- package/dist/drivers/web/tools/snapshot-filter/snapshot-filter.js.map +1 -0
- package/dist/drivers/web/tools/snapshot-filter/tree-reconstructor.d.ts.map +1 -0
- package/dist/drivers/web/tools/snapshot-filter/tree-reconstructor.js.map +1 -0
- package/dist/drivers/web/tools/snapshot-service.d.ts +31 -0
- package/dist/drivers/web/tools/snapshot-service.d.ts.map +1 -0
- package/dist/{tools/browser → drivers/web/tools}/snapshot-service.js +36 -7
- package/dist/drivers/web/tools/snapshot-service.js.map +1 -0
- package/dist/drivers/web/tools/tool.d.ts +76 -0
- package/dist/drivers/web/tools/tool.d.ts.map +1 -0
- package/dist/drivers/web/tools/tool.js +562 -0
- package/dist/drivers/web/tools/tool.js.map +1 -0
- package/dist/drivers/web/tools/transient-state-tracker.d.ts +26 -0
- package/dist/drivers/web/tools/transient-state-tracker.d.ts.map +1 -0
- package/dist/{tools/browser → drivers/web/tools}/transient-state-tracker.js +54 -19
- package/dist/drivers/web/tools/transient-state-tracker.js.map +1 -0
- package/dist/drivers/web/tools/types.d.ts +11 -0
- package/dist/drivers/web/tools/types.d.ts.map +1 -0
- package/dist/drivers/web/tools/types.js +2 -0
- package/dist/drivers/web/tools/types.js.map +1 -0
- package/dist/evidence/atomic-file.d.ts +23 -0
- package/dist/evidence/atomic-file.d.ts.map +1 -0
- package/dist/evidence/atomic-file.js +104 -0
- package/dist/evidence/atomic-file.js.map +1 -0
- package/dist/evidence/layout.d.ts +30 -0
- package/dist/evidence/layout.d.ts.map +1 -0
- package/dist/evidence/layout.js +152 -0
- package/dist/evidence/layout.js.map +1 -0
- package/dist/evidence/retention.d.ts +5 -0
- package/dist/evidence/retention.d.ts.map +1 -0
- package/dist/evidence/retention.js +10 -0
- package/dist/evidence/retention.js.map +1 -0
- package/dist/evidence/store.d.ts +101 -0
- package/dist/evidence/store.d.ts.map +1 -0
- package/dist/evidence/store.js +375 -0
- package/dist/evidence/store.js.map +1 -0
- package/dist/evidence/terminal-finalizer.d.ts +19 -0
- package/dist/evidence/terminal-finalizer.d.ts.map +1 -0
- package/dist/evidence/terminal-finalizer.js +56 -0
- package/dist/evidence/terminal-finalizer.js.map +1 -0
- package/dist/index.d.ts +8 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +11 -1
- package/dist/index.js.map +1 -1
- package/dist/logging/invocation-logger.d.ts +4 -0
- package/dist/logging/invocation-logger.d.ts.map +1 -0
- package/dist/logging/invocation-logger.js +25 -0
- package/dist/logging/invocation-logger.js.map +1 -0
- package/dist/logging/types.d.ts +8 -0
- package/dist/logging/types.d.ts.map +1 -0
- package/dist/logging/types.js +7 -0
- package/dist/logging/types.js.map +1 -0
- package/dist/redaction/diagnostic-sanitizer.d.ts +7 -0
- package/dist/redaction/diagnostic-sanitizer.d.ts.map +1 -0
- package/dist/redaction/diagnostic-sanitizer.js +42 -0
- package/dist/redaction/diagnostic-sanitizer.js.map +1 -0
- package/dist/redaction/redactor.d.ts +40 -0
- package/dist/redaction/redactor.d.ts.map +1 -0
- package/dist/redaction/redactor.js +176 -0
- package/dist/redaction/redactor.js.map +1 -0
- package/dist/redaction/scrub.d.ts +29 -0
- package/dist/redaction/scrub.d.ts.map +1 -0
- package/dist/redaction/scrub.js +54 -0
- package/dist/redaction/scrub.js.map +1 -0
- package/dist/runtime/config.d.ts +15 -0
- package/dist/runtime/config.d.ts.map +1 -0
- package/dist/runtime/config.js +2 -0
- package/dist/runtime/config.js.map +1 -0
- package/dist/runtime/driver-boundary.d.ts +16 -0
- package/dist/runtime/driver-boundary.d.ts.map +1 -0
- package/dist/runtime/driver-boundary.js +71 -0
- package/dist/runtime/driver-boundary.js.map +1 -0
- package/dist/runtime/driver-session.d.ts +9 -0
- package/dist/runtime/driver-session.d.ts.map +1 -0
- package/dist/runtime/driver-session.js +64 -0
- package/dist/runtime/driver-session.js.map +1 -0
- package/dist/runtime/internal-step-evidence.d.ts +30 -0
- package/dist/runtime/internal-step-evidence.d.ts.map +1 -0
- package/dist/runtime/internal-step-evidence.js +82 -0
- package/dist/runtime/internal-step-evidence.js.map +1 -0
- package/dist/runtime/result-adapter.d.ts +9 -0
- package/dist/runtime/result-adapter.d.ts.map +1 -0
- package/dist/runtime/result-adapter.js +77 -0
- package/dist/runtime/result-adapter.js.map +1 -0
- package/dist/runtime/runner.d.ts +35 -88
- package/dist/runtime/runner.d.ts.map +1 -1
- package/dist/runtime/runner.js +60 -74
- package/dist/runtime/runner.js.map +1 -1
- package/dist/runtime/scenario-control.d.ts +40 -0
- package/dist/runtime/scenario-control.d.ts.map +1 -0
- package/dist/runtime/scenario-control.js +94 -0
- package/dist/runtime/scenario-control.js.map +1 -0
- package/dist/runtime/scenario-runner.d.ts +39 -0
- package/dist/runtime/scenario-runner.d.ts.map +1 -0
- package/dist/runtime/scenario-runner.js +273 -0
- package/dist/runtime/scenario-runner.js.map +1 -0
- package/dist/runtime/scenario-state.d.ts +55 -0
- package/dist/runtime/scenario-state.d.ts.map +1 -0
- package/dist/runtime/scenario-state.js +128 -0
- package/dist/runtime/scenario-state.js.map +1 -0
- package/dist/runtime/step-execution.d.ts +24 -14
- package/dist/runtime/step-execution.d.ts.map +1 -1
- package/dist/runtime/step-execution.js +130 -36
- package/dist/runtime/step-execution.js.map +1 -1
- package/dist/runtime/text.d.ts +10 -0
- package/dist/runtime/text.d.ts.map +1 -0
- package/dist/runtime/text.js +25 -0
- package/dist/runtime/text.js.map +1 -0
- package/dist/runtime/types.d.ts +50 -76
- package/dist/runtime/types.d.ts.map +1 -1
- package/dist/runtime/usage-tracker.d.ts +37 -0
- package/dist/runtime/usage-tracker.d.ts.map +1 -0
- package/dist/runtime/usage-tracker.js +110 -0
- package/dist/runtime/usage-tracker.js.map +1 -0
- package/dist/tools/define-agent-tool.d.ts +3 -28
- package/dist/tools/define-agent-tool.d.ts.map +1 -1
- package/dist/tools/define-agent-tool.js +3 -28
- package/dist/tools/define-agent-tool.js.map +1 -1
- package/dist/tools/dispatcher.d.ts +11 -3
- package/dist/tools/dispatcher.d.ts.map +1 -1
- package/dist/tools/dispatcher.js +50 -8
- package/dist/tools/dispatcher.js.map +1 -1
- package/dist/tools/registry.d.ts +8 -10
- package/dist/tools/registry.d.ts.map +1 -1
- package/dist/tools/registry.js +19 -11
- package/dist/tools/registry.js.map +1 -1
- package/dist/tools/step/result-tool.d.ts.map +1 -1
- package/dist/tools/step/result-tool.js +8 -6
- package/dist/tools/step/result-tool.js.map +1 -1
- package/dist/tools/tool-contract.d.ts +1 -1
- package/dist/tools/tool-contract.d.ts.map +1 -1
- package/dist/tools/tool-contract.js.map +1 -1
- package/dist/tools/types.d.ts +77 -6
- package/dist/tools/types.d.ts.map +1 -1
- package/dist/tools/types.js.map +1 -1
- package/docs/CLI.md +59 -0
- package/docs/CONFIGURATION.md +91 -0
- package/docs/DRIVERS.md +75 -0
- package/docs/EVIDENCE.md +62 -0
- package/package.json +51 -59
- package/schemas/checkmate-config.v1.json +176 -0
- package/schemas/describe-result.v1.json +414 -0
- package/schemas/driver-descriptor.v1.json +66 -0
- package/schemas/run-request.v1.json +67 -0
- package/schemas/run-result.v1.json +333 -0
- package/schemas/validation-result.v1.json +86 -0
- package/dist/ai/response-processor.d.ts +0 -20
- package/dist/ai/response-processor.d.ts.map +0 -1
- package/dist/ai/response-processor.js +0 -62
- package/dist/ai/response-processor.js.map +0 -1
- package/dist/ai/token-pricing.d.ts +0 -13
- package/dist/ai/token-pricing.d.ts.map +0 -1
- package/dist/ai/token-pricing.js +0 -348
- package/dist/ai/token-pricing.js.map +0 -1
- package/dist/ai/token-tracker.d.ts +0 -33
- package/dist/ai/token-tracker.d.ts.map +0 -1
- package/dist/ai/token-tracker.js +0 -138
- package/dist/ai/token-tracker.js.map +0 -1
- package/dist/cli/create-examples.d.ts +0 -21
- package/dist/cli/create-examples.d.ts.map +0 -1
- package/dist/cli/create-examples.js +0 -116
- package/dist/cli/create-examples.js.map +0 -1
- package/dist/cli.d.ts +0 -15
- package/dist/cli.d.ts.map +0 -1
- package/dist/cli.js +0 -69
- package/dist/cli.js.map +0 -1
- package/dist/config/runtime-config.d.ts +0 -21
- package/dist/config/runtime-config.d.ts.map +0 -1
- package/dist/config/runtime-config.js +0 -98
- package/dist/config/runtime-config.js.map +0 -1
- package/dist/core.d.ts +0 -8
- package/dist/core.d.ts.map +0 -1
- package/dist/core.js +0 -4
- package/dist/core.js.map +0 -1
- package/dist/integrations/salesforce/authenticator.d.ts +0 -27
- package/dist/integrations/salesforce/authenticator.d.ts.map +0 -1
- package/dist/integrations/salesforce/authenticator.js +0 -27
- package/dist/integrations/salesforce/authenticator.js.map +0 -1
- package/dist/integrations/salesforce/cli-handler.d.ts +0 -14
- package/dist/integrations/salesforce/cli-handler.d.ts.map +0 -1
- package/dist/integrations/salesforce/cli-handler.js +0 -50
- package/dist/integrations/salesforce/cli-handler.js.map +0 -1
- package/dist/logging/index.d.ts +0 -2
- package/dist/logging/index.d.ts.map +0 -1
- package/dist/logging/index.js +0 -4
- package/dist/logging/index.js.map +0 -1
- package/dist/logging/logger.d.ts +0 -5
- package/dist/logging/logger.d.ts.map +0 -1
- package/dist/logging/logger.js +0 -15
- package/dist/logging/logger.js.map +0 -1
- package/dist/playwright.d.ts +0 -102
- package/dist/playwright.d.ts.map +0 -1
- package/dist/playwright.js +0 -116
- package/dist/playwright.js.map +0 -1
- package/dist/runtime/extension.d.ts +0 -274
- package/dist/runtime/extension.d.ts.map +0 -1
- package/dist/runtime/extension.js +0 -171
- package/dist/runtime/extension.js.map +0 -1
- package/dist/salesforce.d.ts +0 -71
- package/dist/salesforce.d.ts.map +0 -1
- package/dist/salesforce.js +0 -73
- package/dist/salesforce.js.map +0 -1
- package/dist/tools/browser/screenshot-service.d.ts.map +0 -1
- package/dist/tools/browser/screenshot-service.js.map +0 -1
- package/dist/tools/browser/snapshot-filter/index.d.ts.map +0 -1
- package/dist/tools/browser/snapshot-filter/index.js.map +0 -1
- package/dist/tools/browser/snapshot-filter/semantic-scorer.d.ts.map +0 -1
- package/dist/tools/browser/snapshot-filter/semantic-scorer.js.map +0 -1
- package/dist/tools/browser/snapshot-filter/snapshot-filter.d.ts +0 -4
- package/dist/tools/browser/snapshot-filter/snapshot-filter.d.ts.map +0 -1
- package/dist/tools/browser/snapshot-filter/snapshot-filter.js +0 -57
- package/dist/tools/browser/snapshot-filter/snapshot-filter.js.map +0 -1
- package/dist/tools/browser/snapshot-filter/tree-reconstructor.d.ts.map +0 -1
- package/dist/tools/browser/snapshot-filter/tree-reconstructor.js.map +0 -1
- package/dist/tools/browser/snapshot-service.d.ts +0 -19
- package/dist/tools/browser/snapshot-service.d.ts.map +0 -1
- package/dist/tools/browser/snapshot-service.js.map +0 -1
- package/dist/tools/browser/tool.d.ts +0 -34
- package/dist/tools/browser/tool.d.ts.map +0 -1
- package/dist/tools/browser/tool.js +0 -226
- package/dist/tools/browser/tool.js.map +0 -1
- package/dist/tools/browser/transient-state-tracker.d.ts +0 -16
- package/dist/tools/browser/transient-state-tracker.d.ts.map +0 -1
- package/dist/tools/browser/transient-state-tracker.js.map +0 -1
- package/dist/tools/salesforce/login-tool.d.ts +0 -7
- package/dist/tools/salesforce/login-tool.d.ts.map +0 -1
- package/dist/tools/salesforce/login-tool.js +0 -25
- package/dist/tools/salesforce/login-tool.js.map +0 -1
- package/docs/EXTENSIONS.md +0 -232
- package/docs/GUIDE.md +0 -465
- package/docs/ROADMAP.md +0 -47
- package/playwright.config.ts +0 -39
- package/test/examples/salesforce/trial-dev-org.spec.ts +0 -82
- package/test/examples/web/website-testing.spec.ts +0 -281
- /package/dist/{tools/browser → drivers/web/tools}/screenshot-service.js +0 -0
- /package/dist/{tools/browser → drivers/web/tools}/snapshot-filter/index.d.ts +0 -0
- /package/dist/{tools/browser → drivers/web/tools}/snapshot-filter/index.js +0 -0
- /package/dist/{tools/browser → drivers/web/tools}/snapshot-filter/semantic-scorer.d.ts +0 -0
- /package/dist/{tools/browser → drivers/web/tools}/snapshot-filter/semantic-scorer.js +0 -0
- /package/dist/{tools/browser → drivers/web/tools}/snapshot-filter/tree-reconstructor.d.ts +0 -0
- /package/dist/{tools/browser → drivers/web/tools}/snapshot-filter/tree-reconstructor.js +0 -0
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# Configuration Reference
|
|
2
|
+
|
|
3
|
+
`checkmate.config.json` is the sole operator-controlled configuration source. It separates approved package registration and logical secret bindings from policy-owned execution behavior.
|
|
4
|
+
|
|
5
|
+
## Lookup
|
|
6
|
+
|
|
7
|
+
- Default: `$PWD/checkmate.config.json`
|
|
8
|
+
- Explicit: `--config <path>`, resolved from `$PWD`
|
|
9
|
+
- Invocation root: always `$PWD`, even when the config file is elsewhere
|
|
10
|
+
- No ancestor search, package metadata fallback, TypeScript config, or low-level environment override layer
|
|
11
|
+
|
|
12
|
+
Relative output paths and driver package resolution remain anchored to the invocation root.
|
|
13
|
+
|
|
14
|
+
## Manifest Sections
|
|
15
|
+
|
|
16
|
+
| Field | Purpose |
|
|
17
|
+
| ----------------- | -------------------------------------------------------------- |
|
|
18
|
+
| `schemaVersion` | Must be `1` |
|
|
19
|
+
| `outputDirectory` | Relative run root; defaults to `.checkmate/runs` |
|
|
20
|
+
| `defaultPolicy` | Policy selected when a request omits `scenario.policy` |
|
|
21
|
+
| `secretBindings` | Logical names mapped to environment variable names |
|
|
22
|
+
| `policies` | Model egress, bounds, evidence, and enabled-driver settings |
|
|
23
|
+
| `drivers` | Stable driver IDs mapped to approved packages and secret slots |
|
|
24
|
+
|
|
25
|
+
Unknown fields are rejected. Validation does not coerce, strip, default inside input documents, or clamp values.
|
|
26
|
+
|
|
27
|
+
## Model Egress
|
|
28
|
+
|
|
29
|
+
Each policy owns:
|
|
30
|
+
|
|
31
|
+
| Field | Meaning |
|
|
32
|
+
| -------------------------- | ------------------------------------------------------- |
|
|
33
|
+
| `provider.id` | `openai` in v1 |
|
|
34
|
+
| `provider.model` | Provider model identifier |
|
|
35
|
+
| `provider.baseUrl` | Optional OpenAI-compatible endpoint |
|
|
36
|
+
| `provider.apiKeyBinding` | Logical binding, never a secret value |
|
|
37
|
+
| `provider.temperature` | Optional non-negative number |
|
|
38
|
+
| `provider.reasoningEffort` | Optional `low`, `medium`, or `high` |
|
|
39
|
+
| `textRedaction` | `on` or `off` for provider-bound text |
|
|
40
|
+
| `allowOpaque` | Whether provider-bound opaque/image context is allowed |
|
|
41
|
+
| `maxStepBytes` | Maximum complete provider request bytes for a step |
|
|
42
|
+
| `maxMessageBytes` | Maximum serialized bytes for one provider-bound message |
|
|
43
|
+
|
|
44
|
+
The complete-request limit covers messages, tools, and JSON framing, not HTTP headers. Requests cannot override any model-egress field.
|
|
45
|
+
|
|
46
|
+
## Bounds
|
|
47
|
+
|
|
48
|
+
| Field | Valid value | Scope |
|
|
49
|
+
| -------------------- | --------------------: | ---------------------------------------------------- |
|
|
50
|
+
| `scenarioTimeoutMs` | Integer >= 1 | Entire worker scenario, including import and startup |
|
|
51
|
+
| `stepTimeoutMs` | Integer >= 1 | One model/tool step |
|
|
52
|
+
| `turnsPerStep` | Integer >= 1 | Model turns per step |
|
|
53
|
+
| `requestTimeoutMs` | Integer >= 1 | One provider request |
|
|
54
|
+
| `maxRetries` | Integer >= 0 | Provider-level retries |
|
|
55
|
+
| `loopMaxRepetitions` | Integer >= 1 | Repeated tool-call pattern bound |
|
|
56
|
+
| `cleanupTimeoutMs` | Integer >= 1 | Shared runner/session cleanup deadline |
|
|
57
|
+
| `budgetTokens` | Optional integer >= 1 | Cumulative scenario token ceiling |
|
|
58
|
+
|
|
59
|
+
A request may only tighten `scenarioTimeoutMs` through `limits.timeoutMs`, and may tighten or introduce `limits.budgetTokens`. Attempts to raise a policy ceiling are invalid. V1 has no USD budget, cost calculation, or pricing table.
|
|
60
|
+
|
|
61
|
+
## Evidence Policy
|
|
62
|
+
|
|
63
|
+
- `retention`: `on`, `retain-on-failure`, or `off`
|
|
64
|
+
- `redaction`: `on` or `off`
|
|
65
|
+
- `allowOpaque`: explicit permission for binary evidence that cannot be text-redacted
|
|
66
|
+
|
|
67
|
+
Requests cannot weaken these settings.
|
|
68
|
+
|
|
69
|
+
## Driver Policy
|
|
70
|
+
|
|
71
|
+
The root `drivers` entry approves a package and maps its descriptor-declared secret slots to logical bindings. `policies.<id>.drivers.<driverId>` enables that registered driver, supplies settings validated against its static descriptor, and provides `tools.allowed`.
|
|
72
|
+
|
|
73
|
+
`["*"]` exposes every descriptor-declared tool. An explicit list exposes only those names. Unknown names, undeclared secret slots, missing slot bindings, unknown drivers, and drivers not enabled by the selected policy are invalid before executable import.
|
|
74
|
+
|
|
75
|
+
## Secret Handling
|
|
76
|
+
|
|
77
|
+
Preflight reads a binding only long enough to verify that its environment value is a non-empty string. It does not return, retain, serialize, log, persist, derive from, or send that value over IPC. The worker resolves values independently for execution. Secret values never enter the prepared plan, request, result, checkpoint, evidence reference, or `describe` output.
|
|
78
|
+
|
|
79
|
+
JavaScript heap erasure is not guaranteed. Registered executable code remains trusted and can access the host environment outside Checkmate's reader.
|
|
80
|
+
|
|
81
|
+
## Fixed Ingestion Limits
|
|
82
|
+
|
|
83
|
+
Checkmate applies fixed parent-side limits before expensive validation, executable import, identity allocation, secret resolution, or worker creation:
|
|
84
|
+
|
|
85
|
+
- 256 KiB each for request, manifest, and descriptor documents
|
|
86
|
+
- Parsed depth 64 and 10,000 aggregate values
|
|
87
|
+
- 100 scenario steps
|
|
88
|
+
- 128 characters for IDs, 256 for scenario names
|
|
89
|
+
- 16 KiB UTF-8 each for action and expectation text
|
|
90
|
+
- 100 diagnostics and 64 KiB of serialized diagnostics
|
|
91
|
+
- 1 MiB serialized worker start frame
|
package/docs/DRIVERS.md
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# Driver Authoring
|
|
2
|
+
|
|
3
|
+
A driver gives the harness a model-visible interaction surface while Checkmate retains scenario sequencing, limits, verdict tools, result categories, evidence policy, and stdout behavior.
|
|
4
|
+
|
|
5
|
+
## Registration And Static Discovery
|
|
6
|
+
|
|
7
|
+
An installed driver package exports both its executable module and `./checkmate-driver.json`. The manifest maps a trusted stable ID to the package. `describe` and `validate` resolve only the descriptor export as data; executable code is imported only after a validated `run` allocates identity and latches `targetMutation` to `possibly-mutated`.
|
|
8
|
+
|
|
9
|
+
The descriptor declares:
|
|
10
|
+
|
|
11
|
+
- `id` and `driverContractVersion: 1`
|
|
12
|
+
- Strict JSON Schemas for request-owned target and policy-owned settings
|
|
13
|
+
- Required local secret slot names
|
|
14
|
+
- The exact runtime tool-name set
|
|
15
|
+
- Permitted evidence kinds, media types, and text/opaque classification
|
|
16
|
+
|
|
17
|
+
At startup, the executable ID and contract version must match the descriptor. Runtime tools must have exact set equality with descriptor tools and may not be duplicated or collide with the harness-owned `pass_test_step` and `fail_test_step` tools.
|
|
18
|
+
|
|
19
|
+
## Executable Contract
|
|
20
|
+
|
|
21
|
+
```typescript
|
|
22
|
+
import type { CheckmateDriverV1 } from '@xoxoai/checkmate/driver'
|
|
23
|
+
|
|
24
|
+
export const checkmateDriver: CheckmateDriverV1 = {
|
|
25
|
+
id: 'database',
|
|
26
|
+
driverContractVersion: 1,
|
|
27
|
+
async start(input) {
|
|
28
|
+
const connection = await connect(input.secrets.read('connection'))
|
|
29
|
+
return {
|
|
30
|
+
tools: databaseTools(connection),
|
|
31
|
+
instructions: ['Use read-only queries to inspect application state.'],
|
|
32
|
+
buildInitialContext: async ({ signal }) => currentSchema(connection, signal),
|
|
33
|
+
handleToolResponses: async ({ signal }) => currentSchema(connection, signal),
|
|
34
|
+
close: async () => connection.close(),
|
|
35
|
+
}
|
|
36
|
+
},
|
|
37
|
+
}
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
`DriverStartInput` supplies validated `target` and `settings`, a slot-limited secret reader, a descriptor-limited evidence sink, an invocation logger, and an abort signal. The session supplies tools, instructions, initial context, post-tool context, and one idempotent close operation.
|
|
41
|
+
|
|
42
|
+
## Tools
|
|
43
|
+
|
|
44
|
+
Use `defineDriverTool()` for Zod-backed tool arguments. Tool wire names remain local names such as `database_query`; they are not prefixed with the driver ID because one scenario has one driver session.
|
|
45
|
+
|
|
46
|
+
A returned `{ status: 'error', response }` is a recoverable target observation. It is sent to the model and the step continues. A thrown or rejected tool operation means the driver boundary is unsafe and terminates the step as `infra / tool-error`. Drivers cannot return assertions; only harness-owned result tools decide whether an expectation passed or failed.
|
|
47
|
+
|
|
48
|
+
## Context
|
|
49
|
+
|
|
50
|
+
Driver context is provider-neutral text or image data. `ephemeral: true` marks current target state that should replace prior ephemeral state after actions. Explicit summaries may be durable. Checkmate applies policy-owned model-egress redaction, opaque-content permission, and byte limits before provider adaptation.
|
|
51
|
+
|
|
52
|
+
Every awaited executable boundary is wrapped by a harness-owned absolute deadline:
|
|
53
|
+
|
|
54
|
+
- Driver module import and `start`: remaining scenario deadline
|
|
55
|
+
- Initial context, tools, and post-tool context: current step deadline
|
|
56
|
+
- Runner teardown and session close: one shared cleanup deadline
|
|
57
|
+
|
|
58
|
+
Abort signals are cooperative. The CLI parent terminates a worker that does not settle within its containment bound.
|
|
59
|
+
|
|
60
|
+
## Secrets, Logs, And Evidence
|
|
61
|
+
|
|
62
|
+
Drivers read only descriptor-declared secret slots. Logger messages are exact-secret and pattern sanitized before leaving the worker. Direct writes to stdout or stderr are discarded.
|
|
63
|
+
|
|
64
|
+
Drivers submit evidence by declared kind and media type, never by path. The sink validates attribution and lifecycle, enforces 10 MiB candidate and 64 MiB per-invocation accepted-buffer limits, applies redaction and retention, writes atomically, and publishes a reference only after durability confirmation.
|
|
65
|
+
|
|
66
|
+
## Built-In Web Driver
|
|
67
|
+
|
|
68
|
+
`@xoxoai/checkmate/driver-web` owns Playwright browser, context, pages, snapshots, screenshots, tab/popup state, dialogs, transient DOM state, network recording, and fourteen browser tools. It accepts target `{ "baseUrl": "https://..." }` and these optional policy settings:
|
|
69
|
+
|
|
70
|
+
- `headless`
|
|
71
|
+
- `snapshotFilter`
|
|
72
|
+
- `snapshotTopPercent` in `(0, 100]`
|
|
73
|
+
- `screenshotsInModelContext`
|
|
74
|
+
|
|
75
|
+
Importing the root contract or driver-authoring entry does not load Playwright. Only selecting and executing the web driver imports the browser runtime.
|
package/docs/EVIDENCE.md
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# Evidence And Durability
|
|
2
|
+
|
|
3
|
+
Every valid run owns one invocation directory beneath the manifest's `outputDirectory`.
|
|
4
|
+
|
|
5
|
+
```text
|
|
6
|
+
.checkmate/runs/<utc>-<scenario-slug>-<run-id>/
|
|
7
|
+
|-- invocation.json
|
|
8
|
+
|-- checkpoint.json
|
|
9
|
+
|-- evidence/
|
|
10
|
+
| |-- harness/steps/001-<step-slug>/transcript.md
|
|
11
|
+
| `-- driver/<driver-id>/...
|
|
12
|
+
`-- result.json
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
The 16-character lowercase hexadecimal `runId` is generated from eight random bytes. It identifies one attempt and is not a request hash. Concurrent runs, including runs with the same scenario ID, receive separate directories and mutable state.
|
|
16
|
+
|
|
17
|
+
## Compact Result
|
|
18
|
+
|
|
19
|
+
The final envelope always carries compact routing and diagnosis data: top-level status/category/reason, target mutation state, timing, driver and policy identity, effective limits, aggregate tokens, every step state, compact tool-call summaries, evidence completeness, references, and diagnostics.
|
|
20
|
+
|
|
21
|
+
Transcripts, snapshots, screenshots, and driver-specific artifacts remain outside the envelope. Follow only `evidence.references`; never infer filenames or promote `checkpoint.json` to a result.
|
|
22
|
+
|
|
23
|
+
## Retention
|
|
24
|
+
|
|
25
|
+
| Policy | Passed step | Failed step |
|
|
26
|
+
| ------------------- | ------------------------------ | ------------------------------ |
|
|
27
|
+
| `on` | Retain selected heavy evidence | Retain selected heavy evidence |
|
|
28
|
+
| `retain-on-failure` | Discard heavy evidence | Retain selected heavy evidence |
|
|
29
|
+
| `off` | Discard heavy evidence | Discard heavy evidence |
|
|
30
|
+
|
|
31
|
+
Compact result data is retained independently of this table. Opaque evidence is discarded unless the operator also sets `allowOpaque: true`.
|
|
32
|
+
|
|
33
|
+
`evidence.state: "complete"` means the selected policy was fulfilled, including when it required no heavy artifacts. `partial` means an optional capture or persistence operation failed. A driver is not required to produce every declared evidence kind on every run.
|
|
34
|
+
|
|
35
|
+
## Redaction
|
|
36
|
+
|
|
37
|
+
With `redaction: "on"`, Checkmate applies exact resolved-secret replacement, credential-pattern recognition, and field-aware content redaction before data becomes durable or enters terminal IPC. JSON and YAML evidence is parsed, redacted structurally, and deterministically reserialized. Malformed structured evidence is rejected.
|
|
38
|
+
|
|
39
|
+
Structural metadata is copied exactly and must not contain confidential values: IDs, status/category/reason values, diagnostic codes and paths, timestamps, effective limits, usage, evidence kinds/media types/producers/paths, and driver/tool identities.
|
|
40
|
+
|
|
41
|
+
With `redaction: "off"`, raw harness-owned content and accepted evidence bytes are preserved by explicit operator choice. The API result, `result.json`, terminal IPC, and CLI stdout may therefore contain raw values. Out-of-band diagnostics remain sanitized and raw worker streams remain discarded under both settings.
|
|
42
|
+
|
|
43
|
+
## Capture Limits
|
|
44
|
+
|
|
45
|
+
- One candidate: at most 10 MiB before and after transformation
|
|
46
|
+
- Accepted pending buffers: at most 64 MiB per invocation
|
|
47
|
+
- Counters are released when candidates are committed or discarded
|
|
48
|
+
- Concurrent in-process invocations do not share a global allowance
|
|
49
|
+
|
|
50
|
+
An embedding host running `N` concurrent calls may retain approximately `N * 64 MiB` plus driver and model overhead. The caller owns admission control.
|
|
51
|
+
|
|
52
|
+
## Atomic Commitment
|
|
53
|
+
|
|
54
|
+
Files are written to a restrictive same-directory temporary file, flushed, renamed, and followed by directory synchronization where supported. Evidence references become visible only after this sequence succeeds. A post-rename synchronization failure is durability-uncertain and publishes no reference even if the destination is observable.
|
|
55
|
+
|
|
56
|
+
`invocation.json` is mandatory before executable driver startup. `checkpoint.json` is an atomically replaced partial diagnostic snapshot, not proof of process liveness, a resume point, or a terminal result. `result.json` is the only durable terminal marker. Parent pre-execution operational errors and containment results do not claim a durable terminal file.
|
|
57
|
+
|
|
58
|
+
Optional evidence failure preserves the established verdict, marks evidence partial, and appends a diagnostic. Checkpoint failure appends a diagnostic without changing verdict or evidence completeness. Failure to commit final `result.json` changes the top-level route to `infra / result-write-failed`, preserves completed semantic state, and returns an explicitly uncommitted envelope through the API or CLI when it fits the IPC bound.
|
|
59
|
+
|
|
60
|
+
## Byte Identity
|
|
61
|
+
|
|
62
|
+
Committed `result.json` uses deterministic two-space JSON with recursively sorted object keys, preserved array order, UTF-8 without a BOM, and one trailing line feed. The in-process finalized object is parsed from those bytes. The CLI verifies and writes the same bytes unchanged. Parent-authored containment has stdout bytes only and claims no durable result.
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@xoxoai/checkmate",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "
|
|
3
|
+
"version": "0.5.1",
|
|
4
|
+
"description": "Agent-first test automation harness with versioned requests, bounded execution, and durable evidence",
|
|
5
5
|
"homepage": "https://github.com/dawiddiwad/checkmate",
|
|
6
6
|
"repository": {
|
|
7
7
|
"type": "git",
|
|
@@ -10,24 +10,21 @@
|
|
|
10
10
|
"keywords": [
|
|
11
11
|
"test-automation",
|
|
12
12
|
"e2e-testing",
|
|
13
|
-
"playwright",
|
|
14
|
-
"openai",
|
|
15
13
|
"ai-testing",
|
|
16
|
-
"
|
|
17
|
-
"
|
|
18
|
-
"
|
|
19
|
-
"
|
|
20
|
-
"groq"
|
|
14
|
+
"agents",
|
|
15
|
+
"test-harness",
|
|
16
|
+
"cli",
|
|
17
|
+
"playwright"
|
|
21
18
|
],
|
|
22
19
|
"files": [
|
|
23
20
|
"bin",
|
|
24
21
|
"dist",
|
|
25
|
-
"
|
|
26
|
-
"docs/
|
|
27
|
-
"docs/
|
|
28
|
-
"
|
|
22
|
+
"schemas",
|
|
23
|
+
"docs/CLI.md",
|
|
24
|
+
"docs/CONFIGURATION.md",
|
|
25
|
+
"docs/DRIVERS.md",
|
|
26
|
+
"docs/EVIDENCE.md",
|
|
29
27
|
"README.md",
|
|
30
|
-
"test/examples",
|
|
31
28
|
"LICENSE"
|
|
32
29
|
],
|
|
33
30
|
"bin": {
|
|
@@ -36,34 +33,37 @@
|
|
|
36
33
|
"publishConfig": {
|
|
37
34
|
"access": "public"
|
|
38
35
|
},
|
|
39
|
-
"main": "dist/
|
|
40
|
-
"types": "dist/
|
|
36
|
+
"main": "dist/index.js",
|
|
37
|
+
"types": "dist/index.d.ts",
|
|
41
38
|
"type": "module",
|
|
42
39
|
"exports": {
|
|
43
40
|
".": {
|
|
44
|
-
"types": "./dist/
|
|
45
|
-
"default": "./dist/
|
|
41
|
+
"types": "./dist/index.d.ts",
|
|
42
|
+
"default": "./dist/index.js"
|
|
46
43
|
},
|
|
47
|
-
"./
|
|
48
|
-
"types": "./dist/
|
|
49
|
-
"default": "./dist/
|
|
44
|
+
"./driver": {
|
|
45
|
+
"types": "./dist/driver.d.ts",
|
|
46
|
+
"default": "./dist/driver.js"
|
|
50
47
|
},
|
|
51
|
-
"./
|
|
52
|
-
"types": "./dist/
|
|
53
|
-
"default": "./dist/
|
|
48
|
+
"./driver-web": {
|
|
49
|
+
"types": "./dist/drivers/web/index.d.ts",
|
|
50
|
+
"default": "./dist/drivers/web/index.js"
|
|
54
51
|
},
|
|
55
|
-
"./
|
|
56
|
-
|
|
57
|
-
"default": "./dist/salesforce.js"
|
|
58
|
-
}
|
|
52
|
+
"./driver-web/checkmate-driver.json": "./dist/drivers/web/checkmate-driver.json",
|
|
53
|
+
"./schemas/*": "./schemas/*"
|
|
59
54
|
},
|
|
60
55
|
"scripts": {
|
|
61
56
|
"checkmate:install": "npm install && npx playwright install",
|
|
62
|
-
"build": "npm run validation:check && npm run build:
|
|
57
|
+
"build": "npm run validation:check && npm run build:package",
|
|
63
58
|
"build:dist": "tsc -p tsconfig.build.json",
|
|
64
|
-
"
|
|
65
|
-
"
|
|
66
|
-
"
|
|
59
|
+
"build:package": "npm run clean && npm run build:dist && node scripts/copy-package-assets.mjs",
|
|
60
|
+
"clean": "rm -rf dist schemas",
|
|
61
|
+
"prepack": "npm run build:package",
|
|
62
|
+
"phase:verify": "node test/package/lifecycle.mjs",
|
|
63
|
+
"test:package:active": "node test/package/active-exports.mjs",
|
|
64
|
+
"test:acceptance:agent": "node test/acceptance/agent-workflow.mjs",
|
|
65
|
+
"test:e2e:ollama": "node test/acceptance/ollama-cli.mjs",
|
|
66
|
+
"validation:check": "npm run compile:check && npm run lint:check && npm run format:check && npm run test:unit:run",
|
|
67
67
|
"compile:check": "npx tsc --noEmit",
|
|
68
68
|
"lint:check": "eslint .",
|
|
69
69
|
"lint:fix": "eslint . --fix",
|
|
@@ -73,41 +73,33 @@
|
|
|
73
73
|
"test:unit:ui": "vitest --ui --config src/test/vitest.config.ts",
|
|
74
74
|
"test:unit:coverage": "vitest --coverage --config src/test/vitest.config.ts",
|
|
75
75
|
"test:unit:run": "vitest run --config src/test/vitest.config.ts",
|
|
76
|
-
"test:
|
|
77
|
-
"test:salesforce": "npm run build:dist && npx playwright test --project=salesforce",
|
|
78
|
-
"test:web:example": "npm run build:dist && npx playwright test --project=web --grep=ollama",
|
|
79
|
-
"show:report": "npx playwright show-report test-reports/html"
|
|
76
|
+
"test:package:final": "node test/package/final-surface.mjs"
|
|
80
77
|
},
|
|
81
78
|
"author": "Dawid Dobrowolski SoftQA",
|
|
82
79
|
"license": "MIT",
|
|
83
80
|
"dependencies": {
|
|
84
|
-
"@huggingface/transformers": "^4.0
|
|
85
|
-
"
|
|
86
|
-
"
|
|
87
|
-
"
|
|
88
|
-
"
|
|
89
|
-
"
|
|
90
|
-
"yaml": "^2.
|
|
91
|
-
"zod": "^4.
|
|
92
|
-
},
|
|
93
|
-
"peerDependencies": {
|
|
94
|
-
"@playwright/test": "^1.59.1"
|
|
81
|
+
"@huggingface/transformers": "^4.2.0",
|
|
82
|
+
"ajv": "^8.20.0",
|
|
83
|
+
"ajv-formats": "^3.0.1",
|
|
84
|
+
"openai": "^7.10.0",
|
|
85
|
+
"playwright": "^1.63.0",
|
|
86
|
+
"sharp": "^0.35.4",
|
|
87
|
+
"yaml": "^2.9.0",
|
|
88
|
+
"zod": "^4.5.4"
|
|
95
89
|
},
|
|
96
90
|
"devDependencies": {
|
|
97
91
|
"@eslint/js": "^10.0.1",
|
|
98
|
-
"@
|
|
99
|
-
"@
|
|
100
|
-
"@
|
|
101
|
-
"
|
|
102
|
-
"
|
|
103
|
-
"dotenv": "^17.4.1",
|
|
104
|
-
"eslint": "^10.2.0",
|
|
92
|
+
"@types/node": "^26.5.0",
|
|
93
|
+
"@vitest/coverage-v8": "^5.0.0",
|
|
94
|
+
"@vitest/ui": "^5.0.0",
|
|
95
|
+
"dotenv": "^17.4.2",
|
|
96
|
+
"eslint": "^10.10.0",
|
|
105
97
|
"eslint-config-prettier": "^10.1.8",
|
|
106
|
-
"prettier": "^3.
|
|
107
|
-
"tsx": "^4.
|
|
108
|
-
"typescript": "^
|
|
109
|
-
"typescript-eslint": "^8.
|
|
110
|
-
"vitest": "^
|
|
98
|
+
"prettier": "^3.9.6",
|
|
99
|
+
"tsx": "^4.23.13",
|
|
100
|
+
"typescript": "^6.0.3",
|
|
101
|
+
"typescript-eslint": "^8.70.0",
|
|
102
|
+
"vitest": "^5.0.0"
|
|
111
103
|
},
|
|
112
104
|
"directories": {
|
|
113
105
|
"doc": "docs",
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://checkmate.dev/schemas/checkmate-config.v1.json",
|
|
4
|
+
"title": "Checkmate configuration version 1",
|
|
5
|
+
"type": "object",
|
|
6
|
+
"additionalProperties": false,
|
|
7
|
+
"required": ["schemaVersion", "defaultPolicy", "secretBindings", "policies", "drivers"],
|
|
8
|
+
"properties": {
|
|
9
|
+
"schemaVersion": { "const": 1 },
|
|
10
|
+
"outputDirectory": {
|
|
11
|
+
"type": "string",
|
|
12
|
+
"minLength": 1,
|
|
13
|
+
"maxLength": 1024,
|
|
14
|
+
"pattern": "^(?!/)(?!.*(?:^|/)\\.\\.?(?:/|$))[^\\u0000-\\u001f/]+(?:/[^\\u0000-\\u001f/]+)*$"
|
|
15
|
+
},
|
|
16
|
+
"defaultPolicy": { "$ref": "#/$defs/id" },
|
|
17
|
+
"secretBindings": {
|
|
18
|
+
"type": "object",
|
|
19
|
+
"propertyNames": { "$ref": "#/$defs/id" },
|
|
20
|
+
"additionalProperties": { "$ref": "#/$defs/secretBinding" }
|
|
21
|
+
},
|
|
22
|
+
"policies": {
|
|
23
|
+
"type": "object",
|
|
24
|
+
"minProperties": 1,
|
|
25
|
+
"propertyNames": { "$ref": "#/$defs/id" },
|
|
26
|
+
"additionalProperties": { "$ref": "#/$defs/policy" }
|
|
27
|
+
},
|
|
28
|
+
"drivers": {
|
|
29
|
+
"type": "object",
|
|
30
|
+
"minProperties": 1,
|
|
31
|
+
"propertyNames": { "$ref": "#/$defs/id" },
|
|
32
|
+
"additionalProperties": { "$ref": "#/$defs/driverRegistration" }
|
|
33
|
+
}
|
|
34
|
+
},
|
|
35
|
+
"$defs": {
|
|
36
|
+
"id": {
|
|
37
|
+
"type": "string",
|
|
38
|
+
"minLength": 1,
|
|
39
|
+
"maxLength": 128,
|
|
40
|
+
"pattern": "^(?!.*[\\u0000-\\u001f]).*$"
|
|
41
|
+
},
|
|
42
|
+
"secretBinding": {
|
|
43
|
+
"type": "object",
|
|
44
|
+
"additionalProperties": false,
|
|
45
|
+
"required": ["source", "name"],
|
|
46
|
+
"properties": {
|
|
47
|
+
"source": { "const": "environment" },
|
|
48
|
+
"name": {
|
|
49
|
+
"type": "string",
|
|
50
|
+
"minLength": 1,
|
|
51
|
+
"maxLength": 256,
|
|
52
|
+
"pattern": "^(?!.*[\\u0000-\\u001f]).*$"
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
},
|
|
56
|
+
"provider": {
|
|
57
|
+
"type": "object",
|
|
58
|
+
"additionalProperties": false,
|
|
59
|
+
"required": ["id", "model", "apiKeyBinding"],
|
|
60
|
+
"properties": {
|
|
61
|
+
"id": { "const": "openai" },
|
|
62
|
+
"model": {
|
|
63
|
+
"type": "string",
|
|
64
|
+
"minLength": 1,
|
|
65
|
+
"maxLength": 256,
|
|
66
|
+
"pattern": "^(?!.*[\\u0000-\\u001f]).*$"
|
|
67
|
+
},
|
|
68
|
+
"baseUrl": { "type": "string", "format": "uri" },
|
|
69
|
+
"apiKeyBinding": { "$ref": "#/$defs/id" },
|
|
70
|
+
"temperature": { "type": "number", "minimum": 0 },
|
|
71
|
+
"reasoningEffort": { "enum": ["low", "medium", "high"] }
|
|
72
|
+
}
|
|
73
|
+
},
|
|
74
|
+
"modelEgress": {
|
|
75
|
+
"type": "object",
|
|
76
|
+
"additionalProperties": false,
|
|
77
|
+
"required": ["provider", "textRedaction", "allowOpaque", "maxStepBytes", "maxMessageBytes"],
|
|
78
|
+
"properties": {
|
|
79
|
+
"provider": { "$ref": "#/$defs/provider" },
|
|
80
|
+
"textRedaction": { "enum": ["on", "off"] },
|
|
81
|
+
"allowOpaque": { "type": "boolean" },
|
|
82
|
+
"maxStepBytes": { "type": "integer", "minimum": 1 },
|
|
83
|
+
"maxMessageBytes": { "type": "integer", "minimum": 1 }
|
|
84
|
+
}
|
|
85
|
+
},
|
|
86
|
+
"bounds": {
|
|
87
|
+
"type": "object",
|
|
88
|
+
"additionalProperties": false,
|
|
89
|
+
"required": [
|
|
90
|
+
"scenarioTimeoutMs",
|
|
91
|
+
"stepTimeoutMs",
|
|
92
|
+
"turnsPerStep",
|
|
93
|
+
"requestTimeoutMs",
|
|
94
|
+
"maxRetries",
|
|
95
|
+
"loopMaxRepetitions",
|
|
96
|
+
"cleanupTimeoutMs"
|
|
97
|
+
],
|
|
98
|
+
"properties": {
|
|
99
|
+
"scenarioTimeoutMs": { "type": "integer", "minimum": 1 },
|
|
100
|
+
"stepTimeoutMs": { "type": "integer", "minimum": 1 },
|
|
101
|
+
"turnsPerStep": { "type": "integer", "minimum": 1 },
|
|
102
|
+
"requestTimeoutMs": { "type": "integer", "minimum": 1 },
|
|
103
|
+
"maxRetries": { "type": "integer", "minimum": 0 },
|
|
104
|
+
"loopMaxRepetitions": { "type": "integer", "minimum": 1 },
|
|
105
|
+
"cleanupTimeoutMs": { "type": "integer", "minimum": 1 },
|
|
106
|
+
"budgetTokens": { "type": "integer", "minimum": 1 }
|
|
107
|
+
}
|
|
108
|
+
},
|
|
109
|
+
"evidence": {
|
|
110
|
+
"type": "object",
|
|
111
|
+
"additionalProperties": false,
|
|
112
|
+
"required": ["retention", "redaction", "allowOpaque"],
|
|
113
|
+
"properties": {
|
|
114
|
+
"retention": { "enum": ["on", "retain-on-failure", "off"] },
|
|
115
|
+
"redaction": { "enum": ["on", "off"] },
|
|
116
|
+
"allowOpaque": { "type": "boolean" }
|
|
117
|
+
}
|
|
118
|
+
},
|
|
119
|
+
"toolPolicy": {
|
|
120
|
+
"type": "object",
|
|
121
|
+
"additionalProperties": false,
|
|
122
|
+
"required": ["allowed"],
|
|
123
|
+
"properties": {
|
|
124
|
+
"allowed": {
|
|
125
|
+
"type": "array",
|
|
126
|
+
"minItems": 1,
|
|
127
|
+
"uniqueItems": true,
|
|
128
|
+
"items": { "$ref": "#/$defs/id" }
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
},
|
|
132
|
+
"policyDriver": {
|
|
133
|
+
"type": "object",
|
|
134
|
+
"additionalProperties": false,
|
|
135
|
+
"required": ["settings", "tools"],
|
|
136
|
+
"properties": {
|
|
137
|
+
"settings": { "type": "object" },
|
|
138
|
+
"tools": { "$ref": "#/$defs/toolPolicy" }
|
|
139
|
+
}
|
|
140
|
+
},
|
|
141
|
+
"policy": {
|
|
142
|
+
"type": "object",
|
|
143
|
+
"additionalProperties": false,
|
|
144
|
+
"required": ["modelEgress", "bounds", "evidence", "drivers"],
|
|
145
|
+
"properties": {
|
|
146
|
+
"modelEgress": { "$ref": "#/$defs/modelEgress" },
|
|
147
|
+
"bounds": { "$ref": "#/$defs/bounds" },
|
|
148
|
+
"evidence": { "$ref": "#/$defs/evidence" },
|
|
149
|
+
"drivers": {
|
|
150
|
+
"type": "object",
|
|
151
|
+
"minProperties": 1,
|
|
152
|
+
"propertyNames": { "$ref": "#/$defs/id" },
|
|
153
|
+
"additionalProperties": { "$ref": "#/$defs/policyDriver" }
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
},
|
|
157
|
+
"driverRegistration": {
|
|
158
|
+
"type": "object",
|
|
159
|
+
"additionalProperties": false,
|
|
160
|
+
"required": ["package", "secrets"],
|
|
161
|
+
"properties": {
|
|
162
|
+
"package": {
|
|
163
|
+
"type": "string",
|
|
164
|
+
"minLength": 1,
|
|
165
|
+
"maxLength": 256,
|
|
166
|
+
"pattern": "^(?!.*[\\u0000-\\u001f]).*$"
|
|
167
|
+
},
|
|
168
|
+
"secrets": {
|
|
169
|
+
"type": "object",
|
|
170
|
+
"propertyNames": { "$ref": "#/$defs/id" },
|
|
171
|
+
"additionalProperties": { "$ref": "#/$defs/id" }
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
}
|