@gfdelarue/testkit 0.4.5 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +724 -495
- package/bin/testkit-windows-owner-arm64.exe +0 -0
- package/bin/testkit-windows-owner-x64.exe +0 -0
- package/dist/app/bugs/ci-report.js +2 -2
- package/dist/app/bugs/ci-summary.js +3 -3
- package/dist/app/bugs/draft.js +3 -3
- package/dist/app/bugs/file.js +1 -1
- package/dist/app/bugs/gate.js +1 -1
- package/dist/app/bugs/guide.js +3 -3
- package/dist/app/bugs/issue.js +4 -4
- package/dist/app/bugs/known-failures.js +1 -1
- package/dist/app/bugs/record.js +1 -1
- package/dist/app/bugs/run-reading.js +1 -1
- package/dist/app/bugs/supplied-reproduce.js +1 -0
- package/dist/app/campaigns/campaign.js +1 -1
- package/dist/app/campaigns/index.js +2 -2
- package/dist/app/cleanup/index.js +1 -1
- package/dist/app/db-schema/prepare.js +1 -1
- package/dist/app/destroy/index.js +1 -1
- package/dist/app/discover/index.js +2 -2
- package/dist/app/doctor/index.js +1 -1
- package/dist/app/environments/config-driven.js +1 -1
- package/dist/app/environments/environment-file.js +1 -0
- package/dist/app/environments/file-setup.js +1 -0
- package/dist/app/environments/routing.js +1 -1
- package/dist/app/environments/static-values.js +1 -0
- package/dist/app/environments/supplied-input.js +1 -0
- package/dist/app/environments/supplied.js +1 -1
- package/dist/app/environments/task-databases.js +2 -0
- package/dist/app/inspect/artifacts.js +1 -1
- package/dist/app/inspect/index.js +1 -1
- package/dist/app/lint/index.js +1 -1
- package/dist/app/lint/native.js +1 -1
- package/dist/app/lint/options.js +1 -1
- package/dist/app/lint/rules/engine-imports.js +1 -1
- package/dist/app/lint/rules/syntax.js +1 -1
- package/dist/app/lint/rules/unit.js +1 -1
- package/dist/app/reports/prepare.js +1 -1
- package/dist/app/run/caller-redaction.js +1 -0
- package/dist/app/run/engine-execution-failures.js +1 -1
- package/dist/app/run/engine-execution-reporting.js +1 -1
- package/dist/app/run/engine-execution.js +1 -1
- package/dist/app/run/engine-orchestration.js +1 -1
- package/dist/app/run/environment.js +1 -1
- package/dist/app/run/failure-details.js +1 -1
- package/dist/app/run/live-run.js +1 -1
- package/dist/app/run/orchestrator.js +1 -1
- package/dist/app/run/planning.js +1 -1
- package/dist/app/run/request.js +1 -1
- package/dist/app/run/results.js +1 -1
- package/dist/app/run/run-events.js +1 -1
- package/dist/app/run/run-finalization.js +1 -1
- package/dist/app/run/scheduler/estimates.js +1 -1
- package/dist/app/run/scheduler/identity.js +1 -1
- package/dist/app/run/scheduler/index.js +1 -1
- package/dist/app/run/setup/runtime-node-setup-protocol.js +1 -1
- package/dist/app/run/setup/runtime-node-setup-worker.js +1 -1
- package/dist/app/run/setup/runtime-node-setup.js +2 -2
- package/dist/app/run/task-batches.js +1 -1
- package/dist/app/run/worker-loop.js +1 -1
- package/dist/app/status/model.js +1 -1
- package/dist/app/status/runs.js +1 -1
- package/dist/app/typecheck/index.js +4 -4
- package/dist/cli/command-catalog.js +1 -1
- package/dist/cli/command-flags.js +1 -1
- package/dist/cli/commands/artifacts.js +2 -2
- package/dist/cli/commands/bug/file.js +2 -2
- package/dist/cli/commands/bug/record.js +2 -2
- package/dist/cli/commands/discover.js +1 -1
- package/dist/cli/commands/guard/unsafe-entrypoint.js +1 -1
- package/dist/cli/commands/run.js +2 -2
- package/dist/cli/commands/typecheck.js +1 -1
- package/dist/cli/commands/workspace.js +1 -1
- package/dist/cli/framework/errors.js +1 -1
- package/dist/cli/framework/execute.js +7 -6
- package/dist/cli/framework/parser.js +1 -1
- package/dist/cli/operations/run/operation.js +2 -2
- package/dist/cli/renderers/bug/text.js +1 -1
- package/dist/cli/renderers/discover/report.js +1 -1
- package/dist/cli/renderers/run/events.js +2 -2
- package/dist/cli/renderers/run/formatting.js +1 -1
- package/dist/cli/renderers/run/status.js +1 -1
- package/dist/cli/renderers/run/text-reporter.js +24 -22
- package/dist/cli/terminal/colors.js +1 -1
- package/dist/cli/terminal/layout.js +2 -2
- package/dist/config/authoring/database-fixture-discovery.js +1 -0
- package/dist/config/authoring/http-methods.js +1 -1
- package/dist/config/authoring/index.js +1 -1
- package/dist/config/loader/config-loader.js +1 -1
- package/dist/config/loader/decode-discovery.js +1 -1
- package/dist/config/loader/decode-execution.js +1 -1
- package/dist/config/loader/dependencies.js +1 -1
- package/dist/config/loader/discover-tests.js +1 -1
- package/dist/config/loader/discovery-summary.js +1 -1
- package/dist/config/loader/discovery.js +2 -2
- package/dist/config/loader/env.js +1 -1
- package/dist/config/loader/file-metadata.js +1 -1
- package/dist/config/loader/groups.js +2 -1
- package/dist/config/loader/index.js +1 -1
- package/dist/config/loader/load.js +1 -1
- package/dist/config/loader/managed-configs.js +1 -1
- package/dist/config/loader/owned-discovery.js +1 -0
- package/dist/config/loader/path-policy.js +1 -1
- package/dist/config/loader/runtime-isolation.js +1 -1
- package/dist/config/loader/runtime.js +1 -1
- package/dist/config/loader/skip-config.js +1 -1
- package/dist/config/loader/task-policy.js +1 -0
- package/dist/config/loader/test-selection.js +1 -1
- package/dist/config/loader/validation.js +1 -1
- package/dist/database/admin.js +2 -2
- package/dist/database/bindings.js +1 -0
- package/dist/database/cleanup.js +1 -1
- package/dist/database/configured-steps.js +1 -1
- package/dist/database/docker-metadata.js +3 -3
- package/dist/database/docker.js +3 -3
- package/dist/database/index.js +1 -1
- package/dist/database/local-postgres.js +1 -1
- package/dist/database/naming.js +1 -1
- package/dist/database/postgres/connection-check.js +1 -0
- package/dist/database/postgres/protocol.js +2 -0
- package/dist/database/schema-operations.js +1 -1
- package/dist/database/schema-source.js +3 -3
- package/dist/database/source-schema-env.js +1 -0
- package/dist/database/state-files.js +1 -1
- package/dist/database/state.js +1 -1
- package/dist/database/steps.js +3 -3
- package/dist/domain/file-timeout.js +1 -1
- package/dist/domain/identifiers.js +1 -1
- package/dist/domain/retired-test-types.js +1 -0
- package/dist/domain/runtime-isolation-contract.js +1 -1
- package/dist/domain/suite-selection.js +1 -1
- package/dist/domain/test-types.js +1 -1
- package/dist/engines/adapters/evidence-files.js +1 -1
- package/dist/engines/adapters/execution.js +1 -1
- package/dist/engines/adapters/json-bounds.js +2 -0
- package/dist/engines/adapters/native-report.js +1 -1
- package/dist/engines/api/execution.js +1 -1
- package/dist/engines/api/index.js +1 -1
- package/dist/engines/api/runtime-artifact.js +1 -1
- package/dist/engines/catalog.js +1 -1
- package/dist/engines/contracts/environment-values.js +1 -0
- package/dist/engines/definitions.js +1 -1
- package/dist/engines/environment/handles.js +1 -0
- package/dist/engines/environment/index.js +1 -0
- package/dist/engines/factory.js +1 -1
- package/dist/engines/installations/node-vitest.js +1 -1
- package/dist/engines/installations/package-resolution.js +1 -1
- package/dist/engines/installations/playwright.js +1 -1
- package/dist/engines/managed-module-resolution.js +8 -4
- package/dist/engines/manager.js +1 -1
- package/dist/engines/node-vitest/adapter/adapter.js +1 -1
- package/dist/engines/node-vitest/adapter/driver.js +2 -0
- package/dist/engines/node-vitest/adapter/evidence.js +2 -2
- package/dist/engines/node-vitest/adapter/failure-details.js +4 -2
- package/dist/engines/node-vitest/adapter/reporter.js +1 -1
- package/dist/engines/node-vitest/adapter/workspace.js +17 -195
- package/dist/engines/node-vitest/inventory/index.js +2 -0
- package/dist/engines/node-vitest/inventory/list-files.js +3 -0
- package/dist/engines/node-vitest/runtime/data/index.js +1 -1
- package/dist/engines/node-vitest/runtime/data/report.js +5 -6
- package/dist/engines/node-vitest/runtime/data/scenario.js +2 -2
- package/dist/engines/node-vitest/runtime/runtime-input.js +1 -1
- package/dist/engines/node-vitest/selection.js +1 -0
- package/dist/engines/playwright/adapter/adapter.js +1 -1
- package/dist/engines/playwright/adapter/config.js +3 -3
- package/dist/engines/playwright/adapter/evidence-validation.js +1 -1
- package/dist/engines/playwright/adapter/evidence.js +1 -1
- package/dist/engines/playwright/adapter/reporter.js +1 -1
- package/dist/engines/playwright/runtime/config-file.js +1 -1
- package/dist/engines/playwright/runtime/config.js +1 -1
- package/dist/engines/playwright/runtime/managed-test.js +1 -0
- package/dist/engines/playwright/runtime/runtime-input.js +29 -10
- package/dist/engines/runtime-input-file.js +1 -1
- package/dist/environment/lifecycle-controller.js +1 -1
- package/dist/environment/lifecycle.js +1 -1
- package/dist/environment/ready-environment.js +1 -1
- package/dist/environment/run-environment.js +1 -1
- package/dist/environment/run-stopping.js +1 -1
- package/dist/environment/runtime-contexts.js +1 -1
- package/dist/environment/runtime-manager.js +1 -1
- package/dist/environment/runtime-planning.js +1 -1
- package/dist/environment/runtime-preparation.js +1 -1
- package/dist/environment/services.js +3 -3
- package/dist/environment/template.js +1 -1
- package/dist/evidence/artifact-config.js +1 -1
- package/dist/evidence/artifacts.js +2 -2
- package/dist/evidence/bundles.js +1 -1
- package/dist/evidence/environment-record.js +1 -1
- package/dist/evidence/index.js +1 -1
- package/dist/evidence/maintenance.js +1 -0
- package/dist/evidence/read-run.js +1 -1
- package/dist/evidence/reading.js +1 -1
- package/dist/evidence/redaction.js +4 -0
- package/dist/evidence/reporting.js +1 -1
- package/dist/evidence/run-model.js +1 -0
- package/dist/evidence/storage.js +2 -2
- package/dist/evidence/timing.js +1 -1
- package/dist/kernel/error-serialization.js +1 -1
- package/dist/kernel/known-keys.js +1 -0
- package/dist/kernel/redaction-targets.js +1 -0
- package/dist/kernel/redaction.js +4 -1
- package/dist/kernel/unknown-values.js +1 -1
- package/dist/process/environment.js +1 -1
- package/dist/process/execa-bundle.json +3 -3
- package/dist/process/files/json-file.js +1 -1
- package/dist/process/host/platform.js +1 -1
- package/dist/process/host-process.js +1 -1
- package/dist/process/index.js +1 -1
- package/dist/process/locks.js +4 -4
- package/dist/process/port-listener.js +2 -0
- package/dist/process/process-group.js +1 -1
- package/dist/process/process.js +32 -32
- package/dist/process/readiness.js +1 -1
- package/dist/process/ready-deadline.js +1 -0
- package/dist/process/runs/ownership.js +1 -1
- package/dist/process/runs/stop.js +1 -1
- package/dist/process/service-processes.js +1 -1
- package/dist/process/windows-cli-owner.js +3 -3
- package/dist/process/windows-native.js +1 -1
- package/dist/repo/fingerprint-policy.js +1 -1
- package/dist/repo/managed-git.js +1 -1
- package/dist/repo/physical-tree.js +1 -1
- package/dist/repo/state.js +4 -4
- package/dist/steps/build-config.js +1 -1
- package/dist/steps/configured-step-testkit-modules.js +1 -1
- package/dist/steps/configured-steps.js +1 -1
- package/dist/steps/environment.js +1 -1
- package/dist/steps/index.js +1 -1
- package/dist/toolchains/index.js +3 -3
- package/dist/toolchains/semver.js +1 -1
- package/dist/toolchains/vitest/index.js +1 -1
- package/dist/tracker/redaction.js +1 -2
- package/dist/{config/runtime/build-assets.d.ts → types/config-build-assets.d.ts} +5 -3
- package/dist/{database/steps.d.ts → types/config-database-steps.d.ts} +7 -7
- package/dist/{config/runtime/next-runtime-tsconfig.d.ts → types/config-next-runtime-tsconfig.d.ts} +5 -3
- package/dist/types/config.d.ts +422 -0
- package/dist/types/drizzle.d.ts +13 -0
- package/dist/{config/runtime/env/index.d.ts → types/env.d.ts} +22 -20
- package/dist/types/environment.d.ts +46 -0
- package/dist/types/index.d.ts +21 -0
- package/dist/types/node-vitest-data.d.ts +191 -0
- package/dist/types/playwright.d.ts +130 -0
- package/dist/types/tooling-vitest.d.ts +13 -0
- package/dist/types/workspace.d.ts +633 -0
- package/docs/bug-records.md +35 -40
- package/docs/engines.md +259 -0
- package/docs/release-notes.md +2116 -0
- package/docs/test-types.md +37 -85
- package/package.json +22 -32
- package/testkit-config-schema.md +391 -260
- package/dist/app/bugs/areas.d.ts +0 -38
- package/dist/app/bugs/assessment.d.ts +0 -9
- package/dist/app/bugs/categories.d.ts +0 -41
- package/dist/app/bugs/ci-report.d.ts +0 -21
- package/dist/app/bugs/ci-summary.d.ts +0 -13
- package/dist/app/bugs/draft.d.ts +0 -70
- package/dist/app/bugs/facts.d.ts +0 -49
- package/dist/app/bugs/file.d.ts +0 -45
- package/dist/app/bugs/gate.d.ts +0 -22
- package/dist/app/bugs/guide.d.ts +0 -20
- package/dist/app/bugs/issue.d.ts +0 -18
- package/dist/app/bugs/known-failures.d.ts +0 -49
- package/dist/app/bugs/record.d.ts +0 -14
- package/dist/app/bugs/run-reading.d.ts +0 -51
- package/dist/app/bugs/store.d.ts +0 -34
- package/dist/app/campaigns/brief.d.ts +0 -7
- package/dist/app/campaigns/campaign.d.ts +0 -14
- package/dist/app/campaigns/index.d.ts +0 -38
- package/dist/app/campaigns/store.d.ts +0 -5
- package/dist/app/campaigns/summary.d.ts +0 -13
- package/dist/app/cleanup/index.d.ts +0 -52
- package/dist/app/db-schema/prepare.d.ts +0 -11
- package/dist/app/db-schema/refresh.d.ts +0 -17
- package/dist/app/db-schema/verify.d.ts +0 -11
- package/dist/app/destroy/index.d.ts +0 -16
- package/dist/app/discover/index.d.ts +0 -8
- package/dist/app/doctor/index.d.ts +0 -17
- package/dist/app/environments/config-driven.d.ts +0 -5
- package/dist/app/environments/routing.d.ts +0 -6
- package/dist/app/environments/supplied.d.ts +0 -3
- package/dist/app/file-selection.d.ts +0 -1
- package/dist/app/inspect/artifacts.d.ts +0 -30
- package/dist/app/inspect/index.d.ts +0 -112
- package/dist/app/lint/index.d.ts +0 -28
- package/dist/app/lint/native.d.ts +0 -10
- package/dist/app/lint/options.d.ts +0 -66
- package/dist/app/lint/policy.d.ts +0 -21
- package/dist/app/lint/rules/data.d.ts +0 -3
- package/dist/app/lint/rules/data.js +0 -1
- package/dist/app/lint/rules/engine-imports.d.ts +0 -3
- package/dist/app/lint/rules/syntax.d.ts +0 -4
- package/dist/app/lint/rules/unit.d.ts +0 -3
- package/dist/app/lint/shared.d.ts +0 -34
- package/dist/app/package-inspection.d.ts +0 -20
- package/dist/app/reports/files.d.ts +0 -13
- package/dist/app/reports/prepare.d.ts +0 -20
- package/dist/app/reports/publish.d.ts +0 -21
- package/dist/app/run/endpoint-readiness.d.ts +0 -43
- package/dist/app/run/endpoint-readiness.js +0 -1
- package/dist/app/run/engine-evidence-persistence.d.ts +0 -39
- package/dist/app/run/engine-execution-cleanup.d.ts +0 -8
- package/dist/app/run/engine-execution-cleanup.js +0 -1
- package/dist/app/run/engine-execution-failures.d.ts +0 -41
- package/dist/app/run/engine-execution-observer.d.ts +0 -21
- package/dist/app/run/engine-execution-reporting.d.ts +0 -17
- package/dist/app/run/engine-execution.d.ts +0 -65
- package/dist/app/run/engine-failure-stack.d.ts +0 -1
- package/dist/app/run/engine-orchestration.d.ts +0 -30
- package/dist/app/run/environment.d.ts +0 -33
- package/dist/app/run/failure-details.d.ts +0 -37
- package/dist/app/run/live-run.d.ts +0 -29
- package/dist/app/run/operation-progress.d.ts +0 -13
- package/dist/app/run/orchestrator.d.ts +0 -75
- package/dist/app/run/planning-types.d.ts +0 -68
- package/dist/app/run/planning.d.ts +0 -14
- package/dist/app/run/request.d.ts +0 -30
- package/dist/app/run/results.d.ts +0 -161
- package/dist/app/run/run-events.d.ts +0 -144
- package/dist/app/run/run-finalization.d.ts +0 -56
- package/dist/app/run/run-guards.d.ts +0 -3
- package/dist/app/run/runtime-failures.d.ts +0 -18
- package/dist/app/run/runtime-fixtures.d.ts +0 -2
- package/dist/app/run/runtime-task-types.d.ts +0 -20
- package/dist/app/run/scheduler/estimates.d.ts +0 -32
- package/dist/app/run/scheduler/identity.d.ts +0 -9
- package/dist/app/run/scheduler/index.d.ts +0 -124
- package/dist/app/run/scheduler/locks.d.ts +0 -6
- package/dist/app/run/scheduler/observations.d.ts +0 -20
- package/dist/app/run/selection.d.ts +0 -9
- package/dist/app/run/setup/runtime-node-setup-errors.d.ts +0 -13
- package/dist/app/run/setup/runtime-node-setup-protocol.d.ts +0 -85
- package/dist/app/run/setup/runtime-node-setup-worker.d.ts +0 -1
- package/dist/app/run/setup/runtime-node-setup.d.ts +0 -48
- package/dist/app/run/task-batches.d.ts +0 -23
- package/dist/app/run/task-leases.d.ts +0 -15
- package/dist/app/run/worker-loop.d.ts +0 -66
- package/dist/app/setup/playwright.d.ts +0 -22
- package/dist/app/status/index.d.ts +0 -9
- package/dist/app/status/model.d.ts +0 -29
- package/dist/app/status/runs.d.ts +0 -99
- package/dist/app/stop/index.d.ts +0 -14
- package/dist/app/tracker/index.d.ts +0 -13
- package/dist/app/tracker/remote.d.ts +0 -13
- package/dist/app/tracker/token.d.ts +0 -7
- package/dist/app/typecheck/index.d.ts +0 -63
- package/dist/cli/command-catalog.d.ts +0 -156
- package/dist/cli/command-flags.d.ts +0 -19
- package/dist/cli/command-registry.d.ts +0 -3
- package/dist/cli/commands/artifacts.d.ts +0 -48
- package/dist/cli/commands/bug/file.d.ts +0 -16
- package/dist/cli/commands/bug/guide.d.ts +0 -5
- package/dist/cli/commands/bug/record.d.ts +0 -9
- package/dist/cli/commands/campaign/finish.d.ts +0 -10
- package/dist/cli/commands/campaign/start.d.ts +0 -7
- package/dist/cli/commands/campaign/status.d.ts +0 -13
- package/dist/cli/commands/ci/report.d.ts +0 -15
- package/dist/cli/commands/cleanup.d.ts +0 -13
- package/dist/cli/commands/db/schema/refresh.d.ts +0 -2
- package/dist/cli/commands/db/schema/verify.d.ts +0 -2
- package/dist/cli/commands/destroy.d.ts +0 -9
- package/dist/cli/commands/discover.d.ts +0 -2
- package/dist/cli/commands/doctor.d.ts +0 -2
- package/dist/cli/commands/guard/unsafe-entrypoint.d.ts +0 -2
- package/dist/cli/commands/lint.d.ts +0 -2
- package/dist/cli/commands/report/prepare.d.ts +0 -12
- package/dist/cli/commands/report/publish.d.ts +0 -13
- package/dist/cli/commands/run/status.d.ts +0 -5
- package/dist/cli/commands/run/stop.d.ts +0 -7
- package/dist/cli/commands/run.d.ts +0 -9
- package/dist/cli/commands/setup/playwright.d.ts +0 -2
- package/dist/cli/commands/status.d.ts +0 -5
- package/dist/cli/commands/stop.d.ts +0 -12
- package/dist/cli/commands/typecheck.d.ts +0 -2
- package/dist/cli/commands/workspace.d.ts +0 -2
- package/dist/cli/entrypoint.d.ts +0 -1
- package/dist/cli/framework/command.d.ts +0 -21
- package/dist/cli/framework/definitions.d.ts +0 -100
- package/dist/cli/framework/errors.d.ts +0 -6
- package/dist/cli/framework/execute.d.ts +0 -14
- package/dist/cli/framework/help.d.ts +0 -11
- package/dist/cli/framework/index.d.ts +0 -3
- package/dist/cli/framework/parser.d.ts +0 -6
- package/dist/cli/index.d.ts +0 -4
- package/dist/cli/json-errors.d.ts +0 -3
- package/dist/cli/operations/db/schema/refresh/operation.d.ts +0 -7
- package/dist/cli/operations/db/schema/verify/operation.d.ts +0 -7
- package/dist/cli/operations/run/operation.d.ts +0 -26
- package/dist/cli/process-signal.d.ts +0 -1
- package/dist/cli/renderers/bug/text.d.ts +0 -8
- package/dist/cli/renderers/campaign/text.d.ts +0 -2
- package/dist/cli/renderers/ci/text.d.ts +0 -2
- package/dist/cli/renderers/cleanup/format.d.ts +0 -14
- package/dist/cli/renderers/db-schema/text.d.ts +0 -4
- package/dist/cli/renderers/destroy/text.d.ts +0 -4
- package/dist/cli/renderers/discover/report.d.ts +0 -7
- package/dist/cli/renderers/discover/text.d.ts +0 -3
- package/dist/cli/renderers/doctor/text.d.ts +0 -2
- package/dist/cli/renderers/lint/text.d.ts +0 -2
- package/dist/cli/renderers/run/events.d.ts +0 -8
- package/dist/cli/renderers/run/failure.d.ts +0 -8
- package/dist/cli/renderers/run/formatting.d.ts +0 -79
- package/dist/cli/renderers/run/status.d.ts +0 -6
- package/dist/cli/renderers/run/text-reporter.d.ts +0 -19
- package/dist/cli/renderers/status/text.d.ts +0 -2
- package/dist/cli/renderers/stop/text.d.ts +0 -7
- package/dist/cli/renderers/typecheck/text.d.ts +0 -2
- package/dist/cli/terminal/colors.d.ts +0 -10
- package/dist/cli/terminal/layout.d.ts +0 -11
- package/dist/cli/version-drift.d.ts +0 -25
- package/dist/config/authoring/build-config-contract.d.ts +0 -27
- package/dist/config/authoring/configured-step-contract.d.ts +0 -18
- package/dist/config/authoring/http-execution-contract.d.ts +0 -70
- package/dist/config/authoring/http-methods.d.ts +0 -18
- package/dist/config/authoring/index.d.ts +0 -372
- package/dist/config/authoring/preset-env.d.ts +0 -19
- package/dist/config/authoring/runtime.d.ts +0 -64
- package/dist/config/authoring/runtime.js +0 -1
- package/dist/config/loader/bugs.d.ts +0 -15
- package/dist/config/loader/config-loader.d.ts +0 -9
- package/dist/config/loader/database-materialization.d.ts +0 -19
- package/dist/config/loader/database-roles.d.ts +0 -2
- package/dist/config/loader/database.d.ts +0 -55
- package/dist/config/loader/decode-discovery.d.ts +0 -5
- package/dist/config/loader/decode-execution.d.ts +0 -7
- package/dist/config/loader/dependencies.d.ts +0 -7
- package/dist/config/loader/discover-tests.d.ts +0 -111
- package/dist/config/loader/discovery-summary.d.ts +0 -29
- package/dist/config/loader/discovery.d.ts +0 -53
- package/dist/config/loader/env.d.ts +0 -4
- package/dist/config/loader/execution-model.d.ts +0 -15
- package/dist/config/loader/file-metadata.d.ts +0 -19
- package/dist/config/loader/groups.d.ts +0 -33
- package/dist/config/loader/index.d.ts +0 -10
- package/dist/config/loader/load.d.ts +0 -89
- package/dist/config/loader/managed-configs.d.ts +0 -23
- package/dist/config/loader/path-policy.d.ts +0 -9
- package/dist/config/loader/paths.d.ts +0 -5
- package/dist/config/loader/runtime-isolation.d.ts +0 -44
- package/dist/config/loader/runtime.d.ts +0 -16
- package/dist/config/loader/skip-config.d.ts +0 -44
- package/dist/config/loader/test-selection.d.ts +0 -15
- package/dist/config/loader/validation.d.ts +0 -15
- package/dist/config/loader/values.d.ts +0 -3
- package/dist/config/loader/values.js +0 -1
- package/dist/database/activity-gate.d.ts +0 -22
- package/dist/database/admin.d.ts +0 -31
- package/dist/database/cleanup.d.ts +0 -36
- package/dist/database/configured-steps.d.ts +0 -27
- package/dist/database/constants.d.ts +0 -11
- package/dist/database/docker-metadata.d.ts +0 -3
- package/dist/database/docker.d.ts +0 -65
- package/dist/database/drizzle/index.d.ts +0 -13
- package/dist/database/fingerprint.d.ts +0 -12
- package/dist/database/index.d.ts +0 -39
- package/dist/database/lifecycle-cleanup.d.ts +0 -8
- package/dist/database/local-postgres.d.ts +0 -71
- package/dist/database/naming.d.ts +0 -17
- package/dist/database/postgres/index.d.ts +0 -71
- package/dist/database/progress.d.ts +0 -6
- package/dist/database/roles.d.ts +0 -19
- package/dist/database/runtime-config.d.ts +0 -16
- package/dist/database/runtime-reservations.d.ts +0 -19
- package/dist/database/schema-operations.d.ts +0 -18
- package/dist/database/schema-source.d.ts +0 -96
- package/dist/database/source-refresh-lock.d.ts +0 -8
- package/dist/database/source-url.d.ts +0 -21
- package/dist/database/state-files.d.ts +0 -13
- package/dist/database/state.d.ts +0 -26
- package/dist/database/template-steps.d.ts +0 -39
- package/dist/domain/file-timeout.d.ts +0 -25
- package/dist/domain/identifiers.d.ts +0 -9
- package/dist/domain/operation-display.d.ts +0 -10
- package/dist/domain/operation-reason.d.ts +0 -5
- package/dist/domain/postgres-route.d.ts +0 -4
- package/dist/domain/runtime-identity.d.ts +0 -1
- package/dist/domain/runtime-isolation-contract.d.ts +0 -5
- package/dist/domain/service-name.d.ts +0 -2
- package/dist/domain/suite-selection.d.ts +0 -20
- package/dist/domain/test-outcomes.d.ts +0 -2
- package/dist/domain/test-types.d.ts +0 -16
- package/dist/engines/adapters/evidence-files.d.ts +0 -32
- package/dist/engines/adapters/evidence-identity.d.ts +0 -12
- package/dist/engines/adapters/execution.d.ts +0 -41
- package/dist/engines/adapters/native-report.d.ts +0 -25
- package/dist/engines/adapters/node-startup-cache.d.ts +0 -8
- package/dist/engines/adapters/workspace.d.ts +0 -4
- package/dist/engines/api/capabilities.d.ts +0 -21
- package/dist/engines/api/capabilities.js +0 -1
- package/dist/engines/api/evidence.d.ts +0 -38
- package/dist/engines/api/execution-observer.d.ts +0 -8
- package/dist/engines/api/execution-scope.d.ts +0 -17
- package/dist/engines/api/execution.d.ts +0 -101
- package/dist/engines/api/index.d.ts +0 -5
- package/dist/engines/api/k6-runtime-input.d.ts +0 -26
- package/dist/engines/api/k6-runtime-input.js +0 -1
- package/dist/engines/api/runtime-artifact.d.ts +0 -19
- package/dist/engines/bundle-cache.d.ts +0 -32
- package/dist/engines/catalog.d.ts +0 -27
- package/dist/engines/contracts/fixture-manifest.d.ts +0 -34
- package/dist/engines/contracts/json-bounds.d.ts +0 -36
- package/dist/engines/contracts/json-bounds.js +0 -2
- package/dist/engines/contracts/k6-repository-configuration.d.ts +0 -12
- package/dist/engines/contracts/k6-repository-configuration.js +0 -1
- package/dist/engines/contracts/test-context.d.ts +0 -15
- package/dist/engines/contracts/test-context.js +0 -1
- package/dist/engines/definitions.d.ts +0 -23
- package/dist/engines/factory.d.ts +0 -6
- package/dist/engines/generated-performance.d.ts +0 -1
- package/dist/engines/installation-catalog.d.ts +0 -27
- package/dist/engines/installation-catalog.js +0 -1
- package/dist/engines/installations/k6.d.ts +0 -98
- package/dist/engines/installations/k6.js +0 -1
- package/dist/engines/installations/node-vitest.d.ts +0 -33
- package/dist/engines/installations/package-requirements.d.ts +0 -28
- package/dist/engines/installations/package-resolution.d.ts +0 -51
- package/dist/engines/installations/playwright-resolution-child.d.ts +0 -1
- package/dist/engines/installations/playwright.d.ts +0 -75
- package/dist/engines/installations/types.d.ts +0 -10
- package/dist/engines/k6/adapter/bundle.d.ts +0 -25
- package/dist/engines/k6/adapter/bundle.js +0 -112
- package/dist/engines/k6/adapter/evidence.d.ts +0 -15
- package/dist/engines/k6/adapter/evidence.js +0 -4
- package/dist/engines/k6/adapter/executor.d.ts +0 -13
- package/dist/engines/k6/adapter/executor.js +0 -1
- package/dist/engines/k6/adapter/index.d.ts +0 -1
- package/dist/engines/k6/adapter/index.js +0 -1
- package/dist/engines/k6/adapter/input.d.ts +0 -21
- package/dist/engines/k6/adapter/input.js +0 -1
- package/dist/engines/k6/adapter/native-summary.d.ts +0 -11
- package/dist/engines/k6/adapter/native-summary.js +0 -1
- package/dist/engines/k6/adapter/repository-configuration.d.ts +0 -2
- package/dist/engines/k6/adapter/repository-configuration.js +0 -1
- package/dist/engines/k6/authoring-type-paths.d.ts +0 -3
- package/dist/engines/k6/authoring-type-paths.js +0 -1
- package/dist/engines/k6/runtime/ambient-types/bundle-manifest.json +0 -146
- package/dist/engines/k6/runtime/ambient-types/k6/LICENSE +0 -21
- package/dist/engines/k6/runtime/ambient-types/k6/browser/index.d.ts +0 -6332
- package/dist/engines/k6/runtime/ambient-types/k6/crypto/index.d.ts +0 -218
- package/dist/engines/k6/runtime/ambient-types/k6/data/index.d.ts +0 -16
- package/dist/engines/k6/runtime/ambient-types/k6/encoding/index.d.ts +0 -46
- package/dist/engines/k6/runtime/ambient-types/k6/execution/index.d.ts +0 -127
- package/dist/engines/k6/runtime/ambient-types/k6/experimental/csv/index.d.ts +0 -96
- package/dist/engines/k6/runtime/ambient-types/k6/experimental/fs/index.d.ts +0 -104
- package/dist/engines/k6/runtime/ambient-types/k6/experimental/redis/index.d.ts +0 -821
- package/dist/engines/k6/runtime/ambient-types/k6/experimental/streams/index.d.ts +0 -198
- package/dist/engines/k6/runtime/ambient-types/k6/global.d.ts +0 -83
- package/dist/engines/k6/runtime/ambient-types/k6/html/index.d.ts +0 -1325
- package/dist/engines/k6/runtime/ambient-types/k6/http/index.d.ts +0 -806
- package/dist/engines/k6/runtime/ambient-types/k6/index.d.ts +0 -126
- package/dist/engines/k6/runtime/ambient-types/k6/metrics/index.d.ts +0 -66
- package/dist/engines/k6/runtime/ambient-types/k6/net/grpc/index.d.ts +0 -247
- package/dist/engines/k6/runtime/ambient-types/k6/options/index.d.ts +0 -459
- package/dist/engines/k6/runtime/ambient-types/k6/package.json +0 -79
- package/dist/engines/k6/runtime/ambient-types/k6/secrets/index.d.ts +0 -36
- package/dist/engines/k6/runtime/ambient-types/k6/timers/index.d.ts +0 -47
- package/dist/engines/k6/runtime/ambient-types/k6/websockets/index.d.ts +0 -324
- package/dist/engines/k6/runtime/ambient-types/k6/ws/index.d.ts +0 -283
- package/dist/engines/k6/runtime/ambient-types/xk6-sql/LICENSE +0 -201
- package/dist/engines/k6/runtime/ambient-types/xk6-sql/index.d.ts +0 -397
- package/dist/engines/k6/runtime/ambient-types/xk6-sql/package.json +0 -15
- package/dist/engines/k6/runtime/artifacts.d.ts +0 -2
- package/dist/engines/k6/runtime/artifacts.js +0 -1
- package/dist/engines/k6/runtime/checks.d.ts +0 -17
- package/dist/engines/k6/runtime/checks.js +0 -1
- package/dist/engines/k6/runtime/dal-fixtures.d.ts +0 -24
- package/dist/engines/k6/runtime/dal-fixtures.js +0 -1
- package/dist/engines/k6/runtime/dal-suite.d.ts +0 -24
- package/dist/engines/k6/runtime/dal-suite.js +0 -1
- package/dist/engines/k6/runtime/dal.d.ts +0 -10
- package/dist/engines/k6/runtime/dal.js +0 -1
- package/dist/engines/k6/runtime/fixtures.d.ts +0 -3
- package/dist/engines/k6/runtime/fixtures.js +0 -1
- package/dist/engines/k6/runtime/http-assertions.d.ts +0 -19
- package/dist/engines/k6/runtime/http-assertions.js +0 -1
- package/dist/engines/k6/runtime/http-checks.d.ts +0 -19
- package/dist/engines/k6/runtime/http-checks.js +0 -1
- package/dist/engines/k6/runtime/http-suite-runtime.d.ts +0 -103
- package/dist/engines/k6/runtime/http-suite-runtime.js +0 -1
- package/dist/engines/k6/runtime/http.d.ts +0 -15
- package/dist/engines/k6/runtime/http.js +0 -1
- package/dist/engines/k6/runtime/index.d.ts +0 -11
- package/dist/engines/k6/runtime/index.js +0 -1
- package/dist/engines/k6/runtime/input.d.ts +0 -3
- package/dist/engines/k6/runtime/input.js +0 -1
- package/dist/engines/k6/runtime/k6-repository-configuration.d.ts +0 -9
- package/dist/engines/k6/runtime/k6-repository-configuration.js +0 -1
- package/dist/engines/k6/runtime/node-setup.d.ts +0 -13
- package/dist/engines/k6/runtime/node-setup.js +0 -1
- package/dist/engines/k6/runtime/runtime-globals.d.ts +0 -10
- package/dist/engines/k6/runtime/runtime.d.ts +0 -54
- package/dist/engines/k6/runtime/runtime.js +0 -1
- package/dist/engines/k6/runtime/shared/assertion-details.d.ts +0 -19
- package/dist/engines/k6/runtime/shared/assertion-details.js +0 -2
- package/dist/engines/k6/runtime/shared/db-normalization.d.ts +0 -19
- package/dist/engines/k6/runtime/shared/db-normalization.js +0 -1
- package/dist/engines/k6/runtime/shared/error-body.d.ts +0 -3
- package/dist/engines/k6/runtime/shared/error-body.js +0 -1
- package/dist/engines/k6/runtime/shared/file-data.d.ts +0 -1
- package/dist/engines/k6/runtime/shared/file-data.js +0 -1
- package/dist/engines/k6/runtime/shared/fixture-engine.d.ts +0 -8
- package/dist/engines/k6/runtime/shared/fixture-engine.js +0 -2
- package/dist/engines/k6/runtime/shared/http-check-plan.d.ts +0 -29
- package/dist/engines/k6/runtime/shared/http-check-plan.js +0 -1
- package/dist/engines/k6/runtime/shared/http-parsing.d.ts +0 -10
- package/dist/engines/k6/runtime/shared/http-parsing.js +0 -3
- package/dist/engines/k6/runtime/shared/ordering.d.ts +0 -2
- package/dist/engines/k6/runtime/shared/ordering.js +0 -1
- package/dist/engines/k6/runtime/shared/properties.d.ts +0 -1
- package/dist/engines/k6/runtime/shared/properties.js +0 -1
- package/dist/engines/k6/runtime/suite.d.ts +0 -4
- package/dist/engines/k6/runtime/suite.js +0 -1
- package/dist/engines/k6/runtime/types.d.ts +0 -244
- package/dist/engines/k6/runtime/types.js +0 -1
- package/dist/engines/k6/runtime/xk6-sql-driver-postgres.d.ts +0 -7
- package/dist/engines/k6/runtime/xk6-sql-driver-postgres.js +0 -1
- package/dist/engines/managed-module-resolution-policy-input.d.ts +0 -8
- package/dist/engines/managed-module-resolution-policy.d.ts +0 -25
- package/dist/engines/managed-module-resolution.d.ts +0 -13
- package/dist/engines/manager.d.ts +0 -25
- package/dist/engines/node-vitest/adapter/adapter.d.ts +0 -16
- package/dist/engines/node-vitest/adapter/consumer-typescript-paths.d.ts +0 -17
- package/dist/engines/node-vitest/adapter/consumer-typescript-paths.js +0 -3
- package/dist/engines/node-vitest/adapter/evidence.d.ts +0 -33
- package/dist/engines/node-vitest/adapter/failure-details.d.ts +0 -3
- package/dist/engines/node-vitest/adapter/index.d.ts +0 -1
- package/dist/engines/node-vitest/adapter/reporter.d.ts +0 -7
- package/dist/engines/node-vitest/adapter/resolution.d.ts +0 -12
- package/dist/engines/node-vitest/adapter/resolution.js +0 -1
- package/dist/engines/node-vitest/adapter/workspace.d.ts +0 -19
- package/dist/engines/node-vitest/runtime/contract/index.d.ts +0 -64
- package/dist/engines/node-vitest/runtime/contract/index.js +0 -1
- package/dist/engines/node-vitest/runtime/data/http.d.ts +0 -4
- package/dist/engines/node-vitest/runtime/data/http.js +0 -1
- package/dist/engines/node-vitest/runtime/data/index.d.ts +0 -8
- package/dist/engines/node-vitest/runtime/data/primitives.d.ts +0 -1
- package/dist/engines/node-vitest/runtime/data/report.d.ts +0 -3
- package/dist/engines/node-vitest/runtime/data/runner.d.ts +0 -4
- package/dist/engines/node-vitest/runtime/data/runner.js +0 -1
- package/dist/engines/node-vitest/runtime/data/scenario.d.ts +0 -11
- package/dist/engines/node-vitest/runtime/data/types.d.ts +0 -384
- package/dist/engines/node-vitest/runtime/runtime-input.d.ts +0 -64
- package/dist/engines/playwright/adapter/adapter.d.ts +0 -18
- package/dist/engines/playwright/adapter/attachments.d.ts +0 -26
- package/dist/engines/playwright/adapter/config.d.ts +0 -45
- package/dist/engines/playwright/adapter/evidence-validation.d.ts +0 -9
- package/dist/engines/playwright/adapter/evidence.d.ts +0 -32
- package/dist/engines/playwright/adapter/failure-details.d.ts +0 -2
- package/dist/engines/playwright/adapter/index.d.ts +0 -3
- package/dist/engines/playwright/adapter/reporter.d.ts +0 -12
- package/dist/engines/playwright/runtime/config-file.d.ts +0 -2
- package/dist/engines/playwright/runtime/config.d.ts +0 -24
- package/dist/engines/playwright/runtime/index.d.ts +0 -4
- package/dist/engines/playwright/runtime/runtime-input.d.ts +0 -36
- package/dist/engines/playwright/runtime/sandbox.d.ts +0 -84
- package/dist/engines/runtime-input-file.d.ts +0 -27
- package/dist/environment/lifecycle-artifacts.d.ts +0 -7
- package/dist/environment/lifecycle-controller.d.ts +0 -60
- package/dist/environment/lifecycle.d.ts +0 -32
- package/dist/environment/maintenance.d.ts +0 -68
- package/dist/environment/planning-types.d.ts +0 -33
- package/dist/environment/ports.d.ts +0 -10
- package/dist/environment/ready-environment.d.ts +0 -16
- package/dist/environment/run-environment.d.ts +0 -39
- package/dist/environment/run-stopping.d.ts +0 -13
- package/dist/environment/runtime-contexts.d.ts +0 -37
- package/dist/environment/runtime-manager.d.ts +0 -87
- package/dist/environment/runtime-planning.d.ts +0 -16
- package/dist/environment/runtime-preparation.d.ts +0 -30
- package/dist/environment/runtime-service.d.ts +0 -12
- package/dist/environment/services.d.ts +0 -56
- package/dist/environment/state.d.ts +0 -15
- package/dist/environment/status-runtime.d.ts +0 -53
- package/dist/environment/template.d.ts +0 -164
- package/dist/evidence/artifact-config.d.ts +0 -14
- package/dist/evidence/artifact-validation.d.ts +0 -4
- package/dist/evidence/artifacts.d.ts +0 -75
- package/dist/evidence/bundles.d.ts +0 -27
- package/dist/evidence/ci-provenance.d.ts +0 -9
- package/dist/evidence/contracts.d.ts +0 -154
- package/dist/evidence/environment-record.d.ts +0 -22
- package/dist/evidence/file-evidence.d.ts +0 -23
- package/dist/evidence/finalization.d.ts +0 -9
- package/dist/evidence/index.d.ts +0 -8
- package/dist/evidence/logs.d.ts +0 -31
- package/dist/evidence/operation-timings.d.ts +0 -60
- package/dist/evidence/read-bundle.d.ts +0 -4
- package/dist/evidence/read-fields.d.ts +0 -21
- package/dist/evidence/read-provenance.d.ts +0 -2
- package/dist/evidence/read-redaction.d.ts +0 -2
- package/dist/evidence/read-retention.d.ts +0 -2
- package/dist/evidence/read-run.d.ts +0 -2
- package/dist/evidence/reading.d.ts +0 -28
- package/dist/evidence/reporting-types.d.ts +0 -349
- package/dist/evidence/reporting.d.ts +0 -3
- package/dist/evidence/secure-file-copy.d.ts +0 -24
- package/dist/evidence/setup-progress.d.ts +0 -63
- package/dist/evidence/step-output.d.ts +0 -16
- package/dist/evidence/storage-inventory.d.ts +0 -27
- package/dist/evidence/storage.d.ts +0 -25
- package/dist/evidence/timing.d.ts +0 -23
- package/dist/file.d.ts +0 -11
- package/dist/index.d.ts +0 -4
- package/dist/kernel/error-serialization.d.ts +0 -40
- package/dist/kernel/redaction.d.ts +0 -2
- package/dist/kernel/unknown-values.d.ts +0 -11
- package/dist/kernel/work-metrics.d.ts +0 -6
- package/dist/process/capture-output.d.ts +0 -7
- package/dist/process/command-shell.d.ts +0 -1
- package/dist/process/environment-overlay.d.ts +0 -17
- package/dist/process/environment.d.ts +0 -33
- package/dist/process/files/contained-path.d.ts +0 -8
- package/dist/process/files/directory-status.d.ts +0 -14
- package/dist/process/files/dotenv.d.ts +0 -2
- package/dist/process/files/file-snapshot.d.ts +0 -34
- package/dist/process/files/json-file.d.ts +0 -11
- package/dist/process/files/paths.d.ts +0 -3
- package/dist/process/files/product-dir.d.ts +0 -1
- package/dist/process/files/temp-dir.d.ts +0 -11
- package/dist/process/host/metadata.d.ts +0 -6
- package/dist/process/host/platform.d.ts +0 -57
- package/dist/process/host-command.d.ts +0 -4
- package/dist/process/host-inspection-command.d.ts +0 -13
- package/dist/process/host-process.d.ts +0 -19
- package/dist/process/index.d.ts +0 -9
- package/dist/process/lifecycle-fence.d.ts +0 -10
- package/dist/process/locks.d.ts +0 -28
- package/dist/process/process-contracts.d.ts +0 -78
- package/dist/process/process-group.d.ts +0 -12
- package/dist/process/process-identity.d.ts +0 -27
- package/dist/process/process-limits.d.ts +0 -6
- package/dist/process/process.d.ts +0 -12
- package/dist/process/readiness.d.ts +0 -43
- package/dist/process/recovery.d.ts +0 -3
- package/dist/process/runs/inspection.d.ts +0 -5
- package/dist/process/runs/manifest.d.ts +0 -86
- package/dist/process/runs/ownership.d.ts +0 -11
- package/dist/process/runs/product-state.d.ts +0 -2
- package/dist/process/runs/schema.d.ts +0 -3
- package/dist/process/runs/stop.d.ts +0 -19
- package/dist/process/runtime-socket.d.ts +0 -4
- package/dist/process/service-processes.d.ts +0 -13
- package/dist/process/shutdown.d.ts +0 -10
- package/dist/process/stoppable-settlement.d.ts +0 -26
- package/dist/process/url-redaction.d.ts +0 -1
- package/dist/process/windows-cli-owner.d.ts +0 -2
- package/dist/process/windows-native.d.ts +0 -12
- package/dist/process/windows-process-identity.d.ts +0 -4
- package/dist/repo/ci-environment.d.ts +0 -7
- package/dist/repo/fingerprint-policy.d.ts +0 -13
- package/dist/repo/git-path.d.ts +0 -2
- package/dist/repo/history.d.ts +0 -17
- package/dist/repo/managed-git.d.ts +0 -10
- package/dist/repo/metadata.d.ts +0 -3
- package/dist/repo/physical-tree.d.ts +0 -37
- package/dist/repo/publish-directory.d.ts +0 -20
- package/dist/repo/remote.d.ts +0 -6
- package/dist/repo/state.d.ts +0 -26
- package/dist/repo/tree-listing.d.ts +0 -10
- package/dist/steps/build-config.d.ts +0 -34
- package/dist/steps/configured-step-testkit-modules.d.ts +0 -19
- package/dist/steps/configured-steps.d.ts +0 -51
- package/dist/steps/environment.d.ts +0 -82
- package/dist/steps/index.d.ts +0 -2
- package/dist/steps/module-runner.d.ts +0 -1
- package/dist/steps/run.d.ts +0 -36
- package/dist/toolchains/archive.d.ts +0 -53
- package/dist/toolchains/esbuild.d.ts +0 -9
- package/dist/toolchains/host-node-support.d.ts +0 -3
- package/dist/toolchains/index.d.ts +0 -114
- package/dist/toolchains/managed-node-releases.d.ts +0 -28
- package/dist/toolchains/node-cache.d.ts +0 -20
- package/dist/toolchains/node-integrity.d.ts +0 -18
- package/dist/toolchains/node-provisioning.d.ts +0 -29
- package/dist/toolchains/semver.d.ts +0 -13
- package/dist/toolchains/shared-operation.d.ts +0 -11
- package/dist/toolchains/timeouts.d.ts +0 -1
- package/dist/toolchains/vitest/index.d.ts +0 -13
- package/dist/tracker/gitea.d.ts +0 -14
- package/dist/tracker/github.d.ts +0 -14
- package/dist/tracker/http.d.ts +0 -33
- package/dist/tracker/index.d.ts +0 -4
- package/dist/tracker/redaction.d.ts +0 -4
- package/dist/tracker/requests.d.ts +0 -17
- package/dist/tracker/responses.d.ts +0 -24
- package/dist/tracker/types.d.ts +0 -51
- package/docs/engine-architecture.md +0 -682
- /package/dist/engines/{k6/runtime/runtime-globals.js → api/profiles.js} +0 -0
package/README.md
CHANGED
|
@@ -4,17 +4,19 @@
|
|
|
4
4
|
|
|
5
5
|
# @gfdelarue/testkit
|
|
6
6
|
|
|
7
|
-
`@gfdelarue/testkit` is
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
7
|
+
`@gfdelarue/testkit` is a test runner and CLI for coding agents and CI. It runs
|
|
8
|
+
a repository's ordinary Vitest test files, such as `src/sum.test.ts`, as they
|
|
9
|
+
are, and typed `*.testkit.ts` suites for Playwright and Node. It assigns
|
|
10
|
+
tests to groups by directory, starts only the local services a group uses,
|
|
11
|
+
provisions Docker-managed local Postgres databases when a group needs one,
|
|
12
|
+
schedules the files, and collects results and artifacts. It starts no service
|
|
13
|
+
for a test whose group uses none. Coding agents use Testkit to write tests
|
|
14
|
+
and hunt bugs, and CI uses it to run those tests. It needs no account or
|
|
15
|
+
server.
|
|
13
16
|
|
|
14
17
|
Testkit exists to replace false confidence with inspectable execution evidence.
|
|
15
|
-
It
|
|
16
|
-
|
|
17
|
-
fixtures, and inputs supplied to the run; this is what passed, failed, was
|
|
18
|
+
It makes claims that deserve trust: these tests genuinely executed; these were
|
|
19
|
+
the runtimes, fixtures, and inputs supplied to the run; this is what passed, failed, was
|
|
18
20
|
skipped, or did not run; and this is the evidence behind the result. Testkit
|
|
19
21
|
retains provenance, logs, and verifiable artifacts so those claims can be
|
|
20
22
|
audited rather than accepted on the strength of a green check mark.
|
|
@@ -41,8 +43,9 @@ npm run testkit
|
|
|
41
43
|
|
|
42
44
|
The command exits zero only when at least one test file completed and every
|
|
43
45
|
required file passed. Test failures, infrastructure failures, interrupted or
|
|
44
|
-
not-run files,
|
|
45
|
-
|
|
46
|
+
not-run files, a skipped file, and an empty selection all fail the gate. A file
|
|
47
|
+
that a `skip` rule or its own `skip` metadata skips fails the gate even when
|
|
48
|
+
every other file passes, and `--allow-empty` doesn't change that. `npm run testkit -- --allow-empty` is an explicit escape hatch for a job
|
|
46
49
|
whose empty selection is intentional; it should not be the default CI command.
|
|
47
50
|
|
|
48
51
|
Testkit writes the immutable, verifiable run bundle under
|
|
@@ -111,6 +114,23 @@ only when the test fails. It stops when tracked files have uncommitted changes,
|
|
|
111
114
|
when no remote branch contains the commit, or when an open issue already covers
|
|
112
115
|
the test. The token needs permission to create issues and labels.
|
|
113
116
|
|
|
117
|
+
`testkit bug record` saves the `--vitest-config`, `--vitest-project`, and
|
|
118
|
+
`--vitest-runner` it was given in the draft, and asks for `--vitest-project`
|
|
119
|
+
when two Vitest projects run the test's file. `testkit bug file` reruns with
|
|
120
|
+
them, and the issue's reproduce command carries them. With
|
|
121
|
+
`TESTKIT_REQUIRE_COMPLETE_SUITE=1`, it reruns every file the selection lists,
|
|
122
|
+
then reads the recorded failure.
|
|
123
|
+
|
|
124
|
+
`testkit bug file` takes the same [supplied values](#run-against-a-supplied-environment)
|
|
125
|
+
as `testkit run` and reruns the test on them. The issue's reproduce command
|
|
126
|
+
carries those flags, so `testkit bug file` refuses inputs that would put a raw
|
|
127
|
+
secret in it. It takes `--env-from` and `--environment-file`, and takes
|
|
128
|
+
`--base-url` and `--endpoint` only for URLs without credentials, a query, or a
|
|
129
|
+
fragment. It refuses `--database` and `--env`; pass those values through
|
|
130
|
+
`--env-from` or the environment file. The command runs from the repository
|
|
131
|
+
root, so a relative `--environment-file` needs `testkit bug file` run from
|
|
132
|
+
there too.
|
|
133
|
+
|
|
114
134
|
### Name the product's areas
|
|
115
135
|
|
|
116
136
|
Add `areas` to `testkit.config.ts` so that each issue names the part of the
|
|
@@ -243,28 +263,27 @@ the report to the chosen platform.
|
|
|
243
263
|
|
|
244
264
|
## Execution engines
|
|
245
265
|
|
|
246
|
-
Testkit has
|
|
247
|
-
|
|
248
|
-
- `
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
runtime.
|
|
266
|
+
Testkit has two built-in engines:
|
|
267
|
+
|
|
268
|
+
- `node-vitest` runs plain Vitest test files with Node and Vitest.
|
|
269
|
+
- `playwright` runs `ui` files with Playwright and managed Chromium.
|
|
270
|
+
|
|
271
|
+
A plain Vitest test file needs no Testkit suffix, directory, metadata, or
|
|
272
|
+
wrapper. It keeps its native `vitest` imports and its repository paths, has no
|
|
273
|
+
test type, and runs under the engine's general `node-vitest` profile. Testkit
|
|
274
|
+
does not evaluate `testkit.config.ts` in the test's process, and it installs the
|
|
275
|
+
values of the services the test's group uses before the test's imports
|
|
276
|
+
evaluate. See [Plain Vitest test files](#plain-vitest-test-files) for which
|
|
277
|
+
files Testkit runs.
|
|
278
|
+
|
|
279
|
+
Every typed `*.testkit.ts` file must export `defineFile({ engine: "..." })`. The
|
|
280
|
+
filename suffix supplies the public test type; Testkit validates that explicit
|
|
281
|
+
pair and derives a runner profile. It never infers a default engine for a typed
|
|
282
|
+
file or falls back to a different engine.
|
|
283
|
+
|
|
284
|
+
Consumers declare only `@gfdelarue/testkit`, which installs the exact versions
|
|
285
|
+
of the tools its engines run. A product's own copies of Vitest, Playwright, or
|
|
286
|
+
their tools don't change what Testkit runs.
|
|
268
287
|
|
|
269
288
|
Playwright browser installation is explicit:
|
|
270
289
|
|
|
@@ -273,28 +292,17 @@ npx @gfdelarue/testkit setup playwright
|
|
|
273
292
|
npx @gfdelarue/testkit setup playwright --with-deps # CI and fresh hosts
|
|
274
293
|
```
|
|
275
294
|
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
output
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
the AGPL corresponding source archive and licenses. Runtime evidence records
|
|
288
|
-
the package name and version and the k6 version observed by executing the
|
|
289
|
-
installed binary. The
|
|
290
|
-
[testkit-k6 README](https://github.com/gfdelarue/testkit-k6#readme) documents
|
|
291
|
-
the reproducible build, the expected executable digests, and the license
|
|
292
|
-
notes.
|
|
293
|
-
|
|
294
|
-
Managed task environment is engine input rather than ambient host state. k6
|
|
295
|
-
suites read it through native `__ENV`; Node/Vitest data and contract callbacks
|
|
296
|
-
read immutable, enumerable `context.env`; and Playwright suites can read the
|
|
297
|
-
same contract from `readPlaywrightContext()?.env`. Managed Node processes also
|
|
295
|
+
Normal runs never install or download a browser.
|
|
296
|
+
[Installation](docs/engines.md#installation) covers what an install contains,
|
|
297
|
+
the supported platforms, and how to recover a broken one.
|
|
298
|
+
[Typechecking](docs/engines.md#typechecking) covers the supported TypeScript
|
|
299
|
+
versions and `tsconfig.playwright.json`.
|
|
300
|
+
|
|
301
|
+
Testkit bounds the output it captures from each command it runs, and asks the
|
|
302
|
+
command and its descendants to stop on cancellation or timeout.
|
|
303
|
+
|
|
304
|
+
Managed task environment is engine input. Playwright suites read it from
|
|
305
|
+
`readPlaywrightContext()?.env`. Managed Node processes also
|
|
298
306
|
expose each value as an immutable, non-enumerable `process.env.KEY` property so
|
|
299
307
|
libraries that read `process.env` directly see it, but non-Node child processes
|
|
300
308
|
do not receive consumer values unless the suite explicitly passes them.
|
|
@@ -304,21 +312,23 @@ reserves `APPDATA`, `ComSpec`, `HOME`, `LANG`, `LC_ALL`, `LOCALAPPDATA`, `PATH`,
|
|
|
304
312
|
`PATHEXT`, `SystemRoot`, `TEMP`, `TMP`, `TMPDIR`, `USERPROFILE`,
|
|
305
313
|
`XDG_CACHE_HOME`, `XDG_CONFIG_HOME`, and `XDG_DATA_HOME`. In addition:
|
|
306
314
|
|
|
307
|
-
-
|
|
308
|
-
|
|
309
|
-
|
|
315
|
+
- Node/Vitest reserves `BASE_URL`, `FORCE_COLOR`, `NODE_OPTIONS`, `NO_COLOR`,
|
|
316
|
+
`UPDATE_SNAPSHOT`, `VITEST_POOL_ID`, `VITEST_WORKER_ID`, and the `TESTKIT_`
|
|
317
|
+
keys it sets itself, listed in
|
|
318
|
+
[Environment](docs/engines.md#environment);
|
|
319
|
+
other `TESTKIT_` and `VITEST_` keys, such as
|
|
320
|
+
`TESTKIT_REQUIRE_COMPLETE_SUITE`, pass through, though Vitest's own `TEST`,
|
|
321
|
+
`VITEST`, `VITEST_MODE`, `FORCE_TTY`, `MODE`, `DEV`, `PROD`, and `SSR`
|
|
322
|
+
replace a group's values;
|
|
310
323
|
- Playwright reserves the `TESTKIT_`, `PLAYWRIGHT_`, `PWTEST_`, and `PW_TEST_`
|
|
311
324
|
prefixes plus `BASE_URL`, `FORCE_COLOR`, `NODE_OPTIONS`, and `NO_COLOR`.
|
|
312
325
|
|
|
313
326
|
Case-variant duplicates and invalid names or NUL-containing values fail instead
|
|
314
|
-
of being normalized.
|
|
315
|
-
environment-block character units. Node/Vitest accepts at most 512 entries and
|
|
327
|
+
of being normalized. Node/Vitest accepts at most 512 entries and
|
|
316
328
|
4 MiB of UTF-8 key/value data. Playwright accepts at most 512 entries and 2 MiB
|
|
317
|
-
of UTF-8 key/value data.
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
[Engine Architecture](docs/engine-architecture.md#managed-environment) for the
|
|
321
|
-
complete transport and aggregate-input limits.
|
|
329
|
+
of UTF-8 key/value data. [Environment](docs/engines.md#environment) lists the
|
|
330
|
+
complete limits, and [Named endpoints and databases](#named-endpoints-and-databases)
|
|
331
|
+
covers the environment API.
|
|
322
332
|
|
|
323
333
|
## Usage
|
|
324
334
|
|
|
@@ -335,28 +345,27 @@ npx @gfdelarue/testkit discover --json > .testkit/discovery.json
|
|
|
335
345
|
npx @gfdelarue/testkit discover --output .testkit/discovery.json
|
|
336
346
|
|
|
337
347
|
# Filter by type
|
|
338
|
-
npx @gfdelarue/testkit --type int
|
|
339
|
-
npx @gfdelarue/testkit --type dal
|
|
340
|
-
npx @gfdelarue/testkit --type e2e
|
|
341
|
-
npx @gfdelarue/testkit --type int,e2e,dal
|
|
342
348
|
npx @gfdelarue/testkit --type ui
|
|
349
|
+
npx @gfdelarue/testkit --type plain
|
|
343
350
|
|
|
344
|
-
#
|
|
351
|
+
# Run up to 8 test tasks at once
|
|
345
352
|
npx @gfdelarue/testkit --task-concurrency 8
|
|
346
353
|
|
|
347
354
|
# One file-level wall clock budget for every suite file
|
|
348
355
|
npx @gfdelarue/testkit --file-timeout-seconds 60
|
|
349
356
|
|
|
350
|
-
# Specific
|
|
351
|
-
npx @gfdelarue/testkit --
|
|
352
|
-
npx @gfdelarue/testkit --
|
|
353
|
-
npx @gfdelarue/testkit --
|
|
357
|
+
# Specific group / suite
|
|
358
|
+
npx @gfdelarue/testkit --group frontend --type ui -s navigation
|
|
359
|
+
npx @gfdelarue/testkit --group api -s health
|
|
360
|
+
npx @gfdelarue/testkit --group api --type plain
|
|
361
|
+
npx @gfdelarue/testkit --type ui -s ui:checkout
|
|
354
362
|
|
|
355
363
|
# Exact file
|
|
356
|
-
npx @gfdelarue/testkit --type
|
|
364
|
+
npx @gfdelarue/testkit --type ui --file __testkit__/checkout.ui.testkit.ts
|
|
365
|
+
npx @gfdelarue/testkit run --file src/sum.test.ts
|
|
357
366
|
|
|
358
367
|
# Temporarily ignore repo-declared skip rules
|
|
359
|
-
npx @gfdelarue/testkit --ignore-skip-rules --file
|
|
368
|
+
npx @gfdelarue/testkit --ignore-skip-rules --file src/billing.test.ts
|
|
360
369
|
|
|
361
370
|
# Lifecycle
|
|
362
371
|
npx @gfdelarue/testkit status
|
|
@@ -371,14 +380,27 @@ npx @gfdelarue/testkit db schema refresh --service api
|
|
|
371
380
|
npx @gfdelarue/testkit db schema verify --service api
|
|
372
381
|
```
|
|
373
382
|
|
|
383
|
+
`--type ui` and typed suite selectors such as `ui:checkout` select typed files
|
|
384
|
+
only. `--type plain` selects plain Vitest test files only. A run without
|
|
385
|
+
`--type`, or with `--type all`, runs both. `--type` narrows the other filters,
|
|
386
|
+
so `--group api --type plain` runs the `api` group's plain files and none of its
|
|
387
|
+
UI files.
|
|
388
|
+
|
|
389
|
+
`run`, `discover`, `artifacts`, `workspace`, and `typecheck` select test groups
|
|
390
|
+
with `--group`. `status`, `destroy`, `cleanup`, and `db schema` take `--service`
|
|
391
|
+
to name a service.
|
|
392
|
+
|
|
374
393
|
Database cleanup is scoped to the selected product. `cleanup --resources --global`
|
|
375
394
|
is retired; use `--dir <product> cleanup --resources` for each product. After
|
|
376
395
|
deleting a product directory, remove its orphaned containers and volumes
|
|
377
396
|
explicitly through Docker.
|
|
378
397
|
|
|
379
398
|
Use `execution.taskConcurrency` in config and `--task-concurrency` on the CLI.
|
|
380
|
-
|
|
381
|
-
|
|
399
|
+
A task is a typed file, a batch of typed files, or one Vitest invocation of
|
|
400
|
+
plain files. Plain files in one invocation run with Vitest's file parallelism,
|
|
401
|
+
which `taskConcurrency` doesn't limit.
|
|
402
|
+
`execution.workers`, `presets.nodeNext({ workers })` and `--workers` are
|
|
403
|
+
deprecated aliases. When both names are
|
|
382
404
|
provided in config or on the CLI, their values must match. The CLI value overrides
|
|
383
405
|
config after those checks. See the [execution settings](testkit-config-schema.md#execution).
|
|
384
406
|
|
|
@@ -386,15 +408,125 @@ Batch `run` output stays intentionally short: one line per completed file,
|
|
|
386
408
|
concise failure blocks, and a final summary. Service logs, captured runtime
|
|
387
409
|
output, and emitted artifacts are persisted under `.testkit/results/`.
|
|
388
410
|
|
|
411
|
+
Text output is coloured in a terminal and plain otherwise. As in Node,
|
|
412
|
+
`NO_COLOR` turns colour off and `FORCE_COLOR` overrides it: `0` or `false`
|
|
413
|
+
turns colour off and any other value turns it on.
|
|
414
|
+
|
|
389
415
|
Testkit seals an immutable local run bundle. `testkit run` prints
|
|
390
416
|
`Run ID: <id>` after sealing in text output and returns `runId` in `--json`
|
|
391
417
|
output, for passed and failed runs.
|
|
392
418
|
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
419
|
+
Testkit starts the files that took longest in earlier runs first, so slow
|
|
420
|
+
files start while workers are free. The run document records the order under
|
|
421
|
+
`planning`.
|
|
422
|
+
|
|
423
|
+
### Run against a supplied environment
|
|
424
|
+
|
|
425
|
+
`testkit run` can take the values its tests need instead of provisioning them.
|
|
426
|
+
Every selected group then runs on those values alone. Testkit starts none of
|
|
427
|
+
the group's services, creates no database, and runs no `.setup.*` module beside
|
|
428
|
+
a test. It also ignores the env values and env files those services declare, so
|
|
429
|
+
supply every value the tests read. Groups, selection, skip rules, and other
|
|
430
|
+
policy work as before.
|
|
431
|
+
|
|
432
|
+
```bash
|
|
433
|
+
npx @gfdelarue/testkit run --group checks \
|
|
434
|
+
--base-url http://127.0.0.1:3000 \
|
|
435
|
+
--endpoint api=http://127.0.0.1:3001 \
|
|
436
|
+
--database store=postgres://app:password@127.0.0.1:5432/store \
|
|
437
|
+
--env FEATURE_FLAG=on \
|
|
438
|
+
--env-from API_TOKEN=CI_API_TOKEN
|
|
439
|
+
|
|
440
|
+
npx @gfdelarue/testkit run --environment-file supplied.json
|
|
441
|
+
```
|
|
442
|
+
|
|
443
|
+
- `--base-url URL` supplies the default endpoint, which tests receive as
|
|
444
|
+
`BASE_URL`.
|
|
445
|
+
- `--endpoint NAME=URL` supplies a named endpoint, usually named after the
|
|
446
|
+
service it stands in for.
|
|
447
|
+
- `--database NAME=URL` supplies a named PostgreSQL database.
|
|
448
|
+
- `--env NAME=VALUE` supplies an environment value.
|
|
449
|
+
- `--env-from NAME=HOST_KEY` supplies the value of host variable `HOST_KEY`
|
|
450
|
+
under `NAME`.
|
|
451
|
+
- `--environment-file PATH` reads values, prepared setup data, and handle
|
|
452
|
+
descriptors from a file.
|
|
453
|
+
|
|
454
|
+
Every flag except `--base-url` and `--environment-file` can be repeated. Tests
|
|
455
|
+
read named values with `endpoint(name)` and `database(name)` from
|
|
456
|
+
`@gfdelarue/testkit/environment`. An `--env`, `--env-from`, or file `env` entry
|
|
457
|
+
named `BASE_URL` or `DATABASE_URL` sets the default endpoint or database, and an
|
|
458
|
+
empty database URL means no database. Without an explicit default, a group's
|
|
459
|
+
`BASE_URL` and `DATABASE_URL` are those of the first service in its `uses` that
|
|
460
|
+
has a supplied endpoint or database. Either way the default is also available
|
|
461
|
+
by the group's name, as in a configured run, so it may not differ from a named
|
|
462
|
+
endpoint or database with that name.
|
|
463
|
+
|
|
464
|
+
Testkit reads host variables only through `--env-from`. A `BASE_URL` or
|
|
465
|
+
`DATABASE_URL` already set in the shell does not switch a run to supplied
|
|
466
|
+
values. Two inputs may define the same value only if they agree. Any other
|
|
467
|
+
repeat, such as `--base-url` and `--env BASE_URL` with different URLs, or a
|
|
468
|
+
missing `--env-from` host variable, fails the run before it starts. Before a
|
|
469
|
+
test starts, each supplied endpoint must give a valid HTTP response. A 4xx or
|
|
470
|
+
5xx response counts as valid.
|
|
471
|
+
|
|
472
|
+
Each supplied database must also accept a connection before a test starts.
|
|
473
|
+
Testkit connects, signs in with the URL's credentials, runs `SELECT 1`, and
|
|
474
|
+
disconnects. It handles trust, password, md5, and SCRAM-SHA-256 authentication.
|
|
475
|
+
TLS follows the URL's `sslmode` as in libpq. The default, `prefer`, tries TLS
|
|
476
|
+
first and then a plain connection, and `allow` tries them the other way round.
|
|
477
|
+
`verify-ca` and `verify-full` need `sslrootcert`, set to a CA file or to
|
|
478
|
+
`system`, and `sslrootcert=system` alone means `verify-full`. `host=/path`
|
|
479
|
+
connects through the Unix socket in that directory and ignores `sslmode`. The
|
|
480
|
+
check reads only the URL. It ignores `PG*` variables, `~/.pgpass`, and
|
|
481
|
+
`~/.postgresql/root.crt`, and it presents no client certificate. A database
|
|
482
|
+
that is unreachable, fails TLS, rejects the credentials, does not exist, or is
|
|
483
|
+
not PostgreSQL fails the run before any test starts. The message names the
|
|
484
|
+
database, never its URL.
|
|
485
|
+
|
|
486
|
+
The environment file is JSON:
|
|
487
|
+
|
|
488
|
+
```json
|
|
489
|
+
{
|
|
490
|
+
"version": 1,
|
|
491
|
+
"baseUrl": "http://127.0.0.1:3000",
|
|
492
|
+
"databaseUrl": "postgres://app:password@127.0.0.1:5432/app",
|
|
493
|
+
"endpoints": { "api": "http://127.0.0.1:3001" },
|
|
494
|
+
"databases": { "store": "postgres://app:password@127.0.0.1:5432/store" },
|
|
495
|
+
"env": { "FEATURE_FLAG": "on" },
|
|
496
|
+
"ui": {},
|
|
497
|
+
"setupData": { "actor": "Grace" },
|
|
498
|
+
"handles": {
|
|
499
|
+
"webhook": {
|
|
500
|
+
"kind": "http-capture",
|
|
501
|
+
"url": "http://127.0.0.1:41000/webhook",
|
|
502
|
+
"requestsUrl": "http://127.0.0.1:41000/__testkit__/<capability>/requests",
|
|
503
|
+
"resetUrl": "http://127.0.0.1:41000/__testkit__/<capability>/requests"
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
```
|
|
508
|
+
|
|
509
|
+
Only `version` is required, and this Testkit reads version 1. `baseUrl`,
|
|
510
|
+
`databaseUrl`, and `ui` may be `null`, as an environment rendered by Testkit
|
|
511
|
+
writes them, and `null` there counts as absent. Tests receive
|
|
512
|
+
`setupData` as the data a setup module would have returned: `{}` when the file
|
|
513
|
+
leaves it out, and `null` when the file sets it to `null`. `handles` entries must each name a `kind`, and tests reach them with
|
|
514
|
+
[`handle(name)`](#handles). An `http-capture` entry holds the `url`,
|
|
515
|
+
`requestsUrl`, and `resetUrl` that a capture sidecar reports, and gives the
|
|
516
|
+
test the same handle a setup module's capture would. Other kinds are handed on,
|
|
517
|
+
and `handle()` throws `EnvironmentHandleError` with `"unsupported"` for them. Any other field, such as
|
|
518
|
+
service or database configuration, is rejected. Errors name the file and the
|
|
519
|
+
field and do not repeat the file's contents.
|
|
520
|
+
|
|
521
|
+
A run records the values its tests received in this shape. Each acquisition in
|
|
522
|
+
`run.json`'s `environment` section has them under `values`, with secrets and
|
|
523
|
+
capture capabilities redacted. Its `setupData` is the supplied setup data or
|
|
524
|
+
what the file's setup module returned, and `{}` without either. Sealing redacts
|
|
525
|
+
setup data under a key that names a secret or that `redact` lists, and records
|
|
526
|
+
the rest as returned, so list any other sensitive key in `redact`. With
|
|
527
|
+
`"version": 1` added, an
|
|
528
|
+
acquisition's `values` are an environment file that gives a second run the
|
|
529
|
+
same values, unless redaction changed one of them.
|
|
398
530
|
|
|
399
531
|
## Tooling Adapters
|
|
400
532
|
|
|
@@ -430,23 +562,17 @@ export default defineConfig({
|
|
|
430
562
|
// playwright.config.ts
|
|
431
563
|
import { defineTestkitConfig, devices } from "@gfdelarue/testkit/playwright";
|
|
432
564
|
|
|
433
|
-
export default defineTestkitConfig(
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
},
|
|
438
|
-
{
|
|
439
|
-
dotenvFiles: [".env.local"],
|
|
440
|
-
},
|
|
441
|
-
);
|
|
565
|
+
export default defineTestkitConfig({
|
|
566
|
+
testDir: "./__testkit__",
|
|
567
|
+
projects: [{ name: "chromium", use: { ...devices["Desktop Chrome"] } }],
|
|
568
|
+
});
|
|
442
569
|
```
|
|
443
570
|
|
|
444
571
|
Managed UI suites use `@gfdelarue/testkit/playwright` for Testkit's Playwright
|
|
445
572
|
helpers and re-exported authoring API. They may also import exact public exports
|
|
446
573
|
from the managed `@playwright/test`, `playwright`, and `playwright-core`
|
|
447
|
-
packages.
|
|
448
|
-
|
|
449
|
-
different compile-time API.
|
|
574
|
+
packages. Those imports resolve to Testkit's pinned versions at run time and
|
|
575
|
+
when typechecking, whatever Playwright version the product installs.
|
|
450
576
|
|
|
451
577
|
For scripts and app-runtime code, `@gfdelarue/testkit/env` provides shared helpers
|
|
452
578
|
for managed runtime detection, dotenv loading, and local-database safety:
|
|
@@ -541,25 +667,27 @@ and attempts under `environment.services`. A config that sets `local.port`, the
|
|
|
541
667
|
`port` option of `app.node` or `app.next`, or a literal port in `local.baseUrl`
|
|
542
668
|
or `local.readyUrl` fails to load; delete it and use `{port}`.
|
|
543
669
|
|
|
544
|
-
Every test file declares its execution engine and any file-local
|
|
545
|
-
metadata next to the test
|
|
670
|
+
Every typed test file declares its execution engine and any file-local
|
|
671
|
+
scheduling metadata next to the test. A plain Vitest test file declares
|
|
672
|
+
neither:
|
|
546
673
|
|
|
547
674
|
```ts
|
|
548
675
|
import { defineFile } from "@gfdelarue/testkit";
|
|
549
676
|
|
|
550
677
|
export const testkit = defineFile({
|
|
551
|
-
engine: "
|
|
678
|
+
engine: "node-vitest",
|
|
552
679
|
skip: "Billing is currently unavailable locally",
|
|
553
|
-
locks: ["global-worker-loop"],
|
|
554
680
|
});
|
|
555
681
|
```
|
|
556
682
|
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
683
|
+
A file can also name `locks`.
|
|
684
|
+
[Give each test its own data](#give-each-test-its-own-data) says when to use
|
|
685
|
+
one.
|
|
686
|
+
|
|
687
|
+
In a typed file, the engine value is a required literal. The supported public
|
|
688
|
+
authoring path is `@gfdelarue/testkit/playwright`. Playwright suites may also
|
|
689
|
+
import exact public APIs directly from Testkit's managed `@playwright/test`,
|
|
690
|
+
`playwright`, and `playwright-core` packages.
|
|
563
691
|
There are no legacy aliases or implicit-runner compatibility paths.
|
|
564
692
|
|
|
565
693
|
`testkit.config.ts` is optional. Without it, Testkit runs every test file with
|
|
@@ -584,10 +712,9 @@ for:
|
|
|
584
712
|
- repo-declared suite/file skip policies with explicit reasons
|
|
585
713
|
|
|
586
714
|
`runtime.prepare` is the generic build-once hook for shared runtimes. It runs
|
|
587
|
-
once per runtime generation before local services start,
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
stable build-and-start flows.
|
|
715
|
+
once per runtime generation before local services start, and runs again when
|
|
716
|
+
its declared inputs change. This is the right way to move expensive browser
|
|
717
|
+
targets from `next dev` / watch mode to stable build-and-start flows.
|
|
591
718
|
|
|
592
719
|
Setup steps (`runtime.prepare`, database `template`, and other Testkit-run
|
|
593
720
|
Node commands) inherit `NODE_OPTIONS`. `TESTKIT_SETUP_NODE_OPTIONS` appends
|
|
@@ -598,19 +725,14 @@ setup processes, so set it deliberately.
|
|
|
598
725
|
Testkit-owned Docker resources are labelled at creation time so they can be
|
|
599
726
|
found again even if product-local `.testkit` state is deleted. `testkit destroy`
|
|
600
727
|
removes labelled resources for the current product after service state is
|
|
601
|
-
destroyed.
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
immutable volume ID or conditional-delete API, so direct Docker-daemon
|
|
610
|
-
principals are part of the host trust boundary and must not replace that
|
|
611
|
-
unguessable name during the final removal call. Testkit serializes database
|
|
612
|
-
mutations for the same product; independent products can proceed concurrently.
|
|
613
|
-
Use `cleanup --resources --dry-run` to preview the selected product's cleanup.
|
|
728
|
+
destroyed. It leaves run results under `.testkit/results` and other files
|
|
729
|
+
under `.testkit`, such as the compiled config and timings, in place. `testkit cleanup --resources` cleans stale resources for the current
|
|
730
|
+
product. Both commands remove only the current product's resources and leave
|
|
731
|
+
other products' alone, including their idle caches. Host-local Postgres data
|
|
732
|
+
lives in a Testkit-labelled named Docker volume, so cleanup can find and remove
|
|
733
|
+
it. Database changes for one product run one at a time; other products don't
|
|
734
|
+
wait for them. Use `cleanup --resources --dry-run` to preview the selected
|
|
735
|
+
product's cleanup.
|
|
614
736
|
|
|
615
737
|
`database.template` is the database-side equivalent for reusable template DB
|
|
616
738
|
state. When `database.sourceSchema` is configured, Testkit treats the configured
|
|
@@ -622,34 +744,16 @@ schema still matches the source dump. If local replay differs, Testkit refreshes
|
|
|
622
744
|
from the source once for the current cache key and retries. If it still differs,
|
|
623
745
|
the run fails with schema diagnostics under `.testkit/results/schema`.
|
|
624
746
|
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
the
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
`.gitignore` files are respected; host-global excludes, `core.excludesFile`, and
|
|
636
|
-
`.git/info/exclude` do not alter Testkit's cache identity.
|
|
637
|
-
|
|
638
|
-
Fingerprint Git commands disable replacement objects and optional locks and do
|
|
639
|
-
not read system or user Git configuration. Git path records are decoded from
|
|
640
|
-
their byte-preserving quoted form and must be valid UTF-8. Uninitialized
|
|
641
|
-
gitlinks and special filesystem nodes fail explicitly instead of being assigned
|
|
642
|
-
an incomplete or ambiguous identity. Testkit reads the complete HEAD, index,
|
|
643
|
-
untracked, submodule, and physical snapshot twice and fails if the observations
|
|
644
|
-
differ.
|
|
645
|
-
|
|
646
|
-
Physical reads open regular files by descriptor and verify every pathname
|
|
647
|
-
component before and after use; on POSIX, ancestor directory descriptors remain
|
|
648
|
-
open during the read. Node does not expose a portable `openat`/Windows reparse-
|
|
649
|
-
handle traversal API, so this detects ordinary replacement races but is not a
|
|
650
|
-
security boundary against a same-user adversary able to perform a perfectly
|
|
651
|
-
timed replace-and-restore ABA attack. Run Testkit against source trees that are
|
|
652
|
-
not concurrently controlled by an untrusted local process.
|
|
747
|
+
Testkit keys the source schema cache on the repository's state. A clean
|
|
748
|
+
worktree shares the entry for its commit. A dirty worktree or a directory
|
|
749
|
+
outside Git gets its own entry, keyed on the content of its files.
|
|
750
|
+
|
|
751
|
+
The key changes when a file's bytes, type, or executable mode changes, including
|
|
752
|
+
changes Git's own status can hide, such as sparse checkout or line-ending
|
|
753
|
+
normalization, and when a submodule moves. Untracked files are included unless
|
|
754
|
+
the repository's `.gitignore` files exclude them; global or per-clone Git
|
|
755
|
+
excludes don't count. A path Testkit can't read, or a tree that changes while
|
|
756
|
+
Testkit reads it, fails the run instead of producing a key.
|
|
653
757
|
|
|
654
758
|
Testkit-owned outputs are always excluded from fingerprints: `.testkit/`,
|
|
655
759
|
and `.next-testkit/`. `testkit.config.ts` is normal repo
|
|
@@ -674,22 +778,16 @@ Template setup executes in three explicit phases:
|
|
|
674
778
|
|
|
675
779
|
Schema drift is checked after each successful `migrate` and `seed` step, and
|
|
676
780
|
`verify` only runs once local replay matches the source schema. Source refreshes
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
PgBouncer/pooler URLs fail closed; configure a direct source URL for those
|
|
688
|
-
providers. Concurrent refreshes for the same service and cache key are
|
|
689
|
-
serialized with a cache-local lock so multiple Testkit processes do not
|
|
690
|
-
stampede the source database. Testkit also maintains
|
|
691
|
-
`.testkit/db/<service>/source-schemas/index.json` and prunes old inactive cache
|
|
692
|
-
entries automatically.
|
|
781
|
+
dump the schema only, in Docker, so seed and reference data never enter the
|
|
782
|
+
baseline and no host `pg_dump` is needed. Keep schema-changing setup in its own
|
|
783
|
+
step where possible; a single command that changes schema and then fails before
|
|
784
|
+
exiting cannot be refreshed at the midpoint.
|
|
785
|
+
|
|
786
|
+
A source schema refresh uses one connection. A Neon pooled source URL is
|
|
787
|
+
rewritten to the matching direct Neon endpoint. Other PgBouncer or pooler URLs
|
|
788
|
+
fail; configure a direct source URL for those providers. Refreshes of the same
|
|
789
|
+
cache entry run one at a time, even from separate Testkit processes, and
|
|
790
|
+
Testkit prunes old inactive cache entries automatically.
|
|
693
791
|
|
|
694
792
|
For most repos, prefer declarative step objects directly inside
|
|
695
793
|
`database.postgres({ template: ... })` and `runtime.prepare.steps`.
|
|
@@ -699,10 +797,9 @@ The supported shapes are:
|
|
|
699
797
|
- `{ kind: "sql-file", path: "..." }`
|
|
700
798
|
- `{ kind: "module", target: "file.ts#exportName" }`
|
|
701
799
|
|
|
702
|
-
Local files and ordinary external packages remain valid module targets.
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
private Testkit subpath fails before preparation or database setup starts.
|
|
800
|
+
Local files and ordinary external packages remain valid module targets. An
|
|
801
|
+
unsupported or private `@gfdelarue/testkit` target fails when the config loads,
|
|
802
|
+
before preparation or database setup starts.
|
|
706
803
|
|
|
707
804
|
`runtime.toolchain` is the first-class way to make those prepare/start commands
|
|
708
805
|
run under the correct Node toolchain instead of whatever `node`/`npm` happened
|
|
@@ -720,16 +817,12 @@ to launch `testkit`. Node toolchains support:
|
|
|
720
817
|
- `package.json#packageManager`
|
|
721
818
|
- `package.json#engines.npm`
|
|
722
819
|
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
rejected. `runtime.toolchain.cwd` is always a normalized product-relative path
|
|
730
|
-
and cannot contain a parent-directory segment. The trusted official Linux Node
|
|
731
|
-
archives are glibc builds; `install: "download"` fails before fetch or cache
|
|
732
|
-
mutation on musl instead of pretending those binaries are compatible.
|
|
820
|
+
`install: "download"` supports the Node versions and platforms Testkit lists,
|
|
821
|
+
and fails before downloading for any other. Downloads are cached under
|
|
822
|
+
`.testkit/_toolchains/node`, and a modified cache entry fails instead of
|
|
823
|
+
running. `runtime.toolchain.cwd` is a product-relative path and cannot contain
|
|
824
|
+
a parent-directory segment. The official Linux Node archives are glibc builds,
|
|
825
|
+
so `install: "download"` fails on musl before it downloads anything.
|
|
733
826
|
|
|
734
827
|
Example:
|
|
735
828
|
|
|
@@ -758,6 +851,77 @@ artifact/log/setup files. Payloads are SHA-256 addressed under
|
|
|
758
851
|
removing their logical per-run paths. `.testkit/results/latest.json` remains a
|
|
759
852
|
convenience projection; it is not the historical authority.
|
|
760
853
|
|
|
854
|
+
`run.json` is schema 20. `groups` holds each group's suites and files.
|
|
855
|
+
`environment` is the run's environment record. For each acquisition, it holds:
|
|
856
|
+
|
|
857
|
+
- the source that prepared it: `configured`, `static`, `supplied`, or `none`;
|
|
858
|
+
- the tasks it served and the services it started;
|
|
859
|
+
- each file's setup;
|
|
860
|
+
- how it was released;
|
|
861
|
+
- the values its tests received.
|
|
862
|
+
|
|
863
|
+
It also holds each service's start attempts, the service and setup logs, the
|
|
864
|
+
run's capacity and database backend, and the values tests asked for that the
|
|
865
|
+
environment lacked. Runs that Testkit 0.4.x stored as schema 19 stay readable
|
|
866
|
+
through `testkit artifacts`, `ci report`, known-failure checks, and the
|
|
867
|
+
workspace API.
|
|
868
|
+
|
|
869
|
+
Sealing replaces secrets with `<redacted>` in the run record and in every text
|
|
870
|
+
file of the bundle. A key names a secret when it holds `PASSWORD`, `TOKEN`,
|
|
871
|
+
`SECRET`, `PRIVATE_KEY`, `AUTH`, `COOKIE`, `SESSION`, or `API_KEY` in any letter
|
|
872
|
+
case. Sealing redacts:
|
|
873
|
+
|
|
874
|
+
- A `KEY=value` assignment under such a key, an `Authorization`,
|
|
875
|
+
`Proxy-Authorization`, `Cookie`, or `Set-Cookie` header value, a JSON string
|
|
876
|
+
value under such a key, even when the key is spelled with escapes, a
|
|
877
|
+
single-quoted `'password': '…'` value, and a URL's password, wherever they
|
|
878
|
+
appear. A quoted value ends at its closing quote and keeps its quotes, and a
|
|
879
|
+
value that starts with an auth scheme such as `Bearer` ends with its line.
|
|
880
|
+
A secret in one JSON value leaves the record's other values as they are.
|
|
881
|
+
- The value of each such key, and of each key that `redact` names, in the
|
|
882
|
+
environment Testkit runs in. When the value is a URL, its password counts
|
|
883
|
+
too.
|
|
884
|
+
- The values the run's environment resolved: each value under such a key or a
|
|
885
|
+
key that `redact` names, and the password of each URL. A PostgreSQL URL's
|
|
886
|
+
password may also be a `password` query parameter, as libpq reads it. This
|
|
887
|
+
covers config `env`, env files, `local.env`, dependency services, base,
|
|
888
|
+
readiness, UI, and database URLs, `--env`, `--env-from`, and an
|
|
889
|
+
`--environment-file`. A value under such a key that is a URL counts only for
|
|
890
|
+
its password.
|
|
891
|
+
- The unguessable segment in each [HTTP capture's](#handles) operation URLs,
|
|
892
|
+
from a setup module's capture or an `--environment-file` handle.
|
|
893
|
+
|
|
894
|
+
Name a secret's key in `redact` when it holds none of the words above:
|
|
895
|
+
|
|
896
|
+
```ts
|
|
897
|
+
export default defineConfig({
|
|
898
|
+
redact: ["STRIPE_SK", "PGPASS"],
|
|
899
|
+
});
|
|
900
|
+
```
|
|
901
|
+
|
|
902
|
+
Testkit matches the names in any letter case, in a configured run and a
|
|
903
|
+
supplied run alike. `redact` adds keys whose values sealing redacts; the forms
|
|
904
|
+
in the first item still use the words above.
|
|
905
|
+
|
|
906
|
+
A value shorter than eight characters is not redacted on its own. Sealing also
|
|
907
|
+
redacts a value's JSON-escaped forms, each line of a value that spans lines,
|
|
908
|
+
and keys in the free-form data and metadata that tests attach. When output cuts a secret short and marks
|
|
909
|
+
the cut with `…` or `...`, sealing redacts the part that shows. `testkit run
|
|
910
|
+
--json` and the run's progress output apply the same redaction.
|
|
911
|
+
|
|
912
|
+
Some secrets stay visible:
|
|
913
|
+
|
|
914
|
+
- What a setup module prints goes to the terminal as written.
|
|
915
|
+
- Sealing does not look inside compressed or binary files, such as a Playwright
|
|
916
|
+
`trace.zip`. It refuses to seal a binary file that holds one of the values
|
|
917
|
+
above in plain bytes.
|
|
918
|
+
- When a run's process is killed before it seals the run, `testkit stop`,
|
|
919
|
+
`testkit cleanup`, or the next run seals it from another process. That
|
|
920
|
+
process does not have the values the run's environment resolved or the
|
|
921
|
+
`redact` list, so it redacts only the first item and its own environment's
|
|
922
|
+
values under the words above. The bundle's `archive.redaction.status` is then
|
|
923
|
+
`source-targets-unavailable` instead of `applied`.
|
|
924
|
+
|
|
761
925
|
Use the artifact archive config to opt into pruning or atomic publication to
|
|
762
926
|
an external directory. Local retention is indefinite by default:
|
|
763
927
|
|
|
@@ -801,254 +965,376 @@ Programmatic consumers can import `listRunBundles`, `loadRunBundleArtifact`,
|
|
|
801
965
|
|
|
802
966
|
## Authoring
|
|
803
967
|
|
|
804
|
-
###
|
|
968
|
+
### Named endpoints and databases
|
|
805
969
|
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
970
|
+
`@gfdelarue/testkit/environment` gives a test the URL and database of each
|
|
971
|
+
service its group uses, by the service's name:
|
|
972
|
+
|
|
973
|
+
```ts
|
|
974
|
+
import { database, endpoint } from "@gfdelarue/testkit/environment";
|
|
975
|
+
|
|
976
|
+
const response = await fetch(new URL("/health", endpoint("api")));
|
|
977
|
+
const reportsDatabase = database("reports");
|
|
978
|
+
```
|
|
979
|
+
|
|
980
|
+
Both functions return strings. A service's database is the one Testkit
|
|
981
|
+
provisions for it, or the `DATABASE_URL` its `env` declares. The module works
|
|
982
|
+
in Vitest and Playwright files and in the modules they import.
|
|
983
|
+
Testkit installs the values before any of that code loads, so module-level code
|
|
984
|
+
and native hooks can read them, such as a Playwright config and its
|
|
985
|
+
`globalSetup` or a Vitest `beforeAll`.
|
|
986
|
+
|
|
987
|
+
A name with no value throws `EnvironmentValueMissingError`, with `kind`,
|
|
988
|
+
`valueName`, and the `available` names. `valueName` is `null` when the missing
|
|
989
|
+
value is the run's default URL. Outside a Testkit run every call throws it.
|
|
990
|
+
|
|
991
|
+
A test that fails on this error, or a file whose imports throw it, ends as an
|
|
992
|
+
infrastructure failure in Vitest and Playwright. The run fails, and
|
|
993
|
+
`testkit bug file` refuses to file it. A test that catches the error and
|
|
994
|
+
carries on is unaffected. When the run gives a test no base URL, a Playwright
|
|
995
|
+
`page.goto()` to a relative URL ends the same way.
|
|
996
|
+
|
|
997
|
+
`BASE_URL` and `DATABASE_URL` hold the values of the first service
|
|
998
|
+
in `uses` that has each one. Playwright gives `BASE_URL` and `baseURL` in
|
|
999
|
+
canonical form, so an origin ends with `/`. `endpoint()` returns the URL
|
|
1000
|
+
unchanged in every engine.
|
|
1001
|
+
|
|
1002
|
+
The group's default endpoint and database are also available by the group's
|
|
1003
|
+
name, whether the run starts its services or takes supplied values. For a group
|
|
1004
|
+
`checks` that uses `api`, `endpoint("checks")` and `endpoint("api")` return the
|
|
1005
|
+
same URL. A group whose default differs from a used service's value with the
|
|
1006
|
+
group's name, such as a group `api` with `uses: ["web", "api"]`, gets no
|
|
1007
|
+
values: its files end as an infrastructure failure that names the group, and
|
|
1008
|
+
other groups still run.
|
|
1009
|
+
|
|
1010
|
+
### Handles
|
|
1011
|
+
|
|
1012
|
+
`handle(name)` from `@gfdelarue/testkit/environment` gives a test an object for
|
|
1013
|
+
something its environment prepared. Today that is an HTTP capture, which a
|
|
1014
|
+
[setup module](#node-setup-beside-a-test) starts with `sidecars.httpCapture()`.
|
|
1015
|
+
The handle is named after the capture's `id`:
|
|
809
1016
|
|
|
810
1017
|
```js
|
|
811
|
-
// tests/webhook.
|
|
1018
|
+
// tests/webhook.test.setup.mjs
|
|
812
1019
|
export async function setup({ sidecars }) {
|
|
813
|
-
|
|
814
|
-
return {
|
|
815
|
-
data: { webhookUrl: capture.url, requestsUrl: capture.requestsUrl },
|
|
816
|
-
};
|
|
1020
|
+
await sidecars.httpCapture({ id: "webhook", path: "/webhook" });
|
|
817
1021
|
}
|
|
818
1022
|
```
|
|
819
1023
|
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
`serviceName`, `testFile`, and `sidecars`. The worker shares the host process's
|
|
824
|
-
permissions. It is not a sandbox for untrusted code.
|
|
825
|
-
|
|
826
|
-
Return plain JSON, or `{ data, cleanup }` with an optional cleanup function.
|
|
827
|
-
Testkit passes the data to the engine as `setupData` and records it with the
|
|
828
|
-
setup module path and sidecar handles. Bare `null` or `undefined`, and a
|
|
829
|
-
cleanup-only result, give empty data. `{ data: null }` preserves `null`.
|
|
830
|
-
Data must encode to at most 1 MiB of JSON. Invalid data reports its JSON path,
|
|
831
|
-
such as `$.items[1]`, without invoking property getters.
|
|
832
|
-
|
|
833
|
-
Setup shares the task's absolute deadline. On failure or cancellation, Testkit
|
|
834
|
-
runs registered cleanup and closes its sidecars. Once setup succeeds, those
|
|
835
|
-
resources remain available until task cleanup. Cleanup callbacks run in reverse
|
|
836
|
-
registration order. The runner uses the task timeout for cleanup, capped at
|
|
837
|
-
60 seconds. A hung worker is terminated so it cannot keep its listeners open.
|
|
838
|
-
|
|
839
|
-
Thrown setup errors, deadline expiry, and invalid data fail the task with the
|
|
840
|
-
setup module path. Consumer cleanup errors fail the task; Testkit cleanup or
|
|
841
|
-
worker failures count as infrastructure failures. Cleanup remains idempotent.
|
|
842
|
-
HTTP capture retains at most 100 requests by default, configurable up to 1,000.
|
|
843
|
-
Each body defaults to 1 MiB, configurable up to 8 MiB, and retained bodies total
|
|
844
|
-
at most 8 MiB per listener. A setup can create up to 16 listeners, each with at
|
|
845
|
-
most 256 connections.
|
|
1024
|
+
```ts
|
|
1025
|
+
// tests/webhook.test.ts
|
|
1026
|
+
import { handle } from "@gfdelarue/testkit/environment";
|
|
846
1027
|
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
1028
|
+
const webhook = handle("webhook");
|
|
1029
|
+
await placeOrder({ callbackUrl: webhook.url });
|
|
1030
|
+
const [delivery] = await webhook.requests();
|
|
1031
|
+
await webhook.reset();
|
|
1032
|
+
```
|
|
850
1033
|
|
|
851
|
-
|
|
1034
|
+
A capture handle has `kind` (`"http-capture"`), `url` (where the product sends
|
|
1035
|
+
its requests), `requests()`, which returns the captured requests oldest first,
|
|
1036
|
+
and `reset()`, which forgets them. Both methods return a Promise in
|
|
1037
|
+
Vitest and Playwright. Each call fails after 10 seconds if the capture hasn't
|
|
1038
|
+
answered in full.
|
|
1039
|
+
|
|
1040
|
+
A handle is a reference, not a connection. It can list or reset its own capture
|
|
1041
|
+
and nothing else. The capture's operation URLs hold an unguessable segment, so
|
|
1042
|
+
holding the handle is the authority. Testkit keeps that segment out of the
|
|
1043
|
+
run's results, artifacts, and events, and the capture refuses any other
|
|
1044
|
+
operation. When Testkit releases the file's environment, the capture closes and
|
|
1045
|
+
the handle expires. Calling a handle never starts a sidecar or a service.
|
|
1046
|
+
|
|
1047
|
+
- A name with no handle throws `EnvironmentValueMissingError` with `kind`
|
|
1048
|
+
`"handle"`, and the test ends as an infrastructure failure, as for a missing
|
|
1049
|
+
endpoint.
|
|
1050
|
+
- Every capture created without an `id` gets the id `http-capture`. If two do,
|
|
1051
|
+
`handle("http-capture")` throws `EnvironmentHandleAmbiguousError` with the
|
|
1052
|
+
`count`. Give each capture a distinct `id`.
|
|
1053
|
+
- A call that the capture refuses, such as one from a stale or forged
|
|
1054
|
+
reference, throws `EnvironmentHandleError` with `reason` `"refused"`. A call
|
|
1055
|
+
whose capture no longer answers throws it with `"unavailable"`, and a
|
|
1056
|
+
descriptor this Testkit cannot use throws it with `"invalid"` or
|
|
1057
|
+
`"unsupported"`. These end the test as ordinary failures.
|
|
1058
|
+
|
|
1059
|
+
### Give each test its own data
|
|
1060
|
+
|
|
1061
|
+
Testkit runs test files at the same time. Files on one runtime share its
|
|
1062
|
+
services and its database. Write each test so that it passes beside any other
|
|
1063
|
+
test, in any order.
|
|
1064
|
+
|
|
1065
|
+
- Create the rows the test needs, and filter every query and update to them.
|
|
1066
|
+
Don't depend on rows that another test creates, and don't count, update, or
|
|
1067
|
+
delete across a whole table. Every test sees the rows from `template.seed`,
|
|
1068
|
+
so read them and leave them unchanged.
|
|
1069
|
+
- When the test ends, leave its rows in place. Testkit resets the databases it
|
|
1070
|
+
provisions. By default, every run starts from a fresh clone of the template
|
|
1071
|
+
database. See [`database.reset`](testkit-config-schema.md#database).
|
|
1072
|
+
- Clean up only what Testkit doesn't own, such as an account in a third-party
|
|
1073
|
+
sandbox. Register the cleanup as soon as the test creates the account. Give
|
|
1074
|
+
the account a unique name, so that a leftover from a cleanup that never ran
|
|
1075
|
+
can't collide with the next run.
|
|
852
1076
|
|
|
853
1077
|
```ts
|
|
854
|
-
import {
|
|
855
|
-
import {
|
|
1078
|
+
import { randomUUID } from "node:crypto";
|
|
1079
|
+
import { endpoint } from "@gfdelarue/testkit/environment";
|
|
1080
|
+
import { expect, onTestFinished, test } from "vitest";
|
|
856
1081
|
|
|
857
|
-
|
|
1082
|
+
import { sandbox } from "../payments-sandbox.js";
|
|
858
1083
|
|
|
859
|
-
|
|
860
|
-
const
|
|
861
|
-
|
|
862
|
-
});
|
|
1084
|
+
test("a refund reaches the sandbox customer", async () => {
|
|
1085
|
+
const customer = await sandbox.createCustomer({ email: `refund-${randomUUID()}@example.test` });
|
|
1086
|
+
onTestFinished(() => sandbox.deleteCustomer(customer.id));
|
|
863
1087
|
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
1088
|
+
const response = await fetch(new URL("/api/refunds", endpoint("api")), {
|
|
1089
|
+
method: "POST",
|
|
1090
|
+
headers: { "content-type": "application/json" },
|
|
1091
|
+
body: JSON.stringify({ customerId: customer.id, amountCents: 500 }),
|
|
1092
|
+
});
|
|
1093
|
+
expect(response.status).toBe(201);
|
|
1094
|
+
});
|
|
867
1095
|
```
|
|
868
1096
|
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
1097
|
+
A [Node setup module](#node-setup-beside-a-test) returns its cleanup with its
|
|
1098
|
+
data. Testkit runs only the cleanup that `setup` returns. If a step in `setup`
|
|
1099
|
+
can throw after `setup` creates something, undo the creation before rethrowing.
|
|
872
1100
|
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
1101
|
+
A test that has to count or change a whole table needs a database that other
|
|
1102
|
+
tests don't write to. Give the test a group of its own, and have the group use
|
|
1103
|
+
a service with its own database. Below, the `ledger` group owns the test files
|
|
1104
|
+
under `src/ledger`, and its tests get the database of `ledger-db`, built from
|
|
1105
|
+
the same schema:
|
|
876
1106
|
|
|
877
1107
|
```ts
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
1108
|
+
services: {
|
|
1109
|
+
api: app.node({
|
|
1110
|
+
// ...
|
|
1111
|
+
}),
|
|
1112
|
+
"ledger-db": database.fixture({
|
|
1113
|
+
template: { migrate: [{ kind: "sql-file", path: "db/schema.sql" }] },
|
|
1114
|
+
}),
|
|
1115
|
+
},
|
|
1116
|
+
groups: {
|
|
1117
|
+
api: { roots: ["src"], uses: ["api"] },
|
|
1118
|
+
ledger: { roots: ["src/ledger"], uses: ["ledger-db"] },
|
|
1119
|
+
},
|
|
1120
|
+
```
|
|
884
1121
|
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
({ actor, actors, req }) => {
|
|
891
|
-
req.get("/api/auth/session");
|
|
892
|
-
req("GET", "/api/auth/session");
|
|
893
|
-
actor?.req.get("/api/auth/session");
|
|
894
|
-
req.as("outsider").get("/api/auth/session");
|
|
895
|
-
actors.get("reviewer").rawReq.get("/api/auth/session");
|
|
896
|
-
},
|
|
897
|
-
);
|
|
1122
|
+
`locks` is an escape hatch for a resource that can't be duplicated, such as a
|
|
1123
|
+
payment provider's sandbox account with one webhook URL that some tests change.
|
|
1124
|
+
Files that name the same lock never run at the same time within one run.
|
|
1125
|
+
Testkit doesn't share locks between runs, such as two CI jobs. Two tests that
|
|
1126
|
+
need a lock because they share rows need their own rows instead.
|
|
898
1127
|
|
|
899
|
-
|
|
900
|
-
export const
|
|
901
|
-
export default suite.exec;
|
|
1128
|
+
```ts
|
|
1129
|
+
export const testkit = defineFile({ engine: "node-vitest", locks: ["payments-sandbox-webhook"] });
|
|
902
1130
|
```
|
|
903
1131
|
|
|
904
|
-
|
|
905
|
-
// __testkit__/helpers/session.ts
|
|
906
|
-
import type { AuthAdapter, AuthSessionBundle } from "@gfdelarue/testkit/k6";
|
|
1132
|
+
### Node setup beside a test
|
|
907
1133
|
|
|
908
|
-
|
|
1134
|
+
A test such as `tests/webhook.test.ts` can have a sibling
|
|
1135
|
+
`tests/webhook.test.setup.mjs`. Testkit runs its named `setup` export, or its
|
|
1136
|
+
default export, before starting the test engine:
|
|
1137
|
+
|
|
1138
|
+
```js
|
|
1139
|
+
// tests/webhook.test.setup.mjs
|
|
1140
|
+
export async function setup({ sidecars }) {
|
|
1141
|
+
const capture = await sidecars.httpCapture({ path: "/webhook" });
|
|
909
1142
|
return {
|
|
910
|
-
|
|
911
|
-
const actors = Object.fromEntries(
|
|
912
|
-
actorNames.map((actorName, actorIndex) => [
|
|
913
|
-
actorName,
|
|
914
|
-
{ actorIndex, actorName, session: login(http, env.BASE, actorName) },
|
|
915
|
-
]),
|
|
916
|
-
);
|
|
917
|
-
return { actors, primaryActor: actorNames[0] };
|
|
918
|
-
},
|
|
919
|
-
headers(bundle, { actor }) {
|
|
920
|
-
const session = bundle?.actors[actor ?? bundle.primaryActor ?? ""]?.session;
|
|
921
|
-
return session ? { Authorization: `Bearer ${session.jwt}` } : {};
|
|
922
|
-
},
|
|
1143
|
+
data: { webhookUrl: capture.url, requestsUrl: capture.requestsUrl },
|
|
923
1144
|
};
|
|
924
1145
|
}
|
|
925
1146
|
```
|
|
926
1147
|
|
|
927
|
-
|
|
928
|
-
the
|
|
929
|
-
|
|
930
|
-
|
|
1148
|
+
Setup is part of preparing the file's environment. Testkit runs it once, after
|
|
1149
|
+
it acquires the services the file's group uses and before the test engine
|
|
1150
|
+
starts. Testkit never batches a Playwright file that has setup with other
|
|
1151
|
+
Playwright files, and a plain Vitest file that has setup runs in its own Vitest
|
|
1152
|
+
invocation. Files without setup still batch.
|
|
1153
|
+
|
|
1154
|
+
Setup runs with its own JavaScript state. It can use Node APIs, prepare
|
|
1155
|
+
external resources, and start HTTP capture listeners. Tests
|
|
1156
|
+
reach a capture by its `id` through [`handle()`](#handles). A capture's
|
|
1157
|
+
`requestsUrl` and `resetUrl` hold an unguessable segment that Testkit keeps out
|
|
1158
|
+
of the run's records, and its capture `path` may not start with `/__testkit__/`.
|
|
1159
|
+
The context contains `deadlineMs`, `signal`, `env`, `leaseDir`, `productDir`,
|
|
1160
|
+
`serviceName`, `testFile`, and `sidecars`. `serviceName` holds the name of the
|
|
1161
|
+
test's group. Setup has the same permissions as the Testkit process. It is not
|
|
1162
|
+
a sandbox for untrusted code.
|
|
931
1163
|
|
|
932
|
-
|
|
1164
|
+
A run on [supplied values](#run-against-a-supplied-environment) does not run
|
|
1165
|
+
setup modules. Its tests receive the environment file's `setupData` instead.
|
|
933
1166
|
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
return fixtureScope.seed("widget", "primary", { name: "Primary Widget" }, () => {
|
|
943
|
-
const widgetId = fixtureScope.id("widget");
|
|
944
|
-
db.exec(`
|
|
945
|
-
INSERT INTO widgets (id, name)
|
|
946
|
-
VALUES ('${widgetId}', 'Primary Widget')
|
|
947
|
-
`);
|
|
948
|
-
return { widgetId };
|
|
949
|
-
});
|
|
950
|
-
},
|
|
951
|
-
}));
|
|
952
|
-
|
|
953
|
-
const suite = defineDalSuite({ fixtures }, ({ db, fixtureScope, fixtures }) => {
|
|
954
|
-
const widget = fixtures.widget();
|
|
955
|
-
db.query(`select id from widgets where id = '${widget.widgetId}'`);
|
|
956
|
-
fixtureScope.records();
|
|
957
|
-
});
|
|
1167
|
+
Return plain JSON, or `{ data, cleanup }` with an optional cleanup function.
|
|
1168
|
+
Testkit passes the data to the engine as `setupData` and records it with the
|
|
1169
|
+
setup module path and sidecar handles. Tests read it with `setupData()` from
|
|
1170
|
+
`@gfdelarue/testkit/environment`, which returns `{}` when the file has no setup
|
|
1171
|
+
module and throws outside a Testkit run. Bare `null` or `undefined`, and a
|
|
1172
|
+
cleanup-only result, give empty data. `{ data: null }` preserves `null`.
|
|
1173
|
+
Data must encode to at most 1 MiB of JSON. Invalid data reports its JSON path,
|
|
1174
|
+
such as `$.items[1]`.
|
|
958
1175
|
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
1176
|
+
Setup shares the task's absolute deadline. On failure or cancellation, Testkit
|
|
1177
|
+
runs registered cleanup and closes its sidecars. Once setup succeeds, those
|
|
1178
|
+
resources remain available until the test and its native hooks finish and
|
|
1179
|
+
Testkit releases the file's environment. Cleanup callbacks run in reverse
|
|
1180
|
+
registration order. The runner uses the task timeout for cleanup, capped at
|
|
1181
|
+
60 seconds. Testkit stops a setup that hangs past that limit, which closes its
|
|
1182
|
+
listeners.
|
|
963
1183
|
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
1184
|
+
Thrown setup errors, deadline expiry, and invalid data fail the task with the
|
|
1185
|
+
setup module path. Consumer cleanup errors fail the task; a failure in
|
|
1186
|
+
Testkit's own cleanup, or a setup that stops unexpectedly, counts as an
|
|
1187
|
+
infrastructure failure. Cleanup remains idempotent.
|
|
1188
|
+
HTTP capture retains at most 100 requests by default, configurable up to 1,000.
|
|
1189
|
+
Each body defaults to 1 MiB, configurable up to 8 MiB, and retained bodies total
|
|
1190
|
+
at most 8 MiB per listener. A setup can create up to 16 listeners, each with at
|
|
1191
|
+
most 256 connections.
|
|
971
1192
|
|
|
972
|
-
|
|
973
|
-
|
|
1193
|
+
Discovery tries sibling `.setup.mjs`, sibling `.setup.js`, then the matching
|
|
1194
|
+
`.mjs` and `.js` files in `__setup__`, in that order. For the example above,
|
|
1195
|
+
the fallback is `tests/__setup__/webhook.test.mjs`.
|
|
974
1196
|
|
|
975
|
-
|
|
976
|
-
- `fixtureScope.slug(label)`
|
|
977
|
-
- `fixtureScope.email(label)`
|
|
978
|
-
- `fixtureScope.string(label, options)`
|
|
979
|
-
- `fixtureScope.token(label)`
|
|
980
|
-
- `fixtureScope.seed(kind, key, signature, create)`
|
|
981
|
-
- `fixtureScope.records()`
|
|
1197
|
+
### HTTP and persistence checks
|
|
982
1198
|
|
|
983
|
-
|
|
1199
|
+
Use plain Vitest files for ordinary HTTP, service, and database checks. Read
|
|
1200
|
+
configured URLs with `endpoint(name)` and `database(name)` from
|
|
1201
|
+
`@gfdelarue/testkit/environment`, then use `fetch` or your application's
|
|
1202
|
+
database client.
|
|
984
1203
|
|
|
985
|
-
|
|
986
|
-
- identical reseeds reuse the same seeded value
|
|
987
|
-
- conflicting reseeds fail immediately
|
|
988
|
-
- dependency cycles fail immediately
|
|
989
|
-
- seeded fixture records are persisted as a `testkit.dal-fixtures` artifact
|
|
1204
|
+
### Record data-flow evidence
|
|
990
1205
|
|
|
991
|
-
|
|
1206
|
+
`dataFlow(context)` from `@gfdelarue/testkit/node-vitest/data` records what a
|
|
1207
|
+
plain Vitest test sees data do: metrics, counts, bytes, durations, money,
|
|
1208
|
+
ratios, sequences, records, actors, systems, transitions, events, findings,
|
|
1209
|
+
external calls, and cache, LLM, and RPC usage. Budgets check the recorded
|
|
1210
|
+
values, for example that a metric stays under a limit or that a sequence was
|
|
1211
|
+
recorded.
|
|
992
1212
|
|
|
993
1213
|
```ts
|
|
994
|
-
import {
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
const upload = req.multipart.post("/api/v1/uploads", {
|
|
1008
|
-
fields: { name: "fixture" },
|
|
1009
|
-
files: [
|
|
1010
|
-
{
|
|
1011
|
-
field: "file",
|
|
1012
|
-
data: "hello",
|
|
1013
|
-
filename: "fixture.txt",
|
|
1014
|
-
contentType: "text/plain",
|
|
1015
|
-
},
|
|
1016
|
-
],
|
|
1214
|
+
import { endpoint } from "@gfdelarue/testkit/environment";
|
|
1215
|
+
import { dataFlow } from "@gfdelarue/testkit/node-vitest/data";
|
|
1216
|
+
import { test } from "vitest";
|
|
1217
|
+
|
|
1218
|
+
test("lists one page of orders", async (context) => {
|
|
1219
|
+
const flow = dataFlow(context);
|
|
1220
|
+
const response = await fetch(`${endpoint("api")}/orders?page=1`);
|
|
1221
|
+
const body = await response.text();
|
|
1222
|
+
flow.recordHttpRequest({
|
|
1223
|
+
method: "GET",
|
|
1224
|
+
path: "/orders",
|
|
1225
|
+
status: response.status,
|
|
1226
|
+
responseBytes: Buffer.byteLength(body),
|
|
1017
1227
|
});
|
|
1018
|
-
|
|
1019
|
-
expect.status(upload, 201, "upload is accepted");
|
|
1020
|
-
parse.safeJson(upload);
|
|
1228
|
+
flow.expectMetric("http.response.bytes").atMost(64_000);
|
|
1021
1229
|
});
|
|
1022
1230
|
```
|
|
1023
1231
|
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
`waitFor()` consumes the file budget configured by `execution.fileTimeoutSeconds`.
|
|
1037
|
-
Consumers should not set local timeout values in test files.
|
|
1038
|
-
|
|
1039
|
-
```ts
|
|
1040
|
-
import { parse, waitFor } from "@gfdelarue/testkit/k6";
|
|
1041
|
-
|
|
1042
|
-
const response = waitFor(
|
|
1043
|
-
() => req.get("/api/v1/jobs/123"),
|
|
1044
|
-
(res) => parse.json(res).data?.status === "completed",
|
|
1045
|
-
{ description: "job 123 to complete" },
|
|
1046
|
-
);
|
|
1047
|
-
```
|
|
1232
|
+
- `dataFlow` needs the context of a running test. Every call in one attempt
|
|
1233
|
+
returns the same recorder, and a call without that context throws.
|
|
1234
|
+
- The library reads no URL, environment value, fixture file, or Testkit config.
|
|
1235
|
+
Read named URLs from `@gfdelarue/testkit/environment`, log in from test code,
|
|
1236
|
+
and read files with `fs`.
|
|
1237
|
+
- A failed budget fails its test when the test finishes. The failure detail has
|
|
1238
|
+
the kind `data-flow-budget` and a key made of the test's full title and the
|
|
1239
|
+
budget.
|
|
1240
|
+
- Testkit writes `data-flow.json` and `data-flow.md` for a file whose tests
|
|
1241
|
+
recorded something. Each such test is one scenario, from its final attempt. A
|
|
1242
|
+
test that records nothing adds nothing, and a file whose tests record
|
|
1243
|
+
nothing gets no data-flow artifacts.
|
|
1048
1244
|
|
|
1049
1245
|
## Discovery
|
|
1050
1246
|
|
|
1051
|
-
|
|
1247
|
+
Testkit discovers plain Vitest test files and typed suites.
|
|
1248
|
+
|
|
1249
|
+
### Plain Vitest test files
|
|
1250
|
+
|
|
1251
|
+
Testkit runs plain Vitest test files with the repository's own Vitest and its
|
|
1252
|
+
config, as `vitest run` does from the product directory. It loads the root
|
|
1253
|
+
`vitest.config.*` or `vite.config.*` the way Vitest does and lists the files
|
|
1254
|
+
that config's projects select, without running global setup. Nested configs and
|
|
1255
|
+
package scripts have no effect. Without a config, Vitest's defaults select files
|
|
1256
|
+
that match `**/*.{test,spec}.?(c|m)[jt]s?(x)`, outside `node_modules` and
|
|
1257
|
+
`.git`.
|
|
1258
|
+
|
|
1259
|
+
`discover`, `run`, `typecheck`, `bug record`, and `bug file` take three flags
|
|
1260
|
+
that change the selection:
|
|
1261
|
+
|
|
1262
|
+
- `--vitest-config <path>` uses that config, relative to the product
|
|
1263
|
+
directory. A missing file, or one outside the product directory, fails before
|
|
1264
|
+
any test runs.
|
|
1265
|
+
- `--vitest-project <name>` runs only that project. An unknown name fails and
|
|
1266
|
+
lists the projects.
|
|
1267
|
+
- `--vitest-runner managed` runs plain files with Testkit's own Vitest and its
|
|
1268
|
+
default rules, without the repository's config.
|
|
1269
|
+
|
|
1270
|
+
Testkit supports Vitest 4.x. Another major, a broken installation, a root
|
|
1271
|
+
Vitest config that fails to load, or a Vitest that `package.json` declares but
|
|
1272
|
+
that is not installed fails before any test runs, naming the version and path.
|
|
1273
|
+
`--vitest-runner managed` avoids the repository's Vitest. A run that selects
|
|
1274
|
+
only typed files, with `--type ui` or with typed `--file` paths, `bug record` or
|
|
1275
|
+
`bug file` for a typed test, and the commands that select no tests (`status`,
|
|
1276
|
+
`run status`, `doctor`, `destroy`, `cleanup`, and `db schema`) don't load the
|
|
1277
|
+
repository's Vitest, so they still work. A product that neither installs nor
|
|
1278
|
+
declares Vitest has no plain test files.
|
|
1279
|
+
|
|
1280
|
+
Plain files run in one Vitest invocation per environment their groups give them.
|
|
1281
|
+
Groups that give their files the same values share one, such as `root` and the
|
|
1282
|
+
other groups that use no services. A group that uses a service Testkit starts
|
|
1283
|
+
gets an invocation of its own.
|
|
1284
|
+
Inside each invocation, root global setup, teardown, and the repository's
|
|
1285
|
+
reporters run as they do under `vitest run`, and Testkit's reporter runs beside
|
|
1286
|
+
them, so a run calls them once for each invocation. An invocation's deadline is
|
|
1287
|
+
the sum of its files' timeouts; Vitest's own test and hook timeouts apply inside
|
|
1288
|
+
it. A file that two projects select runs in both, its result combines both
|
|
1289
|
+
runs, and its evidence names each run's project. A file with a setup module,
|
|
1290
|
+
locks, or isolation other than `shared` runs in its own Vitest invocation with
|
|
1291
|
+
its own environment. Invocations run one after another unless
|
|
1292
|
+
`--task-concurrency` or `execution.taskConcurrency` is above 1. To run one group's
|
|
1293
|
+
files alone, select it with `--group`.
|
|
1294
|
+
|
|
1295
|
+
Plain files see the host environment Testkit started with: `HOME`, `PATH`,
|
|
1296
|
+
temporary and XDG directories, locale, `CI`, `FORCE_COLOR`, `NO_COLOR`, and any
|
|
1297
|
+
other variable reach config loading, setup files, and the tests as they do
|
|
1298
|
+
under `vitest run`. Unlike `vitest run`, ambient `BASE_URL` and `DATABASE_URL`
|
|
1299
|
+
never reach them; those come only from the group's environment, from
|
|
1300
|
+
provisioned services or supplied values. Testkit loads no env files for plain
|
|
1301
|
+
files, and Vitest's own env handling and `test.env` work as usual. Testkit puts
|
|
1302
|
+
its own `--import` in front of the shell's `NODE_OPTIONS`. Preloads in it see the
|
|
1303
|
+
shell's environment and the group's values; as under `vitest run`, Vitest sets
|
|
1304
|
+
`TEST`, `VITEST`, and `NODE_ENV` after them. `--vitest-runner managed` leaves the
|
|
1305
|
+
shell's `NODE_OPTIONS` out.
|
|
1306
|
+
|
|
1307
|
+
When Vitest detects a coding agent from the environment, such as `CLAUDECODE`,
|
|
1308
|
+
`AI_AGENT`, or `CURSOR_AGENT`, and the repository's Vitest config names no
|
|
1309
|
+
reporter, Vitest uses its agent reporter. That reporter prints no console output
|
|
1310
|
+
from a passing test, so the output is missing from the file's
|
|
1311
|
+
`node-vitest-stdout` artifact. A failing test's output still appears. Name a
|
|
1312
|
+
reporter in the Vitest config, such as `reporters: ["default"]`, to keep it.
|
|
1313
|
+
|
|
1314
|
+
Two settings narrow the files Vitest selects: `discovery.exclude` in `testkit.config.ts`, and
|
|
1315
|
+
a declared group's `exclude`. Testkit's built-in exclusions for typed files,
|
|
1316
|
+
such as `dist`, `build`, and `coverage`, and `discovery.roots` do not apply to
|
|
1317
|
+
plain files.
|
|
1318
|
+
|
|
1319
|
+
A plain file belongs to the group whose roots contain it, or to the implicit
|
|
1320
|
+
`root` group, which uses no service. Testkit starts no service for a plain
|
|
1321
|
+
file in `root`.
|
|
1322
|
+
|
|
1323
|
+
The plain files in one directory form a suite named after that directory,
|
|
1324
|
+
relative to the group's root. In `root`, `--suite src` selects
|
|
1325
|
+
`src/sum.test.ts` and `src/sum.spec.ts`. A file directly in a group's root names
|
|
1326
|
+
its own suite, so `sum.test.ts` in a group rooted at `src` is in suite `sum`.
|
|
1327
|
+
When two suites in a group would share a name, Testkit lengthens the later
|
|
1328
|
+
ones with parent directories, such as `api-src`, and numbers them if that is
|
|
1329
|
+
not enough. `testkit typecheck` checks plain TypeScript test files and skips
|
|
1330
|
+
plain JavaScript ones, as `tsc` does without `allowJs`. It adds the TypeScript
|
|
1331
|
+
files in each Vitest project's `setupFiles` to the program, so a test can use
|
|
1332
|
+
the matchers a setup file declares, such as those from
|
|
1333
|
+
`@testing-library/jest-dom/vitest`.
|
|
1334
|
+
|
|
1335
|
+
### Typed suites
|
|
1336
|
+
|
|
1337
|
+
`testkit` discovers typed suites from `__testkit__/` directories.
|
|
1052
1338
|
|
|
1053
1339
|
Place each `__testkit__/` directory as a peer of the thing being tested. The
|
|
1054
1340
|
target is the smallest owned directory or package surface that honestly contains
|
|
@@ -1077,67 +1363,66 @@ production owner when a narrower owner exists.
|
|
|
1077
1363
|
|
|
1078
1364
|
Concrete layouts:
|
|
1079
1365
|
|
|
1080
|
-
- `src/api/routes/__testkit__/auth-me.int.testkit.ts`
|
|
1081
|
-
- `src/db/__testkit__/sessions-count-type.dal.testkit.ts`
|
|
1082
1366
|
- `frontend/src/components/layout/__testkit__/navigation.ui.testkit.ts`
|
|
1083
|
-
- `src/internal/handler/repos/__testkit__/crud.int.testkit.ts`
|
|
1084
|
-
|
|
1085
|
-
`testkit` uses these suffixes automatically:
|
|
1086
1367
|
|
|
1087
|
-
|
|
1088
|
-
- `*.e2e.testkit.ts`
|
|
1089
|
-
- `*.contract.testkit.ts`
|
|
1090
|
-
- `*.data.testkit.ts`
|
|
1091
|
-
- `*.dal.testkit.ts`
|
|
1092
|
-
- `*.load.testkit.ts`
|
|
1093
|
-
- `*.ui.testkit.ts`
|
|
1368
|
+
`testkit` uses the `*.ui.testkit.ts` suffix automatically.
|
|
1094
1369
|
|
|
1095
|
-
The suffix never chooses an engine. Each discovered file must also
|
|
1096
|
-
root-package `defineFile(...)` declaration with a supported literal
|
|
1370
|
+
The suffix never chooses an engine. Each discovered typed file must also
|
|
1371
|
+
contain one root-package `defineFile(...)` declaration with a supported literal
|
|
1372
|
+
engine.
|
|
1097
1373
|
Unsupported pairs fail discovery; for example, a `*.ui.testkit.ts` file cannot
|
|
1098
|
-
declare `engine: "
|
|
1374
|
+
declare `engine: "node-vitest"`.
|
|
1099
1375
|
|
|
1100
1376
|
See [docs/test-types.md](docs/test-types.md) for the canonical type model.
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1377
|
+
|
|
1378
|
+
Test groups own test files by directory. Declare them under `groups` in
|
|
1379
|
+
`testkit.config.ts`. Each group lists its repo-relative `roots` and the services
|
|
1380
|
+
its tests `use`:
|
|
1381
|
+
|
|
1382
|
+
```ts
|
|
1383
|
+
export default defineConfig({
|
|
1384
|
+
services: {
|
|
1385
|
+
api: app.node({ cwd: "services/api", start: "node dist/server.js" }),
|
|
1386
|
+
web: app.next({ cwd: "apps/web", dependsOn: ["api"] }),
|
|
1387
|
+
},
|
|
1388
|
+
groups: {
|
|
1389
|
+
api: { roots: ["services/api"], uses: ["api"] },
|
|
1390
|
+
web: { roots: ["apps/web"], uses: ["web", "api"] },
|
|
1391
|
+
},
|
|
1392
|
+
});
|
|
1393
|
+
```
|
|
1394
|
+
|
|
1395
|
+
- A file belongs to the group with the deepest root that contains it. When two
|
|
1396
|
+
groups have roots at the same depth, discovery reports `ambiguous_test`.
|
|
1397
|
+
- Files that no group's roots claim belong to the implicit group `root`, which
|
|
1398
|
+
uses no service. With no `testkit.config.ts`, every file belongs to `root`.
|
|
1399
|
+
- A run starts the services that the selected files' groups use, with their
|
|
1400
|
+
`dependsOn`, and nothing else. A test gets `BASE_URL`, `DATABASE_URL`, and
|
|
1401
|
+
`env` values only from the services its group uses. When two of them set the
|
|
1402
|
+
same name, the first in `uses` wins. Each used service's URL and database
|
|
1403
|
+
are also available by name; see
|
|
1404
|
+
[Named endpoints and databases](#named-endpoints-and-databases).
|
|
1405
|
+
- `skip`, `requirements`, and `runtimeIsolation` are group fields. See the
|
|
1406
|
+
[config reference](testkit-config-schema.md#groups).
|
|
1407
|
+
|
|
1408
|
+
Services own no test files. A config with `services` and no `groups` key
|
|
1409
|
+
fails to load when a service's `local.cwd` or `discovery.roots` holds a typed
|
|
1410
|
+
test file, which that service owned before 0.6.0. The error lists the `groups`
|
|
1411
|
+
entries that keep those files with their services. `status`, `destroy`,
|
|
1412
|
+
`cleanup`, and `db schema` still run on it. Otherwise every file belongs to
|
|
1413
|
+
`root`.
|
|
1129
1414
|
|
|
1130
1415
|
Suite names are inferred from the colocated path:
|
|
1131
1416
|
|
|
1132
|
-
- `auth/__testkit__/*.
|
|
1133
|
-
- `routes/__testkit__/
|
|
1417
|
+
- `auth/__testkit__/*.ui.testkit.ts` => `auth`
|
|
1418
|
+
- `routes/__testkit__/checkout.ui.testkit.ts` => `routes`
|
|
1134
1419
|
|
|
1135
1420
|
Discovery is also a first-class CLI/API surface:
|
|
1136
1421
|
|
|
1137
1422
|
- `testkit discover`
|
|
1138
|
-
- human-first compact output with
|
|
1423
|
+
- human-first compact output with group -> type -> suite -> file hierarchy
|
|
1139
1424
|
- `testkit discover --output-mode verbose`
|
|
1140
|
-
- explicit paths, IDs, locks,
|
|
1425
|
+
- group roots and uses, explicit paths, IDs, locks, and skip reasons
|
|
1141
1426
|
- `testkit discover --json`
|
|
1142
1427
|
- machine-readable output with stable enums, canonical paths, and summary data
|
|
1143
1428
|
- `testkit discover --output .testkit/discovery.json`
|
|
@@ -1166,7 +1451,7 @@ identifier plus canonical metadata such as:
|
|
|
1166
1451
|
- `path`
|
|
1167
1452
|
- `service`
|
|
1168
1453
|
- `suiteName`
|
|
1169
|
-
- `testType
|
|
1454
|
+
- `testType`, absent for a plain Vitest test file
|
|
1170
1455
|
- `engineId`
|
|
1171
1456
|
- `profileId`
|
|
1172
1457
|
- `contract`
|
|
@@ -1177,8 +1462,10 @@ identifier plus canonical metadata such as:
|
|
|
1177
1462
|
- `displayName`
|
|
1178
1463
|
|
|
1179
1464
|
Discovery output has no cross-run counts, last outcomes, durations, or trends.
|
|
1180
|
-
|
|
1181
|
-
|
|
1465
|
+
Testkit 0.2.1 removed `testkit.status.json`, `.testkit/history/`, and
|
|
1466
|
+
discovery's `history`, `files[].history`, and `summary.latestResults` fields.
|
|
1467
|
+
To check one run, read its bundle under `.testkit/results/runs/<run-id>/`.
|
|
1468
|
+
Testkit ignores an old `.testkit/history/` directory and leaves it in place.
|
|
1182
1469
|
|
|
1183
1470
|
## Local Databases
|
|
1184
1471
|
|
|
@@ -1191,13 +1478,10 @@ client image instead of using host-installed binaries, which keeps local runs
|
|
|
1191
1478
|
consistent across Linux, macOS, and native Windows. Override the client image
|
|
1192
1479
|
with `TESTKIT_POSTGRES_CLIENT_IMAGE` when needed.
|
|
1193
1480
|
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
the same under Docker Desktop, rootless Docker, and Docker-in-Docker. Clients
|
|
1199
|
-
for external databases, such as `sourceSchema` URLs, still connect from the
|
|
1200
|
-
platform's host address.
|
|
1481
|
+
Template steps, schema snapshots, and `database.steps` helpers work the same
|
|
1482
|
+
under Docker Desktop, rootless Docker, and Docker-in-Docker. Clients for
|
|
1483
|
+
external databases, such as `sourceSchema` URLs, connect from the platform's
|
|
1484
|
+
host address, so those databases must be reachable from there.
|
|
1201
1485
|
|
|
1202
1486
|
- normal test runs always use transient host-managed Postgres
|
|
1203
1487
|
- template databases are cached
|
|
@@ -1213,7 +1497,8 @@ The service process receives its provisioned database as `DATABASE_URL`. Every
|
|
|
1213
1497
|
task gets the same `DATABASE_URL` when its service declares a database, and its
|
|
1214
1498
|
service's URL as `BASE_URL` when the service declares a server, whatever its
|
|
1215
1499
|
engine. The provisioned `DATABASE_URL` replaces one that the service's `env`
|
|
1216
|
-
declares.
|
|
1500
|
+
declares. A test whose group uses several services with databases reads each
|
|
1501
|
+
one with `database(name)` from `@gfdelarue/testkit/environment`. Declared `env` values can also use the `{dbUrl}`, `{dbHost}`,
|
|
1217
1502
|
`{dbPort}`, `{dbName}`, `{dbUser}`, `{dbPassword}`, and `{dbSsl}` tokens, with
|
|
1218
1503
|
`:serviceName` to read the database of a service listed in `dependsOn`. Testkit
|
|
1219
1504
|
resolves them from the database provisioned for the current run. They are
|
|
@@ -1224,59 +1509,3 @@ schema cache. `db schema verify` prepares local templates and verifies local
|
|
|
1224
1509
|
replay against the cached/refreshed source schema. `--skip-schema-source-verify`
|
|
1225
1510
|
is available as a narrow escape hatch when users need to run tests while schema
|
|
1226
1511
|
verification is temporarily blocked.
|
|
1227
|
-
|
|
1228
|
-
## Development Tests
|
|
1229
|
-
|
|
1230
|
-
```bash
|
|
1231
|
-
npm test
|
|
1232
|
-
npm run test:unit
|
|
1233
|
-
npm run test:integration
|
|
1234
|
-
npm run test:system
|
|
1235
|
-
npm run test:live
|
|
1236
|
-
npm run test:integration -- fixture-package
|
|
1237
|
-
```
|
|
1238
|
-
|
|
1239
|
-
The four lanes use Vitest directly. Unit tests (source and pure tooling) run in
|
|
1240
|
-
parallel; integration and system files run serially; live tests are explicitly
|
|
1241
|
-
opt-in and serial. Pass ordinary Vitest filters after npm's `--` separator.
|
|
1242
|
-
`npm test` runs the first three lanes and never collects live cases.
|
|
1243
|
-
|
|
1244
|
-
Live tests cover credentials, external services, network-acquired runtimes,
|
|
1245
|
-
alternate toolchains, and Docker/platform matrices. An explicitly selected live
|
|
1246
|
-
test fails clearly when its prerequisites are unavailable; it does not silently
|
|
1247
|
-
skip.
|
|
1248
|
-
|
|
1249
|
-
Repository commands use npm's native timing support, enabled project-wide by
|
|
1250
|
-
the tracked `.npmrc`. Build, lint, and release modes support `help`; release
|
|
1251
|
-
provides `bump`, `check`, `consumer-lock`, and `plan` modes.
|
|
1252
|
-
A push to `master` publishes only when it raises `version` in `package.json`
|
|
1253
|
-
and the new version is above npm's latest. To release, run
|
|
1254
|
-
`npm run release -- bump <version>` and merge the pull request once its checks
|
|
1255
|
-
pass. The
|
|
1256
|
-
[publish workflow](https://github.com/gfdelarue/testkit/blob/master/docs/release-gates.md)
|
|
1257
|
-
publishes that exact version through npm trusted publishing and tags the
|
|
1258
|
-
merged commit. Any other merge publishes nothing.
|
|
1259
|
-
|
|
1260
|
-
The static audit retains package/lock/runtime policy, engine dependency/import
|
|
1261
|
-
intent, retired engine/product references, minimal repository traversal, the
|
|
1262
|
-
dependency direction between top-level `src/` modules, and the test execution
|
|
1263
|
-
boundary. An import outside the direction table in
|
|
1264
|
-
`scripts/lint/audits/import-direction.ts` fails lint. The table may allow
|
|
1265
|
-
directions that no import uses. Test execution code may import only test
|
|
1266
|
-
execution and shared kernel code, apart from a committed list of existing
|
|
1267
|
-
imports that only shrinks. The
|
|
1268
|
-
[dependency direction notes](https://github.com/gfdelarue/testkit/blob/master/docs/dependency-direction.md)
|
|
1269
|
-
describe both rules. Other implementation-shape audits stay out of lint.
|
|
1270
|
-
|
|
1271
|
-
JavaScript modules under `test/fixtures/**` describe fixture behavior and keep
|
|
1272
|
-
their fixture-owned paths. Outside fixtures, only `bin/testkit.mjs` and the
|
|
1273
|
-
documented raw Node process probe remain `.mjs`; the probe is a program, not a
|
|
1274
|
-
Vitest suite.
|
|
1275
|
-
|
|
1276
|
-
GitHub Actions run a narrow installed Node/Vitest consumer smoke on dedicated
|
|
1277
|
-
Orthrus Ubuntu x64 and Windows x64 self-hosted runners for pull requests from
|
|
1278
|
-
this repository and manual dispatches. Fork pull requests skip this workflow's
|
|
1279
|
-
jobs. Local `npm test`,
|
|
1280
|
-
`npm run test:unit`, `npm run test:integration`, and `npm run test:system` remain
|
|
1281
|
-
the full developer entrypoints. PostgreSQL and Docker compatibility are explicit
|
|
1282
|
-
opt-in dispatcher cases. The smoke workflow is not five-platform release proof.
|