@gfdelarue/testkit 0.5.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +590 -165
- package/bin/testkit-windows-owner-arm64.exe +0 -0
- package/bin/testkit-windows-owner-x64.exe +0 -0
- package/dist/app/bugs/batch.js +1 -0
- package/dist/app/bugs/ci-report.js +2 -2
- package/dist/app/bugs/ci-summary.js +3 -3
- package/dist/app/bugs/draft.js +2 -2
- package/dist/app/bugs/file.js +1 -1
- package/dist/app/bugs/guide.js +1 -1
- package/dist/app/bugs/record.js +2 -2
- package/dist/app/bugs/run-reading.js +1 -1
- package/dist/app/bugs/store.js +1 -1
- package/dist/app/campaigns/index.js +2 -2
- package/dist/app/campaigns/summary.js +1 -1
- package/dist/app/cleanup/index.js +1 -1
- package/dist/app/db-schema/prepare.js +1 -1
- package/dist/app/db-schema/refresh.js +1 -1
- package/dist/app/db-schema/verify.js +1 -1
- package/dist/app/destroy/index.js +1 -1
- package/dist/app/discover/index.js +2 -2
- package/dist/app/environments/config-driven.js +1 -1
- package/dist/app/environments/file-setup.js +1 -1
- package/dist/app/environments/routing.js +1 -1
- package/dist/app/environments/static-values.js +1 -1
- package/dist/app/environments/supplied.js +1 -1
- package/dist/app/inspect/index.js +1 -1
- package/dist/app/lint/native.js +1 -1
- package/dist/app/lint/options.js +1 -1
- package/dist/app/lint/shared.js +1 -1
- package/dist/app/run/engine-orchestration.js +1 -1
- package/dist/app/run/environment.js +1 -1
- package/dist/app/run/live-run.js +1 -1
- package/dist/app/run/orchestrator.js +1 -1
- package/dist/app/run/planning.js +1 -1
- package/dist/app/run/request.js +1 -1
- package/dist/app/run/results.js +1 -1
- package/dist/app/run/run-finalization.js +1 -1
- package/dist/app/run/run-finisher.js +1 -0
- package/dist/app/run/setup/runtime-node-setup-protocol.js +1 -1
- package/dist/app/run/setup/runtime-node-setup-worker.js +1 -1
- package/dist/app/run/setup/runtime-node-setup.js +2 -2
- package/dist/app/run/task-batches.js +1 -1
- package/dist/app/run/worker-loop.js +1 -1
- package/dist/app/status/model.js +1 -1
- package/dist/app/status/runs.js +1 -1
- package/dist/app/stop/index.js +1 -1
- package/dist/app/tracker/index.js +1 -1
- package/dist/app/tracker/token.js +2 -2
- package/dist/app/typecheck/index.js +4 -4
- package/dist/cli/command-catalog.js +1 -1
- package/dist/cli/command-flags.js +1 -1
- package/dist/cli/commands/artifacts.js +2 -2
- package/dist/cli/commands/bug/file.js +3 -2
- package/dist/cli/commands/discover.js +1 -1
- package/dist/cli/commands/guard/unsafe-entrypoint.js +1 -1
- package/dist/cli/commands/run.js +2 -2
- package/dist/cli/commands/typecheck.js +1 -1
- package/dist/cli/commands/workspace.js +1 -1
- package/dist/cli/framework/help.js +5 -5
- package/dist/cli/framework/parser.js +1 -1
- package/dist/cli/json-errors.js +1 -1
- package/dist/cli/renderers/bug/text.js +1 -1
- package/dist/cli/renderers/run/formatting.js +1 -1
- package/dist/config/authoring/database-fixture-discovery.js +1 -0
- package/dist/config/authoring/index.js +1 -1
- package/dist/config/loader/config-loader.js +1 -1
- package/dist/config/loader/decode-discovery.js +1 -1
- package/dist/config/loader/discover-tests.js +1 -1
- package/dist/config/loader/discovery-result.js +1 -0
- package/dist/config/loader/discovery.js +2 -2
- package/dist/config/loader/file-metadata.js +1 -1
- package/dist/config/loader/groups.js +2 -1
- package/dist/config/loader/index.js +1 -1
- package/dist/config/loader/load.js +1 -1
- package/dist/config/loader/managed-configs.js +1 -1
- package/dist/config/loader/owned-discovery.js +1 -1
- package/dist/config/loader/runtime-isolation.js +1 -1
- package/dist/config/loader/skip-config.js +1 -1
- package/dist/config/loader/test-selection.js +1 -1
- package/dist/database/admin.js +5 -2
- package/dist/database/call-caps.js +1 -0
- package/dist/database/cleanup.js +1 -1
- package/dist/database/configured-steps.js +1 -1
- package/dist/database/constants.js +1 -1
- package/dist/database/docker-endpoint.js +1 -0
- package/dist/database/docker-metadata.js +2 -2
- package/dist/database/docker.js +5 -5
- package/dist/database/index.js +2 -1
- package/dist/database/lifecycle-cleanup.js +1 -1
- package/dist/database/local-postgres.js +4 -1
- package/dist/database/naming.js +1 -1
- package/dist/database/operation-deadline.js +1 -0
- package/dist/database/postgres/connection-check.js +1 -1
- package/dist/database/postgres/index.js +2 -2
- package/dist/database/postgres/owned-exec.js +27 -0
- package/dist/database/postgres/protocol.js +2 -2
- package/dist/database/provider.js +1 -0
- package/dist/database/roles.js +2 -2
- package/dist/database/schema-operations.js +5 -1
- package/dist/database/schema-source.js +3 -3
- package/dist/database/steps.js +1 -1
- package/dist/database/template-steps.js +4 -4
- package/dist/domain/identifiers.js +1 -1
- package/dist/domain/retired-test-types.js +1 -1
- package/dist/domain/suite-selection.js +1 -1
- package/dist/domain/test-types.js +1 -1
- package/dist/engines/adapters/evidence-files.js +1 -1
- package/dist/engines/adapters/execution.js +1 -1
- package/dist/engines/catalog.js +1 -1
- package/dist/engines/definitions.js +1 -1
- package/dist/engines/environment/index.js +1 -1
- package/dist/engines/installations/node-vitest.js +1 -1
- package/dist/engines/installations/playwright.js +1 -1
- package/dist/engines/managed-module-resolution.js +6 -2
- package/dist/engines/node-vitest/adapter/adapter.js +1 -1
- package/dist/engines/node-vitest/adapter/driver.js +2 -2
- package/dist/engines/node-vitest/adapter/evidence.js +2 -2
- package/dist/engines/node-vitest/adapter/workspace.js +8 -24
- package/dist/engines/node-vitest/inventory/index.js +2 -2
- package/dist/engines/node-vitest/inventory/list-files.js +3 -3
- package/dist/engines/node-vitest/runtime/data/index.js +1 -1
- package/dist/engines/node-vitest/runtime/data/report.js +1 -1
- package/dist/engines/node-vitest/runtime/data/scenario.js +2 -2
- package/dist/engines/node-vitest/runtime/runtime-input.js +1 -1
- package/dist/engines/playwright/adapter/adapter.js +1 -1
- package/dist/engines/playwright/adapter/browser-network-evidence.js +4 -0
- package/dist/engines/playwright/adapter/config.js +16 -6
- package/dist/engines/playwright/adapter/evidence.js +2 -1
- package/dist/engines/playwright/adapter/failure-details.js +2 -2
- package/dist/engines/playwright/adapter/reporter.js +1 -1
- package/dist/engines/playwright/runtime/browser-network.js +1 -0
- package/dist/engines/playwright/runtime/browser-traffic.js +1 -0
- package/dist/engines/playwright/runtime/chromium-environment.js +1 -0
- package/dist/engines/playwright/runtime/emulation.js +1 -0
- package/dist/engines/playwright/runtime/index.js +1 -1
- package/dist/engines/playwright/runtime/managed-test.js +1 -0
- package/dist/engines/playwright/runtime/runtime-input.js +2 -2
- package/dist/environment/lifecycle-controller.js +1 -1
- package/dist/environment/lifecycle.js +1 -1
- package/dist/environment/maintenance.js +1 -1
- package/dist/environment/run-environment.js +1 -1
- package/dist/environment/run-stopping.js +1 -1
- package/dist/environment/runtime-manager.js +1 -1
- package/dist/evidence/artifact-config.js +1 -1
- package/dist/evidence/artifacts.js +5 -2
- package/dist/evidence/bundles.js +2 -1
- package/dist/evidence/compressed-artifacts.js +1 -0
- package/dist/evidence/environment-record.js +1 -1
- package/dist/evidence/index.js +1 -1
- package/dist/evidence/live-run-summary.js +1 -0
- package/dist/evidence/local-paths.js +1 -0
- package/dist/evidence/maintenance.js +1 -0
- package/dist/evidence/operation-timings.js +1 -1
- package/dist/evidence/read-run.js +1 -1
- package/dist/evidence/redaction.js +4 -4
- package/dist/evidence/run-evidence.js +1 -0
- package/dist/evidence/setup-progress.js +1 -1
- package/dist/evidence/storage-inventory.js +1 -1
- package/dist/evidence/storage.js +2 -2
- package/dist/kernel/known-keys.js +1 -0
- package/dist/kernel/redaction-targets.js +1 -1
- package/dist/kernel/redaction.js +7 -4
- package/dist/process/engine-environment-keys.js +1 -0
- package/dist/process/environment-overlay.js +1 -1
- package/dist/process/environment.js +1 -1
- package/dist/process/execa-bundle.json +3 -3
- package/dist/process/files/evidence-writes.js +1 -0
- package/dist/process/files/product-dir.js +1 -1
- package/dist/process/host-process.js +1 -1
- package/dist/process/index.js +1 -1
- package/dist/process/lifecycle-fence.js +1 -1
- package/dist/process/lock-progress.js +2 -0
- package/dist/process/locks.js +4 -4
- package/dist/process/process-identity.js +1 -1
- package/dist/process/process.js +35 -35
- package/dist/process/recovery.js +1 -1
- package/dist/process/runs/manifest.js +1 -1
- package/dist/process/runs/ownership.js +1 -1
- package/dist/process/runs/stop.js +1 -1
- package/dist/process/shutdown.js +1 -1
- package/dist/process/windows-native.js +1 -1
- package/dist/repo/fingerprint-policy.js +1 -1
- package/dist/repo/history.js +7 -4
- package/dist/repo/managed-git.js +1 -1
- package/dist/repo/state.js +4 -4
- package/dist/steps/configured-step-testkit-modules.js +1 -1
- package/dist/steps/environment.js +1 -1
- package/dist/toolchains/index.js +2 -2
- package/dist/toolchains/node-provisioning.js +2 -2
- package/dist/toolchains/vitest/index.js +1 -1
- package/dist/tracker/gitea.js +1 -1
- package/dist/tracker/github.js +1 -1
- package/dist/tracker/http.js +1 -1
- package/dist/tracker/ownership.js +1 -0
- package/dist/tracker/redaction.js +1 -1
- package/dist/tracker/responses.js +1 -1
- package/dist/types/config.d.ts +7 -86
- package/dist/types/environment.d.ts +6 -0
- package/dist/types/node-vitest-data.d.ts +2 -218
- package/dist/types/playwright.d.ts +21 -0
- package/dist/types/workspace.d.ts +30 -14
- package/docs/bug-records.md +19 -18
- package/docs/engines.md +115 -21
- package/docs/release-notes.md +2846 -0
- package/docs/test-types.md +19 -69
- package/package.json +15 -6
- package/testkit-config-schema.md +101 -170
- package/dist/app/lint/rules/data.js +0 -1
- package/dist/config/authoring/runtime.js +0 -1
- package/dist/config/loader/task-policy.js +0 -1
- package/dist/engines/node-vitest/runtime/contract/index.js +0 -1
- package/dist/engines/node-vitest/runtime/data/http.js +0 -1
- package/dist/engines/node-vitest/runtime/data/runner.js +0 -1
- package/dist/environment/lifecycle-artifacts.js +0 -1
- package/dist/types/node-vitest-contract.d.ts +0 -75
package/docs/test-types.md
CHANGED
|
@@ -11,12 +11,15 @@ so does any Node-side test that reads its environment from `process.env` or
|
|
|
11
11
|
[Plain Vitest test files](../README.md#plain-vitest-test-files) for which files
|
|
12
12
|
Testkit runs.
|
|
13
13
|
|
|
14
|
-
|
|
15
|
-
|
|
14
|
+
A plain Vitest test records data-flow evidence, such as metrics, sequences,
|
|
15
|
+
records, and budgets, with `dataFlow(context)` from
|
|
16
|
+
`@gfdelarue/testkit/node-vitest/data`. Recording evidence gives the test no
|
|
17
|
+
type and no setup. See
|
|
18
|
+
[Record data-flow evidence](../README.md#record-data-flow-evidence).
|
|
19
|
+
|
|
20
|
+
Typed files carry one product-level test type in their filename suffix:
|
|
16
21
|
|
|
17
22
|
- `ui`: browser-level UI checks for user-visible pages and interactions.
|
|
18
|
-
- `contract`: Node-backed service/runtime contract checks that need Node APIs but are not data-flow evidence suites.
|
|
19
|
-
- `data`: data-flow and data-contract checks for throughput, payload shape, conversation/request traces, and cost/latency budgets.
|
|
20
23
|
|
|
21
24
|
Consumers choose the type that describes the product behavior and declare the
|
|
22
25
|
execution engine in each file's `defineFile(...)` metadata. Testkit validates
|
|
@@ -24,12 +27,10 @@ that pair against its engine catalog and resolves the exact runner profile.
|
|
|
24
27
|
|
|
25
28
|
The current catalog is deliberately explicit:
|
|
26
29
|
|
|
27
|
-
| Test type | Engine | Runner profile
|
|
28
|
-
| ------------------ | ------------- |
|
|
29
|
-
| none, a plain file | `node-vitest` | `node-vitest`
|
|
30
|
-
| `
|
|
31
|
-
| `contract` | `node-vitest` | `node-vitest-contract` |
|
|
32
|
-
| `ui` | `playwright` | `playwright-ui` |
|
|
30
|
+
| Test type | Engine | Runner profile |
|
|
31
|
+
| ------------------ | ------------- | --------------- |
|
|
32
|
+
| none, a plain file | `node-vitest` | `node-vitest` |
|
|
33
|
+
| `ui` | `playwright` | `playwright-ui` |
|
|
33
34
|
|
|
34
35
|
Every typed file must import `defineFile` from the root package and export one
|
|
35
36
|
direct metadata declaration:
|
|
@@ -37,7 +38,7 @@ direct metadata declaration:
|
|
|
37
38
|
```ts
|
|
38
39
|
import { defineFile } from "@gfdelarue/testkit";
|
|
39
40
|
|
|
40
|
-
export const testkit = defineFile({ engine: "
|
|
41
|
+
export const testkit = defineFile({ engine: "playwright" });
|
|
41
42
|
```
|
|
42
43
|
|
|
43
44
|
The engine is not inferred from the suffix, imports, or repo configuration. A
|
|
@@ -46,9 +47,9 @@ error. The profile is derived from the validated pair and is carried unchanged
|
|
|
46
47
|
through scheduling, results, and immutable run evidence.
|
|
47
48
|
|
|
48
49
|
Run selection by `--file`, `--group`, `--type`, or `--suite` narrows the files
|
|
49
|
-
whose engine metadata is read and validated.
|
|
50
|
-
|
|
51
|
-
|
|
50
|
+
whose engine metadata is read and validated. `--type ui` and typed suite
|
|
51
|
+
selectors select typed files only, `--type plain` selects plain files only, and
|
|
52
|
+
`--type all` selects both. Testkit uses discovered paths, canonical type
|
|
52
53
|
suffixes, and suite names to select candidates; each selected typed file must
|
|
53
54
|
still declare a valid engine. Invalid metadata in an unselected file
|
|
54
55
|
does not block that focused run. Root configuration, service dependencies, and
|
|
@@ -56,11 +57,7 @@ group skip, requirement, and isolation selectors remain validated against
|
|
|
56
57
|
the complete discovered project. `discover`, `doctor`, and `typecheck` validate
|
|
57
58
|
metadata across the whole project.
|
|
58
59
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
- `*.ui.testkit.ts`
|
|
62
|
-
- `*.contract.testkit.ts`
|
|
63
|
-
- `*.data.testkit.ts`
|
|
60
|
+
The canonical filename suffix is `*.ui.testkit.ts`.
|
|
64
61
|
|
|
65
62
|
UI tests may import Playwright's public browser primitives directly from the
|
|
66
63
|
managed native package:
|
|
@@ -72,56 +69,9 @@ import { expect, test } from "@playwright/test";
|
|
|
72
69
|
export const testkit = defineFile({ engine: "playwright" });
|
|
73
70
|
```
|
|
74
71
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
import { defineFile } from "@gfdelarue/testkit";
|
|
79
|
-
import { dataScenario } from "@gfdelarue/testkit/node-vitest/data";
|
|
80
|
-
import { expect } from "vitest";
|
|
81
|
-
|
|
82
|
-
export const testkit = defineFile({ engine: "node-vitest" });
|
|
83
|
-
```
|
|
84
|
-
|
|
85
|
-
Contract suites import Node-backed contract primitives from Testkit:
|
|
86
|
-
|
|
87
|
-
```ts
|
|
88
|
-
import { defineFile } from "@gfdelarue/testkit";
|
|
89
|
-
import { contractScenario } from "@gfdelarue/testkit/node-vitest/contract";
|
|
90
|
-
import { expect } from "vitest";
|
|
91
|
-
|
|
92
|
-
export const testkit = defineFile({ engine: "node-vitest" });
|
|
93
|
-
```
|
|
94
|
-
|
|
95
|
-
Use `contract` when the behavior under test is a service/runtime boundary that
|
|
96
|
-
needs Node APIs, fixtures, file IO, or richer protocol checks, but does not need
|
|
97
|
-
Testkit data-flow evidence. Contract files are native Vitest suites:
|
|
98
|
-
`contractScenario(...)` provides the managed Testkit contract context, while
|
|
99
|
-
direct public runner APIs remain available for native behavior such as
|
|
100
|
-
`test.skip(...)` and `test.todo(...)`. Contract suites run as Node processes
|
|
101
|
-
managed by Testkit and should talk to configured service URLs.
|
|
102
|
-
|
|
103
|
-
Use `data` when the behavior under test is the shape, cost, volume, latency, or
|
|
104
|
-
movement of data through a real product flow. A data suite is not a generic
|
|
105
|
-
Vitest migration target: write `dataScenario(...)` cases that record Testkit
|
|
106
|
-
structured data-flow evidence with the provided data context, such as metrics,
|
|
107
|
-
budgets, traces, fixtures, HTTP payload measurements, cache behavior, LLM usage,
|
|
108
|
-
actors, systems, records, transitions, external calls, and reconciliation
|
|
109
|
-
evidence.
|
|
110
|
-
Data scenarios use Vitest directly for assertions, hooks, and mocking.
|
|
111
|
-
`dataScenario(...)` owns case registration and evidence: lint reports native
|
|
112
|
-
registration imports, and the runtime rejects cases registered any other way.
|
|
113
|
-
Weak bookkeeping such as a single `count(...)`, `event(...)`, or `finding(...)`
|
|
114
|
-
is useful as supporting detail, but it is not enough by itself to make a suite a
|
|
115
|
-
data-flow suite.
|
|
116
|
-
|
|
117
|
-
Data suites run as Node processes managed by Testkit's `node-vitest` engine and
|
|
118
|
-
should use configured service URLs for data-flow evidence. If a check requires
|
|
119
|
-
service-local loopback, `serviceContextUrl`, DOM rendering, browser automation,
|
|
120
|
-
or raw app-server process access, use a plain Vitest test, `contract`, or `ui` instead.
|
|
121
|
-
|
|
122
|
-
Use a plain Vitest test file for unit tests and other Node-side checks that need
|
|
123
|
-
no Testkit context, `contract` for typed service or protocol contracts, and `ui`
|
|
124
|
-
when the browser is part of the assertion. Plain Vitest files can exercise HTTP,
|
|
72
|
+
Use a plain Vitest test file for unit tests, other Node-side checks, and
|
|
73
|
+
service or protocol contracts, and `ui` when the browser is part of the
|
|
74
|
+
assertion. Plain Vitest files can exercise HTTP,
|
|
125
75
|
persistence, and service-spanning flows without a type label.
|
|
126
76
|
|
|
127
77
|
The retired `@gfdelarue/testkit/runtime`, `@gfdelarue/testkit/data`,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gfdelarue/testkit",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.1",
|
|
4
4
|
"description": "Test orchestration platform for product acceptance and QA workflows",
|
|
5
5
|
"license": "SEE LICENSE IN LICENSE",
|
|
6
6
|
"type": "module",
|
|
@@ -50,10 +50,6 @@
|
|
|
50
50
|
"types": "./dist/types/node-vitest-data.d.ts",
|
|
51
51
|
"default": "./dist/engines/node-vitest/runtime/data/index.js"
|
|
52
52
|
},
|
|
53
|
-
"./node-vitest/contract": {
|
|
54
|
-
"types": "./dist/types/node-vitest-contract.d.ts",
|
|
55
|
-
"default": "./dist/engines/node-vitest/runtime/contract/index.js"
|
|
56
|
-
},
|
|
57
53
|
"./workspace": {
|
|
58
54
|
"types": "./dist/types/workspace.d.ts",
|
|
59
55
|
"default": "./dist/app/inspect/index.js"
|
|
@@ -88,6 +84,7 @@
|
|
|
88
84
|
"dist/",
|
|
89
85
|
"docs/bug-records.md",
|
|
90
86
|
"docs/engines.md",
|
|
87
|
+
"docs/release-notes.md",
|
|
91
88
|
"docs/test-types.md",
|
|
92
89
|
"testkit-config-schema.md"
|
|
93
90
|
],
|
|
@@ -139,7 +136,19 @@
|
|
|
139
136
|
"yauzl": "3.4.0"
|
|
140
137
|
},
|
|
141
138
|
"overrides": {
|
|
142
|
-
"
|
|
139
|
+
"@verdaccio/config": {
|
|
140
|
+
"js-yaml": "5.4.1"
|
|
141
|
+
},
|
|
142
|
+
"@verdaccio/hooks": {
|
|
143
|
+
"handlebars": "4.7.10"
|
|
144
|
+
},
|
|
145
|
+
"express": {
|
|
146
|
+
"qs": "6.16.0"
|
|
147
|
+
},
|
|
148
|
+
"postcss": "8.5.24",
|
|
149
|
+
"verdaccio": {
|
|
150
|
+
"compression": "1.8.2"
|
|
151
|
+
}
|
|
143
152
|
},
|
|
144
153
|
"engines": {
|
|
145
154
|
"node": ">=22.15.0 <23.0.0 || >=24.14.1 <25.0.0",
|
package/testkit-config-schema.md
CHANGED
|
@@ -75,7 +75,7 @@ file-local metadata of a typed file may also declare skip and lock intent:
|
|
|
75
75
|
import { defineFile } from "@gfdelarue/testkit";
|
|
76
76
|
|
|
77
77
|
export const testkit = defineFile({
|
|
78
|
-
engine: "
|
|
78
|
+
engine: "playwright",
|
|
79
79
|
skip: "Billing is currently unavailable locally",
|
|
80
80
|
locks: ["payments-sandbox-webhook"],
|
|
81
81
|
});
|
|
@@ -94,12 +94,10 @@ literal values `"node-vitest"` or `"playwright"`.
|
|
|
94
94
|
The filename suffix determines the test type, and each engine and test type
|
|
95
95
|
pair runs under exactly one runner profile:
|
|
96
96
|
|
|
97
|
-
| Test type | Engine | Profile
|
|
98
|
-
| ------------------ | ------------- |
|
|
99
|
-
| none, a plain file | `node-vitest` | `node-vitest`
|
|
100
|
-
| `
|
|
101
|
-
| `contract` | `node-vitest` | `node-vitest-contract` |
|
|
102
|
-
| `ui` | `playwright` | `playwright-ui` |
|
|
97
|
+
| Test type | Engine | Profile |
|
|
98
|
+
| ------------------ | ------------- | --------------- |
|
|
99
|
+
| none, a plain file | `node-vitest` | `node-vitest` |
|
|
100
|
+
| `ui` | `playwright` | `playwright-ui` |
|
|
103
101
|
|
|
104
102
|
For a typed file, missing metadata, dynamic values, and unsupported
|
|
105
103
|
engine/test-type pairs are discovery errors. There is no default-engine or legacy-runner compatibility
|
|
@@ -121,36 +119,45 @@ runtime-policy modules under `src/` just to cooperate with managed runs.
|
|
|
121
119
|
|
|
122
120
|
## Top-Level Fields
|
|
123
121
|
|
|
122
|
+
The top-level keys are `areas`, `artifacts`, `bugs`, `discovery`, `execution`,
|
|
123
|
+
`fingerprints`, `groups`, `lint`, `redact`, `services`, `toolchains`, and `ui`.
|
|
124
|
+
Testkit refuses a config with any other top-level key. The error names the key
|
|
125
|
+
and the closest known key when one is close.
|
|
126
|
+
|
|
124
127
|
### `execution`
|
|
125
128
|
|
|
126
129
|
Default execution settings for the repo.
|
|
127
130
|
|
|
128
|
-
- `taskConcurrency`: maximum concurrent test tasks across the run (default `1`)
|
|
131
|
+
- `taskConcurrency`: maximum concurrent test tasks across the run (default `1`).
|
|
132
|
+
A typed file, a batch of typed files, or one Vitest invocation of plain files
|
|
133
|
+
is one task. Plain files in one invocation run with Vitest's file
|
|
134
|
+
parallelism, which the Vitest config's `fileParallelism` and `maxWorkers`
|
|
135
|
+
control.
|
|
129
136
|
- `fileTimeoutSeconds`
|
|
130
137
|
|
|
131
|
-
`execution.workers` and `presets.nodeNext({ workers })`
|
|
132
|
-
|
|
138
|
+
`execution.workers` and `presets.nodeNext({ workers })` are deprecated aliases.
|
|
139
|
+
Use `taskConcurrency` in new config.
|
|
133
140
|
If both names are present, their values must match.
|
|
134
141
|
|
|
135
|
-
`--task-concurrency` overrides the configured value. The CLI alias
|
|
136
|
-
also
|
|
137
|
-
|
|
142
|
+
`--task-concurrency` overrides the configured value. The deprecated CLI alias
|
|
143
|
+
`--workers` also works; contradictory CLI aliases are rejected before applying
|
|
144
|
+
CLI precedence. Contradictory config aliases are rejected even
|
|
138
145
|
when the CLI supplies an override.
|
|
139
146
|
|
|
140
147
|
Run documents keep the configured `run.workers` field. Observed
|
|
141
148
|
`workerCount`, Playwright's own `workers`, `runtime.instances` and
|
|
142
149
|
`runtime.maxConcurrentTasks` keep their existing meanings.
|
|
143
150
|
|
|
144
|
-
`fileTimeoutSeconds` is the single per-file wall clock budget for the repo.
|
|
145
|
-
helpers like `waitFor()` consume this budget automatically.
|
|
151
|
+
`fileTimeoutSeconds` is the single per-file wall clock budget for the repo.
|
|
146
152
|
|
|
147
153
|
### `lint`
|
|
148
154
|
|
|
149
155
|
Optional `testkit lint` settings. Omit it to run the default rules.
|
|
150
156
|
|
|
151
157
|
- `rules`: map of rule name to `true`, `false`, or an options object.
|
|
152
|
-
`engineImports`, `configImports`,
|
|
153
|
-
`testkitBoundary` default to disabled.
|
|
158
|
+
`engineImports`, `configImports`, and `uiSpecShape` default to enabled; `unitTestBoundaries` and
|
|
159
|
+
`testkitBoundary` default to disabled. `dataSuiteContract` was removed in
|
|
160
|
+
0.6.0, and `testkit lint` rejects a config that sets it.
|
|
154
161
|
- Rule options: `enabled` (default `true`), `include`, `exclude`. `testkitBoundary`
|
|
155
162
|
also accepts `allowed` and `trackedOnly`.
|
|
156
163
|
- `ui`: `maxLines` (default `220`) and `maxTests` (default `8`) for UI suites.
|
|
@@ -158,16 +165,11 @@ Optional `testkit lint` settings. Omit it to run the default rules.
|
|
|
158
165
|
Enabled-rule findings and syntax errors fail the command. There are no severity
|
|
159
166
|
levels. The rules report:
|
|
160
167
|
|
|
161
|
-
- `engineImports`: a value import the suite's engine can't load, such as
|
|
162
|
-
in a
|
|
168
|
+
- `engineImports`: a value import the suite's engine can't load, such as Vitest
|
|
169
|
+
in a UI suite. Type-only imports
|
|
163
170
|
are allowed.
|
|
164
171
|
- `configImports`: `testkit.config.ts` importing a repo-local `__testkit__`
|
|
165
172
|
helper module.
|
|
166
|
-
- `dataSuiteContract`: a data suite that doesn't register its cases through
|
|
167
|
-
`dataScenario` from `@gfdelarue/testkit/node-vitest/data`, registers them
|
|
168
|
-
through Vitest's `test`, `it`, `describe`, `suite`, or `bench`, or records
|
|
169
|
-
only bookkeeping such as `count(...)`, `event(...)`, and `finding(...)`
|
|
170
|
-
without product data-flow evidence such as `http(...)` or `record(...)`.
|
|
171
173
|
- `uiSpecShape`: a UI suite longer than `ui.maxLines` lines or with more than
|
|
172
174
|
`ui.maxTests` tests.
|
|
173
175
|
- `unitTestBoundaries`: test doubles, module replacement, `process.env` reads,
|
|
@@ -193,7 +195,11 @@ areas: {
|
|
|
193
195
|
|
|
194
196
|
- The key is the area id: lowercase letters, digits, and single hyphens, at
|
|
195
197
|
most 40 characters.
|
|
196
|
-
- `name`: one line of 1 to 60 characters. It becomes the label's description
|
|
198
|
+
- `name`: one line of 1 to 60 characters. It becomes the label's description,
|
|
199
|
+
and the issue carries it. Testkit doesn't rewrite it, so it must hold no
|
|
200
|
+
text that Testkit redacts as a secret. `testkit bug file` stops when the
|
|
201
|
+
fault's area has such a name. [Issue body](docs/bug-records.md#issue-body)
|
|
202
|
+
lists what Testkit redacts.
|
|
197
203
|
- `paths`: at least one glob. Globs are relative to the repository root and
|
|
198
204
|
written with `/`. A leading `!` or `/` and a `..` segment are rejected.
|
|
199
205
|
|
|
@@ -277,7 +283,7 @@ runs. The backend falls back to `TESTKIT_BACKEND_BASE_URL` and then
|
|
|
277
283
|
### `services`
|
|
278
284
|
|
|
279
285
|
Map of service name to service config. A service describes what Testkit can
|
|
280
|
-
start.
|
|
286
|
+
start. It owns no test files; [`groups`](#groups) assigns them.
|
|
281
287
|
|
|
282
288
|
### `groups`
|
|
283
289
|
|
|
@@ -324,21 +330,21 @@ Files that no group's roots claim belong to the implicit group `root`, which
|
|
|
324
330
|
uses no service. `root` is reserved. With no `testkit.config.ts`, every file
|
|
325
331
|
belongs to `root`, and `groups: {}` gives every file to `root`.
|
|
326
332
|
|
|
327
|
-
|
|
328
|
-
service's `skip`, `requirements`, or `runtimeIsolation`
|
|
329
|
-
the group field to use.
|
|
333
|
+
Testkit assigns no files by a service's `discovery`. With `groups` set, it
|
|
334
|
+
rejects a service's `discovery`, `skip`, `requirements`, or `runtimeIsolation`
|
|
335
|
+
with a message naming the group field to use. The `discovery` that
|
|
336
|
+
`database.fixture()` adds by default is the exception.
|
|
330
337
|
|
|
331
|
-
####
|
|
338
|
+
#### Configs without `groups`
|
|
332
339
|
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
removed; declare `groups` instead.
|
|
340
|
+
Before 0.6.0, a config with `services` and no `groups` key gave each service
|
|
341
|
+
the typed `*.testkit.ts` files under its `local.cwd` and `discovery.roots`. Such
|
|
342
|
+
a config now fails to load when a service's directories hold a typed file. The
|
|
343
|
+
error lists a `groups` entry for each of those services, with its `roots`,
|
|
344
|
+
`uses`, and any `exclude`, so the files keep their services. `status`,
|
|
345
|
+
`destroy`, `cleanup`, and `db schema` still run on such a config. A group also
|
|
346
|
+
owns the plain Vitest files under its roots, which a service never did. When no
|
|
347
|
+
service's directories hold a typed file, every file belongs to `root`.
|
|
342
348
|
|
|
343
349
|
### `toolchains`
|
|
344
350
|
|
|
@@ -378,48 +384,12 @@ download mode fails on a musl host before downloading anything.
|
|
|
378
384
|
- `package.json#packageManager`
|
|
379
385
|
- `package.json#engines.npm`
|
|
380
386
|
|
|
381
|
-
### `profiles
|
|
382
|
-
|
|
383
|
-
Named HTTP profiles for Node/Vitest data suites, selected through
|
|
384
|
-
`flow.http({ profile })`.
|
|
385
|
-
|
|
386
|
-
Node/Vitest data profiles may carry `auth`, `headers`, and
|
|
387
|
-
`rawHeaders` functions. `auth.setup` or `auth.setupAsync`
|
|
388
|
-
returns a session bundle of named actors; `headers` maps the bundle and the
|
|
389
|
-
requested actor to request headers.
|
|
390
|
-
|
|
391
|
-
```ts
|
|
392
|
-
interface SessionBundle {
|
|
393
|
-
actors: Record<string, { session: { jwt: string } }>;
|
|
394
|
-
primaryActor: string;
|
|
395
|
-
}
|
|
396
|
-
|
|
397
|
-
profiles: {
|
|
398
|
-
http: {
|
|
399
|
-
default: {
|
|
400
|
-
auth: {
|
|
401
|
-
async setupAsync({ env, http }): Promise<SessionBundle> {
|
|
402
|
-
const login = await http.post(`${env.BASE}/api/v1/auth/login`, body);
|
|
403
|
-
const jwt = login.cookies?.session?.[0]?.value;
|
|
404
|
-
if (!jwt) throw new Error("Login response did not set the session cookie");
|
|
405
|
-
return { actors: { primary: { session: { jwt } } }, primaryActor: "primary" };
|
|
406
|
-
},
|
|
407
|
-
headers(setupData): Record<string, string> {
|
|
408
|
-
const bundle = setupData as SessionBundle | null;
|
|
409
|
-
return bundle
|
|
410
|
-
? { Authorization: `Bearer ${bundle.actors[bundle.primaryActor].session.jwt}` }
|
|
411
|
-
: {};
|
|
412
|
-
},
|
|
413
|
-
},
|
|
414
|
-
},
|
|
415
|
-
},
|
|
416
|
-
}
|
|
417
|
-
```
|
|
387
|
+
### `profiles`
|
|
418
388
|
|
|
419
|
-
|
|
420
|
-
test
|
|
421
|
-
|
|
422
|
-
|
|
389
|
+
Testkit 0.6.0 removed HTTP profiles. A config that sets `profiles` fails to
|
|
390
|
+
load. A test logs in with `fetch` or its own client against the URL from
|
|
391
|
+
`@gfdelarue/testkit/environment`, as
|
|
392
|
+
[Record data-flow evidence](README.md#record-data-flow-evidence) shows.
|
|
423
393
|
|
|
424
394
|
## Service Fields
|
|
425
395
|
|
|
@@ -594,10 +564,9 @@ to the matching `{db*:api}` token. `env.values` entries override expanded names.
|
|
|
594
564
|
### `envFile` / `envFiles`
|
|
595
565
|
|
|
596
566
|
Optional extra env files loaded into the declared service/task environment for
|
|
597
|
-
migrations, seeds, services, and every execution engine. Node/Vitest
|
|
598
|
-
|
|
599
|
-
`readPlaywrightContext().env`.
|
|
600
|
-
non-enumerable compatibility overlay, not the consumer-input contract.
|
|
567
|
+
migrations, seeds, services, and every execution engine. Node/Vitest tests read
|
|
568
|
+
task values from `process.env`, and managed Playwright contexts expose them
|
|
569
|
+
through `readPlaywrightContext().env`.
|
|
601
570
|
|
|
602
571
|
Testkit reads a service's env files only when a selected group uses the service
|
|
603
572
|
or a used service depends on it. Loading the config reads none of them, so a
|
|
@@ -626,10 +595,10 @@ Environment names are validated case-insensitively. All engines reserve
|
|
|
626
595
|
`XDG_CONFIG_HOME`, and `XDG_DATA_HOME`. Engine-specific reservations and limits
|
|
627
596
|
are:
|
|
628
597
|
|
|
629
|
-
| Engine | Additional reservations
|
|
630
|
-
| ------------- |
|
|
631
|
-
| `node-vitest` | exact keys `BASE_URL`, `FORCE_COLOR`, `NODE_OPTIONS`, `NO_COLOR`, `UPDATE_SNAPSHOT`, `VITEST_POOL_ID`, `VITEST_WORKER_ID`, and the `TESTKIT_` keys it sets
|
|
632
|
-
| `playwright` | prefixes `TESTKIT_`, `PLAYWRIGHT_`, `PWTEST_`, `PW_TEST_`; exact keys `BASE_URL`, `FORCE_COLOR`, `NODE_OPTIONS`, `NO_COLOR`
|
|
598
|
+
| Engine | Additional reservations | Consumer environment limit |
|
|
599
|
+
| ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------ |
|
|
600
|
+
| `node-vitest` | exact keys `BASE_URL`, `FORCE_COLOR`, `NODE_OPTIONS`, `NO_COLOR`, `UPDATE_SNAPSHOT`, `VITEST_POOL_ID`, `VITEST_WORKER_ID`, and the `TESTKIT_` keys it sets | 512 entries; 4 MiB of UTF-8 key/value data |
|
|
601
|
+
| `playwright` | prefixes `TESTKIT_`, `PLAYWRIGHT_`, `PWTEST_`, `PW_TEST_`; exact keys `BASE_URL`, `DBUS_SESSION_BUS_ADDRESS`, `FC_FONTATIONS`, `FORCE_COLOR`, `NODE_OPTIONS`, `NO_COLOR` | 512 entries; 2 MiB of UTF-8 key/value data |
|
|
633
602
|
|
|
634
603
|
Empty names, names containing `=` or NUL, values containing NUL, and
|
|
635
604
|
case-variant duplicates are rejected. The limits apply after env files and
|
|
@@ -743,13 +712,37 @@ service provisioned in the same runtime instance, as `DATABASE_URL` and in
|
|
|
743
712
|
|
|
744
713
|
### `runtime`
|
|
745
714
|
|
|
746
|
-
|
|
715
|
+
Service-local runtime instances and their preparation.
|
|
716
|
+
|
|
717
|
+
```ts
|
|
718
|
+
runtime: {
|
|
719
|
+
instances: 1,
|
|
720
|
+
maxConcurrentTasks: 4,
|
|
721
|
+
prepare: {
|
|
722
|
+
inputs: ["frontend/src", "frontend/public", "frontend/package.json"],
|
|
723
|
+
steps: [{ kind: "command", run: "npm run build", cwd: "frontend" }],
|
|
724
|
+
},
|
|
725
|
+
}
|
|
726
|
+
```
|
|
747
727
|
|
|
748
728
|
- `instances`
|
|
729
|
+
- number of reusable local runtime instances for that service graph
|
|
749
730
|
- `maxConcurrentTasks`
|
|
731
|
+
- maximum number of concurrent file tasks admitted to each runtime instance
|
|
732
|
+
- omit it to leave concurrency bounded only by the outer `taskConcurrency` setting
|
|
750
733
|
- `toolchain`
|
|
734
|
+
- `prepare`
|
|
735
|
+
- optional runtime-generation preparation hook
|
|
736
|
+
- runs once per runtime instance before local services start
|
|
737
|
+
- fingerprints declared `inputs` plus referenced step files/module inputs
|
|
738
|
+
- use this for build-once/start-many shared runtimes such as `next build` +
|
|
739
|
+
`next start`
|
|
751
740
|
- `prepare.inputs`
|
|
741
|
+
- extra repo-relative inputs that should invalidate prepared artifacts when
|
|
742
|
+
they change
|
|
752
743
|
- `prepare.steps`
|
|
744
|
+
- ordered lifecycle steps using the same step DSL as `database.template`
|
|
745
|
+
- each step supports `cwd` and `inputs`
|
|
753
746
|
|
|
754
747
|
`runtime.toolchain` may be either:
|
|
755
748
|
|
|
@@ -766,19 +759,20 @@ Changing the resolved Node or npm version invalidates prepared artifacts, so
|
|
|
766
759
|
|
|
767
760
|
### `skip`
|
|
768
761
|
|
|
769
|
-
Skip rules for a group's files, set as `groups.<name>.skip`.
|
|
770
|
-
ownership a service's `skip` applies to the files it owns. Skipped files remain
|
|
762
|
+
Skip rules for a group's files, set as `groups.<name>.skip`. Skipped files remain
|
|
771
763
|
visible in run artifacts with `status: "skipped"` and the configured reason.
|
|
772
764
|
|
|
773
765
|
Prefer file-local `export const testkit = defineFile({ ... })` metadata
|
|
774
|
-
for per-file skips and locks. Repo-level `skip` is mainly for
|
|
775
|
-
policy.
|
|
766
|
+
for per-file skips and locks in a typed file. Repo-level `skip` is mainly for
|
|
767
|
+
broad suite-level policy. A plain Vitest file has no file-local metadata, so its
|
|
768
|
+
skips, locks, and isolation go in `skip.files`, `requirements.files`, and
|
|
769
|
+
`runtimeIsolation.files`.
|
|
776
770
|
|
|
777
771
|
```ts
|
|
778
772
|
skip: {
|
|
779
773
|
files: [
|
|
780
774
|
{
|
|
781
|
-
path: "
|
|
775
|
+
path: "src/billing/billing.test.ts",
|
|
782
776
|
reason: "Billing is currently unavailable",
|
|
783
777
|
},
|
|
784
778
|
],
|
|
@@ -796,59 +790,26 @@ skip: {
|
|
|
796
790
|
- `files[].reason`
|
|
797
791
|
- required non-empty explanation recorded in artifacts
|
|
798
792
|
- `suites[].selector`
|
|
799
|
-
- existing suite selector syntax, including typed selectors like `
|
|
800
|
-
and `ui:billing`
|
|
793
|
+
- existing suite selector syntax, including typed selectors like `ui:billing`
|
|
801
794
|
- `suites[].reason`
|
|
802
795
|
- required non-empty explanation recorded in artifacts
|
|
803
796
|
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
Service-local runtime instance settings.
|
|
809
|
-
|
|
810
|
-
```ts
|
|
811
|
-
runtime: {
|
|
812
|
-
instances: 1,
|
|
813
|
-
maxConcurrentTasks: 4,
|
|
814
|
-
prepare: {
|
|
815
|
-
inputs: ["frontend/src", "frontend/public", "frontend/package.json"],
|
|
816
|
-
steps: [{ kind: "command", run: "npm run build", cwd: "frontend" }],
|
|
817
|
-
},
|
|
818
|
-
}
|
|
819
|
-
```
|
|
820
|
-
|
|
821
|
-
- `instances`
|
|
822
|
-
- number of reusable local runtime instances for that service graph
|
|
823
|
-
- `maxConcurrentTasks`
|
|
824
|
-
- maximum number of concurrent file tasks admitted to each runtime instance
|
|
825
|
-
- omit it to leave concurrency bounded only by the outer `taskConcurrency` setting
|
|
826
|
-
- `prepare`
|
|
827
|
-
- optional runtime-generation preparation hook
|
|
828
|
-
- runs once per runtime instance before local services start
|
|
829
|
-
- fingerprints declared `inputs` plus referenced step files/module inputs
|
|
830
|
-
- use this for build-once/start-many shared runtimes such as `next build` +
|
|
831
|
-
`next start`
|
|
832
|
-
- `prepare.inputs`
|
|
833
|
-
- extra repo-relative inputs that should invalidate prepared artifacts when
|
|
834
|
-
they change
|
|
835
|
-
- `prepare.steps`
|
|
836
|
-
- ordered lifecycle steps using the same step DSL as `database.template`
|
|
837
|
-
- each step supports `cwd` and `inputs`
|
|
797
|
+
A run that skips a selected file fails, even when every other file passes, and
|
|
798
|
+
`--allow-empty` doesn't change that. Use `--ignore-skip-rules` on the CLI to run
|
|
799
|
+
skipped files anyway.
|
|
838
800
|
|
|
839
801
|
### `requirements`
|
|
840
802
|
|
|
841
|
-
Task lock rules for a group's files, set as `groups.<name>.requirements`.
|
|
842
|
-
legacy ownership a service's `requirements` apply to the files it owns.
|
|
803
|
+
Task lock rules for a group's files, set as `groups.<name>.requirements`.
|
|
843
804
|
|
|
844
805
|
```ts
|
|
845
806
|
requirements: {
|
|
846
807
|
suites: [
|
|
847
|
-
{ selector: "
|
|
808
|
+
{ selector: "ui:checkout", locks: ["payments-sandbox-webhook"] },
|
|
848
809
|
],
|
|
849
810
|
files: [
|
|
850
811
|
{
|
|
851
|
-
path: "src/api/routes/
|
|
812
|
+
path: "src/api/routes/payout-webhook.test.ts",
|
|
852
813
|
locks: ["payments-sandbox-webhook"],
|
|
853
814
|
},
|
|
854
815
|
],
|
|
@@ -861,7 +822,7 @@ wait for each other. Use a lock for a resource that can't be duplicated, never
|
|
|
861
822
|
for tests that share rows.
|
|
862
823
|
|
|
863
824
|
- `suites[].selector`
|
|
864
|
-
- typed selector like `
|
|
825
|
+
- typed selector like `ui:billing`
|
|
865
826
|
- `suites[].locks`
|
|
866
827
|
- named locks required by matching files
|
|
867
828
|
- `files[].path`
|
|
@@ -872,15 +833,14 @@ for tests that share rows.
|
|
|
872
833
|
### `runtimeIsolation`
|
|
873
834
|
|
|
874
835
|
What a group's files share with each other in a run, set as
|
|
875
|
-
`groups.<name>.runtimeIsolation`.
|
|
876
|
-
`runtimeIsolation` applies to the files it owns. A file can set its own with
|
|
836
|
+
`groups.<name>.runtimeIsolation`. A file can set its own with
|
|
877
837
|
`defineFile({ runtimeIsolation: "resetBefore" })`.
|
|
878
838
|
|
|
879
839
|
```ts
|
|
880
840
|
runtimeIsolation: {
|
|
881
841
|
default: "shared",
|
|
882
|
-
suites: [{ selector: "
|
|
883
|
-
files: [{ path: "
|
|
842
|
+
suites: [{ selector: "ui:admin", policy: "resetBefore" }],
|
|
843
|
+
files: [{ path: "src/orders/purge.test.ts", policy: "resetBefore" }],
|
|
884
844
|
}
|
|
885
845
|
```
|
|
886
846
|
|
|
@@ -957,44 +917,15 @@ Typed suites are discovered from the filesystem:
|
|
|
957
917
|
- a file belongs to the group with the deepest matching root in
|
|
958
918
|
`groups.<name>.roots`
|
|
959
919
|
- files that no group claims belong to the implicit group `root`
|
|
960
|
-
- under legacy ownership, service groups take their roots from `local.cwd` and
|
|
961
|
-
`discovery.roots`
|
|
962
|
-
|
|
963
|
-
Supported suffixes:
|
|
964
920
|
|
|
965
|
-
|
|
966
|
-
- `*.data.testkit.ts`
|
|
967
|
-
- `*.ui.testkit.ts`
|
|
921
|
+
The supported suffix is `*.ui.testkit.ts`.
|
|
968
922
|
|
|
969
923
|
Every discovered typed file must also export `defineFile({ engine: "..." })`
|
|
970
924
|
from the root package. The suffix supplies `testType`; the required engine declaration
|
|
971
925
|
supplies `engineId`; and Testkit derives `profileId` from the catalog. None of
|
|
972
926
|
those identities are silently rewritten later in the run.
|
|
973
927
|
|
|
974
|
-
|
|
975
|
-
declares `engine: "node-vitest"`, uses
|
|
976
|
-
`@gfdelarue/testkit/node-vitest/contract`, runs through
|
|
977
|
-
`node-vitest-contract`, and is the right place for checks that need Node APIs
|
|
978
|
-
but are not data-flow evidence suites. Contract files are native Vitest suites:
|
|
979
|
-
`contractScenario(...)` provides managed Testkit context when needed, while
|
|
980
|
-
direct public Vitest runner APIs remain valid.
|
|
981
|
-
|
|
982
|
-
`*.data.testkit.ts` is a Testkit-owned data-flow/data-contract suite type. It
|
|
983
|
-
declares `engine: "node-vitest"`, uses
|
|
984
|
-
`@gfdelarue/testkit/node-vitest/data`, records evidence through
|
|
985
|
-
`dataScenario(...)`, and runs through `node-vitest-data`. Data suites may use
|
|
986
|
-
non-registering public Vitest APIs such as `expect`, but `dataScenario(...)`
|
|
987
|
-
owns case registration. Ordinary Node/Vitest assertions, DOM/browser checks,
|
|
988
|
-
and service-local loopback checks belong in the appropriate non-data suite
|
|
989
|
-
type.
|
|
990
|
-
|
|
991
|
-
Examples:
|
|
992
|
-
|
|
993
|
-
- `src/api/routes/__testkit__/auth/me.contract.testkit.ts`
|
|
994
|
-
- `src/api/routes/__testkit__/runtime.contract.testkit.ts`
|
|
995
|
-
- `src/api/routes/__testkit__/agent-flow.data.testkit.ts`
|
|
996
|
-
- `src/db/__testkit__/sessions/count-type.contract.testkit.ts`
|
|
997
|
-
- `frontend/__testkit__/navigation/navigation.ui.testkit.ts`
|
|
928
|
+
Example: `frontend/__testkit__/navigation/navigation.ui.testkit.ts`.
|
|
998
929
|
|
|
999
930
|
`testkit discover` exposes that inventory directly without running suites:
|
|
1000
931
|
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
import{AST_NODE_TYPES as r}from"@typescript-eslint/types";import{binding as p,memberName as w,stringValue as E}from"../policy.js";const T=new Set(["addFinding","actor","budget","bytes","count","duration","event","externalCall","expectMetric","expectSequence","fixture","finding","http","metric","money","ratio","reconcile","record","recordCache","recordHttpRequest","recordLlmUsage","recordRpc","sequence","sequenceBudget","system","transition"]),N=new Set(["actor","bytes","duration","externalCall","http","money","ratio","reconcile","record","recordCache","recordHttpRequest","recordLlmUsage","recordRpc","sequence","system","transition"]),f=new Set(["bench","describe","it","suite","test","BenchmarkRunner","TestRunner","EvaluatedModules"]);export function createDataPolicy({context:o,file:C,report:n}){if(!C.endsWith(".data.testkit.ts"))return{};const d=new Set,h=new Set,v=new Map,g=new Map;let y=!1,b=!1;const u=(e,t,a="vitest-runner-import")=>{n(e,a,t,`Data suites cannot use ${t} for Vitest case registration; declare cases with dataScenario(...)`)},x=e=>{for(const t of o.sourceCode.getAncestors(e).reverse())if(t.type===r.ArrowFunctionExpression||t.type===r.FunctionExpression){const a=g.get(t);if(a)return a}},m=e=>e.startsWith("vitest/")&&e!=="vitest/package.json"||e==="@gfdelarue/testkit/node-vitest/contract",k=e=>{E(e)?.includes("{serviceContextUrl:")&&n(e,"service-namespace-url","serviceContextUrl","Data suites must not use serviceContextUrl placeholders because they run outside the deployed service namespace")};return{ImportDeclaration(e){const t=e.source.value;if(t==="@playwright/test"&&n(e,"browser-runtime-import",t,"Browser/runtime driving belongs in ui suites, not data suites"),(t.startsWith("@testing-library/")||t==="jsdom"||t==="happy-dom")&&n(e,"dom-runtime-import","dom-runtime","DOM/rendering tests belong in ui suites, not data suites"),e.importKind==="type")return;const a=e.specifiers.filter(s=>s.type!==r.ImportSpecifier||s.importKind!=="type");if(!(e.specifiers.length&&!a.length)){m(t)&&u(e,t),t==="@gfdelarue/testkit/node-vitest/data"&&(y=!0);for(const s of a){const i=o.sourceCode.getDeclaredVariables(e).find(l=>l.name===s.local.name);if(t==="vitest"&&s.type===r.ImportNamespaceSpecifier&&i&&h.add(i),s.type!==r.ImportSpecifier)continue;const c=s.imported.type===r.Identifier?s.imported.name:s.imported.value;t==="vitest"&&f.has(c)&&u(s,c),t==="@gfdelarue/testkit/node-vitest/data"&&c==="dataScenario"&&i&&d.add(i)}}},ExportNamedDeclaration(e){if(!e.source||e.exportKind==="type")return;const t=e.specifiers.filter(a=>a.exportKind!=="type");if(!(e.specifiers.length&&!t.length)&&(m(e.source.value)&&u(e,e.source.value),e.source.value==="vitest"))for(const a of t){const s=a.local.type===r.Identifier?a.local.name:a.local.value;f.has(s)&&u(a,s)}},ExportAllDeclaration(e){e.exportKind!=="type"&&m(e.source.value)&&u(e,e.source.value)},MemberExpression(e){if(e.object.type!==r.Identifier)return;const t=p(o,e.object),a=w(e);t&&h.has(t)&&a&&f.has(a)&&u(e,a,"vitest-runner-call")},CallExpression(e){const t=e.callee.type===r.Identifier?p(o,e.callee):null;if(t&&d.has(t)){b=!0;const i=e.arguments.at(-1);if(!i||i.type!==r.ArrowFunctionExpression&&i.type!==r.FunctionExpression){n(e,"missing-scenario-callback","dataScenario","dataScenario(...) must provide a callback that receives the Testkit data context");return}const c=i.params[0]?.type===r.AssignmentPattern?i.params[0].left:i.params.at(0),l=c?.type===r.Identifier?o.sourceCode.getDeclaredVariables(i).find(D=>D.name===c.name):null;if(!l){n(e,"missing-data-context","dataScenario","dataScenario callbacks must accept the Testkit data context and record data evidence");return}const S={callback:i,context:l,contextName:l.name,methods:new Set,assertion:!1};v.set(e,S),g.set(i,S);return}const a=o.sourceCode.getText(e.callee);e.callee.type===r.MemberExpression&&a.includes(".serviceContextUrl")&&n(e,"service-namespace-url","serviceContextUrl","Data suites run in the managed Node.js runner, not inside a service namespace; use baseUrl/public endpoints or move service-namespace checks to contract");const s=x(e);if(s&&(/^(expect|assert)(?:\(|\.)/u.test(a)&&(s.assertion=!0),e.callee.type===r.MemberExpression&&e.callee.object.type===r.Identifier&&p(o,e.callee.object)===s.context)){const i=w(e.callee);i&&T.has(i)&&s.methods.add(i)}},"CallExpression:exit"(e){const t=v.get(e);if(!t)return;const{callback:a,contextName:s,methods:i,assertion:c}=t;i.size?[...i].some(l=>N.has(l))||(n(a,"weak-data-evidence",s,"Data scenarios must record product data-flow evidence such as http, bytes, sequence, record, transition, system, externalCall, recordCache, recordLlmUsage, or reconcile; ordinary Node/Vitest-style checks belong in *.contract.testkit.ts with @gfdelarue/testkit/node-vitest/contract"),c&&n(a,"vitest-shaped-data-suite",s,"This dataScenario looks like an ordinary assertion test with weak data bookkeeping; move Node-backed contract checks to *.contract.testkit.ts or model the product data flow with structured evidence")):n(a,"missing-data-evidence",s,"Data scenarios must record Testkit-owned data-flow evidence with the data context, such as http, bytes, sequence, record, transition, system, externalCall, recordCache, recordLlmUsage, or reconcile")},ThrowStatement(e){const t=x(e);t&&(t.assertion=!0)},Literal:k,TemplateLiteral:k,"Program:exit"(e){y?d.size?b||n(e,"missing-data-scenario","dataScenario","Data suites must declare at least one dataScenario(...)"):n(e,"missing-data-scenario-import","dataScenario","Data suites must import dataScenario from @gfdelarue/testkit/node-vitest/data"):n(e,"missing-data-import","@gfdelarue/testkit/node-vitest/data","Data suites must import dataScenario from @gfdelarue/testkit/node-vitest/data")}}}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
let o=null;export function registerRepoConfig(n){o=n??null}export function getRepoConfig(){return o}export function clearRepoConfig(){o=null}export function resolveHttpProfile(n){if(!n)return null;const t=getRepoConfig();if(!t)return null;const r=t.profiles?.http?.[n];if(!r)throw new Error(`Unknown testkit HTTP profile "${n}"`);return r}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
import{matchesSuiteSelectors as a}from"../../domain/suite-selection.js";import{normalizePath as l}from"./paths.js";import{resolveRuntimeIsolationPolicy as i}from"./runtime-isolation.js";export function resolveTaskPolicy(o,t,r){return{locks:m(o,t,r),runtimeIsolation:i({testkit:o},t,r)}}function m(o,t,r){const n=l(r),s=new Set;for(const e of o.requirements.suites)if(a(t.testType,t.name,[e.selector]))for(const c of e.locks)s.add(c);for(const e of o.requirements.fileLocksByPath.get(n)??[])s.add(e);for(const e of o.fileMetadataByPath.get(n)?.locks??[])s.add(e);return[...s].sort()}
|