@gfdelarue/testkit 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. package/README.md +136 -110
  2. package/dist/app/bugs/draft.js +2 -2
  3. package/dist/app/bugs/guide.js +1 -1
  4. package/dist/app/bugs/record.js +2 -2
  5. package/dist/app/campaigns/index.js +2 -2
  6. package/dist/app/destroy/index.js +1 -1
  7. package/dist/app/discover/index.js +2 -2
  8. package/dist/app/environments/static-values.js +1 -1
  9. package/dist/app/environments/supplied.js +1 -1
  10. package/dist/app/inspect/index.js +1 -1
  11. package/dist/app/lint/native.js +1 -1
  12. package/dist/app/lint/options.js +1 -1
  13. package/dist/app/run/engine-orchestration.js +1 -1
  14. package/dist/app/run/environment.js +1 -1
  15. package/dist/app/run/orchestrator.js +1 -1
  16. package/dist/app/run/request.js +1 -1
  17. package/dist/app/run/run-finalization.js +1 -1
  18. package/dist/app/run/task-batches.js +1 -1
  19. package/dist/app/run/worker-loop.js +1 -1
  20. package/dist/app/status/model.js +1 -1
  21. package/dist/app/status/runs.js +1 -1
  22. package/dist/app/typecheck/index.js +4 -4
  23. package/dist/cli/command-catalog.js +1 -1
  24. package/dist/cli/command-flags.js +1 -1
  25. package/dist/cli/commands/artifacts.js +2 -2
  26. package/dist/cli/commands/discover.js +1 -1
  27. package/dist/cli/commands/guard/unsafe-entrypoint.js +1 -1
  28. package/dist/cli/commands/run.js +2 -2
  29. package/dist/cli/commands/typecheck.js +1 -1
  30. package/dist/cli/commands/workspace.js +1 -1
  31. package/dist/config/authoring/database-fixture-discovery.js +1 -0
  32. package/dist/config/authoring/index.js +1 -1
  33. package/dist/config/loader/config-loader.js +1 -1
  34. package/dist/config/loader/decode-discovery.js +1 -1
  35. package/dist/config/loader/discover-tests.js +1 -1
  36. package/dist/config/loader/discovery.js +2 -2
  37. package/dist/config/loader/file-metadata.js +1 -1
  38. package/dist/config/loader/groups.js +2 -1
  39. package/dist/config/loader/load.js +1 -1
  40. package/dist/config/loader/managed-configs.js +1 -1
  41. package/dist/config/loader/owned-discovery.js +1 -1
  42. package/dist/config/loader/runtime-isolation.js +1 -1
  43. package/dist/config/loader/skip-config.js +1 -1
  44. package/dist/database/admin.js +2 -2
  45. package/dist/database/configured-steps.js +1 -1
  46. package/dist/domain/identifiers.js +1 -1
  47. package/dist/domain/retired-test-types.js +1 -1
  48. package/dist/domain/suite-selection.js +1 -1
  49. package/dist/domain/test-types.js +1 -1
  50. package/dist/engines/catalog.js +1 -1
  51. package/dist/engines/definitions.js +1 -1
  52. package/dist/engines/environment/index.js +1 -1
  53. package/dist/engines/installations/node-vitest.js +1 -1
  54. package/dist/engines/installations/playwright.js +1 -1
  55. package/dist/engines/managed-module-resolution.js +6 -2
  56. package/dist/engines/node-vitest/adapter/adapter.js +1 -1
  57. package/dist/engines/node-vitest/adapter/driver.js +2 -2
  58. package/dist/engines/node-vitest/adapter/evidence.js +2 -2
  59. package/dist/engines/node-vitest/adapter/workspace.js +8 -24
  60. package/dist/engines/node-vitest/inventory/index.js +2 -2
  61. package/dist/engines/node-vitest/inventory/list-files.js +3 -3
  62. package/dist/engines/node-vitest/runtime/data/index.js +1 -1
  63. package/dist/engines/node-vitest/runtime/data/report.js +1 -1
  64. package/dist/engines/node-vitest/runtime/data/scenario.js +2 -2
  65. package/dist/engines/node-vitest/runtime/runtime-input.js +1 -1
  66. package/dist/engines/playwright/adapter/adapter.js +1 -1
  67. package/dist/engines/playwright/adapter/config.js +2 -2
  68. package/dist/engines/playwright/runtime/managed-test.js +1 -0
  69. package/dist/environment/lifecycle.js +1 -1
  70. package/dist/environment/run-environment.js +1 -1
  71. package/dist/evidence/artifact-config.js +1 -1
  72. package/dist/evidence/maintenance.js +1 -0
  73. package/dist/evidence/read-run.js +1 -1
  74. package/dist/kernel/known-keys.js +1 -0
  75. package/dist/process/execa-bundle.json +1 -1
  76. package/dist/repo/fingerprint-policy.js +1 -1
  77. package/dist/steps/configured-step-testkit-modules.js +1 -1
  78. package/dist/toolchains/index.js +2 -2
  79. package/dist/toolchains/vitest/index.js +1 -1
  80. package/dist/types/config.d.ts +7 -86
  81. package/dist/types/environment.d.ts +6 -0
  82. package/dist/types/node-vitest-data.d.ts +2 -218
  83. package/dist/types/workspace.d.ts +27 -11
  84. package/docs/bug-records.md +3 -8
  85. package/docs/engines.md +63 -17
  86. package/docs/release-notes.md +2116 -0
  87. package/docs/test-types.md +19 -69
  88. package/package.json +2 -5
  89. package/testkit-config-schema.md +65 -128
  90. package/dist/app/lint/rules/data.js +0 -1
  91. package/dist/config/authoring/runtime.js +0 -1
  92. package/dist/engines/node-vitest/runtime/contract/index.js +0 -1
  93. package/dist/engines/node-vitest/runtime/data/http.js +0 -1
  94. package/dist/engines/node-vitest/runtime/data/runner.js +0 -1
  95. package/dist/types/node-vitest-contract.d.ts +0 -75
@@ -0,0 +1,2116 @@
1
+ # 0.6.0
2
+
3
+ 0.6.0 removes the `contract` and `data` test types and legacy service
4
+ ownership, and refuses config keys that Testkit doesn't read. To upgrade from
5
+ 0.5.0, work through these steps in order:
6
+
7
+ 1. If `testkit.config.ts` has `services` and no `groups`, add the groups that
8
+ the load error lists before you rename any typed file. The error lists a
9
+ group only for a service that still owns a typed file. Then, with or without
10
+ that step, move each service's `skip`, `requirements`, and
11
+ `runtimeIsolation` to its group. Move its `discovery.roots` and
12
+ `discovery.exclude` to the group's `roots` and `exclude`, as paths from the
13
+ product directory, and delete its `discovery`. See
14
+ [Remove legacy service ownership](#remove-legacy-service-ownership).
15
+ 2. Replace `--service` with `--group` on `run`, `discover`, `artifacts`, and
16
+ `workspace`, even if the config already has `groups`. See the same section.
17
+ 3. Turn each `*.contract.testkit.ts` file into a plain Vitest file. See
18
+ [Remove the contract test type](#remove-the-contract-test-type).
19
+ 4. Turn each `*.data.testkit.ts` file into a plain Vitest file that calls
20
+ `dataFlow(context)`, and delete `profiles` from `testkit.config.ts`. See
21
+ [Remove the data test type](#remove-the-data-test-type).
22
+ 5. Delete each config key that the load error names, or move its value to the
23
+ key that applies. See
24
+ [A config with an unknown key fails to load](#a-config-with-an-unknown-key-fails-to-load).
25
+ 6. Remove leftover `int`, `e2e`, `dal`, and `load` selectors. Delete or migrate
26
+ leftover files of those types, and files that declare `engine: "k6"`. 0.5.0
27
+ stopped running them, and 0.6.0 no longer flags them. See
28
+ [Testkit no longer recognises the types and engine that 0.5.0 removed](#testkit-no-longer-recognises-the-types-and-engine-that-050-removed).
29
+
30
+ A config that sets `profiles` or an unknown top-level key fails to load before
31
+ Testkit checks its groups. If that happens at step 1, delete those keys as
32
+ steps 4 and 5 say, and load the config again.
33
+
34
+ Upgrading from 0.4.x? Work through [0.4.5](#045) and then [0.5.0](#050) first.
35
+ 0.4.5 makes `local.port` a config error. 0.5.0 adds `groups`, which step 1
36
+ needs, and removes the `int`, `e2e`, `dal`, and `load` types, whose files 0.6.0
37
+ ignores without a warning.
38
+
39
+ ## Breaking changes
40
+
41
+ ### Remove legacy service ownership
42
+
43
+ Only `groups` assign test files to services
44
+ ([gfdelarue/testkit#241](https://github.com/gfdelarue/testkit/issues/241)).
45
+ Before 0.6.0, a config with `services` and no `groups` key gave each service the
46
+ typed files under its `local.cwd` and `discovery.roots`, and printed a notice.
47
+ That legacy ownership is gone. A config that relied on it fails to load and
48
+ lists the groups to add:
49
+
50
+ ```text
51
+ testkit.config.ts has services and no groups. Testkit 0.6.0 removed legacy service ownership, so services no longer own the test files under their directories. Add these groups to keep each typed test file with the service that owned it:
52
+
53
+ groups: {
54
+ api: { roots: ["services/api"], uses: ["api"] },
55
+ web: { roots: ["apps/web"], uses: ["web"] },
56
+ },
57
+
58
+ A group also owns the plain Vitest files under its roots.
59
+ Migrate with "Remove legacy service ownership" in node_modules/@gfdelarue/testkit/docs/release-notes.md.
60
+ ```
61
+
62
+ - The error lists each service that owned a typed file, or excluded one under
63
+ its directories, with that service's `roots`, `uses`, and `exclude` as 0.5.0
64
+ resolved them.
65
+ - `run`, `typecheck`, and `discover --strict` stop with it. `discover` without
66
+ `--strict` reports it as a `config_invalid` diagnostic. `status`,
67
+ `run status`, `destroy`, `cleanup`, and `db schema` still run, so you can
68
+ stop 0.5.0 services before you migrate.
69
+ - A config whose services held no typed file, such as one with only a database
70
+ service, loads, and every file belongs to `root`.
71
+
72
+ Migrate a two-service config:
73
+
74
+ ```ts
75
+ // Before
76
+ export default defineConfig({
77
+ services: {
78
+ api: app.node({ cwd: "services/api", start: "node dist/server.js", runtimeIsolation: "resetBefore" }),
79
+ web: app.next({ cwd: "apps/web", dependsOn: ["api"] }),
80
+ },
81
+ });
82
+ ```
83
+
84
+ ```ts
85
+ // After
86
+ export default defineConfig({
87
+ services: {
88
+ api: app.node({ cwd: "services/api", start: "node dist/server.js" }),
89
+ web: app.next({ cwd: "apps/web", dependsOn: ["api"] }),
90
+ },
91
+ groups: {
92
+ api: { roots: ["services/api"], uses: ["api"], runtimeIsolation: "resetBefore" },
93
+ web: { roots: ["apps/web"], uses: ["web"] },
94
+ },
95
+ });
96
+ ```
97
+
98
+ - Move a service's `skip`, `requirements`, and `runtimeIsolation` to its group.
99
+ `ServiceConfig` no longer declares them, and beside `groups` Testkit rejects
100
+ them on a service with a message that names the group field.
101
+ - Move a service's `discovery.roots` and `discovery.exclude` to its group's
102
+ `roots` and `exclude`, and delete the service's `discovery`. A service's
103
+ `discovery` paths are relative to its `local.cwd`, and a group's are relative
104
+ to the product directory. The root `__testkit__` of a service whose
105
+ `local.cwd` is `services/api` becomes `services/api/__testkit__`. The error
106
+ above prints them that way. Beside `groups`, Testkit rejects a service's
107
+ `discovery` with a message that names the group's fields:
108
+
109
+ ```text
110
+ services.api.discovery applies to tests, and services own no tests when groups is set; move its roots and exclude to groups.api.roots and groups.api.exclude, as paths from the product directory
111
+ ```
112
+
113
+ The `discovery` that `database.fixture()` adds by default is still accepted.
114
+
115
+ - A group's roots also claim plain Vitest files, which a service never did, and
116
+ a run starts the group's services for them. A service with no
117
+ `discovery.roots` and `local.cwd: "."` gets the root `.`, so its group owns
118
+ every plain file. To keep plain files in `root`, narrow the roots to the
119
+ directories that hold typed files.
120
+
121
+ `run`, `discover`, `artifacts`, and `workspace` no longer take `--service` as an
122
+ alias for `--group`:
123
+
124
+ ```text
125
+ Testkit 0.6.0 removed --service from run, discover, artifacts, and workspace; use --group api
126
+ ```
127
+
128
+ `status`, `destroy`, `cleanup`, and `db schema` keep `--service`, which names a
129
+ service there.
130
+
131
+ A discovered group's `source` is `groups` or `implicit`, and `TestGroupSource`
132
+ no longer includes `services`. `inspectWorkspace()` from
133
+ `@gfdelarue/testkit/workspace` no longer takes a `notice` callback.
134
+
135
+ ### Remove the contract test type
136
+
137
+ This release removes the `contract` test type. A contract file becomes a plain
138
+ Vitest file, which Testkit runs as it is. These are removed:
139
+
140
+ - the `*.contract.testkit.ts` suffix and the `node-vitest-contract` profile
141
+ - the `@gfdelarue/testkit/node-vitest/contract` export, with
142
+ `contractScenario`, `readContractContext`, `createHttpClient`, and their
143
+ types
144
+
145
+ Wherever you can name `contract`, Testkit fails with a message that names the
146
+ release and this section:
147
+
148
+ ```text
149
+ __testkit__/health.contract.testkit.ts: Testkit 0.6.0 removed the contract test type. Migrate with "Remove the contract test type" in node_modules/@gfdelarue/testkit/docs/release-notes.md.
150
+ ```
151
+
152
+ - `discover`, `run`, and `typecheck` stop on a leftover
153
+ `*.contract.testkit.ts` file unless a group excludes it, and `lint` and
154
+ `doctor` stop on any leftover file. A run stops even when the repository's
155
+ plain files would pass.
156
+ - `testkit run contract`, `--type contract`, `--suite contract:<name>`, and
157
+ `contract:` selectors in `skip`, `requirements`, and `runtimeIsolation` stop
158
+ with the same message.
159
+ - `bug file` refuses an unfiled draft for a contract file. For a draft it
160
+ already filed, it still returns the issue.
161
+ - `status`, `run status`, `destroy`, `cleanup`, and `db schema` still run while
162
+ leftover files remain.
163
+ - Stored runs whose suites are `contract` on `node-vitest-contract` stay
164
+ readable through `testkit artifacts`, `ci report`, and the workspace API.
165
+
166
+ Migrate each contract file:
167
+
168
+ | In a contract file | In the plain Vitest file |
169
+ | ------------------------------------------------------------------------------------ | ------------------------------------------------------------------- |
170
+ | `import { contractScenario, expect } from "@gfdelarue/testkit/node-vitest/contract"` | `import { expect, test } from "vitest"` |
171
+ | `contractScenario(name, run)` | `test(name, run)` |
172
+ | `contractScenario(name, { timeoutMs }, run)` | `test(name, run, timeoutMs)` |
173
+ | `export const testkit = defineFile({ engine: "node-vitest" })` | Delete it and its import |
174
+ | `name.contract.testkit.ts` | A name your Vitest config includes, such as `name.test.ts` |
175
+ | `name.contract.testkit.setup.mjs` | `name.test.setup.mjs` |
176
+ | `context.baseUrl` and `context.http()` | `endpoint(name)` from `@gfdelarue/testkit/environment`, and `fetch` |
177
+ | `context.env` | `process.env` |
178
+ | `context.setupData` | `setupData()` from `@gfdelarue/testkit/environment` |
179
+ | `context.fixture("a.json")` | `readFileSync(new URL("./__fixtures__/a.json", import.meta.url))` |
180
+ | `context.serviceName`, `task`, `engineId`, and `profileId` | Nothing. Vitest owns test identity. |
181
+
182
+ A plain file has no `defineFile` metadata. Move its `skip`, `locks`, and
183
+ `runtimeIsolation` to the group's `skip.files`, `requirements.files`, and
184
+ `runtimeIsolation.files` in `testkit.config.ts`. A plain file gets services only
185
+ from a declared group. See
186
+ [Remove legacy service ownership](#remove-legacy-service-ownership).
187
+
188
+ A typed file's tests and hooks could run until the file's timeout. Under the
189
+ repository's Vitest, a plain file gets Vitest's own limits: 5 seconds for a test
190
+ and 10 seconds for a hook, unless the Vitest config sets `testTimeout` and
191
+ `hookTimeout`. Give a slow test its timeout, as in `test(name, run, timeoutMs)`,
192
+ or set the limits in the config. Under `--vitest-runner managed`, Testkit sets
193
+ them to the remaining time of the Vitest invocation, which starts with the sum
194
+ of its files' timeouts.
195
+
196
+ Each typed file was its own task, so with the default `taskConcurrency` of 1,
197
+ typed files ran one at a time. Plain files that share a Vitest invocation run
198
+ with Vitest's file parallelism, which `taskConcurrency` doesn't limit. If
199
+ migrated tests can't run side by side, give each its own rows, or set
200
+ `fileParallelism: false` or `maxWorkers` in the Vitest config. Under
201
+ `--vitest-runner managed`, they run one at a time.
202
+
203
+ Renaming a test file changes the path that bug issues name. A failure is known
204
+ only when an open issue's `test.file` equals the failing file's path. After you
205
+ rename a file, edit `test.file` in the facts block of each open issue that
206
+ names it, so that `bugs.knownFailures: "allow"` still treats its failures as
207
+ known. Issues and bug drafts that record `testType: "contract"` stay readable.
208
+
209
+ ### Remove the data test type
210
+
211
+ This release removes the `data` test type. Its recording and budgets are now a
212
+ library that a plain Vitest test calls: `dataFlow(context)` from
213
+ `@gfdelarue/testkit/node-vitest/data`
214
+ ([gfdelarue/testkit#240](https://github.com/gfdelarue/testkit/issues/240)).
215
+ These are removed:
216
+
217
+ - the `*.data.testkit.ts` suffix, the `node-vitest-data` profile, and the data
218
+ runner
219
+ - `dataScenario`, `createHttpClient`, `readDataContext`, and the `expect`
220
+ re-export from `@gfdelarue/testkit/node-vitest/data`, with the types of the
221
+ scenario context, the HTTP client, its actors, and fixtures
222
+ - the `dataSuiteContract` lint rule
223
+ - HTTP profiles: `profiles.http` in `testkit.config.ts`, and
224
+ `registerRepoConfig`, `getRepoConfig`, `clearRepoConfig`, `RepoConfig`, the
225
+ `HttpProfile*` types, `HttpRequestParams`, `RuntimeCookie`, `RuntimeEnv`, and
226
+ `RuntimeHeaders` from `@gfdelarue/testkit/config`
227
+ - `*.data.testkit.ts` from the `testkitExcludes()` globs of
228
+ `@gfdelarue/testkit/tooling/vitest`
229
+ - the `description` field of a scenario in `data-flow.json`, which only
230
+ `dataScenario` options could set
231
+
232
+ Wherever you can name `data`, Testkit fails with a message that names the
233
+ release and this section:
234
+
235
+ ```text
236
+ __testkit__/flow.data.testkit.ts: Testkit 0.6.0 removed the data test type. Migrate with "Remove the data test type" in node_modules/@gfdelarue/testkit/docs/release-notes.md.
237
+ ```
238
+
239
+ Testkit stops in the same places as for `contract`, above. A config that sets
240
+ `profiles` fails to load, and `testkit lint` rejects a config that sets
241
+ `lint.rules.dataSuiteContract`, with messages that name this section. Stored
242
+ runs whose suites are `data` on `node-vitest-data`, and issues whose facts
243
+ block says `"testType": "data"`, stay readable.
244
+
245
+ Migrate each data file:
246
+
247
+ | In a data file | In the plain Vitest file |
248
+ | ------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------- |
249
+ | `import { dataScenario, expect } from "@gfdelarue/testkit/node-vitest/data"` | `import { dataFlow } from "@gfdelarue/testkit/node-vitest/data"` and `import { expect, test } from "vitest"` |
250
+ | `dataScenario(name, (flow) => { ... })` | `test(name, (context) => { const flow = dataFlow(context); ... })` |
251
+ | `dataScenario(name, { timeoutMs }, run)` | `test(name, run, timeoutMs)` |
252
+ | A helper that takes `DataScenarioContext` | A helper that takes `DataFlow` from `@gfdelarue/testkit/node-vitest/data` |
253
+ | `export const testkit = defineFile({ engine: "node-vitest" })` | Delete it and its import |
254
+ | `name.data.testkit.ts` | A name your Vitest config includes, such as `name.test.ts` |
255
+ | `flow.http()`, `flow.http({ profile })`, `createHttpClient`, and HTTP actor sessions | `fetch`, or your own client, against `endpoint(name)` from `@gfdelarue/testkit/environment`, then `flow.recordHttpRequest(...)` |
256
+ | `profiles.http` in `testkit.config.ts` | Log in from test code, and delete `profiles` |
257
+ | `flow.fixture(name)` | `readFileSync(...)`, and `flow.count(...)` and `flow.bytes(...)` if the test should record the file |
258
+ | `flow.baseUrl`, `flow.env`, and `readDataContext()` | `endpoint(name)` and `database(name)` from `@gfdelarue/testkit/environment`, and `process.env` |
259
+ | `flow.setupData` | `setupData()` from `@gfdelarue/testkit/environment` |
260
+ | `flow.serviceName` and `flow.task` | Nothing. Vitest owns test identity. |
261
+ | `lint.rules.dataSuiteContract` | Delete it |
262
+ | `--type data` or `--suite data:<name>` | `--type plain`, with `--group` or `--file` to narrow it |
263
+
264
+ A Shell test, before and after, abridged to one budget. Both versions leave out
265
+ the `record`, `bytes`, and `count` calls before it and the `message` option of
266
+ `equals`:
267
+
268
+ ```ts
269
+ // Before: src/browser/__testkit__/browser-profile-history-work.data.testkit.ts
270
+ import { defineFile } from "@gfdelarue/testkit";
271
+ import { dataScenario } from "@gfdelarue/testkit/node-vitest/data";
272
+
273
+ export const testkit = defineFile({ engine: "node-vitest" });
274
+
275
+ dataScenario("shell BrowserTab profile and history path stays model-owned", async (flow) => {
276
+ const measurement = await measureBrowserProfileHistoryWork();
277
+ flow.expectMetric("shell.browser-profile-history.profileModelFiles").equals(2);
278
+ });
279
+ ```
280
+
281
+ ```ts
282
+ // After: src/browser/__testkit__/browser-profile-history-work.test.ts
283
+ import { dataFlow } from "@gfdelarue/testkit/node-vitest/data";
284
+ import { test } from "vitest";
285
+
286
+ test("shell BrowserTab profile and history path stays model-owned", async (context) => {
287
+ const flow = dataFlow(context);
288
+ const measurement = await measureBrowserProfileHistoryWork();
289
+ flow.expectMetric("shell.browser-profile-history.profileModelFiles").equals(2);
290
+ });
291
+ ```
292
+
293
+ The migrated file keeps its metrics, budgets, `data-flow.json`,
294
+ `data-flow-report.md`, and `data-flow-budget` failure details. As for a contract
295
+ file, move its `defineFile` metadata to the group, give slow tests a timeout,
296
+ check that its tests can run side by side, and after renaming it, edit
297
+ `test.file` in the open issues that name it. See
298
+ [Remove the contract test type](#remove-the-contract-test-type).
299
+
300
+ Testkit no longer imports `testkit.config.ts` into any test process. A data test
301
+ that relied on a side effect of that import, such as a value the config set in
302
+ `process.env`, has to do that work itself.
303
+
304
+ ### A config with an unknown key fails to load
305
+
306
+ `testkit.config.ts` used to load with keys that Testkit doesn't read, so a
307
+ misspelled key such as `executoin` had no effect and no warning
308
+ ([gfdelarue/testkit#218](https://github.com/gfdelarue/testkit/issues/218)).
309
+ Testkit now refuses such a config when it loads it, whether the file calls
310
+ `defineConfig` or exports a plain object. The error names each unknown key, and
311
+ the closest known key when one is close:
312
+
313
+ ```text
314
+ testkit.config.ts has unknown keys "executoin" (mispelled "execution"?), "madeUpKey"; it accepts areas, artifacts, bugs, discovery, execution, fingerprints, groups, lint, redact, services, toolchains, ui
315
+ ```
316
+
317
+ The same check now runs inside these blocks:
318
+
319
+ - `discovery`, at the top level
320
+ - each `groups` entry
321
+ - `skip`, `requirements`, and `runtimeIsolation`, and the rules in their
322
+ `files` and `suites` lists, in a group
323
+ - `artifacts`, `artifacts.retention`, and `artifacts.publication`
324
+ - `fingerprints`
325
+ - each `toolchains` entry, and a service's inline `runtime.toolchain`
326
+
327
+ `execution`, `lint`, `bugs`, and each `areas` entry already rejected unknown
328
+ keys. A config that sets `profiles` still fails with the message in
329
+ [Remove the data test type](#remove-the-data-test-type).
330
+
331
+ A key that Testkit never read never took effect. Delete it, or move its value
332
+ to the key that applies. For example, `global: { fileTimeoutSeconds: 120 }`
333
+ never set a timeout; `execution: { fileTimeoutSeconds: 120 }` does.
334
+
335
+ ### Testkit no longer recognises the types and engine that 0.5.0 removed
336
+
337
+ 0.5.0 named the release and its migration when it found an `int`, `e2e`, `dal`,
338
+ or `load` file or selector, or a file that declared `engine: "k6"`. One minor
339
+ release later, those checks are gone:
340
+
341
+ - Discovery ignores a leftover `*.int.testkit.ts`, `*.e2e.testkit.ts`,
342
+ `*.dal.testkit.ts`, or `*.load.testkit.ts` file, and `discover`, `run`,
343
+ `lint`, `typecheck`, and `doctor` no longer reject it. If the repository's
344
+ Vitest config selects such a file, Testkit runs it as a plain Vitest file,
345
+ and `bug record` records a bug for it without a test type.
346
+ - `--type int`, `--suite int:<name>`, and `int:` selectors in `skip`,
347
+ `requirements`, and `runtimeIsolation` fail as unknown types, such as
348
+ `Unknown type "int"`. Unless the Vitest config selects such a file,
349
+ `run --file` with it fails because it matches no suite.
350
+ - Stored runs and bug drafts that name these types stay readable, and
351
+ `bug file` still returns the issue of a filed draft.
352
+ - A typed file that declares `engine: "k6"` fails as any unknown engine does:
353
+ `unsupported engine "k6"; expected one of: node-vitest, playwright`.
354
+
355
+ ## New behaviour
356
+
357
+ ### A plain Vitest file with per-file needs runs in its own invocation
358
+
359
+ In 0.5.0, a plain Vitest file with a setup module, locks that differ from the
360
+ other plain files', or isolation other than `shared` stopped the run when other
361
+ plain files ran with it. Such a file now runs in its own Vitest invocation, with
362
+ its own environment, beside the invocation that runs the rest. Its setup module
363
+ runs before its invocation starts, its locks hold for that invocation, and
364
+ `resetBefore` gives it its own database clone. Root global setup, teardown, and
365
+ the repository's reporters run once for each invocation.
366
+
367
+ ### A bare `testkit run` runs plain files from groups that use different services
368
+
369
+ In 0.5.0, a run stopped before it started anything when its plain Vitest files
370
+ belonged to groups that gave them different services or values
371
+ ([gfdelarue/testkit#378](https://github.com/gfdelarue/testkit/issues/378)). A
372
+ repository with unit tests in `root` and plain files in a service-backed group
373
+ couldn't pass the one-command CI gate the README recommends. Now plain files
374
+ run in one Vitest invocation per environment their groups give them, each
375
+ against its own group's services. Groups that give their files the same values
376
+ share one, such as `root` and the other groups that use no services. A group
377
+ that uses a service Testkit starts gets an invocation of its own. Root global setup, teardown, and the
378
+ repository's reporters run once for each invocation, so a run calls them once
379
+ more for each extra invocation. Invocations run one after another unless
380
+ `--task-concurrency` or `execution.taskConcurrency` is above 1.
381
+
382
+ ### `--type plain` selects only plain Vitest files
383
+
384
+ With `ui` the only test type, a run could select UI files with `--type ui` but
385
+ had no way to select plain Vitest test files without them
386
+ ([gfdelarue/testkit#380](https://github.com/gfdelarue/testkit/issues/380)).
387
+ `run` and `discover` now accept `--type plain`, which selects plain files only.
388
+ It narrows `--group`, `--suite`, and `--file` like any other type, so
389
+ `--group api --type plain` runs the `api` group's plain files and none of its UI
390
+ files. `testkit run plain` does the same as `testkit run --type plain`.
391
+
392
+ `--type ui,plain` selects the same files as `--type all`. A run whose `--type`
393
+ names `plain` reports it in `run.types`, as it does `ui`.
394
+
395
+ ### `typecheck` reads Vitest setup files and takes `--group`
396
+
397
+ `testkit typecheck` built the Node/Vitest program from the plain test files and
398
+ what they import. A file in the Vitest config's `setupFiles` is named only as a
399
+ string, so the program never reached it. Matchers that a setup file declares,
400
+ such as `toBeInTheDocument` from `@testing-library/jest-dom/vitest`, failed with
401
+ `TS2339` although `tsc` and `vitest run` accepted them
402
+ ([gfdelarue/testkit#379](https://github.com/gfdelarue/testkit/issues/379)).
403
+
404
+ Each plain file's program now includes the TypeScript setup files of its Vitest
405
+ project, and `typecheck` checks them like the test files. A setup file with a
406
+ type error now fails `typecheck`. Files whose projects load different setup
407
+ files go to separate programs, and the label of a program with setup files names
408
+ them, as in `node-vitest with tests/setup.ts`. A JavaScript setup file, or a package named
409
+ directly in `setupFiles`, still adds no types, so import the package from a
410
+ TypeScript setup file.
411
+
412
+ `typecheck --group <name>` checks the config and the files that group owns. A
413
+ group's setup files are those of the Vitest projects its files belong to.
414
+
415
+ ### `setupData()` reads a file's setup data
416
+
417
+ `setupData()` from `@gfdelarue/testkit/environment` returns the data the file's
418
+ setup module returned, or the `setupData` of a supplied environment file. It
419
+ works in plain Vitest files and Playwright files. A file
420
+ without setup data gets `{}`. Outside a Testkit run, `setupData()` throws.
421
+
422
+ ### A run's result no longer depends on post-run maintenance
423
+
424
+ After a run seals its evidence, Testkit tidies the product's local state:
425
+
426
+ - it removes other runs' stale manifests;
427
+ - it removes the product's unreferenced database containers and volumes;
428
+ - it stops the idle retained database.
429
+
430
+ That maintenance now runs after the seal and has its own result:
431
+
432
+ - A failure no longer changes the run's status, exit code, or sealed evidence. `testkit run` prints `Notice: Post-run maintenance failed: <reason>. The run's result is unchanged.` on stderr.
433
+ - `.testkit/results/maintenance.json` records the latest run's maintenance result. `testkit status` shows a warning while that result is failed.
434
+ - When Testkit can't read part of the `.testkit` tree to inventory its storage, `run.json` records `"storage": { "testkit": { "status": "failed", "error": "…" } }` and the run keeps its result.
435
+
436
+ A run that fails to release its own resources, such as dropping its task databases, still fails. A group whose tests passed now reports that failure as an infrastructure failure, `INFRA`, instead of a test failure, `FAIL`. So does a failure while sealing the run.
437
+
438
+ ### Retired-type errors name the installed release notes
439
+
440
+ The package now ships `docs/release-notes.md`. Each error that points at a
441
+ migration names the section and the installed file, such as
442
+ `Migrate with "Remove the contract test type" in node_modules/@gfdelarue/testkit/docs/release-notes.md`.
443
+ It used to name `docs/release-notes.md#…`, which the package didn't ship.
444
+
445
+ ## Fixes
446
+
447
+ ### `campaign finish` lists plain Vitest tests
448
+
449
+ The summary's "Testkit tests added or changed" list held only `*.testkit.ts`
450
+ files, so it left out the plain Vitest test files that Testkit has run since
451
+ 0.5.0. It now also lists changed files named like a Vitest test, such as
452
+ `refund.test.ts` or `refund.spec.js`.
453
+
454
+ ### `typecheck` uses the repository's `lib` for plain files
455
+
456
+ The Node/Vitest program set `lib` to ES2022 in place of the `lib` of the
457
+ repository's tsconfig. A plain file that runs in a jsdom or happy-dom
458
+ environment failed `typecheck` with `Cannot find name 'document'`, or, when a
459
+ package brought in the DOM library alone, with `TS2488` on a `NodeList` spread,
460
+ although the repository's own `tsc` accepted it
461
+ ([gfdelarue/testkit#398](https://github.com/gfdelarue/testkit/issues/398)).
462
+
463
+ A Node/Vitest program now uses the `lib` of the tsconfig it extends, and
464
+ ES2022 only when there is no tsconfig. A test that uses a library its tsconfig
465
+ leaves out now fails `typecheck` as it fails `tsc`.
466
+
467
+ ### A UI file's launch options keep managed Chromium
468
+
469
+ A UI file that set `launchOptions` with `test.use` lost the `executablePath`
470
+ of the Chromium that Testkit manages, because the file's value replaced the
471
+ config's. Playwright then started its headless shell where one was installed,
472
+ and failed to launch where none was. The headless shell never restores a page
473
+ from the back/forward cache, so a file couldn't test that
474
+ ([gfdelarue/testkit#373](https://github.com/gfdelarue/testkit/issues/373)).
475
+
476
+ Testkit now sets the managed `executablePath` after the file's launch options,
477
+ whether the file imports `test` from `@playwright/test`, `playwright/test`, or
478
+ `@gfdelarue/testkit/playwright`. The file's other options still apply.
479
+ [Browser launch options](engines.md#browser-launch-options) shows how a file
480
+ turns on the back/forward cache.
481
+
482
+ ### A group's `NODE_ENV` reaches plain files under the managed runner
483
+
484
+ Under `--vitest-runner managed`, a service that set a key Vitest or Vite writes
485
+ itself failed the Node/Vitest files in the groups that use it. Such keys
486
+ include `NODE_ENV`, `TEST`, `VITEST_MODE`, `SSR`, and a `VITE_` value that a
487
+ `.env` file also sets. With `NODE_ENV` or `VITEST_MODE`, each file took 60
488
+ seconds to fail, the time Vitest waits for a worker that never starts
489
+ ([gfdelarue/testkit#384](https://github.com/gfdelarue/testkit/issues/384)).
490
+ These keys now behave as under the repository's Vitest: tests see the group's
491
+ `NODE_ENV` and `VITE_` values, and Vitest's own `TEST`, `VITEST`,
492
+ `VITEST_MODE`, `FORCE_TTY`, `MODE`, `DEV`, `PROD`, and `SSR`.
493
+
494
+ When the Node/Vitest driver or a Vitest worker fails while it installs the
495
+ run's input, its files now fail at once with `NODE_VITEST_START_FAILED` and the
496
+ cause, instead of after Vitest's 60-second wait:
497
+
498
+ ```text
499
+ Vitest could not start: Node/Vitest runtime input size changed after preparation
500
+ ```
501
+
502
+ ### A UI file can import from a `.tsx` module that contains JSX
503
+
504
+ A UI file that imported anything from a `.tsx` module of the product that
505
+ contains JSX failed before its tests ran:
506
+
507
+ ```text
508
+ Error: Playwright consumers may not import managed package file outside exact public exports: playwright (resolved /…/node_modules/playwright/jsx-runtime from file:///…/src/view.tsx)
509
+ ```
510
+
511
+ Playwright compiles the module's JSX to import its runtime by an absolute path,
512
+ `<playwright>/jsx-runtime`, instead of by the specifier `playwright/jsx-runtime`
513
+ ([gfdelarue/testkit#390](https://github.com/gfdelarue/testkit/issues/390)).
514
+ Testkit now reads a path in a managed Playwright package that names one of the
515
+ package's exact exports as that export, so the runtime resolves to Testkit's
516
+ pinned Playwright.
517
+
518
+ ### A test's data-flow evidence comes from its final repeat
519
+
520
+ When a repository's `aroundEach` hook failed a later repeat of a test before it
521
+ called `runTest`, the test's `data-flow.json` entry and `data-flow-budget`
522
+ details came from the earlier repeat
523
+ ([gfdelarue/testkit#359](https://github.com/gfdelarue/testkit/issues/359)). Only
524
+ Vitest 4.1's `aroundEach` could do this. `dataFlow` now records the test's
525
+ configured repeats, and Testkit keeps a test's evidence only from its final
526
+ repeat.
527
+
528
+ ### A cancelled `sql-file` step stops
529
+
530
+ A `sql-file` database step ignored its caller's cancellation and its step
531
+ deadline, so its Postgres client ran until the SQL finished or the client's own
532
+ 30-minute timeout passed, even after the run was cancelled
533
+ ([gfdelarue/testkit#381](https://github.com/gfdelarue/testkit/issues/381)). It
534
+ now stops as `command` and `module` steps do, when its caller cancels it or its
535
+ step deadline passes, and its server query ends with it. The step deadline
536
+ defaults to 60 minutes, so a `sql-file` step that 0.5.0 stopped at 30 minutes
537
+ can now run for up to 60.
538
+
539
+ ### A cancelled admin query says when its server query may still run
540
+
541
+ When Testkit cancelled a database admin query that runs through `docker exec`,
542
+ or found it stalled, and then failed to end the query's server backend, the
543
+ error named only the cancellation while the server query kept running
544
+ ([gfdelarue/testkit#382](https://github.com/gfdelarue/testkit/issues/382)). The
545
+ error now also names the failed termination and its reason, and Testkit doesn't
546
+ retry that attempt.
547
+
548
+ # 0.5.0
549
+
550
+ ## Remove k6
551
+
552
+ This release removes the k6 engine and the `int`, `e2e`, `dal`, and `load` test
553
+ types. Testkit runs plain Vitest files, typed Vitest `contract` and `data`
554
+ files, and Playwright `ui` files.
555
+
556
+ Wherever you can name a retired type, Testkit fails with a message that names
557
+ the release and this section:
558
+
559
+ ```text
560
+ __testkit__/health.int.testkit.ts: Testkit 0.5.0 removed the int test type. Migrate with docs/release-notes.md#remove-k6.
561
+ ```
562
+
563
+ - Discovery rejects a file of a retired type unless a group excludes it. A
564
+ file that still declares `engine: "k6"` fails the same way, naming the k6
565
+ engine.
566
+ - `--type`, `--suite`, and the selectors in `skip`, `requirements`, and
567
+ `runtimeIsolation` reject a retired type.
568
+ - `lint`, `typecheck`, and `doctor` stop on a retired file with the same
569
+ message.
570
+ - `bug file` refuses an unfiled draft for a retired type. For a draft it
571
+ already filed, it still returns the issue.
572
+ - `status`, `run status`, `destroy`, `cleanup`, and `db schema` still run while
573
+ retired files remain, so you can inspect and clean up local state before you
574
+ migrate.
575
+ - Stored runs and bug drafts that name a retired type stay readable.
576
+
577
+ These are removed with the engine:
578
+
579
+ - the `@gfdelarue/testkit/k6` export and the k6 binary packages
580
+ - the k6 HTTP and DAL helpers and the k6 execution options
581
+ - the synchronous response types `RuntimeHttpClient`, `RuntimeResponse*`,
582
+ `RuntimeHttpTrace*`, and `RuntimeHeaderValue`, from the
583
+ `@gfdelarue/testkit/config` and `@gfdelarue/testkit/node-vitest/data`
584
+ declarations
585
+ - the `dalParallelSafety` lint rule. Delete it from `lint.rules`, because
586
+ Testkit rejects an unknown rule.
587
+
588
+ To migrate, move HTTP, database, and service-flow checks to plain Vitest files.
589
+ Read named endpoints and databases from `@gfdelarue/testkit/environment`, and
590
+ use `fetch` or your application's database client. `contract` and `data` files
591
+ keep their APIs, HTTP profiles still configure the `data` HTTP clients, and
592
+ `ui` files still run on Playwright. Testkit has no replacement for load tests.
593
+
594
+ ## Plain Vitest test files run
595
+
596
+ Testkit runs a repository's ordinary Vitest test files, such as
597
+ `src/sum.test.ts` and `src/sum.spec.ts`, as they are
598
+ ([gfdelarue/testkit#237](https://github.com/gfdelarue/testkit/issues/237)):
599
+
600
+ - A plain file needs no Testkit suffix, `__testkit__` directory, `defineFile`,
601
+ or wrapper, and keeps its native `vitest` imports.
602
+ - Testkit finds plain files the way the repository's Vitest config selects
603
+ them, as the next section describes.
604
+ - `discovery.exclude` and a declared group's `exclude` narrow the list.
605
+ Testkit's built-in exclusions, `discovery.roots`, and a legacy service's
606
+ rules apply only to typed files, so a legacy service's `exclude` doesn't drop
607
+ a plain file.
608
+ - A plain file belongs to the implicit `root` group unless a declared group's
609
+ roots claim it, and it gets the values of the services its group uses.
610
+ Testkit starts no service for a plain file in `root`.
611
+ - Plain files have no test type. They run under the general `node-vitest`
612
+ profile, and a bug filed from one records no `testType`. A `--type` that
613
+ names test types and typed suite selectors leave them out; `--type all`
614
+ includes them.
615
+ - The plain files in one directory form a suite named after that directory,
616
+ relative to the group's root, and a file directly in a group's root names
617
+ its own suite. In `discover --json` plain files have no `testType`, and their
618
+ IDs have no type segment: `root|node-vitest|node-vitest|src/sum.test.ts`.
619
+ `summary.byTestType` counts typed files only. `testkit discover` lists them
620
+ under `Vitest`.
621
+ - `typecheck` checks plain TypeScript test files and skips plain JavaScript
622
+ ones.
623
+ - `unitTestBoundaries` stays off by default and only reports. Its messages no
624
+ longer tell a unit test to move to a Testkit suite.
625
+
626
+ A bare `testkit run` in a repository with ordinary Vitest files now runs them
627
+ too. To run typed files only, name their types:
628
+ `--type data,contract,ui`.
629
+
630
+ ## Plain Vitest test files run with the repository's Vitest
631
+
632
+ Plain files run with the repository's own Vitest installation and config
633
+ ([gfdelarue/testkit#238](https://github.com/gfdelarue/testkit/issues/238)):
634
+
635
+ - Testkit loads the root `vitest.config.*` or `vite.config.*` as `vitest run`
636
+ does from the product directory, and lists the files its projects select
637
+ without running global setup. Nested configs and package scripts have no
638
+ effect. The notice that Testkit ignored a root Vitest config is gone.
639
+ - `discover`, `run`, `typecheck`, `bug record`, and `bug file` take
640
+ `--vitest-config <path>`, `--vitest-project <name>`, and
641
+ `--vitest-runner repo|managed`. `managed` keeps the previous behaviour:
642
+ Testkit's own Vitest with its default rules and no repository config.
643
+ - Testkit supports Vitest 4.x. Another major, a broken installation, a root
644
+ Vitest config that fails to load, or a Vitest that the product declares but
645
+ has not installed fails before any test runs, with a message that names the
646
+ path. Most of these messages also suggest `--vitest-runner managed`. A run
647
+ that selects only typed files, with `--type` or with typed `--file` paths,
648
+ `bug record` or `bug file` for a typed test, and the commands that select no
649
+ tests (`status`, `run status`, `doctor`, `destroy`, `cleanup`, and
650
+ `db schema`) don't load the repository's Vitest, so they still work. A
651
+ product that neither installs nor declares Vitest has no plain files.
652
+ - All plain files in a run go to one Vitest invocation. Root global setup,
653
+ teardown, and the repository's reporters run as often as they do under
654
+ `vitest run`, and Testkit's reporter runs beside them. The invocation's
655
+ deadline is the sum of its files' timeouts, and Vitest's own test and hook
656
+ timeouts apply inside it. A file that two projects select runs in both, and
657
+ its evidence names each run's project. Files from different groups run
658
+ together when their groups give them the same services, values, and locks.
659
+ Different services or locks, isolation other than `shared`, or a setup
660
+ module stop the run before Testkit provisions anything; different values
661
+ stop it before Vitest starts. When `testkit stop` cuts the invocation short,
662
+ every file that hasn't finished, started or not, is cancelled.
663
+ - Testkit sets only `allowOnly: false`, `includeTaskLocation: true`,
664
+ `passWithNoTests: false`, and `update: "none"` over the repository's config,
665
+ so a plain file's failure `location` has a line and column. A project that
666
+ sets `includeTaskLocation: false` itself keeps that setting.
667
+ - Plain files see the host environment Testkit started with, from config
668
+ loading to the test, including `FORCE_COLOR`, `NO_COLOR`, and the shell's
669
+ `NODE_OPTIONS`, to which Testkit adds its own `--import`. Unlike
670
+ `vitest run`, ambient `BASE_URL` and `DATABASE_URL` never reach them; those
671
+ come only from the group's environment. Testkit loads no env files for them.
672
+ - Node/Vitest processes no longer receive `TESTKIT_INTERNAL_FILE_DEADLINE_MS`
673
+ or `TESTKIT_INTERNAL_FILE_TIMEOUT_SECONDS`, which nothing in them read.
674
+ - Testkit's own output honours `NO_COLOR` in a terminal and treats
675
+ `FORCE_COLOR=0` or `FORCE_COLOR=false` as no colour, as Node does.
676
+ - Node/Vitest reserves the exact keys it sets itself instead of the whole
677
+ `TESTKIT_` and `VITEST_` prefixes, so a policy key such as
678
+ `TESTKIT_REQUIRE_COMPLETE_SUITE` reaches the run from either environment
679
+ source.
680
+ - `bug record` saves the Vitest selectors it was given in the local draft, and
681
+ asks for `--vitest-project` when two projects run the test's file.
682
+ `bug file` reruns with them, and the issue's reproduce command carries them.
683
+ With `TESTKIT_REQUIRE_COMPLETE_SUITE=1`, `bug file` reruns every file the
684
+ selection lists before it reads the recorded failure.
685
+ - Typed node-vitest files keep Testkit's own Vitest and its authoring entry
686
+ points. Testkit no longer checks which packages their imports resolve to.
687
+ - Typed node-vitest files now resolve `paths` from their nearest
688
+ `tsconfig.json`, through Vite, instead of from the product root's
689
+ `tsconfig.json` or `tsconfig.build.json`. A repository whose `paths` are only
690
+ in `tsconfig.build.json` must move them into a `tsconfig.json`.
691
+ - Engine evidence names only the Vitest version in `directPackages`, and its
692
+ `installationKind` is `repository-package` when the repository's Vitest ran.
693
+
694
+ ## A run that stops on an error names it
695
+
696
+ `testkit run` names an error that stops the whole run rather than one test
697
+ file, such as the planning refusal for plain Vitest files that cannot share
698
+ one invocation (#352). Before, the default output showed only
699
+ `Runtime errors: N`, and the message needed `--output-mode debug` or `--json`.
700
+
701
+ - The text output prints `FAIL run error` and the message when the error
702
+ happens.
703
+ - `--output-mode events` writes a `run.error` event with `message`, once for
704
+ each error, not once for each group.
705
+ - The debug summary labels each group's error `error:` instead of
706
+ `worker error:`.
707
+
708
+ ## A failed coverage threshold fails the run, not each file
709
+
710
+ When the repository's Vitest config enables coverage with thresholds and the
711
+ coverage falls short, Vitest exits 1 after its tests pass. Testkit used to
712
+ mark every file in the run as an unexplained infrastructure failure. Each file
713
+ now keeps its own outcome, and the run fails with a run error that carries
714
+ Vitest's message:
715
+
716
+ ```text
717
+ FAIL run error
718
+ Vitest failed the run after its tests passed: ERROR: Coverage for lines (50%) does not meet global threshold (100%)
719
+ ```
720
+
721
+ The thresholds apply to the files Testkit runs, as they do to
722
+ `vitest run <file>`, so `testkit run --file` can fail on coverage while the
723
+ whole suite passes.
724
+
725
+ ## Record data-flow evidence in a plain Vitest test
726
+
727
+ `dataFlow(context)` from `@gfdelarue/testkit/node-vitest/data` gives a plain
728
+ Vitest test the recorder that `dataScenario()` passes to its callback:
729
+
730
+ ```ts
731
+ import { test } from "vitest";
732
+ import { dataFlow } from "@gfdelarue/testkit/node-vitest/data";
733
+
734
+ test("lists one page of orders", async (context) => {
735
+ const flow = dataFlow(context);
736
+ flow.count("orders.queries", 1);
737
+ flow.budget("orders.queries").atMost(1);
738
+ });
739
+ ```
740
+
741
+ - Every call in one attempt returns the same recorder. Without the context of
742
+ a running test, `dataFlow` throws and says how to call it.
743
+ - A failed budget fails the test that declared it when the test finishes.
744
+ - The file's `data-flow.json` lists one scenario for each test that recorded
745
+ data, named by the test's full title and taken from its final attempt. A
746
+ skipped test, or one that records nothing, adds no scenario.
747
+ - `dataScenario()` now runs on `dataFlow()`. A scenario inside a `describe`
748
+ block is named by its full title, the `describe` names and its own name
749
+ joined by spaces. It used to have its own name only.
750
+ - Typed `data` files still register every case with `dataScenario()`.
751
+
752
+ ## Readiness leaves no connection open
753
+
754
+ Testkit no longer holds a connection to a service after its readiness check.
755
+ The check used `fetch`, and when it gave up on a slow attempt and tried again,
756
+ the connection pool could open a connection that never carried a request. A
757
+ Node service that calls `server.close()` on SIGTERM waits for that connection,
758
+ so it ran past its `local.shutdownTimeoutMs` grace, one second by default, and
759
+ Testkit killed it with SIGKILL. Its shutdown work, such as flushing logs, did
760
+ not finish.
761
+
762
+ The readiness check now opens its own connections without keep-alive and
763
+ closes all of them when it returns, whether the service became ready, timed
764
+ out, exited, or the run was cancelled. It sends the headers `fetch` sent, with
765
+ `Connection: close`. It still follows up to 20 redirects and still needs a 2xx
766
+ response. A redirect fails the attempt when its `Location` holds a username or
767
+ password, as with `fetch`. A redirect with more than one `Location` also fails
768
+ the attempt. `fetch` joined the values into one URL instead.
769
+
770
+ Other changes:
771
+
772
+ - A `readyUrl` that holds a username and password now sends them as Basic
773
+ authentication, including after a relative redirect. `fetch` refused such a
774
+ URL, so readiness used to time out.
775
+ - A `readyUrl` on a port that `fetch` blocks, such as 6666, now works.
776
+ - Readiness connects to the service directly. With `NODE_USE_ENV_PROXY` or
777
+ `--use-env-proxy`, it no longer goes through `HTTP_PROXY` or `HTTPS_PROXY`.
778
+
779
+ ## Every service URL answers HTTP before tests start
780
+
781
+ A local service's `baseUrl` now has to answer HTTP before any test receives it.
782
+ Any response counts, including a 4xx or 5xx status, because only the test knows
783
+ which status is wrong. A port that accepts connections but does not answer HTTP
784
+ does not count. Testkit retries until `readyTimeoutMs` runs out and then fails
785
+ the service's startup, so its tasks end `not_run` with an infrastructure
786
+ failure.
787
+
788
+ - A service whose `readyUrl` has the same origin as its `baseUrl` is checked
789
+ only at its `readyUrl`, which still needs a 2xx response. `app.node` and
790
+ `app.next` default to such a `readyUrl`.
791
+ - `readyUrl: false` used to mean the service was ready once it had started. It
792
+ now removes only the 2xx check, and `baseUrl` must still answer HTTP.
793
+ - `local.baseUrl` is optional. A process without an HTTP endpoint, such as a
794
+ worker, declares none: `worker: { local: { start: "node worker.mjs" } }`. It is
795
+ ready once it has started, and its groups get no URL for it.
796
+ `{baseUrl:worker}` fails to resolve, and `run.json` records its `baseUrl` as
797
+ `null`.
798
+ - A `readyUrl` that is not an HTTP URL fails startup at once instead of at
799
+ `readyTimeoutMs`.
800
+
801
+ This breaks a config whose local service never answers HTTP at its `baseUrl`,
802
+ including one with `readyUrl: false`. Delete the service's `baseUrl`. `app.node`
803
+ and `app.next` always set one, so declare such a service as a plain `local`
804
+ block instead.
805
+
806
+ Playwright tasks no longer check their URL themselves. That check opened a TCP
807
+ connection, only for a URL outside the local machine, and ran inside the file
808
+ timeout. The `TESTKIT_HTTP_ENDPOINT_PREFLIGHT` variables and the
809
+ `http_endpoint_readiness` failure detail are gone.
810
+
811
+ ## A port that another process answers no longer counts as ready
812
+
813
+ Testkit releases a service's port just before it starts the service. If another
814
+ process took the port in between and answered HTTP, readiness passed. The
815
+ service then failed to bind and exited, and the run stopped
816
+ ([gfdelarue/testkit#339](https://github.com/gfdelarue/testkit/issues/339)).
817
+ After readiness passes, Testkit now checks that every listener on the port
818
+ belongs to the service. If one doesn't, Testkit stops the service and starts it
819
+ again on a fresh port, within the same three attempts, and records the attempt
820
+ as `another process held the port`. The check runs on Linux and Windows. On
821
+ macOS, and for a listener Testkit can't attribute, readiness counts as before.
822
+
823
+ ## A slow database probe no longer ends readiness
824
+
825
+ Testkit gives a local Postgres container 60 seconds to accept connections and
826
+ probes it with `docker exec pg_isready`, at most 10 seconds per probe. A probe
827
+ that ran past its 10 seconds used to fail the database at once with `Docker
828
+ local Postgres readiness probe ... exceeded its managed deadline`, even with
829
+ most of the 60 seconds left. A busy Docker daemon can make one probe that slow.
830
+ That probe now counts as one failed attempt, and Testkit keeps probing until the
831
+ database answers or the 60 seconds run out. Cancelling the run still stops
832
+ readiness at once.
833
+
834
+ ## A slow database query is no longer cut off at 15 seconds
835
+
836
+ Testkit gave each `docker exec psql` admin query 15 seconds, then failed its
837
+ operation with `Docker Postgres admin query ... exceeded its managed deadline`.
838
+ A `CREATE DATABASE` that waited for a lock on its template, or ran slowly on a
839
+ busy Docker host, failed this way while Postgres was still doing the work
840
+ ([gfdelarue/testkit#334](https://github.com/gfdelarue/testkit/issues/334)).
841
+ A query now runs for as long as its Postgres backend is active. Every 15
842
+ seconds Testkit looks for the query's session in `pg_stat_activity`. When two
843
+ checks in a row miss it, or a check fails, Testkit stops the query, terminates
844
+ its backend, and reports the same error.
845
+
846
+ ## Docker reads go to the CLI after a direct read times out
847
+
848
+ Each Docker metadata operation reads the engine directly for up to 5 seconds,
849
+ then runs the docker CLI for the rest of its deadline. Once a direct read times
850
+ out, later operations in the process go straight to the CLI, instead of losing
851
+ another 5 seconds each. A deadline error now says how long each step took:
852
+
853
+ ```text
854
+ Docker container inventory exceeded its managed deadline: the direct read took 5.0 s, then the docker CLI ran 25.0 s
855
+ ```
856
+
857
+ ## Node/Vitest failures keep their errors
858
+
859
+ These changes apply to every node-vitest file
860
+ ([gfdelarue/testkit#237](https://github.com/gfdelarue/testkit/issues/237)):
861
+
862
+ - A file whose import throws now fails with that error as a `vitest-error`,
863
+ instead of `NODE_VITEST_NO_TESTS`. A file that defines no tests still fails
864
+ with `NODE_VITEST_NO_TESTS`.
865
+ - A `vitest-error` from an import or a suite-level hook such as `beforeAll`
866
+ keeps the error's stack in `stack`, and its `message` now starts with the
867
+ error's name, such as `TypeError:`.
868
+ - A `vitest-test` failure's `message` is the error's name and its whole
869
+ message, as Vitest prints them, such as
870
+ `AssertionError: expected 2 to be 3 // Object.is equality`. The stack moves
871
+ to `stack`. The message used to be the error's stack text. For a test that
872
+ hits its own timeout, that text didn't contain "Test timed out".
873
+ - After retries, a `vitest-test` failure describes the final attempt. The
874
+ earlier attempts' errors stay in the native report.
875
+ - A `vitest-test` failure's `location` gives the line and column where the
876
+ test is declared, beside its path. It used to give the path alone.
877
+ - A missing or invalid native report is an infrastructure failure,
878
+ `NODE_VITEST_REPORT_INVALID`, whatever Vitest's exit code. A nonzero exit
879
+ used to make it a product failure.
880
+ - The `testkit-vitest-report` artifact lists every case with its state and the
881
+ number of attempts Vitest made, so a retried test's attempts are in the run's
882
+ evidence.
883
+
884
+ ## `bug record` and `bug file` check the test a bug names
885
+
886
+ - `bug record` records a bug only for a test that Testkit discovers. It used
887
+ to accept any existing file whose name gave a type, so a draft could name a
888
+ file that `bug file` could not run. A misplaced typed file now stops
889
+ `bug record` with discovery's own error, as it stops `testkit run`.
890
+ - A draft records `testType` only when the test file's name gives one.
891
+ - `bug file` refuses a node-vitest test when another test in the same file has
892
+ the same full title, whether that test passed or failed. An issue names its
893
+ test by file and title, so it could not tell the two apart. Give each test
894
+ its own title, then record the bug again
895
+ ([gfdelarue/testkit#237](https://github.com/gfdelarue/testkit/issues/237)).
896
+
897
+ ## `cleanup --dir` cleans the product it names
898
+
899
+ `testkit cleanup --dir` cleans the product that `--dir` names when that product
900
+ has no config. It used to clean the directory it was run from. A run with
901
+ nothing to run, from a product with no config, wrote its results there too
902
+ ([gfdelarue/testkit#322](https://github.com/gfdelarue/testkit/issues/322)).
903
+
904
+ ## Env files are read only for the services a run uses
905
+
906
+ Loading `testkit.config.ts` no longer reads any env file. Testkit reads a
907
+ service's `envFile`, `envFiles`, or default `.env.testkit` and `.env` files only
908
+ when a selected group uses the service or a used service depends on it. A
909
+ missing or unreadable file of an unused service no longer stops a run. File
910
+ order and precedence are unchanged. A later file overrides an earlier one, `env`
911
+ overrides every file, `local.env` overrides `env`, and a group takes each name
912
+ from the first service in its `uses` that sets it.
913
+
914
+ - A service that declares only `env` or env files starts nothing, and neither
915
+ does a `dependsOn` edge between such services. Their groups get the values
916
+ with no runtime slot, `testkit.runtime.ensure`, graph directory, or teardown.
917
+ Such a service used to take a runtime slot.
918
+ - A group that uses a service now gets the config's `ui` settings even when its
919
+ services declare nothing else. It used to get them only when a service
920
+ declared env values or a dependency.
921
+ - Because nothing starts for it, such a service's `env` and `ui` can use only
922
+ `{service}`, `{stateDir}`, `{productDir}`, `{lease}`, `{leaseDir}`, `{bin}`,
923
+ and `{pathList}`. `{runtime}` and `{prepareDir}` used to resolve against the
924
+ runtime it no longer takes. They and every port, URL, and database token now
925
+ fail the group's tasks with an infrastructure failure that names the token.
926
+ - `defineTestkitConfig` from `@gfdelarue/testkit/playwright` no longer loads
927
+ dotenv files. It used to load `.env.local` outside a Testkit run, and also
928
+ inside one with `onlyWhenAllowed: false`. Its `cwd`, `dotenvFiles`,
929
+ `onlyWhenAllowed`, and `override` options are removed, and passing one throws.
930
+ In a Testkit run, the services a group uses have already provided their
931
+ values when the config loads. To load a file when you run Playwright on its
932
+ own, do it in `playwright.config.ts` with a dotenv library of your own.
933
+ `@gfdelarue/testkit/env` does not resolve inside a managed Playwright run, so
934
+ importing it there fails the run.
935
+ - `archive.declaredEnvironmentKeys` in `run.json` and the bundle provenance lists
936
+ the names a used service declares in `env`. It no longer includes names that
937
+ only an env file sets.
938
+
939
+ ## Test groups
940
+
941
+ `testkit.config.ts` takes a `groups` key. A group owns the test files under its
942
+ `roots` and names the services its tests use:
943
+
944
+ ```ts
945
+ export default defineConfig({
946
+ services: {
947
+ api: app.node({ cwd: "services/api", start: "node dist/server.js" }),
948
+ web: app.next({ cwd: "apps/web", dependsOn: ["api"] }),
949
+ },
950
+ groups: {
951
+ api: { roots: ["services/api"], uses: ["api"] },
952
+ web: { roots: ["apps/web"], uses: ["web", "api"] },
953
+ },
954
+ });
955
+ ```
956
+
957
+ - A file belongs to the group with the deepest root that contains it. Files
958
+ that no group claims belong to the implicit group `root`, which uses no
959
+ service. With no config, every file is in `root`. In a repository without
960
+ services, discovery, file IDs, and run results now name `root` where they
961
+ named `app`.
962
+ - A test gets values only from the services its group uses. When two of them
963
+ set the same name, such as `BASE_URL`, `DATABASE_URL`, or an `env` entry, the
964
+ first in `uses` wins.
965
+ - `skip`, `requirements`, and `runtimeIsolation` belong to the group. Beside
966
+ `groups`, Testkit rejects them on a service with a message naming the group
967
+ field to use, and it ignores a service's `discovery`. `groups: {}` gives
968
+ every file to `root`.
969
+
970
+ A config with `services` and no `groups` key keeps legacy ownership: each
971
+ service owns the typed files under its `local.cwd` and `discovery.roots`. A
972
+ typed file outside every service root now belongs to `root`; discovery used to
973
+ fail it with `unowned_test`. When legacy ownership gives a file to a service,
974
+ `run`, `discover`, `workspace`, and `typecheck` print a notice once on stderr.
975
+ Legacy ownership and the `--service` alias on test-selecting commands will be
976
+ removed
977
+ ([gfdelarue/testkit#241](https://github.com/gfdelarue/testkit/issues/241)).
978
+
979
+ `run`, `discover`, `artifacts`, and `workspace` take `--group <name>`. On those
980
+ four commands `--service` still works as an alias for `--group` and prints a
981
+ notice. Passing both with different names is a usage error. Other commands,
982
+ such as `status`, `destroy`, `cleanup`, and `db schema`, keep `--service` for
983
+ services. `workspace --group` filters discovery and targets but not service
984
+ status, which `workspace --service` used to filter.
985
+
986
+ `discover --json` is schema 10:
987
+
988
+ - `services[]` is `groups[]`. Each group has `name`, `roots`, `uses`, `source`
989
+ (`groups`, `services`, or `implicit`), `suiteCount`, `fileCount`,
990
+ `activeFileCount`, and `skippedFileCount`. `localCwd`, `dependsOn`, and
991
+ `discovered` are gone. `root` is always listed.
992
+ - On files and suites, `service` is `group` and `groupLabel` is `suiteLabel`.
993
+ `dependsOn` and `contract.executionHost` are gone.
994
+ - `filters.service` is `filters.group`, `summary.services` is
995
+ `summary.groups`, and `summary.byService` is `summary.byGroup`.
996
+ `ambiguous_test` diagnostics list `groupNames`. `unowned_test` no longer
997
+ occurs.
998
+ - File and suite IDs keep their format, with the group name first.
999
+
1000
+ `workspace --json` is schema 3. It embeds the schema 10 summary and
1001
+ diagnostics, and its targets carry `group` in place of `service`.
1002
+ `@gfdelarue/testkit/workspace` exports `DiscoveryGroup` in place of
1003
+ `DiscoveryService`, and `discoverTests` and `inspectWorkspace` take a `group`
1004
+ option in place of `service`.
1005
+
1006
+ ## Test ownership
1007
+
1008
+ `testkit discover` and the loader now assign test files to owners through one
1009
+ resolver, which `groups` builds on (#233). Under legacy ownership, owner names
1010
+ and file IDs stay the same, with these changes:
1011
+
1012
+ - `run` and `typecheck` find a Playwright config by walking up from the common
1013
+ parent of the owning service's discovery roots to the product directory. They
1014
+ used to look only in the service's rendered `local.cwd`. A service with
1015
+ `discovery: { roots: ["apps/web/e2e"] }` now uses
1016
+ `apps/web/playwright.config.ts`.
1017
+ - A service named `root` is rejected and must be renamed, because `root` is
1018
+ the implicit group that owns files no other group claims.
1019
+ - A `discovery.exclude` entry that names a single file now drops that file in
1020
+ every mode. Before, it was ignored when the config declared no services, and
1021
+ with services a file outside every service root was reported as
1022
+ `unowned_test`.
1023
+ - A file that two discovery roots of the same service both claim is no longer
1024
+ ambiguous. With `local.cwd: "api"` and `discovery.roots: [".", "../api"]`, a
1025
+ file under `api` used to be reported as `ambiguous_test` naming `api, api`.
1026
+ It now belongs to `api`.
1027
+ - A file that overlapping repository `discovery.roots` both reach, such as
1028
+ `[".", "src"]`, is listed once. Its suite used to list it twice.
1029
+ - Two suites in a group and type no longer get the same name. When short names
1030
+ collided, the fallback could give a suite a numbered name that another
1031
+ directory already had, such as `a-b-2`, and merge the two suites.
1032
+
1033
+ A run builds its tasks from test groups and starts only the services those
1034
+ groups use, with their `dependsOn`. Under legacy ownership each service owns
1035
+ its tests as a group that uses that service, so a run starts the same services
1036
+ as before. Two things change:
1037
+
1038
+ - A task gets values only from the services its group uses. Its endpoints no
1039
+ longer include a `dependsOn` service, so a Playwright test's
1040
+ `readPlaywrightContext().endpoints` lists only its own service. A used
1041
+ service can still pass a dependency's URL on through `env`, for example with
1042
+ `endpoint.serviceContextUrl("api")`.
1043
+ - A service whose selected files are all skipped no longer reserves a runtime
1044
+ instance or database capacity. `run.json` leaves it out of
1045
+ `environment.capacity.instanceCount` and
1046
+ `environment.capacity.resourceBudgets`.
1047
+
1048
+ ## Locks left by a dead process
1049
+
1050
+ Testkit runs, database work, and Node toolchain installs take locks that other
1051
+ Testkit processes on the host respect. A process that dies while it holds or
1052
+ waits for one of these locks leaves a claim behind, and the next process to find
1053
+ the claim removes it. When several processes found the same dead claim, each of
1054
+ them deleted it. On Windows, a process could get `EPERM` while another was still
1055
+ deleting the claim, and its command failed
1056
+ ([gfdelarue/testkit#255](https://github.com/gfdelarue/testkit/issues/255)).
1057
+
1058
+ A process now renames a dead claim out of the lock directory before it deletes
1059
+ it, and only the process whose rename succeeds deletes the claim. The others
1060
+ treat the claim as gone. Two ordinary renames of one directory can both succeed
1061
+ on Windows, so there Testkit's Windows helper makes the rename through a handle
1062
+ that keeps every other process from renaming or deleting the claim until the
1063
+ move is done. A permission error still fails the command. On Windows, another
1064
+ process reading the claim can block the rename for a moment, so Testkit retries
1065
+ it for up to about ten seconds first. A process that dies
1066
+ between the rename and the deletion leaves a `<lock>.dir.reclaiming-<id>`
1067
+ directory next to the lock. It no longer affects the lock, and you can delete
1068
+ it.
1069
+
1070
+ ## Windows locks wait out a briefly held file
1071
+
1072
+ On Windows, an antivirus scanner, an indexer, or another Testkit process
1073
+ checking a lock can hold a lock file open for a moment. Releasing a lock, and
1074
+ creating or removing a lock's directory, failed on the first such `EPERM`,
1075
+ `EBUSY` or `EACCES` error
1076
+ ([gfdelarue/testkit#270](https://github.com/gfdelarue/testkit/issues/270)).
1077
+ A failed release left the lock held until its process exited, so in a
1078
+ long-running process every later wait for that lock timed out. A new
1079
+ acquisition also failed while the previous lock directory was still being
1080
+ deleted.
1081
+
1082
+ These steps now retry for about one and a half seconds on those errors and
1083
+ throw if the error persists. Release still refuses to remove a lock whose owner
1084
+ or ticket has changed, and on Windows it checks that again before every
1085
+ attempt. It then moves the claim out of the lock directory before deleting it,
1086
+ so a process that dies in between leaves a `<lock>.dir.releasing-<id>`
1087
+ directory that no longer affects the lock and that you can delete. Linux and
1088
+ macOS behave as before.
1089
+
1090
+ ## Database bindings stay in memory
1091
+
1092
+ Testkit used to write each runtime database's URL to a `database_url` file in
1093
+ the runtime's state directory, then read the file back for `runtime.prepare`
1094
+ steps, service processes, `{db*}` placeholders, and test values. Each runtime
1095
+ instance now keeps the binding that provisioning returns and passes it to those
1096
+ consumers directly
1097
+ ([gfdelarue/testkit#140](https://github.com/gfdelarue/testkit/issues/140)).
1098
+
1099
+ - `databaseFrom` now supplies a database. A service with
1100
+ `databaseFrom: "api"` gets `api`'s database from the same runtime instance as
1101
+ `DATABASE_URL`, in `{dbUrl}` and the other database placeholders, and in its
1102
+ `runtime.prepare` steps and fingerprint. It used to get no database at all.
1103
+ The named service must provision its database in the same runtime, so list it
1104
+ in `dependsOn`. Otherwise the run fails with an error that says so.
1105
+ - Testkit no longer writes `database_url` or `local_template_database_name` to
1106
+ runtime state directories under `.testkit/_graphs`, and ignores copies that
1107
+ an earlier version left there. A script that read `database_url` should read
1108
+ `DATABASE_URL` or a `{dbUrl}` env value instead.
1109
+ - A database template counts as cached only after its rebuild finishes.
1110
+ Testkit removes the template's cached fingerprint before a rebuild and writes
1111
+ it again at the end. If a process was killed during a rebuild, the old
1112
+ fingerprint used to stay next to a partly built template, and the next run
1113
+ cloned that template.
1114
+
1115
+ ## A file with setup runs in its own acquisition
1116
+
1117
+ Testkit now runs a file's `.setup.mjs`, `.setup.js`, or `__setup__` module as
1118
+ part of preparing the file's environment, after it acquires the services the
1119
+ group uses. Setup still runs once per file, shares the file's deadline, and
1120
+ keeps its sidecars until the test finishes. Cleanup failures keep their
1121
+ classification: a consumer cleanup error fails the test, and a Testkit cleanup
1122
+ error is an infrastructure failure.
1123
+
1124
+ A Playwright file with setup no longer shares a batch with other Playwright
1125
+ files of its group. It gets an acquisition of its own, as a file with locks
1126
+ does. Files without setup still batch.
1127
+ In `run.json`, such a file has no `runtimeBatch`.
1128
+
1129
+ ## Handles: an HTTP capture by name
1130
+
1131
+ Tests can now use an HTTP capture that their file's setup module started,
1132
+ through `handle(name)` from `@gfdelarue/testkit/environment`. A capture's
1133
+ handle is named after its `id` and has `url`, `requests()`, and `reset()`, and
1134
+ both methods return Promises. A run on supplied values gets the same handle
1135
+ from an `http-capture` entry under `handles` in its environment file. See
1136
+ [Handles](../README.md#handles).
1137
+
1138
+ - A capture's `requestsUrl` and `resetUrl` now hold an unguessable segment,
1139
+ `/__testkit__/<capability>/requests`, which Testkit keeps out of the run's
1140
+ results, artifacts, and events. Setups that hand those URLs to their tests
1141
+ keep working. A request to any other `/__testkit__/` path gets `404`, and a
1142
+ method other than `GET` or `DELETE` on the requests URL gets `405`. The old
1143
+ fixed `/__testkit__/requests` path no longer answers.
1144
+ - A capture `path` may not start with `/__testkit__/`.
1145
+ - A missing handle ends the test as an infrastructure failure, like a missing
1146
+ endpoint or database. A name that several captures share throws
1147
+ `EnvironmentHandleAmbiguousError`. A refused, unavailable, invalid, or
1148
+ unsupported handle throws `EnvironmentHandleError` with that `reason`.
1149
+
1150
+ ## Named endpoints and databases
1151
+
1152
+ Tests can read each used service's URL and database by name from the new
1153
+ `@gfdelarue/testkit/environment` module:
1154
+
1155
+ ```ts
1156
+ import { database, endpoint } from "@gfdelarue/testkit/environment";
1157
+
1158
+ const api = endpoint("api");
1159
+ const reports = database("reports");
1160
+ ```
1161
+
1162
+ - `endpoint(name)` returns the URL of the service `name` that the test's group
1163
+ uses, and `database(name)` returns that service's database URL. A service's
1164
+ database is the one Testkit provisions for it, or the `DATABASE_URL` its
1165
+ `env` declares.
1166
+ - The functions work in node-vitest and Playwright files and in the modules
1167
+ they import. Testkit installs the values before the test file and the
1168
+ product code it imports load, so module-level code and native hooks can read
1169
+ them, such as a Playwright config and its `globalSetup` or a Vitest
1170
+ `beforeAll`.
1171
+ - A name with no value throws `EnvironmentValueMissingError`. Its `kind` is
1172
+ `"endpoint"` or `"database"`, `valueName` is the name asked for, or `null`
1173
+ when the missing value is the run's default URL, and `available` lists the
1174
+ names the run provides. Outside a Testkit run every call throws it.
1175
+ - A test that fails on `EnvironmentValueMissingError`, or a file whose imports
1176
+ throw it, now ends as an infrastructure failure, not a test failure, in
1177
+ node-vitest and Playwright. Its failure key is
1178
+ `TESTKIT_ENVIRONMENT_VALUE_MISSING`. The run fails, and `testkit bug file`
1179
+ refuses to file it, as it does for a server that never became ready. A test
1180
+ that catches the error and carries on is unaffected, and a failed query, a
1181
+ failed assertion against a reachable service, or a refused connection stays
1182
+ a test failure.
1183
+ - When the run gives a test no base URL, a Playwright `page.goto()` to a
1184
+ relative URL ends the same way.
1185
+ - `BASE_URL` and `DATABASE_URL` stay. They hold the endpoint and database of the
1186
+ first service in `uses` that has one. Playwright's `BASE_URL` and `baseURL`
1187
+ keep their canonical form, so an origin ends with `/`. `endpoint()` returns
1188
+ the URL unchanged in every engine.
1189
+ - `readPlaywrightContext()` gains `databases`.
1190
+ - `testkit.config.ts` and lifecycle step modules cannot import the module.
1191
+
1192
+ ## Source schema credentials stay out of tests
1193
+
1194
+ Testkit kept `TESTKIT_SCHEMA_SOURCE_DATABASE_URL` and the service's own
1195
+ `TESTKIT_<SERVICE>_SCHEMA_SOURCE_DATABASE_URL` out of the values it gives a
1196
+ group's tests. A name chosen with `database.schema.fromEnv()`, or another
1197
+ service's `TESTKIT_<SERVICE>_SCHEMA_SOURCE_DATABASE_URL`, still reached tests
1198
+ when a used service's `env` set it, so tests could receive a source database
1199
+ URL, often a production database. Tests no longer receive a value under any
1200
+ name that the source schema of a service in their runtime reads, whichever
1201
+ service sets it
1202
+ ([gfdelarue/testkit#279](https://github.com/gfdelarue/testkit/issues/279)).
1203
+
1204
+ ## A suite selector's type is the text before its colon
1205
+
1206
+ The text before the first colon of a suite selector is now always the test
1207
+ type ([gfdelarue/testkit#283](https://github.com/gfdelarue/testkit/issues/283)).
1208
+ A selector such as `Int:auth` or `v2-api:auth` used to name a plain suite with
1209
+ that whole name. It now fails with `Unknown suite selector type`.
1210
+
1211
+ ## A shared database keeps its rows when another runtime instance starts
1212
+
1213
+ With `binding: "shared"`, every runtime instance of a service uses one
1214
+ database. Testkit reset that database each time it set up an instance, and a
1215
+ run sets up an instance the first time a test needs it. An instance set up
1216
+ partway through a run dropped the shared database while tests on other
1217
+ instances were using it
1218
+ ([gfdelarue/testkit#284](https://github.com/gfdelarue/testkit/issues/284)).
1219
+ Postgres terminated their connections, and their rows disappeared.
1220
+
1221
+ Testkit now resets a shared database once per run, when the first instance sets
1222
+ it up. Later instances use it as it is, and so does an instance that Testkit
1223
+ sets up again after a failure. If the database no longer exists, the instance
1224
+ clones it again from the template. `reset: false` and the default `per-runtime`
1225
+ binding behave as before.
1226
+
1227
+ ## `resetBefore` gives a file its own database
1228
+
1229
+ `runtimeIsolation` accepted `shared`, `resetBefore`, `resetAfter`, and
1230
+ `exclusive`, but only batching read it. `resetBefore` and `resetAfter` reset
1231
+ nothing, and `exclusive` didn't keep other files off the runtime
1232
+ ([gfdelarue/testkit#281](https://github.com/gfdelarue/testkit/issues/281)).
1233
+
1234
+ `resetBefore` now works. Before the file runs, Testkit clones each database it
1235
+ provisions that the file's environment reaches, from the template that database
1236
+ came from. That includes a database a used service names in its env, such as
1237
+ `{dbUrl:audit}`. The file's
1238
+ `.setup.*` module, `database()`, `DATABASE_URL`, and `{dbUrl}`-style values in
1239
+ the group's service env point at the clone, and Testkit drops it when the file
1240
+ finishes. A clone took 0.1–0.15 s from an empty template and about 0.35 s from
1241
+ a 117 MB one, on Linux and on the Windows CI runner. Use it for a test that
1242
+ changes a whole table, such as an unfiltered `DELETE` or a migration test.
1243
+
1244
+ Running services keep the runtime's database, so `resetBefore` isolates only
1245
+ the test's own database access. A request the test sends to an API server
1246
+ still reads and writes the shared database. When the server's writes must be
1247
+ isolated too, give the test its own group whose `uses` names a second service
1248
+ entry for the same server. Groups that use the same services share their
1249
+ runtimes.
1250
+
1251
+ A `resetBefore` file on supplied values, or whose environment reaches no
1252
+ database Testkit provisions, ends as an environment failure.
1253
+
1254
+ `resetAfter` and `exclusive` are removed. A config or `defineFile` that sets
1255
+ either fails to load with a message that names the replacement: `resetBefore`
1256
+ on the file that needs a clean database, or `shared` with tests that create
1257
+ and filter by their own rows. This is a breaking removal.
1258
+
1259
+ ## Run tests against supplied values
1260
+
1261
+ `testkit run` takes `--base-url`, `--endpoint NAME=URL`, `--database NAME=URL`,
1262
+ `--env NAME=VALUE`, `--env-from NAME=HOST_KEY`, and `--environment-file PATH`
1263
+ ([gfdelarue/testkit#235](https://github.com/gfdelarue/testkit/issues/235)).
1264
+ With any of them, every selected group runs on the supplied values. Testkit
1265
+ starts none of its services, creates no database, and runs no `.setup.*`
1266
+ module, and the group keeps its name, selection, and policy. Tests read named
1267
+ endpoints and databases through `@gfdelarue/testkit/environment`. The environment
1268
+ file is versioned JSON holding values, the setup data tests receive, and handle
1269
+ descriptors. Conflicting values, a missing `--env-from` host variable, and an
1270
+ invalid or unknown-version file fail before the run starts.
1271
+
1272
+ Before a test receives a supplied endpoint, Testkit checks that it answers
1273
+ HTTP, for up to 30 seconds. Any response counts, including a 404 or 500. An
1274
+ endpoint that refuses connections or never answers HTTP ends the run as an
1275
+ infrastructure failure.
1276
+
1277
+ Before a test receives a supplied database, Testkit connects to it, signs in,
1278
+ and runs `SELECT 1`. A database that is unreachable, fails TLS, rejects the
1279
+ credentials, does not exist, or is not PostgreSQL ends the run as an
1280
+ infrastructure failure, so `testkit bug file` refuses it. Without the check,
1281
+ the test ran and failed on the connection error, which read as a product bug.
1282
+ The check speaks the PostgreSQL protocol itself, with trust, password, md5, and
1283
+ SCRAM-SHA-256 authentication, TLS from `sslmode`, and Unix sockets through
1284
+ `host=/path`.
1285
+ [Run against a supplied environment](../README.md#run-against-a-supplied-environment)
1286
+ describes the flags and the file.
1287
+
1288
+ `testkit bug file` takes the same flags and reruns the recorded test on them.
1289
+ The issue's reproduce command carries `--base-url`, `--endpoint`, `--env-from`,
1290
+ and `--environment-file` as given. `testkit bug file` refuses `--database`,
1291
+ `--env`, and URLs with credentials, a query, or a fragment, so the command never
1292
+ holds a raw secret.
1293
+
1294
+ ## A file reads the same values from every source
1295
+
1296
+ A file now gets the same names and setup data whether the run starts its
1297
+ services or takes supplied values
1298
+ ([gfdelarue/testkit#326](https://github.com/gfdelarue/testkit/issues/326)):
1299
+
1300
+ - A group's default endpoint and database are available by the group's name in
1301
+ a configured run too. For a group `checks` that uses `api`,
1302
+ `endpoint("checks")` used to throw `EnvironmentValueMissingError` unless the
1303
+ values were supplied. Supplied values now also name a default taken from
1304
+ `uses` after the group, not only an explicit one such as `--base-url`.
1305
+ - A run refuses a group whose default differs from a used service's endpoint or
1306
+ database with the group's name, such as a group `api` with
1307
+ `uses: ["web", "api"]`. From either source, that group's files end as an
1308
+ infrastructure failure that names the group, and other groups still run,
1309
+ including groups that use the same services. A supplied run used to fail
1310
+ before it started. Rename the group, or list the service of that name first
1311
+ in `uses`.
1312
+ - Each acquisition in `run.json` records `setupData`: the supplied setup data or
1313
+ what the file's setup module returned, and `{}` without either, which is what
1314
+ tests already received. A configured acquisition used to leave it out.
1315
+ - A configured acquisition no longer records a service's `BASE_URL` env value
1316
+ under `env`. Tests never received it there, because every engine sets
1317
+ `BASE_URL` from the group's default endpoint.
1318
+
1319
+ So an acquisition's recorded `values`, with `"version": 1` added, work as an
1320
+ `--environment-file` that gives a second run the same values.
1321
+
1322
+ ## Sealed runs redact the secrets a run's environment resolved
1323
+
1324
+ Sealing took the secrets it redacts from the environment Testkit runs in. A
1325
+ value that reached a service or a test from `.env.testkit`, another env file,
1326
+ config `env`, or `local.env` stayed in the sealed run whenever a server log, a
1327
+ test's output, or an assertion message printed it
1328
+ ([gfdelarue/testkit#186](https://github.com/gfdelarue/testkit/issues/186)).
1329
+ Values from `--env`, `--env-from`, and `--environment-file` had the same gap.
1330
+
1331
+ A run now keeps, in memory, the values its environment resolves for each
1332
+ service, dependency services included, and for supplied input. Sealing redacts
1333
+ each value at least eight characters long under a key that holds `PASSWORD`,
1334
+ `TOKEN`, `SECRET`, `PRIVATE_KEY`, `AUTH`, `COOKIE`, `SESSION`, or `API_KEY`,
1335
+ and the password of each resolved URL. A configured run and a supplied run that
1336
+ print the same secrets seal the same redactions. Testkit takes a runtime's
1337
+ values before it prepares the runtime's database, and adds the values that
1338
+ depend on the database once it is ready, so a failed database seed, a run
1339
+ stopped during startup and a server that never becomes ready are redacted too.
1340
+
1341
+ - Sealing also redacts a value's JSON-escaped forms, each line of a value that
1342
+ spans lines, keys in the free-form data and metadata that tests attach, and
1343
+ the part of a secret that output shows before a `…` or `...` cut.
1344
+ - `testkit run --json` and the run's progress output apply the same redaction.
1345
+ They used to show unredacted test results.
1346
+ - When a run's process is killed before it seals the run, the process that
1347
+ seals it later does not have these values. It redacts patterns and its own
1348
+ environment only, and the bundle's `archive.redaction.status` reads
1349
+ `source-targets-unavailable` instead of `applied`.
1350
+ - A secret under another key name, such as `STRIPE_SK`, counts when `redact`
1351
+ names its key. Sealing still does not look inside compressed files such as
1352
+ Playwright traces
1353
+ ([gfdelarue/testkit#187](https://github.com/gfdelarue/testkit/issues/187)).
1354
+
1355
+ ## Stored run results redact whole secrets
1356
+
1357
+ Sealing redacted a `KEY=value` secret only up to its first space, and left
1358
+ header and JSON string secrets whole. A test title such as
1359
+ `rejects PASSWORD="first-part example-secret" with 401` was stored as
1360
+ `rejects PASSWORD=<redacted> example-secret" with 401`, in `run.json`,
1361
+ `latest.json`, and the run's JSON artifacts
1362
+ ([gfdelarue/testkit#213](https://github.com/gfdelarue/testkit/issues/213)).
1363
+
1364
+ Sealing, the run's returned result and progress output, and error messages now
1365
+ use the rules `testkit bug file` uses, which `docs/bug-records.md` describes. A
1366
+ quoted value ends at its closing quote and is written as `"<redacted>"`, so
1367
+ redacting the text again changes nothing. Filed issues and CI reports show it
1368
+ the same way. A value that starts with an auth scheme such as `Bearer` ends
1369
+ with its line, and `Authorization`, `Proxy-Authorization`, `Cookie`, and
1370
+ `Set-Cookie` header values and JSON string values under a secret's key are
1371
+ redacted, also when the key is spelled with escapes such as `"\u0074oken"`. A
1372
+ single-quoted `'password': '…'` value, which only error messages redacted
1373
+ before, is redacted by the same rules, and it ends at the next apostrophe that
1374
+ no backslash precedes. A value that starts with `\"` or `\'`, as a quoted
1375
+ value does inside a JSON string, ends at the matching escaped quote. Sealing
1376
+ reads JSON value by value: a JSON artifact, a log line from its first `{` or
1377
+ `[`, and a string that holds JSON. It redacts each decoded string, redacts a
1378
+ string member under a secret's key, and rewrites only the strings it changes,
1379
+ so a secret in one value leaves the record's other values, and numbers, spacing
1380
+ and line endings keep their exact bytes.
1381
+
1382
+ ## Name more secret keys with `redact`
1383
+
1384
+ A secret under a key that holds none of `PASSWORD`, `TOKEN`, `SECRET`,
1385
+ `PRIVATE_KEY`, `AUTH`, `COOKIE`, `SESSION`, or `API_KEY`, such as `STRIPE_SK`,
1386
+ stayed in a sealed run unless it was a URL's password
1387
+ ([gfdelarue/testkit#186](https://github.com/gfdelarue/testkit/issues/186)).
1388
+
1389
+ `redact` in `testkit.config.ts` names more keys:
1390
+
1391
+ ```ts
1392
+ export default defineConfig({
1393
+ redact: ["STRIPE_SK", "PGPASS"],
1394
+ });
1395
+ ```
1396
+
1397
+ Testkit matches the names in any letter case. A value under one counts the way
1398
+ a value under a built-in key does, from every source a run's environment
1399
+ resolves and from the environment Testkit runs in, in a configured run and a
1400
+ supplied run alike. It counts when it is at least eight characters long. A
1401
+ resolved value that is a URL counts only for its password; in the environment
1402
+ Testkit runs in, it counts whole and for its password. The bundle's
1403
+ `sensitiveEnvironmentKeys` lists such a key when the environment Testkit runs
1404
+ in holds it. `redact` doesn't add text forms: `STRIPE_SK=…` in a log is redacted
1405
+ when its value is one Testkit resolved. A config whose `redact` isn't a list of
1406
+ non-empty strings fails to load, and the message names the entry.
1407
+
1408
+ ## A PostgreSQL URL's query password is redacted
1409
+
1410
+ libpq also reads a password from the URL's query string, as in
1411
+ `postgres://db.internal/app?password=…`. Sealing treated only a password before
1412
+ the `@` as a resolved secret, so a test that printed that password on its own
1413
+ left it in the sealed run. A `postgres://` or `postgresql://` URL's `password`
1414
+ query parameter now counts as the URL's password, both raw and percent-decoded.
1415
+
1416
+ ## Retention keeps the objects that every stored run lists
1417
+
1418
+ Retention, and the cleanup after a failed seal, free the stored objects under
1419
+ `.testkit/results/objects` that no run bundle lists. They used to count only the
1420
+ bundles whose `run.json` this Testkit could read, so an object that only an
1421
+ unreadable bundle listed was freed. The bundle's own files stayed readable, but
1422
+ the object store no longer held what its manifest named.
1423
+
1424
+ - An object is now kept while any bundle's `manifest.json` lists it, whether or
1425
+ not Testkit can read that bundle's `run.json`.
1426
+ - While a bundle has no readable manifest, retention frees no object. Its
1427
+ decision records `garbageCollection.status: "not_started"` and names the
1428
+ bundle, and the decision's status is `failed`.
1429
+
1430
+ ## Run documents are schema 20
1431
+
1432
+ `run.json`, `latest.json` and `live.json` move from schema 19 to schema 20.
1433
+ Each field that named the service owning a test now names its group, and the
1434
+ facts that depend on how the environment was provided move into one
1435
+ `environment` record. Testkit still reads schema-19 runs stored by 0.4.x through
1436
+ `testkit artifacts`, `ci report`, known-failure checks and the workspace API,
1437
+ and finalizes a 0.4.x `live.json` as schema 19.
1438
+
1439
+ Renamed fields:
1440
+
1441
+ | Schema 19 | Schema 20 |
1442
+ | ------------------------------- | ----------------------------- |
1443
+ | `services` | `groups` |
1444
+ | `summary.services` | `summary.groups` |
1445
+ | `summary.diagnostics[].service` | `summary.diagnostics[].group` |
1446
+ | `run.serviceFilter` | `run.groupFilter` |
1447
+ | `planning.tasks[].service` | `planning.tasks[].group` |
1448
+
1449
+ Moved into `environment`:
1450
+
1451
+ | Schema 19 | Schema 20 |
1452
+ | -------------------------- | -------------------------------------- |
1453
+ | `run.runtimeInstanceCount` | `environment.capacity.instanceCount` |
1454
+ | `run.runtimeStats` | `environment.capacity.runtimeStats` |
1455
+ | `planning.resourceBudgets` | `environment.capacity.resourceBudgets` |
1456
+ | `run.dbBackend` | `environment.dbBackend` |
1457
+ | `services[].dbBackend` | `environment.groups[].dbBackend` |
1458
+ | `environment.services` | `environment.services` (unchanged) |
1459
+ | `logs` | `environment.logs` |
1460
+ | `setup.operations` | `environment.setupOperations` |
1461
+
1462
+ `environment` also gains these fields:
1463
+
1464
+ - `acquisitions`: one entry for each environment the run acquired. It records:
1465
+ - the source that prepared it: `configured`, `static`, `supplied` or `none`;
1466
+ - the tasks it served, its runtime and services;
1467
+ - each file's setup;
1468
+ - its release outcome;
1469
+ - its timings, as ISO times;
1470
+ - the values its tests received, in the `--environment-file` shape with
1471
+ secrets and capture capabilities redacted.
1472
+ - `cleanup`: the run's cleanup outcome.
1473
+ - `missingValues`: the named values tests asked for that their environment
1474
+ lacked.
1475
+
1476
+ The same rename reaches the run's other outputs:
1477
+
1478
+ - `--output-mode events`:
1479
+ - `run.service_plans` is now `run.group_plans`, with `groups[].group`.
1480
+ - `run.service_skipped` is now `run.group_skipped`, with `group`.
1481
+ - `run.task_planned_skip`, `run.task_started`, `run.task_finished` and
1482
+ `run.runtime_error` carry `group` instead of `service`.
1483
+ - `run.summary` results no longer carry `dbBackend`.
1484
+ - Events about a real service keep `service`: `run.local_service_starting`,
1485
+ `run.toolchain_resolved` and `run.setup_finished`.
1486
+ - The run summary reads `groups N/M passed` and `N/M groups failed`.
1487
+ - `testkit run --json` prints the schema-20 `runArtifact`. Its `results` no
1488
+ longer carry `dbBackend`.
1489
+ - `testkit artifacts --json` names each artifact's `group` instead of its
1490
+ `service`.
1491
+ - In `@gfdelarue/testkit/workspace`:
1492
+ - `loadRunBundleArtifact`, `latestRunArtifact` and `liveRunArtifact` return a
1493
+ schema-19 or schema-20 document.
1494
+ - A run summary counts `groups` instead of `services`.
1495
+
1496
+ A run bundle's `provenance.json` keeps its `scenario.serviceFilter` field, which
1497
+ now holds the group filter. The `--service` flag still selects a group.
1498
+
1499
+ ## One declaration file per entry point
1500
+
1501
+ Each public entry point now has one bundled declaration file under
1502
+ `dist/types/`, and the package ships 14 of its own `.d.ts` files instead of
1503
+ the 483 that 0.4.5 shipped. The root `types` field and each export's `types`
1504
+ condition point at the bundles. A bundle exports the entry point's own names
1505
+ and every interface and type alias its public signatures use, so types that
1506
+ consumers received but could not import, such as `DataEvidence` or
1507
+ `RunBundleManifest`, now have importable names. Public signatures no longer
1508
+ name internal types:
1509
+
1510
+ - `@gfdelarue/testkit/playwright`: `EngineJsonObject`, `EngineJsonValue` and
1511
+ `EngineJsonPrimitive` are now `JsonObject`, `JsonValue` and `JsonPrimitive`.
1512
+ - `@gfdelarue/testkit/config/database-steps`: `DatabaseStepContext` no longer
1513
+ has `ownedContainer`, which Testkit supplies itself.
1514
+ - `@gfdelarue/testkit/workspace`: a run bundle's `archive.retention` is typed
1515
+ as `RunBundleRetentionSummary`, with its status, time, policy and error. The
1516
+ stored block still holds the garbage collection and removal details.
1517
+ `WorkspaceServiceStatus` and `WorkspaceRuntimeGraphStatus` type `product`,
1518
+ `caches` and `desired` directly instead of through the internal status model.
1519
+ - `__testkitRegisterNodeVitestRepositoryConfiguration` is no longer declared.
1520
+ It remains a runtime export for Testkit's generated setup modules.
1521
+ - `UnknownRecord` is gone; the signatures that used it say
1522
+ `Record<string, unknown>`.
1523
+
1524
+ A project that referenced a declaration file by its path under `dist/` must
1525
+ import the package subpath instead. See
1526
+ [gfdelarue/testkit#214](https://github.com/gfdelarue/testkit/issues/214).
1527
+
1528
+ ## Engine documentation in the package
1529
+
1530
+ The package ships `docs/engines.md` in place of `docs/engine-architecture.md`.
1531
+ `docs/engines.md` covers engine authoring, typechecking, the environment a test
1532
+ receives, Playwright attachments, installation, and the supported platforms.
1533
+ The README and the configuration reference no longer describe Testkit's
1534
+ internals, and every relative link in the shipped docs resolves inside the
1535
+ package ([gfdelarue/testkit#214](https://github.com/gfdelarue/testkit/issues/214)).
1536
+
1537
+ # 0.4.5
1538
+
1539
+ ## Service ports come from the OS
1540
+
1541
+ Testkit no longer computes service ports. It used to add a fixed stride to
1542
+ `local.port`, or to the port in `local.baseUrl`, for each runtime instance and
1543
+ graph. Two runs of the same repository on one host computed the same ports and
1544
+ collided, whether they were two CI jobs on one runner or two agent worktrees.
1545
+
1546
+ - Every service in every runtime instance now gets a port from the OS. Testkit
1547
+ binds `127.0.0.1:0` and the same port on `::1`, holds both until just before
1548
+ it spawns the service, and passes the port as `PORT` and `{port}`.
1549
+ - A service that exits before it becomes ready gets a fresh port and starts
1550
+ again, up to three attempts in total. A service that already started and
1551
+ reads the moved service's URL or port restarts with the new value. A retry
1552
+ never gets a port the service already tried. The final error is an
1553
+ infrastructure failure that lists each attempt's port and outcome and the last
1554
+ attempt's output.
1555
+ - `run.json` gains `environment.services`: each service's `runtimeLabel`,
1556
+ `serviceName`, `baseUrl`, `readyUrl`, `port`, and every start attempt, recorded
1557
+ as it happens, including those of a startup that fails. Recorded URLs drop
1558
+ credentials, query values, and fragments.
1559
+ - The prepare fingerprint now covers the environment that `runtime.prepare`
1560
+ steps see, with its templates resolved, instead of the raw env templates and
1561
+ the service's `start`, `baseUrl`, and `readyUrl`. A prepare step that embeds a
1562
+ service URL or port rebuilds on every run.
1563
+
1564
+ This breaks existing configs:
1565
+
1566
+ - `local.port` and the `port` option of `app.node` and `app.next` are config
1567
+ errors. Delete them and use `{port}` in `start`, `baseUrl`, and `readyUrl`.
1568
+ - A literal port in `local.baseUrl` or `local.readyUrl` is a config error, and
1569
+ `local.baseUrl` must contain `{port}`. `app.node` and `app.next` already
1570
+ default to `http://127.0.0.1:{port}`.
1571
+ - A literal URL in another service's env no longer follows that service's port.
1572
+ Use `{baseUrl:<service>}` or `{port:<service>}`.
1573
+ - There is no setting that pins a service to a fixed port.
1574
+
1575
+ ## Stopped Windows commands
1576
+
1577
+ On Windows, a cancelled or timed-out command now waits up to four seconds for
1578
+ its process tree to exit before it returns. It used to return as soon as its
1579
+ root process exited. The Playwright adapter then removed Chromium's profile
1580
+ while terminated browser processes still held it, and the run failed with
1581
+ `EBUSY` ([gfdelarue/testkit#205](https://github.com/gfdelarue/testkit/issues/205)).
1582
+ A terminated process on a loaded host can take longer than four seconds to
1583
+ exit. The command then returns a termination error that names the process, and
1584
+ the run can still end in an infrastructure failure. A process in the tree that
1585
+ Testkit cannot open or terminate is reported the same way, and so is a process
1586
+ whose parent exited before Testkit could confirm that it belongs to the tree.
1587
+ Testkit leaves both running. Once termination fails, the command returns
1588
+ without waiting for output that a surviving process holds open. It then
1589
+ terminates only the root, through the process handle Node opened when it
1590
+ started it. It no longer starts `taskkill` against the root's process id,
1591
+ which another process may have reused.
1592
+
1593
+ `testkit stop` now gives a Windows run its five-second finalization window
1594
+ before it forces the run, as it does on POSIX. It used to force a Windows run
1595
+ as soon as the two-second grace ended, and the run exited with code 1.
1596
+
1597
+ ## Baseline-relative data budgets removed
1598
+
1599
+ `relativeToBaseline()` is removed from the metric budgets that `budget(name)`
1600
+ and `expectMetric(name)` return in `@gfdelarue/testkit/node-vitest/data`. The
1601
+ `DataBaselineBudgetOptions` type is removed with it. Nothing compared the
1602
+ metric with the named run, so these budgets always reported passed. A scenario
1603
+ that still calls `relativeToBaseline()` fails with a `TypeError`, and
1604
+ TypeScript reports the missing method. Use `atMost`, `atLeast`, `lessThan`,
1605
+ `greaterThan`, or `equals` with a fixed value instead.
1606
+
1607
+ Budget records in `data-flow.json` no longer have the `baseline_relative`
1608
+ operator or the `policy` and `thresholdVersion` fields. This is a breaking
1609
+ removal. See [gfdelarue/testkit#224](https://github.com/gfdelarue/testkit/issues/224).
1610
+
1611
+ ## `createDataTrace()` removed
1612
+
1613
+ `createDataTrace()` is removed from `@gfdelarue/testkit/node-vitest/data`. The
1614
+ context it returned never reached the run's report, so its metrics, evidence,
1615
+ and findings were lost, and its budgets were never checked. A failing budget on
1616
+ a trace passed. Record data inside `dataScenario()` and use the context it
1617
+ passes to the callback. This is a breaking removal. See
1618
+ [gfdelarue/testkit#242](https://github.com/gfdelarue/testkit/issues/242).
1619
+
1620
+ ## Bug records: optional `testType`
1621
+
1622
+ For tools that read the facts block in Testkit's issues:
1623
+
1624
+ - `testType` may be absent. Testkit writes it when the test file's name gives a
1625
+ type, and a later release files bugs for test files whose names give none.
1626
+ Read a version 1 block without `testType` as valid. The format version stays
1627
+ `1`.
1628
+ - Every type that a Testkit release has written stays readable: `ui`, `e2e`,
1629
+ `int`, `contract`, `data`, `dal`, and `load`. Issues keep the type they were
1630
+ filed with, so a reader keeps a type after Testkit stops writing it.
1631
+ - `"testType": null` is invalid, as for every other optional field.
1632
+
1633
+ This release still writes `testType` on every issue it files.
1634
+
1635
+ Four category texts no longer assume the product is a server. They are the
1636
+ boundary rules of the `missing-validation` cause and the `workflow-breakdown`
1637
+ impact, and the definition and boundary rule of the
1638
+ `availability-or-performance` impact. Every id and every list's order is
1639
+ unchanged.
1640
+
1641
+ ## Tests that need nothing run with zero setup
1642
+
1643
+ Testkit no longer starts a server that `testkit.config.ts` does not declare, so
1644
+ a config that relied on an inferred server breaks. Before, a service without
1645
+ `local` got `{bin:next} dev -p {port}` when its directory had a `next.config.*`
1646
+ file, `go run ./cmd/server` when it had `cmd/server/`, and
1647
+ `{bin:tsx} watch src/index.ts` when it had `package.json` and `src/`. A repo
1648
+ with no config got the same server for its implicit `app` owner. Loading a
1649
+ config also failed when a suite's type expected a server that Testkit could not
1650
+ resolve. Both are gone. To keep a server that Testkit used to infer, declare
1651
+ it, for example `services: { app: app.node({ entry: "src/index.ts" }) }`.
1652
+
1653
+ - An owner that declares no `local`, `database`, `databaseFrom`, `dependsOn`,
1654
+ `runtime.build`, `runtime.prepare` step, `runtime.toolchain`, or env values
1655
+ gets no setup. Its tasks reserve no port, claim no runtime slot, run no
1656
+ `testkit.runtime.ensure`, and write nothing under `.testkit/_graphs`. A run
1657
+ made only of such tasks records no `environment.services` and does no
1658
+ Postgres or Docker maintenance.
1659
+ - With no config, the implicit `app` owner loads no `.env` or `.env.testkit`
1660
+ file. A service in `testkit.config.ts` still loads them.
1661
+ - Only `data` tests import `testkit.config.ts` in the test process, for their
1662
+ HTTP profiles. In other Node/Vitest tests, `getRepoConfig()` returns `null`.
1663
+ - `testkit status`, `doctor`, `typecheck`, and `destroy` work in a repo whose
1664
+ owners declare nothing. `destroy` lists Docker containers only when a service
1665
+ declares a database or local database state exists.
1666
+
1667
+ ## A test's type no longer decides its setup or its values
1668
+
1669
+ A task now gets everything its service declares, whatever its test type. A
1670
+ `dal` task used to get its service's database, `runtime.build` and
1671
+ `runtime.prepare` steps, and toolchain, but not the server in `local`. That
1672
+ server now starts for `dal` tasks too, and Testkit waits for its `readyUrl`
1673
+ before the test runs. If the server fails to start, the `dal` task ends with an
1674
+ infrastructure failure. A service that declares only a database still starts no
1675
+ server.
1676
+
1677
+ Every task gets `BASE_URL` when its service has a URL and `DATABASE_URL` when
1678
+ its service has a database, on every engine. Tasks other than k6 `dal` used to
1679
+ get `DATABASE_URL` only from declared `env`, and k6 `dal` tasks never got
1680
+ `BASE_URL`. The provisioned `DATABASE_URL` replaces one that the service's
1681
+ `env` declares.
1682
+
1683
+ k6 and Playwright files run without a URL or a database. A file that checks
1684
+ constants, or a Playwright test that builds its page with `page.setContent()`,
1685
+ passes. A file that reads a value its service does not provide fails where it
1686
+ reads it, for example with `BASE_URL env var required` from k6 or
1687
+ `Cannot navigate to invalid URL` from `page.goto("/")`. Testkit records these as
1688
+ product failures.
1689
+
1690
+ Without a URL, a Playwright test gets no `BASE_URL`, and
1691
+ `readPlaywrightContext().baseUrl` is `null`. `PlaywrightTestContext.baseUrl` is
1692
+ now typed `string | null`. Testkit also leaves `use.baseURL` from your
1693
+ `playwright.config.*` in place. With a URL, Testkit's `use.baseURL` still
1694
+ replaces yours.
1695
+
1696
+ `dal` files of one service now run at the same time. Every `dal` task used to
1697
+ take a `database:<service>` lock. The lock kept two `dal` files of one service
1698
+ from overlapping, but not a `dal` file and another test on the same database. If
1699
+ two files share rows that one of them changes, give both the same `locks` entry
1700
+ in `defineFile`.
1701
+
1702
+ ## Test runs
1703
+
1704
+ - `--file-timeout-seconds` now sets each file's timeout. Testkit used to record
1705
+ the flag's value in the results and run each file under the timeout from
1706
+ `testkit.config.ts` (#212).
1707
+ - A task's lease directory, `TESTKIT_LEASE_DIR`, is now under
1708
+ `.testkit/_execution/leases/`. Engine workspaces and the compilation cache
1709
+ are under `.testkit/_execution/node_modules/testkit-engine/`. Testkit removes
1710
+ them when the task or the run ends.
1711
+ - Test processes no longer get `TESTKIT_RUNTIME_ID`. The lease ID still
1712
+ identifies each task. Setup steps and servers that Testkit starts still get
1713
+ `TESTKIT_RUNTIME_ID`.
1714
+
1715
+ # 0.4.1
1716
+
1717
+ ## Bug filing, campaigns, and CI reports
1718
+
1719
+ Testkit now files confirmed bugs as issues on GitHub or Gitea and reports runs
1720
+ there. None of it needs an account or a server.
1721
+
1722
+ - `testkit bug guide` prints the bug record format. `testkit bug record` checks
1723
+ a record and stores a draft. `testkit bug file` runs the recorded test, and
1724
+ files the issue when the test fails.
1725
+ - `testkit campaign start`, `status`, and `finish` give a coding agent a target,
1726
+ list what it recorded, and write the summary for the pull request that adds
1727
+ the tests.
1728
+ - `testkit ci report` posts the last run as one commit status, the job's step
1729
+ summary, and one pull request comment that it updates in place.
1730
+ - `areas` in `testkit.config.ts` maps source paths to the product's areas. Each
1731
+ filed issue gets an `area:<id>` label.
1732
+ - `bugs.knownFailures` decides what a filed bug does to a run. The default,
1733
+ `"fail"`, keeps the gate as it was. With `"allow"`, a run passes when an open
1734
+ issue with the `testkit` label covers every failure.
1735
+
1736
+ The package now ships `docs/bug-records.md`, the reference for the issue
1737
+ format, the facts block, the known-failure rule, the commit status, and the
1738
+ pull request comment.
1739
+
1740
+ Run results under `.testkit/results/` gain failure details.
1741
+
1742
+ - Vitest. An error outside a test is a failure detail of kind `vitest-error`.
1743
+ A file that fails for a reason other than its failed tests keeps its engine
1744
+ failure beside the details of those tests. That reason is a timeout, too much
1745
+ output, a signal, or an exit code other than 1.
1746
+ - Playwright. A `playwright-spec` detail's key now holds the names of the
1747
+ `describe` blocks before the spec's title. A spec that fails in a file other
1748
+ than the selected one is a detail of kind `playwright-other-file-spec`. A
1749
+ `PLAYWRIGHT_SPECS_FAILED` failure carries `runStatus`, `failOnFlakyTests`,
1750
+ `reportErrors`, `attachmentFailures`, `evidenceErrors`, and `exitCode` or
1751
+ `exitSignal` in its details.
1752
+ - k6. A failed assertion's detail carries `labelled`, which says whether the
1753
+ script gave the assertion a label. Two assertions with one key and two titles
1754
+ are two details. A `K6_THRESHOLDS_FAILED` failure carries `failedChecks` and
1755
+ `exitCode` or `exitSignal`. A `K6_RUNTIME_EXCEPTION` failure carries
1756
+ `summaryRead`, `exitCode` or `exitSignal`, and `phase` when the suite threw
1757
+ during `setup` or `exec`. A script that does not load carries no phase.
1758
+
1759
+ A Playwright file whose specs a helper module declares now runs when the helper
1760
+ has the file's own extension. Testkit matched a spec to its file by the module
1761
+ that called `test()`, so such a file failed with
1762
+ `Selected Playwright task did not execute any tests`. A file whose every spec
1763
+ comes from one helper with another extension, such as a `.ts` file and a `.js`
1764
+ helper, still fails that way. Playwright takes the helper for the file's source
1765
+ and leaves the file out of the run.
1766
+
1767
+ # 0.4.0
1768
+
1769
+ ## Testkit Cloud commands removed
1770
+
1771
+ This release removes the `testkit cloud` commands, the `cloud` config key, and
1772
+ the `docs/cloud-control.md` and `docs/testkit-cloud-boundary.md` references.
1773
+ Testkit no longer publishes runs, findings, or campaign reports to Testkit
1774
+ Cloud. `testkit cloud` with any arguments now exits with status 2 and an error
1775
+ that names this release. The package no longer depends on `undici` or `yazl`.
1776
+
1777
+ Testkit ignores a `cloud` key that remains in `testkit.config.ts`.
1778
+ `defineConfig` still accepts the key when the config sets other fields, so
1779
+ `testkit typecheck` passes. A config object annotated as `TestkitConfig` gets a
1780
+ TypeScript unknown-property error. Delete the key.
1781
+
1782
+ Run bundles under `.testkit/results/` are unchanged. `testkit run` seals them
1783
+ and `testkit artifacts` reads them as before, and `artifacts.publication` still
1784
+ copies sealed bundles to a local directory. This is a breaking removal.
1785
+
1786
+ # 0.2.7
1787
+
1788
+ Playwright body attachments above 768 KiB now retain their bytes and test
1789
+ results. Testkit stores attachment bodies in private files and applies the
1790
+ existing 64 MiB individual and 256 MiB aggregate limits to body and path
1791
+ attachments together. Rejected attachments retain named errors alongside spec
1792
+ outcomes, assertion details, and accepted neighboring attachments. Temporary
1793
+ attachment files are removed after collection, including failed and cancelled
1794
+ runs. Text-secret redaction and binary-secret rejection still apply when sealing
1795
+ run evidence.
1796
+
1797
+ Fixed in [gfdelarue/testkit#154](https://github.com/gfdelarue/testkit/pull/154).
1798
+
1799
+ # 0.2.6
1800
+
1801
+ `testkit run --json` now emits one valid JSON document when services or test
1802
+ files are skipped. Human progress messages no longer appear before the result.
1803
+ Run IDs, exit codes, and structured skip information remain available to CI
1804
+ consumers. Compact and debug output retain their existing messages.
1805
+
1806
+ Fixed in [gfdelarue/testkit#147](https://github.com/gfdelarue/testkit/pull/147).
1807
+
1808
+ # 0.2.5
1809
+
1810
+ This release includes the changes merged in
1811
+ [gfdelarue/testkit#134](https://github.com/gfdelarue/testkit/pull/134),
1812
+ [gfdelarue/testkit#135](https://github.com/gfdelarue/testkit/pull/135), and
1813
+ [gfdelarue/testkit#136](https://github.com/gfdelarue/testkit/pull/136).
1814
+ Database schema preparation and Docker commands share their existing operations.
1815
+ Archive handling removes the unused Tar extraction path and shares accounting
1816
+ across the remaining formats.
1817
+
1818
+ ## Embedded OIDC provider removed
1819
+
1820
+ The `auth` config export, including `auth.providers.oidc()`, and the
1821
+ `@gfdelarue/testkit/config/oidc-provider` and
1822
+ `@gfdelarue/testkit/config/oidc-provider-runtime` subpaths are removed.
1823
+ Projects that need a test identity provider must supply one through ordinary
1824
+ managed services and configure their applications to use it. Testkit no longer
1825
+ implements an identity-provider server.
1826
+
1827
+ Consumer-owned API and browser authentication helpers remain supported.
1828
+ This is a breaking removal; external package consumers have not been inventoried.
1829
+
1830
+ # 0.2.4
1831
+
1832
+ The k6 engine now installs from public npm packages. The root manifest pins the
1833
+ five `@gfdelarue/testkit-engine-k6-<target>` packages at 1.0.0, and npm selects
1834
+ the one matching the host together with the shared
1835
+ `@gfdelarue/testkit-engine-k6-source` package. The packages are built and
1836
+ published with provenance from
1837
+ [gfdelarue/testkit-k6](https://github.com/gfdelarue/testkit-k6) on their own
1838
+ version line; the executable remains k6 1.7.1 with xk6-sql. Testkit no longer
1839
+ vendors binaries or builds k6, and installing it no longer needs registry
1840
+ credentials. The resolver checks the installed package's manifest and probes
1841
+ the executable once; a version other than the pinned one fails with
1842
+ `TESTKIT_K6_ENGINE_VERSION_MISMATCH`. Installing with `--omit=optional` is
1843
+ unsupported. The 0.2.x engine packages are deprecated.
1844
+
1845
+ `database.postgres` and `database.fixture` accept `roles`, an array of
1846
+ `{ name, login: false }` declarations. Testkit creates these local roles before
1847
+ source-schema replay, so imported policies can reference roles absent from a
1848
+ fresh cluster. Existing roles must match the ordinary NOLOGIN role contract,
1849
+ including on template cache hits. Conflicts stop provisioning without altering
1850
+ existing roles. Roles persist until the owned PostgreSQL volume is destroyed.
1851
+ This fixes [gfdelarue/testkit#129](https://github.com/gfdelarue/testkit/issues/129).
1852
+
1853
+ ## `auth.fixture` removed
1854
+
1855
+ `@gfdelarue/testkit/config` no longer exports `auth.fixture`, `auth.contracts`,
1856
+ `auth.topologies`, or `auth.profile`. `auth.providers.oidc` remains. The
1857
+ generated-actor profile descriptors, the `auth-fixture` k6 wire kind, and the
1858
+ `__testkitAuthProfile` suite metadata are removed with them. k6 repository
1859
+ profiles are JSON-only; a profile that carries a function fails validation.
1860
+
1861
+ Move signup, login, and header projection into the k6 suite module or a shared
1862
+ `__testkit__/helpers/` module. `defineHttpSuite({ auth: { setup, headers },
1863
+ rawHeaders })` keeps the session bundle, `actor`, `actors`, and `req.as(...)`
1864
+ runtime. Node/Vitest data profiles keep in-process `auth.setup`,
1865
+ `auth.setupAsync`, `headers`, and `rawHeaders` functions on
1866
+ `profiles.http.<name>`. Testkit 0.6.0 removed HTTP profiles; see
1867
+ [Remove the data test type](#remove-the-data-test-type).
1868
+
1869
+ This is a breaking removal. The migrated consumers are the two in-repository
1870
+ system fixtures. External consumers of the retired API have not been inventoried.
1871
+
1872
+ Local service commands on Windows keep their quoted executable paths and
1873
+ arguments, and Testkit creates its temporary directories on Windows even when
1874
+ the path exceeds `MAX_PATH`. Sensitive-text redaction runs in linear time on
1875
+ long input.
1876
+
1877
+ The undocumented `TESTKIT_PSQL_IMAGE` and `TESTKIT_PG_DUMP_IMAGE` aliases and
1878
+ the `TESTKIT_OIDC_CONFIG_JSON` override are removed. Use the documented
1879
+ `TESTKIT_POSTGRES_CLIENT_IMAGE`; the OIDC provider is configured only through
1880
+ the repository configuration. `TESTKIT_CLOUD_URL`, `TESTKIT_SETUP_NODE_OPTIONS`,
1881
+ and the Playwright `TESTKIT_FRONTEND_BASE_URL` and `TESTKIT_BACKEND_BASE_URL`
1882
+ fallbacks are now documented in the README.
1883
+
1884
+ # 0.2.3
1885
+
1886
+ Configured commands on Windows preserve quoted paths and arguments when
1887
+ invoking `cmd.exe`. TypeScript builds can now write prepared output directories
1888
+ without receiving literal quote characters in `--outDir`.
1889
+
1890
+ `testkit typecheck` supports consumer TypeScript 5.9.3 and 6.0.2. Generated
1891
+ configurations clear the deprecated `baseUrl` option while preserving inherited
1892
+ aliases and Testkit's type mappings. This fixes the TS5101 failure reported in
1893
+ [gfdelarue/testkit#117](https://github.com/gfdelarue/testkit/issues/117), including
1894
+ consumers with a separate frontend compiler.
1895
+
1896
+ Database state scans now enumerate only the directories Testkit owns under
1897
+ `.testkit` (`_infra/local-postgres`, `_dbcache`, graph runtime services, and
1898
+ top-level service directories carrying a `database_backend` marker). Unrelated
1899
+ content beneath `.testkit`, such as a consumer virtualenv or the managed Node
1900
+ toolchain's own `npm` links, no longer fails run start or resource cleanup with
1901
+ `Refusing unsafe Testkit database state symlink`. Container and volume
1902
+ references are read only from owned state, so stray files elsewhere no longer
1903
+ keep stale resources alive. This addresses
1904
+ [gfdelarue/testkit#118](https://github.com/gfdelarue/testkit/issues/118).
1905
+
1906
+ Local services accept `local.shutdownTimeoutMs`, a positive finite integer in
1907
+ milliseconds that defaults to 1000. `app.node()` and `app.next()` forward the
1908
+ same option. On POSIX, shutdown sends SIGTERM immediately and SIGKILL only when
1909
+ the allowance expires. Cancellation, repeated stop requests, descendant cleanup,
1910
+ and maintenance recovery share the first deadline. Windows retains forceful
1911
+ `taskkill /T /F`. This fixes
1912
+ [gfdelarue/testkit#116](https://github.com/gfdelarue/testkit/issues/116).
1913
+
1914
+ `stop`, `run stop`, active `cleanup`, and `destroy` let the controller finish
1915
+ service shutdown before escalating. After SIGINT, they wait at least two
1916
+ seconds and until outstanding service deadlines plus three seconds for forced
1917
+ termination have elapsed. They then send the controller SIGTERM, allow five
1918
+ seconds for finalization, and use SIGKILL if it remains active. A controller
1919
+ with no registered services retains the two-second grace and five-second
1920
+ finalization windows.
1921
+
1922
+ The `.scenario.testkit.ts` category, `defineScenarioSuite()` and `--seed` are
1923
+ removed. Use `.e2e.testkit.ts` for product journeys or `.int.testkit.ts` for
1924
+ service checks, and replace the scenario choice/resource/step helpers with
1925
+ ordinary test code. `defineHttpSuite()` remains available. Native k6
1926
+ `options.scenarios`, executors, groups and lifecycle hooks are unchanged.
1927
+
1928
+ Discovery ignores the retired suffix, so migrate those files before upgrading;
1929
+ a mixed run can still pass using its remaining supported files. Saved scenario
1930
+ results remain readable. New run documents omit `scenarioSeed`, and provenance
1931
+ omits `scenario.seed` while retaining its generic run-selection fields.
1932
+
1933
+ # 0.2.2
1934
+
1935
+ PostgreSQL clients for Testkit-owned databases now join the database container's
1936
+ network namespace. Template rebuilds, schema checks and snapshots, and database
1937
+ steps no longer depend on Docker host networking reaching a published port.
1938
+ This addresses the failure reported with Docker Desktop and WSL2 in
1939
+ [gfdelarue/testkit#111](https://github.com/gfdelarue/testkit/issues/111).
1940
+ External `sourceSchema` database URLs retain their existing routing.
1941
+
1942
+ # 0.2.1
1943
+
1944
+ This release adds explicit Cloud campaign reporting and optional local
1945
+ verification policy. It also includes the configuration changes below.
1946
+ Consumers pinned to 0.2.0 keep their existing behavior until they upgrade.
1947
+
1948
+ The pinned Vitest version is 4.1.11, which fixes the `@vitest/mocker` path
1949
+ traversal advisory GHSA-82fw-gwwq-j7x9.
1950
+
1951
+ ## Next app Clerk preset
1952
+
1953
+ `app.next` no longer accepts an `auth` option. The only value it took,
1954
+ `"disabled-clerk"`, set six product-private environment variables and was never
1955
+ documented. Put the variables in `env.values` instead:
1956
+
1957
+ ```ts
1958
+ app.next({
1959
+ port: 3000,
1960
+ env: {
1961
+ values: {
1962
+ DISABLE_CLERK_AUTH: "1",
1963
+ NEXT_PUBLIC_DISABLE_CLERK_AUTH: "1",
1964
+ NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY: "",
1965
+ CLERK_PUBLISHABLE_KEY: "",
1966
+ CLERK_SECRET_KEY: "",
1967
+ CLERK_WEBHOOK_SECRET: "",
1968
+ },
1969
+ },
1970
+ });
1971
+ ```
1972
+
1973
+ ## Fly.io request routing header
1974
+
1975
+ The k6 and Node/Vitest HTTP runtimes no longer read `MACHINE_ID` from the
1976
+ environment or add a `fly-force-instance-id` header to requests. The Fly.io
1977
+ infrastructure was removed earlier and nothing documented the header. The
1978
+ `routeParams` field on `RuntimeEnv`, the `routeHeaders` option on the k6
1979
+ `createHttpClient`, and the matching positional argument on `makeReq` and
1980
+ `makeRawReq` are removed with it. A test that needs a routing header sets it
1981
+ through `headers` or `rawHeaders` on its HTTP profile.
1982
+
1983
+ ## Lint rules are booleans
1984
+
1985
+ `testkit lint` no longer has severity levels or ratchet mode. Each rule is
1986
+ enabled or disabled, enabled-rule findings fail the command, and results depend
1987
+ only on current source. `lint.level` and the rule levels `off`, `warn`,
1988
+ `ratchet`, and `strict` are rejected with migration guidance. Replace them with
1989
+ `true`, `false`, or `{ enabled: boolean }` alongside the existing `include`,
1990
+ `exclude`, `allowed`, and `trackedOnly` options:
1991
+
1992
+ ```ts
1993
+ lint: {
1994
+ rules: {
1995
+ dalParallelSafety: false,
1996
+ unitTestBoundaries: { include: ["src/**/*.test.ts"] },
1997
+ },
1998
+ }
1999
+ ```
2000
+
2001
+ Results and `.testkit/lint/latest.json` drop `level`, `severity`,
2002
+ `ratchetStatus`, `fingerprint`, `notes`, and the warning and debt counts.
2003
+ `base.json` and `ratchet.json` are no longer written. The `TestkitLintLevel`
2004
+ type is removed. Unit-test boundary findings now report every source location
2005
+ instead of one per file and operation.
2006
+
2007
+ ## Environment overrides
2008
+
2009
+ Three undocumented environment variables are no longer read:
2010
+
2011
+ - `TESTKIT_PSQL_IMAGE` and `TESTKIT_PG_DUMP_IMAGE` were aliases for the
2012
+ documented `TESTKIT_POSTGRES_CLIENT_IMAGE`. Rename them.
2013
+ - `TESTKIT_OIDC_CONFIG_JSON` let the environment replace the OIDC provider
2014
+ config, including client secrets and users, at launch. Configure the provider
2015
+ explicitly instead. The embedded provider is removed in 0.2.5;
2016
+ see [Embedded OIDC provider removed](#embedded-oidc-provider-removed).
2017
+
2018
+ `TESTKIT_CLOUD_URL`, `TESTKIT_SETUP_NODE_OPTIONS`, and the Playwright
2019
+ `TESTKIT_FRONTEND_BASE_URL` and `TESTKIT_BACKEND_BASE_URL` fallbacks keep their
2020
+ behavior and are now documented in the README and config schema.
2021
+
2022
+ ## Playwright typecheck support roots
2023
+
2024
+ `testkit typecheck` no longer evaluates Playwright configuration or traverses a
2025
+ separate dependency/provenance graph. It runs native TypeScript over engine-profiled
2026
+ projects. Add `tsconfig.playwright.json` with explicit `files`/`include` entries for
2027
+ string-named setup, teardown, reporter, fixture, and support-project modules.
2028
+ Discovered UI suites and the selected Playwright config remain implicit roots.
2029
+ See the [support-file contract](engines.md#playwright-support-files).
2030
+ Managed runtime enforcement remains authoritative; direct import lint feedback
2031
+ moves to ESLint under #22.
2032
+
2033
+ ## Data-profile Vitest imports
2034
+
2035
+ Data suites now receive restricted Vitest runtime exports. Native case
2036
+ registration is rejected through helpers and aliases as well as direct imports.
2037
+ Data imports omit runner classes, Vitest runtime subpaths, and Testkit contract
2038
+ registration. A native collection check also rejects cases obtained through
2039
+ `import.meta.vitest` or the native global. Assertions, mocks, hooks, and root type-only imports
2040
+ remain supported.
2041
+
2042
+ `testkit typecheck` reports data suites under `node-vitest-data` and checks them
2043
+ separately from contract suites sharing the same tsconfig. Direct-import lint
2044
+ feedback remains; managed imports and native collection checks replace
2045
+ whole-program provenance analysis.
2046
+
2047
+ ## UI authentication fixtures
2048
+
2049
+ The product-specific UI authentication API is deprecated and removed in the next
2050
+ release. Remove `ui.auth(...)` and its authentication configuration from
2051
+ `testkit.config.ts`. The provisioning, login, session-storage, authenticated-shell,
2052
+ and provisioned-sandbox exports from `@gfdelarue/testkit/playwright` are removed.
2053
+ Their identity, session, profile, and sandbox types are removed too.
2054
+
2055
+ Move signup, login, SQL seeding, session extraction, local storage, and
2056
+ application readiness into your own Playwright fixtures. Use native
2057
+ `test.extend`, request contexts, page locators, and browser storage APIs.
2058
+ Testkit retains `createUiSandbox`, runtime context, target safety, and settling
2059
+ helpers. Put generic `frontendBaseUrl`, `backendBaseUrl`, and `allowRemote`
2060
+ settings directly under `ui`, or pass them as sandbox options.
2061
+ The separate `auth.fixture` API is removed too; see above.
2062
+
2063
+ ## Configuration metadata
2064
+
2065
+ Auth and Node toolchain builders keep their existing names and options. Their
2066
+ returned objects no longer carry the redundant `json-session`, `auth-topology`,
2067
+ `testkit-oidc-provider`, or `node` kind fields. Remove reads of those metadata
2068
+ fields and omit `kind: "node"` from inline toolchain configuration. The
2069
+ `auth.topologies` builders are removed with `auth.fixture` (see above).
2070
+
2071
+ Existing database state directories and verified Node toolchain caches remain
2072
+ usable. The database state marker keeps its filename and no longer stores a
2073
+ backend value. Node toolchain environment variables and cache fingerprints
2074
+ retain their existing values.
2075
+
2076
+ ## Local status and history removal
2077
+
2078
+ Testkit now keeps immutable run bundles locally and publishes selected bundles
2079
+ explicitly to Testkit Cloud. These flags and outputs are removed:
2080
+
2081
+ - `testkit run --write-status` and `--allow-partial-status`. Both are unknown flags.
2082
+ - The product-root `testkit.status.json` snapshot and `run` JSON's `statusArtifact` field.
2083
+ - `.testkit/history/tests.json`, its per-test counts, durations, timestamps, and last status.
2084
+ - Discovery JSON's `history`, `files[].history`, and `summary.latestResults` fields.
2085
+ Compact and verbose discovery output no longer render history or pass rates.
2086
+ - Workspace JSON's `history`, `discovery.history`, and
2087
+ `discovery.summary.latestResults` fields. Targets no longer have source `history`;
2088
+ discovery file targets no longer receive a last status from history.
2089
+ - Run planning's `tasks[].historyKey` and the `history` estimate source.
2090
+
2091
+ Remove the retired flags from scripts. Consumers of a single run should verify
2092
+ its bundle at `.testkit/results/runs/<run-id>/`. Consumers of cross-run counts,
2093
+ last outcomes, durations, or trends should use Testkit Cloud. The `status`
2094
+ command still reports current runtime and process state.
2095
+
2096
+ `testkit run` prints the sealed run ID and returns it as `runId` with `--json`,
2097
+ including failed runs. Use `testkit cloud publish --run <run-id>` to publish
2098
+ that exact bundle. Publication has its own exit status. See the
2099
+ [README CI recipe](../README.md) for preserving the test outcome while
2100
+ publishing evidence. The newest-bundle default remains available interactively.
2101
+
2102
+ Old `.testkit/history/` files are ignored and left untouched. Testkit does not
2103
+ migrate their contents. `.testkit/timings.json` and its schema 2 are unchanged.
2104
+ Without usable timings, the scheduler uses a default estimate until execution
2105
+ records a timing. Existing status files are no longer excluded from repository
2106
+ fingerprints; remove them from the product repository if they are obsolete.
2107
+
2108
+ Run schema 19, bundle schema 2, provenance schema 3, and retention schema 1 are
2109
+ unchanged. Retention still prunes bundles and unreferenced objects but no longer
2110
+ appends the write-only `decisions.jsonl` ledger. Legacy ledger metadata in a
2111
+ supplied bundle remains readable and validated when present.
2112
+
2113
+ Cloud receipts follow the current publication API: `decisionReasons` contains
2114
+ `{ ruleId, message }` objects, and `policyId` plus `policyRevision` identify the
2115
+ evaluated policy. They replace the client's stale string reasons and
2116
+ `policyVersion` fields. Text publication output prints each reason's message.