@gfdelarue/testkit 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +136 -110
- package/dist/app/bugs/draft.js +2 -2
- package/dist/app/bugs/guide.js +1 -1
- package/dist/app/bugs/record.js +2 -2
- package/dist/app/campaigns/index.js +2 -2
- package/dist/app/destroy/index.js +1 -1
- package/dist/app/discover/index.js +2 -2
- package/dist/app/environments/static-values.js +1 -1
- package/dist/app/environments/supplied.js +1 -1
- package/dist/app/inspect/index.js +1 -1
- package/dist/app/lint/native.js +1 -1
- package/dist/app/lint/options.js +1 -1
- package/dist/app/run/engine-orchestration.js +1 -1
- package/dist/app/run/environment.js +1 -1
- package/dist/app/run/orchestrator.js +1 -1
- package/dist/app/run/request.js +1 -1
- package/dist/app/run/run-finalization.js +1 -1
- package/dist/app/run/task-batches.js +1 -1
- package/dist/app/run/worker-loop.js +1 -1
- package/dist/app/status/model.js +1 -1
- package/dist/app/status/runs.js +1 -1
- package/dist/app/typecheck/index.js +4 -4
- package/dist/cli/command-catalog.js +1 -1
- package/dist/cli/command-flags.js +1 -1
- package/dist/cli/commands/artifacts.js +2 -2
- package/dist/cli/commands/discover.js +1 -1
- package/dist/cli/commands/guard/unsafe-entrypoint.js +1 -1
- package/dist/cli/commands/run.js +2 -2
- package/dist/cli/commands/typecheck.js +1 -1
- package/dist/cli/commands/workspace.js +1 -1
- package/dist/config/authoring/database-fixture-discovery.js +1 -0
- package/dist/config/authoring/index.js +1 -1
- package/dist/config/loader/config-loader.js +1 -1
- package/dist/config/loader/decode-discovery.js +1 -1
- package/dist/config/loader/discover-tests.js +1 -1
- package/dist/config/loader/discovery.js +2 -2
- package/dist/config/loader/file-metadata.js +1 -1
- package/dist/config/loader/groups.js +2 -1
- package/dist/config/loader/load.js +1 -1
- package/dist/config/loader/managed-configs.js +1 -1
- package/dist/config/loader/owned-discovery.js +1 -1
- package/dist/config/loader/runtime-isolation.js +1 -1
- package/dist/config/loader/skip-config.js +1 -1
- package/dist/database/admin.js +2 -2
- package/dist/database/configured-steps.js +1 -1
- package/dist/domain/identifiers.js +1 -1
- package/dist/domain/retired-test-types.js +1 -1
- package/dist/domain/suite-selection.js +1 -1
- package/dist/domain/test-types.js +1 -1
- package/dist/engines/catalog.js +1 -1
- package/dist/engines/definitions.js +1 -1
- package/dist/engines/environment/index.js +1 -1
- package/dist/engines/installations/node-vitest.js +1 -1
- package/dist/engines/installations/playwright.js +1 -1
- package/dist/engines/managed-module-resolution.js +6 -2
- package/dist/engines/node-vitest/adapter/adapter.js +1 -1
- package/dist/engines/node-vitest/adapter/driver.js +2 -2
- package/dist/engines/node-vitest/adapter/evidence.js +2 -2
- package/dist/engines/node-vitest/adapter/workspace.js +8 -24
- package/dist/engines/node-vitest/inventory/index.js +2 -2
- package/dist/engines/node-vitest/inventory/list-files.js +3 -3
- package/dist/engines/node-vitest/runtime/data/index.js +1 -1
- package/dist/engines/node-vitest/runtime/data/report.js +1 -1
- package/dist/engines/node-vitest/runtime/data/scenario.js +2 -2
- package/dist/engines/node-vitest/runtime/runtime-input.js +1 -1
- package/dist/engines/playwright/adapter/adapter.js +1 -1
- package/dist/engines/playwright/adapter/config.js +2 -2
- package/dist/engines/playwright/runtime/managed-test.js +1 -0
- package/dist/environment/lifecycle.js +1 -1
- package/dist/environment/run-environment.js +1 -1
- package/dist/evidence/artifact-config.js +1 -1
- package/dist/evidence/maintenance.js +1 -0
- package/dist/evidence/read-run.js +1 -1
- package/dist/kernel/known-keys.js +1 -0
- package/dist/process/execa-bundle.json +1 -1
- package/dist/repo/fingerprint-policy.js +1 -1
- package/dist/steps/configured-step-testkit-modules.js +1 -1
- package/dist/toolchains/index.js +2 -2
- package/dist/toolchains/vitest/index.js +1 -1
- package/dist/types/config.d.ts +7 -86
- package/dist/types/environment.d.ts +6 -0
- package/dist/types/node-vitest-data.d.ts +2 -218
- package/dist/types/workspace.d.ts +27 -11
- package/docs/bug-records.md +3 -8
- package/docs/engines.md +63 -17
- package/docs/release-notes.md +2116 -0
- package/docs/test-types.md +19 -69
- package/package.json +2 -5
- package/testkit-config-schema.md +65 -128
- package/dist/app/lint/rules/data.js +0 -1
- package/dist/config/authoring/runtime.js +0 -1
- package/dist/engines/node-vitest/runtime/contract/index.js +0 -1
- package/dist/engines/node-vitest/runtime/data/http.js +0 -1
- package/dist/engines/node-vitest/runtime/data/runner.js +0 -1
- package/dist/types/node-vitest-contract.d.ts +0 -75
package/README.md
CHANGED
|
@@ -43,8 +43,9 @@ npm run testkit
|
|
|
43
43
|
|
|
44
44
|
The command exits zero only when at least one test file completed and every
|
|
45
45
|
required file passed. Test failures, infrastructure failures, interrupted or
|
|
46
|
-
not-run files,
|
|
47
|
-
|
|
46
|
+
not-run files, a skipped file, and an empty selection all fail the gate. A file
|
|
47
|
+
that a `skip` rule or its own `skip` metadata skips fails the gate even when
|
|
48
|
+
every other file passes, and `--allow-empty` doesn't change that. `npm run testkit -- --allow-empty` is an explicit escape hatch for a job
|
|
48
49
|
whose empty selection is intentional; it should not be the default CI command.
|
|
49
50
|
|
|
50
51
|
Testkit writes the immutable, verifiable run bundle under
|
|
@@ -264,8 +265,7 @@ the report to the chosen platform.
|
|
|
264
265
|
|
|
265
266
|
Testkit has two built-in engines:
|
|
266
267
|
|
|
267
|
-
- `node-vitest` runs plain Vitest test files
|
|
268
|
-
`contract` files with Node and Vitest.
|
|
268
|
+
- `node-vitest` runs plain Vitest test files with Node and Vitest.
|
|
269
269
|
- `playwright` runs `ui` files with Playwright and managed Chromium.
|
|
270
270
|
|
|
271
271
|
A plain Vitest test file needs no Testkit suffix, directory, metadata, or
|
|
@@ -301,9 +301,8 @@ versions and `tsconfig.playwright.json`.
|
|
|
301
301
|
Testkit bounds the output it captures from each command it runs, and asks the
|
|
302
302
|
command and its descendants to stop on cancellation or timeout.
|
|
303
303
|
|
|
304
|
-
Managed task environment is engine input.
|
|
305
|
-
|
|
306
|
-
same contract from `readPlaywrightContext()?.env`. Managed Node processes also
|
|
304
|
+
Managed task environment is engine input. Playwright suites read it from
|
|
305
|
+
`readPlaywrightContext()?.env`. Managed Node processes also
|
|
307
306
|
expose each value as an immutable, non-enumerable `process.env.KEY` property so
|
|
308
307
|
libraries that read `process.env` directly see it, but non-Node child processes
|
|
309
308
|
do not receive consumer values unless the suite explicitly passes them.
|
|
@@ -318,7 +317,9 @@ reserves `APPDATA`, `ComSpec`, `HOME`, `LANG`, `LC_ALL`, `LOCALAPPDATA`, `PATH`,
|
|
|
318
317
|
keys it sets itself, listed in
|
|
319
318
|
[Environment](docs/engines.md#environment);
|
|
320
319
|
other `TESTKIT_` and `VITEST_` keys, such as
|
|
321
|
-
`TESTKIT_REQUIRE_COMPLETE_SUITE`, pass through
|
|
320
|
+
`TESTKIT_REQUIRE_COMPLETE_SUITE`, pass through, though Vitest's own `TEST`,
|
|
321
|
+
`VITEST`, `VITEST_MODE`, `FORCE_TTY`, `MODE`, `DEV`, `PROD`, and `SSR`
|
|
322
|
+
replace a group's values;
|
|
322
323
|
- Playwright reserves the `TESTKIT_`, `PLAYWRIGHT_`, `PWTEST_`, and `PW_TEST_`
|
|
323
324
|
prefixes plus `BASE_URL`, `FORCE_COLOR`, `NODE_OPTIONS`, and `NO_COLOR`.
|
|
324
325
|
|
|
@@ -344,12 +345,10 @@ npx @gfdelarue/testkit discover --json > .testkit/discovery.json
|
|
|
344
345
|
npx @gfdelarue/testkit discover --output .testkit/discovery.json
|
|
345
346
|
|
|
346
347
|
# Filter by type
|
|
347
|
-
npx @gfdelarue/testkit --type contract
|
|
348
|
-
npx @gfdelarue/testkit --type data
|
|
349
|
-
npx @gfdelarue/testkit --type contract,data
|
|
350
348
|
npx @gfdelarue/testkit --type ui
|
|
349
|
+
npx @gfdelarue/testkit --type plain
|
|
351
350
|
|
|
352
|
-
#
|
|
351
|
+
# Run up to 8 test tasks at once
|
|
353
352
|
npx @gfdelarue/testkit --task-concurrency 8
|
|
354
353
|
|
|
355
354
|
# One file-level wall clock budget for every suite file
|
|
@@ -357,15 +356,16 @@ npx @gfdelarue/testkit --file-timeout-seconds 60
|
|
|
357
356
|
|
|
358
357
|
# Specific group / suite
|
|
359
358
|
npx @gfdelarue/testkit --group frontend --type ui -s navigation
|
|
360
|
-
npx @gfdelarue/testkit --group api
|
|
361
|
-
npx @gfdelarue/testkit --
|
|
359
|
+
npx @gfdelarue/testkit --group api -s health
|
|
360
|
+
npx @gfdelarue/testkit --group api --type plain
|
|
361
|
+
npx @gfdelarue/testkit --type ui -s ui:checkout
|
|
362
362
|
|
|
363
363
|
# Exact file
|
|
364
|
-
npx @gfdelarue/testkit --type
|
|
364
|
+
npx @gfdelarue/testkit --type ui --file __testkit__/checkout.ui.testkit.ts
|
|
365
365
|
npx @gfdelarue/testkit run --file src/sum.test.ts
|
|
366
366
|
|
|
367
367
|
# Temporarily ignore repo-declared skip rules
|
|
368
|
-
npx @gfdelarue/testkit --ignore-skip-rules --file
|
|
368
|
+
npx @gfdelarue/testkit --ignore-skip-rules --file src/billing.test.ts
|
|
369
369
|
|
|
370
370
|
# Lifecycle
|
|
371
371
|
npx @gfdelarue/testkit status
|
|
@@ -380,15 +380,15 @@ npx @gfdelarue/testkit db schema refresh --service api
|
|
|
380
380
|
npx @gfdelarue/testkit db schema verify --service api
|
|
381
381
|
```
|
|
382
382
|
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
with `--type all`,
|
|
386
|
-
|
|
383
|
+
`--type ui` and typed suite selectors such as `ui:checkout` select typed files
|
|
384
|
+
only. `--type plain` selects plain Vitest test files only. A run without
|
|
385
|
+
`--type`, or with `--type all`, runs both. `--type` narrows the other filters,
|
|
386
|
+
so `--group api --type plain` runs the `api` group's plain files and none of its
|
|
387
|
+
UI files.
|
|
387
388
|
|
|
388
|
-
`run`, `discover`, `artifacts`, and `
|
|
389
|
-
`--group`.
|
|
390
|
-
|
|
391
|
-
take `--service` to name a service.
|
|
389
|
+
`run`, `discover`, `artifacts`, `workspace`, and `typecheck` select test groups
|
|
390
|
+
with `--group`. `status`, `destroy`, `cleanup`, and `db schema` take `--service`
|
|
391
|
+
to name a service.
|
|
392
392
|
|
|
393
393
|
Database cleanup is scoped to the selected product. `cleanup --resources --global`
|
|
394
394
|
is retired; use `--dir <product> cleanup --resources` for each product. After
|
|
@@ -396,8 +396,11 @@ deleting a product directory, remove its orphaned containers and volumes
|
|
|
396
396
|
explicitly through Docker.
|
|
397
397
|
|
|
398
398
|
Use `execution.taskConcurrency` in config and `--task-concurrency` on the CLI.
|
|
399
|
-
|
|
400
|
-
|
|
399
|
+
A task is a typed file, a batch of typed files, or one Vitest invocation of
|
|
400
|
+
plain files. Plain files in one invocation run with Vitest's file parallelism,
|
|
401
|
+
which `taskConcurrency` doesn't limit.
|
|
402
|
+
`execution.workers`, `presets.nodeNext({ workers })` and `--workers` are
|
|
403
|
+
deprecated aliases. When both names are
|
|
401
404
|
provided in config or on the CLI, their values must match. The CLI value overrides
|
|
402
405
|
config after those checks. See the [execution settings](testkit-config-schema.md#execution).
|
|
403
406
|
|
|
@@ -682,15 +685,10 @@ A file can also name `locks`.
|
|
|
682
685
|
one.
|
|
683
686
|
|
|
684
687
|
In a typed file, the engine value is a required literal. The supported public
|
|
685
|
-
authoring
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
from Testkit's managed `vitest`, `@playwright/test`, `playwright`, and
|
|
690
|
-
`playwright-core` packages.
|
|
691
|
-
There are no legacy aliases or implicit-runner compatibility paths. A typed
|
|
692
|
-
Node/Vitest file resolves `paths` from its nearest `tsconfig.json`, through
|
|
693
|
-
Vite's `resolve.tsconfigPaths`.
|
|
688
|
+
authoring path is `@gfdelarue/testkit/playwright`. Playwright suites may also
|
|
689
|
+
import exact public APIs directly from Testkit's managed `@playwright/test`,
|
|
690
|
+
`playwright`, and `playwright-core` packages.
|
|
691
|
+
There are no legacy aliases or implicit-runner compatibility paths.
|
|
694
692
|
|
|
695
693
|
`testkit.config.ts` is optional. Without it, Testkit runs every test file with
|
|
696
694
|
zero setup: it starts no server, provisions no database, and loads no `.env`
|
|
@@ -727,7 +725,8 @@ setup processes, so set it deliberately.
|
|
|
727
725
|
Testkit-owned Docker resources are labelled at creation time so they can be
|
|
728
726
|
found again even if product-local `.testkit` state is deleted. `testkit destroy`
|
|
729
727
|
removes labelled resources for the current product after service state is
|
|
730
|
-
destroyed.
|
|
728
|
+
destroyed. It leaves run results under `.testkit/results` and other files
|
|
729
|
+
under `.testkit`, such as the compiled config and timings, in place. `testkit cleanup --resources` cleans stale resources for the current
|
|
731
730
|
product. Both commands remove only the current product's resources and leave
|
|
732
731
|
other products' alone, including their idle caches. Host-local Postgres data
|
|
733
732
|
lives in a Testkit-labelled named Docker volume, so cleanup can find and remove
|
|
@@ -1016,14 +1015,14 @@ something its environment prepared. Today that is an HTTP capture, which a
|
|
|
1016
1015
|
The handle is named after the capture's `id`:
|
|
1017
1016
|
|
|
1018
1017
|
```js
|
|
1019
|
-
// tests/webhook.
|
|
1018
|
+
// tests/webhook.test.setup.mjs
|
|
1020
1019
|
export async function setup({ sidecars }) {
|
|
1021
1020
|
await sidecars.httpCapture({ id: "webhook", path: "/webhook" });
|
|
1022
1021
|
}
|
|
1023
1022
|
```
|
|
1024
1023
|
|
|
1025
1024
|
```ts
|
|
1026
|
-
// tests/webhook.
|
|
1025
|
+
// tests/webhook.test.ts
|
|
1027
1026
|
import { handle } from "@gfdelarue/testkit/environment";
|
|
1028
1027
|
|
|
1029
1028
|
const webhook = handle("webhook");
|
|
@@ -1077,19 +1076,20 @@ test, in any order.
|
|
|
1077
1076
|
|
|
1078
1077
|
```ts
|
|
1079
1078
|
import { randomUUID } from "node:crypto";
|
|
1080
|
-
import {
|
|
1081
|
-
import {
|
|
1082
|
-
import { expect, onTestFinished } from "vitest";
|
|
1079
|
+
import { endpoint } from "@gfdelarue/testkit/environment";
|
|
1080
|
+
import { expect, onTestFinished, test } from "vitest";
|
|
1083
1081
|
|
|
1084
1082
|
import { sandbox } from "../payments-sandbox.js";
|
|
1085
1083
|
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
contractScenario("a refund reaches the sandbox customer", async ({ http }) => {
|
|
1084
|
+
test("a refund reaches the sandbox customer", async () => {
|
|
1089
1085
|
const customer = await sandbox.createCustomer({ email: `refund-${randomUUID()}@example.test` });
|
|
1090
1086
|
onTestFinished(() => sandbox.deleteCustomer(customer.id));
|
|
1091
1087
|
|
|
1092
|
-
const response = await
|
|
1088
|
+
const response = await fetch(new URL("/api/refunds", endpoint("api")), {
|
|
1089
|
+
method: "POST",
|
|
1090
|
+
headers: { "content-type": "application/json" },
|
|
1091
|
+
body: JSON.stringify({ customerId: customer.id, amountCents: 500 }),
|
|
1092
|
+
});
|
|
1093
1093
|
expect(response.status).toBe(201);
|
|
1094
1094
|
});
|
|
1095
1095
|
```
|
|
@@ -1131,12 +1131,12 @@ export const testkit = defineFile({ engine: "node-vitest", locks: ["payments-san
|
|
|
1131
1131
|
|
|
1132
1132
|
### Node setup beside a test
|
|
1133
1133
|
|
|
1134
|
-
A test such as `tests/webhook.
|
|
1135
|
-
`tests/webhook.
|
|
1136
|
-
|
|
1134
|
+
A test such as `tests/webhook.test.ts` can have a sibling
|
|
1135
|
+
`tests/webhook.test.setup.mjs`. Testkit runs its named `setup` export, or its
|
|
1136
|
+
default export, before starting the test engine:
|
|
1137
1137
|
|
|
1138
1138
|
```js
|
|
1139
|
-
// tests/webhook.
|
|
1139
|
+
// tests/webhook.test.setup.mjs
|
|
1140
1140
|
export async function setup({ sidecars }) {
|
|
1141
1141
|
const capture = await sidecars.httpCapture({ path: "/webhook" });
|
|
1142
1142
|
return {
|
|
@@ -1148,7 +1148,8 @@ export async function setup({ sidecars }) {
|
|
|
1148
1148
|
Setup is part of preparing the file's environment. Testkit runs it once, after
|
|
1149
1149
|
it acquires the services the file's group uses and before the test engine
|
|
1150
1150
|
starts. Testkit never batches a Playwright file that has setup with other
|
|
1151
|
-
Playwright files
|
|
1151
|
+
Playwright files, and a plain Vitest file that has setup runs in its own Vitest
|
|
1152
|
+
invocation. Files without setup still batch.
|
|
1152
1153
|
|
|
1153
1154
|
Setup runs with its own JavaScript state. It can use Node APIs, prepare
|
|
1154
1155
|
external resources, and start HTTP capture listeners. Tests
|
|
@@ -1165,7 +1166,9 @@ setup modules. Its tests receive the environment file's `setupData` instead.
|
|
|
1165
1166
|
|
|
1166
1167
|
Return plain JSON, or `{ data, cleanup }` with an optional cleanup function.
|
|
1167
1168
|
Testkit passes the data to the engine as `setupData` and records it with the
|
|
1168
|
-
setup module path and sidecar handles.
|
|
1169
|
+
setup module path and sidecar handles. Tests read it with `setupData()` from
|
|
1170
|
+
`@gfdelarue/testkit/environment`, which returns `{}` when the file has no setup
|
|
1171
|
+
module and throws outside a Testkit run. Bare `null` or `undefined`, and a
|
|
1169
1172
|
cleanup-only result, give empty data. `{ data: null }` preserves `null`.
|
|
1170
1173
|
Data must encode to at most 1 MiB of JSON. Invalid data reports its JSON path,
|
|
1171
1174
|
such as `$.items[1]`.
|
|
@@ -1189,15 +1192,55 @@ most 256 connections.
|
|
|
1189
1192
|
|
|
1190
1193
|
Discovery tries sibling `.setup.mjs`, sibling `.setup.js`, then the matching
|
|
1191
1194
|
`.mjs` and `.js` files in `__setup__`, in that order. For the example above,
|
|
1192
|
-
the fallback is `tests/__setup__/webhook.
|
|
1195
|
+
the fallback is `tests/__setup__/webhook.test.mjs`.
|
|
1193
1196
|
|
|
1194
1197
|
### HTTP and persistence checks
|
|
1195
1198
|
|
|
1196
1199
|
Use plain Vitest files for ordinary HTTP, service, and database checks. Read
|
|
1197
1200
|
configured URLs with `endpoint(name)` and `database(name)` from
|
|
1198
1201
|
`@gfdelarue/testkit/environment`, then use `fetch` or your application's
|
|
1199
|
-
database client.
|
|
1200
|
-
|
|
1202
|
+
database client.
|
|
1203
|
+
|
|
1204
|
+
### Record data-flow evidence
|
|
1205
|
+
|
|
1206
|
+
`dataFlow(context)` from `@gfdelarue/testkit/node-vitest/data` records what a
|
|
1207
|
+
plain Vitest test sees data do: metrics, counts, bytes, durations, money,
|
|
1208
|
+
ratios, sequences, records, actors, systems, transitions, events, findings,
|
|
1209
|
+
external calls, and cache, LLM, and RPC usage. Budgets check the recorded
|
|
1210
|
+
values, for example that a metric stays under a limit or that a sequence was
|
|
1211
|
+
recorded.
|
|
1212
|
+
|
|
1213
|
+
```ts
|
|
1214
|
+
import { endpoint } from "@gfdelarue/testkit/environment";
|
|
1215
|
+
import { dataFlow } from "@gfdelarue/testkit/node-vitest/data";
|
|
1216
|
+
import { test } from "vitest";
|
|
1217
|
+
|
|
1218
|
+
test("lists one page of orders", async (context) => {
|
|
1219
|
+
const flow = dataFlow(context);
|
|
1220
|
+
const response = await fetch(`${endpoint("api")}/orders?page=1`);
|
|
1221
|
+
const body = await response.text();
|
|
1222
|
+
flow.recordHttpRequest({
|
|
1223
|
+
method: "GET",
|
|
1224
|
+
path: "/orders",
|
|
1225
|
+
status: response.status,
|
|
1226
|
+
responseBytes: Buffer.byteLength(body),
|
|
1227
|
+
});
|
|
1228
|
+
flow.expectMetric("http.response.bytes").atMost(64_000);
|
|
1229
|
+
});
|
|
1230
|
+
```
|
|
1231
|
+
|
|
1232
|
+
- `dataFlow` needs the context of a running test. Every call in one attempt
|
|
1233
|
+
returns the same recorder, and a call without that context throws.
|
|
1234
|
+
- The library reads no URL, environment value, fixture file, or Testkit config.
|
|
1235
|
+
Read named URLs from `@gfdelarue/testkit/environment`, log in from test code,
|
|
1236
|
+
and read files with `fs`.
|
|
1237
|
+
- A failed budget fails its test when the test finishes. The failure detail has
|
|
1238
|
+
the kind `data-flow-budget` and a key made of the test's full title and the
|
|
1239
|
+
budget.
|
|
1240
|
+
- Testkit writes `data-flow.json` and `data-flow.md` for a file whose tests
|
|
1241
|
+
recorded something. Each such test is one scenario, from its final attempt. A
|
|
1242
|
+
test that records nothing adds nothing, and a file whose tests record
|
|
1243
|
+
nothing gets no data-flow artifacts.
|
|
1201
1244
|
|
|
1202
1245
|
## Discovery
|
|
1203
1246
|
|
|
@@ -1228,24 +1271,26 @@ Testkit supports Vitest 4.x. Another major, a broken installation, a root
|
|
|
1228
1271
|
Vitest config that fails to load, or a Vitest that `package.json` declares but
|
|
1229
1272
|
that is not installed fails before any test runs, naming the version and path.
|
|
1230
1273
|
`--vitest-runner managed` avoids the repository's Vitest. A run that selects
|
|
1231
|
-
only typed files, with `--type` or with typed `--file` paths, `bug record` or
|
|
1274
|
+
only typed files, with `--type ui` or with typed `--file` paths, `bug record` or
|
|
1232
1275
|
`bug file` for a typed test, and the commands that select no tests (`status`,
|
|
1233
1276
|
`run status`, `doctor`, `destroy`, `cleanup`, and `db schema`) don't load the
|
|
1234
1277
|
repository's Vitest, so they still work. A product that neither installs nor
|
|
1235
1278
|
declares Vitest has no plain test files.
|
|
1236
1279
|
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1280
|
+
Plain files run in one Vitest invocation per environment their groups give them.
|
|
1281
|
+
Groups that give their files the same values share one, such as `root` and the
|
|
1282
|
+
other groups that use no services. A group that uses a service Testkit starts
|
|
1283
|
+
gets an invocation of its own.
|
|
1284
|
+
Inside each invocation, root global setup, teardown, and the repository's
|
|
1285
|
+
reporters run as they do under `vitest run`, and Testkit's reporter runs beside
|
|
1286
|
+
them, so a run calls them once for each invocation. An invocation's deadline is
|
|
1287
|
+
the sum of its files' timeouts; Vitest's own test and hook timeouts apply inside
|
|
1288
|
+
it. A file that two projects select runs in both, its result combines both
|
|
1289
|
+
runs, and its evidence names each run's project. A file with a setup module,
|
|
1290
|
+
locks, or isolation other than `shared` runs in its own Vitest invocation with
|
|
1291
|
+
its own environment. Invocations run one after another unless
|
|
1292
|
+
`--task-concurrency` or `execution.taskConcurrency` is above 1. To run one group's
|
|
1293
|
+
files alone, select it with `--group`.
|
|
1249
1294
|
|
|
1250
1295
|
Plain files see the host environment Testkit started with: `HOME`, `PATH`,
|
|
1251
1296
|
temporary and XDG directories, locale, `CI`, `FORCE_COLOR`, `NO_COLOR`, and any
|
|
@@ -1259,14 +1304,21 @@ shell's environment and the group's values; as under `vitest run`, Vitest sets
|
|
|
1259
1304
|
`TEST`, `VITEST`, and `NODE_ENV` after them. `--vitest-runner managed` leaves the
|
|
1260
1305
|
shell's `NODE_OPTIONS` out.
|
|
1261
1306
|
|
|
1307
|
+
When Vitest detects a coding agent from the environment, such as `CLAUDECODE`,
|
|
1308
|
+
`AI_AGENT`, or `CURSOR_AGENT`, and the repository's Vitest config names no
|
|
1309
|
+
reporter, Vitest uses its agent reporter. That reporter prints no console output
|
|
1310
|
+
from a passing test, so the output is missing from the file's
|
|
1311
|
+
`node-vitest-stdout` artifact. A failing test's output still appears. Name a
|
|
1312
|
+
reporter in the Vitest config, such as `reporters: ["default"]`, to keep it.
|
|
1313
|
+
|
|
1262
1314
|
Two settings narrow the files Vitest selects: `discovery.exclude` in `testkit.config.ts`, and
|
|
1263
1315
|
a declared group's `exclude`. Testkit's built-in exclusions for typed files,
|
|
1264
|
-
such as `dist`, `build`, and `coverage`, `discovery.roots
|
|
1265
|
-
|
|
1316
|
+
such as `dist`, `build`, and `coverage`, and `discovery.roots` do not apply to
|
|
1317
|
+
plain files.
|
|
1266
1318
|
|
|
1267
1319
|
A plain file belongs to the group whose roots contain it, or to the implicit
|
|
1268
|
-
`root` group, which uses no service.
|
|
1269
|
-
|
|
1320
|
+
`root` group, which uses no service. Testkit starts no service for a plain
|
|
1321
|
+
file in `root`.
|
|
1270
1322
|
|
|
1271
1323
|
The plain files in one directory form a suite named after that directory,
|
|
1272
1324
|
relative to the group's root. In `root`, `--suite src` selects
|
|
@@ -1275,7 +1327,10 @@ its own suite, so `sum.test.ts` in a group rooted at `src` is in suite `sum`.
|
|
|
1275
1327
|
When two suites in a group would share a name, Testkit lengthens the later
|
|
1276
1328
|
ones with parent directories, such as `api-src`, and numbers them if that is
|
|
1277
1329
|
not enough. `testkit typecheck` checks plain TypeScript test files and skips
|
|
1278
|
-
plain JavaScript ones, as `tsc` does without `allowJs`.
|
|
1330
|
+
plain JavaScript ones, as `tsc` does without `allowJs`. It adds the TypeScript
|
|
1331
|
+
files in each Vitest project's `setupFiles` to the program, so a test can use
|
|
1332
|
+
the matchers a setup file declares, such as those from
|
|
1333
|
+
`@testing-library/jest-dom/vitest`.
|
|
1279
1334
|
|
|
1280
1335
|
### Typed suites
|
|
1281
1336
|
|
|
@@ -1308,16 +1363,9 @@ production owner when a narrower owner exists.
|
|
|
1308
1363
|
|
|
1309
1364
|
Concrete layouts:
|
|
1310
1365
|
|
|
1311
|
-
- `src/api/routes/__testkit__/auth-me.contract.testkit.ts`
|
|
1312
|
-
- `src/db/__testkit__/sessions-count-type.contract.testkit.ts`
|
|
1313
1366
|
- `frontend/src/components/layout/__testkit__/navigation.ui.testkit.ts`
|
|
1314
|
-
- `src/internal/handler/repos/__testkit__/crud.contract.testkit.ts`
|
|
1315
|
-
|
|
1316
|
-
`testkit` uses these suffixes automatically:
|
|
1317
1367
|
|
|
1318
|
-
|
|
1319
|
-
- `*.data.testkit.ts`
|
|
1320
|
-
- `*.ui.testkit.ts`
|
|
1368
|
+
`testkit` uses the `*.ui.testkit.ts` suffix automatically.
|
|
1321
1369
|
|
|
1322
1370
|
The suffix never chooses an engine. Each discovered typed file must also
|
|
1323
1371
|
contain one root-package `defineFile(...)` declaration with a supported literal
|
|
@@ -1326,29 +1374,6 @@ Unsupported pairs fail discovery; for example, a `*.ui.testkit.ts` file cannot
|
|
|
1326
1374
|
declare `engine: "node-vitest"`.
|
|
1327
1375
|
|
|
1328
1376
|
See [docs/test-types.md](docs/test-types.md) for the canonical type model.
|
|
1329
|
-
`*.contract.testkit.ts` files are Node-backed service/runtime contract suites.
|
|
1330
|
-
They use `contractScenario(...)` from
|
|
1331
|
-
`@gfdelarue/testkit/node-vitest/contract`, declare `engine: "node-vitest"`, and run
|
|
1332
|
-
through the `node-vitest-contract` profile for checks that need Node APIs but
|
|
1333
|
-
are not data-flow evidence suites. They are native Vitest suites:
|
|
1334
|
-
`contractScenario(...)` is the managed-context convenience, while direct public
|
|
1335
|
-
Vitest runner APIs remain valid for behavior such as `test.skip(...)` and
|
|
1336
|
-
`test.todo(...)`.
|
|
1337
|
-
|
|
1338
|
-
`*.data.testkit.ts` files are Testkit data-flow/data-contract suites, not raw
|
|
1339
|
-
Vitest files with a Testkit suffix. They must use `dataScenario(...)` from
|
|
1340
|
-
`@gfdelarue/testkit/node-vitest/data`, declare `engine: "node-vitest"`, and record
|
|
1341
|
-
structured data-flow evidence through the data context. Supporting
|
|
1342
|
-
`count(...)`/`event(...)` calls are not enough by themselves; model actors,
|
|
1343
|
-
systems, records, transitions, external calls, payloads, budgets, cache/LLM/RPC
|
|
1344
|
-
usage, or reconciliation evidence. Data suites run through the
|
|
1345
|
-
`node-vitest-data` profile. Use a plain Vitest test file for plain Node
|
|
1346
|
-
assertions and HTTP or database integration, `contract` for typed service or
|
|
1347
|
-
protocol contracts, and `ui` when a browser is part of the assertion.
|
|
1348
|
-
Data suites use Vitest directly for assertions, hooks, and mocking.
|
|
1349
|
-
`dataScenario(...)` owns case registration: `testkit lint` reports native
|
|
1350
|
-
registration imports in data suites, and the data runner rejects any case that
|
|
1351
|
-
was not registered through the DSL.
|
|
1352
1377
|
|
|
1353
1378
|
Test groups own test files by directory. Declare them under `groups` in
|
|
1354
1379
|
`testkit.config.ts`. Each group lists its repo-relative `roots` and the services
|
|
@@ -1380,16 +1405,17 @@ export default defineConfig({
|
|
|
1380
1405
|
- `skip`, `requirements`, and `runtimeIsolation` are group fields. See the
|
|
1381
1406
|
[config reference](testkit-config-schema.md#groups).
|
|
1382
1407
|
|
|
1383
|
-
A config with `services` and no `groups` key
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1408
|
+
Services own no test files. A config with `services` and no `groups` key
|
|
1409
|
+
fails to load when a service's `local.cwd` or `discovery.roots` holds a typed
|
|
1410
|
+
test file, which that service owned before 0.6.0. The error lists the `groups`
|
|
1411
|
+
entries that keep those files with their services. `status`, `destroy`,
|
|
1412
|
+
`cleanup`, and `db schema` still run on it. Otherwise every file belongs to
|
|
1413
|
+
`root`.
|
|
1388
1414
|
|
|
1389
1415
|
Suite names are inferred from the colocated path:
|
|
1390
1416
|
|
|
1391
|
-
- `auth/__testkit__/*.
|
|
1392
|
-
- `routes/__testkit__/
|
|
1417
|
+
- `auth/__testkit__/*.ui.testkit.ts` => `auth`
|
|
1418
|
+
- `routes/__testkit__/checkout.ui.testkit.ts` => `routes`
|
|
1393
1419
|
|
|
1394
1420
|
Discovery is also a first-class CLI/API surface:
|
|
1395
1421
|
|
package/dist/app/bugs/draft.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
import G from"node:crypto";import A from"node:path";import{isPublicTestType as j}from"../../domain/identifiers.js";import{isUnknownRecord as g}from"../../kernel/unknown-values.js";import{BUG_CATCHERS as O,BUG_CAUSES as P,BUG_CONFIDENCES as C,BUG_ESCAPES as U,BUG_IMPACTS as F,BUG_SEVERITIES as R,categoryIds as _,isOneOf as E}from"./categories.js";import{BUG_FACTS_TEST_TYPES as z,decodeFaultLines as D,isCommitId as $}from"./facts.js";import{redactBugText as B}from"../../tracker/redaction.js";import{isTimestamp as w}from"../campaigns/campaign.js";export const BUG_DRAFT_SCHEMA_VERSION=1,BUG_TITLE_MAX_LENGTH=200,BUG_SECTION_MAX_LENGTH=12e3;const V=48,L=/^[a-z0-9]+(?:-[a-z0-9]+)*$/u;export const CAMPAIGN_ID_MAX_LENGTH=60;export class BugRecordError extends Error{problems;constructor(e,n){super(`${e}:
|
|
2
2
|
${n.map(s=>`- ${s}`).join(`
|
|
3
|
-
`)}`),this.name="BugRecordError",this.problems=n}}export function isCampaignId(t){return typeof t=="string"&&t.length<=CAMPAIGN_ID_MAX_LENGTH&&L.test(t)}export function testTypeOfFile(t){const e=/\.([a-z0-9]+)\.testkit\.ts$/u.exec(t)?.[1];return j(e)?e:null}function M(t){const e=/\.([a-z0-9]+)\.testkit\.ts$/u.exec(t)?.[1];return z.find(n=>n===e)??null}export function toRepositoryPath(t,e){const n=e.replaceAll("\\","/");if(n.startsWith("/")||/^[a-z]:/iu.test(n))return null;const s=A.posix.normalize(A.posix.join(t,n));return s==="."||s===".."||s.startsWith("../")?null:s}export function toProductPath(t,e){return t===""?e:A.posix.relative(t,e)}export function readBugReport(t,e){if(!g(t))throw new BugRecordError("The bug record is invalid",["it must be a JSON object"]);const n=[],s=[],u=I(t,n,{path:r=>toRepositoryPath(e,r),text:(r,d)=>{const a=B(r);return a!==r&&s.push(d),a}});if(n.length>0)throw new BugRecordError("The bug record is invalid",n);return{redactedFields:s,report:u}}export function createBugDraft(t,e){const n=testTypeOfFile(t.test.file);return{schemaVersion:BUG_DRAFT_SCHEMA_VERSION,id:bugDraftId(t),recordedAt:e.recordedAt.toISOString(),recordedCommit:e.recordedCommit,productPath:e.productPath,caughtBy:e.campaign===null?"manual":"campaign",...e.campaign===null?{}:{campaign:e.campaign},...n===null?{}:{testType:n},...e.vitest===void 0?{}:{vitest:e.vitest},...t}}export function requireOneVitestProject(t,e,n){if(e.length<2||n.project!==null)return;const s=e.map(({project:u})=>JSON.stringify(u)).join(", ");throw new Error(`test.file ${t} runs in the Vitest projects ${s}. Name the one whose run fails with --vitest-project.`)}export function readBugDraft(t){const e="The bug draft is unreadable";if(!g(t))throw new BugRecordError(e,["it must be a JSON object"]);if(t.schemaVersion!==BUG_DRAFT_SCHEMA_VERSION)throw new BugRecordError(e,[`unknown schemaVersion ${JSON.stringify(t.schemaVersion)}`]);const n=[],s=(p,b,
|
|
4
|
-
${t.test.title}`).digest("hex");return`${e||"bug"}-${n.slice(0,8)}`}export function isBugDraftId(t){return typeof t=="string"&&/^[a-z0-9]+(?:-[a-z0-9]+)*-[0-9a-f]{8}$/u.test(t)}export function namesSameTest(t,e){return t.test.file===e.test.file&&t.test.title===e.test.title}function I(t,e,n){const s=(i,o,c,f)=>{const m=i[o];if(typeof m!="string"||m.trim().length===0)return e.push(`${c} must be a non-empty string`),"";const h=m.trim(),l=h.length>f?h:n.text(h,c);return l.length>f&&e.push(`${c} must be at most ${String(f)} characters`),l},u=(i,o,c)=>{const f=i[o],m=typeof f=="string"?f.trim():"",h=m.length>0?n.path(m):null;return h===null?e.push(`${c} must be a path inside the repository`):[m,h].some(l=>B(l)!==l)&&e.push(`${c} is a path that holds text Testkit redacts as a secret`),h??""},r=(i,o,c)=>E(i,o)?o:(e.push(`${c} must be one of ${i.join(", ")}`),i[0]),d=i=>{const o=t[i];return g(o)?o:(e.push(`${i} must be an object`),{})},a=(i,o,c)=>{const f=s(i,o,c,BUG_TITLE_MAX_LENGTH);return/[\r\n]/u.test(f)&&e.push(`${c} must be one line`),f},T=a(t,"title","title"),p=d("test"),
|
|
3
|
+
`)}`),this.name="BugRecordError",this.problems=n}}export function isCampaignId(t){return typeof t=="string"&&t.length<=CAMPAIGN_ID_MAX_LENGTH&&L.test(t)}export function testTypeOfFile(t){const e=/\.([a-z0-9]+)\.testkit\.ts$/u.exec(t)?.[1];return j(e)?e:null}function M(t){const e=/\.([a-z0-9]+)\.testkit\.ts$/u.exec(t)?.[1];return z.find(n=>n===e)??null}export function toRepositoryPath(t,e){const n=e.replaceAll("\\","/");if(n.startsWith("/")||/^[a-z]:/iu.test(n))return null;const s=A.posix.normalize(A.posix.join(t,n));return s==="."||s===".."||s.startsWith("../")?null:s}export function toProductPath(t,e){return t===""?e:A.posix.relative(t,e)}export function readBugReport(t,e){if(!g(t))throw new BugRecordError("The bug record is invalid",["it must be a JSON object"]);const n=[],s=[],u=I(t,n,{path:r=>toRepositoryPath(e,r),text:(r,d)=>{const a=B(r);return a!==r&&s.push(d),a}});if(n.length>0)throw new BugRecordError("The bug record is invalid",n);return{redactedFields:s,report:u}}export function createBugDraft(t,e){const n=testTypeOfFile(t.test.file);return{schemaVersion:BUG_DRAFT_SCHEMA_VERSION,id:bugDraftId(t),recordedAt:e.recordedAt.toISOString(),recordedCommit:e.recordedCommit,productPath:e.productPath,caughtBy:e.campaign===null?"manual":"campaign",...e.campaign===null?{}:{campaign:e.campaign},...n===null?{}:{testType:n},...e.vitest===void 0?{}:{vitest:e.vitest},...t}}export function requireOneVitestProject(t,e,n){if(e.length<2||n.project!==null)return;const s=e.map(({project:u})=>JSON.stringify(u)).join(", ");throw new Error(`test.file ${t} runs in the Vitest projects ${s}. Name the one whose run fails with --vitest-project.`)}export function readBugDraft(t){const e="The bug draft is unreadable";if(!g(t))throw new BugRecordError(e,["it must be a JSON object"]);if(t.schemaVersion!==BUG_DRAFT_SCHEMA_VERSION)throw new BugRecordError(e,[`unknown schemaVersion ${JSON.stringify(t.schemaVersion)}`]);const n=[],s=(p,y,b,x)=>y(p)?p:(n.push(x),b),u=I(t,n,{path:p=>toRepositoryPath("",p),text:p=>p}),r=t.testType===void 0&&testTypeOfFile(u.test.file)===null?null:M(u.test.file);t.testType!==(r??void 0)&&n.push(r===null?"testType must be absent for a test file without a type":"testType does not match test.file");const d=s(t.caughtBy,J,"manual","caughtBy must be campaign or manual"),a=s(t.campaign,k,void 0,"campaign must be a campaign id");d==="campaign"!=(a!==void 0)&&n.push("campaign must be present when caughtBy is campaign and absent otherwise");const T={schemaVersion:BUG_DRAFT_SCHEMA_VERSION,id:s(t.id,isBugDraftId,"","id is not a draft id"),recordedAt:s(t.recordedAt,w,"","recordedAt must be a timestamp"),recordedCommit:s(t.recordedCommit,$,"","recordedCommit must be a full commit id"),productPath:s(t.productPath,N,"","productPath must be a string"),caughtBy:d,...a===void 0?{}:{campaign:a},...r===null?{}:{testType:r},...t.vitest===void 0?{}:{vitest:s(t.vitest,X,{},"vitest must hold runner, config and project text")},...u,...t.filed===void 0?{}:{filed:H(t.filed,n)}};if(n.length>0)throw new BugRecordError(e,n);return T}export function bugDraftId(t){const e=t.title.normalize("NFKD").replace(/\p{M}+/gu,"").toLowerCase().replace(/[^a-z0-9]+/gu,"-").slice(0,V).replace(/^-+|-+$/gu,""),n=G.createHash("sha256").update(`${t.test.file}
|
|
4
|
+
${t.test.title}`).digest("hex");return`${e||"bug"}-${n.slice(0,8)}`}export function isBugDraftId(t){return typeof t=="string"&&/^[a-z0-9]+(?:-[a-z0-9]+)*-[0-9a-f]{8}$/u.test(t)}export function namesSameTest(t,e){return t.test.file===e.test.file&&t.test.title===e.test.title}function I(t,e,n){const s=(i,o,c,f)=>{const m=i[o];if(typeof m!="string"||m.trim().length===0)return e.push(`${c} must be a non-empty string`),"";const h=m.trim(),l=h.length>f?h:n.text(h,c);return l.length>f&&e.push(`${c} must be at most ${String(f)} characters`),l},u=(i,o,c)=>{const f=i[o],m=typeof f=="string"?f.trim():"",h=m.length>0?n.path(m):null;return h===null?e.push(`${c} must be a path inside the repository`):[m,h].some(l=>B(l)!==l)&&e.push(`${c} is a path that holds text Testkit redacts as a secret`),h??""},r=(i,o,c)=>E(i,o)?o:(e.push(`${c} must be one of ${i.join(", ")}`),i[0]),d=i=>{const o=t[i];return g(o)?o:(e.push(`${i} must be an object`),{})},a=(i,o,c)=>{const f=s(i,o,c,BUG_TITLE_MAX_LENGTH);return/[\r\n]/u.test(f)&&e.push(`${c} must be one line`),f},T=a(t,"title","title"),p=d("test"),y=d("fault"),b=d("confidence"),x=u(p,"file","test.file");let S=[1,1];try{S=D(y.lines)}catch(i){e.push(i instanceof Error?i.message:"fault.lines is invalid")}return{title:T,severity:r(R,t.severity,"severity"),expected:s(t,"expected","expected",BUG_SECTION_MAX_LENGTH),authority:s(t,"authority","authority",BUG_SECTION_MAX_LENGTH),observed:s(t,"observed","observed",BUG_SECTION_MAX_LENGTH),rootCause:s(t,"rootCause","rootCause",BUG_SECTION_MAX_LENGTH),test:{file:x,title:a(p,"title","test.title")},fault:{lines:S,path:u(y,"path","fault.path")},cause:r(_(P),t.cause,"cause"),impact:r(_(F),t.impact,"impact"),confidence:{cause:r(C,b.cause,"confidence.cause"),impact:r(C,b.impact,"confidence.impact")},escape:r(_(U),t.escape,"escape")}}function H(t,e){return g(t)&&w(t.filedAt)&&$(t.foundAt)&&typeof t.number=="number"&&Number.isSafeInteger(t.number)&&t.number>=1&&typeof t.url=="string"?{filedAt:t.filedAt,foundAt:t.foundAt,number:t.number,url:t.url}:(e.push("filed must hold filedAt, foundAt, number, and url"),{filedAt:"",foundAt:"",number:0,url:""})}function k(t){return t===void 0||isCampaignId(t)}function J(t){return E(O,t)}function X(t){return g(t)&&Object.values(t).every(N)}function N(t){return typeof t=="string"}
|
package/dist/app/bugs/guide.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import{BUG_CAUSES as i,BUG_CONFIDENCES as u,BUG_ESCAPES as a,BUG_IMPACTS as s,BUG_SEVERITIES as o}from"./categories.js";import{BUG_TITLE_MAX_LENGTH as r}from"./draft.js";import{SEVERITY_LABELS as h}from"./issue.js";export const BUG_RECORD_EXAMPLE={title:"A refund pays out twice when two requests overlap",severity:"high",expected:"A refund pays out once.",authority:'`docs/refunds.md`, "Refunds are idempotent on the order id".',observed:"Two overlapping refund requests for one order both pay out.",rootCause:"`refund()` reads the order's refund state and writes the payout without a lock.",test:{file:"src/checkout/
|
|
1
|
+
import{BUG_CAUSES as i,BUG_CONFIDENCES as u,BUG_ESCAPES as a,BUG_IMPACTS as s,BUG_SEVERITIES as o}from"./categories.js";import{BUG_TITLE_MAX_LENGTH as r}from"./draft.js";import{SEVERITY_LABELS as h}from"./issue.js";export const BUG_RECORD_EXAMPLE={title:"A refund pays out twice when two requests overlap",severity:"high",expected:"A refund pays out once.",authority:'`docs/refunds.md`, "Refunds are idempotent on the order id".',observed:"Two overlapping refund requests for one order both pay out.",rootCause:"`refund()` reads the order's refund state and writes the payout without a lock.",test:{file:"src/checkout/refund.test.ts",title:"a refund pays out once"},fault:{path:"src/checkout/refunds.ts",lines:[41,58]},cause:"race-condition",impact:"financial-error",confidence:{cause:"high",impact:"high"},escape:"no-test"};const c=e=>e.map(t=>`\`${t}\``).join(", "),d=e=>`One of ${c(u)}. How sure you are of the ${e}.`,n=e=>({meaning:`One id from the ${e.field} list below.`,name:e.field});export const BUG_GUIDE={fields:[{name:"title",meaning:`One line of at most ${String(r)} characters that names the broken behaviour. It becomes the issue title.`},{name:"severity",meaning:`One of ${c(o)}. The severities are defined below.`},{name:"expected",meaning:"What the product should do."},{name:"authority",meaning:"The source that says so: documentation, a specification, an API contract, or the code's own stated intent. Name it and quote the part that applies."},{name:"observed",meaning:"What the product does instead."},{name:"rootCause",meaning:"Why the code behaves this way."},{name:"test.file",meaning:"The Testkit test file that fails because of the bug."},{name:"test.title",meaning:`The failure's title. It is one line of at most ${String(r)} characters.`},{name:"fault.path",meaning:"The source file that holds the mistake."},{name:"fault.lines",meaning:"The first and last faulty line as `[start, end]`, counted from 1. One faulty line is `[41, 41]`."},n(i),n(s),{name:"confidence.cause",meaning:d("cause")},{name:"confidence.impact",meaning:d("impact")},n(a)],rules:["Write `test.file` and `fault.path` relative to the product directory, the directory that holds `testkit.config.ts`.","`test.title` must be the failure's title exactly as Testkit reports it. In a Vitest or Playwright file, that is the names of the `describe` blocks around the test and the test's own name, joined by single spaces. When `testkit bug file` rejects the title, its message lists the titles a bug record can use.","Keep credentials, tokens, and passwords out of the record. Testkit redacts the secrets it recognises before it stores the record and again before it files the issue. It files no bug under a failure's title that holds text it redacts, so rename such a test or label such an assertion first."],example:BUG_RECORD_EXAMPLE,severities:o.map(e=>({definition:h[e].description,id:e})),categoryLists:[i,s,a]};export function renderBugGuide(e){return["## Bug record","Write one JSON object for each confirmed bug. Every field is required.",e.fields.map(t=>`- \`${t.name}\`: ${t.meaning}`).join(`
|
|
2
2
|
`),...e.rules,"Example:",["```json",JSON.stringify(e.example,null,2),"```"].join(`
|
|
3
3
|
`),"## Severities",e.severities.map(t=>`- \`${t.id}\`: ${t.definition}`).join(`
|
|
4
4
|
`),...e.categoryLists.flatMap(l)].join(`
|
package/dist/app/bugs/record.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import
|
|
2
|
-
Run \`testkit bug guide\` to see the record format.`,{cause:
|
|
1
|
+
import c from"node:fs";import h from"node:path";import{text as b}from"node:stream/consumers";import{loadTestkitConfig as D}from"../../config/loader/index.js";import{discoverOwnedTests as k}from"../../config/loader/owned-discovery.js";import{isTypedTestFile as T}from"../../domain/identifiers.js";import{resolveVitestSelection as E}from"../../engines/catalog.js";import{posixPath as P}from"../../process/files/paths.js";import{findActiveCampaign as S,readCampaigns as F}from"../campaigns/store.js";import{BugRecordError as $,createBugDraft as x,namesSameTest as C,readBugReport as R,requireOneVitestProject as j,toProductPath as B}from"./draft.js";import{bugDraftFile as O,displayPath as y,displayUnreadableDrafts as J,readStoredDrafts as N,resolveBugCheckout as A,writeJsonFile as I}from"./store.js";export async function recordBug(o){const t=await A(o),{redactedFields:i,report:e}=U(await q(t.cwd,o.file),t.productPath);for(const[r,g]of[["test.file",e.test.file],["fault.path",e.fault.path]])if(c.statSync(h.join(t.repoRoot,g),{throwIfNoEntry:!1})?.isFile()!==!0)throw new Error(`${r} resolves to ${g} from the repository root, and no file is there. Write the path relative to the product directory.`);const{config:a}=await D(t.productDir),f=E(o.vitest??{},t.productDir),l=B(t.productPath,e.test.file),n=await k(t.productDir,a,{plain:!T(l),selection:f}),m=n.problems.at(0)??n.migration;if(m!==null)throw new Error(m);if(!n.project.files.some(r=>r.filePath===l))throw new Error(`test.file resolves to ${e.test.file}, and Testkit does not discover a test there. Run \`testkit discover\` to list the tests it runs.`);j(e.test.file,n.plainFiles.filter(({file:r})=>r===l),f);const d=x(e,{campaign:S(F(t.productDir))?.id??null,productPath:t.productPath,recordedAt:o.at??new Date,recordedCommit:t.headCommit,vitest:V(o.vitest??{},t.productDir,f)}),u=N(t.productDir),p=u.drafts.filter(r=>C(r.draft,d)),w=p.find(r=>r.draft.filed!==void 0)?.draft.filed;if(w!==void 0)throw new Error(`This test already has a filed bug: ${w.url}. Testkit keeps one bug for each test.`);const s=O(t.productDir,d.id),v=u.unreadable.find(r=>r.file===s);if(v!==void 0)throw new Error(`${y(t,s)} exists and Testkit cannot read it: ${v.problem}. Fix or delete the file, then run this again.`);I(s,d);for(const r of p)r.file!==s&&c.rmSync(r.file,{force:!0});return{draft:d,path:y(t,s),redactedFields:i,replaced:p.length>0,unreadableDrafts:J(t,u.unreadable)}}function V({"vitest-project":o,"vitest-runner":t},i,{config:e}){const a={...t===void 0?{}:{"vitest-runner":t},...e===null?{}:{"vitest-config":P(h.relative(c.realpathSync(i),e))},...o===void 0?{}:{"vitest-project":o}};return Object.keys(a).length===0?void 0:a}async function q(o,t){if(t==="-")return{source:"standard input",text:await b(process.stdin)};try{return{source:t,text:c.readFileSync(h.resolve(o,t),"utf8")}}catch(i){throw new Error(`Testkit cannot read the bug record file ${t}.`,{cause:i})}}function U(o,t){let i;try{i=JSON.parse(o.text)}catch(e){throw new Error(`The bug record in ${o.source} is not valid JSON.`,{cause:e})}try{return R(i,t)}catch(e){throw e instanceof $?new Error(`${e.message}
|
|
2
|
+
Run \`testkit bug guide\` to see the record format.`,{cause:e}):e}}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import
|
|
2
|
-
`),
|
|
1
|
+
import A from"node:fs";import c from"node:path";import{readAreaMap as x,readBugPolicy as I}from"../../config/loader/bugs.js";import{PLAIN_TEST_SUFFIX as P}from"../../config/loader/discovery.js";import{loadTestkitConfig as w}from"../../config/loader/index.js";import{listFilesChangedSince as j}from"../../repo/history.js";import{listCommitFiles as T}from"../../repo/tree-listing.js";import{measureAreaMap as E}from"../bugs/areas.js";import{displayPath as D,displayUnreadableDrafts as y,readStoredDrafts as S,resolveBugCheckout as u,writeJsonFile as F,writeTextFile as R}from"../bugs/store.js";import{renderCampaignBrief as B}from"./brief.js";import{defaultCampaignId as $,readCampaignId as L,readCampaignTarget as M}from"./campaign.js";import{campaignDir as k,findActiveCampaign as f,findLastFinishedCampaign as N,readCampaigns as p}from"./store.js";import{renderCampaignSummary as O}from"./summary.js";export async function startCampaign(a){const t=a.at??new Date,r=M(a.target),i=a.id===void 0?$(r,t):L(a.id),e=await u(a),n=f(p(e.productDir));if(n!==null)throw new Error(`Campaign ${n.id} is active in this checkout. Run \`testkit campaign finish\` before you start another.`);const s=k(e.productDir,i);if(A.existsSync(s))throw new Error(`${D(e,s)} already exists. Pass --id to give this campaign another id.`);const m=await v(e),d={schemaVersion:1,id:i,target:r,startedAt:t.toISOString(),startCommit:e.headCommit};return F(c.join(s,"campaign.json"),d),{areas:m,brief:B(d,m,e.productPath),campaign:d}}export async function readCampaignStatus(a){const t=await u(a),r=p(t.productDir),i=f(r),e=S(t.productDir),n=await v(t);return{active:i,areas:{areaCount:n.areas.length,doubleClaimCount:n.doubleClaims.length,unclaimedShare:n.unclaimedShare},drafts:i===null?[]:b(e.drafts,i.id),lastFinished:N(r),unreadableDrafts:y(t,e.unreadable)}}export async function finishCampaign(a){const t=await u(a),r=f(p(t.productDir));if(r===null)throw new Error("No campaign is active in this checkout. Run `testkit campaign start --target <text>` to start one.");const i={...r,finishedAt:(a.at??new Date).toISOString()},e=S(t.productDir),n=b(e.drafts,i.id),s=n.filter(o=>o.filed!==void 0),m=n.filter(o=>o.filed===void 0),{config:d}=await w(t.productDir),l=await j(t.repoRoot,i.startCommit),g=O({campaign:i,filed:s,headCommit:t.headCommit,knownFailures:I(d).knownFailures,testFiles:l===null?null:l.filter(o=>o.endsWith(".testkit.ts")||P.test(o)),unfiled:m,unreadableDrafts:y(t,e.unreadable).map(o=>o.file)}),h=k(t.productDir,i.id),C=c.join(h,"summary.md");return R(C,`${g}
|
|
2
|
+
`),F(c.join(h,"campaign.json"),i),{campaign:i,filed:s,headCommit:t.headCommit,summary:g,summaryPath:D(t,C),unfiled:m}}function b(a,t){return a.map(r=>r.draft).filter(r=>r.campaign===t)}async function v(a){const{config:t}=await w(a.productDir);return E(x(t),await T(a.repoRoot,a.headCommit))}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{filterConfigsByService as
|
|
1
|
+
import{filterConfigsByService as s}from"../../config/loader/managed-configs.js";import{loadConfigContext as n}from"../../config/loader/index.js";import{createEnvironmentMaintenance as i}from"../../environment/maintenance.js";import{hasLocalDatabaseState as d}from"../../database/lifecycle-cleanup.js";export async function destroy(t,e,r={}){await i(t).maintain({kind:"destroy",configs:e,destroyOwnedResources:r.destroyOwnedResources})}export async function destroyEnvironment(t){const e=await n({dir:t.dir,allowUnmigratedServices:!0,plain:!1,retired:[]}),r=s(e.services,t.service||null),o=e.services.some(a=>a.testkit.database!==void 0||a.testkit.databaseFrom!==void 0);return await destroy(e.productDir,r,{destroyOwnedResources:!t.service&&(o||d(e.productDir))}),{ok:!0,productDir:e.productDir,results:r.map(a=>({name:a.name,destroyed:!0}))}}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import{resolveRequestedFiles as
|
|
2
|
-
`),
|
|
1
|
+
import{resolveRequestedFiles as u}from"../file-selection.js";import{resolveVitestSelection as f}from"../../engines/catalog.js";import n from"fs";import t from"path";import{loadDiscoveryContext as p,resolveProductDir as v}from"../../config/loader/index.js";export async function executeDiscoverOperation({output:o,vitest:a,...i},l=process.cwd()){const e=v(l,i.dir),m=u(i.file||[],e,l),{discovery:s}=await p({...i,dir:e,file:m,vitest:f(a??{},e)});let c=null;if(o){const r=t.resolve(e,o);n.mkdirSync(t.dirname(r),{recursive:!0}),n.writeFileSync(r,`${JSON.stringify(s,null,2)}
|
|
2
|
+
`),c=(t.relative(e,r)||t.basename(r)).replaceAll("\\","/")}return{...s,outputLabel:c}}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{environmentTaskIdentities as p,renderSharedEnvironment as
|
|
1
|
+
import{environmentTaskIdentities as p,PREVIEW_LEASE as h,renderSharedEnvironment as w}from"../run/environment.js";import{withEnvFileValues as g}from"../../config/loader/env.js";import{sensitiveKeyMatcher as y}from"../../kernel/redaction-targets.js";import{renderEnvironment as E}from"./config-driven.js";import{prepareFileSetup as v}from"./file-setup.js";import{resetBeforeRefusal as S}from"./task-databases.js";const I=new Set(["bin","lease","leaseDir","pathList","productDir","service","stateDir"]);export function createStaticValueEnvironments(r,c,a=null){const s=new Map,d=e=>{const t=s.get(e.name);if(t!==void 0)return t;const n=g(e);return T(n),s.set(e.name,n),n},l=(e,t,n,o)=>{const i=r.get(e);if(i===void 0)throw new Error(`No services are declared for group "${e}"`);const m={graphKey:t,leaseId:n.id,runtimeId:t,targetNames:[e],services:new Map(i.map(u=>[u.name,{baseUrl:null,database:null,config:d(u)}]))};return E(m,{name:e,uses:i.map(({name:u})=>u)},n,o,y(a?.redactKeys))};return{canAcquire:()=>!0,getGraph:()=>null,getGraphFailure:()=>null,markGraphFailed:()=>{},preview(e){try{return l(e,"",h,()=>{})}catch{return null}},acquire(e,t){const n=a?.acquisition({source:"static",graphKey:e.graphKey,leaseId:t.id,tasks:p(e)});return Promise.resolve().then(()=>{const o=w(e,i=>l(i,e.graphKey,t,m=>a?.addRedactionTarget(m)));return n?.acquired({runtimeId:null,services:[]}),n?.rendered(o),{environment:o,prepare:async(i,m)=>{if(i.runtimeIsolation==="resetBefore")throw S(i,!1);return v(i,m,{environment:o,lease:t,productDir:c,record:n,redact:u=>a?.addRedactionTarget(u)})},release:()=>(n?.released(),Promise.resolve())}}).catch(o=>{throw n?.failed(o),o})}}}function T(r){const c=[...Object.entries(r.testkit.serviceEnv).map(([a,s])=>[`env.${a}`,s]),["ui",r.testkit.ui]];for(const[a,s]of c)for(const d of f(s))if(!I.has(d))throw new Error(`Service "${r.name}" uses {${d}} in ${a}, which needs a running service, and "${r.name}" starts none`)}function f(r){return typeof r=="string"?[...r.matchAll(/\{([a-zA-Z]+)(?::[^}]+)?\}/gu)].map(c=>c[1]):Array.isArray(r)?r.flatMap(f):r!==null&&typeof r=="object"?Object.values(r).flatMap(f):[]}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{createEnvironment as y,environmentTaskIdentities as
|
|
1
|
+
import{createEnvironment as y,environmentTaskIdentities as O,renderSharedEnvironment as R}from"../run/environment.js";import{httpCaptureRedactionTargets as T}from"../run/setup/runtime-node-setup-protocol.js";import{checkPostgresConnection as E,parsePostgresUrl as M}from"../../database/postgres/connection-check.js";import{errorMessage as l}from"../../kernel/error-serialization.js";import{envRedactionTargets as g,jsonRedactionTargets as f,sensitiveKeyMatcher as j,urlRedactionTargets as q}from"../../kernel/redaction-targets.js";import{parseHttpUrl as D,requireHttpResponse as I}from"../../process/readiness.js";import{prepareFileSetup as P}from"./file-setup.js";import{resetBeforeRefusal as S}from"./task-databases.js";const m=3e4;export function createSuppliedEnvironments(e,{productDir:t,record:a=null,signal:o}){const i=j(a?.redactKeys);return{canAcquire:()=>!0,getGraph:()=>null,getGraphFailure:()=>null,markGraphFailed:()=>{},preview(r){try{return suppliedEnvironment(r,Object.hasOwn(e,r)?e[r]:{})}catch{return null}},async acquire(r,s){const u=Object.hasOwn(e,r.targetName),c=a?.acquisition({source:u?"supplied":"none",graphKey:r.graphKey,leaseId:s.id,tasks:O(r)}),b=u?e[r.targetName]:{};for(const n of $(b,i))a?.addRedactionTarget(n);let d;try{d=R(r,n=>suppliedEnvironment(n,Object.hasOwn(e,n)?e[n]:{})),await C(r.targetName,d,o)}catch(n){throw o?.aborted&&n===o.reason?c?.cancelled(n):c?.failed(n),n}return c?.acquired({runtimeId:null,services:[]}),c?.rendered(d),{environment:d,prepare:async(n,w)=>{if(n.runtimeIsolation==="resetBefore")throw S(n,u);return u?null:P(n,w,{environment:d,lease:s,productDir:t,record:c,redact:U=>a?.addRedactionTarget(U)})},release:()=>(c?.released(),Promise.resolve())}}}}function $(e,t){return[...g(e.env??{},t),...[e.baseUrl,e.databaseUrl,...Object.values(e.databases??{}),...Object.values(e.endpoints??{})].flatMap(a=>a?q(a):[]),...f("handles",e.handles??{},t),...Object.values(e.handles??{}).filter(a=>a.kind==="http-capture").flatMap(T),...f("setupData",e.setupData??{},t),...f("ui",e.ui??{},t)]}async function C(e,t,a){const o=h(t.endpoints),i=h(t.databases);for(const[r,s]of o)await p(e,"endpoint",s,()=>D(r),a);for(const[r,s]of i)await p(e,"database",s,()=>{_(r)},a);for(const[r,s]of o)await p(e,"endpoint",s,()=>I(r,{signal:a,timeoutMs:m}),a);for(const[r,s]of i)await p(e,"database",s,()=>E(r,{signal:a,timeoutMs:m}),a)}function h(e){const t=new Map;for(const[a,o]of Object.entries(e))t.has(o)||t.set(o,a);return t}function _(e){try{M(e)}catch(t){throw new Error(`its URL ${l(t)}`,{cause:t})}}async function p(e,t,a,o,i){try{await o()}catch(r){throw i?.aborted?i.reason:new Error(`Supplied ${t} "${a}" for group "${e}" is not usable: ${l(r)}`,{cause:r})}}export function suppliedEnvironment(e,t){return{...y({baseUrl:t.baseUrl??null,databaseUrl:t.databaseUrl||null,databases:t.databases??{},endpoints:t.endpoints??{},env:t.env??{},group:e,...t.setupData===void 0?{}:{setupData:t.setupData},ui:t.ui??null}),...t.handles===void 0?{}:{handles:{...t.handles}}}}
|