@gfdelarue/testkit 0.4.3 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +109 -22
- package/bin/testkit-windows-owner-arm64.exe +0 -0
- package/bin/testkit-windows-owner-x64.exe +0 -0
- package/dist/app/bugs/categories.js +1 -1
- package/dist/app/bugs/draft.js +3 -3
- package/dist/app/bugs/facts.d.ts +3 -2
- package/dist/app/bugs/facts.js +2 -2
- package/dist/app/bugs/guide.js +1 -1
- package/dist/app/bugs/run-reading.js +1 -1
- package/dist/app/db-schema/prepare.js +1 -1
- package/dist/app/destroy/index.js +1 -1
- package/dist/app/doctor/index.js +1 -1
- package/dist/app/environments/config-driven.d.ts +5 -0
- package/dist/app/environments/config-driven.js +1 -0
- package/dist/app/environments/routing.d.ts +6 -0
- package/dist/app/environments/routing.js +1 -0
- package/dist/app/environments/supplied.d.ts +3 -0
- package/dist/app/environments/supplied.js +1 -0
- package/dist/app/file-selection.js +1 -1
- package/dist/app/lint/native.js +1 -1
- package/dist/app/lint/options.d.ts +0 -1
- package/dist/app/lint/options.js +1 -1
- package/dist/app/lint/shared.d.ts +0 -2
- package/dist/app/lint/shared.js +1 -1
- package/dist/app/package-inspection.js +1 -1
- package/dist/app/reports/files.d.ts +13 -0
- package/dist/app/reports/files.js +1 -0
- package/dist/app/reports/prepare.d.ts +20 -0
- package/dist/app/reports/prepare.js +2 -0
- package/dist/app/reports/publish.d.ts +21 -0
- package/dist/app/reports/publish.js +1 -0
- package/dist/app/run/engine-evidence-persistence.js +1 -1
- package/dist/app/run/engine-execution-cleanup.js +1 -1
- package/dist/app/run/engine-execution-failures.js +1 -1
- package/dist/app/run/engine-execution.d.ts +10 -8
- package/dist/app/run/engine-execution.js +1 -1
- package/dist/app/run/engine-orchestration.d.ts +15 -34
- package/dist/app/run/engine-orchestration.js +1 -1
- package/dist/app/run/environment.d.ts +33 -0
- package/dist/app/run/environment.js +1 -0
- package/dist/app/run/failure-details.js +1 -1
- package/dist/app/run/live-run.d.ts +3 -1
- package/dist/app/run/live-run.js +1 -1
- package/dist/app/run/orchestrator.d.ts +8 -1
- package/dist/app/run/orchestrator.js +1 -1
- package/dist/app/run/planning.d.ts +1 -2
- package/dist/app/run/planning.js +1 -1
- package/dist/app/run/results.d.ts +1 -1
- package/dist/app/run/results.js +1 -1
- package/dist/app/run/run-finalization.d.ts +6 -2
- package/dist/app/run/run-finalization.js +1 -1
- package/dist/app/run/runtime-fixtures.js +1 -1
- package/dist/app/run/runtime-task-types.d.ts +1 -1
- package/dist/app/run/scheduler/index.js +1 -1
- package/dist/app/run/scheduler/locks.d.ts +6 -0
- package/dist/app/run/scheduler/locks.js +1 -0
- package/dist/app/run/selection.d.ts +0 -1
- package/dist/app/run/selection.js +1 -1
- package/dist/app/run/setup/runtime-node-setup-protocol.js +1 -1
- package/dist/app/run/setup/runtime-node-setup-worker.js +1 -1
- package/dist/app/run/task-leases.d.ts +15 -0
- package/dist/app/run/task-leases.js +1 -0
- package/dist/app/run/worker-loop.d.ts +13 -4
- package/dist/app/run/worker-loop.js +1 -1
- package/dist/app/tracker/index.d.ts +6 -1
- package/dist/app/tracker/index.js +1 -1
- package/dist/app/typecheck/index.js +4 -4
- package/dist/cli/command-catalog.d.ts +19 -0
- package/dist/cli/command-catalog.js +1 -1
- package/dist/cli/command-registry.js +1 -1
- package/dist/cli/commands/report/prepare.d.ts +12 -0
- package/dist/cli/commands/report/prepare.js +5 -0
- package/dist/cli/commands/report/publish.d.ts +13 -0
- package/dist/cli/commands/report/publish.js +2 -0
- package/dist/cli/json-errors.js +1 -1
- package/dist/cli/operations/run/operation.d.ts +3 -1
- package/dist/cli/operations/run/operation.js +2 -2
- package/dist/cli/renderers/doctor/text.js +1 -1
- package/dist/cli/renderers/run/formatting.js +1 -1
- package/dist/config/authoring/index.d.ts +4 -13
- package/dist/config/authoring/index.js +1 -1
- package/dist/config/authoring/preset-env.js +1 -1
- package/dist/config/loader/config-loader.js +1 -1
- package/dist/config/loader/database.js +1 -1
- package/dist/config/loader/decode-execution.d.ts +0 -2
- package/dist/config/loader/decode-execution.js +1 -1
- package/dist/config/loader/discover-tests.js +1 -1
- package/dist/config/loader/discovery.d.ts +0 -1
- package/dist/config/loader/discovery.js +2 -2
- package/dist/config/loader/env.js +1 -1
- package/dist/config/loader/execution-model.d.ts +0 -1
- package/dist/config/loader/execution-model.js +1 -1
- package/dist/config/loader/file-metadata.d.ts +1 -1
- package/dist/config/loader/file-metadata.js +1 -1
- package/dist/config/loader/groups.d.ts +33 -0
- package/dist/config/loader/groups.js +1 -0
- package/dist/config/loader/load.d.ts +0 -2
- package/dist/config/loader/load.js +1 -1
- package/dist/config/loader/runtime-isolation.js +1 -1
- package/dist/config/loader/runtime.d.ts +0 -9
- package/dist/config/loader/runtime.js +1 -1
- package/dist/config/loader/skip-config.js +1 -1
- package/dist/config/loader/validation.d.ts +1 -2
- package/dist/config/loader/validation.js +1 -1
- package/dist/config/loader/values.d.ts +0 -1
- package/dist/config/loader/values.js +1 -1
- package/dist/database/activity-gate.js +1 -1
- package/dist/database/admin.js +2 -2
- package/dist/database/lifecycle-cleanup.d.ts +1 -0
- package/dist/database/lifecycle-cleanup.js +1 -1
- package/dist/database/local-postgres.js +1 -1
- package/dist/database/naming.js +1 -1
- package/dist/database/schema-operations.d.ts +1 -1
- package/dist/database/schema-operations.js +1 -1
- package/dist/database/schema-source.js +3 -3
- package/dist/database/source-url.d.ts +0 -1
- package/dist/database/source-url.js +2 -2
- package/dist/database/state.js +1 -1
- package/dist/database/steps.js +3 -3
- package/dist/domain/file-timeout.d.ts +0 -3
- package/dist/domain/file-timeout.js +1 -1
- package/dist/domain/operation-display.js +1 -1
- package/dist/domain/test-outcomes.d.ts +0 -1
- package/dist/domain/test-outcomes.js +1 -1
- package/dist/engines/adapters/evidence-identity.js +1 -1
- package/dist/engines/adapters/execution.d.ts +0 -1
- package/dist/engines/adapters/execution.js +1 -1
- package/dist/engines/api/capabilities.d.ts +0 -10
- package/dist/engines/api/capabilities.js +1 -1
- package/dist/engines/api/execution-scope.js +1 -1
- package/dist/engines/api/k6-runtime-input.js +1 -1
- package/dist/engines/bundle-cache.js +1 -1
- package/dist/engines/contracts/fixture-manifest.js +1 -1
- package/dist/engines/definitions.d.ts +1 -0
- package/dist/engines/definitions.js +1 -1
- package/dist/engines/installations/playwright-resolution-child.js +1 -1
- package/dist/engines/installations/playwright.js +1 -1
- package/dist/engines/k6/adapter/evidence.js +4 -4
- package/dist/engines/k6/adapter/executor.js +1 -1
- package/dist/engines/k6/adapter/native-summary.js +1 -1
- package/dist/engines/k6/runtime/checks.js +1 -1
- package/dist/engines/k6/runtime/dal-fixtures.js +1 -1
- package/dist/engines/k6/runtime/dal-suite.js +1 -1
- package/dist/engines/k6/runtime/http-suite-runtime.js +1 -1
- package/dist/engines/k6/runtime/http.js +1 -1
- package/dist/engines/k6/runtime/shared/db-normalization.js +1 -1
- package/dist/engines/k6/runtime/shared/fixture-engine.js +2 -2
- package/dist/engines/k6/runtime/shared/http-parsing.js +3 -3
- package/dist/engines/managed-module-resolution-policy-input.d.ts +0 -1
- package/dist/engines/managed-module-resolution-policy-input.js +1 -1
- package/dist/engines/managed-module-resolution-policy.js +1 -1
- package/dist/engines/node-vitest/adapter/adapter.js +1 -1
- package/dist/engines/node-vitest/adapter/evidence.js +2 -2
- package/dist/engines/node-vitest/adapter/workspace.js +22 -21
- package/dist/engines/node-vitest/runtime/contract/index.js +1 -1
- package/dist/engines/node-vitest/runtime/data/http.js +1 -1
- package/dist/engines/node-vitest/runtime/data/index.d.ts +2 -2
- package/dist/engines/node-vitest/runtime/data/index.js +1 -1
- package/dist/engines/node-vitest/runtime/data/report.js +5 -5
- package/dist/engines/node-vitest/runtime/data/scenario.d.ts +0 -1
- package/dist/engines/node-vitest/runtime/data/scenario.js +2 -2
- package/dist/engines/node-vitest/runtime/data/types.d.ts +1 -29
- package/dist/engines/node-vitest/runtime/runtime-input.js +1 -1
- package/dist/engines/playwright/adapter/adapter.js +1 -1
- package/dist/engines/playwright/adapter/config.d.ts +1 -1
- package/dist/engines/playwright/adapter/config.js +4 -4
- package/dist/engines/playwright/adapter/evidence-validation.d.ts +0 -3
- package/dist/engines/playwright/adapter/evidence-validation.js +1 -1
- package/dist/engines/playwright/adapter/evidence.js +1 -1
- package/dist/engines/playwright/runtime/config-file.js +1 -1
- package/dist/engines/playwright/runtime/config.js +1 -1
- package/dist/engines/playwright/runtime/runtime-input.d.ts +2 -2
- package/dist/engines/playwright/runtime/runtime-input.js +16 -12
- package/dist/environment/lifecycle-artifacts.js +1 -1
- package/dist/environment/lifecycle-controller.d.ts +1 -11
- package/dist/environment/lifecycle-controller.js +1 -1
- package/dist/environment/lifecycle.d.ts +14 -11
- package/dist/environment/lifecycle.js +1 -1
- package/dist/environment/planning-types.d.ts +0 -2
- package/dist/environment/ports.d.ts +10 -0
- package/dist/environment/ports.js +1 -0
- package/dist/environment/ready-environment.d.ts +2 -5
- package/dist/environment/ready-environment.js +1 -1
- package/dist/environment/run-environment.d.ts +1 -0
- package/dist/environment/run-environment.js +1 -1
- package/dist/environment/runtime-contexts.d.ts +11 -7
- package/dist/environment/runtime-contexts.js +1 -1
- package/dist/environment/runtime-manager.d.ts +1 -9
- package/dist/environment/runtime-manager.js +1 -1
- package/dist/environment/runtime-planning.js +1 -1
- package/dist/environment/runtime-preparation.js +1 -1
- package/dist/environment/runtime-service.d.ts +0 -6
- package/dist/environment/services.d.ts +11 -3
- package/dist/environment/services.js +3 -1
- package/dist/environment/state.d.ts +1 -4
- package/dist/environment/state.js +1 -1
- package/dist/environment/template.d.ts +6 -22
- package/dist/environment/template.js +1 -1
- package/dist/evidence/artifact-config.js +1 -1
- package/dist/evidence/artifact-validation.d.ts +1 -2
- package/dist/evidence/artifact-validation.js +1 -1
- package/dist/evidence/artifacts.js +2 -2
- package/dist/evidence/bundles.js +1 -1
- package/dist/evidence/environment-record.d.ts +22 -0
- package/dist/evidence/environment-record.js +1 -0
- package/dist/evidence/index.d.ts +1 -1
- package/dist/evidence/logs.js +2 -2
- package/dist/evidence/operation-timings.js +1 -1
- package/dist/evidence/read-fields.d.ts +0 -1
- package/dist/evidence/read-fields.js +1 -1
- package/dist/evidence/read-run.js +1 -1
- package/dist/evidence/reading.js +1 -1
- package/dist/evidence/reporting-types.d.ts +19 -0
- package/dist/evidence/reporting.d.ts +2 -2
- package/dist/evidence/reporting.js +1 -1
- package/dist/evidence/storage.d.ts +0 -2
- package/dist/evidence/storage.js +2 -2
- package/dist/evidence/timing.js +1 -1
- package/dist/kernel/error-serialization.d.ts +2 -0
- package/dist/kernel/error-serialization.js +1 -1
- package/dist/kernel/unknown-values.d.ts +5 -0
- package/dist/kernel/unknown-values.js +1 -1
- package/dist/process/environment.js +1 -1
- package/dist/process/execa-bundle.json +3 -3
- package/dist/process/files/json-file.d.ts +0 -1
- package/dist/process/files/json-file.js +1 -1
- package/dist/process/files/paths.d.ts +2 -0
- package/dist/process/files/paths.js +1 -1
- package/dist/process/host/metadata.js +1 -1
- package/dist/process/host/platform.js +1 -1
- package/dist/process/host-process.js +1 -1
- package/dist/process/lifecycle-fence.d.ts +1 -2
- package/dist/process/lifecycle-fence.js +1 -1
- package/dist/process/locks.js +2 -2
- package/dist/process/process-group.js +1 -1
- package/dist/process/process-identity.js +1 -1
- package/dist/process/process.js +30 -30
- package/dist/process/readiness.d.ts +7 -1
- package/dist/process/readiness.js +1 -1
- package/dist/process/recovery.js +1 -1
- package/dist/process/runs/manifest.js +1 -1
- package/dist/process/runs/ownership.js +1 -1
- package/dist/process/runs/product-state.js +1 -1
- package/dist/process/runs/stop.js +1 -1
- package/dist/process/stoppable-settlement.d.ts +26 -0
- package/dist/process/stoppable-settlement.js +1 -0
- package/dist/process/url-redaction.d.ts +1 -0
- package/dist/process/url-redaction.js +1 -0
- package/dist/process/windows-native.d.ts +6 -0
- package/dist/process/windows-native.js +1 -1
- package/dist/repo/ci-environment.js +1 -1
- package/dist/repo/fingerprint-policy.d.ts +0 -2
- package/dist/repo/fingerprint-policy.js +1 -1
- package/dist/repo/physical-tree.js +1 -1
- package/dist/repo/publish-directory.d.ts +20 -0
- package/dist/repo/publish-directory.js +3 -0
- package/dist/repo/state.js +4 -4
- package/dist/steps/build-config.d.ts +0 -1
- package/dist/steps/build-config.js +1 -1
- package/dist/steps/configured-steps.js +1 -1
- package/dist/steps/environment.d.ts +0 -2
- package/dist/steps/environment.js +1 -1
- package/dist/steps/index.d.ts +1 -1
- package/dist/steps/run.js +2 -2
- package/dist/toolchains/archive.js +1 -1
- package/dist/toolchains/index.js +3 -3
- package/dist/toolchains/node-provisioning.js +2 -2
- package/dist/toolchains/vitest/index.js +1 -1
- package/docs/bug-records.md +54 -52
- package/docs/engine-architecture.md +39 -11
- package/package.json +1 -1
- package/testkit-config-schema.md +43 -5
- package/dist/app/run/playwright-configuration.d.ts +0 -2
- package/dist/app/run/playwright-configuration.js +0 -1
- package/dist/environment/readiness.d.ts +0 -12
- package/dist/environment/readiness.js +0 -1
package/README.md
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
<p align="center">
|
|
2
|
-
<img src="https://
|
|
2
|
+
<img src="https://turnstone.cloud/assets/social-card.png" alt="AI Testkit logo" width="600">
|
|
3
3
|
</p>
|
|
4
4
|
|
|
5
5
|
# @gfdelarue/testkit
|
|
@@ -174,6 +174,73 @@ claims, how to work, and the bug record format. Give that output to the agent.
|
|
|
174
174
|
summary for the pull request that adds the tests to
|
|
175
175
|
`.testkit/campaigns/<id>/summary.md` and prints it.
|
|
176
176
|
|
|
177
|
+
## Prepare and publish a report bundle
|
|
178
|
+
|
|
179
|
+
Prepare a new directory from your own template, recorded bug IDs, and selected
|
|
180
|
+
run artifacts. All inputs are optional. Repeat `--bug` for related findings and
|
|
181
|
+
`--artifact` for supporting files.
|
|
182
|
+
|
|
183
|
+
```bash
|
|
184
|
+
npx testkit artifacts --run <run-id> --json
|
|
185
|
+
npx testkit report prepare ./draft-report \
|
|
186
|
+
--template ./report-template.md \
|
|
187
|
+
--instructions ./reporting-instructions.md \
|
|
188
|
+
--bug <bug-id> --run <run-id> --artifact <ref.path>
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
Preparation copies the template unchanged to `report.md`, or creates an empty
|
|
192
|
+
file. It puts recorded facts in `context.json` and selected, verified run files
|
|
193
|
+
under `evidence/`. Drafting instructions are printed for the author and are not
|
|
194
|
+
copied into the bundle. Write and review the narrative yourself. Templates have
|
|
195
|
+
no special syntax, and preparation never overwrites an existing directory.
|
|
196
|
+
|
|
197
|
+
Publish that directory, or any directory you prepared yourself, to an explicit
|
|
198
|
+
repository, branch, and bundle path:
|
|
199
|
+
|
|
200
|
+
```bash
|
|
201
|
+
npx testkit report publish ./draft-report \
|
|
202
|
+
--repository https://github.com/your-team/private-reports \
|
|
203
|
+
--branch main --path reports/refund-race --json
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
Use a full Gitea repository URL to publish there. `owner/name` uses the host of
|
|
207
|
+
the checkout's origin. Authentication uses the same environment variables and
|
|
208
|
+
Git credential helper as `testkit bug file`; the token needs repository write
|
|
209
|
+
access. Both commands support `--dir` to select the Testkit checkout and
|
|
210
|
+
`--json` for structured output. Local input paths are relative to your current
|
|
211
|
+
directory. Publication needs Git installed and works without Testkit config.
|
|
212
|
+
|
|
213
|
+
Publication replaces the chosen bundle directory, including removing files
|
|
214
|
+
that are no longer in your local bundle. It preserves other repository paths
|
|
215
|
+
and leaves your checkout, index, and branch untouched. The destination branch
|
|
216
|
+
must exist unless the repository is empty. A concurrent branch update rejects
|
|
217
|
+
the push; review the new branch state before trying again. Fetching requests
|
|
218
|
+
a blob-free snapshot and monitors temporary Git storage against a 256 MiB
|
|
219
|
+
budget, with possible brief overshoot between checks. Use a smaller reports
|
|
220
|
+
repository if it exceeds that budget. An unchanged bundle
|
|
221
|
+
returns its existing revision without creating a commit.
|
|
222
|
+
|
|
223
|
+
The result includes the exact `revision` and a provider `url` pinned to it.
|
|
224
|
+
`--message` sets the commit message. Commits use the standard `GIT_AUTHOR_*` and
|
|
225
|
+
`GIT_COMMITTER_*` identity variables when set, otherwise `Testkit
|
|
226
|
+
<testkit@localhost>`. Keep the revision used for a submission. A failed push can
|
|
227
|
+
have an uncertain result after a lost connection, so check the reported commit
|
|
228
|
+
at the destination before retrying.
|
|
229
|
+
|
|
230
|
+
Nested text and binary files keep their bytes and names, without line-ending
|
|
231
|
+
conversion or Git LFS filters. Publication accepts ordinary files up to 16 MiB
|
|
232
|
+
each, 64 MiB total, and 1000 files. Directory depth is limited to 32, with at
|
|
233
|
+
most 128 directories including the repository root and destination parents.
|
|
234
|
+
Symlinks, Git metadata, traversal paths, and
|
|
235
|
+
filenames containing backslashes or line breaks are rejected. Filenames must
|
|
236
|
+
be UTF-8. Empty directories are not stored by Git. Review everything in the
|
|
237
|
+
selected directory before publishing it, including hidden files.
|
|
238
|
+
|
|
239
|
+
Neither command needs Turnstone, a paid feature, or a fixed report format.
|
|
240
|
+
To view the result in Turnstone, connect the destination repository and select
|
|
241
|
+
the same branch and the parent folder, such as `reports`. The author submits
|
|
242
|
+
the report to the chosen platform.
|
|
243
|
+
|
|
177
244
|
## Execution engines
|
|
178
245
|
|
|
179
246
|
Testkit has three explicit built-in engines rather than one implicit runtime:
|
|
@@ -428,7 +495,6 @@ export default defineConfig({
|
|
|
428
495
|
api: app.node({
|
|
429
496
|
cwd: ".",
|
|
430
497
|
entry: "src/index.ts",
|
|
431
|
-
port: 3004,
|
|
432
498
|
envFiles: [".env.testkit"],
|
|
433
499
|
database: database.postgres({
|
|
434
500
|
runtimeConnections: 20,
|
|
@@ -448,7 +514,6 @@ export default defineConfig({
|
|
|
448
514
|
frontend: app.next({
|
|
449
515
|
cwd: "frontend",
|
|
450
516
|
mode: "start",
|
|
451
|
-
port: 3000,
|
|
452
517
|
dependsOn: ["api"],
|
|
453
518
|
envFiles: ["frontend/.env.testkit"],
|
|
454
519
|
env: {
|
|
@@ -466,6 +531,16 @@ export default defineConfig({
|
|
|
466
531
|
});
|
|
467
532
|
```
|
|
468
533
|
|
|
534
|
+
Every service Testkit starts listens on a port the OS assigns for that run.
|
|
535
|
+
Testkit passes it to the service as `PORT` and as `{port}` in `start`, `env`,
|
|
536
|
+
`baseUrl`, and `readyUrl`, so two runs of the same repository on one host, such
|
|
537
|
+
as two CI jobs on a runner or two agent worktrees, never compete for a port. A
|
|
538
|
+
service that exits before it becomes ready gets a fresh port and up to three
|
|
539
|
+
attempts in total. Each run's `run.json` lists every service's actual URL, port,
|
|
540
|
+
and attempts under `environment.services`. A config that sets `local.port`, the
|
|
541
|
+
`port` option of `app.node` or `app.next`, or a literal port in `local.baseUrl`
|
|
542
|
+
or `local.readyUrl` fails to load; delete it and use `{port}`.
|
|
543
|
+
|
|
469
544
|
Every test file declares its execution engine and any file-local scheduling
|
|
470
545
|
metadata next to the test:
|
|
471
546
|
|
|
@@ -487,7 +562,10 @@ from Testkit's managed `vitest`, `@playwright/test`, `playwright`, and
|
|
|
487
562
|
`playwright-core` packages.
|
|
488
563
|
There are no legacy aliases or implicit-runner compatibility paths.
|
|
489
564
|
|
|
490
|
-
`testkit.config.ts` is optional
|
|
565
|
+
`testkit.config.ts` is optional. Without it, Testkit runs every test file with
|
|
566
|
+
zero setup: it starts no server, provisions no database, and loads no `.env`
|
|
567
|
+
file. A test that needs a server or a database gets one from a service that
|
|
568
|
+
`testkit.config.ts` declares. The config is also the primary escape hatch
|
|
491
569
|
for:
|
|
492
570
|
|
|
493
571
|
- worker count and per-file runtime budget
|
|
@@ -666,7 +744,6 @@ toolchains: {
|
|
|
666
744
|
services: {
|
|
667
745
|
frontend: app.next({
|
|
668
746
|
cwd: "frontend",
|
|
669
|
-
port: 3000,
|
|
670
747
|
runtime: {
|
|
671
748
|
toolchain: "frontendNode",
|
|
672
749
|
},
|
|
@@ -884,10 +961,11 @@ export const setup = suite.setup;
|
|
|
884
961
|
export default suite.exec;
|
|
885
962
|
```
|
|
886
963
|
|
|
887
|
-
DAL files
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
964
|
+
DAL files run at the same time as other tests, including other DAL files of the
|
|
965
|
+
same service. If two files share rows that one of them changes, give both the
|
|
966
|
+
same lock, for example `defineFile({ engine: "k6", locks: ["widgets"] })`, and
|
|
967
|
+
Testkit runs them one at a time. Runtime services reserve a configurable
|
|
968
|
+
Postgres connection budget for each active runtime slot. Use `database.postgres({ runtimeConnections })` for the
|
|
891
969
|
service pool size and `database.postgres({ maxConnections,
|
|
892
970
|
reservedConnections })` for the backing Postgres capacity.
|
|
893
971
|
|
|
@@ -1131,8 +1209,11 @@ platform's host address.
|
|
|
1131
1209
|
- template fingerprints are derived automatically from env files, source schema
|
|
1132
1210
|
cache, migrate/seed config, and repo contents
|
|
1133
1211
|
|
|
1134
|
-
The service process receives its provisioned database as `DATABASE_URL`.
|
|
1135
|
-
|
|
1212
|
+
The service process receives its provisioned database as `DATABASE_URL`. Every
|
|
1213
|
+
task gets the same `DATABASE_URL` when its service declares a database, and its
|
|
1214
|
+
service's URL as `BASE_URL` when the service declares a server, whatever its
|
|
1215
|
+
engine. The provisioned `DATABASE_URL` replaces one that the service's `env`
|
|
1216
|
+
declares. Declared `env` values can also use the `{dbUrl}`, `{dbHost}`,
|
|
1136
1217
|
`{dbPort}`, `{dbName}`, `{dbUser}`, `{dbPassword}`, and `{dbSsl}` tokens, with
|
|
1137
1218
|
`:serviceName` to read the database of a service listed in `dependsOn`. Testkit
|
|
1138
1219
|
resolves them from the database provisioned for the current run. They are
|
|
@@ -1167,19 +1248,25 @@ skip.
|
|
|
1167
1248
|
|
|
1168
1249
|
Repository commands use npm's native timing support, enabled project-wide by
|
|
1169
1250
|
the tracked `.npmrc`. Build, lint, and release modes support `help`; release
|
|
1170
|
-
provides `bump` and `
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1251
|
+
provides `bump`, `check`, `consumer-lock`, and `plan` modes.
|
|
1252
|
+
A push to `master` publishes only when it raises `version` in `package.json`
|
|
1253
|
+
and the new version is above npm's latest. To release, run
|
|
1254
|
+
`npm run release -- bump <version>` and merge the pull request once its checks
|
|
1255
|
+
pass. The
|
|
1256
|
+
[publish workflow](https://github.com/gfdelarue/testkit/blob/master/docs/release-gates.md)
|
|
1257
|
+
publishes that exact version through npm trusted publishing and tags the
|
|
1258
|
+
merged commit. Any other merge publishes nothing.
|
|
1175
1259
|
|
|
1176
1260
|
The static audit retains package/lock/runtime policy, engine dependency/import
|
|
1177
|
-
intent, retired engine/product references, minimal repository traversal,
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
import
|
|
1181
|
-
|
|
1182
|
-
|
|
1261
|
+
intent, retired engine/product references, minimal repository traversal, the
|
|
1262
|
+
dependency direction between top-level `src/` modules, and the test execution
|
|
1263
|
+
boundary. An import outside the direction table in
|
|
1264
|
+
`scripts/lint/audits/import-direction.ts` fails lint. The table may allow
|
|
1265
|
+
directions that no import uses. Test execution code may import only test
|
|
1266
|
+
execution and shared kernel code, apart from a committed list of existing
|
|
1267
|
+
imports that only shrinks. The
|
|
1268
|
+
[dependency direction notes](https://github.com/gfdelarue/testkit/blob/master/docs/dependency-direction.md)
|
|
1269
|
+
describe both rules. Other implementation-shape audits stay out of lint.
|
|
1183
1270
|
|
|
1184
1271
|
JavaScript modules under `test/fixtures/**` describe fixture behavior and keep
|
|
1185
1272
|
their fixture-owned paths. Outside fixtures, only `bin/testkit.mjs` and the
|
|
Binary file
|
|
Binary file
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export const BUG_FACTS_VERSION=1;function e(i,t,o=""){return{boundaryRule:o,definition:t,id:i}}export const BUG_CAUSES={field:"cause",question:"What kind of mistake is in the code?",assignmentRule:"A bug gets exactly one cause. When a bug holds several mistakes, classify the earliest one in the causal chain. Use `other` when none fits.",categories:[e("missing-validation","The code never checks an input or a state precondition that the domain rules require: range, format, existence, uniqueness, cross-field consistency, or lifecycle state. Includes a validator that exists and accepts invalid input.","Earliest mistake wins. If application code never tested the condition, it is `missing-validation` even when a database constraint or the runtime later throws and the caller sees
|
|
1
|
+
export const BUG_FACTS_VERSION=1;function e(i,t,o=""){return{boundaryRule:o,definition:t,id:i}}export const BUG_CAUSES={field:"cause",question:"What kind of mistake is in the code?",assignmentRule:"A bug gets exactly one cause. When a bug holds several mistakes, classify the earliest one in the causal chain. Use `other` when none fits.",categories:[e("missing-validation","The code never checks an input or a state precondition that the domain rules require: range, format, existence, uniqueness, cross-field consistency, or lifecycle state. Includes a validator that exists and accepts invalid input.","Earliest mistake wins. If application code never tested the condition, it is `missing-validation` even when a database constraint or the runtime later throws and the caller sees an unexpected error. If the unchecked value is null, absent, empty, zero, or a default, use `null-or-default-mishandling`."),e("mishandled-error","A failure was detected and signalled, and the handling is wrong: not caught, translated to the wrong error, swallowed, ignored by the UI, or followed by a wrong retry or recovery.","The failure must already be signalled by something that checked. If nothing in the application checked and a lower layer failed, it is `missing-validation`."),e("race-condition","Correctness depends on how overlapping operations interleave: check-then-act without a lock or constraint, a lost update, or a late asynchronous completion that overwrites newer state.","Needs two overlapping operations to reproduce. If one sequential flow reproduces it, it is `stale-state` or `missing-validation`."),e("unbounded-or-inefficient-work","Results are correct and the cost is unbounded or needlessly repeated: unpaginated reads, N+1 queries, a missing or redundant index, serial waits, or loading everything and then slicing.","Correct results at excessive cost. If the result is wrong, it is `incorrect-query-or-selection`."),e("incorrect-query-or-selection","The code identifies the wrong set of records: a missing or wrong predicate, unstable ordering, a truncated page treated as complete, duplicate rows from a join, or a lookup key that omits a distinguishing part.","A missing predicate is `missing-access-control` only when it separates actors or tenants with different rights. The right records with wrong arithmetic on them is `incorrect-logic`."),e("missing-access-control","A permission, tenant, ownership, or separation-of-duties check is absent or wrong.","Use it when the missing check is about who the actor is. A missing check about what the data or state is belongs in `missing-validation`."),e("null-or-default-mishandling","Null, absent, empty, zero, or default values are conflated or not handled: null treated as omitted, a zero or an empty collection treated as missing, a null dereference, a wrong fallback default, or a caller that omits an argument and silently gets the default.","Wins over `missing-validation` and `incorrect-logic` whenever the triggering value is null, absent, empty, zero, or a default."),e("stale-state","Held state, such as a cache, local UI state, a derived or denormalised value, or leftover fields, is not refreshed, reset, persisted, or tied to the right identity when the thing it mirrors changes.","If the omission leaves old values visible where current ones should be, it is `stale-state`. If the missing action leaves nothing stale, it is `omitted-step`. If it reproduces only with overlapping operations, it is `race-condition`."),e("interface-mismatch","Data is lost or misread where it crosses between representations or components. One component's mapping drops, overwrites, hardcodes, or mis-assigns fields, or two components that must agree disagree on shape, type, field names, allowed values, semantics, or which copy of the data is authoritative.","Use it when the values were right before the crossing. Wrong computation on values that arrived intact is `incorrect-logic`. A held copy that goes out of date after its source changes is `stale-state`. An omitted argument that falls back to a default is `null-or-default-mishandling`."),e("omitted-step","A required follow-on action is not performed on some path: an audit record, a notification, a dispatch, a finalisation, or an alternate route that bypasses the shared routine.","If the missing thing is a check, it is `missing-validation`. If its absence leaves old values on display, it is `stale-state`."),e("incorrect-logic","The algorithm, arithmetic, date or time handling, rounding, boundary, comparison, boolean condition, parsing, or order of operations is wrong.","The default for wrong logic on valid, present values. If the trigger is a null, empty, zero, or default value, use `null-or-default-mishandling`. If the fault is which records were chosen, use `incorrect-query-or-selection`."),e("other","No category above describes the mistake.")]},BUG_IMPACTS={field:"impact",question:"What is at stake for the business?",assignmentRule:"A bug gets exactly one impact. The list is in precedence order, and the first type that applies wins. Classify the impact the test demonstrated, not a downstream one you expect. Use `other` when none fits.",categories:[e("unauthorized-access","Someone can see or change something they are not entitled to, including getting past a separation-of-duties rule.","The actor lacks the right. If an entitled actor skips a mandated control or leaves no evidence, it is `audit-or-compliance-gap`."),e("financial-error","A monetary amount, rate, currency, limit, payment or settlement state, or a money-committing decision is wrong where it is stored, charged, paid, or decided.","A money figure that is only displayed wrongly, with the stored value and the decision intact, is `wrong-information-shown`."),e("audit-or-compliance-gap","Required audit evidence or attribution is missing, duplicated, or false, or a mandated control did not run.","Use it when the harm is to evidence or to a regulatory or policy control. A skipped ordinary process step is `workflow-breakdown`."),e("data-loss-or-corruption","Stored data, or work the user entered, is destroyed, overwritten, duplicated, or left invalid or inconsistent.","If the action silently did nothing and prior stored state is intact, it is `blocked-operation`."),e("workflow-breakdown","A business process, such as a workflow, task, schedule, notification, or gate, is skipped, stalled, duplicated, or advanced wrongly.","Use it when the process goes wrong after the triggering call returns. A single user action that fails in front of the user is `blocked-operation`."),e("availability-or-performance","The product is slow, unavailable, or exhausts resources in a way that affects more than the triggering call or grows with data volume.","One call that fails for one caller is `blocked-operation`."),e("blocked-operation","A legitimate action fails, returns an unusable or misleading error, or silently does not apply, with prior stored state intact.","The user was trying to do something and could not. If they were only reading and what they read was wrong, it is `wrong-information-shown`."),e("wrong-information-shown","Displayed or returned information is wrong, out of date, incomplete, or duplicated, with nothing stored wrongly.","Last in precedence. If the wrong value is also persisted, it is `data-loss-or-corruption`."),e("other","No type above describes what is at stake.")]},BUG_ESCAPES={field:"escape",question:"Why did the tests that existed before yours not catch the bug?",assignmentRule:"A bug gets exactly one reason. Judge the tests that were there before yours. Use `other` when none fits.",categories:[e("no-test","No existing test runs the faulty behaviour.","If any existing test reaches the faulty code path, use another reason."),e("untested-input","A test runs the behaviour and never with the input, state, or data volume that triggers the fault.","One sequential flow with the right input reproduces the bug. If it needs overlapping operations, it is `untested-interleaving`."),e("untested-interleaving","A test runs the behaviour in sequence, and the fault needs overlapping operations or a particular timing.","The triggering input is already covered, and only the overlap is missing."),e("wrong-assertion","A test runs the triggering case against the real code and passes, because its assertion is missing, too weak, or asserts the broken behaviour.","The faulty code ran in the test. If a stand-in ran in its place, it is `test-double`."),e("test-double","A test runs the triggering case against a test double or a fixture that hides the fault in the real component.","The real component would have failed the test."),e("other","No reason above applies.")]},BUG_SEVERITIES=["critical","high","medium","low"],BUG_CONFIDENCES=["high","low"],BUG_CATCHERS=["campaign","manual"];export function categoryIds(i){return i.categories.map(t=>t.id)}export function isOneOf(i,t){return typeof t=="string"&&i.some(o=>o===t)}
|
package/dist/app/bugs/draft.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
import G from"node:crypto";import T from"node:path";import{isPublicTestType as P}from"../../domain/identifiers.js";import{isUnknownRecord as A}from"../../kernel/unknown-values.js";import{BUG_CATCHERS as O,BUG_CAUSES as R,BUG_CONFIDENCES as B,BUG_ESCAPES as U,BUG_IMPACTS as D,BUG_SEVERITIES as F,categoryIds as C,isOneOf as E}from"./categories.js";import{decodeFaultLines as z,isCommitId as $}from"./facts.js";import{redactBugText as S}from"../../tracker/redaction.js";export const BUG_DRAFT_SCHEMA_VERSION=1,BUG_TITLE_MAX_LENGTH=200,BUG_SECTION_MAX_LENGTH=12e3;const L=48,M=/^[a-z0-9]+(?:-[a-z0-9]+)*$/u;export const CAMPAIGN_ID_MAX_LENGTH=60;export class BugRecordError extends Error{problems;constructor(e,n){super(`${e}:
|
|
2
|
-
${n.map(
|
|
3
|
-
`)}`),this.name="BugRecordError",this.problems=n}}export function isCampaignId(t){return typeof t=="string"&&t.length<=CAMPAIGN_ID_MAX_LENGTH&&M.test(t)}export function testTypeOfFile(t){const e=/\.([a-z0-9]+)\.testkit\.ts$/u.exec(t)?.[1];return P(e)?e:null}export function toRepositoryPath(t,e){const n=e.replaceAll("\\","/");if(n.startsWith("/")||/^[a-z]:/iu.test(n))return null;const
|
|
4
|
-
${t.test.title}`).digest("hex");return`${e||"bug"}-${n.slice(0,8)}`}export function isBugDraftId(t){return typeof t=="string"&&/^[a-z0-9]+(?:-[a-z0-9]+)*-[0-9a-f]{8}$/u.test(t)}export function namesSameTest(t,e){return t.test.file===e.test.file&&t.test.title===e.test.title}function I(t,e,n){const
|
|
2
|
+
${n.map(s=>`- ${s}`).join(`
|
|
3
|
+
`)}`),this.name="BugRecordError",this.problems=n}}export function isCampaignId(t){return typeof t=="string"&&t.length<=CAMPAIGN_ID_MAX_LENGTH&&M.test(t)}export function testTypeOfFile(t){const e=/\.([a-z0-9]+)\.testkit\.ts$/u.exec(t)?.[1];return P(e)?e:null}export function toRepositoryPath(t,e){const n=e.replaceAll("\\","/");if(n.startsWith("/")||/^[a-z]:/iu.test(n))return null;const s=T.posix.normalize(T.posix.join(t,n));return s==="."||s===".."||s.startsWith("../")?null:s}export function toProductPath(t,e){return t===""?e:T.posix.relative(t,e)}export function readBugReport(t,e){if(!A(t))throw new BugRecordError("The bug record is invalid",["it must be a JSON object"]);const n=[],s=[],p=I(t,n,{path:i=>toRepositoryPath(e,i),text:(i,f)=>{const u=S(i);return u!==i&&s.push(f),u}});if(n.length>0)throw new BugRecordError("The bug record is invalid",n);return{redactedFields:s,report:p}}export function createBugDraft(t,e){const n=testTypeOfFile(t.test.file);if(n===null)throw new BugRecordError("The bug record is invalid",[w("test.file")]);return{schemaVersion:BUG_DRAFT_SCHEMA_VERSION,id:bugDraftId(t),recordedAt:e.recordedAt.toISOString(),recordedCommit:e.recordedCommit,productPath:e.productPath,caughtBy:e.campaign===null?"manual":"campaign",...e.campaign===null?{}:{campaign:e.campaign},testType:n,...t}}export function readBugDraft(t){const e="The bug draft is unreadable";if(!A(t))throw new BugRecordError(e,["it must be a JSON object"]);if(t.schemaVersion!==BUG_DRAFT_SCHEMA_VERSION)throw new BugRecordError(e,[`unknown schemaVersion ${JSON.stringify(t.schemaVersion)}`]);const n=[],s=(a,b,y,l)=>b(a)?a:(n.push(l),y),p=I(t,n,{path:a=>toRepositoryPath("",a),text:a=>a}),i=testTypeOfFile(p.test.file)??"int";t.testType!==i&&n.push("testType does not match test.file");const f=s(t.caughtBy,V,"manual","caughtBy must be campaign or manual"),u=s(t.campaign,H,void 0,"campaign must be a campaign id");f==="campaign"!=(u!==void 0)&&n.push("campaign must be present when caughtBy is campaign and absent otherwise");const x={schemaVersion:BUG_DRAFT_SCHEMA_VERSION,id:s(t.id,isBugDraftId,"","id is not a draft id"),recordedAt:s(t.recordedAt,N,"","recordedAt must be a timestamp"),recordedCommit:s(t.recordedCommit,$,"","recordedCommit must be a full commit id"),productPath:s(t.productPath,k,"","productPath must be a string"),caughtBy:f,...u===void 0?{}:{campaign:u},testType:i,...p,...t.filed===void 0?{}:{filed:j(t.filed,n)}};if(n.length>0)throw new BugRecordError(e,n);return x}export function bugDraftId(t){const e=t.title.normalize("NFKD").replace(/\p{M}+/gu,"").toLowerCase().replace(/[^a-z0-9]+/gu,"-").slice(0,L).replace(/^-+|-+$/gu,""),n=G.createHash("sha256").update(`${t.test.file}
|
|
4
|
+
${t.test.title}`).digest("hex");return`${e||"bug"}-${n.slice(0,8)}`}export function isBugDraftId(t){return typeof t=="string"&&/^[a-z0-9]+(?:-[a-z0-9]+)*-[0-9a-f]{8}$/u.test(t)}export function namesSameTest(t,e){return t.test.file===e.test.file&&t.test.title===e.test.title}function I(t,e,n){const s=(r,o,c,d)=>{const m=r[o];if(typeof m!="string"||m.trim().length===0)return e.push(`${c} must be a non-empty string`),"";const h=m.trim(),g=h.length>d?h:n.text(h,c);return g.length>d&&e.push(`${c} must be at most ${String(d)} characters`),g},p=(r,o,c)=>{const d=r[o],m=typeof d=="string"?d.trim():"",h=m.length>0?n.path(m):null;return h===null?e.push(`${c} must be a path inside the repository`):[m,h].some(g=>S(g)!==g)&&e.push(`${c} is a path that holds text Testkit redacts as a secret`),h??""},i=(r,o,c)=>E(r,o)?o:(e.push(`${c} must be one of ${r.join(", ")}`),r[0]),f=r=>{const o=t[r];return A(o)?o:(e.push(`${r} must be an object`),{})},u=(r,o,c)=>{const d=s(r,o,c,BUG_TITLE_MAX_LENGTH);return/[\r\n]/u.test(d)&&e.push(`${c} must be one line`),d},x=u(t,"title","title"),a=f("test"),b=f("fault"),y=f("confidence"),l=p(a,"file","test.file");l!==""&&testTypeOfFile(l)===null&&e.push(w("test.file"));let _=[1,1];try{_=z(b.lines)}catch(r){e.push(r instanceof Error?r.message:"fault.lines is invalid")}return{title:x,severity:i(F,t.severity,"severity"),expected:s(t,"expected","expected",BUG_SECTION_MAX_LENGTH),authority:s(t,"authority","authority",BUG_SECTION_MAX_LENGTH),observed:s(t,"observed","observed",BUG_SECTION_MAX_LENGTH),rootCause:s(t,"rootCause","rootCause",BUG_SECTION_MAX_LENGTH),test:{file:l,title:u(a,"title","test.title")},fault:{lines:_,path:p(b,"path","fault.path")},cause:i(C(R),t.cause,"cause"),impact:i(C(D),t.impact,"impact"),confidence:{cause:i(B,y.cause,"confidence.cause"),impact:i(B,y.impact,"confidence.impact")},escape:i(C(U),t.escape,"escape")}}function j(t,e){return A(t)&&N(t.filedAt)&&$(t.foundAt)&&typeof t.number=="number"&&Number.isSafeInteger(t.number)&&t.number>=1&&typeof t.url=="string"?{filedAt:t.filedAt,foundAt:t.foundAt,number:t.number,url:t.url}:(e.push("filed must hold filedAt, foundAt, number, and url"),{filedAt:"",foundAt:"",number:0,url:""})}function H(t){return t===void 0||isCampaignId(t)}function V(t){return E(O,t)}function k(t){return typeof t=="string"}function N(t){return typeof t=="string"&&!Number.isNaN(Date.parse(t))}function w(t){return`${t} must be a Testkit test file, such as refund.int.testkit.ts`}
|
package/dist/app/bugs/facts.d.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import { type PublicTestType } from "../../domain/identifiers.js";
|
|
2
1
|
import { BUG_FACTS_VERSION, type BugCatcher, type BugCause, type BugConfidence, type BugEscape, type BugImpact } from "./categories.js";
|
|
3
2
|
export declare const FACTS_START_MARKER = "<!-- testkit-facts:start -->";
|
|
4
3
|
export declare const FACTS_END_MARKER = "<!-- testkit-facts:end -->";
|
|
4
|
+
export declare const BUG_FACTS_TEST_TYPES: readonly ["ui", "e2e", "int", "contract", "data", "dal", "load"];
|
|
5
|
+
export type BugFactsTestType = (typeof BUG_FACTS_TEST_TYPES)[number];
|
|
5
6
|
export interface BugFactsArea {
|
|
6
7
|
readonly id: string;
|
|
7
8
|
readonly name: string;
|
|
@@ -22,7 +23,7 @@ export interface BugFacts {
|
|
|
22
23
|
readonly escape?: BugEscape;
|
|
23
24
|
readonly caughtBy: BugCatcher;
|
|
24
25
|
readonly campaign?: string;
|
|
25
|
-
readonly testType
|
|
26
|
+
readonly testType?: BugFactsTestType;
|
|
26
27
|
readonly test: {
|
|
27
28
|
readonly file: string;
|
|
28
29
|
readonly title?: string;
|
package/dist/app/bugs/facts.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import{isAreaId as
|
|
2
|
-
`)}export function readFactsBlock(e){const t=e.lastIndexOf(FACTS_START_MARKER);if(t===-1)return{status:"missing"};const
|
|
1
|
+
import{isAreaId as w}from"../../config/loader/bugs.js";import{isUnknownRecord as m}from"../../kernel/unknown-values.js";import{BUG_CATCHERS as E,BUG_CAUSES as S,BUG_CONFIDENCES as p,BUG_ESCAPES as b,BUG_FACTS_VERSION as h,BUG_IMPACTS as T,categoryIds as s,isOneOf as A}from"./categories.js";export const FACTS_START_MARKER="<!-- testkit-facts:start -->",FACTS_END_MARKER="<!-- testkit-facts:end -->";const B=/^(?:[0-9a-f]{40}|[0-9a-f]{64})$/u;export const BUG_FACTS_TEST_TYPES=Object.freeze(["ui","e2e","int","contract","data","dal","load"]);export function renderFactsBlock(e){return["<details>","<summary>Testkit facts</summary>","",FACTS_START_MARKER,"","```json",JSON.stringify(l(e),null,2),"```","",FACTS_END_MARKER,"","</details>"].join(`
|
|
2
|
+
`)}export function readFactsBlock(e){const t=e.lastIndexOf(FACTS_START_MARKER);if(t===-1)return{status:"missing"};const r=t+FACTS_START_MARKER.length,n=e.indexOf(FACTS_END_MARKER,r);if(n===-1)return{status:"invalid",reason:"the facts block has no end marker"};const d=/```[^\n]*\r?\n([\s\S]*?)\r?\n[ \t]*```/u.exec(e.slice(r,n));if(d===null)return{status:"invalid",reason:"the facts block has no fenced code block"};let u;try{u=JSON.parse(d[1])}catch{return{status:"invalid",reason:"the facts block is not valid JSON"}}try{return{facts:_(u),status:"ok"}}catch(f){return{status:"invalid",reason:f instanceof Error?f.message:"the facts block is invalid"}}}export function isCommitId(e){return typeof e=="string"&&B.test(e)}function _(e){if(!m(e))throw new Error("the facts block is not a JSON object");if(e.v!==h)throw new Error(`unknown facts version ${JSON.stringify(e.v)}`);const t=c(e.confidence,"confidence"),r=c(e.test,"test");return l({v:h,cause:o(s(S),e.cause,"cause"),impact:o(s(T),e.impact,"impact"),confidence:{cause:o(p,t.cause,"confidence.cause"),impact:o(p,t.impact,"confidence.impact")},escape:i(e.escape,n=>o(s(b),n,"escape")),caughtBy:o(E,e.caughtBy,"caughtBy"),campaign:i(e.campaign,n=>a(n,"campaign")),testType:i(e.testType,n=>o(BUG_FACTS_TEST_TYPES,n,"testType")),test:{file:a(r.file,"test.file"),title:i(r.title,n=>a(n,"test.title"))},foundAt:y(e.foundAt,"foundAt"),fault:i(e.fault,k),area:i(e.area,O),introducedBy:i(e.introducedBy,n=>y(n,"introducedBy")),introducedMethod:i(e.introducedMethod,n=>o(["blame"],n,"introducedMethod"))})}function k(e){const t=c(e,"fault");return{lines:i(t.lines,decodeFaultLines),path:a(t.path,"fault.path")}}export function decodeFaultLines(e){if(!Array.isArray(e)||e.length!==2)throw new Error("fault.lines must be [start, end]");const t=e,[r,n]=t;if(!g(r)||!g(n)||r>n)throw new Error("fault.lines must be two line numbers counted from 1, with start no greater than end");return[r,n]}function O(e){const t=c(e,"area");if(!w(t.id))throw new Error("area.id is not an area id");const r=t.share;if(typeof r!="number"||!Number.isFinite(r)||r<0||r>1)throw new Error("area.share must be a number from 0 to 1");return{id:t.id,name:a(t.name,"area.name"),share:r}}function l(e){return{v:e.v,cause:e.cause,impact:e.impact,confidence:{cause:e.confidence.cause,impact:e.confidence.impact},...e.escape===void 0?{}:{escape:e.escape},caughtBy:e.caughtBy,...e.campaign===void 0?{}:{campaign:e.campaign},...e.testType===void 0?{}:{testType:e.testType},test:{file:e.test.file,...e.test.title===void 0?{}:{title:e.test.title}},foundAt:e.foundAt,...e.fault===void 0?{}:{fault:{path:e.fault.path,...e.fault.lines===void 0?{}:{lines:e.fault.lines}}},...e.area===void 0?{}:{area:{id:e.area.id,name:e.area.name,share:e.area.share}},...e.introducedBy===void 0?{}:{introducedBy:e.introducedBy},...e.introducedMethod===void 0?{}:{introducedMethod:e.introducedMethod}}}function i(e,t){return e===void 0?void 0:t(e)}function c(e,t){if(!m(e))throw new Error(`${t} must be an object`);return e}function a(e,t){if(typeof e!="string"||e.trim().length===0)throw new Error(`${t} must be a non-empty string`);return e}function o(e,t,r){if(!A(e,t))throw new Error(`${r} must be one of ${e.join(", ")}`);return t}function y(e,t){if(!isCommitId(e))throw new Error(`${t} must be a full lowercase commit id`);return e}function g(e){return typeof e=="number"&&Number.isSafeInteger(e)&&e>=1}
|
package/dist/app/bugs/guide.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import{BUG_CAUSES as s,BUG_CONFIDENCES as c,BUG_ESCAPES as a,BUG_IMPACTS as n,BUG_SEVERITIES as o}from"./categories.js";import{BUG_TITLE_MAX_LENGTH as r}from"./draft.js";import{SEVERITY_LABELS as d}from"./issue.js";export const BUG_RECORD_EXAMPLE={title:"A refund pays out twice when two requests overlap",severity:"high",expected:"A refund pays out once.",authority:'`docs/refunds.md`, "Refunds are idempotent on the order id".',observed:"Two overlapping refund requests for one order both pay out.",rootCause:"`refund()` reads the order's refund state and writes the payout without a lock.",test:{file:"src/checkout/__testkit__/refund.
|
|
1
|
+
import{BUG_CAUSES as s,BUG_CONFIDENCES as c,BUG_ESCAPES as a,BUG_IMPACTS as n,BUG_SEVERITIES as o}from"./categories.js";import{BUG_TITLE_MAX_LENGTH as r}from"./draft.js";import{SEVERITY_LABELS as d}from"./issue.js";export const BUG_RECORD_EXAMPLE={title:"A refund pays out twice when two requests overlap",severity:"high",expected:"A refund pays out once.",authority:'`docs/refunds.md`, "Refunds are idempotent on the order id".',observed:"Two overlapping refund requests for one order both pay out.",rootCause:"`refund()` reads the order's refund state and writes the payout without a lock.",test:{file:"src/checkout/__testkit__/refund.int.testkit.ts",title:"a refund pays out once"},fault:{path:"src/checkout/refunds.ts",lines:[41,58]},cause:"race-condition",impact:"financial-error",confidence:{cause:"high",impact:"high"},escape:"no-test"};const h=e=>e.map(t=>`\`${t}\``).join(", "),l=e=>`One of ${h(c)}. How sure you are of the ${e}.`,i=e=>({meaning:`One id from the ${e.field} list below.`,name:e.field});export const BUG_GUIDE={fields:[{name:"title",meaning:`One line of at most ${String(r)} characters that names the broken behaviour. It becomes the issue title.`},{name:"severity",meaning:`One of ${h(o)}. The severities are defined below.`},{name:"expected",meaning:"What the product should do."},{name:"authority",meaning:"The source that says so: documentation, a specification, an API contract, or the code's own stated intent. Name it and quote the part that applies."},{name:"observed",meaning:"What the product does instead."},{name:"rootCause",meaning:"Why the code behaves this way."},{name:"test.file",meaning:"The Testkit test file that fails because of the bug."},{name:"test.title",meaning:`The failure's title, or what the script tests when a k6 script fails as a whole. It is one line of at most ${String(r)} characters.`},{name:"fault.path",meaning:"The source file that holds the mistake."},{name:"fault.lines",meaning:"The first and last faulty line as `[start, end]`, counted from 1. One faulty line is `[41, 41]`."},i(s),i(n),{name:"confidence.cause",meaning:l("cause")},{name:"confidence.impact",meaning:l("impact")},i(a)],rules:["Write `test.file` and `fault.path` relative to the product directory, the directory that holds `testkit.config.ts`.","`test.title` must be the failure's title exactly as Testkit reports it. In a Vitest or Playwright file, that is the names of the `describe` blocks around the test and the test's own name, joined by single spaces. In a k6 file, that is the label of the failed assertion. A failed assertion on a response that has no label is named by its request and its default title, such as `GET /health > status is 201`. When `testkit bug file` rejects the title, its message lists the titles a bug record can use.","A k6 script that fails as a whole reports a script failure, which has no title. It fails that way when it calls `fail()`, throws while it runs, fails a k6 `check()`, or fails a threshold. When the script reports no named failure, write in `test.title` what the script tests. Testkit then files one issue that covers the whole file. While that issue is open, every named failure in the file and its script failure are known.","Keep credentials, tokens, and passwords out of the record. Testkit redacts the secrets it recognises before it stores the record and again before it files the issue. It files no bug under a failure's title that holds text it redacts, so rename such a test or label such an assertion first."],example:BUG_RECORD_EXAMPLE,severities:o.map(e=>({definition:d[e].description,id:e})),categoryLists:[s,n,a]};export function renderBugGuide(e){return["## Bug record","Write one JSON object for each confirmed bug. Every field is required.",e.fields.map(t=>`- \`${t.name}\`: ${t.meaning}`).join(`
|
|
2
2
|
`),...e.rules,"Example:",["```json",JSON.stringify(e.example,null,2),"```"].join(`
|
|
3
3
|
`),"## Severities",e.severities.map(t=>`- \`${t.id}\`: ${t.definition}`).join(`
|
|
4
4
|
`),...e.categoryLists.flatMap(u)].join(`
|
|
@@ -1 +1 @@
|
|
|
1
|
-
const h=new Set(["engine-failure","playwright-other-file-spec","runtime-exception","vitest-error"]),k=1,c=99,E=0;export function readRun(e,i){if(e.finalizationFailure!==void 0)return o(e.finalizationFailure.message??"Testkit could not finish the run");if(e.run.status==="interrupted")return o("the run was interrupted");const{cleanup:t,publication:u}=e.archive??{};if(u?.status==="failed"&&u.required===!0)return o("Testkit could not publish the run's results");if(t?.status==="failed")return o("Testkit could not clean up after the run");const a=e.services.flatMap(n=>n.errors??[]).at(0);if(a!==void 0)return o(a);const r=e.services.flatMap(n=>n.suites.flatMap(p=>p.files??[]));if(r.some(n=>n.infraFailed===!0||n.blockedByRuntime===!0))return o("test infrastructure failure");if(r.some(n=>n.status==="not_run"))return o("one or more required test files did not run");if(r.some(n=>n.status==="skipped"))return o("one or more selected test files were skipped");const l=r.find(n=>n.status!=="passed"&&n.status!=="failed");if(l!==void 0)return o(`${l.path} ended as ${l.status}`);const s=r.filter(n=>n.status==="failed");return s.length>0?{failures:s.flatMap(n=>
|
|
1
|
+
import{isUnknownRecord as f}from"../../kernel/unknown-values.js";const h=new Set(["engine-failure","playwright-other-file-spec","runtime-exception","vitest-error"]),k=1,c=99,E=0;export function readRun(e,i){if(e.finalizationFailure!==void 0)return o(e.finalizationFailure.message??"Testkit could not finish the run");if(e.run.status==="interrupted")return o("the run was interrupted");const{cleanup:t,publication:u}=e.archive??{};if(u?.status==="failed"&&u.required===!0)return o("Testkit could not publish the run's results");if(t?.status==="failed")return o("Testkit could not clean up after the run");const a=e.services.flatMap(n=>n.errors??[]).at(0);if(a!==void 0)return o(a);const r=e.services.flatMap(n=>n.suites.flatMap(p=>p.files??[]));if(r.some(n=>n.infraFailed===!0||n.blockedByRuntime===!0))return o("test infrastructure failure");if(r.some(n=>n.status==="not_run"))return o("one or more required test files did not run");if(r.some(n=>n.status==="skipped"))return o("one or more selected test files were skipped");const l=r.find(n=>n.status!=="passed"&&n.status!=="failed");if(l!==void 0)return o(`${l.path} ended as ${l.status}`);const s=r.filter(n=>n.status==="failed");return s.length>0?{failures:s.flatMap(n=>_(n,i)),kind:"tests-failed"}:e.run.status!=="passed"?o(r.length===0?"no runnable test files executed":"the run did not pass"):{kind:"passed",passedFiles:r.length}}function _(e,i){const t=i===""?e.path:`${i}/${e.path}`,u=e.failureDetails??[],a=u.filter(s=>!h.has(s.kind??"")),r=[...new Set(a.map(y))].map(s=>({file:t,title:s})),l=u.filter(s=>s.kind==="engine-failure"||s.kind==="vitest-error").map(s=>s.kind==="vitest-error"?"unexplained":T(s,a));return l.includes("unexplained")||r.length===0&&!l.includes("script")?[...r,{file:t,title:null}]:l.includes("script")?[...r,{file:t,script:!0,title:null}]:r}function y(e){const i=e.key?.trim()??"";return i===""?e.title:e.kind==="playwright-spec"||e.labelled===!1?i:e.title}function T(e,i){const t=f(e.actual)&&f(e.actual.details)?e.actual.details:null;if(t===null)return"unexplained";if(e.key==="K6_RUNTIME_EXCEPTION")return t.phase==="exec"&&t.summaryRead===!0&&t.exitSignal===void 0&&(t.exitCode===E||t.exitCode===c)?"script":"unexplained";if(e.key==="PLAYWRIGHT_SPECS_FAILED"){const u=d(i.filter(s=>s.kind==="playwright-spec")),a=f(t.stats)?t.stats:{},r=t.failOnFlakyTests!==!1&&a.flaky!==0;return a.unexpected===u&&t.reportErrors===0&&t.attachmentFailures===0&&t.evidenceErrors===0&&t.runStatus==="failed"&&t.exitCode===k&&typeof t.failOnFlakyTests=="boolean"&&!r?"restated":"unexplained"}if(e.key==="K6_THRESHOLDS_FAILED"){if(t.exitCode!==c)return"unexplained";const u=t.thresholds;if(!Array.isArray(u))return"unexplained";const a=u;return a.length>0&&a.every(l=>typeof l=="string"&&l.startsWith("checks("))&&i.length>0&&t.failedChecks===d(i)?"restated":"script"}return"unexplained"}function d(e){return e.reduce((i,t)=>i+(typeof t.count=="number"?t.count:1),0)}function o(e){return{kind:"run-failed",reason:e}}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import
|
|
1
|
+
import f from"node:path";import{resolveProductDir as g}from"../../config/loader/index.js";import{collectRequiredConfigs as p,loadManagedConfigs as d,resolveTargetConfig as u,topologicallySortConfigs as v}from"../../config/loader/managed-configs.js";import{runtimePlanning as C}from"../../environment/runtime-planning.js";import{portsOf as w,releasePorts as h,reservePorts as P}from"../../environment/ports.js";export async function prepareSchemaOperation(o,m){const t=g(process.cwd(),o.dir),{configs:n}=await d({dir:t}),e=u(n,o.service||null),i=p(n,e.name);return{productDir:t,serviceName:e.name,async resolveConfigs(l){const s=await P(i.filter(r=>r.testkit.local).map(r=>r.name));await h(s.values());const a=C.resolveRuntimeInstanceConfigs(i,"runtime-1",f.join(l,"runtime-1"),{graphDirName:`${m}-${e.name}`,ports:w(s)}),c=a.find(r=>r.name===e.name);if(!c)throw new Error(`Resolved runtime config missing target service "${e.name}"`);return{configs:v(a),target:c}}}}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{filterConfigsByService as
|
|
1
|
+
import{filterConfigsByService as a}from"../../config/loader/managed-configs.js";import{loadConfigContext as s}from"../../config/loader/index.js";import{createEnvironmentMaintenance as i}from"../../environment/maintenance.js";import{hasLocalDatabaseState as d}from"../../database/lifecycle-cleanup.js";export async function destroy(t,e,r={}){await i(t).maintain({kind:"destroy",configs:e,destroyOwnedResources:r.destroyOwnedResources})}export async function destroyEnvironment(t){const e=await s({dir:t.dir}),r=a(e.configs,t.service||null),n=e.configs.some(o=>o.testkit.database!==void 0||o.testkit.databaseFrom!==void 0);return await destroy(e.productDir,r,{destroyOwnedResources:!t.service&&(n||d(e.productDir))}),{ok:!0,productDir:e.productDir,results:r.map(o=>({name:o.name,destroyed:!0}))}}
|
package/dist/app/doctor/index.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
import m from"path";import{randomUUID as v}from"node:crypto";import{loadDiscoveryContext as k,loadConfigContext as w}from"../../config/loader/index.js";import{runLintForConfig as D}from"../lint/index.js";export async function runDoctor(e={}){const t=[],r=e.dir?m.resolve(process.cwd(),e.dir):process.cwd(),s=await w({dir:r,configImportCacheSalt:v()});t.push({code:"config-load",level:"pass",message:"Loaded testkit config and service config"});const l=await k({dir:r,diagnostics:"report"},s),
|
|
1
|
+
import m from"path";import{randomUUID as v}from"node:crypto";import{loadDiscoveryContext as k,loadConfigContext as w}from"../../config/loader/index.js";import{runLintForConfig as D}from"../lint/index.js";import{isUnknownRecord as a}from"../../kernel/unknown-values.js";export async function runDoctor(e={}){const t=[],r=e.dir?m.resolve(process.cwd(),e.dir):process.cwd(),s=await w({dir:r,configImportCacheSalt:v()});t.push({code:"config-load",level:"pass",message:"Loaded testkit config and service config"});const l=await k({dir:r,diagnostics:"report"},s),g=l.discovery,f=C(g,"discovery result"),u=d(f.diagnostics,"discovery result diagnostics"),p=d(f.files,"discovery result files"),c=u.filter(S);t.push({code:"discovery",level:c.length===0?"pass":"fail",message:c.length===0?`Discovered ${String(p.length)} testkit files without diagnostics`:`Discovery reported ${String(c.length)} error diagnostics`,details:c});const o=await D(n(s,"config"),{dir:r});if(t.push({code:"lint",level:o.ok?"pass":"fail",message:o.ok?`No Testkit lint violations across ${String(o.summary.testkitFiles)} suite file(s)`:`Found ${String(o.summary.violations)} Testkit lint violation(s)`,details:o.violations}),e.typecheck!==!1)try{const{runTestkitTypecheckForContext:i}=await import("../typecheck/index.js"),y=await i(s,l.discovery,{dir:r}),h=d(n(y,"results"),"typecheck results");t.push({code:"typecheck",level:"pass",message:`Typechecked ${String(h.length)} generated program(s)`})}catch(i){t.push({code:"typecheck",level:"fail",message:$(i)})}return{ok:!t.some(i=>i.level==="fail"),productDir:r,checks:t}}function S(e){return a(e)&&Reflect.get(e,"severity")==="error"}function $(e){const t=n(e,"result"),r=n(t,"stderr");if(typeof r=="string"&&r.length>0)return r;const s=n(t,"stdout");return typeof s=="string"&&s.length>0?s:e instanceof Error?e.message:String(e)}function d(e,t){if(!Array.isArray(e))throw new Error(`${t} must be an array`);return e}function C(e,t){if(!a(e))throw new Error(`${t} must be an object`);return e}function n(e,t){return a(e)?Reflect.get(e,t):void 0}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { Environment, EnvironmentEntry, EnvironmentLease } from "../run/environment.js";
|
|
2
|
+
import type { ReadyEnvironment } from "../../environment/ready-environment.js";
|
|
3
|
+
import type { ProvisionedRun } from "../../environment/run-environment.js";
|
|
4
|
+
export declare function createConfigDrivenEnvironments(run: ProvisionedRun): EnvironmentEntry;
|
|
5
|
+
export declare function renderEnvironment(ready: Readonly<ReadyEnvironment>, targetName: string, lease: Readonly<EnvironmentLease>): Environment;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{buildTaskExecutionEnv as f,buildTemplateContext as u,finalizeSerializable as d}from"../../steps/index.js";import{resolveServiceCwd as g}from"../../config/loader/index.js";export function createConfigDrivenEnvironments(t){return{canAcquire:e=>t.canAcquire(e),getGraph:e=>t.getGraph(e),getGraphFailure:e=>t.getGraphFailure(e),markGraphFailed:(e,r)=>{t.markGraphFailed(e,r)},async acquire(e,r){const n=await t.acquire({...e,leaseId:r.id}),i=o=>t.release(n.leaseId,o);try{return{environment:renderEnvironment(n,e.targetName,r),nativeConfigDirectory:p(n,e.targetName),release:i}}catch(o){throw await i({invalidate:!1}),o}}}}export function renderEnvironment(t,e,r){const n=t.services.get(e);if(!n)throw new Error(`Runtime instance is missing target config "${e}"`);const i={leaseDir:r.directory,leaseId:r.id},o=Object.fromEntries([...t.services.values()].flatMap(({config:l,baseUrl:s})=>typeof s=="string"&&s.length>0?[[l.name,s]]:[])),c=f(n.config,i,{},null,{includeTestkitControls:!1,inheritProcessEnv:!1}),a=n.config.testkit.ui?m(d(n.config.testkit.ui,u(n.config,i))):null;return{baseUrl:Object.hasOwn(o,e)?o[e]:null,databaseUrl:n.database?.url??null,endpoints:o,env:v(E(c),e),ui:a}}function p(t,e){const r=t.services.get(e)?.config;if(!r)throw new Error(`Runtime instance is missing target config "${e}"`);return g(r.productDir,r.testkit.local?.cwd)}function v(t,e){const r=`TESTKIT_${e.toUpperCase().replace(/[^A-Z0-9]+/g,"_")}_SCHEMA_SOURCE_DATABASE_URL`,n=new Set(["TESTKIT_SCHEMA_SOURCE_DATABASE_URL",r]);return Object.fromEntries(Object.entries(t).filter(([i])=>!n.has(i.toUpperCase())))}function E(t){return Object.fromEntries(Object.entries(t).filter(e=>typeof e[1]=="string"))}function m(t){const e=JSON.stringify(t);if(typeof e!="string")return null;const r=JSON.parse(e);return r!==null&&typeof r=="object"&&!Array.isArray(r)?r:null}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { Environment, EnvironmentEntry } from "../run/environment.js";
|
|
2
|
+
import type { NormalizedServiceConfig } from "../../config/loader/index.js";
|
|
3
|
+
export declare function declaresProvisioning(runtimeConfigs: readonly NormalizedServiceConfig[]): boolean;
|
|
4
|
+
export declare function routeEnvironments(sourcesByGraph: ReadonlyMap<string, EnvironmentEntry>): EnvironmentEntry;
|
|
5
|
+
export type EnvironmentHold = (owner: string, environment: Environment) => Promise<void>;
|
|
6
|
+
export declare function holdBeforeRelease(entry: EnvironmentEntry, hold: EnvironmentHold): EnvironmentEntry;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export function declaresProvisioning(o){return o.some(({testkit:r})=>r.local!==void 0||r.database!==void 0||r.databaseFrom!==void 0||r.dependsOn.length>0||r.runtime.build!==null||r.runtime.prepare.steps.length>0||r.runtime.toolchain!==null||Object.keys(r.serviceEnv).length>0)}export function routeEnvironments(o){const r=e=>{const n=o.get(e);if(n===void 0)throw new Error(`No environment source for runtime graph "${e}"`);return n};return{acquire:(e,n)=>r(e.graphKey).acquire(e,n),canAcquire:e=>r(e.graphKey).canAcquire(e),getGraph:e=>r(e).getGraph(e),getGraphFailure:e=>r(e).getGraphFailure(e),markGraphFailed:(e,n)=>{r(e).markGraphFailed(e,n)}}}export function holdBeforeRelease(o,r){return{...o,async acquire(e,n){const a=await o.acquire(e,n);let i=null;return{environment:a.environment,nativeConfigDirectory:a.nativeConfigDirectory,async release(l){const d=i===null;i??=Promise.resolve().then(()=>r(e.targetName,a.environment)).then(()=>({failed:!1}),u=>({error:u,failed:!0}));const t=await i,c=d&&t.failed;try{await a.release(l)}catch(u){if(!c)throw u}if(c)throw t.error}}}}}
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
import type { Environment, EnvironmentEntry } from "../run/environment.js";
|
|
2
|
+
export type SuppliedEnvironment = Partial<Environment>;
|
|
3
|
+
export declare function createSuppliedEnvironments(nativeConfigDirectories: Readonly<Record<string, string>>, supplied?: Readonly<Record<string, SuppliedEnvironment>>): EnvironmentEntry;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export function createSuppliedEnvironments(n,r={}){return{canAcquire:()=>!0,getGraph:()=>null,getGraphFailure:()=>null,markGraphFailed:()=>{},acquire(e){const t=Object.hasOwn(r,e.targetName)?r[e.targetName]:{};return Object.hasOwn(n,e.targetName)?Promise.resolve({environment:a(e.targetName,t),nativeConfigDirectory:n[e.targetName],release:()=>Promise.resolve()}):Promise.reject(new Error(`No working directory for owner "${e.targetName}"`))}}}function a(n,r){const e=r.baseUrl??null;return{baseUrl:e,databaseUrl:r.databaseUrl??null,endpoints:{...r.endpoints,...e===null?{}:{[n]:e}},env:{...r.env},ui:r.ui??null}}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import
|
|
1
|
+
import n from"node:path";import{posixPath as u}from"../process/files/paths.js";export function resolveRequestedFiles(e,r,s=process.cwd()){const t=p(e),i=d(r,"product"),a=d(s,"invocation"),o=[],l=new Set;for(const f of t){const c=m(f,i,a);l.has(c)||(l.add(c),o.push(c))}return o}function m(e,r,s){const t=String(e),i=n.isAbsolute(t)?[t]:[n.resolve(s,t),n.resolve(r,t)];for(const a of i){const o=u(n.relative(r,a));if(!o.startsWith("../")&&o!==".."&&!n.isAbsolute(o))return o.replace(/^\.\/+/,"")}return u(t).replace(/^\.\/+/,"")}function p(e){if(!Array.isArray(e))throw new TypeError("Requested test files must be an array");return e}function d(e,r){if(typeof e!="string")throw new TypeError(`Testkit ${r} directory must be a string`);return e}
|
package/dist/app/lint/native.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
import y from"node:path";import{LINT_RULE_NAMES as _}from"./options.js";import{compareFindings as $,findingKey as B,isTypeScriptLikePath as M,matchesAnyPathPattern as h,normalizeFinding as c,
|
|
1
|
+
import y from"node:path";import{posixPath as x}from"../../process/files/paths.js";import{LINT_RULE_NAMES as _}from"./options.js";import{compareFindings as $,findingKey as B,isTypeScriptLikePath as M,matchesAnyPathPattern as h,normalizeFinding as c,SOURCE_IGNORES as F}from"./shared.js";import{createSyntaxPolicy as R}from"./rules/syntax.js";import{createDataPolicy as U}from"./rules/data.js";import{createUnitPolicy as A}from"./rules/unit.js";import{createEngineImportPolicy as q}from"./rules/engine-imports.js";export async function lintProject({productDir:i,files:n,trackedFiles:a,options:s}){const T=await K(),[{ESLint:k},v]=T;if(k.version!=="10.7.0"||v.version!=="8.65.0")throw new Error("Testkit lint requires its managed eslint@10.7.0 and @typescript-eslint/parser@8.65.0 installation");const I=new Map,w=[],f=_.filter(e=>s.rules[e].enabled),L=Object.fromEntries(f.filter(e=>e!=="testkitBoundary").map(e=>{const t=s.rules[e],r={meta:{type:"problem",schema:[],messages:{violation:"{{message}}"}},defaultOptions:[],create(o){return(()=>{const S=x(y.relative(i,o.filename)),u={context:o,file:S,report:(l,b,C,g)=>{const O=c({file:S,ruleName:e,ruleId:t.ruleId,kind:b,target:C,message:g,line:l.loc.start.line,snippet:o.sourceCode.getText(l).split(/\r?\n/u)[0]});I.set(E(o.filename,l.loc.start.line,l.loc.start.column+1,g),O),o.report({node:l,messageId:"violation",data:{message:g}})},options:s,literals:w};return e==="engineImports"?q(u):e==="dataSuiteContract"?U(u):e==="unitTestBoundaries"?A(u):R(e,u,n)})()}};return[t.ruleId,r]})),N=new k({cwd:i,overrideConfigFile:!0,allowInlineConfig:!1,errorOnUnmatchedPattern:!1,globInputPaths:!1,warnIgnored:!1,overrideConfig:[{ignores:F},{files:["**/*.{js,jsx,mjs,cjs,ts,tsx,mts,cts}"],languageOptions:{parser:v,ecmaVersion:"latest",sourceType:"module"},plugins:{testkit:{rules:L}}},...f.filter(e=>e!=="testkitBoundary").map(e=>{const t=s.rules[e];return{files:t.include.length?t.include:["**/*.{js,jsx,mjs,cjs,ts,tsx,mts,cts}"],ignores:t.exclude,rules:{[`testkit/${t.ruleId}`]:"error"}}})]}),P=[],j=n.filter(M);if(j.length){const e=j.map(r=>y.join(i,r)),t=await N.lintFiles(e);P.push(...t)}const d=P.flatMap(e=>e.messages.map(t=>{if(t.ruleId===null)return c({file:x(y.relative(i,e.filePath)),line:t.line,ruleName:"syntax",ruleId:"syntax",kind:"parse-error",target:t.message,message:t.message});const r=I.get(E(e.filePath,t.line,t.column,t.message));if(!r)throw new Error(`Missing Testkit finding for ${t.ruleId}: ${t.message}`);return r}));if(f.includes("testkitBoundary")){const e=s.rules.testkitBoundary,t=a&&new Set(a);for(const r of n)e.trackedOnly&&t&&!t.has(r)||!z(r,e)||!r.split("/").some(o=>o.toLowerCase().includes("testkit"))||h(r,e.allowed??[])||d.push(c({file:r,ruleName:"testkitBoundary",ruleId:e.ruleId,kind:"testkit-path",target:r,message:"Tracked Testkit plumbing must live in testkit.config.* or __testkit__/ suites, or move into @gfdelarue/testkit"}))}const p=new Map;for(const e of w){const t=`${e.field}:${e.value}`,r=p.get(t)??[];r.push(e),p.set(t,r)}for(const[e,t]of p)if(!(new Set(t.map(r=>r.file)).size<2))for(const r of t)d.push(c({...r,ruleName:"dalParallelSafety",ruleId:"dal-parallel-safety",kind:"duplicate-literal",target:e,message:`Duplicate ${r.field} literal "${r.value}" appears across DAL specs; use fixture-derived namespaced values`}));const m=new Map;for(const e of d.sort($)){const t=B(e);m.has(t)||m.set(t,e)}return[...m.values()]}function E(i,n,a,s){return[i,n,a,s].join("\0")}function z(i,n){return(!n.include.length||h(i,n.include))&&!h(i,n.exclude)}function K(){return Promise.all([import("eslint"),import("@typescript-eslint/parser")])}
|
|
@@ -32,7 +32,6 @@ export interface NormalizedLintOptions {
|
|
|
32
32
|
export declare const LINT_RULE_NAMES: readonly ["engineImports", "uiSpecShape", "dalParallelSafety", "dataSuiteContract", "configImports", "testkitBoundary", "unitTestBoundaries"];
|
|
33
33
|
export type LintRuleName = (typeof LINT_RULE_NAMES)[number];
|
|
34
34
|
export type LintRulePolicies = Record<LintRuleName, LintRulePolicy>;
|
|
35
|
-
export declare function isLintRuleName(value: string): value is LintRuleName;
|
|
36
35
|
export declare const RULES: {
|
|
37
36
|
readonly engineImports: {
|
|
38
37
|
readonly ruleId: "engine-imports";
|
package/dist/app/lint/options.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
import{DEFAULT_TESTKIT_BOUNDARY_ALLOWED as a
|
|
1
|
+
import{DEFAULT_TESTKIT_BOUNDARY_ALLOWED as a}from"./shared.js";import{posixPath as E}from"../../process/files/paths.js";import{isUnknownRecord as l}from"../../kernel/unknown-values.js";export const LINT_RULE_NAMES=["engineImports","uiSpecShape","dalParallelSafety","dataSuiteContract","configImports","testkitBoundary","unitTestBoundaries"];const b=new Set(["rules","ui"]),p=new Set(["enabled","exclude","include"]),m=new Set([...p,"allowed","trackedOnly"]),S=new Set(["maxLines","maxTests"]),u="lint levels were removed; set a rule to true or false, or use { enabled: boolean }";export const RULES={engineImports:{ruleId:"engine-imports",defaultEnabled:!0},uiSpecShape:{ruleId:"ui-spec-shape",defaultEnabled:!0},dalParallelSafety:{ruleId:"dal-parallel-safety",defaultEnabled:!0},dataSuiteContract:{ruleId:"data-suite-contract",defaultEnabled:!0},configImports:{ruleId:"config-imports",defaultEnabled:!0},testkitBoundary:{ruleId:"testkit-boundary",defaultEnabled:!1},unitTestBoundaries:{ruleId:"unit-test-boundaries",defaultEnabled:!1}};const I=220,O=8;export function normalizeLintConfig(e={}){const n=l(e)?e.lint??{}:{};return normalizeLintOptions(n)}export function normalizeLintOptions(e={}){if(!l(e))throw new Error("lint config must be an object");if(Object.prototype.hasOwnProperty.call(e,"level"))throw new Error(`lint.level was removed; ${u}`);f(e,b,"lint");const n=e.rules??{};if(!l(n))throw new Error("lint.rules must be an object");for(const i of Object.keys(n)){if(i==="missingImports")throw new Error("lint.rules.missingImports was removed; use testkit typecheck for TypeScript import diagnostics");if(!Object.prototype.hasOwnProperty.call(RULES,i))throw new Error(`Unknown lint rule "${i}"`)}const t=i=>{const s=RULES[i];return T({ruleName:i,ruleId:s.ruleId,value:n[i],defaultEnabled:s.defaultEnabled})},r={engineImports:t("engineImports"),configImports:t("configImports"),dalParallelSafety:t("dalParallelSafety"),dataSuiteContract:t("dataSuiteContract"),testkitBoundary:t("testkitBoundary"),uiSpecShape:t("uiSpecShape"),unitTestBoundaries:t("unitTestBoundaries")},o=e.ui===void 0?{}:e.ui;if(!l(o))throw new Error("lint.ui must be an object");return f(o,S,"lint.ui"),{rules:r,ui:{maxLines:w(o.maxLines,I,"lint.ui.maxLines"),maxTests:w(o.maxTests,O,"lint.ui.maxTests")}}}function T({ruleName:e,ruleId:n,value:t,defaultEnabled:r}){if(t===void 0)return d({ruleName:e,ruleId:n,enabled:r});if(typeof t=="boolean")return d({ruleName:e,ruleId:n,enabled:t});if(!l(t))throw new Error(`lint.rules.${e} must be true, false, or a rule config object; ${u}`);if(Object.prototype.hasOwnProperty.call(t,"level"))throw new Error(`lint.rules.${e}.level was removed; ${u}`);return f(t,e==="testkitBoundary"?m:p,`lint.rules.${e}`),d({ruleName:e,ruleId:n,enabled:y(t.enabled,`lint.rules.${e}.enabled`)??!0,include:c(t.include,`lint.rules.${e}.include`),exclude:c(t.exclude,`lint.rules.${e}.exclude`),allowed:e==="testkitBoundary"?h(t.allowed):void 0,trackedOnly:e==="testkitBoundary"?y(t.trackedOnly,`lint.rules.${e}.trackedOnly`):void 0})}function d({ruleName:e,ruleId:n,enabled:t,include:r=[],exclude:o=[],allowed:i,trackedOnly:s}){return{ruleName:e,ruleId:n,enabled:t,include:r,exclude:o,...e==="testkitBoundary"?{allowed:i||a}:{},...e==="testkitBoundary"?{trackedOnly:s!==!1}:{}}}function c(e,n){if(e==null)return[];if(!Array.isArray(e))throw new Error(`${n} must be an array`);return e.map((t,r)=>{const o=typeof t=="string"?t.trim():"";if(!o)throw new Error(`${n}[${String(r)}] must be a non-empty string`);return E(o)})}function h(e){if(e==null)return a;const n=c(e,"lint.rules.testkitBoundary.allowed");return n.length>0?n:a}function w(e,n,t){if(e===void 0)return n;if(typeof e!="number"||!Number.isInteger(e)||e<=0)throw new Error(`${t} must be a positive integer`);return e}function y(e,n){if(e!==void 0){if(typeof e!="boolean")throw new Error(`${n} must be a boolean`);return e}}function f(e,n,t){for(const r of Object.keys(e))if(!n.has(r))throw new Error(`Unknown ${t} option "${r}"`)}
|
|
@@ -24,8 +24,6 @@ export declare const SOURCE_IGNORES: string[];
|
|
|
24
24
|
export declare const DEFAULT_TESTKIT_BOUNDARY_ALLOWED: string[];
|
|
25
25
|
export declare function collectCurrentFiles(productDir: string, gitAvailable: boolean): Promise<string[]>;
|
|
26
26
|
export declare function collectTrackedFiles(productDir: string): Promise<string[] | null>;
|
|
27
|
-
export declare function normalizePath(value: string): string;
|
|
28
|
-
export declare function isObjectRecord(value: unknown): value is Record<string, unknown>;
|
|
29
27
|
export declare function isSourceLikePath(file: string): boolean;
|
|
30
28
|
export declare function isTypeScriptLikePath(file: string): boolean;
|
|
31
29
|
export declare function isTestkitSuitePath(file: string): boolean;
|
package/dist/app/lint/shared.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
import
|
|
1
|
+
import n from"node:fs";import a from"node:path";import{minimatch as l}from"minimatch";import{glob as p}from"tinyglobby";import{TEST_TYPE_ORDER as u}from"../../domain/test-types.js";import{runManagedGitCommand as c}from"../../repo/managed-git.js";import{posixPath as i}from"../../process/files/paths.js";export const DEFAULT_EXCLUDED_DIRS=[".git",".next",".next-testkit",".testkit","coverage","dist","node_modules","playwright-report"],SOURCE_PATTERNS=["**/*.{js,jsx,mjs,cjs,ts,tsx,mts,cts,json}"],SOURCE_IGNORES=DEFAULT_EXCLUDED_DIRS.map(t=>`**/${t}/**`),DEFAULT_TESTKIT_BOUNDARY_ALLOWED=["testkit.config.ts","testkit.config.mts","testkit.config.mjs","testkit.config.js","**/__testkit__/**"];export async function collectCurrentFiles(t,e){const o=e?await c(t,["ls-files","--cached","--others","--exclude-standard","-z"]):void 0,s=o?.exitCode===0?o.stdout.split("\0").filter(r=>r&&n.existsSync(a.join(t,r))):await p(SOURCE_PATTERNS,{cwd:t,ignore:SOURCE_IGNORES,dot:!0,followSymbolicLinks:!1});return[...new Set(s.map(i))].filter(r=>isSourceLikePath(r)&&!isExcludedRelativePath(r)).sort((r,m)=>r.localeCompare(m))}export async function collectTrackedFiles(t){const e=await c(t,["ls-files","-z"]);return e.exitCode!==0?null:e.stdout.split("\0").map(o=>i(o)).filter(Boolean).filter(o=>n.existsSync(a.join(t,o))).sort((o,s)=>o.localeCompare(s))}export function isSourceLikePath(t){return/\.(?:[cm]?[jt]sx?|json)$/u.test(t)}export function isTypeScriptLikePath(t){return/\.[cm]?[jt]sx?$/u.test(t)}export function isTestkitSuitePath(t){return u.some(e=>t.endsWith(`.${e}.testkit.ts`))}export function isExcludedRelativePath(t){return SOURCE_IGNORES.some(e=>l(t,e,{dot:!0}))}export function matchesAnyPathPattern(t,e){return e.some(o=>l(t,o,{dot:!0}))}export function compareFindings(t,e){return t.file.localeCompare(e.file)||t.line-e.line||t.ruleId.localeCompare(e.ruleId)||t.kind.localeCompare(e.kind)||t.target.localeCompare(e.target)}export function normalizeFinding(t){return{ruleName:t.ruleName,ruleId:t.ruleId,kind:t.kind||"violation",target:t.target||"",file:i(t.file),line:t.line||1,message:t.message,...t.snippet?{snippet:t.snippet}:{}}}export function findingKey(t){return[t.file,String(t.line),t.ruleId,t.kind,t.target].join("\0")}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import p from"node:fs";import r from"node:path";import{fileURLToPath as
|
|
1
|
+
import p from"node:fs";import r from"node:path";import{fileURLToPath as v}from"node:url";import{satisfiesSemver as P}from"../toolchains/semver.js";import{readPackageMetadata as D}from"../process/host/metadata.js";import{isUnknownRecord as f}from"../kernel/unknown-values.js";const S=["dependencies","devDependencies","optionalDependencies","peerDependencies"];export function testkitIdentity(){return{description:"Test orchestration platform for product acceptance and QA workflows",name:"@gfdelarue/testkit",version:D().version}}export function inspectPackageVersionDrift(e={}){const n=e.currentPackageRoot||j(e.binUrl||import.meta.url),t=a(r.join(n,"package.json"));if(!t?.version)throw new Error(`Unable to read the current Testkit version from ${n}`);const c=e.cwd||process.cwd(),s=e.dir?r.resolve(c,e.dir):c;if(l(s,n))return null;const g=a(r.join(s,"package.json")),o=h(g),d=r.join(s,"node_modules","@gfdelarue","testkit"),i=a(r.join(d,"package.json"));return!o&&!i||i&&l(d,n)?null:i?.version&&i.version!==t.version?{productDir:s,currentPackageRoot:n,currentVersion:t.version,productPackageRoot:d,productVersion:i.version,declaredSpec:o}:!i&&o&&w(o)&&!P(t.version,m(o))?{productDir:s,currentPackageRoot:n,currentVersion:t.version,productPackageRoot:null,productVersion:null,declaredSpec:o}:null}function j(e){const n=v(e);return R(r.dirname(n))}function R(e){let n=r.resolve(e);for(;;){if(a(r.join(n,"package.json"))?.name==="@gfdelarue/testkit")return n;const c=r.dirname(n);if(c===n)return r.resolve(e);n=c}}function h(e=null){for(const n of S){const t=e?.[n]?.["@gfdelarue/testkit"];if(t?.trim())return t.trim()}return null}function a(e){try{const n=JSON.parse(p.readFileSync(e,"utf8"));return y(n)}catch{return null}}function l(e,n){try{return p.realpathSync(e)===p.realpathSync(n)}catch{return r.resolve(e)===r.resolve(n)}}function w(e){const n=m(e);return/^(?:[<>]=?|[\^~=])?\s*\d/u.test(n)}function m(e){return e.trim().replace(/^workspace:/u,"")}function y(e){return f(e)?{dependencies:u(e.dependencies),devDependencies:u(e.devDependencies),name:k(e.name),optionalDependencies:u(e.optionalDependencies),peerDependencies:u(e.peerDependencies),version:k(e.version)}:null}function u(e){if(!f(e))return;const n=Object.entries(e).filter(t=>typeof t[1]=="string");return Object.fromEntries(n)}function k(e){return typeof e=="string"?e:void 0}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export declare const REPORT_FILE_BYTES: number;
|
|
2
|
+
export declare const REPORT_BYTES: number;
|
|
3
|
+
export declare const REPORT_FILES = 1000;
|
|
4
|
+
export declare const REPORT_TREES = 128;
|
|
5
|
+
export interface ReportFile {
|
|
6
|
+
readonly path: string;
|
|
7
|
+
readonly absolutePath: string;
|
|
8
|
+
readonly executable: boolean;
|
|
9
|
+
readonly size: number;
|
|
10
|
+
}
|
|
11
|
+
export declare function requireReportPath(value: string): string;
|
|
12
|
+
export declare function copyReportFile(source: string, destination: string, root?: string): number;
|
|
13
|
+
export declare function snapshotReport(source: string, destination: string, maxDirectories?: number): ReportFile[];
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import o from"node:fs";import c from"node:path";import{copyRegularFileSecurely as g}from"../../evidence/secure-file-copy.js";import{sameFileSnapshot as b}from"../../process/files/file-snapshot.js";export const REPORT_FILE_BYTES=16*1024*1024,REPORT_BYTES=64*1024*1024,REPORT_FILES=1e3,REPORT_TREES=128;export function requireReportPath(t){if(!t||Buffer.byteLength(t)>4096||Buffer.from(t).toString("utf8")!==t||/[\\\0\r\n]/u.test(t)||c.posix.normalize(t)!==t||t.startsWith("/")||t.split("/").some(r=>!r||r==="."||r===".."||r.toLowerCase()===".git"))throw new Error("Use a relative report path without traversal, Git metadata, backslashes, or line breaks.");return t}export function copyReportFile(t,r,f){return o.mkdirSync(c.dirname(r),{recursive:!0}),g(t,r,{maxBytes:REPORT_FILE_BYTES,sizeLimitDescription:"16 MiB report file limit",sourceRoot:f}).sizeBytes}export function snapshotReport(t,r,f=REPORT_TREES){const i=[],R=o.realpathSync(t);let l=0,d=0;function y(e,n,w){const p=o.lstatSync(e,{bigint:!0});if(!p.isDirectory()||p.isSymbolicLink())throw new Error("Reports must contain ordinary directories and files, not links.");if(w>32||++d>f)throw new Error("The report exceeds its directory traversal limit.");for(const s of o.readdirSync(e,{withFileTypes:!0,encoding:"buffer"})){const m=s.name.toString("utf8");if(!Buffer.from(m).equals(s.name))throw new Error("Report filenames must be UTF-8.");const a=requireReportPath(n+m),h=c.join(e,m);if(s.isDirectory()){y(h,a+"/",w+1);continue}if(!s.isFile())throw new Error(`Report entry ${JSON.stringify(a)} is not an ordinary file.`);if(i.length>=REPORT_FILES)throw new Error("A report may contain at most 1000 files.");const S=o.lstatSync(h).mode,u=c.join(r,a),E=copyReportFile(h,u,R);if(l+=E,l>REPORT_BYTES)throw new Error("A report may contain at most 64 MiB of files.");i.push({path:a,absolutePath:u,executable:(S&73)!==0,size:E})}if(!b(p,o.lstatSync(e,{bigint:!0})))throw new Error("The report directory changed while it was being read. Try again with a stable directory.")}if(y(t,"",0),i.length===0)throw new Error("The report directory contains no files. Git does not store empty directories.");return i.sort((e,n)=>e.path<n.path?-1:e.path>n.path?1:0)}
|