@overkill-dev/test 0.0.13 → 0.0.15

package/readme.md

@@ -6,13 +6,70 @@ This package ships the public `overkill` binary and the staged root authoring
 facade. The binary parses the minimal command surface and delegates
 execution to `@overkill-dev/run/command-line`.
 
+## Quick start
+
+Install the standard distribution:
+
+```sh
+npm install -D @overkill-dev/test
+```
+
+Create `overkill.config.ts`:
+
+```ts
+import { defineConfig } from '@overkill-dev/test/config';
+
+export const config = defineConfig({
+    profiles: {
+        microtest: {
+            testFamily: 'microtest',
+            files: {
+                include: [ 'source/**/*.test.ts' ]
+            }
+        }
+    }
+});
+```
+
+Create `source/calculator.test.ts`:
+
+```ts
+import { suite, test } from '@overkill-dev/test';
+
+function add(left: number, right: number): number {
+    return left + right;
+}
+
+export const testNode = suite('calculator', [
+    test('adds numbers', (scope) => {
+        scope.assert.equal(add(2, 3), 5);
+
+        return scope.assert.collect();
+    })
+]);
+```
+
+Run the configured test files:
+
+```sh
+npx overkill run
+```
+
+## Package surface
+
 Current root runtime exports:
 
 - `test`
+- `skippedTest`
 - `suite`
 - `table`
+- `defineHarness`
 - `defineMacro`
 - `defineParameterizedTestBody`
+- `createTranscript`
+- `recordSink`
+- `recordAsyncSink`
+- `transcriptUsage`
 - `doubleUsage`
 - `rule`
 - `testDouble`
@@ -34,14 +91,19 @@ Standard subpaths:
   `createGithubActionsOutputRenderer`.
 - `@overkill-dev/test/assert` re-exports assertion-extension helpers from
   `@overkill-dev/assert`.
-- `@overkill-dev/test/bench`, `@overkill-dev/test/resources`, and
-  `@overkill-dev/test/baselines` are reserved. They currently export only
-  `unavailable()`.
+- `@overkill-dev/test/compatibility` exports `throwingTest` for explicit
+  throwable-style test authoring.
+- `@overkill-dev/test/resources` re-exports typed resource and runtime
+  descriptors from `@overkill-dev/resources`, including
+  `createTemporaryDirectoryResource(...)`, and adds `withRuntime(...)`,
+  `withResource(...)`, and `withResources(...)` for runner-aware binding.
+- `@overkill-dev/test/bench` and `@overkill-dev/test/baselines` are reserved.
+  They currently export only `unavailable()`.
 
 Implemented root authoring forms:
 
 ```ts
-import { doubleUsage, suite, table, test, testDouble } from '@overkill-dev/test';
+import { doubleUsage, skippedTest, suite, table, test, testDouble } from '@overkill-dev/test';
 
 export const testNode = suite('users', [
     test('loads user', (scope) => {
@@ -61,7 +123,8 @@ export const testNode = suite('users', [
             scope.assert.true(canLoadUser(scope.parameters));
             return scope.assert.collect();
         }
-    })
+    }),
+    skippedTest('loads platform user', 'requires linux')
 ]);
 ```
 
@@ -69,21 +132,40 @@ The root doubles exports are the current lightweight public surface from
 `@overkill-dev/doubles`. Import the leaf package directly when documenting or
 testing doubles package ownership.
 
-Nodes created through this root facade derive `metadata.kind: 'microtest'`.
-High-level authoring metadata accepts `tags` and `extra`; engine-owned fields
-such as `kind`, `runtimes`, and `ownership` stay outside the root facade.
+Nodes created through this root facade are family-neutral. The selected runner
+profile owns family policy, capture support, resource policy, and timeout
+validation.
+
+Use the object form when attaching annotations or controls. Annotations on the
+exported top-level `testNode` apply to the whole module's test tree.
+
+`skippedTest(title, reason)` creates a visible leaf test with a mandatory
+reason. It is discovered, listed, reported as skipped, and never runs user
+code. Object form is `skippedTest({ title, annotations, controls, reason })`.
 
-Use the object form when attaching node metadata. Metadata on the exported
-top-level `testNode` applies to the whole module's test tree.
+`throwingTest` is available from the compatibility subpath when a test should
+pass by completing normally instead of returning `scope.assert.collect()`:
+
+```ts
+import { throwingTest } from '@overkill-dev/test/compatibility';
+
+export const testNode = throwingTest('legacy assertion', (scope) => {
+    scope.assert.equal(add(2, 3), 5);
+});
+```
+
+The throwing body receives `scope.assert`, `scope.require`, and `scope.signal`.
+It does not expose `scope.plan` or `scope.assert.collect`.
 
 ```ts
 export const testNode = suite({
     title: 'users',
-    metadata: { tags: [ 'auth' ] },
+    annotations: { tags: [ 'auth' ] },
     children: [
         test({
             title: 'loads user',
-            metadata: { tags: [ 'critical' ] },
+            annotations: { tags: [ 'critical' ] },
+            controls: { timeoutMilliseconds: 1_000 },
             body(scope) {
                 scope.assert.equal(loadUser('42').name, 'Ada');
                 return scope.assert.collect();
@@ -98,6 +180,81 @@ body receives the original row value as `scope.parameters`; default row titles
 are `case 1`, `case 2`, and so on. Reachable tables must contain at least two
 rows.
 
+`defineHarness` creates reusable test-side harness constructors:
+
+```ts
+const userHarness = defineHarness({
+    loadUser: () => testDouble.resolves<() => Promise<User>>(user)
+}, (parts) => {
+    return {
+        subject: createUserService({ loadUser: parts.loadUser }),
+        ...parts
+    };
+});
+
+const harness = userHarness.create();
+```
+
+Object-form part factories run fresh for each `create()` call. Sparse
+overrides replace final part values; overridden factories are not called.
+Function-form harnesses support richer sync or async setup:
+
+```ts
+const renderUser = defineHarness(async (overrides: {
+    readonly loadUser?: () => Promise<User>;
+}) => {
+    const loadUser = overrides.loadUser ?? testDouble.resolves<() => Promise<User>>(user);
+
+    return {
+        loadUser,
+        rendered: await render(<UserPage loadUser={loadUser} />)
+    };
+});
+```
+
+`createTranscript` records ordered interaction entries as tuple values:
+
+```ts
+const transcript = createTranscript<readonly [kind: 'state', value: number]>();
+
+transcript.record('state', 1);
+
+scope.assert(transcriptUsage.exactly, transcript, [
+    [ 'state', 1 ]
+]);
+```
+
+`transcript.sink(kind)` creates a void test double that records each call as
+`[kind, ...args]`. The double call history and transcript entries reset
+independently.
+
+```ts
+const transcript = createTranscript<readonly [kind: 'warn', message: string]>();
+const warn = transcript.sink<(message: string) => void>('warn');
+
+warn('retrying');
+
+scope.assert(transcriptUsage.contains, transcript, [ 'warn', 'retrying' ]);
+scope.assert(doubleUsage.calledOnceWith, warn, [ 'retrying' ]);
+```
+
+`recordSink` and `recordAsyncSink` subscribe immediately and return disposable
+transcripts for callback-based sources:
+
+```ts
+const states = recordSink<readonly [kind: 'state', value: number]>((record) => {
+    return store.subscribe((value) => {
+        record('state', value);
+    });
+});
+
+const events = recordAsyncSink<readonly [kind: 'event', value: Event]>((record) => {
+    return queue.listen((value) => {
+        record('event', value);
+    });
+});
+```
+
 `defineMacro` preserves source locations for reusable test-node factories:
 
 ```ts
@@ -129,8 +286,112 @@ export const testNode = suite('users', [
 ]);
 ```
 
-`createTestFacade` creates another narrow authoring surface for one test
-family:
+Runtime handles stay behind the resources subpath:
+
+```ts
+import { createTestFacade } from '@overkill-dev/test';
+import {
+    composeRuntimes,
+    createTemporaryDirectoryResource,
+    defineResource,
+    defineRuntime,
+    withResource,
+    withRuntime
+} from '@overkill-dev/test/resources';
+
+const api = createTestFacade({ annotations: { tags: [ 'api' ] } });
+const scratch = createTemporaryDirectoryResource('scratch');
+const database = defineResource({
+    name: 'database',
+    scope: 'per-file',
+    requirements: [],
+    dependencies: {},
+    acquire() {
+        return openDatabase();
+    },
+    deserializeHandle(payload) {
+        return { url: payload.url };
+    },
+    dispose(database) {
+        return database.close();
+    },
+    serializeHandle(database) {
+        return { url: database.url };
+    }
+});
+
+const runtime = defineRuntime({
+    name: 'api',
+    dimensions: {},
+    resources: { database, scratch },
+    requirements: []
+});
+
+api.test(
+    'loads user',
+    withRuntime(runtime, (scope) => {
+        scope.assert.true(scope.signal instanceof AbortSignal);
+        scope.assert.equal(typeof scope.runtimes.api.database, 'object');
+        return scope.assert.collect();
+    })
+);
+
+api.test(
+    'writes scratch output',
+    withResource(scratch, (scope) => {
+        scope.assert.true(scope.signal instanceof AbortSignal);
+        scope.assert.true(scope.resources.scratch.path.length > 0);
+        return scope.assert.collect();
+    })
+);
+```
+
+Resource wrappers attach descriptors for runner collection and acquire handles
+when the body executes. `withRuntime(...)` exposes handles at
+`scope.runtimes.<runtimeName>`, while `withResource(...)` and
+`withResources(...)` expose handles at `scope.resources.<resourceKey>`.
+`composeRuntimes(...)` combines leaf runtimes and runtime matrices into one
+graph while keeping child runtime names as the public `scope.runtimes` keys.
+Nested resource wrappers compose into one per-case acquisition graph before
+the body runs when no runner lifecycle policy is active. During normal
+runner execution, wrappers declare descriptors and the runner acquires them at
+their declared `per-run`, `per-file`, `per-suite`, `per-case`, or
+`shared-per-worker` lifetime boundary. Duplicate public `scope.resources` keys or public
+`scope.runtimes` names are rejected before planning; resource keys inside
+different runtime scopes remain namespaced by their runtime name.
+`per-run` resources require `serializeHandle(...)` and
+`deserializeHandle(...)`; `per-file` and `per-suite` resources may use those
+hooks when the owner handle and consumer handle differ.
+Microtest profiles reject collected resource descriptors before execution.
+
+Test bodies receive async-control methods on `scope`:
+
+| Method               | Purpose                                                   |
+| -------------------- | --------------------------------------------------------- |
+| `drainMicrotasks()`  | Wait for already-scheduled Promise and microtask work.    |
+| `yieldToNextTurn()`  | Yield through one Node event-loop turn.                   |
+| `settleAsyncWork()`  | Run a bounded set of microtask and next-turn checkpoints. |
+| `startInFlight(...)` | Start background Promise work now and observe it later.   |
+| `cleanup(...)`       | Register sync or async teardown for test-owned resources. |
+
+```ts
+test('logs background failures', async (scope) => {
+    const run = scope.startInFlight(() => worker.run());
+
+    await scope.yieldToNextTurn();
+
+    await run.rejects({ message: 'boom' });
+    scope.assert.equal(logger.error.interactionCount, 1);
+    return scope.assert.collect();
+});
+```
+
+`settleAsyncWork()` is intentionally bounded. Use it for finite queue
+cascades, not as proof that all background work is complete. Work started with
+`startInFlight(...)` must settle and be observed before the test ends.
+
+`createTestFacade` creates another narrow authoring surface without selecting
+a test family:
 
 ```ts
 import { createTestFacade } from '@overkill-dev/test';
@@ -139,17 +400,84 @@ export const {
     defineMacro,
     defineParameterizedTestBody,
     runIfMain,
+    skippedTest,
     suite,
     table,
     test
 } = createTestFacade({
-    metadata: { tags: [ 'integration' ] },
-    testFamily: 'integration'
+    annotations: { tags: [ 'api' ] },
+    controls: { capture: 'live' }
 });
 ```
 
+Facades can also bind one runtime graph, direct resources, and project-owned
+scope conveniences:
+
+```ts
+import { createTestFacade } from '@overkill-dev/test';
+import { apiRuntime, scratch } from '#tests/resources';
+
+export const { test } = createTestFacade({
+    runtime: apiRuntime,
+    resources: { scratch },
+    mapScope(scope) {
+        return {
+            apiUrl: scope.runtimes.api.server.url,
+            scratchPath: scope.resources.scratch.path
+        };
+    }
+});
+
+test('loads users', (scope) => {
+    scope.assert.true(scope.apiUrl.length > 0);
+    scope.assert.true(scope.scratchPath.length > 0);
+    return scope.assert.collect();
+});
+```
+
+`mapScope(...)` sees the base test scope plus facade-bound runtime and
+resource handles. Body-specific `withRuntime(...)`, `withResource(...)`, and
+`withResources(...)` wrappers still compose into the final body scope, but
+they are not inputs to the facade mapper.
+
+`mapScope(...)` may add project-owned convenience properties only. It must not
+return reserved scope keys: `assert`, `require`, `plan`, `collect`, `cleanup`,
+`signal`, `drainMicrotasks`, `settleAsyncWork`, `startInFlight`,
+`yieldToNextTurn`, `runtimes`, `resources`, or `parameters`. Known collisions
+are rejected by TypeScript, and dynamic mapper results are validated at
+runtime before the mapped scope is composed.
+
 The returned facade contains authoring helpers only. Assertions and doubles
 are imported alongside it instead of being registered into the facade.
+Facades and root helpers accept `controls.capture` as authored data. The
+selected profile decides whether that control is valid for a run.
+
+Default tests should keep importing from `@overkill-dev/test`. When a project
+has a custom facade, it can re-export that facade through a stable local alias:
+
+```json
+{
+    "imports": {
+        "#tests/custom-authoring": "./testing/custom-authoring.ts"
+    }
+}
+```
+
+```ts
+import { createTestFacade } from '@overkill-dev/test';
+
+export const { suite, test } = createTestFacade({
+    annotations: { tags: [ 'custom-authoring' ] }
+});
+```
+
+```ts
+import { suite, test } from '#tests/custom-authoring';
+```
+
+Native Node `package.json#imports` aliases must start with `#`. A project can
+use an alias such as `@tests/my-facade` only when its TypeScript or bundler
+tooling owns that resolution.
 
 Direct Node execution:
 
@@ -179,8 +507,11 @@ Supported command-line surface:
 - `--config <path>`
 - `--file <path>`
 - `--filter <expr>`
+- `--order <seeded|lexical>`
+- `--seed <n>`
 - `--title <text>`
 - `--profile <name>`
+- `--no-capture`
 - `--measure-resource-usage`
 - `--resource-budget <name=value>`
 - `--with-locations`
@@ -191,10 +522,18 @@ Supported command-line surface:
 `residentSetGrowthBytesPerSecond`. Supplying a resource budget enables
 resource usage measurement for that run.
 
+`--no-capture` applies to `run` only. It passes stdout and stderr through live
+for capture-capable profiles and is invalid for microtest profiles.
+
 `--filter`, `--title`, and `--file` apply the same run selection to `run` and
 `list`. `--filter` supports `=`, `~`, `:`, `!`, `|`, and parentheses over
 `tag`, `runtime`, `owner`, `stability`, `file`, `title`, `suite`, and `params`.
 
+Runs use seeded ordering by default. Pass `--seed <n>` to reproduce a shuffle,
+or `--order lexical` to use deterministic source-stable order. `runIfMain(...)`
+also uses seeded order and reports the generated seed through run facts and
+first-party reporters.
+
 When no paths are supplied, `run` and `list` discover files from the selected
 profile's `files.include` and `files.exclude` policy. Explicit file paths run
 directly. Directory paths filter the selected profile's discovered files and

package/sbom.cdx.json

@@ -16,115 +16,128 @@
         "component": {
             "type": "library",
             "name": "@overkill-dev/test",
-            "version": "0.0.13",
-            "bom-ref": "pkg:npm/@overkill-dev/test@0.0.13",
-            "purl": "pkg:npm/@overkill-dev/test@0.0.13"
+            "version": "0.0.15",
+            "bom-ref": "pkg:npm/@overkill-dev/test@0.0.15",
+            "purl": "pkg:npm/@overkill-dev/test@0.0.15"
         }
     },
     "components": [
         {
             "type": "library",
             "name": "@overkill-dev/assert",
-            "version": "0.0.29",
-            "bom-ref": "pkg:npm/@overkill-dev/assert@0.0.29",
+            "version": "0.0.31",
+            "bom-ref": "pkg:npm/@overkill-dev/assert@0.0.31",
             "scope": "required",
             "licenses": [
                 {
                     "expression": "MIT"
                 }
             ],
-            "purl": "pkg:npm/@overkill-dev/assert@0.0.29"
+            "purl": "pkg:npm/@overkill-dev/assert@0.0.31"
         },
         {
             "type": "library",
             "name": "@overkill-dev/doubles",
-            "version": "0.0.27",
-            "bom-ref": "pkg:npm/@overkill-dev/doubles@0.0.27",
+            "version": "0.0.29",
+            "bom-ref": "pkg:npm/@overkill-dev/doubles@0.0.29",
             "scope": "required",
             "licenses": [
                 {
                     "expression": "MIT"
                 }
             ],
-            "purl": "pkg:npm/@overkill-dev/doubles@0.0.27"
+            "purl": "pkg:npm/@overkill-dev/doubles@0.0.29"
         },
         {
             "type": "library",
             "name": "@overkill-dev/engine",
-            "version": "0.0.28",
-            "bom-ref": "pkg:npm/@overkill-dev/engine@0.0.28",
+            "version": "0.0.39",
+            "bom-ref": "pkg:npm/@overkill-dev/engine@0.0.39",
             "scope": "required",
             "licenses": [
                 {
                     "expression": "MIT"
                 }
             ],
-            "purl": "pkg:npm/@overkill-dev/engine@0.0.28"
+            "purl": "pkg:npm/@overkill-dev/engine@0.0.39"
         },
         {
             "type": "library",
             "name": "@overkill-dev/output-renderer-github-actions",
-            "version": "0.0.18",
-            "bom-ref": "pkg:npm/@overkill-dev/output-renderer-github-actions@0.0.18",
+            "version": "0.0.20",
+            "bom-ref": "pkg:npm/@overkill-dev/output-renderer-github-actions@0.0.20",
             "scope": "required",
             "licenses": [
                 {
                     "expression": "MIT"
                 }
             ],
-            "purl": "pkg:npm/@overkill-dev/output-renderer-github-actions@0.0.18"
+            "purl": "pkg:npm/@overkill-dev/output-renderer-github-actions@0.0.20"
         },
         {
             "type": "library",
             "name": "@overkill-dev/reporter-brief",
-            "version": "0.0.18",
-            "bom-ref": "pkg:npm/@overkill-dev/reporter-brief@0.0.18",
+            "version": "0.0.20",
+            "bom-ref": "pkg:npm/@overkill-dev/reporter-brief@0.0.20",
             "scope": "required",
             "licenses": [
                 {
                     "expression": "MIT"
                 }
             ],
-            "purl": "pkg:npm/@overkill-dev/reporter-brief@0.0.18"
+            "purl": "pkg:npm/@overkill-dev/reporter-brief@0.0.20"
         },
         {
             "type": "library",
             "name": "@overkill-dev/reporter-dot",
-            "version": "0.0.29",
-            "bom-ref": "pkg:npm/@overkill-dev/reporter-dot@0.0.29",
+            "version": "0.0.31",
+            "bom-ref": "pkg:npm/@overkill-dev/reporter-dot@0.0.31",
             "scope": "required",
             "licenses": [
                 {
                     "expression": "MIT"
                 }
             ],
-            "purl": "pkg:npm/@overkill-dev/reporter-dot@0.0.29"
+            "purl": "pkg:npm/@overkill-dev/reporter-dot@0.0.31"
         },
         {
             "type": "library",
             "name": "@overkill-dev/reporter-line",
-            "version": "0.0.29",
-            "bom-ref": "pkg:npm/@overkill-dev/reporter-line@0.0.29",
+            "version": "0.0.31",
+            "bom-ref": "pkg:npm/@overkill-dev/reporter-line@0.0.31",
             "scope": "required",
             "licenses": [
                 {
                     "expression": "MIT"
                 }
             ],
-            "purl": "pkg:npm/@overkill-dev/reporter-line@0.0.29"
+            "purl": "pkg:npm/@overkill-dev/reporter-line@0.0.31"
+        },
+        {
+            "type": "library",
+            "name": "@overkill-dev/resources",
+            "version": "0.0.2",
+            "bom-ref": "pkg:npm/@overkill-dev/resources@0.0.2",
+            "scope": "required",
+            "licenses": [
+                {
+                    "expression": "MIT"
+                }
+            ],
+            "purl": "pkg:npm/@overkill-dev/resources@0.0.2"
         },
         {
             "type": "library",
             "name": "@overkill-dev/run",
-            "version": "0.0.33",
-            "bom-ref": "pkg:npm/@overkill-dev/run@0.0.33",
+            "version": "0.0.51",
+            "bom-ref": "pkg:npm/@overkill-dev/run@0.0.51",
             "scope": "required",
             "licenses": [
                 {
                     "expression": "MIT"
                 }
             ],
-            "purl": "pkg:npm/@overkill-dev/run@0.0.33"
+            "purl": "pkg:npm/@overkill-dev/run@0.0.51"
         },
         {
             "type": "library",
@@ -142,40 +155,44 @@
     ],
     "dependencies": [
         {
-            "ref": "pkg:npm/@overkill-dev/assert@0.0.29"
+            "ref": "pkg:npm/@overkill-dev/assert@0.0.31"
+        },
+        {
+            "ref": "pkg:npm/@overkill-dev/doubles@0.0.29"
         },
         {
-            "ref": "pkg:npm/@overkill-dev/doubles@0.0.27"
+            "ref": "pkg:npm/@overkill-dev/engine@0.0.39"
         },
         {
-            "ref": "pkg:npm/@overkill-dev/engine@0.0.28"
+            "ref": "pkg:npm/@overkill-dev/output-renderer-github-actions@0.0.20"
         },
         {
-            "ref": "pkg:npm/@overkill-dev/output-renderer-github-actions@0.0.18"
+            "ref": "pkg:npm/@overkill-dev/reporter-brief@0.0.20"
         },
         {
-            "ref": "pkg:npm/@overkill-dev/reporter-brief@0.0.18"
+            "ref": "pkg:npm/@overkill-dev/reporter-dot@0.0.31"
         },
         {
-            "ref": "pkg:npm/@overkill-dev/reporter-dot@0.0.29"
+            "ref": "pkg:npm/@overkill-dev/reporter-line@0.0.31"
         },
         {
-            "ref": "pkg:npm/@overkill-dev/reporter-line@0.0.29"
+            "ref": "pkg:npm/@overkill-dev/resources@0.0.2"
         },
         {
-            "ref": "pkg:npm/@overkill-dev/run@0.0.33"
+            "ref": "pkg:npm/@overkill-dev/run@0.0.51"
         },
         {
-            "ref": "pkg:npm/@overkill-dev/test@0.0.13",
+            "ref": "pkg:npm/@overkill-dev/test@0.0.15",
             "dependsOn": [
-                "pkg:npm/@overkill-dev/assert@0.0.29",
-                "pkg:npm/@overkill-dev/doubles@0.0.27",
-                "pkg:npm/@overkill-dev/engine@0.0.28",
-                "pkg:npm/@overkill-dev/output-renderer-github-actions@0.0.18",
-                "pkg:npm/@overkill-dev/reporter-brief@0.0.18",
-                "pkg:npm/@overkill-dev/reporter-dot@0.0.29",
-                "pkg:npm/@overkill-dev/reporter-line@0.0.29",
-                "pkg:npm/@overkill-dev/run@0.0.33",
+                "pkg:npm/@overkill-dev/assert@0.0.31",
+                "pkg:npm/@overkill-dev/doubles@0.0.29",
+                "pkg:npm/@overkill-dev/engine@0.0.39",
+                "pkg:npm/@overkill-dev/output-renderer-github-actions@0.0.20",
+                "pkg:npm/@overkill-dev/reporter-brief@0.0.20",
+                "pkg:npm/@overkill-dev/reporter-dot@0.0.31",
+                "pkg:npm/@overkill-dev/reporter-line@0.0.31",
+                "pkg:npm/@overkill-dev/resources@0.0.2",
+                "pkg:npm/@overkill-dev/run@0.0.51",
                 "pkg:npm/cmd-ts@0.15.0"
             ]
         },