@creative-introvert/tons-of-tests 2.0.0 → 3.0.0-rc.0

package/README.md

@@ -26,7 +26,8 @@ A specialized test runner for predictive functions, focusing on scenarios where
 
 Use Node.js 24 or newer and Effect 4. The Effect packages in this repository
 use the same exact version. The SQLite driver uses Node's built-in SQLite.
-Existing database files and stored test hashes keep the same format.
+This release requires a fresh database. Set `dbPath` to a new file and run the
+suite again. Old database formats are not upgraded automatically.
 
 ### Installation
 
@@ -46,13 +47,21 @@ Define your test suite:
 // my-test-suite.ts
 import {Classify} from '@creative-introvert/tons-of-tests';
 import * as CLI from '@creative-introvert/tons-of-tests-cli';
-import {Effect} from 'effect';
+import {Effect, Schema} from 'effect';
 
 const myFunction = (input: number) => Promise.resolve(input * 1.7);
+const classify = Classify.makeClassify<number, number>({
+    // Treat an absolute error of up to 0.4 as a match.
+    isEqual: (a, b) => Math.abs(b - a) <= 0.4,
+});
 
 CLI.run({
     testSuite: {
         name: 'with-cli-simple',
+        schemas: {
+            testCase: {input: Schema.Finite, expected: Schema.Finite},
+            evaluation: {result: Schema.Finite, expected: Schema.Finite},
+        },
         testCases: [
             {input: 0, expected: 0},
             {input: 1, expected: 2},
@@ -60,12 +69,9 @@ CLI.run({
             {input: 3, expected: 4},
             {input: 4, expected: 5},
         ],
-        // `makeClassify` customises how observed vs. expected results are
-        // labelled. `isEqual` decides TP/TN vs FP/FN — here we tolerate a
-        // 0.4 absolute error, so `1.7` passes for `2`, etc.
-        classify: Classify.makeClassify({
-            isEqual: (a, b) => Math.abs(b - a) <= 0.4,
-        }),
+        evaluate: (result, expected) => [
+            {key: 'result', result, expected, label: classify(result, expected)},
+        ],
         program: (input: number) => Effect.promise(() => myFunction(input)),
     },
     dbPath: 'with-cli-simple.db',
@@ -73,7 +79,18 @@ CLI.run({
 });
 ```
 
-> Omit `classify` to get default strict equality; then every non-exact result is labelled FP.
+Omit `evaluate` to compare the whole output with the expected value using
+`Schema.toEquivalence`. Reuse the same schema instance for
+`schemas.evaluation.result` and `schemas.evaluation.expected`. The default
+evaluator returns one result with key `result`. It treats `null` and `undefined`
+as missing values for classification.
+
+`Classify.makeClassify` returns a label for one comparison. Use it inside
+`evaluate` to choose other matching rules or compare values of different types.
+
+The runner keeps the first case for each input/expected-value hash. It warns
+and skips later duplicates before calling `program`, then continues the suite.
+Tags do not change a case's identity; the first case's tags are kept.
 
 `testCases` can also be an Effect `Stream`, which is useful for large or generated suites that are consumed lazily by the runner:
 
@@ -83,6 +100,10 @@ import {Stream} from 'effect';
 CLI.run({
     testSuite: {
         name: 'with-cli-stream',
+        schemas: {
+            testCase: {input: Schema.Finite, expected: Schema.Finite},
+            evaluation: {result: Schema.Finite, expected: Schema.Finite},
+        },
         testCases: Stream.fromIterable([
             {input: 0, expected: 0},
             {input: 1, expected: 2},
@@ -93,99 +114,211 @@ CLI.run({
 });
 ```
 
-#### Summarize
+### Multiple Results From One Test Case
+
+Use `evaluate` to track parts of one program output separately. The runner calls
+`program` once per test case, then passes its output and the case's expected
+value to `evaluate`.
 
+For a BANT inference, the evaluator can return a result for each of budget,
+authority, need, and timeline:
+
+```ts
+import {Classify, type Test} from '@creative-introvert/tons-of-tests';
+import {Array, Schema} from 'effect';
+
+const Field = Schema.NullOr(Schema.String);
+const Bant = Schema.Struct({
+    budget: Field,
+    authority: Field,
+    need: Field,
+    timeline: Field,
+});
+type Bant = typeof Bant.Type;
+
+const fields = ['budget', 'authority', 'need', 'timeline'] as const;
+const classify = Classify.makeClassify({isEqual: Schema.toEquivalence(Field)});
+
+const evaluate: Test.Evaluate<Bant, Bant, string | null, string | null> = (
+    output,
+    expected,
+) =>
+    Array.map(fields, key => ({
+        key,
+        result: output[key],
+        expected: expected[key],
+        label: classify(output[key], expected[key]),
+    }));
 ```
-pnpx tsx my-test-suite.ts summarize
 
-┌───────────────────────────────────────────────────────────────────────────┐
-│ SUMMARY                                                                   │
-├─────┬──────────┬────────┬──────┬───────┬──────────┬────────┬──────────────┤
-│ #/∑ │ hash     │ ms     │ tags │ input │ expected │ label₀ │ diff result₀ │
-├─────┼──────────┼────────┼──────┼───────┼──────────┼────────┼──────────────┤
-│ 1/5 │ bd04cb2c │ 0.69ms │      │ 0     │ 0        │ TP     │              │
-├─────┼──────────┼────────┼──────┼───────┼──────────┼────────┼──────────────┤
-│ 2/5 │ 562e2cca │ 0.36ms │      │ 1     │ 2        │ TP     │ 2 => 1.7     │
-├─────┼──────────┼────────┼──────┼───────┼──────────┼────────┼──────────────┤
-│ 3/5 │ a5afd52f │ 3.30ms │      │ 2     │ 3        │ TP     │ 3 => 3.4     │
-├─────┼──────────┼────────┼──────┼───────┼──────────┼────────┼──────────────┤
-│ 4/5 │ 5f7f8725 │ 1.19ms │      │ 3     │ 4        │ FP     │ 4 => 5.1     │
-├─────┼──────────┼────────┼──────┼───────┼──────────┼────────┼──────────────┤
-│ 5/5 │ 6cc26923 │ 1.22ms │      │ 4     │ 5        │ FP     │ 5 => 6.8     │
-├─────┼──────────┼────────┼──────┼───────┼──────────┼────────┼──────────────┤
-│ #/∑ │ hash     │ ms     │ tags │ input │ expected │ label₀ │ diff result₀ │
-└─────┴──────────┴────────┴──────┴───────┴──────────┴────────┴──────────────┘
-
-
-┌────────────────────────────────────────────────────────────────────┐
-│ STATS                                                              │
-├───┬────┬────┬────┬────┬───────────┬────────┬──────────┬────────────┤
-│ ∑ │ TP │ TN │ FP │ FN │ precision │ recall │ timeMean │ timeMedian │
-├───┼────┼────┼────┼────┼───────────┼────────┼──────────┼────────────┤
-│ 5 │ 3  │ 0  │ 2  │ 0  │ 0.60      │ 1.00   │ 1.35ms   │ 1.19ms     │
-├───┼────┼────┼────┼────┼───────────┼────────┼──────────┼────────────┤
-│ ∑ │ TP │ TN │ FP │ FN │ precision │ recall │ timeMean │ timeMedian │
-└───┴────┴────┴────┴────┴───────────┴────────┴──────────┴────────────┘
+Pass `evaluate` alongside `program` in the test suite. See the runnable
+[multiple-results.ts example](./workspace/examples/src/with-cli/multiple-results.ts)
+for a complete suite using a fixed response in place of an LLM call.
+
+`evaluate` is a pure function. It must return a non-empty readonly array of
+`Test.EvaluationResult` values. Each value has a `key`, `result`, `expected`, and
+`label`. Keys must be non-empty and unique within the case. Keep keys stable
+across runs so the CLI can match each result to its previous value.
+
+The CLI shows one row per result in the existing table. Rows from the same case
+stay next to each other and share the same `hash`. The `key` column identifies
+each result. The hash comes from the original input and expected value; each
+result is matched across runs by that hash and its key.
+
+For the BANT example, these columns show the shared case hash (other columns
+are omitted here):
+
+```text
+┌──────────┬───────────┬───────┐
+│ hash     │ key       │ label │
+├──────────┼───────────┼───────┤
+│ 688c8d1f │ budget    │ FP    │
+├──────────┼───────────┼───────┤
+│ 688c8d1f │ authority │ TP    │
+├──────────┼───────────┼───────┤
+│ 688c8d1f │ need      │ TP    │
+├──────────┼───────────┼───────┤
+│ 688c8d1f │ timeline  │ FN    │
+└──────────┴───────────┴───────┘
 ```
 
-#### Summarize With Labels Filter
+The statistics count cases and results separately. Labels, precision, and recall
+count visible results. Timing statistics count each visible case once, even when
+it has several result rows.
 
-```
-pnpx tsx my-test-suite.ts summarize --labels TP
+This suite needs a schema for the whole BANT expectation and schemas for the
+individual field values:
 
-┌───────────────────────────────────────────────────────────────────────────┐
-│ SUMMARY                                                                   │
-├─────┬──────────┬────────┬──────┬───────┬──────────┬────────┬──────────────┤
-│ #/∑ │ hash     │ ms     │ tags │ input │ expected │ label₀ │ diff result₀ │
-├─────┼──────────┼────────┼──────┼───────┼──────────┼────────┼──────────────┤
-│ 1/3 │ bd04cb2c │ 0.52ms │      │ 0     │ 0        │ TP     │              │
-├─────┼──────────┼────────┼──────┼───────┼──────────┼────────┼──────────────┤
-│ 2/3 │ 562e2cca │ 0.41ms │      │ 1     │ 2        │ TP     │ 2 => 1.7     │
-├─────┼──────────┼────────┼──────┼───────┼──────────┼────────┼──────────────┤
-│ 3/3 │ a5afd52f │ 0.74ms │      │ 2     │ 3        │ TP     │ 3 => 3.4     │
-├─────┼──────────┼────────┼──────┼───────┼──────────┼────────┼──────────────┤
-│ #/∑ │ hash     │ ms     │ tags │ input │ expected │ label₀ │ diff result₀ │
-└─────┴──────────┴────────┴──────┴───────┴──────────┴────────┴──────────────┘
-
-
-┌────────────────────────────────────────────────────────────────────┐
-│ STATS                                                              │
-├───┬────┬────┬────┬────┬───────────┬────────┬──────────┬────────────┤
-│ ∑ │ TP │ TN │ FP │ FN │ precision │ recall │ timeMean │ timeMedian │
-├───┼────┼────┼────┼────┼───────────┼────────┼──────────┼────────────┤
-│ 5 │ 3  │ 0  │ 2  │ 0  │ 0.60      │ 1.00   │ 1.23ms   │ 1.28ms     │
-├───┼────┼────┼────┼────┼───────────┼────────┼──────────┼────────────┤
-│ ∑ │ TP │ TN │ FP │ FN │ precision │ recall │ timeMean │ timeMedian │
-└───┴────┴────┴────┴────┴───────────┴────────┴──────────┴────────────┘
+```ts
+const schemas = {
+    testCase: {input: Schema.String, expected: Bant},
+    evaluation: {result: Field, expected: Field},
+};
 ```
 
-#### Summarize With Tag Filters
+### Value Schemas
+
+Every suite requires `schemas`. The fields describe the values used at each
+stage:
+
+| Field | Value |
+| --- | --- |
+| `testCase.input` | Original input passed to `program` and saved with each result. |
+| `testCase.expected` | Whole expected value passed to `evaluate` and used in the case hash. |
+| `evaluation.result` | Result returned by `evaluate`. |
+| `evaluation.expected` | Expected value returned by `evaluate`. |
+
+The runner derives JSON codecs with Effect's `Schema.toCodecJson`. Built-in
+schemas such as `Schema.BigInt` and `Schema.Date` provide JSON encodings.
+Custom codecs can choose other encodings. Their encoded and decoded types may
+differ. Schemas must not require Effect services.
+
+`program`, `evaluate`, and fresh results keep the original values. Saved values
+are encoded and loaded values are decoded with the same schemas. The whole
+program output needs no storage schema when `evaluate` selects smaller values.
 
-`--tags` includes results matching ANY of the listed tags (OR):
+Encodings must be deterministic. The runner rejects undeclared object fields
+and checks that decoding the JSON text restores an equivalent value according
+to `Schema.toEquivalence`. These checks run before hashing or saving. For
+example, an untagged bigint/string union can encode both `123n` and `"123"` as
+the same JSON string. Use an encoding that records which type it contains.
 
+Case hashes cover the encoded original input and whole expected value.
+Result IDs also cover the evaluation key, label, and encoded result values.
+Object keys are sorted; array order remains significant. Changing an encoding
+can change hashes. Existing data is not migrated.
+
+Tables and visual diffs display encoded JSON. Comparisons between runs use
+schema equality on decoded values. Saved labels are not re-evaluated.
+
+Conversion failures use `Error.TestValueError`. It includes the suite,
+one-based case number, evaluation key when available, field, operation, and
+original `SchemaError` cause. A failed CLI execution stays incomplete and cannot
+be committed or reused with `--cached`.
+
+Direct repository calls also require schemas:
+
+```ts
+yield* repository.withTestRun(
+    run,
+    repository.insertTestResults(results, run, schemas),
+);
+const saved = repository.getTestResultsStream(run, schemas);
 ```
-pnpx tsx my-test-suite.ts summarize --tags foo,bar
+
+`Show.summarize({testRun, schemas})` returns an Effect containing the table text.
+`Show.prepareSummary(results, {schemas, suite})` prepares table chunks inside
+`Effect.scoped`. Both report conversion failures in the Effect error channel.
+The schema-free `Classify.defaultIsEqual` and `Test.makeSha256` exports have
+been removed.
+
+### CLI Commands
+
+The CLI reads and compares results in batches. It writes formatted rows to a
+temporary file, then reads them back to print the table with the correct column
+widths. Timing statistics use temporary SQLite tables, including the exact
+median. Temporary files and tables are removed after use or failure.
+
+This avoids keeping both full runs and the full table text in JavaScript memory.
+The runner still keeps one hash per distinct case to detect duplicates. Arrays
+you supply as `testCases` also stay in memory; use a stream to read cases as
+needed. Temporary storage grows with the report size. The existing commands,
+flags, table layout, and statistics stay the same.
+
+#### Summarize
+
+```sh
+pnpx tsx my-test-suite.ts summarize
 ```
 
-`--all-tags` includes results matching ALL listed tags (AND):
+#### Filter Results
+
+Both `summarize` and `diff` accept these filters:
+
+| Flag | Includes a result when… |
+| --- | --- |
+| `--keys budget,need` | Its key is `budget` or `need`. |
+| `--labels FP,FN` | Its current label is `FP` or `FN`. |
+| `--tags enterprise,german` | Its case has either tag. |
+| `--all-tags enterprise,german` | Its case has both tags. |
+
+Different flags combine with AND. For example, show budget errors from
+enterprise cases:
 
+```sh
+pnpx tsx bant-suite.ts summarize --keys budget --labels FP,FN --tags enterprise
 ```
-pnpx tsx my-test-suite.ts summarize --all-tags foo,bar
+
+For the basic example, show matching results:
+
+```sh
+pnpx tsx my-test-suite.ts summarize --labels TP
 ```
 
-Both flags compose with `--labels`.
+Filters change the displayed rows and statistics. The CLI runs and saves all
+evaluations before applying them, so filtering does not discard saved results
+or reduce the number of program calls. No matches produces an empty view;
+`summarize` also reports zero counts.
 
 #### Summarize From Cache
 
-`--cached` reads the most recently inserted test results from the local DB
-instead of re-running the program. This is useful when you want to re-format
-or re-filter without incurring the cost of a full run:
+After the whole suite finishes successfully, `--cached` reads its saved results
+without re-running the program or evaluator. This also works for an empty
+suite. If the previous attempt failed or was interrupted, the CLI clears its
+partial results and runs the suite again.
 
-```
-pnpx tsx my-test-suite.ts summarize --cached
+Use the cache to inspect other keys without making another inference call:
+
+```sh
+pnpx tsx bant-suite.ts summarize --cached --keys budget,need
 ```
 
 `diff --cached` behaves the same way for the current-run side of the diff.
+The cache belongs to the suite name. It does not detect changes to the program
+or test cases; omit `--cached` when you want to run them again.
+
+After upgrading, use a fresh database and create a new baseline. See
+[database setup](./MIGRATION_GUIDE_V1.md#database-setup).
 
 #### Write Test Results
 
@@ -193,9 +326,14 @@ pnpx tsx my-test-suite.ts summarize --cached
 pnpx tsx my-test-suite.ts commit
 ```
 
+`commit` saves the current results as the baseline for later comparisons.
+It rejects a tracked run that failed or is still running. Changing the
+baseline and creating the next current run happen in one database transaction.
+
 #### Diff
 
 The diff command compares your current test results with previously committed results. This is useful when you've made changes to either:
+
 - Your test suite's inputs or expected values
 - The implementation of the function being tested
 
@@ -205,67 +343,40 @@ index 21cd713..ab1b6dc 100644
 --- a/my-test-suite.ts
 +++ b/my-test-suite.ts
 @@ -1,7 +1,7 @@
- import * as CLI from '@creative-introvert/tots-cli';
+ import * as CLI from '@creative-introvert/tons-of-tests-cli';
  import {Effect} from 'effect';
  
 -const myFunction = (input: number) => Promise.resolve(input * 1.7);
 +const myFunction = (input: number) => Promise.resolve(input * 2);
  
- void CLI.run({
+ CLI.run({
      testSuite: {
 ```
 
-```
+```sh
 pnpx tsx my-test-suite.ts diff
-# or, if you want it to process.exit(1) on diff
+# Exit with code 1 when the filtered view contains differences.
 pnpx tsx my-test-suite.ts diff --exit-on-diff
-
-┌─────────────────────────────────────────────────────────────────────────────────────────────────────┐
-│ SUMMARY                                                                                             │
-├─────┬──────────┬────────┬──────┬───────┬──────────┬────────┬──────────────┬─────────┬───────────────┤
-│ #/∑ │ hash     │ ms     │ tags │ input │ expected │ label₀ │ diff result₀ │ label₋₁ │ diff result₋₁ │
-├─────┼──────────┼────────┼──────┼───────┼──────────┼────────┼──────────────┼─────────┼───────────────┤
-│ 1/4 │ 562e2cca │ 0.23ms │      │ 1     │ 2        │ TP     │              │ FP      │ 2 => 1.7      │
-├─────┼──────────┼────────┼──────┼───────┼──────────┼────────┼──────────────┼─────────┼───────────────┤
-│ 2/4 │ a5afd52f │ 3.66ms │      │ 2     │ 3        │ FP     │ 3 => 4       │ FP      │ 3 => 3.4      │
-├─────┼──────────┼────────┼──────┼───────┼──────────┼────────┼──────────────┼─────────┼───────────────┤
-│ 3/4 │ 5f7f8725 │ 1.19ms │      │ 3     │ 4        │ FP     │ 4 => 6       │ FP      │ 4 => 5.1      │
-├─────┼──────────┼────────┼──────┼───────┼──────────┼────────┼──────────────┼─────────┼───────────────┤
-│ 4/4 │ 6cc26923 │ 0.98ms │      │ 4     │ 5        │ FP     │ 5 => 8       │ FP      │ 5 => 6.8      │
-├─────┼──────────┼────────┼──────┼───────┼──────────┼────────┼──────────────┼─────────┼───────────────┤
-│ #/∑ │ hash     │ ms     │ tags │ input │ expected │ label₀ │ diff result₀ │ label₋₁ │ diff result₋₁ │
-└─────┴──────────┴────────┴──────┴───────┴──────────┴────────┴──────────────┴─────────┴───────────────┘
-
-
-┌────────────────────────────────────────────────────────────────────┐
-│ STATS                                                              │
-├───┬────┬────┬────┬────┬───────────┬────────┬──────────┬────────────┤
-│ ∑ │ TP │ TN │ FP │ FN │ precision │ recall │ timeMean │ timeMedian │
-├───┼────┼────┼────┼────┼───────────┼────────┼──────────┼────────────┤
-│ 5 │ 2  │ 0  │ 3  │ 0  │ 0.40      │ 1.00   │ 1.30ms   │ 0.98ms     │
-├───┼────┼────┼────┼────┼───────────┼────────┼──────────┼────────────┤
-│ ∑ │ TP │ TN │ FP │ FN │ precision │ recall │ timeMean │ timeMedian │
-└───┴────┴────┴────┴────┴───────────┴────────┴──────────┴────────────┘
-
-
-┌────────────────────────────────────────┐
-│ DIFF                                   │
-├────┬────┬────┬────┬───────────┬────────┤
-│ TP │ TN │ FP │ FN │ precision │ recall │
-├────┼────┼────┼────┼───────────┼────────┤
-│ 1  │ 0  │ -1 │ 0  │ 0.20      │ 0.00   │
-├────┼────┼────┼────┼───────────┼────────┤
-│ TP │ TN │ FP │ FN │ precision │ recall │
-└────┴────┴────┴────┴───────────┴────────┘
 ```
 
-Checkout `workspace/examples/src/with-cli` for more examples.
+For a suite with several results per case, select the keys to compare:
 
+```sh
+pnpx tsx bant-suite.ts diff --keys budget --labels FP,FN
 ```
-pnpx tsx <file-path>
-# e.g.
-pnpx tsx ./workspace/examples/src/with-cli/simple-run.ts
-pnpx tsx ./workspace/examples/src/with-cli/simple-effect.ts
+
+`diff` matches results by case hash and key before applying filters. It includes
+new results and changes to a result's value, expected value, or label. A current
+`FP` result keeps its previous value and label even when that previous label
+does not match `--labels`.
+
+See [workspace/examples/src/with-cli](./workspace/examples/src/with-cli) for
+more examples. From the repository root, run:
+
+```sh
+direnv exec . pnpm exec tsx workspace/examples/src/with-cli/simple-run.ts summarize
+direnv exec . pnpm exec tsx workspace/examples/src/with-cli/simple-effect.ts
+direnv exec . pnpm exec tsx workspace/examples/src/with-cli/multiple-results.ts summarize --keys budget,need
 ```
 
 ### Using the CLI programmatically
@@ -305,6 +416,30 @@ CLI.effect(
 );
 ```
 
+Each `CLI.effect` call opens and closes its own database connection. To share
+one repository across several commands, use `CLI.effectNoDeps` and provide
+the services once around the whole program. This also lets an in-memory
+database keep its results between commands:
+
+```ts
+import * as CLI from '@creative-introvert/tons-of-tests-cli';
+import {NodeRuntime} from '@effect/platform-node';
+import {Effect} from 'effect';
+
+const config = {testSuite, dbPath: ':memory:'};
+
+Effect.gen(function* () {
+    yield* CLI.effectNoDeps(config, ['node', 'script', 'summarize']);
+    yield* CLI.effectNoDeps(config, ['node', 'script', 'commit']);
+}).pipe(
+    Effect.provide(CLI.buildLayer(config)),
+    NodeRuntime.runMain,
+);
+```
+
+`effectNoDeps` uses the supplied platform services and `TestRepository`.
+Its `config.dbPath` is only used if you pass the config to `buildLayer`.
+
 If you need the last committed test-run hash from inside another script, use
 the dedicated helper:
 
@@ -318,16 +453,10 @@ const hash = await CLI.getLastTestRunHash({
 // hash: string | null
 ```
 
-If you compose the CLI's `diff` programmatically (via `_diff` or a custom
-harness), `diff --exit-on-diff` fails with a typed `DiffNonEmpty` error
-when the current run has visible diffs:
-
-```ts
-import {DiffNonEmpty} from '@creative-introvert/tons-of-tests-cli';
-
-// ... compose _diff into your own Effect pipeline, then:
-Effect.catchTag('DiffNonEmpty', () => /* handle */);
-```
+When you use `CLI.effect` or `CLI.effectNoDeps`, `diff --exit-on-diff` fails with a typed
+`DiffNonEmpty` error when the current run has visible diffs. Handle it with
+`Effect.catchTag('DiffNonEmpty', handler)`; `error.count` gives the number of
+visible changed results.
 
 At the process entry point (`CLI.run`), `DiffNonEmpty` is translated to
 `process.exitCode = 1` — the user-facing "exit with code 1 on diff"

package/dist/cjs/Classify.js

@@ -3,22 +3,25 @@
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
-exports.values = exports.makeClassify = exports.defaultIsNil = exports.defaultIsEqual = exports.Stats = exports.LabelSchema = void 0;
+exports.values = exports.makeClassify = exports.defaultIsNil = exports.Stats = exports.LabelSchema = void 0;
 var _effect = /*#__PURE__*/require("effect");
 var internal = /*#__PURE__*/_interopRequireWildcard(/*#__PURE__*/require("./internal/Classify.js"));
 var cache;
 function _interopRequireWildcard(e, t) { if (!t && e && e.__esModule) return e; var r, c, a = Object.defineProperty, n = { __proto__: null, default: e }; if (Object(e) !== e) return n; if (cache || "function" != typeof WeakMap || (cache = new WeakMap()), cache) { if (cache.has(e)) return cache.get(e); cache.set(e, n); } for (c in e) "default" !== c && {}.hasOwnProperty.call(e, c) && ((r = a && Object.getOwnPropertyDescriptor(e, c)) && (r.get || r.set) ? a(n, c, r) : n[c] = e[c]); return n; }
 const LabelSchema = exports.LabelSchema = internal.LabelSchema;
 const values = exports.values = internal.values;
-const defaultIsEqual = exports.defaultIsEqual = internal.defaultIsEqual;
 const defaultIsNil = exports.defaultIsNil = internal.defaultIsNil;
 const makeClassify = exports.makeClassify = internal.makeClassify;
+/** Counts selected results; timing statistics count each selected case once. */
 class Stats extends /*#__PURE__*/_effect.Schema.Class('Stats')({
   TP: _effect.Schema.Finite,
   TN: _effect.Schema.Finite,
   FP: _effect.Schema.Finite,
   FN: _effect.Schema.Finite,
+  /** Number of evaluation results. */
   total: _effect.Schema.Finite,
+  /** Number of distinct original test cases represented by those results. */
+  testCases: _effect.Schema.Int,
   precision: _effect.Schema.Finite,
   recall: _effect.Schema.Finite,
   timeMin: /*#__PURE__*/_effect.Schema.Option(_effect.Schema.Finite),
@@ -33,6 +36,7 @@ class Stats extends /*#__PURE__*/_effect.Schema.Class('Stats')({
       FP: 0,
       FN: 0,
       total: 0,
+      testCases: 0,
       precision: 0,
       recall: 0,
       timeMin: _effect.Option.none(),

package/dist/cjs/Classify.js.map

@@ -1 +1 @@
-{"version":3,"file":"Classify.js","names":["_effect","require","internal","_interopRequireWildcard","cache","e","t","__esModule","r","c","a","Object","defineProperty","n","__proto__","default","WeakMap","has","get","set","hasOwnProperty","call","getOwnPropertyDescriptor","LabelSchema","exports","values","defaultIsEqual","defaultIsNil","makeClassify","Stats","Schema","Class","TP","Finite","TN","FP","FN","total","precision","recall","timeMin","Option","timeMax","timeMedian","timeMean","empty","none"],"sources":["../../src/Classify.ts"],"sourcesContent":[null],"mappings":";;;;;;AAAA,IAAAA,OAAA,gBAAAC,OAAA;AAEA,IAAAC,QAAA,gBAAAC,uBAAA,cAAAF,OAAA;AAAmD,IAAAG,KAAA;AAAA,SAAAD,wBAAAE,CAAA,EAAAC,CAAA,SAAAA,CAAA,IAAAD,CAAA,IAAAA,CAAA,CAAAE,UAAA,SAAAF,CAAA,MAAAG,CAAA,EAAAC,CAAA,EAAAC,CAAA,GAAAC,MAAA,CAAAC,cAAA,EAAAC,CAAA,KAAAC,SAAA,QAAAC,OAAA,EAAAV,CAAA,QAAAM,MAAA,CAAAN,CAAA,MAAAA,CAAA,SAAAQ,CAAA,MAAAT,KAAA,yBAAAY,OAAA,KAAAZ,KAAA,OAAAY,OAAA,KAAAZ,KAAA,QAAAA,KAAA,CAAAa,GAAA,CAAAZ,CAAA,UAAAD,KAAA,CAAAc,GAAA,CAAAb,CAAA,GAAAD,KAAA,CAAAe,GAAA,CAAAd,CAAA,EAAAQ,CAAA,UAAAJ,CAAA,IAAAJ,CAAA,gBAAAI,CAAA,OAAAW,cAAA,CAAAC,IAAA,CAAAhB,CAAA,EAAAI,CAAA,OAAAD,CAAA,GAAAE,CAAA,IAAAC,MAAA,CAAAW,wBAAA,CAAAjB,CAAA,EAAAI,CAAA,OAAAD,CAAA,CAAAU,GAAA,IAAAV,CAAA,CAAAW,GAAA,IAAAT,CAAA,CAAAG,CAAA,EAAAJ,CAAA,EAAAD,CAAA,IAAAK,CAAA,CAAAJ,CAAA,IAAAJ,CAAA,CAAAI,CAAA,WAAAI,CAAA;AAE5C,MAAMU,WAAW,GAAAC,OAAA,CAAAD,WAAA,GAAGrB,QAAQ,CAACqB,WAAW;AAGxC,MAAME,MAAM,GAAAD,OAAA,CAAAC,MAAA,GAAGvB,QAAQ,CAACuB,MAAM;AAC9B,MAAMC,cAAc,GAAAF,OAAA,CAAAE,cAAA,GAAGxB,QAAQ,CAACwB,cAAc;AAC9C,MAAMC,YAAY,GAAAH,OAAA,CAAAG,YAAA,GAAGzB,QAAQ,CAACyB,YAAY;AAC1C,MAAMC,YAAY,GAAAJ,OAAA,CAAAI,YAAA,GAAG1B,QAAQ,CAAC0B,YAAY;AAI3C,MAAOC,KAAM,sBAAQC,cAAM,CAACC,KAAK,CAAQ,OAAO,CAAC,CAAC;EACpDC,EAAE,EAAEF,cAAM,CAACG,MAAM;EACjBC,EAAE,EAAEJ,cAAM,CAACG,MAAM;EACjBE,EAAE,EAAEL,cAAM,CAACG,MAAM;EACjBG,EAAE,EAAEN,cAAM,CAACG,MAAM;EACjBI,KAAK,EAAEP,cAAM,CAACG,MAAM;EACpBK,SAAS,EAAER,cAAM,CAACG,MAAM;EACxBM,MAAM,EAAET,cAAM,CAACG,MAAM;EACrBO,OAAO,eAAEV,cAAM,CAACW,MAAM,CAACX,cAAM,CAACG,MAAM,CAAC;EACrCS,OAAO,eAAEZ,cAAM,CAACW,MAAM,CAACX,cAAM,CAACG,MAAM,CAAC;EACrCU,UAAU,eAAEb,cAAM,CAACW,MAAM,CAACX,cAAM,CAACG,MAAM,CAAC;EACxCW,QAAQ,eAAEd,cAAM,CAACW,MAAM,CAACX,cAAM,CAACG,MAAM;CACxC,CAAC;EACE,OAAOY,KAAKA,CAAA;IACR,OAAO,IAAIhB,KAAK,CAAC;MACbG,EAAE,EAAE,CAAC;MACLE,EAAE,EAAE,CAAC;MACLC,EAAE,EAAE,CAAC;MACLC,EAAE,EAAE,CAAC;MACLC,KAAK,EAAE,CAAC;MACRC,SAAS,EAAE,CAAC;MACZC,MAAM,EAAE,CAAC;MACTC,OAAO,EAAEC,cAAM,CAACK,IAAI,EAAE;MACtBJ,OAAO,EAAED,cAAM,CAACK,IAAI,EAAE;MACtBH,UAAU,EAAEF,cAAM,CAACK,IAAI,EAAE;MACzBF,QAAQ,EAAEH,cAAM,CAACK,IAAI;KACxB,CAAC;EACN;;AACHtB,OAAA,CAAAK,KAAA,GAAAA,KAAA","ignoreList":[]}
+{"version":3,"file":"Classify.js","names":["_effect","require","internal","_interopRequireWildcard","cache","e","t","__esModule","r","c","a","Object","defineProperty","n","__proto__","default","WeakMap","has","get","set","hasOwnProperty","call","getOwnPropertyDescriptor","LabelSchema","exports","values","defaultIsNil","makeClassify","Stats","Schema","Class","TP","Finite","TN","FP","FN","total","testCases","Int","precision","recall","timeMin","Option","timeMax","timeMedian","timeMean","empty","none"],"sources":["../../src/Classify.ts"],"sourcesContent":[null],"mappings":";;;;;;AAAA,IAAAA,OAAA,gBAAAC,OAAA;AAEA,IAAAC,QAAA,gBAAAC,uBAAA,cAAAF,OAAA;AAAmD,IAAAG,KAAA;AAAA,SAAAD,wBAAAE,CAAA,EAAAC,CAAA,SAAAA,CAAA,IAAAD,CAAA,IAAAA,CAAA,CAAAE,UAAA,SAAAF,CAAA,MAAAG,CAAA,EAAAC,CAAA,EAAAC,CAAA,GAAAC,MAAA,CAAAC,cAAA,EAAAC,CAAA,KAAAC,SAAA,QAAAC,OAAA,EAAAV,CAAA,QAAAM,MAAA,CAAAN,CAAA,MAAAA,CAAA,SAAAQ,CAAA,MAAAT,KAAA,yBAAAY,OAAA,KAAAZ,KAAA,OAAAY,OAAA,KAAAZ,KAAA,QAAAA,KAAA,CAAAa,GAAA,CAAAZ,CAAA,UAAAD,KAAA,CAAAc,GAAA,CAAAb,CAAA,GAAAD,KAAA,CAAAe,GAAA,CAAAd,CAAA,EAAAQ,CAAA,UAAAJ,CAAA,IAAAJ,CAAA,gBAAAI,CAAA,OAAAW,cAAA,CAAAC,IAAA,CAAAhB,CAAA,EAAAI,CAAA,OAAAD,CAAA,GAAAE,CAAA,IAAAC,MAAA,CAAAW,wBAAA,CAAAjB,CAAA,EAAAI,CAAA,OAAAD,CAAA,CAAAU,GAAA,IAAAV,CAAA,CAAAW,GAAA,IAAAT,CAAA,CAAAG,CAAA,EAAAJ,CAAA,EAAAD,CAAA,IAAAK,CAAA,CAAAJ,CAAA,IAAAJ,CAAA,CAAAI,CAAA,WAAAI,CAAA;AAE5C,MAAMU,WAAW,GAAAC,OAAA,CAAAD,WAAA,GAAGrB,QAAQ,CAACqB,WAAW;AAGxC,MAAME,MAAM,GAAAD,OAAA,CAAAC,MAAA,GAAGvB,QAAQ,CAACuB,MAAM;AAC9B,MAAMC,YAAY,GAAAF,OAAA,CAAAE,YAAA,GAAGxB,QAAQ,CAACwB,YAAY;AAC1C,MAAMC,YAAY,GAAAH,OAAA,CAAAG,YAAA,GAAGzB,QAAQ,CAACyB,YAAY;AAIjD;AACM,MAAOC,KAAM,sBAAQC,cAAM,CAACC,KAAK,CAAQ,OAAO,CAAC,CAAC;EACpDC,EAAE,EAAEF,cAAM,CAACG,MAAM;EACjBC,EAAE,EAAEJ,cAAM,CAACG,MAAM;EACjBE,EAAE,EAAEL,cAAM,CAACG,MAAM;EACjBG,EAAE,EAAEN,cAAM,CAACG,MAAM;EACjB;EACAI,KAAK,EAAEP,cAAM,CAACG,MAAM;EACpB;EACAK,SAAS,EAAER,cAAM,CAACS,GAAG;EACrBC,SAAS,EAAEV,cAAM,CAACG,MAAM;EACxBQ,MAAM,EAAEX,cAAM,CAACG,MAAM;EACrBS,OAAO,eAAEZ,cAAM,CAACa,MAAM,CAACb,cAAM,CAACG,MAAM,CAAC;EACrCW,OAAO,eAAEd,cAAM,CAACa,MAAM,CAACb,cAAM,CAACG,MAAM,CAAC;EACrCY,UAAU,eAAEf,cAAM,CAACa,MAAM,CAACb,cAAM,CAACG,MAAM,CAAC;EACxCa,QAAQ,eAAEhB,cAAM,CAACa,MAAM,CAACb,cAAM,CAACG,MAAM;CACxC,CAAC;EACE,OAAOc,KAAKA,CAAA;IACR,OAAO,IAAIlB,KAAK,CAAC;MACbG,EAAE,EAAE,CAAC;MACLE,EAAE,EAAE,CAAC;MACLC,EAAE,EAAE,CAAC;MACLC,EAAE,EAAE,CAAC;MACLC,KAAK,EAAE,CAAC;MACRC,SAAS,EAAE,CAAC;MACZE,SAAS,EAAE,CAAC;MACZC,MAAM,EAAE,CAAC;MACTC,OAAO,EAAEC,cAAM,CAACK,IAAI,EAAE;MACtBJ,OAAO,EAAED,cAAM,CAACK,IAAI,EAAE;MACtBH,UAAU,EAAEF,cAAM,CAACK,IAAI,EAAE;MACzBF,QAAQ,EAAEH,cAAM,CAACK,IAAI;KACxB,CAAC;EACN;;AACHvB,OAAA,CAAAI,KAAA,GAAAA,KAAA","ignoreList":[]}

package/dist/cjs/RepositoryError.js

@@ -3,42 +3,33 @@
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
-exports.TestRunNotCreated = exports.RepositoryWriteError = exports.RepositoryQueryError = exports.RepositoryDecodeError = void 0;
-var _SqlError = /*#__PURE__*/require("effect/sql/SqlError");
+exports.TestRunNotFound = exports.TestRunNotCreated = exports.TestRunIncomplete = exports.TestBaselineNotFound = void 0;
+Object.defineProperty(exports, "TestValueError", {
+  enumerable: true,
+  get: function () {
+    return _TestValueError.TestValueError;
+  }
+});
 var _effect = /*#__PURE__*/require("effect");
-const SqlCause = /*#__PURE__*/_effect.Schema.Union([/*#__PURE__*/_effect.Schema.instanceOf(_SqlError.SqlError), /*#__PURE__*/_effect.Schema.instanceOf(_SqlError.ResultLengthMismatch)]);
-class RepositoryQueryError extends /*#__PURE__*/_effect.Schema.TaggedError()('RepositoryQueryError', {
-  operation: _effect.Schema.String,
-  cause: /*#__PURE__*/_effect.Schema.instanceOf(_SqlError.SqlError)
-}) {
-  static fromSql = operation => cause => new RepositoryQueryError({
-    operation,
-    cause
-  });
-}
-exports.RepositoryQueryError = RepositoryQueryError;
-class RepositoryWriteError extends /*#__PURE__*/_effect.Schema.TaggedError()('RepositoryWriteError', {
-  operation: _effect.Schema.String,
-  cause: SqlCause
-}) {
-  static from = operation => cause => new RepositoryWriteError({
-    operation,
-    cause
-  });
-}
-exports.RepositoryWriteError = RepositoryWriteError;
-class RepositoryDecodeError extends /*#__PURE__*/_effect.Schema.TaggedError()('RepositoryDecodeError', {
-  operation: _effect.Schema.String,
-  cause: /*#__PURE__*/_effect.Schema.instanceOf(_effect.Schema.SchemaError)
-}) {
-  static from = operation => cause => new RepositoryDecodeError({
-    operation,
-    cause
-  });
-}
-exports.RepositoryDecodeError = RepositoryDecodeError;
+var _TestValueError = /*#__PURE__*/require("./TestValueError.js");
 class TestRunNotCreated extends /*#__PURE__*/_effect.Schema.TaggedError()('TestRunNotCreated', {
   testSuiteName: _effect.Schema.String
 }) {}
+/** A complete saved run is required for this operation. */
 exports.TestRunNotCreated = TestRunNotCreated;
+class TestRunIncomplete extends /*#__PURE__*/_effect.Schema.TaggedError()('TestRunIncomplete', {
+  testSuiteName: _effect.Schema.String,
+  testRunId: _effect.Schema.Int,
+  variant: _effect.Schema.String
+}) {}
+exports.TestRunIncomplete = TestRunIncomplete;
+class TestRunNotFound extends /*#__PURE__*/_effect.Schema.TaggedError()('TestRunNotFound', {
+  testRunId: _effect.Schema.Int
+}) {}
+exports.TestRunNotFound = TestRunNotFound;
+class TestBaselineNotFound extends /*#__PURE__*/_effect.Schema.TaggedError()('TestBaselineNotFound', {
+  testSuiteName: _effect.Schema.String,
+  variant: _effect.Schema.String
+}) {}
+exports.TestBaselineNotFound = TestBaselineNotFound;
 //# sourceMappingURL=RepositoryError.js.map