@creative-introvert/tons-of-tests 2.0.0 → 2.1.0

package/README.md

@@ -26,7 +26,8 @@ A specialized test runner for predictive functions, focusing on scenarios where
 
 Use Node.js 24 or newer and Effect 4. The Effect packages in this repository
 use the same exact version. The SQLite driver uses Node's built-in SQLite.
-Existing database files and stored test hashes keep the same format.
+Existing database files are migrated when opened. Previously stored results
+receive the key `result`; their test case hashes stay the same.
 
 ### Installation
 
@@ -49,6 +50,10 @@ import * as CLI from '@creative-introvert/tons-of-tests-cli';
 import {Effect} from 'effect';
 
 const myFunction = (input: number) => Promise.resolve(input * 1.7);
+const classify = Classify.makeClassify<number, number>({
+    // Treat an absolute error of up to 0.4 as a match.
+    isEqual: (a, b) => Math.abs(b - a) <= 0.4,
+});
 
 CLI.run({
     testSuite: {
@@ -60,12 +65,9 @@ CLI.run({
             {input: 3, expected: 4},
             {input: 4, expected: 5},
         ],
-        // `makeClassify` customises how observed vs. expected results are
-        // labelled. `isEqual` decides TP/TN vs FP/FN — here we tolerate a
-        // 0.4 absolute error, so `1.7` passes for `2`, etc.
-        classify: Classify.makeClassify({
-            isEqual: (a, b) => Math.abs(b - a) <= 0.4,
-        }),
+        evaluate: (result, expected) => [
+            {key: 'result', result, expected, label: classify(result, expected)},
+        ],
         program: (input: number) => Effect.promise(() => myFunction(input)),
     },
     dbPath: 'with-cli-simple.db',
@@ -73,7 +75,10 @@ CLI.run({
 });
 ```
 
-> Omit `classify` to get default strict equality; then every non-exact result is labelled FP.
+Omit `evaluate` to compare the whole output with the expected value using
+`Classify.defaultIsEqual`. The default evaluator returns one result with key
+`result`. `Classify.makeClassify` returns a label for one comparison; use it
+inside `evaluate` when you want to choose how values match.
 
 `testCases` can also be an Effect `Stream`, which is useful for large or generated suites that are consumed lazily by the runner:
 
@@ -93,96 +98,146 @@ CLI.run({
 });
 ```
 
-#### Summarize
+### Multiple Results From One Test Case
 
+Use `evaluate` to track parts of one program output separately. The runner calls
+`program` once per test case, then passes its output and the case's expected
+value to `evaluate`.
+
+For a BANT inference, the evaluator can return a result for each of budget,
+authority, need, and timeline:
+
+```ts
+import {Classify, type Test} from '@creative-introvert/tons-of-tests';
+import {Array} from 'effect';
+
+type Bant = {
+    readonly budget: string | null;
+    readonly authority: string | null;
+    readonly need: string | null;
+    readonly timeline: string | null;
+};
+
+const fields = ['budget', 'authority', 'need', 'timeline'] as const;
+const classify = Classify.makeClassify({isEqual: Classify.defaultIsEqual});
+
+const evaluate: Test.Evaluate<Bant, Bant, string | null, string | null> = (
+    output,
+    expected,
+) =>
+    Array.map(fields, key => ({
+        key,
+        result: output[key],
+        expected: expected[key],
+        label: classify(output[key], expected[key]),
+    }));
 ```
-pnpx tsx my-test-suite.ts summarize
 
-┌───────────────────────────────────────────────────────────────────────────┐
-│ SUMMARY                                                                   │
-├─────┬──────────┬────────┬──────┬───────┬──────────┬────────┬──────────────┤
-│ #/∑ │ hash     │ ms     │ tags │ input │ expected │ label₀ │ diff result₀ │
-├─────┼──────────┼────────┼──────┼───────┼──────────┼────────┼──────────────┤
-│ 1/5 │ bd04cb2c │ 0.69ms │      │ 0     │ 0        │ TP     │              │
-├─────┼──────────┼────────┼──────┼───────┼──────────┼────────┼──────────────┤
-│ 2/5 │ 562e2cca │ 0.36ms │      │ 1     │ 2        │ TP     │ 2 => 1.7     │
-├─────┼──────────┼────────┼──────┼───────┼──────────┼────────┼──────────────┤
-│ 3/5 │ a5afd52f │ 3.30ms │      │ 2     │ 3        │ TP     │ 3 => 3.4     │
-├─────┼──────────┼────────┼──────┼───────┼──────────┼────────┼──────────────┤
-│ 4/5 │ 5f7f8725 │ 1.19ms │      │ 3     │ 4        │ FP     │ 4 => 5.1     │
-├─────┼──────────┼────────┼──────┼───────┼──────────┼────────┼──────────────┤
-│ 5/5 │ 6cc26923 │ 1.22ms │      │ 4     │ 5        │ FP     │ 5 => 6.8     │
-├─────┼──────────┼────────┼──────┼───────┼──────────┼────────┼──────────────┤
-│ #/∑ │ hash     │ ms     │ tags │ input │ expected │ label₀ │ diff result₀ │
-└─────┴──────────┴────────┴──────┴───────┴──────────┴────────┴──────────────┘
-
-
-┌────────────────────────────────────────────────────────────────────┐
-│ STATS                                                              │
-├───┬────┬────┬────┬────┬───────────┬────────┬──────────┬────────────┤
-│ ∑ │ TP │ TN │ FP │ FN │ precision │ recall │ timeMean │ timeMedian │
-├───┼────┼────┼────┼────┼───────────┼────────┼──────────┼────────────┤
-│ 5 │ 3  │ 0  │ 2  │ 0  │ 0.60      │ 1.00   │ 1.35ms   │ 1.19ms     │
-├───┼────┼────┼────┼────┼───────────┼────────┼──────────┼────────────┤
-│ ∑ │ TP │ TN │ FP │ FN │ precision │ recall │ timeMean │ timeMedian │
-└───┴────┴────┴────┴────┴───────────┴────────┴──────────┴────────────┘
+Pass `evaluate` alongside `program` in the test suite. See the runnable
+[multiple-results.ts example](./workspace/examples/src/with-cli/multiple-results.ts)
+for a complete suite using a fixed response in place of an LLM call.
+
+`evaluate` is a pure function. It must return a non-empty readonly array of
+`Test.EvaluationResult` values. Each value has a `key`, `result`, `expected`, and
+`label`. Keys must be non-empty and unique within the case. Keep keys stable
+across runs so the CLI can match each result to its previous value.
+
+The CLI shows one row per result in the existing table. Rows from the same case
+stay next to each other and share the same `hash`. The `key` column identifies
+each result. The hash comes from the original input and expected value; each
+result is matched across runs by that hash and its key.
+
+For the BANT example, these columns show the shared case hash (other columns
+are omitted here):
+
+```text
+┌──────────┬───────────┬───────┐
+│ hash     │ key       │ label │
+├──────────┼───────────┼───────┤
+│ 688c8d1f │ budget    │ FP    │
+├──────────┼───────────┼───────┤
+│ 688c8d1f │ authority │ TP    │
+├──────────┼───────────┼───────┤
+│ 688c8d1f │ need      │ TP    │
+├──────────┼───────────┼───────┤
+│ 688c8d1f │ timeline  │ FN    │
+└──────────┴───────────┴───────┘
 ```
 
-#### Summarize With Labels Filter
+The statistics count cases and results separately. Labels, precision, and recall
+count visible results. Timing statistics count each visible case once, even when
+it has several result rows.
 
-```
-pnpx tsx my-test-suite.ts summarize --labels TP
+If you supply `testSuite.schemas`, `input` describes the original input.
+`result` and `expected` describe the values returned by `evaluate`. For this
+BANT example, both use `Schema.NullOr(Schema.String)` because each result is one
+field, even though the program returns a full BANT object.
+
+#### Updating Suites That Use `classify`
+
+Replace the suite's `classify` property with an `evaluate` callback. Keep the
+classifier and wrap its label in a result, as shown in the basic example:
 
-┌───────────────────────────────────────────────────────────────────────────┐
-│ SUMMARY                                                                   │
-├─────┬──────────┬────────┬──────┬───────┬──────────┬────────┬──────────────┤
-│ #/∑ │ hash     │ ms     │ tags │ input │ expected │ label₀ │ diff result₀ │
-├─────┼──────────┼────────┼──────┼───────┼──────────┼────────┼──────────────┤
-│ 1/3 │ bd04cb2c │ 0.52ms │      │ 0     │ 0        │ TP     │              │
-├─────┼──────────┼────────┼──────┼───────┼──────────┼────────┼──────────────┤
-│ 2/3 │ 562e2cca │ 0.41ms │      │ 1     │ 2        │ TP     │ 2 => 1.7     │
-├─────┼──────────┼────────┼──────┼───────┼──────────┼────────┼──────────────┤
-│ 3/3 │ a5afd52f │ 0.74ms │      │ 2     │ 3        │ TP     │ 3 => 3.4     │
-├─────┼──────────┼────────┼──────┼───────┼──────────┼────────┼──────────────┤
-│ #/∑ │ hash     │ ms     │ tags │ input │ expected │ label₀ │ diff result₀ │
-└─────┴──────────┴────────┴──────┴───────┴──────────┴────────┴──────────────┘
-
-
-┌────────────────────────────────────────────────────────────────────┐
-│ STATS                                                              │
-├───┬────┬────┬────┬────┬───────────┬────────┬──────────┬────────────┤
-│ ∑ │ TP │ TN │ FP │ FN │ precision │ recall │ timeMean │ timeMedian │
-├───┼────┼────┼────┼────┼───────────┼────────┼──────────┼────────────┤
-│ 5 │ 3  │ 0  │ 2  │ 0  │ 0.60      │ 1.00   │ 1.23ms   │ 1.28ms     │
-├───┼────┼────┼────┼────┼───────────┼────────┼──────────┼────────────┤
-│ ∑ │ TP │ TN │ FP │ FN │ precision │ recall │ timeMean │ timeMedian │
-└───┴────┴────┴────┴────┴───────────┴────────┴──────────┴────────────┘
+```ts
+evaluate: (result, expected) => [
+    {key: 'result', result, expected, label: classify(result, expected)},
+],
 ```
 
-#### Summarize With Tag Filters
+Suites that use the default comparison and omit `classify` need no changes.
+Cached results keep their saved values and labels. Run the suite again after
+changing `evaluate` to produce new results.
 
-`--tags` includes results matching ANY of the listed tags (OR):
+The same `schemas` decode current and previous runs. If you change the stored
+result or expected-value shape and your schemas cannot read older values, use
+a new suite name for the new shape.
 
+### CLI Commands
+
+#### Summarize
+
+```sh
+pnpx tsx my-test-suite.ts summarize
 ```
-pnpx tsx my-test-suite.ts summarize --tags foo,bar
-```
 
-`--all-tags` includes results matching ALL listed tags (AND):
+#### Filter Results
+
+Both `summarize` and `diff` accept these filters:
+
+| Flag | Includes a result when… |
+| --- | --- |
+| `--keys budget,need` | Its key is `budget` or `need`. |
+| `--labels FP,FN` | Its current label is `FP` or `FN`. |
+| `--tags enterprise,german` | Its case has either tag. |
+| `--all-tags enterprise,german` | Its case has both tags. |
+
+Different flags combine with AND. For example, show budget errors from
+enterprise cases:
 
+```sh
+pnpx tsx bant-suite.ts summarize --keys budget --labels FP,FN --tags enterprise
 ```
-pnpx tsx my-test-suite.ts summarize --all-tags foo,bar
+
+For the basic example, show matching results:
+
+```sh
+pnpx tsx my-test-suite.ts summarize --labels TP
 ```
 
-Both flags compose with `--labels`.
+Filters change the displayed rows and statistics. The CLI runs and saves all
+evaluations before applying them, so filtering does not discard saved results
+or reduce the number of program calls. No matches produces an empty view;
+`summarize` also reports zero counts.
 
 #### Summarize From Cache
 
-`--cached` reads the most recently inserted test results from the local DB
-instead of re-running the program. This is useful when you want to re-format
-or re-filter without incurring the cost of a full run:
+When results for the current run are already saved, `--cached` reads them
+without re-running the program or evaluator. If no cached results exist, it
+runs and saves the suite. After the first run, use it to inspect other keys
+without making another inference call:
 
-```
-pnpx tsx my-test-suite.ts summarize --cached
+```sh
+pnpx tsx bant-suite.ts summarize --cached --keys budget,need
 ```
 
 `diff --cached` behaves the same way for the current-run side of the diff.
@@ -196,6 +251,7 @@ pnpx tsx my-test-suite.ts commit
 #### Diff
 
 The diff command compares your current test results with previously committed results. This is useful when you've made changes to either:
+
 - Your test suite's inputs or expected values
 - The implementation of the function being tested
 
@@ -205,67 +261,40 @@ index 21cd713..ab1b6dc 100644
 --- a/my-test-suite.ts
 +++ b/my-test-suite.ts
 @@ -1,7 +1,7 @@
- import * as CLI from '@creative-introvert/tots-cli';
+ import * as CLI from '@creative-introvert/tons-of-tests-cli';
  import {Effect} from 'effect';
  
 -const myFunction = (input: number) => Promise.resolve(input * 1.7);
 +const myFunction = (input: number) => Promise.resolve(input * 2);
  
- void CLI.run({
+ CLI.run({
      testSuite: {
 ```
 
-```
+```sh
 pnpx tsx my-test-suite.ts diff
-# or, if you want it to process.exit(1) on diff
+# Exit with code 1 when the filtered view contains differences.
 pnpx tsx my-test-suite.ts diff --exit-on-diff
-
-┌─────────────────────────────────────────────────────────────────────────────────────────────────────┐
-│ SUMMARY                                                                                             │
-├─────┬──────────┬────────┬──────┬───────┬──────────┬────────┬──────────────┬─────────┬───────────────┤
-│ #/∑ │ hash     │ ms     │ tags │ input │ expected │ label₀ │ diff result₀ │ label₋₁ │ diff result₋₁ │
-├─────┼──────────┼────────┼──────┼───────┼──────────┼────────┼──────────────┼─────────┼───────────────┤
-│ 1/4 │ 562e2cca │ 0.23ms │      │ 1     │ 2        │ TP     │              │ FP      │ 2 => 1.7      │
-├─────┼──────────┼────────┼──────┼───────┼──────────┼────────┼──────────────┼─────────┼───────────────┤
-│ 2/4 │ a5afd52f │ 3.66ms │      │ 2     │ 3        │ FP     │ 3 => 4       │ FP      │ 3 => 3.4      │
-├─────┼──────────┼────────┼──────┼───────┼──────────┼────────┼──────────────┼─────────┼───────────────┤
-│ 3/4 │ 5f7f8725 │ 1.19ms │      │ 3     │ 4        │ FP     │ 4 => 6       │ FP      │ 4 => 5.1      │
-├─────┼──────────┼────────┼──────┼───────┼──────────┼────────┼──────────────┼─────────┼───────────────┤
-│ 4/4 │ 6cc26923 │ 0.98ms │      │ 4     │ 5        │ FP     │ 5 => 8       │ FP      │ 5 => 6.8      │
-├─────┼──────────┼────────┼──────┼───────┼──────────┼────────┼──────────────┼─────────┼───────────────┤
-│ #/∑ │ hash     │ ms     │ tags │ input │ expected │ label₀ │ diff result₀ │ label₋₁ │ diff result₋₁ │
-└─────┴──────────┴────────┴──────┴───────┴──────────┴────────┴──────────────┴─────────┴───────────────┘
-
-
-┌────────────────────────────────────────────────────────────────────┐
-│ STATS                                                              │
-├───┬────┬────┬────┬────┬───────────┬────────┬──────────┬────────────┤
-│ ∑ │ TP │ TN │ FP │ FN │ precision │ recall │ timeMean │ timeMedian │
-├───┼────┼────┼────┼────┼───────────┼────────┼──────────┼────────────┤
-│ 5 │ 2  │ 0  │ 3  │ 0  │ 0.40      │ 1.00   │ 1.30ms   │ 0.98ms     │
-├───┼────┼────┼────┼────┼───────────┼────────┼──────────┼────────────┤
-│ ∑ │ TP │ TN │ FP │ FN │ precision │ recall │ timeMean │ timeMedian │
-└───┴────┴────┴────┴────┴───────────┴────────┴──────────┴────────────┘
-
-
-┌────────────────────────────────────────┐
-│ DIFF                                   │
-├────┬────┬────┬────┬───────────┬────────┤
-│ TP │ TN │ FP │ FN │ precision │ recall │
-├────┼────┼────┼────┼───────────┼────────┤
-│ 1  │ 0  │ -1 │ 0  │ 0.20      │ 0.00   │
-├────┼────┼────┼────┼───────────┼────────┤
-│ TP │ TN │ FP │ FN │ precision │ recall │
-└────┴────┴────┴────┴───────────┴────────┘
 ```
 
-Checkout `workspace/examples/src/with-cli` for more examples.
+For a suite with several results per case, select the keys to compare:
 
+```sh
+pnpx tsx bant-suite.ts diff --keys budget --labels FP,FN
 ```
-pnpx tsx <file-path>
-# e.g.
-pnpx tsx ./workspace/examples/src/with-cli/simple-run.ts
-pnpx tsx ./workspace/examples/src/with-cli/simple-effect.ts
+
+`diff` matches results by case hash and key before applying filters. It includes
+new results and changes to a result's value, expected value, or label. A current
+`FP` result keeps its previous value and label even when that previous label
+does not match `--labels`.
+
+See [workspace/examples/src/with-cli](./workspace/examples/src/with-cli) for
+more examples. From the repository root, run:
+
+```sh
+direnv exec . pnpm exec tsx workspace/examples/src/with-cli/simple-run.ts summarize
+direnv exec . pnpm exec tsx workspace/examples/src/with-cli/simple-effect.ts
+direnv exec . pnpm exec tsx workspace/examples/src/with-cli/multiple-results.ts summarize --keys budget,need
 ```
 
 ### Using the CLI programmatically
@@ -318,16 +347,10 @@ const hash = await CLI.getLastTestRunHash({
 // hash: string | null
 ```
 
-If you compose the CLI's `diff` programmatically (via `_diff` or a custom
-harness), `diff --exit-on-diff` fails with a typed `DiffNonEmpty` error
-when the current run has visible diffs:
-
-```ts
-import {DiffNonEmpty} from '@creative-introvert/tons-of-tests-cli';
-
-// ... compose _diff into your own Effect pipeline, then:
-Effect.catchTag('DiffNonEmpty', () => /* handle */);
-```
+When you use `CLI.effect`, `diff --exit-on-diff` fails with a typed
+`DiffNonEmpty` error when the current run has visible diffs. Handle it with
+`Effect.catchTag('DiffNonEmpty', handler)`; `error.count` gives the number of
+visible changed results.
 
 At the process entry point (`CLI.run`), `DiffNonEmpty` is translated to
 `process.exitCode = 1` — the user-facing "exit with code 1 on diff"

package/dist/cjs/Classify.js

@@ -13,12 +13,16 @@ const values = exports.values = internal.values;
 const defaultIsEqual = exports.defaultIsEqual = internal.defaultIsEqual;
 const defaultIsNil = exports.defaultIsNil = internal.defaultIsNil;
 const makeClassify = exports.makeClassify = internal.makeClassify;
+/** Counts selected results; timing statistics count each selected case once. */
 class Stats extends /*#__PURE__*/_effect.Schema.Class('Stats')({
   TP: _effect.Schema.Finite,
   TN: _effect.Schema.Finite,
   FP: _effect.Schema.Finite,
   FN: _effect.Schema.Finite,
+  /** Number of evaluation results. */
   total: _effect.Schema.Finite,
+  /** Number of distinct original test cases represented by those results. */
+  testCases: _effect.Schema.Int,
   precision: _effect.Schema.Finite,
   recall: _effect.Schema.Finite,
   timeMin: /*#__PURE__*/_effect.Schema.Option(_effect.Schema.Finite),
@@ -33,6 +37,7 @@ class Stats extends /*#__PURE__*/_effect.Schema.Class('Stats')({
       FP: 0,
       FN: 0,
       total: 0,
+      testCases: 0,
       precision: 0,
       recall: 0,
       timeMin: _effect.Option.none(),

package/dist/cjs/Classify.js.map

@@ -1 +1 @@
-{"version":3,"file":"Classify.js","names":["_effect","require","internal","_interopRequireWildcard","cache","e","t","__esModule","r","c","a","Object","defineProperty","n","__proto__","default","WeakMap","has","get","set","hasOwnProperty","call","getOwnPropertyDescriptor","LabelSchema","exports","values","defaultIsEqual","defaultIsNil","makeClassify","Stats","Schema","Class","TP","Finite","TN","FP","FN","total","precision","recall","timeMin","Option","timeMax","timeMedian","timeMean","empty","none"],"sources":["../../src/Classify.ts"],"sourcesContent":[null],"mappings":";;;;;;AAAA,IAAAA,OAAA,gBAAAC,OAAA;AAEA,IAAAC,QAAA,gBAAAC,uBAAA,cAAAF,OAAA;AAAmD,IAAAG,KAAA;AAAA,SAAAD,wBAAAE,CAAA,EAAAC,CAAA,SAAAA,CAAA,IAAAD,CAAA,IAAAA,CAAA,CAAAE,UAAA,SAAAF,CAAA,MAAAG,CAAA,EAAAC,CAAA,EAAAC,CAAA,GAAAC,MAAA,CAAAC,cAAA,EAAAC,CAAA,KAAAC,SAAA,QAAAC,OAAA,EAAAV,CAAA,QAAAM,MAAA,CAAAN,CAAA,MAAAA,CAAA,SAAAQ,CAAA,MAAAT,KAAA,yBAAAY,OAAA,KAAAZ,KAAA,OAAAY,OAAA,KAAAZ,KAAA,QAAAA,KAAA,CAAAa,GAAA,CAAAZ,CAAA,UAAAD,KAAA,CAAAc,GAAA,CAAAb,CAAA,GAAAD,KAAA,CAAAe,GAAA,CAAAd,CAAA,EAAAQ,CAAA,UAAAJ,CAAA,IAAAJ,CAAA,gBAAAI,CAAA,OAAAW,cAAA,CAAAC,IAAA,CAAAhB,CAAA,EAAAI,CAAA,OAAAD,CAAA,GAAAE,CAAA,IAAAC,MAAA,CAAAW,wBAAA,CAAAjB,CAAA,EAAAI,CAAA,OAAAD,CAAA,CAAAU,GAAA,IAAAV,CAAA,CAAAW,GAAA,IAAAT,CAAA,CAAAG,CAAA,EAAAJ,CAAA,EAAAD,CAAA,IAAAK,CAAA,CAAAJ,CAAA,IAAAJ,CAAA,CAAAI,CAAA,WAAAI,CAAA;AAE5C,MAAMU,WAAW,GAAAC,OAAA,CAAAD,WAAA,GAAGrB,QAAQ,CAACqB,WAAW;AAGxC,MAAME,MAAM,GAAAD,OAAA,CAAAC,MAAA,GAAGvB,QAAQ,CAACuB,MAAM;AAC9B,MAAMC,cAAc,GAAAF,OAAA,CAAAE,cAAA,GAAGxB,QAAQ,CAACwB,cAAc;AAC9C,MAAMC,YAAY,GAAAH,OAAA,CAAAG,YAAA,GAAGzB,QAAQ,CAACyB,YAAY;AAC1C,MAAMC,YAAY,GAAAJ,OAAA,CAAAI,YAAA,GAAG1B,QAAQ,CAAC0B,YAAY;AAI3C,MAAOC,KAAM,sBAAQC,cAAM,CAACC,KAAK,CAAQ,OAAO,CAAC,CAAC;EACpDC,EAAE,EAAEF,cAAM,CAACG,MAAM;EACjBC,EAAE,EAAEJ,cAAM,CAACG,MAAM;EACjBE,EAAE,EAAEL,cAAM,CAACG,MAAM;EACjBG,EAAE,EAAEN,cAAM,CAACG,MAAM;EACjBI,KAAK,EAAEP,cAAM,CAACG,MAAM;EACpBK,SAAS,EAAER,cAAM,CAACG,MAAM;EACxBM,MAAM,EAAET,cAAM,CAACG,MAAM;EACrBO,OAAO,eAAEV,cAAM,CAACW,MAAM,CAACX,cAAM,CAACG,MAAM,CAAC;EACrCS,OAAO,eAAEZ,cAAM,CAACW,MAAM,CAACX,cAAM,CAACG,MAAM,CAAC;EACrCU,UAAU,eAAEb,cAAM,CAACW,MAAM,CAACX,cAAM,CAACG,MAAM,CAAC;EACxCW,QAAQ,eAAEd,cAAM,CAACW,MAAM,CAACX,cAAM,CAACG,MAAM;CACxC,CAAC;EACE,OAAOY,KAAKA,CAAA;IACR,OAAO,IAAIhB,KAAK,CAAC;MACbG,EAAE,EAAE,CAAC;MACLE,EAAE,EAAE,CAAC;MACLC,EAAE,EAAE,CAAC;MACLC,EAAE,EAAE,CAAC;MACLC,KAAK,EAAE,CAAC;MACRC,SAAS,EAAE,CAAC;MACZC,MAAM,EAAE,CAAC;MACTC,OAAO,EAAEC,cAAM,CAACK,IAAI,EAAE;MACtBJ,OAAO,EAAED,cAAM,CAACK,IAAI,EAAE;MACtBH,UAAU,EAAEF,cAAM,CAACK,IAAI,EAAE;MACzBF,QAAQ,EAAEH,cAAM,CAACK,IAAI;KACxB,CAAC;EACN;;AACHtB,OAAA,CAAAK,KAAA,GAAAA,KAAA","ignoreList":[]}
+{"version":3,"file":"Classify.js","names":["_effect","require","internal","_interopRequireWildcard","cache","e","t","__esModule","r","c","a","Object","defineProperty","n","__proto__","default","WeakMap","has","get","set","hasOwnProperty","call","getOwnPropertyDescriptor","LabelSchema","exports","values","defaultIsEqual","defaultIsNil","makeClassify","Stats","Schema","Class","TP","Finite","TN","FP","FN","total","testCases","Int","precision","recall","timeMin","Option","timeMax","timeMedian","timeMean","empty","none"],"sources":["../../src/Classify.ts"],"sourcesContent":[null],"mappings":";;;;;;AAAA,IAAAA,OAAA,gBAAAC,OAAA;AAEA,IAAAC,QAAA,gBAAAC,uBAAA,cAAAF,OAAA;AAAmD,IAAAG,KAAA;AAAA,SAAAD,wBAAAE,CAAA,EAAAC,CAAA,SAAAA,CAAA,IAAAD,CAAA,IAAAA,CAAA,CAAAE,UAAA,SAAAF,CAAA,MAAAG,CAAA,EAAAC,CAAA,EAAAC,CAAA,GAAAC,MAAA,CAAAC,cAAA,EAAAC,CAAA,KAAAC,SAAA,QAAAC,OAAA,EAAAV,CAAA,QAAAM,MAAA,CAAAN,CAAA,MAAAA,CAAA,SAAAQ,CAAA,MAAAT,KAAA,yBAAAY,OAAA,KAAAZ,KAAA,OAAAY,OAAA,KAAAZ,KAAA,QAAAA,KAAA,CAAAa,GAAA,CAAAZ,CAAA,UAAAD,KAAA,CAAAc,GAAA,CAAAb,CAAA,GAAAD,KAAA,CAAAe,GAAA,CAAAd,CAAA,EAAAQ,CAAA,UAAAJ,CAAA,IAAAJ,CAAA,gBAAAI,CAAA,OAAAW,cAAA,CAAAC,IAAA,CAAAhB,CAAA,EAAAI,CAAA,OAAAD,CAAA,GAAAE,CAAA,IAAAC,MAAA,CAAAW,wBAAA,CAAAjB,CAAA,EAAAI,CAAA,OAAAD,CAAA,CAAAU,GAAA,IAAAV,CAAA,CAAAW,GAAA,IAAAT,CAAA,CAAAG,CAAA,EAAAJ,CAAA,EAAAD,CAAA,IAAAK,CAAA,CAAAJ,CAAA,IAAAJ,CAAA,CAAAI,CAAA,WAAAI,CAAA;AAE5C,MAAMU,WAAW,GAAAC,OAAA,CAAAD,WAAA,GAAGrB,QAAQ,CAACqB,WAAW;AAGxC,MAAME,MAAM,GAAAD,OAAA,CAAAC,MAAA,GAAGvB,QAAQ,CAACuB,MAAM;AAC9B,MAAMC,cAAc,GAAAF,OAAA,CAAAE,cAAA,GAAGxB,QAAQ,CAACwB,cAAc;AAC9C,MAAMC,YAAY,GAAAH,OAAA,CAAAG,YAAA,GAAGzB,QAAQ,CAACyB,YAAY;AAC1C,MAAMC,YAAY,GAAAJ,OAAA,CAAAI,YAAA,GAAG1B,QAAQ,CAAC0B,YAAY;AAIjD;AACM,MAAOC,KAAM,sBAAQC,cAAM,CAACC,KAAK,CAAQ,OAAO,CAAC,CAAC;EACpDC,EAAE,EAAEF,cAAM,CAACG,MAAM;EACjBC,EAAE,EAAEJ,cAAM,CAACG,MAAM;EACjBE,EAAE,EAAEL,cAAM,CAACG,MAAM;EACjBG,EAAE,EAAEN,cAAM,CAACG,MAAM;EACjB;EACAI,KAAK,EAAEP,cAAM,CAACG,MAAM;EACpB;EACAK,SAAS,EAAER,cAAM,CAACS,GAAG;EACrBC,SAAS,EAAEV,cAAM,CAACG,MAAM;EACxBQ,MAAM,EAAEX,cAAM,CAACG,MAAM;EACrBS,OAAO,eAAEZ,cAAM,CAACa,MAAM,CAACb,cAAM,CAACG,MAAM,CAAC;EACrCW,OAAO,eAAEd,cAAM,CAACa,MAAM,CAACb,cAAM,CAACG,MAAM,CAAC;EACrCY,UAAU,eAAEf,cAAM,CAACa,MAAM,CAACb,cAAM,CAACG,MAAM,CAAC;EACxCa,QAAQ,eAAEhB,cAAM,CAACa,MAAM,CAACb,cAAM,CAACG,MAAM;CACxC,CAAC;EACE,OAAOc,KAAKA,CAAA;IACR,OAAO,IAAIlB,KAAK,CAAC;MACbG,EAAE,EAAE,CAAC;MACLE,EAAE,EAAE,CAAC;MACLC,EAAE,EAAE,CAAC;MACLC,EAAE,EAAE,CAAC;MACLC,KAAK,EAAE,CAAC;MACRC,SAAS,EAAE,CAAC;MACZE,SAAS,EAAE,CAAC;MACZC,MAAM,EAAE,CAAC;MACTC,OAAO,EAAEC,cAAM,CAACK,IAAI,EAAE;MACtBJ,OAAO,EAAED,cAAM,CAACK,IAAI,EAAE;MACtBH,UAAU,EAAEF,cAAM,CAACK,IAAI,EAAE;MACzBF,QAAQ,EAAEH,cAAM,CAACK,IAAI;KACxB,CAAC;EACN;;AACHxB,OAAA,CAAAK,KAAA,GAAAA,KAAA","ignoreList":[]}

package/dist/cjs/Test.js

@@ -3,12 +3,16 @@
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
-exports.runCollectRecord = exports.makeSha256 = exports.diff = exports.all = void 0;
+exports.runCollectRecord = exports.resultKey = exports.makeSha256 = exports.filterResults = exports.diff = exports.all = void 0;
 var internal = /*#__PURE__*/_interopRequireWildcard(/*#__PURE__*/require("./internal/Test.js"));
 var cache;
 function _interopRequireWildcard(e, t) { if (!t && e && e.__esModule) return e; var r, c, a = Object.defineProperty, n = { __proto__: null, default: e }; if (Object(e) !== e) return n; if (cache || "function" != typeof WeakMap || (cache = new WeakMap()), cache) { if (cache.has(e)) return cache.get(e); cache.set(e, n); } for (c in e) "default" !== c && {}.hasOwnProperty.call(e, c) && ((r = a && Object.getOwnPropertyDescriptor(e, c)) && (r.get || r.set) ? a(n, c, r) : n[c] = e[c]); return n; }
 const all = exports.all = internal.all;
 const diff = exports.diff = internal.diff;
 const makeSha256 = exports.makeSha256 = internal.makeSha256;
+/** Builds the lookup key for one evaluation within its original test case. */
+const resultKey = exports.resultKey = internal.resultKey;
 const runCollectRecord = exports.runCollectRecord = internal.runCollectRecord;
+/** Keeps matching results and recalculates counts and case timing statistics. */
+const filterResults = exports.filterResults = internal.filterResults;
 //# sourceMappingURL=Test.js.map

package/dist/cjs/Test.js.map

@@ -1 +1 @@
-{"version":3,"file":"Test.js","names":["internal","_interopRequireWildcard","require","cache","e","t","__esModule","r","c","a","Object","defineProperty","n","__proto__","default","WeakMap","has","get","set","hasOwnProperty","call","getOwnPropertyDescriptor","all","exports","diff","makeSha256","runCollectRecord"],"sources":["../../src/Test.ts"],"sourcesContent":[null],"mappings":";;;;;;AAGA,IAAAA,QAAA,gBAAAC,uBAAA,cAAAC,OAAA;AAA+C,IAAAC,KAAA;AAAA,SAAAF,wBAAAG,CAAA,EAAAC,CAAA,SAAAA,CAAA,IAAAD,CAAA,IAAAA,CAAA,CAAAE,UAAA,SAAAF,CAAA,MAAAG,CAAA,EAAAC,CAAA,EAAAC,CAAA,GAAAC,MAAA,CAAAC,cAAA,EAAAC,CAAA,KAAAC,SAAA,QAAAC,OAAA,EAAAV,CAAA,QAAAM,MAAA,CAAAN,CAAA,MAAAA,CAAA,SAAAQ,CAAA,MAAAT,KAAA,yBAAAY,OAAA,KAAAZ,KAAA,OAAAY,OAAA,KAAAZ,KAAA,QAAAA,KAAA,CAAAa,GAAA,CAAAZ,CAAA,UAAAD,KAAA,CAAAc,GAAA,CAAAb,CAAA,GAAAD,KAAA,CAAAe,GAAA,CAAAd,CAAA,EAAAQ,CAAA,UAAAJ,CAAA,IAAAJ,CAAA,gBAAAI,CAAA,OAAAW,cAAA,CAAAC,IAAA,CAAAhB,CAAA,EAAAI,CAAA,OAAAD,CAAA,GAAAE,CAAA,IAAAC,MAAA,CAAAW,wBAAA,CAAAjB,CAAA,EAAAI,CAAA,OAAAD,CAAA,CAAAU,GAAA,IAAAV,CAAA,CAAAW,GAAA,IAAAT,CAAA,CAAAG,CAAA,EAAAJ,CAAA,EAAAD,CAAA,IAAAK,CAAA,CAAAJ,CAAA,IAAAJ,CAAA,CAAAI,CAAA,WAAAI,CAAA;AAiExC,MAAMU,GAAG,GAAAC,OAAA,CAAAD,GAAA,GAAGtB,QAAQ,CAACsB,GAAG;AACxB,MAAME,IAAI,GAAAD,OAAA,CAAAC,IAAA,GAAGxB,QAAQ,CAACwB,IAAI;AAC1B,MAAMC,UAAU,GAAAF,OAAA,CAAAE,UAAA,GAAGzB,QAAQ,CAACyB,UAAU;AACtC,MAAMC,gBAAgB,GAAAH,OAAA,CAAAG,gBAAA,GAAG1B,QAAQ,CAAC0B,gBAAgB","ignoreList":[]}
+{"version":3,"file":"Test.js","names":["internal","_interopRequireWildcard","require","cache","e","t","__esModule","r","c","a","Object","defineProperty","n","__proto__","default","WeakMap","has","get","set","hasOwnProperty","call","getOwnPropertyDescriptor","all","exports","diff","makeSha256","resultKey","runCollectRecord","filterResults"],"sources":["../../src/Test.ts"],"sourcesContent":[null],"mappings":";;;;;;AAGA,IAAAA,QAAA,gBAAAC,uBAAA,cAAAC,OAAA;AAA+C,IAAAC,KAAA;AAAA,SAAAF,wBAAAG,CAAA,EAAAC,CAAA,SAAAA,CAAA,IAAAD,CAAA,IAAAA,CAAA,CAAAE,UAAA,SAAAF,CAAA,MAAAG,CAAA,EAAAC,CAAA,EAAAC,CAAA,GAAAC,MAAA,CAAAC,cAAA,EAAAC,CAAA,KAAAC,SAAA,QAAAC,OAAA,EAAAV,CAAA,QAAAM,MAAA,CAAAN,CAAA,MAAAA,CAAA,SAAAQ,CAAA,MAAAT,KAAA,yBAAAY,OAAA,KAAAZ,KAAA,OAAAY,OAAA,KAAAZ,KAAA,QAAAA,KAAA,CAAAa,GAAA,CAAAZ,CAAA,UAAAD,KAAA,CAAAc,GAAA,CAAAb,CAAA,GAAAD,KAAA,CAAAe,GAAA,CAAAd,CAAA,EAAAQ,CAAA,UAAAJ,CAAA,IAAAJ,CAAA,gBAAAI,CAAA,OAAAW,cAAA,CAAAC,IAAA,CAAAhB,CAAA,EAAAI,CAAA,OAAAD,CAAA,GAAAE,CAAA,IAAAC,MAAA,CAAAW,wBAAA,CAAAjB,CAAA,EAAAI,CAAA,OAAAD,CAAA,CAAAU,GAAA,IAAAV,CAAA,CAAAW,GAAA,IAAAT,CAAA,CAAAG,CAAA,EAAAJ,CAAA,EAAAD,CAAA,IAAAK,CAAA,CAAAJ,CAAA,IAAAJ,CAAA,CAAAI,CAAA,WAAAI,CAAA;AAyGxC,MAAMU,GAAG,GAAAC,OAAA,CAAAD,GAAA,GAAGtB,QAAQ,CAACsB,GAAG;AACxB,MAAME,IAAI,GAAAD,OAAA,CAAAC,IAAA,GAAGxB,QAAQ,CAACwB,IAAI;AAC1B,MAAMC,UAAU,GAAAF,OAAA,CAAAE,UAAA,GAAGzB,QAAQ,CAACyB,UAAU;AAC7C;AACO,MAAMC,SAAS,GAAAH,OAAA,CAAAG,SAAA,GAAG1B,QAAQ,CAAC0B,SAAS;AACpC,MAAMC,gBAAgB,GAAAJ,OAAA,CAAAI,gBAAA,GAAG3B,QAAQ,CAAC2B,gBAAgB;AACzD;AACO,MAAMC,aAAa,GAAAL,OAAA,CAAAK,aAAA,GAAG5B,QAAQ,CAAC4B,aAAa","ignoreList":[]}

package/dist/cjs/Test.repository.types.js.map

@@ -1 +1 @@
-{"version":3,"file":"Test.repository.types.js","names":["_effect","require","TestRun","Schema","Class","id","Int","name","String","hash","NullOr","exports"],"sources":["../../src/Test.repository.types.ts"],"sourcesContent":[null],"mappings":";;;;;;AAAA,IAAAA,OAAA,gBAAAC,OAAA;AAkBM,MAAOC,OAAQ,sBAAQC,cAAM,CAACC,KAAK,CAAU,SAAS,CAAC,CAAC;EAC1DC,EAAE,EAAEF,cAAM,CAACG,GAAG;EACdC,IAAI,EAAEJ,cAAM,CAACK,MAAM;EACnBC,IAAI,eAAEN,cAAM,CAACO,MAAM,CAACP,cAAM,CAACK,MAAM;CACpC,CAAC;AAAGG,OAAA,CAAAT,OAAA,GAAAA,OAAA","ignoreList":[]}
+{"version":3,"file":"Test.repository.types.js","names":["_effect","require","TestRun","Schema","Class","id","Int","name","String","hash","NullOr","exports"],"sources":["../../src/Test.repository.types.ts"],"sourcesContent":[null],"mappings":";;;;;;AAAA,IAAAA,OAAA,gBAAAC,OAAA;AAoBM,MAAOC,OAAQ,sBAAQC,cAAM,CAACC,KAAK,CAAU,SAAS,CAAC,CAAC;EAC1DC,EAAE,EAAEF,cAAM,CAACG,GAAG;EACdC,IAAI,EAAEJ,cAAM,CAACK,MAAM;EACnBC,IAAI,eAAEN,cAAM,CAACO,MAAM,CAACP,cAAM,CAACK,MAAM;CACpC,CAAC;AAAGG,OAAA,CAAAT,OAAA,GAAAA,OAAA","ignoreList":[]}