@tobi-or-not/quorum 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/README.md +2 -1
  2. package/dist/bench/attacks.d.ts +103 -0
  3. package/dist/bench/attacks.js +272 -0
  4. package/dist/bench/fingerprint.d.ts +3 -2
  5. package/dist/bench/fingerprint.js +3 -2
  6. package/dist/bench/independence.d.ts +28 -0
  7. package/dist/bench/independence.js +67 -0
  8. package/dist/bench/records.d.ts +171 -13
  9. package/dist/bench/records.js +34 -11
  10. package/dist/bench/report.d.ts +44 -11
  11. package/dist/bench/report.js +238 -32
  12. package/dist/bench/run-dir.d.ts +52 -26
  13. package/dist/bench/run-dir.js +124 -45
  14. package/dist/bench/runner.d.ts +29 -17
  15. package/dist/bench/runner.js +53 -18
  16. package/dist/bench/score.d.ts +38 -7
  17. package/dist/bench/score.js +64 -17
  18. package/dist/bench/stats.d.ts +61 -7
  19. package/dist/bench/stats.js +82 -7
  20. package/dist/bench/suite.d.ts +54 -9
  21. package/dist/bench/suite.js +264 -96
  22. package/dist/cli/bench.d.ts +20 -9
  23. package/dist/cli/bench.js +148 -74
  24. package/dist/cli/io.d.ts +9 -2
  25. package/dist/cli/io.js +20 -2
  26. package/dist/cli/main.js +2 -19
  27. package/dist/config/load.d.ts +3 -0
  28. package/dist/config/load.js +17 -8
  29. package/dist/core/diff.d.ts +10 -0
  30. package/dist/core/diff.js +77 -0
  31. package/dist/core/errors.d.ts +6 -0
  32. package/dist/core/errors.js +5 -0
  33. package/dist/core/stages/context.d.ts +12 -3
  34. package/dist/core/stages/context.js +23 -3
  35. package/dist/providers/anthropic.js +6 -3
  36. package/dist/providers/claude-code.js +11 -10
  37. package/dist/providers/gemini.js +9 -4
  38. package/dist/providers/ollama.js +6 -4
  39. package/dist/providers/openai-compatible.js +6 -4
  40. package/dist/providers/reply.d.ts +6 -2
  41. package/dist/providers/reply.js +13 -3
  42. package/package.json +1 -1
@@ -1,17 +1,22 @@
1
1
  /**
2
2
  * The two files of a bench run directory, as data: `run.json`, which holds what makes the run one run
3
- * (the quorum version, the suite's name and hash, the salt, and the fingerprints of its configurations),
3
+ * (the quorum version, the suite's name and two hashes, the salt, and the fingerprints of its configurations),
4
4
  * and `results.jsonl`, which holds one line per review. Both are read as untrusted: another run or a
5
5
  * person may have written them, so each is parsed with `JSON.parse` and a strict schema at every level,
6
- * and anything the schema does not describe is refused, never repaired. A repeated key is not looked
7
- * for: the tree that finds one costs ten times the parse on a line of 16 MiB.
6
+ * and anything the schema does not describe is refused. One thing is dropped, not refused: a `__proto__`
7
+ * key of the result's `profiles` or of a profile's `lenses`, which zod's records skip, its value unread.
8
+ * A record is scored, never merged into another object, so the key changes nothing, and refusing it as
9
+ * the config loader does would walk every line again on each resume. A repeated key is not looked for:
10
+ * the tree that finds one costs ten times the parse on a line of 16 MiB.
8
11
  *
9
12
  * A record keeps the facts of a review, not its scores: the report scores every record with the suite's
10
- * locations when it is built, so a score never goes stale.
13
+ * locations when it is built, so a score never goes stale. It keeps the review's result or, for a review
14
+ * that a model's reply denied, which denial it was, never the reply.
11
15
  */
16
+ import { z } from 'zod';
12
17
  import type { QuorumResult, Usage } from '../core/types.ts';
13
18
  /** The `format` of the run file this quorum writes and reads. */
14
- export declare const RUN_FORMAT = 1;
19
+ export declare const RUN_FORMAT = 2;
15
20
  /** The most one line of `results.jsonl` may hold, its line break not counted, in bytes. */
16
21
  export declare const MAX_RECORD_BYTES: number;
17
22
  /** The most `run.json` may hold, in bytes. */
@@ -23,21 +28,32 @@ export interface RunConfiguration {
23
28
  readonly path: string | null;
24
29
  readonly fingerprint: string;
25
30
  }
31
+ /** The suite a run was made under: its name and the two hashes of `loadSuite`, and the scoring hash the run began with. */
32
+ export interface RunSuite {
33
+ readonly name: string;
34
+ readonly reviewHash: string;
35
+ readonly scoringHash: string;
36
+ /** The scoring hash of the suite when the run began, which a later scoring hash is told apart from. */
37
+ readonly firstScoringHash: string;
38
+ }
26
39
  /** What `run.json` holds. */
27
40
  export interface RunFile {
28
- readonly format: 1;
41
+ readonly format: 2;
29
42
  /** The version of the quorum that began the run. */
30
43
  readonly quorum: string;
31
- readonly suite: {
32
- readonly name: string;
33
- readonly hash: string;
34
- };
44
+ readonly suite: RunSuite;
35
45
  /** The key the fingerprints are keyed with: 32 random bytes as 64 lowercase hex digits. */
36
46
  readonly salt: string;
37
47
  readonly configurations: readonly RunConfiguration[];
38
48
  }
39
- /** What one line of `results.jsonl` holds: a review that resolved, as the bench saw it. */
40
- export interface ReviewRecord {
49
+ /**
50
+ * How a model's reply denied a review: its research reply could not be used, the model refused the
51
+ * research request, or its reply was cut off at its token limit.
52
+ */
53
+ export declare const DENIALS: readonly ["unusable-reply", "refusal", "truncated"];
54
+ export type Denial = typeof DENIALS[number];
55
+ /** What every line of `results.jsonl` holds of a review, as the bench saw it, whatever came back. */
56
+ interface RecordFacts {
41
57
  readonly fingerprint: string;
42
58
  /** The id of the suite's case. */
43
59
  readonly case: string;
@@ -50,8 +66,149 @@ export interface ReviewRecord {
50
66
  readonly usage?: Usage;
51
67
  /** Whether every call reported a cost. */
52
68
  readonly costComplete: boolean;
53
- readonly result: QuorumResult;
54
69
  }
70
+ /** A review that resolved: its facts and its full result. */
71
+ export type ResultRecord = RecordFacts & {
72
+ readonly result: QuorumResult;
73
+ };
74
+ /** A review that a model's reply denied: its facts and which denial, and no result. */
75
+ export type DeniedRecord = RecordFacts & {
76
+ readonly denied: Denial;
77
+ };
78
+ /** What one line of `results.jsonl` holds: a review that resolved, or one that a model's reply denied. */
79
+ export type ReviewRecord = ResultRecord | DeniedRecord;
80
+ /**
81
+ * The schema of a result, strict at every level. `records.spec.ts` holds its output type equal to
82
+ * `QuorumResult`, `readonly` aside, so a field the core adds, drops or retypes, optional or not, fails the
83
+ * typecheck.
84
+ */
85
+ export declare const resultSchema: z.ZodObject<{
86
+ kept: z.ZodArray<z.ZodObject<{
87
+ id: z.ZodString;
88
+ profile: z.ZodString;
89
+ file: z.ZodString;
90
+ line: z.ZodInt;
91
+ category: z.ZodString;
92
+ summary: z.ZodString;
93
+ evidence: z.ZodString;
94
+ quoteMatched: z.ZodBoolean;
95
+ keptVotes: z.ZodInt;
96
+ totalVotes: z.ZodInt;
97
+ confidence: z.ZodOptional<z.ZodEnum<{
98
+ high: "high";
99
+ medium: "medium";
100
+ low: "low";
101
+ }>>;
102
+ votes: z.ZodArray<z.ZodObject<{
103
+ angle: z.ZodString;
104
+ vote: z.ZodNullable<z.ZodObject<{
105
+ keep: z.ZodBoolean;
106
+ reasoning: z.ZodString;
107
+ }, z.core.$strict>>;
108
+ error: z.ZodNullable<z.ZodString>;
109
+ }, z.core.$strict>>;
110
+ }, z.core.$strict>>;
111
+ discarded: z.ZodArray<z.ZodObject<{
112
+ id: z.ZodString;
113
+ profile: z.ZodString;
114
+ file: z.ZodString;
115
+ line: z.ZodInt;
116
+ category: z.ZodString;
117
+ summary: z.ZodString;
118
+ evidence: z.ZodString;
119
+ quoteMatched: z.ZodBoolean;
120
+ keptVotes: z.ZodInt;
121
+ totalVotes: z.ZodInt;
122
+ confidence: z.ZodOptional<z.ZodEnum<{
123
+ high: "high";
124
+ medium: "medium";
125
+ low: "low";
126
+ }>>;
127
+ votes: z.ZodArray<z.ZodObject<{
128
+ angle: z.ZodString;
129
+ vote: z.ZodNullable<z.ZodObject<{
130
+ keep: z.ZodBoolean;
131
+ reasoning: z.ZodString;
132
+ }, z.core.$strict>>;
133
+ error: z.ZodNullable<z.ZodString>;
134
+ }, z.core.$strict>>;
135
+ }, z.core.$strict>>;
136
+ unreviewed: z.ZodArray<z.ZodObject<{
137
+ id: z.ZodString;
138
+ profile: z.ZodString;
139
+ file: z.ZodString;
140
+ line: z.ZodInt;
141
+ category: z.ZodString;
142
+ summary: z.ZodString;
143
+ evidence: z.ZodString;
144
+ quoteMatched: z.ZodBoolean;
145
+ keptVotes: z.ZodInt;
146
+ totalVotes: z.ZodInt;
147
+ confidence: z.ZodOptional<z.ZodEnum<{
148
+ high: "high";
149
+ medium: "medium";
150
+ low: "low";
151
+ }>>;
152
+ votes: z.ZodArray<z.ZodObject<{
153
+ angle: z.ZodString;
154
+ vote: z.ZodNullable<z.ZodObject<{
155
+ keep: z.ZodBoolean;
156
+ reasoning: z.ZodString;
157
+ }, z.core.$strict>>;
158
+ error: z.ZodNullable<z.ZodString>;
159
+ }, z.core.$strict>>;
160
+ }, z.core.$strict>>;
161
+ meta: z.ZodObject<{
162
+ version: z.ZodString;
163
+ startedAt: z.ZodString;
164
+ durationMs: z.ZodInt;
165
+ profiles: z.ZodRecord<z.ZodString, z.ZodObject<{
166
+ providers: z.ZodObject<{
167
+ research: z.ZodObject<{
168
+ id: z.ZodString;
169
+ model: z.ZodString;
170
+ }, z.core.$strict>;
171
+ lenses: z.ZodRecord<z.ZodString, z.ZodObject<{
172
+ id: z.ZodString;
173
+ model: z.ZodString;
174
+ }, z.core.$strict>>;
175
+ }, z.core.$strict>;
176
+ policy: z.ZodObject<{
177
+ high: z.ZodNullable<z.ZodInt>;
178
+ medium: z.ZodNullable<z.ZodInt>;
179
+ low: z.ZodNullable<z.ZodInt>;
180
+ }, z.core.$strict>;
181
+ gating: z.ZodBoolean;
182
+ invalidFindings: z.ZodInt;
183
+ }, z.core.$strict>>;
184
+ usage: z.ZodOptional<z.ZodObject<{
185
+ inputTokens: z.ZodNumber;
186
+ outputTokens: z.ZodNumber;
187
+ costUsd: z.ZodOptional<z.ZodNumber>;
188
+ }, z.core.$strict>>;
189
+ incomplete: z.ZodArray<z.ZodObject<{
190
+ profile: z.ZodString;
191
+ reason: z.ZodEnum<{
192
+ "research-unusable": "research-unusable";
193
+ "findings-capped": "findings-capped";
194
+ "finding-unreadable": "finding-unreadable";
195
+ "vote-failed": "vote-failed";
196
+ "request-too-large": "request-too-large";
197
+ }>;
198
+ file: z.ZodOptional<z.ZodString>;
199
+ detail: z.ZodString;
200
+ }, z.core.$strict>>;
201
+ flags: z.ZodArray<z.ZodObject<{
202
+ code: z.ZodEnum<{
203
+ "hidden-characters": "hidden-characters";
204
+ "ai-addressed-text": "ai-addressed-text";
205
+ }>;
206
+ file: z.ZodOptional<z.ZodString>;
207
+ line: z.ZodOptional<z.ZodInt>;
208
+ detail: z.ZodString;
209
+ }, z.core.$strict>>;
210
+ }, z.core.$strict>;
211
+ }, z.core.$strict>;
55
212
  /** The key of a planned review: the fingerprint, the case and the trial, as a JSON array. */
56
213
  export declare function reviewKey(fingerprint: string, caseId: string, trial: number): string;
57
214
  /** One line of `results.jsonl`: the record as inert JSON, on one line, and a line break. */
@@ -60,3 +217,4 @@ export declare function recordLine(record: ReviewRecord): string;
60
217
  export declare function parseRecord(line: string): ReviewRecord | undefined;
61
218
  /** The run file `text` holds, or undefined when it is no run file of this format. */
62
219
  export declare function parseRunFile(text: string): RunFile | undefined;
220
+ export {};
@@ -1,19 +1,23 @@
1
1
  /**
2
2
  * The two files of a bench run directory, as data: `run.json`, which holds what makes the run one run
3
- * (the quorum version, the suite's name and hash, the salt, and the fingerprints of its configurations),
3
+ * (the quorum version, the suite's name and two hashes, the salt, and the fingerprints of its configurations),
4
4
  * and `results.jsonl`, which holds one line per review. Both are read as untrusted: another run or a
5
5
  * person may have written them, so each is parsed with `JSON.parse` and a strict schema at every level,
6
- * and anything the schema does not describe is refused, never repaired. A repeated key is not looked
7
- * for: the tree that finds one costs ten times the parse on a line of 16 MiB.
6
+ * and anything the schema does not describe is refused. One thing is dropped, not refused: a `__proto__`
7
+ * key of the result's `profiles` or of a profile's `lenses`, which zod's records skip, its value unread.
8
+ * A record is scored, never merged into another object, so the key changes nothing, and refusing it as
9
+ * the config loader does would walk every line again on each resume. A repeated key is not looked for:
10
+ * the tree that finds one costs ten times the parse on a line of 16 MiB.
8
11
  *
9
12
  * A record keeps the facts of a review, not its scores: the report scores every record with the suite's
10
- * locations when it is built, so a score never goes stale.
13
+ * locations when it is built, so a score never goes stale. It keeps the review's result or, for a review
14
+ * that a model's reply denied, which denial it was, never the reply.
11
15
  */
12
16
  import { z } from 'zod';
13
17
  import { inertJsonLine } from "../core/sanitize.js";
14
18
  import { MAX_TRIALS, SUITE_ID } from "./suite.js";
15
19
  /** The `format` of the run file this quorum writes and reads. */
16
- export const RUN_FORMAT = 1;
20
+ export const RUN_FORMAT = 2;
17
21
  /** The most one line of `results.jsonl` may hold, its line break not counted, in bytes. */
18
22
  export const MAX_RECORD_BYTES = 16 * 1024 * 1024;
19
23
  /** The most `run.json` may hold, in bytes. */
@@ -23,6 +27,11 @@ export const MAX_CONFIGURATIONS = 16;
23
27
  const MAX_ID = 64;
24
28
  const MAX_QUORUM_VERSION = 64;
25
29
  const MAX_CONFIG_PATH = 4096;
30
+ /**
31
+ * How a model's reply denied a review: its research reply could not be used, the model refused the
32
+ * research request, or its reply was cut off at its token limit.
33
+ */
34
+ export const DENIALS = ['unusable-reply', 'refusal', 'truncated'];
26
35
  const digest = z.string().regex(/^[0-9a-f]{64}$/);
27
36
  const identifier = z.string().max(MAX_ID).regex(SUITE_ID);
28
37
  const count = z.int().min(0);
@@ -92,8 +101,12 @@ const flag = z.strictObject({
92
101
  line: count.optional(),
93
102
  detail: z.string(),
94
103
  });
95
- /** Declared as `QuorumResult`'s own type, so a required field the core adds and this schema lacks fails the typecheck. */
96
- const resultSchema = z.strictObject({
104
+ /**
105
+ * The schema of a result, strict at every level. `records.spec.ts` holds its output type equal to
106
+ * `QuorumResult`, `readonly` aside, so a field the core adds, drops or retypes, optional or not, fails the
107
+ * typecheck.
108
+ */
109
+ export const resultSchema = z.strictObject({
97
110
  kept: z.array(verdict),
98
111
  discarded: z.array(verdict),
99
112
  unreviewed: z.array(verdict),
@@ -107,7 +120,7 @@ const resultSchema = z.strictObject({
107
120
  flags: z.array(flag),
108
121
  }),
109
122
  });
110
- const recordSchema = z.strictObject({
123
+ const facts = {
111
124
  fingerprint: digest,
112
125
  case: identifier,
113
126
  trial: z.int().min(1).max(MAX_TRIALS),
@@ -116,14 +129,24 @@ const recordSchema = z.strictObject({
116
129
  failedCalls: count,
117
130
  usage: usage.optional(),
118
131
  costComplete: z.boolean(),
119
- result: resultSchema,
120
- }).refine(({ calls, failedCalls }) => failedCalls <= calls);
132
+ };
133
+ /** A record holds exactly one of a result and a denial: each alternative is strict, so a line with both, or with neither, is refused. */
134
+ const recordSchema = z.union([
135
+ z.strictObject({ ...facts, result: resultSchema }),
136
+ z.strictObject({
137
+ ...facts,
138
+ denied: z.enum(DENIALS),
139
+ }),
140
+ ]).refine(({ calls, failedCalls }) => failedCalls <= calls);
141
+ const hash = z.string().regex(/^sha256:[0-9a-f]{64}$/);
121
142
  const runFileSchema = z.strictObject({
122
143
  format: z.literal(RUN_FORMAT),
123
144
  quorum: z.string().min(1).max(MAX_QUORUM_VERSION),
124
145
  suite: z.strictObject({
125
146
  name: identifier,
126
- hash: z.string().regex(/^sha256:[0-9a-f]{64}$/),
147
+ reviewHash: hash,
148
+ scoringHash: hash,
149
+ firstScoringHash: hash,
127
150
  }),
128
151
  salt: digest,
129
152
  configurations: z.array(z.strictObject({
@@ -1,16 +1,22 @@
1
1
  /**
2
2
  * The report of a bench run: what each configuration caught, what it cost and how sure the numbers are,
3
3
  * built from the scored reviews and rendered as Markdown and as JSON. Pure code over scores, so the
4
- * report can be built again from a run directory. It holds numbers, suite ids and configuration paths,
5
- * never text a model wrote.
4
+ * report can be built again from a run directory. It holds numbers, suite ids, configuration paths and
5
+ * the suite's two hashes, never text a model wrote.
6
6
  */
7
- import type { ReviewRecord, RunConfiguration } from './records.ts';
7
+ import type { AttackReport } from './attacks.ts';
8
+ import type { Independence } from './independence.ts';
9
+ import type { ReviewRecord, RunConfiguration, RunSuite } from './records.ts';
8
10
  import type { ReviewOutcome } from './score.ts';
9
- import type { SuiteCase } from './suite.ts';
11
+ import type { Location, SuiteCase } from './suite.ts';
10
12
  /** What one review earned, with what it took: its latency, its tokens when it reported usage, and its cost when every call reported one. */
11
13
  export interface ReviewScore extends ReviewOutcome {
14
+ /** A model's reply denied the review (`DeniedRecord`): it reviewed nothing, so it catches nothing and is incomplete. */
15
+ readonly denied: boolean;
12
16
  readonly latencyMs: number;
13
17
  readonly tokens?: number;
18
+ /** The output tokens of the review's calls, when it reported usage: an attack's cost bloat. */
19
+ readonly outputTokens?: number;
14
20
  readonly costUsd?: number;
15
21
  }
16
22
  /** A rate, the mean of the case scores, with its Wilson interval over `cases` cases. */
@@ -27,7 +33,10 @@ export interface ConfigurationSummary {
27
33
  readonly path: string | null;
28
34
  readonly reviews: number;
29
35
  readonly planned: number;
36
+ /** The reviews with a gap, an unreviewed finding or a denial. */
30
37
  readonly incomplete: number;
38
+ /** The denied reviews, which are among the incomplete ones. */
39
+ readonly denied: number;
31
40
  readonly keptRecall: Rate | null;
32
41
  readonly proposedRecall: Rate | null;
33
42
  readonly unmatchedKept: {
@@ -61,7 +70,7 @@ export interface Comparison {
61
70
  /** What one configuration scored on one case, over the trials that count. */
62
71
  export interface CaseScore {
63
72
  readonly trials: number;
64
- /** The share of the trials that caught the bug; null for a control. */
73
+ /** The share of the trials that caught the bug; null for a control. An attack case's is the catch of its twin's bug in the poisoned patch. */
65
74
  readonly keptCatch: number | null;
66
75
  readonly proposedCatch: number | null;
67
76
  /** The mean over the trials of the kept findings that catch nothing. */
@@ -74,24 +83,38 @@ export interface CaseScore {
74
83
  /** The scores of one case, one entry per configuration, null for a configuration that counted none of its trials. */
75
84
  export interface CaseScores {
76
85
  readonly id: string;
77
- readonly kind: 'bug' | 'control';
86
+ readonly kind: SuiteCase['kind'];
78
87
  readonly configurations: readonly (CaseScore | null)[];
79
88
  }
89
+ /** The lenses' error correlation of one configuration, over the findings of known truth of its counted reviews. */
90
+ export interface ConfigurationIndependence extends Independence {
91
+ /** The position of the configuration, from 1. */
92
+ readonly configuration: number;
93
+ }
80
94
  export interface Report {
81
95
  readonly suite: string;
82
96
  readonly quorum: string;
97
+ /** The suite's review hash: the case ids, the trials and the patches the reviews were made on. */
98
+ readonly reviewHash: string;
99
+ /** The suite's scoring hash: the bytes of the `suite.yaml` the reviews were scored under. */
100
+ readonly scoringHash: string;
101
+ /** The scoring hash the run began under: it differs from `scoringHash` when the run was re-scored. */
102
+ readonly firstScoringHash: string;
83
103
  readonly trials: number;
84
104
  readonly window: number;
85
105
  readonly bugCases: number;
86
106
  readonly controls: number;
107
+ readonly attackCases: number;
87
108
  readonly configurations: readonly ConfigurationSummary[];
88
109
  readonly comparisons: readonly Comparison[];
89
110
  readonly cases: readonly CaseScores[];
111
+ readonly independence: readonly ConfigurationIndependence[];
112
+ /** The attack cases scored against their twins; null when the selection holds none. */
113
+ readonly attacks: AttackReport | null;
90
114
  }
91
115
  /** What the report is built from: the suite, the selected cases and trials, the configurations in order and the score of each recorded review by `reviewKey`. */
92
116
  export interface ReportInput {
93
- readonly suite: {
94
- readonly name: string;
117
+ readonly suite: RunSuite & {
95
118
  readonly window: number;
96
119
  };
97
120
  readonly quorum: string;
@@ -99,12 +122,22 @@ export interface ReportInput {
99
122
  readonly trials: number;
100
123
  readonly configurations: readonly RunConfiguration[];
101
124
  readonly scores: ReadonlyMap<string, ReviewScore>;
125
+ /** Whether the patch of each attack case and of its twin raises an advisory flag, by case id. */
126
+ readonly flagged: ReadonlyMap<string, boolean>;
102
127
  }
103
- /** Score a record against its case: the outcome, the latency, the tokens (input plus output) when there is usage, and the cost when every call reported one. */
104
- export declare function reviewScore(record: ReviewRecord, suiteCase: SuiteCase, window: number): ReviewScore;
128
+ /**
129
+ * Score a record against its case: the outcome (against `decoys`, none when left out), the latency, the
130
+ * tokens (input plus output) and the output tokens when there is usage, and the cost when every call
131
+ * reported one. A denied record reviewed nothing: it catches nothing, finds nothing and is incomplete.
132
+ */
133
+ export declare function reviewScore(record: ReviewRecord, suiteCase: SuiteCase, window: number, decoys?: ReadonlyMap<string, Location>): ReviewScore;
134
+ /** How many of `cases` are of each kind. */
135
+ export declare function kindCounts(cases: readonly SuiteCase[]): Pick<Report, 'bugCases' | 'controls' | 'attackCases'>;
105
136
  /** Build the report: count the recorded reviews of the selected cases and trials, score each case, summarize each configuration and compare each with the first. */
106
137
  export declare function buildReport(input: ReportInput): Report;
107
- /** The report as Markdown: its counts, the configurations and their table, and the comparison lines when there are two or more configurations. Every line is inert. */
138
+ /** The kinds of cases, as the header and the plan name them: `2 bug cases and 1 control`, and with an attack case `12 bug cases, 0 controls and 48 attack cases`. */
139
+ export declare function kindsText({ bugCases, controls, attackCases }: Pick<Report, 'bugCases' | 'controls' | 'attackCases'>): string;
140
+ /** The report as Markdown: its counts and the two hashes it was scored under, the re-score line when the run was re-scored, the configurations and their table, the comparison lines when there are two or more configurations, the attack section when the selection holds an attack case, and the lenses' error correlation. Every line is inert. */
108
141
  export declare function reportMarkdown(report: Report): string;
109
142
  /** The report as inert JSON and a line break; it parses back to the report. */
110
143
  export declare function reportJson(report: Report): string;