@gaunt-sloth/batch 2.0.0-alpha.20 → 2.0.0-alpha.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +4 -0
- package/dist/index.js +6 -0
- package/dist/index.js.map +1 -1
- package/dist/reporters/drive.d.ts +14 -0
- package/dist/reporters/drive.js +49 -0
- package/dist/reporters/drive.js.map +1 -0
- package/dist/reporters/registry.d.ts +11 -0
- package/dist/reporters/registry.js +26 -0
- package/dist/reporters/registry.js.map +1 -0
- package/dist/reporters/reporterTypes.d.ts +36 -0
- package/dist/reporters/reporterTypes.js +2 -0
- package/dist/reporters/reporterTypes.js.map +1 -0
- package/dist/reporters/textReporter.d.ts +10 -0
- package/dist/reporters/textReporter.js +50 -0
- package/dist/reporters/textReporter.js.map +1 -0
- package/package.json +3 -3
package/dist/index.d.ts
CHANGED
|
@@ -14,5 +14,9 @@ export { writeEvalOutput } from '#src/evalOutput.js';
|
|
|
14
14
|
export type { AdkAgentTarget, AgUiAgentTarget, DeterministicCheckResult, EvalCase, EvalCaseResult, EvalExpectation, EvalSuite, EvalSuiteSummary, EvalTarget, GthAgentTarget, EvalTurn, EvalTurnResult, JudgeFn, JudgeOutcome, JudgeVerdict, RunConversationFn, TurnRunOutcome, } from '#src/evalTypes.js';
|
|
15
15
|
export type { RunEvalSuiteOptions } from '#src/evalRunner.js';
|
|
16
16
|
export { DEFAULT_EVAL_PASS_THRESHOLD } from '#src/evalTypes.js';
|
|
17
|
+
export { resolveReporters, availableReporterNames } from '#src/reporters/registry.js';
|
|
18
|
+
export { driveReporters } from '#src/reporters/drive.js';
|
|
19
|
+
export { createTextReporter } from '#src/reporters/textReporter.js';
|
|
20
|
+
export type { EvalReporter, EvalReporterFactory, EvalRunContext, NamedReporter, } from '#src/reporters/reporterTypes.js';
|
|
17
21
|
export { runWorkflow } from '#src/workflow/runWorkflow.js';
|
|
18
22
|
export type { WorkflowAgentOptions, WorkflowContext, RunWorkflowOptions, } from '#src/workflow/runWorkflow.js';
|
package/dist/index.js
CHANGED
|
@@ -11,6 +11,12 @@ export { judgeEvalCase, buildJudgeMessages, EvalVerdictSchema, EVAL_JUDGE_DEFAUL
|
|
|
11
11
|
export { runEvalSuite } from '#src/evalRunner.js';
|
|
12
12
|
export { writeEvalOutput } from '#src/evalOutput.js';
|
|
13
13
|
export { DEFAULT_EVAL_PASS_THRESHOLD } from '#src/evalTypes.js';
|
|
14
|
+
// BATCH-19 — the `gth eval` reporter facility (A1 seam). These are the public plugin contract an
|
|
15
|
+
// out-of-core `@gaunt-sloth/eval-reporter-*` package implements, exported from the package root so a
|
|
16
|
+
// reporter package can type its factory against ONE import.
|
|
17
|
+
export { resolveReporters, availableReporterNames } from '#src/reporters/registry.js';
|
|
18
|
+
export { driveReporters } from '#src/reporters/drive.js';
|
|
19
|
+
export { createTextReporter } from '#src/reporters/textReporter.js';
|
|
14
20
|
// BATCH-3 — the `gth workflow` host: runs a local JS orchestration script that drives one or more
|
|
15
21
|
// LLM calls through a small WorkflowContext.
|
|
16
22
|
export { runWorkflow } from '#src/workflow/runWorkflow.js';
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxE,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAUlD,OAAO,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AAEpD,gGAAgG;AAChG,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AACrE,OAAO,EACL,aAAa,EACb,kBAAkB,EAClB,iBAAiB,EACjB,6BAA6B,GAC9B,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAqBrD,OAAO,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AAEhE,kGAAkG;AAClG,6CAA6C;AAC7C,OAAO,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxE,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAUlD,OAAO,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AAEpD,gGAAgG;AAChG,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AACrE,OAAO,EACL,aAAa,EACb,kBAAkB,EAClB,iBAAiB,EACjB,6BAA6B,GAC9B,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAqBrD,OAAO,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AAEhE,iGAAiG;AACjG,qGAAqG;AACrG,4DAA4D;AAC5D,OAAO,EAAE,gBAAgB,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAC;AACtF,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AAQpE,kGAAkG;AAClG,6CAA6C;AAC7C,OAAO,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { EvalSuiteSummary } from '#src/evalTypes.js';
|
|
2
|
+
import type { EvalRunContext, NamedReporter } from '#src/reporters/reporterTypes.js';
|
|
3
|
+
/**
|
|
4
|
+
* Drive reporters over one completed run's results, in lifecycle order:
|
|
5
|
+
* onSuiteStart → onCellResult per case (summary.cases order) → onSuiteEnd. A reporter hook that
|
|
6
|
+
* throws is caught and surfaced via displayWarning — a reporter must NEVER change the eval exit code
|
|
7
|
+
* or abort the run (the exit code is classifyEvalExit's job alone). Never throws.
|
|
8
|
+
*
|
|
9
|
+
* The lifecycle is phase-grouped across reporters (every reporter's onSuiteStart, then every
|
|
10
|
+
* reporter's onCellResult per case, then every reporter's onSuiteEnd) so a shared surface like the
|
|
11
|
+
* console is never interleaved out of order. With the single built-in `text` reporter this
|
|
12
|
+
* reproduces the former `printSummary` output byte-for-byte.
|
|
13
|
+
*/
|
|
14
|
+
export declare function driveReporters(reporters: NamedReporter[], summary: EvalSuiteSummary, ctx: EvalRunContext): Promise<void>;
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { displayWarning } from '@gaunt-sloth/core/utils/consoleUtils.js';
|
|
2
|
+
/** Run one reporter hook, swallowing any error (thrown OR a rejected Promise, the JUnit reporter's
|
|
3
|
+
* real failure mode) into a warning. A reporter must NEVER change the eval exit code or abort the
|
|
4
|
+
* run (the exit code is `classifyEvalExit`'s job alone), so a failing hook is contained here and the
|
|
5
|
+
* run continues. The warning names WHICH reporter's WHICH hook failed — ambiguous once multiple
|
|
6
|
+
* reporters (text + junit + custom) are real. */
|
|
7
|
+
async function runHookQuietly(reporterName, hook, run) {
|
|
8
|
+
try {
|
|
9
|
+
await run();
|
|
10
|
+
}
|
|
11
|
+
catch (error) {
|
|
12
|
+
displayWarning(`eval reporter "${reporterName}" ${hook} hook failed: ` +
|
|
13
|
+
(error instanceof Error ? error.message : String(error)));
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Drive reporters over one completed run's results, in lifecycle order:
|
|
18
|
+
* onSuiteStart → onCellResult per case (summary.cases order) → onSuiteEnd. A reporter hook that
|
|
19
|
+
* throws is caught and surfaced via displayWarning — a reporter must NEVER change the eval exit code
|
|
20
|
+
* or abort the run (the exit code is classifyEvalExit's job alone). Never throws.
|
|
21
|
+
*
|
|
22
|
+
* The lifecycle is phase-grouped across reporters (every reporter's onSuiteStart, then every
|
|
23
|
+
* reporter's onCellResult per case, then every reporter's onSuiteEnd) so a shared surface like the
|
|
24
|
+
* console is never interleaved out of order. With the single built-in `text` reporter this
|
|
25
|
+
* reproduces the former `printSummary` output byte-for-byte.
|
|
26
|
+
*/
|
|
27
|
+
export async function driveReporters(reporters, summary, ctx) {
|
|
28
|
+
for (const { name, reporter } of reporters) {
|
|
29
|
+
const onSuiteStart = reporter.onSuiteStart;
|
|
30
|
+
if (onSuiteStart) {
|
|
31
|
+
await runHookQuietly(name, 'onSuiteStart', () => onSuiteStart.call(reporter, ctx));
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
for (const result of summary.cases) {
|
|
35
|
+
for (const { name, reporter } of reporters) {
|
|
36
|
+
const onCellResult = reporter.onCellResult;
|
|
37
|
+
if (onCellResult) {
|
|
38
|
+
await runHookQuietly(name, 'onCellResult', () => onCellResult.call(reporter, result, ctx));
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
for (const { name, reporter } of reporters) {
|
|
43
|
+
const onSuiteEnd = reporter.onSuiteEnd;
|
|
44
|
+
if (onSuiteEnd) {
|
|
45
|
+
await runHookQuietly(name, 'onSuiteEnd', () => onSuiteEnd.call(reporter, summary, ctx));
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
//# sourceMappingURL=drive.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"drive.js","sourceRoot":"","sources":["../../src/reporters/drive.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,yCAAyC,CAAC;AAIzE;;;;iDAIiD;AACjD,KAAK,UAAU,cAAc,CAC3B,YAAoB,EACpB,IAAY,EACZ,GAA+B;IAE/B,IAAI,CAAC;QACH,MAAM,GAAG,EAAE,CAAC;IACd,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,cAAc,CACZ,kBAAkB,YAAY,KAAK,IAAI,gBAAgB;YACrD,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAC3D,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,SAA0B,EAC1B,OAAyB,EACzB,GAAmB;IAEnB,KAAK,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,SAAS,EAAE,CAAC;QAC3C,MAAM,YAAY,GAAG,QAAQ,CAAC,YAAY,CAAC;QAC3C,IAAI,YAAY,EAAE,CAAC;YACjB,MAAM,cAAc,CAAC,IAAI,EAAE,cAAc,EAAE,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC;QACrF,CAAC;IACH,CAAC;IAED,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnC,KAAK,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,SAAS,EAAE,CAAC;YAC3C,MAAM,YAAY,GAAG,QAAQ,CAAC,YAAY,CAAC;YAC3C,IAAI,YAAY,EAAE,CAAC;gBACjB,MAAM,cAAc,CAAC,IAAI,EAAE,cAAc,EAAE,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC;YAC7F,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,SAAS,EAAE,CAAC;QAC3C,MAAM,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC;QACvC,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,cAAc,CAAC,IAAI,EAAE,YAAY,EAAE,GAAG,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;QAC1F,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { EvalReporterFactory, NamedReporter } from '#src/reporters/reporterTypes.js';
|
|
2
|
+
/** Resolve reporter names to named instances ({@link NamedReporter}). `custom` (A2 — the bundled
|
|
3
|
+
* JUnit reporter and any config-declared user reporters) overlays/extends the built-ins through this
|
|
4
|
+
* ONE path; a `custom` key colliding with a built-in name wins (config beats built-in). A name found
|
|
5
|
+
* in neither throws with the available list (the command maps that to exit 2). Reporters are
|
|
6
|
+
* instantiated per call (fresh state per run). Each is paired with the name it was selected under so
|
|
7
|
+
* {@link driveReporters} can name a failing reporter in its contained-error warning. */
|
|
8
|
+
export declare function resolveReporters(names: string[], custom?: Record<string, EvalReporterFactory>): NamedReporter[];
|
|
9
|
+
/** The names `resolveReporters` accepts, given the same `custom` map — used to build the error
|
|
10
|
+
* message on an unknown name (and, in A2, to list reporters in help/errors). */
|
|
11
|
+
export declare function availableReporterNames(custom?: Record<string, EvalReporterFactory>): string[];
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { createTextReporter } from '#src/reporters/textReporter.js';
|
|
2
|
+
/** The built-in reporters, keyed by the name a user (later, A2) selects with `--reporter`. A1 ships
|
|
3
|
+
* only `text` (the former `printSummary`); A2 adds a JUnit reporter and a custom-reporter map. */
|
|
4
|
+
const BUILTIN_REPORTERS = { text: createTextReporter };
|
|
5
|
+
/** Resolve reporter names to named instances ({@link NamedReporter}). `custom` (A2 — the bundled
|
|
6
|
+
* JUnit reporter and any config-declared user reporters) overlays/extends the built-ins through this
|
|
7
|
+
* ONE path; a `custom` key colliding with a built-in name wins (config beats built-in). A name found
|
|
8
|
+
* in neither throws with the available list (the command maps that to exit 2). Reporters are
|
|
9
|
+
* instantiated per call (fresh state per run). Each is paired with the name it was selected under so
|
|
10
|
+
* {@link driveReporters} can name a failing reporter in its contained-error warning. */
|
|
11
|
+
export function resolveReporters(names, custom = {}) {
|
|
12
|
+
const registry = { ...BUILTIN_REPORTERS, ...custom };
|
|
13
|
+
return names.map((name) => {
|
|
14
|
+
const factory = registry[name];
|
|
15
|
+
if (!factory) {
|
|
16
|
+
throw new Error(`unknown reporter "${name}". Available reporters: ${availableReporterNames(custom).join(', ')}`);
|
|
17
|
+
}
|
|
18
|
+
return { name, reporter: factory() };
|
|
19
|
+
});
|
|
20
|
+
}
|
|
21
|
+
/** The names `resolveReporters` accepts, given the same `custom` map — used to build the error
|
|
22
|
+
* message on an unknown name (and, in A2, to list reporters in help/errors). */
|
|
23
|
+
export function availableReporterNames(custom = {}) {
|
|
24
|
+
return Object.keys({ ...BUILTIN_REPORTERS, ...custom });
|
|
25
|
+
}
|
|
26
|
+
//# sourceMappingURL=registry.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"registry.js","sourceRoot":"","sources":["../../src/reporters/registry.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AAEpE;kGACkG;AAClG,MAAM,iBAAiB,GAAwC,EAAE,IAAI,EAAE,kBAAkB,EAAE,CAAC;AAE5F;;;;;wFAKwF;AACxF,MAAM,UAAU,gBAAgB,CAC9B,KAAe,EACf,MAAM,GAAwC,EAAE;IAEhD,MAAM,QAAQ,GAAwC,EAAE,GAAG,iBAAiB,EAAE,GAAG,MAAM,EAAE,CAAC;IAC1F,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC/B,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CACb,qBAAqB,IAAI,2BAA2B,sBAAsB,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAChG,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE,CAAC;IACvC,CAAC,CAAC,CAAC;AACL,CAAC;AAED;gFACgF;AAChF,MAAM,UAAU,sBAAsB,CAAC,MAAM,GAAwC,EAAE;IACrF,OAAO,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,iBAAiB,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;AAC1D,CAAC"}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type { EvalCaseResult, EvalSuiteSummary } from '#src/evalTypes.js';
|
|
2
|
+
/** Immutable per-run context handed to every reporter hook. Carries what a reporter needs beyond the
|
|
3
|
+
* result objects themselves. `suitePath`/`outputDir` are already known to the command; passing them
|
|
4
|
+
* now (unused by the text reporter's console output except outputDir) sets up A2 (JUnit
|
|
5
|
+
* <testsuite name>) and B (per-suite output dirs) without another interface change. */
|
|
6
|
+
export interface EvalRunContext {
|
|
7
|
+
/** The suite file path as given on the CLI. */
|
|
8
|
+
suitePath: string;
|
|
9
|
+
/** The directory this run's structured output (results.json etc.) is written to. */
|
|
10
|
+
outputDir: string;
|
|
11
|
+
/** The BATCH-10-Task-2 self-describing judge line, when a separate judge profile is in effect. */
|
|
12
|
+
judgeNotice?: {
|
|
13
|
+
profile: string;
|
|
14
|
+
model?: string;
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
/** A reporter renders an eval run. Every hook is optional — a reporter implements only what it needs
|
|
18
|
+
* (the text reporter uses all three; a future file reporter may use only onSuiteEnd). Hooks are
|
|
19
|
+
* driven in lifecycle order by {@link driveReporters}. A reporter MUST NOT be able to fail the run:
|
|
20
|
+
* the driver catches any hook error and surfaces it as a warning (see driveReporters). */
|
|
21
|
+
export interface EvalReporter {
|
|
22
|
+
onSuiteStart?(ctx: EvalRunContext): void | Promise<void>;
|
|
23
|
+
onCellResult?(result: EvalCaseResult, ctx: EvalRunContext): void | Promise<void>;
|
|
24
|
+
onSuiteEnd?(summary: EvalSuiteSummary, ctx: EvalRunContext): void | Promise<void>;
|
|
25
|
+
}
|
|
26
|
+
/** Reporters are created per run (they may accumulate state, e.g. a JUnit doc), so the registry
|
|
27
|
+
* stores factories, not instances. */
|
|
28
|
+
export type EvalReporterFactory = () => EvalReporter;
|
|
29
|
+
/** A resolved reporter paired with the name it was selected under. {@link resolveReporters} returns
|
|
30
|
+
* these (not bare {@link EvalReporter}s) so {@link driveReporters} can name WHICH reporter's hook
|
|
31
|
+
* failed in the contained-error warning — with multiple reporters real (A2: text + junit + custom),
|
|
32
|
+
* a bare "onCellResult hook failed" is ambiguous. */
|
|
33
|
+
export interface NamedReporter {
|
|
34
|
+
name: string;
|
|
35
|
+
reporter: EvalReporter;
|
|
36
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reporterTypes.js","sourceRoot":"","sources":["../../src/reporters/reporterTypes.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { EvalReporter } from '#src/reporters/reporterTypes.js';
|
|
2
|
+
/**
|
|
3
|
+
* The built-in default reporter: the human-readable, `review`-flavored summary — one PASS/FAIL line
|
|
4
|
+
* per cell (failures carry their reasons), then a suite-total line. Doesn't replicate `review`'s
|
|
5
|
+
* exact `REVIEW RATING` block format, just its spirit (a scannable verdict, not a wall of JSON).
|
|
6
|
+
*
|
|
7
|
+
* A1: this is a byte-for-byte port of `evalCommand.ts`'s former `printSummary` onto the reporter
|
|
8
|
+
* lifecycle — same lines, same order, same `display`/`displayWarning`/`displaySuccess` channels.
|
|
9
|
+
*/
|
|
10
|
+
export declare function createTextReporter(): EvalReporter;
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { display, displaySuccess, displayWarning } from '@gaunt-sloth/core/utils/consoleUtils.js';
|
|
2
|
+
/**
|
|
3
|
+
* The built-in default reporter: the human-readable, `review`-flavored summary — one PASS/FAIL line
|
|
4
|
+
* per cell (failures carry their reasons), then a suite-total line. Doesn't replicate `review`'s
|
|
5
|
+
* exact `REVIEW RATING` block format, just its spirit (a scannable verdict, not a wall of JSON).
|
|
6
|
+
*
|
|
7
|
+
* A1: this is a byte-for-byte port of `evalCommand.ts`'s former `printSummary` onto the reporter
|
|
8
|
+
* lifecycle — same lines, same order, same `display`/`displayWarning`/`displaySuccess` channels.
|
|
9
|
+
*/
|
|
10
|
+
export function createTextReporter() {
|
|
11
|
+
return {
|
|
12
|
+
onSuiteStart(ctx) {
|
|
13
|
+
// BATCH-10 Task 2: when a separate judge profile is in effect, lead with a single
|
|
14
|
+
// self-describing line so a captured run records which model graded it (reproducibility).
|
|
15
|
+
// Emitted only for a separate judge — the default SUT-as-judge run prints exactly as before.
|
|
16
|
+
if (ctx.judgeNotice) {
|
|
17
|
+
display(`Judge: profile "${ctx.judgeNotice.profile}"` +
|
|
18
|
+
(ctx.judgeNotice.model ? ` (model: ${ctx.judgeNotice.model})` : ''));
|
|
19
|
+
}
|
|
20
|
+
},
|
|
21
|
+
onCellResult(result) {
|
|
22
|
+
// BATCH-12: an identity-matrix cell tags its line with the identity it ran under, so the two
|
|
23
|
+
// rows a case produces per (case × identity) are distinguishable. A no-identities cell prints
|
|
24
|
+
// exactly as before (just the case id).
|
|
25
|
+
const label = result.identity ? `${result.id} [${result.identity}]` : result.id;
|
|
26
|
+
if (result.verdict === 'PASS') {
|
|
27
|
+
display(`PASS ${label}`);
|
|
28
|
+
}
|
|
29
|
+
else {
|
|
30
|
+
displayWarning(`FAIL ${label} — ${result.reasons.join('; ') || 'no reason recorded'}`);
|
|
31
|
+
}
|
|
32
|
+
},
|
|
33
|
+
onSuiteEnd(summary, ctx) {
|
|
34
|
+
// M1: X/Y counts CELLS in a matrix run (one per case × identity) — e.g. `2/2` for 1 case × 2
|
|
35
|
+
// identities — so a bare "case(s)" would misreport the denominator. Use an identity-aware
|
|
36
|
+
// noun: "case(s)" for a no-identities run (unchanged), "cell(s)" once any cell carries an
|
|
37
|
+
// identity.
|
|
38
|
+
const isMatrix = summary.cases.some((caseResult) => caseResult.identity !== undefined);
|
|
39
|
+
const noun = isMatrix ? 'cell(s)' : 'case(s)';
|
|
40
|
+
const verdictLine = `EVAL RESULT: ${summary.passed}/${summary.total} ${noun} passed`;
|
|
41
|
+
if (summary.failed === 0) {
|
|
42
|
+
displaySuccess(`${verdictLine}. Results written to ${ctx.outputDir}`);
|
|
43
|
+
}
|
|
44
|
+
else {
|
|
45
|
+
displayWarning(`${verdictLine}, ${summary.failed} failed. Results written to ${ctx.outputDir}`);
|
|
46
|
+
}
|
|
47
|
+
},
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
//# sourceMappingURL=textReporter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"textReporter.js","sourceRoot":"","sources":["../../src/reporters/textReporter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,yCAAyC,CAAC;AAIlG;;;;;;;GAOG;AACH,MAAM,UAAU,kBAAkB;IAChC,OAAO;QACL,YAAY,CAAC,GAAmB;YAC9B,kFAAkF;YAClF,0FAA0F;YAC1F,6FAA6F;YAC7F,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC;gBACpB,OAAO,CACL,mBAAmB,GAAG,CAAC,WAAW,CAAC,OAAO,GAAG;oBAC3C,CAAC,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,WAAW,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CACtE,CAAC;YACJ,CAAC;QACH,CAAC;QAED,YAAY,CAAC,MAAsB;YACjC,6FAA6F;YAC7F,8FAA8F;YAC9F,wCAAwC;YACxC,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,EAAE,KAAK,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;YAChF,IAAI,MAAM,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;gBAC9B,OAAO,CAAC,SAAS,KAAK,EAAE,CAAC,CAAC;YAC5B,CAAC;iBAAM,CAAC;gBACN,cAAc,CAAC,SAAS,KAAK,MAAM,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,oBAAoB,EAAE,CAAC,CAAC;YAC1F,CAAC;QACH,CAAC;QAED,UAAU,CAAC,OAAyB,EAAE,GAAmB;YACvD,6FAA6F;YAC7F,0FAA0F;YAC1F,0FAA0F;YAC1F,YAAY;YACZ,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC;YACvF,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;YAC9C,MAAM,WAAW,GAAG,gBAAgB,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,KAAK,IAAI,IAAI,SAAS,CAAC;YACrF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACzB,cAAc,CAAC,GAAG,WAAW,wBAAwB,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC;YACxE,CAAC;iBAAM,CAAC;gBACN,cAAc,CACZ,GAAG,WAAW,KAAK,OAAO,CAAC,MAAM,+BAA+B,GAAG,CAAC,SAAS,EAAE,CAChF,CAAC;YACJ,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gaunt-sloth/batch",
|
|
3
|
-
"version": "2.0.0-alpha.
|
|
3
|
+
"version": "2.0.0-alpha.22",
|
|
4
4
|
"description": "Batch matrix runtime for Gaunt Sloth (BATCH-1): run a prompt-executable over a matrix of models and/or content-bound inputs",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Andrew Kondratev",
|
|
@@ -37,8 +37,8 @@
|
|
|
37
37
|
"@langchain/core": "^1.2.3",
|
|
38
38
|
"yaml": "^2.9.0",
|
|
39
39
|
"zod": "^4.4.3",
|
|
40
|
-
"@gaunt-sloth/agent": "2.0.0-alpha.
|
|
41
|
-
"@gaunt-sloth/core": "2.0.0-alpha.
|
|
40
|
+
"@gaunt-sloth/agent": "2.0.0-alpha.22",
|
|
41
|
+
"@gaunt-sloth/core": "2.0.0-alpha.22"
|
|
42
42
|
},
|
|
43
43
|
"files": [
|
|
44
44
|
"./dist/*"
|