@gaunt-sloth/batch 2.0.0-alpha.24 → 2.0.0-alpha.26

package/dist/evalTypes.js

@@ -4,4 +4,102 @@
  * Kept as our own constant (not imported) since review's is private to that module and coupled to
  * middleware/tool-call/artifact-store plumbing that doesn't fit a plain structured-output call. */
 export const DEFAULT_EVAL_PASS_THRESHOLD = 6;
+/**
+ * BATCH-25 Half B — the stable marker a `rater` rationale carries when the §8 hardline floor refuses
+ * the command: the one gradeable form of "this never reaches a shell, whatever the rater said".
+ *
+ * It is a RATIONALE marker rather than a label or an action on purpose. The floor is not a rung
+ * decision: it fires at execution time inside the shell tool, under every rung, after the approvals
+ * gate has already returned. It therefore has no representation in the rater's label/action
+ * vocabulary, and inventing one here would be this package forming an opinion about a mechanism it
+ * does not own. Instead it lands in the text, where a `forced_by: hardline-floor` case (or a plain
+ * `must_contain`) grades it.
+ *
+ * It lives in this module rather than beside its producer so the SUITE PARSER can desugar
+ * `forced_by:` without importing `raterTarget.js` — which would pull `@gaunt-sloth/agent` and core's
+ * rater into the module graph of every suite parse.
+ */
+export const HARDLINE_REFUSAL_MARKER = 'hardline floor: refused';
+/**
+ * BATCH-25 Half B — the deterministic mechanisms of the approvals gate that a case can assert fired,
+ * spelled EXACTLY as the approvals corpus spells them in its own `forced_by` / `floor_refuses`
+ * fields, so transcribing a corpus case is a copy rather than a translation.
+ *
+ * ## Why this exists (the I1 finding)
+ *
+ * A model-free case cannot be graded on its `action`. With no verdict, core's decision mapping
+ * substitutes its fail-closed one and returns the SAME action for every command at a rated rung —
+ * `ls -la`, `rm -rf /` and `rm -rf $(echo /)` are indistinguishable. `expect_action: escalate` on a
+ * deterministic case therefore passes for a directory listing, and would still pass with the floor
+ * and both preflights deleted: a constant, not a regression gate.
+ *
+ * What DOES discriminate is **which deterministic mechanism fired**, which the target reports in the
+ * rationale, and which is exactly what the corpus records. So a deterministic case asserts that.
+ */
+export const FORCED_BY_MECHANISMS = [
+    'hardline-floor',
+    'ambiguity-preflight',
+    'script-env-leak-preflight',
+];
+/** The prefix a rationale carries for a decision a preflight forced, followed by the mechanism. */
+export const FORCED_BY_MARKER = 'forced by';
+/**
+ * The exact substring a `forced_by: <mechanism>` assertion looks for in the rationale. The floor's
+ * is {@link HARDLINE_REFUSAL_MARKER} — the marker it already emits — so there is one marker per
+ * mechanism and no second spelling of the same claim.
+ */
+export const FORCED_BY_ASSERTIONS = {
+    'hardline-floor': HARDLINE_REFUSAL_MARKER,
+    'ambiguity-preflight': `${FORCED_BY_MARKER}: ambiguity-preflight`,
+    'script-env-leak-preflight': `${FORCED_BY_MARKER}: script-env-leak-preflight`,
+};
+/**
+ * Which of {@link FORCED_BY_MECHANISMS} are **preflights of the decision mapping** — the ones that
+ * can only be observed by handing the gate a rating for them to OVERRIDE.
+ *
+ * **This is the single statement of the rule, and everything else reads it through
+ * {@link mechanismNeedsPermissiveRating}.** It is kept here, next to the mechanism list, rather than
+ * in `raterTarget.js`, so the runner can consult it without pulling `@gaunt-sloth/agent` and core's
+ * rater into the module graph — and it is pinned by a unit test to equal the target's own probe
+ * table, so the two cannot drift.
+ *
+ * ## Why the floor is not on this list (measured, CFG-28)
+ *
+ * The two preflights live INSIDE `mapVerdictToAction`, and CFG-28 narrowed them to raise only an
+ * outcome that sits below the deterministic floor. Give that mapping no verdict and it substitutes
+ * core's fail-closed one, which is *at* the floor — so no preflight rewrites anything, every command
+ * comes back with the identical placeholder sentence, and the assertion is a constant again. Handing
+ * it a permissive rating is not a hypothetical: **overriding a permissive rating is the entire job of
+ * a preflight**, so it is the one path on which the mechanism is real.
+ *
+ * The §8 hardline floor is the opposite by design. The corpus defines `floor_refuses` as a refusal
+ * *at execution time*, "regardless of the rating, the approval, or the allow-list", and explicitly
+ * "a SEPARATE assertion from `action`". The target reads it from `checkHardline`, which never sees a
+ * verdict — so a rating cannot change the answer, and supplying one buys nothing. It also costs:
+ * `mapVerdictToAction` does not consult the floor, so a permissive rating on `rm -rf /` returns
+ * `approve`, which would move the action column of **5 of the corpus's 6 `fl-*` cases** off the
+ * `escalate` they author. Measured over all 22 deterministic corpus cases; see the fix-round-2
+ * section of `handoff/task-1-report.md`.
+ *
+ * ## The property that makes this self-justifying
+ *
+ * When the claimed preflight really does fire, the rating does not move the action — every corpus
+ * case that claims one (`ob-02`…`ob-05`, `el-01`, `el-02`) is `escalate` with or without it. When it
+ * does not fire, the marker and the action go red **together**. That is the discrimination, not a
+ * side effect of it.
+ */
+export const PREFLIGHT_MECHANISMS = [
+    'ambiguity-preflight',
+    'script-env-leak-preflight',
+];
+/**
+ * Must a round claiming this mechanism be driven with a permissive rating for the mechanism to be
+ * observable at all? See {@link PREFLIGHT_MECHANISMS} for the whole rule and why the floor is
+ * excluded. `undefined` (the round claims no mechanism) is `false`: an ordinary `model_free` case
+ * must keep going through the gate with no verdict, or its action would change.
+ */
+export function mechanismNeedsPermissiveRating(mechanism) {
+    return (mechanism !== undefined &&
+        PREFLIGHT_MECHANISMS.includes(mechanism));
+}
 //# sourceMappingURL=evalTypes.js.map

package/dist/evalTypes.js.map

@@ -1 +1 @@
-{"version":3,"file":"evalTypes.js","sourceRoot":"","sources":["../src/evalTypes.ts"],"names":[],"mappings":"AASA;;;;mGAImG;AACnG,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,CAAC"}
+{"version":3,"file":"evalTypes.js","sourceRoot":"","sources":["../src/evalTypes.ts"],"names":[],"mappings":"AAgBA;;;;mGAImG;AACnG,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,CAAC;AAE7C;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,yBAAyB,CAAC;AAEjE;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG;IAClC,gBAAgB;IAChB,qBAAqB;IACrB,2BAA2B;CACnB,CAAC;AAKX,mGAAmG;AACnG,MAAM,CAAC,MAAM,gBAAgB,GAAG,WAAW,CAAC;AAE5C;;;;GAIG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAsC;IACrE,gBAAgB,EAAE,uBAAuB;IACzC,qBAAqB,EAAE,GAAG,gBAAgB,uBAAuB;IACjE,2BAA2B,EAAE,GAAG,gBAAgB,6BAA6B;CAC9E,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG;IAClC,qBAAqB;IACrB,2BAA2B;CACoB,CAAC;AAWlD;;;;;GAKG;AACH,MAAM,UAAU,8BAA8B,CAAC,SAAwC;IACrF,OAAO,CACL,SAAS,KAAK,SAAS;QACtB,oBAAqD,CAAC,QAAQ,CAAC,SAAS,CAAC,CAC3E,CAAC;AACJ,CAAC"}

package/dist/index.d.ts

@@ -4,7 +4,7 @@ export { parseOverFile } from '#src/parseOver.js';
 export { runBatchMatrix, buildBatchSummary } from '#src/BatchRunner.js';
 export { writeBatchOutput } from '#src/output.js';
 export type { BatchRunnerOptions, BatchSummary, CellResult, CellRunOutcome, MatrixCell, MatrixRow, RunCellFn, ToolResultRecord, } from '#src/types.js';
-export { DEFAULT_CONCURRENCY } from '#src/types.js';
+export { DEFAULT_CELL_CONCURRENCY } from '#src/types.js';
 export { parseEvalSuite } from '#src/evalSuite.js';
 export { runDeterministicChecks } from '#src/deterministicChecks.js';
 export { judgeEvalCase, buildJudgeMessages, EvalVerdictSchema, EVAL_JUDGE_DEFAULT_TIMEOUT_MS, } from '#src/judge.js';
@@ -14,9 +14,23 @@ export { writeEvalOutput } from '#src/evalOutput.js';
 export type { AdkAgentTarget, AgUiAgentTarget, DeterministicCheckResult, EvalCase, EvalCaseResult, EvalExpectation, EvalSuite, EvalSuiteSummary, EvalTarget, GthAgentTarget, EvalTurn, EvalTurnResult, JudgeFn, JudgeOutcome, JudgeVerdict, RunConversationFn, TurnRunOutcome, } from '#src/evalTypes.js';
 export type { RunEvalSuiteOptions } from '#src/evalRunner.js';
 export { DEFAULT_EVAL_PASS_THRESHOLD } from '#src/evalTypes.js';
+export { extractClassificationValue, buildConfusionMatrix, collectTags, readRaw, } from '#src/classification.js';
+export { buildClassificationReport } from '#src/classificationReport.js';
+export { computeMetric, parseMetricPredicate, evaluatePredicate, evaluatePredicates, formatTally, } from '#src/metrics.js';
+export { renderClassificationReport, renderConfusionMatrix, renderMetric, } from '#src/classificationRender.js';
+export { buildBlindExport, diffRelabel, renderRelabelDiff } from '#src/blindExport.js';
+export type { BlindExport, BlindExportCase, RelabelDiff, RelabelEntry } from '#src/blindExport.js';
+export { expandSweep, deepMerge, renderComparison, diffRuns, renderRunDiff, } from '#src/evalCompare.js';
+export type { ComparisonColumn, RunDiff, RunDiffEntry, SweepCell } from '#src/evalCompare.js';
+export { UNRECOGNIZED_LABEL, NO_EXPECTATION } from '#src/classificationTypes.js';
+export type { ClassificationExtractor, ClassifiedCell, EvalCaseClassification, EvalClassificationReport, EvalClassificationSpec, EvalConfusionMatrix, EvalMetricCoverage, EvalMetricResult, EvalMetricSpec, EvalMetricTally, MetricField, MetricPredicate, } from '#src/classificationTypes.js';
+export type { ClassifyOutcome, ClassifyRequest, EvalSweep, EvalSweepValue, RaterTarget, RunClassifyFn, } from '#src/evalTypes.js';
+export { buildRaterClassifier, HARDLINE_REFUSAL_MARKER, NO_RATING_CALL_MARKER, } from '#src/raterTarget.js';
+export type { RaterClassifierOptions } from '#src/raterTarget.js';
 export { resolveReporters, availableReporterNames } from '#src/reporters/registry.js';
 export { driveReporters } from '#src/reporters/drive.js';
 export { createTextReporter } from '#src/reporters/textReporter.js';
 export type { EvalReporter, EvalReporterFactory, EvalRunContext, NamedReporter, } from '#src/reporters/reporterTypes.js';
 export { runWorkflow } from '#src/workflow/runWorkflow.js';
+export { DEFAULT_WORKFLOW_CONCURRENCY } from '#src/types.js';
 export type { WorkflowAgentOptions, WorkflowContext, RunWorkflowOptions, } from '#src/workflow/runWorkflow.js';

package/dist/index.js

@@ -3,7 +3,7 @@ export { bindCellContent } from '#src/interpolate.js';
 export { parseOverFile } from '#src/parseOver.js';
 export { runBatchMatrix, buildBatchSummary } from '#src/BatchRunner.js';
 export { writeBatchOutput } from '#src/output.js';
-export { DEFAULT_CONCURRENCY } from '#src/types.js';
+export { DEFAULT_CELL_CONCURRENCY } from '#src/types.js';
 // BATCH-2 — `gth eval`'s suite parsing, deterministic checks, judge, runner, and output writer.
 export { parseEvalSuite } from '#src/evalSuite.js';
 export { runDeterministicChecks } from '#src/deterministicChecks.js';
@@ -11,6 +11,18 @@ export { judgeEvalCase, buildJudgeMessages, EvalVerdictSchema, EVAL_JUDGE_DEFAUL
 export { runEvalSuite } from '#src/evalRunner.js';
 export { writeEvalOutput } from '#src/evalOutput.js';
 export { DEFAULT_EVAL_PASS_THRESHOLD } from '#src/evalTypes.js';
+// BATCH-25 — the classifier eval facility: label/action assertions, the confusion matrix, declared
+// metrics with corpus-wide denominators, blind export, and the sweep/run-over-run comparison.
+export { extractClassificationValue, buildConfusionMatrix, collectTags, readRaw, } from '#src/classification.js';
+export { buildClassificationReport } from '#src/classificationReport.js';
+export { computeMetric, parseMetricPredicate, evaluatePredicate, evaluatePredicates, formatTally, } from '#src/metrics.js';
+export { renderClassificationReport, renderConfusionMatrix, renderMetric, } from '#src/classificationRender.js';
+export { buildBlindExport, diffRelabel, renderRelabelDiff } from '#src/blindExport.js';
+export { expandSweep, deepMerge, renderComparison, diffRuns, renderRunDiff, } from '#src/evalCompare.js';
+export { UNRECOGNIZED_LABEL, NO_EXPECTATION } from '#src/classificationTypes.js';
+// BATCH-25 Half B — the `rater` target: the one implementation of the `RunClassifyFn` seam, which
+// drives gth's own approvals rater over a corpus of commands.
+export { buildRaterClassifier, HARDLINE_REFUSAL_MARKER, NO_RATING_CALL_MARKER, } from '#src/raterTarget.js';
 // BATCH-19 — the `gth eval` reporter facility (A1 seam). These are the public plugin contract an
 // out-of-core `@gaunt-sloth/eval-reporter-*` package implements, exported from the package root so a
 // reporter package can type its factory against ONE import.
@@ -20,4 +32,5 @@ export { createTextReporter } from '#src/reporters/textReporter.js';
 // BATCH-3 — the `gth workflow` host: runs a local JS orchestration script that drives one or more
 // LLM calls through a small WorkflowContext.
 export { runWorkflow } from '#src/workflow/runWorkflow.js';
+export { DEFAULT_WORKFLOW_CONCURRENCY } from '#src/types.js';
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxE,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAWlD,OAAO,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AAEpD,gGAAgG;AAChG,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AACrE,OAAO,EACL,aAAa,EACb,kBAAkB,EAClB,iBAAiB,EACjB,6BAA6B,GAC9B,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAqBrD,OAAO,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AAEhE,iGAAiG;AACjG,qGAAqG;AACrG,4DAA4D;AAC5D,OAAO,EAAE,gBAAgB,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAC;AACtF,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AAQpE,kGAAkG;AAClG,6CAA6C;AAC7C,OAAO,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxE,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAWlD,OAAO,EAAE,wBAAwB,EAAE,MAAM,eAAe,CAAC;AAEzD,gGAAgG;AAChG,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AACrE,OAAO,EACL,aAAa,EACb,kBAAkB,EAClB,iBAAiB,EACjB,6BAA6B,GAC9B,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAqBrD,OAAO,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AAEhE,mGAAmG;AACnG,8FAA8F;AAC9F,OAAO,EACL,0BAA0B,EAC1B,oBAAoB,EACpB,WAAW,EACX,OAAO,GACR,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,yBAAyB,EAAE,MAAM,8BAA8B,CAAC;AACzE,OAAO,EACL,aAAa,EACb,oBAAoB,EACpB,iBAAiB,EACjB,kBAAkB,EAClB,WAAW,GACZ,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,0BAA0B,EAC1B,qBAAqB,EACrB,YAAY,GACb,MAAM,8BAA8B,CAAC;AACtC,OAAO,EAAE,gBAAgB,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAEvF,OAAO,EACL,WAAW,EACX,SAAS,EACT,gBAAgB,EAChB,QAAQ,EACR,aAAa,GACd,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAwBjF,kGAAkG;AAClG,8DAA8D;AAC9D,OAAO,EACL,oBAAoB,EACpB,uBAAuB,EACvB,qBAAqB,GACtB,MAAM,qBAAqB,CAAC;AAG7B,iGAAiG;AACjG,qGAAqG;AACrG,4DAA4D;AAC5D,OAAO,EAAE,gBAAgB,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAC;AACtF,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AAQpE,kGAAkG;AAClG,6CAA6C;AAC7C,OAAO,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAC3D,OAAO,EAAE,4BAA4B,EAAE,MAAM,eAAe,CAAC"}

package/dist/metrics.d.ts

@@ -0,0 +1,50 @@
+import type { ClassifiedCell, EvalClassificationSpec, EvalMetricResult, EvalMetricSpec, EvalMetricTally, MetricPredicate } from '#src/classificationTypes.js';
+/**
+ * Parse ONE predicate string. Predicates come as a LIST and are ANDed; there is no `or`, no nesting
+ * and no parentheses — see {@link MetricPredicate} for why that limit is deliberate.
+ *
+ * Accepted forms:
+ * - `expected.label != safe` · `actual.action == approve`
+ * - `actual.label == expected.label` (field-to-field — this is how accuracy is written)
+ * - `expected.label in [destructive, catastrophic, attack]` · `actual.action not in [approve]`
+ * - `has_tag(injection)` · `not has_tag(negotiation)`
+ *
+ * The literal `none` matches a cell where the field is absent.
+ *
+ * Every literal is validated against the suite's declared enum. That check is the point: a typo'd
+ * value would make the predicate unsatisfiable and the metric a permanent, trusted zero.
+ *
+ * @throws Error with an actionable message (the offending text, the field, and the declared enum).
+ */
+export declare function parseMetricPredicate(raw: string, spec: EvalClassificationSpec, where: string): MetricPredicate;
+/** Evaluate ONE predicate against one cell. */
+export declare function evaluatePredicate(predicate: MetricPredicate, cell: ClassifiedCell): boolean;
+/** All predicates in a list must hold (ANDed). An empty list is vacuously true — that is what "the
+ * whole corpus" means for an absent `over:`. */
+export declare function evaluatePredicates(predicates: MetricPredicate[], cell: ClassifiedCell): boolean;
+/**
+ * Compute one declared metric over the corpus — overall and per tag — together with everything that
+ * bounds what it can see.
+ *
+ * The denominator is every SCORED cell satisfying `over` (every scored cell when `over` is absent).
+ * The numerator is the cells in that denominator that also satisfy `where`.
+ *
+ * Warnings, in the order they matter:
+ * - **numerator outside the denominator** — the throwaway's exact bug shape: cases that satisfy what
+ *   the metric is counting but sit outside what it divides by, so the metric cannot see them;
+ * - **subset denominator** — fired on the evaluated count, so it catches both a narrowing `over:`
+ *   and cells lost to SUT failures;
+ * - **excluded cells** — cells that never produced a classification at all;
+ * - **empty denominator** — the metric measured nothing, so its value is `null` rather than a
+ *   flattering zero.
+ *
+ * @param cells Every cell in the suite, scored and unscored alike (the unscored ones are what make
+ *   the coverage numbers honest).
+ */
+export declare function computeMetric(spec: EvalMetricSpec, cells: ClassifiedCell[], tags: string[]): EvalMetricResult;
+/** True when this metric's thresholds are absolute COUNTS rather than fractions. The two forms are
+ * mutually exclusive per metric (enforced at parse time), so one flag decides the whole reading. */
+export declare function isCountGate(spec: EvalMetricSpec): boolean;
+/** Format a tally for human output: `3/29 (10.3%)`, or `n/a (0 cases)` for an empty denominator —
+ * which must never render as `0.0%`. */
+export declare function formatTally(tally: EvalMetricTally): string;