@tuned-tensor/local 0.2.9 → 0.3.0

package/dist/evaluation.d.ts

@@ -1,83 +1,33 @@
 import { type BehaviorSpecExample, type EvalReport, type EvalSplit, type LocalRunnerConfig } from "./contracts.js";
 import type { LocalRunReporter } from "./run-reporter.js";
-/**
- * Token-overlap F1 between expected and actual output (bag-of-words, case and
- * whitespace insensitive). This is a cheap deterministic reference-similarity
- * signal for free-text tasks where exact match is always 0 and an LLM judge
- * can be noisy; both scoring paths keep working unchanged alongside it.
- */
+/** Cheap deterministic free-text similarity reported beside exact match. */
 export declare function tokenF1(expected: string, actual: string): number;
-/** Derives a deterministic 32-bit seed from an arbitrary string (FNV-1a). */
+/** Deterministic 32-bit FNV-1a seed. */
 export declare function deriveSampleSeed(value: string): number;
-/**
- * Selects `count` examples with a seeded shuffle so that truncated evaluation
- * is a deterministic random sample instead of a dataset-order prefix. The
- * selected examples keep their original relative order, so baseline and
- * candidate evaluations that use the same seed score identical examples.
- */
 export declare function sampleExamples<T>(examples: T[], count: number, seed: number): T[];
 /**
- * Deterministically splits spec examples into a training split and an eval
- * holdout so that default spec runs do not evaluate on their own training
- * data. The holdout is a seeded random sample of about `holdoutRatio` of the
- * examples, with at least 1 holdout and at least 1 training example. Both
- * splits preserve the original example order. With fewer than 2 examples the
- * holdout is empty and callers should fall back to training-set evaluation.
+ * Inline specs get a deterministic holdout. A real run with fewer than two
+ * examples is rejected by the shared validator before this function is used.
  */
 export declare function splitSpecExamples<T>(examples: T[], seed: number, holdoutRatio?: number): {
     train: T[];
     holdout: T[];
 };
+export declare const INFERENCE_PROTOCOL_VERSION = 2;
 /**
- * Builds the judge messages. The spec's compiled system message (system
- * prompt, guidelines, and constraints) is forwarded as task_instructions so
- * the judge scores conformance to the task, not just similarity to the
- * reference. Without it, a judge treats `expected` as a fact checklist and
- * systematically penalizes outputs trained toward a different style (for
- * example concise summaries) even when they follow the spec.
- */
-export declare function buildJudgeMessages(args: {
-    prompt: string;
-    expected: string;
-    actual: string;
-    taskInstructions?: string;
-}): Array<{
-    role: "system" | "user";
-    content: string;
-}>;
-export interface EvaluationJudgeAvailability {
-    available: boolean;
-    reason?: string;
-}
-export declare function evaluationJudgeAvailability(config: LocalRunnerConfig): EvaluationJudgeAvailability;
-/**
- * Fail before model inference when judge scoring cannot run. Exact-match
- * fallback is allowed only when the user opted into it explicitly in config.
- */
-export declare function assertEvaluationScoringReady(config: LocalRunnerConfig, scoringMode?: "exact_match" | "llm_judge" | "json_fields"): EvaluationJudgeAvailability;
-export type RegressionCategory = "factual" | "omission" | "style" | "fallback" | "other";
-/**
- * Coarse category for a judge reasoning string so comparison reports can
- * answer "what kind of worse?" without re-reading every example. Factual
- * problems dominate omissions, which dominate style notes; fallback marks
- * examples that were never judge-scored (their score is not comparable to
- * judged ones).
- */
-export declare function classifyJudgeReasoning(reasoning: string | null, scoredBy?: string): RegressionCategory;
-/**
- * Cache key for a baseline evaluation. Baseline outputs are deterministic for
- * a given model, example set, and generation settings, and judge scores only
- * depend on those plus the scoring configuration, so re-running a spec with
- * an unchanged baseline can reuse the previous report instead of paying for
- * inference and judge calls again. The package version participates so rubric
- * or evaluator changes invalidate old entries.
+ * Deterministic local scoring needs no network credentials. This function is
+ * kept as the shared preflight hook used by validate, doctor, and run.
  */
+export declare function assertEvaluationScoringReady(config: LocalRunnerConfig): void;
 export declare function baselineCacheKey(args: {
     modelId: string;
     baseModelRevision?: string;
     sourceFingerprint?: string;
     system: string;
     examples: BehaviorSpecExample[];
+    evalExamplesTotal: number;
+    evalSplit?: EvalSplit;
+    evalSampleSeed?: number | null;
     config: LocalRunnerConfig;
     packageVersion: string;
 }): string;
@@ -98,26 +48,16 @@ export declare function evaluateExamples(args: {
     sampleSeed?: number;
     shouldCancel?: () => boolean | Promise<boolean>;
 }): Promise<EvalReport>;
-export declare function rescoreEvalReport(args: {
-    report: EvalReport;
-    config: LocalRunnerConfig;
-    outputPath: string;
-    system?: string;
-    reporter?: LocalRunReporter;
-}): Promise<EvalReport>;
 export declare function compareEvalReports(baseline: EvalReport, candidate: EvalReport): {
     avg_score_delta: number;
     pass_rate_delta: number;
     exact_match_rate_delta: number;
     token_f1_delta: number;
-    judge_only_avg_score_delta: number | null;
     regressions: number;
     improvements: number;
-    regression_taxonomy: Record<RegressionCategory, number> | undefined;
     regressed_examples: {
         prompt: string;
         old_score: number;
         new_score: number;
-        category: RegressionCategory;
     }[];
 };