@plumloom/cli 0.1.0

package/dist/output/colors.js

@@ -0,0 +1,21 @@
+import chalk from 'chalk';
+/**
+ * Semantic colour vocabulary for the line-based CLI.
+ *
+ * Colour carries meaning, never decoration: pass/positive is green, warning is
+ * yellow, failure or regression is red, identifiers and metadata are dim, and
+ * user input is cyan. Chalk disables itself for pipes, `NO_COLOR`, and
+ * non-TTY streams, so the same renderers stay pipe-safe.
+ */
+export const paint = {
+    pass: (value) => chalk.green(value),
+    warn: (value) => chalk.yellow(value),
+    fail: (value) => chalk.red(value),
+    /** Identifiers and other reference metadata recede behind the content. */
+    meta: (value) => chalk.dim(value),
+    /** Echoed user input. */
+    input: (value) => chalk.cyan(value),
+    heading: (value) => chalk.bold(value),
+    header: (value) => chalk.dim.bold(value),
+};
+//# sourceMappingURL=colors.js.map

package/dist/output/colors.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"colors.js","sourceRoot":"","sources":["../../src/output/colors.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,KAAK,GAAG;IACnB,IAAI,EAAE,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC;IACnD,IAAI,EAAE,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC;IACpD,IAAI,EAAE,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC;IACjD,0EAA0E;IAC1E,IAAI,EAAE,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC;IACjD,yBAAyB;IACzB,KAAK,EAAE,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;IACnD,OAAO,EAAE,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;IACrD,MAAM,EAAE,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC;CAChD,CAAC"}

package/dist/output/digest.d.ts

@@ -0,0 +1,165 @@
+import type { JsonObject } from '../domain/common.js';
+import type { Evaluation, EvaluationResults, ScenarioResults } from '../domain/types.js';
+/**
+ * Deterministic, bounded digests over already-validated Autoeval payloads.
+ *
+ * These helpers only read fields that are present and omit everything else, so
+ * a digest can never assert a detail the backend did not return. They perform
+ * no network access and no inference.
+ */
+export declare const MAX_DIGEST_CHARS = 4000;
+/**
+ * Summarises the validated evaluation configuration — the evaluation's inputs —
+ * for display and reasoning. The sections shown depend on the context type, so
+ * the consumer sees the material the judge actually scored.
+ */
+export declare function summarizeEvaluationConfiguration(evaluation: Evaluation): string;
+type ModelScore = {
+    name: string;
+    mean: number;
+    sampleSize?: number;
+    runs?: number;
+    isPrimary?: boolean;
+};
+/** Per-model overall means from the model-performance payload. */
+export declare function scenarioModelScores(modelPerformance: JsonObject): ModelScore[];
+/** One metric of one model, with the interval the payload reported. */
+export type ScenarioMetricScore = {
+    metric: string;
+    mean: number;
+    ci95?: number;
+    ci95Lower?: number;
+    ci95Upper?: number;
+    sampleSize?: number;
+    insufficientSamples?: boolean;
+};
+export type ScenarioModelMetrics = {
+    model: string;
+    isPrimary?: boolean;
+    runs?: number;
+    metrics: ScenarioMetricScore[];
+};
+/**
+ * Per-metric means for each scored model. `overall` is an aggregate of the
+ * other metrics, so it is excluded here and reported separately as the headline
+ * score. Metrics the payload marks without data are dropped, never defaulted.
+ */
+export declare function scenarioMetricScores(modelPerformance: JsonObject): ScenarioModelMetrics[];
+export type ScenarioTestCaseScore = {
+    name: string;
+    index: number;
+    scores: {
+        model: string;
+        mean: number;
+        ci95?: number;
+        insufficientSamples?: boolean;
+    }[];
+};
+/**
+ * Per-test-case means in the order the backend indexed them, so repeated runs
+ * of the same evaluation always list the cases identically.
+ */
+export declare function scenarioTestCaseScores(scenarioComparison: JsonObject): ScenarioTestCaseScore[];
+export type ScenarioHeadline = {
+    mean?: number;
+    ci95?: number;
+    ci95Lower?: number;
+    ci95Upper?: number;
+    runs?: number;
+    testCases?: number;
+    primaryModel?: string;
+};
+/** Headline numbers for a scenario run, read straight from the payloads. */
+export declare function scenarioHeadline(modelPerformance: JsonObject, scenarioComparison: JsonObject): ScenarioHeadline;
+export type ScenarioConvergence = {
+    headline?: string;
+    consistencyTarget?: string;
+    observedVariation?: number;
+    autoStopRun?: number;
+    runsPlanned?: number;
+    judgesPerOutput?: number;
+    targetMet?: boolean;
+};
+/**
+ * The Evaluation Status card's own numbers. Nothing here is derived: when the
+ * backend does not report convergence detail the block is omitted entirely,
+ * because a variation figure recomputed from a standard deviation would be a
+ * different statistic wearing the same label.
+ */
+export declare function scenarioConvergence(...payloads: readonly (JsonObject | undefined)[]): ScenarioConvergence;
+/**
+ * The reliability figures the evaluation service reports for a multi-run scenario.
+ *
+ * `multiRun` is decided by the presence of `achievedConsistency`, and the
+ * stability verdict compares that reported figure against the reported target
+ * with the reported operator. Nothing here is recomputed from standard
+ * deviations, and the coarse `consistency_level` enum is deliberately ignored.
+ */
+export type ScenarioReliability = {
+    multiRun: boolean;
+    achievedConsistency?: number;
+    targetConsistency?: number;
+    targetOperator?: string;
+    stable?: boolean;
+    runsCompleted?: number;
+    runsPlanned?: number;
+    stoppedAtRun?: number;
+    autoStopTriggered?: boolean;
+    maxRunsReached?: boolean;
+};
+/** Reliability detail read from the run status payload, or `multiRun: false`. */
+export declare function scenarioReliability(status: JsonObject | undefined): ScenarioReliability;
+export type ScenarioOutput = {
+    name: string;
+    index: number;
+    input?: string;
+    responses: {
+        model: string;
+        response?: string;
+        overallScore?: number;
+        runNumber?: number;
+        runCount?: number;
+        metrics: {
+            metric: string;
+            score: number;
+        }[];
+        tokens?: number;
+        cost?: number;
+        latencyMs?: number;
+    }[];
+    detailsUrl?: string;
+};
+/** Per-test-case model outputs, shown only when the user asks for them. */
+export declare function scenarioOutputs(modelResponses: JsonObject): ScenarioOutput[];
+/** Summarises validated scenario results for display, memory, and reasoning. */
+export declare function summarizeScenarioResults(results: ScenarioResults): string;
+export type MetricScore = {
+    name: string;
+    score: number;
+};
+/** Per-metric scores from a conversation or agent-trace session payload. */
+export declare function sessionMetricScores(input: unknown): MetricScore[];
+/** Per-dimension scores from a trajectory payload. */
+export declare function trajectoryMetricScores(input: unknown): MetricScore[];
+/** Every scored area of a result payload, whatever its context type. */
+export declare function resultMetricScores(results: EvaluationResults): MetricScore[];
+/**
+ * Deterministic ranking of the lowest-scoring areas, so a "what should I fix
+ * first" answer is ordered by measured evidence rather than model preference.
+ */
+export declare function weakestAreasLine(results: EvaluationResults): string | undefined;
+/**
+ * Builds the single interpretation prompt: the user's own question plus the
+ * digests gathered while answering it (evaluation inputs and result outputs).
+ * The question is kept so the answering model addresses it instead of being handed data
+ * with nothing to answer.
+ */
+export declare function buildInterpretationMessage(question: string, digests: readonly string[], maxChars: number): string;
+/**
+ * Deterministic answer used when no usable interpretation is returned.
+ * It only repeats headline facts already present in the digests, so the user
+ * still gets an answer instead of a failure message.
+ */
+export declare function fallbackInterpretation(digests: readonly string[]): string | undefined;
+export {};
+//# sourceMappingURL=digest.d.ts.map

package/dist/output/digest.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"digest.d.ts","sourceRoot":"","sources":["../../src/output/digest.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,KAAK,EAAE,UAAU,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAGzF;;;;;;GAMG;AAEH,eAAO,MAAM,gBAAgB,OAAQ,CAAC;AAmOtC;;;;GAIG;AACH,wBAAgB,gCAAgC,CAAC,UAAU,EAAE,UAAU,GAAG,MAAM,CAoC/E;AAED,KAAK,UAAU,GAAG;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB,CAAC;AAMF,kEAAkE;AAClE,wBAAgB,mBAAmB,CAAC,gBAAgB,EAAE,UAAU,GAAG,UAAU,EAAE,CAoB9E;AAED,uEAAuE;AACvE,MAAM,MAAM,mBAAmB,GAAG;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,mBAAmB,CAAC,EAAE,OAAO,CAAC;CAC/B,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,mBAAmB,EAAE,CAAC;CAChC,CAAC;AAEF;;;;GAIG;AACH,wBAAgB,oBAAoB,CAAC,gBAAgB,EAAE,UAAU,GAAG,oBAAoB,EAAE,CAwCzF;AAED,MAAM,MAAM,qBAAqB,GAAG;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,mBAAmB,CAAC,EAAE,OAAO,CAAA;KAAE,EAAE,CAAC;CACzF,CAAC;AAEF;;;GAGG;AACH,wBAAgB,sBAAsB,CAAC,kBAAkB,EAAE,UAAU,GAAG,qBAAqB,EAAE,CA4B9F;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,4EAA4E;AAC5E,wBAAgB,gBAAgB,CAC9B,gBAAgB,EAAE,UAAU,EAC5B,kBAAkB,EAAE,UAAU,GAC7B,gBAAgB,CAuBlB;AAED,MAAM,MAAM,mBAAmB,GAAG;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB,CAAC;AAIF;;;;;GAKG;AACH,wBAAgB,mBAAmB,CACjC,GAAG,QAAQ,EAAE,SAAS,CAAC,UAAU,GAAG,SAAS,CAAC,EAAE,GAC/C,mBAAmB,CAgCrB;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,mBAAmB,GAAG;IAChC,QAAQ,EAAE,OAAO,CAAC;IAClB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B,CAAC;AAWF,iFAAiF;AACjF,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,UAAU,GAAG,SAAS,GAAG,mBAAmB,CA+CvF;AAED,MAAM,MAAM,cAAc,GAAG;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,EAAE;QACT,KAAK,EAAE,MAAM,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE;YAAE,MAAM,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAA;SAAE,EAAE,CAAC;QAC7C,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,EAAE,CAAC;IACJ,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,CAAC;AAIF,2EAA2E;AAC3E,wBAAgB,eAAe,CAAC,cAAc,EAAE,UAAU,GAAG,cAAc,EAAE,CAoE5E;AA2HD,gFAAgF;AAChF,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,eAAe,GAAG,MAAM,CAkBzE;AAED,MAAM,MAAM,WAAW,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAIF,4EAA4E;AAC5E,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,OAAO,GAAG,WAAW,EAAE,CAUjE;AAED,sDAAsD;AACtD,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,OAAO,GAAG,WAAW,EAAE,CAUpE;AAED,wEAAwE;AACxE,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,iBAAiB,GAAG,WAAW,EAAE,CAY5E;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,iBAAiB,GAAG,MAAM,GAAG,SAAS,CAO/E;AAED;;;;;GAKG;AACH,wBAAgB,0BAA0B,CACxC,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,SAAS,MAAM,EAAE,EAC1B,QAAQ,EAAE,MAAM,GACf,MAAM,CAWR;AAKD;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,GAAG,SAAS,CAQrF"}