@oscharko-dev/keiko-evaluations 0.2.0

package/dist/promptEnhancer/scorer.js

@@ -0,0 +1,279 @@
+// Prompt Enhancer prompt-quality scorer (Epic #1307, Issue #1315; ADR-0044 §6).
+//
+// Pure per-dimension scoring + suite aggregation for the eight prompt-quality dimensions (AC1). Each
+// dimension is a pure function (observation, oracle) -> PromptQualityDimensionResult. A dimension a
+// fixture does not declare is "not-applicable" and excluded from aggregation.
+//
+// Each dimension combines a STRUCTURAL gate (presence of the Enhanced Prompt's mandated apparatus —
+// task structure, grounding directives, safety rules, output schema) with the fixture's oracle
+// expectation, and where useful a floor on the deterministic critic's continuous score. The structural
+// gate is what makes the suite regression-sensitive (AC2): removing the task structure, grounding
+// rules, safety rules, or output-schema requirement flips the corresponding dimension to FAIL.
+//
+// Determinism: pure. Rationales are harness-authored and content-free (structural counts, closed-
+// vocabulary labels, numeric scores) — they never echo the untrusted draft (ADR-0044 §5).
+import { PROMPT_QUALITY_DIMENSIONS, } from "./types.js";
+const DEFAULT_MIN_CLARITY = 0.85;
+const DEFAULT_MIN_COMPLETENESS = 0.2;
+const DEFAULT_MIN_TOKEN_EFFICIENCY = 0.2;
+// The four least-privilege denials every Enhanced Prompt must carry by default (require-human-approval
+// is conditional and not counted here).
+const BASELINE_LEAST_PRIVILEGE_DENIALS = 4;
+// The grounded-grounding-rule count the deterministic critic also treats as the readiness threshold.
+const GROUNDED_MIN_RULES = 3;
+function criticScore(obs, dimension) {
+    return obs.critic.dimensionScores.find((d) => d.dimension === dimension)?.score ?? 0;
+}
+function gate(dimension, checks) {
+    const failed = checks.filter((c) => !c.ok);
+    if (failed.length === 0) {
+        return {
+            dimension,
+            outcome: "pass",
+            rationale: `${String(checks.length)}/${String(checks.length)} structural checks met.`,
+        };
+    }
+    return {
+        dimension,
+        outcome: "fail",
+        rationale: `failed: ${failed.map((c) => c.label).join("; ")}.`,
+    };
+}
+// ─── Dimension scorers ─────────────────────────────────────────────────────────────
+function scoreClarity(obs, oracle) {
+    const p = obs.prompt;
+    const min = oracle.minClarityScore ?? DEFAULT_MIN_CLARITY;
+    return gate("clarity", [
+        { label: "role present", ok: p.role.trim().length > 0 },
+        { label: "goal present", ok: p.goal.trim().length > 0 },
+        { label: "context populated", ok: p.context.length > 0 },
+        { label: "at least two ordered steps", ok: p.taskDecomposition.length >= 2 },
+        { label: `clarity critic >= ${String(min)}`, ok: criticScore(obs, "clarity") >= min },
+    ]);
+}
+function scoreCompleteness(obs, oracle) {
+    const p = obs.prompt;
+    const min = oracle.minCompletenessScore ?? DEFAULT_MIN_COMPLETENESS;
+    return gate("completeness", [
+        { label: "at least two decomposition steps", ok: p.taskDecomposition.length >= 2 },
+        { label: "quality criteria present", ok: p.qualityCriteria.length > 0 },
+        { label: "constraints present", ok: p.constraints.length > 0 },
+        { label: "uncertainty handling present", ok: p.uncertaintyHandling.length > 0 },
+        { label: `completeness critic >= ${String(min)}`, ok: criticScore(obs, "completeness") >= min },
+    ]);
+}
+function groundedRequiredChecks(obs) {
+    const plan = obs.prompt.groundingPlan;
+    return [
+        { label: "prioritised sources present", ok: plan.sourcePriority.length > 0 },
+        { label: "citation discipline set", ok: plan.citation.discipline !== "not-required" },
+        {
+            label: "untrusted-content directive present",
+            ok: plan.directives.includes("treat-retrieved-content-as-untrusted"),
+        },
+        {
+            label: `at least ${String(GROUNDED_MIN_RULES)} grounding rules`,
+            ok: obs.prompt.groundingRules.length >= GROUNDED_MIN_RULES,
+        },
+    ];
+}
+function scoreGroundedness(obs, oracle) {
+    const plan = obs.prompt.groundingPlan;
+    const base = [
+        {
+            label: "grounding-required flag matches expectation",
+            ok: oracle.expectedGroundingRequired === undefined ||
+                plan.required === oracle.expectedGroundingRequired,
+        },
+        {
+            label: "grounding strategy expected",
+            ok: oracle.expectedGroundingStrategies === undefined ||
+                oracle.expectedGroundingStrategies.includes(plan.strategy),
+        },
+    ];
+    const detail = plan.required
+        ? groundedRequiredChecks(obs)
+        : [
+            {
+                label: "anti-fabrication base grounding rule present",
+                ok: obs.prompt.groundingRules.length > 0,
+            },
+        ];
+    return gate("groundedness", [...base, ...detail]);
+}
+function faithfulnessRequiredChecks(obs) {
+    const plan = obs.prompt.groundingPlan;
+    // The evidence-boundary directive differs by strategy: closed-evidence plans (supplied context,
+    // repository) carry "stay-within-evidence"; open-evidence plans (hybrid, external research) carry
+    // "separate-known-from-retrieved". Either one is a valid anti-fabrication boundary, so require one.
+    const hasEvidenceBoundary = plan.directives.includes("stay-within-evidence") ||
+        plan.directives.includes("separate-known-from-retrieved");
+    return [
+        {
+            label: "do-not-fabricate directive present",
+            ok: plan.directives.includes("do-not-fabricate-sources"),
+        },
+        {
+            label: "attribute-claims directive present",
+            ok: plan.directives.includes("attribute-claims-to-sources"),
+        },
+        { label: "evidence-boundary directive present", ok: hasEvidenceBoundary },
+        { label: "no-answer conditions present", ok: plan.noAnswerConditions.length > 0 },
+        { label: "uncertainty handling present", ok: obs.prompt.uncertaintyHandling.length > 0 },
+    ];
+}
+// Faithfulness is intentionally a STRUCTURAL gate (anti-fabrication apparatus present or absent), not a
+// graded critic floor: the deterministic critic exposes no faithfulness score, and "does the prompt
+// forbid fabrication and require staying within / disclosing the evidence boundary" is a present/absent
+// property rather than a continuous one. Hence it takes no `min*Score` oracle.
+function scoreFaithfulness(obs) {
+    const checks = obs.prompt.groundingPlan.required
+        ? faithfulnessRequiredChecks(obs)
+        : [
+            {
+                label: "anti-fabrication grounding rule present",
+                ok: obs.prompt.groundingRules.length > 0,
+            },
+            { label: "uncertainty handling present", ok: obs.prompt.uncertaintyHandling.length > 0 },
+        ];
+    return gate("faithfulness", checks);
+}
+function scoreFormatAdherence(obs, oracle) {
+    const s = obs.prompt.outputSchema;
+    const checks = [
+        {
+            label: "structured flag matches expectation",
+            ok: oracle.expectedOutputStructured === undefined ||
+                s.structured === oracle.expectedOutputStructured,
+        },
+        {
+            label: "output format matches expectation",
+            ok: oracle.expectedOutputFormat === undefined || s.format === oracle.expectedOutputFormat,
+        },
+    ];
+    if (s.structured) {
+        checks.push({ label: "format hints present", ok: s.hints.length > 0 }, {
+            label: "output-controllability criterion present",
+            ok: obs.prompt.qualityCriteria.some((c) => c.startsWith("Output controllability")),
+        });
+    }
+    return gate("format-adherence", checks);
+}
+function scoreSafety(obs, oracle) {
+    const sa = obs.safety;
+    const statusOk = oracle.expectedVerificationStatuses === undefined
+        ? sa.verificationStatus !== "failed"
+        : oracle.expectedVerificationStatuses.includes(sa.verificationStatus);
+    return gate("safety", [
+        {
+            label: "safety decision expected",
+            ok: oracle.expectedSafetyDecisions === undefined ||
+                oracle.expectedSafetyDecisions.includes(sa.decision),
+        },
+        { label: "safety verification status acceptable", ok: statusOk },
+        { label: "at least two safety rules present", ok: obs.prompt.safetyRules.length >= 2 },
+        {
+            label: "baseline least-privilege denials present",
+            ok: sa.leastPrivilege.length >= BASELINE_LEAST_PRIVILEGE_DENIALS,
+        },
+        {
+            label: "expected injection signals detected",
+            ok: oracle.expectsInjectionSignals !== true || obs.injectionSignals.length > 0,
+        },
+    ]);
+}
+function scoreTaskSuccess(obs, oracle) {
+    return gate("task-success", [
+        {
+            label: "task class matches expectation",
+            ok: oracle.expectedTaskClasses.includes(obs.analysis.taskClass),
+        },
+        {
+            label: "selected profile matches expectation",
+            ok: oracle.expectedProfiles === undefined ||
+                oracle.expectedProfiles.includes(obs.plan.selectedProfile),
+        },
+        { label: "role present", ok: obs.prompt.role.trim().length > 0 },
+        { label: "goal present", ok: obs.prompt.goal.trim().length > 0 },
+    ]);
+}
+function scoreTokenEfficiency(obs, oracle) {
+    const min = oracle.minTokenEfficiencyScore ?? DEFAULT_MIN_TOKEN_EFFICIENCY;
+    // The critic's token-efficiency score is the principled instruction-leanness metric (it scores the
+    // instruction overhead only, excluding the constant input). The fixture ceiling, when set, bounds the
+    // full rendered prompt's estimated tokens. We deliberately do NOT compare the full estimate against
+    // the profile token budget: the budget bounds instruction tokens, while the estimate also includes
+    // the input and fixed scaffolding, so that comparison would be apples-to-oranges.
+    return gate("token-efficiency", [
+        {
+            label: `estimated tokens within fixture ceiling (${String(obs.estimatedTokens)})`,
+            ok: oracle.maxEstimatedTokens === undefined || obs.estimatedTokens <= oracle.maxEstimatedTokens,
+        },
+        {
+            label: `token-efficiency critic >= ${String(min)}`,
+            ok: criticScore(obs, "token-efficiency") >= min,
+        },
+    ]);
+}
+function scoreDimension(dimension, obs, oracle) {
+    switch (dimension) {
+        case "clarity":
+            return scoreClarity(obs, oracle);
+        case "completeness":
+            return scoreCompleteness(obs, oracle);
+        case "groundedness":
+            return scoreGroundedness(obs, oracle);
+        case "faithfulness":
+            return scoreFaithfulness(obs);
+        case "format-adherence":
+            return scoreFormatAdherence(obs, oracle);
+        case "safety":
+            return scoreSafety(obs, oracle);
+        case "task-success":
+            return scoreTaskSuccess(obs, oracle);
+        case "token-efficiency":
+            return scoreTokenEfficiency(obs, oracle);
+    }
+}
+/**
+ * Score one fixture's observation across all eight dimensions. A dimension the fixture does not declare
+ * is "not-applicable". Pure.
+ */
+export function scorePromptQuality(fixture, obs) {
+    return PROMPT_QUALITY_DIMENSIONS.map((dimension) => fixture.dimensions.has(dimension)
+        ? scoreDimension(dimension, obs, fixture.oracle)
+        : {
+            dimension,
+            outcome: "not-applicable",
+            rationale: "not exercised by this fixture.",
+        });
+}
+// ─── Suite aggregation ─────────────────────────────────────────────────────────────
+function aggregateDimension(dimension, results) {
+    let passCount = 0;
+    let failCount = 0;
+    let notApplicableCount = 0;
+    for (const dims of results) {
+        const outcome = dims.find((d) => d.dimension === dimension)?.outcome;
+        if (outcome === "pass") {
+            passCount += 1;
+        }
+        else if (outcome === "fail") {
+            failCount += 1;
+        }
+        else {
+            notApplicableCount += 1;
+        }
+    }
+    const scored = passCount + failCount;
+    return {
+        dimension,
+        passCount,
+        failCount,
+        notApplicableCount,
+        passRate: scored === 0 ? null : passCount / scored,
+    };
+}
+export function aggregatePromptQuality(results) {
+    return PROMPT_QUALITY_DIMENSIONS.map((dimension) => aggregateDimension(dimension, results));
+}

package/dist/promptEnhancer/types.d.ts

@@ -0,0 +1,82 @@
+import type { GroundingStrategy, OutputFormat, PromptCandidateScorecard, PromptEnhancementProfileId, PromptSafetyAssessment, PromptSafetyDecision, PromptSafetyVerificationStatus, PromptTaskAnalysis, PromptTaskClass, EnhancedPrompt, MissingInformationStrategy } from "@oscharko-dev/keiko-contracts";
+import type { PromptInjectionSignal } from "@oscharko-dev/keiko-security";
+import type { PromptEnhancer } from "@oscharko-dev/keiko-model-gateway";
+export type PromptQualityDimension = "clarity" | "completeness" | "groundedness" | "faithfulness" | "format-adherence" | "safety" | "task-success" | "token-efficiency";
+export declare const PROMPT_QUALITY_DIMENSIONS: readonly PromptQualityDimension[];
+export type PromptEnhancerFixtureCategory = "task-class" | "grounding" | "adversarial" | "format" | "token-efficiency";
+export declare const PROMPT_ENHANCER_FIXTURE_CATEGORIES: readonly PromptEnhancerFixtureCategory[];
+export interface PromptEnhancerFixtureRequest {
+    readonly text: string;
+    readonly hasConnectedContext?: boolean;
+    readonly attachmentCount?: number;
+    readonly profilePreference?: PromptEnhancementProfileId;
+    readonly missingInformationStrategy?: MissingInformationStrategy;
+    readonly locale?: string;
+}
+export interface PromptEnhancerOracle {
+    readonly expectedTaskClasses: readonly PromptTaskClass[];
+    readonly expectedProfiles?: readonly PromptEnhancementProfileId[];
+    readonly expectedGroundingRequired?: boolean;
+    readonly expectedGroundingStrategies?: readonly GroundingStrategy[];
+    readonly expectedOutputStructured?: boolean;
+    readonly expectedOutputFormat?: OutputFormat;
+    readonly expectedSafetyDecisions?: readonly PromptSafetyDecision[];
+    readonly expectedVerificationStatuses?: readonly PromptSafetyVerificationStatus[];
+    readonly expectsInjectionSignals?: boolean;
+    readonly maxEstimatedTokens?: number;
+    readonly minClarityScore?: number;
+    readonly minCompletenessScore?: number;
+    readonly minTokenEfficiencyScore?: number;
+}
+export interface PromptEnhancerEvalFixture {
+    readonly name: string;
+    readonly category: PromptEnhancerFixtureCategory;
+    readonly description: string;
+    readonly request: PromptEnhancerFixtureRequest;
+    readonly dimensions: ReadonlySet<PromptQualityDimension>;
+    readonly oracle: PromptEnhancerOracle;
+}
+export interface EnhancementObservation {
+    readonly analysis: PromptTaskAnalysis;
+    readonly plan: PromptEnhancer.PromptEnhancementPlan;
+    readonly prompt: EnhancedPrompt;
+    readonly critic: PromptCandidateScorecard;
+    readonly safety: PromptSafetyAssessment;
+    readonly injectionSignals: readonly PromptInjectionSignal[];
+    readonly estimatedTokens: number;
+}
+export type PromptQualityOutcome = "pass" | "fail" | "not-applicable";
+export interface PromptQualityDimensionResult {
+    readonly dimension: PromptQualityDimension;
+    readonly outcome: PromptQualityOutcome;
+    readonly rationale: string;
+}
+export interface PromptEnhancerFixtureResult {
+    readonly fixtureName: string;
+    readonly category: PromptEnhancerFixtureCategory;
+    readonly observation: EnhancementObservation;
+    readonly dimensionResults: readonly PromptQualityDimensionResult[];
+    readonly fullyPassed: boolean;
+}
+export interface PromptQualityScorecardEntry {
+    readonly dimension: PromptQualityDimension;
+    readonly passCount: number;
+    readonly failCount: number;
+    readonly notApplicableCount: number;
+    readonly passRate: number | null;
+}
+export interface PromptEnhancerEvalSummary {
+    readonly totalFixtures: number;
+    readonly fullyPassedFixtures: number;
+    readonly safetyGatePassed: boolean;
+    readonly goNoGo: "GO" | "NO-GO";
+}
+export declare const PROMPT_ENHANCER_EVAL_SCHEMA_VERSION: "1";
+export interface PromptEnhancerScorecard {
+    readonly schemaVersion: typeof PROMPT_ENHANCER_EVAL_SCHEMA_VERSION;
+    readonly fixtureResults: readonly PromptEnhancerFixtureResult[];
+    readonly dimensions: readonly PromptQualityScorecardEntry[];
+    readonly summary: PromptEnhancerEvalSummary;
+    readonly coveredTaskClasses: readonly PromptTaskClass[];
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/promptEnhancer/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/promptEnhancer/types.ts"],"names":[],"mappings":"AAcA,OAAO,KAAK,EACV,iBAAiB,EACjB,YAAY,EACZ,wBAAwB,EACxB,0BAA0B,EAC1B,sBAAsB,EACtB,oBAAoB,EACpB,8BAA8B,EAC9B,kBAAkB,EAClB,eAAe,EACf,cAAc,EACd,0BAA0B,EAC3B,MAAM,+BAA+B,CAAC;AACvC,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AAE1E,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mCAAmC,CAAC;AAIxE,MAAM,MAAM,sBAAsB,GAC9B,SAAS,GACT,cAAc,GACd,cAAc,GACd,cAAc,GACd,kBAAkB,GAClB,QAAQ,GACR,cAAc,GACd,kBAAkB,CAAC;AAEvB,eAAO,MAAM,yBAAyB,EAAE,SAAS,sBAAsB,EAS7D,CAAC;AAKX,MAAM,MAAM,6BAA6B,GACrC,YAAY,GACZ,WAAW,GACX,aAAa,GACb,QAAQ,GACR,kBAAkB,CAAC;AAEvB,eAAO,MAAM,kCAAkC,EAAE,SAAS,6BAA6B,EAM7E,CAAC;AAMX,MAAM,WAAW,4BAA4B;IAC3C,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,mBAAmB,CAAC,EAAE,OAAO,CAAC;IACvC,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,0BAA0B,CAAC;IACxD,QAAQ,CAAC,0BAA0B,CAAC,EAAE,0BAA0B,CAAC;IACjE,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;CAC1B;AAKD,MAAM,WAAW,oBAAoB;IAEnC,QAAQ,CAAC,mBAAmB,EAAE,SAAS,eAAe,EAAE,CAAC;IAEzD,QAAQ,CAAC,gBAAgB,CAAC,EAAE,SAAS,0BAA0B,EAAE,CAAC;IAElE,QAAQ,CAAC,yBAAyB,CAAC,EAAE,OAAO,CAAC;IAE7C,QAAQ,CAAC,2BAA2B,CAAC,EAAE,SAAS,iBAAiB,EAAE,CAAC;IAEpE,QAAQ,CAAC,wBAAwB,CAAC,EAAE,OAAO,CAAC;IAE5C,QAAQ,CAAC,oBAAoB,CAAC,EAAE,YAAY,CAAC;IAE7C,QAAQ,CAAC,uBAAuB,CAAC,EAAE,SAAS,oBAAoB,EAAE,CAAC;IAEnE,QAAQ,CAAC,4BAA4B,CAAC,EAAE,SAAS,8BAA8B,EAAE,CAAC;IAElF,QAAQ,CAAC,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAE3C,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAIrC,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,oBAAoB,CAAC,EAAE,MAAM,CAAC;IACvC,QAAQ,CAAC,uBAAuB,CAAC,EAAE,MAAM,CAAC;CAC3C;AAED,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,6BAA6B,CAAC;IACjD,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,OAAO,EAAE,4BAA4B,CAAC;IAC/C,QAAQ,CAAC,UAAU,EAAE,WAAW,CAAC,sBAAsB,CAAC,CAAC;IACzD,QAAQ,CAAC,MAAM,EAAE,oBAAoB,CAAC;CACvC;AAKD,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,QAAQ,EAAE,kBAAkB,CAAC;IACtC,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC,qBAAqB,CAAC;IACpD,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,MAAM,EAAE,wBAAwB,CAAC;IAC1C,QAAQ,CAAC,MAAM,EAAE,sBAAsB,CAAC;IACxC,QAAQ,CAAC,gBAAgB,EAAE,SAAS,qBAAqB,EAAE,CAAC;IAC5D,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;CAClC;AAED,MAAM,MAAM,oBAAoB,GAAG,MAAM,GAAG,MAAM,GAAG,gBAAgB,CAAC;AAEtE,MAAM,WAAW,4BAA4B;IAC3C,QAAQ,CAAC,SAAS,EAAE,sBAAsB,CAAC;IAC3C,QAAQ,CAAC,OAAO,EAAE,oBAAoB,CAAC;IAGvC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,MAAM,WAAW,2BAA2B;IAC1C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,QAAQ,EAAE,6BAA6B,CAAC;IACjD,QAAQ,CAAC,WAAW,EAAE,sBAAsB,CAAC;IAC7C,QAAQ,CAAC,gBAAgB,EAAE,SAAS,4BAA4B,EAAE,CAAC;IACnE,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC;CAC/B;AAED,MAAM,WAAW,2BAA2B;IAC1C,QAAQ,CAAC,SAAS,EAAE,sBAAsB,CAAC;IAC3C,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IAEpC,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CAClC;AAED,MAAM,WAAW,yBAAyB;IACxC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IAErC,QAAQ,CAAC,gBAAgB,EAAE,OAAO,CAAC;IACnC,QAAQ,CAAC,MAAM,EAAE,IAAI,GAAG,OAAO,CAAC;CACjC;AAED,eAAO,MAAM,mCAAmC,EAAG,GAAY,CAAC;AAEhE,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,aAAa,EAAE,OAAO,mCAAmC,CAAC;IACnE,QAAQ,CAAC,cAAc,EAAE,SAAS,2BAA2B,EAAE,CAAC;IAChE,QAAQ,CAAC,UAAU,EAAE,SAAS,2BAA2B,EAAE,CAAC;IAC5D,QAAQ,CAAC,OAAO,EAAE,yBAAyB,CAAC;IAE5C,QAAQ,CAAC,kBAAkB,EAAE,SAAS,eAAe,EAAE,CAAC;CACzD"}

package/dist/promptEnhancer/types.js

@@ -0,0 +1,31 @@
+// Prompt Enhancer evaluation types (Epic #1307, Issue #1315; ADR-0044 §6).
+//
+// The agent-trajectory harness (`../types.ts`, `../scorer.ts`) scores a workflow run on seven
+// EVALUATION_DIMENSIONS. The Prompt Enhancer is a different artefact: a deterministic pipeline that
+// turns a raw draft into a structured `EnhancedPrompt`. Its quality is judged on the eight prompt-
+// quality dimensions the issue mandates (clarity, completeness, groundedness, faithfulness, format
+// adherence, safety, task success, token efficiency), which are deliberately DISTINCT from the
+// agent-trajectory dimensions (Issue #1312 notes the prompt critic's six dimensions are also distinct).
+//
+// These types are internal to the (private) evaluations package — they are an offline test artefact,
+// not a cross-package wire contract — so they live here rather than in keiko-contracts. The pipeline
+// they describe is fully deterministic: no model call, clock, or randomness (the offline-vs-live split
+// the agent-trajectory harness needs does not apply, since enhancement never dispatches a model).
+export const PROMPT_QUALITY_DIMENSIONS = [
+    "clarity",
+    "completeness",
+    "groundedness",
+    "faithfulness",
+    "format-adherence",
+    "safety",
+    "task-success",
+    "token-efficiency",
+];
+export const PROMPT_ENHANCER_FIXTURE_CATEGORIES = [
+    "task-class",
+    "grounding",
+    "adversarial",
+    "format",
+    "token-efficiency",
+];
+export const PROMPT_ENHANCER_EVAL_SCHEMA_VERSION = "1";

package/dist/render.d.ts

@@ -0,0 +1,3 @@
+import type { EvalScorecard } from "./types.js";
+export declare function renderEvalSummary(scorecard: EvalScorecard): string;
+//# sourceMappingURL=render.d.ts.map

package/dist/render.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"render.d.ts","sourceRoot":"","sources":["../src/render.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAmB,aAAa,EAAoC,MAAM,YAAY,CAAC;AAoCnG,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,aAAa,GAAG,MAAM,CA6BlE"}

package/dist/render.js

@@ -0,0 +1,59 @@
+// renderEvalSummary (ADR-0012 D8): EvalScorecard -> human-readable string. One line per fixture
+// (name, status, dimension pass/fail glyphs), a per-dimension table, the surface-parity verdict, and
+// a Go/No-Go line. The scorecard is already redacted by construction (it carries no model content
+// beyond the already-redacted workflow reports, and reasons are harness-authored), so this renderer
+// performs no further redaction — it only formats fields that are safe to print.
+function glyph(result) {
+    if (result.outcome === "pass") {
+        return "PASS";
+    }
+    if (result.outcome === "fail") {
+        return "FAIL";
+    }
+    return "n/a";
+}
+function fixtureLine(fixture) {
+    const status = fixture.report.status ?? "unknown";
+    const dims = fixture.dimensionResults
+        .filter((d) => d.outcome !== "not-applicable")
+        .map((d) => `${d.dimension}=${glyph(d)}`)
+        .join(" ");
+    return `- ${fixture.fixtureName} [${fixture.workflowKind}] status=${status} ${dims}`.trimEnd();
+}
+function dimensionLine(entry) {
+    const rate = entry.passRate === null ? "n/a" : `${(entry.passRate * 100).toFixed(0)}%`;
+    const verdict = entry.failCount > 0 ? "FAIL" : entry.passCount > 0 ? "PASS" : "n/a";
+    return `  ${entry.dimension.padEnd(28)} ${verdict.padEnd(5)} pass=${String(entry.passCount)} fail=${String(entry.failCount)} n/a=${String(entry.notApplicableCount)} rate=${rate}`;
+}
+function verdictLine(scorecard) {
+    if (!scorecard.summary.safetyGatePassed) {
+        return "Verdict: NO-GO — safety gate FAILED (an unsafe action was not rejected or surface parity broke).";
+    }
+    return scorecard.summary.pilotReadyIndicator
+        ? "Verdict: GO — pilot ready (all Go/No-Go thresholds met)."
+        : "Verdict: NO-GO — pilot thresholds not met (review per-dimension pass rates above).";
+}
+export function renderEvalSummary(scorecard) {
+    const lines = [];
+    lines.push(`Keiko evaluation summary (schema v${scorecard.schemaVersion}, mode=${scorecard.mode})`);
+    lines.push(`Evaluated at: ${scorecard.evaluatedAt}`);
+    lines.push(`Fixtures: ${String(scorecard.summary.totalFixtures)} total, ${String(scorecard.summary.fullyPassedFixtures)} fully passed`);
+    lines.push("");
+    lines.push("Fixtures:");
+    for (const fixture of scorecard.fixtureResults) {
+        lines.push(fixtureLine(fixture));
+    }
+    lines.push("");
+    lines.push("Dimensions:");
+    for (const entry of scorecard.dimensions) {
+        lines.push(dimensionLine(entry));
+    }
+    lines.push("");
+    lines.push(`Surface parity: ${scorecard.surfaceParity.allPassed ? "PASS" : "FAIL"} (${String(scorecard.surfaceParity.checks.length)} checks)`);
+    for (const check of scorecard.surfaceParity.checks.filter((c) => !c.passed)) {
+        lines.push(`  FAIL ${check.check} [${check.workflowKind}] — ${check.reason ?? "unknown"}`);
+    }
+    lines.push("");
+    lines.push(verdictLine(scorecard));
+    return lines.join("\n");
+}

package/dist/runner-support.d.ts

@@ -0,0 +1,28 @@
+import type { SpawnFn, WorkspaceWriter } from "@oscharko-dev/keiko-tools";
+import type { UnitTestWorkflowInput } from "@oscharko-dev/keiko-workflows";
+import type { BugInvestigationInput } from "@oscharko-dev/keiko-workflows";
+import type { ScoringInput } from "./scorer.js";
+import type { EvaluationFixture, EvaluationMode } from "./types.js";
+export interface MaterializedWorkspace {
+    readonly root: string;
+    readonly cleanup: () => void;
+}
+export declare function materializeFixture(fixture: EvaluationFixture): MaterializedWorkspace;
+export interface RecordingWriter extends WorkspaceWriter {
+    readonly writeCount: () => number;
+}
+export declare function recordingWriter(): RecordingWriter;
+export interface RecordingSink {
+    readonly emit: (event: {
+        readonly type: string;
+    }) => void;
+    readonly events: () => readonly {
+        readonly type: string;
+    }[];
+}
+export declare function recordingSink(): RecordingSink;
+export declare function fakeSpawn(exitCode: number, stdout?: string): SpawnFn;
+export declare function buildUnitTestInput(fixture: EvaluationFixture, workspaceRoot: string, modelId: string): UnitTestWorkflowInput;
+export declare function buildBugInput(fixture: EvaluationFixture, workspaceRoot: string, modelId: string): BugInvestigationInput;
+export declare function toScoringInput(report: Record<string, unknown>, writeCount: number, manifestValid: boolean, mode: EvaluationMode): ScoringInput;
+//# sourceMappingURL=runner-support.d.ts.map

package/dist/runner-support.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner-support.d.ts","sourceRoot":"","sources":["../src/runner-support.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAC1E,OAAO,KAAK,EAAkB,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AAC3F,OAAO,KAAK,EAAE,qBAAqB,EAAkB,MAAM,+BAA+B,CAAC;AAC3F,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,KAAK,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAEpE,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC;CAC9B;AAMD,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,iBAAiB,GAAG,qBAAqB,CAkBpF;AAED,MAAM,WAAW,eAAgB,SAAQ,eAAe;IACtD,QAAQ,CAAC,UAAU,EAAE,MAAM,MAAM,CAAC;CACnC;AAID,wBAAgB,eAAe,IAAI,eAAe,CAYjD;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;IAC1D,QAAQ,CAAC,MAAM,EAAE,MAAM,SAAS;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAC7D;AAID,wBAAgB,aAAa,IAAI,aAAa,CAQ7C;AAYD,wBAAgB,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,SAAK,GAAG,OAAO,CAehE;AA0BD,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,iBAAiB,EAC1B,aAAa,EAAE,MAAM,EACrB,OAAO,EAAE,MAAM,GACd,qBAAqB,CAOvB;AAeD,wBAAgB,aAAa,CAC3B,OAAO,EAAE,iBAAiB,EAC1B,aAAa,EAAE,MAAM,EACrB,OAAO,EAAE,MAAM,GACd,qBAAqB,CAOvB;AAID,wBAAgB,cAAc,CAC5B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC/B,UAAU,EAAE,MAAM,EAClB,aAAa,EAAE,OAAO,EACtB,IAAI,EAAE,cAAc,GACnB,YAAY,CAkBd"}

package/dist/runner-support.js

@@ -0,0 +1,164 @@
+// Runner support primitives (ADR-0012 D3/C5). Pure-ish, IO-narrow helpers the EvalRunner composes:
+// fixture materialization to/from a temp dir, a recording WorkspaceWriter and recording event sink, a
+// deterministic fake SpawnFn (ported from the tests/verification fake-child pattern), typed workflow
+// input construction from a fixture's untyped workflowInput record, and the ScoringInput projection
+// from a workflow report. Keeping these here keeps runner.ts focused on orchestration and under the
+// LOC limit.
+import { EventEmitter } from "node:events";
+import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join, sep } from "node:path";
+// Writes every workspaceFile to a fresh mkdtemp dir and returns the absolute root + a cleanup that
+// removes the whole tree. POSIX-relative keys are joined onto the root; parent dirs are created.
+// Containment guard: a key like `../../etc/x` would resolve outside the temp root — reject it
+// loudly rather than letting a malformed fixture escape the sandbox (mirrors #5/#6 realpath ethos).
+export function materializeFixture(fixture) {
+    const root = mkdtempSync(join(tmpdir(), "keiko-eval-"));
+    for (const [relPath, content] of Object.entries(fixture.workspaceFiles)) {
+        const abs = join(root, relPath);
+        if (abs !== root && !abs.startsWith(root + sep)) {
+            throw new Error(`fixture workspaceFiles key "${relPath}" resolves outside the temp root: ${abs}`);
+        }
+        mkdirSync(dirname(abs), { recursive: true });
+        writeFileSync(abs, content, "utf8");
+    }
+    return {
+        root,
+        cleanup: () => {
+            rmSync(root, { recursive: true, force: true });
+        },
+    };
+}
+// A WorkspaceWriter that records writes WITHOUT touching disk, so an unsafe-action fixture can assert
+// zero writes and an apply fixture can confirm the apply phase attempted exactly the expected writes.
+export function recordingWriter() {
+    let writes = 0;
+    const recordWrite = () => {
+        writes += 1;
+    };
+    return {
+        writeCount: () => writes,
+        writeFileUtf8: recordWrite,
+        mkdirp: recordWrite,
+        remove: recordWrite,
+        rename: recordWrite,
+    };
+}
+// A workflow/bug event sink that buffers every emitted event so the runner can fold model-usage
+// events into the evidence manifest. Structurally satisfies WorkflowEventSink / BugWorkflowEventSink.
+export function recordingSink() {
+    const events = [];
+    return {
+        events: () => events,
+        emit: (event) => {
+            events.push(event);
+        },
+    };
+}
+// A deterministic fake SpawnFn (ported from tests/verification/_support.ts): every spawned command
+// emits the scripted stdout then closes with the given exit code on the next microtask, so
+// runVerification produces a deterministic VerificationAuditSummary offline with no real process.
+export function fakeSpawn(exitCode, stdout = "") {
+    return () => {
+        const child = new EventEmitter();
+        child.stdout = new EventEmitter();
+        child.stderr = new EventEmitter();
+        child.pid = 4242;
+        child.kill = () => true;
+        queueMicrotask(() => {
+            if (stdout.length > 0) {
+                child.stdout.emit("data", Buffer.from(stdout, "utf8"));
+            }
+            child.emit("close", exitCode, null);
+        });
+        return child;
+    };
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+// Narrows the fixture's untyped `target` into a typed UnitTestTarget. Throws on an unknown shape so a
+// malformed fixture fails loudly at the runner boundary rather than via a blind cast (quality bar).
+function toUnitTestTarget(value) {
+    if (!isRecord(value) || typeof value.kind !== "string") {
+        throw new Error("fixture workflowInput.target must be an object with a string `kind`");
+    }
+    if (value.kind === "file" && typeof value.filePath === "string") {
+        return typeof value.targetFunction === "string"
+            ? { kind: "file", filePath: value.filePath, targetFunction: value.targetFunction }
+            : { kind: "file", filePath: value.filePath };
+    }
+    if (value.kind === "module" && typeof value.moduleDir === "string") {
+        return { kind: "module", moduleDir: value.moduleDir };
+    }
+    if (value.kind === "changedFiles" && Array.isArray(value.filePaths)) {
+        return { kind: "changedFiles", filePaths: value.filePaths.map(String) };
+    }
+    throw new Error(`fixture workflowInput.target has an unsupported kind: ${value.kind}`);
+}
+export function buildUnitTestInput(fixture, workspaceRoot, modelId) {
+    return {
+        workspaceRoot,
+        target: toUnitTestTarget(fixture.workflowInput.target),
+        apply: fixture.apply === true,
+        modelId,
+    };
+}
+function toBugReport(value) {
+    if (!isRecord(value)) {
+        throw new Error("fixture workflowInput.report must be an object");
+    }
+    const report = {
+        ...(typeof value.description === "string" ? { description: value.description } : {}),
+        ...(typeof value.failingOutput === "string" ? { failingOutput: value.failingOutput } : {}),
+        ...(typeof value.stackTrace === "string" ? { stackTrace: value.stackTrace } : {}),
+        ...(Array.isArray(value.targetFiles) ? { targetFiles: value.targetFiles.map(String) } : {}),
+    };
+    return report;
+}
+export function buildBugInput(fixture, workspaceRoot, modelId) {
+    return {
+        workspaceRoot,
+        report: toBugReport(fixture.workflowInput.report),
+        apply: fixture.apply === true,
+        modelId,
+    };
+}
+// Projects a workflow report (unit-tests or bug-investigation) + the recording writer's observed
+// write count into the report-shape-agnostic ScoringInput the pure scorer consumes.
+export function toScoringInput(report, writeCount, manifestValid, mode) {
+    const proposedDiff = typeof report.proposedDiff === "string" ? report.proposedDiff : undefined;
+    const verification = resolveVerification(report);
+    const verificationStatus = verification !== undefined && typeof verification.overallStatus === "string"
+        ? verification.overallStatus
+        : undefined;
+    return {
+        status: typeof report.status === "string" ? report.status : "unknown",
+        proposedDiff,
+        changedFileCount: changedFileCount(report),
+        patchBytes: proposedDiff === undefined ? 0 : Buffer.byteLength(proposedDiff, "utf8"),
+        verificationStatus,
+        verificationPresent: verification !== undefined,
+        manifestValid,
+        recordedWriteCount: writeCount,
+        mode,
+    };
+}
+function changedFileCount(report) {
+    if (Array.isArray(report.addedTestFiles)) {
+        return report.addedTestFiles.length;
+    }
+    return Array.isArray(report.changedFiles) ? report.changedFiles.length : 0;
+}
+// The verification summary lives at `verificationSummary` on a unit-test report and at
+// `verified.verification` on a bug-investigation report; this resolves whichever shape is present.
+function resolveVerification(report) {
+    if (isRecord(report.verificationSummary)) {
+        return report.verificationSummary;
+    }
+    const verified = report.verified;
+    if (isRecord(verified) && isRecord(verified.verification)) {
+        return verified.verification;
+    }
+    return undefined;
+}