@indemn/cli 1.0.0 → 1.1.0

package/src/report/scoring.ts

@@ -0,0 +1,281 @@
+// src/report/scoring.ts — Scoring utilities ported from dashboard
+// Pure arithmetic, no browser dependencies
+
+// ---------------------------------------------------------------------------
+// Thresholds
+// ---------------------------------------------------------------------------
+
+export const SCORE_THRESHOLDS = {
+  SUCCESS: 0.70,
+  WARNING: 0.50,
+} as const;
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface ScoreMetrics {
+  criteriaRate: number | null;       // 0-1, null when no data
+  rubricRate: number | null;         // 0-1, null when no rubric data
+  criteriaPassed: number;
+  criteriaTotal: number;
+  rubricRulesPassed: number;
+  rubricRulesTotal: number;
+}
+
+export type ScoreColor = 'success' | 'warning' | 'error';
+
+export interface ScoreColorClasses {
+  text: string;
+  bg: string;
+  bar: string;
+  hex: string;
+}
+
+// Minimal types for scoring functions
+export interface ScoringRun {
+  status?: string;
+  passed?: number;
+  completed?: number;
+  criteria_passed?: number;
+  criteria_total?: number;
+  rubric_rules_passed?: number;
+  rubric_rules_total?: number;
+  component_scores?: Record<string, { score?: number; total?: number; passed?: number }>;
+}
+
+export interface ScoringResult {
+  criteria_scores?: Array<{ passed: boolean }>;
+  rubric_scores?: Array<{ passed: boolean }>;
+}
+
+// V1 matrix types
+export type Severity = 'High' | 'Medium' | 'Low';
+
+export interface EvaluatorInfo {
+  key: string;
+  name: string;
+  severity: Severity;
+  prompt?: string;
+  model?: string;
+}
+
+export interface MatrixCell {
+  score: number;
+  comment: string | null;
+  passed: boolean;
+}
+
+export interface MatrixRow {
+  resultId: string;
+  testCaseId: string;
+  question: string;
+  response: string;
+  cells: Record<string, MatrixCell>;
+  avgScore: number;
+}
+
+export interface MatrixData {
+  evaluators: EvaluatorInfo[];
+  rows: MatrixRow[];
+  columnAverages: Record<string, number>;
+  overallAverage: number;
+}
+
+export interface WeightConfig {
+  High: number;
+  Medium: number;
+  Low: number;
+}
+
+export const DEFAULT_WEIGHTS: WeightConfig = {
+  High: 2,
+  Medium: 1,
+  Low: 0.5,
+};
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function sumComponentScores(cs: Record<string, { score?: number; total?: number; passed?: number }>): { passed: number; total: number } {
+  const scopes = ['prompt', 'knowledge_base', 'function', 'general'] as const;
+  let passed = 0;
+  let total = 0;
+  for (const scope of scopes) {
+    passed += cs[scope]?.passed ?? 0;
+    total += cs[scope]?.total ?? 0;
+  }
+  return { passed, total };
+}
+
+// ---------------------------------------------------------------------------
+// Run-level scoring
+// ---------------------------------------------------------------------------
+
+export function getRunScoreMetrics(run: ScoringRun): ScoreMetrics {
+  let criteriaPassed: number;
+  let criteriaTotal: number;
+  let rubricRulesPassed: number;
+  let rubricRulesTotal: number;
+
+  if (run.criteria_total !== undefined && run.criteria_total > 0) {
+    criteriaPassed = run.criteria_passed ?? 0;
+    criteriaTotal = run.criteria_total;
+  } else {
+    criteriaPassed = run.passed ?? 0;
+    criteriaTotal = run.completed ?? 0;
+  }
+
+  const hasNewRubricFields = run.rubric_rules_total !== undefined;
+  if (hasNewRubricFields) {
+    rubricRulesPassed = run.rubric_rules_passed ?? 0;
+    rubricRulesTotal = run.rubric_rules_total ?? 0;
+  } else {
+    const cs = run.component_scores;
+    if (cs) {
+      const sums = sumComponentScores(cs);
+      rubricRulesPassed = sums.passed;
+      rubricRulesTotal = sums.total;
+    } else {
+      rubricRulesPassed = 0;
+      rubricRulesTotal = 0;
+    }
+  }
+
+  const criteriaRate = criteriaTotal > 0 ? criteriaPassed / criteriaTotal : null;
+  const rubricRate = rubricRulesTotal > 0 ? rubricRulesPassed / rubricRulesTotal : null;
+
+  return { criteriaRate, rubricRate, criteriaPassed, criteriaTotal, rubricRulesPassed, rubricRulesTotal };
+}
+
+export function getRunCriteriaScore(run: ScoringRun): number | null {
+  const { criteriaRate } = getRunScoreMetrics(run);
+  return criteriaRate !== null ? Math.round(criteriaRate * 100) : null;
+}
+
+export function getRunRubricScore(run: ScoringRun): number | null {
+  const { rubricRate } = getRunScoreMetrics(run);
+  return rubricRate !== null ? Math.round(rubricRate * 100) : null;
+}
+
+// ---------------------------------------------------------------------------
+// Result-level scoring
+// ---------------------------------------------------------------------------
+
+export function getResultsScoreMetrics(results: ScoringResult[]): ScoreMetrics {
+  if (results.length === 0) {
+    return { criteriaRate: null, rubricRate: null, criteriaPassed: 0, criteriaTotal: 0, rubricRulesPassed: 0, rubricRulesTotal: 0 };
+  }
+
+  const allCriteria = results.flatMap((r) => r.criteria_scores || []);
+  const criteriaTotal = allCriteria.length;
+  const criteriaPassed = allCriteria.filter((c) => c.passed).length;
+  const criteriaRate = criteriaTotal > 0 ? criteriaPassed / criteriaTotal : null;
+
+  const allRubricScores = results.flatMap((r) => r.rubric_scores || []);
+  const rubricRulesTotal = allRubricScores.length;
+  const rubricRulesPassed = allRubricScores.filter((r) => r.passed).length;
+  const rubricRate = rubricRulesTotal > 0 ? rubricRulesPassed / rubricRulesTotal : null;
+
+  return { criteriaRate, rubricRate, criteriaPassed, criteriaTotal, rubricRulesPassed, rubricRulesTotal };
+}
+
+// ---------------------------------------------------------------------------
+// Aggregate pass rate
+// ---------------------------------------------------------------------------
+
+export function getAggregatePassRate(runs: ScoringRun[]): number | null {
+  const completed = runs.filter((r) => r.status === 'completed');
+  if (completed.length === 0) return null;
+
+  let totalPassed = 0;
+  let totalChecks = 0;
+  for (const run of completed) {
+    if (run.criteria_total !== undefined && run.criteria_total > 0) {
+      totalPassed += run.criteria_passed ?? 0;
+      totalChecks += run.criteria_total;
+    } else {
+      totalPassed += run.passed ?? 0;
+      totalChecks += run.completed ?? 0;
+    }
+  }
+  return totalChecks > 0 ? totalPassed / totalChecks : null;
+}
+
+// ---------------------------------------------------------------------------
+// Color utilities
+// ---------------------------------------------------------------------------
+
+export function getScoreColor(rate: number | null | undefined): ScoreColor {
+  if (rate === null || rate === undefined) return 'error';
+  if (rate >= SCORE_THRESHOLDS.SUCCESS) return 'success';
+  if (rate >= SCORE_THRESHOLDS.WARNING) return 'warning';
+  return 'error';
+}
+
+export function getScoreColorClasses(rate: number | null | undefined): ScoreColorClasses {
+  const color = getScoreColor(rate);
+  switch (color) {
+    case 'success':
+      return { text: 'text-success', bg: 'bg-success/10', bar: 'bg-success', hex: '#22c55e' };
+    case 'warning':
+      return { text: 'text-warning', bg: 'bg-warning/10', bar: 'bg-warning', hex: '#f59e0b' };
+    case 'error':
+      return { text: 'text-error', bg: 'bg-error/10', bar: 'bg-error', hex: '#ef4444' };
+  }
+}
+
+export function getScoreHexColor(rate: number | null | undefined): string {
+  return getScoreColorClasses(rate).hex;
+}
+
+// ---------------------------------------------------------------------------
+// Weighted score calculation (V1 reports)
+// ---------------------------------------------------------------------------
+
+export function calculateWeightedScores(
+  matrixData: MatrixData,
+  weights: WeightConfig,
+): {
+  overall: number;
+  bySeverity: Record<Severity, { score: number; count: number }>;
+} {
+  const bySeverity: Record<Severity, { total: number; weight: number; count: number }> = {
+    High: { total: 0, weight: 0, count: 0 },
+    Medium: { total: 0, weight: 0, count: 0 },
+    Low: { total: 0, weight: 0, count: 0 },
+  };
+
+  for (const evaluator of matrixData.evaluators) {
+    const avg = matrixData.columnAverages[evaluator.key] || 0;
+    const weight = weights[evaluator.severity];
+    bySeverity[evaluator.severity].total += avg * weight;
+    bySeverity[evaluator.severity].weight += weight;
+    bySeverity[evaluator.severity].count++;
+  }
+
+  let overallTotal = 0;
+  let overallWeight = 0;
+
+  const result: Record<Severity, { score: number; count: number }> = {
+    High: { score: 0, count: 0 },
+    Medium: { score: 0, count: 0 },
+    Low: { score: 0, count: 0 },
+  };
+
+  for (const severity of ['High', 'Medium', 'Low'] as Severity[]) {
+    const data = bySeverity[severity];
+    result[severity] = {
+      score: data.weight > 0 ? data.total / data.weight : 0,
+      count: data.count,
+    };
+    overallTotal += data.total;
+    overallWeight += data.weight;
+  }
+
+  return {
+    overall: overallWeight > 0 ? overallTotal / overallWeight : 0,
+    bySeverity: result,
+  };
+}

package/src/report/styles.ts

@@ -0,0 +1,138 @@
+// src/report/styles.ts — Brand styles and font registration for PDF reports
+import { Font, StyleSheet } from '@react-pdf/renderer';
+import { fileURLToPath } from 'url';
+import { dirname, join } from 'path';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const assetsDir = join(__dirname, 'assets');
+
+// Register Barlow font family from bundled files
+Font.register({
+  family: 'Barlow',
+  fonts: [
+    { src: join(assetsDir, 'fonts', 'Barlow-Regular.ttf'), fontWeight: 400 },
+    { src: join(assetsDir, 'fonts', 'Barlow-Medium.ttf'), fontWeight: 500 },
+    { src: join(assetsDir, 'fonts', 'Barlow-SemiBold.ttf'), fontWeight: 600 },
+    { src: join(assetsDir, 'fonts', 'Barlow-Bold.ttf'), fontWeight: 700 },
+  ],
+});
+
+export const logoPath = join(assetsDir, 'Indemn_PrimaryLogo_Iris.png');
+
+// Brand colors from Indemn Brand Guide
+export const colors = {
+  // Primary brand colors
+  iris: '#4752a3',
+  lilac: '#a67cb7',
+  eggplant: '#1e2553',
+  white: '#ffffff',
+  // Supporting brand colors
+  lime: '#e0da67',
+  olive: '#2a2b1a',
+  // Derived UI colors
+  background: '#f4f3f8',
+  muted: '#6b7094',
+  border: '#d4d7e3',
+  // Semantic colors
+  success: '#22c55e',
+  error: '#ef4444',
+  warning: '#f59e0b',
+};
+
+// Shared styles
+export const styles = StyleSheet.create({
+  page: {
+    fontFamily: 'Barlow',
+    fontSize: 11,
+    color: colors.eggplant,
+    backgroundColor: colors.white,
+    padding: 40,
+  },
+  pageHeader: {
+    flexDirection: 'row',
+    justifyContent: 'space-between',
+    alignItems: 'center',
+    marginBottom: 24,
+    paddingBottom: 12,
+    borderBottomWidth: 1,
+    borderBottomColor: colors.border,
+  },
+  pageTitle: {
+    fontSize: 18,
+    fontWeight: 600,
+    color: colors.iris,
+  },
+  pageNumber: {
+    fontSize: 9,
+    color: colors.muted,
+  },
+  sectionTitle: {
+    fontSize: 14,
+    fontWeight: 600,
+    color: colors.iris,
+    marginBottom: 12,
+  },
+  bodyText: {
+    fontSize: 11,
+    fontWeight: 400,
+    color: colors.eggplant,
+    lineHeight: 1.5,
+    marginBottom: 8,
+  },
+  caption: {
+    fontSize: 9,
+    fontWeight: 500,
+    color: colors.muted,
+  },
+  card: {
+    backgroundColor: colors.background,
+    borderRadius: 6,
+    padding: 16,
+    marginBottom: 12,
+  },
+  metricValue: {
+    fontSize: 28,
+    fontWeight: 700,
+    color: colors.eggplant,
+  },
+  metricLabel: {
+    fontSize: 10,
+    fontWeight: 500,
+    color: colors.muted,
+    marginTop: 4,
+  },
+  row: {
+    flexDirection: 'row',
+  },
+  col: {
+    flex: 1,
+  },
+  footer: {
+    position: 'absolute',
+    bottom: 30,
+    left: 40,
+    right: 40,
+    flexDirection: 'row',
+    justifyContent: 'space-between',
+    alignItems: 'center',
+    paddingTop: 12,
+    borderTopWidth: 1,
+    borderTopColor: colors.border,
+  },
+  // Evaluation-specific styles
+  scorePass: {
+    color: colors.success,
+  },
+  scoreFail: {
+    color: colors.error,
+  },
+  severityHigh: {
+    color: '#3a4389', // Dark iris
+  },
+  severityMedium: {
+    color: colors.iris, // Standard iris
+  },
+  severityLow: {
+    color: '#7c85c0', // Light iris
+  },
+});

package/tsconfig.json

@@ -12,7 +12,8 @@
     "resolveJsonModule": true,
     "declaration": true,
     "declarationMap": true,
-    "sourceMap": true
+    "sourceMap": true,
+    "jsx": "react-jsx"
   },
   "include": ["src/**/*"],
   "exclude": ["node_modules", "dist", "test"]