gravito-eval 0.1.0

package/dist/src/types.d.ts

@@ -0,0 +1,131 @@
+/**
+ * Gravito Eval — Core Types
+ *
+ * Shared type definitions for the evaluation framework.
+ * All modules import from here — no circular dependencies.
+ */
+export declare const ISSUE_CATEGORIES: readonly ["conversion", "navigation", "visual_hierarchy", "trust", "content", "compliance", "performance"];
+export type IssueCategory = (typeof ISSUE_CATEGORIES)[number];
+export declare const SEVERITY_LEVELS: readonly ["low", "medium", "high", "critical"];
+export type SeverityLevel = (typeof SEVERITY_LEVELS)[number];
+/**
+ * A normalized finding from either an AI system or a human auditor.
+ * This is the universal input format for gravito-eval.
+ */
+export interface Finding {
+    /** Unique identifier */
+    id: string;
+    /** Human-readable description of the issue */
+    description: string;
+    /** Issue category */
+    category: IssueCategory;
+    /** Severity level */
+    severity: SeverityLevel;
+    /** Where the issue was found (page, component, URL, etc.) */
+    location?: string;
+    /** Optional keywords for matching */
+    keywords?: string[];
+}
+export interface MatchPair {
+    aiIssue: Finding;
+    humanIssue: Finding;
+    similarity: number;
+    matchType: "strict" | "cross_category" | "conceptual";
+}
+export interface MatchResult {
+    matched: MatchPair[];
+    aiOnly: Finding[];
+    humanOnly: Finding[];
+}
+export interface MultiPassMatchResult {
+    strictMatches: MatchPair[];
+    crossCategoryMatches: MatchPair[];
+    conceptualMatches: MatchPair[];
+    aiOnly: Finding[];
+    humanOnly: Finding[];
+    summary: {
+        total_ai: number;
+        total_human: number;
+        strict_matched: number;
+        cross_category_matched: number;
+        conceptual_matched: number;
+        total_matched: number;
+        ai_only: number;
+        human_only: number;
+    };
+}
+export declare const ADJUDICATION_LABELS: readonly ["VALID", "INVALID", "DUPLICATE", "LOW_VALUE"];
+export type AdjudicationLabel = (typeof ADJUDICATION_LABELS)[number];
+export interface Adjudication {
+    /** ID of the AI-only finding being adjudicated */
+    findingId: string;
+    /** Verdict */
+    label: AdjudicationLabel;
+    /** Reasoning */
+    reasoning?: string;
+}
+export interface DetectionMetrics {
+    recall: number;
+    precision: number;
+    f1: number;
+    matchedCount: number;
+    totalAI: number;
+    totalHuman: number;
+}
+export interface RankingMetrics {
+    top3Overlap: number;
+    top5Overlap: number;
+    spearmanCorrelation: number;
+}
+export interface SeverityMetrics {
+    weightedKappa: number;
+    meanAbsoluteError: number;
+    confusionMatrix: Record<string, Record<string, number>>;
+}
+export interface NovelSignalMetrics {
+    totalAiOnly: number;
+    validCount: number;
+    invalidCount: number;
+    duplicateCount: number;
+    lowValueCount: number;
+    validatedNovelRate: number;
+    systemStrength: "WEAK" | "MODERATE" | "STRONG" | "DIFFERENTIATED";
+}
+export interface ConfidenceInterval {
+    mean: number;
+    lowerBound: number;
+    upperBound: number;
+}
+export interface EvalResult {
+    detection: DetectionMetrics;
+    ranking: RankingMetrics;
+    severity: SeverityMetrics;
+    novelSignal?: NovelSignalMetrics;
+    matchBreakdown: {
+        strict: number;
+        crossCategory: number;
+        conceptual: number;
+    };
+    /** Individual match pairs for detailed inspection */
+    matches: MatchPair[];
+    /** Findings only in AI output (novel/unmatched) */
+    aiOnly: Finding[];
+    /** Findings only in human output (missed by AI) */
+    humanOnly: Finding[];
+    adjustedPrecision?: number;
+    verdict: "PASS" | "PARTIAL" | "FAIL" | "INSUFFICIENT_DATA";
+}
+export interface ConfidenceFactors {
+    signal_strength: number;
+    cross_signal_support: number;
+    pattern_repetition: number;
+    rule_determinism: number;
+    clarity_of_evidence: number;
+}
+export interface ScoredFinding extends Finding {
+    confidence: number;
+    factors: ConfidenceFactors;
+    isSubjective: boolean;
+    signalCount: number;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/src/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,eAAO,MAAM,gBAAgB,4GAQnB,CAAC;AAEX,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,gBAAgB,CAAC,CAAC,MAAM,CAAC,CAAC;AAE9D,eAAO,MAAM,eAAe,gDAAiD,CAAC;AAE9E,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,eAAe,CAAC,CAAC,MAAM,CAAC,CAAC;AAE7D;;;GAGG;AACH,MAAM,WAAW,OAAO;IACtB,wBAAwB;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,8CAA8C;IAC9C,WAAW,EAAE,MAAM,CAAC;IACpB,qBAAqB;IACrB,QAAQ,EAAE,aAAa,CAAC;IACxB,qBAAqB;IACrB,QAAQ,EAAE,aAAa,CAAC;IACxB,6DAA6D;IAC7D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,qCAAqC;IACrC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AAID,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,OAAO,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,QAAQ,GAAG,gBAAgB,GAAG,YAAY,CAAC;CACvD;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,SAAS,EAAE,CAAC;IACrB,MAAM,EAAE,OAAO,EAAE,CAAC;IAClB,SAAS,EAAE,OAAO,EAAE,CAAC;CACtB;AAED,MAAM,WAAW,oBAAoB;IACnC,aAAa,EAAE,SAAS,EAAE,CAAC;IAC3B,oBAAoB,EAAE,SAAS,EAAE,CAAC;IAClC,iBAAiB,EAAE,SAAS,EAAE,CAAC;IAC/B,MAAM,EAAE,OAAO,EAAE,CAAC;IAClB,SAAS,EAAE,OAAO,EAAE,CAAC;IACrB,OAAO,EAAE;QACP,QAAQ,EAAE,MAAM,CAAC;QACjB,WAAW,EAAE,MAAM,CAAC;QACpB,cAAc,EAAE,MAAM,CAAC;QACvB,sBAAsB,EAAE,MAAM,CAAC;QAC/B,kBAAkB,EAAE,MAAM,CAAC;QAC3B,aAAa,EAAE,MAAM,CAAC;QACtB,OAAO,EAAE,MAAM,CAAC;QAChB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAID,eAAO,MAAM,mBAAmB,yDAKtB,CAAC;AAEX,MAAM,MAAM,iBAAiB,GAAG,CAAC,OAAO,mBAAmB,CAAC,CAAC,MAAM,CAAC,CAAC;AAErE,MAAM,WAAW,YAAY;IAC3B,kDAAkD;IAClD,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc;IACd,KAAK,EAAE,iBAAiB,CAAC;IACzB,gBAAgB;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAID,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,EAAE,EAAE,MAAM,CAAC;IACX,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,eAAe;IAC9B,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CACzD;AAED,MAAM,WAAW,kBAAkB;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,cAAc,EAAE,MAAM,GAAG,UAAU,GAAG,QAAQ,GAAG,gBAAgB,CAAC;CACnE;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,gBAAgB,CAAC;IAC5B,OAAO,EAAE,cAAc,CAAC;IACxB,QAAQ,EAAE,eAAe,CAAC;IAC1B,WAAW,CAAC,EAAE,kBAAkB,CAAC;IACjC,cAAc,EAAE;QACd,MAAM,EAAE,MAAM,CAAC;QACf,aAAa,EAAE,MAAM,CAAC;QACtB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,qDAAqD;IACrD,OAAO,EAAE,SAAS,EAAE,CAAC;IACrB,mDAAmD;IACnD,MAAM,EAAE,OAAO,EAAE,CAAC;IAClB,mDAAmD;IACnD,SAAS,EAAE,OAAO,EAAE,CAAC;IACrB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,OAAO,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,mBAAmB,CAAC;CAC5D;AAID,MAAM,WAAW,iBAAiB;IAChC,eAAe,EAAE,MAAM,CAAC;IACxB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,gBAAgB,EAAE,MAAM,CAAC;IACzB,mBAAmB,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,aAAc,SAAQ,OAAO;IAC5C,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,iBAAiB,CAAC;IAC3B,YAAY,EAAE,OAAO,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;CACrB"}

package/dist/src/types.js

@@ -0,0 +1,28 @@
+"use strict";
+/**
+ * Gravito Eval — Core Types
+ *
+ * Shared type definitions for the evaluation framework.
+ * All modules import from here — no circular dependencies.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ADJUDICATION_LABELS = exports.SEVERITY_LEVELS = exports.ISSUE_CATEGORIES = void 0;
+// ─── Finding Types ────────────────────────────────────────────────────────
+exports.ISSUE_CATEGORIES = [
+    "conversion",
+    "navigation",
+    "visual_hierarchy",
+    "trust",
+    "content",
+    "compliance",
+    "performance",
+];
+exports.SEVERITY_LEVELS = ["low", "medium", "high", "critical"];
+// ─── Adjudication Types ───────────────────────────────────────────────────
+exports.ADJUDICATION_LABELS = [
+    "VALID",
+    "INVALID",
+    "DUPLICATE",
+    "LOW_VALUE",
+];
+//# sourceMappingURL=types.js.map

package/dist/src/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;AAEH,6EAA6E;AAEhE,QAAA,gBAAgB,GAAG;IAC9B,YAAY;IACZ,YAAY;IACZ,kBAAkB;IAClB,OAAO;IACP,SAAS;IACT,YAAY;IACZ,aAAa;CACL,CAAC;AAIE,QAAA,eAAe,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,CAAU,CAAC;AAwD9E,6EAA6E;AAEhE,QAAA,mBAAmB,GAAG;IACjC,OAAO;IACP,SAAS;IACT,WAAW;IACX,WAAW;CACH,CAAC"}

package/examples/basic/input.json

@@ -0,0 +1,76 @@
+{
+  "aiFindings": [
+    {
+      "id": "ai-1",
+      "description": "Missing primary CTA button on homepage — no clear action for visitors to take",
+      "category": "conversion",
+      "severity": "high",
+      "keywords": ["CTA", "button", "action", "homepage", "conversion"]
+    },
+    {
+      "id": "ai-2",
+      "description": "Weak value proposition headline — does not explain what the product does or its core benefit",
+      "category": "content",
+      "severity": "high",
+      "keywords": ["headline", "value proposition", "product", "benefit", "explain"]
+    },
+    {
+      "id": "ai-3",
+      "description": "Confusing visual hierarchy — competing elements with similar weight make page hard to scan",
+      "category": "visual_hierarchy",
+      "severity": "medium",
+      "keywords": ["hierarchy", "layout", "confusing", "importance", "scanning"]
+    },
+    {
+      "id": "ai-4",
+      "description": "Inconsistent tone of voice across product and marketing pages",
+      "category": "content",
+      "severity": "low",
+      "keywords": ["tone", "voice", "consistency", "messaging"]
+    },
+    {
+      "id": "ai-5",
+      "description": "No trust indicators visible above the fold — missing testimonials or customer logos",
+      "category": "trust",
+      "severity": "medium",
+      "keywords": ["trust", "testimonials", "logos", "social proof"]
+    },
+    {
+      "id": "ai-6",
+      "description": "Navigation labels overlap in meaning — users cannot distinguish between sections",
+      "category": "navigation",
+      "severity": "medium",
+      "keywords": ["navigation", "labels", "overlap", "confusing", "sections"]
+    }
+  ],
+  "humanFindings": [
+    {
+      "id": "human-1",
+      "description": "No clear action button on the homepage — visitors don't know what to do next",
+      "category": "conversion",
+      "severity": "high",
+      "keywords": ["action", "button", "homepage", "CTA", "conversion"]
+    },
+    {
+      "id": "human-2",
+      "description": "Confusing layout — hard to tell what's most important on the page, hierarchy unclear",
+      "category": "visual_hierarchy",
+      "severity": "medium",
+      "keywords": ["layout", "confusing", "hierarchy", "importance"]
+    },
+    {
+      "id": "human-3",
+      "description": "Headline doesn't explain what the product does — weak value proposition",
+      "category": "content",
+      "severity": "high",
+      "keywords": ["headline", "product", "explain", "value proposition"]
+    },
+    {
+      "id": "human-4",
+      "description": "Footer links are broken — several lead to 404 pages",
+      "category": "navigation",
+      "severity": "medium",
+      "keywords": ["footer", "links", "broken", "404"]
+    }
+  ]
+}

package/examples/basic/run.ts

@@ -0,0 +1,33 @@
+/**
+ * Basic Example — Simplest possible evaluation
+ *
+ * Run: npx ts-node examples/basic/run.ts
+ * Or:  gravito-eval run examples/basic
+ */
+
+import { evaluate } from "../../src";
+import data from "./input.json";
+
+const result = evaluate(data.aiFindings as any, data.humanFindings as any);
+
+console.log("Gravito Eval Results\n");
+console.log(`Recall: ${Math.round(result.detection.recall * 100)}%`);
+console.log(`Precision: ${Math.round(result.detection.precision * 100)}%`);
+console.log(`F1: ${Math.round(result.detection.f1 * 100)}%`);
+console.log();
+console.log(`Top-3 Agreement: ${Math.round(result.ranking.top3Overlap * 100)}%`);
+if (result.novelSignal) {
+  console.log(`Novel Signal: ${Math.round(result.novelSignal.validatedNovelRate * 100)}% (validated)`);
+}
+console.log();
+console.log("Interpretation:");
+if (result.detection.recall >= 0.7) {
+  console.log("- Strong alignment with human judgment");
+} else if (result.detection.recall >= 0.5) {
+  console.log("- Moderate alignment — some human findings missed");
+} else {
+  console.log("- Low alignment — many human findings missed");
+}
+if (result.novelSignal && result.novelSignal.validatedNovelRate >= 0.25) {
+  console.log("- Additional issues detected beyond baseline");
+}

package/package.json

@@ -0,0 +1,50 @@
+{
+  "name": "gravito-eval",
+  "version": "0.1.0",
+  "description": "Measure how closely AI decisions match human judgment — and where they add new signal.",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "bin": {
+    "gravito-eval": "dist/cli/index.js"
+  },
+  "files": [
+    "dist",
+    "examples",
+    "README.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "clean": "rm -rf dist",
+    "prepublishOnly": "npm run clean && npm run build"
+  },
+  "keywords": [
+    "ai-evaluation",
+    "alignment",
+    "calibration",
+    "precision",
+    "recall",
+    "f1-score",
+    "semantic-matching",
+    "novel-signal",
+    "human-ai-comparison",
+    "evaluation-framework"
+  ],
+  "author": "Gravito <hello@gravitoai.com>",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/samuelrkestenbaum-dot/gravito-eval.git"
+  },
+  "homepage": "https://gravito.ai",
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "devDependencies": {
+    "@types/node": "^25.5.0",
+    "typescript": "^5.4.0",
+    "vitest": "^1.6.0"
+  }
+}