@iamu-dev/iamu 0.2.0-beta.1 → 0.2.0-beta.10

package/dist/eval/gemini-walker.d.ts

@@ -0,0 +1,46 @@
+/**
+ * Autonomous Gemini Walker Runner
+ *
+ * Runs an LLM (Gemini 2.5 Flash via REST) against realistic product fixtures
+ * using the real `iamu` CLI driving surface, and grades the resulting walkthrough
+ * using the Walkthrough Evaluation Scorecard.
+ */
+import type { CanonicalWalkthrough } from "@iamu-dev/protocol";
+export interface GeminiRunnerOptions {
+    scenarioSlug: string;
+    targetUrl: string;
+    projectDir: string;
+    binPath: string;
+    cdpPort: number;
+    apiKey?: string;
+    googleCloudProject?: string;
+    googleCloudLocation?: string;
+    model?: string;
+    maxTurns?: number;
+}
+export interface GeminiCallTurn {
+    role: "user" | "model";
+    parts: Array<{
+        text: string;
+        thoughtSignature?: string;
+    } | {
+        functionCall: {
+            name: string;
+            args: Record<string, unknown>;
+        };
+        thoughtSignature?: string;
+    } | {
+        functionResponse: {
+            name: string;
+            response: Record<string, unknown>;
+        };
+    }>;
+}
+/**
+ * Execute an autonomous Gemini walker walkthrough run against a live fixture app.
+ */
+export declare function runGeminiWalkthrough(opts: GeminiRunnerOptions): Promise<{
+    runId: string;
+    canonical: CanonicalWalkthrough;
+    turns: number;
+}>;

package/dist/eval/walkthrough-evaluator.d.ts

@@ -0,0 +1,61 @@
+import type { CanonicalWalkthrough } from "@iamu-dev/protocol";
+export interface GroundTruthDefect {
+    id: string;
+    category: "compliance" | "functional" | "ux";
+    keywords: string[];
+    description: string;
+}
+export interface ScenarioEvalContract {
+    scenarioSlug: string;
+    expectedState: "populated" | "empty" | "mismatch" | "auth_gated";
+    optimalSteps: number;
+    expectedDefects?: GroundTruthDefect[];
+    unseededDataKeywords?: string[];
+    allowedSurfaces?: string[];
+    expectedApproximationTarget?: string;
+    authRequired?: boolean;
+}
+export interface WalkthroughEvaluationScorecard {
+    scenarioSlug: string;
+    runId: string;
+    expectedState: string;
+    defectRecall: {
+        matched: number;
+        expected: number;
+        rate: number;
+    };
+    defectPrecision: {
+        legitimate: number;
+        total: number;
+        rate: number;
+    };
+    falsePositiveRateEnvironment: {
+        count: number;
+        totalFindings: number;
+        rate: number;
+    };
+    preconditionAdherence: {
+        adhered: boolean;
+        reason: string;
+    };
+    trajectory: {
+        totalSteps: number;
+        optimalSteps: number;
+        driftCount: number;
+        stepWasteIndex: number;
+    };
+    authTransition: {
+        detected: boolean;
+        transitionSteps: number;
+        success: boolean;
+    };
+    passed: boolean;
+    summary: string;
+}
+/**
+ * Evaluates a completed canonical walkthrough against ground-truth contracts
+ * for compliance detection, empty-state false positives, semantic squinting, and auth.
+ */
+export declare function evaluateWalkthroughRun(canonical: CanonicalWalkthrough, contract: ScenarioEvalContract): WalkthroughEvaluationScorecard;
+/** Formats an array of evaluation scorecards into an ASCII scorecard table */
+export declare function formatEvaluationScorecardTable(scorecards: WalkthroughEvaluationScorecard[]): string;