agent-scenario-loop 0.1.0

package/dist/runner/live-comparison.d.ts

@@ -0,0 +1,67 @@
+type LiveProfileForComparison = {
+    label: string;
+    runDir: string;
+    runId: string;
+    scenarioId: string;
+};
+type ComparisonMetricStatus = 'better' | 'worse' | 'unchanged' | 'inconclusive';
+type ComparisonMetricHighlight = {
+    baseline: number | boolean | null;
+    current: number | boolean | null;
+    delta: number | null;
+    name: string;
+    status: Exclude<ComparisonMetricStatus, 'unchanged'>;
+    unit: string;
+};
+type ComparisonMetricSummary = {
+    counts: Record<ComparisonMetricStatus, number>;
+    notableMetrics: ComparisonMetricHighlight[];
+};
+type LiveComparisonResult = {
+    baselineDir: string | null;
+    comparisonDir: string | null;
+    label: string;
+    metricSummary?: ComparisonMetricSummary;
+    reason: string | null;
+    runId: string;
+    scenarioId: string;
+    status: 'better' | 'worse' | 'unchanged' | 'mixed' | 'inconclusive' | 'skipped';
+    summaryPath: string | null;
+};
+type CompareLiveProfilesOptions = {
+    outputDir: string;
+    profiles: LiveProfileForComparison[];
+};
+/**
+ * Checks whether a parsed CLI flag is enabled.
+ *
+ * @param {unknown} value
+ * @returns {boolean}
+ */
+declare function isEnabledFlag(value: unknown): boolean;
+/**
+ * Builds a compact agent-readable summary from a detailed comparison artifact.
+ *
+ * @param {Record<string, unknown>} comparison
+ * @returns {ComparisonMetricSummary | undefined}
+ */
+declare function buildComparisonMetricSummary(comparison: Record<string, unknown>): ComparisonMetricSummary | undefined;
+/**
+ * Writes comparison artifacts for one already-passed live profile.
+ *
+ * @param {{outputDir: string, profile: LiveProfileForComparison}} options
+ * @returns {Promise<LiveComparisonResult>}
+ */
+declare function compareLiveProfileToLatest({ outputDir, profile, }: {
+    outputDir: string;
+    profile: LiveProfileForComparison;
+}): Promise<LiveComparisonResult>;
+/**
+ * Compares live proof profiles against the latest trusted prior run for each scenario.
+ *
+ * @param {CompareLiveProfilesOptions} options
+ * @returns {Promise<LiveComparisonResult[]>}
+ */
+declare function compareLiveProfilesToLatest({ outputDir, profiles, }: CompareLiveProfilesOptions): Promise<LiveComparisonResult[]>;
+export { buildComparisonMetricSummary, compareLiveProfileToLatest, compareLiveProfilesToLatest, isEnabledFlag, };
+export type { CompareLiveProfilesOptions, LiveComparisonResult, LiveProfileForComparison, ComparisonMetricHighlight, ComparisonMetricSummary, };

package/dist/runner/live-comparison.js

@@ -0,0 +1,147 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildComparisonMetricSummary = buildComparisonMetricSummary;
+exports.compareLiveProfileToLatest = compareLiveProfileToLatest;
+exports.compareLiveProfilesToLatest = compareLiveProfilesToLatest;
+exports.isEnabledFlag = isEnabledFlag;
+const path = require('node:path');
+const { createArtifactLayout } = require('../core/artifact-layout');
+const { buildAgentSummaryMarkdown } = require('../core/agent-summary');
+const { readRunArtifacts } = require('../core/comparison');
+const { writeJsonArtifact, writeTextArtifact } = require('../core/artifact-writer');
+const { SCHEMAS } = require('../core/schema-validator');
+const { compareLatestTrustedRun } = require('./compare-latest');
+/**
+ * Checks whether a parsed CLI flag is enabled.
+ *
+ * @param {unknown} value
+ * @returns {boolean}
+ */
+function isEnabledFlag(value) {
+    return value === true || value === 'true';
+}
+/**
+ * Reports whether compare-latest failed only because no baseline exists yet.
+ *
+ * @param {unknown} error
+ * @returns {error is Error}
+ */
+function isMissingPriorRunError(error) {
+    return error instanceof Error && error.message.startsWith('No trusted prior run found');
+}
+/**
+ * Builds a compact agent-readable summary from a detailed comparison artifact.
+ *
+ * @param {Record<string, unknown>} comparison
+ * @returns {ComparisonMetricSummary | undefined}
+ */
+function buildComparisonMetricSummary(comparison) {
+    if (!Array.isArray(comparison.metricComparisons)) {
+        return undefined;
+    }
+    const counts = {
+        better: 0,
+        worse: 0,
+        unchanged: 0,
+        inconclusive: 0,
+    };
+    const notableMetrics = [];
+    for (const metric of comparison.metricComparisons) {
+        if (!metric || typeof metric !== 'object') {
+            continue;
+        }
+        const record = metric;
+        const status = record.status;
+        if (status !== 'better' && status !== 'worse' && status !== 'unchanged' && status !== 'inconclusive') {
+            continue;
+        }
+        counts[status] += 1;
+        if (status === 'unchanged') {
+            continue;
+        }
+        notableMetrics.push({
+            baseline: typeof record.baseline === 'number' || typeof record.baseline === 'boolean' ? record.baseline : null,
+            current: typeof record.current === 'number' || typeof record.current === 'boolean' ? record.current : null,
+            delta: typeof record.delta === 'number' ? record.delta : null,
+            name: typeof record.name === 'string' ? record.name : 'unknown metric',
+            status,
+            unit: typeof record.unit === 'string' ? record.unit : 'unknown',
+        });
+    }
+    return {
+        counts,
+        notableMetrics,
+    };
+}
+/**
+ * Writes comparison artifacts for one already-passed live profile.
+ *
+ * @param {{outputDir: string, profile: LiveProfileForComparison}} options
+ * @returns {Promise<LiveComparisonResult>}
+ */
+async function compareLiveProfileToLatest({ outputDir, profile, }) {
+    try {
+        const result = compareLatestTrustedRun({
+            currentDir: profile.runDir,
+            rootDir: outputDir,
+            scenarioId: profile.scenarioId,
+        });
+        const comparisonDir = path.join(outputDir, 'comparisons', profile.scenarioId, profile.runId);
+        const layout = createArtifactLayout({ outputDir: comparisonDir });
+        await writeJsonArtifact({
+            filePath: layout.comparison,
+            value: result.comparison,
+            schema: SCHEMAS.comparison,
+            label: 'Comparison artifact',
+        });
+        const current = readRunArtifacts(profile.runDir);
+        await writeTextArtifact({
+            filePath: layout.agentSummary,
+            content: buildAgentSummaryMarkdown({
+                health: current.health,
+                verdict: current.verdict,
+                comparison: result.comparison,
+            }),
+        });
+        const metricSummary = buildComparisonMetricSummary(result.comparison);
+        return {
+            baselineDir: result.baselineDir,
+            comparisonDir,
+            label: profile.label,
+            ...(metricSummary ? { metricSummary } : {}),
+            reason: null,
+            runId: profile.runId,
+            scenarioId: profile.scenarioId,
+            status: String(result.comparison.comparisonStatus),
+            summaryPath: layout.agentSummary,
+        };
+    }
+    catch (error) {
+        if (!isMissingPriorRunError(error)) {
+            throw error;
+        }
+        return {
+            baselineDir: null,
+            comparisonDir: null,
+            label: profile.label,
+            reason: error.message,
+            runId: profile.runId,
+            scenarioId: profile.scenarioId,
+            status: 'skipped',
+            summaryPath: null,
+        };
+    }
+}
+/**
+ * Compares live proof profiles against the latest trusted prior run for each scenario.
+ *
+ * @param {CompareLiveProfilesOptions} options
+ * @returns {Promise<LiveComparisonResult[]>}
+ */
+async function compareLiveProfilesToLatest({ outputDir, profiles, }) {
+    const comparisons = [];
+    for (const profile of profiles) {
+        comparisons.push(await compareLiveProfileToLatest({ outputDir, profile }));
+    }
+    return comparisons;
+}

package/dist/runner/live-ios.d.ts

@@ -0,0 +1,137 @@
+#!/usr/bin/env node
+type CliArgs = import('./ios-simctl').CliArgs;
+type IosGenericLiveOptions = {
+    agentDeviceExecutor?: import('./agent-device').CommandExecutor;
+    argentExecutor?: import('./argent').CommandExecutor;
+    delay?: (ms: number) => Promise<void>;
+    executor?: import('./ios-simctl').CommandExecutor;
+};
+type IosGenericLiveResult = {
+    aggregateSummary: import('./live-proof-summary').LiveProofSummaryResult;
+    outputDir: string;
+    preflightDir: string;
+    profileDir: string;
+};
+type SkippedInteractionProof = import('./live-proof-summary').LiveProofSkippedInteractionProofPointer;
+/**
+ * Prints CLI usage.
+ *
+ * @param {{write: (message: string) => unknown}} [output]
+ * @returns {void}
+ */
+declare function usage(output?: {
+    write: (message: string) => unknown;
+}): void;
+/**
+ * Reads a JSON object from disk.
+ *
+ * @param {string} filePath
+ * @returns {Record<string, unknown>}
+ */
+declare function readJson(filePath: string): Record<string, unknown>;
+/**
+ * Resolves a stable scenario id from a scenario file.
+ *
+ * @param {{scenario: Record<string, unknown>, scenarioPath: string}} options
+ * @returns {string}
+ */
+declare function resolveScenarioId({ scenario, scenarioPath, }: {
+    scenario: Record<string, unknown>;
+    scenarioPath: string;
+}): string;
+/**
+ * Resolves the iOS bundle id from explicit input or ASL config.
+ *
+ * @param {{args: CliArgs, config: Record<string, unknown>}} options
+ * @returns {string | null}
+ */
+declare function resolveIosBundleId({ args, config, }: {
+    args: CliArgs;
+    config: Record<string, unknown>;
+}): string | null;
+/**
+ * Converts an optional run suffix into a path-safe segment.
+ *
+ * @param {unknown} value
+ * @returns {string | null}
+ */
+declare function normalizeRunSuffix(value: unknown): string | null;
+/**
+ * Applies an optional suffix to deterministic run ids.
+ *
+ * @param {string} baseRunId
+ * @param {string | null} suffix
+ * @returns {string}
+ */
+declare function buildRunId(baseRunId: string, suffix: string | null): string;
+/**
+ * Throws if a profile or interaction proof failed before aggregate writing.
+ *
+ * @param {{kind: string, runDir: string, health: Record<string, unknown>, verdict?: Record<string, unknown>}} options
+ * @returns {void}
+ */
+declare function assertPassedRun({ health, kind, runDir, verdict, }: {
+    health: Record<string, unknown>;
+    kind: string;
+    runDir: string;
+    verdict?: Record<string, unknown>;
+}): void;
+/**
+ * Reports whether profile evidence is trusted enough to run sidecar interaction proofs.
+ *
+ * @param {{health: Record<string, unknown>, verdict: Record<string, unknown>}} run
+ * @returns {boolean}
+ */
+declare function isTrustedProfileRun({ health, verdict, }: {
+    health: Record<string, unknown>;
+    verdict: Record<string, unknown>;
+}): boolean;
+/**
+ * Builds skipped sidecar pointers for requested runners when the profile gate failed.
+ *
+ * @param {{requestedRunners: string[], runIdsByRunner: Record<string, string>, scenarioId: string, profileHealthStatus: unknown, profileVerdictStatus: unknown}} options
+ * @returns {SkippedInteractionProof[]}
+ */
+declare function buildSkippedInteractionProofs({ profileHealthStatus, profileVerdictStatus, requestedRunners, runIdsByRunner, scenarioId, }: {
+    profileHealthStatus: unknown;
+    profileVerdictStatus: unknown;
+    requestedRunners: string[];
+    runIdsByRunner: Record<string, string>;
+    scenarioId: string;
+}): SkippedInteractionProof[];
+/**
+ * Throws after aggregate writing when the live proof itself failed.
+ *
+ * @param {IosGenericLiveResult} result
+ * @returns {void}
+ */
+declare function assertAggregatePassed(result: IosGenericLiveResult): void;
+/**
+ * Throws after aggregate proof writing when fail-on-regression should gate the run.
+ *
+ * @param {{result: IosGenericLiveResult, comparisons: Array<{label: string, status: string}>}} options
+ * @returns {void}
+ */
+declare function assertNoRegressedComparisons({ comparisons, result, }: {
+    comparisons: Array<{
+        label: string;
+        status: string;
+    }>;
+    result: IosGenericLiveResult;
+}): void;
+/**
+ * Runs a generic one-scenario iOS live proof.
+ *
+ * @param {CliArgs} args
+ * @param {IosGenericLiveOptions} [options]
+ * @returns {Promise<IosGenericLiveResult>}
+ */
+declare function runIosLiveProof(args: CliArgs, options?: IosGenericLiveOptions): Promise<IosGenericLiveResult>;
+/**
+ * Runs the generic iOS live CLI.
+ *
+ * @returns {Promise<void>}
+ */
+declare function main(): Promise<void>;
+export { assertNoRegressedComparisons, assertAggregatePassed, assertPassedRun, buildRunId, buildSkippedInteractionProofs, isTrustedProfileRun, main, normalizeRunSuffix, readJson, resolveIosBundleId, resolveScenarioId, runIosLiveProof, usage, };
+export type { IosGenericLiveOptions, IosGenericLiveResult, };