agent-scenario-loop 0.1.0

package/dist/runner/compare.js

@@ -0,0 +1,157 @@
+#!/usr/bin/env node
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.assertNoRegressedComparison = assertNoRegressedComparison;
+exports.isEnabledFlag = isEnabledFlag;
+exports.main = main;
+exports.parseArgs = parseArgs;
+exports.resolveOutput = resolveOutput;
+exports.usage = usage;
+const path = require('node:path');
+const { buildAgentSummaryMarkdown } = require('../core/agent-summary');
+const { compareRunDirectories, readRunArtifacts } = require('../core/comparison');
+const { createArtifactLayout } = require('../core/artifact-layout');
+const { writeJsonArtifact, writeTextArtifact } = require('../core/artifact-writer');
+const { SCHEMAS } = require('../core/schema-validator');
+const { hasHelpFlag, writeUsage } = require('./cli');
+/**
+ * Prints CLI usage to stderr.
+ *
+ * @returns {void}
+ */
+function usage(output = process.stderr) {
+    writeUsage([
+        'Usage: asl-compare --baseline <run-dir> --current <run-dir> [--out <comparison.json|run-dir>] [--fail-on-regression]',
+        '',
+        'Without --out, prints comparison.json to stdout.',
+        'When --out points at a directory, writes comparison.json and agent-summary.md there.',
+        'Use --fail-on-regression to exit nonzero after writing evidence when comparisonStatus is worse.',
+    ], output);
+}
+/**
+ * Parses `--key value` arguments for the comparison CLI.
+ *
+ * @param {string[]} argv
+ * @returns {CliArgs}
+ */
+function parseArgs(argv) {
+    const args = {};
+    for (let index = 0; index < argv.length; index += 1) {
+        const token = argv[index];
+        if (!token || !token.startsWith('--')) {
+            continue;
+        }
+        const key = token.slice(2);
+        const value = argv[index + 1];
+        if (value && !value.startsWith('--')) {
+            args[key] = value;
+            index += 1;
+        }
+        else {
+            args[key] = true;
+        }
+    }
+    return args;
+}
+/**
+ * Resolves `--out` as either an explicit JSON file or a run directory.
+ *
+ * @param {string} out
+ * @returns {{comparisonPath: string, summaryPath: string | null, printedPath: string}}
+ */
+function resolveOutput(out) {
+    const resolved = path.resolve(out);
+    if (path.extname(resolved) === '.json') {
+        return {
+            comparisonPath: resolved,
+            summaryPath: null,
+            printedPath: resolved,
+        };
+    }
+    const layout = createArtifactLayout({ outputDir: resolved });
+    return {
+        comparisonPath: layout.comparison,
+        summaryPath: layout.agentSummary,
+        printedPath: resolved,
+    };
+}
+/**
+ * Returns whether a boolean CLI flag was provided.
+ *
+ * @param {unknown} value
+ * @returns {boolean}
+ */
+function isEnabledFlag(value) {
+    return value === true || value === 'true';
+}
+/**
+ * Throws when a comparison result should fail a strict regression gate.
+ *
+ * @param {{comparison: Record<string, unknown>, evidencePath?: string}} options
+ * @returns {void}
+ */
+function assertNoRegressedComparison({ comparison, evidencePath, }) {
+    if (comparison.comparisonStatus !== 'worse') {
+        return;
+    }
+    const runId = typeof comparison.runId === 'string' ? comparison.runId : 'current run';
+    const evidenceHint = evidencePath ? ` Inspect ${evidencePath}.` : '';
+    throw new Error(`Comparison regressed for ${runId}.${evidenceHint}`);
+}
+/**
+ * Runs the compare CLI.
+ *
+ * @returns {Promise<void>}
+ */
+async function main() {
+    const argv = process.argv.slice(2);
+    if (hasHelpFlag(argv)) {
+        usage(process.stdout);
+        return;
+    }
+    const args = parseArgs(argv);
+    if (typeof args.baseline !== 'string' || typeof args.current !== 'string') {
+        usage();
+        process.exitCode = 1;
+        return;
+    }
+    const baselineDir = path.resolve(args.baseline);
+    const currentDir = path.resolve(args.current);
+    const comparison = compareRunDirectories({ baselineDir, currentDir });
+    const failOnRegression = isEnabledFlag(args['fail-on-regression']);
+    if (typeof args.out === 'string' && args.out.length > 0) {
+        const { comparisonPath, summaryPath, printedPath } = resolveOutput(args.out);
+        await writeJsonArtifact({
+            filePath: comparisonPath,
+            value: comparison,
+            schema: SCHEMAS.comparison,
+            label: 'Comparison artifact',
+        });
+        if (summaryPath) {
+            const current = readRunArtifacts(currentDir);
+            await writeTextArtifact({
+                filePath: summaryPath,
+                content: buildAgentSummaryMarkdown({
+                    health: current.health,
+                    verdict: current.verdict,
+                    comparison,
+                }),
+            });
+        }
+        process.stdout.write(`${printedPath}\n`);
+        if (failOnRegression) {
+            assertNoRegressedComparison({ comparison, evidencePath: printedPath });
+        }
+        return;
+    }
+    process.stdout.write(`${JSON.stringify(comparison, null, 2)}\n`);
+    if (failOnRegression) {
+        assertNoRegressedComparison({ comparison });
+    }
+}
+if (require.main === module) {
+    main().catch((error) => {
+        console.error(error instanceof Error ? error.message : String(error));
+        process.exitCode = 1;
+    });
+}

package/dist/runner/demo-loop.d.ts

@@ -0,0 +1,45 @@
+#!/usr/bin/env node
+type CliArgs = {
+    out?: string | boolean;
+    [key: string]: string | boolean | undefined;
+};
+type DemoLoopResult = {
+    baselineRunDir: string;
+    comparison: Record<string, unknown>;
+    currentRunDir: string;
+    outputDir: string;
+    preflightDir: string;
+};
+/**
+ * Prints CLI usage.
+ *
+ * @param {{write: (message: string) => unknown}} [output]
+ * @returns {void}
+ */
+declare function usage(output?: {
+    write: (message: string) => unknown;
+}): void;
+/**
+ * Parses `--key value` arguments for the fixture demo loop.
+ *
+ * @param {string[]} argv
+ * @returns {CliArgs}
+ */
+declare function parseArgs(argv: string[]): CliArgs;
+/**
+ * Writes a fixture-backed preflight, profile history, current profile, and latest-trusted comparison.
+ *
+ * @param {{outputDir?: string}} options
+ * @returns {Promise<DemoLoopResult>}
+ */
+declare function runDemoLoop({ outputDir }?: {
+    outputDir?: string;
+}): Promise<DemoLoopResult>;
+/**
+ * Runs the fixture demo CLI.
+ *
+ * @returns {Promise<void>}
+ */
+declare function main(): Promise<void>;
+export { main, parseArgs, runDemoLoop, usage, };
+export type { CliArgs, DemoLoopResult, };

package/dist/runner/demo-loop.js

@@ -0,0 +1,170 @@
+#!/usr/bin/env node
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.main = main;
+exports.parseArgs = parseArgs;
+exports.runDemoLoop = runDemoLoop;
+exports.usage = usage;
+const path = require('node:path');
+const { buildAgentSummaryMarkdown } = require('../core/agent-summary');
+const { createArtifactLayout } = require('../core/artifact-layout');
+const { writeJsonArtifact, writeTextArtifact } = require('../core/artifact-writer');
+const { readRunArtifacts } = require('../core/comparison');
+const { SCHEMAS } = require('../core/schema-validator');
+const { buildPlanArtifacts } = require('./check-plan');
+const { hasHelpFlag, writeUsage } = require('./cli');
+const { compareLatestTrustedRun } = require('./compare-latest');
+const { runProfileIos } = require('./profile-ios');
+/**
+ * Prints CLI usage.
+ *
+ * @param {{write: (message: string) => unknown}} [output]
+ * @returns {void}
+ */
+function usage(output = process.stderr) {
+    writeUsage([
+        'Usage: asl-demo-loop [--out <dir>]',
+        '',
+        'Runs the fixture preflight, baseline/current profile logs, and comparison without a simulator.',
+    ], output);
+}
+/**
+ * Parses `--key value` arguments for the fixture demo loop.
+ *
+ * @param {string[]} argv
+ * @returns {CliArgs}
+ */
+function parseArgs(argv) {
+    const args = {};
+    for (let index = 0; index < argv.length; index += 1) {
+        const token = argv[index];
+        if (!token || !token.startsWith('--')) {
+            continue;
+        }
+        const key = token.slice(2);
+        const value = argv[index + 1];
+        if (value && !value.startsWith('--')) {
+            args[key] = value;
+            index += 1;
+        }
+        else {
+            args[key] = true;
+        }
+    }
+    return args;
+}
+/**
+ * Writes a fixture-backed preflight, profile history, current profile, and latest-trusted comparison.
+ *
+ * @param {{outputDir?: string}} options
+ * @returns {Promise<DemoLoopResult>}
+ */
+async function runDemoLoop({ outputDir = path.resolve('artifacts/demo-loop') } = {}) {
+    const root = process.cwd();
+    const resolvedOutputDir = path.resolve(outputDir);
+    const preflightDir = path.join(resolvedOutputDir, 'preflight', 'app-startup');
+    const profileRoot = path.join(resolvedOutputDir, 'profile-runs');
+    const configPath = path.join(root, 'core/config-template.json');
+    const profileScenarioPath = path.join(root, 'examples/scenarios/ios/app-startup.json');
+    const mobileScenarioPath = path.join(root, 'examples/scenarios/mobile/app-startup.json');
+    const runnerPath = path.join(root, 'examples/runners/xcodebuildmcp-ios.json');
+    const baselineLogPath = path.join(root, 'examples/event-logs/app-startup-baseline.log');
+    const currentLogPath = path.join(root, 'examples/event-logs/app-startup-current.log');
+    const preflight = await buildPlanArtifacts({
+        scenarioPath: mobileScenarioPath,
+        runnerPath,
+        platform: 'ios',
+        runId: 'demo-preflight',
+    });
+    const preflightLayout = createArtifactLayout({ outputDir: preflightDir });
+    await writeJsonArtifact({
+        filePath: preflightLayout.health,
+        value: preflight.health,
+        schema: SCHEMAS.health,
+        label: 'Health artifact',
+    });
+    await writeJsonArtifact({
+        filePath: preflightLayout.verdict,
+        value: preflight.verdict,
+        schema: SCHEMAS.verdict,
+        label: 'Verdict artifact',
+    });
+    await writeTextArtifact({
+        filePath: preflightLayout.agentSummary,
+        content: preflight.agentSummary,
+    });
+    await writeJsonArtifact({
+        filePath: preflightLayout.plannerCompatibility,
+        value: preflight.compatibility,
+        schema: {
+            type: 'object',
+            additionalProperties: true,
+        },
+        label: 'Planner compatibility artifact',
+    });
+    await runProfileIos({
+        config: configPath,
+        events: baselineLogPath,
+        out: profileRoot,
+        scenario: profileScenarioPath,
+        'run-id': 'demo-baseline',
+    });
+    const current = await runProfileIos({
+        config: configPath,
+        events: currentLogPath,
+        out: profileRoot,
+        scenario: profileScenarioPath,
+        'run-id': 'demo-current',
+    });
+    const latestComparison = compareLatestTrustedRun({
+        rootDir: profileRoot,
+        scenarioId: 'app-startup',
+        currentDir: current.runDir,
+    });
+    const currentLayout = createArtifactLayout({ outputDir: current.runDir });
+    await writeJsonArtifact({
+        filePath: currentLayout.comparison,
+        value: latestComparison.comparison,
+        schema: SCHEMAS.comparison,
+        label: 'Comparison artifact',
+    });
+    const currentArtifacts = readRunArtifacts(current.runDir);
+    await writeTextArtifact({
+        filePath: currentLayout.agentSummary,
+        content: buildAgentSummaryMarkdown({
+            health: currentArtifacts.health,
+            verdict: currentArtifacts.verdict,
+            comparison: latestComparison.comparison,
+        }),
+    });
+    return {
+        baselineRunDir: latestComparison.baselineDir,
+        comparison: latestComparison.comparison,
+        currentRunDir: current.runDir,
+        outputDir: resolvedOutputDir,
+        preflightDir,
+    };
+}
+/**
+ * Runs the fixture demo CLI.
+ *
+ * @returns {Promise<void>}
+ */
+async function main() {
+    const argv = process.argv.slice(2);
+    if (hasHelpFlag(argv)) {
+        usage(process.stdout);
+        return;
+    }
+    const args = parseArgs(argv);
+    const result = await runDemoLoop({
+        ...(typeof args.out === 'string' ? { outputDir: args.out } : {}),
+    });
+    process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
+}
+if (require.main === module) {
+    main().catch((error) => {
+        console.error(error instanceof Error ? error.message : String(error));
+        process.exitCode = 1;
+    });
+}

package/dist/runner/example-android-live.d.ts

@@ -0,0 +1,137 @@
+#!/usr/bin/env node
+type CliArgs = import('./android-adb').CliArgs;
+type LiveComparisonResult = import('./live-comparison').LiveComparisonResult;
+type LiveProofSummaryResult = import('./live-proof-summary').LiveProofSummaryResult;
+type AndroidLiveProofOptions = {
+    agentDeviceExecutor?: import('./agent-device').CommandExecutor;
+    argentExecutor?: import('./argent').CommandExecutor;
+    delay?: (ms: number) => Promise<void>;
+    executor?: import('./android-adb').CommandExecutor;
+    packageRoot?: string;
+};
+type AndroidInteractionProof = {
+    label: string;
+    runDir: string;
+    runId: string;
+    runnerId: string;
+    scenarioId: string;
+};
+type AndroidSkippedInteractionProof = import('./live-proof-summary').LiveProofSkippedInteractionProofPointer;
+type AndroidLiveProfile = {
+    healthStatus?: string;
+    label: string;
+    runDir: string;
+    runId: string;
+    scenario: string;
+    scenarioId: string;
+    verdictStatus?: string;
+};
+type AndroidLiveProofResult = {
+    aggregateSummary: LiveProofSummaryResult;
+    comparisons: LiveComparisonResult[];
+    interactionProofs: AndroidInteractionProof[];
+    outputDir: string;
+    preflightDir: string;
+    profiles: AndroidLiveProfile[];
+    skippedInteractionProofs: AndroidSkippedInteractionProof[];
+};
+type RegressionGateOptions = {
+    platformLabel: string;
+    result: AndroidLiveProofResult;
+};
+/**
+ * Prints CLI usage.
+ *
+ * @param {{write: (message: string) => unknown}} [output]
+ * @returns {void}
+ */
+declare function usage(output?: {
+    write: (message: string) => unknown;
+}): void;
+/**
+ * Resolves the Android target used by every example proof lane.
+ *
+ * @param {CliArgs} args
+ * @returns {string}
+ */
+declare function resolveAndroidSerial(args: CliArgs): string;
+/**
+ * Converts a caller-provided run suffix into a path-safe run-id segment.
+ *
+ * @param {unknown} value
+ * @returns {string | null}
+ */
+declare function normalizeRunSuffix(value: unknown): string | null;
+/**
+ * Applies the optional run suffix while preserving deterministic defaults.
+ *
+ * @param {string} baseRunId
+ * @param {string | null} suffix
+ * @returns {string}
+ */
+declare function buildLiveRunId(baseRunId: string, suffix: string | null): string;
+/**
+ * Builds the comparison lane suffix for enabled interaction proofs.
+ *
+ * @param {string[]} runnerIds
+ * @returns {string}
+ */
+declare function buildInteractionComparisonLane(runnerIds: string[]): string;
+/**
+ * Reports whether profile evidence is healthy enough to trust sidecar proofs and comparisons.
+ *
+ * @param {{health: Record<string, unknown>, verdict: Record<string, unknown>}} run
+ * @returns {boolean}
+ */
+declare function isTrustedProfileRun({ health, verdict, }: {
+    health: Record<string, unknown>;
+    verdict: Record<string, unknown>;
+}): boolean;
+/**
+ * Builds skipped sidecar pointers for requested example runners when the profile batch failed.
+ *
+ * @param {{failedProfiles: AndroidLiveProfile[], requestedRunners: string[], runIdsByRunner: Record<string, string>}} options
+ * @returns {AndroidSkippedInteractionProof[]}
+ */
+declare function buildSkippedInteractionProofs({ failedProfiles, requestedRunners, runIdsByRunner, }: {
+    failedProfiles: AndroidLiveProfile[];
+    requestedRunners: string[];
+    runIdsByRunner: Record<string, string>;
+}): AndroidSkippedInteractionProof[];
+/**
+ * Throws after aggregate proof writing when the live proof itself failed.
+ *
+ * @param {AndroidLiveProofResult} result
+ * @returns {void}
+ */
+declare function assertAggregatePassed(result: AndroidLiveProofResult): void;
+/**
+ * Throws after aggregate proof writing when fail-on-regression should gate the run.
+ *
+ * @param {RegressionGateOptions} options
+ * @returns {void}
+ */
+declare function assertNoRegressedComparisons({ platformLabel, result, }: RegressionGateOptions): void;
+/**
+ * Runs adb preflight plus all canonical Android example live profiles.
+ *
+ * @param {CliArgs} args
+ * @param {AndroidLiveProofOptions} [options]
+ * @returns {Promise<AndroidLiveProofResult>}
+ */
+declare function runExampleAndroidLiveProof(args: CliArgs, options?: AndroidLiveProofOptions): Promise<AndroidLiveProofResult>;
+/**
+ * Formats the live proof result for humans and agents.
+ *
+ * @param {AndroidLiveProofResult} result
+ * @returns {string}
+ */
+declare function formatResult(result: AndroidLiveProofResult): string;
+/**
+ * Runs the example Android live proof CLI.
+ *
+ * @returns {Promise<void>}
+ */
+declare function main(): Promise<void>;
+export { assertAggregatePassed, formatResult, assertNoRegressedComparisons, buildLiveRunId, buildSkippedInteractionProofs, buildInteractionComparisonLane, isTrustedProfileRun, main, normalizeRunSuffix, resolveAndroidSerial, runExampleAndroidLiveProof, usage, };
+export type { AndroidLiveProofOptions, AndroidInteractionProof, AndroidLiveProofResult, AndroidLiveProfile, };