@telora/factory 0.4.5

package/dist/crash-recovery.d.ts

@@ -0,0 +1,83 @@
+/**
+ * Factory crash recovery: orphaned worktree detection, stale session cleanup,
+ * and stuck instance recovery.
+ *
+ * When the factory orchestrator crashes (SIGKILL, OOM, power loss), worktrees
+ * and DB records persist with no in-memory tracking. On restart, this module:
+ *
+ * 1. Scans `.telora/factory-worktrees/` for directories not tracked in memory.
+ * 2. Resets factory_sessions stuck in 'running' to 'failed'.
+ * 3. Resets factory_instances stuck in 'building'/'gating' to 'paused' with
+ *    an escalation about unclean shutdown.
+ * 4. Commits WIP changes in orphaned worktrees to preserve in-flight work.
+ *
+ * Runs after config init and before the poll loop starts, so the factory
+ * starts in a clean, known state.
+ *
+ * Mirrors the recovery patterns from packages/daemon/src/crash-recovery.ts.
+ */
+import type { FactoryConfig } from './types.js';
+export interface OrphanedWorktree {
+    /** Absolute path to the worktree directory on disk. */
+    worktreePath: string;
+    /** Git branch name derived from the worktree entry (if available). */
+    branchName: string | null;
+}
+export interface RecoveryAction {
+    type: 'worktree_wip_committed' | 'session_reset' | 'instance_paused' | 'escalation_created' | 'execution_unit_reset';
+    /** Entity ID (worktree path, session ID, instance ID, or execution unit ID). */
+    entityId: string;
+    /** Human-readable description of what was done. */
+    description: string;
+}
+export interface RecoveryResult {
+    /** Number of orphaned worktrees found on disk. */
+    orphanedWorktreesFound: number;
+    /** Number of WIP commits created for orphaned worktrees. */
+    wipCommitsCreated: number;
+    /** Number of sessions reset from 'running' to 'failed'. */
+    sessionsReset: number;
+    /** Number of instances reset from 'building'/'gating' to 'paused'. */
+    instancesPaused: number;
+    /** Number of escalations created for paused instances. */
+    escalationsCreated: number;
+    /** Number of execution units reset from 'assigned'/'running' to 'idle'. */
+    executionUnitsReset: number;
+    /** Detailed log of all recovery actions taken. */
+    actions: RecoveryAction[];
+    /** Non-fatal errors encountered during recovery. */
+    errors: string[];
+}
+/**
+ * Parse `git worktree list --porcelain` output to build a map of
+ * worktree path -> branch name for all registered worktrees.
+ */
+export declare function parseWorktreeList(repoPath: string): Map<string, string>;
+/**
+ * Scan the factory worktree directory for directories that exist on disk.
+ *
+ * Returns all subdirectories found in `factoryWorktreeDir`, each with
+ * its git branch name (if git tracks it) or null.
+ *
+ * The caller determines which worktrees are "orphaned" by comparing
+ * against the in-memory active instances map (which is empty on fresh start).
+ */
+export declare function scanFactoryWorktrees(config: FactoryConfig): OrphanedWorktree[];
+/**
+ * Run full crash recovery on factory orchestrator startup.
+ *
+ * Should be called after config init and API client setup, but before
+ * the poll loop starts. Safe to call multiple times -- idempotent by
+ * design. A second run with no crashed state finds nothing to recover.
+ *
+ * Recovery steps:
+ * 1. Scan for orphaned worktrees and WIP-commit uncommitted changes.
+ * 2. Reset factory_sessions stuck in 'running' to 'failed'.
+ * 3. Reset factory_instances stuck in 'building'/'gating' to 'paused'.
+ * 4. Create escalations for paused instances.
+ *
+ * @param config - Factory configuration.
+ * @returns      A RecoveryResult summarizing all actions taken.
+ */
+export declare function recoverFromCrash(config: FactoryConfig): Promise<RecoveryResult>;
+//# sourceMappingURL=crash-recovery.d.ts.map

package/dist/crash-recovery.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"crash-recovery.d.ts","sourceRoot":"","sources":["../src/crash-recovery.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAKH,OAAO,KAAK,EAAE,aAAa,EAAmB,MAAM,YAAY,CAAC;AAYjE,MAAM,WAAW,gBAAgB;IAC/B,uDAAuD;IACvD,YAAY,EAAE,MAAM,CAAC;IACrB,sEAAsE;IACtE,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,wBAAwB,GAAG,eAAe,GAAG,iBAAiB,GAAG,oBAAoB,GAAG,sBAAsB,CAAC;IACrH,gFAAgF;IAChF,QAAQ,EAAE,MAAM,CAAC;IACjB,mDAAmD;IACnD,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,kDAAkD;IAClD,sBAAsB,EAAE,MAAM,CAAC;IAC/B,4DAA4D;IAC5D,iBAAiB,EAAE,MAAM,CAAC;IAC1B,2DAA2D;IAC3D,aAAa,EAAE,MAAM,CAAC;IACtB,sEAAsE;IACtE,eAAe,EAAE,MAAM,CAAC;IACxB,0DAA0D;IAC1D,kBAAkB,EAAE,MAAM,CAAC;IAC3B,2EAA2E;IAC3E,mBAAmB,EAAE,MAAM,CAAC;IAC5B,kDAAkD;IAClD,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,oDAAoD;IACpD,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAyDD;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAiBvE;AAED;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,aAAa,GAAG,gBAAgB,EAAE,CA6C9E;AAgWD;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,gBAAgB,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,cAAc,CAAC,CA0FrF"}

package/dist/crash-recovery.js

@@ -0,0 +1,522 @@
+/**
+ * Factory crash recovery: orphaned worktree detection, stale session cleanup,
+ * and stuck instance recovery.
+ *
+ * When the factory orchestrator crashes (SIGKILL, OOM, power loss), worktrees
+ * and DB records persist with no in-memory tracking. On restart, this module:
+ *
+ * 1. Scans `.telora/factory-worktrees/` for directories not tracked in memory.
+ * 2. Resets factory_sessions stuck in 'running' to 'failed'.
+ * 3. Resets factory_instances stuck in 'building'/'gating' to 'paused' with
+ *    an escalation about unclean shutdown.
+ * 4. Commits WIP changes in orphaned worktrees to preserve in-flight work.
+ *
+ * Runs after config init and before the poll loop starts, so the factory
+ * starts in a clean, known state.
+ *
+ * Mirrors the recovery patterns from packages/daemon/src/crash-recovery.ts.
+ */
+import { execFileSync } from 'node:child_process';
+import { existsSync, readdirSync, statSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { commitWip } from './git-factory.js';
+import { updateInstanceStatus, getActiveInstances } from './queries/instances.js';
+import { updateFactorySession } from './queries/sessions.js';
+import { createFactoryEscalation } from './queries/escalations.js';
+import { getExecutionUnitsByInstance, updateExecutionUnit } from './queries/execution-units.js';
+import { productLabel } from './product-config.js';
+// ============================================================================
+// Git helpers (synchronous, same pattern as git-factory.ts)
+// ============================================================================
+/** Default timeout for git CLI operations (30 seconds -- read-only ops are fast). */
+const GIT_TIMEOUT_MS = 30_000;
+/**
+ * Run a git command synchronously and return a structured result.
+ * Never throws -- callers inspect `result.success`.
+ */
+function runGit(args, cwd) {
+    try {
+        const output = execFileSync('git', args, {
+            cwd,
+            encoding: 'utf-8',
+            stdio: ['pipe', 'pipe', 'pipe'],
+            timeout: GIT_TIMEOUT_MS,
+        });
+        return { success: true, output: output.trim(), error: '' };
+    }
+    catch (err) {
+        const execErr = err;
+        if (execErr.killed || execErr.signal === 'SIGTERM') {
+            return {
+                success: false,
+                output: execErr.stdout?.toString() ?? '',
+                error: `Git operation timed out after ${GIT_TIMEOUT_MS}ms: git ${args.join(' ')}`,
+            };
+        }
+        return {
+            success: false,
+            output: execErr.stdout?.toString() ?? '',
+            error: execErr.stderr?.toString() ?? execErr.message ?? 'Unknown git error',
+        };
+    }
+}
+// ============================================================================
+// Worktree scanning
+// ============================================================================
+/**
+ * Parse `git worktree list --porcelain` output to build a map of
+ * worktree path -> branch name for all registered worktrees.
+ */
+export function parseWorktreeList(repoPath) {
+    const result = runGit(['worktree', 'list', '--porcelain'], repoPath);
+    const map = new Map();
+    if (!result.success)
+        return map;
+    let currentPath = null;
+    for (const line of result.output.split('\n')) {
+        if (line.startsWith('worktree ')) {
+            currentPath = line.slice('worktree '.length);
+        }
+        else if (line.startsWith('branch refs/heads/') && currentPath) {
+            map.set(currentPath, line.slice('branch refs/heads/'.length));
+            currentPath = null;
+        }
+        else if (line === '') {
+            currentPath = null;
+        }
+    }
+    return map;
+}
+/**
+ * Scan the factory worktree directory for directories that exist on disk.
+ *
+ * Returns all subdirectories found in `factoryWorktreeDir`, each with
+ * its git branch name (if git tracks it) or null.
+ *
+ * The caller determines which worktrees are "orphaned" by comparing
+ * against the in-memory active instances map (which is empty on fresh start).
+ */
+export function scanFactoryWorktrees(config) {
+    // Parse git worktree list from all configured product repos
+    const pathToBranch = new Map();
+    for (const product of config.products) {
+        const worktreeMap = parseWorktreeList(product.repoPath);
+        for (const [wtPath, branch] of worktreeMap) {
+            pathToBranch.set(resolve(wtPath), branch);
+        }
+    }
+    // Scan each product's factory worktree directory
+    const orphans = [];
+    for (const product of config.products) {
+        const factoryDir = resolve(product.repoPath, '.telora', 'factory-worktrees');
+        if (!existsSync(factoryDir)) {
+            continue;
+        }
+        const entries = readdirSync(factoryDir);
+        const subdirs = entries.filter((entry) => {
+            const fullPath = join(factoryDir, entry);
+            try {
+                return statSync(fullPath).isDirectory();
+            }
+            catch {
+                return false;
+            }
+        });
+        for (const subdir of subdirs) {
+            const fullPath = resolve(join(factoryDir, subdir));
+            const branchName = pathToBranch.get(fullPath) ?? null;
+            orphans.push({
+                worktreePath: fullPath,
+                branchName,
+            });
+        }
+    }
+    if (orphans.length === 0) {
+        console.log('[crash-recovery] No orphaned factory worktrees found across any product');
+    }
+    return orphans;
+}
+// ============================================================================
+// Recovery: orphaned worktrees
+// ============================================================================
+/**
+ * WIP-commit any uncommitted changes in orphaned factory worktrees.
+ *
+ * This preserves in-flight work that was not committed before the crash.
+ * The worktrees are left in place -- they will be reused when the instance
+ * resumes (after being reset to 'paused').
+ */
+function recoverOrphanedWorktrees(orphans) {
+    let wipCommitsCreated = 0;
+    const actions = [];
+    for (const orphan of orphans) {
+        const label = orphan.branchName ?? 'unknown-branch';
+        try {
+            const sha = commitWip(orphan.worktreePath, `crash-recovery: ${label}`);
+            if (sha) {
+                wipCommitsCreated++;
+                actions.push({
+                    type: 'worktree_wip_committed',
+                    entityId: orphan.worktreePath,
+                    description: `WIP commit ${sha.slice(0, 8)} created for orphaned worktree (branch: ${label})`,
+                });
+                console.log(`[crash-recovery] WIP commit ${sha.slice(0, 8)} for orphaned worktree: ${orphan.worktreePath}`);
+            }
+            else {
+                console.log(`[crash-recovery] Orphaned worktree has no uncommitted changes: ${orphan.worktreePath}`);
+            }
+        }
+        catch (err) {
+            console.warn(`[crash-recovery] Failed to WIP-commit orphaned worktree ${orphan.worktreePath}: ` +
+                `${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    return { wipCommitsCreated, actions };
+}
+// ============================================================================
+// Recovery: stale sessions
+// ============================================================================
+/**
+ * Reset factory sessions stuck in 'running' or 'starting' status to 'failed'.
+ *
+ * After a crash, any sessions that were 'running' are no longer actually
+ * running -- their processes died with the orchestrator. We query the API
+ * for all instances that might have active sessions and reset them.
+ */
+async function recoverStaleSessions(config) {
+    const sessionsReset = 0;
+    const actions = [];
+    const errors = [];
+    // We need to find sessions stuck in 'running' status. Since we don't have
+    // a direct "get all running sessions" query, we use getActiveSessionsForInstance
+    // for each instance that might have stale sessions. We get the instances from
+    // the pending query (which returns instances that aren't in terminal states).
+    //
+    // However, getPendingInstances only returns 'pending' status. For crash
+    // recovery we need instances in 'building' and 'gating' states too.
+    // Those will be handled via the instance recovery below, which also
+    // resets their sessions. But we still want a catch-all for any sessions
+    // that might be orphaned.
+    //
+    // The approach: reset sessions for instances we know about from worktree
+    // scanning. The instance recovery step will handle the rest.
+    // Note: Session reset for specific instances happens in recoverStuckInstances.
+    // This function serves as documentation of the strategy. The actual per-instance
+    // session reset is co-located with the instance status reset for atomicity.
+    void config; // Config may be used in future for direct session queries
+    return { sessionsReset, actions, errors };
+}
+// ============================================================================
+// Recovery: stuck instances
+// ============================================================================
+/**
+ * Find and recover factory instances stuck in 'building' or 'gating' status.
+ *
+ * After an unclean shutdown, these instances had active processing that
+ * was interrupted. We:
+ * 1. Reset their status to 'paused'.
+ * 2. Mark their active sessions as 'failed'.
+ * 3. Create an escalation to notify the human operator.
+ */
+async function recoverStuckInstances(config) {
+    let instancesPaused = 0;
+    let sessionsReset = 0;
+    let escalationsCreated = 0;
+    const actions = [];
+    const errors = [];
+    // Query non-terminal instances (designing, planning, building, paused) and
+    // filter for those stuck in active processing states after the crash.
+    // Query instances across all configured products (per-product isolation)
+    const instances = [];
+    for (const product of config.products) {
+        const pLabel = productLabel(product);
+        try {
+            const productInstances = await getActiveInstances(product.id);
+            if (productInstances.length > 0) {
+                console.log(`[crash-recovery] [${pLabel}] Found ${productInstances.length} active instance(s)`);
+            }
+            instances.push(...productInstances);
+        }
+        catch (err) {
+            const msg = `[${pLabel}] Failed to query instances for crash recovery: ${err instanceof Error ? err.message : String(err)}`;
+            console.warn(`[crash-recovery] ${msg}`);
+            errors.push(msg);
+            // continue to next product
+        }
+    }
+    // Filter for instances that are stuck in active processing states
+    // Note: 'gating' was removed as an instance-level status -- gates run
+    // inline per work unit during the 'building' phase.
+    const stuckStatuses = new Set(['building']);
+    const stuckInstances = instances.filter((inst) => stuckStatuses.has(inst.status));
+    if (stuckInstances.length === 0) {
+        console.log('[crash-recovery] No stuck instances found in building status');
+        return { instancesPaused, sessionsReset, escalationsCreated, actions, errors };
+    }
+    console.log(`[crash-recovery] Found ${stuckInstances.length} instance(s) stuck in building/gating status`);
+    // Build a reverse map: instance product_id -> product label for per-instance logging
+    const productLabelByProductId = new Map();
+    for (const product of config.products) {
+        productLabelByProductId.set(product.id, productLabel(product));
+    }
+    for (const instance of stuckInstances) {
+        const instLabel = productLabelByProductId.get(instance.productId ?? '') ?? '';
+        const instPrefix = instLabel ? `[crash-recovery] [${instLabel}]` : '[crash-recovery]';
+        // Step 1: Reset active sessions to 'failed'
+        try {
+            const { getActiveSessionsForInstance } = await import('./queries/sessions.js');
+            const activeSessions = await getActiveSessionsForInstance(instance.id);
+            for (const session of activeSessions) {
+                try {
+                    await updateFactorySession(session.id, {
+                        status: 'failed',
+                        endedAt: new Date().toISOString(),
+                    });
+                    sessionsReset++;
+                    actions.push({
+                        type: 'session_reset',
+                        entityId: session.id,
+                        description: `Session reset from '${session.status}' to 'failed' (instance ${instance.id}, crash recovery)`,
+                    });
+                }
+                catch (err) {
+                    const msg = `Failed to reset session ${session.id}: ${err instanceof Error ? err.message : String(err)}`;
+                    console.warn(`${instPrefix} ${msg}`);
+                    errors.push(msg);
+                }
+            }
+        }
+        catch (err) {
+            const msg = `Failed to query sessions for instance ${instance.id}: ${err instanceof Error ? err.message : String(err)}`;
+            console.warn(`${instPrefix} ${msg}`);
+            errors.push(msg);
+        }
+        // Step 2: Reset instance status to 'paused'
+        try {
+            await updateInstanceStatus(instance.id, 'paused', {
+                escalationReason: 'Unclean shutdown recovery',
+                escalationMessage: `Factory orchestrator crashed or was killed while instance was in '${instance.status}' status. ` +
+                    'Instance has been paused for manual review. Active sessions have been marked as failed. ' +
+                    'Resume or cancel this instance from the control panel.',
+                escalatedAt: new Date().toISOString(),
+            });
+            instancesPaused++;
+            actions.push({
+                type: 'instance_paused',
+                entityId: instance.id,
+                description: `Instance reset from '${instance.status}' to 'paused' (crash recovery)`,
+            });
+            console.log(`${instPrefix} Instance ${instance.id} reset: ${instance.status} -> paused`);
+        }
+        catch (err) {
+            const msg = `Failed to pause instance ${instance.id}: ${err instanceof Error ? err.message : String(err)}`;
+            console.error(`${instPrefix} ${msg}`);
+            errors.push(msg);
+            continue; // Skip escalation if we couldn't even pause the instance
+        }
+        // Step 3: Create escalation
+        try {
+            await createFactoryEscalation(instance.id, 'Unclean shutdown: factory orchestrator crashed', `Instance was in '${instance.status}' status when the factory orchestrator shut down unexpectedly. ` +
+                `The instance has been paused and its active sessions marked as failed. ` +
+                `Please review the instance state and either resume or cancel it.`);
+            escalationsCreated++;
+            actions.push({
+                type: 'escalation_created',
+                entityId: instance.id,
+                description: `Escalation created for instance paused during crash recovery (was '${instance.status}')`,
+            });
+            console.log(`${instPrefix} Created escalation for instance ${instance.id}`);
+        }
+        catch (err) {
+            const msg = `Failed to create escalation for instance ${instance.id}: ${err instanceof Error ? err.message : String(err)}`;
+            console.warn(`${instPrefix} ${msg}`);
+            errors.push(msg);
+        }
+    }
+    return { instancesPaused, sessionsReset, escalationsCreated, actions, errors };
+}
+// ============================================================================
+// Recovery: orphaned execution units
+// ============================================================================
+/**
+ * Reset execution units stuck in 'assigned' or 'running' status to 'idle'.
+ *
+ * After a crash, these units had active sessions that died with the daemon.
+ * We query ALL active (non-terminal) instances and reset their orphaned units.
+ * This runs BEFORE instance adoption so the scheduler sees clean state.
+ */
+async function recoverOrphanedExecutionUnits(config) {
+    let executionUnitsReset = 0;
+    const actions = [];
+    const errors = [];
+    // Query instances across all configured products (per-product isolation)
+    const instances = [];
+    // Build a reverse map: product id -> product label for per-instance logging
+    const productLabelById = new Map();
+    for (const product of config.products) {
+        const pLabel = productLabel(product);
+        productLabelById.set(product.id, pLabel);
+        try {
+            const productInstances = await getActiveInstances(product.id);
+            instances.push(...productInstances);
+        }
+        catch (err) {
+            const msg = `[${pLabel}] Failed to query instances for execution unit recovery: ${err instanceof Error ? err.message : String(err)}`;
+            console.warn(`[crash-recovery] ${msg}`);
+            errors.push(msg);
+            // continue to next product
+        }
+    }
+    // Only building instances have execution units
+    const buildingInstances = instances.filter((inst) => inst.status === 'building');
+    if (buildingInstances.length === 0) {
+        console.log('[crash-recovery] No building instances found -- skipping execution unit recovery');
+        return { executionUnitsReset, actions, errors };
+    }
+    for (const instance of buildingInstances) {
+        const instLabel = productLabelById.get(instance.productId ?? '') ?? '';
+        const instPrefix = instLabel ? `[crash-recovery] [${instLabel}]` : '[crash-recovery]';
+        let units;
+        try {
+            units = await getExecutionUnitsByInstance(instance.id);
+        }
+        catch (err) {
+            const msg = `Failed to query execution units for instance ${instance.id}: ${err instanceof Error ? err.message : String(err)}`;
+            console.warn(`${instPrefix} ${msg}`);
+            errors.push(msg);
+            continue;
+        }
+        const orphanedUnits = units.filter((u) => u.status === 'assigned' || u.status === 'running');
+        for (const unit of orphanedUnits) {
+            try {
+                await updateExecutionUnit(unit.id, {
+                    status: 'idle',
+                    assignedStrategyId: null,
+                    claudeSessionId: null,
+                    assignedAt: null,
+                });
+                executionUnitsReset++;
+                actions.push({
+                    type: 'execution_unit_reset',
+                    entityId: unit.id,
+                    description: `Execution unit reset from '${unit.status}' to 'idle' ` +
+                        `(instance ${instance.id}, slot ${unit.slotIndex}, crash recovery)`,
+                });
+                console.log(`${instPrefix} Reset execution unit ${unit.id} (slot ${unit.slotIndex}) ` +
+                    `from '${unit.status}' to 'idle' (instance ${instance.id})`);
+            }
+            catch (err) {
+                const msg = `Failed to reset execution unit ${unit.id}: ${err instanceof Error ? err.message : String(err)}`;
+                console.warn(`${instPrefix} ${msg}`);
+                errors.push(msg);
+            }
+        }
+    }
+    if (executionUnitsReset > 0) {
+        console.log(`[crash-recovery] Reset ${executionUnitsReset} orphaned execution unit(s) to idle`);
+    }
+    else {
+        console.log('[crash-recovery] No orphaned execution units found');
+    }
+    return { executionUnitsReset, actions, errors };
+}
+// ============================================================================
+// Main recovery orchestrator
+// ============================================================================
+/**
+ * Run full crash recovery on factory orchestrator startup.
+ *
+ * Should be called after config init and API client setup, but before
+ * the poll loop starts. Safe to call multiple times -- idempotent by
+ * design. A second run with no crashed state finds nothing to recover.
+ *
+ * Recovery steps:
+ * 1. Scan for orphaned worktrees and WIP-commit uncommitted changes.
+ * 2. Reset factory_sessions stuck in 'running' to 'failed'.
+ * 3. Reset factory_instances stuck in 'building'/'gating' to 'paused'.
+ * 4. Create escalations for paused instances.
+ *
+ * @param config - Factory configuration.
+ * @returns      A RecoveryResult summarizing all actions taken.
+ */
+export async function recoverFromCrash(config) {
+    console.log('[crash-recovery] Starting factory crash recovery scan...');
+    const result = {
+        orphanedWorktreesFound: 0,
+        wipCommitsCreated: 0,
+        sessionsReset: 0,
+        instancesPaused: 0,
+        escalationsCreated: 0,
+        executionUnitsReset: 0,
+        actions: [],
+        errors: [],
+    };
+    // Step 1: Scan for orphaned worktrees and WIP-commit uncommitted changes
+    try {
+        const orphans = scanFactoryWorktrees(config);
+        result.orphanedWorktreesFound = orphans.length;
+        if (orphans.length > 0) {
+            console.log(`[crash-recovery] Found ${orphans.length} worktree(s) across factory-worktrees directories`);
+            const worktreeResult = recoverOrphanedWorktrees(orphans);
+            result.wipCommitsCreated = worktreeResult.wipCommitsCreated;
+            result.actions.push(...worktreeResult.actions);
+        }
+    }
+    catch (err) {
+        const msg = `Failed during worktree scan: ${err instanceof Error ? err.message : String(err)}`;
+        console.error(`[crash-recovery] ${msg}`);
+        result.errors.push(msg);
+    }
+    // Step 2: Reset stale sessions (catch-all for sessions not tied to instances)
+    try {
+        const sessionResult = await recoverStaleSessions(config);
+        result.sessionsReset += sessionResult.sessionsReset;
+        result.actions.push(...sessionResult.actions);
+        result.errors.push(...sessionResult.errors);
+    }
+    catch (err) {
+        const msg = `Failed during stale session recovery: ${err instanceof Error ? err.message : String(err)}`;
+        console.error(`[crash-recovery] ${msg}`);
+        result.errors.push(msg);
+    }
+    // Step 3: Reset orphaned execution units (assigned/running -> idle)
+    // This runs BEFORE instance adoption so the scheduler sees clean state.
+    try {
+        const euResult = await recoverOrphanedExecutionUnits(config);
+        result.executionUnitsReset = euResult.executionUnitsReset;
+        result.actions.push(...euResult.actions);
+        result.errors.push(...euResult.errors);
+    }
+    catch (err) {
+        const msg = `Failed during execution unit recovery: ${err instanceof Error ? err.message : String(err)}`;
+        console.error(`[crash-recovery] ${msg}`);
+        result.errors.push(msg);
+    }
+    // Step 4: Reset stuck instances (building/gating -> paused) and their sessions
+    try {
+        const instanceResult = await recoverStuckInstances(config);
+        result.instancesPaused = instanceResult.instancesPaused;
+        result.sessionsReset += instanceResult.sessionsReset;
+        result.escalationsCreated = instanceResult.escalationsCreated;
+        result.actions.push(...instanceResult.actions);
+        result.errors.push(...instanceResult.errors);
+    }
+    catch (err) {
+        const msg = `Failed during stuck instance recovery: ${err instanceof Error ? err.message : String(err)}`;
+        console.error(`[crash-recovery] ${msg}`);
+        result.errors.push(msg);
+    }
+    // Summary
+    console.log(`[crash-recovery] Factory recovery complete: ` +
+        `${result.orphanedWorktreesFound} worktree(s) found, ` +
+        `${result.wipCommitsCreated} WIP commit(s), ` +
+        `${result.sessionsReset} session(s) reset, ` +
+        `${result.executionUnitsReset} execution unit(s) reset, ` +
+        `${result.instancesPaused} instance(s) paused, ` +
+        `${result.escalationsCreated} escalation(s) created`);
+    if (result.errors.length > 0) {
+        console.warn(`[crash-recovery] ${result.errors.length} non-fatal error(s) during recovery`);
+    }
+    return result;
+}
+//# sourceMappingURL=crash-recovery.js.map

package/dist/crash-recovery.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crash-recovery.js","sourceRoot":"","sources":["../src/crash-recovery.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC5D,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAE1C,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAClF,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAE,uBAAuB,EAAE,MAAM,0BAA0B,CAAC;AACnE,OAAO,EAAE,2BAA2B,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AAChG,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAwCnD,+EAA+E;AAC/E,4DAA4D;AAC5D,+EAA+E;AAE/E,qFAAqF;AACrF,MAAM,cAAc,GAAG,MAAM,CAAC;AAQ9B;;;GAGG;AACH,SAAS,MAAM,CAAC,IAAc,EAAE,GAAW;IACzC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,EAAE,IAAI,EAAE;YACvC,GAAG;YACH,QAAQ,EAAE,OAAO;YACjB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;YAC/B,OAAO,EAAE,cAAc;SACxB,CAAC,CAAC;QACH,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;IAC7D,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,MAAM,OAAO,GAAG,GAMf,CAAC;QAEF,IAAI,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YACnD,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE;gBACxC,KAAK,EAAE,iCAAiC,cAAc,WAAW,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;aAClF,CAAC;QACJ,CAAC;QAED,OAAO;YACL,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE;YACxC,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,IAAI,OAAO,CAAC,OAAO,IAAI,mBAAmB;SAC5E,CAAC;IACJ,CAAC;AACH,CAAC;AAED,+EAA+E;AAC/E,oBAAoB;AACpB,+EAA+E;AAE/E;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,QAAgB;IAChD,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,UAAU,EAAE,MAAM,EAAE,aAAa,CAAC,EAAE,QAAQ,CAAC,CAAC;IACrE,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAC;IACtC,IAAI,CAAC,MAAM,CAAC,OAAO;QAAE,OAAO,GAAG,CAAC;IAEhC,IAAI,WAAW,GAAkB,IAAI,CAAC;IACtC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAC7C,IAAI,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;YACjC,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QAC/C,CAAC;aAAM,IAAI,IAAI,CAAC,UAAU,CAAC,oBAAoB,CAAC,IAAI,WAAW,EAAE,CAAC;YAChE,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC;YAC9D,WAAW,GAAG,IAAI,CAAC;QACrB,CAAC;aAAM,IAAI,IAAI,KAAK,EAAE,EAAE,CAAC;YACvB,WAAW,GAAG,IAAI,CAAC;QACrB,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,oBAAoB,CAAC,MAAqB;IACxD,4DAA4D;IAC5D,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC/C,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACtC,MAAM,WAAW,GAAG,iBAAiB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACxD,KAAK,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,WAAW,EAAE,CAAC;YAC3C,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;QAC5C,CAAC;IACH,CAAC;IAED,iDAAiD;IACjD,MAAM,OAAO,GAAuB,EAAE,CAAC;IACvC,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACtC,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,EAAE,SAAS,EAAE,mBAAmB,CAAC,CAAC;QAE7E,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC5B,SAAS;QACX,CAAC;QAED,MAAM,OAAO,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC;QACxC,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;YACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YACzC,IAAI,CAAC;gBACH,OAAO,QAAQ,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;YAC1C,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC;YACnD,MAAM,UAAU,GAAG,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;YAEtD,OAAO,CAAC,IAAI,CAAC;gBACX,YAAY,EAAE,QAAQ;gBACtB,UAAU;aACX,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,CAAC,GAAG,CAAC,yEAAyE,CAAC,CAAC;IACzF,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,+EAA+E;AAC/E,+BAA+B;AAC/B,+EAA+E;AAE/E;;;;;;GAMG;AACH,SAAS,wBAAwB,CAC/B,OAA2B;IAE3B,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAC1B,MAAM,OAAO,GAAqB,EAAE,CAAC;IAErC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,IAAI,gBAAgB,CAAC;QAEpD,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC,YAAY,EAAE,mBAAmB,KAAK,EAAE,CAAC,CAAC;YACvE,IAAI,GAAG,EAAE,CAAC;gBACR,iBAAiB,EAAE,CAAC;gBACpB,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,wBAAwB;oBAC9B,QAAQ,EAAE,MAAM,CAAC,YAAY;oBAC7B,WAAW,EAAE,cAAc,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,2CAA2C,KAAK,GAAG;iBAC9F,CAAC,CAAC;gBACH,OAAO,CAAC,GAAG,CACT,+BAA+B,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,2BAA2B,MAAM,CAAC,YAAY,EAAE,CAC/F,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,GAAG,CACT,kEAAkE,MAAM,CAAC,YAAY,EAAE,CACxF,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,IAAI,CACV,2DAA2D,MAAM,CAAC,YAAY,IAAI;gBAClF,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACtD,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,EAAE,iBAAiB,EAAE,OAAO,EAAE,CAAC;AACxC,CAAC;AAED,+EAA+E;AAC/E,2BAA2B;AAC3B,+EAA+E;AAE/E;;;;;;GAMG;AACH,KAAK,UAAU,oBAAoB,CACjC,MAAqB;IAErB,MAAM,aAAa,GAAG,CAAC,CAAC;IACxB,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,0EAA0E;IAC1E,iFAAiF;IACjF,8EAA8E;IAC9E,8EAA8E;IAC9E,EAAE;IACF,wEAAwE;IACxE,oEAAoE;IACpE,oEAAoE;IACpE,wEAAwE;IACxE,0BAA0B;IAC1B,EAAE;IACF,yEAAyE;IACzE,6DAA6D;IAE7D,+EAA+E;IAC/E,iFAAiF;IACjF,4EAA4E;IAE5E,KAAK,MAAM,CAAC,CAAC,0DAA0D;IAEvE,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AAC5C,CAAC;AAED,+EAA+E;AAC/E,4BAA4B;AAC5B,+EAA+E;AAE/E;;;;;;;;GAQG;AACH,KAAK,UAAU,qBAAqB,CAClC,MAAqB;IAQrB,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,kBAAkB,GAAG,CAAC,CAAC;IAC3B,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,2EAA2E;IAC3E,sEAAsE;IAEtE,yEAAyE;IACzE,MAAM,SAAS,GAAsB,EAAE,CAAC;IACxC,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;QACrC,IAAI,CAAC;YACH,MAAM,gBAAgB,GAAG,MAAM,kBAAkB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAC9D,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAChC,OAAO,CAAC,GAAG,CAAC,qBAAqB,MAAM,WAAW,gBAAgB,CAAC,MAAM,qBAAqB,CAAC,CAAC;YAClG,CAAC;YACD,SAAS,CAAC,IAAI,CAAC,GAAG,gBAAgB,CAAC,CAAC;QACtC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,IAAI,MAAM,mDAAmD,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5H,OAAO,CAAC,IAAI,CAAC,oBAAoB,GAAG,EAAE,CAAC,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACjB,2BAA2B;QAC7B,CAAC;IACH,CAAC;IAED,kEAAkE;IAClE,sEAAsE;IACtE,oDAAoD;IACpD,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;IAC5C,MAAM,cAAc,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;IAElF,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,CAAC,GAAG,CAAC,8DAA8D,CAAC,CAAC;QAC5E,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,kBAAkB,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;IACjF,CAAC;IAED,OAAO,CAAC,GAAG,CACT,0BAA0B,cAAc,CAAC,MAAM,8CAA8C,CAC9F,CAAC;IAEF,qFAAqF;IACrF,MAAM,uBAAuB,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC1D,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACtC,uBAAuB,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;IACjE,CAAC;IAED,KAAK,MAAM,QAAQ,IAAI,cAAc,EAAE,CAAC;QACtC,MAAM,SAAS,GAAG,uBAAuB,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QAC9E,MAAM,UAAU,GAAG,SAAS,CAAC,CAAC,CAAC,qBAAqB,SAAS,GAAG,CAAC,CAAC,CAAC,kBAAkB,CAAC;QAEtF,4CAA4C;QAC5C,IAAI,CAAC;YACH,MAAM,EAAE,4BAA4B,EAAE,GAAG,MAAM,MAAM,CAAC,uBAAuB,CAAC,CAAC;YAC/E,MAAM,cAAc,GAAG,MAAM,4BAA4B,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YAEvE,KAAK,MAAM,OAAO,IAAI,cAAc,EAAE,CAAC;gBACrC,IAAI,CAAC;oBACH,MAAM,oBAAoB,CAAC,OAAO,CAAC,EAAE,EAAE;wBACrC,MAAM,EAAE,QAAQ;wBAChB,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;qBAClC,CAAC,CAAC;oBACH,aAAa,EAAE,CAAC;oBAChB,OAAO,CAAC,IAAI,CAAC;wBACX,IAAI,EAAE,eAAe;wBACrB,QAAQ,EAAE,OAAO,CAAC,EAAE;wBACpB,WAAW,EAAE,uBAAuB,OAAO,CAAC,MAAM,2BAA2B,QAAQ,CAAC,EAAE,mBAAmB;qBAC5G,CAAC,CAAC;gBACL,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,MAAM,GAAG,GAAG,2BAA2B,OAAO,CAAC,EAAE,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;oBACzG,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;oBACrC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACnB,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,yCAAyC,QAAQ,CAAC,EAAE,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YACxH,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;YACrC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;QAED,4CAA4C;QAC5C,IAAI,CAAC;YACH,MAAM,oBAAoB,CAAC,QAAQ,CAAC,EAAE,EAAE,QAAQ,EAAE;gBAChD,gBAAgB,EAAE,2BAA2B;gBAC7C,iBAAiB,EACf,qEAAqE,QAAQ,CAAC,MAAM,YAAY;oBAChG,0FAA0F;oBAC1F,wDAAwD;gBAC1D,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACtC,CAAC,CAAC;YACH,eAAe,EAAE,CAAC;YAClB,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,iBAAiB;gBACvB,QAAQ,EAAE,QAAQ,CAAC,EAAE;gBACrB,WAAW,EAAE,wBAAwB,QAAQ,CAAC,MAAM,gCAAgC;aACrF,CAAC,CAAC;YACH,OAAO,CAAC,GAAG,CACT,GAAG,UAAU,aAAa,QAAQ,CAAC,EAAE,WAAW,QAAQ,CAAC,MAAM,YAAY,CAC5E,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,4BAA4B,QAAQ,CAAC,EAAE,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3G,OAAO,CAAC,KAAK,CAAC,GAAG,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;YACtC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACjB,SAAS,CAAC,yDAAyD;QACrE,CAAC;QAED,4BAA4B;QAC5B,IAAI,CAAC;YACH,MAAM,uBAAuB,CAC3B,QAAQ,CAAC,EAAE,EACX,gDAAgD,EAChD,oBAAoB,QAAQ,CAAC,MAAM,iEAAiE;gBACpG,yEAAyE;gBACzE,kEAAkE,CACnE,CAAC;YACF,kBAAkB,EAAE,CAAC;YACrB,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,oBAAoB;gBAC1B,QAAQ,EAAE,QAAQ,CAAC,EAAE;gBACrB,WAAW,EAAE,sEAAsE,QAAQ,CAAC,MAAM,IAAI;aACvG,CAAC,CAAC;YACH,OAAO,CAAC,GAAG,CACT,GAAG,UAAU,oCAAoC,QAAQ,CAAC,EAAE,EAAE,CAC/D,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,4CAA4C,QAAQ,CAAC,EAAE,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3H,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;YACrC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IAED,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,kBAAkB,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AACjF,CAAC;AAED,+EAA+E;AAC/E,qCAAqC;AACrC,+EAA+E;AAE/E;;;;;;GAMG;AACH,KAAK,UAAU,6BAA6B,CAC1C,MAAqB;IAErB,IAAI,mBAAmB,GAAG,CAAC,CAAC;IAC5B,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,yEAAyE;IACzE,MAAM,SAAS,GAAsB,EAAE,CAAC;IACxC,4EAA4E;IAC5E,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAkB,CAAC;IACnD,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;QACrC,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QACzC,IAAI,CAAC;YACH,MAAM,gBAAgB,GAAG,MAAM,kBAAkB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAC9D,SAAS,CAAC,IAAI,CAAC,GAAG,gBAAgB,CAAC,CAAC;QACtC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,IAAI,MAAM,4DAA4D,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YACrI,OAAO,CAAC,IAAI,CAAC,oBAAoB,GAAG,EAAE,CAAC,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACjB,2BAA2B;QAC7B,CAAC;IACH,CAAC;IAED,+CAA+C;IAC/C,MAAM,iBAAiB,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC;IAEjF,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,kFAAkF,CAAC,CAAC;QAChG,OAAO,EAAE,mBAAmB,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;IAClD,CAAC;IAED,KAAK,MAAM,QAAQ,IAAI,iBAAiB,EAAE,CAAC;QACzC,MAAM,SAAS,GAAG,gBAAgB,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QACvE,MAAM,UAAU,GAAG,SAAS,CAAC,CAAC,CAAC,qBAAqB,SAAS,GAAG,CAAC,CAAC,CAAC,kBAAkB,CAAC;QAEtF,IAAI,KAAK,CAAC;QACV,IAAI,CAAC;YACH,KAAK,GAAG,MAAM,2BAA2B,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QACzD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,gDAAgD,QAAQ,CAAC,EAAE,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/H,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;YACrC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACjB,SAAS;QACX,CAAC;QAED,MAAM,aAAa,GAAG,KAAK,CAAC,MAAM,CAChC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,UAAU,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS,CACzD,CAAC;QAEF,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;YACjC,IAAI,CAAC;gBACH,MAAM,mBAAmB,CAAC,IAAI,CAAC,EAAE,EAAE;oBACjC,MAAM,EAAE,MAAM;oBACd,kBAAkB,EAAE,IAAI;oBACxB,eAAe,EAAE,IAAI;oBACrB,UAAU,EAAE,IAAI;iBACjB,CAAC,CAAC;gBACH,mBAAmB,EAAE,CAAC;gBACtB,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,sBAAsB;oBAC5B,QAAQ,EAAE,IAAI,CAAC,EAAE;oBACjB,WAAW,EAAE,8BAA8B,IAAI,CAAC,MAAM,cAAc;wBAClE,aAAa,QAAQ,CAAC,EAAE,UAAU,IAAI,CAAC,SAAS,mBAAmB;iBACtE,CAAC,CAAC;gBACH,OAAO,CAAC,GAAG,CACT,GAAG,UAAU,yBAAyB,IAAI,CAAC,EAAE,UAAU,IAAI,CAAC,SAAS,IAAI;oBACzE,SAAS,IAAI,CAAC,MAAM,yBAAyB,QAAQ,CAAC,EAAE,GAAG,CAC5D,CAAC;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,GAAG,GAAG,kCAAkC,IAAI,CAAC,EAAE,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC7G,OAAO,CAAC,IAAI,CAAC,GAAG,UAAU,IAAI,GAAG,EAAE,CAAC,CAAC;gBACrC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,mBAAmB,GAAG,CAAC,EAAE,CAAC;QAC5B,OAAO,CAAC,GAAG,CACT,0BAA0B,mBAAmB,qCAAqC,CACnF,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;IACpE,CAAC;IAED,OAAO,EAAE,mBAAmB,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AAClD,CAAC;AAED,+EAA+E;AAC/E,6BAA6B;AAC7B,+EAA+E;AAE/E;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,MAAqB;IAC1D,OAAO,CAAC,GAAG,CAAC,0DAA0D,CAAC,CAAC;IAExE,MAAM,MAAM,GAAmB;QAC7B,sBAAsB,EAAE,CAAC;QACzB,iBAAiB,EAAE,CAAC;QACpB,aAAa,EAAE,CAAC;QAChB,eAAe,EAAE,CAAC;QAClB,kBAAkB,EAAE,CAAC;QACrB,mBAAmB,EAAE,CAAC;QACtB,OAAO,EAAE,EAAE;QACX,MAAM,EAAE,EAAE;KACX,CAAC;IAEF,yEAAyE;IACzE,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;QAC7C,MAAM,CAAC,sBAAsB,GAAG,OAAO,CAAC,MAAM,CAAC;QAE/C,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,OAAO,CAAC,GAAG,CACT,0BAA0B,OAAO,CAAC,MAAM,mDAAmD,CAC5F,CAAC;YACF,MAAM,cAAc,GAAG,wBAAwB,CAAC,OAAO,CAAC,CAAC;YACzD,MAAM,CAAC,iBAAiB,GAAG,cAAc,CAAC,iBAAiB,CAAC;YAC5D,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,GAAG,GAAG,gCAAgC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/F,OAAO,CAAC,KAAK,CAAC,oBAAoB,GAAG,EAAE,CAAC,CAAC;QACzC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED,8EAA8E;IAC9E,IAAI,CAAC;QACH,MAAM,aAAa,GAAG,MAAM,oBAAoB,CAAC,MAAM,CAAC,CAAC;QACzD,MAAM,CAAC,aAAa,IAAI,aAAa,CAAC,aAAa,CAAC;QACpD,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;QAC9C,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IAC9C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,GAAG,GAAG,yCAAyC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QACxG,OAAO,CAAC,KAAK,CAAC,oBAAoB,GAAG,EAAE,CAAC,CAAC;QACzC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED,oEAAoE;IACpE,wEAAwE;IACxE,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,6BAA6B,CAAC,MAAM,CAAC,CAAC;QAC7D,MAAM,CAAC,mBAAmB,GAAG,QAAQ,CAAC,mBAAmB,CAAC;QAC1D,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC;QACzC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;IACzC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,GAAG,GAAG,0CAA0C,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QACzG,OAAO,CAAC,KAAK,CAAC,oBAAoB,GAAG,EAAE,CAAC,CAAC;QACzC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED,+EAA+E;IAC/E,IAAI,CAAC;QACH,MAAM,cAAc,GAAG,MAAM,qBAAqB,CAAC,MAAM,CAAC,CAAC;QAC3D,MAAM,CAAC,eAAe,GAAG,cAAc,CAAC,eAAe,CAAC;QACxD,MAAM,CAAC,aAAa,IAAI,cAAc,CAAC,aAAa,CAAC;QACrD,MAAM,CAAC,kBAAkB,GAAG,cAAc,CAAC,kBAAkB,CAAC;QAC9D,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;QAC/C,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IAC/C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,GAAG,GAAG,0CAA0C,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QACzG,OAAO,CAAC,KAAK,CAAC,oBAAoB,GAAG,EAAE,CAAC,CAAC;QACzC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED,UAAU;IACV,OAAO,CAAC,GAAG,CACT,8CAA8C;QAC9C,GAAG,MAAM,CAAC,sBAAsB,sBAAsB;QACtD,GAAG,MAAM,CAAC,iBAAiB,kBAAkB;QAC7C,GAAG,MAAM,CAAC,aAAa,qBAAqB;QAC5C,GAAG,MAAM,CAAC,mBAAmB,4BAA4B;QACzD,GAAG,MAAM,CAAC,eAAe,uBAAuB;QAChD,GAAG,MAAM,CAAC,kBAAkB,wBAAwB,CACrD,CAAC;IAEF,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,OAAO,CAAC,IAAI,CACV,oBAAoB,MAAM,CAAC,MAAM,CAAC,MAAM,qCAAqC,CAC9E,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}