@waymakeros/cli 2.0.0

package/dist/sync/sync-engine.js

@@ -0,0 +1,875 @@
+/**
+ * Waymaker Sync - Sync Engine
+ * Main orchestration engine for bidirectional sync between IDE and Commander
+ *
+ * Best Practices Implemented:
+ * 1. Persistent Index - tracks synced files across daemon restarts
+ * 2. Checksum-based change detection - skips unchanged files
+ * 3. Retry with exponential backoff - handles transient failures
+ * 4. Atomic index writes - prevents corruption
+ * 5. Conflict detection - warns when IDE and Commander diverge
+ */
+import * as path from 'path';
+import * as fs from 'fs/promises';
+import { glob } from 'glob';
+import { EventEmitter } from 'events';
+import { ConfigManager } from './config-manager.js';
+import { MarkdownParser } from './markdown-parser.js';
+import { TrinityMapper } from './trinity-mapper.js';
+import { FileWatcher } from './file-watcher.js';
+import { FolderStatusMapper } from './folder-status-mapper.js';
+import { SyncIndexManager } from './sync-index.js';
+/** Retry configuration */
+const RETRY_CONFIG = {
+    maxRetries: 3,
+    baseDelayMs: 1000,
+    maxDelayMs: 10000,
+};
+export class SyncEngine extends EventEmitter {
+    configManager;
+    parser;
+    mapper = null; // Initialized in initialize()
+    watcher = null;
+    folderStatusMapper;
+    indexManager;
+    projectPath;
+    running = false;
+    startedAt = null;
+    syncCount = 0;
+    errorCount = 0;
+    lastSyncAt = null;
+    recentErrors = [];
+    syncStates = new Map();
+    initialScanComplete = false;
+    constructor(projectPath) {
+        super();
+        this.projectPath = projectPath;
+        this.configManager = new ConfigManager(projectPath);
+        this.parser = new MarkdownParser();
+        this.folderStatusMapper = new FolderStatusMapper();
+        this.indexManager = new SyncIndexManager(projectPath);
+    }
+    /**
+     * Initialize sync engine
+     */
+    async initialize() {
+        console.error('[SyncEngine] Initializing...');
+        // Load configuration
+        await this.configManager.load();
+        const config = this.configManager.getConfig();
+        console.error('[SyncEngine] Configuration loaded');
+        console.error(`[SyncEngine] Workspace: ${config.workspace_id}`);
+        console.error(`[SyncEngine] Project: ${config.project_id}`);
+        console.error(`[SyncEngine] Taskboard: ${config.sync.taskboard_id || 'not set'}`);
+        console.error(`[SyncEngine] Mode: ${config.sync.mode}`);
+        // Load persistent sync index
+        await this.indexManager.load();
+        if (config.sync.taskboard_id) {
+            this.indexManager.setTaskboardId(config.sync.taskboard_id);
+        }
+        const indexStats = this.indexManager.getStats();
+        console.error(`[SyncEngine] Index loaded: ${indexStats.filesCount} files tracked, ${indexStats.totalSyncs} total syncs`);
+        // Initialize TrinityMapper with FolderService configuration
+        const folderServiceConfig = {
+            projectId: config.project_id,
+            workspaceId: config.workspace_id,
+            organizationId: config.organization_id,
+            userId: 'sync-engine', // TBD: Get from auth context
+            projectRoot: this.projectPath,
+        };
+        this.mapper = new TrinityMapper(undefined, folderServiceConfig);
+        console.error('[SyncEngine] TrinityMapper initialized with FolderService');
+    }
+    /**
+     * Start sync daemon
+     */
+    async start() {
+        if (this.running) {
+            throw new Error('Sync engine is already running');
+        }
+        await this.initialize();
+        const config = this.configManager.getConfig();
+        if (!config.sync.enabled) {
+            throw new Error('Sync is disabled in configuration');
+        }
+        if (!config.sync.taskboard_id) {
+            console.error('[SyncEngine] WARNING: No taskboard_id configured. Some sync operations may fail.');
+        }
+        // Create file watcher
+        this.watcher = new FileWatcher({
+            watchPatterns: config.sync.watch_patterns,
+            ignorePatterns: config.sync.ignore_patterns,
+            projectPath: this.projectPath,
+            debounceMs: 1000,
+        });
+        // Set up event handlers
+        this.watcher.on('change', (event) => {
+            this.handleFileChange(event);
+        });
+        this.watcher.on('ready', () => {
+            console.error('[SyncEngine] File watcher ready');
+            this.emit('ready');
+        });
+        this.watcher.on('error', (error) => {
+            console.error('[SyncEngine] Watcher error:', error);
+            this.handleError(error.message, 'watcher');
+        });
+        // Start watching
+        this.watcher.start();
+        this.running = true;
+        this.startedAt = new Date();
+        console.error('[SyncEngine] ✅ Sync engine started');
+        console.error(`[SyncEngine] Watching ${config.sync.watch_patterns.length} patterns`);
+        this.emit('started');
+    }
+    /**
+     * Stop sync daemon
+     */
+    async stop() {
+        if (!this.running) {
+            return;
+        }
+        console.error('[SyncEngine] Stopping sync engine...');
+        // Flush pending index writes
+        await this.indexManager.flush();
+        console.error('[SyncEngine] Index saved');
+        // Stop file watcher
+        if (this.watcher) {
+            await this.watcher.stop();
+            this.watcher = null;
+        }
+        this.running = false;
+        this.startedAt = null;
+        console.error('[SyncEngine] ✅ Sync engine stopped');
+        this.emit('stopped');
+    }
+    /**
+     * Handle file change events
+     */
+    async handleFileChange(event) {
+        const config = this.configManager.getConfig();
+        const mode = config.sync.mode;
+        // Skip if mode is commander-to-ide only
+        if (mode === 'commander-to-ide') {
+            return;
+        }
+        console.error(`[SyncEngine] Processing ${event.type}: ${event.path}`);
+        try {
+            // Handle file moves specially - detect status change
+            if (event.type === 'move' && event.oldPath) {
+                await this.handleFileMove(event);
+                return;
+            }
+            // Use retry logic for file changes too
+            const result = await this.syncWithRetry(event.path);
+            if (result.success) {
+                this.syncCount++;
+                this.lastSyncAt = new Date();
+                // Update sync state
+                const checksum = this.indexManager.calculateChecksum(await this.parser.parseFile(path.join(this.projectPath, event.path)).then(p => p.content));
+                // Update in-memory state
+                this.syncStates.set(event.path, {
+                    filePath: event.path,
+                    lastSyncedAt: new Date(),
+                    lastModifiedAt: event.timestamp,
+                    commanderId: result.commanderId || null,
+                    documentType: result.documentType,
+                    checksum,
+                });
+                // Update persistent index
+                if (result.operation !== 'skip') {
+                    const parsed = await this.parser.parseFile(path.join(this.projectPath, event.path));
+                    this.indexManager.updateFileState(event.path, {
+                        checksum,
+                        documentType: result.documentType,
+                        taskId: parsed.frontmatter.sync?.task_id || result.commanderId || undefined,
+                        documentId: parsed.frontmatter.sync?.document_id || undefined,
+                        layerId: parsed.frontmatter.sync?.layer_id || undefined,
+                        fileModifiedAt: event.timestamp,
+                    });
+                }
+                this.emit('sync-success', result);
+                console.error(`[SyncEngine] ✅ ${result.message}`);
+            }
+            else {
+                if (result.operation !== 'skip') {
+                    this.handleError(result.error || result.message, event.path);
+                }
+            }
+        }
+        catch (error) {
+            this.handleError(error instanceof Error ? error.message : String(error), event.path);
+        }
+    }
+    /**
+     * Handle file move events - update task status based on folder change
+     */
+    async handleFileMove(event) {
+        if (!event.oldPath) {
+            // Can't process without old path, fall back to normal sync
+            await this.syncFileToCommander(event.path);
+            return;
+        }
+        const absolutePath = path.join(this.projectPath, event.path);
+        // Detect status change from folder paths
+        const statusChange = this.folderStatusMapper.detectStatusChange(event.oldPath, event.path);
+        if (statusChange) {
+            console.error(`[SyncEngine] Status change detected: ${statusChange.oldStatus.statusSection} → ${statusChange.newStatus.statusSection}`);
+            // Parse the file to get task_id
+            const parsed = await this.parser.parseFile(absolutePath);
+            if (parsed.frontmatter.sync?.task_id) {
+                // Update task status in Commander
+                const taskId = parsed.frontmatter.sync.task_id;
+                const newStatus = statusChange.newStatus.statusSection;
+                console.error(`[SyncEngine] Updating task ${taskId} status to: ${newStatus}`);
+                try {
+                    await this.updateTaskStatus(taskId, newStatus);
+                    // Update sync state for new path
+                    const checksum = this.indexManager.calculateChecksum(parsed.content);
+                    const oldState = this.syncStates.get(event.oldPath);
+                    const docType = oldState?.documentType || 'task';
+                    // Update in-memory state
+                    this.syncStates.delete(event.oldPath);
+                    this.syncStates.set(event.path, {
+                        filePath: event.path,
+                        lastSyncedAt: new Date(),
+                        lastModifiedAt: event.timestamp,
+                        commanderId: taskId,
+                        documentType: docType,
+                        checksum,
+                    });
+                    // Update persistent index (remove old path, add new)
+                    this.indexManager.removeFile(event.oldPath);
+                    this.indexManager.updateFileState(event.path, {
+                        checksum,
+                        documentType: docType,
+                        taskId,
+                        documentId: parsed.frontmatter.sync?.document_id || undefined,
+                        layerId: parsed.frontmatter.sync?.layer_id || undefined,
+                        fileModifiedAt: event.timestamp,
+                    });
+                    this.syncCount++;
+                    this.lastSyncAt = new Date();
+                    const result = {
+                        success: true,
+                        operation: 'update',
+                        documentType: 'task',
+                        filePath: event.path,
+                        commanderId: taskId,
+                        message: `Task status updated: ${statusChange.oldStatus.statusSection} → ${newStatus}`,
+                    };
+                    this.emit('sync-success', result);
+                    console.error(`[SyncEngine] ✅ ${result.message}`);
+                }
+                catch (error) {
+                    this.handleError(error instanceof Error ? error.message : String(error), event.path);
+                }
+            }
+            else {
+                // No task_id, do a full sync to create/update
+                console.error(`[SyncEngine] No task_id in file, performing full sync`);
+                await this.syncFileToCommander(event.path);
+            }
+        }
+        else {
+            // No status change (move within same status folder), just update path metadata
+            console.error(`[SyncEngine] File moved but status unchanged, syncing content`);
+            await this.syncFileToCommander(event.path);
+        }
+    }
+    /**
+     * Update task status in Commander
+     */
+    async updateTaskStatus(taskId, newStatus) {
+        // Get API key from env or auth.json
+        let apiKey = process.env.WAYMAKER_API_KEY;
+        if (!apiKey) {
+            const authPath = path.join(process.env.HOME || '', '.waymaker', 'auth.json');
+            try {
+                const authData = JSON.parse(await fs.readFile(authPath, 'utf-8'));
+                apiKey = authData.accessToken;
+            }
+            catch {
+                // Ignore auth file read errors
+            }
+        }
+        if (!apiKey) {
+            throw new Error('Authentication required. Run: waymaker auth login');
+        }
+        const baseUrl = process.env.WAYMAKER_APPS_URL || 'https://apps.waymakerone.com';
+        const response = await fetch(`${baseUrl}/functions/v1/commander-task-operations`, {
+            method: 'POST',
+            headers: {
+                'Authorization': `Bearer ${apiKey}`,
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                action: 'update-task',
+                id: taskId,
+                status: newStatus,
+            }),
+        });
+        if (!response.ok) {
+            const errorText = await response.text();
+            throw new Error(`Failed to update task status: ${response.status} ${errorText}`);
+        }
+        const result = await response.json();
+        if (!result.success) {
+            throw new Error(result.message || 'Failed to update task status');
+        }
+    }
+    /**
+     * Sync a single file to Commander
+     */
+    async syncFileToCommander(relativePath) {
+        const config = this.configManager.getConfig();
+        const absolutePath = path.join(this.projectPath, relativePath);
+        if (!config.sync.taskboard_id) {
+            return {
+                success: false,
+                operation: 'skip',
+                documentType: 'document',
+                filePath: relativePath,
+                message: 'No taskboard_id configured',
+                error: 'Configuration error: sync.taskboard_id is required',
+            };
+        }
+        if (!this.mapper) {
+            throw new Error('Sync engine not initialized - call initialize() first');
+        }
+        // Check if file was just written by ReverseSync (prevent sync loop)
+        try {
+            const parsed = await this.parser.parseFile(absolutePath);
+            const commanderModifiedAt = parsed.frontmatter.commander_modified_at;
+            if (commanderModifiedAt) {
+                const modifiedTime = new Date(commanderModifiedAt).getTime();
+                const now = Date.now();
+                const timeDiff = now - modifiedTime;
+                // If Commander modified this file within the last 10 seconds, skip syncing back
+                // This prevents the sync loop between ReverseSync and SyncEngine
+                if (timeDiff < 10000) {
+                    console.log(`[SyncEngine] ✅ Skipping recently synced file (${Math.round(timeDiff / 1000)}s ago from Commander)`);
+                    return {
+                        success: true,
+                        operation: 'skip',
+                        documentType: 'document',
+                        filePath: relativePath,
+                        message: 'Skipped - recently synced from Commander',
+                    };
+                }
+            }
+        }
+        catch (error) {
+            // If parsing fails, continue with sync attempt
+            console.error(`[SyncEngine] Warning: Could not check commander_modified_at: ${error}`);
+        }
+        return await this.mapper.syncToCommander({
+            filePath: absolutePath,
+            taskboardId: config.sync.taskboard_id,
+            projectId: config.project_id,
+        });
+    }
+    /**
+     * Sync with retry logic and exponential backoff
+     * Handles transient failures like network timeouts
+     */
+    async syncWithRetry(relativePath) {
+        let lastError = null;
+        for (let attempt = 0; attempt <= RETRY_CONFIG.maxRetries; attempt++) {
+            try {
+                const result = await this.syncFileToCommander(relativePath);
+                // If sync succeeded or was skipped (not an error), return
+                if (result.success || result.operation === 'skip') {
+                    return result;
+                }
+                // Check if error is retryable
+                const errorMsg = result.error || result.message;
+                if (!this.isRetryableError(errorMsg)) {
+                    return result; // Non-retryable error, return immediately
+                }
+                lastError = new Error(errorMsg);
+            }
+            catch (error) {
+                lastError = error instanceof Error ? error : new Error(String(error));
+                // Check if error is retryable
+                if (!this.isRetryableError(lastError.message)) {
+                    throw lastError;
+                }
+            }
+            // If we have more retries, wait with exponential backoff
+            if (attempt < RETRY_CONFIG.maxRetries) {
+                const delay = Math.min(RETRY_CONFIG.baseDelayMs * Math.pow(2, attempt), RETRY_CONFIG.maxDelayMs);
+                console.error(`[SyncEngine] Retry ${attempt + 1}/${RETRY_CONFIG.maxRetries} for ${relativePath} in ${delay}ms`);
+                await this.sleep(delay);
+            }
+        }
+        // All retries exhausted
+        return {
+            success: false,
+            operation: 'skip',
+            documentType: 'document',
+            filePath: relativePath,
+            message: `Failed after ${RETRY_CONFIG.maxRetries} retries`,
+            error: lastError?.message || 'Unknown error',
+        };
+    }
+    /**
+     * Check if an error is retryable (transient failures)
+     */
+    isRetryableError(errorMsg) {
+        const retryablePatterns = [
+            /timeout/i,
+            /ETIMEDOUT/i,
+            /ECONNRESET/i,
+            /ECONNREFUSED/i,
+            /ENOTFOUND/i,
+            /network/i,
+            /socket hang up/i,
+            /503/i, // Service Unavailable
+            /502/i, // Bad Gateway
+            /504/i, // Gateway Timeout
+            /429/i, // Too Many Requests
+        ];
+        return retryablePatterns.some(pattern => pattern.test(errorMsg));
+    }
+    /**
+     * Sleep helper for retry delays
+     */
+    sleep(ms) {
+        return new Promise(resolve => setTimeout(resolve, ms));
+    }
+    /**
+     * Handle sync errors
+     */
+    handleError(error, filePath) {
+        this.errorCount++;
+        const errorEntry = {
+            timestamp: new Date(),
+            error,
+            filePath,
+        };
+        this.recentErrors.push(errorEntry);
+        // Keep only last 10 errors
+        if (this.recentErrors.length > 10) {
+            this.recentErrors.shift();
+        }
+        console.error(`[SyncEngine] ❌ Error in ${filePath}: ${error}`);
+        this.emit('error', errorEntry);
+    }
+    /**
+     * Get sync daemon status
+     */
+    getStatus() {
+        const stats = this.watcher?.getStats();
+        const indexStats = this.indexManager.getStats();
+        return {
+            running: this.running,
+            startedAt: this.startedAt,
+            filesWatched: stats?.filesWatched ?? 0,
+            lastSyncAt: this.lastSyncAt,
+            syncCount: this.syncCount,
+            errorCount: this.errorCount,
+            recentErrors: this.recentErrors,
+            indexStats,
+        };
+    }
+    /**
+     * Get sync state for a file
+     */
+    getSyncState(filePath) {
+        return this.syncStates.get(filePath) || null;
+    }
+    /**
+     * Get all sync states
+     */
+    getAllSyncStates() {
+        return this.syncStates;
+    }
+    /**
+     * Manually trigger sync for a specific file
+     */
+    async syncFile(relativePath) {
+        return await this.syncFileToCommander(relativePath);
+    }
+    /**
+     * Sync all existing files matching watch patterns
+     * This is the "initial scan" that runs on daemon start
+     */
+    async syncAll(options = {}) {
+        const startTime = Date.now();
+        const { batchSize = 50, concurrency = 5, dryRun = false, force = false, } = options;
+        const config = this.configManager.getConfig();
+        if (!config.sync.taskboard_id) {
+            return {
+                totalFiles: 0,
+                synced: 0,
+                skipped: 0,
+                failed: 1,
+                results: [],
+                errors: [{ filePath: '', error: 'No taskboard_id configured' }],
+                duration: Date.now() - startTime,
+            };
+        }
+        console.error('[SyncEngine] Starting initial scan...');
+        console.error(`[SyncEngine] Options: batchSize=${batchSize}, concurrency=${concurrency}, dryRun=${dryRun}`);
+        console.error(`[SyncEngine] Taskboard ID: ${config.sync.taskboard_id}`);
+        console.error(`[SyncEngine] Workspace ID: ${config.workspace_id}`);
+        console.error(`[SyncEngine] Organization ID: ${config.organization_id}`);
+        console.error(`[SyncEngine] Project ID: ${config.project_id}`);
+        // Step 1: Discover and create all folders first (including empty ones)
+        // This ensures folder structure parity between IDE and Commander
+        console.error('[SyncEngine] Step 1: Discovering directories...');
+        const directories = await this.discoverDirectories(config.sync.watch_patterns);
+        console.error(`[SyncEngine] Found ${directories.length} directories to sync`);
+        if (!dryRun && this.mapper) {
+            console.error('[SyncEngine] Creating folder structure in Commander...');
+            const folderService = this.mapper.getFolderService();
+            if (folderService) {
+                let foldersCreated = 0;
+                for (const dir of directories) {
+                    try {
+                        await folderService.ensureFolderPath(dir);
+                        foldersCreated++;
+                    }
+                    catch (error) {
+                        console.error(`[SyncEngine] ⚠️ Failed to create folder ${dir}:`, error);
+                    }
+                }
+                console.error(`[SyncEngine] ✅ Folder structure synced (${foldersCreated} directories)`);
+            }
+        }
+        // Step 2: Discover all markdown files matching watch patterns
+        console.error('[SyncEngine] Step 2: Discovering files...');
+        const files = await this.discoverFiles(config.sync.watch_patterns);
+        console.error(`[SyncEngine] Found ${files.length} files to scan`);
+        if (dryRun) {
+            console.error('[SyncEngine] DRY RUN - no files will be synced');
+            return {
+                totalFiles: files.length,
+                synced: 0,
+                skipped: files.length,
+                failed: 0,
+                results: files.map(f => ({
+                    success: true,
+                    operation: 'skip',
+                    documentType: 'document',
+                    filePath: f,
+                    message: 'Dry run - skipped',
+                })),
+                errors: [],
+                duration: Date.now() - startTime,
+            };
+        }
+        const results = [];
+        const errors = [];
+        let synced = 0;
+        let skipped = 0;
+        let failed = 0;
+        // Process files in batches
+        for (let i = 0; i < files.length; i += batchSize) {
+            const batch = files.slice(i, i + batchSize);
+            console.error(`[SyncEngine] Processing batch ${Math.floor(i / batchSize) + 1}/${Math.ceil(files.length / batchSize)} (${batch.length} files)`);
+            // Process batch with concurrency limit
+            const batchResults = await this.processBatch(batch, concurrency, force);
+            for (const result of batchResults) {
+                results.push(result);
+                if (result.success) {
+                    if (result.operation === 'skip') {
+                        skipped++;
+                    }
+                    else {
+                        synced++;
+                        this.syncCount++;
+                    }
+                }
+                else {
+                    failed++;
+                    // Use handleError to populate recentErrors (was missing!)
+                    this.handleError(result.error || result.message, result.filePath);
+                    errors.push({
+                        filePath: result.filePath,
+                        error: result.error || result.message,
+                    });
+                }
+            }
+            // Save index after each batch (incremental saves - don't lose progress on crash)
+            await this.indexManager.flush();
+            console.error(`[SyncEngine] Index saved after batch (${Math.min(i + batchSize, files.length)}/${files.length} files processed)`);
+            // Emit progress event
+            this.emit('sync-progress', {
+                total: files.length,
+                processed: Math.min(i + batchSize, files.length),
+                synced,
+                skipped,
+                failed,
+            });
+        }
+        this.lastSyncAt = new Date();
+        this.initialScanComplete = true;
+        // Record full sync in persistent index
+        this.indexManager.recordFullSync();
+        await this.indexManager.flush(); // Ensure index is saved
+        const duration = Date.now() - startTime;
+        console.error(`[SyncEngine] Initial scan complete in ${duration}ms`);
+        console.error(`[SyncEngine] Results: ${synced} synced, ${skipped} skipped, ${failed} failed`);
+        this.emit('sync-all-complete', { totalFiles: files.length, synced, skipped, failed, duration });
+        return {
+            totalFiles: files.length,
+            synced,
+            skipped,
+            failed,
+            results,
+            errors,
+            duration,
+        };
+    }
+    /**
+     * Discover all directories matching watch patterns (including empty ones)
+     * This ensures folder structure parity between IDE and Commander
+     */
+    async discoverDirectories(patterns) {
+        const allDirs = new Set();
+        const config = this.configManager.getConfig();
+        const ignorePatterns = config.sync.ignore_patterns || [];
+        for (const pattern of patterns) {
+            try {
+                // Get base directory from pattern (e.g., "docs/**/*.md" → "docs")
+                const baseParts = pattern.split('*')[0].split('/').filter(p => p);
+                const baseDir = baseParts.join('/') || '.';
+                const absoluteBaseDir = path.join(this.projectPath, baseDir);
+                // Check if base directory exists
+                try {
+                    await fs.access(absoluteBaseDir);
+                }
+                catch {
+                    continue; // Base directory doesn't exist, skip
+                }
+                // Walk directory tree to find all subdirectories
+                const walkDir = async (dir) => {
+                    try {
+                        const entries = await fs.readdir(dir, { withFileTypes: true });
+                        for (const entry of entries) {
+                            if (entry.isDirectory()) {
+                                const fullPath = path.join(dir, entry.name);
+                                const relativePath = path.relative(this.projectPath, fullPath);
+                                // Check ignore patterns
+                                const shouldIgnore = ignorePatterns.some(ignorePattern => {
+                                    // Simple glob matching for common patterns
+                                    if (ignorePattern.includes('**')) {
+                                        const prefix = ignorePattern.split('**')[0];
+                                        return relativePath.startsWith(prefix.replace(/\/$/, ''));
+                                    }
+                                    return relativePath.includes(ignorePattern.replace(/\*\*/g, '').replace(/\//g, ''));
+                                });
+                                if (!shouldIgnore) {
+                                    allDirs.add(relativePath);
+                                    await walkDir(fullPath); // Recurse
+                                }
+                            }
+                        }
+                    }
+                    catch (error) {
+                        console.error(`[SyncEngine] Error reading directory ${dir}:`, error);
+                    }
+                };
+                await walkDir(absoluteBaseDir);
+            }
+            catch (error) {
+                console.error(`[SyncEngine] Error discovering directories for pattern ${pattern}:`, error);
+            }
+        }
+        // Sort for consistent processing order (parents before children)
+        return Array.from(allDirs).sort();
+    }
+    /**
+     * Discover all markdown files matching patterns
+     */
+    async discoverFiles(patterns) {
+        const allFiles = new Set();
+        for (const pattern of patterns) {
+            try {
+                const absolutePattern = path.join(this.projectPath, pattern);
+                const matches = await glob(absolutePattern, {
+                    nodir: true,
+                    ignore: ['**/node_modules/**', '**/.git/**', '**/dist/**'],
+                });
+                for (const match of matches) {
+                    if (match.endsWith('.md')) {
+                        // Store as relative path
+                        const relativePath = path.relative(this.projectPath, match);
+                        allFiles.add(relativePath);
+                    }
+                }
+            }
+            catch (error) {
+                console.error(`[SyncEngine] Error globbing pattern ${pattern}:`, error);
+            }
+        }
+        // Sort files for consistent processing order
+        return Array.from(allFiles).sort();
+    }
+    /**
+     * Process a batch of files with concurrency limit
+     * Uses Promise.allSettled to prevent one bad file from crashing entire batch
+     */
+    async processBatch(files, concurrency, force) {
+        const results = [];
+        // Process in chunks of 'concurrency' size
+        for (let i = 0; i < files.length; i += concurrency) {
+            const chunk = files.slice(i, i + concurrency);
+            const chunkPromises = chunk.map(file => this.syncFileWithCheck(file, force));
+            // Use allSettled to handle individual failures gracefully
+            const settledResults = await Promise.allSettled(chunkPromises);
+            for (let j = 0; j < settledResults.length; j++) {
+                const settled = settledResults[j];
+                const filePath = chunk[j];
+                if (settled.status === 'fulfilled') {
+                    results.push(settled.value);
+                }
+                else {
+                    // Promise rejected - create error result instead of crashing
+                    console.error(`[SyncEngine] ❌ Unhandled error in ${filePath}:`, settled.reason);
+                    results.push({
+                        success: false,
+                        operation: 'skip',
+                        documentType: 'document',
+                        filePath,
+                        message: `Unhandled error: ${settled.reason?.message || String(settled.reason)}`,
+                        error: settled.reason?.message || String(settled.reason),
+                    });
+                }
+            }
+        }
+        return results;
+    }
+    /**
+     * Sync a file with checksum verification to avoid unnecessary syncs
+     * Uses persistent index and retry logic with exponential backoff
+     */
+    async syncFileWithCheck(relativePath, force) {
+        try {
+            const absolutePath = path.join(this.projectPath, relativePath);
+            // Read and parse file
+            let content;
+            let fileStat = null;
+            try {
+                content = await fs.readFile(absolutePath, 'utf-8');
+                fileStat = await fs.stat(absolutePath);
+            }
+            catch {
+                // File was deleted - remove from index
+                this.indexManager.removeFile(relativePath);
+                return {
+                    success: false,
+                    operation: 'skip',
+                    documentType: 'document',
+                    filePath: relativePath,
+                    message: 'File not found or unreadable',
+                    error: 'File not found',
+                };
+            }
+            // Calculate checksum
+            const checksum = this.indexManager.calculateChecksum(content);
+            // Check persistent index - skip if unchanged (unless forced)
+            if (!force && !this.indexManager.needsSync(relativePath, checksum)) {
+                const indexState = this.indexManager.getFileState(relativePath);
+                return {
+                    success: true,
+                    operation: 'skip',
+                    documentType: (indexState?.documentType || 'document'),
+                    filePath: relativePath,
+                    message: 'Already synced (checksum match in index)',
+                };
+            }
+            // Also check in-memory state for recent syncs (within same session)
+            const existingState = this.syncStates.get(relativePath);
+            if (!force && existingState && existingState.checksum === checksum) {
+                return {
+                    success: true,
+                    operation: 'skip',
+                    documentType: existingState.documentType,
+                    filePath: relativePath,
+                    message: 'Already synced (checksum match)',
+                };
+            }
+            // Parse file - auto-init will happen in trinity-mapper if no sync block
+            const parsed = await this.parser.parseFile(absolutePath);
+            // Only skip if explicitly disabled (sync.enabled: false)
+            if (parsed.frontmatter.sync?.enabled === false) {
+                console.error(`[SyncEngine] Skipped (sync.enabled: false): ${relativePath}`);
+                return {
+                    success: true,
+                    operation: 'skip',
+                    documentType: 'document',
+                    filePath: relativePath,
+                    message: 'Sync disabled via frontmatter (sync.enabled: false)',
+                };
+            }
+            // Files without sync block will be auto-initialized by trinity-mapper
+            // Don't skip them here - let them flow through to get frontmatter added
+            // Check for potential conflicts
+            const fileModifiedAt = fileStat ? new Date(fileStat.mtimeMs) : new Date();
+            const conflictCheck = this.indexManager.detectPotentialConflicts(relativePath, checksum, fileModifiedAt);
+            if (conflictCheck.hasConflict) {
+                console.error(`[SyncEngine] ⚠️  Potential conflict detected for ${relativePath}: ${conflictCheck.reason}`);
+                this.emit('conflict', { filePath: relativePath, reason: conflictCheck.reason });
+            }
+            // Extract status from folder path
+            const statusFromPath = this.folderStatusMapper.getStatusFromPath(relativePath);
+            console.error(`[SyncEngine] ${relativePath} -> status: ${statusFromPath.statusSection} (${statusFromPath.confidence})`);
+            // Sync to Commander with retry logic
+            const result = await this.syncWithRetry(relativePath);
+            // Update both in-memory state and persistent index on success
+            if (result.success && result.operation !== 'skip') {
+                // Update in-memory state
+                this.syncStates.set(relativePath, {
+                    filePath: relativePath,
+                    lastSyncedAt: new Date(),
+                    lastModifiedAt: fileModifiedAt,
+                    commanderId: result.commanderId || null,
+                    documentType: result.documentType,
+                    checksum,
+                });
+                // Update persistent index
+                this.indexManager.updateFileState(relativePath, {
+                    checksum,
+                    documentType: result.documentType,
+                    taskId: parsed.frontmatter.sync?.task_id || result.commanderId || undefined,
+                    documentId: parsed.frontmatter.sync?.document_id || undefined,
+                    layerId: parsed.frontmatter.sync?.layer_id || undefined,
+                    fileModifiedAt,
+                });
+            }
+            return result;
+        }
+        catch (error) {
+            return {
+                success: false,
+                operation: 'skip',
+                documentType: 'document',
+                filePath: relativePath,
+                message: error instanceof Error ? error.message : String(error),
+                error: error instanceof Error ? error.message : String(error),
+            };
+        }
+    }
+    /**
+     * Get folder status mapper instance
+     */
+    getFolderStatusMapper() {
+        return this.folderStatusMapper;
+    }
+    /**
+     * Check if initial scan is complete
+     */
+    isInitialScanComplete() {
+        return this.initialScanComplete;
+    }
+    /**
+     * Check if engine is running
+     */
+    isRunning() {
+        return this.running;
+    }
+}
+//# sourceMappingURL=sync-engine.js.map