@n8n-as-code/sync 0.5.0

package/dist/services/watcher.js

@@ -0,0 +1,655 @@
+import fs from 'fs';
+import path from 'path';
+import EventEmitter from 'events';
+import * as chokidar from 'chokidar';
+import { WorkflowSanitizer } from './workflow-sanitizer.js';
+import { HashUtils } from './hash-utils.js';
+import { WorkflowSyncStatus } from '../types.js';
+/**
+ * Watcher - State Observation Component
+ *
+ * Responsibilities:
+ * 1. File System Watch with debounce
+ * 2. Remote Polling with lightweight strategy
+ * 3. Canonical Hashing (SHA-256 of sorted JSON)
+ * 4. Status Matrix Calculation (3-way comparison)
+ * 5. State Persistence (only component that writes to .n8n-state.json)
+ *
+ * Never performs synchronization actions - only observes reality.
+ */
+export class Watcher extends EventEmitter {
+    watcher = null;
+    pollInterval = null;
+    client;
+    directory;
+    pollIntervalMs;
+    syncInactive;
+    ignoredTags;
+    stateFilePath;
+    isConnected = true;
+    isInitializing = false;
+    // Internal state tracking
+    localHashes = new Map(); // filename -> hash
+    remoteHashes = new Map(); // workflowId -> hash
+    fileToIdMap = new Map(); // filename -> workflowId
+    idToFileMap = new Map(); // workflowId -> filename
+    lastKnownStatuses = new Map(); // workflowId or filename -> status
+    // Concurrency control
+    isPaused = new Set(); // IDs for which observation is paused
+    syncInProgress = new Set(); // IDs currently being synced
+    pausedFilenames = new Set(); // Filenames for which observation is paused (for workflows without ID yet)
+    // Lightweight polling cache
+    remoteTimestamps = new Map(); // workflowId -> updatedAt
+    constructor(client, options) {
+        super();
+        this.client = client;
+        this.directory = options.directory;
+        this.pollIntervalMs = options.pollIntervalMs;
+        this.syncInactive = options.syncInactive;
+        this.ignoredTags = options.ignoredTags;
+        this.stateFilePath = path.join(this.directory, '.n8n-state.json');
+    }
+    async start() {
+        if (this.watcher || this.pollInterval)
+            return;
+        this.isInitializing = true;
+        // Initial scan - throw error if connection fails on startup
+        try {
+            await this.refreshRemoteState();
+        }
+        catch (error) {
+            // Check if it's a connection error
+            const isConnectionError = error.code === 'ECONNREFUSED' ||
+                error.code === 'ENOTFOUND' ||
+                error.code === 'ETIMEDOUT' ||
+                error.message?.includes('fetch failed') ||
+                error.message?.includes('ECONNREFUSED') ||
+                error.message?.includes('ENOTFOUND') ||
+                error.cause?.code === 'ECONNREFUSED';
+            if (isConnectionError) {
+                this.isInitializing = false;
+                // On startup, throw the error to prevent initialization
+                throw new Error('Cannot connect to n8n instance. Please check if n8n is running and the host URL is correct.');
+            }
+            // For other errors, re-throw
+            this.isInitializing = false;
+            throw error;
+        }
+        await this.refreshLocalState();
+        this.isInitializing = false;
+        // Local Watch with debounce
+        this.watcher = chokidar.watch(this.directory, {
+            ignored: [
+                /(^|[\/\\])\../, // Hidden files
+                '**/_archive/**', // Archive folder (strictly ignored)
+                '**/.n8n-state.json' // State file
+            ],
+            persistent: true,
+            ignoreInitial: true,
+            awaitWriteFinish: { stabilityThreshold: 500, pollInterval: 100 } // 500ms debounce
+        });
+        this.watcher
+            .on('add', (p) => this.onLocalChange(p))
+            .on('change', (p) => this.onLocalChange(p))
+            .on('unlink', (p) => this.onLocalDelete(p));
+        // Remote Poll
+        if (this.pollIntervalMs > 0) {
+            this.pollInterval = setInterval(() => this.refreshRemoteState(), this.pollIntervalMs);
+        }
+        this.emit('ready');
+    }
+    stop() {
+        if (this.watcher) {
+            this.watcher.close();
+            this.watcher = null;
+        }
+        if (this.pollInterval) {
+            clearInterval(this.pollInterval);
+            this.pollInterval = null;
+        }
+    }
+    /**
+     * Pause observation for a workflow during sync operations
+     */
+    pauseObservation(workflowId) {
+        this.isPaused.add(workflowId);
+    }
+    /**
+     * Resume observation after sync operations
+     */
+    resumeObservation(workflowId) {
+        this.isPaused.delete(workflowId);
+        // Don't force refresh here - let the normal polling cycle handle it
+        // Forcing a refresh after every sync can cause loops in auto-sync mode
+        // because the remote state might not be immediately consistent with local state
+    }
+    /**
+     * Pause observation for a filename (for workflows without ID yet)
+     */
+    pauseObservationByFilename(filename) {
+        this.pausedFilenames.add(filename);
+    }
+    /**
+     * Resume observation for a filename
+     */
+    resumeObservationByFilename(filename) {
+        this.pausedFilenames.delete(filename);
+    }
+    /**
+     * Mark a workflow as being synced (prevents race conditions)
+     */
+    markSyncInProgress(workflowId) {
+        this.syncInProgress.add(workflowId);
+    }
+    /**
+     * Mark a workflow as no longer being synced
+     */
+    markSyncComplete(workflowId) {
+        this.syncInProgress.delete(workflowId);
+    }
+    async onLocalChange(filePath) {
+        const filename = path.basename(filePath);
+        if (!filename.endsWith('.json'))
+            return;
+        const content = this.readJsonFile(filePath);
+        if (!content)
+            return;
+        // Check if filename is paused (for workflows without ID)
+        if (this.pausedFilenames.has(filename)) {
+            return;
+        }
+        const workflowId = content.id || this.fileToIdMap.get(filename);
+        if (workflowId && (this.isPaused.has(workflowId) || this.syncInProgress.has(workflowId))) {
+            return;
+        }
+        // IMPORTANT: Hash is calculated on the SANITIZED version
+        // This means versionId, versionCounter, pinData, etc. are ignored
+        // The file on disk can contain these fields, but they won't affect the hash
+        const clean = WorkflowSanitizer.cleanForStorage(content);
+        const hash = this.computeHash(clean);
+        this.localHashes.set(filename, hash);
+        if (workflowId) {
+            this.fileToIdMap.set(filename, workflowId);
+            this.idToFileMap.set(workflowId, filename);
+        }
+        this.broadcastStatus(filename, workflowId);
+    }
+    async onLocalDelete(filePath) {
+        const filename = path.basename(filePath);
+        let workflowId = this.fileToIdMap.get(filename);
+        // If workflowId not found via filename mapping, try to find it via state
+        if (!workflowId) {
+            const state = this.loadState();
+            for (const [id, stateData] of Object.entries(state.workflows)) {
+                const mappedFilename = this.idToFileMap.get(id);
+                if (mappedFilename === filename) {
+                    workflowId = id;
+                    break;
+                }
+            }
+        }
+        if (workflowId && (this.isPaused.has(workflowId) || this.syncInProgress.has(workflowId))) {
+            return;
+        }
+        // CRITICAL: Per spec 5.3 DELETED_LOCALLY - Archive Remote to _archive/ IMMEDIATELY
+        // This happens BEFORE user confirmation, to ensure we have a backup
+        if (workflowId) {
+            const remoteHash = this.remoteHashes.get(workflowId);
+            const lastSyncedHash = this.getLastSyncedHash(workflowId);
+            // Only archive if remote exists and matches last synced (true local deletion)
+            if (remoteHash && remoteHash === lastSyncedHash) {
+                try {
+                    // Fetch remote workflow content
+                    const remoteWorkflow = await this.client.getWorkflow(workflowId);
+                    if (remoteWorkflow) {
+                        // Create archive directory if it doesn't exist
+                        const archiveDir = path.join(this.directory, '.archive');
+                        if (!fs.existsSync(archiveDir)) {
+                            fs.mkdirSync(archiveDir, { recursive: true });
+                        }
+                        // Save to archive with timestamp
+                        const clean = WorkflowSanitizer.cleanForStorage(remoteWorkflow);
+                        const archivePath = path.join(archiveDir, `${Date.now()}_${filename}`);
+                        fs.writeFileSync(archivePath, JSON.stringify(clean, null, 2));
+                        console.log(`[Watcher] Archived remote workflow to: ${archivePath}`);
+                    }
+                }
+                catch (error) {
+                    console.warn(`[Watcher] Failed to archive remote workflow ${workflowId}:`, error);
+                    // Continue anyway - deletion detection should still work
+                }
+            }
+        }
+        this.localHashes.delete(filename);
+        this.broadcastStatus(filename, workflowId);
+    }
+    async refreshLocalState() {
+        if (!fs.existsSync(this.directory)) {
+            console.log(`[DEBUG] refreshLocalState: Directory missing: ${this.directory}`);
+            // Clear all local hashes since directory doesn't exist
+            this.localHashes.clear();
+            return;
+        }
+        const files = fs.readdirSync(this.directory).filter(f => f.endsWith('.json') && !f.startsWith('.'));
+        const currentFiles = new Set(files);
+        // Remove entries for files that no longer exist
+        for (const filename of this.localHashes.keys()) {
+            if (!currentFiles.has(filename)) {
+                this.localHashes.delete(filename);
+                const workflowId = this.fileToIdMap.get(filename);
+                if (workflowId) {
+                    // Broadcast status change for deleted file
+                    this.broadcastStatus(filename, workflowId);
+                }
+            }
+        }
+        // Add/update entries for existing files
+        for (const filename of files) {
+            const filePath = path.join(this.directory, filename);
+            const content = this.readJsonFile(filePath);
+            if (content) {
+                const clean = WorkflowSanitizer.cleanForStorage(content);
+                const hash = this.computeHash(clean);
+                this.localHashes.set(filename, hash);
+                if (content.id) {
+                    this.fileToIdMap.set(filename, content.id);
+                    this.idToFileMap.set(content.id, filename);
+                }
+            }
+        }
+    }
+    /**
+     * Lightweight polling strategy:
+     * 1. Fetch only IDs and updatedAt timestamps
+     * 2. Compare with cached timestamps
+     * 3. Fetch full content only if timestamp changed
+     */
+    async refreshRemoteState() {
+        try {
+            const remoteWorkflows = await this.client.getAllWorkflows();
+            this.isConnected = true;
+            const currentRemoteIds = new Set();
+            for (const wf of remoteWorkflows) {
+                if (this.shouldIgnore(wf))
+                    continue;
+                if (this.isPaused.has(wf.id) || this.syncInProgress.has(wf.id))
+                    continue;
+                currentRemoteIds.add(wf.id);
+                // CRITICAL: Use ID-based mapping instead of name-based filename generation
+                // This prevents issues when filename != workflow.name
+                let filename = this.idToFileMap.get(wf.id);
+                // If no mapping exists, try to find the file by scanning local files
+                if (!filename) {
+                    filename = this.findFilenameByWorkflowId(wf.id);
+                }
+                // If still not found, generate filename from name (new remote workflow)
+                if (!filename) {
+                    filename = `${this.safeName(wf.name)}.json`;
+                }
+                this.idToFileMap.set(wf.id, filename);
+                this.fileToIdMap.set(filename, wf.id);
+                // Check if we need to fetch full content
+                const cachedTimestamp = this.remoteTimestamps.get(wf.id);
+                const needsFullFetch = !cachedTimestamp ||
+                    (wf.updatedAt && wf.updatedAt !== cachedTimestamp);
+                if (needsFullFetch) {
+                    try {
+                        const fullWf = await this.client.getWorkflow(wf.id);
+                        if (fullWf) {
+                            const clean = WorkflowSanitizer.cleanForStorage(fullWf);
+                            const hash = this.computeHash(clean);
+                            this.remoteHashes.set(wf.id, hash);
+                            if (wf.updatedAt) {
+                                this.remoteTimestamps.set(wf.id, wf.updatedAt);
+                            }
+                            this.broadcastStatus(filename, wf.id);
+                        }
+                    }
+                    catch (e) {
+                        console.warn(`[Watcher] Could not fetch workflow ${wf.id}:`, e);
+                    }
+                }
+                else {
+                    // Timestamp unchanged, use cached hash
+                    const cachedHash = this.remoteHashes.get(wf.id);
+                    if (cachedHash) {
+                        this.broadcastStatus(filename, wf.id);
+                    }
+                }
+            }
+            // Prune remoteHashes for deleted workflows
+            for (const id of this.remoteHashes.keys()) {
+                if (!currentRemoteIds.has(id)) {
+                    this.remoteHashes.delete(id);
+                    this.remoteTimestamps.delete(id);
+                    const filename = this.idToFileMap.get(id);
+                    if (filename)
+                        this.broadcastStatus(filename, id);
+                }
+            }
+        }
+        catch (error) {
+            // Check if it's a connection error
+            const isConnectionError = error.code === 'ECONNREFUSED' ||
+                error.code === 'ENOTFOUND' ||
+                error.code === 'ETIMEDOUT' ||
+                error.message?.includes('fetch failed') ||
+                error.message?.includes('ECONNREFUSED') ||
+                error.message?.includes('ENOTFOUND') ||
+                error.cause?.code === 'ECONNREFUSED';
+            if (isConnectionError) {
+                this.isConnected = false;
+                // Stop polling to avoid spamming errors
+                if (this.pollInterval) {
+                    clearInterval(this.pollInterval);
+                    this.pollInterval = null;
+                }
+                // Emit a specific connection error
+                this.emit('connection-lost', new Error('Lost connection to n8n instance. Please check if n8n is running.'));
+            }
+            else {
+                // For other errors, just emit the error
+                this.emit('error', error);
+            }
+            // Re-throw so that start() can catch it on initial call
+            throw error;
+        }
+    }
+    /**
+     * Finalize sync - update base state after successful sync operation
+     * Called by SyncEngine after PULL/PUSH completes
+     */
+    async finalizeSync(workflowId) {
+        let filename = this.idToFileMap.get(workflowId);
+        // If workflow not tracked yet (first sync of local-only workflow),
+        // scan directory to find the file with this ID
+        if (!filename) {
+            const files = fs.readdirSync(this.directory).filter(f => f.endsWith('.json') && !f.startsWith('.'));
+            for (const file of files) {
+                const filePath = path.join(this.directory, file);
+                const content = this.readJsonFile(filePath);
+                if (content?.id === workflowId) {
+                    filename = file;
+                    // Initialize tracking for this workflow
+                    this.fileToIdMap.set(filename, workflowId);
+                    this.idToFileMap.set(workflowId, filename);
+                    break;
+                }
+            }
+            if (!filename) {
+                throw new Error(`Cannot finalize sync: workflow ${workflowId} not found in directory`);
+            }
+        }
+        // Get current reality
+        const filePath = path.join(this.directory, filename);
+        const content = this.readJsonFile(filePath);
+        if (!content) {
+            throw new Error(`Cannot finalize sync: local file not found for ${workflowId}`);
+        }
+        const clean = WorkflowSanitizer.cleanForStorage(content);
+        const computedHash = this.computeHash(clean);
+        // After a successful sync, local and remote should be identical
+        // Use the computed hash for both
+        const localHash = computedHash;
+        const remoteHash = computedHash;
+        // Update caches
+        this.localHashes.set(filename, localHash);
+        this.remoteHashes.set(workflowId, remoteHash);
+        // Update base state
+        await this.updateWorkflowState(workflowId, localHash);
+        // Broadcast new IN_SYNC status
+        this.broadcastStatus(filename, workflowId);
+    }
+    /**
+     * Update workflow state in .n8n-state.json
+     * Only this component writes to the state file
+     */
+    async updateWorkflowState(id, hash) {
+        const state = this.loadState();
+        state.workflows[id] = {
+            lastSyncedHash: hash,
+            lastSyncedAt: new Date().toISOString()
+        };
+        this.saveState(state);
+    }
+    /**
+     * Remove workflow from state file
+     * Called after deletion confirmation
+     */
+    async removeWorkflowState(id) {
+        const state = this.loadState();
+        delete state.workflows[id];
+        this.saveState(state);
+        // Clean up internal tracking
+        const filename = this.idToFileMap.get(id);
+        if (filename) {
+            this.fileToIdMap.delete(filename);
+        }
+        this.idToFileMap.delete(id);
+        this.remoteHashes.delete(id);
+        this.remoteTimestamps.delete(id);
+    }
+    /**
+     * Load state from .n8n-state.json
+     */
+    loadState() {
+        if (fs.existsSync(this.stateFilePath)) {
+            try {
+                const data = JSON.parse(fs.readFileSync(this.stateFilePath, 'utf-8'));
+                if (!data.workflows) {
+                    data.workflows = {};
+                }
+                return data;
+            }
+            catch (e) {
+                console.warn('Could not read state file, using empty state');
+            }
+        }
+        return { workflows: {} };
+    }
+    /**
+     * Save state to .n8n-state.json
+     */
+    saveState(state) {
+        fs.writeFileSync(this.stateFilePath, JSON.stringify(state, null, 2));
+    }
+    /**
+     * Compute canonical hash for content
+     */
+    computeHash(content) {
+        return HashUtils.computeHash(content);
+    }
+    broadcastStatus(filename, workflowId) {
+        if (this.isInitializing)
+            return;
+        const status = this.calculateStatus(filename, workflowId);
+        const key = workflowId || filename;
+        const lastStatus = this.lastKnownStatuses.get(key);
+        if (status !== lastStatus) {
+            this.lastKnownStatuses.set(key, status);
+            this.emit('statusChange', {
+                filename,
+                workflowId,
+                status
+            });
+        }
+    }
+    calculateStatus(filename, workflowId) {
+        if (!workflowId)
+            workflowId = this.fileToIdMap.get(filename);
+        const localHash = this.localHashes.get(filename);
+        const remoteHash = workflowId ? this.remoteHashes.get(workflowId) : undefined;
+        // If we are disconnected and don't have a remote hash, don't claim it's deleted
+        if (!this.isConnected && !remoteHash && workflowId) {
+            return WorkflowSyncStatus.IN_SYNC; // Treat as in-sync or unknown to avoid "deleted" panic
+        }
+        // Get base state
+        const state = this.loadState();
+        const baseState = workflowId ? state.workflows[workflowId] : undefined;
+        const lastSyncedHash = baseState?.lastSyncedHash;
+        // Implementation of 4.2 Status Logic Matrix
+        if (localHash && !lastSyncedHash && !remoteHash)
+            return WorkflowSyncStatus.EXIST_ONLY_LOCALLY;
+        if (remoteHash && !lastSyncedHash && !localHash)
+            return WorkflowSyncStatus.EXIST_ONLY_REMOTELY;
+        if (localHash && remoteHash && localHash === remoteHash)
+            return WorkflowSyncStatus.IN_SYNC;
+        if (lastSyncedHash) {
+            // Check deletions first (they take precedence over modifications)
+            if (!localHash && remoteHash === lastSyncedHash)
+                return WorkflowSyncStatus.DELETED_LOCALLY;
+            if (!remoteHash && localHash === lastSyncedHash)
+                return WorkflowSyncStatus.DELETED_REMOTELY;
+            // Then check modifications
+            const localModified = localHash !== lastSyncedHash;
+            const remoteModified = remoteHash && remoteHash !== lastSyncedHash;
+            if (localModified && remoteModified)
+                return WorkflowSyncStatus.CONFLICT;
+            if (localModified && remoteHash === lastSyncedHash)
+                return WorkflowSyncStatus.MODIFIED_LOCALLY;
+            if (remoteModified && localHash === lastSyncedHash)
+                return WorkflowSyncStatus.MODIFIED_REMOTELY;
+        }
+        // Fallback for edge cases
+        return WorkflowSyncStatus.CONFLICT;
+    }
+    shouldIgnore(wf) {
+        if (!this.syncInactive && !wf.active)
+            return true;
+        if (wf.tags) {
+            const hasIgnoredTag = wf.tags.some(t => this.ignoredTags.includes(t.name.toLowerCase()));
+            if (hasIgnoredTag)
+                return true;
+        }
+        return false;
+    }
+    safeName(name) {
+        return name.replace(/[\/\\:]/g, '_').replace(/\s+/g, ' ').trim();
+    }
+    /**
+     * Find local file that contains a specific workflow ID
+     * Used when we have an ID but no filename mapping yet (e.g., after file rename)
+     */
+    findFilenameByWorkflowId(workflowId) {
+        if (!fs.existsSync(this.directory)) {
+            return undefined;
+        }
+        const files = fs.readdirSync(this.directory)
+            .filter(f => f.endsWith('.json') && !f.startsWith('.'));
+        for (const file of files) {
+            const content = this.readJsonFile(path.join(this.directory, file));
+            if (content?.id === workflowId) {
+                return file;
+            }
+        }
+        return undefined;
+    }
+    readJsonFile(filePath) {
+        try {
+            return JSON.parse(fs.readFileSync(filePath, 'utf8'));
+        }
+        catch {
+            return null;
+        }
+    }
+    getFileToIdMap() {
+        return this.fileToIdMap;
+    }
+    getStatusMatrix() {
+        const results = new Map();
+        const state = this.loadState();
+        // 1. Process all local files
+        for (const [filename, hash] of this.localHashes.entries()) {
+            const workflowId = this.fileToIdMap.get(filename);
+            const status = this.calculateStatus(filename, workflowId);
+            results.set(filename, {
+                id: workflowId || '',
+                name: filename.replace('.json', ''),
+                filename: filename,
+                status: status,
+                active: true
+            });
+        }
+        // 2. Process all remote workflows not yet in results
+        for (const [workflowId, remoteHash] of this.remoteHashes.entries()) {
+            const filename = this.idToFileMap.get(workflowId) || `${workflowId}.json`;
+            if (!results.has(filename)) {
+                const status = this.calculateStatus(filename, workflowId);
+                results.set(filename, {
+                    id: workflowId,
+                    name: filename.replace('.json', ''),
+                    filename: filename,
+                    status: status,
+                    active: true
+                });
+            }
+        }
+        // 3. Process tracked but deleted workflows
+        for (const id of Object.keys(state.workflows)) {
+            const filename = this.idToFileMap.get(id) || `${id}.json`;
+            if (!results.has(filename)) {
+                const status = this.calculateStatus(filename, id);
+                results.set(filename, {
+                    id,
+                    name: filename.replace('.json', ''),
+                    filename,
+                    status,
+                    active: true
+                });
+            }
+        }
+        return Array.from(results.values()).sort((a, b) => a.name.localeCompare(b.name));
+    }
+    /**
+     * Get last synced hash for a workflow
+     */
+    getLastSyncedHash(workflowId) {
+        const state = this.loadState();
+        return state.workflows[workflowId]?.lastSyncedHash;
+    }
+    /**
+     * Update remote hash cache (for SyncEngine use)
+     * @internal
+     */
+    setRemoteHash(workflowId, hash) {
+        this.remoteHashes.set(workflowId, hash);
+    }
+    /**
+     * Get all tracked workflow IDs
+     */
+    getTrackedWorkflowIds() {
+        const state = this.loadState();
+        return Object.keys(state.workflows);
+    }
+    /**
+     * Update workflow ID in state (when a workflow is re-created with a new ID)
+     */
+    async updateWorkflowId(oldId, newId) {
+        const state = this.loadState();
+        // Migrate state from old ID to new ID
+        if (state.workflows[oldId]) {
+            state.workflows[newId] = state.workflows[oldId];
+            delete state.workflows[oldId];
+            this.saveState(state);
+        }
+        // Update internal mappings
+        const filename = this.idToFileMap.get(oldId);
+        if (filename) {
+            this.idToFileMap.delete(oldId);
+            this.idToFileMap.set(newId, filename);
+            this.fileToIdMap.set(filename, newId);
+        }
+        // Update hash maps
+        const remoteHash = this.remoteHashes.get(oldId);
+        if (remoteHash) {
+            this.remoteHashes.delete(oldId);
+            this.remoteHashes.set(newId, remoteHash);
+        }
+        const timestamp = this.remoteTimestamps.get(oldId);
+        if (timestamp) {
+            this.remoteTimestamps.delete(oldId);
+            this.remoteTimestamps.set(newId, timestamp);
+        }
+    }
+}
+//# sourceMappingURL=watcher.js.map