@stoneforge/quarry 1.13.0 → 1.14.0

package/dist/external-sync/sync-engine.js

@@ -30,11 +30,14 @@ import { createTimestamp } from '@stoneforge/core';
 import { createHash } from 'crypto';
 import { computeContentHashSync } from '../sync/hash.js';
 import { taskToExternalTask, externalTaskToTaskUpdates, getFieldMapConfigForProvider } from './adapters/task-sync-adapter.js';
+import { documentToExternalDocumentInput, externalDocumentToDocumentUpdates, computeExternalDocumentHash, resolveDocumentLibraryPaths } from './adapters/document-sync-adapter.js';
 // ============================================================================
 // Settings Keys
 // ============================================================================
 /** Key prefix for sync cursor settings */
 const SYNC_CURSOR_KEY_PREFIX = 'external_sync.cursor';
+/** Number of documents to push concurrently (balances throughput vs API rate limits) */
+const PUSH_CONCURRENCY = 3;
 /**
  * Build a settings key for a sync cursor.
  * Format: external_sync.cursor.<provider>.<project>.<adapterType>
@@ -118,47 +121,65 @@ export class SyncEngine {
         const errors = [];
         const now = createTimestamp();
         // Step 1: Find elements to push
-        const elements = await this.findLinkedElements(options);
-        // Step 2: Process each element
-        for (const element of elements) {
-            const syncState = getExternalSyncState(element.metadata);
-            if (!syncState) {
-                // No sync state — shouldn't happen since we filtered for it, but guard
-                skipped.push(1);
-                continue;
-            }
-            // Skip if direction is pull-only
-            if (syncState.direction === 'pull') {
-                skipped.push(1);
-                continue;
-            }
-            // Skip closed/tombstone tasks — they're done and shouldn't sync.
-            // If a task is later reopened (status changes away from closed/tombstone),
-            // it will be picked up again naturally since the filter no longer applies.
-            const elementStatus = element.status;
-            if (elementStatus === 'closed' || elementStatus === 'tombstone') {
-                skipped.push(1);
-                continue;
-            }
-            try {
-                const result = await this.pushElement(element, syncState, options, now);
-                if (result === 'pushed') {
-                    pushed.push(1);
+        let elements = await this.findLinkedElements(options);
+        // Step 1.5: Batch-resolve library paths for all document elements
+        // This avoids N+1 queries by resolving all paths upfront
+        const documentElements = elements.filter(el => {
+            const syncState = getExternalSyncState(el.metadata);
+            return syncState?.adapterType === 'document';
+        });
+        const libraryPaths = documentElements.length > 0
+            ? await resolveDocumentLibraryPaths(this.api, documentElements.map(el => el.id))
+            : new Map();
+        // Step 1.6: Filter out documents without a library unless includeNoLibrary is set
+        let noLibrarySkipped = 0;
+        if (!options.includeNoLibrary && documentElements.length > 0) {
+            const beforeCount = elements.length;
+            elements = elements.filter(el => {
+                const syncState = getExternalSyncState(el.metadata);
+                if (syncState?.adapterType !== 'document')
+                    return true; // keep non-documents
+                return libraryPaths.has(el.id);
+            });
+            noLibrarySkipped = beforeCount - elements.length;
+        }
+        // Notify caller of total count before processing begins
+        options.onBeforeProcess?.(elements.length);
+        // Step 2: Process elements concurrently in batches
+        let processedCount = 0;
+        // Notify caller of total element count on first progress callback
+        options.onProgress?.(0, elements.length);
+        for (let i = 0; i < elements.length; i += PUSH_CONCURRENCY) {
+            const batch = elements.slice(i, i + PUSH_CONCURRENCY);
+            const results = await Promise.allSettled(batch.map(element => this.pushSingleElement(element, options, now, libraryPaths)));
+            for (let j = 0; j < results.length; j++) {
+                const result = results[j];
+                if (result.status === 'fulfilled') {
+                    const outcome = result.value;
+                    if (outcome === 'pushed') {
+                        pushed.push(1);
+                    }
+                    else if (outcome === 'skipped') {
+                        skipped.push(1);
+                    }
                 }
-                else if (result === 'skipped') {
-                    skipped.push(1);
+                else {
+                    // rejected — extract error info
+                    const element = batch[j];
+                    const syncState = getExternalSyncState(element.metadata);
+                    const err = result.reason;
+                    errors.push({
+                        elementId: element.id,
+                        externalId: syncState?.externalId ?? '',
+                        provider: syncState?.provider ?? 'unknown',
+                        project: syncState?.project ?? 'unknown',
+                        message: err instanceof Error ? err.message : String(err),
+                        retryable: isRetryableError(err),
+                    });
                 }
             }
-            catch (err) {
-                errors.push({
-                    elementId: element.id,
-                    externalId: syncState.externalId,
-                    provider: syncState.provider,
-                    project: syncState.project,
-                    message: err instanceof Error ? err.message : String(err),
-                    retryable: isRetryableError(err),
-                });
-            }
+            processedCount += batch.length;
+            options.onProgress?.(processedCount, elements.length);
         }
         return buildResult({
             pushed: pushed.length,
@@ -168,8 +189,43 @@ export class SyncEngine {
             errors,
             provider: this.getPrimaryProvider(),
             project: this.getPrimaryProject(),
+            noLibrarySkipped,
         });
     }
+    /**
+     * Process a single element for push — validates sync state, checks skip conditions,
+     * and routes to the appropriate push method (pushElement or pushDocument).
+     *
+     * Extracted from the push() loop body to enable concurrent execution via Promise.allSettled.
+     *
+     * @returns 'pushed' if changes were sent, 'skipped' if element was skipped
+     */
+    async pushSingleElement(element, options, now, libraryPaths = new Map()) {
+        const syncState = getExternalSyncState(element.metadata);
+        if (!syncState) {
+            // No sync state — shouldn't happen since we filtered for it, but guard
+            return 'skipped';
+        }
+        // Skip if direction is pull-only
+        if (syncState.direction === 'pull') {
+            return 'skipped';
+        }
+        // Skip closed/tombstone tasks and archived documents — they're done
+        // and shouldn't sync. If an element is later reopened/reactivated,
+        // it will be picked up again naturally since the filter no longer applies.
+        const elementStatus = element.status;
+        if (elementStatus === 'closed' || elementStatus === 'tombstone' || elementStatus === 'archived') {
+            return 'skipped';
+        }
+        // Route by adapter type: tasks → pushElement(), documents → pushDocument()
+        if (syncState.adapterType === 'document') {
+            const libraryPath = libraryPaths.get(element.id);
+            return this.pushDocument(element, syncState, options, now, libraryPath);
+        }
+        else {
+            return this.pushElement(element, syncState, options, now);
+        }
+    }
     /**
      * Push a single element to its external service.
      *
@@ -219,6 +275,55 @@ export class SyncEngine {
         });
         return 'pushed';
     }
+    /**
+     * Push a single document to its external service.
+     *
+     * Parallel to pushElement(), but uses document-specific conversion
+     * (documentToExternalDocumentInput) and the DocumentSyncAdapter.
+     *
+     * @returns 'pushed' if changes were sent, 'skipped' if no changes detected
+     */
+    async pushDocument(element, syncState, options, now, libraryPath) {
+        // Check for actual content change via hash (skip when force is true)
+        const currentHash = computeContentHashSync(element).hash;
+        if (!options.force && syncState.lastPushedHash && currentHash === syncState.lastPushedHash) {
+            return 'skipped';
+        }
+        // Verify events have occurred since last push (skip when force is true)
+        if (!options.force && syncState.lastPushedAt) {
+            const events = await this.api.listEvents({
+                elementId: element.id,
+                eventType: ['updated'],
+                after: syncState.lastPushedAt,
+            });
+            if (events.length === 0) {
+                return 'skipped';
+            }
+        }
+        // Dry run — report but don't actually push
+        if (options.dryRun) {
+            return 'pushed';
+        }
+        // Get the document adapter for this element's provider
+        const adapter = this.getDocumentAdapter(syncState.provider);
+        if (!adapter) {
+            throw new Error(`No document adapter found for provider '${syncState.provider}'`);
+        }
+        // Build external document input using the shared field mapping utilities
+        const docInput = documentToExternalDocumentInput(element, libraryPath);
+        // Push to external service
+        await adapter.updatePage(syncState.project, syncState.externalId, docInput);
+        // Update sync state on element
+        const updatedSyncState = {
+            ...syncState,
+            lastPushedAt: now,
+            lastPushedHash: currentHash,
+        };
+        await this.api.update(element.id, {
+            metadata: setExternalSyncState(element.metadata, updatedSyncState),
+        });
+        return 'pushed';
+    }
     // --------------------------------------------------------------------------
     // Pull — pull externally-changed items into Stoneforge
     // --------------------------------------------------------------------------
@@ -243,28 +348,58 @@ export class SyncEngine {
         const conflicts = [];
         const errors = [];
         const now = createTimestamp();
-        // Get all task adapters from the registry
-        const adapterEntries = this.registry.getAdaptersOfType('task');
-        for (const { provider, adapter } of adapterEntries) {
-            const taskAdapter = adapter;
-            const providerConfig = this.getProviderConfig(provider.name);
-            const project = providerConfig?.defaultProject;
-            if (!project) {
-                // No project configured for this provider — skip
-                continue;
-            }
-            try {
-                const result = await this.pullFromProvider(provider, taskAdapter, project, options, now, conflicts, errors);
-                pulled.push(result.pulled);
-                skipped.push(result.skipped);
+        const adapterFilter = options.adapterTypes && options.adapterTypes.length > 0
+            ? new Set(options.adapterTypes)
+            : null;
+        // Get all task adapters from the registry (skip if filtered out)
+        if (!adapterFilter || adapterFilter.has('task')) {
+            const taskAdapterEntries = this.registry.getAdaptersOfType('task');
+            for (const { provider, adapter } of taskAdapterEntries) {
+                const taskAdapter = adapter;
+                const providerConfig = this.getProviderConfig(provider.name);
+                const project = providerConfig?.defaultProject;
+                if (!project) {
+                    // No project configured for this provider — skip
+                    continue;
+                }
+                try {
+                    const result = await this.pullFromProvider(provider, taskAdapter, project, options, now, conflicts, errors);
+                    pulled.push(result.pulled);
+                    skipped.push(result.skipped);
+                }
+                catch (err) {
+                    errors.push({
+                        provider: provider.name,
+                        project,
+                        message: err instanceof Error ? err.message : String(err),
+                        retryable: isRetryableError(err),
+                    });
+                }
             }
-            catch (err) {
-                errors.push({
-                    provider: provider.name,
-                    project,
-                    message: err instanceof Error ? err.message : String(err),
-                    retryable: isRetryableError(err),
-                });
+        }
+        // Get all document adapters from the registry (skip if filtered out)
+        if (!adapterFilter || adapterFilter.has('document')) {
+            const docAdapterEntries = this.registry.getAdaptersOfType('document');
+            for (const { provider, adapter } of docAdapterEntries) {
+                const docAdapter = adapter;
+                const providerConfig = this.getProviderConfig(provider.name);
+                const project = providerConfig?.defaultProject;
+                if (!project) {
+                    continue;
+                }
+                try {
+                    const result = await this.pullDocumentsFromProvider(provider, docAdapter, project, options, now, conflicts, errors);
+                    pulled.push(result.pulled);
+                    skipped.push(result.skipped);
+                }
+                catch (err) {
+                    errors.push({
+                        provider: provider.name,
+                        project,
+                        message: err instanceof Error ? err.message : String(err),
+                        retryable: isRetryableError(err),
+                    });
+                }
             }
         }
         return buildResult({
@@ -441,6 +576,206 @@ export class SyncEngine {
         return 'pulled';
     }
     // --------------------------------------------------------------------------
+    // Pull — document-specific helpers
+    // --------------------------------------------------------------------------
+    /**
+     * Pull document changes from a specific provider.
+     * Parallel to pullFromProvider(), but uses the DocumentSyncAdapter.
+     */
+    async pullDocumentsFromProvider(provider, adapter, project, options, now, conflicts, errors) {
+        let pulledCount = 0;
+        let skippedCount = 0;
+        // Get the sync cursor for this provider+project for documents
+        const syncCursor = this.getSyncCursor(provider.name, project, 'document');
+        // Fetch changed documents since cursor
+        const allExternalDocs = await adapter.listPagesSince(project, syncCursor);
+        // Skip external documents with empty or whitespace-only titles
+        const externalDocs = allExternalDocs.filter((doc) => doc.title && doc.title.trim().length > 0);
+        // Get all locally-linked document elements for matching
+        const linkedElements = await this.findLinkedElementsForProvider(provider.name, project);
+        // Filter to only document-type linked elements
+        const linkedDocuments = linkedElements.filter((el) => {
+            const state = getExternalSyncState(el.metadata);
+            return state && state.adapterType === 'document';
+        });
+        // Build a map of externalId → local element for fast lookup
+        const linkedByExternalId = new Map();
+        for (const el of linkedDocuments) {
+            const state = getExternalSyncState(el.metadata);
+            if (state) {
+                linkedByExternalId.set(state.externalId, el);
+            }
+        }
+        for (const externalDoc of externalDocs) {
+            // If taskIds filter is set (used generically for element IDs), check if this external doc matches
+            if (options.taskIds) {
+                const localEl = linkedByExternalId.get(externalDoc.externalId);
+                if (!localEl || !options.taskIds.includes(localEl.id)) {
+                    skippedCount++;
+                    continue;
+                }
+            }
+            try {
+                const result = await this.pullDocumentItem(provider, project, externalDoc, linkedByExternalId, options, now, conflicts);
+                if (result === 'pulled') {
+                    pulledCount++;
+                }
+                else if (result === 'skipped') {
+                    skippedCount++;
+                }
+                else if (result === 'created') {
+                    pulledCount++;
+                }
+            }
+            catch (err) {
+                errors.push({
+                    externalId: externalDoc.externalId,
+                    provider: provider.name,
+                    project,
+                    message: err instanceof Error ? err.message : String(err),
+                    retryable: isRetryableError(err),
+                });
+            }
+        }
+        // Update sync cursor (unless dry run)
+        if (!options.dryRun && externalDocs.length > 0) {
+            this.setSyncCursor(provider.name, project, 'document', now);
+        }
+        return { pulled: pulledCount, skipped: skippedCount };
+    }
+    /**
+     * Pull a single external document into Stoneforge.
+     * Parallel to pullItem(), but uses document-specific conversion.
+     *
+     * @returns 'pulled' if local element was updated, 'skipped' if no changes,
+     *          'created' if a new document was created
+     */
+    async pullDocumentItem(provider, project, externalDoc, linkedByExternalId, options, now, conflicts) {
+        const localElement = linkedByExternalId.get(externalDoc.externalId);
+        if (!localElement) {
+            // Unlinked external document — create new document if --all flag is set
+            if (options.all) {
+                if (options.dryRun) {
+                    return 'created';
+                }
+                await this.createDocumentFromExternal(provider, project, externalDoc, now);
+                return 'created';
+            }
+            return 'skipped';
+        }
+        // Linked element — check for actual change via hash
+        const syncState = getExternalSyncState(localElement.metadata);
+        // Skip if direction is push-only
+        if (syncState.direction === 'push') {
+            return 'skipped';
+        }
+        // Skip updates to archived documents
+        const localStatus = localElement.status;
+        if (localStatus === 'archived') {
+            return 'skipped';
+        }
+        // Compute a hash of the external document content to detect real changes
+        const remoteContentKey = computeExternalDocumentHash(externalDoc);
+        if (syncState.lastPulledHash && remoteContentKey === syncState.lastPulledHash) {
+            return 'skipped';
+        }
+        // Check for conflict: has local also changed since last pull?
+        const localHash = computeContentHashSync(localElement).hash;
+        const localChanged = syncState.lastPushedHash !== undefined && localHash !== syncState.lastPushedHash;
+        if (localChanged) {
+            // Both sides changed — use conflict resolver
+            const resolution = this.conflictResolver.resolve(localElement, externalDoc, this.defaultStrategy);
+            conflicts.push({
+                elementId: localElement.id,
+                externalId: externalDoc.externalId,
+                provider: provider.name,
+                project,
+                localUpdatedAt: localElement.updatedAt,
+                remoteUpdatedAt: externalDoc.updatedAt,
+                strategy: this.defaultStrategy,
+                resolved: resolution.resolved,
+                winner: resolution.resolved ? resolution.winner : undefined,
+            });
+            if (!resolution.resolved) {
+                // Manual resolution needed — tag the element
+                if (!options.dryRun) {
+                    const tags = localElement.tags.includes('sync-conflict')
+                        ? localElement.tags
+                        : [...localElement.tags, 'sync-conflict'];
+                    await this.api.update(localElement.id, { tags });
+                }
+                return 'skipped';
+            }
+            if (resolution.winner === 'local') {
+                // Local wins — skip the pull, but update the pulled hash
+                if (!options.dryRun) {
+                    const updatedSyncState = {
+                        ...syncState,
+                        lastPulledAt: now,
+                        lastPulledHash: remoteContentKey,
+                    };
+                    await this.api.update(localElement.id, {
+                        metadata: setExternalSyncState(localElement.metadata, updatedSyncState),
+                    });
+                }
+                return 'skipped';
+            }
+            // Remote wins — fall through to apply remote changes
+        }
+        // Dry run — report but don't actually apply
+        if (options.dryRun) {
+            return 'pulled';
+        }
+        // Apply remote changes to local document using document field mapping.
+        // Pass the existing document for diff mode (only changed fields are returned).
+        const updates = externalDocumentToDocumentUpdates(externalDoc, localElement);
+        const updatedSyncState = {
+            ...syncState,
+            lastPulledAt: now,
+            lastPulledHash: remoteContentKey,
+        };
+        const mergedMetadata = setExternalSyncState({ ...localElement.metadata }, updatedSyncState);
+        await this.api.update(localElement.id, {
+            ...updates,
+            metadata: mergedMetadata,
+        });
+        return 'pulled';
+    }
+    /**
+     * Create a new Stoneforge document from an unlinked external document.
+     */
+    async createDocumentFromExternal(provider, project, externalDoc, now) {
+        const syncState = {
+            provider: provider.name,
+            project,
+            externalId: externalDoc.externalId,
+            url: externalDoc.url,
+            lastPulledAt: now,
+            lastPulledHash: computeExternalDocumentHash(externalDoc),
+            direction: 'bidirectional',
+            adapterType: 'document',
+        };
+        // Use the document field mapping for conversion
+        const docUpdates = externalDocumentToDocumentUpdates(externalDoc);
+        const createInput = {
+            type: 'document',
+            title: docUpdates.title ?? externalDoc.title,
+            content: docUpdates.content ?? externalDoc.content,
+            contentType: docUpdates.contentType ?? 'markdown',
+            status: 'active',
+            category: 'other',
+            version: 1,
+            previousVersionId: null,
+            immutable: false,
+            tags: [],
+            externalRef: externalDoc.url,
+            createdBy: 'system',
+            metadata: { _externalSync: syncState },
+        };
+        const element = await this.api.create(createInput);
+        return element;
+    }
+    // --------------------------------------------------------------------------
     // Sync — bidirectional (push then pull)
     // --------------------------------------------------------------------------
     /**
@@ -459,7 +794,7 @@ export class SyncEngine {
             success: pushResult.success && pullResult.success,
             provider: pushResult.provider || pullResult.provider,
             project: pushResult.project || pullResult.project,
-            adapterType: 'task',
+            adapterType: pushResult.adapterType || pullResult.adapterType,
             pushed: pushResult.pushed,
             pulled: pullResult.pulled,
             skipped: pushResult.skipped + pullResult.skipped,
@@ -478,27 +813,43 @@ export class SyncEngine {
      * - Otherwise, returns tasks with _externalSync that have been updated
      */
     async findLinkedElements(options) {
+        const adapterFilter = options.adapterTypes && options.adapterTypes.length > 0
+            ? new Set(options.adapterTypes)
+            : null;
         if (options.taskIds && options.taskIds.length > 0) {
-            // Fetch specific tasks
+            // Fetch specific elements (tasks or documents)
             const elements = [];
             for (const id of options.taskIds) {
                 const element = await this.api.get(id);
-                if (element && getExternalSyncState(element.metadata)) {
-                    elements.push(element);
+                if (element) {
+                    const state = getExternalSyncState(element.metadata);
+                    if (state && (!adapterFilter || adapterFilter.has(state.adapterType))) {
+                        elements.push(element);
+                    }
                 }
             }
             return elements;
         }
-        // Fetch all tasks, then filter to those with _externalSync metadata
-        const allTasks = await this.api.list({ type: 'task' });
-        return allTasks.filter((el) => getExternalSyncState(el.metadata) !== undefined);
+        // Fetch elements based on adapter type filter
+        const collections = [];
+        if (!adapterFilter || adapterFilter.has('task')) {
+            collections.push(await this.api.list({ type: 'task' }));
+        }
+        if (!adapterFilter || adapterFilter.has('document')) {
+            collections.push(await this.api.list({ type: 'document' }));
+        }
+        return collections.flat().filter((el) => {
+            const state = getExternalSyncState(el.metadata);
+            return state !== undefined && (!adapterFilter || adapterFilter.has(state.adapterType));
+        });
     }
     /**
      * Find elements linked to a specific provider and project.
      */
     async findLinkedElementsForProvider(providerName, project) {
         const allTasks = await this.api.list({ type: 'task' });
-        return allTasks.filter((el) => {
+        const allDocs = await this.api.list({ type: 'document' });
+        return [...allTasks, ...allDocs].filter((el) => {
             const state = getExternalSyncState(el.metadata);
             return state && state.provider === providerName && state.project === project;
         });
@@ -674,6 +1025,15 @@ export class SyncEngine {
             return undefined;
         return provider.getTaskAdapter?.();
     }
+    /**
+     * Get a DocumentSyncAdapter for a given provider name.
+     */
+    getDocumentAdapter(providerName) {
+        const provider = this.registry.get(providerName);
+        if (!provider)
+            return undefined;
+        return provider.getDocumentAdapter?.();
+    }
     /**
      * Get the ProviderConfig for a given provider name.
      */
@@ -730,10 +1090,14 @@ function isRetryableError(err) {
     const message = err.message.toLowerCase();
     return (message.includes('rate limit') ||
         message.includes('timeout') ||
+        message.includes('gateway timeout') ||
+        message.includes('bad gateway') ||
         message.includes('econnrefused') ||
         message.includes('enotfound') ||
         message.includes('network') ||
+        message.includes('502') ||
         message.includes('503') ||
+        message.includes('504') ||
         message.includes('429'));
 }
 /**
@@ -750,6 +1114,9 @@ function buildResult(params) {
         skipped: params.skipped,
         conflicts: params.conflicts,
         errors: params.errors,
+        ...(params.noLibrarySkipped !== undefined && params.noLibrarySkipped > 0
+            ? { noLibrarySkipped: params.noLibrarySkipped }
+            : {}),
     };
 }
 // ============================================================================
@@ -782,4 +1149,6 @@ function buildResult(params) {
 export function createSyncEngine(config) {
     return new SyncEngine(config);
 }
+// Export constants for testing
+export { PUSH_CONCURRENCY };
 //# sourceMappingURL=sync-engine.js.map