@magicpages/ghost-typesense-core 1.9.1 → 1.9.3

package/dist/index.d.mts

@@ -35,6 +35,26 @@ declare class GhostTypesenseManager {
      * Fetch all posts from Ghost and index them in Typesense
      */
     indexAllPosts(): Promise<void>;
+    /**
+     * Index documents in batches with retry logic and backpressure handling
+     * @private
+     */
+    private indexDocumentsBatched;
+    /**
+     * Process a single batch with retry logic and backpressure handling
+     * @private
+     */
+    private processBatchWithRetry;
+    /**
+     * Retry failed batches with smaller batch sizes
+     * @private
+     */
+    private retryFailedBatches;
+    /**
+     * Sleep utility for backoff delays
+     * @private
+     */
+    private sleep;
     /**
      * Index a single post in Typesense
      */

package/dist/index.d.ts

@@ -35,6 +35,26 @@ declare class GhostTypesenseManager {
      * Fetch all posts from Ghost and index them in Typesense
      */
     indexAllPosts(): Promise<void>;
+    /**
+     * Index documents in batches with retry logic and backpressure handling
+     * @private
+     */
+    private indexDocumentsBatched;
+    /**
+     * Process a single batch with retry logic and backpressure handling
+     * @private
+     */
+    private processBatchWithRetry;
+    /**
+     * Retry failed batches with smaller batch sizes
+     * @private
+     */
+    private retryFailedBatches;
+    /**
+     * Sleep utility for backoff delays
+     * @private
+     */
+    private sleep;
     /**
      * Index a single post in Typesense
      */

package/dist/index.js

@@ -41,9 +41,10 @@ var GhostTypesenseManager = class {
     this.typesense = new import_typesense.Client({
       nodes: config.typesense.nodes,
       apiKey: config.typesense.apiKey,
-      connectionTimeoutSeconds: config.typesense.connectionTimeoutSeconds,
-      retryIntervalSeconds: config.typesense.retryIntervalSeconds,
-      numRetries: 3
+      connectionTimeoutSeconds: config.typesense.connectionTimeoutSeconds || 3600,
+      // 60 minutes for bulk operations
+      retryIntervalSeconds: config.typesense.retryIntervalSeconds || 2,
+      numRetries: 5
     });
   }
   /**
@@ -163,23 +164,154 @@ var GhostTypesenseManager = class {
     }
     console.log(`Found ${allPosts.length} posts to index`);
     const documents = allPosts.map((post) => this.transformPost(post));
-    try {
-      const collection = this.typesense.collections(this.collectionName);
-      const results = await Promise.all(
-        documents.map(
-          (doc) => collection.documents().upsert(doc).then(() => ({ success: true, id: doc.id })).catch((error) => ({ success: false, id: doc.id, error: error.message }))
-        )
-      );
-      const succeeded = results.filter((r) => r.success).length;
-      const failed = results.filter((r) => !r.success).length;
-      console.log(`Indexing complete: ${succeeded} succeeded, ${failed} failed`);
-      if (failed > 0) {
-        console.log("Failed documents:", results.filter((r) => !r.success));
+    await this.indexDocumentsBatched(documents);
+  }
+  /**
+   * Index documents in batches with retry logic and backpressure handling
+   * @private
+   */
+  async indexDocumentsBatched(documents) {
+    const batchSize = this.config.typesense.batchSize || 200;
+    const maxConcurrentBatches = this.config.typesense.maxConcurrentBatches || 12;
+    const batches = [];
+    for (let i = 0; i < documents.length; i += batchSize) {
+      batches.push(documents.slice(i, i + batchSize));
+    }
+    console.log(`Processing ${documents.length} documents in ${batches.length} batches (batch size: ${batchSize})`);
+    const collection = this.typesense.collections(this.collectionName);
+    let totalSucceeded = 0;
+    let totalFailed = 0;
+    const failedBatches = [];
+    for (let i = 0; i < batches.length; i += maxConcurrentBatches) {
+      const batchGroup = batches.slice(i, i + maxConcurrentBatches);
+      const batchPromises = batchGroup.map(async (batch, batchIndex) => {
+        const actualBatchIndex = i + batchIndex;
+        return this.processBatchWithRetry(collection, batch, actualBatchIndex, batches.length);
+      });
+      const results = await Promise.allSettled(batchPromises);
+      results.forEach((result, batchIndex) => {
+        const actualBatchIndex = i + batchIndex;
+        if (result.status === "fulfilled") {
+          totalSucceeded += result.value.succeeded;
+          totalFailed += result.value.failed;
+          if (result.value.error) {
+            failedBatches.push({
+              batchIndex: actualBatchIndex,
+              documents: batchGroup[batchIndex],
+              error: result.value.error
+            });
+          }
+        } else {
+          const batchSize2 = batchGroup[batchIndex].length;
+          totalFailed += batchSize2;
+          failedBatches.push({
+            batchIndex: actualBatchIndex,
+            documents: batchGroup[batchIndex],
+            error: result.reason?.message || "Unknown batch error"
+          });
+        }
+      });
+      console.log(`Progress: ${Math.min(i + maxConcurrentBatches, batches.length)}/${batches.length} batch groups processed`);
+    }
+    console.log(`Indexing complete: ${totalSucceeded} succeeded, ${totalFailed} failed`);
+    if (failedBatches.length > 0) {
+      console.log(`Retrying ${failedBatches.length} failed batches with smaller batch size...`);
+      const retryResults = await this.retryFailedBatches(collection, failedBatches);
+      totalSucceeded += retryResults.succeeded;
+      totalFailed = totalFailed - retryResults.retryAttempted + retryResults.failed;
+      console.log(`Final result: ${totalSucceeded} succeeded, ${totalFailed} failed`);
+    }
+    if (totalFailed > 0) {
+      console.log(`\u26A0\uFE0F  ${totalFailed} documents failed to index. Consider running sync again or checking server capacity.`);
+    }
+  }
+  /**
+   * Process a single batch with retry logic and backpressure handling
+   * @private
+   */
+  async processBatchWithRetry(collection, documents, batchIndex, totalBatches) {
+    const maxRetries = 3;
+    let lastError = "";
+    for (let attempt = 1; attempt <= maxRetries; attempt++) {
+      try {
+        console.log(`Processing batch ${batchIndex + 1}/${totalBatches} (${documents.length} docs) - attempt ${attempt}`);
+        const result = await collection.documents().import(documents, {
+          action: "upsert",
+          batch_size: documents.length,
+          return_doc: false,
+          return_id: false
+        });
+        const succeeded = result.filter((r) => r.success === true).length;
+        const failed = documents.length - succeeded;
+        if (failed > 0) {
+          console.log(`Batch ${batchIndex + 1}: ${succeeded} succeeded, ${failed} failed`);
+        }
+        return { succeeded, failed };
+      } catch (error) {
+        lastError = error.message || error;
+        if (error.httpStatus === 503 || lastError.includes("503") || lastError.includes("Not Ready")) {
+          const backoffDelay = Math.min(1e3 * Math.pow(2, attempt), 1e4);
+          console.log(`Batch ${batchIndex + 1}: Server overload (503), retrying in ${backoffDelay}ms...`);
+          await this.sleep(backoffDelay);
+          continue;
+        }
+        if (lastError.includes("timeout") || lastError.includes("ECONNABORTED")) {
+          const backoffDelay = Math.min(2e3 * attempt, 8e3);
+          console.log(`Batch ${batchIndex + 1}: Timeout error, retrying in ${backoffDelay}ms...`);
+          await this.sleep(backoffDelay);
+          continue;
+        }
+        if (attempt < maxRetries) {
+          const backoffDelay = 1e3 * attempt;
+          console.log(`Batch ${batchIndex + 1}: Error (${lastError}), retrying in ${backoffDelay}ms...`);
+          await this.sleep(backoffDelay);
+          continue;
+        }
       }
-    } catch (error) {
-      console.error("Indexing error:", error);
-      throw error;
     }
+    console.error(`Batch ${batchIndex + 1} failed after ${maxRetries} attempts: ${lastError}`);
+    return { succeeded: 0, failed: documents.length, error: lastError };
+  }
+  /**
+   * Retry failed batches with smaller batch sizes
+   * @private
+   */
+  async retryFailedBatches(collection, failedBatches) {
+    let succeeded = 0;
+    let failed = 0;
+    let retryAttempted = 0;
+    for (const failedBatch of failedBatches) {
+      retryAttempted += failedBatch.documents.length;
+      const smallBatches = [];
+      for (let i = 0; i < failedBatch.documents.length; i += 50) {
+        smallBatches.push(failedBatch.documents.slice(i, i + 50));
+      }
+      for (const smallBatch of smallBatches) {
+        try {
+          const result = await collection.documents().import(smallBatch, {
+            action: "upsert",
+            batch_size: smallBatch.length,
+            return_doc: false,
+            return_id: false
+          });
+          const batchSucceeded = result.filter((r) => r.success === true).length;
+          succeeded += batchSucceeded;
+          failed += smallBatch.length - batchSucceeded;
+        } catch (error) {
+          console.error(`Small batch retry failed: ${error.message || error}`);
+          failed += smallBatch.length;
+        }
+        await this.sleep(500);
+      }
+    }
+    return { succeeded, failed, retryAttempted };
+  }
+  /**
+   * Sleep utility for backoff delays
+   * @private
+   */
+  sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
   }
   /**
    * Index a single post in Typesense

package/dist/index.mjs

@@ -17,9 +17,10 @@ var GhostTypesenseManager = class {
     this.typesense = new Client({
       nodes: config.typesense.nodes,
       apiKey: config.typesense.apiKey,
-      connectionTimeoutSeconds: config.typesense.connectionTimeoutSeconds,
-      retryIntervalSeconds: config.typesense.retryIntervalSeconds,
-      numRetries: 3
+      connectionTimeoutSeconds: config.typesense.connectionTimeoutSeconds || 3600,
+      // 60 minutes for bulk operations
+      retryIntervalSeconds: config.typesense.retryIntervalSeconds || 2,
+      numRetries: 5
     });
   }
   /**
@@ -139,23 +140,154 @@ var GhostTypesenseManager = class {
     }
     console.log(`Found ${allPosts.length} posts to index`);
     const documents = allPosts.map((post) => this.transformPost(post));
-    try {
-      const collection = this.typesense.collections(this.collectionName);
-      const results = await Promise.all(
-        documents.map(
-          (doc) => collection.documents().upsert(doc).then(() => ({ success: true, id: doc.id })).catch((error) => ({ success: false, id: doc.id, error: error.message }))
-        )
-      );
-      const succeeded = results.filter((r) => r.success).length;
-      const failed = results.filter((r) => !r.success).length;
-      console.log(`Indexing complete: ${succeeded} succeeded, ${failed} failed`);
-      if (failed > 0) {
-        console.log("Failed documents:", results.filter((r) => !r.success));
+    await this.indexDocumentsBatched(documents);
+  }
+  /**
+   * Index documents in batches with retry logic and backpressure handling
+   * @private
+   */
+  async indexDocumentsBatched(documents) {
+    const batchSize = this.config.typesense.batchSize || 200;
+    const maxConcurrentBatches = this.config.typesense.maxConcurrentBatches || 12;
+    const batches = [];
+    for (let i = 0; i < documents.length; i += batchSize) {
+      batches.push(documents.slice(i, i + batchSize));
+    }
+    console.log(`Processing ${documents.length} documents in ${batches.length} batches (batch size: ${batchSize})`);
+    const collection = this.typesense.collections(this.collectionName);
+    let totalSucceeded = 0;
+    let totalFailed = 0;
+    const failedBatches = [];
+    for (let i = 0; i < batches.length; i += maxConcurrentBatches) {
+      const batchGroup = batches.slice(i, i + maxConcurrentBatches);
+      const batchPromises = batchGroup.map(async (batch, batchIndex) => {
+        const actualBatchIndex = i + batchIndex;
+        return this.processBatchWithRetry(collection, batch, actualBatchIndex, batches.length);
+      });
+      const results = await Promise.allSettled(batchPromises);
+      results.forEach((result, batchIndex) => {
+        const actualBatchIndex = i + batchIndex;
+        if (result.status === "fulfilled") {
+          totalSucceeded += result.value.succeeded;
+          totalFailed += result.value.failed;
+          if (result.value.error) {
+            failedBatches.push({
+              batchIndex: actualBatchIndex,
+              documents: batchGroup[batchIndex],
+              error: result.value.error
+            });
+          }
+        } else {
+          const batchSize2 = batchGroup[batchIndex].length;
+          totalFailed += batchSize2;
+          failedBatches.push({
+            batchIndex: actualBatchIndex,
+            documents: batchGroup[batchIndex],
+            error: result.reason?.message || "Unknown batch error"
+          });
+        }
+      });
+      console.log(`Progress: ${Math.min(i + maxConcurrentBatches, batches.length)}/${batches.length} batch groups processed`);
+    }
+    console.log(`Indexing complete: ${totalSucceeded} succeeded, ${totalFailed} failed`);
+    if (failedBatches.length > 0) {
+      console.log(`Retrying ${failedBatches.length} failed batches with smaller batch size...`);
+      const retryResults = await this.retryFailedBatches(collection, failedBatches);
+      totalSucceeded += retryResults.succeeded;
+      totalFailed = totalFailed - retryResults.retryAttempted + retryResults.failed;
+      console.log(`Final result: ${totalSucceeded} succeeded, ${totalFailed} failed`);
+    }
+    if (totalFailed > 0) {
+      console.log(`\u26A0\uFE0F  ${totalFailed} documents failed to index. Consider running sync again or checking server capacity.`);
+    }
+  }
+  /**
+   * Process a single batch with retry logic and backpressure handling
+   * @private
+   */
+  async processBatchWithRetry(collection, documents, batchIndex, totalBatches) {
+    const maxRetries = 3;
+    let lastError = "";
+    for (let attempt = 1; attempt <= maxRetries; attempt++) {
+      try {
+        console.log(`Processing batch ${batchIndex + 1}/${totalBatches} (${documents.length} docs) - attempt ${attempt}`);
+        const result = await collection.documents().import(documents, {
+          action: "upsert",
+          batch_size: documents.length,
+          return_doc: false,
+          return_id: false
+        });
+        const succeeded = result.filter((r) => r.success === true).length;
+        const failed = documents.length - succeeded;
+        if (failed > 0) {
+          console.log(`Batch ${batchIndex + 1}: ${succeeded} succeeded, ${failed} failed`);
+        }
+        return { succeeded, failed };
+      } catch (error) {
+        lastError = error.message || error;
+        if (error.httpStatus === 503 || lastError.includes("503") || lastError.includes("Not Ready")) {
+          const backoffDelay = Math.min(1e3 * Math.pow(2, attempt), 1e4);
+          console.log(`Batch ${batchIndex + 1}: Server overload (503), retrying in ${backoffDelay}ms...`);
+          await this.sleep(backoffDelay);
+          continue;
+        }
+        if (lastError.includes("timeout") || lastError.includes("ECONNABORTED")) {
+          const backoffDelay = Math.min(2e3 * attempt, 8e3);
+          console.log(`Batch ${batchIndex + 1}: Timeout error, retrying in ${backoffDelay}ms...`);
+          await this.sleep(backoffDelay);
+          continue;
+        }
+        if (attempt < maxRetries) {
+          const backoffDelay = 1e3 * attempt;
+          console.log(`Batch ${batchIndex + 1}: Error (${lastError}), retrying in ${backoffDelay}ms...`);
+          await this.sleep(backoffDelay);
+          continue;
+        }
       }
-    } catch (error) {
-      console.error("Indexing error:", error);
-      throw error;
     }
+    console.error(`Batch ${batchIndex + 1} failed after ${maxRetries} attempts: ${lastError}`);
+    return { succeeded: 0, failed: documents.length, error: lastError };
+  }
+  /**
+   * Retry failed batches with smaller batch sizes
+   * @private
+   */
+  async retryFailedBatches(collection, failedBatches) {
+    let succeeded = 0;
+    let failed = 0;
+    let retryAttempted = 0;
+    for (const failedBatch of failedBatches) {
+      retryAttempted += failedBatch.documents.length;
+      const smallBatches = [];
+      for (let i = 0; i < failedBatch.documents.length; i += 50) {
+        smallBatches.push(failedBatch.documents.slice(i, i + 50));
+      }
+      for (const smallBatch of smallBatches) {
+        try {
+          const result = await collection.documents().import(smallBatch, {
+            action: "upsert",
+            batch_size: smallBatch.length,
+            return_doc: false,
+            return_id: false
+          });
+          const batchSucceeded = result.filter((r) => r.success === true).length;
+          succeeded += batchSucceeded;
+          failed += smallBatch.length - batchSucceeded;
+        } catch (error) {
+          console.error(`Small batch retry failed: ${error.message || error}`);
+          failed += smallBatch.length;
+        }
+        await this.sleep(500);
+      }
+    }
+    return { succeeded, failed, retryAttempted };
+  }
+  /**
+   * Sleep utility for backoff delays
+   * @private
+   */
+  sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
   }
   /**
    * Index a single post in Typesense

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@magicpages/ghost-typesense-core",
-  "version": "1.9.1",
+  "version": "1.9.3",
   "description": "Core functionality for Ghost-Typesense integration",
   "author": "MagicPages",
   "license": "MIT",
@@ -24,7 +24,7 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
-    "@magicpages/ghost-typesense-config": "^1.9.1",
+    "@magicpages/ghost-typesense-config": "^1.9.3",
     "@ts-ghost/content-api": "^4.0.6",
     "@ts-ghost/core-api": "^4.0.6",
     "typesense": "^1.7.2",