@coderule/mcp 1.3.0 → 1.4.0

package/dist/cli.js

@@ -5,7 +5,7 @@ import { createHash } from 'crypto';
 import envPaths from 'env-paths';
 import pino from 'pino';
 import Database from 'better-sqlite3';
-import { Qulite, enqueueFsEvent } from '@coderule/qulite';
+import { Qulite, JobStatus, enqueueFsEvent } from '@coderule/qulite';
 import { CoderuleClients, ASTHttpClient, SyncHttpClient } from '@coderule/clients';
 import fs2 from 'fs';
 import { Worker } from 'worker_threads';
@@ -29,7 +29,7 @@ var DEFAULT_HEARTBEAT_CHECK_INTERVAL_MS = 5e3;
 var DEFAULT_QUEUE_POLL_INTERVAL_MS = 500;
 var DEFAULT_HASH_BATCH_SIZE = 32;
 var DEFAULT_MAX_SNAPSHOT_ATTEMPTS = 5;
-var DEFAULT_HTTP_TIMEOUT_MS = 3e4;
+var DEFAULT_HTTP_TIMEOUT_MS = 12e4;
 
 // src/config/Configurator.ts
 var DEFAULT_RETRIEVAL_FORMATTER = "standard";
@@ -534,6 +534,16 @@ var Outbox = class {
     if (purged > 0) {
       this.log.warn({ purged }, "Purged legacy fs_control jobs without kind");
     }
+    try {
+      const counts = {
+        pending: this.queue.countByStatus(JobStatus.Pending),
+        processing: this.queue.countByStatus(JobStatus.Processing),
+        done: this.queue.countByStatus(JobStatus.Done),
+        failed: this.queue.countByStatus(JobStatus.Failed)
+      };
+      this.log.debug({ counts }, "Outbox initialized");
+    } catch {
+    }
   }
   getQueue() {
     return this.queue;
@@ -1045,44 +1055,109 @@ function computeSnapshot(filesRepo) {
     totalSize
   };
 }
-async function uploadMissing(rootPath, missing, syncClient, logger2) {
-  if (!missing || missing.length === 0) return;
-  const map = /* @__PURE__ */ new Map();
-  for (const missingFile of missing) {
-    const absPath = path.join(rootPath, missingFile.file_path);
+async function withRetries(op, logger2, context, maxAttempts) {
+  let attempt = 0;
+  while (true) {
     try {
-      const buffer = await fs4.readFile(absPath);
-      map.set(missingFile.file_hash, {
-        path: missingFile.file_path,
-        content: buffer
-      });
-    } catch (error) {
+      return await op();
+    } catch (err) {
+      attempt += 1;
+      if (attempt >= maxAttempts) {
+        logger2.error(
+          { err, ...context, attempt },
+          "Operation failed after retries"
+        );
+        throw err;
+      }
+      const delay = Math.min(15e3, 1e3 * 2 ** (attempt - 1));
       logger2.warn(
-        { err: error, relPath: missingFile.file_path },
-        "Failed to read missing file content"
+        { err, ...context, attempt, delay },
+        "Operation failed; retrying"
       );
+      await sleep(delay);
+    }
+  }
+}
+async function uploadMissing(rootPath, missing, syncClient, logger2, maxAttempts, chunkSize = 64) {
+  if (!missing || missing.length === 0) return;
+  const total = missing.length;
+  const chunks = [];
+  for (let i = 0; i < total; i += chunkSize) {
+    chunks.push(missing.slice(i, i + chunkSize));
+  }
+  for (let idx = 0; idx < chunks.length; idx++) {
+    const list = chunks[idx];
+    const map = /* @__PURE__ */ new Map();
+    for (const missingFile of list) {
+      const absPath = path.join(rootPath, missingFile.file_path);
+      try {
+        const buffer = await fs4.readFile(absPath);
+        map.set(missingFile.file_hash, {
+          path: missingFile.file_path,
+          content: buffer
+        });
+      } catch (error) {
+        logger2.warn(
+          { err: error, relPath: missingFile.file_path },
+          "Failed to read missing file content"
+        );
+      }
     }
+    if (map.size === 0) continue;
+    await withRetries(
+      () => syncClient.uploadFileContent(map),
+      logger2,
+      {
+        op: "uploadFileContent",
+        chunkIndex: idx + 1,
+        chunks: chunks.length,
+        files: map.size
+      },
+      maxAttempts
+    );
   }
-  if (map.size === 0) return;
-  await syncClient.uploadFileContent(map);
 }
-async function ensureSnapshotCreated(rootPath, computation, syncClient, logger2) {
+async function ensureSnapshotCreated(rootPath, computation, syncClient, logger2, options) {
   const { snapshotHash, files } = computation;
-  let status = await syncClient.checkSnapshotStatus(snapshotHash);
+  const maxAttempts = options?.maxAttempts ?? 5;
+  const uploadChunkSize = options?.uploadChunkSize ?? 64;
+  let status = await withRetries(
+    () => syncClient.checkSnapshotStatus(snapshotHash),
+    logger2,
+    { op: "checkSnapshotStatus", snapshotHash },
+    maxAttempts
+  );
   if (status.status === "READY") {
     logger2.info({ snapshotHash }, "Snapshot already READY");
     return;
   }
   if (status.status === "NOT_FOUND" || status.status === "MISSING_CONTENT") {
-    status = await syncClient.createSnapshot(snapshotHash, files);
+    status = await withRetries(
+      () => syncClient.createSnapshot(snapshotHash, files),
+      logger2,
+      { op: "createSnapshot", snapshotHash },
+      maxAttempts
+    );
   }
   if (status.status === "MISSING_CONTENT" && status.missing_files?.length) {
     logger2.info(
       { missing: status.missing_files.length },
       "Uploading missing file content"
     );
-    await uploadMissing(rootPath, status.missing_files, syncClient, logger2);
-    status = await syncClient.createSnapshot(snapshotHash, files);
+    await uploadMissing(
+      rootPath,
+      status.missing_files,
+      syncClient,
+      logger2,
+      maxAttempts,
+      uploadChunkSize
+    );
+    status = await withRetries(
+      () => syncClient.createSnapshot(snapshotHash, files),
+      logger2,
+      { op: "createSnapshot", snapshotHash },
+      maxAttempts
+    );
   }
   let attempt = 0;
   while (status.status !== "READY") {
@@ -1092,13 +1167,24 @@ async function ensureSnapshotCreated(rootPath, computation, syncClient, logger2)
     const delay = Math.min(5e3, 1e3 * Math.max(1, 2 ** attempt));
     await sleep(delay);
     attempt += 1;
-    status = await syncClient.checkSnapshotStatus(snapshotHash);
+    status = await withRetries(
+      () => syncClient.checkSnapshotStatus(snapshotHash),
+      logger2,
+      { op: "checkSnapshotStatus", snapshotHash },
+      maxAttempts
+    );
   }
   logger2.info({ snapshotHash }, "Snapshot READY");
 }
-async function publishSnapshot(rootPath, filesRepo, snapshotsRepo, syncClient, logger2) {
+async function publishSnapshot(rootPath, filesRepo, snapshotsRepo, syncClient, logger2, options) {
   const computation = computeSnapshot(filesRepo);
-  await ensureSnapshotCreated(rootPath, computation, syncClient, logger2);
+  await ensureSnapshotCreated(
+    rootPath,
+    computation,
+    syncClient,
+    logger2,
+    options
+  );
   const createdAt = Date.now();
   snapshotsRepo.insert(
     computation.snapshotHash,
@@ -1138,7 +1224,8 @@ async function runInitialSyncPipeline(runtime) {
     runtime.filesRepo,
     runtime.snapshotsRepo,
     runtime.clients.sync,
-    syncLogger
+    syncLogger,
+    { maxAttempts: runtime.config.maxSnapshotAttempts }
   );
   return result;
 }
@@ -1541,7 +1628,8 @@ var ServiceRunner = class {
         this.runtime.filesRepo,
         this.runtime.snapshotsRepo,
         this.runtime.clients.sync,
-        log
+        log,
+        { maxAttempts: this.runtime.config.maxSnapshotAttempts }
       );
       this.runtime.outbox.ack(job.id, this.fsControlLeaseOwner);
       this.state.updateSnapshotReady(result.createdAt);
@@ -1622,15 +1710,27 @@ async function runService(params) {
   try {
     runner = new ServiceRunner(runtime);
     await runner.prepareWatcher(true);
-    const initial = await runInitialSyncPipeline(runtime);
-    runtime.logger.info(
-      {
-        snapshotHash: initial.snapshotHash,
-        filesCount: initial.filesCount
-      },
-      "Initial sync completed; entering continuous mode"
-    );
-    runner.recordInitialSnapshot(initial.createdAt);
+    let initialCreatedAt;
+    try {
+      const initial = await runInitialSyncPipeline(runtime);
+      runtime.logger.info(
+        {
+          snapshotHash: initial.snapshotHash,
+          filesCount: initial.filesCount
+        },
+        "Initial sync completed; entering continuous mode"
+      );
+      initialCreatedAt = initial.createdAt;
+    } catch (error) {
+      runtime.logger.warn(
+        { err: error },
+        "Initial sync failed; enqueuing snapshot job and continuing"
+      );
+      runtime.outbox.enqueueSnapshot(runtime.config.rootId, 0);
+    }
+    if (initialCreatedAt) {
+      runner.recordInitialSnapshot(initialCreatedAt);
+    }
     await runner.startLoops();
     await runner.enableWatcherProcessing();
     runtime.logger.info("Coderule scanner service is running");