@agent8/deploy 1.3.1 → 1.4.1

package/bin/agent8.js

@@ -148,12 +148,18 @@ if (fs.existsSync(crossrampFilePath)) {
 const filesToUpload = [];
 const newHashes = {};
 
-// Calculate file hash
-const calculateFileHash = (filePath) => {
-  const fileContent = fs.readFileSync(filePath);
-  return crypto.createHash("md5").update(fileContent).digest("hex");
-};
+// Calculate file hash by streaming — a large asset must not be buffered
+// whole just to be hashed (RSS high-water was dominated by this).
+const calculateFileHash = (filePath) =>
+  new Promise((resolve, reject) => {
+    const hash = crypto.createHash("md5");
+    fs.createReadStream(filePath)
+      .on("data", (chunk) => hash.update(chunk))
+      .on("end", () => resolve(hash.digest("hex")))
+      .on("error", reject);
+  });
 
+const main = async () => {
 // Check for server.js (priority: ./server.js, fallback: ./server/dist/server.js)
 let serverFilePath = path.resolve(process.cwd(), "server.js");
 if (!fs.existsSync(serverFilePath)) {
@@ -171,7 +177,7 @@ if (fs.existsSync(serverFilePath)) {
     const serverContent = fs.readFileSync(serverFilePath, "utf8");
     eval(`(function() { ${serverContent}; })`)();
 
-    const hash = calculateFileHash(serverFilePath);
+    const hash = await calculateFileHash(serverFilePath);
     newHashes["server.js"] = hash;
 
     if (prodMode || hash !== deployedHashes["server.js"]) {
@@ -179,6 +185,7 @@ if (fs.existsSync(serverFilePath)) {
         filePath: serverFilePath,
         uploadPath: "",
         fileName: "server.js",
+        deployKey: "server.js",
       });
     }
   } catch (error) {
@@ -191,7 +198,10 @@ if (!previewMode) {
   // Check for dist directory
   const distDirPath = path.resolve(process.cwd(), "dist");
   if (fs.existsSync(distDirPath) && fs.statSync(distDirPath).isDirectory()) {
-    const processDistFiles = (dirPath, basePath = "") => {
+    // Collect paths synchronously, then hash sequentially via streams so at
+    // most one file's read window is in memory at a time.
+    const distEntries = [];
+    const collectDistFiles = (dirPath, basePath = "") => {
       const files = fs.readdirSync(dirPath);
 
       files.forEach((file) => {
@@ -203,23 +213,28 @@ if (!previewMode) {
           .replace(/\\/g, "/");
 
         if (fs.statSync(fullPath).isDirectory()) {
-          processDistFiles(fullPath, relativePath);
+          collectDistFiles(fullPath, relativePath);
         } else {
-          const hash = calculateFileHash(fullPath);
-          newHashes[deployKey] = hash;
-
-          if (prodMode || hash !== deployedHashes[deployKey]) {
-            filesToUpload.push({
-              filePath: fullPath,
-              uploadPath: uploadPath === "." ? "" : uploadPath,
-              fileName: path.basename(file),
-            });
-          }
+          distEntries.push({ fullPath, uploadPath, deployKey });
         }
       });
     };
 
-    processDistFiles(distDirPath);
+    collectDistFiles(distDirPath);
+
+    for (const entry of distEntries) {
+      const hash = await calculateFileHash(entry.fullPath);
+      newHashes[entry.deployKey] = hash;
+
+      if (prodMode || hash !== deployedHashes[entry.deployKey]) {
+        filesToUpload.push({
+          filePath: entry.fullPath,
+          uploadPath: entry.uploadPath === "." ? "" : entry.uploadPath,
+          fileName: path.basename(entry.fullPath),
+          deployKey: entry.deployKey,
+        });
+      }
+    }
   }
 }
 
@@ -229,32 +244,51 @@ if (filesToUpload.length === 0) {
       "No file changes detected. Setting up CrossRamp configuration..."
     );
 
-    setupCrossRamp(targetVerse, crossrampConfig).then((success) => {
-      if (success) {
-        console.log("CrossRamp setup completed.");
-      }
-      process.exit(success ? 0 : 1);
-    });
+    const success = await setupCrossRamp(targetVerse, crossrampConfig);
+    if (success) {
+      console.log("CrossRamp setup completed.");
+    }
+    process.exit(success ? 0 : 1);
   } else {
     console.log("No changes detected. Nothing to deploy.");
     process.exit(0);
   }
-  return;
 }
 
 console.log(`Found ${filesToUpload.length} files to upload...`);
 
-// Upload files in parallel
-const uploadPromises = filesToUpload.map((fileInfo) => {
+// Upload files with bounded concurrency, streaming each file from disk.
+// The previous implementation buffered every file with readFileSync and
+// started all uploads at once, so peak memory was roughly the whole dist
+// size times 2-3 (raw buffer + multipart framing + axios serialization) —
+// measured 2GB+ for asset-heavy games, inside 4GB containers. Streaming
+// with knownLength keeps the request identical on the wire (same multipart
+// framing, exact Content-Length) while holding only a few in-flight
+// buffers in memory.
+const UPLOAD_CONCURRENCY = 5;
+
+const uploadFile = (fileInfo) => {
   const { filePath, uploadPath, fileName } = fileInfo;
   const form = new FormData();
-  const fileContent = fs.readFileSync(filePath);
-  form.append("file", fileContent, fileName);
+  form.append("file", fs.createReadStream(filePath), {
+    filename: fileName,
+    knownLength: fs.statSync(filePath).size,
+  });
   form.append("path", uploadPath || "/");
 
   return axios
     .post(`${endpoint}/verses/${targetVerse}/files`, form, {
-      headers: getAuthHeaders(form.getHeaders()),
+      headers: {
+        ...getAuthHeaders(form.getHeaders()),
+        "Content-Length": form.getLengthSync(),
+      },
+      // maxRedirects: 0 forces the plain http adapter. With redirects
+      // enabled, follow-redirects buffers the ENTIRE request body in memory
+      // (to replay it after a redirect), which silently defeats streaming.
+      // The upload endpoint never redirects POSTs, so nothing is lost.
+      maxRedirects: 0,
+      maxBodyLength: Infinity,
+      maxContentLength: Infinity,
     })
     .then(() => {
       console.log(`Uploaded: ${uploadPath}/${fileName}`);
@@ -267,30 +301,74 @@ const uploadPromises = filesToUpload.map((fileInfo) => {
       );
       return false;
     });
+};
+
+// Minimal worker pool: at most `concurrency` uploads in flight, results in
+// input order, never rejects (uploadFile already maps errors to false).
+const uploadAll = async (files, concurrency) => {
+  const results = new Array(files.length);
+  let next = 0;
+  const workers = Array.from(
+    { length: Math.min(concurrency, files.length) },
+    async () => {
+      while (next < files.length) {
+        const i = next++;
+        results[i] = await uploadFile(files[i]);
+      }
+    }
+  );
+  await Promise.all(workers);
+  return results;
+};
+
+const results = await uploadAll(filesToUpload, UPLOAD_CONCURRENCY);
+const successCount = results.filter(Boolean).length;
+console.log(
+  `Deployment complete: ${successCount}/${filesToUpload.length} files uploaded successfully.`
+);
+
+// A failed upload must not be recorded as deployed — otherwise the next
+// incremental run sees a matching hash, skips the file, and the verse stays
+// broken until a --prod redeploy. Revert each failed file's hash to its last
+// deployed value (the server still holds that content), or drop it entirely
+// if it was never deployed, so the next run retries it.
+const failedKeys = [];
+results.forEach((ok, i) => {
+  if (ok) return;
+  const { deployKey } = filesToUpload[i];
+  failedKeys.push(deployKey);
+  if (deployKey in deployedHashes) {
+    newHashes[deployKey] = deployedHashes[deployKey];
+  } else {
+    delete newHashes[deployKey];
+  }
 });
 
-Promise.all(uploadPromises)
-  .then(async (results) => {
-    const successCount = results.filter(Boolean).length;
-    console.log(
-      `Deployment complete: ${successCount}/${filesToUpload.length} files uploaded successfully.`
+// CrossRamp setup if config exists and all files uploaded successfully
+if (crossrampConfig && successCount === filesToUpload.length) {
+  console.log("Setting up CrossRamp configuration...");
+  const success = await setupCrossRamp(targetVerse, crossrampConfig);
+  if (!success) {
+    console.warn(
+      "Warning: CrossRamp setup failed, but files were deployed"
     );
+  }
+}
 
-    // CrossRamp setup if config exists and all files uploaded successfully
-    if (crossrampConfig && successCount === filesToUpload.length) {
-      console.log("Setting up CrossRamp configuration...");
-      const success = await setupCrossRamp(targetVerse, crossrampConfig);
-      if (!success) {
-        console.warn(
-          "Warning: CrossRamp setup failed, but files were deployed"
-        );
-      }
-    }
+// Save new hashes to .deployed file
+fs.writeFileSync(deployedFilePath, JSON.stringify(newHashes, null, 2));
+console.log("Updated .deployed file with new file hashes");
 
-    // Save new hashes to .deployed file
-    fs.writeFileSync(deployedFilePath, JSON.stringify(newHashes, null, 2));
-    console.log("Updated .deployed file with new file hashes");
-  })
-  .catch((error) => {
-    console.error("Deployment failed:", error.message);
-  });
+if (failedKeys.length > 0) {
+  console.error(
+    `Deployment incomplete: ${failedKeys.length} file(s) failed to upload and will be retried on the next run:`
+  );
+  failedKeys.forEach((key) => console.error(`  - ${key}`));
+  process.exit(1);
+}
+};
+
+main().catch((error) => {
+  console.error("Deployment failed:", error.message);
+  process.exit(1);
+});

package/package.json

@@ -1,8 +1,7 @@
 {
   "name": "@agent8/deploy",
-  "version": "1.3.1",
+  "version": "1.4.1",
   "description": "A CLI tool for running agent8 commands",
-  "main": "index.js",
   "bin": {
     "agent8": "./bin/agent8.js"
   },
@@ -13,5 +12,8 @@
     "axios": "^1.7.9",
     "dotenv": "^16.4.7",
     "form-data": "^4.0.1"
-  }
+  },
+  "files": [
+    "bin"
+  ]
 }