mnemospark 0.9.0 → 0.9.2

package/dist/index.js

@@ -3024,10 +3024,12 @@ import {
   randomBytes as randomBytesNode,
   randomUUID as randomUUID3
 } from "crypto";
-import { createReadStream as createReadStream2, statfsSync } from "fs";
+import { createReadStream as createReadStream2, createWriteStream as createWriteStream2, statfsSync } from "fs";
 import { lstat, mkdir as mkdir5, readFile as readFile3, readdir as readdir2, rm, stat as stat2, writeFile as writeFile3 } from "fs/promises";
-import { homedir as homedir6 } from "os";
+import { homedir as homedir6, tmpdir } from "os";
 import { basename as basename2, dirname as dirname5, join as join8, resolve as resolve2 } from "path";
+import { Readable } from "stream";
+import { finished } from "stream/promises";
 import { privateKeyToAccount as privateKeyToAccount5 } from "viem/accounts";
 
 // src/cloud-ls-format.ts
@@ -3817,6 +3819,7 @@ var DEFAULT_BACKUP_DIR = join8(homedir6(), BACKUP_DIR_SUBPATH);
 var BLOCKRUN_WALLET_KEY_SUBPATH = join8(".openclaw", "blockrun", "wallet.key");
 var MNEMOSPARK_WALLET_KEY_SUBPATH = join8(".openclaw", "mnemospark", "wallet", "wallet.key");
 var INLINE_UPLOAD_MAX_BYTES = 45e5;
+var NODE_FS_MAX_READFILE_BYTES = 2147483648;
 var PAYMENT_CRON_SCHEDULE = "0 0 1 * *";
 var TAR_OVERHEAD_BYTES = 10 * 1024 * 1024;
 var QUOTE_VALIDITY_USER_NOTE = "Quotes are valid for one hour. Please run price-storage again if you need a new quote.";
@@ -4740,6 +4743,39 @@ function encryptAesGcm(plaintext, key, randomFn = randomBytesNode) {
   const tag = cipher.getAuthTag();
   return Buffer.concat([nonce, ciphertext, tag]);
 }
+async function encryptPlaintextFileToAesGcmPath(plaintextPath, dek, outPath, randomFn = randomBytesNode) {
+  if (dek.length !== 32) {
+    throw new Error("Expected 32-byte AES key");
+  }
+  const nonce = randomFn(AES_GCM_NONCE_BYTES);
+  const cipher = createCipheriv("aes-256-gcm", dek, nonce);
+  const writeStream = createWriteStream2(outPath, { flags: "w" });
+  writeStream.write(nonce);
+  try {
+    for await (const chunk of createReadStream2(plaintextPath)) {
+      const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+      const out = cipher.update(buf);
+      if (out.length) {
+        await new Promise((resolve3, reject) => {
+          writeStream.write(out, (err) => err ? reject(err) : resolve3());
+        });
+      }
+    }
+    const final = cipher.final();
+    const tag = cipher.getAuthTag();
+    await new Promise((resolve3, reject) => {
+      writeStream.write(final, (err) => err ? reject(err) : resolve3());
+    });
+    await new Promise((resolve3, reject) => {
+      writeStream.write(tag, (err) => err ? reject(err) : resolve3());
+    });
+    writeStream.end();
+    await finished(writeStream);
+  } catch (err) {
+    writeStream.destroy();
+    throw err;
+  }
+}
 async function loadOrCreateKek(walletAddress, homeDir) {
   const keyPath = resolveWalletKekPath(walletAddress, homeDir);
   await mkdir5(dirname5(keyPath), { recursive: true });
@@ -4756,11 +4792,42 @@ async function loadOrCreateKek(walletAddress, homeDir) {
   return { kek: generated, keyPath };
 }
 async function prepareUploadPayload(archivePath, walletAddress, homeDir) {
-  const plaintext = await readFile3(archivePath);
+  const archiveStat = await stat2(archivePath);
+  if (!archiveStat.isFile()) {
+    throw new Error(`Cannot read backup archive: not a file (${archivePath}).`);
+  }
   const { kek, keyPath } = await loadOrCreateKek(walletAddress, homeDir);
   const dek = randomBytesNode(32);
-  const encryptedContent = encryptAesGcm(plaintext, dek);
   const wrappedDek = encryptAesGcm(dek, kek);
+  if (archiveStat.size >= NODE_FS_MAX_READFILE_BYTES) {
+    const encryptedTempPath = join8(tmpdir(), `mnemospark-upload-${randomUUID3()}.enc`);
+    try {
+      await encryptPlaintextFileToAesGcmPath(archivePath, dek, encryptedTempPath);
+      const encStat = await stat2(encryptedTempPath);
+      const payloadHash2 = await sha256File(encryptedTempPath);
+      const payload2 = {
+        mode: "presigned",
+        content_base64: void 0,
+        content_sha256: payloadHash2,
+        content_length_bytes: encStat.size,
+        wrapped_dek: wrappedDek.toString("base64"),
+        encryption_algorithm: "AES-256-GCM",
+        bucket_name_hint: bucketNameForWallet(walletAddress),
+        key_store_path_hint: keyPath
+      };
+      return {
+        payload: payload2,
+        encryptedContent: null,
+        encryptedTempPath
+      };
+    } catch (err) {
+      await rm(encryptedTempPath, { force: true }).catch(() => {
+      });
+      throw err;
+    }
+  }
+  const plaintext = await readFile3(archivePath);
+  const encryptedContent = encryptAesGcm(plaintext, dek);
   const payloadHash = sha256Buffer(encryptedContent);
   const payload = {
     mode: encryptedContent.length <= INLINE_UPLOAD_MAX_BYTES ? "inline" : "presigned",
@@ -4777,7 +4844,17 @@ async function prepareUploadPayload(archivePath, walletAddress, homeDir) {
     encryptedContent
   };
 }
-async function uploadPresignedObjectIfNeeded(uploadResponse, uploadMode, encryptedContent, fetchImpl = fetch) {
+function presignedPutBodyInit(encryptedContent, encryptedTempPath) {
+  if (encryptedTempPath?.trim()) {
+    const body = Readable.toWeb(createReadStream2(encryptedTempPath));
+    return { body, duplex: "half" };
+  }
+  if (encryptedContent) {
+    return { body: new Uint8Array(encryptedContent) };
+  }
+  throw new Error("Cannot upload storage object: missing encrypted payload body.");
+}
+async function uploadPresignedObjectIfNeeded(uploadResponse, uploadMode, encryptedContent, encryptedTempPath, fetchImpl = fetch, presignedStreamContentLengthBytes) {
   if (!uploadResponse.upload_url) {
     if (uploadMode === "presigned") {
       throw new Error("Cannot upload storage object: missing presigned upload URL.");
@@ -4788,24 +4865,39 @@ async function uploadPresignedObjectIfNeeded(uploadResponse, uploadMode, encrypt
   if (!headers.has("content-type")) {
     headers.set("content-type", "application/octet-stream");
   }
-  const putBody = new Uint8Array(encryptedContent);
-  const firstAttempt = await fetchImpl(uploadResponse.upload_url, {
+  if (encryptedTempPath?.trim()) {
+    const len = presignedStreamContentLengthBytes ?? (await stat2(encryptedTempPath.trim())).size;
+    if (!headers.has("content-length")) {
+      headers.set("content-length", String(len));
+    }
+  }
+  const { body, duplex } = presignedPutBodyInit(encryptedContent, encryptedTempPath);
+  const firstInit = {
     method: "PUT",
     headers,
-    body: putBody,
+    body,
     redirect: "manual"
-  });
+  };
+  if (duplex) {
+    firstInit.duplex = duplex;
+  }
+  const firstAttempt = await fetchImpl(uploadResponse.upload_url, firstInit);
   if (firstAttempt.ok) {
     return;
   }
   if ((firstAttempt.status === 307 || firstAttempt.status === 308) && firstAttempt.headers.has("location")) {
     const location = firstAttempt.headers.get("location")?.trim();
     if (location) {
-      const redirectedAttempt = await fetchImpl(location, {
+      const retryBody = presignedPutBodyInit(encryptedContent, encryptedTempPath);
+      const retryInit = {
         method: "PUT",
         headers,
-        body: putBody
-      });
+        body: retryBody.body
+      };
+      if (retryBody.duplex) {
+        retryInit.duplex = retryBody.duplex;
+      }
+      const redirectedAttempt = await fetchImpl(location, retryInit);
       if (redirectedAttempt.ok) {
         return;
       }
@@ -6218,6 +6310,7 @@ operation-id: ${operationId}`,
       executionContext.forcedOperationId ?? idempotencyKeyFn(),
       executionContext.forcedTraceId
     );
+    let preparedPayload;
     try {
       const loggedQuote = await datastore.findQuoteById(parsed.uploadRequest.quote_id);
       if (!loggedQuote) {
@@ -6289,7 +6382,7 @@ operation-id: ${operationId}`,
           isError: true
         };
       }
-      const preparedPayload = await prepareUploadPayload(
+      preparedPayload = await prepareUploadPayload(
         archivePath,
         parsed.uploadRequest.wallet_address,
         mnemosparkHomeDir
@@ -6333,7 +6426,9 @@ operation-id: ${operationId}`,
         uploadResponse,
         preparedPayload.payload.mode,
         preparedPayload.encryptedContent,
-        fetchImpl
+        preparedPayload.encryptedTempPath,
+        fetchImpl,
+        preparedPayload.encryptedTempPath ? preparedPayload.payload.content_length_bytes : void 0
       );
       let finalizedUploadResponse = uploadResponse;
       if (preparedPayload.payload.mode === "presigned" && uploadResponse.confirmation_required === true) {
@@ -6470,6 +6565,11 @@ operation-id: ${operationId}`,
         text: uploadErrorMessage ?? "Cannot upload storage object",
         isError: true
       };
+    } finally {
+      if (preparedPayload?.encryptedTempPath) {
+        await rm(preparedPayload.encryptedTempPath, { force: true }).catch(() => {
+        });
+      }
     }
   }
   if (parsed.mode === "ls") {