mnemospark 0.8.3 → 0.9.1

package/README.md

@@ -59,9 +59,11 @@ Use via `/mnemospark_cloud ...` in OpenClaw chat.
 ### Get a storage quote
 
 ```text
-/mnemospark_cloud price-storage --wallet-address <addr> --object-id <id> --object-id-hash <sha256> --gb <gb> --provider <provider> --region <region>
+/mnemospark_cloud price-storage --wallet-address <addr> --object-id <id> --object-id-hash <sha256> --gb <gb> --provider aws --region us-east-1
 ```
 
+Use other regions by changing `--provider` and `--region` (defaults: `aws` / `us-east-1`).
+
 ### Upload using quote
 
 ```text

package/dist/cli.js

@@ -2973,10 +2973,12 @@ import {
   randomBytes as randomBytesNode,
   randomUUID as randomUUID3
 } from "crypto";
-import { createReadStream as createReadStream2, statfsSync } from "fs";
+import { createReadStream as createReadStream2, createWriteStream as createWriteStream2, statfsSync } from "fs";
 import { lstat, mkdir as mkdir5, readFile as readFile3, readdir as readdir2, rm, stat as stat2, writeFile as writeFile3 } from "fs/promises";
-import { homedir as homedir6 } from "os";
+import { homedir as homedir6, tmpdir } from "os";
 import { basename as basename2, dirname as dirname5, join as join8, resolve as resolve2 } from "path";
+import { Readable } from "stream";
+import { finished } from "stream/promises";
 import { privateKeyToAccount as privateKeyToAccount5 } from "viem/accounts";
 
 // src/cloud-ls-format.ts
@@ -3374,6 +3376,7 @@ async function createCloudDatastore(homeDir) {
     addOperationsColumn("subagent_session_id", "TEXT");
     addOperationsColumn("timeout_seconds", "INTEGER");
     addOperationsColumn("cancel_requested_at", "TEXT");
+    addOperationsColumn("result_text", "TEXT");
     nextDb.prepare(
       `INSERT INTO schema_migrations(version, applied_at)
        VALUES(?, ?)
@@ -3565,8 +3568,8 @@ async function createCloudDatastore(homeDir) {
         const ts = nowIso();
         const terminalStatuses = /* @__PURE__ */ new Set(["succeeded", "failed", "cancelled", "timed_out"]);
         db.prepare(
-          `INSERT INTO operations(operation_id, type, object_id, quote_id, trace_id, orchestrator, subagent_session_id, timeout_seconds, cancel_requested_at, status, error_code, error_message, started_at, finished_at, created_at, updated_at)
-           VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+          `INSERT INTO operations(operation_id, type, object_id, quote_id, trace_id, orchestrator, subagent_session_id, timeout_seconds, cancel_requested_at, status, error_code, error_message, result_text, started_at, finished_at, created_at, updated_at)
+           VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
            ON CONFLICT(operation_id) DO UPDATE SET
              type=excluded.type,
              object_id=COALESCE(excluded.object_id, operations.object_id),
@@ -3579,6 +3582,7 @@ async function createCloudDatastore(homeDir) {
              status=excluded.status,
              error_code=excluded.error_code,
              error_message=excluded.error_message,
+             result_text=COALESCE(excluded.result_text, operations.result_text),
              started_at=COALESCE(excluded.started_at, operations.started_at),
              finished_at=COALESCE(excluded.finished_at, operations.finished_at),
              updated_at=excluded.updated_at`
@@ -3595,6 +3599,7 @@ async function createCloudDatastore(homeDir) {
           row.status,
           row.error_code,
           row.error_message,
+          row.result_text ?? null,
           row.status === "started" ? ts : null,
           terminalStatuses.has(row.status) ? ts : null,
           ts,
@@ -3604,7 +3609,7 @@ async function createCloudDatastore(homeDir) {
     },
     findOperationById: async (operationId) => safe(() => {
       const row = db.prepare(
-        `SELECT operation_id, type, object_id, quote_id, trace_id, orchestrator, subagent_session_id, timeout_seconds, cancel_requested_at, status, error_code, error_message, started_at, finished_at, updated_at
+        `SELECT operation_id, type, object_id, quote_id, trace_id, orchestrator, subagent_session_id, timeout_seconds, cancel_requested_at, status, error_code, error_message, result_text, started_at, finished_at, updated_at
              FROM operations
              WHERE operation_id = ?
              LIMIT 1`
@@ -3763,6 +3768,7 @@ var DEFAULT_BACKUP_DIR = join8(homedir6(), BACKUP_DIR_SUBPATH);
 var BLOCKRUN_WALLET_KEY_SUBPATH = join8(".openclaw", "blockrun", "wallet.key");
 var MNEMOSPARK_WALLET_KEY_SUBPATH = join8(".openclaw", "mnemospark", "wallet", "wallet.key");
 var INLINE_UPLOAD_MAX_BYTES = 45e5;
+var NODE_FS_MAX_READFILE_BYTES = 2147483648;
 var PAYMENT_CRON_SCHEDULE = "0 0 1 * *";
 var TAR_OVERHEAD_BYTES = 10 * 1024 * 1024;
 var QUOTE_VALIDITY_USER_NOTE = "Quotes are valid for one hour. Please run price-storage again if you need a new quote.";
@@ -3809,8 +3815,8 @@ var CLOUD_HELP_TEXT = [
   "  Purpose: create a local tar+gzip archive under ~/.openclaw/mnemospark/backup (filename from sanitized friendly name) and record metadata in SQLite for later price-storage and upload.",
   "  Required: " + REQUIRED_BACKUP,
   "",
-  "\u2022 `/mnemospark_cloud price-storage --wallet-address <addr> --object-id <id> --object-id-hash <hash> --gb <gb> --provider <provider> --region <region>`",
-  "  Purpose: request a storage quote before upload.",
+  "\u2022 `/mnemospark_cloud price-storage --wallet-address <addr> --object-id <id> --object-id-hash <hash> --gb <gb> --provider aws --region us-east-1`",
+  "  Purpose: request a storage quote before upload (defaults shown; override `--provider` / `--region` for other regions).",
   "  Required: " + REQUIRED_PRICE_STORAGE,
   "",
   "\u2022 `/mnemospark_cloud upload --quote-id <quote-id> --wallet-address <addr> --object-id <id> --object-id-hash <hash> [--name <friendly-name>] [--async] [--orchestrator <inline|subagent>] [--timeout-seconds <n>]`",
@@ -4686,6 +4692,39 @@ function encryptAesGcm(plaintext, key, randomFn = randomBytesNode) {
   const tag = cipher.getAuthTag();
   return Buffer.concat([nonce, ciphertext, tag]);
 }
+async function encryptPlaintextFileToAesGcmPath(plaintextPath, dek, outPath, randomFn = randomBytesNode) {
+  if (dek.length !== 32) {
+    throw new Error("Expected 32-byte AES key");
+  }
+  const nonce = randomFn(AES_GCM_NONCE_BYTES);
+  const cipher = createCipheriv("aes-256-gcm", dek, nonce);
+  const writeStream = createWriteStream2(outPath, { flags: "w" });
+  writeStream.write(nonce);
+  try {
+    for await (const chunk of createReadStream2(plaintextPath)) {
+      const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+      const out = cipher.update(buf);
+      if (out.length) {
+        await new Promise((resolve3, reject) => {
+          writeStream.write(out, (err) => err ? reject(err) : resolve3());
+        });
+      }
+    }
+    const final = cipher.final();
+    const tag = cipher.getAuthTag();
+    await new Promise((resolve3, reject) => {
+      writeStream.write(final, (err) => err ? reject(err) : resolve3());
+    });
+    await new Promise((resolve3, reject) => {
+      writeStream.write(tag, (err) => err ? reject(err) : resolve3());
+    });
+    writeStream.end();
+    await finished(writeStream);
+  } catch (err) {
+    writeStream.destroy();
+    throw err;
+  }
+}
 async function loadOrCreateKek(walletAddress, homeDir) {
   const keyPath = resolveWalletKekPath(walletAddress, homeDir);
   await mkdir5(dirname5(keyPath), { recursive: true });
@@ -4702,11 +4741,42 @@ async function loadOrCreateKek(walletAddress, homeDir) {
   return { kek: generated, keyPath };
 }
 async function prepareUploadPayload(archivePath, walletAddress, homeDir) {
-  const plaintext = await readFile3(archivePath);
+  const archiveStat = await stat2(archivePath);
+  if (!archiveStat.isFile()) {
+    throw new Error(`Cannot read backup archive: not a file (${archivePath}).`);
+  }
   const { kek, keyPath } = await loadOrCreateKek(walletAddress, homeDir);
   const dek = randomBytesNode(32);
-  const encryptedContent = encryptAesGcm(plaintext, dek);
   const wrappedDek = encryptAesGcm(dek, kek);
+  if (archiveStat.size >= NODE_FS_MAX_READFILE_BYTES) {
+    const encryptedTempPath = join8(tmpdir(), `mnemospark-upload-${randomUUID3()}.enc`);
+    try {
+      await encryptPlaintextFileToAesGcmPath(archivePath, dek, encryptedTempPath);
+      const encStat = await stat2(encryptedTempPath);
+      const payloadHash2 = await sha256File(encryptedTempPath);
+      const payload2 = {
+        mode: "presigned",
+        content_base64: void 0,
+        content_sha256: payloadHash2,
+        content_length_bytes: encStat.size,
+        wrapped_dek: wrappedDek.toString("base64"),
+        encryption_algorithm: "AES-256-GCM",
+        bucket_name_hint: bucketNameForWallet(walletAddress),
+        key_store_path_hint: keyPath
+      };
+      return {
+        payload: payload2,
+        encryptedContent: null,
+        encryptedTempPath
+      };
+    } catch (err) {
+      await rm(encryptedTempPath, { force: true }).catch(() => {
+      });
+      throw err;
+    }
+  }
+  const plaintext = await readFile3(archivePath);
+  const encryptedContent = encryptAesGcm(plaintext, dek);
   const payloadHash = sha256Buffer(encryptedContent);
   const payload = {
     mode: encryptedContent.length <= INLINE_UPLOAD_MAX_BYTES ? "inline" : "presigned",
@@ -4723,7 +4793,17 @@ async function prepareUploadPayload(archivePath, walletAddress, homeDir) {
     encryptedContent
   };
 }
-async function uploadPresignedObjectIfNeeded(uploadResponse, uploadMode, encryptedContent, fetchImpl = fetch) {
+function presignedPutBodyInit(encryptedContent, encryptedTempPath) {
+  if (encryptedTempPath?.trim()) {
+    const body = Readable.toWeb(createReadStream2(encryptedTempPath));
+    return { body, duplex: "half" };
+  }
+  if (encryptedContent) {
+    return { body: new Uint8Array(encryptedContent) };
+  }
+  throw new Error("Cannot upload storage object: missing encrypted payload body.");
+}
+async function uploadPresignedObjectIfNeeded(uploadResponse, uploadMode, encryptedContent, encryptedTempPath, fetchImpl = fetch) {
   if (!uploadResponse.upload_url) {
     if (uploadMode === "presigned") {
       throw new Error("Cannot upload storage object: missing presigned upload URL.");
@@ -4734,24 +4814,33 @@ async function uploadPresignedObjectIfNeeded(uploadResponse, uploadMode, encrypt
   if (!headers.has("content-type")) {
     headers.set("content-type", "application/octet-stream");
   }
-  const putBody = new Uint8Array(encryptedContent);
-  const firstAttempt = await fetchImpl(uploadResponse.upload_url, {
+  const { body, duplex } = presignedPutBodyInit(encryptedContent, encryptedTempPath);
+  const firstInit = {
     method: "PUT",
     headers,
-    body: putBody,
+    body,
     redirect: "manual"
-  });
+  };
+  if (duplex) {
+    firstInit.duplex = duplex;
+  }
+  const firstAttempt = await fetchImpl(uploadResponse.upload_url, firstInit);
   if (firstAttempt.ok) {
     return;
   }
   if ((firstAttempt.status === 307 || firstAttempt.status === 308) && firstAttempt.headers.has("location")) {
     const location = firstAttempt.headers.get("location")?.trim();
     if (location) {
-      const redirectedAttempt = await fetchImpl(location, {
+      const retryBody = presignedPutBodyInit(encryptedContent, encryptedTempPath);
+      const retryInit = {
         method: "PUT",
         headers,
-        body: putBody
-      });
+        body: retryBody.body
+      };
+      if (retryBody.duplex) {
+        retryInit.duplex = retryBody.duplex;
+      }
+      const redirectedAttempt = await fetchImpl(location, retryInit);
       if (redirectedAttempt.ok) {
         return;
       }
@@ -4879,9 +4968,11 @@ function formatPriceStorageUserMessage(quote, localArchiveHint) {
 function quoteLookupMatchesPriceStorageResponse(lookup, quote) {
   return lookup.quoteId === quote.quote_id && lookup.walletAddress.trim().toLowerCase() === quote.addr.trim().toLowerCase() && lookup.objectId === quote.object_id && lookup.objectIdHash.toLowerCase() === quote.object_id_hash.toLowerCase() && lookup.storagePrice === quote.storage_price && lookup.provider === quote.provider && lookup.location === quote.location;
 }
+var DEFAULT_BACKUP_QUOTE_PROVIDER = "aws";
+var DEFAULT_BACKUP_QUOTE_REGION = "us-east-1";
 function formatBackupSuccessUserMessage(result, walletAddress, friendlyName) {
   const hash = result.objectIdHash.replace(/\s/g, "");
-  const priceStorageLine = `/mnemospark_cloud price-storage --wallet-address \`${walletAddress}\` --object-id \`${result.objectId}\` --object-id-hash \`${hash}\` --gb \`${result.objectSizeGb}\` --provider <provider> --region <region>`;
+  const priceStorageLine = `/mnemospark_cloud price-storage --wallet-address \`${walletAddress}\` --object-id \`${result.objectId}\` --object-id-hash \`${hash}\` --gb \`${result.objectSizeGb}\` --provider ${DEFAULT_BACKUP_QUOTE_PROVIDER} --region ${DEFAULT_BACKUP_QUOTE_REGION}`;
   return [
     `Backup archive: \`${result.archivePath}\``,
     "",
@@ -4890,10 +4981,12 @@ function formatBackupSuccessUserMessage(result, walletAddress, friendlyName) {
     `object-id-hash: ${hash}`,
     `object-size: ${result.objectSizeGb}`,
     "",
-    "Next, request a storage quote. Replace `<provider>` and `<region>` (one line):",
+    "Next, request a storage quote.",
     "",
     priceStorageLine,
     "",
+    `The default region is ${DEFAULT_BACKUP_QUOTE_REGION}. Change the command parameters to switch regions (not required).`,
+    "",
     "Region examples (merge into the command above):",
     "North America: `--provider aws --region us-east-1`",
     "Europe: `--provider aws --region eu-north-1`",
@@ -5407,8 +5500,8 @@ async function runCloudCommandHandler(ctx, options, executionContext = {}) {
   const datastore = await createCloudDatastore(mnemosparkHomeDir);
   const terminalOperationStatuses = /* @__PURE__ */ new Set(["succeeded", "failed", "cancelled", "timed_out"]);
   const isTerminalOperationStatus = (status) => terminalOperationStatuses.has(status);
-  const formatOperationStatus = (operation) => ({
-    text: [
+  const formatOperationStatus = (operation) => {
+    const meta = [
       `operation-id: ${operation.operation_id}`,
       `type: ${operation.type}`,
       `status: ${operation.status}`,
@@ -5419,9 +5512,15 @@ async function runCloudCommandHandler(ctx, options, executionContext = {}) {
       operation.timeout_seconds ? `timeout-seconds: ${operation.timeout_seconds}` : null,
       operation.error_code ? `error-code: ${operation.error_code}` : null,
       operation.error_message ? `error-message: ${operation.error_message}` : null
-    ].filter((v) => Boolean(v)).join("\n"),
-    isError: operation.status === "failed" || operation.status === "cancelled" || operation.status === "timed_out"
-  });
+    ].filter((v) => Boolean(v)).join("\n");
+    const withResult = operation.status === "succeeded" && operation.result_text?.trim() ? `${meta}
+
+${operation.result_text}` : meta;
+    return {
+      text: withResult,
+      isError: operation.status === "failed" || operation.status === "cancelled" || operation.status === "timed_out"
+    };
+  };
   if (parsed.mode === "op-status") {
     let operation = await datastore.findOperationById(parsed.operationId);
     if (!operation) {
@@ -5758,7 +5857,7 @@ async function runCloudCommandHandler(ctx, options, executionContext = {}) {
                 mnemosparkHomeDir
               );
             },
-            onCompleted: async (sessionId) => {
+            onCompleted: async (sessionId, result) => {
               await datastore.upsertOperation({
                 operation_id: operationId,
                 type: opType,
@@ -5770,7 +5869,8 @@ async function runCloudCommandHandler(ctx, options, executionContext = {}) {
                 timeout_seconds: timeoutSeconds,
                 status: "succeeded",
                 error_code: null,
-                error_message: null
+                error_message: null,
+                result_text: result.isError ? null : result.text
               });
               await emitOperationEventBestEffort(
                 "operation.completed",
@@ -5947,7 +6047,8 @@ operation-id: ${operationId}`,
         orchestrator: "inline",
         status: result.isError ? "failed" : "succeeded",
         error_code: result.isError ? "ASYNC_FAILED" : null,
-        error_message: result.isError ? result.text : null
+        error_message: result.isError ? result.text : null,
+        result_text: result.isError ? null : result.text
       });
       await emitOperationEventBestEffort(
         "operation.completed",
@@ -6027,7 +6128,7 @@ operation-id: ${operationId}`,
       await emitCloudEventBestEffort(
         "backup.completed",
         {
-          operation_id: randomUUID3(),
+          operation_id: executionContext.forcedOperationId?.trim() || randomUUID3(),
           object_id: result.objectId,
           status: "succeeded",
           details: {
@@ -6152,6 +6253,7 @@ operation-id: ${operationId}`,
       executionContext.forcedOperationId ?? idempotencyKeyFn(),
       executionContext.forcedTraceId
     );
+    let preparedPayload;
     try {
       const loggedQuote = await datastore.findQuoteById(parsed.uploadRequest.quote_id);
       if (!loggedQuote) {
@@ -6223,7 +6325,7 @@ operation-id: ${operationId}`,
           isError: true
         };
       }
-      const preparedPayload = await prepareUploadPayload(
+      preparedPayload = await prepareUploadPayload(
         archivePath,
         parsed.uploadRequest.wallet_address,
         mnemosparkHomeDir
@@ -6267,6 +6369,7 @@ operation-id: ${operationId}`,
         uploadResponse,
         preparedPayload.payload.mode,
         preparedPayload.encryptedContent,
+        preparedPayload.encryptedTempPath,
         fetchImpl
       );
       let finalizedUploadResponse = uploadResponse;
@@ -6404,6 +6507,11 @@ operation-id: ${operationId}`,
         text: uploadErrorMessage ?? "Cannot upload storage object",
         isError: true
       };
+    } finally {
+      if (preparedPayload?.encryptedTempPath) {
+        await rm(preparedPayload.encryptedTempPath, { force: true }).catch(() => {
+        });
+      }
     }
   }
   if (parsed.mode === "ls") {