@remnic/cli 1.0.24 → 1.0.25

package/dist/index.js

@@ -5,6 +5,7 @@ import path11 from "path";
 import { createDecipheriv, createHash } from "crypto";
 import * as childProcess2 from "child_process";
 import { fileURLToPath as fileURLToPath4 } from "url";
+import { gzipSync } from "zlib";
 import {
   parseConfig,
   isOpenaiApiKeyDisabled,
@@ -6648,6 +6649,7 @@ function offlineEndpoint(remoteUrl, pathname, params = {}) {
   return url.toString();
 }
 var OFFLINE_SYNC_REQUEST_TIMEOUT_DEFAULT_MS = 15 * 6e4;
+var OFFLINE_SYNC_SNAPSHOT_BASE_POST_PREFERRED_MAX_BODY_BYTES = 16 * 1024 * 1024;
 function parseOfflineSyncRequestTimeoutMs(raw, fallback = OFFLINE_SYNC_REQUEST_TIMEOUT_DEFAULT_MS) {
   if (raw === void 0 || raw.trim().length === 0) return fallback;
   const parsed = Number(raw);
@@ -6746,29 +6748,130 @@ async function fetchOfflineJson(url, token, init = {}) {
     }
   );
 }
+async function parseOfflineSnapshotStreamResponse(response) {
+  if (!response.body) {
+    throw new Error("offline sync snapshot stream response omitted body");
+  }
+  const reader = response.body.getReader();
+  const decoder = new TextDecoder();
+  let buffered = "";
+  let header = null;
+  const files = [];
+  const handleLine = (line) => {
+    if (line.trim().length === 0) return;
+    const parsed = JSON.parse(line);
+    if (parsed.type === "snapshot") {
+      if (header) throw new Error("offline sync snapshot stream repeated header");
+      header = {
+        ...typeof parsed.namespace === "string" && parsed.namespace.length > 0 ? { namespace: parsed.namespace } : {},
+        format: parsed.format,
+        schemaVersion: parsed.schemaVersion,
+        createdAt: parsed.createdAt,
+        sourceId: parsed.sourceId,
+        includeTranscripts: parsed.includeTranscripts
+      };
+      return;
+    }
+    if (parsed.type === "file") {
+      if (!header) throw new Error("offline sync snapshot stream file arrived before header");
+      files.push(parsed.file);
+      return;
+    }
+    throw new Error("offline sync snapshot stream contained unknown event");
+  };
+  for (; ; ) {
+    const { value, done } = await reader.read();
+    if (done) break;
+    buffered += decoder.decode(value, { stream: true });
+    for (; ; ) {
+      const newline = buffered.indexOf("\n");
+      if (newline < 0) break;
+      const line = buffered.slice(0, newline);
+      buffered = buffered.slice(newline + 1);
+      handleLine(line);
+    }
+  }
+  buffered += decoder.decode();
+  handleLine(buffered);
+  const finalHeader = header;
+  if (!finalHeader) throw new Error("offline sync snapshot stream omitted header");
+  const snapshot = normalizeOfflineSyncSnapshot({
+    format: finalHeader.format,
+    schemaVersion: finalHeader.schemaVersion,
+    createdAt: finalHeader.createdAt,
+    sourceId: finalHeader.sourceId,
+    includeTranscripts: finalHeader.includeTranscripts,
+    files
+  });
+  return {
+    ...finalHeader.namespace ? { namespace: finalHeader.namespace } : {},
+    ...snapshot
+  };
+}
+async function fetchOfflineSnapshotStream(args) {
+  const url = offlineEndpoint(args.remoteUrl, "/remnic/v1/offline-sync/snapshot-stream", {
+    namespace: args.namespace,
+    include_transcripts: args.includeTranscripts ? "true" : "false",
+    content: "false"
+  });
+  return fetchOfflineWithResponse(
+    url,
+    args.token,
+    {},
+    {},
+    async (response) => {
+      if (!response.ok) {
+        await throwOfflineResponseError(response, url, {}, "offline sync snapshot-stream request");
+      }
+      return parseOfflineSnapshotStreamResponse(response);
+    }
+  );
+}
 async function fetchOfflineSnapshot(args) {
+  let tryStreamSnapshot = false;
   if (args.includeContent === false && args.baseFiles && args.baseFiles.length > 0) {
-    const postBody = offlineSnapshotBasePostBody({
-      namespace: args.namespace,
-      includeTranscripts: args.includeTranscripts,
-      baseFiles: args.baseFiles,
-      baseCapturedAt: args.baseCapturedAt
-    });
-    if (offlineSnapshotBasePostBodyFits(postBody)) {
-      try {
-        return await fetchOfflineJson(
-          offlineEndpoint(args.remoteUrl, "/remnic/v1/offline-sync/snapshot"),
-          args.token,
-          {
-            method: "POST",
-            body: postBody
-          }
-        );
-      } catch (error) {
-        if (!isOfflineSnapshotPostFallbackError(error)) throw error;
+    if (args.baseFiles.length > OFFLINE_SYNC_SNAPSHOT_BASE_POST_MAX_FILES) {
+      tryStreamSnapshot = true;
+    } else {
+      const postBody = offlineSnapshotBasePostBody({
+        namespace: args.namespace,
+        includeTranscripts: args.includeTranscripts,
+        baseFiles: args.baseFiles,
+        baseCapturedAt: args.baseCapturedAt
+      });
+      const postRequest = offlineSnapshotBasePostRequest(postBody);
+      if (postRequest) {
+        const postRequestUsesGzip = new Headers(postRequest.headers).get("content-encoding")?.toLowerCase() === "gzip";
+        try {
+          return await fetchOfflineJson(
+            offlineEndpoint(args.remoteUrl, "/remnic/v1/offline-sync/snapshot"),
+            args.token,
+            {
+              method: "POST",
+              ...postRequest
+            }
+          );
+        } catch (error) {
+          if (!isOfflineSnapshotPostFallbackError(error, { compressed: postRequestUsesGzip })) throw error;
+          tryStreamSnapshot = true;
+        }
+      } else {
+        tryStreamSnapshot = true;
       }
     }
   }
+  if (tryStreamSnapshot) {
+    try {
+      return await fetchOfflineSnapshotStream({
+        remoteUrl: args.remoteUrl,
+        token: args.token,
+        namespace: args.namespace,
+        includeTranscripts: args.includeTranscripts
+      });
+    } catch (error) {
+      if (!isOfflineSnapshotStreamFallbackError(error)) throw error;
+    }
+  }
   return fetchOfflineJson(
     offlineEndpoint(args.remoteUrl, "/remnic/v1/offline-sync/snapshot", {
       namespace: args.namespace,
@@ -6788,11 +6891,36 @@ function offlineSnapshotBasePostBody(args) {
   });
 }
 function offlineSnapshotBasePostBodyFits(body) {
-  return Buffer.byteLength(body, "utf-8") <= OFFLINE_SYNC_SNAPSHOT_BASE_MAX_BODY_BYTES;
+  const bytes = Buffer.byteLength(body, "utf-8");
+  return bytes <= OFFLINE_SYNC_SNAPSHOT_BASE_MAX_BODY_BYTES && bytes <= OFFLINE_SYNC_SNAPSHOT_BASE_POST_PREFERRED_MAX_BODY_BYTES;
+}
+var OFFLINE_SYNC_SNAPSHOT_BASE_POST_MAX_FILES = 5e4;
+function offlineSnapshotBasePostRequest(body) {
+  const bytes = Buffer.byteLength(body, "utf-8");
+  if (bytes > OFFLINE_SYNC_SNAPSHOT_BASE_MAX_BODY_BYTES) return null;
+  if (bytes <= OFFLINE_SYNC_SNAPSHOT_BASE_POST_PREFERRED_MAX_BODY_BYTES) {
+    return { body };
+  }
+  const compressed = gzipSync(body);
+  if (compressed.byteLength > OFFLINE_SYNC_SNAPSHOT_BASE_POST_PREFERRED_MAX_BODY_BYTES) {
+    return null;
+  }
+  return {
+    body: compressed,
+    headers: {
+      "content-encoding": "gzip"
+    }
+  };
+}
+function isOfflineSnapshotPostFallbackError(error, options = {}) {
+  const message = error instanceof Error ? error.message : String(error);
+  if (/offline-sync\/snapshot\b.* returned (404|405|413)\b/.test(message)) return true;
+  if (!options.compressed) return false;
+  return /offline-sync\/snapshot\b.* returned (400|415)\b/.test(message) && /\b(unsupported_content_encoding|invalid_gzip_body|invalid_json)\b/.test(message);
 }
-function isOfflineSnapshotPostFallbackError(error) {
+function isOfflineSnapshotStreamFallbackError(error) {
   const message = error instanceof Error ? error.message : String(error);
-  return /offline-sync\/snapshot\b.* returned (404|405|413)\b/.test(message);
+  return /offline-sync\/snapshot-stream\b.* returned (404|405)\b/.test(message);
 }
 async function fetchOfflineFiles(args) {
   return fetchOfflineJson(
@@ -11273,6 +11401,8 @@ export {
   OFFLINE_SYNC_DIRECT_PUSH_MIN_BYTES,
   OFFLINE_SYNC_FILE_CONTENT_UPLOAD_CHUNK_BYTES,
   OFFLINE_SYNC_REQUEST_TIMEOUT_DEFAULT_MS,
+  OFFLINE_SYNC_SNAPSHOT_BASE_POST_MAX_FILES,
+  OFFLINE_SYNC_SNAPSHOT_BASE_POST_PREFERRED_MAX_BODY_BYTES,
   TAXONOMY_RESOLVE_BOOLEAN_FLAGS,
   TAXONOMY_RESOLVE_VALUE_FLAGS,
   __benchDatasetTestHooks,
@@ -11284,6 +11414,7 @@ export {
   chunkOfflineFileContentBatches,
   directHydrateLargeOfflineFiles,
   extractXrayRawArgs,
+  fetchOfflineSnapshot,
   formatOfflineLargeFilePushFailureMessage,
   formatOfflineRequestForError,
   getBenchUsageText,
@@ -11293,6 +11424,7 @@ export {
   offlinePartialHydrationForPaths,
   offlineSnapshotBasePostBody,
   offlineSnapshotBasePostBodyFits,
+  offlineSnapshotBasePostRequest,
   offlineSnapshotContentFilesForApply,
   parseBenchArgs,
   parseCapsuleForkArgs,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remnic/cli",
-  "version": "1.0.24",
+  "version": "1.0.25",
   "description": "CLI for Remnic memory — init, query, doctor, daemon management",
   "type": "module",
   "main": "dist/index.js",
@@ -26,9 +26,9 @@
   },
   "dependencies": {
     "yaml": "^2.4.2",
+    "@remnic/plugin-pi": "^1.0.9",
     "@remnic/server": "^1.0.5",
-    "@remnic/plugin-pi": "^1.0.10",
-    "@remnic/core": "^1.1.30"
+    "@remnic/core": "^1.1.31"
   },
   "peerDependencies": {
     "@remnic/bench": "^1.0.0",
@@ -80,8 +80,8 @@
     "@remnic/import-claude": "0.1.0",
     "@remnic/import-gemini": "0.1.0",
     "@remnic/import-lossless-claw": "0.1.1",
-    "@remnic/import-mem0": "0.1.0",
-    "@remnic/import-supermemory": "0.1.2"
+    "@remnic/import-supermemory": "0.1.2",
+    "@remnic/import-mem0": "0.1.0"
   },
   "license": "MIT",
   "repository": {