@remnic/cli 1.0.10 → 1.0.12

package/dist/index.js

@@ -88,10 +88,12 @@ import {
   formatProcedureStatsText,
   parseXrayCliOptions,
   renderXray,
+  OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES,
   applyOfflineSyncSnapshot,
   buildOfflineSyncChangeset,
   buildOfflineSyncSnapshot,
   defaultOfflineSyncStatePath,
+  normalizeOfflineSyncSnapshot,
   offlineSyncStateFromSnapshot,
   readOfflineSyncState,
   summarizeOfflineSyncChangeset,
@@ -6667,6 +6669,63 @@ async function fetchOfflineFiles(args) {
     }
   );
 }
+var OFFLINE_SYNC_DIRECT_HYDRATE_MIN_BYTES = 16 * 1024 * 1024;
+function parseOfflineHeaderNumber(headers, name) {
+  const raw = headers.get(name);
+  if (raw === null) throw new Error(`offline file content response omitted ${name}`);
+  const parsed = Number(raw);
+  if (!Number.isFinite(parsed) || parsed < 0) {
+    throw new Error(`offline file content response had invalid ${name}: ${raw}`);
+  }
+  return parsed;
+}
+async function fetchOfflineFileContentChunk(args) {
+  const response = await fetch(
+    offlineEndpoint(args.remoteUrl, "/remnic/v1/offline-sync/file-content"),
+    {
+      method: "POST",
+      headers: {
+        authorization: `Bearer ${args.token}`,
+        "content-type": "application/json"
+      },
+      body: JSON.stringify({
+        namespace: args.namespace,
+        includeTranscripts: args.includeTranscripts,
+        path: args.path,
+        offset: args.offset,
+        length: args.length
+      })
+    }
+  );
+  if (!response.ok) {
+    let detail = "";
+    try {
+      detail = await response.text();
+    } catch {
+      detail = "";
+    }
+    throw new Error(
+      `offline sync file-content request failed: ${response.status} ${response.statusText}${detail ? ` - ${detail.slice(0, 500)}` : ""}`
+    );
+  }
+  const encodedPath = response.headers.get("x-remnic-file-path");
+  const relPath = encodedPath ? decodeURIComponent(encodedPath) : args.path;
+  const content = Buffer.from(await response.arrayBuffer());
+  const chunkBytes = parseOfflineHeaderNumber(response.headers, "x-remnic-chunk-bytes");
+  const sha256 = response.headers.get("x-remnic-file-sha256") ?? void 0;
+  if (content.length !== chunkBytes) {
+    throw new Error(`offline file content response length mismatch for ${relPath}`);
+  }
+  return {
+    path: relPath,
+    ...sha256 ? { sha256 } : {},
+    bytes: parseOfflineHeaderNumber(response.headers, "x-remnic-file-bytes"),
+    mtimeMs: parseOfflineHeaderNumber(response.headers, "x-remnic-file-mtime-ms"),
+    offset: parseOfflineHeaderNumber(response.headers, "x-remnic-chunk-offset"),
+    chunkBytes,
+    content
+  };
+}
 function resolvedOfflineSnapshotNamespace(snapshot, requestedNamespace) {
   const resolved = typeof snapshot.namespace === "string" && snapshot.namespace.trim().length > 0 ? snapshot.namespace.trim() : void 0;
   return resolved ?? requestedNamespace;
@@ -6713,6 +6772,86 @@ function offlineSnapshotContentPathsForApply(options) {
   }
   return paths.sort();
 }
+function shouldDirectHydrateOfflineFile(options) {
+  if (options.incoming.bytes < OFFLINE_SYNC_DIRECT_HYDRATE_MIN_BYTES) return false;
+  if (options.current?.sha256 === options.incoming.sha256) return false;
+  if (options.current && options.base && options.current.sha256 === options.base.sha256) {
+    return true;
+  }
+  return !options.current && !options.base;
+}
+function resolveOfflineDirectHydrationPath(memoryDir, relPath) {
+  const base = path11.resolve(memoryDir);
+  const target = path11.resolve(base, relPath);
+  const relative = path11.relative(base, target);
+  if (relative === "" || relative === ".." || relative.startsWith(`..${path11.sep}`) || path11.isAbsolute(relative)) {
+    throw new Error(`offline sync direct hydration path escapes memory dir: ${relPath}`);
+  }
+  return target;
+}
+async function fetchOfflineFileContent(args) {
+  const chunks = [];
+  const hash = createHash("sha256");
+  let offset = 0;
+  while (offset < args.expected.bytes) {
+    const chunk = await fetchOfflineFileContentChunk({
+      remoteUrl: args.remoteUrl,
+      token: args.token,
+      namespace: args.namespace,
+      includeTranscripts: args.includeTranscripts,
+      path: args.expected.path,
+      offset,
+      length: Math.min(
+        OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES,
+        args.expected.bytes - offset
+      )
+    });
+    if (chunk.path !== args.expected.path || chunk.sha256 !== void 0 && chunk.sha256 !== args.expected.sha256 || chunk.bytes !== args.expected.bytes || chunk.mtimeMs !== args.expected.mtimeMs || chunk.offset !== offset || chunk.chunkBytes !== chunk.content.length) {
+      throw new Error(`remote file changed while fetching offline content: ${args.expected.path}`);
+    }
+    if (chunk.chunkBytes === 0) {
+      throw new Error(`remote offline content chunk was empty before EOF: ${args.expected.path}`);
+    }
+    chunks.push(chunk.content);
+    hash.update(chunk.content);
+    offset += chunk.chunkBytes;
+  }
+  const content = Buffer.concat(chunks, offset);
+  const digest = hash.digest("hex");
+  if (digest !== args.expected.sha256 || content.length !== args.expected.bytes) {
+    throw new Error(`remote offline content checksum mismatch for ${args.expected.path}`);
+  }
+  return content;
+}
+async function directHydrateLargeOfflineFiles(args) {
+  if (!args.writeFile) return /* @__PURE__ */ new Set();
+  const snapshot = normalizeOfflineSyncSnapshot(args.snapshot);
+  const base = offlineFileStateMap(args.baseFiles);
+  const current = offlineFileStateMap(args.currentFiles);
+  const hydrated = /* @__PURE__ */ new Set();
+  const candidates = snapshot.files.filter((incoming) => shouldDirectHydrateOfflineFile({
+    incoming,
+    base: base.get(incoming.path),
+    current: current.get(incoming.path)
+  })).sort((left, right) => right.bytes - left.bytes || left.path.localeCompare(right.path));
+  for (const incoming of candidates) {
+    const content = await fetchOfflineFileContent({
+      remoteUrl: args.remoteUrl,
+      token: args.token,
+      namespace: args.namespace,
+      includeTranscripts: args.includeTranscripts,
+      expected: incoming
+    });
+    await args.writeFile({
+      root: args.memoryDir,
+      path: incoming.path,
+      filePath: resolveOfflineDirectHydrationPath(args.memoryDir, incoming.path),
+      content
+    });
+    hydrated.add(incoming.path);
+  }
+  return hydrated;
+}
 function chunkOfflineFilePaths(paths) {
   const chunks = [];
   let current = [];
@@ -6910,6 +7049,24 @@ async function runOfflineSyncOnce(options) {
     includeTranscripts: options.includeTranscripts,
     readFile: storageIo.readFile
   });
+  const directHydratedPaths = await directHydrateLargeOfflineFiles({
+    remoteUrl: options.remoteUrl,
+    token: options.token,
+    namespace: syncNamespace,
+    includeTranscripts: options.includeTranscripts,
+    snapshot: remoteSnapshotMetadata,
+    baseFiles,
+    currentFiles: currentSnapshot.files,
+    memoryDir: options.memoryDir,
+    writeFile: storageIo.writeFile
+  });
+  const applyCurrentSnapshot = directHydratedPaths.size > 0 ? await buildOfflineSyncSnapshot({
+    root: options.memoryDir,
+    sourceId: localOfflineSourceId(options.memoryDir),
+    includeContent: false,
+    includeTranscripts: options.includeTranscripts,
+    readFile: storageIo.readFile
+  }) : currentSnapshot;
   const remoteSnapshot = await hydrateOfflineSnapshotContent({
     remoteUrl: options.remoteUrl,
     token: options.token,
@@ -6917,7 +7074,7 @@ async function runOfflineSyncOnce(options) {
     includeTranscripts: options.includeTranscripts,
     snapshot: remoteSnapshotMetadata,
     baseFiles,
-    currentFiles: currentSnapshot.files
+    currentFiles: applyCurrentSnapshot.files
   });
   const resolvedNamespace = resolvedOfflineSnapshotNamespace(remoteSnapshot, syncNamespace);
   let pull;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remnic/cli",
-  "version": "1.0.10",
+  "version": "1.0.12",
   "description": "CLI for Remnic memory — init, query, doctor, daemon management",
   "type": "module",
   "main": "dist/index.js",
@@ -26,9 +26,9 @@
   },
   "dependencies": {
     "yaml": "^2.4.2",
-    "@remnic/plugin-pi": "^1.0.0",
+    "@remnic/core": "^1.1.18",
     "@remnic/server": "^1.0.5",
-    "@remnic/core": "^1.1.16"
+    "@remnic/plugin-pi": "^1.0.0"
   },
   "peerDependencies": {
     "@remnic/bench": "^1.0.0",
@@ -73,12 +73,12 @@
   "devDependencies": {
     "tsup": "^8.5.1",
     "typescript": "^5.9.3",
-    "@remnic/bench": "1.0.1",
     "@remnic/export-weclone": "1.0.1",
-    "@remnic/import-chatgpt": "0.1.0",
+    "@remnic/bench": "1.0.1",
     "@remnic/import-weclone": "1.0.1",
-    "@remnic/import-claude": "0.1.0",
+    "@remnic/import-chatgpt": "0.1.0",
     "@remnic/import-gemini": "0.1.0",
+    "@remnic/import-claude": "0.1.0",
     "@remnic/import-lossless-claw": "0.1.1",
     "@remnic/import-mem0": "0.1.0",
     "@remnic/import-supermemory": "0.1.2"