@remnic/cli 9.54.4 → 9.54.5

package/dist/index.js

@@ -299,13 +299,14 @@ import {
 } from "@remnic/core/reconcile/manifest.js";
 
 // src/offline-storage-io.ts
-import { mkdtemp, readdir, lstat, rm } from "fs/promises";
+import { createDecipheriv, createHash } from "crypto";
 import fs3 from "fs";
+import { lstat, mkdtemp, readdir, rm } from "fs/promises";
 import path from "path";
-import { createHash, createDecipheriv } from "crypto";
 import {
   OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES,
-  StorageManager
+  StorageManager,
+  createSupportPassportPrivateFileExclusion
 } from "@remnic/core";
 import { OFFLINE_DECRYPT_STAGING_DIR_PREFIX } from "@remnic/core/offline-sync-exclude-globs";
 import {
@@ -326,6 +327,28 @@ import {
   readHeader,
   secureStoreDir
 } from "@remnic/core/secure-store";
+var OFFLINE_SYNC_EXCLUSION_CONCURRENCY = 16;
+function resolveOfflineDirectHydrationPath(memoryDir, relPath) {
+  const base = path.resolve(memoryDir);
+  const target = path.resolve(base, relPath);
+  const relative = path.relative(base, target);
+  if (relative === "" || relative === ".." || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
+    throw new Error(`offline sync direct hydration path escapes memory dir: ${relPath}`);
+  }
+  return target;
+}
+async function filterOfflineSyncBaseFiles(memoryDir, files, excludeFile) {
+  const excluded = new Array(files.length);
+  for (let offset = 0; offset < files.length; offset += OFFLINE_SYNC_EXCLUSION_CONCURRENCY) {
+    await Promise.all(
+      files.slice(offset, offset + OFFLINE_SYNC_EXCLUSION_CONCURRENCY).map(async (file, index) => {
+        const filePath = resolveOfflineDirectHydrationPath(memoryDir, file.path);
+        excluded[offset + index] = await excludeFile({ root: memoryDir, path: file.path, filePath });
+      })
+    );
+  }
+  return files.filter((_file, index) => excluded[index] === false);
+}
 async function createConfiguredOfflineStorage(memoryDir, secureStoreEncryptOnWrite = true) {
   const storage = new StorageManager(memoryDir);
   const header = await readHeader(memoryDir);
@@ -365,6 +388,7 @@ async function createOfflineStorageIo(memoryDir, configuredStorage) {
   await cleanupOrphanedOfflineDecryptStaging(memoryDir);
   const { storage, secureStoreKey } = configuredStorage ?? await createConfiguredOfflineStorage(memoryDir);
   return {
+    excludeFile: createSupportPassportPrivateFileExclusion(storage),
     readFile: async ({ filePath }) => storage.readOfflineSyncFile(filePath),
     readDeletionRevisions: () => storage.readDeletionRevisions(),
     readFileDigest: async ({ filePath }) => {
@@ -471,15 +495,9 @@ async function* readEncryptedOfflineFileChunks(options) {
   if (envelopeVersion !== ENVELOPE_VERSION) {
     throw new Error(`secure-store envelope has unsupported version ${envelopeVersion}: ${options.filePath}`);
   }
-  const salt = envelopeHeader.subarray(
-    ENVELOPE_LAYOUT.salt,
-    ENVELOPE_LAYOUT.salt + ENVELOPE_SALT_LENGTH
-  );
+  const salt = envelopeHeader.subarray(ENVELOPE_LAYOUT.salt, ENVELOPE_LAYOUT.salt + ENVELOPE_SALT_LENGTH);
   const iv = envelopeHeader.subarray(ENVELOPE_LAYOUT.iv, ENVELOPE_LAYOUT.iv + IV_LENGTH);
-  const authTag = envelopeHeader.subarray(
-    ENVELOPE_LAYOUT.authTag,
-    ENVELOPE_LAYOUT.authTag + AUTH_TAG_LENGTH
-  );
+  const authTag = envelopeHeader.subarray(ENVELOPE_LAYOUT.authTag, ENVELOPE_LAYOUT.authTag + AUTH_TAG_LENGTH);
   const aadCandidates = offlineFileAadCandidates(options.filePath, options.memoryDir);
   let lastError;
   for (const aad of aadCandidates) {
@@ -498,9 +516,7 @@ async function* readEncryptedOfflineFileChunks(options) {
           highWaterMark: options.chunkSize
         });
         for await (const encryptedChunk of stream) {
-          const plain = decipher.update(
-            Buffer.isBuffer(encryptedChunk) ? encryptedChunk : Buffer.from(encryptedChunk)
-          );
+          const plain = decipher.update(Buffer.isBuffer(encryptedChunk) ? encryptedChunk : Buffer.from(encryptedChunk));
           if (plain.length > 0 && !output.write(plain)) {
             await new Promise((resolve2, reject) => {
               output.once("drain", resolve2);
@@ -1194,10 +1210,14 @@ async function computeConvergePlan(options = {}) {
     for (const rootInfo of roots) {
       const ns = rootInfo.namespace;
       namespacesToPlan.add(ns);
+      const io = await createOfflineStorageIo(rootInfo.rootDir);
       const snapshot = await buildOfflineSyncSnapshotFromBase({
         root: rootInfo.rootDir,
         sourceId: "local",
-        includeContent: false
+        includeContent: false,
+        readFile: io.readFile,
+        readFileDigest: io.readFileDigest,
+        excludeFile: io.excludeFile
       });
       const files = snapshot.files.filter((record2) => !isInternalRemnicStatePath3(record2.path)).map((record2) => ({
         path: record2.path,
@@ -1207,7 +1227,6 @@ async function computeConvergePlan(options = {}) {
       }));
       localMap.set(ns, files);
       const evidence = await readLocalTombstoneEvidence(rootInfo.rootDir);
-      const io = await createOfflineStorageIo(rootInfo.rootDir);
       let manifestReadFailed = false;
       const manifest = await buildReconcileManifest({
         files,
@@ -11747,15 +11766,6 @@ function offlineDirectPushFiles(options) {
     return current.sha256 !== base.get(current.path)?.sha256;
   }).sort((left, right) => right.bytes - left.bytes || left.path.localeCompare(right.path));
 }
-function resolveOfflineDirectHydrationPath(memoryDir, relPath) {
-  const base = path17.resolve(memoryDir);
-  const target = path17.resolve(base, relPath);
-  const relative = path17.relative(base, target);
-  if (relative === "" || relative === ".." || relative.startsWith(`..${path17.sep}`) || path17.isAbsolute(relative)) {
-    throw new Error(`offline sync direct hydration path escapes memory dir: ${relPath}`);
-  }
-  return target;
-}
 var OFFLINE_SYNC_FILE_CONTENT_UPLOAD_CHUNK_BYTES = OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES4;
 async function pushOfflineFileContent(args) {
   if (args.readFileChunks) {
@@ -12355,6 +12365,7 @@ async function runOfflineSyncOnce(options) {
     options.secureStoreEncryptOnWrite
   );
   const storageIo = await createOfflineStorageIo(options.memoryDir, offlineStorage);
+  const syncBaseFiles = await filterOfflineSyncBaseFiles(options.memoryDir, baseFiles, storageIo.excludeFile);
   const localSourceId = localOfflineSourceId(options.memoryDir);
   await drainOfflineSyncImpressions(options.memoryDir, options);
   await drainPendingLifecycleForOfflineSync(
@@ -12369,21 +12380,22 @@ async function runOfflineSyncOnce(options) {
   const currentSnapshotForPush = await buildOfflineSyncSnapshotFromBase2({
     root: options.memoryDir,
     sourceId: localSourceId,
-    baseFiles,
+    baseFiles: syncBaseFiles,
     baseCapturedAt,
     includeContent: false,
     includeTranscripts: options.includeTranscripts,
     readFile: storageIo.readFile,
     readFileDigest: storageIo.readFileDigest,
+    excludeFile: storageIo.excludeFile,
     userExcludeRegexps: options.userExcludeRegexps
   });
   const pendingSummary = summarizeOfflineSyncPendingFiles({
-    baseFiles,
+    baseFiles: syncBaseFiles,
     currentFiles: currentSnapshotForPush.files,
     includeTranscripts: options.includeTranscripts,
     userExcludeRegexps: options.userExcludeRegexps
   });
-  const baseByPath = offlineFileStateMap(baseFiles);
+  const baseByPath = offlineFileStateMap(syncBaseFiles);
   let directPushAppliedUpserts = 0;
   let directPushSkipped = 0;
   let directPushNamespace;
@@ -12393,7 +12405,7 @@ async function runOfflineSyncOnce(options) {
   const directPushFailures = [];
   for (const file of offlineDirectPushFiles({
     currentFiles: currentSnapshotForPush.files,
-    baseFiles
+    baseFiles: syncBaseFiles
   })) {
     if (options.skipLargeFilePaths?.has(file.path)) {
       continue;
@@ -12471,7 +12483,7 @@ async function runOfflineSyncOnce(options) {
     const resolvedNamespace2 = partial?.resolvedNamespace ?? resolvedOfflineSnapshotNamespace({ namespace: pushed?.namespace ?? "" }, syncNamespace);
     const stateWritePaths2 = stateWritePathsFor(resolvedNamespace2);
     const nextBaseFiles = advanceOfflineBaseFilesForSuccessfulPush({
-      baseFiles,
+      baseFiles: syncBaseFiles,
       currentFiles: currentSnapshotForPush.files,
       directPushedPaths: [...directPushedPaths],
       hydratedFiles: partial?.hydratedFiles,
@@ -12518,12 +12530,13 @@ async function runOfflineSyncOnce(options) {
   let currentSnapshotForChangeset = directPushedPaths.size > 0 ? await buildOfflineSyncSnapshotFromBase2({
     root: options.memoryDir,
     sourceId: localSourceId,
-    baseFiles,
+    baseFiles: syncBaseFiles,
     baseCapturedAt,
     includeContent: false,
     includeTranscripts: options.includeTranscripts,
     readFile: storageIo.readFile,
     readFileDigest: storageIo.readFileDigest,
+    excludeFile: storageIo.excludeFile,
     userExcludeRegexps: options.userExcludeRegexps
   }) : currentSnapshotForPush;
   let changesetRetryCount = 0;
@@ -12533,7 +12546,7 @@ async function runOfflineSyncOnce(options) {
         root: options.memoryDir,
         sourceId: localSourceId,
         currentFiles: currentSnapshotForChangeset.files,
-        baseFiles,
+        baseFiles: syncBaseFiles,
         // 3-strikes skipped large files must be excluded here too — the
         // direct-push loop skips them, but without this line the changeset
         // path would still try to upsert them inline (Cursor review, PR
@@ -12571,12 +12584,13 @@ async function runOfflineSyncOnce(options) {
       currentSnapshotForChangeset = await buildOfflineSyncSnapshotFromBase2({
         root: options.memoryDir,
         sourceId: localSourceId,
-        baseFiles,
+        baseFiles: syncBaseFiles,
         baseCapturedAt,
         includeContent: false,
         includeTranscripts: options.includeTranscripts,
         readFile: storageIo.readFile,
         readFileDigest: storageIo.readFileDigest,
+        excludeFile: storageIo.excludeFile,
         userExcludeRegexps: options.userExcludeRegexps
       });
     }
@@ -12614,7 +12628,7 @@ async function runOfflineSyncOnce(options) {
       namespace: syncNamespace,
       includeTranscripts: options.includeTranscripts,
       includeContent: false,
-      baseFiles,
+      baseFiles: syncBaseFiles,
       baseCapturedAt
     });
   } catch (error) {
@@ -12626,12 +12640,13 @@ async function runOfflineSyncOnce(options) {
     currentSnapshot = await buildOfflineSyncSnapshotFromBase2({
       root: options.memoryDir,
       sourceId: localSourceId,
-      baseFiles,
+      baseFiles: syncBaseFiles,
       baseCapturedAt,
       includeContent: false,
       includeTranscripts: options.includeTranscripts,
       readFile: storageIo.readFile,
-      readFileDigest: storageIo.readFileDigest
+      readFileDigest: storageIo.readFileDigest,
+      excludeFile: storageIo.excludeFile
     });
   } catch (error) {
     if (pushed) return writePartialPushState(error);
@@ -12649,7 +12664,7 @@ async function runOfflineSyncOnce(options) {
       namespace: syncNamespace,
       includeTranscripts: options.includeTranscripts,
       snapshot: remoteSnapshotMetadata,
-      baseFiles,
+      baseFiles: syncBaseFiles,
       currentFiles: currentSnapshot.files,
       memoryDir: options.memoryDir,
       readFile: storageIo.readFile,
@@ -12706,7 +12721,7 @@ async function runOfflineSyncOnce(options) {
       namespace: syncNamespace,
       includeTranscripts: options.includeTranscripts,
       snapshot: remoteSnapshotMetadata,
-      baseFiles,
+      baseFiles: syncBaseFiles,
       currentFiles: applyCurrentSnapshot.files,
       deferredPaths: [...remoteDeferredPaths],
       missingContentDeferredPaths: remoteDeferredPaths
@@ -12724,7 +12739,7 @@ async function runOfflineSyncOnce(options) {
     pull = await applyOfflineSyncSnapshot({
       root: options.memoryDir,
       snapshot: remoteSnapshot,
-      baseFiles,
+      baseFiles: syncBaseFiles,
       currentFiles: latestApplySnapshot.files,
       deferredPaths: [...remoteDeferredPaths],
       allowMissingConflictContent: true,
@@ -12752,7 +12767,7 @@ async function runOfflineSyncOnce(options) {
         namespace: syncNamespace,
         includeTranscripts: options.includeTranscripts,
         snapshot: remoteSnapshotMetadata,
-        baseFiles,
+        baseFiles: syncBaseFiles,
         currentFiles: applyCurrentSnapshot.files,
         deferredPaths: [...remoteDeferredPaths],
         missingContentDeferredPaths: remoteDeferredPaths
@@ -12771,7 +12786,7 @@ async function runOfflineSyncOnce(options) {
       pull = await applyOfflineSyncSnapshot({
         root: options.memoryDir,
         snapshot: retrySnapshot,
-        baseFiles,
+        baseFiles: syncBaseFiles,
         currentFiles: latestRetryApplySnapshot.files,
         deferredPaths: [...remoteDeferredPaths],
         allowMissingConflictContent: true,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remnic/cli",
-  "version": "9.54.4",
+  "version": "9.54.5",
   "description": "CLI for Remnic memory — init, query, doctor, daemon management",
   "type": "module",
   "main": "dist/index.js",
@@ -26,25 +26,25 @@
   },
   "dependencies": {
     "yaml": "^2.4.2",
-    "@remnic/plugin-pi": "^9.54.4",
-    "@remnic/core": "^9.54.4",
-    "@remnic/server": "^9.54.4"
+    "@remnic/plugin-pi": "^9.54.5",
+    "@remnic/server": "^9.54.5",
+    "@remnic/core": "^9.54.5"
   },
   "peerDependencies": {
-    "@remnic/bench": "^9.54.4",
-    "@remnic/plugin-openclaw": "^9.54.4",
-    "@remnic/export-weclone": "^9.54.4",
-    "@remnic/import-weclone": "^9.54.4",
-    "@remnic/import-chatgpt": "^9.54.4",
-    "@remnic/import-claude": "^9.54.4",
-    "@remnic/import-gemini": "^9.54.4",
-    "@remnic/import-lossless-claw": "^9.54.4",
-    "@remnic/import-mem0": "^9.54.4",
-    "@remnic/import-supermemory": "^9.54.4",
-    "@remnic/connector-limitless": "^9.54.4",
-    "@remnic/connector-bee": "^9.54.4",
-    "@remnic/connector-omi": "^9.54.4",
-    "@remnic/capture-audio": "^9.54.4"
+    "@remnic/bench": "^9.54.5",
+    "@remnic/plugin-openclaw": "^9.54.5",
+    "@remnic/export-weclone": "^9.54.5",
+    "@remnic/import-weclone": "^9.54.5",
+    "@remnic/import-chatgpt": "^9.54.5",
+    "@remnic/import-claude": "^9.54.5",
+    "@remnic/import-gemini": "^9.54.5",
+    "@remnic/import-lossless-claw": "^9.54.5",
+    "@remnic/import-mem0": "^9.54.5",
+    "@remnic/import-supermemory": "^9.54.5",
+    "@remnic/connector-limitless": "^9.54.5",
+    "@remnic/connector-bee": "^9.54.5",
+    "@remnic/connector-omi": "^9.54.5",
+    "@remnic/capture-audio": "^9.54.5"
   },
   "peerDependenciesMeta": {
     "@remnic/bench": {
@@ -93,19 +93,19 @@
   "devDependencies": {
     "tsup": "^8.5.1",
     "typescript": "^5.9.3",
-    "@remnic/bench": "9.54.4",
-    "@remnic/plugin-openclaw": "9.54.4",
-    "@remnic/export-weclone": "9.54.4",
-    "@remnic/import-weclone": "9.54.4",
-    "@remnic/import-chatgpt": "9.54.4",
-    "@remnic/import-lossless-claw": "9.54.4",
-    "@remnic/import-gemini": "9.54.4",
-    "@remnic/import-mem0": "9.54.4",
-    "@remnic/import-supermemory": "9.54.4",
-    "@remnic/connector-limitless": "9.54.4",
-    "@remnic/import-claude": "9.54.4",
-    "@remnic/connector-bee": "9.54.4",
-    "@remnic/connector-omi": "9.54.4"
+    "@remnic/bench": "9.54.5",
+    "@remnic/plugin-openclaw": "9.54.5",
+    "@remnic/export-weclone": "9.54.5",
+    "@remnic/import-weclone": "9.54.5",
+    "@remnic/import-chatgpt": "9.54.5",
+    "@remnic/import-gemini": "9.54.5",
+    "@remnic/import-lossless-claw": "9.54.5",
+    "@remnic/import-mem0": "9.54.5",
+    "@remnic/import-claude": "9.54.5",
+    "@remnic/import-supermemory": "9.54.5",
+    "@remnic/connector-limitless": "9.54.5",
+    "@remnic/connector-bee": "9.54.5",
+    "@remnic/connector-omi": "9.54.5"
   },
   "license": "MIT",
   "repository": {