@remnic/cli 1.0.15 → 1.0.17

package/dist/index.js

@@ -2,7 +2,7 @@
 import fs7 from "fs";
 import os from "os";
 import path11 from "path";
-import { createHash } from "crypto";
+import { createDecipheriv, createHash } from "crypto";
 import * as childProcess2 from "child_process";
 import { fileURLToPath as fileURLToPath4 } from "url";
 import {
@@ -95,8 +95,8 @@ import {
   defaultOfflineSyncStatePath,
   normalizeOfflineSyncSnapshot,
   offlineSyncStateFromSnapshot,
+  readOfflineSyncFileContentChunk,
   readOfflineSyncState,
-  summarizeOfflineSyncChangeset,
   summarizeOfflineSyncPendingChanges,
   writeOfflineSyncState,
   buildActionConfidenceInputFromOptions,
@@ -106,7 +106,24 @@ import {
   forkCapsule,
   readForkLineage
 } from "@remnic/core";
-import { keyring, readHeader, secureStoreDir } from "@remnic/core/secure-store";
+import {
+  AUTH_TAG_LENGTH,
+  ENVELOPE_HEADER_SIZE,
+  ENVELOPE_LAYOUT,
+  ENVELOPE_SALT_LENGTH,
+  ENVELOPE_VERSION,
+  FILE_FORMAT_FLAGS,
+  FILE_FORMAT_VERSION,
+  IV_LENGTH,
+  MAGIC_BYTES,
+  MAGIC_HEADER_SIZE,
+  SecureStoreLockedError,
+  filePathAad,
+  isEncryptedFile,
+  keyring,
+  readHeader,
+  secureStoreDir
+} from "@remnic/core/secure-store";
 
 // src/optional-module-loader.ts
 function isSpecifierNotFoundError(err, specifier) {
@@ -6728,6 +6745,41 @@ async function fetchOfflineFileContentChunk(args) {
     content
   };
 }
+async function postOfflineFileContentChunk(args) {
+  const response = await fetch(
+    offlineEndpoint(args.remoteUrl, "/remnic/v1/offline-sync/apply-file-content", {
+      namespace: args.namespace
+    }),
+    {
+      method: "POST",
+      headers: {
+        authorization: `Bearer ${args.token}`,
+        "content-type": "application/octet-stream",
+        "x-remnic-include-transcripts": args.includeTranscripts ? "true" : "false",
+        "x-remnic-source-id": encodeURIComponent(args.sourceId),
+        "x-remnic-file-path": encodeURIComponent(args.file.path),
+        "x-remnic-file-sha256": args.file.sha256,
+        "x-remnic-file-bytes": String(args.file.bytes),
+        "x-remnic-file-mtime-ms": String(args.file.mtimeMs),
+        "x-remnic-chunk-offset": String(args.offset),
+        ...args.baseSha256 ? { "x-remnic-base-sha256": args.baseSha256 } : {}
+      },
+      body: new Blob([new Uint8Array(args.content)])
+    }
+  );
+  if (!response.ok) {
+    let detail = "";
+    try {
+      detail = await response.text();
+    } catch {
+      detail = "";
+    }
+    throw new Error(
+      `offline sync apply-file-content request failed: ${response.status} ${response.statusText}${detail ? ` - ${detail.slice(0, 500)}` : ""}`
+    );
+  }
+  return await response.json();
+}
 function resolvedOfflineSnapshotNamespace(snapshot, requestedNamespace) {
   const resolved = typeof snapshot.namespace === "string" && snapshot.namespace.trim().length > 0 ? snapshot.namespace.trim() : void 0;
   return resolved ?? requestedNamespace;
@@ -6782,6 +6834,13 @@ function shouldDirectHydrateOfflineFile(options) {
   }
   return !options.current && !options.base;
 }
+function offlineDirectPushFiles(options) {
+  const base = offlineFileStateMap(options.baseFiles);
+  return options.currentFiles.filter((current) => {
+    if (current.bytes < OFFLINE_SYNC_DIRECT_HYDRATE_MIN_BYTES) return false;
+    return current.sha256 !== base.get(current.path)?.sha256;
+  }).sort((left, right) => right.bytes - left.bytes || left.path.localeCompare(right.path));
+}
 function resolveOfflineDirectHydrationPath(memoryDir, relPath) {
   const base = path11.resolve(memoryDir);
   const target = path11.resolve(base, relPath);
@@ -6791,6 +6850,133 @@ function resolveOfflineDirectHydrationPath(memoryDir, relPath) {
   }
   return target;
 }
+async function pushOfflineFileContent(args) {
+  if (args.readFileChunks) {
+    return pushOfflineFileContentFromChunkReader(args);
+  }
+  let offset = 0;
+  let finalResult = null;
+  while (offset < args.file.bytes || args.file.bytes === 0 && offset === 0) {
+    const chunk = await readOfflineSyncFileContentChunk({
+      root: args.memoryDir,
+      path: args.file.path,
+      offset,
+      length: Math.min(
+        OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES,
+        Math.max(1, args.file.bytes - offset)
+      ),
+      includeTranscripts: args.includeTranscripts,
+      readFile: args.readFile
+    });
+    if (chunk.path !== args.file.path || chunk.sha256 !== void 0 && chunk.sha256 !== args.file.sha256 || chunk.bytes !== args.file.bytes || chunk.mtimeMs !== args.file.mtimeMs || chunk.offset !== offset) {
+      throw new Error(`local file changed while pushing offline content: ${args.file.path}`);
+    }
+    if (chunk.chunkBytes === 0 && args.file.bytes > 0) {
+      throw new Error(`local offline content chunk was empty before EOF: ${args.file.path}`);
+    }
+    finalResult = await postOfflineFileContentChunk({
+      remoteUrl: args.remoteUrl,
+      token: args.token,
+      namespace: args.namespace,
+      includeTranscripts: args.includeTranscripts,
+      sourceId: args.sourceId,
+      file: args.file,
+      baseSha256: args.baseSha256,
+      offset,
+      content: chunk.content
+    });
+    if (finalResult.conflict) {
+      return finalResult;
+    }
+    offset += chunk.chunkBytes;
+    if (args.file.bytes === 0) break;
+  }
+  if (!finalResult?.done) {
+    throw new Error(`offline sync large-file push did not finish for ${args.file.path}`);
+  }
+  return finalResult;
+}
+async function pushOfflineFileContentFromChunkReader(args) {
+  const filePath = resolveOfflineDirectHydrationPath(args.memoryDir, args.file.path);
+  const stat = fs7.statSync(filePath);
+  if (stat.mtimeMs !== args.file.mtimeMs) {
+    throw new Error(`local file changed while pushing offline content: ${args.file.path}`);
+  }
+  const hash = createHash("sha256");
+  const chunks = args.readFileChunks({
+    root: path11.resolve(args.memoryDir),
+    path: args.file.path,
+    filePath,
+    chunkSize: OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES
+  });
+  let offset = 0;
+  let pending = null;
+  let finalResult = null;
+  for await (const rawChunk of chunks) {
+    const chunk = Buffer.from(rawChunk);
+    if (chunk.length === 0) continue;
+    if (chunk.length > OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES) {
+      throw new Error(`local offline content chunk exceeds max size: ${args.file.path}`);
+    }
+    if (pending) {
+      hash.update(pending);
+      finalResult = await postOfflineFileContentChunk({
+        remoteUrl: args.remoteUrl,
+        token: args.token,
+        namespace: args.namespace,
+        includeTranscripts: args.includeTranscripts,
+        sourceId: args.sourceId,
+        file: args.file,
+        baseSha256: args.baseSha256,
+        offset,
+        content: pending
+      });
+      if (finalResult.conflict) {
+        return finalResult;
+      }
+      offset += pending.length;
+    }
+    pending = chunk;
+  }
+  if (pending) hash.update(pending);
+  const finalBytes = offset + (pending?.length ?? 0);
+  const digest = hash.digest("hex");
+  if (digest !== args.file.sha256 || finalBytes !== args.file.bytes) {
+    throw new Error(`local file changed while pushing offline content: ${args.file.path}`);
+  }
+  if (pending) {
+    finalResult = await postOfflineFileContentChunk({
+      remoteUrl: args.remoteUrl,
+      token: args.token,
+      namespace: args.namespace,
+      includeTranscripts: args.includeTranscripts,
+      sourceId: args.sourceId,
+      file: args.file,
+      baseSha256: args.baseSha256,
+      offset,
+      content: pending
+    });
+    if (finalResult.conflict) {
+      return finalResult;
+    }
+  } else if (args.file.bytes === 0) {
+    finalResult = await postOfflineFileContentChunk({
+      remoteUrl: args.remoteUrl,
+      token: args.token,
+      namespace: args.namespace,
+      includeTranscripts: args.includeTranscripts,
+      sourceId: args.sourceId,
+      file: args.file,
+      baseSha256: args.baseSha256,
+      offset: 0,
+      content: Buffer.alloc(0)
+    });
+  }
+  if (!finalResult?.done) {
+    throw new Error(`offline sync large-file push did not finish for ${args.file.path}`);
+  }
+  return finalResult;
+}
 async function fetchOfflineFileContent(args) {
   const chunks = [];
   const hash = createHash("sha256");
@@ -6978,19 +7164,137 @@ function waitForOfflineInterval(ms, setCancel) {
 async function createOfflineStorageIo(memoryDir) {
   const storage = new StorageManager(memoryDir);
   const header = await readHeader(memoryDir);
+  let secureStoreKey = null;
   if (header) {
     storage.setSecureStoreRequired(true);
     const key = keyring.getKey(secureStoreDir(memoryDir));
     if (key) {
       storage.setSecureStoreKey(key);
+      secureStoreKey = key;
     }
   }
   return {
     readFile: async ({ filePath }) => storage.readOfflineSyncFile(filePath),
+    readFileChunks: ({ filePath, chunkSize }) => readOfflineSyncFileChunks({
+      filePath,
+      memoryDir,
+      secureStoreKey,
+      chunkSize
+    }),
     writeFile: async ({ filePath, content }) => storage.writeOfflineSyncFile(filePath, content),
     deleteFile: async ({ filePath }) => storage.deleteOfflineSyncFile(filePath)
   };
 }
+async function* readOfflineSyncFileChunks(options) {
+  const header = await readFilePrefix(options.filePath, MAGIC_HEADER_SIZE);
+  if (!isEncryptedFile(header)) {
+    yield* readPlainOfflineFileChunks(options.filePath, options.chunkSize);
+    return;
+  }
+  if (!options.secureStoreKey) {
+    throw new SecureStoreLockedError(
+      `secure-store is locked \u2014 cannot read encrypted file at ${options.filePath}. Run \`remnic secure-store unlock\` to decrypt.`
+    );
+  }
+  yield* readEncryptedOfflineFileChunks({
+    filePath: options.filePath,
+    memoryDir: options.memoryDir,
+    key: options.secureStoreKey,
+    chunkSize: options.chunkSize
+  });
+}
+async function readFilePrefix(filePath, length) {
+  const handle = await fs7.promises.open(filePath, "r");
+  try {
+    const out = Buffer.alloc(length);
+    const { bytesRead } = await handle.read(out, 0, length, 0);
+    return out.subarray(0, bytesRead);
+  } finally {
+    await handle.close();
+  }
+}
+async function* readPlainOfflineFileChunks(filePath, chunkSize) {
+  const stream = fs7.createReadStream(filePath, { highWaterMark: chunkSize });
+  for await (const chunk of stream) {
+    yield Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+  }
+}
+async function* readEncryptedOfflineFileChunks(options) {
+  const header = await readFilePrefix(options.filePath, MAGIC_HEADER_SIZE + ENVELOPE_HEADER_SIZE);
+  if (header.length < MAGIC_HEADER_SIZE + ENVELOPE_HEADER_SIZE || !isEncryptedFile(header)) {
+    throw new Error(`secure-store encrypted file is truncated: ${options.filePath}`);
+  }
+  const version = header.readUInt8(MAGIC_BYTES.length);
+  const flags = header.readUInt8(MAGIC_BYTES.length + 1);
+  if (version !== FILE_FORMAT_VERSION) {
+    throw new Error(`secure-store file has unsupported version ${version}: ${options.filePath}`);
+  }
+  if (flags !== FILE_FORMAT_FLAGS) {
+    throw new Error(`secure-store file has unsupported flags 0x${flags.toString(16)}: ${options.filePath}`);
+  }
+  const envelopeHeader = header.subarray(MAGIC_HEADER_SIZE);
+  const envelopeVersion = envelopeHeader.readUInt8(ENVELOPE_LAYOUT.version);
+  if (envelopeVersion !== ENVELOPE_VERSION) {
+    throw new Error(`secure-store envelope has unsupported version ${envelopeVersion}: ${options.filePath}`);
+  }
+  const salt = envelopeHeader.subarray(
+    ENVELOPE_LAYOUT.salt,
+    ENVELOPE_LAYOUT.salt + ENVELOPE_SALT_LENGTH
+  );
+  const iv = envelopeHeader.subarray(ENVELOPE_LAYOUT.iv, ENVELOPE_LAYOUT.iv + IV_LENGTH);
+  const authTag = envelopeHeader.subarray(
+    ENVELOPE_LAYOUT.authTag,
+    ENVELOPE_LAYOUT.authTag + AUTH_TAG_LENGTH
+  );
+  const decipher = createDecipheriv("aes-256-gcm", options.key, iv, {
+    authTagLength: AUTH_TAG_LENGTH
+  });
+  decipher.setAuthTag(authTag);
+  decipher.setAAD(Buffer.concat([secureStoreEnvelopeHeaderAad(salt), filePathAad(options.filePath, options.memoryDir)]));
+  let pending = Buffer.alloc(0);
+  const stream = fs7.createReadStream(options.filePath, {
+    start: MAGIC_HEADER_SIZE + ENVELOPE_HEADER_SIZE,
+    highWaterMark: options.chunkSize
+  });
+  for await (const encryptedChunk of stream) {
+    const plain = decipher.update(Buffer.isBuffer(encryptedChunk) ? encryptedChunk : Buffer.from(encryptedChunk));
+    if (plain.length > 0) {
+      pending = Buffer.concat([pending, plain], pending.length + plain.length);
+    }
+    while (pending.length >= options.chunkSize) {
+      yield pending.subarray(0, options.chunkSize);
+      pending = pending.subarray(options.chunkSize);
+    }
+  }
+  const finalPlain = decipher.final();
+  if (finalPlain.length > 0) {
+    pending = Buffer.concat([pending, finalPlain], pending.length + finalPlain.length);
+  }
+  while (pending.length >= options.chunkSize) {
+    yield pending.subarray(0, options.chunkSize);
+    pending = pending.subarray(options.chunkSize);
+  }
+  if (pending.length > 0) yield pending;
+}
+function secureStoreEnvelopeHeaderAad(salt) {
+  const out = Buffer.alloc(1 + ENVELOPE_SALT_LENGTH);
+  out.writeUInt8(ENVELOPE_VERSION, 0);
+  Buffer.from(salt).copy(out, 1);
+  return out;
+}
+function formatOfflineLargeFilePushFailureMessage(failures) {
+  const paths = failures.slice(0, 5).map((failure) => `${failure.path}: ${failure.error}`).join("; ");
+  const suffix = failures.length > 5 ? `; +${failures.length - 5} more` : "";
+  return `offline sync large-file push failed for ${failures.length} file${failures.length === 1 ? "" : "s"}: ${paths}${suffix}`;
+}
+var OfflineLargeFilePushError = class extends Error {
+  failures;
+  constructor(failures) {
+    super(formatOfflineLargeFilePushFailureMessage(failures));
+    this.name = "OfflineLargeFilePushError";
+    this.failures = failures;
+  }
+};
 async function runOfflineSyncOnce(options) {
   fs7.mkdirSync(options.memoryDir, { recursive: true });
   let activeStatePath = options.statePath;
@@ -7030,20 +7334,90 @@ async function runOfflineSyncOnce(options) {
   }
   const baseFiles = priorState?.baseFiles ?? [];
   const storageIo = await createOfflineStorageIo(options.memoryDir);
+  const localSourceId = localOfflineSourceId(options.memoryDir);
+  const pendingSummary = await summarizeOfflineSyncPendingChanges({
+    root: options.memoryDir,
+    sourceId: localSourceId,
+    baseFiles,
+    includeTranscripts: options.includeTranscripts,
+    readFile: storageIo.readFile
+  });
+  const currentSnapshotForPush = await buildOfflineSyncSnapshot({
+    root: options.memoryDir,
+    sourceId: localSourceId,
+    includeContent: false,
+    includeTranscripts: options.includeTranscripts,
+    readFile: storageIo.readFile
+  });
+  const baseByPath = offlineFileStateMap(baseFiles);
+  let directPushAppliedUpserts = 0;
+  let directPushSkipped = 0;
+  let directPushNamespace;
+  const directPushConflicts = [];
+  const directPushedPaths = /* @__PURE__ */ new Set();
+  const directPushFailures = [];
+  for (const file of offlineDirectPushFiles({
+    currentFiles: currentSnapshotForPush.files,
+    baseFiles
+  })) {
+    let result;
+    try {
+      result = await pushOfflineFileContent({
+        remoteUrl: options.remoteUrl,
+        token: options.token,
+        namespace: syncNamespace,
+        includeTranscripts: options.includeTranscripts,
+        memoryDir: options.memoryDir,
+        sourceId: localSourceId,
+        file,
+        baseSha256: baseByPath.get(file.path)?.sha256,
+        readFile: storageIo.readFile,
+        readFileChunks: storageIo.readFileChunks
+      });
+    } catch (error) {
+      directPushFailures.push({
+        path: file.path,
+        error: error instanceof Error ? error.message : String(error)
+      });
+      continue;
+    }
+    directPushNamespace = result.namespace ?? directPushNamespace;
+    directPushedPaths.add(file.path);
+    if (result.conflict) {
+      directPushConflicts.push({
+        path: result.conflict.path,
+        reason: result.conflict.reason,
+        ...result.conflict.conflictPath ? { conflictPath: result.conflict.conflictPath } : {}
+      });
+    } else {
+      if (result.applied) directPushAppliedUpserts += 1;
+      if (result.skipped) directPushSkipped += 1;
+    }
+  }
+  if (directPushFailures.length > 0) {
+    throw new OfflineLargeFilePushError(directPushFailures);
+  }
   const changeset = await buildOfflineSyncChangeset({
     root: options.memoryDir,
-    sourceId: localOfflineSourceId(options.memoryDir),
+    sourceId: localSourceId,
     baseFiles,
+    excludePaths: [...directPushedPaths],
     includeTranscripts: options.includeTranscripts,
     readFile: storageIo.readFile
   });
-  const pendingSummary = summarizeOfflineSyncChangeset(changeset);
-  const pushed = changeset.changes.length > 0 ? await pushOfflineChanges({
+  const pushedInline = changeset.changes.length > 0 ? await pushOfflineChanges({
     remoteUrl: options.remoteUrl,
     token: options.token,
     namespace: syncNamespace,
     changeset
   }) : null;
+  const pushed = directPushedPaths.size > 0 || pushedInline ? {
+    namespace: pushedInline?.namespace ?? directPushNamespace ?? syncNamespace ?? "",
+    appliedUpserts: (pushedInline?.appliedUpserts ?? 0) + directPushAppliedUpserts,
+    appliedDeletes: pushedInline?.appliedDeletes ?? 0,
+    skipped: (pushedInline?.skipped ?? 0) + directPushSkipped,
+    conflicts: [...directPushConflicts, ...pushedInline?.conflicts ?? []]
+  } : null;
   const remoteSnapshotMetadata = await fetchOfflineSnapshot({
     remoteUrl: options.remoteUrl,
     token: options.token,
@@ -7053,7 +7427,7 @@ async function runOfflineSyncOnce(options) {
   });
   const currentSnapshot = await buildOfflineSyncSnapshot({
     root: options.memoryDir,
-    sourceId: localOfflineSourceId(options.memoryDir),
+    sourceId: localSourceId,
     includeContent: false,
     includeTranscripts: options.includeTranscripts,
     readFile: storageIo.readFile
@@ -7071,7 +7445,7 @@ async function runOfflineSyncOnce(options) {
   });
   const applyCurrentSnapshot = directHydratedPaths.size > 0 ? await buildOfflineSyncSnapshot({
     root: options.memoryDir,
-    sourceId: localOfflineSourceId(options.memoryDir),
+    sourceId: localSourceId,
     includeContent: false,
     includeTranscripts: options.includeTranscripts,
     readFile: storageIo.readFile
@@ -10238,6 +10612,7 @@ export {
   buildQueryRecallRequest,
   chunkOfflineFileContentBatches,
   extractXrayRawArgs,
+  formatOfflineLargeFilePushFailureMessage,
   getBenchUsageText,
   hasFlag,
   main,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@remnic/cli",
-  "version": "1.0.15",
+  "version": "1.0.17",
   "description": "CLI for Remnic memory — init, query, doctor, daemon management",
   "type": "module",
   "main": "dist/index.js",
@@ -26,9 +26,9 @@
   },
   "dependencies": {
     "yaml": "^2.4.2",
-    "@remnic/plugin-pi": "^1.0.1",
-    "@remnic/server": "^1.0.5",
-    "@remnic/core": "^1.1.21"
+    "@remnic/plugin-pi": "^1.0.3",
+    "@remnic/core": "^1.1.23",
+    "@remnic/server": "^1.0.5"
   },
   "peerDependencies": {
     "@remnic/bench": "^1.0.0",
@@ -77,8 +77,8 @@
     "@remnic/export-weclone": "1.0.1",
     "@remnic/import-weclone": "1.0.1",
     "@remnic/import-chatgpt": "0.1.0",
-    "@remnic/import-claude": "0.1.0",
     "@remnic/import-gemini": "0.1.0",
+    "@remnic/import-claude": "0.1.0",
     "@remnic/import-lossless-claw": "0.1.1",
     "@remnic/import-mem0": "0.1.0",
     "@remnic/import-supermemory": "0.1.2"