@remnic/core 1.1.22 → 1.1.24

package/dist/{chunk-FF46Q3SN.js → chunk-AMVN77EU.js}

@@ -10,7 +10,7 @@ import {
 import {
   MAGIC_HEADER_SIZE,
   isEncryptedFile
-} from "./chunk-SH5S7XYD.js";
+} from "./chunk-MXFBBHJU.js";
 import {
   parseFlexibleIsoTimestamp
 } from "./chunk-P7FMDTKL.js";
@@ -24,6 +24,7 @@ import {
   readdir,
   readFile,
   rename,
+  rm,
   stat,
   unlink,
   writeFile
@@ -33,34 +34,13 @@ var OFFLINE_SYNC_SNAPSHOT_FORMAT = "remnic.offline-sync.snapshot.v1";
 var OFFLINE_SYNC_CHANGESET_FORMAT = "remnic.offline-sync.changeset.v1";
 var OFFLINE_SYNC_STATE_VERSION = 1;
 var OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES = 64 * 1024 * 1024;
+var OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES = 8 * 1024 * 1024;
+var OFFLINE_SYNC_APPLY_MAX_BODY_BYTES = 16 * 1024 * 1024;
 var SYNC_INTERNAL_DIR = ".offline-sync";
+var OFFLINE_SYNC_UPLOAD_STAGING_MAX_AGE_MS = 24 * 60 * 60 * 1e3;
 var EXCLUDED_FILE_NAMES = /* @__PURE__ */ new Set([
   ".sync-state.json"
 ]);
-var DERIVED_RUNTIME_STATE_BASENAMES = /* @__PURE__ */ new Set([
-  ".artifact-write-version.log",
-  ".memory-status-version.log",
-  "fact-hashes.ready",
-  "fact-hashes.txt",
-  "buffer-surprise-ledger.jsonl",
-  "buffer.json",
-  "embeddings.json",
-  "entity-mention-index.json",
-  "index_tags.json",
-  "index_time.json",
-  "last_graph_recall.json",
-  "last_intent.json",
-  "last_qmd_recall.json",
-  "last_recall.json",
-  "lcm.sqlite",
-  "lcm.sqlite-shm",
-  "lcm.sqlite-wal",
-  "memory-lifecycle-ledger.jsonl",
-  "memory-projection.sqlite",
-  "memory-projection.sqlite-shm",
-  "memory-projection.sqlite-wal",
-  "recall_impressions.jsonl"
-]);
 var EXCLUDED_FILE_PREFIXES = [
   ".remnic-sync.",
   ".remnic-sync-state."
@@ -281,7 +261,6 @@ function shouldExcludeRelPath(relPosix, includeTranscripts) {
   const parts = relPosix.split("/");
   if (parts.some((part) => DEFAULT_TRANSFER_EXCLUDE_DIRS.has(part))) return true;
   if (parts.some((part) => part === SYNC_INTERNAL_DIR)) return true;
-  if (isDerivedRuntimeStatePath(parts)) return true;
   if (!includeTranscripts && parts[0] === "transcripts") return true;
   const basename = parts[parts.length - 1] ?? "";
   if (isCanonicalRuntimeStatePath(parts) && basename.includes(".tmp-")) return true;
@@ -291,11 +270,7 @@ function shouldExcludeRelPath(relPosix, includeTranscripts) {
 function shouldIgnoreIncomingRuntimePath(relPosix) {
   const parts = relPosix.split("/");
   const basename = parts[parts.length - 1] ?? "";
-  return isDerivedRuntimeStatePath(parts) || isCanonicalRuntimeStatePath(parts) && basename.includes(".tmp-");
-}
-function isDerivedRuntimeStatePath(parts) {
-  const basename = parts[parts.length - 1] ?? "";
-  return isCanonicalRuntimeStatePath(parts) && DERIVED_RUNTIME_STATE_BASENAMES.has(basename);
+  return isCanonicalRuntimeStatePath(parts) && basename.includes(".tmp-");
 }
 function isCanonicalRuntimeStatePath(parts) {
   if (parts[0] === "state") return true;
@@ -477,6 +452,9 @@ async function readOfflineSyncFileContentChunk(options) {
 }
 async function buildOfflineSyncChangeset(options) {
   const includeTranscripts = options.includeTranscripts !== false;
+  const excludedPaths = new Set(
+    (options.excludePaths ?? []).map((relPath) => normalizeRelativePath(relPath, "excludePaths[]"))
+  );
   const base = byPath(filterBaseFilesForMode(
     normalizeFileStates(options.baseFiles),
     includeTranscripts
@@ -492,6 +470,7 @@ async function buildOfflineSyncChangeset(options) {
   const currentMap = byPath(current.files);
   const changes = [];
   for (const relPath of unionPaths(base, currentMap)) {
+    if (excludedPaths.has(relPath)) continue;
     const baseEntry = base.get(relPath);
     const currentEntry = currentMap.get(relPath);
     if (currentEntry && currentEntry.sha256 !== baseEntry?.sha256) {
@@ -874,6 +853,352 @@ async function writeSafeFile(root, relPath, content, writeFileHook) {
     throw error;
   }
 }
+async function applyOfflineSyncFileContentChunk(options) {
+  const root = await ensureSyncRoot(options.root, "applyOfflineSyncFileContentChunk");
+  const sourceId = normalizeSourceId(options.sourceId, "sourceId");
+  const relPath = normalizeRelativePath(options.path, "path");
+  const includeTranscripts = options.includeTranscripts !== false;
+  if (shouldExcludeRelPath(relPath, includeTranscripts)) {
+    throw new Error(`offline sync file content path is excluded: ${relPath}`);
+  }
+  const sha256 = assertSha256(options.sha256, "sha256");
+  const bytes = assertNonNegativeInteger(options.bytes, "bytes");
+  const mtimeMs = assertNonNegativeFinite(options.mtimeMs, "mtimeMs");
+  const offset = options.offset === void 0 ? 0 : assertNonNegativeInteger(options.offset, "offset");
+  const baseSha256 = options.baseSha256 === void 0 ? void 0 : assertSha256(options.baseSha256, "baseSha256");
+  if (!Buffer.isBuffer(options.content)) {
+    throw new Error("content must be a Buffer");
+  }
+  if (options.content.length > OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES) {
+    throw new Error(
+      `content chunk must be ${OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES} bytes or fewer`
+    );
+  }
+  if (bytes > 0 && options.content.length === 0) {
+    throw new Error("content chunk must be non-empty before EOF");
+  }
+  if (offset > bytes || offset + options.content.length > bytes) {
+    throw new Error(`content chunk range exceeds declared file size for ${relPath}`);
+  }
+  if (options.writeFile && !options.writeFileChunks) {
+    throw new Error("offline sync upload storage hooks require writeFileChunks");
+  }
+  if (options.writeFile && !options.writeStagingFile) {
+    throw new Error("offline sync upload storage hooks require writeStagingFile");
+  }
+  if (offset === 0) {
+    await pruneOfflineUploadStaging(root);
+  }
+  const upload = await writeOfflineUploadChunk({
+    root,
+    sourceId,
+    relPath,
+    sha256,
+    bytes,
+    offset,
+    content: options.content,
+    readFile: options.readFile,
+    writeFile: options.writeFile,
+    writeStagingFile: options.writeStagingFile
+  });
+  const done = offset + options.content.length === bytes;
+  const baseResult = {
+    path: relPath,
+    sha256,
+    bytes,
+    mtimeMs,
+    offset,
+    chunkBytes: options.content.length,
+    done
+  };
+  if (!done) {
+    return {
+      ...baseResult,
+      applied: false,
+      skipped: false
+    };
+  }
+  const digest = await digestOfflineUploadStagingContent({
+    root,
+    upload,
+    readFile: options.readFile
+  });
+  if (digest.sha256 !== sha256 || digest.bytes !== bytes) {
+    await cleanupOfflineUpload(upload).catch(() => {
+    });
+    throw new Error(`offline sync upload checksum mismatch for ${relPath}`);
+  }
+  const currentSnapshot = await buildOfflineSyncSnapshotForPaths({
+    root: root.abs,
+    sourceId: "local",
+    paths: [relPath],
+    includeContent: false,
+    includeTranscripts,
+    readFile: options.readFile
+  });
+  const currentFile = currentSnapshot.files[0];
+  const uploadedState = {
+    path: relPath,
+    sha256,
+    bytes,
+    mtimeMs
+  };
+  try {
+    if (currentFile?.sha256 === sha256) {
+      return {
+        ...baseResult,
+        applied: false,
+        skipped: true,
+        currentFile: toFileState(currentFile)
+      };
+    }
+    if (!baseSha256 && currentFile) {
+      const conflict = await recordConflict({
+        root,
+        relPath,
+        reason: "remote_exists_for_local_create",
+        localSha256: currentFile.sha256,
+        incomingSha256: sha256,
+        writeConflictCopies: false,
+        sourceId,
+        writeFile: options.writeFile
+      });
+      return {
+        ...baseResult,
+        applied: false,
+        skipped: false,
+        conflict,
+        currentFile: toFileState(currentFile)
+      };
+    }
+    if (baseSha256 && currentFile?.sha256 !== baseSha256) {
+      const conflict = await recordConflict({
+        root,
+        relPath,
+        reason: currentFile ? "remote_changed_for_local_update" : "remote_deleted_for_local_update",
+        baseSha256,
+        localSha256: currentFile?.sha256,
+        incomingSha256: sha256,
+        writeConflictCopies: false,
+        sourceId,
+        writeFile: options.writeFile
+      });
+      return {
+        ...baseResult,
+        applied: false,
+        skipped: false,
+        conflict,
+        ...currentFile ? { currentFile: toFileState(currentFile) } : {}
+      };
+    }
+    await writeSafeFileFromUpload(root, relPath, upload, options.readFile, options.writeFileChunks);
+    return {
+      ...baseResult,
+      applied: true,
+      skipped: false,
+      currentFile: uploadedState
+    };
+  } finally {
+    await cleanupOfflineUpload(upload).catch(() => {
+    });
+  }
+}
+function offlineUploadRelPath(options) {
+  const key = hashText([
+    options.sourceId,
+    options.relPath,
+    options.sha256,
+    String(options.bytes)
+  ].join("\0"));
+  return `${SYNC_INTERNAL_DIR}/uploads/${key}.part`;
+}
+async function offlineUploadPath(root, options) {
+  const relPath = offlineUploadRelPath(options);
+  return {
+    kind: "single",
+    relPath,
+    filePath: await resolveSafeArchiveTarget(root, relPath)
+  };
+}
+async function offlineUploadChunkPath(root, options) {
+  const uploadRelPath = offlineUploadRelPath(options);
+  const relPath = `${uploadRelPath}/${String(options.offset).padStart(20, "0")}.part`;
+  return {
+    kind: "chunks",
+    relPath,
+    filePath: await resolveSafeArchiveTarget(root, relPath)
+  };
+}
+async function writeOfflineUploadChunk(options) {
+  if ((options.writeFile || options.writeStagingFile) && !options.readFile) {
+    throw new Error("offline sync upload chunk storage hooks require readFile");
+  }
+  const uploadRoot = {
+    ...await offlineUploadPath(options.root, options),
+    kind: "chunks"
+  };
+  if (options.offset === 0) {
+    await rm(uploadRoot.filePath, { recursive: true, force: true }).catch(() => {
+    });
+  } else {
+    const existing = await stat(uploadRoot.filePath).catch((error) => {
+      if (error.code === "ENOENT") return null;
+      throw error;
+    });
+    if (!existing || !existing.isDirectory()) {
+      throw new Error(`offline sync upload is missing initial chunk for ${options.relPath}`);
+    }
+  }
+  const chunk = await offlineUploadChunkPath(options.root, { ...options, offset: options.offset });
+  const writeStagingFile = options.writeStagingFile ?? options.writeFile;
+  if (writeStagingFile) {
+    await writeOfflineUploadContent({
+      root: options.root,
+      relPath: chunk.relPath,
+      filePath: chunk.filePath,
+      content: options.content,
+      writeFile: writeStagingFile
+    });
+    return uploadRoot;
+  }
+  await mkdir(path.dirname(chunk.filePath), { recursive: true });
+  const existingChunk = await lstat(chunk.filePath).catch((error) => {
+    if (error.code === "ENOENT") return null;
+    throw error;
+  });
+  if (existingChunk?.isSymbolicLink()) {
+    throw new Error(`offline sync upload chunk is a symlink: ${chunk.relPath}`);
+  }
+  await writeFile(chunk.filePath, options.content, { mode: 384 });
+  return uploadRoot;
+}
+async function pruneOfflineUploadStaging(root) {
+  const uploadsRelPath = `${SYNC_INTERNAL_DIR}/uploads`;
+  const uploadsPath = await resolveSafeArchiveTarget(root, uploadsRelPath);
+  const entries = await readdir(uploadsPath, { withFileTypes: true }).catch((error) => {
+    if (error.code === "ENOENT") return [];
+    throw error;
+  });
+  const now = Date.now();
+  await Promise.all(entries.map(async (entry) => {
+    if (!/^[a-f0-9]{64}\.part$/i.test(entry.name)) return;
+    const relPath = `${uploadsRelPath}/${entry.name}`;
+    const filePath = await resolveSafeArchiveTarget(root, relPath);
+    const info = await lstat(filePath).catch((error) => {
+      if (error.code === "ENOENT") return null;
+      throw error;
+    });
+    if (!info) return;
+    if (now - info.mtimeMs <= OFFLINE_SYNC_UPLOAD_STAGING_MAX_AGE_MS) return;
+    await rm(filePath, { recursive: true, force: true });
+  }));
+}
+async function* readOfflineUploadStagingChunks(options) {
+  if (options.upload.kind === "single") {
+    yield await readOfflineUploadContent({
+      root: options.root,
+      relPath: options.upload.relPath,
+      filePath: options.upload.filePath,
+      readFile: options.readFile
+    });
+    return;
+  }
+  const entries = await readdir(options.upload.filePath);
+  const chunkNames = entries.filter((entry) => /^\d{20}\.part$/.test(entry)).sort();
+  if (chunkNames.length === 0) {
+    throw new Error(`offline sync upload is missing chunks for ${options.upload.relPath}`);
+  }
+  let expectedOffset = 0;
+  for (const chunkName of chunkNames) {
+    const offset = Number(chunkName.slice(0, 20));
+    if (!Number.isSafeInteger(offset) || offset !== expectedOffset) {
+      throw new Error(
+        `offline sync upload offset mismatch for ${options.upload.relPath}: expected ${expectedOffset}, got ${offset}`
+      );
+    }
+    const relPath = `${options.upload.relPath}/${chunkName}`;
+    const filePath = await resolveSafeArchiveTarget(options.root, relPath);
+    const content = await readOfflineUploadContent({
+      root: options.root,
+      relPath,
+      filePath,
+      readFile: options.readFile
+    });
+    expectedOffset += content.length;
+    yield content;
+  }
+}
+async function digestOfflineUploadStagingContent(options) {
+  const hash = createHash("sha256");
+  let bytes = 0;
+  for await (const chunk of readOfflineUploadStagingChunks(options)) {
+    hash.update(chunk);
+    bytes += chunk.length;
+  }
+  return { sha256: hash.digest("hex"), bytes };
+}
+async function writeSafeFileFromUpload(root, relPath, upload, readFile2, writeFileChunks) {
+  const target = await resolveSafeArchiveTarget(root, relPath);
+  const chunks = readOfflineUploadStagingChunks({ root, upload, readFile: readFile2 });
+  if (writeFileChunks) {
+    await writeFileChunks({ root: root.abs, path: relPath, filePath: target, chunks });
+    return;
+  }
+  await mkdir(path.dirname(target), { recursive: true });
+  const tmp = path.join(
+    path.dirname(target),
+    `.remnic-sync.${process.pid}.${randomUUID()}.tmp`
+  );
+  const handle = await open(tmp, "w", 384);
+  try {
+    for await (const chunk of chunks) {
+      if (chunk.length > 0) await handle.write(chunk);
+    }
+    await handle.close();
+    const targetStat = await lstat(target).catch((error) => {
+      if (error.code === "ENOENT") return null;
+      throw error;
+    });
+    if (targetStat?.isSymbolicLink()) {
+      throw new Error(`offline sync target is a symlink: ${relPath}`);
+    }
+    await rename(tmp, target);
+  } catch (error) {
+    await handle.close().catch(() => {
+    });
+    await unlink(tmp).catch(() => {
+    });
+    throw error;
+  }
+}
+async function cleanupOfflineUpload(upload) {
+  if (upload.kind === "chunks") {
+    await rm(upload.filePath, { recursive: true, force: true });
+    return;
+  }
+  await unlink(upload.filePath).catch((error) => {
+    if (error.code === "ENOENT") return;
+    throw error;
+  });
+}
+async function readOfflineUploadContent(options) {
+  if (options.readFile) {
+    return options.readFile({
+      root: options.root.abs,
+      path: options.relPath,
+      filePath: options.filePath
+    });
+  }
+  return readFile(options.filePath);
+}
+async function writeOfflineUploadContent(options) {
+  await options.writeFile({
+    root: options.root.abs,
+    path: options.relPath,
+    filePath: options.filePath,
+    content: options.content
+  });
+}
 async function deleteSafeFile(root, relPath, deleteFile) {
   const target = await resolveSafeArchiveTarget(root, relPath);
   if (deleteFile) {
@@ -974,6 +1299,8 @@ export {
   OFFLINE_SYNC_CHANGESET_FORMAT,
   OFFLINE_SYNC_STATE_VERSION,
   OFFLINE_SYNC_FILE_CONTENT_MAX_CHUNK_BYTES,
+  OFFLINE_SYNC_FILE_CONTENT_TRANSFER_CHUNK_BYTES,
+  OFFLINE_SYNC_APPLY_MAX_BODY_BYTES,
   normalizeOfflineSyncSnapshot,
   normalizeOfflineSyncChangeset,
   buildOfflineSyncSnapshot,
@@ -984,6 +1311,7 @@ export {
   summarizeOfflineSyncPendingChanges,
   applyOfflineSyncSnapshot,
   applyOfflineSyncChangeset,
+  applyOfflineSyncFileContentChunk,
   defaultOfflineSyncStatePath,
   readOfflineSyncState,
   writeOfflineSyncState,
@@ -991,4 +1319,4 @@ export {
   normalizeOfflineSyncState,
   fileStatesFromSnapshot
 };
-//# sourceMappingURL=chunk-FF46Q3SN.js.map
+//# sourceMappingURL=chunk-AMVN77EU.js.map