@remnic/core 9.45.3 → 9.45.5

package/src/access-service-observe-scope.test.ts

@@ -243,6 +243,166 @@ test("#2128 pending observe preparation is a force-flush barrier", async () => {
   assert.equal(waited, true);
 });
 
+test("#2206 scoped wait ignores unresolved preparations with a different hint", async () => {
+  const tracker = new PendingObserveExtractionTracker();
+  const target = tracker.reserve("opaque-session", "projectTag:project-a");
+  const unrelated = tracker.reserve("opaque-session", "projectTag:project-b");
+  let settled = false;
+  const waitPromise = tracker
+    .wait(
+      "opaque-session",
+      "alice",
+      "alice-project-a",
+      undefined,
+      undefined,
+      "projectTag:project-a",
+    )
+    .then(() => {
+      settled = true;
+    });
+
+  target.release();
+  await new Promise<void>((resolve) => setImmediate(resolve));
+  const settledBeforeUnrelatedRelease = settled;
+  unrelated.release();
+  await waitPromise;
+
+  assert.equal(settledBeforeUnrelatedRelease, true);
+});
+
+test("#2206 scoped wait fences a reservation that later resolves into the flushed scope", async () => {
+  const tracker = new PendingObserveExtractionTracker();
+  const reservation = tracker.reserve("opaque-session", "cwd:/workspace/project/nested");
+  const unrelated = tracker.reserve("opaque-session", "cwd:/workspace/other/nested");
+
+  await tracker.wait(
+    "opaque-session",
+    "alice",
+    "alice-project",
+    undefined,
+    undefined,
+    "projectTag:Acme/Webshop",
+  );
+  assert.equal(reservation.isCancelled(), false, "the raw hint mismatch initially excludes the reservation");
+
+  reservation.setScope("alice", "alice-project");
+  unrelated.setScope("alice", "alice-other-project");
+
+  assert.equal(
+    reservation.isCancelled(),
+    true,
+    "a reservation existing at the wait boundary cannot enter the flushed scope later",
+  );
+  assert.equal(unrelated.isCancelled(), false, "the fence must not cancel a different resolved scope");
+  unrelated.release();
+  reservation.release();
+});
+
+test("#2206 resolved scope overrides a mismatched preparation hint", async () => {
+  const tracker = new PendingObserveExtractionTracker();
+  const preparation = tracker.reserve("opaque-session", "cwd:/workspace/project/src");
+  preparation.setScope("alice", "alice-project");
+  let settled = false;
+  const waitPromise = tracker
+    .wait(
+      "opaque-session",
+      "alice",
+      "alice-project",
+      undefined,
+      undefined,
+      "projectTag:Acme/Webshop",
+    )
+    .then(() => {
+      settled = true;
+    });
+
+  await Promise.resolve();
+  assert.equal(settled, false);
+  preparation.release();
+  await waitPromise;
+  assert.equal(settled, true);
+});
+
+test("#2206 resolved scope mismatch overrides an identical preparation hint", async () => {
+  const tracker = new PendingObserveExtractionTracker();
+  const unrelated = tracker.reserve("opaque-session", "projectTag:project-a");
+  unrelated.setScope("alice", "alice-project-b");
+  let settled = false;
+  const waitPromise = tracker
+    .wait(
+      "opaque-session",
+      "alice",
+      "alice-project-a",
+      undefined,
+      undefined,
+      "projectTag:project-a",
+    )
+    .then(() => {
+      settled = true;
+    });
+
+  await new Promise<void>((resolve) => setImmediate(resolve));
+  const settledBeforeUnrelatedRelease = settled;
+  unrelated.release();
+  await waitPromise;
+
+  assert.equal(settledBeforeUnrelatedRelease, true);
+});
+
+test("#2206 unresolved preparation without a hint remains a conservative barrier", async () => {
+  const tracker = new PendingObserveExtractionTracker();
+  const preparation = tracker.reserve("opaque-session");
+  let settled = false;
+  const waitPromise = tracker
+    .wait(
+      "opaque-session",
+      "alice",
+      "alice-project",
+      undefined,
+      undefined,
+      "projectTag:project-a",
+    )
+    .then(() => {
+      settled = true;
+    });
+
+  await Promise.resolve();
+  assert.equal(settled, false);
+  preparation.release();
+  await waitPromise;
+  assert.equal(settled, true);
+});
+
+test("#2206 raw-hint cancellation only cancels matching preparations", async () => {
+  const tracker = new PendingObserveExtractionTracker();
+  const unresolved = tracker.reserve("opaque-session", "projectTag:project-a");
+  const resolved = tracker.reserve("opaque-session", "projectTag:project-a");
+  const unrelated = tracker.reserve("opaque-session", "projectTag:project-b");
+  resolved.setScope("alice", "alice-project-a");
+  let releaseExtraction!: () => void;
+  const extraction = new Promise<void>((resolve) => {
+    releaseExtraction = resolve;
+  });
+  const unrelatedController = new AbortController();
+  tracker.track(
+    tracker.key("opaque-session", "alice", "alice-project-b"),
+    extraction,
+    unrelatedController,
+  );
+
+  tracker.cancelPreparations("opaque-session", "projectTag:project-a");
+
+  assert.equal(unresolved.isCancelled(), true);
+  assert.equal(resolved.isCancelled(), true);
+  assert.equal(unrelated.isCancelled(), false);
+  assert.equal(unrelatedController.signal.aborted, false);
+  unresolved.release();
+  resolved.release();
+  unrelated.release();
+  releaseExtraction();
+  await extraction;
+});
+
 test("#2128 scoped observe preparation cancellation preserves another project", () => {
   const tracker = new PendingObserveExtractionTracker();
   const projectA = tracker.reserve("opaque-session", "projectTag:project-a");
@@ -256,7 +416,6 @@ test("#2128 scoped observe preparation cancellation preserves another project",
   projectB.release();
 });
 
-
 test("#2128 cancellation matches resolved scope despite a different request hint", () => {
   const tracker = new PendingObserveExtractionTracker();
   const preparation = tracker.reserve("session-z", "cwd:/workspace/project/src");

package/src/access-service-offline-file-content.test.ts

@@ -1,8 +1,58 @@
 import assert from "node:assert/strict";
+import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
 import test from "node:test";
 
 import { EngramAccessInputError, EngramAccessService } from "./access-service.js";
-import { OFFLINE_SYNC_MAX_MTIME_MS } from "./offline-sync.js";
+import { OFFLINE_SYNC_CHANGESET_FORMAT, OFFLINE_SYNC_MAX_MTIME_MS } from "./offline-sync.js";
+import { isEncryptedFile } from "./secure-store/index.js";
+import { ContentHashIndex } from "./storage/content-hash-index.js";
+import { StorageManager } from "./storage.js";
+import { withEntityCanonicalMutationLock } from "./storage/entity-canonical-id-lock.js";
+
+class RawEntityMutationProbeStorage extends StorageManager {
+  readonly probeEntitiesDir: string;
+  writeEntered = false;
+  deleteEntered = false;
+
+  constructor(memoryDir: string) {
+    super(memoryDir);
+    this.probeEntitiesDir = path.join(memoryDir, "entities");
+  }
+
+  override async readMemoryByPath(filePath: string) {
+    if (path.dirname(filePath) === this.probeEntitiesDir) return null;
+    return super.readMemoryByPath(filePath);
+  }
+
+  protected override tombstoneBlockedCaptureIndexOptions() {
+    const options = super.tombstoneBlockedCaptureIndexOptions();
+    return {
+      ...options,
+      parseMemory: (filePath: string, content: Buffer) =>
+        path.dirname(filePath) === this.probeEntitiesDir
+          ? null
+          : options.parseMemory?.(filePath, content) ?? null,
+    };
+  }
+
+  protected override async writeManagedStorageFile(
+    filePath: string,
+    write: () => Promise<void>,
+  ): Promise<void> {
+    this.writeEntered = true;
+    await super.writeManagedStorageFile(filePath, write);
+  }
+
+  protected override async deleteManagedStorageFile(
+    filePath: string,
+    deletionMtimeMs?: number | null,
+  ): Promise<boolean> {
+    this.deleteEntered = true;
+    return super.deleteManagedStorageFile(filePath, deletionMtimeMs);
+  }
+}
 
 function createOfflineService(): EngramAccessService {
   return new EngramAccessService({
@@ -18,6 +68,241 @@ function createOfflineService(): EngramAccessService {
   } as any);
 }
 
+function memoryFile(options: {
+  id: string;
+  body: string;
+  status?: string;
+  contentHash?: string;
+}): string {
+  return [
+    "---",
+    `id: ${options.id}`,
+    "category: fact",
+    ...(options.status ? [`status: ${options.status}`] : []),
+    ...(options.contentHash ? [`contentHash: ${options.contentHash}`] : []),
+    "---",
+    options.body,
+  ].join("\n");
+}
+
+function createManifestService(options: {
+  root: string;
+  storage: StorageManager;
+  namespacesEnabled?: boolean;
+}): { service: EngramAccessService; getStorageCalls: string[] } {
+  const getStorageCalls: string[] = [];
+  const service = new EngramAccessService({
+    config: {
+      memoryDir: options.root,
+      namespacesEnabled: options.namespacesEnabled === true,
+      defaultNamespace: "global",
+      sharedNamespace: "shared",
+      namespacePolicies: options.namespacesEnabled
+        ? [{ name: "team", readPrincipals: ["reader"], writePrincipals: [] }]
+        : [],
+      offlineSyncExcludes: [],
+    },
+    namespaceCatalog: {
+      async listNamespaces() {
+        return [];
+      },
+    },
+    async drainPendingRecallImpressions() {
+      return { folded: false, pendingDeferred: false };
+    },
+    async getStorage(namespace: string) {
+      getStorageCalls.push(namespace);
+      return options.storage;
+    },
+  } as unknown as ConstructorParameters<typeof EngramAccessService>[0]);
+  return { service, getStorageCalls };
+}
+
+test("offline manifest streams body-free active, archived, old, bad, and encrypted memory rows lazily", async () => {
+  const root = await mkdtemp(path.join(os.tmpdir(), "remnic-access-manifest-"));
+  try {
+    const activeBody = "active body must never leave the service";
+    const archivedBody = "archived body must never leave the service";
+    const oldBody = "legacy body must never leave the service";
+    const encryptedBody = "encrypted body must never leave the service";
+    const activeContentHash = ContentHashIndex.computeHash(activeBody);
+    const files = new Map<string, string>([
+      [
+        "facts/active.md",
+        memoryFile({
+          id: "active-memory",
+          body: activeBody,
+          status: "active",
+          contentHash: activeContentHash,
+        }),
+      ],
+      [
+        "archive/facts/archived.md",
+        memoryFile({ id: "archived-memory", body: archivedBody, status: "active" }),
+      ],
+      ["facts/old.md", memoryFile({ id: "old-memory", body: oldBody })],
+      ["facts/bad.md", `not valid memory frontmatter\n${activeBody}`],
+    ]);
+    for (const [relativePath, content] of files) {
+      const filePath = path.join(root, relativePath);
+      await mkdir(path.dirname(filePath), { recursive: true });
+      await writeFile(filePath, content);
+    }
+    const storage = new StorageManager(root);
+    await storage.ensureDirectories();
+    storage.setSecureStoreKey(Buffer.alloc(32, 11));
+    storage.setSecureStoreRequired(true);
+    const encryptedRelativePath = "facts/encrypted.md";
+    await storage.writeOfflineSyncFile(
+      path.join(root, encryptedRelativePath),
+      Buffer.from(memoryFile({ id: "encrypted-memory", body: encryptedBody, status: "active" })),
+    );
+    assert.equal(isEncryptedFile(await readFile(path.join(root, encryptedRelativePath))), true);
+
+    let digestReads = 0;
+    let bodyReads = 0;
+    const digestOfflineSyncFile = storage.digestOfflineSyncFile.bind(storage);
+    const readOfflineSyncFile = storage.readOfflineSyncFile.bind(storage);
+    storage.digestOfflineSyncFile = async (filePath: string) => {
+      digestReads += 1;
+      return digestOfflineSyncFile(filePath);
+    };
+    storage.readOfflineSyncFile = async (filePath: string) => {
+      bodyReads += 1;
+      return readOfflineSyncFile(filePath);
+    };
+    const { service } = createManifestService({ root, storage });
+
+    const manifest = await service.offlineSyncManifestStream();
+    assert.equal(manifest.format, "remnic-reconcile-manifest");
+    assert.equal(manifest.schemaVersion, 1);
+    assert.equal(Array.isArray(manifest.files), false);
+    assert.equal(digestReads, 0);
+    assert.equal(bodyReads, 0);
+
+    const iterator = manifest.files[Symbol.asyncIterator]();
+    const first = await iterator.next();
+    assert.equal(first.done, false);
+    assert.equal(first.value?.path, "archive/facts/archived.md");
+    assert.equal(digestReads, 1);
+    assert.equal(bodyReads, 1);
+
+    const rows = first.value ? [first.value] : [];
+    for await (const row of { [Symbol.asyncIterator]: () => iterator }) rows.push(row);
+    const rowsByPath = new Map(rows.map((row) => [row.path, row]));
+    assert.deepEqual(rowsByPath.get("facts/active.md")?.memory, {
+      id: "active-memory",
+      category: "fact",
+      contentHash: activeContentHash,
+      status: "active",
+    });
+    assert.notEqual(rowsByPath.get("facts/active.md")?.sha256, activeContentHash);
+    assert.equal(rowsByPath.get("archive/facts/archived.md")?.memory?.status, "archived");
+    assert.equal(
+      rowsByPath.get("facts/old.md")?.memory?.contentHash,
+      ContentHashIndex.computeHash(oldBody),
+    );
+    assert.equal(rowsByPath.get("facts/old.md")?.memory?.status, "active");
+    assert.equal(rowsByPath.get("facts/bad.md")?.memory, undefined);
+    assert.equal(rowsByPath.get(encryptedRelativePath)?.memory?.id, "encrypted-memory");
+    for (const row of rows) {
+      assert.equal("content" in row, false);
+      assert.equal("contentBase64" in row, false);
+      assert.doesNotMatch(JSON.stringify(row), /body must never leave the service/);
+      assert.equal(typeof row.sha256, "string");
+      assert.equal(typeof row.bytes, "number");
+      assert.equal(typeof row.mtimeMs, "number");
+    }
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});
+
+test("offline manifest uses snapshot namespace authorization and storage routing", async () => {
+  const root = await mkdtemp(path.join(os.tmpdir(), "remnic-access-manifest-namespace-"));
+  try {
+    const storage = new StorageManager(path.join(root, "namespaces", "team"));
+    await storage.ensureDirectories();
+    const { service, getStorageCalls } = createManifestService({
+      root,
+      storage,
+      namespacesEnabled: true,
+    });
+    await assert.rejects(
+      () => service.offlineSyncManifestStream({ namespace: "team", principal: "outsider" }),
+      /namespace is not readable: team/,
+    );
+    const manifest = await service.offlineSyncManifestStream({
+      namespace: "team",
+      principal: "reader",
+      includeTranscripts: false,
+    });
+    assert.equal(manifest.namespace, "team");
+    assert.deepEqual(getStorageCalls, ["team"]);
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});
+
+test("offline file-transfer endpoints reject internal Remnic state paths", async () => {
+  const root = await mkdtemp(path.join(os.tmpdir(), "remnic-access-internal-state-"));
+  try {
+    const storage = new StorageManager(root);
+    await storage.ensureDirectories();
+    const { service } = createManifestService({ root, storage });
+    const internalPath = ".remnic/state/converge-cursors/peer.json";
+    const internalFilePath = path.join(root, internalPath);
+    await mkdir(path.dirname(internalFilePath), { recursive: true });
+    await writeFile(internalFilePath, "cursor");
+
+    await assert.rejects(
+      () => service.offlineSyncFiles({ paths: [internalPath] }),
+      (error) =>
+        error instanceof EngramAccessInputError
+        && /offline sync snapshot path is excluded/.test(error.message),
+    );
+    await assert.rejects(
+      () => service.offlineSyncFileContent({ path: internalPath, offset: 0, length: 1 }),
+      (error) =>
+        error instanceof EngramAccessInputError
+        && /offline sync file content path is excluded/.test(error.message),
+    );
+    await assert.rejects(
+      () => service.offlineSyncApplyFileContent({
+        sourceId: "peer",
+        path: internalPath,
+        sha256: "a".repeat(64),
+        bytes: 0,
+        mtimeMs: 0,
+        offset: 0,
+        content: Buffer.alloc(0),
+      }),
+      (error) =>
+        error instanceof EngramAccessInputError
+        && /offline sync file content path is excluded/.test(error.message),
+    );
+    const applyResult = await service.offlineSyncApply({
+      changeset: {
+        format: OFFLINE_SYNC_CHANGESET_FORMAT,
+        schemaVersion: 1,
+        createdAt: new Date().toISOString(),
+        sourceId: "peer",
+        includeTranscripts: true,
+        changes: [{
+          type: "delete",
+          path: internalPath,
+          baseSha256: "b".repeat(64),
+        }],
+      },
+    });
+    assert.equal(applyResult.appliedDeletes, 0);
+    assert.equal(await readFile(internalFilePath, "utf8"), "cursor");
+
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});
+
 test("offline apply-file-content reports invalid metadata as input errors", async () => {
   const service = createOfflineService();
   await assert.rejects(
@@ -92,3 +377,62 @@ test("offline convergence completion refreshes authorized namespaces as one batc
   assert.deepEqual(result, { namespaces: ["global"], refreshed: true });
   assert.deepEqual(refreshed, [["global"]]);
 });
+
+test("offline raw entity writes wait for the canonical mutation lock", async () => {
+  const root = await mkdtemp(path.join(os.tmpdir(), "remnic-offline-entity-write-lock-"));
+  let writePromise: Promise<void> | undefined;
+  try {
+    await new StorageManager(root).ensureDirectories();
+    const storage = new RawEntityMutationProbeStorage(root);
+    const target = path.join(root, "entities", "project-replicated.md");
+    const content = Buffer.from(
+      "---\nid: project-replicated\n---\n\n# Replicated\n\n**Type:** project\n\nReplicated bytes.\n",
+      "utf8",
+    );
+
+    await withEntityCanonicalMutationLock(path.join(root, "state"), async () => {
+      writePromise = storage.writeOfflineSyncFile(target, content);
+      await new Promise<void>((resolve) => setImmediate(resolve));
+      assert.equal(
+        storage.writeEntered,
+        false,
+        "the raw entity write must not enter its managed-file mutation while the canonical lock is held",
+      );
+    });
+
+    assert.ok(writePromise);
+    await writePromise;
+    assert.deepEqual(await readFile(target), content);
+  } finally {
+    await writePromise?.catch(() => undefined);
+    await rm(root, { recursive: true, force: true });
+  }
+});
+
+test("offline raw entity deletes wait for the canonical mutation lock", async () => {
+  const root = await mkdtemp(path.join(os.tmpdir(), "remnic-offline-entity-delete-lock-"));
+  let deletePromise: Promise<void> | undefined;
+  try {
+    await new StorageManager(root).ensureDirectories();
+    const target = path.join(root, "entities", "project-replicated.md");
+    await writeFile(target, "# Replicated\n", "utf8");
+    const storage = new RawEntityMutationProbeStorage(root);
+
+    await withEntityCanonicalMutationLock(path.join(root, "state"), async () => {
+      deletePromise = storage.deleteOfflineSyncFile(target, null);
+      await new Promise<void>((resolve) => setImmediate(resolve));
+      assert.equal(
+        storage.deleteEntered,
+        false,
+        "the raw entity delete must not enter its managed-file mutation while the canonical lock is held",
+      );
+    });
+
+    assert.ok(deletePromise);
+    await deletePromise;
+    await assert.rejects(() => readFile(target));
+  } finally {
+    await deletePromise?.catch(() => undefined);
+    await rm(root, { recursive: true, force: true });
+  }
+});