@remnic/core 10.0.49 → 10.0.51

package/src/extraction-span-wire.test.ts

@@ -34,3 +34,31 @@ test("provider schema omits span in off mode and keeps it in on/shadow (issue #2
   assert.deepEqual(extractionProviderJsonSchema("on"), extractionProviderJsonSchema("shadow"));
   assert.notEqual(extractionProviderJsonSchema("off"), extractionProviderJsonSchema("on"));
 });
+
+// Providers receive this as the extraction response_format. zod 3 advertised
+// additionalProperties:false on every object; the zod 4 port must too.
+type SchemaNode = {
+  type?: string;
+  properties?: Record<string, SchemaNode>;
+  items?: SchemaNode;
+  anyOf?: SchemaNode[];
+  additionalProperties?: boolean | SchemaNode;
+};
+
+function openObjectSchemas(node: SchemaNode, path: string, found: string[]): void {
+  if (node.type === "object" && node.properties && node.additionalProperties !== false) found.push(path);
+  for (const [name, child] of Object.entries(node.properties ?? {})) {
+    openObjectSchemas(child, `${path}.${name}`, found);
+  }
+  if (node.items) openObjectSchemas(node.items, `${path}[]`, found);
+  node.anyOf?.forEach((child, index) => openObjectSchemas(child, `${path}|${index}`, found));
+}
+
+test("provider schema closes every object schema with additionalProperties false", () => {
+  for (const mode of ["off", "on"] as const) {
+    const open: string[] = [];
+    // SAFETY: extractionProviderJsonSchema returns a JSON Schema document; SchemaNode names the keys walked here.
+    openObjectSchemas(extractionProviderJsonSchema(mode) as SchemaNode, "$", open);
+    assert.deepEqual(open, [], `spanMode=${mode}: objects that let a provider emit extra keys`);
+  }
+});

package/src/extraction-span-wire.ts

@@ -5,7 +5,7 @@
  * schema is mode-aware so off-mode structured output cannot advertise `span`.
  */
 
-import { zodToJsonSchema } from "zod-to-json-schema";
+import { z } from "zod";
 import type { ExtractionSpanConfig } from "./extraction-span-config.js";
 import { parseSpanOffsets, type ParsedSpan } from "./extraction-span.js";
 import { ExtractionProviderOutputSchema } from "./schemas.js";
@@ -19,13 +19,41 @@ export function applySpanMode(opts: {
   return parseSpanOffsets(opts.text, { start: opts.start, end: opts.end }, opts.enabled);
 }
 
-const JSON_SCHEMA_OPTS = { $refStrategy: "none" as const, effectStrategy: "input" as const };
+/** The parts of a JSON Schema node the extraction schema uses. */
+type SchemaNode = {
+  type?: string;
+  properties?: Record<string, SchemaNode>;
+  items?: SchemaNode;
+  anyOf?: SchemaNode[];
+  additionalProperties?: boolean | SchemaNode;
+};
 
-const EXTRACTION_PROVIDER_JSON_SCHEMA_WITH_SPAN = zodToJsonSchema(
-  ExtractionProviderOutputSchema,
-  JSON_SCHEMA_OPTS,
-) as Record<string, unknown>;
+/**
+ * `io: "input"` leaves objects open (zod accepts extra input keys), but providers
+ * must be told to emit no extra keys: zod-to-json-schema advertised
+ * additionalProperties:false on every object. Close any object schema that
+ * does not already state its own.
+ */
+function closeObjectSchemas(node: SchemaNode): SchemaNode {
+  if (node.type === "object" && node.properties && node.additionalProperties === undefined) {
+    node.additionalProperties = false;
+  }
+  Object.values(node.properties ?? {}).forEach(closeObjectSchemas);
+  if (node.items) closeObjectSchemas(node.items);
+  node.anyOf?.forEach(closeObjectSchemas);
+  return node;
+}
 
+// zod 4 ships toJSONSchema natively; zod-to-json-schema emits an empty
+// document for v4 schemas, so the dependency is dropped here. `io: "input"`
+// matches the old effectStrategy: "input" (advertise the input shape of
+// transforms/preprocess to providers).
+// SAFETY: z.toJSONSchema returns a JSON-Schema document — always an object at
+// the root; the record shape is what omitFactSpan walks.
+const EXTRACTION_PROVIDER_JSON_SCHEMA_WITH_SPAN: SchemaNode = closeObjectSchemas(
+  // SAFETY: z.toJSONSchema returns a JSON Schema document; SchemaNode names the keys this file reads.
+  z.toJSONSchema(ExtractionProviderOutputSchema, { target: "draft-7", io: "input" }) as SchemaNode,
+);
 type FactItems = {
   properties?: Record<string, unknown>;
   required?: string[];

package/src/index.ts

@@ -657,7 +657,7 @@ export {
   type ParsedBriefingWindow,
   type BriefingFormatValue,
 } from "./briefing.js";
-
+export { withBriefingDeadline, BriefingTimeoutError, BRIEFING_DEADLINE_MS } from "./briefing-deadline.js";
 // ---------------------------------------------------------------------------
 // Binary lifecycle management (#367)
 // ---------------------------------------------------------------------------

package/src/location/tagging.ts

@@ -21,7 +21,7 @@
 
 import type { StorageManager } from "../index.js";
 import { log } from "../logger.js";
-import type { MemoryFile } from "../types.js";
+import type { BriefingResult, MemoryFile } from "../types.js";
 import type { WearableConversation } from "../wearables/types.js";
 import { localDayKey, placeDurations } from "./intervals.js";
 import {
@@ -294,23 +294,25 @@ export async function briefingLocationSection(
   );
 }
 
-export async function applyBriefingLocationContext<
-  T extends { markdown: string; json: Record<string, unknown> },
->(
-  result: T,
+/**
+ * Append the opt-in location section to a briefing. The section is read BEFORE
+ * `build` runs: buildBriefing drains correction notifications at most once, so
+ * a deadline during this read must not lose them.
+ */
+export async function buildWithBriefingLocation<T extends Pick<BriefingResult, "markdown" | "json">>(
   includeLocation: boolean | undefined,
   memoryDir: string,
   windowStartUtc: string,
   location: LocationConfig,
+  build: () => Promise<T>,
 ): Promise<T> {
-  if (includeLocation !== true || !locationTaggingEnabled(location)) return result;
-  const section = await briefingLocationSection(memoryDir, windowStartUtc, location);
+  const section =
+    includeLocation === true && locationTaggingEnabled(location)
+      ? await briefingLocationSection(memoryDir, windowStartUtc, location)
+      : null;
+  const result = await build();
   if (!section) return result;
-  return {
-    ...result,
-    markdown: `${result.markdown}\n\n${section}\n`,
-    json: { ...result.json, locationContext: section },
-  };
+  return { ...result, markdown: `${result.markdown}\n\n${section}\n`, json: { ...result.json, locationContext: section } };
 }
 
 /**

package/src/offline-sync-file-io.ts

@@ -77,9 +77,16 @@ export async function plainFileDigest(filePath: string): Promise<string> {
   return hash.digest("hex");
 }
 
-import { lstat, utimes } from "node:fs/promises";
+import { lstat, realpath, utimes } from "node:fs/promises";
 import { isCensusMtimeMs } from "./census-validation.js";
-import { resolveSafeArchiveTarget, type SafeArchiveRoot } from "./transfer/fs-utils.js";
+import path from "node:path";
+import {
+  fromPosixRelPath,
+  isPathInsideRoot,
+  resolveSafeArchiveTarget,
+  type SafeArchiveRoot,
+  validateArchiveRelativePath,
+} from "./transfer/fs-utils.js";
 
 /** Sync-internal directory spooling partial uploads and coordination state. */
 export const SYNC_INTERNAL_DIR = ".offline-sync";
@@ -124,3 +131,55 @@ export async function setSafeFileMtime(
   await utimes(target, mtime, mtime);
   return true;
 }
+
+/**
+ * True when a regular (non-symlink) file exists at the archive-relative path.
+ * A symlink is not a regular local file and reads as absent, so callers treat
+ * it like any other missing path instead of failing the whole push.
+ */
+export async function offlineSyncRegularFilePresent(
+  root: SafeArchiveRoot,
+  relPath: string,
+): Promise<boolean> {
+  const target = path.resolve(root.abs, fromPosixRelPath(validateArchiveRelativePath(relPath, root.errorPrefix)));
+  if (!isPathInsideRoot(root.abs, target)) {
+    throw new Error(`${root.errorPrefix}: record path escapes target root: ${relPath}`);
+  }
+  const st = await lstat(target).catch((error: NodeJS.ErrnoException) => {
+    // SAFETY: Node fs errors carry `code`; only ENOENT means absent, all other
+    // errno codes must propagate so census uncertainty fails closed.
+    if (error.code === "ENOENT") return null;
+    throw error;
+  });
+  if (!st || st.isSymbolicLink() || !st.isFile()) return false;
+  // A regular file still has to resolve inside the root (a symlinked parent directory escapes it).
+  await resolveSafeArchiveTarget(root, relPath);
+  // The census skips a symlinked directory, so a file reached only through one is absent here too.
+  return (await realpath(target)) === path.join(root.real, path.relative(root.abs, target));
+}
+
+/** A path the census observed vanished before its digest was read: a retryable race, never a confirmed delete. */
+export class OfflineSyncCensusRaceError extends Error {
+  constructor(readonly relPath: string, cause?: unknown) {
+    super(
+      `offline sync census race: ${relPath} vanished during digest; retry the snapshot`,
+      cause === undefined ? undefined : { cause },
+    );
+    this.name = "OfflineSyncCensusRaceError";
+  }
+}
+
+/** The message the CLI push retry loop matches: it defers the path and rebuilds the census. */
+export function changedWhileBuildingChangesetError(relPath: string, cause?: unknown): Error {
+  return new Error(
+    `offline sync file changed while building changeset: ${relPath}`,
+    cause === undefined ? undefined : { cause },
+  );
+}
+
+/** A `.catch` handler: a census race while building a changeset upsert stays retryable. */
+export function censusRaceAsChangedWhileBuilding(relPath: string): (error: Error) => never {
+  return (error) => {
+    throw error instanceof OfflineSyncCensusRaceError ? changedWhileBuildingChangesetError(relPath, error) : error;
+  };
+}

package/src/offline-sync.test.ts

@@ -1,6 +1,6 @@
 import assert from "node:assert/strict";
 import { createHash } from "node:crypto";
-import { chmod, mkdir, mkdtemp, readdir, readFile, rm, stat, utimes, writeFile } from "node:fs/promises";
+import { chmod, mkdir, mkdtemp, readdir, readFile, rm, stat, symlink, utimes, writeFile } from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
 import test from "node:test";
@@ -3714,33 +3714,37 @@ test("applyOfflineSyncSnapshot consumes readIncomingFile for records without inl
   }
 });
 
-test("buildOfflineSyncSnapshot skips a file that vanishes during the walk", async () => {
+test("buildOfflineSyncSnapshot fails closed when a file vanishes during digest", async () => {
   const root = await tempDir("remnic-offline-vanish");
   try {
     await write(root, "facts/keep.md", "keep");
     await write(root, "facts/gone.md", "gone");
-    const snapshot = await buildOfflineSyncSnapshot({
-      root,
-      sourceId: "local",
-      includeContent: false,
-      readFileDigest: async ({ path: relPath, filePath }) => {
-        if (relPath.endsWith("gone.md")) {
-          await rm(filePath);
-          throw Object.assign(new Error("ENOENT: vanished"), { code: "ENOENT" });
-        }
-        const content = await readFile(filePath);
-        return {
-          sha256: createHash("sha256").update(content).digest("hex"),
-          bytes: content.length,
-        };
-      },
-    });
-    assert.deepEqual(snapshot.files.map((file) => file.path), ["facts/keep.md"]);
+    await assert.rejects(
+      () => buildOfflineSyncSnapshot({
+        root,
+        sourceId: "local",
+        includeContent: false,
+        readFileDigest: async ({ path: relPath, filePath }) => {
+          if (relPath.endsWith("gone.md")) {
+            await rm(filePath);
+            throw Object.assign(new Error("ENOENT: vanished"), { code: "ENOENT" });
+          }
+          const content = await readFile(filePath);
+          return {
+            sha256: createHash("sha256").update(content).digest("hex"),
+            bytes: content.length,
+          };
+        },
+      }),
+      /census race|vanished during digest|retry the snapshot/i,
+    );
+    assert.equal(await readUtf8(root, "facts/keep.md"), "keep");
   } finally {
     await rm(root, { recursive: true, force: true });
   }
 });
 
+
 test("base snapshot fails when a base-known file vanishes before the fast-base stat", async () => {
   const root = await tempDir("remnic-offline-base-vanish");
   try {
@@ -3770,7 +3774,7 @@ test("base snapshot fails when a base-known file vanishes before the fast-base s
   }
 });
 
-test("base snapshot omits a brand-new file that vanishes during the walk", async () => {
+test("base snapshot omits a brand-new file that vanishes before the fast-base stat", async () => {
   const root = await tempDir("remnic-offline-new-vanish");
   try {
     await write(root, "facts/keep.md", "keep");
@@ -3798,6 +3802,7 @@ test("base snapshot omits a brand-new file that vanishes during the walk", async
   }
 });
 
+
 test("snapshot fails when an embedding generation member vanishes during the walk", async () => {
   const root = await tempDir("remnic-offline-shard-vanish");
   try {
@@ -3822,3 +3827,291 @@ test("snapshot fails when an embedding generation member vanishes during the wal
     await rm(root, { recursive: true, force: true });
   }
 });
+
+test("a transient snapshot omission must not emit a delete for a shared-base file", async () => {
+  const root = await tempDir("remnic-offline-uncertain-delete");
+  try {
+    await write(root, "facts/keep.md", "keep");
+    await write(root, "facts/race.md", "race-v1");
+    const base = await buildOfflineSyncSnapshot({
+      root,
+      sourceId: "local",
+      includeContent: false,
+    });
+    // Live census fail-closes on digest ENOENT. A stale/incomplete currentFiles
+    // list is still possible from an older client cache — that path must not
+    // emit a confirmed delete when the file is back on disk.
+    const incompleteFiles = base.files.filter((file) => file.path !== "facts/race.md");
+    assert.deepEqual(
+      incompleteFiles.map((file) => file.path),
+      ["facts/keep.md"],
+      "fixture: the race path is absent from the incomplete currentFiles view",
+    );
+    await write(root, "facts/race.md", "race-v1");
+    const changeset = await buildOfflineSyncChangesetFromSnapshot({
+      root,
+      sourceId: "local",
+      currentFiles: incompleteFiles,
+      baseFiles: base.files,
+    });
+    assert.equal(
+      changeset.changes.filter((change) => change.path === "facts/race.md" && change.type === "delete").length,
+      0,
+      "an unreadable/recreated path must not become a confirmed delete against the shared base",
+    );
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});
+
+test("apply must not overwrite a recreated local file omitted from the local census", async () => {
+  const root = await tempDir("remnic-offline-uncertain-overwrite");
+  try {
+    const recreated = "recreated local bytes";
+    const incoming = "incoming remote bytes";
+    await write(root, "facts/race.md", recreated);
+    const incomingSha = createHash("sha256").update(incoming).digest("hex");
+    // Local census omitted the recreated path (stale/racy currentFiles).
+    // Incoming is base-less, so the apply path would treat it as a pure create.
+    const result = await applyOfflineSyncSnapshot({
+      root,
+      currentFiles: [],
+      baseFiles: [],
+      snapshot: {
+        format: OFFLINE_SYNC_SNAPSHOT_FORMAT,
+        schemaVersion: 1,
+        createdAt: new Date().toISOString(),
+        sourceId: "remote",
+        includeTranscripts: false,
+        files: [{
+          path: "facts/race.md",
+          sha256: incomingSha,
+          bytes: Buffer.byteLength(incoming),
+          mtimeMs: 2,
+          contentBase64: Buffer.from(incoming).toString("base64"),
+        }],
+      },
+    });
+    assert.equal(
+      await readUtf8(root, "facts/race.md"),
+      recreated,
+      "recreated local content must survive when the local census omitted it",
+    );
+    assert.equal(result.upserted, 0, "must not publish over an uncertain local path");
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});
+
+test("changeset apply must not overwrite a recreated local file omitted by controlled digest census", async () => {
+  // Public applyOfflineSyncChangeset builds local currentFiles via
+  // buildOfflineSyncSnapshotForPaths when currentFiles is omitted. A controlled
+  // digest that recreates local bytes then throws ENOENT must fail closed at
+  // the shared census boundary — never export confirmed absence and overwrite.
+  const root = await tempDir("remnic-offline-changeset-uncertain-overwrite");
+  try {
+    const recreated = "recreated local bytes";
+    const incoming = "incoming remote bytes";
+    const incomingSha = createHash("sha256").update(incoming).digest("hex");
+    await write(root, "facts/race.md", recreated);
+
+    await assert.rejects(
+      () => applyOfflineSyncChangeset({
+        root,
+        // No currentFiles: exercise the public snapshotForPaths census path.
+        returnCurrentFiles: false,
+        writeConflictCopies: false,
+        readFileDigest: async ({ path: relPath, filePath }) => {
+          if (relPath === "facts/race.md") {
+            await writeFile(filePath, recreated);
+            throw Object.assign(new Error("ENOENT: vanished during census"), { code: "ENOENT" });
+          }
+          const content = await readFile(filePath);
+          return {
+            sha256: createHash("sha256").update(content).digest("hex"),
+            bytes: content.length,
+          };
+        },
+        changeset: {
+          format: "remnic.offline-sync.changeset.v1",
+          schemaVersion: 1,
+          createdAt: new Date().toISOString(),
+          sourceId: "remote",
+          includeTranscripts: false,
+          changes: [{
+            type: "upsert",
+            path: "facts/race.md",
+            file: {
+              path: "facts/race.md",
+              sha256: incomingSha,
+              bytes: Buffer.byteLength(incoming),
+              mtimeMs: 2,
+              contentBase64: Buffer.from(incoming).toString("base64"),
+            },
+          }],
+        },
+      }),
+      /census race|vanished during digest|retry the snapshot/i,
+    );
+
+    assert.equal(
+      await readUtf8(root, "facts/race.md"),
+      recreated,
+      "recreated local content must survive a fail-closed public changeset census race",
+    );
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});
+
+test("sender snapshot omission from digest race must not delete receiver shared-base file", async () => {
+  // Root premise: sender digest race must not export an apparently-complete
+  // snapshot that omits the path. Fail closed at the shared census boundary so
+  // no consumer can assign confirmed absence. Sender and receiver bytes stay.
+  const senderRoot = await tempDir("remnic-offline-sender-uncertain-omit");
+  const receiverRoot = await tempDir("remnic-offline-receiver-shared-base");
+  try {
+    const good = "unchanged good fact bytes";
+    const keep = "keep peer";
+    await write(senderRoot, "facts/race.md", good);
+    await write(senderRoot, "facts/keep.md", keep);
+    await write(receiverRoot, "facts/race.md", good);
+    await write(receiverRoot, "facts/keep.md", keep);
+
+    const sharedBase = await buildOfflineSyncSnapshot({
+      root: receiverRoot,
+      sourceId: "shared",
+      includeContent: false,
+    });
+    assert.ok(
+      sharedBase.files.some((file) => file.path === "facts/race.md"),
+      "fixture: shared base includes the good race path",
+    );
+
+    await assert.rejects(
+      () => buildOfflineSyncSnapshot({
+        root: senderRoot,
+        sourceId: "sender",
+        includeContent: false,
+        readFileDigest: async ({ path: relPath, filePath }) => {
+          if (relPath === "facts/race.md") {
+            await writeFile(filePath, good);
+            throw Object.assign(new Error("ENOENT: vanished during sender census"), { code: "ENOENT" });
+          }
+          const content = await readFile(filePath);
+          return {
+            sha256: createHash("sha256").update(content).digest("hex"),
+            bytes: content.length,
+          };
+        },
+      }),
+      /census race|vanished during digest|retry the snapshot/i,
+    );
+
+    assert.equal(await readUtf8(senderRoot, "facts/race.md"), good, "sender bytes preserved");
+    assert.equal(await readUtf8(receiverRoot, "facts/race.md"), good, "receiver bytes preserved without a false snapshot");
+    assert.equal(await readUtf8(receiverRoot, "facts/keep.md"), keep);
+  } finally {
+    await rm(senderRoot, { recursive: true, force: true });
+    await rm(receiverRoot, { recursive: true, force: true });
+  }
+});
+
+
+test("a file vanishing during the changeset upsert read keeps the retryable changed-while-building message", async () => {
+  const root = await mkdtemp(path.join(os.tmpdir(), "remnic-offline-changeset-vanish-"));
+  try {
+    const rel = "facts/volatile.md";
+    const bytes = Buffer.from("volatile bytes\n");
+    await mkdir(path.join(root, "facts"), { recursive: true });
+    await writeFile(path.join(root, rel), bytes);
+    const currentFile = {
+      path: rel,
+      sha256: createHash("sha256").update(bytes).digest("hex"),
+      bytes: bytes.byteLength,
+      mtimeMs: 1,
+    };
+    await assert.rejects(
+      buildOfflineSyncChangesetFromSnapshot({
+        root,
+        sourceId: "laptop",
+        baseFiles: [],
+        currentFiles: [currentFile],
+        readFile: async () => {
+          await rm(path.join(root, rel), { force: true });
+          throw Object.assign(new Error("ENOENT: file vanished"), { code: "ENOENT" });
+        },
+      }),
+      (error: Error) => {
+        // The CLI retry loop matches this exact prefix to defer the path and rebuild the census.
+        assert.equal(error.message, `offline sync file changed while building changeset: ${rel}`);
+        return true;
+      },
+    );
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});
+
+test("a tracked file replaced by a symlink is a local delete, not a push-blocking error", async () => {
+  const root = await mkdtemp(path.join(os.tmpdir(), "remnic-offline-symlink-replaced-"));
+  const outside = await mkdtemp(path.join(os.tmpdir(), "remnic-offline-symlink-target-"));
+  try {
+    const rel = "facts/tracked.md";
+    const bytes = Buffer.from("tracked bytes\n");
+    await mkdir(path.join(root, "facts"), { recursive: true });
+    await writeFile(path.join(outside, "target.md"), "outside\n");
+    await symlink(path.join(outside, "target.md"), path.join(root, rel));
+    const baseFile = {
+      path: rel,
+      sha256: createHash("sha256").update(bytes).digest("hex"),
+      bytes: bytes.byteLength,
+      mtimeMs: 1,
+    };
+    const changeset = await buildOfflineSyncChangesetFromSnapshot({
+      root,
+      sourceId: "laptop",
+      baseFiles: [baseFile],
+      currentFiles: [],
+    });
+    assert.deepEqual(
+      changeset.changes.map((change) => [change.type, change.path]),
+      [["delete", rel]],
+      "a symlink is not a regular local file: it reads as a delete, as before the census change",
+    );
+  } finally {
+    await rm(root, { recursive: true, force: true });
+    await rm(outside, { recursive: true, force: true });
+  }
+});
+
+test("a tracked file under a directory replaced by an in-root symlink is a local delete", async () => {
+  const root = await mkdtemp(path.join(os.tmpdir(), "remnic-offline-symlink-parent-"));
+  try {
+    const rel = "facts/tracked.md";
+    const bytes = Buffer.from("tracked bytes\n");
+    await mkdir(path.join(root, "elsewhere"), { recursive: true });
+    await writeFile(path.join(root, "elsewhere", "tracked.md"), bytes);
+    // The census skips a symlinked directory, so facts/tracked.md is not in it.
+    await symlink(path.join(root, "elsewhere"), path.join(root, "facts"));
+    const baseFile = {
+      path: rel,
+      sha256: createHash("sha256").update(bytes).digest("hex"),
+      bytes: bytes.byteLength,
+      mtimeMs: 1,
+    };
+    const changeset = await buildOfflineSyncChangesetFromSnapshot({
+      root,
+      sourceId: "laptop",
+      baseFiles: [baseFile],
+      currentFiles: [],
+    });
+    assert.deepEqual(
+      changeset.changes.map((change) => [change.type, change.path]),
+      [["delete", rel]],
+      "a path reached only through a symlinked parent is absent, as the census sees it",
+    );
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});