@remnic/core 9.3.630 → 9.3.632

package/src/wearables/pipeline.test.ts

@@ -544,14 +544,15 @@ test("zero provider conversations never clobber an existing transcript", async (
   }
 });
 
-test("page-capped days carry a visible partial marker and keep warning", async () => {
+test("looping pagination cursors stop with a partial marker and keep warning", async () => {
   const memoryDir = mkdtempSync(path.join(tmpdir(), "remnic-pipeline-"));
   try {
     const conversation = makeConversation("c1", "2026-06-11", [
       { speaker: "user", isWearer: true, text: "First chunk of a very long recorded day of conversations." },
       { speaker: "Speaker 2", text: "Indeed, the recordings just keep going on and on today." },
     ]);
-    // A connector that always reports another page (pathological).
+    // A connector whose cursor loops forever (pathological) — cycle
+    // detection must stop it; no page-count cap truncates real data.
     const endlessConnector = {
       id: "testsource",
       displayName: "Test Source",
@@ -567,14 +568,96 @@ test("page-capped days carry a visible partial marker and keep warning", async (
     };
     const { deps, written } = makeDeps(memoryDir);
     const first = await syncWearableSource(endlessConnector, settings(), config(), { days: 1 }, deps);
-    assert.ok(first.warnings.some((warning) => warning.includes("stopped paginating")));
+    assert.ok(first.warnings.some((warning) => warning.includes("repeated cursor")));
     assert.equal(written.length, 1);
     assert.match(written[0].serialized, /pagination safety cap reached/);
 
     // Identical second sync: file unchanged (no rewrite), warning persists.
     const second = await syncWearableSource(endlessConnector, settings(), config(), { days: 1 }, deps);
     assert.equal(second.transcriptsWritten.length, 0);
-    assert.ok(second.warnings.some((warning) => warning.includes("stopped paginating")));
+    assert.ok(second.warnings.some((warning) => warning.includes("repeated cursor")));
+  } finally {
+    rmSync(memoryDir, { recursive: true, force: true });
+  }
+});
+
+test("looping providers that re-serve rows never duplicate conversations", async () => {
+  const memoryDir = mkdtempSync(path.join(tmpdir(), "remnic-pipeline-"));
+  try {
+    const conversation = makeConversation("c1", "2026-06-11", [
+      { speaker: "user", isWearer: true, text: "A conversation the provider keeps re-serving on every page." },
+      { speaker: "Speaker 2", text: "The looping pagination should still store this exactly once." },
+    ]);
+    // Pathological: every page returns the SAME row and loops the cursor.
+    const reServingConnector = {
+      id: "testsource",
+      displayName: "Test Source",
+      async verifyAuth() {
+        return { ok: true };
+      },
+      async fetchConversations() {
+        return { conversations: [conversation], nextCursor: "loop" };
+      },
+    };
+    const { deps, written } = makeDeps(memoryDir);
+    const summary = await syncWearableSource(
+      reServingConnector,
+      settings(),
+      config(),
+      { days: 1 },
+      deps,
+    );
+    assert.equal(summary.conversations, 1, "re-served rows collapse by conversation id");
+    assert.ok(summary.warnings.some((warning) => warning.includes("repeated cursor")));
+    assert.equal(written.length, 1);
+    const occurrences = written[0].serialized.split("conversation c1").length - 1;
+    assert.equal(occurrences, 1, "day file stores the conversation exactly once");
+  } finally {
+    rmSync(memoryDir, { recursive: true, force: true });
+  }
+});
+
+test("long provider days paginate fully — no page-count truncation", async () => {
+  const memoryDir = mkdtempSync(path.join(tmpdir(), "remnic-pipeline-"));
+  try {
+    // 60 pages (beyond the old 50-page cap), each with one conversation.
+    const PAGES = 60;
+    const longDayConnector = {
+      id: "testsource",
+      displayName: "Test Source",
+      async verifyAuth() {
+        return { ok: true };
+      },
+      async fetchConversations(opts: { cursor?: string | null }) {
+        const page = opts.cursor ? Number(opts.cursor) : 0;
+        return {
+          conversations: [
+            makeConversation(`c${page + 1}`, "2026-06-11", [
+              { speaker: "user", isWearer: true, text: `Recorded conversation segment number ${page + 1} from the long day.` },
+              { speaker: "Speaker 2", text: "Acknowledged, that part of the day was captured as well." },
+            ]),
+          ],
+          nextCursor: page + 1 < PAGES ? String(page + 1) : null,
+        };
+      },
+    };
+    const { deps } = makeDeps(memoryDir);
+    const summary = await syncWearableSource(
+      longDayConnector,
+      settings(),
+      config(),
+      { days: 1 },
+      deps,
+    );
+    assert.equal(summary.conversations, PAGES, "every page's conversation synced");
+    assert.equal(
+      summary.warnings.filter(
+        (warning) =>
+          warning.includes("stopped paginating") || warning.includes("repeated cursor"),
+      ).length,
+      0,
+      "a real long day is never truncated or warned",
+    );
   } finally {
     rmSync(memoryDir, { recursive: true, force: true });
   }

package/src/wearables/pipeline.ts

@@ -52,10 +52,13 @@ import type {
   WearablesConfig,
 } from "./types.js";
 
-/** Safety cap on pages fetched per day window. */
-const MAX_PAGES_PER_DAY = 50;
-/** Safety cap on native-memory pages per sync. */
-const MAX_NATIVE_PAGES = 20;
+/**
+ * Pathological-provider backstop on pagination loops. Day fetches are
+ * intentionally UNLIMITED for real data — a full day must never be
+ * truncated — so runaway protection comes from cursor-cycle detection
+ * plus this far-out-of-band ceiling, not from a content-sized cap.
+ */
+const PAGE_SAFETY_CEILING = 10_000;
 /** Default lookback window (today + yesterday) for unscoped syncs. */
 const DEFAULT_SYNC_DAYS = 2;
 const MAX_SYNC_DAYS = 90;
@@ -186,23 +189,42 @@ async function fetchAllConversationsForDate(
   signal: AbortSignal | undefined,
   warnings: string[],
 ): Promise<{ conversations: WearableConversation[]; partial: boolean }> {
-  const conversations: WearableConversation[] = [];
+  // Keyed by conversation id: a looping or overlapping provider can
+  // re-serve rows it already returned, and appending blindly would
+  // store the same conversation twice in the day file (Cursor review
+  // on PR #1464). Map keeps first-seen order; a re-served id replaces
+  // its entry in place, so the provider's LATEST version of a
+  // conversation wins — exactly what the current-day refresh wants.
+  const byId = new Map<string, WearableConversation>();
   let cursor: string | null | undefined = undefined;
-  for (let page = 0; page < MAX_PAGES_PER_DAY; page++) {
+  const seenCursors = new Set<string>();
+  const collect = () => [...byId.values()];
+  for (let page = 0; page < PAGE_SAFETY_CEILING; page++) {
     const result = await connector.fetchConversations({
       date,
       timezone,
       cursor,
       signal,
     });
-    conversations.push(...result.conversations);
-    if (!result.nextCursor) return { conversations, partial: false };
+    for (const conversation of result.conversations) {
+      byId.set(conversation.id, conversation);
+    }
+    if (!result.nextCursor) return { conversations: collect(), partial: false };
+    // A repeated cursor means the provider's pagination is looping —
+    // following it again would refetch the same page forever.
+    if (seenCursors.has(result.nextCursor)) {
+      warnings.push(
+        `${connector.id}: provider pagination repeated cursor on ${date} — stopped to avoid an infinite loop; day may be partially synced (every sync refetches and re-warns while the provider misbehaves)`,
+      );
+      return { conversations: collect(), partial: true };
+    }
+    seenCursors.add(result.nextCursor);
     cursor = result.nextCursor;
   }
   warnings.push(
-    `${connector.id}: stopped paginating ${date} after ${MAX_PAGES_PER_DAY} pages — day may be partially synced (every sync refetches and re-warns until the provider day fits the cap)`,
+    `${connector.id}: stopped paginating ${date} after the ${PAGE_SAFETY_CEILING}-page safety ceiling — day may be partially synced (every sync refetches and re-warns while this persists)`,
   );
-  return { conversations, partial: true };
+  return { conversations: collect(), partial: true };
 }
 
 /** Visible marker appended to day files whose fetch hit the page cap. */
@@ -521,7 +543,8 @@ export async function syncWearableSource(
           : {}),
       };
       let cursor: string | null | undefined = undefined;
-      for (let page = 0; page < MAX_NATIVE_PAGES; page++) {
+      const seenNativeCursors = new Set<string>();
+      for (let page = 0; page < PAGE_SAFETY_CEILING; page++) {
         const result = await connector.fetchNativeMemories({
           cursor,
           signal: options.signal,
@@ -538,10 +561,17 @@ export async function syncWearableSource(
         importedNativeIds.push(...imported.importedIds);
         for (const id of imported.importedIds) alreadyImported.add(id);
         if (!result.nextCursor) break;
+        if (seenNativeCursors.has(result.nextCursor)) {
+          summary.warnings.push(
+            `${connector.id}: provider pagination repeated cursor during native-memory import — stopped to avoid an infinite loop; remaining items import on the next sync`,
+          );
+          break;
+        }
+        seenNativeCursors.add(result.nextCursor);
         cursor = result.nextCursor;
-        if (page === MAX_NATIVE_PAGES - 1) {
+        if (page === PAGE_SAFETY_CEILING - 1) {
           summary.warnings.push(
-            `${connector.id}: stopped native-memory import after ${MAX_NATIVE_PAGES} pages — remaining items import on the next sync`,
+            `${connector.id}: stopped native-memory import at the ${PAGE_SAFETY_CEILING}-page safety ceiling — remaining items import on the next sync`,
           );
         }
       }

package/src/wearables/types.ts

@@ -176,7 +176,9 @@ export interface WearableSourceSettings {
   /** Drop extracted facts scored below this importance level. */
   minImportance: ImportanceLevel;
   /**
-   * Cap on memories created per source per day. 0 disables the cap.
+   * Cap on memories created per source per day. 0 (the default)
+   * disables the cap — the smart trust pipeline is the quality gate,
+   * and a count cap would drop real memories on busy days.
    */
   maxMemoriesPerDay: number;
   /**
@@ -250,6 +252,23 @@ export interface WearablesConfig {
    * Default false.
    */
   digestEnabled: boolean;
+  /**
+   * Periodically refresh transcripts in-process (long-lived hosts).
+   * Every tick re-syncs `autoSyncDays` ending today — existing day
+   * files included, so the current day keeps growing while the
+   * wearable records. Default true.
+   */
+  autoSyncEnabled: boolean;
+  /** Minutes between auto-sync ticks (1-1440). Default 15. */
+  autoSyncIntervalMinutes: number;
+  /** Rolling window (days ending today) per tick (1-90). Default 2. */
+  autoSyncDays: number;
+  /**
+   * Once-per-local-day deep pass window (days, 0-90) picking up late
+   * uploads and provider re-processing. 0 disables; otherwise must be
+   * >= autoSyncDays. Default 7.
+   */
+  autoSyncDeepDays: number;
   /** Correction rules from config (merged with CLI-managed rules). */
   corrections: WearableCorrectionRule[];
   /** Per-source settings, keyed by connector id. */