@remnic/core 9.15.2 → 9.16.0

package/src/activity/pipeline.ts

@@ -0,0 +1,223 @@
+import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
+import { randomUUID } from "node:crypto";
+import path from "node:path";
+
+import { displayErrorDetail } from "../runtime/better-sqlite.js";
+
+import {
+  activityDayWindow,
+  activityDigestPath,
+  composeActivityDigestBody,
+  composeActivityDigestMeta,
+  isValidActivityDate,
+  serializeActivityDigest,
+} from "./digest.js";
+import type { ActivityStore } from "./store.js";
+import type { ActivitySnapshot, ActivitySourceClient } from "./types.js";
+
+const MAX_SYNC_PAGES = 10_000;
+
+/**
+ * Sync cursors are scoped per (machine, local day): each date resumes its own
+ * pagination independently, so a multi-day window never inherits an earlier
+ * day's cursor (which could skip backfill). Encoded into the store's single
+ * key column with a NUL separator that cannot occur in a machine label.
+ */
+export function activityCursorKey(machine: string, date: string): string {
+  return `${machine}\u0000${date}`;
+}
+
+// Serializes the list -> compose -> write digest section per (memoryDir, date)
+// so two sources syncing the same day cannot lose-update each other's digest:
+// the last writer in the chain lists every row present and wins.
+const digestLocks = new Map<string, Promise<unknown>>();
+
+function withDigestLock<T>(key: string, fn: () => Promise<T>): Promise<T> {
+  const prior = digestLocks.get(key) ?? Promise.resolve();
+  const next = prior.then(fn, fn);
+  const tail = next.then(
+    () => undefined,
+    () => undefined,
+  );
+  digestLocks.set(key, tail);
+  void tail.then(() => {
+    if (digestLocks.get(key) === tail) digestLocks.delete(key);
+  });
+  return next;
+}
+
+export interface ActivitySyncOptions {
+  date: string;
+  timezone: string;
+  memoryDir: string;
+  store: ActivityStore;
+  signal?: AbortSignal;
+  /** Runaway-pagination guard; defaults to MAX_SYNC_PAGES. */
+  maxPages?: number;
+  /**
+   * Search-index refresh, run after a digest is (re)written so the durable
+   * markdown becomes discoverable (AGENTS.md rule 31; the digest lives in the
+   * QMD collection root but bypasses the extraction->persist->index pipeline).
+   * Host-agnostic: the caller injects the core index seam (SearchBackend.update).
+   * Best-effort — a failure never fails the sync (rows/digest are durable and
+   * index on the next update); it surfaces as `reindexError`.
+   */
+  afterWrites?: (signal?: AbortSignal) => Promise<void>;
+}
+
+export interface ActivitySyncResult {
+  machine: string;
+  fetched: number;
+  inserted: number;
+  duplicates: number;
+  cursor: string | null;
+  digestWritten: boolean;
+  /** Set when the post-write reindex threw; the sync still succeeded. */
+  reindexError?: string;
+}
+
+function snapshotForMachine(snapshot: ActivitySnapshot, machine: string): ActivitySnapshot {
+  return {
+    machine,
+    capturedAtUtc: snapshot.capturedAtUtc,
+    app: snapshot.app,
+    windowTitle: snapshot.windowTitle,
+    ...(snapshot.browserUrl === undefined ? {} : { browserUrl: snapshot.browserUrl }),
+    text: snapshot.text,
+    textSource: snapshot.textSource,
+    contentHash: snapshot.contentHash,
+    ...(snapshot.simhash === undefined ? {} : { simhash: snapshot.simhash }),
+  };
+}
+
+async function writeDigestIfChanged(memoryDir: string, date: string, serialized: string): Promise<boolean> {
+  const target = activityDigestPath(memoryDir, date);
+  try {
+    if ((await readFile(target, "utf8")) === serialized) return false;
+  } catch (error: unknown) {
+    if (!(error instanceof Error) || !("code" in error) || error.code !== "ENOENT") throw error;
+  }
+
+  await mkdir(path.dirname(target), { recursive: true, mode: 0o700 });
+  const temporary = `${target}.${process.pid}.${randomUUID()}.tmp`;
+  await writeFile(temporary, serialized, { encoding: "utf8", mode: 0o600 });
+  await rename(temporary, target);
+  return true;
+}
+
+/**
+ * Pull one source's local-day snapshots and persist its cursor only after every
+ * page and the derived digest are durable. Replays after a partial failure are
+ * safe because ActivityStore deduplicates an exact captured snapshot.
+ */
+export async function syncActivitySource(
+  source: ActivitySourceClient,
+  options: ActivitySyncOptions,
+): Promise<ActivitySyncResult> {
+  if (!isValidActivityDate(options.date)) {
+    throw new RangeError(`Invalid activity date "${options.date}"; expected a real YYYY-MM-DD day.`);
+  }
+  if (source.machineLabel.trim().length === 0) {
+    throw new RangeError("activity source machine label must not be empty");
+  }
+  const maxPages = options.maxPages ?? MAX_SYNC_PAGES;
+  if (!Number.isInteger(maxPages) || maxPages < 1) {
+    throw new RangeError("activity sync maxPages must be a positive integer");
+  }
+
+  const cursorKey = activityCursorKey(source.machineLabel, options.date);
+  let cursor = options.store.getCursor(cursorKey);
+  let pages = 0;
+  let fetched = 0;
+  let inserted = 0;
+  let duplicates = 0;
+  const seenCursors = new Set<string>();
+  let completed = false;
+
+  while (pages < maxPages) {
+    options.signal?.throwIfAborted();
+    const page = await source.fetchSnapshots({
+      date: options.date,
+      timezone: options.timezone,
+      cursor,
+      signal: options.signal,
+    });
+    pages += 1;
+    fetched += page.snapshots.length;
+
+    for (const snapshot of page.snapshots) {
+      // Persist every row keyed by its own timestamp: an out-of-window snapshot
+      // (replay file, timezone-boundary bug) is durable under its own day and
+      // never lost, while the day digest below is a window query so it never
+      // pollutes options.date. A malformed instant is rejected loudly here.
+      const result = options.store.insertSnapshot(snapshotForMachine(snapshot, source.machineLabel));
+      if (result.inserted) inserted += 1;
+      else duplicates += 1;
+    }
+    if (page.nextCursor === null) {
+      completed = true;
+      break;
+    }
+    if (typeof page.nextCursor !== "string" || page.nextCursor.length === 0) {
+      throw new TypeError("activity source returned an invalid cursor");
+    }
+    if (seenCursors.has(page.nextCursor)) {
+      throw new Error("activity source returned a repeated cursor");
+    }
+    seenCursors.add(page.nextCursor);
+    cursor = page.nextCursor;
+  }
+
+  // Only a loop that ran out of its page budget without seeing a terminal
+  // null cursor is a runaway; completing on the final allowed page is normal.
+  if (!completed) {
+    throw new Error(`activity source exceeded ${maxPages} pages`);
+  }
+
+  // Re-check abort before the durable write section: stop() may have fired after
+  // the last page returned, or while this date waited behind digestLocks. An
+  // aborted tick must not compose/write a digest or advance the cursor.
+  options.signal?.throwIfAborted();
+  const digestWritten = await withDigestLock(`${options.memoryDir}\u0000${options.date}`, async () => {
+    options.signal?.throwIfAborted();
+    const { startUtc, endUtc } = activityDayWindow(options.date, options.timezone);
+    const snapshots = options.store.listSnapshotsForDay(null, startUtc, endUtc);
+    const body = composeActivityDigestBody(options.date, options.timezone, snapshots);
+    const machines = [...new Set(snapshots.map((snapshot) => snapshot.machine))].sort((a, b) =>
+      a < b ? -1 : a > b ? 1 : 0,
+    );
+    const serialized = serializeActivityDigest(composeActivityDigestMeta(options.date, machines, snapshots, body), body);
+    return writeDigestIfChanged(options.memoryDir, options.date, serialized);
+  });
+
+  options.signal?.throwIfAborted();
+  options.store.setCursor(cursorKey, cursor);
+
+  // Rows and digest are durable and the cursor has advanced; refresh the search
+  // index so the new digest is discoverable (rule 31). Best-effort: a failure
+  // is reported, not thrown — the data indexes on the next update.
+  let reindexError: string | undefined;
+  if (digestWritten && options.afterWrites !== undefined) {
+    try {
+      await options.afterWrites(options.signal);
+    } catch (error: unknown) {
+      // reindexError is part of the exported ActivitySyncResult. Keep our
+      // controlled QMD strict messages, but reduce opaque backend errors (which
+      // can embed absolute paths / loader stacks) to a sanitized name+code.
+      reindexError =
+        error instanceof Error && error.message.startsWith("QMD ")
+          ? error.message
+          : displayErrorDetail(error) || "reindex failed";
+    }
+  }
+
+  return {
+    machine: source.machineLabel,
+    fetched,
+    inserted,
+    duplicates,
+    cursor,
+    digestWritten,
+    ...(reindexError === undefined ? {} : { reindexError }),
+  };
+}

package/src/activity/reindex.test.ts

@@ -0,0 +1,97 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { refreshActivityIndex, type ActivityIndexRefresher } from "./reindex.js";
+
+function refresher(overrides: Partial<ActivityIndexRefresher>, calls: string[]): ActivityIndexRefresher {
+  return {
+    async update() {
+      calls.push("update");
+    },
+    ...overrides,
+  };
+}
+
+test("refreshActivityIndex prefers the forced+strict collection refresh", async () => {
+  const calls: string[] = [];
+  const qmd = refresher(
+    {
+      async updateStrict() {
+        calls.push("updateStrict");
+      },
+      async updateCollectionStrict(collection: string) {
+        calls.push(`updateCollectionStrict:${collection}`);
+      },
+    },
+    calls,
+  );
+
+  await refreshActivityIndex(qmd, "openclaw-engram");
+  assert.deepEqual(calls, ["updateCollectionStrict:openclaw-engram"], "forces past the fail-open min-interval gate");
+});
+
+test("refreshActivityIndex falls back to updateStrict, then update", async () => {
+  const strictCalls: string[] = [];
+  await refreshActivityIndex(
+    refresher(
+      {
+        async updateStrict() {
+          strictCalls.push("updateStrict");
+        },
+      },
+      strictCalls,
+    ),
+    "openclaw-engram",
+  );
+  assert.deepEqual(strictCalls, ["updateStrict"], "uses strict global refresh when no collection-strict path exists");
+
+  const plainCalls: string[] = [];
+  await refreshActivityIndex(refresher({}, plainCalls), "openclaw-engram");
+  assert.deepEqual(plainCalls, ["update"], "falls back to plain update when no strict path exists");
+});
+
+test("refreshActivityIndex skips collection-strict for an empty collection name", async () => {
+  const calls: string[] = [];
+  await refreshActivityIndex(
+    refresher(
+      {
+        async updateStrict() {
+          calls.push("updateStrict");
+        },
+        async updateCollectionStrict(collection: string) {
+          calls.push(`updateCollectionStrict:${collection}`);
+        },
+      },
+      calls,
+    ),
+    "   ",
+  );
+  assert.deepEqual(calls, ["updateStrict"], "no collection -> strict global refresh, never a blank-collection call");
+});
+
+test("refreshActivityIndex propagates a strict-refresh failure (no fake success)", async () => {
+  const calls: string[] = [];
+  const qmd = refresher(
+    {
+      async updateCollectionStrict() {
+        calls.push("updateCollectionStrict");
+        throw new Error("QMD update skipped by per-collection failure backoff");
+      },
+    },
+    calls,
+  );
+  await assert.rejects(refreshActivityIndex(qmd, "openclaw-engram"), /failure backoff/);
+});
+
+test("refreshActivityIndex forwards the abort signal to the backend refresh", async () => {
+  const controller = new AbortController();
+  let seenSignal: AbortSignal | undefined;
+  const qmd: ActivityIndexRefresher = {
+    async update() {},
+    async updateCollectionStrict(_collection, execution) {
+      seenSignal = execution?.signal;
+    },
+  };
+  await refreshActivityIndex(qmd, "openclaw-engram", controller.signal);
+  assert.equal(seenSignal, controller.signal, "the tick's abort signal reaches the QMD refresh");
+});

package/src/activity/reindex.ts

@@ -0,0 +1,40 @@
+/**
+ * Forced, non-fake search-index refresh for direct activity digest writes
+ * (issue #1899; AGENTS.md rule 31). The digest is written straight to the QMD
+ * collection root, bypassing the extraction->persist->index pipeline, so it must
+ * explicitly trigger a refresh — and that refresh must actually run.
+ *
+ * Ordinary `SearchBackend.update()` is fail-open: it silently returns when the
+ * backend is unavailable, in failure backoff, or inside its min-interval gate.
+ * A freshly written digest hit by that gate would stay unsearchable until an
+ * unrelated reindex, and the per-day digest-unchanged guard means the next tick
+ * won't retry. So prefer the forced/strict contracts: `updateCollectionStrict`
+ * bypasses the min-interval gate (force) AND throws on a genuine failure instead
+ * of reporting a fake success, then `updateStrict`, then plain `update`.
+ */
+
+import type { SearchExecutionOptions } from "../search/port.js";
+
+/** The subset of the search backend the activity reindex needs. */
+export interface ActivityIndexRefresher {
+  update(execution?: SearchExecutionOptions): Promise<void>;
+  updateStrict?(execution?: SearchExecutionOptions): Promise<void>;
+  updateCollectionStrict?(collection: string, execution?: SearchExecutionOptions): Promise<void>;
+}
+
+export async function refreshActivityIndex(
+  qmd: ActivityIndexRefresher,
+  collection: string,
+  signal?: AbortSignal,
+): Promise<void> {
+  const execution = signal === undefined ? undefined : { signal };
+  if (typeof qmd.updateCollectionStrict === "function" && collection.trim().length > 0) {
+    await qmd.updateCollectionStrict(collection, execution);
+    return;
+  }
+  if (typeof qmd.updateStrict === "function") {
+    await qmd.updateStrict(execution);
+    return;
+  }
+  await qmd.update(execution);
+}