@remnic/core 9.15.3 → 9.16.1

package/src/activity/source-client.ts

@@ -0,0 +1,159 @@
+import { validateActivityBaseUrl } from "./config.js";
+import { displayErrorDetail } from "../runtime/better-sqlite.js";
+import type { ActivitySnapshot, ActivitySnapshotPage, ActivitySourceCheck, ActivitySourceClient } from "./types.js";
+
+/** Abort a stalled capture-daemon request rather than hang the whole sync. */
+const DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
+
+export interface ActivityHttpSourceClientOptions {
+  machineLabel: string;
+  baseUrl: string;
+  token?: string;
+  /** Per-request timeout in ms; defaults to DEFAULT_REQUEST_TIMEOUT_MS. */
+  timeoutMs?: number;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null;
+}
+
+function stringField(value: Record<string, unknown>, field: string): string {
+  const result = value[field];
+  if (typeof result !== "string" || result.length === 0) {
+    throw new TypeError(`activity source response has no valid ${field}`);
+  }
+  return result;
+}
+
+/**
+ * Content fields (app/windowTitle/text) may be legitimately empty: a foreground
+ * window can be untitled or have no extractable text. Require the string TYPE
+ * (a missing/non-string field still fails loudly) but tolerate "", matching the
+ * durable store's contract — else a blank window poisons the day cursor by
+ * failing every replay of the same page.
+ */
+function contentStringField(value: Record<string, unknown>, field: string): string {
+  const result = value[field];
+  if (typeof result !== "string") {
+    throw new TypeError(`activity source response has invalid ${field}`);
+  }
+  return result;
+}
+
+function optionalStringField(value: Record<string, unknown>, field: string): string | undefined {
+  const result = value[field];
+  if (result === undefined || result === null) return undefined;
+  if (typeof result !== "string") throw new TypeError(`activity source response has invalid ${field}`);
+  return result;
+}
+
+function snapshotFromWire(value: unknown, machine: string): ActivitySnapshot {
+  if (!isRecord(value)) throw new TypeError("activity source returned a non-object snapshot");
+  const textSource = stringField(value, "textSource");
+  if (textSource !== "ax" && textSource !== "ocr") {
+    throw new TypeError("activity source response has invalid textSource");
+  }
+  const browserUrl = optionalStringField(value, "browserUrl");
+  const simhash = optionalStringField(value, "simhash");
+  return {
+    machine,
+    capturedAtUtc: stringField(value, "capturedAtUtc"),
+    app: contentStringField(value, "app"),
+    windowTitle: contentStringField(value, "windowTitle"),
+    ...(browserUrl === undefined ? {} : { browserUrl }),
+    text: contentStringField(value, "text"),
+    textSource,
+    contentHash: stringField(value, "contentHash"),
+    ...(simhash === undefined ? {} : { simhash }),
+  };
+}
+
+function responsePage(value: unknown, machine: string): ActivitySnapshotPage {
+  if (!isRecord(value) || !Array.isArray(value.snapshots)) {
+    throw new TypeError("activity source response has no snapshots array");
+  }
+  // A missing/undefined nextCursor (field omitted, not JSON null) is normal
+  // end-of-pagination — only a present non-string value is invalid.
+  const nextCursor = value.nextCursor === undefined ? null : value.nextCursor;
+  if (nextCursor !== null && typeof nextCursor !== "string") {
+    throw new TypeError("activity source response has invalid nextCursor");
+  }
+  return {
+    snapshots: value.snapshots.map((snapshot) => snapshotFromWire(snapshot, machine)),
+    nextCursor,
+  };
+}
+
+function requestUrl(baseUrl: string, path: string, query: Record<string, string | undefined>): string {
+  const url = new URL(path, baseUrl);
+  const parts = Object.entries(query)
+    .filter((entry): entry is [string, string] => entry[1] !== undefined)
+    .map(([key, value]) => `${encodeURIComponent(key)}=${encodeURIComponent(value)}`);
+  url.search = parts.join("&");
+  return url.toString();
+}
+
+export class ActivityHttpSourceClient implements ActivitySourceClient {
+  readonly machineLabel: string;
+  private readonly baseUrl: string;
+  private readonly token: string | undefined;
+  private readonly timeoutMs: number;
+
+  constructor(options: ActivityHttpSourceClientOptions) {
+    if (options.machineLabel.trim().length === 0) throw new RangeError("activity source machine label must not be empty");
+    if (options.timeoutMs !== undefined && (!Number.isFinite(options.timeoutMs) || options.timeoutMs <= 0)) {
+      throw new RangeError("activity source timeoutMs must be a positive number");
+    }
+    this.machineLabel = options.machineLabel;
+    this.baseUrl = validateActivityBaseUrl(options.baseUrl).toString();
+    this.token = options.token;
+    this.timeoutMs = options.timeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS;
+  }
+
+  private async request(url: string, signal?: AbortSignal): Promise<Response> {
+    // Bound every request so a stalled daemon aborts instead of hanging the
+    // sync forever; honor the caller's signal too when one is supplied.
+    const timeout = AbortSignal.timeout(this.timeoutMs);
+    const composed = signal === undefined ? timeout : AbortSignal.any([signal, timeout]);
+    const response = await fetch(url, {
+      headers: this.token === undefined ? undefined : { authorization: `Bearer ${this.token}` },
+      signal: composed,
+    });
+    if (!response.ok) throw new Error(`activity source HTTP ${response.status}`);
+    return response;
+  }
+
+  async verify(signal?: AbortSignal): Promise<ActivitySourceCheck> {
+    try {
+      await this.request(requestUrl(this.baseUrl, "/v1/health", {}), signal);
+      return { ok: true };
+    } catch (error: unknown) {
+      // The controlled `HTTP <status>` message is safe to surface verbatim;
+      // anything else (network/runtime fetch errors that can embed hostnames or
+      // absolute paths) is reduced to a sanitized name+code via
+      // displayErrorDetail, matching the wearables operator-facing path.
+      const detail =
+        error instanceof Error && error.message.startsWith("activity source HTTP ")
+          ? error.message.slice("activity source ".length)
+          : displayErrorDetail(error) || "request failed";
+      return { ok: false, detail };
+    }
+  }
+
+  async fetchSnapshots(opts: {
+    date: string;
+    timezone: string;
+    cursor?: string | null;
+    signal?: AbortSignal;
+  }): Promise<ActivitySnapshotPage> {
+    const response = await this.request(
+      requestUrl(this.baseUrl, "/v1/snapshots", {
+        date: opts.date,
+        timezone: opts.timezone,
+        cursor: opts.cursor ?? undefined,
+      }),
+      opts.signal,
+    );
+    return responsePage(await response.json(), this.machineLabel);
+  }
+}

package/src/activity/sync-registration.test.ts

@@ -0,0 +1,134 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import type { ActivitySyncRunSummary } from "./runner.js";
+import type { ActivitySyncSchedulerOptions } from "./scheduler.js";
+import {
+  ActivitySyncRegistrar,
+  type ActivitySyncRegistrarDeps,
+  type ActivitySyncSchedulerLike,
+} from "./sync-registration.js";
+
+class FakeScheduler implements ActivitySyncSchedulerLike {
+  starts = 0;
+  stops = 0;
+  readonly options: ActivitySyncSchedulerOptions;
+  constructor(options: ActivitySyncSchedulerOptions) {
+    this.options = options;
+  }
+  start(): unknown {
+    this.starts += 1;
+    return { registered: true, intervalMs: this.options.intervalMs ?? 0 };
+  }
+  async stop(): Promise<void> {
+    this.stops += 1;
+  }
+}
+
+function makeDeps(overrides: Partial<ActivitySyncRegistrarDeps> = {}): {
+  deps: ActivitySyncRegistrarDeps;
+  created: FakeScheduler[];
+  state: { retries: number };
+} {
+  const created: FakeScheduler[] = [];
+  const state = { retries: 0 };
+  const deps: ActivitySyncRegistrarDeps = {
+    config: { enabled: true, timezone: "UTC", syncDays: 1, autoSyncIntervalMinutes: 15, sources: [] },
+    memoryDir: "/tmp/remnic-activity-registrar",
+    qmdCollection: "openclaw-engram",
+    secureStoreEnabled: false,
+    getQmd: () => ({ update: async () => undefined }),
+    requestReindexRetry: () => {
+      state.retries += 1;
+    },
+    createScheduler: (options) => {
+      const s = new FakeScheduler(options);
+      created.push(s);
+      return s;
+    },
+    ...overrides,
+  };
+  return { deps, created, state };
+}
+
+test("register arms a scheduler on a live signal", async () => {
+  const { deps, created } = makeDeps();
+  const registrar = new ActivitySyncRegistrar(deps);
+  await registrar.register(new AbortController().signal);
+  assert.equal(created.length, 1, "exactly one scheduler is constructed");
+  assert.equal(created[0]?.starts, 1, "the scheduler is started");
+  assert.ok(registrar.armed, "the registrar reports armed");
+});
+
+test("register on an aborted signal never arms and stops any prior scheduler", async () => {
+  const { deps, created } = makeDeps();
+  const registrar = new ActivitySyncRegistrar(deps);
+  await registrar.register(new AbortController().signal);
+  const prior = created[0];
+  const aborted = AbortSignal.abort();
+  await registrar.register(aborted);
+  assert.equal(created.length, 1, "no replacement scheduler is constructed while aborted");
+  assert.equal(prior?.stops, 1, "the prior scheduler is stopped before bailing");
+  assert.equal(registrar.armed, false, "the registrar is disarmed after an aborted re-init");
+});
+
+test("register re-arms by stopping the prior scheduler first", async () => {
+  const { deps, created } = makeDeps();
+  const registrar = new ActivitySyncRegistrar(deps);
+  await registrar.register(new AbortController().signal);
+  await registrar.register(new AbortController().signal);
+  assert.equal(created.length, 2, "a fresh scheduler is armed on re-register");
+  assert.equal(created[0]?.stops, 1, "the prior scheduler is drained");
+  assert.equal(created[1]?.starts, 1, "the replacement scheduler is started");
+  assert.ok(registrar.armed);
+});
+
+test("register refuses to arm under a secure store", async () => {
+  const { deps, created } = makeDeps({ secureStoreEnabled: true });
+  const registrar = new ActivitySyncRegistrar(deps);
+  await registrar.register(new AbortController().signal);
+  assert.equal(created.length, 0, "no scheduler is armed when at-rest encryption is on");
+  assert.equal(registrar.armed, false);
+});
+
+test("dispose latches teardown and drains the scheduler", async () => {
+  const { deps, created } = makeDeps();
+  const registrar = new ActivitySyncRegistrar(deps);
+  await registrar.register(new AbortController().signal);
+  await registrar.dispose();
+  assert.equal(created[0]?.stops, 1, "dispose drains the armed scheduler");
+  assert.equal(registrar.armed, false, "the registrar is disarmed after dispose");
+});
+
+test("a reindex failure queues a retry, but never after teardown", async () => {
+  const box = makeDeps();
+  const registrar = new ActivitySyncRegistrar(box.deps);
+  await registrar.register(new AbortController().signal);
+  const onRun = box.created[0]?.options.onRun as (s: ActivitySyncRunSummary) => void;
+  const failed = { machine: "m", ran: true, fetched: 0, inserted: 0, duplicates: 0, reindexErrorCount: 2 } as unknown as ActivitySyncRunSummary;
+  onRun(failed);
+  assert.equal(box.state.retries, 1, "a failed reindex queues one maintenance retry");
+  await registrar.dispose();
+  onRun(failed);
+  assert.equal(box.state.retries, 1, "a draining tick's onRun does not re-arm maintenance after teardown");
+});
+
+test("abort landing during construction stops the freshly-started scheduler", async () => {
+  const controller = new AbortController();
+  const created: FakeScheduler[] = [];
+  const { deps } = makeDeps({
+    createScheduler: (options) => {
+      // Simulate the abort arriving in the window between the guard and start().
+      controller.abort();
+      const s = new FakeScheduler(options);
+      created.push(s);
+      return s;
+    },
+  });
+  const registrar = new ActivitySyncRegistrar(deps);
+  await registrar.register(controller.signal);
+  assert.equal(created.length, 1, "the scheduler is constructed before the abort is observed");
+  assert.equal(created[0]?.starts, 1, "it is started");
+  assert.equal(created[0]?.stops, 1, "then immediately stopped once the abort is seen");
+  assert.equal(registrar.armed, false, "the registrar does not retain an aborted scheduler");
+});

package/src/activity/sync-registration.ts

@@ -0,0 +1,139 @@
+/**
+ * Activity (screen-capture) sync lifecycle owner (issue #1900).
+ *
+ * The MaintenanceScheduler used to inline the arm / re-arm / teardown wiring
+ * for the periodic activity sync, which made the lifecycle invariants easy to
+ * break one edit at a time. This module centralizes them in one testable place:
+ *
+ *  - Re-arm and teardown BOTH stop any prior scheduler first, so a repeated
+ *    registration never orphans a running interval.
+ *  - An aborted deferred-init signal means teardown began. `destroy()` aborts
+ *    the init signal *before* it disposes, so an aborted signal here also
+ *    implies `dispose()` may already have latched teardown. `register()` never
+ *    arms nor clears the latch while aborted — even when the abort lands
+ *    mid-drain — so a late re-init cannot resurrect a torn-down instance.
+ *  - The teardown latch gates the reindex-failure retry, so a draining tick's
+ *    `onRun` cannot queue QMD maintenance after teardown.
+ *  - Activity digests + SQLite are written in the clear, so the registrar
+ *    refuses to arm under a secure store rather than silently bypass opted-in
+ *    at-rest encryption.
+ *
+ * Host-agnostic: the scheduler factory is injectable, so tests drive the whole
+ * lifecycle against a fake scheduler without real timers.
+ */
+
+import { refreshActivityIndex, type ActivityIndexRefresher } from "./reindex.js";
+import type { ActivitySyncRunSummary } from "./runner.js";
+import { ActivitySyncScheduler, type ActivitySyncSchedulerOptions } from "./scheduler.js";
+import type { ActivityConfig } from "./types.js";
+import { log } from "../logger.js";
+
+/** The scheduler surface the registrar drives (injectable for tests). */
+export interface ActivitySyncSchedulerLike {
+  start(): unknown;
+  stop(): Promise<void>;
+}
+
+export interface ActivitySyncRegistrarDeps {
+  /** Activity sync settings; the master default-off flag lives here. */
+  readonly config: ActivityConfig;
+  /** Memory root the digests + SQLite store live under. */
+  readonly memoryDir: string;
+  /** QMD collection the day digest is (strictly) reindexed into. */
+  readonly qmdCollection: string;
+  /**
+   * True when at-rest encryption is configured. Activity writes are not yet
+   * encrypted, so the registrar refuses to arm rather than leak plaintext.
+   */
+  readonly secureStoreEnabled: boolean;
+  /** Live QMD accessor — the host may swap backends after construction. */
+  readonly getQmd: () => ActivityIndexRefresher;
+  /** Debounced/singleflighted QMD retry used when a tick's reindex fails. */
+  readonly requestReindexRetry: () => void;
+  /** Scheduler factory (injectable for tests); defaults to the real scheduler. */
+  readonly createScheduler?: (options: ActivitySyncSchedulerOptions) => ActivitySyncSchedulerLike;
+}
+
+export class ActivitySyncRegistrar {
+  private scheduler: ActivitySyncSchedulerLike | null = null;
+  /** Latched by dispose() so a draining tick's onRun cannot re-arm maintenance. */
+  private disposed = false;
+  private readonly createScheduler: (options: ActivitySyncSchedulerOptions) => ActivitySyncSchedulerLike;
+
+  constructor(private readonly deps: ActivitySyncRegistrarDeps) {
+    this.createScheduler = deps.createScheduler ?? ((options) => new ActivitySyncScheduler(options));
+  }
+
+  /** True while a scheduler is armed (test/telemetry observability). */
+  get armed(): boolean {
+    return this.scheduler !== null;
+  }
+
+  /**
+   * Arm (or re-arm) the periodic sync for the current lifecycle. Idempotent and
+   * safe to call repeatedly; always stops a prior scheduler first.
+   */
+  async register(signal: AbortSignal): Promise<void> {
+    // Re-arm or teardown: stop any prior scheduler before anything else, so a
+    // repeated registration never orphans a running interval.
+    const prior = this.scheduler;
+    this.scheduler = null;
+    await prior?.stop();
+    // Teardown began (possibly during the drain above). Never arm nor clear the
+    // latch: dispose() may already have set it, and it must stay set.
+    if (signal.aborted) return;
+    // Opted-in at-rest encryption must not be silently bypassed: activity
+    // digests + SQLite are written in the clear, so refuse to run under it.
+    if (this.deps.secureStoreEnabled && this.deps.config.enabled) {
+      log.warn(
+        "activity sync disabled: secure store is enabled but activity digests/SQLite are not encrypted at rest",
+      );
+      return;
+    }
+    // Committed to a fresh lifecycle: clear the latch so this scheduler's
+    // reindex retries fire again.
+    this.disposed = false;
+    try {
+      const scheduler = this.createScheduler({
+        config: this.deps.config,
+        memoryDir: this.deps.memoryDir,
+        intervalMs: this.deps.config.autoSyncIntervalMinutes * 60_000,
+        // Strict refresh after each digest write (rule 31): bypasses the
+        // min-interval gate and throws on a real failure.
+        reindexSearch: (sig) => refreshActivityIndex(this.deps.getQmd(), this.deps.qmdCollection, sig),
+        // A failed reindex (QMD down) leaves the next unchanged-digest tick
+        // skipping afterWrites, so route the retry through the durable,
+        // throttled QMD path — but never after teardown.
+        onRun: (summary: ActivitySyncRunSummary) => {
+          if (summary.reindexErrorCount > 0 && !this.disposed) {
+            log.warn(
+              `activity sync: ${summary.reindexErrorCount} source(s) had a failed reindex; queuing a maintenance retry`,
+            );
+            this.deps.requestReindexRetry();
+          }
+        },
+      });
+      this.scheduler = scheduler;
+      scheduler.start();
+      // Close the race where abort fires between the guard above and start().
+      if (signal.aborted) {
+        await scheduler.stop();
+        this.scheduler = null;
+      }
+    } catch (err) {
+      log.debug(`activity sync scheduler start failed (non-fatal): ${err}`);
+    }
+  }
+
+  /**
+   * Latch teardown and drain the scheduler. Latching first ensures a draining
+   * tick's onRun cannot queue a retry; the drain is awaited so no in-flight
+   * tick keeps writing after teardown.
+   */
+  async dispose(): Promise<void> {
+    this.disposed = true;
+    const drain = this.scheduler?.stop();
+    this.scheduler = null;
+    await drain;
+  }
+}

package/src/activity/types.ts

@@ -84,3 +84,20 @@ export interface ActivitySourceClient {
     signal?: AbortSignal;
   }): Promise<ActivitySnapshotPage>;
 }
+
+/** One trusted capture daemon that feeds this memory directory. */
+export interface ActivitySourceConfig {
+  machineLabel: string;
+  baseUrl: string;
+  token?: string;
+}
+
+/** Opt-in activity synchronization settings. */
+export interface ActivityConfig {
+  enabled: boolean;
+  timezone: string;
+  syncDays: number;
+  /** Periodic auto-sync cadence in minutes (default 15). */
+  autoSyncIntervalMinutes: number;
+  sources: ActivitySourceConfig[];
+}

package/src/config.test.ts

@@ -2348,3 +2348,22 @@ test("parseConfig bounded-state knobs coerce valid strings, preserve 0, and reje
     assert.throws(() => parseConfig({ recallImpressionsRotateKeep: 1001 }), /between 1 and 1000/);
     assert.throws(() => parseConfig({ recallImpressionsRotateKeep: "1000000" }), /between 1 and 1000/);
   }));
+
+test("parseConfig forwards activity source settings", () => {
+  const config = parseConfig({
+    activity: {
+      enabled: "true",
+      timezone: "America/Chicago",
+      syncDays: "2",
+      sources: [{ machineLabel: "fixture-machine", baseUrl: "http://127.0.0.1:4319" }],
+    },
+  });
+
+  assert.deepEqual(config.activity, {
+    enabled: true,
+    timezone: "America/Chicago",
+    syncDays: 2,
+    autoSyncIntervalMinutes: 15,
+    sources: [{ machineLabel: "fixture-machine", baseUrl: "http://127.0.0.1:4319" }],
+  });
+});

package/src/config.ts

@@ -51,6 +51,7 @@ import {
   resolveNamespaceCatalogEnabled,
 } from "./emit-legacy-tools.js";
 import { parseWearablesConfig } from "./wearables/config.js";
+import { parseActivityConfig } from "./activity/config.js";
 import { parseProvenanceConfig } from "./provenance.js";
 import { parseBoundedJsonlStateConfig } from "./bounded-jsonl-state.js";
 import { parseCodingKnowledgeConfig } from "./coding/coding-knowledge-config.js";
@@ -1172,6 +1173,7 @@ export function parseConfig(
 
   // Wearable transcript ingestion (Limitless / Bee / Omi) — delegated to the wearables module.
   const wearables = parseWearablesConfig(cfg.wearables);
+  const activity = parseActivityConfig(cfg.activity);
 
   const provenance = parseProvenanceConfig(cfg.provenance);
   // Coding-agent project/branch scoping (issue #569)
@@ -2335,6 +2337,7 @@ export function parseConfig(
     dreamsPhases,
     procedural,
     wearables,
+    activity,
     provenance,
     // At-rest encryption (issue #690 PR 3/4)
     // coerceBool handles CLI string inputs: `--config secureStoreEnabled=true`