@archipelagolab/lobi 1.0.0

package/src/matrix/monitor/inbound-dedupe.test.ts

@@ -0,0 +1,146 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { createMatrixInboundEventDeduper } from "./inbound-dedupe.js";
+
+describe("Matrix inbound event dedupe", () => {
+  const tempDirs: string[] = [];
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+    vi.useRealTimers();
+    for (const dir of tempDirs.splice(0)) {
+      fs.rmSync(dir, { recursive: true, force: true });
+    }
+  });
+
+  function createStoragePath(): string {
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-matrix-inbound-dedupe-"));
+    tempDirs.push(dir);
+    return path.join(dir, "inbound-dedupe.json");
+  }
+
+  const auth = {
+    accountId: "ops",
+    homeserver: "https://matrix.example.org",
+    userId: "@bot:example.org",
+    accessToken: "token",
+    deviceId: "DEVICE",
+  } as const;
+
+  it("persists committed events across restarts", async () => {
+    const storagePath = createStoragePath();
+    const first = await createMatrixInboundEventDeduper({
+      auth: auth as never,
+      storagePath,
+    });
+
+    expect(first.claimEvent({ roomId: "!room:example.org", eventId: "$event-1" })).toBe(true);
+    await first.commitEvent({
+      roomId: "!room:example.org",
+      eventId: "$event-1",
+    });
+    await first.stop();
+
+    const second = await createMatrixInboundEventDeduper({
+      auth: auth as never,
+      storagePath,
+    });
+    expect(second.claimEvent({ roomId: "!room:example.org", eventId: "$event-1" })).toBe(false);
+  });
+
+  it("does not persist released pending claims", async () => {
+    const storagePath = createStoragePath();
+    const first = await createMatrixInboundEventDeduper({
+      auth: auth as never,
+      storagePath,
+    });
+
+    expect(first.claimEvent({ roomId: "!room:example.org", eventId: "$event-2" })).toBe(true);
+    first.releaseEvent({ roomId: "!room:example.org", eventId: "$event-2" });
+    await first.stop();
+
+    const second = await createMatrixInboundEventDeduper({
+      auth: auth as never,
+      storagePath,
+    });
+    expect(second.claimEvent({ roomId: "!room:example.org", eventId: "$event-2" })).toBe(true);
+  });
+
+  it("prunes expired and overflowed entries on load", async () => {
+    const storagePath = createStoragePath();
+    fs.writeFileSync(
+      storagePath,
+      JSON.stringify({
+        version: 1,
+        entries: [
+          { key: "!room:example.org|$old", ts: 10 },
+          { key: "!room:example.org|$keep-1", ts: 90 },
+          { key: "!room:example.org|$keep-2", ts: 95 },
+          { key: "!room:example.org|$keep-3", ts: 100 },
+        ],
+      }),
+      "utf8",
+    );
+
+    const deduper = await createMatrixInboundEventDeduper({
+      auth: auth as never,
+      storagePath,
+      ttlMs: 20,
+      maxEntries: 2,
+      nowMs: () => 100,
+    });
+
+    expect(deduper.claimEvent({ roomId: "!room:example.org", eventId: "$old" })).toBe(true);
+    expect(deduper.claimEvent({ roomId: "!room:example.org", eventId: "$keep-1" })).toBe(true);
+    expect(deduper.claimEvent({ roomId: "!room:example.org", eventId: "$keep-2" })).toBe(false);
+    expect(deduper.claimEvent({ roomId: "!room:example.org", eventId: "$keep-3" })).toBe(false);
+  });
+
+  it("retains replayed backlog events based on processing time", async () => {
+    const storagePath = createStoragePath();
+    let now = 100;
+    const first = await createMatrixInboundEventDeduper({
+      auth: auth as never,
+      storagePath,
+      ttlMs: 20,
+      nowMs: () => now,
+    });
+
+    expect(first.claimEvent({ roomId: "!room:example.org", eventId: "$backlog" })).toBe(true);
+    await first.commitEvent({
+      roomId: "!room:example.org",
+      eventId: "$backlog",
+    });
+    await first.stop();
+
+    now = 110;
+    const second = await createMatrixInboundEventDeduper({
+      auth: auth as never,
+      storagePath,
+      ttlMs: 20,
+      nowMs: () => now,
+    });
+    expect(second.claimEvent({ roomId: "!room:example.org", eventId: "$backlog" })).toBe(false);
+  });
+
+  it("treats stop persistence failures as best-effort cleanup", async () => {
+    const blockingPath = createStoragePath();
+    fs.writeFileSync(blockingPath, "blocking file", "utf8");
+    const deduper = await createMatrixInboundEventDeduper({
+      auth: auth as never,
+      storagePath: path.join(blockingPath, "nested", "inbound-dedupe.json"),
+    });
+
+    expect(deduper.claimEvent({ roomId: "!room:example.org", eventId: "$persist-fail" })).toBe(
+      true,
+    );
+    await deduper.commitEvent({
+      roomId: "!room:example.org",
+      eventId: "$persist-fail",
+    });
+
+    await expect(deduper.stop()).resolves.toBeUndefined();
+  });
+});

package/src/matrix/monitor/inbound-dedupe.ts

@@ -0,0 +1,267 @@
+import { readJsonFileWithFallback, writeJsonFileAtomically } from "openclaw/plugin-sdk/json-store";
+import { createAsyncLock } from "../async-lock.js";
+import { resolveMatrixStateFilePath } from "../client/storage.js";
+import type { MatrixAuth } from "../client/types.js";
+import { LogService } from "../sdk/logger.js";
+
+const INBOUND_DEDUPE_FILENAME = "inbound-dedupe.json";
+const STORE_VERSION = 1;
+const DEFAULT_MAX_ENTRIES = 20_000;
+const DEFAULT_TTL_MS = 30 * 24 * 60 * 60 * 1000;
+const PERSIST_DEBOUNCE_MS = 250;
+
+type StoredMatrixInboundDedupeEntry = {
+  key: string;
+  ts: number;
+};
+
+type StoredMatrixInboundDedupeState = {
+  version: number;
+  entries: StoredMatrixInboundDedupeEntry[];
+};
+
+export type MatrixInboundEventDeduper = {
+  claimEvent: (params: { roomId: string; eventId: string }) => boolean;
+  commitEvent: (params: { roomId: string; eventId: string }) => Promise<void>;
+  releaseEvent: (params: { roomId: string; eventId: string }) => void;
+  flush: () => Promise<void>;
+  stop: () => Promise<void>;
+};
+
+function normalizeEventPart(value: string): string {
+  return value.trim();
+}
+
+function buildEventKey(params: { roomId: string; eventId: string }): string {
+  const roomId = normalizeEventPart(params.roomId);
+  const eventId = normalizeEventPart(params.eventId);
+  return roomId && eventId ? `${roomId}|${eventId}` : "";
+}
+
+function resolveInboundDedupeStatePath(params: {
+  auth: MatrixAuth;
+  env?: NodeJS.ProcessEnv;
+  stateDir?: string;
+}): string {
+  return resolveMatrixStateFilePath({
+    auth: params.auth,
+    env: params.env,
+    stateDir: params.stateDir,
+    filename: INBOUND_DEDUPE_FILENAME,
+  });
+}
+
+function normalizeTimestamp(raw: unknown): number | null {
+  if (typeof raw !== "number" || !Number.isFinite(raw)) {
+    return null;
+  }
+  return Math.max(0, Math.floor(raw));
+}
+
+function pruneSeenEvents(params: {
+  seen: Map<string, number>;
+  ttlMs: number;
+  maxEntries: number;
+  nowMs: number;
+}) {
+  const { seen, ttlMs, maxEntries, nowMs } = params;
+  if (ttlMs > 0) {
+    const cutoff = nowMs - ttlMs;
+    for (const [key, ts] of seen) {
+      if (ts < cutoff) {
+        seen.delete(key);
+      }
+    }
+  }
+  const max = Math.max(0, Math.floor(maxEntries));
+  if (max <= 0) {
+    seen.clear();
+    return;
+  }
+  while (seen.size > max) {
+    const oldestKey = seen.keys().next().value;
+    if (typeof oldestKey !== "string") {
+      break;
+    }
+    seen.delete(oldestKey);
+  }
+}
+
+function toStoredState(params: {
+  seen: Map<string, number>;
+  ttlMs: number;
+  maxEntries: number;
+  nowMs: number;
+}): StoredMatrixInboundDedupeState {
+  pruneSeenEvents(params);
+  return {
+    version: STORE_VERSION,
+    entries: Array.from(params.seen.entries()).map(([key, ts]) => ({ key, ts })),
+  };
+}
+
+async function readStoredState(
+  storagePath: string,
+): Promise<StoredMatrixInboundDedupeState | null> {
+  const { value } = await readJsonFileWithFallback<StoredMatrixInboundDedupeState | null>(
+    storagePath,
+    null,
+  );
+  if (value?.version !== STORE_VERSION || !Array.isArray(value.entries)) {
+    return null;
+  }
+  return value;
+}
+
+export async function createMatrixInboundEventDeduper(params: {
+  auth: MatrixAuth;
+  env?: NodeJS.ProcessEnv;
+  stateDir?: string;
+  storagePath?: string;
+  ttlMs?: number;
+  maxEntries?: number;
+  nowMs?: () => number;
+}): Promise<MatrixInboundEventDeduper> {
+  const nowMs = params.nowMs ?? (() => Date.now());
+  const ttlMs =
+    typeof params.ttlMs === "number" && Number.isFinite(params.ttlMs)
+      ? Math.max(0, Math.floor(params.ttlMs))
+      : DEFAULT_TTL_MS;
+  const maxEntries =
+    typeof params.maxEntries === "number" && Number.isFinite(params.maxEntries)
+      ? Math.max(0, Math.floor(params.maxEntries))
+      : DEFAULT_MAX_ENTRIES;
+  const storagePath =
+    params.storagePath ??
+    resolveInboundDedupeStatePath({
+      auth: params.auth,
+      env: params.env,
+      stateDir: params.stateDir,
+    });
+
+  const seen = new Map<string, number>();
+  const pending = new Set<string>();
+  const persistLock = createAsyncLock();
+
+  try {
+    const stored = await readStoredState(storagePath);
+    for (const entry of stored?.entries ?? []) {
+      if (!entry || typeof entry.key !== "string") {
+        continue;
+      }
+      const key = entry.key.trim();
+      const ts = normalizeTimestamp(entry.ts);
+      if (!key || ts === null) {
+        continue;
+      }
+      seen.set(key, ts);
+    }
+    pruneSeenEvents({ seen, ttlMs, maxEntries, nowMs: nowMs() });
+  } catch (err) {
+    LogService.warn("MatrixInboundDedupe", "Failed loading Matrix inbound dedupe store:", err);
+  }
+
+  let dirty = false;
+  let persistTimer: NodeJS.Timeout | null = null;
+  let persistPromise: Promise<void> | null = null;
+
+  const persist = async () => {
+    dirty = false;
+    const payload = toStoredState({
+      seen,
+      ttlMs,
+      maxEntries,
+      nowMs: nowMs(),
+    });
+    try {
+      await persistLock(async () => {
+        await writeJsonFileAtomically(storagePath, payload);
+      });
+    } catch (err) {
+      dirty = true;
+      throw err;
+    }
+  };
+
+  const flush = async (): Promise<void> => {
+    if (persistTimer) {
+      clearTimeout(persistTimer);
+      persistTimer = null;
+    }
+    for (;;) {
+      if (!dirty && !persistPromise) {
+        break;
+      }
+      if (dirty && !persistPromise) {
+        persistPromise = persist().finally(() => {
+          persistPromise = null;
+        });
+      }
+      await persistPromise;
+    }
+  };
+
+  const schedulePersist = () => {
+    dirty = true;
+    if (persistTimer) {
+      return;
+    }
+    persistTimer = setTimeout(() => {
+      persistTimer = null;
+      void flush().catch((err) => {
+        LogService.warn(
+          "MatrixInboundDedupe",
+          "Failed persisting Matrix inbound dedupe store:",
+          err,
+        );
+      });
+    }, PERSIST_DEBOUNCE_MS);
+    persistTimer.unref?.();
+  };
+
+  return {
+    claimEvent: ({ roomId, eventId }) => {
+      const key = buildEventKey({ roomId, eventId });
+      if (!key) {
+        return true;
+      }
+      pruneSeenEvents({ seen, ttlMs, maxEntries, nowMs: nowMs() });
+      if (seen.has(key) || pending.has(key)) {
+        return false;
+      }
+      pending.add(key);
+      return true;
+    },
+    commitEvent: async ({ roomId, eventId }) => {
+      const key = buildEventKey({ roomId, eventId });
+      if (!key) {
+        return;
+      }
+      pending.delete(key);
+      const ts = nowMs();
+      seen.delete(key);
+      seen.set(key, ts);
+      pruneSeenEvents({ seen, ttlMs, maxEntries, nowMs: nowMs() });
+      schedulePersist();
+    },
+    releaseEvent: ({ roomId, eventId }) => {
+      const key = buildEventKey({ roomId, eventId });
+      if (!key) {
+        return;
+      }
+      pending.delete(key);
+    },
+    flush,
+    stop: async () => {
+      try {
+        await flush();
+      } catch (err) {
+        LogService.warn(
+          "MatrixInboundDedupe",
+          "Failed to flush Matrix inbound dedupe store during stop():",
+          err,
+        );
+      }
+    },
+  };
+}