@archipelagolab/lobi 1.0.1 → 1.0.6

package/src/matrix/monitor/inbound-dedupe.test.ts

@@ -1,146 +0,0 @@
-import fs from "node:fs";
-import os from "node:os";
-import path from "node:path";
-import { afterEach, describe, expect, it, vi } from "vitest";
-import { createMatrixInboundEventDeduper } from "./inbound-dedupe.js";
-
-describe("Matrix inbound event dedupe", () => {
-  const tempDirs: string[] = [];
-
-  afterEach(() => {
-    vi.restoreAllMocks();
-    vi.useRealTimers();
-    for (const dir of tempDirs.splice(0)) {
-      fs.rmSync(dir, { recursive: true, force: true });
-    }
-  });
-
-  function createStoragePath(): string {
-    const dir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-matrix-inbound-dedupe-"));
-    tempDirs.push(dir);
-    return path.join(dir, "inbound-dedupe.json");
-  }
-
-  const auth = {
-    accountId: "ops",
-    homeserver: "https://matrix.example.org",
-    userId: "@bot:example.org",
-    accessToken: "token",
-    deviceId: "DEVICE",
-  } as const;
-
-  it("persists committed events across restarts", async () => {
-    const storagePath = createStoragePath();
-    const first = await createMatrixInboundEventDeduper({
-      auth: auth as never,
-      storagePath,
-    });
-
-    expect(first.claimEvent({ roomId: "!room:example.org", eventId: "$event-1" })).toBe(true);
-    await first.commitEvent({
-      roomId: "!room:example.org",
-      eventId: "$event-1",
-    });
-    await first.stop();
-
-    const second = await createMatrixInboundEventDeduper({
-      auth: auth as never,
-      storagePath,
-    });
-    expect(second.claimEvent({ roomId: "!room:example.org", eventId: "$event-1" })).toBe(false);
-  });
-
-  it("does not persist released pending claims", async () => {
-    const storagePath = createStoragePath();
-    const first = await createMatrixInboundEventDeduper({
-      auth: auth as never,
-      storagePath,
-    });
-
-    expect(first.claimEvent({ roomId: "!room:example.org", eventId: "$event-2" })).toBe(true);
-    first.releaseEvent({ roomId: "!room:example.org", eventId: "$event-2" });
-    await first.stop();
-
-    const second = await createMatrixInboundEventDeduper({
-      auth: auth as never,
-      storagePath,
-    });
-    expect(second.claimEvent({ roomId: "!room:example.org", eventId: "$event-2" })).toBe(true);
-  });
-
-  it("prunes expired and overflowed entries on load", async () => {
-    const storagePath = createStoragePath();
-    fs.writeFileSync(
-      storagePath,
-      JSON.stringify({
-        version: 1,
-        entries: [
-          { key: "!room:example.org|$old", ts: 10 },
-          { key: "!room:example.org|$keep-1", ts: 90 },
-          { key: "!room:example.org|$keep-2", ts: 95 },
-          { key: "!room:example.org|$keep-3", ts: 100 },
-        ],
-      }),
-      "utf8",
-    );
-
-    const deduper = await createMatrixInboundEventDeduper({
-      auth: auth as never,
-      storagePath,
-      ttlMs: 20,
-      maxEntries: 2,
-      nowMs: () => 100,
-    });
-
-    expect(deduper.claimEvent({ roomId: "!room:example.org", eventId: "$old" })).toBe(true);
-    expect(deduper.claimEvent({ roomId: "!room:example.org", eventId: "$keep-1" })).toBe(true);
-    expect(deduper.claimEvent({ roomId: "!room:example.org", eventId: "$keep-2" })).toBe(false);
-    expect(deduper.claimEvent({ roomId: "!room:example.org", eventId: "$keep-3" })).toBe(false);
-  });
-
-  it("retains replayed backlog events based on processing time", async () => {
-    const storagePath = createStoragePath();
-    let now = 100;
-    const first = await createMatrixInboundEventDeduper({
-      auth: auth as never,
-      storagePath,
-      ttlMs: 20,
-      nowMs: () => now,
-    });
-
-    expect(first.claimEvent({ roomId: "!room:example.org", eventId: "$backlog" })).toBe(true);
-    await first.commitEvent({
-      roomId: "!room:example.org",
-      eventId: "$backlog",
-    });
-    await first.stop();
-
-    now = 110;
-    const second = await createMatrixInboundEventDeduper({
-      auth: auth as never,
-      storagePath,
-      ttlMs: 20,
-      nowMs: () => now,
-    });
-    expect(second.claimEvent({ roomId: "!room:example.org", eventId: "$backlog" })).toBe(false);
-  });
-
-  it("treats stop persistence failures as best-effort cleanup", async () => {
-    const blockingPath = createStoragePath();
-    fs.writeFileSync(blockingPath, "blocking file", "utf8");
-    const deduper = await createMatrixInboundEventDeduper({
-      auth: auth as never,
-      storagePath: path.join(blockingPath, "nested", "inbound-dedupe.json"),
-    });
-
-    expect(deduper.claimEvent({ roomId: "!room:example.org", eventId: "$persist-fail" })).toBe(
-      true,
-    );
-    await deduper.commitEvent({
-      roomId: "!room:example.org",
-      eventId: "$persist-fail",
-    });
-
-    await expect(deduper.stop()).resolves.toBeUndefined();
-  });
-});

package/src/matrix/monitor/inbound-dedupe.ts

@@ -1,267 +0,0 @@
-import { readJsonFileWithFallback, writeJsonFileAtomically } from "openclaw/plugin-sdk/json-store";
-import { createAsyncLock } from "../async-lock.js";
-import { resolveMatrixStateFilePath } from "../client/storage.js";
-import type { MatrixAuth } from "../client/types.js";
-import { LogService } from "../sdk/logger.js";
-
-const INBOUND_DEDUPE_FILENAME = "inbound-dedupe.json";
-const STORE_VERSION = 1;
-const DEFAULT_MAX_ENTRIES = 20_000;
-const DEFAULT_TTL_MS = 30 * 24 * 60 * 60 * 1000;
-const PERSIST_DEBOUNCE_MS = 250;
-
-type StoredMatrixInboundDedupeEntry = {
-  key: string;
-  ts: number;
-};
-
-type StoredMatrixInboundDedupeState = {
-  version: number;
-  entries: StoredMatrixInboundDedupeEntry[];
-};
-
-export type MatrixInboundEventDeduper = {
-  claimEvent: (params: { roomId: string; eventId: string }) => boolean;
-  commitEvent: (params: { roomId: string; eventId: string }) => Promise<void>;
-  releaseEvent: (params: { roomId: string; eventId: string }) => void;
-  flush: () => Promise<void>;
-  stop: () => Promise<void>;
-};
-
-function normalizeEventPart(value: string): string {
-  return value.trim();
-}
-
-function buildEventKey(params: { roomId: string; eventId: string }): string {
-  const roomId = normalizeEventPart(params.roomId);
-  const eventId = normalizeEventPart(params.eventId);
-  return roomId && eventId ? `${roomId}|${eventId}` : "";
-}
-
-function resolveInboundDedupeStatePath(params: {
-  auth: MatrixAuth;
-  env?: NodeJS.ProcessEnv;
-  stateDir?: string;
-}): string {
-  return resolveMatrixStateFilePath({
-    auth: params.auth,
-    env: params.env,
-    stateDir: params.stateDir,
-    filename: INBOUND_DEDUPE_FILENAME,
-  });
-}
-
-function normalizeTimestamp(raw: unknown): number | null {
-  if (typeof raw !== "number" || !Number.isFinite(raw)) {
-    return null;
-  }
-  return Math.max(0, Math.floor(raw));
-}
-
-function pruneSeenEvents(params: {
-  seen: Map<string, number>;
-  ttlMs: number;
-  maxEntries: number;
-  nowMs: number;
-}) {
-  const { seen, ttlMs, maxEntries, nowMs } = params;
-  if (ttlMs > 0) {
-    const cutoff = nowMs - ttlMs;
-    for (const [key, ts] of seen) {
-      if (ts < cutoff) {
-        seen.delete(key);
-      }
-    }
-  }
-  const max = Math.max(0, Math.floor(maxEntries));
-  if (max <= 0) {
-    seen.clear();
-    return;
-  }
-  while (seen.size > max) {
-    const oldestKey = seen.keys().next().value;
-    if (typeof oldestKey !== "string") {
-      break;
-    }
-    seen.delete(oldestKey);
-  }
-}
-
-function toStoredState(params: {
-  seen: Map<string, number>;
-  ttlMs: number;
-  maxEntries: number;
-  nowMs: number;
-}): StoredMatrixInboundDedupeState {
-  pruneSeenEvents(params);
-  return {
-    version: STORE_VERSION,
-    entries: Array.from(params.seen.entries()).map(([key, ts]) => ({ key, ts })),
-  };
-}
-
-async function readStoredState(
-  storagePath: string,
-): Promise<StoredMatrixInboundDedupeState | null> {
-  const { value } = await readJsonFileWithFallback<StoredMatrixInboundDedupeState | null>(
-    storagePath,
-    null,
-  );
-  if (value?.version !== STORE_VERSION || !Array.isArray(value.entries)) {
-    return null;
-  }
-  return value;
-}
-
-export async function createMatrixInboundEventDeduper(params: {
-  auth: MatrixAuth;
-  env?: NodeJS.ProcessEnv;
-  stateDir?: string;
-  storagePath?: string;
-  ttlMs?: number;
-  maxEntries?: number;
-  nowMs?: () => number;
-}): Promise<MatrixInboundEventDeduper> {
-  const nowMs = params.nowMs ?? (() => Date.now());
-  const ttlMs =
-    typeof params.ttlMs === "number" && Number.isFinite(params.ttlMs)
-      ? Math.max(0, Math.floor(params.ttlMs))
-      : DEFAULT_TTL_MS;
-  const maxEntries =
-    typeof params.maxEntries === "number" && Number.isFinite(params.maxEntries)
-      ? Math.max(0, Math.floor(params.maxEntries))
-      : DEFAULT_MAX_ENTRIES;
-  const storagePath =
-    params.storagePath ??
-    resolveInboundDedupeStatePath({
-      auth: params.auth,
-      env: params.env,
-      stateDir: params.stateDir,
-    });
-
-  const seen = new Map<string, number>();
-  const pending = new Set<string>();
-  const persistLock = createAsyncLock();
-
-  try {
-    const stored = await readStoredState(storagePath);
-    for (const entry of stored?.entries ?? []) {
-      if (!entry || typeof entry.key !== "string") {
-        continue;
-      }
-      const key = entry.key.trim();
-      const ts = normalizeTimestamp(entry.ts);
-      if (!key || ts === null) {
-        continue;
-      }
-      seen.set(key, ts);
-    }
-    pruneSeenEvents({ seen, ttlMs, maxEntries, nowMs: nowMs() });
-  } catch (err) {
-    LogService.warn("MatrixInboundDedupe", "Failed loading Matrix inbound dedupe store:", err);
-  }
-
-  let dirty = false;
-  let persistTimer: NodeJS.Timeout | null = null;
-  let persistPromise: Promise<void> | null = null;
-
-  const persist = async () => {
-    dirty = false;
-    const payload = toStoredState({
-      seen,
-      ttlMs,
-      maxEntries,
-      nowMs: nowMs(),
-    });
-    try {
-      await persistLock(async () => {
-        await writeJsonFileAtomically(storagePath, payload);
-      });
-    } catch (err) {
-      dirty = true;
-      throw err;
-    }
-  };
-
-  const flush = async (): Promise<void> => {
-    if (persistTimer) {
-      clearTimeout(persistTimer);
-      persistTimer = null;
-    }
-    for (;;) {
-      if (!dirty && !persistPromise) {
-        break;
-      }
-      if (dirty && !persistPromise) {
-        persistPromise = persist().finally(() => {
-          persistPromise = null;
-        });
-      }
-      await persistPromise;
-    }
-  };
-
-  const schedulePersist = () => {
-    dirty = true;
-    if (persistTimer) {
-      return;
-    }
-    persistTimer = setTimeout(() => {
-      persistTimer = null;
-      void flush().catch((err) => {
-        LogService.warn(
-          "MatrixInboundDedupe",
-          "Failed persisting Matrix inbound dedupe store:",
-          err,
-        );
-      });
-    }, PERSIST_DEBOUNCE_MS);
-    persistTimer.unref?.();
-  };
-
-  return {
-    claimEvent: ({ roomId, eventId }) => {
-      const key = buildEventKey({ roomId, eventId });
-      if (!key) {
-        return true;
-      }
-      pruneSeenEvents({ seen, ttlMs, maxEntries, nowMs: nowMs() });
-      if (seen.has(key) || pending.has(key)) {
-        return false;
-      }
-      pending.add(key);
-      return true;
-    },
-    commitEvent: async ({ roomId, eventId }) => {
-      const key = buildEventKey({ roomId, eventId });
-      if (!key) {
-        return;
-      }
-      pending.delete(key);
-      const ts = nowMs();
-      seen.delete(key);
-      seen.set(key, ts);
-      pruneSeenEvents({ seen, ttlMs, maxEntries, nowMs: nowMs() });
-      schedulePersist();
-    },
-    releaseEvent: ({ roomId, eventId }) => {
-      const key = buildEventKey({ roomId, eventId });
-      if (!key) {
-        return;
-      }
-      pending.delete(key);
-    },
-    flush,
-    stop: async () => {
-      try {
-        await flush();
-      } catch (err) {
-        LogService.warn(
-          "MatrixInboundDedupe",
-          "Failed to flush Matrix inbound dedupe store during stop():",
-          err,
-        );
-      }
-    },
-  };
-}