@archipelagolab/lobi 1.0.0

package/src/matrix/client/create-client.test.ts

@@ -0,0 +1,115 @@
+import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
+
+const ensureMatrixSdkLoggingConfiguredMock = vi.hoisted(() => vi.fn());
+const resolveValidatedMatrixHomeserverUrlMock = vi.hoisted(() => vi.fn());
+const maybeMigrateLegacyStorageMock = vi.hoisted(() => vi.fn(async () => undefined));
+const resolveMatrixStoragePathsMock = vi.hoisted(() => vi.fn());
+const writeStorageMetaMock = vi.hoisted(() => vi.fn());
+const MatrixClientMock = vi.hoisted(() => vi.fn());
+
+vi.mock("./logging.js", () => ({
+  ensureMatrixSdkLoggingConfigured: ensureMatrixSdkLoggingConfiguredMock,
+}));
+
+vi.mock("./config.js", () => ({
+  resolveValidatedMatrixHomeserverUrl: resolveValidatedMatrixHomeserverUrlMock,
+}));
+
+vi.mock("./storage.js", () => ({
+  maybeMigrateLegacyStorage: maybeMigrateLegacyStorageMock,
+  resolveMatrixStoragePaths: resolveMatrixStoragePathsMock,
+  writeStorageMeta: writeStorageMetaMock,
+}));
+
+vi.mock("../sdk.js", () => ({
+  MatrixClient: MatrixClientMock,
+}));
+
+let createMatrixClient: typeof import("./create-client.js").createMatrixClient;
+
+describe("createMatrixClient", () => {
+  const storagePaths = {
+    rootDir: "/tmp/openclaw-matrix-create-client-test",
+    storagePath: "/tmp/openclaw-matrix-create-client-test/storage.json",
+    recoveryKeyPath: "/tmp/openclaw-matrix-create-client-test/recovery.key",
+    idbSnapshotPath: "/tmp/openclaw-matrix-create-client-test/idb.snapshot",
+    metaPath: "/tmp/openclaw-matrix-create-client-test/storage-meta.json",
+    accountKey: "default",
+    tokenHash: "token-hash",
+  };
+
+  beforeAll(async () => {
+    ({ createMatrixClient } = await import("./create-client.js"));
+  });
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    ensureMatrixSdkLoggingConfiguredMock.mockReturnValue(undefined);
+    resolveValidatedMatrixHomeserverUrlMock.mockResolvedValue("https://matrix.example.org");
+    resolveMatrixStoragePathsMock.mockReturnValue(storagePaths);
+    MatrixClientMock.mockImplementation(function MockMatrixClient() {
+      return {
+        stop: vi.fn(),
+      };
+    });
+  });
+
+  it("persists storage metadata by default", async () => {
+    await createMatrixClient({
+      homeserver: "https://matrix.example.org",
+      userId: "@bot:example.org",
+      accessToken: "tok",
+    });
+
+    expect(writeStorageMetaMock).toHaveBeenCalledWith({
+      storagePaths,
+      homeserver: "https://matrix.example.org",
+      userId: "@bot:example.org",
+      accountId: undefined,
+      deviceId: undefined,
+    });
+    expect(resolveMatrixStoragePathsMock).toHaveBeenCalledTimes(1);
+    expect(MatrixClientMock).toHaveBeenCalledWith("https://matrix.example.org", "tok", {
+      userId: "@bot:example.org",
+      password: undefined,
+      deviceId: undefined,
+      encryption: undefined,
+      localTimeoutMs: undefined,
+      initialSyncLimit: undefined,
+      storagePath: storagePaths.storagePath,
+      recoveryKeyPath: storagePaths.recoveryKeyPath,
+      idbSnapshotPath: storagePaths.idbSnapshotPath,
+      cryptoDatabasePrefix: "openclaw-matrix-default-token-hash",
+      autoBootstrapCrypto: undefined,
+      ssrfPolicy: undefined,
+      dispatcherPolicy: undefined,
+    });
+  });
+
+  it("skips persistent storage wiring when persistence is disabled", async () => {
+    await createMatrixClient({
+      homeserver: "https://matrix.example.org",
+      userId: "@bot:example.org",
+      accessToken: "tok",
+      persistStorage: false,
+    });
+
+    expect(resolveMatrixStoragePathsMock).not.toHaveBeenCalled();
+    expect(writeStorageMetaMock).not.toHaveBeenCalled();
+    expect(MatrixClientMock).toHaveBeenCalledWith("https://matrix.example.org", "tok", {
+      userId: "@bot:example.org",
+      password: undefined,
+      deviceId: undefined,
+      encryption: undefined,
+      localTimeoutMs: undefined,
+      initialSyncLimit: undefined,
+      storagePath: undefined,
+      recoveryKeyPath: undefined,
+      idbSnapshotPath: undefined,
+      cryptoDatabasePrefix: undefined,
+      autoBootstrapCrypto: undefined,
+      ssrfPolicy: undefined,
+      dispatcherPolicy: undefined,
+    });
+  });
+});

package/src/matrix/client/create-client.ts

@@ -0,0 +1,101 @@
+import fs from "node:fs";
+import type { PinnedDispatcherPolicy } from "openclaw/plugin-sdk/infra-runtime";
+import { normalizeOptionalString } from "openclaw/plugin-sdk/text-runtime";
+import type { SsrFPolicy } from "../../runtime-api.js";
+import type { MatrixClient } from "../sdk.js";
+import { resolveValidatedMatrixHomeserverUrl } from "./config.js";
+import {
+  maybeMigrateLegacyStorage,
+  resolveMatrixStoragePaths,
+  writeStorageMeta,
+} from "./storage.js";
+
+type MatrixCreateClientRuntimeDeps = {
+  MatrixClient: typeof import("../sdk.js").MatrixClient;
+  ensureMatrixSdkLoggingConfigured: typeof import("./logging.js").ensureMatrixSdkLoggingConfigured;
+};
+
+let matrixCreateClientRuntimeDepsPromise: Promise<MatrixCreateClientRuntimeDeps> | undefined;
+
+async function loadMatrixCreateClientRuntimeDeps(): Promise<MatrixCreateClientRuntimeDeps> {
+  matrixCreateClientRuntimeDepsPromise ??= Promise.all([
+    import("../sdk.js"),
+    import("./logging.js"),
+  ]).then(([sdkModule, loggingModule]) => ({
+    MatrixClient: sdkModule.MatrixClient,
+    ensureMatrixSdkLoggingConfigured: loggingModule.ensureMatrixSdkLoggingConfigured,
+  }));
+  return await matrixCreateClientRuntimeDepsPromise;
+}
+
+export async function createMatrixClient(params: {
+  homeserver: string;
+  userId?: string;
+  accessToken: string;
+  password?: string;
+  deviceId?: string;
+  persistStorage?: boolean;
+  encryption?: boolean;
+  localTimeoutMs?: number;
+  initialSyncLimit?: number;
+  accountId?: string | null;
+  autoBootstrapCrypto?: boolean;
+  allowPrivateNetwork?: boolean;
+  ssrfPolicy?: SsrFPolicy;
+  dispatcherPolicy?: PinnedDispatcherPolicy;
+}): Promise<MatrixClient> {
+  const { MatrixClient, ensureMatrixSdkLoggingConfigured } =
+    await loadMatrixCreateClientRuntimeDeps();
+  ensureMatrixSdkLoggingConfigured();
+  const homeserver = await resolveValidatedMatrixHomeserverUrl(params.homeserver, {
+    dangerouslyAllowPrivateNetwork: params.allowPrivateNetwork,
+  });
+  const matrixClientUserId = normalizeOptionalString(params.userId);
+  const userId = matrixClientUserId ?? "unknown";
+  const persistStorage = params.persistStorage !== false;
+  const storagePaths = persistStorage
+    ? resolveMatrixStoragePaths({
+        homeserver,
+        userId,
+        accessToken: params.accessToken,
+        accountId: params.accountId,
+        deviceId: params.deviceId,
+        env: process.env,
+      })
+    : null;
+
+  if (storagePaths) {
+    await maybeMigrateLegacyStorage({
+      storagePaths,
+      env: process.env,
+    });
+    fs.mkdirSync(storagePaths.rootDir, { recursive: true });
+    writeStorageMeta({
+      storagePaths,
+      homeserver,
+      userId,
+      accountId: params.accountId,
+      deviceId: params.deviceId,
+    });
+  }
+
+  const cryptoDatabasePrefix = storagePaths
+    ? `openclaw-matrix-${storagePaths.accountKey}-${storagePaths.tokenHash}`
+    : undefined;
+
+  return new MatrixClient(homeserver, params.accessToken, {
+    userId: matrixClientUserId,
+    password: params.password,
+    deviceId: params.deviceId,
+    encryption: params.encryption,
+    localTimeoutMs: params.localTimeoutMs,
+    initialSyncLimit: params.initialSyncLimit,
+    storagePath: storagePaths?.storagePath,
+    recoveryKeyPath: storagePaths?.recoveryKeyPath,
+    idbSnapshotPath: storagePaths?.idbSnapshotPath,
+    cryptoDatabasePrefix,
+    autoBootstrapCrypto: params.autoBootstrapCrypto,
+    ssrfPolicy: params.ssrfPolicy,
+    dispatcherPolicy: params.dispatcherPolicy,
+  });
+}

package/src/matrix/client/env-auth.ts

@@ -0,0 +1,6 @@
+export {
+  hasReadyMatrixEnvAuth,
+  resolveGlobalMatrixEnvConfig,
+  resolveMatrixEnvAuthReadiness,
+  resolveScopedMatrixEnvConfig,
+} from "./config.js";

package/src/matrix/client/file-sync-store.test.ts

@@ -0,0 +1,265 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import type { ISyncResponse } from "@archipelagolab/lobi-js-sdk";
+import * as jsonStore from "openclaw/plugin-sdk/json-store";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { FileBackedMatrixSyncStore } from "./file-sync-store.js";
+
+function createSyncResponse(nextBatch: string): ISyncResponse {
+  return {
+    next_batch: nextBatch,
+    rooms: {
+      join: {
+        "!room:example.org": {
+          summary: {
+            "m.heroes": [],
+          },
+          state: { events: [] },
+          timeline: {
+            events: [
+              {
+                content: {
+                  body: "hello",
+                  msgtype: "m.text",
+                },
+                event_id: "$message",
+                origin_server_ts: 1,
+                sender: "@user:example.org",
+                type: "m.room.message",
+              },
+            ],
+            prev_batch: "t0",
+          },
+          ephemeral: { events: [] },
+          account_data: { events: [] },
+          unread_notifications: {},
+        },
+      },
+      invite: {},
+      leave: {},
+      knock: {},
+    },
+    account_data: {
+      events: [
+        {
+          content: { theme: "dark" },
+          type: "com.openclaw.test",
+        },
+      ],
+    },
+  };
+}
+
+function createDeferred() {
+  let resolve!: () => void;
+  const promise = new Promise<void>((resolvePromise) => {
+    resolve = resolvePromise;
+  });
+  return { promise, resolve };
+}
+
+describe("FileBackedMatrixSyncStore", () => {
+  const tempDirs: string[] = [];
+
+  function createStoragePath(): string {
+    const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-matrix-sync-store-"));
+    tempDirs.push(tempDir);
+    return path.join(tempDir, "bot-storage.json");
+  }
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+    vi.useRealTimers();
+    for (const dir of tempDirs.splice(0)) {
+      fs.rmSync(dir, { recursive: true, force: true });
+    }
+  });
+
+  it("persists sync data so restart resumes from the saved cursor", async () => {
+    const storagePath = createStoragePath();
+
+    const firstStore = new FileBackedMatrixSyncStore(storagePath);
+    expect(firstStore.hasSavedSync()).toBe(false);
+    await firstStore.setSyncData(createSyncResponse("s123"));
+    await firstStore.flush();
+
+    const secondStore = new FileBackedMatrixSyncStore(storagePath);
+    expect(secondStore.hasSavedSync()).toBe(true);
+    await expect(secondStore.getSavedSyncToken()).resolves.toBe("s123");
+
+    const savedSync = await secondStore.getSavedSync();
+    expect(savedSync?.nextBatch).toBe("s123");
+    expect(savedSync?.accountData).toEqual([
+      {
+        content: { theme: "dark" },
+        type: "com.openclaw.test",
+      },
+    ]);
+    expect(savedSync?.roomsData.join?.["!room:example.org"]).toBeTruthy();
+    expect(secondStore.hasSavedSyncFromCleanShutdown()).toBe(false);
+  });
+
+  it("claims current-token storage ownership when sync state is persisted", async () => {
+    const storagePath = createStoragePath();
+    const rootDir = path.dirname(storagePath);
+    fs.writeFileSync(
+      path.join(rootDir, "storage-meta.json"),
+      JSON.stringify({
+        homeserver: "https://matrix.example.org",
+        userId: "@bot:example.org",
+        accountId: "default",
+        accessTokenHash: "token-hash",
+        deviceId: null,
+      }),
+      "utf8",
+    );
+
+    const store = new FileBackedMatrixSyncStore(storagePath);
+    await store.setSyncData(createSyncResponse("claimed-token"));
+    await store.flush();
+
+    const meta = JSON.parse(fs.readFileSync(path.join(rootDir, "storage-meta.json"), "utf8")) as {
+      currentTokenStateClaimed?: boolean;
+    };
+    expect(meta.currentTokenStateClaimed).toBe(true);
+  });
+
+  it("only treats sync state as restart-safe after a clean shutdown persist", async () => {
+    const storagePath = createStoragePath();
+
+    const firstStore = new FileBackedMatrixSyncStore(storagePath);
+    await firstStore.setSyncData(createSyncResponse("s123"));
+    await firstStore.flush();
+
+    const afterDirtyPersist = new FileBackedMatrixSyncStore(storagePath);
+    expect(afterDirtyPersist.hasSavedSync()).toBe(true);
+    expect(afterDirtyPersist.hasSavedSyncFromCleanShutdown()).toBe(false);
+
+    firstStore.markCleanShutdown();
+    await firstStore.flush();
+
+    const afterCleanShutdown = new FileBackedMatrixSyncStore(storagePath);
+    expect(afterCleanShutdown.hasSavedSync()).toBe(true);
+    expect(afterCleanShutdown.hasSavedSyncFromCleanShutdown()).toBe(true);
+  });
+
+  it("clears the clean-shutdown marker once fresh sync data arrives", async () => {
+    const storagePath = createStoragePath();
+
+    const firstStore = new FileBackedMatrixSyncStore(storagePath);
+    await firstStore.setSyncData(createSyncResponse("s123"));
+    firstStore.markCleanShutdown();
+    await firstStore.flush();
+
+    const restartedStore = new FileBackedMatrixSyncStore(storagePath);
+    expect(restartedStore.hasSavedSyncFromCleanShutdown()).toBe(true);
+
+    await restartedStore.setSyncData(createSyncResponse("s456"));
+    await restartedStore.flush();
+
+    const afterNewSync = new FileBackedMatrixSyncStore(storagePath);
+    expect(afterNewSync.hasSavedSync()).toBe(true);
+    expect(afterNewSync.hasSavedSyncFromCleanShutdown()).toBe(false);
+    await expect(afterNewSync.getSavedSyncToken()).resolves.toBe("s456");
+  });
+
+  it("coalesces background persistence until the debounce window elapses", async () => {
+    vi.useFakeTimers();
+    const storagePath = createStoragePath();
+    const writeSpy = vi.spyOn(jsonStore, "writeJsonFileAtomically").mockResolvedValue();
+
+    const store = new FileBackedMatrixSyncStore(storagePath);
+    await store.setSyncData(createSyncResponse("s111"));
+    await store.setSyncData(createSyncResponse("s222"));
+    await store.storeClientOptions({ lazyLoadMembers: true });
+
+    expect(writeSpy).not.toHaveBeenCalled();
+
+    await vi.advanceTimersByTimeAsync(249);
+    expect(writeSpy).not.toHaveBeenCalled();
+
+    await vi.advanceTimersByTimeAsync(1);
+    await Promise.resolve();
+    expect(writeSpy).toHaveBeenCalledTimes(1);
+    expect(writeSpy).toHaveBeenCalledWith(
+      storagePath,
+      expect.objectContaining({
+        savedSync: expect.objectContaining({
+          nextBatch: "s222",
+        }),
+        clientOptions: {
+          lazyLoadMembers: true,
+        },
+      }),
+    );
+
+    await store.flush();
+  });
+
+  it("waits for an in-flight persist when shutdown flush runs", async () => {
+    vi.useFakeTimers();
+    const storagePath = createStoragePath();
+    const writeDeferred = createDeferred();
+    const writeSpy = vi
+      .spyOn(jsonStore, "writeJsonFileAtomically")
+      .mockImplementation(async () => writeDeferred.promise);
+
+    const store = new FileBackedMatrixSyncStore(storagePath);
+    await store.setSyncData(createSyncResponse("s777"));
+    await vi.advanceTimersByTimeAsync(250);
+
+    let flushCompleted = false;
+    const flushPromise = store.flush().then(() => {
+      flushCompleted = true;
+    });
+
+    await Promise.resolve();
+    expect(writeSpy).toHaveBeenCalledTimes(1);
+    expect(flushCompleted).toBe(false);
+
+    writeDeferred.resolve();
+    await flushPromise;
+    expect(flushCompleted).toBe(true);
+  });
+
+  it("persists client options alongside sync state", async () => {
+    const storagePath = createStoragePath();
+
+    const firstStore = new FileBackedMatrixSyncStore(storagePath);
+    await firstStore.storeClientOptions({ lazyLoadMembers: true });
+    await firstStore.flush();
+
+    const secondStore = new FileBackedMatrixSyncStore(storagePath);
+    await expect(secondStore.getClientOptions()).resolves.toEqual({ lazyLoadMembers: true });
+  });
+
+  it("loads legacy raw sync payloads from bot-storage.json", async () => {
+    const storagePath = createStoragePath();
+
+    fs.writeFileSync(
+      storagePath,
+      JSON.stringify({
+        next_batch: "legacy-token",
+        rooms: {
+          join: {},
+        },
+        account_data: {
+          events: [],
+        },
+      }),
+      "utf8",
+    );
+
+    const store = new FileBackedMatrixSyncStore(storagePath);
+    expect(store.hasSavedSync()).toBe(true);
+    await expect(store.getSavedSyncToken()).resolves.toBe("legacy-token");
+    await expect(store.getSavedSync()).resolves.toMatchObject({
+      nextBatch: "legacy-token",
+      roomsData: {
+        join: {},
+      },
+      accountData: [],
+    });
+  });
+});