agent-relay 3.1.10 → 3.1.12

package/packages/openclaw/src/__tests__/gateway-poll-fallback.test.ts

@@ -0,0 +1,467 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+const eventHandlers: Record<string, Array<(...args: unknown[]) => void>> = {};
+
+function registerHandler(event: string) {
+  return (handler: (...args: unknown[]) => void) => {
+    if (!eventHandlers[event]) eventHandlers[event] = [];
+    eventHandlers[event].push(handler);
+    return () => {
+      eventHandlers[event] = eventHandlers[event].filter((entry) => entry !== handler);
+    };
+  };
+}
+
+function fireEvent(event: string, ...args: unknown[]) {
+  for (const handler of eventHandlers[event] ?? []) {
+    handler(...args);
+  }
+}
+
+const mockAgentClient = {
+  connect: vi.fn(),
+  disconnect: vi.fn().mockResolvedValue(undefined),
+  subscribe: vi.fn(),
+  channels: {
+    join: vi.fn().mockResolvedValue({ ok: true }),
+    create: vi.fn().mockResolvedValue({ name: 'general' }),
+  },
+  on: {
+    connected: registerHandler('connected'),
+    messageCreated: registerHandler('messageCreated'),
+    threadReply: registerHandler('threadReply'),
+    dmReceived: registerHandler('dmReceived'),
+    groupDmReceived: registerHandler('groupDmReceived'),
+    commandInvoked: registerHandler('commandInvoked'),
+    reactionAdded: registerHandler('reactionAdded'),
+    reactionRemoved: registerHandler('reactionRemoved'),
+    reconnecting: registerHandler('reconnecting'),
+    disconnected: registerHandler('disconnected'),
+    error: registerHandler('error'),
+  },
+};
+
+const registerOrGet = vi.fn().mockResolvedValue({ name: 'test-claw', token: 'tok_test' });
+const registerOrRotate = vi.fn().mockResolvedValue({ name: 'test-claw', token: 'tok_rotated' });
+
+const fsMocks = vi.hoisted(() => ({
+  readFile: vi.fn(),
+  writeFile: vi.fn(),
+  rename: vi.fn(),
+  mkdir: vi.fn(),
+  chmod: vi.fn(),
+}));
+
+const { readFile, writeFile, rename, mkdir } = fsMocks;
+
+vi.mock('@relaycast/sdk', () => ({
+  RelayCast: vi.fn().mockImplementation(() => ({
+    agents: {
+      registerOrGet,
+      registerOrRotate,
+    },
+    as: vi.fn().mockReturnValue(mockAgentClient),
+  })),
+}));
+
+vi.mock('../spawn/manager.js', () => ({
+  SpawnManager: vi.fn().mockImplementation(() => ({
+    size: 0,
+    spawn: vi.fn(),
+    release: vi.fn(),
+    releaseByName: vi.fn(),
+    releaseAll: vi.fn().mockResolvedValue(undefined),
+    list: vi.fn().mockReturnValue([]),
+    get: vi.fn(),
+  })),
+}));
+
+vi.mock('node:fs/promises', () => ({
+  ...fsMocks,
+}));
+
+vi.mock('node:fs', () => ({
+  existsSync: vi.fn().mockReturnValue(false),
+}));
+
+vi.mock('node:http', async (importOriginal) => {
+  const actual = await importOriginal<typeof import('node:http')>();
+  return {
+    ...actual,
+    createServer: vi.fn().mockReturnValue({
+      listen: vi.fn((_port: number, _host: string, cb: () => void) => cb()),
+      close: vi.fn((cb?: () => void) => cb?.()),
+      on: vi.fn(),
+      address: vi.fn().mockReturnValue({ port: 18790 }),
+    }),
+  };
+});
+
+import { InboundGateway } from '../gateway.js';
+
+function response(status: number, body: unknown, headers?: Record<string, string>) {
+  const normalized = Object.fromEntries(
+    Object.entries(headers ?? {}).map(([key, value]) => [key.toLowerCase(), value])
+  );
+  return {
+    ok: status >= 200 && status < 300,
+    status,
+    headers: {
+      get: (name: string) => normalized[name.toLowerCase()] ?? null,
+    },
+    json: vi.fn().mockResolvedValue(body),
+  };
+}
+
+function pendingFetch(init?: RequestInit): Promise<never> {
+  return new Promise((_resolve, reject) => {
+    const signal = init?.signal as AbortSignal | undefined;
+    signal?.addEventListener('abort', () => reject(new Error('aborted')), { once: true });
+  });
+}
+
+function createGateway(
+  pollFallbackOverrides: {
+    wsFailureThreshold?: number;
+    timeoutSeconds?: number;
+    limit?: number;
+    initialCursor?: string;
+    probeWs?: {
+      enabled?: boolean;
+      intervalMs?: number;
+      stableGraceMs?: number;
+    };
+  } = {}
+) {
+  const sendMessage = vi.fn().mockResolvedValue({ event_id: 'evt_out_1' });
+  const gateway = new InboundGateway({
+    config: {
+      apiKey: 'rk_live_test',
+      clawName: 'test-claw',
+      baseUrl: 'http://127.0.0.1:8888',
+      channels: ['general'],
+      transport: {
+        pollFallback: {
+          enabled: true,
+          wsFailureThreshold: 1,
+          timeoutSeconds: 1,
+          ...pollFallbackOverrides,
+          probeWs: {
+            enabled: true,
+            intervalMs: 5_000,
+            stableGraceMs: 10,
+            ...pollFallbackOverrides.probeWs,
+          },
+        },
+      },
+    },
+    relaySender: { sendMessage },
+  });
+  return { gateway, sendMessage };
+}
+
+describe('InboundGateway poll fallback', () => {
+  const fetchMock = vi.fn();
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    vi.useRealTimers();
+    vi.stubGlobal('fetch', fetchMock);
+    for (const key of Object.keys(eventHandlers)) {
+      eventHandlers[key] = [];
+    }
+    readFile.mockReset();
+    readFile.mockRejectedValue(Object.assign(new Error('ENOENT'), { code: 'ENOENT' }));
+    writeFile.mockReset();
+    writeFile.mockResolvedValue(undefined);
+    rename.mockReset();
+    rename.mockResolvedValue(undefined);
+    mkdir.mockReset();
+    mkdir.mockResolvedValue(undefined);
+  });
+
+  afterEach(async () => {
+    vi.unstubAllGlobals();
+    vi.useRealTimers();
+  });
+
+  it('falls back to poll and persists the committed cursor after successful delivery', async () => {
+    fetchMock
+      .mockResolvedValueOnce(
+        response(200, {
+          events: [
+            {
+              id: 'evt_poll_1',
+              sequence: 1,
+              timestamp: '2026-03-06T04:00:00Z',
+              payload: {
+                type: 'message.created',
+                channel: 'general',
+                message: {
+                  id: 'msg_1',
+                  agentName: 'alice',
+                  text: 'hello from poll',
+                },
+              },
+            },
+          ],
+          nextCursor: 'cursor_1',
+          hasMore: false,
+        })
+      )
+      .mockImplementation((_input, init) => pendingFetch(init));
+
+    const { gateway, sendMessage } = createGateway();
+    await gateway.start();
+
+    fireEvent('error', new Error('proxy blocked'));
+
+    await vi.waitFor(() => {
+      expect(sendMessage).toHaveBeenCalledTimes(1);
+    });
+
+    expect(sendMessage.mock.calls[0][0].text).toBe('[relaycast:general] @alice: hello from poll\n(reply with: post_message channel="general" or reply_to_thread message_id="msg_1")');
+    expect(String(fetchMock.mock.calls[0]?.[0])).toContain('/messages/poll');
+    expect(String(fetchMock.mock.calls[0]?.[0])).toContain('cursor=0');
+    expect(writeFile).toHaveBeenCalledWith(
+      expect.stringContaining('inbound-cursor.json.tmp'),
+      expect.stringContaining('"cursor": "cursor_1"'),
+      'utf-8'
+    );
+
+    await gateway.stop();
+  });
+
+  it('resets a stale cursor on 409 and resumes from the initial cursor', async () => {
+    readFile.mockResolvedValueOnce(
+      JSON.stringify({
+        cursor: 'stale_cursor',
+        lastSequence: 41,
+        recentEventIds: [],
+        updatedAt: '2026-03-06T03:59:00Z',
+      })
+    );
+
+    fetchMock
+      .mockImplementationOnce(async (input) => {
+        expect(String(input)).toContain('cursor=stale_cursor');
+        return response(409, {});
+      })
+      .mockImplementationOnce(async (input) => {
+        expect(String(input)).toContain('cursor=0');
+        return response(200, {
+          events: [
+            {
+              id: 'evt_poll_42',
+              sequence: 42,
+              timestamp: '2026-03-06T04:01:00Z',
+              payload: {
+                type: 'message.created',
+                channel: 'general',
+                message: {
+                  id: 'msg_42',
+                  agentName: 'alice',
+                  text: 'resumed after reset',
+                },
+              },
+            },
+          ],
+          nextCursor: 'cursor_42',
+          hasMore: false,
+        });
+      })
+      .mockImplementation((_input, init) => pendingFetch(init));
+
+    const { gateway, sendMessage } = createGateway();
+    await gateway.start();
+
+    fireEvent('disconnected');
+
+    await vi.waitFor(() => {
+      expect(sendMessage).toHaveBeenCalledTimes(1);
+    });
+
+    expect(sendMessage.mock.calls[0][0].text).toBe('[relaycast:general] @alice: resumed after reset\n(reply with: post_message channel="general" or reply_to_thread message_id="msg_42")');
+    expect(writeFile).toHaveBeenCalledWith(
+      expect.stringContaining('inbound-cursor.json.tmp'),
+      expect.stringContaining('"cursor": "cursor_42"'),
+      'utf-8'
+    );
+
+    await gateway.stop();
+  });
+
+  it('promotes back to WS after a stable recovery window', async () => {
+    vi.useFakeTimers();
+
+    fetchMock
+      .mockResolvedValueOnce(
+        response(200, {
+          events: [],
+          nextCursor: 'cursor_0',
+          hasMore: false,
+        })
+      )
+      .mockImplementationOnce((_input, init) => pendingFetch(init))
+      .mockResolvedValueOnce(
+        response(200, {
+          events: [],
+          nextCursor: 'cursor_0',
+          hasMore: false,
+        })
+      )
+      .mockImplementation((_input, init) => pendingFetch(init));
+
+    const { gateway, sendMessage } = createGateway();
+    await gateway.start();
+
+    fireEvent('error', new Error('proxy blocked'));
+
+    await vi.waitFor(() => {
+      expect(fetchMock).toHaveBeenCalled();
+    });
+
+    fireEvent('connected');
+    await vi.advanceTimersByTimeAsync(1_100);
+
+    fireEvent('messageCreated', {
+      type: 'message.created',
+      channel: 'general',
+      message: {
+        id: 'msg_ws_1',
+        agentName: 'bob',
+        text: 'back on ws',
+      },
+    });
+
+    await vi.waitFor(() => {
+      expect(sendMessage).toHaveBeenCalledTimes(1);
+    });
+
+    expect(sendMessage.mock.calls[0][0].text).toBe('[relaycast:general] @bob: back on ws\n(reply with: post_message channel="general" or reply_to_thread message_id="msg_ws_1")');
+
+    await gateway.stop();
+  });
+
+  it('processes WS messages during RECOVERING_WS before promotion completes', async () => {
+    vi.useFakeTimers();
+
+    fetchMock
+      .mockResolvedValueOnce(
+        response(200, {
+          events: [],
+          nextCursor: 'cursor_0',
+          hasMore: false,
+        })
+      )
+      .mockImplementation((_input, init) => pendingFetch(init));
+
+    const { gateway, sendMessage } = createGateway({
+      probeWs: {
+        stableGraceMs: 5_000,
+      },
+    });
+    await gateway.start();
+
+    fireEvent('error', new Error('proxy blocked'));
+
+    await vi.waitFor(() => {
+      expect(fetchMock).toHaveBeenCalled();
+    });
+
+    fireEvent('connected');
+    await vi.advanceTimersByTimeAsync(100);
+
+    fireEvent('messageCreated', {
+      type: 'message.created',
+      channel: 'general',
+      message: {
+        id: 'msg_ws_recovering',
+        agentName: 'carol',
+        text: 'delivered during recovery',
+      },
+    });
+
+    await vi.waitFor(() => {
+      expect(sendMessage).toHaveBeenCalledTimes(1);
+    });
+
+    expect(sendMessage.mock.calls[0][0].text).toBe('[relaycast:general] @carol: delivered during recovery\n(reply with: post_message channel="general" or reply_to_thread message_id="msg_ws_recovering")');
+
+    await gateway.stop();
+  });
+
+  it('does not redeliver a message that was already committed in poll mode after WS recovery', async () => {
+    vi.useFakeTimers();
+
+    fetchMock
+      .mockResolvedValueOnce(
+        response(200, {
+          events: [
+            {
+              id: 'evt_poll_1',
+              sequence: 1,
+              timestamp: '2026-03-06T04:00:00Z',
+              payload: {
+                type: 'message.created',
+                channel: 'general',
+                message: {
+                  id: 'msg_1',
+                  agentName: 'alice',
+                  text: 'hello from poll',
+                },
+              },
+            },
+          ],
+          nextCursor: 'cursor_1',
+          hasMore: false,
+        })
+      )
+      .mockImplementationOnce((_input, init) => pendingFetch(init))
+      .mockResolvedValueOnce(
+        response(200, {
+          events: [],
+          nextCursor: 'cursor_1',
+          hasMore: false,
+        })
+      )
+      .mockImplementation((_input, init) => pendingFetch(init));
+
+    const { gateway, sendMessage } = createGateway();
+    await gateway.start();
+
+    fireEvent('error', new Error('proxy blocked'));
+
+    await vi.waitFor(() => {
+      expect(sendMessage).toHaveBeenCalledTimes(1);
+    });
+
+    fireEvent('connected');
+    await vi.advanceTimersByTimeAsync(1_100);
+
+    fireEvent('messageCreated', {
+      type: 'message.created',
+      channel: 'general',
+      message: {
+        id: 'msg_1',
+        agentName: 'alice',
+        text: 'hello from poll',
+      },
+    });
+
+    await vi.advanceTimersByTimeAsync(0);
+    expect(sendMessage).toHaveBeenCalledTimes(1);
+
+    await gateway.stop();
+  });
+
+  it('uses a single jitter pass for 429 responses without Retry-After', async () => {
+    vi.spyOn(Math, 'random').mockReturnValue(0);
+    fetchMock.mockResolvedValueOnce(response(429, {}));
+
+    const { gateway } = createGateway();
+    const delayMs = await (gateway as any).pollOnce(1);
+
+    expect(delayMs).toBe(550);
+  });
+});

package/packages/openclaw/src/__tests__/gateway-threads.test.ts

@@ -80,6 +80,8 @@ vi.mock('node:fs/promises', () => ({
   readFile: vi.fn().mockResolvedValue('{"spawns":[]}'),
   writeFile: vi.fn().mockResolvedValue(undefined),
   mkdir: vi.fn().mockResolvedValue(undefined),
+  rename: vi.fn().mockResolvedValue(undefined),
+  chmod: vi.fn().mockResolvedValue(undefined),
 }));
 
 vi.mock('node:fs', () => ({
@@ -109,10 +111,7 @@ import { InboundGateway } from '../gateway.js';
 // Helpers
 // ---------------------------------------------------------------------------
 
-function createGateway(overrides?: {
-  clawName?: string;
-  channels?: string[];
-}) {
+function createGateway(overrides?: { clawName?: string; channels?: string[] }) {
   const sendMessage = vi.fn().mockResolvedValue({ event_id: 'evt_1' });
   const gateway = new InboundGateway({
     config: {
@@ -163,7 +162,7 @@ describe('InboundGateway — thread reply injection', () => {
       });
 
       const call = sendMessage.mock.calls[0][0];
-      expect(call.text).toBe('[relaycast:general] @alice: hello world');
+      expect(call.text).toBe('[relaycast:general] @alice: hello world\n(reply with: post_message channel="general" or reply_to_thread message_id="msg_1")');
       expect(call.text).not.toContain('[thread]');
 
       await gateway.stop();
@@ -189,7 +188,7 @@ describe('InboundGateway — thread reply injection', () => {
       });
 
       const call = sendMessage.mock.calls[0][0];
-      expect(call.text).toBe('[thread] [relaycast:general] @bob: replying in thread');
+      expect(call.text).toBe('[thread] [relaycast:general] @bob: replying in thread\n(reply with: reply_to_thread message_id="msg_parent_1")');
 
       await gateway.stop();
     });
@@ -330,10 +329,10 @@ describe('InboundGateway — thread reply injection', () => {
       });
 
       const firstCall = sendMessage.mock.calls[0][0];
-      expect(firstCall.text).toBe('[relaycast:general] @frank: original message');
+      expect(firstCall.text).toBe('[relaycast:general] @frank: original message\n(reply with: post_message channel="general" or reply_to_thread message_id="msg_600")');
 
       const secondCall = sendMessage.mock.calls[1][0];
-      expect(secondCall.text).toBe('[thread] [relaycast:general] @grace: reply to frank');
+      expect(secondCall.text).toBe('[thread] [relaycast:general] @grace: reply to frank\n(reply with: reply_to_thread message_id="msg_600")');
 
       await gateway.stop();
     });
@@ -386,7 +385,7 @@ describe('InboundGateway — thread reply injection', () => {
       });
 
       const call = sendMessage.mock.calls[0][0];
-      expect(call.text).toBe('[relaycast:dm] @alice: hey there');
+      expect(call.text).toBe('[relaycast:dm] @alice: hey there\n(reply with: send_dm to="alice")');
 
       await gateway.stop();
     });
@@ -459,7 +458,7 @@ describe('InboundGateway — thread reply injection', () => {
       });
 
       const call = sendMessage.mock.calls[0][0];
-      expect(call.text).toBe('[relaycast:groupdm] @carol: group message');
+      expect(call.text).toBe('[relaycast:groupdm] @carol: group message\n(reply with: send_dm to="carol")');
 
       await gateway.stop();
     });
@@ -485,7 +484,7 @@ describe('InboundGateway — thread reply injection', () => {
       });
 
       const call = sendMessage.mock.calls[0][0];
-      expect(call.text).toBe('[relaycast:command:general] @dave /deploy production --force');
+      expect(call.text).toBe('[relaycast:command:general] @dave /deploy production --force\n(command invocation \u2014 respond with: post_message channel="general")');
 
       await gateway.stop();
     });
@@ -509,7 +508,7 @@ describe('InboundGateway — thread reply injection', () => {
       });
 
       const call = sendMessage.mock.calls[0][0];
-      expect(call.text).toBe('[relaycast:command:general] @eve /status');
+      expect(call.text).toBe('[relaycast:command:general] @eve /status\n(command invocation \u2014 respond with: post_message channel="general")');
 
       await gateway.stop();
     });
@@ -552,7 +551,9 @@ describe('InboundGateway — thread reply injection', () => {
       });
 
       const call = sendMessage.mock.calls[0][0];
-      expect(call.text).toBe('[relaycast:reaction] @eve reacted thumbsup to message msg_800 (soft notification, no action required)');
+      expect(call.text).toBe(
+        '[relaycast:reaction] @eve reacted thumbsup to message msg_800 (soft notification, no action required)'
+      );
 
       await gateway.stop();
     });
@@ -573,7 +574,9 @@ describe('InboundGateway — thread reply injection', () => {
       });
 
       const call = sendMessage.mock.calls[0][0];
-      expect(call.text).toBe('[relaycast:reaction] @frank removed rocket from message msg_900 (soft notification, no action required)');
+      expect(call.text).toBe(
+        '[relaycast:reaction] @frank removed rocket from message msg_900 (soft notification, no action required)'
+      );
 
       await gateway.stop();
     });
@@ -785,7 +788,7 @@ describe('InboundGateway — thread reply injection', () => {
       });
 
       const call = sendMessage.mock.calls[0][0];
-      expect(call.text).toBe('[relaycast:dm] @alice: dm format test');
+      expect(call.text).toBe('[relaycast:dm] @alice: dm format test\n(reply with: send_dm to="alice")');
 
       await gateway.stop();
     });
@@ -809,7 +812,7 @@ describe('InboundGateway — thread reply injection', () => {
       });
 
       const call = sendMessage.mock.calls[0][0];
-      expect(call.text).toBe('[relaycast:groupdm] @bob: group dm format test');
+      expect(call.text).toBe('[relaycast:groupdm] @bob: group dm format test\n(reply with: send_dm to="bob")');
 
       await gateway.stop();
     });
@@ -833,7 +836,7 @@ describe('InboundGateway — thread reply injection', () => {
       });
 
       const call = sendMessage.mock.calls[0][0];
-      expect(call.text).toBe('[relaycast:command:general] @carol /build --prod');
+      expect(call.text).toBe('[relaycast:command:general] @carol /build --prod\n(command invocation \u2014 respond with: post_message channel="general")');
 
       await gateway.stop();
     });
@@ -854,7 +857,9 @@ describe('InboundGateway — thread reply injection', () => {
       });
 
       const call = sendMessage.mock.calls[0][0];
-      expect(call.text).toBe('[relaycast:reaction] @dave reacted fire to message msg_fmt_react (soft notification, no action required)');
+      expect(call.text).toBe(
+        '[relaycast:reaction] @dave reacted fire to message msg_fmt_react (soft notification, no action required)'
+      );
 
       await gateway.stop();
     });
@@ -879,7 +884,7 @@ describe('InboundGateway — thread reply injection', () => {
       });
 
       const call = sendMessage.mock.calls[0][0];
-      expect(call.text).toBe('[thread] [relaycast:general] @eve: thread format test');
+      expect(call.text).toBe('[thread] [relaycast:general] @eve: thread format test\n(reply with: reply_to_thread message_id="msg_fmt_parent")');
 
       await gateway.stop();
     });
@@ -904,7 +909,7 @@ describe('InboundGateway — thread reply injection', () => {
       });
 
       const call = sendMessage.mock.calls[0][0];
-      expect(call.text).toBe('[relaycast:general] @frank: channel format test');
+      expect(call.text).toBe('[relaycast:general] @frank: channel format test\n(reply with: post_message channel="general" or reply_to_thread message_id="msg_fmt_chan")');
 
       await gateway.stop();
     });
@@ -914,8 +919,11 @@ describe('InboundGateway — thread reply injection', () => {
     it('should skip messages already being processed', async () => {
       // Use a slow sendMessage to simulate a message still being processed
       let resolveFirst: (() => void) | null = null;
-      const firstCallPromise = new Promise<void>((r) => { resolveFirst = r; });
-      const sendMessage = vi.fn()
+      const firstCallPromise = new Promise<void>((r) => {
+        resolveFirst = r;
+      });
+      const sendMessage = vi
+        .fn()
         .mockImplementationOnce(async () => {
           // Block until we manually resolve
           await firstCallPromise;
@@ -971,6 +979,48 @@ describe('InboundGateway — thread reply injection', () => {
 
       await gateway.stop();
     });
+
+    it('should retry a replayed message after a failed delivery', async () => {
+      const sendMessage = vi
+        .fn()
+        .mockResolvedValueOnce({ event_id: 'unsupported_operation' })
+        .mockResolvedValueOnce({ event_id: 'evt_retry_ok' });
+
+      const gateway = new InboundGateway({
+        config: {
+          apiKey: 'rk_live_test',
+          clawName: 'test-claw',
+          baseUrl: 'https://api.relaycast.dev',
+          channels: ['general'],
+        },
+        relaySender: { sendMessage },
+      });
+      await gateway.start();
+
+      const event = {
+        type: 'message.created',
+        channel: 'general',
+        message: {
+          id: 'msg_retry_1',
+          agentName: 'alice',
+          text: 'retry me',
+          attachments: [],
+        },
+      };
+
+      fireEvent('messageCreated', event);
+      await vi.waitFor(() => {
+        expect(sendMessage).toHaveBeenCalledTimes(1);
+      });
+      await new Promise((resolve) => setTimeout(resolve, 0));
+
+      fireEvent('messageCreated', event);
+      await vi.waitFor(() => {
+        expect(sendMessage).toHaveBeenCalledTimes(2);
+      });
+
+      await gateway.stop();
+    });
   });
 
   describe('handleInbound when not running', () => {
@@ -1101,7 +1151,7 @@ describe('InboundGateway — thread reply injection', () => {
       });
 
       const call = sendMessage.mock.calls[0][0];
-      expect(call.text).toBe('[relaycast:general] @alice: normalization test');
+      expect(call.text).toBe('[relaycast:general] @alice: normalization test\n(reply with: post_message channel="general" or reply_to_thread message_id="msg_norm_1")');
 
       await gateway.stop();
     });