@vellumai/assistant 0.3.3 → 0.3.5

package/src/__tests__/call-orchestrator.test.ts

@@ -27,9 +27,18 @@ mock.module('../util/logger.js', () => ({
     }),
 }));
 
+// ── User reference mock ──────────────────────────────────────────────
+
+let mockUserReference = 'my human';
+
+mock.module('../config/user-reference.js', () => ({
+  resolveUserReference: () => mockUserReference,
+}));
+
 // ── Config mock ─────────────────────────────────────────────────────
 
 let mockCallModel: string | undefined = undefined;
+let mockDisclosure: { enabled: boolean; text: string } = { enabled: false, text: '' };
 
 mock.module('../config/loader.js', () => ({
   getConfig: () => ({
@@ -41,7 +50,7 @@ mock.module('../config/loader.js', () => ({
       userConsultTimeoutSeconds: 90,
       userConsultationTimeoutSeconds: 90,
       silenceTimeoutSeconds: 30,
-      disclosure: { enabled: false, text: '' },
+      disclosure: mockDisclosure,
       safety: { denyCategories: [] },
       model: mockCallModel,
     },
@@ -197,6 +206,8 @@ describe('call-orchestrator', () => {
   beforeEach(() => {
     resetTables();
     mockCallModel = undefined;
+    mockUserReference = 'my human';
+    mockDisclosure = { enabled: false, text: '' };
     // Reset the stream mock to default behaviour
     mockStreamFn.mockImplementation(() => createMockStream(['Hello', ' there']));
   });
@@ -280,6 +291,30 @@ describe('call-orchestrator', () => {
     orchestrator.destroy();
   });
 
+  test('strips USER_ANSWERED and USER_INSTRUCTION markers from spoken output', async () => {
+    mockStreamFn.mockImplementation(() =>
+      createMockStream([
+        'Thanks for waiting. ',
+        '[USER_ANSWERED: The guardian said 3 PM works.] ',
+        '[USER_INSTRUCTION: Keep this short.] ',
+        'I can confirm 3 PM works.',
+      ]),
+    );
+    const { relay, orchestrator } = setupOrchestrator();
+
+    await orchestrator.handleCallerUtterance('Any update?');
+
+    const allText = relay.sentTokens.map((t) => t.token).join('');
+    expect(allText).toContain('Thanks for waiting.');
+    expect(allText).toContain('I can confirm 3 PM works.');
+    expect(allText).not.toContain('[USER_ANSWERED:');
+    expect(allText).not.toContain('[USER_INSTRUCTION:');
+    expect(allText).not.toContain('USER_ANSWERED');
+    expect(allText).not.toContain('USER_INSTRUCTION');
+
+    orchestrator.destroy();
+  });
+
   // ── END_CALL pattern ──────────────────────────────────────────────
 
   test('END_CALL pattern: detects marker, calls endSession, updates status to completed', async () => {
@@ -414,6 +449,166 @@ describe('call-orchestrator', () => {
     orchestrator.destroy();
   });
 
+  test('LLM APIUserAbortError: treats as expected abort without technical-issue fallback', async () => {
+    mockStreamFn.mockImplementation(() => {
+      const emitter = new EventEmitter();
+      return {
+        on: (event: string, handler: (...args: unknown[]) => void) => {
+          emitter.on(event, handler);
+          return { on: () => ({ on: () => ({}) }) };
+        },
+        finalMessage: () => {
+          const err = new Error('user abort');
+          err.name = 'APIUserAbortError';
+          return Promise.reject(err);
+        },
+      };
+    });
+
+    const { relay, orchestrator } = setupOrchestrator();
+    await orchestrator.handleCallerUtterance('Hello');
+
+    const errorTokens = relay.sentTokens.filter((t) => t.token.includes('technical issue'));
+    expect(errorTokens.length).toBe(0);
+    expect(orchestrator.getState()).toBe('idle');
+
+    orchestrator.destroy();
+  });
+
+  test('stale superseded turn errors do not emit technical-issue fallback', async () => {
+    let callCount = 0;
+    mockStreamFn.mockImplementation(() => {
+      callCount++;
+      if (callCount === 1) {
+        const emitter = new EventEmitter();
+        return {
+          on: (event: string, handler: (...args: unknown[]) => void) => {
+            emitter.on(event, handler);
+            return { on: () => ({ on: () => ({}) }) };
+          },
+          finalMessage: () =>
+            new Promise((_, reject) => {
+              setTimeout(() => reject(new Error('stale stream failure')), 20);
+            }),
+        };
+      }
+      return createMockStream(['Second turn response.']);
+    });
+
+    const { relay, orchestrator } = setupOrchestrator();
+
+    const firstTurnPromise = orchestrator.handleCallerUtterance('First utterance');
+    // Allow the first turn to enter runLlm before the second utterance interrupts it.
+    await new Promise((r) => setTimeout(r, 5));
+    const secondTurnPromise = orchestrator.handleCallerUtterance('Second utterance');
+
+    await Promise.all([firstTurnPromise, secondTurnPromise]);
+
+    const allTokens = relay.sentTokens.map((t) => t.token).join('');
+    expect(allTokens).toContain('Second turn response.');
+    expect(allTokens).not.toContain('technical issue');
+
+    orchestrator.destroy();
+  });
+
+  test('rapid caller barge-in coalesces contiguous user turns for role alternation', async () => {
+    let callCount = 0;
+    mockStreamFn.mockImplementation((...args: unknown[]) => {
+      callCount++;
+      if (callCount === 1) {
+        const emitter = new EventEmitter();
+        const options = args[1] as { signal?: AbortSignal } | undefined;
+        return {
+          on: (event: string, handler: (...evtArgs: unknown[]) => void) => {
+            emitter.on(event, handler);
+            return { on: () => ({ on: () => ({}) }) };
+          },
+          finalMessage: () =>
+            new Promise((_, reject) => {
+              options?.signal?.addEventListener('abort', () => {
+                const err = new Error('aborted');
+                err.name = 'AbortError';
+                reject(err);
+              }, { once: true });
+            }),
+        };
+      }
+
+      const firstArg = args[0] as { messages: Array<{ role: string; content: string }> };
+      const roles = firstArg.messages.map((m) => m.role);
+      for (let i = 1; i < roles.length; i++) {
+        expect(!(roles[i - 1] === 'user' && roles[i] === 'user')).toBe(true);
+      }
+      const userMessages = firstArg.messages.filter((m) => m.role === 'user');
+      const lastUser = userMessages[userMessages.length - 1];
+      expect(lastUser?.content).toContain('First caller utterance');
+      expect(lastUser?.content).toContain('Second caller utterance');
+      return createMockStream(['Merged turn handled.']);
+    });
+
+    const { relay, orchestrator } = setupOrchestrator();
+    const firstTurnPromise = orchestrator.handleCallerUtterance('First caller utterance');
+    await new Promise((r) => setTimeout(r, 5));
+    const secondTurnPromise = orchestrator.handleCallerUtterance('Second caller utterance');
+
+    await Promise.all([firstTurnPromise, secondTurnPromise]);
+
+    const allTokens = relay.sentTokens.map((t) => t.token).join('');
+    expect(allTokens).toContain('Merged turn handled.');
+
+    orchestrator.destroy();
+  });
+
+  test('interrupt then next caller prompt still preserves role alternation', async () => {
+    let callCount = 0;
+    mockStreamFn.mockImplementation((...args: unknown[]) => {
+      callCount++;
+      if (callCount === 1) {
+        const emitter = new EventEmitter();
+        const options = args[1] as { signal?: AbortSignal } | undefined;
+        return {
+          on: (event: string, handler: (...evtArgs: unknown[]) => void) => {
+            emitter.on(event, handler);
+            return { on: () => ({ on: () => ({}) }) };
+          },
+          finalMessage: () =>
+            new Promise((_, reject) => {
+              options?.signal?.addEventListener('abort', () => {
+                const err = new Error('aborted');
+                err.name = 'AbortError';
+                reject(err);
+              }, { once: true });
+            }),
+        };
+      }
+
+      const firstArg = args[0] as { messages: Array<{ role: string; content: string }> };
+      const roles = firstArg.messages.map((m) => m.role);
+      for (let i = 1; i < roles.length; i++) {
+        expect(!(roles[i - 1] === 'user' && roles[i] === 'user')).toBe(true);
+      }
+      const userMessages = firstArg.messages.filter((m) => m.role === 'user');
+      const lastUser = userMessages[userMessages.length - 1];
+      expect(lastUser?.content).toContain('First caller utterance');
+      expect(lastUser?.content).toContain('Second caller utterance');
+      return createMockStream(['Post-interrupt response.']);
+    });
+
+    const { relay, orchestrator } = setupOrchestrator();
+    const firstTurnPromise = orchestrator.handleCallerUtterance('First caller utterance');
+    await new Promise((r) => setTimeout(r, 5));
+    orchestrator.handleInterrupt();
+    const secondTurnPromise = orchestrator.handleCallerUtterance('Second caller utterance');
+
+    await Promise.all([firstTurnPromise, secondTurnPromise]);
+
+    const allTokens = relay.sentTokens.map((t) => t.token).join('');
+    expect(allTokens).toContain('Post-interrupt response.');
+    expect(allTokens).not.toContain('technical issue');
+
+    orchestrator.destroy();
+  });
+
   test('handleUserAnswer: returns false when not in waiting_on_user state', async () => {
     const { orchestrator } = setupOrchestrator();
 
@@ -435,6 +630,87 @@ describe('call-orchestrator', () => {
     orchestrator.destroy();
   });
 
+  test('handleInterrupt: increments llmRunVersion to suppress stale turn side effects', async () => {
+    // Use a stream whose finalMessage resolves immediately but whose
+    // continuation (the code after `await stream.finalMessage()`) will
+    // run asynchronously. This simulates the race where the promise
+    // microtask is queued right as handleInterrupt fires.
+    mockStreamFn.mockImplementation(() => {
+      const emitter = new EventEmitter();
+      return {
+        on: (event: string, handler: (...args: unknown[]) => void) => {
+          emitter.on(event, handler);
+          return { on: () => ({ on: () => ({}) }) };
+        },
+        finalMessage: () => {
+          // Emit some tokens synchronously
+          emitter.emit('text', 'Stale response that should be suppressed.');
+          return Promise.resolve({
+            content: [{ type: 'text', text: 'Stale response that should be suppressed.' }],
+          });
+        },
+      };
+    });
+
+    const { relay, orchestrator } = setupOrchestrator();
+
+    // Start an LLM turn (don't await — we want to interrupt mid-flight)
+    const turnPromise = orchestrator.handleCallerUtterance('Hello');
+
+    // Interrupt immediately. Because finalMessage resolves as a microtask,
+    // its continuation hasn't run yet. handleInterrupt increments
+    // llmRunVersion so the continuation's isCurrentRun check will fail.
+    orchestrator.handleInterrupt();
+
+    // Let the stale turn's microtask continuation execute
+    await turnPromise;
+
+    // The orchestrator should remain idle — the stale turn must not
+    // have pushed state to waiting_on_user or any other post-turn state.
+    expect(orchestrator.getState()).toBe('idle');
+
+    // No technical-issue fallback should have been sent
+    const errorTokens = relay.sentTokens.filter((t) => t.token.includes('technical issue'));
+    expect(errorTokens.length).toBe(0);
+
+    // endSession should NOT have been called by the stale turn
+    expect(relay.endCalled).toBe(false);
+
+    orchestrator.destroy();
+  });
+
+  test('handleInterrupt: sends turn terminator when interrupting active speech', async () => {
+    mockStreamFn.mockImplementation((...args: unknown[]) => {
+      const emitter = new EventEmitter();
+      const options = args[1] as { signal?: AbortSignal } | undefined;
+      return {
+        on: (event: string, handler: (...evtArgs: unknown[]) => void) => {
+          emitter.on(event, handler);
+          return { on: () => ({ on: () => ({}) }) };
+        },
+        finalMessage: () =>
+          new Promise((_, reject) => {
+            options?.signal?.addEventListener('abort', () => {
+              const err = new Error('aborted');
+              err.name = 'AbortError';
+              reject(err);
+            }, { once: true });
+          }),
+      };
+    });
+
+    const { relay, orchestrator } = setupOrchestrator();
+    const turnPromise = orchestrator.handleCallerUtterance('Start speaking');
+    await new Promise((r) => setTimeout(r, 5));
+    orchestrator.handleInterrupt();
+    await turnPromise;
+
+    const endTurnMarkers = relay.sentTokens.filter((t) => t.token === '' && t.last === true);
+    expect(endTurnMarkers.length).toBeGreaterThan(0);
+
+    orchestrator.destroy();
+  });
+
   // ── destroy ───────────────────────────────────────────────────────
 
   test('destroy: unregisters orchestrator', () => {
@@ -622,4 +898,118 @@ describe('call-orchestrator', () => {
 
     orchestrator.destroy();
   });
+
+  // ── System prompt: identity phrasing ────────────────────────────────
+
+  test('system prompt contains resolved user reference (default)', async () => {
+    mockStreamFn.mockImplementation((...args: unknown[]) => {
+      const firstArg = args[0] as { system: string };
+      expect(firstArg.system).toContain('on behalf of my human');
+      return createMockStream(['Hello.']);
+    });
+
+    const { orchestrator } = setupOrchestrator();
+    await orchestrator.handleCallerUtterance('Hi');
+    orchestrator.destroy();
+  });
+
+  test('system prompt contains resolved user reference when set to a name', async () => {
+    mockUserReference = 'John';
+    mockStreamFn.mockImplementation((...args: unknown[]) => {
+      const firstArg = args[0] as { system: string };
+      expect(firstArg.system).toContain('on behalf of John');
+      return createMockStream(['Hello John\'s contact.']);
+    });
+
+    const { orchestrator } = setupOrchestrator();
+    await orchestrator.handleCallerUtterance('Hi');
+    orchestrator.destroy();
+  });
+
+  test('system prompt does not hardcode "your user" in the opening line', async () => {
+    mockUserReference = 'Alice';
+    mockStreamFn.mockImplementation((...args: unknown[]) => {
+      const firstArg = args[0] as { system: string };
+      expect(firstArg.system).not.toContain('on behalf of your user');
+      expect(firstArg.system).toContain('on behalf of Alice');
+      return createMockStream(['Hi there.']);
+    });
+
+    const { orchestrator } = setupOrchestrator();
+    await orchestrator.handleCallerUtterance('Hello');
+    orchestrator.destroy();
+  });
+
+  test('system prompt includes assistant identity bias rule', async () => {
+    mockStreamFn.mockImplementation((...args: unknown[]) => {
+      const firstArg = args[0] as { system: string };
+      expect(firstArg.system).toContain('refer to yourself as an assistant');
+      expect(firstArg.system).toContain('Avoid the phrase "AI assistant" unless directly asked');
+      return createMockStream(['Sure thing.']);
+    });
+
+    const { orchestrator } = setupOrchestrator();
+    await orchestrator.handleCallerUtterance('Hi');
+    orchestrator.destroy();
+  });
+
+  test('assistant identity rule appears before disclosure rule in prompt', async () => {
+    mockStreamFn.mockImplementation((...args: unknown[]) => {
+      const firstArg = args[0] as { system: string };
+      const prompt = firstArg.system;
+      const identityIdx = prompt.indexOf('refer to yourself as an assistant');
+      const disclosureIdx = prompt.indexOf('Be concise');
+      expect(identityIdx).toBeGreaterThan(-1);
+      expect(disclosureIdx).toBeGreaterThan(-1);
+      expect(identityIdx).toBeLessThan(disclosureIdx);
+      return createMockStream(['OK.']);
+    });
+
+    const { orchestrator } = setupOrchestrator();
+    await orchestrator.handleCallerUtterance('Test');
+    orchestrator.destroy();
+  });
+
+  test('system prompt uses disclosure text when disclosure is enabled', async () => {
+    mockDisclosure = {
+      enabled: true,
+      text: 'At the very beginning of the call, introduce yourself as an assistant calling on behalf of the person you represent. Do not say "AI assistant".',
+    };
+    mockStreamFn.mockImplementation((...args: unknown[]) => {
+      const firstArg = args[0] as { system: string };
+      expect(firstArg.system).toContain('introduce yourself as an assistant calling on behalf of the person you represent');
+      expect(firstArg.system).toContain('Do not say "AI assistant"');
+      return createMockStream(['Hello, I am calling on behalf of my human.']);
+    });
+
+    const { orchestrator } = setupOrchestrator();
+    await orchestrator.handleCallerUtterance('Who is this?');
+    orchestrator.destroy();
+  });
+
+  test('system prompt falls back to "Begin the conversation naturally" when disclosure is disabled', async () => {
+    mockDisclosure = { enabled: false, text: '' };
+    mockStreamFn.mockImplementation((...args: unknown[]) => {
+      const firstArg = args[0] as { system: string };
+      expect(firstArg.system).toContain('Begin the conversation naturally');
+      expect(firstArg.system).not.toContain('introduce yourself as an assistant calling on behalf of the person');
+      return createMockStream(['Hello there.']);
+    });
+
+    const { orchestrator } = setupOrchestrator();
+    await orchestrator.handleCallerUtterance('Hi');
+    orchestrator.destroy();
+  });
+
+  test('system prompt does not use "AI assistant" as a self-identity label', async () => {
+    mockStreamFn.mockImplementation((...args: unknown[]) => {
+      const firstArg = args[0] as { system: string };
+      expect(firstArg.system).not.toMatch(/(?:you are|call yourself|introduce yourself as).*AI assistant/i);
+      return createMockStream(['Got it.']);
+    });
+
+    const { orchestrator } = setupOrchestrator();
+    await orchestrator.handleCallerUtterance('Hello');
+    orchestrator.destroy();
+  });
 });

package/src/__tests__/call-routes-http.test.ts

@@ -80,10 +80,10 @@ mock.module('../calls/twilio-provider.js', () => ({
 
 // Mock Twilio config
 mock.module('../calls/twilio-config.js', () => ({
-  getTwilioConfig: () => ({
+  getTwilioConfig: (assistantId?: string) => ({
     accountSid: 'AC_test',
     authToken: 'test_token',
-    phoneNumber: '+15550001111',
+    phoneNumber: assistantId === 'asst-alpha' ? '+15550009999' : '+15550001111',
     webhookBaseUrl: 'https://test.example.com',
     wssBaseUrl: 'wss://test.example.com',
   }),
@@ -168,6 +168,10 @@ describe('runtime call routes — HTTP layer', () => {
     return `http://127.0.0.1:${port}/v1/calls${path}`;
   }
 
+  function assistantCallsUrl(assistantId: string, path = ''): string {
+    return `http://127.0.0.1:${port}/v1/assistants/${assistantId}/calls${path}`;
+  }
+
   // ── POST /v1/calls/start ────────────────────────────────────────────
 
   test('POST /v1/calls/start returns 201 with call session', async () => {
@@ -222,6 +226,27 @@ describe('runtime call routes — HTTP layer', () => {
     await stopServer();
   });
 
+  test('POST /v1/assistants/:assistantId/calls/start uses assistant-scoped caller number', async () => {
+    await startServer();
+    ensureConversation('conv-start-scoped-1');
+
+    const res = await fetch(assistantCallsUrl('asst-alpha', '/start'), {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json', ...AUTH_HEADERS },
+      body: JSON.stringify({
+        phoneNumber: '+15559997777',
+        task: 'Check order status',
+        conversationId: 'conv-start-scoped-1',
+      }),
+    });
+
+    expect(res.status).toBe(201);
+    const body = await res.json() as { fromNumber: string };
+    expect(body.fromNumber).toBe('+15550009999');
+
+    await stopServer();
+  });
+
   test('POST /v1/calls/start returns 400 for invalid phone number', async () => {
     await startServer();
     ensureConversation('conv-start-2');