@vellumai/assistant 0.3.2 → 0.3.4

package/src/__tests__/call-orchestrator.test.ts

@@ -27,6 +27,14 @@ mock.module('../util/logger.js', () => ({
     }),
 }));
 
+// ── User reference mock ──────────────────────────────────────────────
+
+let mockUserReference = 'my human';
+
+mock.module('../config/user-reference.js', () => ({
+  resolveUserReference: () => mockUserReference,
+}));
+
 // ── Config mock ─────────────────────────────────────────────────────
 
 let mockCallModel: string | undefined = undefined;
@@ -197,6 +205,7 @@ describe('call-orchestrator', () => {
   beforeEach(() => {
     resetTables();
     mockCallModel = undefined;
+    mockUserReference = 'my human';
     // Reset the stream mock to default behaviour
     mockStreamFn.mockImplementation(() => createMockStream(['Hello', ' there']));
   });
@@ -414,6 +423,166 @@ describe('call-orchestrator', () => {
     orchestrator.destroy();
   });
 
+  test('LLM APIUserAbortError: treats as expected abort without technical-issue fallback', async () => {
+    mockStreamFn.mockImplementation(() => {
+      const emitter = new EventEmitter();
+      return {
+        on: (event: string, handler: (...args: unknown[]) => void) => {
+          emitter.on(event, handler);
+          return { on: () => ({ on: () => ({}) }) };
+        },
+        finalMessage: () => {
+          const err = new Error('user abort');
+          err.name = 'APIUserAbortError';
+          return Promise.reject(err);
+        },
+      };
+    });
+
+    const { relay, orchestrator } = setupOrchestrator();
+    await orchestrator.handleCallerUtterance('Hello');
+
+    const errorTokens = relay.sentTokens.filter((t) => t.token.includes('technical issue'));
+    expect(errorTokens.length).toBe(0);
+    expect(orchestrator.getState()).toBe('idle');
+
+    orchestrator.destroy();
+  });
+
+  test('stale superseded turn errors do not emit technical-issue fallback', async () => {
+    let callCount = 0;
+    mockStreamFn.mockImplementation(() => {
+      callCount++;
+      if (callCount === 1) {
+        const emitter = new EventEmitter();
+        return {
+          on: (event: string, handler: (...args: unknown[]) => void) => {
+            emitter.on(event, handler);
+            return { on: () => ({ on: () => ({}) }) };
+          },
+          finalMessage: () =>
+            new Promise((_, reject) => {
+              setTimeout(() => reject(new Error('stale stream failure')), 20);
+            }),
+        };
+      }
+      return createMockStream(['Second turn response.']);
+    });
+
+    const { relay, orchestrator } = setupOrchestrator();
+
+    const firstTurnPromise = orchestrator.handleCallerUtterance('First utterance');
+    // Allow the first turn to enter runLlm before the second utterance interrupts it.
+    await new Promise((r) => setTimeout(r, 5));
+    const secondTurnPromise = orchestrator.handleCallerUtterance('Second utterance');
+
+    await Promise.all([firstTurnPromise, secondTurnPromise]);
+
+    const allTokens = relay.sentTokens.map((t) => t.token).join('');
+    expect(allTokens).toContain('Second turn response.');
+    expect(allTokens).not.toContain('technical issue');
+
+    orchestrator.destroy();
+  });
+
+  test('rapid caller barge-in coalesces contiguous user turns for role alternation', async () => {
+    let callCount = 0;
+    mockStreamFn.mockImplementation((...args: unknown[]) => {
+      callCount++;
+      if (callCount === 1) {
+        const emitter = new EventEmitter();
+        const options = args[1] as { signal?: AbortSignal } | undefined;
+        return {
+          on: (event: string, handler: (...evtArgs: unknown[]) => void) => {
+            emitter.on(event, handler);
+            return { on: () => ({ on: () => ({}) }) };
+          },
+          finalMessage: () =>
+            new Promise((_, reject) => {
+              options?.signal?.addEventListener('abort', () => {
+                const err = new Error('aborted');
+                err.name = 'AbortError';
+                reject(err);
+              }, { once: true });
+            }),
+        };
+      }
+
+      const firstArg = args[0] as { messages: Array<{ role: string; content: string }> };
+      const roles = firstArg.messages.map((m) => m.role);
+      for (let i = 1; i < roles.length; i++) {
+        expect(!(roles[i - 1] === 'user' && roles[i] === 'user')).toBe(true);
+      }
+      const userMessages = firstArg.messages.filter((m) => m.role === 'user');
+      const lastUser = userMessages[userMessages.length - 1];
+      expect(lastUser?.content).toContain('First caller utterance');
+      expect(lastUser?.content).toContain('Second caller utterance');
+      return createMockStream(['Merged turn handled.']);
+    });
+
+    const { relay, orchestrator } = setupOrchestrator();
+    const firstTurnPromise = orchestrator.handleCallerUtterance('First caller utterance');
+    await new Promise((r) => setTimeout(r, 5));
+    const secondTurnPromise = orchestrator.handleCallerUtterance('Second caller utterance');
+
+    await Promise.all([firstTurnPromise, secondTurnPromise]);
+
+    const allTokens = relay.sentTokens.map((t) => t.token).join('');
+    expect(allTokens).toContain('Merged turn handled.');
+
+    orchestrator.destroy();
+  });
+
+  test('interrupt then next caller prompt still preserves role alternation', async () => {
+    let callCount = 0;
+    mockStreamFn.mockImplementation((...args: unknown[]) => {
+      callCount++;
+      if (callCount === 1) {
+        const emitter = new EventEmitter();
+        const options = args[1] as { signal?: AbortSignal } | undefined;
+        return {
+          on: (event: string, handler: (...evtArgs: unknown[]) => void) => {
+            emitter.on(event, handler);
+            return { on: () => ({ on: () => ({}) }) };
+          },
+          finalMessage: () =>
+            new Promise((_, reject) => {
+              options?.signal?.addEventListener('abort', () => {
+                const err = new Error('aborted');
+                err.name = 'AbortError';
+                reject(err);
+              }, { once: true });
+            }),
+        };
+      }
+
+      const firstArg = args[0] as { messages: Array<{ role: string; content: string }> };
+      const roles = firstArg.messages.map((m) => m.role);
+      for (let i = 1; i < roles.length; i++) {
+        expect(!(roles[i - 1] === 'user' && roles[i] === 'user')).toBe(true);
+      }
+      const userMessages = firstArg.messages.filter((m) => m.role === 'user');
+      const lastUser = userMessages[userMessages.length - 1];
+      expect(lastUser?.content).toContain('First caller utterance');
+      expect(lastUser?.content).toContain('Second caller utterance');
+      return createMockStream(['Post-interrupt response.']);
+    });
+
+    const { relay, orchestrator } = setupOrchestrator();
+    const firstTurnPromise = orchestrator.handleCallerUtterance('First caller utterance');
+    await new Promise((r) => setTimeout(r, 5));
+    orchestrator.handleInterrupt();
+    const secondTurnPromise = orchestrator.handleCallerUtterance('Second caller utterance');
+
+    await Promise.all([firstTurnPromise, secondTurnPromise]);
+
+    const allTokens = relay.sentTokens.map((t) => t.token).join('');
+    expect(allTokens).toContain('Post-interrupt response.');
+    expect(allTokens).not.toContain('technical issue');
+
+    orchestrator.destroy();
+  });
+
   test('handleUserAnswer: returns false when not in waiting_on_user state', async () => {
     const { orchestrator } = setupOrchestrator();
 
@@ -435,6 +604,87 @@ describe('call-orchestrator', () => {
     orchestrator.destroy();
   });
 
+  test('handleInterrupt: increments llmRunVersion to suppress stale turn side effects', async () => {
+    // Use a stream whose finalMessage resolves immediately but whose
+    // continuation (the code after `await stream.finalMessage()`) will
+    // run asynchronously. This simulates the race where the promise
+    // microtask is queued right as handleInterrupt fires.
+    mockStreamFn.mockImplementation(() => {
+      const emitter = new EventEmitter();
+      return {
+        on: (event: string, handler: (...args: unknown[]) => void) => {
+          emitter.on(event, handler);
+          return { on: () => ({ on: () => ({}) }) };
+        },
+        finalMessage: () => {
+          // Emit some tokens synchronously
+          emitter.emit('text', 'Stale response that should be suppressed.');
+          return Promise.resolve({
+            content: [{ type: 'text', text: 'Stale response that should be suppressed.' }],
+          });
+        },
+      };
+    });
+
+    const { relay, orchestrator } = setupOrchestrator();
+
+    // Start an LLM turn (don't await — we want to interrupt mid-flight)
+    const turnPromise = orchestrator.handleCallerUtterance('Hello');
+
+    // Interrupt immediately. Because finalMessage resolves as a microtask,
+    // its continuation hasn't run yet. handleInterrupt increments
+    // llmRunVersion so the continuation's isCurrentRun check will fail.
+    orchestrator.handleInterrupt();
+
+    // Let the stale turn's microtask continuation execute
+    await turnPromise;
+
+    // The orchestrator should remain idle — the stale turn must not
+    // have pushed state to waiting_on_user or any other post-turn state.
+    expect(orchestrator.getState()).toBe('idle');
+
+    // No technical-issue fallback should have been sent
+    const errorTokens = relay.sentTokens.filter((t) => t.token.includes('technical issue'));
+    expect(errorTokens.length).toBe(0);
+
+    // endSession should NOT have been called by the stale turn
+    expect(relay.endCalled).toBe(false);
+
+    orchestrator.destroy();
+  });
+
+  test('handleInterrupt: sends turn terminator when interrupting active speech', async () => {
+    mockStreamFn.mockImplementation((...args: unknown[]) => {
+      const emitter = new EventEmitter();
+      const options = args[1] as { signal?: AbortSignal } | undefined;
+      return {
+        on: (event: string, handler: (...evtArgs: unknown[]) => void) => {
+          emitter.on(event, handler);
+          return { on: () => ({ on: () => ({}) }) };
+        },
+        finalMessage: () =>
+          new Promise((_, reject) => {
+            options?.signal?.addEventListener('abort', () => {
+              const err = new Error('aborted');
+              err.name = 'AbortError';
+              reject(err);
+            }, { once: true });
+          }),
+      };
+    });
+
+    const { relay, orchestrator } = setupOrchestrator();
+    const turnPromise = orchestrator.handleCallerUtterance('Start speaking');
+    await new Promise((r) => setTimeout(r, 5));
+    orchestrator.handleInterrupt();
+    await turnPromise;
+
+    const endTurnMarkers = relay.sentTokens.filter((t) => t.token === '' && t.last === true);
+    expect(endTurnMarkers.length).toBeGreaterThan(0);
+
+    orchestrator.destroy();
+  });
+
   // ── destroy ───────────────────────────────────────────────────────
 
   test('destroy: unregisters orchestrator', () => {
@@ -622,4 +872,75 @@ describe('call-orchestrator', () => {
 
     orchestrator.destroy();
   });
+
+  // ── System prompt: identity phrasing ────────────────────────────────
+
+  test('system prompt contains resolved user reference (default)', async () => {
+    mockStreamFn.mockImplementation((...args: unknown[]) => {
+      const firstArg = args[0] as { system: string };
+      expect(firstArg.system).toContain('on behalf of my human');
+      return createMockStream(['Hello.']);
+    });
+
+    const { orchestrator } = setupOrchestrator();
+    await orchestrator.handleCallerUtterance('Hi');
+    orchestrator.destroy();
+  });
+
+  test('system prompt contains resolved user reference when set to a name', async () => {
+    mockUserReference = 'John';
+    mockStreamFn.mockImplementation((...args: unknown[]) => {
+      const firstArg = args[0] as { system: string };
+      expect(firstArg.system).toContain('on behalf of John');
+      return createMockStream(['Hello John\'s contact.']);
+    });
+
+    const { orchestrator } = setupOrchestrator();
+    await orchestrator.handleCallerUtterance('Hi');
+    orchestrator.destroy();
+  });
+
+  test('system prompt does not hardcode "your user" in the opening line', async () => {
+    mockUserReference = 'Alice';
+    mockStreamFn.mockImplementation((...args: unknown[]) => {
+      const firstArg = args[0] as { system: string };
+      expect(firstArg.system).not.toContain('on behalf of your user');
+      expect(firstArg.system).toContain('on behalf of Alice');
+      return createMockStream(['Hi there.']);
+    });
+
+    const { orchestrator } = setupOrchestrator();
+    await orchestrator.handleCallerUtterance('Hello');
+    orchestrator.destroy();
+  });
+
+  test('system prompt includes assistant identity bias rule', async () => {
+    mockStreamFn.mockImplementation((...args: unknown[]) => {
+      const firstArg = args[0] as { system: string };
+      expect(firstArg.system).toContain('refer to yourself as an assistant');
+      expect(firstArg.system).toContain('Avoid the phrase "AI assistant" unless directly asked');
+      return createMockStream(['Sure thing.']);
+    });
+
+    const { orchestrator } = setupOrchestrator();
+    await orchestrator.handleCallerUtterance('Hi');
+    orchestrator.destroy();
+  });
+
+  test('assistant identity rule appears before disclosure rule in prompt', async () => {
+    mockStreamFn.mockImplementation((...args: unknown[]) => {
+      const firstArg = args[0] as { system: string };
+      const prompt = firstArg.system;
+      const identityIdx = prompt.indexOf('refer to yourself as an assistant');
+      const disclosureIdx = prompt.indexOf('Be concise');
+      expect(identityIdx).toBeGreaterThan(-1);
+      expect(disclosureIdx).toBeGreaterThan(-1);
+      expect(identityIdx).toBeLessThan(disclosureIdx);
+      return createMockStream(['OK.']);
+    });
+
+    const { orchestrator } = setupOrchestrator();
+    await orchestrator.handleCallerUtterance('Test');
+    orchestrator.destroy();
+  });
 });