kimaki 0.4.76 → 0.4.78

package/src/queue-advanced-e2e-setup.ts

@@ -330,6 +330,271 @@ export function createDeterministicMatchers(): DeterministicMatcher[] {
     },
   }
 
+  // Model responds with text + tool call, then after tool result the
+  // follow-up matcher responds with text. This creates two assistant messages:
+  // first with finish="tool-calls" + completed, second with finish="stop".
+  // Reproduces the bug where the first message gets no footer even though
+  // it completed normally (isAssistantMessageNaturalCompletion rejects
+  // finish="tool-calls").
+  const toolCallFooterMatcher: DeterministicMatcher = {
+    id: 'tool-call-footer',
+    priority: 108,
+    when: {
+      lastMessageRole: 'user',
+      latestUserTextIncludes: 'TOOL_CALL_FOOTER_MARKER',
+    },
+    then: {
+      parts: [
+        { type: 'stream-start', warnings: [] },
+        { type: 'text-start', id: 'tool-call-footer-text' },
+        { type: 'text-delta', id: 'tool-call-footer-text', delta: 'running tool' },
+        { type: 'text-end', id: 'tool-call-footer-text' },
+        {
+          type: 'tool-call',
+          toolCallId: 'tool-call-footer-bash',
+          toolName: 'bash',
+          input: JSON.stringify({
+            command: 'echo tool-call-footer-test',
+            description: 'Echo for footer test',
+          }),
+        },
+        {
+          type: 'finish',
+          finishReason: 'tool-calls',
+          usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
+        },
+      ],
+    },
+  }
+
+  const toolCallFooterFollowupMatcher: DeterministicMatcher = {
+    id: 'tool-call-footer-followup',
+    priority: 109,
+    when: {
+      lastMessageRole: 'tool',
+      latestUserTextIncludes: 'TOOL_CALL_FOOTER_MARKER',
+    },
+    then: {
+      parts: [
+        { type: 'stream-start', warnings: [] },
+        { type: 'text-start', id: 'tool-call-footer-followup' },
+        { type: 'text-delta', id: 'tool-call-footer-followup', delta: 'tool call completed' },
+        { type: 'text-end', id: 'tool-call-footer-followup' },
+        {
+          type: 'finish',
+          finishReason: 'stop',
+          usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
+        },
+      ],
+    },
+  }
+
+  // Multi-step tool chain: model emits text + 3 parallel tool calls in one
+  // response (finish="tool-calls"). All tools complete, then the follow-up
+  // matcher responds with final text (finish="stop"). This creates 2 assistant
+  // messages — one with finish="tool-calls" + completed, one with finish="stop".
+  // With the naive fix (allowing tool-calls as natural completion), we'd get
+  // 2 footers. Only the final text response should get a footer.
+  const multiToolMatcher: DeterministicMatcher = {
+    id: 'multi-tool',
+    priority: 115,
+    when: {
+      lastMessageRole: 'user',
+      latestUserTextIncludes: 'MULTI_TOOL_FOOTER_MARKER',
+    },
+    then: {
+      parts: [
+        { type: 'stream-start', warnings: [] },
+        { type: 'text-start', id: 'multi-tool-text' },
+        { type: 'text-delta', id: 'multi-tool-text', delta: 'investigating the issue' },
+        { type: 'text-end', id: 'multi-tool-text' },
+        {
+          type: 'tool-call',
+          toolCallId: 'multi-tool-bash-1',
+          toolName: 'bash',
+          input: JSON.stringify({
+            command: 'echo search-done',
+            description: 'Search codebase',
+          }),
+        },
+        {
+          type: 'tool-call',
+          toolCallId: 'multi-tool-bash-2',
+          toolName: 'bash',
+          input: JSON.stringify({
+            command: 'echo read-done',
+            description: 'Read config file',
+          }),
+        },
+        {
+          type: 'tool-call',
+          toolCallId: 'multi-tool-bash-3',
+          toolName: 'bash',
+          input: JSON.stringify({
+            command: 'echo fix-done',
+            description: 'Apply fix',
+          }),
+        },
+        {
+          type: 'finish',
+          finishReason: 'tool-calls',
+          usage: { inputTokens: 10, outputTokens: 15, totalTokens: 25 },
+        },
+      ],
+    },
+  }
+
+  const multiToolFollowupMatcher: DeterministicMatcher = {
+    id: 'multi-tool-followup',
+    priority: 114,
+    when: {
+      latestUserTextIncludes: 'MULTI_TOOL_FOOTER_MARKER',
+      rawPromptIncludes: 'investigating the issue',
+    },
+    then: {
+      parts: [
+        { type: 'stream-start', warnings: [] },
+        { type: 'text-start', id: 'multi-tool-followup-text' },
+        { type: 'text-delta', id: 'multi-tool-followup-text', delta: 'all done, fixed 3 files' },
+        { type: 'text-end', id: 'multi-tool-followup-text' },
+        {
+          type: 'finish',
+          finishReason: 'stop',
+          usage: { inputTokens: 30, outputTokens: 10, totalTokens: 40 },
+        },
+      ],
+    },
+  }
+
+  // Multi-step sequential tool chain: 3 separate tool-call steps (each a
+  // separate assistant message with finish="tool-calls"), then a final text
+  // response. This creates 4 assistant messages total. Without proper
+  // deferred footer logic, each tool-call step would emit its own footer,
+  // producing 3 spurious footers before the real one.
+  //
+  // Flow: user → step1 (text + tool-call) → tool result →
+  //       step2 (text + tool-call) → tool result →
+  //       step3 (text + tool-call) → tool result →
+  //       final text (finish="stop")
+  //
+  // Matcher priority ensures each step fires in order: the highest-priority
+  // matcher that matches wins, and each step's rawPromptIncludes check only
+  // matches once the previous step's output text is in the conversation.
+  const multiStepChainInitMatcher: DeterministicMatcher = {
+    id: 'multi-step-chain-init',
+    priority: 119,
+    when: {
+      lastMessageRole: 'user',
+      latestUserTextIncludes: 'MULTI_STEP_CHAIN_MARKER',
+    },
+    then: {
+      parts: [
+        { type: 'stream-start', warnings: [] },
+        { type: 'text-start', id: 'chain-step1-text' },
+        { type: 'text-delta', id: 'chain-step1-text', delta: 'chain step 1: reading config' },
+        { type: 'text-end', id: 'chain-step1-text' },
+        {
+          type: 'tool-call',
+          toolCallId: 'chain-step1-bash',
+          toolName: 'bash',
+          input: JSON.stringify({
+            command: 'echo chain-step-1-output',
+            description: 'Read config',
+          }),
+        },
+        {
+          type: 'finish',
+          finishReason: 'tool-calls',
+          usage: { inputTokens: 5, outputTokens: 10, totalTokens: 15 },
+        },
+      ],
+    },
+  }
+
+  const multiStepChainStep2Matcher: DeterministicMatcher = {
+    id: 'multi-step-chain-step2',
+    priority: 120,
+    when: {
+      latestUserTextIncludes: 'MULTI_STEP_CHAIN_MARKER',
+      rawPromptIncludes: 'chain step 1: reading config',
+    },
+    then: {
+      parts: [
+        { type: 'stream-start', warnings: [] },
+        { type: 'text-start', id: 'chain-step2-text' },
+        { type: 'text-delta', id: 'chain-step2-text', delta: 'chain step 2: analyzing results' },
+        { type: 'text-end', id: 'chain-step2-text' },
+        {
+          type: 'tool-call',
+          toolCallId: 'chain-step2-bash',
+          toolName: 'bash',
+          input: JSON.stringify({
+            command: 'echo chain-step-2-output',
+            description: 'Analyze results',
+          }),
+        },
+        {
+          type: 'finish',
+          finishReason: 'tool-calls',
+          usage: { inputTokens: 15, outputTokens: 10, totalTokens: 25 },
+        },
+      ],
+    },
+  }
+
+  const multiStepChainStep3Matcher: DeterministicMatcher = {
+    id: 'multi-step-chain-step3',
+    priority: 121,
+    when: {
+      latestUserTextIncludes: 'MULTI_STEP_CHAIN_MARKER',
+      rawPromptIncludes: 'chain step 2: analyzing results',
+    },
+    then: {
+      parts: [
+        { type: 'stream-start', warnings: [] },
+        { type: 'text-start', id: 'chain-step3-text' },
+        { type: 'text-delta', id: 'chain-step3-text', delta: 'chain step 3: applying fix' },
+        { type: 'text-end', id: 'chain-step3-text' },
+        {
+          type: 'tool-call',
+          toolCallId: 'chain-step3-bash',
+          toolName: 'bash',
+          input: JSON.stringify({
+            command: 'echo chain-step-3-output',
+            description: 'Apply fix',
+          }),
+        },
+        {
+          type: 'finish',
+          finishReason: 'tool-calls',
+          usage: { inputTokens: 25, outputTokens: 10, totalTokens: 35 },
+        },
+      ],
+    },
+  }
+
+  const multiStepChainFinalMatcher: DeterministicMatcher = {
+    id: 'multi-step-chain-final',
+    priority: 122,
+    when: {
+      latestUserTextIncludes: 'MULTI_STEP_CHAIN_MARKER',
+      rawPromptIncludes: 'chain step 3: applying fix',
+    },
+    then: {
+      parts: [
+        { type: 'stream-start', warnings: [] },
+        { type: 'text-start', id: 'chain-final-text' },
+        { type: 'text-delta', id: 'chain-final-text', delta: 'chain complete: all 3 steps done' },
+        { type: 'text-end', id: 'chain-final-text' },
+        {
+          type: 'finish',
+          finishReason: 'stop',
+          usage: { inputTokens: 35, outputTokens: 5, totalTokens: 40 },
+        },
+      ],
+    },
+  }
+
   return [
     slowAbortMatcher,
     typingRepulseMatcher,
@@ -338,7 +603,15 @@ export function createDeterministicMatchers(): DeterministicMatcher[] {
     questionToolMatcher,
     permissionTypingMatcher,
     permissionTypingFollowupMatcher,
+    multiToolMatcher,
+    multiToolFollowupMatcher,
+    multiStepChainInitMatcher,
+    multiStepChainStep2Matcher,
+    multiStepChainStep3Matcher,
+    multiStepChainFinalMatcher,
     raceFinalReplyMatcher,
+    toolCallFooterMatcher,
+    toolCallFooterFollowupMatcher,
     toolFollowupMatcher,
     userReplyMatcher,
   ]

package/src/queue-advanced-footer.e2e.test.ts

@@ -362,4 +362,215 @@ e2eTest('queue advanced: footer emission', () => {
     },
     15_000,
   )
+
+  test(
+    'tool-call assistant message gets footer when it completes normally',
+    async () => {
+      // Reproduces the bug: model responds with text + tool call,
+      // finish="tool-calls", message gets completed timestamp. Then the tool
+      // result triggers a follow-up text response in a second assistant message.
+      // The second message gets a footer, but the first (tool-call) message
+      // should ALSO get a footer since it completed normally.
+      // This matches the real-world scenario where an agent calls a bash tool
+      // (e.g. `kimaki send`) and then follows up with a summary text.
+      await ctx.discord.channel(TEXT_CHANNEL_ID).user(TEST_USER_ID).sendMessage({
+        content: 'TOOL_CALL_FOOTER_MARKER',
+      })
+
+      const thread = await ctx.discord.channel(TEXT_CHANNEL_ID).waitForThread({
+        timeout: 4_000,
+        predicate: (t) => {
+          return t.name === 'TOOL_CALL_FOOTER_MARKER'
+        },
+      })
+
+      const th = ctx.discord.thread(thread.id)
+
+      // Wait for the follow-up text response after tool completion.
+      // The tool call completes and the model follows up with a second
+      // assistant message containing text.
+      await waitForBotReplyAfterUserMessage({
+        discord: ctx.discord,
+        threadId: thread.id,
+        userId: TEST_USER_ID,
+        userMessageIncludes: 'TOOL_CALL_FOOTER_MARKER',
+        timeout: 4_000,
+      })
+
+      // Wait for at least one footer to appear
+      await waitForFooterMessage({
+        discord: ctx.discord,
+        threadId: thread.id,
+        timeout: 4_000,
+      })
+
+      // Poll until both footers have arrived — the first footer (after the
+      // tool-call step) and the second (after the text follow-up) are emitted
+      // by sequential handleNaturalAssistantCompletion calls but the second
+      // may not have hit the Discord thread by the time we first check.
+      const deadline = Date.now() + 4_000
+      let footerCount = 0
+      while (Date.now() < deadline) {
+        const msgs = await th.getMessages()
+        footerCount = msgs.filter((m) => {
+          return m.author.id === ctx.discord.botUserId
+            && m.content.startsWith('*')
+            && m.content.includes('⋅')
+        }).length
+        if (footerCount >= 2) {
+          break
+        }
+        await new Promise((resolve) => {
+          setTimeout(resolve, 100)
+        })
+      }
+
+      expect(await th.text()).toMatchInlineSnapshot(`
+        "--- from: user (queue-advanced-tester)
+        TOOL_CALL_FOOTER_MARKER
+        --- from: assistant (TestBot)
+        ⬥ running tool
+        ⬥ ok
+        *project ⋅ main ⋅ Ns ⋅ N% ⋅ deterministic-v2*"
+      `)
+
+      // Only ONE footer at the end — the tool-call step's footer is NOT
+      // emitted mid-turn. The final text follow-up gets the footer.
+      expect(footerCount).toBe(1)
+    },
+    10_000,
+  )
+
+  test(
+    'multi-step tool chain should only have one footer at the end',
+    async () => {
+      // Model does 3 sequential tool calls (each a separate assistant message
+      // with finish="tool-calls") then a final text response. Only the final
+      // text response should get a footer — intermediate tool-call steps
+      // should NOT get footers since they're mid-turn work.
+      await ctx.discord.channel(TEXT_CHANNEL_ID).user(TEST_USER_ID).sendMessage({
+        content: 'MULTI_TOOL_FOOTER_MARKER',
+      })
+
+      const thread = await ctx.discord.channel(TEXT_CHANNEL_ID).waitForThread({
+        timeout: 4_000,
+        predicate: (t) => {
+          return t.name === 'MULTI_TOOL_FOOTER_MARKER'
+        },
+      })
+
+      const th = ctx.discord.thread(thread.id)
+
+      // Wait for the final text response after all 3 tool steps
+      await waitForBotMessageContaining({
+        discord: ctx.discord,
+        threadId: thread.id,
+        userId: TEST_USER_ID,
+        text: 'all done, fixed 3 files',
+        timeout: 4_000,
+      })
+
+      // Wait for the footer after the final response
+      await waitForFooterMessage({
+        discord: ctx.discord,
+        threadId: thread.id,
+        timeout: 4_000,
+      })
+
+      // Give any spurious extra footers time to arrive
+      await new Promise((resolve) => {
+        setTimeout(resolve, 500)
+      })
+
+      const messages = await th.getMessages()
+      const footerCount = messages.filter((m) => {
+        return m.author.id === ctx.discord.botUserId
+          && m.content.startsWith('*')
+          && m.content.includes('⋅')
+      }).length
+
+      expect(await th.text()).toMatchInlineSnapshot(`
+        "--- from: user (queue-advanced-tester)
+        MULTI_TOOL_FOOTER_MARKER
+        --- from: assistant (TestBot)
+        ⬥ investigating the issue
+        ⬥ all done, fixed 3 files
+        *project ⋅ main ⋅ Ns ⋅ N% ⋅ deterministic-v2*"
+      `)
+
+      // Only ONE footer should appear — after the final text response.
+      // Intermediate tool-call steps should NOT get footers.
+      expect(footerCount).toBe(1)
+    },
+    10_000,
+  )
+
+  test(
+    '3 sequential tool-call steps produce exactly 1 footer, not 3',
+    async () => {
+      // This is the most obvious reproduction of the multi-footer bug:
+      // the model runs 3 sequential tool-call steps (each a SEPARATE
+      // assistant message with finish="tool-calls"), then a final text.
+      // With a naive fix that treats tool-calls as natural completions,
+      // you'd see 4 footers (one per assistant message). Only the final
+      // text response should produce a footer.
+      await ctx.discord.channel(TEXT_CHANNEL_ID).user(TEST_USER_ID).sendMessage({
+        content: 'MULTI_STEP_CHAIN_MARKER',
+      })
+
+      const thread = await ctx.discord.channel(TEXT_CHANNEL_ID).waitForThread({
+        timeout: 4_000,
+        predicate: (t) => {
+          return t.name === 'MULTI_STEP_CHAIN_MARKER'
+        },
+      })
+
+      const th = ctx.discord.thread(thread.id)
+
+      // Wait for the final text after all 3 sequential tool steps
+      await waitForBotMessageContaining({
+        discord: ctx.discord,
+        threadId: thread.id,
+        userId: TEST_USER_ID,
+        text: 'chain complete: all 3 steps done',
+        timeout: 8_000,
+      })
+
+      // Wait for footer
+      await waitForFooterMessage({
+        discord: ctx.discord,
+        threadId: thread.id,
+        timeout: 4_000,
+      })
+
+      // Give any spurious extra footers time to arrive
+      await new Promise((resolve) => {
+        setTimeout(resolve, 500)
+      })
+
+      const messages = await th.getMessages()
+      const footerCount = messages.filter((m) => {
+        return m.author.id === ctx.discord.botUserId
+          && m.content.startsWith('*')
+          && m.content.includes('⋅')
+      }).length
+
+      expect(await th.text()).toMatchInlineSnapshot(`
+        "--- from: user (queue-advanced-tester)
+        MULTI_STEP_CHAIN_MARKER
+        --- from: assistant (TestBot)
+        ⬥ chain step 1: reading config
+        ⬥ chain step 2: analyzing results
+        ⬥ chain step 3: applying fix
+        ⬥ chain complete: all 3 steps done
+        *project ⋅ main ⋅ Ns ⋅ N% ⋅ deterministic-v2*"
+      `)
+
+      // The critical assertion: only 1 footer at the very end.
+      // With the naive "allow tool-calls as natural completion" fix,
+      // this would be 4 (one per assistant message). We want 1.
+      expect(footerCount).toBe(1)
+    },
+    15_000,
+  )
 })