@myrialabs/clopen 0.0.1

package/frontend/lib/types/native-ui.ts

@@ -0,0 +1,73 @@
+/**
+ * Frontend Native UI Types
+ * For dialog, select dropdown, and context menu overlays
+ */
+
+// Dialog types
+export interface BrowserDialogEvent {
+	sessionId: string;
+	dialogId: string;
+	type: 'alert' | 'confirm' | 'prompt' | 'beforeunload';
+	message: string;
+	defaultValue?: string;
+	timestamp: number;
+}
+
+export interface BrowserPrintEvent {
+	sessionId: string;
+	timestamp: number;
+}
+
+// Select dropdown types
+export interface BrowserSelectOption {
+	index: number;
+	value: string;
+	text: string;
+	selected: boolean;
+	disabled?: boolean;
+}
+
+export interface BrowserSelectInfo {
+	sessionId: string;
+	selectId: string;
+	x: number;
+	y: number;
+	boundingBox: {
+		x: number;
+		y: number;
+		width: number;
+		height: number;
+	};
+	options: BrowserSelectOption[];
+	selectedIndex: number;
+	timestamp: number;
+}
+
+// Context menu types
+export interface BrowserContextMenuItem {
+	id: string;
+	label: string;
+	enabled: boolean;
+	type?: 'normal' | 'separator' | 'submenu';
+	icon?: string;
+	submenu?: BrowserContextMenuItem[];
+}
+
+export interface BrowserContextMenuInfo {
+	sessionId: string;
+	menuId: string;
+	x: number;
+	y: number;
+	items: BrowserContextMenuItem[];
+	elementInfo: {
+		tagName: string;
+		isLink: boolean;
+		isImage: boolean;
+		isInput: boolean;
+		isTextSelected: boolean;
+		linkUrl?: string;
+		imageUrl?: string;
+		inputType?: string;
+	};
+	timestamp: number;
+}

package/frontend/lib/utils/chat/date-separator.ts

@@ -0,0 +1,39 @@
+import type { SDKMessageFormatter } from '$shared/types/database/schema';
+
+// Interface for date separator items
+export interface DateSeparatorItem {
+  type: 'message' | 'date';
+  data: any;
+  key: string;
+}
+
+// Add date separators to messages
+// startIndex: global offset for stable keys when using windowed slices
+export function addDateSeparators(messages: SDKMessageFormatter[], startIndex: number = 0): DateSeparatorItem[] {
+  const result: DateSeparatorItem[] = [];
+  let lastDate: string | null = null;
+
+  messages.forEach((message, index) => {
+    const createdAt = message.metadata?.created_at || new Date().toISOString();
+    const messageDate = new Date(createdAt).toDateString();
+
+    // Add date separator if date has changed
+    if (messageDate !== lastDate) {
+      result.push({
+        type: 'date',
+        data: createdAt,
+        key: `date-${messageDate}`
+      });
+      lastDate = messageDate;
+    }
+
+    // Use global index for stable keys across window shifts
+    result.push({
+      type: 'message',
+      data: message,
+      key: `message-${startIndex + index}`
+    });
+  });
+
+  return result;
+}

package/frontend/lib/utils/chat/message-grouper.ts

@@ -0,0 +1,219 @@
+import type { SDKMessageFormatter } from '$shared/types/database/schema';
+import {
+  shouldFilterMessage,
+  extractToolUses,
+  extractToolResults
+} from './message-processor';
+import { processToolMessage } from './tool-handler';
+
+// Tool group for mapping tool_use with tool_result
+export interface ToolGroup {
+  toolUseMessage: SDKMessageFormatter;
+  toolResultMessage: SDKMessageFormatter | null;
+}
+
+// Background bash session data
+export interface BackgroundBashData {
+  bashToolId: string;
+  bashOutputs: any[];
+}
+
+// Processed message type
+export type ProcessedMessage = SDKMessageFormatter;
+
+// Group tool_use and tool_result messages together
+export function groupMessages(messages: SDKMessageFormatter[]): {
+  groups: ProcessedMessage[],
+  toolUseMap: Map<string, ToolGroup>
+} {
+  const groups: ProcessedMessage[] = [];
+  const toolUseMap = new Map<string, ToolGroup>();
+
+  messages.forEach(message => {
+    // Skip messages that should be filtered
+    if (shouldFilterMessage(message)) {
+      return;
+    }
+
+    // Handle assistant messages with tool_use
+    if (message.type === 'assistant' && 'message' in message && message.message?.content) {
+      const toolUses = extractToolUses(message.message.content);
+
+      if (toolUses.length > 0) {
+        // Store tool_use messages for grouping
+        toolUses.forEach((toolUse: any) => {
+          if (toolUse.id) {
+            toolUseMap.set(toolUse.id, {
+              toolUseMessage: message,
+              toolResultMessage: null
+            });
+          }
+        });
+        groups.push(message as ProcessedMessage);
+      } else {
+        groups.push(message as ProcessedMessage);
+      }
+    }
+    // Handle user messages with tool_result
+    else if (message.type === 'user' && 'message' in message && message.message?.content) {
+      const toolResults = extractToolResults(message.message.content);
+
+      if (toolResults.length > 0) {
+        // Group tool_result with corresponding tool_use
+        toolResults.forEach((toolResult: any) => {
+          if (toolResult.tool_use_id && toolUseMap.has(toolResult.tool_use_id)) {
+            const group = toolUseMap.get(toolResult.tool_use_id);
+            if (group) {
+              group.toolResultMessage = message;
+            }
+          }
+        });
+        // Don't add tool_result messages separately
+      } else {
+        // Regular user message
+        groups.push(message as ProcessedMessage);
+      }
+    }
+    // Include stream_event and other messages
+    else {
+      groups.push(message as ProcessedMessage);
+    }
+  });
+
+  return { groups, toolUseMap };
+}
+
+// Add tool results to messages
+export function embedToolResults(
+  groups: ProcessedMessage[],
+  toolUseMap: Map<string, ToolGroup>
+): ProcessedMessage[] {
+  // Track background bash sessions
+  const backgroundBashMap = trackBackgroundBashSessions(groups, toolUseMap);
+
+  // Create combined messages with tool_use including $result property
+  return groups.map(message => {
+    if (message.type === 'assistant' && 'message' in message && message.message?.content) {
+      const toolUses = extractToolUses(message.message.content);
+
+      if (toolUses.length > 0) {
+        const processedMessage = processToolMessage(
+          message,
+          toolUseMap,
+          backgroundBashMap
+        );
+        return processedMessage;
+      }
+    }
+
+    return message;
+  });
+}
+
+// Track background bash sessions and their outputs
+function trackBackgroundBashSessions(
+  groups: ProcessedMessage[],
+  toolUseMap: Map<string, ToolGroup>
+): Map<string, BackgroundBashData> {
+  const backgroundBashMap = new Map<string, BackgroundBashData>();
+
+  groups.forEach(message => {
+    if (message.type === 'assistant' && 'message' in message && message.message?.content) {
+      const contentArray = Array.isArray(message.message.content)
+        ? message.message.content
+        : [message.message.content];
+
+      contentArray.forEach((item) => {
+        if (typeof item === 'object' && item && 'type' in item && item.type === 'tool_use') {
+          const toolUse = item as any;
+          // Check for Bash with run_in_background
+          if (toolUse.name === 'Bash' && toolUse.input &&
+              typeof toolUse.input === 'object' &&
+              'run_in_background' in toolUse.input &&
+              toolUse.input.run_in_background) {
+            trackBackgroundBash(toolUse, toolUseMap, backgroundBashMap);
+          }
+          // Collect all BashOutput results
+          else if (toolUse.name === 'BashOutput' && toolUse.input &&
+                   typeof toolUse.input === 'object' &&
+                   'bash_id' in toolUse.input) {
+            trackBashOutput(toolUse, toolUseMap, backgroundBashMap);
+          }
+        }
+      });
+    }
+  });
+
+  return backgroundBashMap;
+}
+
+function trackBackgroundBash(
+  item: any,
+  toolUseMap: Map<string, ToolGroup>,
+  backgroundBashMap: Map<string, BackgroundBashData>
+): void {
+  const toolId = item.id;
+  if (!toolId || !toolUseMap.has(toolId)) return;
+
+  const group = toolUseMap.get(toolId);
+  if (!group?.toolResultMessage) return;
+
+  const resultMessage = group.toolResultMessage as any;
+  const resultContent = resultMessage.message ?
+    (Array.isArray(resultMessage.message.content) ? resultMessage.message.content : [resultMessage.message.content]) : [];
+
+  const toolResult = resultContent.find((resultItem: any) =>
+    typeof resultItem === 'object' &&
+    resultItem &&
+    'type' in resultItem &&
+    resultItem.type === 'tool_result' &&
+    'tool_use_id' in resultItem &&
+    resultItem.tool_use_id === toolId
+  ) as any | undefined;
+
+  if (toolResult?.content && typeof toolResult.content === 'string') {
+    // Extract bash ID from "Command running in background with ID: xxxxx"
+    const idMatch = toolResult.content.match(/Command running in background with ID:\s*(\w+)/);
+    if (idMatch) {
+      const bashId = idMatch[1];
+      backgroundBashMap.set(bashId, {
+        bashToolId: toolId,
+        bashOutputs: []
+      });
+    }
+  }
+}
+
+function trackBashOutput(
+  item: any,
+  toolUseMap: Map<string, ToolGroup>,
+  backgroundBashMap: Map<string, BackgroundBashData>
+): void {
+  const bashId = (item.input as any).bash_id as string;
+  const toolId = item.id;
+
+  if (!toolId || !toolUseMap.has(toolId)) return;
+
+  const group = toolUseMap.get(toolId);
+  if (!group?.toolResultMessage) return;
+
+  const resultMessage = group.toolResultMessage as any;
+  const resultContent = resultMessage.message ?
+    (Array.isArray(resultMessage.message.content) ? resultMessage.message.content : [resultMessage.message.content]) : [];
+
+  const toolResult = resultContent.find((resultItem: any) =>
+    typeof resultItem === 'object' &&
+    resultItem &&
+    'type' in resultItem &&
+    resultItem.type === 'tool_result' &&
+    'tool_use_id' in resultItem &&
+    resultItem.tool_use_id === toolId
+  ) as any | undefined;
+
+  if (toolResult && backgroundBashMap.has(bashId)) {
+    const bashData = backgroundBashMap.get(bashId);
+    if (bashData) {
+      bashData.bashOutputs.push(toolResult);
+    }
+  }
+}

package/frontend/lib/utils/chat/message-processor.ts

@@ -0,0 +1,135 @@
+import type { SDKMessageFormatter } from '$shared/types/database/schema';
+import type { ToolInput } from '$shared/types/messaging';
+
+// Content that can be either single or array
+export type MessageContentArray = any[] | any | string;
+
+// Helper type guards
+export function isToolUseBlock(item: unknown): boolean {
+  return (
+    typeof item === 'object' &&
+    item !== null &&
+    'type' in item &&
+    (item as any).type === 'tool_use'
+  );
+}
+
+export function isToolResultBlock(item: unknown): boolean {
+  return (
+    typeof item === 'object' &&
+    item !== null &&
+    'type' in item &&
+    (item as any).type === 'tool_result'
+  );
+}
+
+// Ensure content is always an array
+export function normalizeContent(content: MessageContentArray): any[] {
+  if (Array.isArray(content)) {
+    return content;
+  }
+  if (typeof content === 'string') {
+    return [{ type: 'text', text: content }];
+  }
+  return [content];
+}
+
+// List of tool names that should include tool_result in content
+export const TOOLS_WITH_RESULTS: ToolInput['name'][] = [
+  'Bash',
+  'TaskOutput',
+  'Edit',
+  'ExitPlanMode',
+  'Glob',
+  'Grep',
+  'KillShell',
+  'ListMcpResources',
+  'NotebookEdit',
+  'ReadMcpResource',
+  'Read',
+  'Task',
+  'TodoWrite',
+  'WebFetch',
+  'WebSearch',
+  'Write'
+];
+
+// Tools that should be hidden from display
+export const HIDDEN_TOOLS: ToolInput['name'][] = [
+  'TaskOutput',
+  'TodoWrite'
+];
+
+// Check if a message should be filtered out
+export function shouldFilterMessage(message: SDKMessageFormatter): boolean {
+  // Skip system and result type messages
+  if (message.type === 'system' || message.type === 'result') {
+    return true;
+  }
+
+  // Filter out stream_event messages with no partial text
+  if (message.type === 'stream_event') {
+    if (!('partialText' in message) || !message.partialText) {
+      return true;
+    }
+  }
+
+  // Filter out assistant messages with no visible content
+  if (message.type === 'assistant') {
+    // No message property at all
+    if (!('message' in message) || !message.message) {
+      return true;
+    }
+
+    const content = message.message.content;
+
+    // No content or falsy content
+    if (!content) {
+      return true;
+    }
+
+    if (Array.isArray(content)) {
+      // Empty content array
+      if (content.length === 0) {
+        return true;
+      }
+      // Check if there's any meaningful visible content
+      const hasNonEmptyText = content.some((item: any) =>
+        item.type === 'text' && item.text && item.text.trim().length > 0
+      );
+      const hasToolUse = content.some((item: any) => item.type === 'tool_use');
+      // Filter if no meaningful text and no tool_use blocks
+      if (!hasNonEmptyText && !hasToolUse) {
+        return true;
+      }
+    }
+  }
+
+  return false;
+}
+
+// Extract tool uses from message content
+export function extractToolUses(content: MessageContentArray): any[] {
+  const contentArray = normalizeContent(content);
+  return contentArray.filter(isToolUseBlock);
+}
+
+// Extract tool results from message content
+export function extractToolResults(content: MessageContentArray): any[] {
+  const contentArray = normalizeContent(content);
+  return contentArray.filter(isToolResultBlock);
+}
+
+// Check if a tool should have its result embedded
+export function shouldEmbedResult(toolName: string): boolean {
+  // Custom MCP tools always embed results (they start with 'mcp__')
+  if (toolName.startsWith('mcp__')) {
+    return true;
+  }
+  return TOOLS_WITH_RESULTS.includes(toolName as ToolInput['name']);
+}
+
+// Check if a tool should be hidden
+export function shouldHideTool(toolName: string): boolean {
+  return HIDDEN_TOOLS.includes(toolName as ToolInput['name']);
+}

package/frontend/lib/utils/chat/tool-handler.ts

@@ -0,0 +1,161 @@
+import {
+  shouldEmbedResult,
+  shouldHideTool
+} from './message-processor';
+import type {
+  ProcessedMessage,
+  ToolGroup,
+  BackgroundBashData
+} from './message-grouper';
+
+// Extended ToolUse with embedded result
+export interface ToolUseWithResult {
+  type: 'tool_use';
+  id: string;
+  name: string;
+  input: any;
+  $result?: any;
+}
+
+// Process a tool message with embedded results
+export function processToolMessage(
+  message: ProcessedMessage,
+  toolUseMap: Map<string, ToolGroup>,
+  backgroundBashMap: Map<string, BackgroundBashData>
+): ProcessedMessage {
+  const messageAny = message as any;
+  const content = messageAny.message?.content ?
+    (Array.isArray(messageAny.message.content) ? messageAny.message.content : [messageAny.message.content]) : [];
+
+  // Create modified content with embedded tool_result in tool_use objects
+  const modifiedContent = content
+    .map((item: any): any => {
+      if (typeof item === 'object' && item && 'type' in item && item.type === 'tool_use') {
+        return processToolUse(item, toolUseMap, backgroundBashMap);
+      }
+      return item;
+    })
+    .filter((item: any) => item !== null); // Remove null items (hidden tools)
+
+  // Return modified message with embedded tool_results
+  return {
+    ...message,
+    message: {
+      ...messageAny.message,
+      content: modifiedContent
+    }
+  } as ProcessedMessage;
+}
+
+// Process individual tool_use item
+function processToolUse(
+  item: any,
+  toolUseMap: Map<string, ToolGroup>,
+  backgroundBashMap: Map<string, BackgroundBashData>
+): ToolUseWithResult | null {
+  // Hide certain tools completely
+  if (shouldHideTool(item.name)) {
+    return null;
+  }
+
+  // Special handling for Bash with run_in_background
+  if (item.name === 'Bash' && item.input &&
+      typeof item.input === 'object' &&
+      'run_in_background' in item.input &&
+      item.input.run_in_background && item.id) {
+    return handleBackgroundBash(item, toolUseMap, backgroundBashMap);
+  }
+
+  // Regular tool handling
+  if (item.id && item.name && shouldEmbedResult(item.name) && toolUseMap.has(item.id)) {
+    return handleRegularTool(item, toolUseMap);
+  }
+
+  return item;
+}
+
+// Handle background bash commands
+function handleBackgroundBash(
+  item: any,
+  toolUseMap: Map<string, ToolGroup>,
+  backgroundBashMap: Map<string, BackgroundBashData>
+): ToolUseWithResult {
+  const group = toolUseMap.get(item.id);
+  if (!group?.toolResultMessage) return item;
+
+  const resultMessage = group.toolResultMessage as any;
+  const resultContent = resultMessage.message ?
+    (Array.isArray(resultMessage.message.content) ? resultMessage.message.content : [resultMessage.message.content]) : [];
+
+  const toolResult = findToolResult(resultContent, item.id);
+
+  if (!toolResult?.content || typeof toolResult.content !== 'string') return item;
+
+  // Extract bash ID and check for BashOutput
+  const idMatch = toolResult.content.match(/Command running in background with ID:\s*(\w+)/);
+  if (!idMatch) return item;
+
+  const bashId = idMatch[1];
+  const bashData = backgroundBashMap.get(bashId);
+
+  if (bashData && bashData.bashOutputs.length > 0) {
+    // Use the last BashOutput result
+    const lastOutput = bashData.bashOutputs[bashData.bashOutputs.length - 1];
+    return {
+      ...item,
+      $result: {
+        ...toolResult,
+        content: lastOutput.content || ""
+      }
+    } as ToolUseWithResult;
+  } else {
+    // No BashOutput found, clear the content
+    return {
+      ...item,
+      $result: {
+        ...toolResult,
+        content: ""
+      }
+    } as ToolUseWithResult;
+  }
+}
+
+// Handle regular tools
+function handleRegularTool(
+  item: any,
+  toolUseMap: Map<string, ToolGroup>
+): ToolUseWithResult {
+  const group = toolUseMap.get(item.id);
+  if (!group || !group.toolResultMessage) return item;
+
+  const resultMessage = group.toolResultMessage as any;
+  const resultContent = resultMessage.message ?
+    (Array.isArray(resultMessage.message.content) ? resultMessage.message.content : [resultMessage.message.content]) : [];
+
+  const toolResult = findToolResult(resultContent, item.id);
+
+  if (toolResult) {
+    // Embed tool_result as $result property in tool_use object
+    return {
+      ...item,
+      $result: toolResult
+    } as ToolUseWithResult;
+  }
+
+  return item;
+}
+
+// Helper to find tool result by id
+function findToolResult(
+  content: any[],
+  toolUseId: string
+): any {
+  return content.find((resultItem: any) =>
+    typeof resultItem === 'object' &&
+    resultItem !== null &&
+    'type' in resultItem &&
+    resultItem.type === 'tool_result' &&
+    'tool_use_id' in resultItem &&
+    resultItem.tool_use_id === toolUseId
+  );
+}