@runtypelabs/persona 3.20.0 → 3.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2851,3 +2851,524 @@ describe('AgentWidgetClient.resumeFlow', () => {
2851
2851
  });
2852
2852
  });
2853
2853
 
2854
+ // ============================================================================
2855
+ // agent_media Event Handling
2856
+ // ============================================================================
2857
+
2858
+ describe('AgentWidgetClient - agent_media events', () => {
2859
+ const collectMediaMessages = (events: AgentWidgetEvent[]): AgentWidgetMessage[] => {
2860
+ const byId = new Map<string, AgentWidgetMessage>();
2861
+ for (const event of events) {
2862
+ if (event.type === 'message' && event.message.id.startsWith('agent-media-')) {
2863
+ byId.set(event.message.id, event.message);
2864
+ }
2865
+ }
2866
+ return Array.from(byId.values());
2867
+ };
2868
+
2869
+ it('renders a base64 image (AI SDK v6 type:"media") as a synthetic message', async () => {
2870
+ const execId = 'exec_media_image';
2871
+ global.fetch = createAgentStreamFetch([
2872
+ sseEvent('agent_start', {
2873
+ executionId: execId, agentId: 'virtual', agentName: 'Test',
2874
+ maxTurns: 1, startedAt: new Date().toISOString(), seq: 1,
2875
+ }),
2876
+ sseEvent('agent_iteration_start', {
2877
+ executionId: execId, iteration: 1, maxTurns: 1,
2878
+ startedAt: new Date().toISOString(), seq: 2,
2879
+ }),
2880
+ sseEvent('agent_tool_start', {
2881
+ executionId: execId, iteration: 1, toolCallId: 'tc_shot',
2882
+ toolName: 'browser:screenshot', startedAt: new Date().toISOString(), seq: 3,
2883
+ }),
2884
+ sseEvent('agent_tool_complete', {
2885
+ executionId: execId, iteration: 1, toolCallId: 'tc_shot',
2886
+ toolName: 'browser:screenshot', completedAt: new Date().toISOString(), seq: 4,
2887
+ }),
2888
+ sseEvent('agent_media', {
2889
+ executionId: execId, iteration: 1, toolCallId: 'tc_shot',
2890
+ toolName: 'browser:screenshot',
2891
+ media: [
2892
+ { type: 'media', data: 'iVBORw==', mediaType: 'image/png' },
2893
+ ],
2894
+ seq: 5,
2895
+ }),
2896
+ sseEvent('agent_complete', {
2897
+ executionId: execId, agentId: 'virtual', success: true,
2898
+ iterations: 1, stopReason: 'max_iterations',
2899
+ completedAt: new Date().toISOString(), seq: 6,
2900
+ }),
2901
+ ]);
2902
+
2903
+ const client = new AgentWidgetClient({
2904
+ apiUrl: 'http://localhost:8000',
2905
+ agent: { name: 'Test', model: 'openai:gpt-4o-mini', systemPrompt: 'test' },
2906
+ });
2907
+ const events: AgentWidgetEvent[] = [];
2908
+ await client.dispatch(
2909
+ { messages: [{ id: 'usr_1', role: 'user', content: 'Snap', createdAt: new Date().toISOString() }] },
2910
+ (e) => events.push(e)
2911
+ );
2912
+
2913
+ const mediaMessages = collectMediaMessages(events);
2914
+ expect(mediaMessages).toHaveLength(1);
2915
+ const msg = mediaMessages[0]!;
2916
+ expect(msg.id).toMatch(/^agent-media-tc_shot-\d+$/);
2917
+ expect(msg.role).toBe('assistant');
2918
+ expect(msg.streaming).toBe(false);
2919
+ expect(msg.contentParts).toBeDefined();
2920
+ expect(msg.contentParts).toHaveLength(1);
2921
+ const part = msg.contentParts![0];
2922
+ expect(part.type).toBe('image');
2923
+ if (part.type === 'image') {
2924
+ expect(part.image).toBe('data:image/png;base64,iVBORw==');
2925
+ expect(part.mimeType).toBe('image/png');
2926
+ }
2927
+ expect(msg.agentMetadata?.executionId).toBe(execId);
2928
+ expect(msg.agentMetadata?.iteration).toBe(1);
2929
+ });
2930
+
2931
+ it('renders a hosted image (AI SDK v3/v4 type:"image-url") as a synthetic message', async () => {
2932
+ const execId = 'exec_media_url';
2933
+ global.fetch = createAgentStreamFetch([
2934
+ sseEvent('agent_start', { executionId: execId, agentId: 'virtual', agentName: 'Test', maxTurns: 1, startedAt: new Date().toISOString(), seq: 1 }),
2935
+ sseEvent('agent_iteration_start', { executionId: execId, iteration: 1, maxTurns: 1, startedAt: new Date().toISOString(), seq: 2 }),
2936
+ sseEvent('agent_media', {
2937
+ executionId: execId, iteration: 1, toolCallId: 'tc_dalle', toolName: 'dalle',
2938
+ media: [{ type: 'image-url', url: 'https://r2.example.com/img.png' }],
2939
+ seq: 3,
2940
+ }),
2941
+ sseEvent('agent_complete', { executionId: execId, agentId: 'virtual', success: true, iterations: 1, stopReason: 'max_iterations', completedAt: new Date().toISOString(), seq: 4 }),
2942
+ ]);
2943
+
2944
+ const client = new AgentWidgetClient({
2945
+ apiUrl: 'http://localhost:8000',
2946
+ agent: { name: 'Test', model: 'openai:gpt-4o-mini', systemPrompt: 'test' },
2947
+ });
2948
+ const events: AgentWidgetEvent[] = [];
2949
+ await client.dispatch(
2950
+ { messages: [{ id: 'usr_1', role: 'user', content: 'Generate', createdAt: new Date().toISOString() }] },
2951
+ (e) => events.push(e)
2952
+ );
2953
+
2954
+ const mediaMessages = collectMediaMessages(events);
2955
+ expect(mediaMessages).toHaveLength(1);
2956
+ const part = mediaMessages[0]!.contentParts![0];
2957
+ expect(part.type).toBe('image');
2958
+ if (part.type === 'image') {
2959
+ expect(part.image).toBe('https://r2.example.com/img.png');
2960
+ expect(part.mimeType).toBeUndefined();
2961
+ }
2962
+ });
2963
+
2964
+ it('preserves mediaType on image-url parts when provided', async () => {
2965
+ const execId = 'exec_media_url_typed';
2966
+ global.fetch = createAgentStreamFetch([
2967
+ sseEvent('agent_start', { executionId: execId, agentId: 'virtual', agentName: 'Test', maxTurns: 1, startedAt: new Date().toISOString(), seq: 1 }),
2968
+ sseEvent('agent_iteration_start', { executionId: execId, iteration: 1, maxTurns: 1, startedAt: new Date().toISOString(), seq: 2 }),
2969
+ sseEvent('agent_media', {
2970
+ executionId: execId, iteration: 1, toolCallId: 'tc_dalle', toolName: 'dalle',
2971
+ media: [{ type: 'image-url', url: 'https://r2.example.com/img.png', mediaType: 'image/png' }],
2972
+ seq: 3,
2973
+ }),
2974
+ sseEvent('agent_complete', { executionId: execId, agentId: 'virtual', success: true, iterations: 1, stopReason: 'max_iterations', completedAt: new Date().toISOString(), seq: 4 }),
2975
+ ]);
2976
+
2977
+ const client = new AgentWidgetClient({
2978
+ apiUrl: 'http://localhost:8000',
2979
+ agent: { name: 'Test', model: 'openai:gpt-4o-mini', systemPrompt: 'test' },
2980
+ });
2981
+ const events: AgentWidgetEvent[] = [];
2982
+ await client.dispatch(
2983
+ { messages: [{ id: 'usr_1', role: 'user', content: 'Generate', createdAt: new Date().toISOString() }] },
2984
+ (e) => events.push(e)
2985
+ );
2986
+
2987
+ const part = collectMediaMessages(events)[0]!.contentParts![0];
2988
+ expect(part.type).toBe('image');
2989
+ if (part.type === 'image') {
2990
+ expect(part.image).toBe('https://r2.example.com/img.png');
2991
+ expect(part.mimeType).toBe('image/png');
2992
+ }
2993
+ });
2994
+
2995
+ it('renders a base64 audio part with mediaType', async () => {
2996
+ const execId = 'exec_media_audio';
2997
+ global.fetch = createAgentStreamFetch([
2998
+ sseEvent('agent_start', { executionId: execId, agentId: 'virtual', agentName: 'Test', maxTurns: 1, startedAt: new Date().toISOString(), seq: 1 }),
2999
+ sseEvent('agent_iteration_start', { executionId: execId, iteration: 1, maxTurns: 1, startedAt: new Date().toISOString(), seq: 2 }),
3000
+ sseEvent('agent_media', {
3001
+ executionId: execId, iteration: 1, toolCallId: 'tc_tts', toolName: 'elevenlabs-tts',
3002
+ media: [{ type: 'media', data: 'AAAA', mediaType: 'audio/mpeg' }],
3003
+ seq: 3,
3004
+ }),
3005
+ sseEvent('agent_complete', { executionId: execId, agentId: 'virtual', success: true, iterations: 1, stopReason: 'max_iterations', completedAt: new Date().toISOString(), seq: 4 }),
3006
+ ]);
3007
+
3008
+ const client = new AgentWidgetClient({
3009
+ apiUrl: 'http://localhost:8000',
3010
+ agent: { name: 'Test', model: 'openai:gpt-4o-mini', systemPrompt: 'test' },
3011
+ });
3012
+ const events: AgentWidgetEvent[] = [];
3013
+ await client.dispatch(
3014
+ { messages: [{ id: 'usr_1', role: 'user', content: 'Speak', createdAt: new Date().toISOString() }] },
3015
+ (e) => events.push(e)
3016
+ );
3017
+
3018
+ const mediaMessages = collectMediaMessages(events);
3019
+ expect(mediaMessages).toHaveLength(1);
3020
+ const part = mediaMessages[0]!.contentParts![0];
3021
+ expect(part.type).toBe('audio');
3022
+ if (part.type === 'audio') {
3023
+ expect(part.audio).toBe('data:audio/mpeg;base64,AAAA');
3024
+ expect(part.mimeType).toBe('audio/mpeg');
3025
+ }
3026
+ });
3027
+
3028
+ it('routes file-url parts by mediaType (audio/video/file)', async () => {
3029
+ const execId = 'exec_media_file_url';
3030
+ global.fetch = createAgentStreamFetch([
3031
+ sseEvent('agent_start', { executionId: execId, agentId: 'virtual', agentName: 'Test', maxTurns: 1, startedAt: new Date().toISOString(), seq: 1 }),
3032
+ sseEvent('agent_iteration_start', { executionId: execId, iteration: 1, maxTurns: 1, startedAt: new Date().toISOString(), seq: 2 }),
3033
+ sseEvent('agent_media', {
3034
+ executionId: execId, iteration: 1, toolCallId: 'tc_files', toolName: 'multi',
3035
+ media: [
3036
+ { type: 'file-url', url: 'https://example.com/a.mp3', mediaType: 'audio/mpeg' },
3037
+ { type: 'file-url', url: 'https://example.com/v.mp4', mediaType: 'video/mp4' },
3038
+ { type: 'file-url', url: 'https://example.com/r.pdf', mediaType: 'application/pdf' },
3039
+ ],
3040
+ seq: 3,
3041
+ }),
3042
+ sseEvent('agent_complete', { executionId: execId, agentId: 'virtual', success: true, iterations: 1, stopReason: 'max_iterations', completedAt: new Date().toISOString(), seq: 4 }),
3043
+ ]);
3044
+
3045
+ const client = new AgentWidgetClient({
3046
+ apiUrl: 'http://localhost:8000',
3047
+ agent: { name: 'Test', model: 'openai:gpt-4o-mini', systemPrompt: 'test' },
3048
+ });
3049
+ const events: AgentWidgetEvent[] = [];
3050
+ await client.dispatch(
3051
+ { messages: [{ id: 'usr_1', role: 'user', content: 'Hi', createdAt: new Date().toISOString() }] },
3052
+ (e) => events.push(e)
3053
+ );
3054
+
3055
+ const parts = collectMediaMessages(events)[0]!.contentParts!;
3056
+ expect(parts).toHaveLength(3);
3057
+ expect(parts[0].type).toBe('audio');
3058
+ expect(parts[1].type).toBe('video');
3059
+ expect(parts[2].type).toBe('file');
3060
+ if (parts[0].type === 'audio') expect(parts[0].audio).toBe('https://example.com/a.mp3');
3061
+ if (parts[1].type === 'video') expect(parts[1].video).toBe('https://example.com/v.mp4');
3062
+ if (parts[2].type === 'file') {
3063
+ expect(parts[2].data).toBe('https://example.com/r.pdf');
3064
+ expect(parts[2].mimeType).toBe('application/pdf');
3065
+ expect(parts[2].filename).toBe('attachment.pdf');
3066
+ }
3067
+ });
3068
+
3069
+ it('renders mixed media parts in a single message', async () => {
3070
+ const execId = 'exec_media_mixed';
3071
+ global.fetch = createAgentStreamFetch([
3072
+ sseEvent('agent_start', { executionId: execId, agentId: 'virtual', agentName: 'Test', maxTurns: 1, startedAt: new Date().toISOString(), seq: 1 }),
3073
+ sseEvent('agent_iteration_start', { executionId: execId, iteration: 1, maxTurns: 1, startedAt: new Date().toISOString(), seq: 2 }),
3074
+ sseEvent('agent_media', {
3075
+ executionId: execId, iteration: 1, toolCallId: 'tc_mix', toolName: 'multi',
3076
+ media: [
3077
+ { type: 'media', data: 'IMG', mediaType: 'image/png' },
3078
+ { type: 'image-url', url: 'https://example.com/dalle.png' },
3079
+ { type: 'media', data: 'FILE', mediaType: 'application/pdf' },
3080
+ ],
3081
+ seq: 3,
3082
+ }),
3083
+ sseEvent('agent_complete', { executionId: execId, agentId: 'virtual', success: true, iterations: 1, stopReason: 'max_iterations', completedAt: new Date().toISOString(), seq: 4 }),
3084
+ ]);
3085
+
3086
+ const client = new AgentWidgetClient({
3087
+ apiUrl: 'http://localhost:8000',
3088
+ agent: { name: 'Test', model: 'openai:gpt-4o-mini', systemPrompt: 'test' },
3089
+ });
3090
+ const events: AgentWidgetEvent[] = [];
3091
+ await client.dispatch(
3092
+ { messages: [{ id: 'usr_1', role: 'user', content: 'Hi', createdAt: new Date().toISOString() }] },
3093
+ (e) => events.push(e)
3094
+ );
3095
+
3096
+ const mediaMessages = collectMediaMessages(events);
3097
+ expect(mediaMessages).toHaveLength(1);
3098
+ const parts = mediaMessages[0]!.contentParts!;
3099
+ expect(parts).toHaveLength(3);
3100
+ expect(parts[0].type).toBe('image');
3101
+ expect(parts[1].type).toBe('image');
3102
+ expect(parts[2].type).toBe('file');
3103
+ if (parts[2].type === 'file') {
3104
+ expect(parts[2].data).toBe('data:application/pdf;base64,FILE');
3105
+ expect(parts[2].filename).toBe('attachment.pdf');
3106
+ }
3107
+ });
3108
+
3109
+ it('inserts media between tool bubble and the next text turn', async () => {
3110
+ const execId = 'exec_media_order';
3111
+ global.fetch = createAgentStreamFetch([
3112
+ sseEvent('agent_start', { executionId: execId, agentId: 'virtual', agentName: 'Test', maxTurns: 1, startedAt: new Date().toISOString(), seq: 1 }),
3113
+ sseEvent('agent_iteration_start', { executionId: execId, iteration: 1, maxTurns: 1, startedAt: new Date().toISOString(), seq: 2 }),
3114
+ sseEvent('agent_turn_start', { executionId: execId, iteration: 1, turnIndex: 0, role: 'assistant', turnId: 'turn_1', seq: 3 }),
3115
+ sseEvent('agent_turn_delta', { executionId: execId, iteration: 1, delta: 'Calling tool...', contentType: 'text', turnId: 'turn_1', seq: 4 }),
3116
+ sseEvent('agent_tool_start', { executionId: execId, iteration: 1, toolCallId: 'tc_1', toolName: 'browser:screenshot', startedAt: new Date().toISOString(), seq: 5 }),
3117
+ sseEvent('agent_tool_complete', { executionId: execId, iteration: 1, toolCallId: 'tc_1', toolName: 'browser:screenshot', completedAt: new Date().toISOString(), seq: 6 }),
3118
+ sseEvent('agent_media', {
3119
+ executionId: execId, iteration: 1, toolCallId: 'tc_1', toolName: 'browser:screenshot',
3120
+ media: [{ type: 'media', data: 'PNG', mediaType: 'image/png' }],
3121
+ seq: 7,
3122
+ }),
3123
+ sseEvent('agent_turn_start', { executionId: execId, iteration: 1, turnIndex: 1, role: 'assistant', turnId: 'turn_2', seq: 8 }),
3124
+ sseEvent('agent_turn_delta', { executionId: execId, iteration: 1, delta: 'Done!', contentType: 'text', turnId: 'turn_2', seq: 9 }),
3125
+ sseEvent('agent_complete', { executionId: execId, agentId: 'virtual', success: true, iterations: 1, stopReason: 'max_iterations', completedAt: new Date().toISOString(), seq: 10 }),
3126
+ ]);
3127
+
3128
+ const client = new AgentWidgetClient({
3129
+ apiUrl: 'http://localhost:8000',
3130
+ agent: { name: 'Test', model: 'openai:gpt-4o-mini', systemPrompt: 'test' },
3131
+ });
3132
+ const events: AgentWidgetEvent[] = [];
3133
+ await client.dispatch(
3134
+ { messages: [{ id: 'usr_1', role: 'user', content: 'Take a snap', createdAt: new Date().toISOString() }] },
3135
+ (e) => events.push(e)
3136
+ );
3137
+
3138
+ // Collect the most-recent state of every emitted message by id.
3139
+ const latest = new Map<string, AgentWidgetMessage>();
3140
+ for (const e of events) {
3141
+ if (e.type === 'message') latest.set(e.message.id, e.message);
3142
+ }
3143
+ const ordered = Array.from(latest.values()).sort(
3144
+ (a, b) => (a.sequence ?? 0) - (b.sequence ?? 0)
3145
+ );
3146
+
3147
+ // Find the media bubble plus the assistant text bubble that came AFTER it
3148
+ const mediaMsg = ordered.find((m) => m.id.startsWith('agent-media-tc_1-'));
3149
+ expect(mediaMsg).toBeDefined();
3150
+ const followingText = ordered.find(
3151
+ (m) =>
3152
+ m.role === 'assistant' &&
3153
+ !m.variant &&
3154
+ !m.id.startsWith('agent-media-') &&
3155
+ (m.sequence ?? 0) > (mediaMsg!.sequence ?? 0)
3156
+ );
3157
+ expect(followingText).toBeDefined();
3158
+ expect(followingText!.content).toBe('Done!');
3159
+ });
3160
+
3161
+ it('skips malformed media parts that have neither data nor url', async () => {
3162
+ const execId = 'exec_media_empty';
3163
+ global.fetch = createAgentStreamFetch([
3164
+ sseEvent('agent_start', { executionId: execId, agentId: 'virtual', agentName: 'Test', maxTurns: 1, startedAt: new Date().toISOString(), seq: 1 }),
3165
+ sseEvent('agent_iteration_start', { executionId: execId, iteration: 1, maxTurns: 1, startedAt: new Date().toISOString(), seq: 2 }),
3166
+ sseEvent('agent_media', {
3167
+ executionId: execId, iteration: 1, toolCallId: 'tc_x', toolName: 'noop',
3168
+ media: [
3169
+ { type: 'media', mediaType: 'image/png' }, // missing data
3170
+ { type: 'image-url' }, // missing url
3171
+ { type: 'unknown-shape', payload: 'whatever' }, // unknown discriminator
3172
+ ],
3173
+ seq: 3,
3174
+ }),
3175
+ sseEvent('agent_complete', { executionId: execId, agentId: 'virtual', success: true, iterations: 1, stopReason: 'max_iterations', completedAt: new Date().toISOString(), seq: 4 }),
3176
+ ]);
3177
+
3178
+ const client = new AgentWidgetClient({
3179
+ apiUrl: 'http://localhost:8000',
3180
+ agent: { name: 'Test', model: 'openai:gpt-4o-mini', systemPrompt: 'test' },
3181
+ });
3182
+ const events: AgentWidgetEvent[] = [];
3183
+ await client.dispatch(
3184
+ { messages: [{ id: 'usr_1', role: 'user', content: 'Hi', createdAt: new Date().toISOString() }] },
3185
+ (e) => events.push(e)
3186
+ );
3187
+
3188
+ expect(collectMediaMessages(events)).toHaveLength(0);
3189
+ });
3190
+
3191
+ it('produces unique ids for repeated agent_media events on the same toolCallId', async () => {
3192
+ const execId = 'exec_media_repeat';
3193
+ global.fetch = createAgentStreamFetch([
3194
+ sseEvent('agent_start', { executionId: execId, agentId: 'virtual', agentName: 'Test', maxTurns: 1, startedAt: new Date().toISOString(), seq: 1 }),
3195
+ sseEvent('agent_iteration_start', { executionId: execId, iteration: 1, maxTurns: 1, startedAt: new Date().toISOString(), seq: 2 }),
3196
+ sseEvent('agent_media', {
3197
+ executionId: execId, iteration: 1, toolCallId: 'tc_repeat', toolName: 'multi',
3198
+ media: [{ type: 'media', data: 'A', mediaType: 'image/png' }],
3199
+ seq: 3,
3200
+ }),
3201
+ sseEvent('agent_media', {
3202
+ executionId: execId, iteration: 1, toolCallId: 'tc_repeat', toolName: 'multi',
3203
+ media: [{ type: 'media', data: 'B', mediaType: 'image/png' }],
3204
+ seq: 4,
3205
+ }),
3206
+ sseEvent('agent_complete', { executionId: execId, agentId: 'virtual', success: true, iterations: 1, stopReason: 'max_iterations', completedAt: new Date().toISOString(), seq: 5 }),
3207
+ ]);
3208
+
3209
+ const client = new AgentWidgetClient({
3210
+ apiUrl: 'http://localhost:8000',
3211
+ agent: { name: 'Test', model: 'openai:gpt-4o-mini', systemPrompt: 'test' },
3212
+ });
3213
+ const events: AgentWidgetEvent[] = [];
3214
+ await client.dispatch(
3215
+ { messages: [{ id: 'usr_1', role: 'user', content: 'Hi', createdAt: new Date().toISOString() }] },
3216
+ (e) => events.push(e)
3217
+ );
3218
+
3219
+ const mediaMessages = collectMediaMessages(events);
3220
+ expect(mediaMessages).toHaveLength(2);
3221
+ expect(mediaMessages[0]!.id).not.toBe(mediaMessages[1]!.id);
3222
+ // Both messages should preserve the toolCallId in the id for traceability
3223
+ for (const m of mediaMessages) {
3224
+ expect(m.id).toMatch(/^agent-media-tc_repeat-\d+$/);
3225
+ }
3226
+ // First message keeps its first content part; not overwritten by the second
3227
+ const first = mediaMessages.find((m) => {
3228
+ const p = m.contentParts?.[0];
3229
+ return p?.type === 'image' && p.image === 'data:image/png;base64,A';
3230
+ });
3231
+ expect(first).toBeDefined();
3232
+ });
3233
+
3234
+ it('seals an in-flight assistant text bubble before splitting on agent_media', async () => {
3235
+ const execId = 'exec_media_seal';
3236
+ global.fetch = createAgentStreamFetch([
3237
+ sseEvent('agent_start', { executionId: execId, agentId: 'virtual', agentName: 'Test', maxTurns: 1, startedAt: new Date().toISOString(), seq: 1 }),
3238
+ sseEvent('agent_iteration_start', { executionId: execId, iteration: 1, maxTurns: 1, startedAt: new Date().toISOString(), seq: 2 }),
3239
+ sseEvent('agent_turn_start', { executionId: execId, iteration: 1, turnIndex: 0, role: 'assistant', turnId: 'turn_1', seq: 3 }),
3240
+ sseEvent('agent_turn_delta', { executionId: execId, iteration: 1, delta: 'Streaming...', contentType: 'text', turnId: 'turn_1', seq: 4 }),
3241
+ // Media arrives mid-stream — earlier text bubble is still streaming.
3242
+ sseEvent('agent_media', {
3243
+ executionId: execId, iteration: 1, toolCallId: 'tc_seal', toolName: 'shot',
3244
+ media: [{ type: 'media', data: 'PNG', mediaType: 'image/png' }],
3245
+ seq: 5,
3246
+ }),
3247
+ sseEvent('agent_complete', { executionId: execId, agentId: 'virtual', success: true, iterations: 1, stopReason: 'max_iterations', completedAt: new Date().toISOString(), seq: 6 }),
3248
+ ]);
3249
+
3250
+ const client = new AgentWidgetClient({
3251
+ apiUrl: 'http://localhost:8000',
3252
+ agent: { name: 'Test', model: 'openai:gpt-4o-mini', systemPrompt: 'test' },
3253
+ });
3254
+ const events: AgentWidgetEvent[] = [];
3255
+ await client.dispatch(
3256
+ { messages: [{ id: 'usr_1', role: 'user', content: 'Go', createdAt: new Date().toISOString() }] },
3257
+ (e) => events.push(e)
3258
+ );
3259
+
3260
+ const latest = new Map<string, AgentWidgetMessage>();
3261
+ for (const e of events) {
3262
+ if (e.type === 'message') latest.set(e.message.id, e.message);
3263
+ }
3264
+
3265
+ // The pre-media assistant bubble must be sealed (no orphan typing indicator).
3266
+ const orphan = Array.from(latest.values()).find(
3267
+ (m) =>
3268
+ m.role === 'assistant' &&
3269
+ !m.variant &&
3270
+ !m.id.startsWith('agent-media-') &&
3271
+ m.content === 'Streaming...'
3272
+ );
3273
+ expect(orphan).toBeDefined();
3274
+ expect(orphan!.streaming).toBe(false);
3275
+ });
3276
+
3277
+ it('routes audio parts case-insensitively (RFC 7231)', async () => {
3278
+ const execId = 'exec_media_case';
3279
+ global.fetch = createAgentStreamFetch([
3280
+ sseEvent('agent_start', { executionId: execId, agentId: 'virtual', agentName: 'Test', maxTurns: 1, startedAt: new Date().toISOString(), seq: 1 }),
3281
+ sseEvent('agent_iteration_start', { executionId: execId, iteration: 1, maxTurns: 1, startedAt: new Date().toISOString(), seq: 2 }),
3282
+ sseEvent('agent_media', {
3283
+ executionId: execId, iteration: 1, toolCallId: 'tc_case', toolName: 'tts',
3284
+ // Non-canonical casing should still land in the audio bucket, not the file bucket.
3285
+ media: [{ type: 'media', data: 'AAAA', mediaType: 'Audio/MPEG' }],
3286
+ seq: 3,
3287
+ }),
3288
+ sseEvent('agent_complete', { executionId: execId, agentId: 'virtual', success: true, iterations: 1, stopReason: 'max_iterations', completedAt: new Date().toISOString(), seq: 4 }),
3289
+ ]);
3290
+
3291
+ const client = new AgentWidgetClient({
3292
+ apiUrl: 'http://localhost:8000',
3293
+ agent: { name: 'Test', model: 'openai:gpt-4o-mini', systemPrompt: 'test' },
3294
+ });
3295
+ const events: AgentWidgetEvent[] = [];
3296
+ await client.dispatch(
3297
+ { messages: [{ id: 'usr_1', role: 'user', content: 'Speak', createdAt: new Date().toISOString() }] },
3298
+ (e) => events.push(e)
3299
+ );
3300
+
3301
+ const part = collectMediaMessages(events)[0]!.contentParts![0];
3302
+ expect(part.type).toBe('audio');
3303
+ if (part.type === 'audio') {
3304
+ // mediaType is canonicalized to lowercase for both routing and storage.
3305
+ expect(part.mimeType).toBe('audio/mpeg');
3306
+ expect(part.audio).toBe('data:audio/mpeg;base64,AAAA');
3307
+ }
3308
+ });
3309
+
3310
+ it('renders a base64 text/csv attachment as a file part (not silently dropped)', async () => {
3311
+ const execId = 'exec_media_csv';
3312
+ global.fetch = createAgentStreamFetch([
3313
+ sseEvent('agent_start', { executionId: execId, agentId: 'virtual', agentName: 'Test', maxTurns: 1, startedAt: new Date().toISOString(), seq: 1 }),
3314
+ sseEvent('agent_iteration_start', { executionId: execId, iteration: 1, maxTurns: 1, startedAt: new Date().toISOString(), seq: 2 }),
3315
+ sseEvent('agent_media', {
3316
+ executionId: execId, iteration: 1, toolCallId: 'tc_csv', toolName: 'export',
3317
+ media: [{ type: 'media', data: 'YSxiCjEsMg==', mediaType: 'text/csv' }],
3318
+ seq: 3,
3319
+ }),
3320
+ sseEvent('agent_complete', { executionId: execId, agentId: 'virtual', success: true, iterations: 1, stopReason: 'max_iterations', completedAt: new Date().toISOString(), seq: 4 }),
3321
+ ]);
3322
+
3323
+ const client = new AgentWidgetClient({
3324
+ apiUrl: 'http://localhost:8000',
3325
+ agent: { name: 'Test', model: 'openai:gpt-4o-mini', systemPrompt: 'test' },
3326
+ });
3327
+ const events: AgentWidgetEvent[] = [];
3328
+ await client.dispatch(
3329
+ { messages: [{ id: 'usr_1', role: 'user', content: 'Export', createdAt: new Date().toISOString() }] },
3330
+ (e) => events.push(e)
3331
+ );
3332
+
3333
+ const part = collectMediaMessages(events)[0]!.contentParts![0];
3334
+ expect(part.type).toBe('file');
3335
+ if (part.type === 'file') {
3336
+ expect(part.mimeType).toBe('text/csv');
3337
+ expect(part.filename).toBe('attachment.csv');
3338
+ expect(part.data).toBe('data:text/csv;base64,YSxiCjEsMg==');
3339
+ }
3340
+ });
3341
+
3342
+ it('defaults missing mediaType on a type:"media" part to application/octet-stream', async () => {
3343
+ const execId = 'exec_media_no_type';
3344
+ global.fetch = createAgentStreamFetch([
3345
+ sseEvent('agent_start', { executionId: execId, agentId: 'virtual', agentName: 'Test', maxTurns: 1, startedAt: new Date().toISOString(), seq: 1 }),
3346
+ sseEvent('agent_iteration_start', { executionId: execId, iteration: 1, maxTurns: 1, startedAt: new Date().toISOString(), seq: 2 }),
3347
+ sseEvent('agent_media', {
3348
+ executionId: execId, iteration: 1, toolCallId: 'tc_blob', toolName: 'opaque',
3349
+ // mediaType is empty — should not produce a malformed `data:;base64,...` URI.
3350
+ media: [{ type: 'media', data: 'AAAA', mediaType: '' }],
3351
+ seq: 3,
3352
+ }),
3353
+ sseEvent('agent_complete', { executionId: execId, agentId: 'virtual', success: true, iterations: 1, stopReason: 'max_iterations', completedAt: new Date().toISOString(), seq: 4 }),
3354
+ ]);
3355
+
3356
+ const client = new AgentWidgetClient({
3357
+ apiUrl: 'http://localhost:8000',
3358
+ agent: { name: 'Test', model: 'openai:gpt-4o-mini', systemPrompt: 'test' },
3359
+ });
3360
+ const events: AgentWidgetEvent[] = [];
3361
+ await client.dispatch(
3362
+ { messages: [{ id: 'usr_1', role: 'user', content: 'Hi', createdAt: new Date().toISOString() }] },
3363
+ (e) => events.push(e)
3364
+ );
3365
+
3366
+ const part = collectMediaMessages(events)[0]!.contentParts![0];
3367
+ expect(part.type).toBe('file');
3368
+ if (part.type === 'file') {
3369
+ expect(part.mimeType).toBe('application/octet-stream');
3370
+ expect(part.data).toBe('data:application/octet-stream;base64,AAAA');
3371
+ }
3372
+ });
3373
+ });
3374
+