@tanstack/ai 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/esm/activities/chat/adapter.d.ts +6 -1
- package/dist/esm/activities/chat/adapter.js.map +1 -1
- package/dist/esm/activities/chat/index.d.ts +8 -0
- package/dist/esm/activities/chat/index.js +78 -17
- package/dist/esm/activities/chat/index.js.map +1 -1
- package/dist/esm/activities/chat/middleware/compose.d.ts +3 -1
- package/dist/esm/activities/chat/middleware/compose.js +79 -1
- package/dist/esm/activities/chat/middleware/compose.js.map +1 -1
- package/dist/esm/activities/error-payload.d.ts +12 -0
- package/dist/esm/activities/error-payload.js +25 -0
- package/dist/esm/activities/error-payload.js.map +1 -0
- package/dist/esm/activities/generateAudio/adapter.d.ts +62 -0
- package/dist/esm/activities/generateAudio/adapter.js +14 -0
- package/dist/esm/activities/generateAudio/adapter.js.map +1 -0
- package/dist/esm/activities/generateAudio/index.d.ts +74 -0
- package/dist/esm/activities/generateAudio/index.js +80 -0
- package/dist/esm/activities/generateAudio/index.js.map +1 -0
- package/dist/esm/activities/generateImage/adapter.d.ts +1 -1
- package/dist/esm/activities/generateImage/adapter.js +1 -1
- package/dist/esm/activities/generateImage/adapter.js.map +1 -1
- package/dist/esm/activities/generateImage/index.d.ts +7 -0
- package/dist/esm/activities/generateImage/index.js +19 -3
- package/dist/esm/activities/generateImage/index.js.map +1 -1
- package/dist/esm/activities/generateSpeech/adapter.d.ts +1 -1
- package/dist/esm/activities/generateSpeech/adapter.js +1 -1
- package/dist/esm/activities/generateSpeech/adapter.js.map +1 -1
- package/dist/esm/activities/generateSpeech/index.d.ts +10 -3
- package/dist/esm/activities/generateSpeech/index.js +32 -3
- package/dist/esm/activities/generateSpeech/index.js.map +1 -1
- package/dist/esm/activities/generateTranscription/adapter.d.ts +1 -1
- package/dist/esm/activities/generateTranscription/adapter.js +1 -1
- package/dist/esm/activities/generateTranscription/adapter.js.map +1 -1
- package/dist/esm/activities/generateTranscription/index.d.ts +10 -3
- package/dist/esm/activities/generateTranscription/index.js +43 -13
- package/dist/esm/activities/generateTranscription/index.js.map +1 -1
- package/dist/esm/activities/generateVideo/index.d.ts +7 -0
- package/dist/esm/activities/generateVideo/index.js +55 -13
- package/dist/esm/activities/generateVideo/index.js.map +1 -1
- package/dist/esm/activities/index.d.ts +5 -2
- package/dist/esm/activities/index.js +11 -6
- package/dist/esm/activities/index.js.map +1 -1
- package/dist/esm/activities/stream-generation-result.js +5 -6
- package/dist/esm/activities/stream-generation-result.js.map +1 -1
- package/dist/esm/activities/summarize/index.d.ts +7 -0
- package/dist/esm/activities/summarize/index.js +54 -19
- package/dist/esm/activities/summarize/index.js.map +1 -1
- package/dist/esm/adapter-internals.d.ts +4 -0
- package/dist/esm/adapter-internals.js +9 -0
- package/dist/esm/adapter-internals.js.map +1 -0
- package/dist/esm/index.d.ts +5 -2
- package/dist/esm/index.js +5 -0
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/logger/console-logger.d.ts +11 -0
- package/dist/esm/logger/console-logger.js +27 -0
- package/dist/esm/logger/console-logger.js.map +1 -0
- package/dist/esm/logger/internal-logger.d.ts +33 -0
- package/dist/esm/logger/internal-logger.js +69 -0
- package/dist/esm/logger/internal-logger.js.map +1 -0
- package/dist/esm/logger/resolve.d.ts +14 -0
- package/dist/esm/logger/resolve.js +54 -0
- package/dist/esm/logger/resolve.js.map +1 -0
- package/dist/esm/logger/types.d.ts +75 -0
- package/dist/esm/stream-to-response.js +3 -8
- package/dist/esm/stream-to-response.js.map +1 -1
- package/dist/esm/types.d.ts +97 -6
- package/package.json +6 -2
- package/skills/ai-core/SKILL.md +5 -3
- package/skills/ai-core/debug-logging/SKILL.md +263 -0
- package/skills/ai-core/media-generation/SKILL.md +154 -10
- package/src/activities/chat/adapter.ts +6 -1
- package/src/activities/chat/index.ts +104 -22
- package/src/activities/chat/middleware/compose.ts +84 -1
- package/src/activities/error-payload.ts +35 -0
- package/src/activities/generateAudio/adapter.ts +89 -0
- package/src/activities/generateAudio/index.ts +224 -0
- package/src/activities/generateImage/adapter.ts +1 -1
- package/src/activities/generateImage/index.ts +29 -3
- package/src/activities/generateSpeech/adapter.ts +1 -1
- package/src/activities/generateSpeech/index.ts +51 -9
- package/src/activities/generateTranscription/adapter.ts +1 -1
- package/src/activities/generateTranscription/index.ts +72 -17
- package/src/activities/generateVideo/index.ts +72 -13
- package/src/activities/index.ts +22 -0
- package/src/activities/stream-generation-result.ts +6 -7
- package/src/activities/summarize/index.ts +66 -20
- package/src/adapter-internals.ts +8 -0
- package/src/index.ts +13 -0
- package/src/logger/console-logger.ts +49 -0
- package/src/logger/internal-logger.ts +107 -0
- package/src/logger/resolve.ts +72 -0
- package/src/logger/types.ts +78 -0
- package/src/stream-to-response.ts +5 -10
- package/src/types.ts +110 -5
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: ai-core/media-generation
|
|
3
3
|
description: >
|
|
4
|
-
Image, video, speech (TTS), and transcription generation using
|
|
4
|
+
Image, audio, video, speech (TTS), and transcription generation using
|
|
5
5
|
activity-specific adapters: generateImage() with openaiImage/geminiImage,
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
useGenerateImage,
|
|
6
|
+
generateAudio() with geminiAudio/falAudio, generateVideo() with async
|
|
7
|
+
polling, generateSpeech() with openaiSpeech, generateTranscription() with
|
|
8
|
+
openaiTranscription. React hooks: useGenerateImage, useGenerateAudio,
|
|
9
|
+
useGenerateSpeech, useTranscription, useGenerateVideo.
|
|
9
10
|
TanStack Start server function integration with toServerSentEventsResponse.
|
|
10
11
|
type: sub-skill
|
|
11
12
|
library: tanstack-ai
|
|
@@ -14,9 +15,11 @@ sources:
|
|
|
14
15
|
- 'TanStack/ai:docs/media/generations.md'
|
|
15
16
|
- 'TanStack/ai:docs/media/generation-hooks.md'
|
|
16
17
|
- 'TanStack/ai:docs/media/image-generation.md'
|
|
18
|
+
- 'TanStack/ai:docs/media/audio-generation.md'
|
|
17
19
|
- 'TanStack/ai:docs/media/video-generation.md'
|
|
18
20
|
- 'TanStack/ai:docs/media/text-to-speech.md'
|
|
19
21
|
- 'TanStack/ai:docs/media/transcription.md'
|
|
22
|
+
- 'TanStack/ai:docs/advanced/debug-logging.md'
|
|
20
23
|
---
|
|
21
24
|
|
|
22
25
|
# Media Generation
|
|
@@ -186,7 +189,40 @@ Result shape: `ImageGenerationResult` with `images` array where each entry
|
|
|
186
189
|
has `b64Json?`, `url?`, and `revisedPrompt?`. OpenAI image URLs expire
|
|
187
190
|
after 1 hour -- download or display immediately.
|
|
188
191
|
|
|
189
|
-
### 2.
|
|
192
|
+
### 2. Audio Generation (Music, Sound Effects)
|
|
193
|
+
|
|
194
|
+
Distinct from TTS — `generateAudio()` produces non-speech audio content.
|
|
195
|
+
Supported adapters: `geminiAudio` (Lyria 3 Pro / Lyria 3 Clip) and
|
|
196
|
+
`falAudio` (MiniMax Music, DiffRhythm, Stable Audio, ElevenLabs SFX, etc.).
|
|
197
|
+
|
|
198
|
+
```typescript
|
|
199
|
+
import { generateAudio } from '@tanstack/ai'
|
|
200
|
+
import { falAudio } from '@tanstack/ai-fal'
|
|
201
|
+
|
|
202
|
+
const result = await generateAudio({
|
|
203
|
+
adapter: falAudio('fal-ai/diffrhythm'),
|
|
204
|
+
prompt: 'An upbeat electronic track with synths',
|
|
205
|
+
duration: 10,
|
|
206
|
+
})
|
|
207
|
+
|
|
208
|
+
// result.audio.url or result.audio.b64Json (provider-dependent)
|
|
209
|
+
// result.audio.contentType e.g. "audio/mpeg"
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
Client hook:
|
|
213
|
+
|
|
214
|
+
```tsx
|
|
215
|
+
import { useGenerateAudio, fetchServerSentEvents } from '@tanstack/ai-react'
|
|
216
|
+
|
|
217
|
+
const { generate, result, isLoading } = useGenerateAudio({
|
|
218
|
+
connection: fetchServerSentEvents('/api/generate/audio'),
|
|
219
|
+
})
|
|
220
|
+
|
|
221
|
+
// Trigger: generate({ prompt: 'Upbeat synths', duration: 10 })
|
|
222
|
+
// Play: <audio src={result.audio.url} controls />
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
### 3. Text-to-Speech
|
|
190
226
|
|
|
191
227
|
Adapter: `openaiSpeech` (tts-1, tts-1-hd, gpt-4o-audio-preview).
|
|
192
228
|
|
|
@@ -220,7 +256,7 @@ const { generate, result, isLoading } = useGenerateSpeech({
|
|
|
220
256
|
// Play: <audio src={`data:audio/${result.format};base64,${result.audio}`} controls />
|
|
221
257
|
```
|
|
222
258
|
|
|
223
|
-
###
|
|
259
|
+
### 4. Audio Transcription
|
|
224
260
|
|
|
225
261
|
Adapter: `openaiTranscription` (whisper-1, gpt-4o-transcribe,
|
|
226
262
|
gpt-4o-mini-transcribe).
|
|
@@ -257,7 +293,7 @@ const { generate, result, isLoading } = useTranscription({
|
|
|
257
293
|
// Trigger: generate({ audio: dataUrl, language: 'en' })
|
|
258
294
|
```
|
|
259
295
|
|
|
260
|
-
###
|
|
296
|
+
### 5. Video Generation (Experimental -- async polling)
|
|
261
297
|
|
|
262
298
|
Video generation uses a jobs/polling architecture. The server creates a job,
|
|
263
299
|
polls for status, and streams updates to the client.
|
|
@@ -454,12 +490,116 @@ for (const img of result.images) {
|
|
|
454
490
|
Not all generation activities support streaming. Passing `stream: true` to
|
|
455
491
|
an activity that does not support it may hang or produce unexpected results.
|
|
456
492
|
Check the activity documentation before enabling streaming. All built-in
|
|
457
|
-
activities (`generateImage`, `
|
|
458
|
-
`generateVideo`, `summarize`) support `stream: true`,
|
|
459
|
-
`useGeneration` setups may not.
|
|
493
|
+
activities (`generateImage`, `generateAudio`, `generateSpeech`,
|
|
494
|
+
`generateTranscription`, `generateVideo`, `summarize`) support `stream: true`,
|
|
495
|
+
but custom `useGeneration` setups may not.
|
|
460
496
|
|
|
461
497
|
> Source: docs/media/generations.md.
|
|
462
498
|
|
|
499
|
+
### e. HIGH: Passing `responseMimeType` or `negativePrompt` to Gemini Lyria
|
|
500
|
+
|
|
501
|
+
Gemini's `GenerateContentConfig` (used by Lyria 3 Pro / Lyria 3 Clip) does
|
|
502
|
+
**not** support `responseMimeType` or `negativePrompt`. Lyria 3 Clip always
|
|
503
|
+
returns 30-second `audio/mp3`; Lyria 3 Pro returns `audio/mp3`. These fields
|
|
504
|
+
are not in `GeminiAudioProviderOptions` — don't reach for them via `as any`.
|
|
505
|
+
|
|
506
|
+
```typescript
|
|
507
|
+
// WRONG — both fields are silently ignored or rejected by the SDK
|
|
508
|
+
generateAudio({
|
|
509
|
+
adapter: geminiAudio('lyria-3-pro-preview'),
|
|
510
|
+
prompt: 'ambient piano',
|
|
511
|
+
modelOptions: {
|
|
512
|
+
responseMimeType: 'audio/wav', // unsupported
|
|
513
|
+
negativePrompt: 'vocals', // unsupported
|
|
514
|
+
} as any,
|
|
515
|
+
})
|
|
516
|
+
|
|
517
|
+
// CORRECT — shape the prompt itself for what you want
|
|
518
|
+
generateAudio({
|
|
519
|
+
adapter: geminiAudio('lyria-3-pro-preview'),
|
|
520
|
+
prompt: 'ambient piano, no vocals',
|
|
521
|
+
})
|
|
522
|
+
```
|
|
523
|
+
|
|
524
|
+
> Source: Gemini API `GenerateContentConfig` type; docs/media/audio-generation.md.
|
|
525
|
+
|
|
526
|
+
### f. MEDIUM: Passing `duration` to Lyria expecting it to control length
|
|
527
|
+
|
|
528
|
+
Lyria 3 Clip is fixed at 30 seconds — the `duration` option is ignored on
|
|
529
|
+
that model. Lyria 3 Pro accepts duration via natural-language in the
|
|
530
|
+
**prompt** ("2-minute ambient track with a 30-second build"), not via the
|
|
531
|
+
`duration` field. `duration` works for fal audio models (mapped to each
|
|
532
|
+
model's native field like `music_length_ms` or `seconds_total`), but not
|
|
533
|
+
for Lyria.
|
|
534
|
+
|
|
535
|
+
```typescript
|
|
536
|
+
// For Lyria: put length guidance in the prompt
|
|
537
|
+
generateAudio({
|
|
538
|
+
adapter: geminiAudio('lyria-3-pro-preview'),
|
|
539
|
+
prompt: 'A 2-minute ambient piano piece with gentle strings',
|
|
540
|
+
// duration: 120 // ← does nothing; rely on the prompt
|
|
541
|
+
})
|
|
542
|
+
|
|
543
|
+
// For fal: duration works and is translated per-model
|
|
544
|
+
generateAudio({
|
|
545
|
+
adapter: falAudio('fal-ai/minimax-music/v2'),
|
|
546
|
+
prompt: 'upbeat synth melody',
|
|
547
|
+
duration: 60, // → music_length_ms: 60_000
|
|
548
|
+
})
|
|
549
|
+
```
|
|
550
|
+
|
|
551
|
+
> Source: Google Lyria 3 docs; docs/media/audio-generation.md.
|
|
552
|
+
|
|
553
|
+
### g. MEDIUM: Gemini TTS multi-speaker with 0 or 3+ speakers
|
|
554
|
+
|
|
555
|
+
`multiSpeakerVoiceConfig.speakerVoiceConfigs` is validated to be length 1 or 2. Passing an empty array or three+ entries throws at the adapter boundary
|
|
556
|
+
(not at Gemini's API) with a clear error. Don't try to work around it with
|
|
557
|
+
`as any`.
|
|
558
|
+
|
|
559
|
+
```typescript
|
|
560
|
+
generateSpeech({
|
|
561
|
+
adapter: geminiSpeech('gemini-2.5-pro-preview-tts'),
|
|
562
|
+
text: '[Alice] Hi. [Bob] Hello!',
|
|
563
|
+
modelOptions: {
|
|
564
|
+
multiSpeakerVoiceConfig: {
|
|
565
|
+
speakerVoiceConfigs: [
|
|
566
|
+
{
|
|
567
|
+
speaker: 'Alice',
|
|
568
|
+
voiceConfig: { prebuiltVoiceConfig: { voiceName: 'Kore' } },
|
|
569
|
+
},
|
|
570
|
+
{
|
|
571
|
+
speaker: 'Bob',
|
|
572
|
+
voiceConfig: { prebuiltVoiceConfig: { voiceName: 'Puck' } },
|
|
573
|
+
},
|
|
574
|
+
],
|
|
575
|
+
},
|
|
576
|
+
},
|
|
577
|
+
})
|
|
578
|
+
```
|
|
579
|
+
|
|
580
|
+
> Source: Gemini TTS adapter validation; CodeRabbit review of PR #463.
|
|
581
|
+
|
|
582
|
+
### h. LOW: Writing a logging middleware to see media chunks flow through
|
|
583
|
+
|
|
584
|
+
Every media activity — `generateAudio`, `generateSpeech`,
|
|
585
|
+
`generateTranscription`, `generateImage`, `generateVideo` — accepts the
|
|
586
|
+
same `debug?: DebugOption` option that `chat()` does. Reach for `debug`
|
|
587
|
+
instead of wiring up logging middleware.
|
|
588
|
+
|
|
589
|
+
```typescript
|
|
590
|
+
// When a speech generation sounds wrong or a transcription returns garbage
|
|
591
|
+
generateSpeech({
|
|
592
|
+
adapter: openaiSpeech('tts-1'),
|
|
593
|
+
text: 'Hello',
|
|
594
|
+
debug: { provider: true, output: true }, // raw SDK chunks + yielded chunks
|
|
595
|
+
})
|
|
596
|
+
```
|
|
597
|
+
|
|
598
|
+
See the `ai-core/debug-logging` sub-skill for full details on categories
|
|
599
|
+
and piping into a custom logger.
|
|
600
|
+
|
|
601
|
+
> Source: docs/advanced/debug-logging.md.
|
|
602
|
+
|
|
463
603
|
---
|
|
464
604
|
|
|
465
605
|
## Cross-References
|
|
@@ -469,3 +609,7 @@ activities (`generateImage`, `generateSpeech`, `generateTranscription`,
|
|
|
469
609
|
images, `openaiSpeech` for speech, `openaiTranscription` for transcription,
|
|
470
610
|
`openaiVideo` for video). The adapter-configuration skill covers provider
|
|
471
611
|
setup, API keys, and model selection.
|
|
612
|
+
- See also: **ai-core/debug-logging/SKILL.md** -- When a media request
|
|
613
|
+
returns unexpected output or fails mid-stream, toggle `debug: true` on
|
|
614
|
+
any `generate*()` call to see request metadata, raw provider chunks, and
|
|
615
|
+
errors. Covers per-category toggling and piping into pino/winston.
|
|
@@ -18,7 +18,12 @@ export interface TextAdapterConfig {
|
|
|
18
18
|
}
|
|
19
19
|
|
|
20
20
|
/**
|
|
21
|
-
* Options for structured output generation
|
|
21
|
+
* Options for structured output generation.
|
|
22
|
+
*
|
|
23
|
+
* The internal logger is threaded through `chatOptions.logger` (inherited from
|
|
24
|
+
* `TextOptions`). Adapter implementations must call `logger.request()` before
|
|
25
|
+
* SDK calls, `logger.provider()` for each chunk received, and `logger.errors()`
|
|
26
|
+
* in catch blocks.
|
|
22
27
|
*/
|
|
23
28
|
export interface StructuredOutputOptions<TProviderOptions extends object> {
|
|
24
29
|
/** Text options for the request */
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
import { devtoolsMiddleware } from '@tanstack/ai-event-client'
|
|
9
9
|
import { stripToSpecMiddleware } from '../../strip-to-spec-middleware'
|
|
10
10
|
import { streamToText } from '../../stream-to-response.js'
|
|
11
|
+
import { resolveDebugOption } from '../../logger/resolve'
|
|
11
12
|
import { LazyToolManager } from './tools/lazy-tool-manager'
|
|
12
13
|
import {
|
|
13
14
|
MiddlewareAbortError,
|
|
@@ -51,6 +52,8 @@ import type {
|
|
|
51
52
|
ChatMiddlewareContext,
|
|
52
53
|
ChatMiddlewarePhase,
|
|
53
54
|
} from './middleware/types'
|
|
55
|
+
import type { InternalLogger } from '../../logger/internal-logger'
|
|
56
|
+
import type { DebugOption } from '../../logger/types'
|
|
54
57
|
import type { ProviderTool } from '../../tools/provider-tool'
|
|
55
58
|
|
|
56
59
|
// ===========================
|
|
@@ -181,6 +184,13 @@ export interface TextActivityOptions<
|
|
|
181
184
|
* Can be used to pass request-scoped data (e.g., user ID, request context).
|
|
182
185
|
*/
|
|
183
186
|
context?: unknown
|
|
187
|
+
/**
|
|
188
|
+
* Enable debug logging. Pass `true` to enable all categories with the default
|
|
189
|
+
* console logger, `false` to silence everything, or a `DebugConfig` object for
|
|
190
|
+
* granular control and/or a custom `Logger`. Defaults to `undefined`, which
|
|
191
|
+
* means only the `errors` category is active.
|
|
192
|
+
*/
|
|
193
|
+
debug?: DebugOption
|
|
184
194
|
}
|
|
185
195
|
|
|
186
196
|
// ===========================
|
|
@@ -293,7 +303,13 @@ class TextEngine<
|
|
|
293
303
|
private middlewareAbortController?: AbortController
|
|
294
304
|
private terminalHookCalled = false
|
|
295
305
|
|
|
296
|
-
|
|
306
|
+
private readonly logger: InternalLogger
|
|
307
|
+
|
|
308
|
+
constructor(
|
|
309
|
+
config: TextEngineConfig<TAdapter, TParams>,
|
|
310
|
+
logger: InternalLogger,
|
|
311
|
+
) {
|
|
312
|
+
this.logger = logger
|
|
297
313
|
this.adapter = config.adapter
|
|
298
314
|
this.params = config.params
|
|
299
315
|
this.systemPrompts = config.params.systemPrompts || []
|
|
@@ -341,7 +357,7 @@ class TextEngine<
|
|
|
341
357
|
...(config.middleware || []),
|
|
342
358
|
stripToSpecMiddleware(),
|
|
343
359
|
]
|
|
344
|
-
this.middlewareRunner = new MiddlewareRunner(allMiddleware)
|
|
360
|
+
this.middlewareRunner = new MiddlewareRunner(allMiddleware, logger)
|
|
345
361
|
this.middlewareAbortController = new AbortController()
|
|
346
362
|
this.middlewareCtx = {
|
|
347
363
|
requestId: this.requestId,
|
|
@@ -393,6 +409,9 @@ class TextEngine<
|
|
|
393
409
|
|
|
394
410
|
async *run(): AsyncGenerator<StreamChunk> {
|
|
395
411
|
this.beforeRun()
|
|
412
|
+
this.logger.agentLoop('run started', {
|
|
413
|
+
conversationId: this.middlewareCtx.conversationId,
|
|
414
|
+
})
|
|
396
415
|
|
|
397
416
|
try {
|
|
398
417
|
// Run initial onConfig (phase = init)
|
|
@@ -417,6 +436,10 @@ class TextEngine<
|
|
|
417
436
|
return
|
|
418
437
|
}
|
|
419
438
|
|
|
439
|
+
this.logger.agentLoop(`iteration=${this.middlewareCtx.iteration}`, {
|
|
440
|
+
iteration: this.middlewareCtx.iteration,
|
|
441
|
+
})
|
|
442
|
+
|
|
420
443
|
await this.beginCycle()
|
|
421
444
|
|
|
422
445
|
if (this.cyclePhase === 'processText') {
|
|
@@ -438,6 +461,10 @@ class TextEngine<
|
|
|
438
461
|
this.endCycle()
|
|
439
462
|
} while (this.shouldContinue())
|
|
440
463
|
|
|
464
|
+
this.logger.agentLoop('run finished', {
|
|
465
|
+
finishReason: this.lastFinishReason,
|
|
466
|
+
})
|
|
467
|
+
|
|
441
468
|
// Call terminal onFinish hook (skip when waiting for client — stream is paused, not finished)
|
|
442
469
|
if (!this.terminalHookCalled && this.toolPhase !== 'wait') {
|
|
443
470
|
this.terminalHookCalled = true
|
|
@@ -460,6 +487,10 @@ class TextEngine<
|
|
|
460
487
|
})
|
|
461
488
|
} else {
|
|
462
489
|
// Genuine error — call onError
|
|
490
|
+
this.logger.errors('chat run failed', {
|
|
491
|
+
error,
|
|
492
|
+
conversationId: this.middlewareCtx.conversationId,
|
|
493
|
+
})
|
|
463
494
|
await this.middlewareRunner.runOnError(this.middlewareCtx, {
|
|
464
495
|
error,
|
|
465
496
|
duration: Date.now() - this.streamStartTime,
|
|
@@ -555,6 +586,18 @@ class TextEngine<
|
|
|
555
586
|
|
|
556
587
|
this.middlewareCtx.phase = 'modelStream'
|
|
557
588
|
|
|
589
|
+
const providerName =
|
|
590
|
+
(this.adapter as { provider?: string }).provider ?? this.adapter.name
|
|
591
|
+
this.logger.request(
|
|
592
|
+
`activity=chat provider=${providerName} model=${this.params.model} messages=${this.messages.length} tools=${this.tools.length} stream=true`,
|
|
593
|
+
{
|
|
594
|
+
provider: providerName,
|
|
595
|
+
model: this.params.model,
|
|
596
|
+
messageCount: this.messages.length,
|
|
597
|
+
toolCount: this.tools.length,
|
|
598
|
+
},
|
|
599
|
+
)
|
|
600
|
+
|
|
558
601
|
for await (const chunk of this.adapter.chatStream({
|
|
559
602
|
model: this.params.model,
|
|
560
603
|
messages: this.messages,
|
|
@@ -566,6 +609,7 @@ class TextEngine<
|
|
|
566
609
|
request: this.effectiveRequest,
|
|
567
610
|
modelOptions,
|
|
568
611
|
systemPrompts: this.systemPrompts,
|
|
612
|
+
logger: this.logger,
|
|
569
613
|
threadId: this.threadId,
|
|
570
614
|
runId: this.runIdOverride,
|
|
571
615
|
})) {
|
|
@@ -585,6 +629,7 @@ class TextEngine<
|
|
|
585
629
|
chunk,
|
|
586
630
|
)
|
|
587
631
|
for (const outputChunk of outputChunks) {
|
|
632
|
+
this.logger.output(`type=${outputChunk.type}`, { chunk: outputChunk })
|
|
588
633
|
yield outputChunk
|
|
589
634
|
this.middlewareCtx.chunkIndex++
|
|
590
635
|
}
|
|
@@ -741,6 +786,10 @@ class TextEngine<
|
|
|
741
786
|
(eventName, data) => this.createCustomEventChunk(eventName, data),
|
|
742
787
|
{
|
|
743
788
|
onBeforeToolCall: async (toolCall, tool, args) => {
|
|
789
|
+
this.logger.tools(`phase=before name=${toolCall.function.name}`, {
|
|
790
|
+
name: toolCall.function.name,
|
|
791
|
+
args,
|
|
792
|
+
})
|
|
744
793
|
const hookCtx = {
|
|
745
794
|
toolCall,
|
|
746
795
|
tool,
|
|
@@ -754,6 +803,10 @@ class TextEngine<
|
|
|
754
803
|
)
|
|
755
804
|
},
|
|
756
805
|
onAfterToolCall: async (info) => {
|
|
806
|
+
this.logger.tools(`phase=after name=${info.toolName}`, {
|
|
807
|
+
name: info.toolName,
|
|
808
|
+
result: info.result,
|
|
809
|
+
})
|
|
757
810
|
await this.middlewareRunner.runOnAfterToolCall(
|
|
758
811
|
this.middlewareCtx,
|
|
759
812
|
info,
|
|
@@ -894,6 +947,10 @@ class TextEngine<
|
|
|
894
947
|
(eventName, data) => this.createCustomEventChunk(eventName, data),
|
|
895
948
|
{
|
|
896
949
|
onBeforeToolCall: async (toolCall, tool, args) => {
|
|
950
|
+
this.logger.tools(`phase=before name=${toolCall.function.name}`, {
|
|
951
|
+
name: toolCall.function.name,
|
|
952
|
+
args,
|
|
953
|
+
})
|
|
897
954
|
const hookCtx = {
|
|
898
955
|
toolCall,
|
|
899
956
|
tool,
|
|
@@ -907,6 +964,10 @@ class TextEngine<
|
|
|
907
964
|
)
|
|
908
965
|
},
|
|
909
966
|
onAfterToolCall: async (info) => {
|
|
967
|
+
this.logger.tools(`phase=after name=${info.toolName}`, {
|
|
968
|
+
name: info.toolName,
|
|
969
|
+
result: info.result,
|
|
970
|
+
})
|
|
910
971
|
await this.middlewareRunner.runOnAfterToolCall(
|
|
911
972
|
this.middlewareCtx,
|
|
912
973
|
info,
|
|
@@ -1491,18 +1552,22 @@ export function chat<
|
|
|
1491
1552
|
async function* runStreamingText(
|
|
1492
1553
|
options: TextActivityOptions<AnyTextAdapter, undefined, true>,
|
|
1493
1554
|
): AsyncIterable<StreamChunk> {
|
|
1494
|
-
const { adapter, middleware, context, ...textOptions } = options
|
|
1555
|
+
const { adapter, middleware, context, debug, ...textOptions } = options
|
|
1495
1556
|
const model = adapter.model
|
|
1557
|
+
const logger = resolveDebugOption(debug)
|
|
1496
1558
|
|
|
1497
|
-
const engine = new TextEngine(
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1559
|
+
const engine = new TextEngine(
|
|
1560
|
+
{
|
|
1561
|
+
adapter,
|
|
1562
|
+
params: { ...textOptions, model, logger } as TextOptions<
|
|
1563
|
+
Record<string, any>,
|
|
1564
|
+
Record<string, any>
|
|
1565
|
+
>,
|
|
1566
|
+
middleware,
|
|
1567
|
+
context,
|
|
1568
|
+
},
|
|
1569
|
+
logger,
|
|
1570
|
+
)
|
|
1506
1571
|
|
|
1507
1572
|
for await (const chunk of engine.run()) {
|
|
1508
1573
|
yield chunk
|
|
@@ -1533,23 +1598,28 @@ function runNonStreamingText(
|
|
|
1533
1598
|
async function runAgenticStructuredOutput<TSchema extends SchemaInput>(
|
|
1534
1599
|
options: TextActivityOptions<AnyTextAdapter, TSchema, boolean>,
|
|
1535
1600
|
): Promise<InferSchemaType<TSchema>> {
|
|
1536
|
-
const { adapter, outputSchema, middleware, context, ...textOptions } =
|
|
1601
|
+
const { adapter, outputSchema, middleware, context, debug, ...textOptions } =
|
|
1602
|
+
options
|
|
1537
1603
|
const model = adapter.model
|
|
1604
|
+
const logger = resolveDebugOption(debug)
|
|
1538
1605
|
|
|
1539
1606
|
if (!outputSchema) {
|
|
1540
1607
|
throw new Error('outputSchema is required for structured output')
|
|
1541
1608
|
}
|
|
1542
1609
|
|
|
1543
1610
|
// Create the engine and run the agentic loop
|
|
1544
|
-
const engine = new TextEngine(
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1611
|
+
const engine = new TextEngine(
|
|
1612
|
+
{
|
|
1613
|
+
adapter,
|
|
1614
|
+
params: { ...textOptions, model, logger } as TextOptions<
|
|
1615
|
+
Record<string, unknown>,
|
|
1616
|
+
Record<string, unknown>
|
|
1617
|
+
>,
|
|
1618
|
+
middleware,
|
|
1619
|
+
context,
|
|
1620
|
+
},
|
|
1621
|
+
logger,
|
|
1622
|
+
)
|
|
1553
1623
|
|
|
1554
1624
|
// Consume the stream to run the agentic loop
|
|
1555
1625
|
for await (const _chunk of engine.run()) {
|
|
@@ -1573,6 +1643,17 @@ async function runAgenticStructuredOutput<TSchema extends SchemaInput>(
|
|
|
1573
1643
|
throw new Error('Failed to convert output schema to JSON Schema')
|
|
1574
1644
|
}
|
|
1575
1645
|
|
|
1646
|
+
const providerName =
|
|
1647
|
+
(adapter as { provider?: string }).provider ?? adapter.name
|
|
1648
|
+
logger.request(
|
|
1649
|
+
`activity=chat-structured provider=${providerName} model=${model} messages=${finalMessages.length}`,
|
|
1650
|
+
{
|
|
1651
|
+
provider: providerName,
|
|
1652
|
+
model,
|
|
1653
|
+
messageCount: finalMessages.length,
|
|
1654
|
+
},
|
|
1655
|
+
)
|
|
1656
|
+
|
|
1576
1657
|
// Call the adapter's structured output method with the conversation context
|
|
1577
1658
|
// The adapter receives JSON Schema and can apply vendor-specific patches
|
|
1578
1659
|
const result = await adapter.structuredOutput({
|
|
@@ -1580,6 +1661,7 @@ async function runAgenticStructuredOutput<TSchema extends SchemaInput>(
|
|
|
1580
1661
|
...structuredTextOptions,
|
|
1581
1662
|
model,
|
|
1582
1663
|
messages: finalMessages,
|
|
1664
|
+
logger,
|
|
1583
1665
|
},
|
|
1584
1666
|
outputSchema: jsonSchema,
|
|
1585
1667
|
})
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { aiEventClient } from '@tanstack/ai-event-client'
|
|
2
2
|
import type { StreamChunk } from '../../../types'
|
|
3
|
+
import type { InternalLogger } from '../../../logger/internal-logger'
|
|
3
4
|
import type {
|
|
4
5
|
AbortInfo,
|
|
5
6
|
AfterToolCallInfo,
|
|
@@ -36,9 +37,14 @@ function instrumentCtx(ctx: ChatMiddlewareContext) {
|
|
|
36
37
|
*/
|
|
37
38
|
export class MiddlewareRunner {
|
|
38
39
|
private readonly middlewares: ReadonlyArray<ChatMiddleware>
|
|
40
|
+
private readonly logger: InternalLogger
|
|
39
41
|
|
|
40
|
-
constructor(
|
|
42
|
+
constructor(
|
|
43
|
+
middlewares: ReadonlyArray<ChatMiddleware>,
|
|
44
|
+
logger: InternalLogger,
|
|
45
|
+
) {
|
|
41
46
|
this.middlewares = middlewares
|
|
47
|
+
this.logger = logger
|
|
42
48
|
}
|
|
43
49
|
|
|
44
50
|
get hasMiddleware(): boolean {
|
|
@@ -63,6 +69,15 @@ export class MiddlewareRunner {
|
|
|
63
69
|
const hasTransform = result !== undefined && result !== null
|
|
64
70
|
if (hasTransform) {
|
|
65
71
|
current = { ...current, ...result }
|
|
72
|
+
if (!skip) {
|
|
73
|
+
this.logger.config(
|
|
74
|
+
`middleware=${mw.name ?? 'unnamed'} keys=${Object.keys(result as object).join(',')}`,
|
|
75
|
+
{
|
|
76
|
+
middleware: mw.name ?? 'unnamed',
|
|
77
|
+
changes: result,
|
|
78
|
+
},
|
|
79
|
+
)
|
|
80
|
+
}
|
|
66
81
|
}
|
|
67
82
|
if (!skip) {
|
|
68
83
|
const base = instrumentCtx(ctx)
|
|
@@ -98,6 +113,10 @@ export class MiddlewareRunner {
|
|
|
98
113
|
const start = Date.now()
|
|
99
114
|
await mw.onStart(ctx)
|
|
100
115
|
if (!skip) {
|
|
116
|
+
this.logger.middleware(
|
|
117
|
+
`hook=onStart middleware=${mw.name ?? 'unnamed'}`,
|
|
118
|
+
{ middleware: mw.name ?? 'unnamed', hook: 'onStart' },
|
|
119
|
+
)
|
|
101
120
|
aiEventClient.emit('middleware:hook:executed', {
|
|
102
121
|
...instrumentCtx(ctx),
|
|
103
122
|
middlewareName: mw.name || 'unnamed',
|
|
@@ -134,10 +153,24 @@ export class MiddlewareRunner {
|
|
|
134
153
|
for (const c of chunks) {
|
|
135
154
|
// Cast: @ag-ui/core Zod passthrough types prevent direct `.type` access
|
|
136
155
|
const chunkType = (c as StreamChunk & { type: string }).type
|
|
156
|
+
if (!skip) {
|
|
157
|
+
this.logger.middleware(
|
|
158
|
+
`hook=onChunk middleware=${mw.name ?? 'unnamed'} in=${chunkType}`,
|
|
159
|
+
{ middleware: mw.name ?? 'unnamed', hook: 'onChunk', in: c },
|
|
160
|
+
)
|
|
161
|
+
}
|
|
137
162
|
const result = await mw.onChunk(ctx, c)
|
|
138
163
|
if (result === null) {
|
|
139
164
|
// Drop this chunk
|
|
140
165
|
if (!skip) {
|
|
166
|
+
this.logger.middleware(
|
|
167
|
+
`hook=onChunk middleware=${mw.name ?? 'unnamed'} in=${chunkType} out=<dropped>`,
|
|
168
|
+
{
|
|
169
|
+
middleware: mw.name ?? 'unnamed',
|
|
170
|
+
hook: 'onChunk',
|
|
171
|
+
dropped: true,
|
|
172
|
+
},
|
|
173
|
+
)
|
|
141
174
|
aiEventClient.emit('middleware:chunk:transformed', {
|
|
142
175
|
...instrumentCtx(ctx),
|
|
143
176
|
middlewareName: mw.name || 'unnamed',
|
|
@@ -154,6 +187,15 @@ export class MiddlewareRunner {
|
|
|
154
187
|
// Expand
|
|
155
188
|
nextChunks.push(...result)
|
|
156
189
|
if (!skip) {
|
|
190
|
+
this.logger.middleware(
|
|
191
|
+
`hook=onChunk middleware=${mw.name ?? 'unnamed'} in=${chunkType} out=[${result.map((r: StreamChunk) => (r as StreamChunk & { type: string }).type).join(',')}]`,
|
|
192
|
+
{
|
|
193
|
+
middleware: mw.name ?? 'unnamed',
|
|
194
|
+
hook: 'onChunk',
|
|
195
|
+
in: c,
|
|
196
|
+
out: result,
|
|
197
|
+
},
|
|
198
|
+
)
|
|
157
199
|
aiEventClient.emit('middleware:chunk:transformed', {
|
|
158
200
|
...instrumentCtx(ctx),
|
|
159
201
|
middlewareName: mw.name || 'unnamed',
|
|
@@ -166,6 +208,15 @@ export class MiddlewareRunner {
|
|
|
166
208
|
// Replace
|
|
167
209
|
nextChunks.push(result)
|
|
168
210
|
if (!skip) {
|
|
211
|
+
this.logger.middleware(
|
|
212
|
+
`hook=onChunk middleware=${mw.name ?? 'unnamed'} in=${chunkType} out=${(result as StreamChunk & { type: string }).type}`,
|
|
213
|
+
{
|
|
214
|
+
middleware: mw.name ?? 'unnamed',
|
|
215
|
+
hook: 'onChunk',
|
|
216
|
+
in: c,
|
|
217
|
+
out: result,
|
|
218
|
+
},
|
|
219
|
+
)
|
|
169
220
|
aiEventClient.emit('middleware:chunk:transformed', {
|
|
170
221
|
...instrumentCtx(ctx),
|
|
171
222
|
middlewareName: mw.name || 'unnamed',
|
|
@@ -197,6 +248,10 @@ export class MiddlewareRunner {
|
|
|
197
248
|
const decision = await mw.onBeforeToolCall(ctx, hookCtx)
|
|
198
249
|
const hasTransform = decision !== undefined && decision !== null
|
|
199
250
|
if (!skip) {
|
|
251
|
+
this.logger.middleware(
|
|
252
|
+
`hook=onBeforeToolCall middleware=${mw.name ?? 'unnamed'}`,
|
|
253
|
+
{ middleware: mw.name ?? 'unnamed', hook: 'onBeforeToolCall' },
|
|
254
|
+
)
|
|
200
255
|
aiEventClient.emit('middleware:hook:executed', {
|
|
201
256
|
...instrumentCtx(ctx),
|
|
202
257
|
middlewareName: mw.name || 'unnamed',
|
|
@@ -227,6 +282,10 @@ export class MiddlewareRunner {
|
|
|
227
282
|
const start = Date.now()
|
|
228
283
|
await mw.onAfterToolCall(ctx, info)
|
|
229
284
|
if (!skip) {
|
|
285
|
+
this.logger.middleware(
|
|
286
|
+
`hook=onAfterToolCall middleware=${mw.name ?? 'unnamed'}`,
|
|
287
|
+
{ middleware: mw.name ?? 'unnamed', hook: 'onAfterToolCall' },
|
|
288
|
+
)
|
|
230
289
|
aiEventClient.emit('middleware:hook:executed', {
|
|
231
290
|
...instrumentCtx(ctx),
|
|
232
291
|
middlewareName: mw.name || 'unnamed',
|
|
@@ -253,6 +312,10 @@ export class MiddlewareRunner {
|
|
|
253
312
|
const start = Date.now()
|
|
254
313
|
await mw.onUsage(ctx, usage)
|
|
255
314
|
if (!skip) {
|
|
315
|
+
this.logger.middleware(
|
|
316
|
+
`hook=onUsage middleware=${mw.name ?? 'unnamed'}`,
|
|
317
|
+
{ middleware: mw.name ?? 'unnamed', hook: 'onUsage' },
|
|
318
|
+
)
|
|
256
319
|
aiEventClient.emit('middleware:hook:executed', {
|
|
257
320
|
...instrumentCtx(ctx),
|
|
258
321
|
middlewareName: mw.name || 'unnamed',
|
|
@@ -279,6 +342,10 @@ export class MiddlewareRunner {
|
|
|
279
342
|
const start = Date.now()
|
|
280
343
|
await mw.onFinish(ctx, info)
|
|
281
344
|
if (!skip) {
|
|
345
|
+
this.logger.middleware(
|
|
346
|
+
`hook=onFinish middleware=${mw.name ?? 'unnamed'}`,
|
|
347
|
+
{ middleware: mw.name ?? 'unnamed', hook: 'onFinish' },
|
|
348
|
+
)
|
|
282
349
|
aiEventClient.emit('middleware:hook:executed', {
|
|
283
350
|
...instrumentCtx(ctx),
|
|
284
351
|
middlewareName: mw.name || 'unnamed',
|
|
@@ -302,6 +369,10 @@ export class MiddlewareRunner {
|
|
|
302
369
|
const start = Date.now()
|
|
303
370
|
await mw.onAbort(ctx, info)
|
|
304
371
|
if (!skip) {
|
|
372
|
+
this.logger.middleware(
|
|
373
|
+
`hook=onAbort middleware=${mw.name ?? 'unnamed'}`,
|
|
374
|
+
{ middleware: mw.name ?? 'unnamed', hook: 'onAbort' },
|
|
375
|
+
)
|
|
305
376
|
aiEventClient.emit('middleware:hook:executed', {
|
|
306
377
|
...instrumentCtx(ctx),
|
|
307
378
|
middlewareName: mw.name || 'unnamed',
|
|
@@ -325,6 +396,10 @@ export class MiddlewareRunner {
|
|
|
325
396
|
const start = Date.now()
|
|
326
397
|
await mw.onError(ctx, info)
|
|
327
398
|
if (!skip) {
|
|
399
|
+
this.logger.middleware(
|
|
400
|
+
`hook=onError middleware=${mw.name ?? 'unnamed'}`,
|
|
401
|
+
{ middleware: mw.name ?? 'unnamed', hook: 'onError' },
|
|
402
|
+
)
|
|
328
403
|
aiEventClient.emit('middleware:hook:executed', {
|
|
329
404
|
...instrumentCtx(ctx),
|
|
330
405
|
middlewareName: mw.name || 'unnamed',
|
|
@@ -352,6 +427,10 @@ export class MiddlewareRunner {
|
|
|
352
427
|
const start = Date.now()
|
|
353
428
|
await mw.onIteration(ctx, info)
|
|
354
429
|
if (!skip) {
|
|
430
|
+
this.logger.middleware(
|
|
431
|
+
`hook=onIteration middleware=${mw.name ?? 'unnamed'}`,
|
|
432
|
+
{ middleware: mw.name ?? 'unnamed', hook: 'onIteration' },
|
|
433
|
+
)
|
|
355
434
|
aiEventClient.emit('middleware:hook:executed', {
|
|
356
435
|
...instrumentCtx(ctx),
|
|
357
436
|
middlewareName: mw.name || 'unnamed',
|
|
@@ -379,6 +458,10 @@ export class MiddlewareRunner {
|
|
|
379
458
|
const start = Date.now()
|
|
380
459
|
await mw.onToolPhaseComplete(ctx, info)
|
|
381
460
|
if (!skip) {
|
|
461
|
+
this.logger.middleware(
|
|
462
|
+
`hook=onToolPhaseComplete middleware=${mw.name ?? 'unnamed'}`,
|
|
463
|
+
{ middleware: mw.name ?? 'unnamed', hook: 'onToolPhaseComplete' },
|
|
464
|
+
)
|
|
382
465
|
aiEventClient.emit('middleware:hook:executed', {
|
|
383
466
|
...instrumentCtx(ctx),
|
|
384
467
|
middlewareName: mw.name || 'unnamed',
|