@cartanova/qgrid-ai-sdk 2.19.2 → 2.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.ko.md +73 -12
- package/README.md +73 -12
- package/dist/index.d.ts +6 -2
- package/dist/index.js +3 -8
- package/dist/realtime-BgodcUud.js +309 -0
- package/dist/realtime-_tFFhM56.d.ts +70 -0
- package/dist/realtime.d.ts +2 -0
- package/dist/realtime.js +1 -0
- package/package.json +7 -3
package/README.ko.md
CHANGED
|
@@ -347,8 +347,9 @@ qgrid는 `grok/*` 모델을 qgrid 대시보드에 등록한 xAI 구독 토큰(
|
|
|
347
347
|
| `qgrid(id)` | `grok/grok-4.7`(카탈로그 기본 모델), `grok/grok-4.7-build-fast`, `grok/grok-4.6`, `grok/grok-4.5`, 그 밖의 `grok/<id>` | `generateText`, `streamText` |
|
|
348
348
|
| `qgrid.image(id)` | `grok/grok-imagine-image`, `grok/grok-imagine-image-2.0` | `generateImage` |
|
|
349
349
|
| `qgrid.video(id)` | `grok/grok-imagine-video`, `grok/grok-imagine-video-1.5`, `grok/grok-imagine-video-1.5-lite` | `experimental_generateVideo` |
|
|
350
|
-
| `qgrid.transcription(id)` | `grok/grok-voice-transcribe-2.0`; OpenAI 구독: `openai/gpt-
|
|
350
|
+
| `qgrid.transcription(id)` | `grok/grok-voice-transcribe-2.0`; OpenAI 구독: `openai/gpt-transcribe`(권장), `openai/gpt-live-transcribe`, `openai/gpt-realtime-whisper`; `openai/gpt-4o-mini-transcribe`, `openai/gpt-4o-transcribe`, `openai/whisper-1`은 OpenAI가 종료하는 2027-02-26까지 | `transcribe` |
|
|
351
351
|
| `qgrid.speech(id)` | `grok/grok-tts`(ID는 request log 라벨로만 쓰임); OpenAI 구독: `openai/gpt-realtime-2.1-mini`, `openai/gpt-realtime-2.1` | `generateSpeech` |
|
|
352
|
+
| `qgrid.realtime(id)`(2.20.0+; 브라우저: `@cartanova/qgrid-ai-sdk/realtime`의 `createQgridRealtime`) | `openai/gpt-live-1` | `experimental_useRealtime`(실험적, WebRTC) |
|
|
352
353
|
|
|
353
354
|
qgrid는 카탈로그에 없는 language model ID를 거부하지 않고 그대로 전달합니다(OpenAI와 같음). 판단은
|
|
354
355
|
subscription backend가 하며 다른 모델로 응답할 수 있으므로, 실제로 실행된 모델은 request log의 응답 모델(또는
|
|
@@ -465,7 +466,7 @@ import {
|
|
|
465
466
|
generateSpeech,
|
|
466
467
|
transcribe,
|
|
467
468
|
} from "ai";
|
|
468
|
-
import { qgrid } from "@cartanova/qgrid-ai-sdk";
|
|
469
|
+
import { qgrid, type QgridGrokVoice } from "@cartanova/qgrid-ai-sdk";
|
|
469
470
|
|
|
470
471
|
const transcript = await transcribe({
|
|
471
472
|
model: qgrid.transcription("grok/grok-voice-transcribe-2.0"),
|
|
@@ -478,7 +479,7 @@ transcript.durationInSeconds;
|
|
|
478
479
|
const speech = await generateSpeech({
|
|
479
480
|
model: qgrid.speech("grok/grok-tts"),
|
|
480
481
|
text: "Your build is ready.",
|
|
481
|
-
voice: "ara",
|
|
482
|
+
voice: "ara" satisfies QgridGrokVoice,
|
|
482
483
|
language: "en",
|
|
483
484
|
outputFormat: "mp3",
|
|
484
485
|
});
|
|
@@ -486,7 +487,7 @@ speech.audio.uint8Array;
|
|
|
486
487
|
```
|
|
487
488
|
|
|
488
489
|
- `transcribe`는 bytes로 오디오 형식을 감지하고 실패하면 `audio/wav`를 씁니다. 서버는 `audio/*`가 아닌 값을 거부합니다. transcript는 들린 단어마다 segment 하나를 담습니다(`segments: [{ text, startSecond, endSecond }]`). Grok은 직접 시간을 재고(2.17.0+), OpenAI는 서버가 내장 정렬기로 transcript를 오디오에 맞춥니다(서버 2.19.0+). 정렬에는 클립 길이의 약 5%만큼 시간이 더 듭니다. 서버에 정렬기가 없거나 클립이 정렬 한도(빽빽한 음성 기준 약 4분)를 넘으면 OpenAI segment는 비어 있습니다.
|
|
489
|
-
- 음성 합성 텍스트는 최대 15,000자입니다. `voice`, `language`, `outputFormat`을 지정하지 않으면 서버 기본값 `ara`, `en`, `mp3`가 쓰입니다. 구독 토큰에 보이는 보이스 28종(여성 9, 남성 19. 예: `ara`, `eve`, `rex`)은 2026-11-09에 모두 영어·mp3로 live 생성했습니다. 다른 language(`@ai-sdk/xai`의 `auto` 포함)와 codec은 그대로 전달되지만 미검증입니다. xAI 목록의 음성 태그(`[pause]`, `[laugh]`, `<whisper>...</whisper>` 등)는 2026-11 확인에서 글자로 읽히지 않았고, `[deadpan]` 같은 ElevenLabs식 감정 태그는 그 목록에 없습니다. `speed`는 말하기 속도입니다(1이 보통, 0.7-1.5 밖의 값은 서버가 거부). 서버가 돌려준 오디오 MIME 타입은 `providerMetadata.qgrid.mediaType`에 있고, AI SDK의 `speech.audio.mediaType`은 bytes에서 추정한 값입니다.
|
|
490
|
+
- 음성 합성 텍스트는 최대 15,000자입니다. `voice`, `language`, `outputFormat`을 지정하지 않으면 서버 기본값 `ara`, `en`, `mp3`가 쓰입니다. 구독 토큰에 보이는 보이스 28종(여성 9, 남성 19. 예: `ara`, `eve`, `rex`. `QGRID_GROK_VOICES`, 타입 `QgridGrokVoice`로 export)은 2026-11-09에 모두 영어·mp3로 live 생성했습니다. 다른 language(`@ai-sdk/xai`의 `auto` 포함)와 codec은 그대로 전달되지만 미검증입니다. xAI 목록의 음성 태그(`[pause]`, `[laugh]`, `<whisper>...</whisper>` 등)는 2026-11 확인에서 글자로 읽히지 않았고, `[deadpan]` 같은 ElevenLabs식 감정 태그는 그 목록에 없습니다. `speed`는 말하기 속도입니다(1이 보통, 0.7-1.5 밖의 값은 서버가 거부). 서버가 돌려준 오디오 MIME 타입은 `providerMetadata.qgrid.mediaType`에 있고, AI SDK의 `speech.audio.mediaType`은 bytes에서 추정한 값입니다.
|
|
490
491
|
- 단어 타이밍: `providerOptions: { xai: { withTimestamps: true } }`(2.16.0+, Grok) 또는 `providerOptions: { qgrid: { withTimestamps: true } }`(2.17.0+, Grok·OpenAI)를 주면 글자별 타이밍이 `providerMetadata.qgrid.audioTimestamps`에 담깁니다. `graphChars`는 입력 글자(공백·문장부호·음성 태그 포함), `graphTimes`는 같은 인덱스의 `[start, end]` 초이고, 전체 길이는 `providerMetadata.qgrid.duration`입니다. `@ai-sdk/xai`가 `providerMetadata.xai`에 담는 모양과 같습니다. `providerMetadata.qgrid.timestampsSource`는 Grok이면 `"provider"`(Grok이 직접 잰 값), OpenAI면 `"aligned"`입니다. OpenAI는 서버가 내장 정렬기로 생성된 오디오에 원문을 맞춰 만듭니다(서버 2.19.0+, 이전 서버는 Grok STT를 거쳤음). 정렬기는 qgrid Docker 이미지에 들어 있고 npm이나 pnpm 설치에서는 `qgrid --install-aligner`로 설치합니다. 정렬기가 없는 서버와 정렬기 한도(빽빽한 음성 기준 약 4분)를 넘는 긴 음성은 `timestampsError`를 돌려줍니다(오류에 그 대본의 한도가 나옴). 타이밍 단계가 실패해도 오디오는 돌려주고, 메타데이터에 `timestampsError`와 AI SDK warning을 남깁니다. 텍스트 정규화로 풀린 토큰은 첫 글자가 전체 구간을 가집니다. Grok mp3에는 앞부분 무음 정보(gapless 태그)가 없어 재생기가 인코더 지연을 그대로 재생하므로, 서버 2.19.0부터 qgrid가 Grok 타이밍에 그 46ms를 더해 실제 재생과 맞춥니다(`@ai-sdk/xai`의 원래 값보다 46ms 늦음).
|
|
491
492
|
|
|
492
493
|
### OpenAI 음성 인식과 합성 (ChatGPT 구독)
|
|
@@ -494,8 +495,10 @@ speech.audio.uint8Array;
|
|
|
494
495
|
`openai/` 음성 ID는 qgrid가 Codex에 쓰는 ChatGPT 구독 토큰으로 OpenAI realtime 세션을 사용합니다. 음성 인식은 Codex의 `/realtime/calls` 흐름을, 음성 합성은 텍스트를 그대로 읽는 realtime 대화 모델을 씁니다(구독에는 `gpt-4o-mini-tts`, `tts-1`이 없습니다). 옵션은 `providerOptions.openai` 아래 `@ai-sdk/openai` 이름을 따르고, `providerOptions.qgrid.tokenName`은 `openai/` 접두사가 필요합니다.
|
|
495
496
|
|
|
496
497
|
```typescript
|
|
498
|
+
import { type QgridOpenAISpeechVoice } from "@cartanova/qgrid-ai-sdk";
|
|
499
|
+
|
|
497
500
|
const transcript = await transcribe({
|
|
498
|
-
model: qgrid.transcription("openai/gpt-
|
|
501
|
+
model: qgrid.transcription("openai/gpt-transcribe"),
|
|
499
502
|
audio: await readFile("meeting.wav"), // PCM16 WAV 또는 raw PCM만
|
|
500
503
|
providerOptions: { openai: { language: "en", prompt: "qgrid, Codex" } },
|
|
501
504
|
});
|
|
@@ -503,7 +506,7 @@ const transcript = await transcribe({
|
|
|
503
506
|
const speech = await generateSpeech({
|
|
504
507
|
model: qgrid.speech("openai/gpt-realtime-2.1-mini"),
|
|
505
508
|
text: "Your build is ready.",
|
|
506
|
-
voice: "marin",
|
|
509
|
+
voice: "marin" satisfies QgridOpenAISpeechVoice,
|
|
507
510
|
outputFormat: "wav",
|
|
508
511
|
providerOptions: { openai: { instructions: "Calm and brisk." } },
|
|
509
512
|
});
|
|
@@ -511,18 +514,75 @@ const speech = await generateSpeech({
|
|
|
511
514
|
|
|
512
515
|
| 모델 | 그대로 전달(`@ai-sdk/openai` 이름) | 변환 | 미지원(경고) |
|
|
513
516
|
| --- | --- | --- | --- |
|
|
514
|
-
| Transcription | `openai.language
|
|
515
|
-
| Speech | `voice`(기본 `marin`; Codex v2 목록 `alloy ash ballad coral echo sage shimmer verse marin cedar`, 그 외는 HTTP 400), `instructions` / `openai.instructions`(원문 그대로 읽기 지시문 뒤에 붙는 스타일 안내) | `outputFormat`은 `wav`(기본, PCM16 24 kHz) 또는 `pcm`; 그 외 값은 `@ai-sdk/openai`가 `mp3`로 되돌리듯 경고와 함께 `wav`로 바뀝니다 | `language`, `speed`, `openai.speed` |
|
|
517
|
+
| Transcription | `openai.language`(`gpt-transcribe`에는 목록 `languages`로 보냄), `openai.prompt`(`gpt-realtime-whisper`는 HTTP 400으로 거부, `whisper-1`은 받지만 업스트림이 무시) | 입력은 PCM16 WAV(`audio/wav`) 또는 raw PCM(`audio/pcm;rate=…;channels=…`)만 가능하며 mp3/webm/ogg는 디코더가 추가되기 전까지 서버가 HTTP 400으로 거부합니다. `durationInSeconds`는 보낸 오디오에서 계산하고 `language`는 요청값을 그대로 돌려줍니다(realtime 세션은 둘 다 보고하지 않음). `segments`에는 정렬한 단어 타이밍이 담깁니다(서버 2.19.0+) | `openai.temperature`, `openai.include`, `openai.timestampGranularities`, `openai.responseFormat`, `openai.chunkingStrategy`, `openai.streaming` |
|
|
518
|
+
| Speech | `voice`(기본 `marin`; Codex v2 목록 `alloy ash ballad coral echo sage shimmer verse marin cedar`, `QGRID_OPENAI_SPEECH_VOICES`로 export; 그 외는 HTTP 400), `instructions` / `openai.instructions`(원문 그대로 읽기 지시문 뒤에 붙는 스타일 안내) | `outputFormat`은 `wav`(기본, PCM16 24 kHz) 또는 `pcm`; 그 외 값은 `@ai-sdk/openai`가 `mp3`로 되돌리듯 경고와 함께 `wav`로 바뀝니다 | `language`, `speed`, `openai.speed` |
|
|
516
519
|
|
|
517
520
|
- 음성 인식은 먼저 Codex realtime call에서 25초 예산으로 실행됩니다(call은 생성 약 30초 뒤 정리됨). 그 안에 전사가 오지 않으면 서버가 직접 realtime 전사 소켓으로 1회 재시도합니다(최대 60초). 19 MiB 인라인 제한에는 24 kHz mono WAV가 약 6.9분 들어갑니다. 6.7분 영어 클립은 첫 경로에서 약 20초에 끝났고, transcript가 더 길게 나온 6.4분 클립은 기한을 넘겨 재시도 경로에서 52초에 끝났습니다.
|
|
518
521
|
- 음성 합성은 TTS 모델이 아니라 텍스트를 읽는 대화 모델입니다. qgrid는 `<script>` 태그 안의 텍스트만 읽고 답하지 말라고 지시합니다. 원문 그대로 읽는지는 짧은 영어 문장과 질문형 영어 캐릭터 대사에서 검증했습니다. 모델이 실제로 말한 transcript를 요청 로그에 함께 남겨 비교할 수 있지만, 한영 혼합 텍스트에서는 transcript가 일치해도 영어 음성을 알아듣기 어려운 경우가 있었습니다.
|
|
522
|
+
- `openai/gpt-transcribe`를 쓰세요. OpenAI는 `gpt-4o-mini-transcribe`, `gpt-4o-transcribe`, `whisper-1`을 2027-02-26에 종료합니다. 2026-11-10 합성 음성 벤치에서 `gpt-transcribe`는 영어 108클립에서 `gpt-4o-transcribe`와 같은 수준이었고(차이는 "21st"를 "twenty-first"로 쓰는 숫자 표기였음), 한영 혼합 18클립의 글자 오류율을 14.5%에서 4.4%로 낮췄습니다. `gpt-4o-transcribe`가 실패한 229초 클립도 끝까지 받아 적었고, `gpt-4o-transcribe`가 없는 말을 지어낸 무음과 잡음에는 아무것도 돌려주지 않았습니다. 학습자의 문법 오류를 고쳐 적을 때가 있으므로("we eat chicken"을 "we ate chicken"으로), 받아쓴 결과만으로 문법을 채점하지 마세요.
|
|
519
523
|
- 모델별 동작은 서버 저장소의 `docs/diagnosis/2026-10-17-openai-realtime-audio-probe.md`에 기록되어 있습니다. 2026-10-17 기준 알려진 한계: `gpt-live-transcribe`, `gpt-realtime-whisper`는 60초 클립의 절반 정도만 반환하므로 짧은 발화에 쓰세요. `gpt-realtime-whisper`는 `openai.prompt`를 거부합니다(서버가 업스트림 호출 전에 HTTP 400). `whisper-1`은 `openai.prompt`를 받지만 무시합니다. 음성 합성은 `gpt-realtime-2.1-mini`가 모든 테스트 문장을 그대로 읽었고(13/13), `gpt-realtime-2.1`은 1회 의역(12/13)했습니다. `gpt-realtime-1.5`는 텍스트를 읽지 않고 답변하는 경우가 잦아(6/13) 2.18.0에서 제외했습니다. 2026-11-06에는 현재 프롬프트로 `gpt-realtime-2.1`과 `gpt-realtime-2.1-mini` 모두 질문형 캐릭터 대사 12/12를 그대로 읽었습니다(이전 프롬프트에서는 `gpt-realtime-2.1`이 7/12를 답변). 2026-11-09에 독립 STT 2종으로 영어 108건(6문장, 보이스 `marin` `cedar` `ballad`, 각 3회)을 판정한 결과 `gpt-realtime-2.1`은 54/54, `gpt-realtime-2.1-mini`는 53/54였고, 실패 1건은 "Sure, let me read that out loud for you exactly as written."으로 시작했습니다. 한영 혼합 문장은 두 모델 모두 끝까지 읽었습니다(각 9/9). 모델이 남기는 transcript에는 이런 서두가 나타나지 않으므로, 정확성이 중요하면 transcript 대신 STT나 `withTimestamps`(정렬이 덧붙은 말을 건너뜀)로 오디오를 확인하세요.
|
|
520
524
|
|
|
521
525
|
### OpenAI 실시간 음성 대화 (GPT-Live)
|
|
522
526
|
|
|
523
|
-
서버 2.19.2+는 ChatGPT 구독 토큰으로 `openai/gpt-live-1` 대화를 엽니다. GPT-Live는 듣는 동안에도 말하는 대화 모델이라 음성 합성과는 다릅니다.
|
|
527
|
+
서버 2.19.2+는 ChatGPT 구독 토큰으로 `openai/gpt-live-1` 대화를 엽니다. GPT-Live는 듣는 동안에도 말하는 대화 모델이라 음성 합성과는 다릅니다.
|
|
528
|
+
|
|
529
|
+
오디오는 qgrid를 지나지 않습니다. 브라우저의 WebRTC 피어가 SDP offer를 만들고, 앱 서버가 이를 `POST /api/qgrid/liveCall`로 보낸 뒤, 돌아온 SDP answer를 브라우저에 적용합니다. 그 뒤 오디오와 이벤트는 브라우저와 OpenAI가 직접 주고받습니다. qgrid는 대화 텍스트와 세션 오디오 시간을 로그에 남기고, 오디오는 저장하지 않습니다.
|
|
530
|
+
|
|
531
|
+
#### AI SDK 실험적 realtime 런타임으로 (SDK 2.20.0+)
|
|
532
|
+
|
|
533
|
+
`qgrid.realtime("openai/gpt-live-1")`은 AI SDK `Experimental_RealtimeModelV4`입니다. 마이크, 재생, WebRTC 피어, 데이터 채널은 AI SDK realtime 세션(`@ai-sdk/react`의 `experimental_useRealtime`, `ai` 7.0.102+)이 맡고, 앱 서버 라우트는 SDP만 교환합니다. AI SDK에서 실험적 API라 `ai` 버전에 따라 바뀔 수 있습니다.
|
|
534
|
+
|
|
535
|
+
```tsx
|
|
536
|
+
// 브라우저: Node 전용 코드가 없는 /realtime 진입점에서 import합니다.
|
|
537
|
+
import { createQgridRealtime } from "@cartanova/qgrid-ai-sdk/realtime";
|
|
538
|
+
import { experimental_useRealtime } from "@ai-sdk/react";
|
|
539
|
+
|
|
540
|
+
// 모듈 최상단에 둡니다. 훅은 model 객체가 바뀌면 세션을 새로 만듭니다.
|
|
541
|
+
const model = createQgridRealtime("openai/gpt-live-1");
|
|
542
|
+
|
|
543
|
+
function LiveConversation() {
|
|
544
|
+
// sessionConfig는 넘기지 않습니다. 보이스와 지시문은 서버 라우트가 정합니다.
|
|
545
|
+
const realtime = experimental_useRealtime({ model, api: { session: "/api/live" } });
|
|
546
|
+
|
|
547
|
+
return (
|
|
548
|
+
<>
|
|
549
|
+
<button onClick={() => realtime.connect()}>시작</button>
|
|
550
|
+
<button onClick={() => realtime.close()}>종료</button>
|
|
551
|
+
<p>{realtime.status}: {realtime.session?.usage?.seconds ?? 0}초</p>
|
|
552
|
+
{realtime.session?.transcripts.map((part, i) => (
|
|
553
|
+
<span key={i}>{part.speaker === "user" ? "🧑 " : ""}{part.delta}</span>
|
|
554
|
+
))}
|
|
555
|
+
</>
|
|
556
|
+
);
|
|
557
|
+
}
|
|
558
|
+
```
|
|
559
|
+
|
|
560
|
+
```typescript
|
|
561
|
+
// 앱 서버 라우트: 훅이 { sdp, sessionConfig }를 보내고 { sdp, sessionId }를 기대합니다.
|
|
562
|
+
import { qgrid, type QgridLiveVoice, type QgridRealtimeProviderOptions } from "@cartanova/qgrid-ai-sdk";
|
|
563
|
+
|
|
564
|
+
app.post("/api/live", async (req, res) => {
|
|
565
|
+
const answer = await qgrid.realtime("openai/gpt-live-1").doCreateWebRTCSession({
|
|
566
|
+
sdp: req.body.sdp,
|
|
567
|
+
// 서버에서 정합니다. 브라우저가 보낸 sessionConfig는 믿지 않습니다.
|
|
568
|
+
sessionConfig: {
|
|
569
|
+
voice: "cove" satisfies QgridLiveVoice,
|
|
570
|
+
instructions: "You are a friendly English tutor.",
|
|
571
|
+
providerOptions: {
|
|
572
|
+
qgrid: { delegation: { model: "openai/gpt-6-luna", webSearch: true } } satisfies QgridRealtimeProviderOptions,
|
|
573
|
+
},
|
|
574
|
+
},
|
|
575
|
+
});
|
|
576
|
+
res.json(answer); // { sdp, sessionId, requestLogId }
|
|
577
|
+
});
|
|
578
|
+
```
|
|
579
|
+
|
|
580
|
+
- `providerOptions.qgrid`(`QgridRealtimeProviderOptions`): `delegation`, `history`, `tokenName`, `logger`이며 아래 HTTP `params`와 같은 필드입니다. `sessionConfig.voice`와 `sessionConfig.instructions`는 `voice`, `instructions`로 보내고, 나머지 `sessionConfig` 필드는 보내지 않습니다. 이 값은 훅이 아니라 서버 호출에서 정하세요. 훅은 `sessionConfig` 객체가 바뀌면 세션을 새로 만들기 때문에, 호출부에 객체를 바로 쓰면 렌더할 때마다 세션이 다시 시작됩니다. `serverUrl`과 `projectName`은 `createQgridRealtime(id, config)` 또는 `QGRID_URL` / `QGRID_PROJECT_NAME`에서 가져옵니다.
|
|
581
|
+
- 음성 모델의 위임은 qgrid 서버가 답합니다. 그래서 각 `delegation-created` 이벤트는 `target: "provider"`로 표시되며, 앱이 돌려보낼 것은 없습니다. 세션은 `delegationMode`를 알리지 않습니다. AI SDK 런타임이 `"provider"`를 거부하고, `"client"`는 앱이 답해야 한다는 뜻이기 때문입니다.
|
|
582
|
+
- 지원하는 명령: `close()`(`session.close`를 보내고 최종 사용량을 기다림)와 `sendEvent({ type: "context-append", content, delegationId })`(음성이 쓸 텍스트 추가, 세션 전체에는 `delegationId: null`). 마이크 끄기, 세션 변경 등 나머지 명령은 `UnsupportedFunctionalityError`를 던집니다.
|
|
583
|
+
- 이벤트: `session.transcripts`에 말한 텍스트가 `transcript-fragment` 이벤트(`speaker`, `delta`, `startMs`, `endMs`)로 쌓입니다. 조각은 턴의 일부이지 턴 전체가 아닙니다. `session.usage.seconds`는 세션 오디오 시간입니다. GPT-Live 원본 이벤트는 각 이벤트의 `raw`에 있습니다.
|
|
524
584
|
|
|
525
|
-
|
|
585
|
+
#### AI SDK 없이 (HTTP)
|
|
526
586
|
|
|
527
587
|
```typescript
|
|
528
588
|
// 브라우저
|
|
@@ -558,9 +618,10 @@ app.post("/live", async (req, res) => {
|
|
|
558
618
|
```
|
|
559
619
|
|
|
560
620
|
- 위임: GPT-Live는 혼자 답할 수 없는 일(검색, 추론)을 클라이언트에 넘기고, 답이 올 때까지 말을 이어갑니다. Codex가 자기 에이전트로 이 요청을 처리하듯 qgrid가 처리합니다. 지금까지의 대화와 함께 세션의 위임 모델로 qgrid query를 돌리고, 음성이 그 답을 말합니다. 세션마다 `params.delegation`으로 고릅니다. `model`(qgrid가 서비스하는 모델, 대시보드 채팅과 같은 목록인 서버 `qgrid.types.ts`의 `QGRID_MODELS`. 기본 `openai/gpt-6-luna`, 목록 밖은 HTTP 400), `effort`(기본 `low`, `medium`, `high`), `webSearch`(제공자의 hosted 웹 검색, `openai/`·`grok/`만, 기본 꺼짐). 위임이 실패하면 짧은 실패 안내를 돌려주어 음성이 기다리지 않고 그 사실을 말하게 합니다. 위임은 각각 `delegation` tool 단계로 기록되고, 백엔드 비용이 실행 비용에 더해집니다.
|
|
561
|
-
-
|
|
621
|
+
- 이전 대화: `params.history`로 세션을 이전 대화에서 시작합니다. 형식은 `[{ role: "user" | "assistant" | "developer", text }]`이고 오래된 것부터 넣습니다. 지난 live 세션의 턴이든 텍스트 채팅 기록이든 똑같이 넣으면, 음성과 위임 모두 이미 나눈 대화로 받아들입니다. 최대 128개, 100만 자까지입니다. 128개를 넘거나 다른 역할을 넣으면 OpenAI가 call은 받지만 세션을 시작하지 않아서, 서버가 미리 HTTP 400으로 막습니다. 세션을 넘겨 대화를 이어 가거나 네트워크가 끊긴 뒤 다시 붙을 때는 지난 세션의 턴을 넣으면 됩니다.
|
|
622
|
+
- 요청 `params`: `sdp`(offer, 필수), `voice`, `instructions`, `history`, `delegation`, 그리고 다른 엔드포인트와 같은 `tokenName`(`openai/` 접두사), `projectName`, `logger`. `model`은 생략하거나 `openai/gpt-live-1`만 됩니다. 다른 값은 HTTP 400입니다.
|
|
562
623
|
- 응답: `sdp`(answer), `callId`, `model`, `voice`, `tokenName`, `requestLogId`. 요청 로그는 세션 내내 `running`이고, 턴이 끝날 때마다 세션 시간이 붙은 대화가 갱신되어 진행 중에도 볼 수 있습니다. 세션이 끝나면 비용 추정치(세션 오디오 분당 $0.05, API 정가)와 함께 마무리됩니다.
|
|
563
|
-
-
|
|
624
|
+
- 음성(`QGRID_LIVE_VOICES`, 타입 `QgridLiveVoice`, `/realtime` 진입점에서도 export): `cove`(기본) `juniper maple spruce ember vale breeze arbor sol aube bubbie haetsal hanul himari leher miko neer porto rindo rio rivage selva sonna tinta viola`. realtime 음성 합성용 음성(`marin` 등)은 쓸 수 없습니다. OpenAI가 call은 받아 주지만 세션이 시작되지 않기 때문에 서버가 HTTP 400으로 막습니다.
|
|
564
625
|
- 데이터 채널 이벤트: `turn.created`, `turn.delta`, `turn.done`이 말하는 대로 각 턴의 `role`과 `transcript`를 전하고, `session.usage.updated`와 `session.closed`가 `usage.audio_duration_ms`를 전합니다. 세션은 시작 2시간 뒤 스스로 끝납니다.
|
|
565
626
|
- 엔드포인트는 브라우저가 아니라 앱 서버에서 부르세요. qgrid에는 호출자 인증이 없고 CORS 헤더도 보내지 않습니다.
|
|
566
627
|
- 업스트림이 요청한 모델 이름을 무시해서 실제 모델은 확인할 수 없습니다. Codex도 qgrid도 `gpt-live-1-codex`를 보냅니다. 호출자가 정의한 도구는 열지 않았습니다. 음성의 위임은 qgrid가 고른 모델로만 처리합니다.
|
package/README.md
CHANGED
|
@@ -357,8 +357,9 @@ Grok model accepts the same factory config as the other models: `serverUrl` (def
|
|
|
357
357
|
| `qgrid(id)` | `grok/grok-4.7` (default catalog model), `grok/grok-4.7-build-fast`, `grok/grok-4.6`, `grok/grok-4.5`, any other `grok/<id>` | `generateText`, `streamText` |
|
|
358
358
|
| `qgrid.image(id)` | `grok/grok-imagine-image`, `grok/grok-imagine-image-2.0` | `generateImage` |
|
|
359
359
|
| `qgrid.video(id)` | `grok/grok-imagine-video`, `grok/grok-imagine-video-1.5`, `grok/grok-imagine-video-1.5-lite` | `experimental_generateVideo` |
|
|
360
|
-
| `qgrid.transcription(id)` | `grok/grok-voice-transcribe-2.0`; OpenAI subscription: `openai/gpt-
|
|
360
|
+
| `qgrid.transcription(id)` | `grok/grok-voice-transcribe-2.0`; OpenAI subscription: `openai/gpt-transcribe` (recommended), `openai/gpt-live-transcribe`, `openai/gpt-realtime-whisper`; `openai/gpt-4o-mini-transcribe`, `openai/gpt-4o-transcribe`, `openai/whisper-1` until OpenAI shuts them down on 2027-02-26 | `transcribe` |
|
|
361
361
|
| `qgrid.speech(id)` | `grok/grok-tts` (the id only labels the request log); OpenAI subscription: `openai/gpt-realtime-2.1-mini`, `openai/gpt-realtime-2.1` | `generateSpeech` |
|
|
362
|
+
| `qgrid.realtime(id)` (2.20.0+; browser: `createQgridRealtime` from `@cartanova/qgrid-ai-sdk/realtime`) | `openai/gpt-live-1` | `experimental_useRealtime` (experimental, WebRTC) |
|
|
362
363
|
|
|
363
364
|
qgrid forwards a language-model id that is not in its catalog unchanged instead of rejecting it,
|
|
364
365
|
as it does for OpenAI; the subscription backend decides. It may serve a different model, so check
|
|
@@ -477,7 +478,7 @@ import {
|
|
|
477
478
|
generateSpeech,
|
|
478
479
|
transcribe,
|
|
479
480
|
} from "ai";
|
|
480
|
-
import { qgrid } from "@cartanova/qgrid-ai-sdk";
|
|
481
|
+
import { qgrid, type QgridGrokVoice } from "@cartanova/qgrid-ai-sdk";
|
|
481
482
|
|
|
482
483
|
const transcript = await transcribe({
|
|
483
484
|
model: qgrid.transcription("grok/grok-voice-transcribe-2.0"),
|
|
@@ -490,7 +491,7 @@ transcript.durationInSeconds;
|
|
|
490
491
|
const speech = await generateSpeech({
|
|
491
492
|
model: qgrid.speech("grok/grok-tts"),
|
|
492
493
|
text: "Your build is ready.",
|
|
493
|
-
voice: "ara",
|
|
494
|
+
voice: "ara" satisfies QgridGrokVoice,
|
|
494
495
|
language: "en",
|
|
495
496
|
outputFormat: "mp3",
|
|
496
497
|
});
|
|
@@ -498,7 +499,7 @@ speech.audio.uint8Array;
|
|
|
498
499
|
```
|
|
499
500
|
|
|
500
501
|
- `transcribe` detects the audio type from its bytes and falls back to `audio/wav`; the server rejects anything that is not `audio/*`. Transcripts carry one segment per heard word (`segments: [{ text, startSecond, endSecond }]`): Grok times them itself (2.17.0+); for OpenAI the server force-aligns the transcript to the audio with its built-in aligner (server 2.19.0+), adding about 5% of the clip length to the request. OpenAI segments are empty when the server lacks the aligner or the clip is too long to align (about 4 minutes of dense speech).
|
|
501
|
-
- Speech text is limited to 15,000 characters. Unset `voice`, `language` and `outputFormat` use the server defaults `ara`, `en` and `mp3`. All 28 voices the subscription token lists (9 female, 19 male, such as `ara`, `eve`, `rex`) were generated live in English as mp3 on 2026-11-09; other languages (including `@ai-sdk/xai`'s `auto`) and codecs are passed through but unverified. Speech tags from xAI's list (`[pause]`, `[laugh]`, `<whisper>...</whisper>`, ...) were not read aloud in a 2026-11 check; ElevenLabs-style emotion tags such as `[deadpan]` are not in that list. `speed` is the speaking rate (1 is normal; the server rejects values outside 0.7-1.5). The server's audio MIME type is in `providerMetadata.qgrid.mediaType`; AI SDK's `speech.audio.mediaType` is sniffed from the bytes.
|
|
502
|
+
- Speech text is limited to 15,000 characters. Unset `voice`, `language` and `outputFormat` use the server defaults `ara`, `en` and `mp3`. All 28 voices the subscription token lists (9 female, 19 male, such as `ara`, `eve`, `rex`; exported as `QGRID_GROK_VOICES`, type `QgridGrokVoice`) were generated live in English as mp3 on 2026-11-09; other languages (including `@ai-sdk/xai`'s `auto`) and codecs are passed through but unverified. Speech tags from xAI's list (`[pause]`, `[laugh]`, `<whisper>...</whisper>`, ...) were not read aloud in a 2026-11 check; ElevenLabs-style emotion tags such as `[deadpan]` are not in that list. `speed` is the speaking rate (1 is normal; the server rejects values outside 0.7-1.5). The server's audio MIME type is in `providerMetadata.qgrid.mediaType`; AI SDK's `speech.audio.mediaType` is sniffed from the bytes.
|
|
502
503
|
- Word timing: `providerOptions: { xai: { withTimestamps: true } }` (2.16.0+, Grok) or `providerOptions: { qgrid: { withTimestamps: true } }` (2.17.0+, Grok or OpenAI) returns per-character timing in `providerMetadata.qgrid.audioTimestamps`: `graphChars` (input characters, spaces, punctuation and speech tags included) and the index-aligned `graphTimes` `[start, end]` seconds, plus `providerMetadata.qgrid.duration`. The shape matches what `@ai-sdk/xai` puts under `providerMetadata.xai`. `providerMetadata.qgrid.timestampsSource` is `"provider"` for Grok, which times its own audio, and `"aligned"` for OpenAI, whose audio the server force-aligns to the text with its built-in aligner (server 2.19.0+; earlier servers used a Grok STT pass). The aligner ships in the qgrid Docker image and `qgrid --install-aligner` installs it on an npm or pnpm install; a server without it returns `timestampsError`, as does speech longer than the aligner's limit (about 4 minutes of dense speech; the error states the limit for that script). If the timing step fails the audio is still returned, with `timestampsError` in the metadata and an AI SDK warning. Text normalization gives an expanded token's whole span to its first character. Grok's mp3 carries no gapless tag, so players keep the encoder delay; since server 2.19.0 qgrid adds those 46 ms to Grok timing so it matches what plays (raw `@ai-sdk/xai` values are 46 ms earlier).
|
|
503
504
|
|
|
504
505
|
### OpenAI transcription and speech (ChatGPT subscription)
|
|
@@ -506,8 +507,10 @@ speech.audio.uint8Array;
|
|
|
506
507
|
`openai/` voice ids run on OpenAI realtime sessions with the same ChatGPT subscription tokens qgrid uses for Codex: transcription through the Codex `/realtime/calls` flow, speech through a realtime conversational model that repeats the text verbatim (there is no `gpt-4o-mini-tts` or `tts-1` on subscription). Options follow `@ai-sdk/openai` under `providerOptions.openai`; `providerOptions.qgrid.tokenName` must carry the `openai/` prefix.
|
|
507
508
|
|
|
508
509
|
```typescript
|
|
510
|
+
import { type QgridOpenAISpeechVoice } from "@cartanova/qgrid-ai-sdk";
|
|
511
|
+
|
|
509
512
|
const transcript = await transcribe({
|
|
510
|
-
model: qgrid.transcription("openai/gpt-
|
|
513
|
+
model: qgrid.transcription("openai/gpt-transcribe"),
|
|
511
514
|
audio: await readFile("meeting.wav"), // PCM16 WAV or raw PCM only
|
|
512
515
|
providerOptions: { openai: { language: "en", prompt: "qgrid, Codex" } },
|
|
513
516
|
});
|
|
@@ -515,7 +518,7 @@ const transcript = await transcribe({
|
|
|
515
518
|
const speech = await generateSpeech({
|
|
516
519
|
model: qgrid.speech("openai/gpt-realtime-2.1-mini"),
|
|
517
520
|
text: "Your build is ready.",
|
|
518
|
-
voice: "marin",
|
|
521
|
+
voice: "marin" satisfies QgridOpenAISpeechVoice,
|
|
519
522
|
outputFormat: "wav",
|
|
520
523
|
providerOptions: { openai: { instructions: "Calm and brisk." } },
|
|
521
524
|
});
|
|
@@ -523,18 +526,75 @@ const speech = await generateSpeech({
|
|
|
523
526
|
|
|
524
527
|
| Model | Faithful (`@ai-sdk/openai` names) | Transformed | Unsupported (warning) |
|
|
525
528
|
| --- | --- | --- | --- |
|
|
526
|
-
| Transcription | `openai.language
|
|
527
|
-
| Speech | `voice` (default `marin`; Codex v2 list `alloy ash ballad coral echo sage shimmer verse marin cedar`, others fail with HTTP 400), `instructions` / `openai.instructions` (style guidance appended to the verbatim-repeat prompt) | `outputFormat` `wav` (default, PCM16 24 kHz) or `pcm`; anything else falls back to `wav` with a warning, as `@ai-sdk/openai` falls back to `mp3` | `language`, `speed`, `openai.speed` |
|
|
529
|
+
| Transcription | `openai.language` (sent as the list `languages` to `gpt-transcribe`), `openai.prompt` (not on `gpt-realtime-whisper`, which rejects it with HTTP 400; accepted but ignored upstream on `whisper-1`) | input must be PCM16 WAV (`audio/wav`) or raw PCM (`audio/pcm;rate=…;channels=…`); mp3/webm/ogg are rejected by the server with HTTP 400 until a decoder lands. `durationInSeconds` is computed from the audio sent; `language` echoes the request (the realtime session reports neither); `segments` carries aligned word timing (server 2.19.0+) | `openai.temperature`, `openai.include`, `openai.timestampGranularities`, `openai.responseFormat`, `openai.chunkingStrategy`, `openai.streaming` |
|
|
530
|
+
| Speech | `voice` (default `marin`; Codex v2 list `alloy ash ballad coral echo sage shimmer verse marin cedar`, exported as `QGRID_OPENAI_SPEECH_VOICES`; others fail with HTTP 400), `instructions` / `openai.instructions` (style guidance appended to the verbatim-repeat prompt) | `outputFormat` `wav` (default, PCM16 24 kHz) or `pcm`; anything else falls back to `wav` with a warning, as `@ai-sdk/openai` falls back to `mp3` | `language`, `speed`, `openai.speed` |
|
|
528
531
|
|
|
529
532
|
- Each transcription first runs on the Codex realtime call with a 25 s budget (the call is torn down about 30 s after creation); if that window closes before the transcript arrives, the server retries once on the direct realtime transcription socket (up to 60 s). The 19 MiB inline limit holds about 6.9 min of 24 kHz mono WAV. A 6.7 min English clip finished on the first path in about 20 s; a 6.4 min clip that produced a longer transcript missed the window and finished on the retry after 52 s.
|
|
530
533
|
- Speech is a conversational model reading text, not a TTS model. qgrid's prompt tells it to read only the text inside `<script>` tags and never answer it. Verbatim fidelity was verified on short English sentences and on question-form English character lines. The model's own transcript of what it said is logged with the request for comparison, but for mixed Korean/English text a matching transcript did not guarantee intelligible English audio.
|
|
534
|
+
- Use `openai/gpt-transcribe`. OpenAI shuts down `gpt-4o-mini-transcribe`, `gpt-4o-transcribe` and `whisper-1` on 2027-02-26. On a 2026-11-10 bench of synthetic speech, `gpt-transcribe` matched `gpt-4o-transcribe` on 108 English clips (the gap was number formatting such as "twenty-first" for "21st"), cut the character error rate on 18 Korean/English mixed clips from 14.5% to 4.4%, transcribed a 229 s clip that `gpt-4o-transcribe` failed, and returned nothing for silence or noise where `gpt-4o-transcribe` invented phrases. It sometimes corrects a learner's grammar ("we eat chicken" came back as "we ate chicken"), so do not grade grammar from the transcript alone.
|
|
531
535
|
- Per-model behavior is recorded in the server repository's `docs/diagnosis/2026-10-17-openai-realtime-audio-probe.md`. Known limits on 2026-10-17: `gpt-live-transcribe` and `gpt-realtime-whisper` returned roughly half of a 60 s clip (use them for short utterances); `gpt-realtime-whisper` rejects `openai.prompt` (the server answers HTTP 400 before any upstream call); `whisper-1` accepts but ignores `openai.prompt`. For speech, `gpt-realtime-2.1-mini` repeated every test text verbatim (13/13), `gpt-realtime-2.1` paraphrased once (12/13); `gpt-realtime-1.5` often answered the text instead of repeating it (6/13) and was removed in 2.18.0. On 2026-11-06, with the current prompt, `gpt-realtime-2.1` and `gpt-realtime-2.1-mini` both read 12/12 question-form character lines verbatim (the earlier prompt let `gpt-realtime-2.1` answer 7/12). On 2026-11-09, judged by two independent STT models over 108 English clips (6 lines, voices `marin` `cedar` `ballad`, 3 takes each), `gpt-realtime-2.1` read 54/54 and `gpt-realtime-2.1-mini` 53/54: one take began with "Sure, let me read that out loud for you exactly as written." Both read every Korean/English mixed line in full (9/9 each). The model's own logged transcript does not show such a preamble, so when fidelity matters, check the audio with STT or `withTimestamps` (its alignment skips the extra words) instead of the transcript.
|
|
532
536
|
|
|
533
537
|
### OpenAI live voice conversation (GPT-Live)
|
|
534
538
|
|
|
535
|
-
Server 2.19.2+ runs `openai/gpt-live-1` conversations on a ChatGPT subscription token. GPT-Live listens and speaks at the same time, so it is a conversation, not speech generation
|
|
539
|
+
Server 2.19.2+ runs `openai/gpt-live-1` conversations on a ChatGPT subscription token. GPT-Live listens and speaks at the same time, so it is a conversation, not speech generation.
|
|
540
|
+
|
|
541
|
+
The audio never passes through qgrid. The browser's own WebRTC peer makes an SDP offer, your server sends it to `POST /api/qgrid/liveCall`, and the browser applies the SDP answer that comes back. Audio and events then flow between the browser and OpenAI. qgrid logs the conversation text and the session's audio time; it stores no audio.
|
|
542
|
+
|
|
543
|
+
#### With AI SDK's experimental realtime runtime (SDK 2.20.0+)
|
|
544
|
+
|
|
545
|
+
`qgrid.realtime("openai/gpt-live-1")` is an AI SDK `Experimental_RealtimeModelV4`. AI SDK's realtime session (`experimental_useRealtime` in `@ai-sdk/react`, `ai` 7.0.102+) then owns the microphone, playback, the WebRTC peer and the data channel; your server route only exchanges the SDP. The API is experimental in AI SDK and may change between `ai` releases.
|
|
546
|
+
|
|
547
|
+
```tsx
|
|
548
|
+
// Browser: import from the /realtime entry, which carries no Node-only code.
|
|
549
|
+
import { createQgridRealtime } from "@cartanova/qgrid-ai-sdk/realtime";
|
|
550
|
+
import { experimental_useRealtime } from "@ai-sdk/react";
|
|
551
|
+
|
|
552
|
+
// Module scope: the hook starts a new session whenever it gets a new model object.
|
|
553
|
+
const model = createQgridRealtime("openai/gpt-live-1");
|
|
554
|
+
|
|
555
|
+
function LiveConversation() {
|
|
556
|
+
// No sessionConfig: the server route sets the voice and instructions.
|
|
557
|
+
const realtime = experimental_useRealtime({ model, api: { session: "/api/live" } });
|
|
558
|
+
|
|
559
|
+
return (
|
|
560
|
+
<>
|
|
561
|
+
<button onClick={() => realtime.connect()}>Start</button>
|
|
562
|
+
<button onClick={() => realtime.close()}>End</button>
|
|
563
|
+
<p>{realtime.status}: {realtime.session?.usage?.seconds ?? 0} s</p>
|
|
564
|
+
{realtime.session?.transcripts.map((part, i) => (
|
|
565
|
+
<span key={i}>{part.speaker === "user" ? "🧑 " : ""}{part.delta}</span>
|
|
566
|
+
))}
|
|
567
|
+
</>
|
|
568
|
+
);
|
|
569
|
+
}
|
|
570
|
+
```
|
|
571
|
+
|
|
572
|
+
```typescript
|
|
573
|
+
// Your server route: the hook posts { sdp, sessionConfig } and expects { sdp, sessionId }.
|
|
574
|
+
import { qgrid, type QgridLiveVoice, type QgridRealtimeProviderOptions } from "@cartanova/qgrid-ai-sdk";
|
|
575
|
+
|
|
576
|
+
app.post("/api/live", async (req, res) => {
|
|
577
|
+
const answer = await qgrid.realtime("openai/gpt-live-1").doCreateWebRTCSession({
|
|
578
|
+
sdp: req.body.sdp,
|
|
579
|
+
// Set on the server; the browser's sessionConfig is not trusted.
|
|
580
|
+
sessionConfig: {
|
|
581
|
+
voice: "cove" satisfies QgridLiveVoice,
|
|
582
|
+
instructions: "You are a friendly English tutor.",
|
|
583
|
+
providerOptions: {
|
|
584
|
+
qgrid: { delegation: { model: "openai/gpt-6-luna", webSearch: true } } satisfies QgridRealtimeProviderOptions,
|
|
585
|
+
},
|
|
586
|
+
},
|
|
587
|
+
});
|
|
588
|
+
res.json(answer); // { sdp, sessionId, requestLogId }
|
|
589
|
+
});
|
|
590
|
+
```
|
|
591
|
+
|
|
592
|
+
- `providerOptions.qgrid` (`QgridRealtimeProviderOptions`): `delegation`, `history`, `tokenName` and `logger`, the same fields as the HTTP `params` below. `sessionConfig.voice` and `sessionConfig.instructions` map to `voice` and `instructions`; other `sessionConfig` fields are not sent. Set them in the server call rather than in the hook: the hook starts a new session whenever it gets a new `sessionConfig` object, so an inline object would restart the session on every render. `serverUrl` and `projectName` come from `createQgridRealtime(id, config)` or `QGRID_URL` / `QGRID_PROJECT_NAME`.
|
|
593
|
+
- qgrid answers the voice's delegations on the server, so each `delegation-created` event has `target: "provider"`: there is nothing for the app to send back. The session reports no `delegationMode`: AI SDK's runtime rejects `"provider"`, and `"client"` would mean the app must answer.
|
|
594
|
+
- Supported commands: `close()` (sends `session.close` and waits for the final usage) and `sendEvent({ type: "context-append", content, delegationId })`, which adds text the voice can use (`delegationId: null` for the whole session). Microphone mute, session updates and the other commands throw `UnsupportedFunctionalityError`.
|
|
595
|
+
- Events: `session.transcripts` holds the spoken text as `transcript-fragment` events (`speaker`, `delta`, `startMs`, `endMs`); fragments are pieces of a turn, not whole turns. `session.usage.seconds` is the session's audio time. The raw GPT-Live events are in each event's `raw`.
|
|
536
596
|
|
|
537
|
-
|
|
597
|
+
#### Without AI SDK (HTTP)
|
|
538
598
|
|
|
539
599
|
```typescript
|
|
540
600
|
// Browser
|
|
@@ -570,9 +630,10 @@ app.post("/live", async (req, res) => {
|
|
|
570
630
|
```
|
|
571
631
|
|
|
572
632
|
- Delegation: GPT-Live hands what it cannot answer itself (lookups, reasoning) to the client and keeps talking until the answer arrives. qgrid answers it, as Codex answers with its own agent: it runs the request as a qgrid query on the session's delegation model, with the conversation so far, and the voice speaks the reply. Choose per session with `params.delegation`: `model` (any model qgrid serves, the list the dashboard chat offers: `QGRID_MODELS` in the server's `qgrid.types.ts`; default `openai/gpt-6-luna`; others are rejected with HTTP 400), `effort` (`low` default, `medium`, `high`) and `webSearch` (the provider's hosted web search, `openai/` and `grok/` only; off by default). A failed delegation is answered with a short failure note so the voice can say so instead of waiting. Each delegation is logged as a `delegation` tool step, and its backend cost is added to the run's cost.
|
|
573
|
-
-
|
|
633
|
+
- History: `params.history` starts the session from an earlier conversation, `[{ role: "user" | "assistant" | "developer", text }]`, oldest first. It can be a past live session's turns or a text chat alike; the voice and its delegations treat it as what was already said. At most 128 items and 1,000,000 characters: past 128 items, or with any other role, OpenAI accepts the call but never starts the session, so the server answers HTTP 400 first. To carry a conversation across sessions (or reconnect after a dropped network), pass the previous session's turns.
|
|
634
|
+
- Request `params`: `sdp` (the offer, required), `voice`, `instructions`, `history`, `delegation`, and the usual `tokenName` (`openai/` prefix), `projectName` and `logger`. `model` may be omitted or `openai/gpt-live-1`; any other value is rejected with HTTP 400.
|
|
574
635
|
- Response: `sdp` (the answer), `callId`, `model`, `voice`, `tokenName` and `requestLogId`. The request log stays `running` for the whole session and its conversation, with each turn's session time, is rewritten after every finished turn, so it can be read while the session runs. When the session ends the run is finished with a cost estimate of $0.05 per minute of session audio (the API list price).
|
|
575
|
-
- Voices: `cove` (default) `juniper maple spruce ember vale breeze arbor sol aube bubbie haetsal hanul himari leher miko neer porto rindo rio rivage selva sonna tinta viola`. The realtime speech voices (`marin` and others) do not apply; the server rejects them with HTTP 400, because OpenAI would accept the call and then never start the session.
|
|
636
|
+
- Voices (`QGRID_LIVE_VOICES`, type `QgridLiveVoice`, also from the `/realtime` entry): `cove` (default) `juniper maple spruce ember vale breeze arbor sol aube bubbie haetsal hanul himari leher miko neer porto rindo rio rivage selva sonna tinta viola`. The realtime speech voices (`marin` and others) do not apply; the server rejects them with HTTP 400, because OpenAI would accept the call and then never start the session.
|
|
576
637
|
- Data channel events: `turn.created`, `turn.delta` and `turn.done` carry each turn's `role` and `transcript` as it is spoken; `session.usage.updated` and `session.closed` carry `usage.audio_duration_ms`. A session ends on its own two hours after it starts.
|
|
577
638
|
- Call the endpoint from your server, not from the browser: qgrid has no caller authentication and sends no CORS headers.
|
|
578
639
|
- The upstream ignores the requested model name, so the serving model cannot be confirmed; Codex itself sends `gpt-live-1-codex`, as qgrid does. Caller-defined tools are not exposed: the voice's delegations run on qgrid's chosen model only.
|
package/dist/index.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { a as createQgridRealtime, c as QGRID_OPENAI_SPEECH_VOICES, d as QgridOpenAISpeechVoice, i as QgridRealtimeProviderOptions, l as QgridGrokVoice, n as QgridRealtimeConfig, o as QGRID_GROK_VOICES, r as QgridRealtimeModel, s as QGRID_LIVE_VOICES, t as QgridLiveRealtimeModel, u as QgridLiveVoice } from "./realtime-_tFFhM56.js";
|
|
1
2
|
import { z } from "zod";
|
|
2
3
|
import { Experimental_VideoModelV3, Experimental_VideoModelV3VideoData, ImageModelV3, LanguageModelV3, SharedV3ProviderMetadata, SpeechModelV3, TranscriptionModelV3 } from "@ai-sdk/provider";
|
|
3
4
|
import { TelemetryOptions } from "ai";
|
|
@@ -43,6 +44,8 @@ type QgridVideoModel = "grok/grok-imagine-video" | "grok/grok-imagine-video-1.5"
|
|
|
43
44
|
type QgridGrokTranscriptionModel = "grok/grok-voice-transcribe-2.0";
|
|
44
45
|
/**
|
|
45
46
|
* OpenAI subscription speech-to-text ids, served through a realtime transcription session.
|
|
47
|
+
* Use `openai/gpt-transcribe` for recordings. OpenAI shuts down
|
|
48
|
+
* `gpt-4o-mini-transcribe`, `gpt-4o-transcribe` and `whisper-1` on 2027-02-26.
|
|
46
49
|
* Known limits (2026-10-17 live matrix): `gpt-live-transcribe` and `gpt-realtime-whisper`
|
|
47
50
|
* returned about half of a 60 s clip; `gpt-realtime-whisper` rejects `openai.prompt` (HTTP 400
|
|
48
51
|
* before any upstream call); `whisper-1` accepts `openai.prompt` without error but the upstream
|
|
@@ -154,10 +157,10 @@ type QgridGrokModel = "grok/grok-4.7" | "grok/grok-4.7-build-fast" | "grok/grok-
|
|
|
154
157
|
*/
|
|
155
158
|
declare const qgridXaiProviderOptionsSchema: z.ZodObject<{
|
|
156
159
|
reasoningEffort: z.ZodOptional<z.ZodEnum<{
|
|
160
|
+
none: "none";
|
|
157
161
|
low: "low";
|
|
158
162
|
medium: "medium";
|
|
159
163
|
high: "high";
|
|
160
|
-
none: "none";
|
|
161
164
|
xhigh: "xhigh";
|
|
162
165
|
}>>;
|
|
163
166
|
maxTurns: z.ZodOptional<z.ZodNumber>;
|
|
@@ -375,6 +378,7 @@ export declare namespace qgrid {
|
|
|
375
378
|
var video: typeof createQgridVideo;
|
|
376
379
|
var transcription: typeof createQgridTranscription;
|
|
377
380
|
var speech: typeof createQgridSpeech;
|
|
381
|
+
var realtime: typeof createQgridRealtime;
|
|
378
382
|
}
|
|
379
383
|
//#endregion
|
|
380
|
-
export { type QgridAnthropicEffort, type QgridAnthropicModel, type QgridAnthropicProviderConfig, type QgridAnthropicProviderOptions, type QgridGrokEffort, type QgridGrokMediaProviderOptions, type QgridGrokModel, type QgridGrokSpeechModel, type QgridGrokTranscriptionModel, type QgridImageGenerationMetadata, type QgridImageModel, type QgridImageProviderConfig, type QgridImageProviderOptions, type QgridLoggerConfig, type QgridMediaProviderConfig, type QgridOpenAIEffort, type QgridOpenAIMediaProviderOptions, type QgridOpenAIModel, type QgridOpenAIProviderConfig, type QgridOpenAIProviderOptions, type QgridOpenAISpeechModel, type QgridOpenAISpeechProviderOptions, type QgridOpenAITranscriptionModel, type QgridOpenAITranscriptionProviderOptions, type QgridProviderConfig, type QgridProviderOptions, type QgridSpeechModel, type QgridSupportedModel, type QgridTranscriptionModel, type QgridVideoModel, type QgridVideoPollOptions, type QgridVideoResult, type QgridXaiImageProviderOptions, type QgridXaiProviderOptions, type QgridXaiTranscriptionProviderOptions, type QgridXaiVideoProviderOptions, qgrid as default };
|
|
384
|
+
export { QGRID_GROK_VOICES, QGRID_LIVE_VOICES, QGRID_OPENAI_SPEECH_VOICES, type QgridAnthropicEffort, type QgridAnthropicModel, type QgridAnthropicProviderConfig, type QgridAnthropicProviderOptions, type QgridGrokEffort, type QgridGrokMediaProviderOptions, type QgridGrokModel, type QgridGrokSpeechModel, type QgridGrokTranscriptionModel, type QgridGrokVoice, type QgridImageGenerationMetadata, type QgridImageModel, type QgridImageProviderConfig, type QgridImageProviderOptions, type QgridLiveRealtimeModel, type QgridLiveVoice, type QgridLoggerConfig, type QgridMediaProviderConfig, type QgridOpenAIEffort, type QgridOpenAIMediaProviderOptions, type QgridOpenAIModel, type QgridOpenAIProviderConfig, type QgridOpenAIProviderOptions, type QgridOpenAISpeechModel, type QgridOpenAISpeechProviderOptions, type QgridOpenAISpeechVoice, type QgridOpenAITranscriptionModel, type QgridOpenAITranscriptionProviderOptions, type QgridProviderConfig, type QgridProviderOptions, type QgridRealtimeConfig, type QgridRealtimeModel, type QgridRealtimeProviderOptions, type QgridSpeechModel, type QgridSupportedModel, type QgridTranscriptionModel, type QgridVideoModel, type QgridVideoPollOptions, type QgridVideoResult, type QgridXaiImageProviderOptions, type QgridXaiProviderOptions, type QgridXaiTranscriptionProviderOptions, type QgridXaiVideoProviderOptions, createQgridRealtime, qgrid as default };
|
package/dist/index.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { c as TEXT_STREAM_ID_PREFIX, i as QGRID_OPENAI_SPEECH_VOICES, l as THREAD_COORD_TTL_MS, n as QGRID_GROK_VOICES, o as PENDING_TOOL_RESULTS_WARNING, r as QGRID_LIVE_VOICES, s as QGRID_PROVIDER_NAME, t as createQgridRealtime, u as TOP_LEVEL_SCHEMA_WARNING } from "./realtime-BgodcUud.js";
|
|
1
2
|
import { z } from "zod";
|
|
2
3
|
import { Agent, fetch as fetch$1 } from "undici";
|
|
3
4
|
//#region src/envelope-stream-parser.ts
|
|
@@ -640,13 +641,6 @@ function serverOrigin(serverUrl) {
|
|
|
640
641
|
}
|
|
641
642
|
}
|
|
642
643
|
//#endregion
|
|
643
|
-
//#region src/qgrid.constant.ts
|
|
644
|
-
const QGRID_PROVIDER_NAME = "qgrid";
|
|
645
|
-
const THREAD_COORD_TTL_MS = 6e5;
|
|
646
|
-
const TEXT_STREAM_ID_PREFIX = "text";
|
|
647
|
-
const TOP_LEVEL_SCHEMA_WARNING = "qgrid server-enforced structured output requires \"object\". Falling back to client-side parsing.";
|
|
648
|
-
const PENDING_TOOL_RESULTS_WARNING = "[qgrid] tool results did not match any pending qgrid run; sending request without runContext";
|
|
649
|
-
//#endregion
|
|
650
644
|
//#region src/utils.ts
|
|
651
645
|
const createRunOutputSchema = z.object({ requestLogId: z.number() });
|
|
652
646
|
const appendStepOutputSchema = z.object({ stepId: z.number() });
|
|
@@ -3075,5 +3069,6 @@ qgrid.image = createQgridImage;
|
|
|
3075
3069
|
qgrid.video = createQgridVideo;
|
|
3076
3070
|
qgrid.transcription = createQgridTranscription;
|
|
3077
3071
|
qgrid.speech = createQgridSpeech;
|
|
3072
|
+
qgrid.realtime = createQgridRealtime;
|
|
3078
3073
|
//#endregion
|
|
3079
|
-
export { QGRID_MAX_REQUEST_BODY_BYTES, QgridMediaHttpError, QgridVideoGenerationError, QgridVideoTimeoutError, QgridXaiOptionsError, createQgridLogger, qgrid as default, qgrid, resumeQgridVideo };
|
|
3074
|
+
export { QGRID_GROK_VOICES, QGRID_LIVE_VOICES, QGRID_MAX_REQUEST_BODY_BYTES, QGRID_OPENAI_SPEECH_VOICES, QgridMediaHttpError, QgridVideoGenerationError, QgridVideoTimeoutError, QgridXaiOptionsError, createQgridLogger, createQgridRealtime, qgrid as default, qgrid, resumeQgridVideo };
|
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { UnsupportedFunctionalityError } from "@ai-sdk/provider";
|
|
3
|
+
//#region src/qgrid.constant.ts
|
|
4
|
+
const QGRID_PROVIDER_NAME = "qgrid";
|
|
5
|
+
const DEFAULT_QGRID_SERVER_URL = "http://localhost:44900";
|
|
6
|
+
const THREAD_COORD_TTL_MS = 6e5;
|
|
7
|
+
const TEXT_STREAM_ID_PREFIX = "text";
|
|
8
|
+
const TOP_LEVEL_SCHEMA_WARNING = "qgrid server-enforced structured output requires \"object\". Falling back to client-side parsing.";
|
|
9
|
+
const PENDING_TOOL_RESULTS_WARNING = "[qgrid] tool results did not match any pending qgrid run; sending request without runContext";
|
|
10
|
+
//#endregion
|
|
11
|
+
//#region src/voices.ts
|
|
12
|
+
/**
|
|
13
|
+
* `voice` for `qgrid.speech("grok/...")`: the /tts voices xAI lists for subscription tokens
|
|
14
|
+
* (2026-11-06; default `ara`). The server passes other ids through to xAI unchecked.
|
|
15
|
+
*/
|
|
16
|
+
const QGRID_GROK_VOICES = {
|
|
17
|
+
female: [
|
|
18
|
+
"ara",
|
|
19
|
+
"aurora",
|
|
20
|
+
"carina",
|
|
21
|
+
"celeste",
|
|
22
|
+
"eve",
|
|
23
|
+
"iris",
|
|
24
|
+
"liora",
|
|
25
|
+
"luna",
|
|
26
|
+
"ursa"
|
|
27
|
+
],
|
|
28
|
+
male: [
|
|
29
|
+
"altair",
|
|
30
|
+
"atlas",
|
|
31
|
+
"castor",
|
|
32
|
+
"cosmo",
|
|
33
|
+
"helios",
|
|
34
|
+
"helix",
|
|
35
|
+
"kepler",
|
|
36
|
+
"leo",
|
|
37
|
+
"lumen",
|
|
38
|
+
"lux",
|
|
39
|
+
"naksh",
|
|
40
|
+
"orion",
|
|
41
|
+
"perseus",
|
|
42
|
+
"rex",
|
|
43
|
+
"rigel",
|
|
44
|
+
"sal",
|
|
45
|
+
"sirius",
|
|
46
|
+
"zagan",
|
|
47
|
+
"zenith"
|
|
48
|
+
]
|
|
49
|
+
};
|
|
50
|
+
/** `voice` for `qgrid.speech("openai/...")` (default `marin`); the server rejects any other with 400. */
|
|
51
|
+
const QGRID_OPENAI_SPEECH_VOICES = [
|
|
52
|
+
"marin",
|
|
53
|
+
"cedar",
|
|
54
|
+
"alloy",
|
|
55
|
+
"ash",
|
|
56
|
+
"ballad",
|
|
57
|
+
"coral",
|
|
58
|
+
"echo",
|
|
59
|
+
"sage",
|
|
60
|
+
"shimmer",
|
|
61
|
+
"verse"
|
|
62
|
+
];
|
|
63
|
+
/**
|
|
64
|
+
* `sessionConfig.voice` for `qgrid.realtime("openai/gpt-live-1")` and `voice` for the HTTP
|
|
65
|
+
* `liveCall` (default `cove`). The server rejects any other: OpenAI would accept the call and never
|
|
66
|
+
* start the session.
|
|
67
|
+
*/
|
|
68
|
+
const QGRID_LIVE_VOICES = [
|
|
69
|
+
"cove",
|
|
70
|
+
"juniper",
|
|
71
|
+
"maple",
|
|
72
|
+
"spruce",
|
|
73
|
+
"ember",
|
|
74
|
+
"vale",
|
|
75
|
+
"breeze",
|
|
76
|
+
"arbor",
|
|
77
|
+
"sol",
|
|
78
|
+
"aube",
|
|
79
|
+
"bubbie",
|
|
80
|
+
"haetsal",
|
|
81
|
+
"hanul",
|
|
82
|
+
"himari",
|
|
83
|
+
"leher",
|
|
84
|
+
"miko",
|
|
85
|
+
"neer",
|
|
86
|
+
"porto",
|
|
87
|
+
"rindo",
|
|
88
|
+
"rio",
|
|
89
|
+
"rivage",
|
|
90
|
+
"selva",
|
|
91
|
+
"sonna",
|
|
92
|
+
"tinta",
|
|
93
|
+
"viola"
|
|
94
|
+
];
|
|
95
|
+
//#endregion
|
|
96
|
+
//#region src/realtime.ts
|
|
97
|
+
const qgridProviderOptionsSchema = z.object({
|
|
98
|
+
tokenName: z.string().optional(),
|
|
99
|
+
logger: z.boolean().optional(),
|
|
100
|
+
history: z.array(z.object({
|
|
101
|
+
role: z.enum([
|
|
102
|
+
"user",
|
|
103
|
+
"assistant",
|
|
104
|
+
"developer"
|
|
105
|
+
]),
|
|
106
|
+
text: z.string()
|
|
107
|
+
})).optional(),
|
|
108
|
+
delegation: z.object({
|
|
109
|
+
model: z.string().optional(),
|
|
110
|
+
effort: z.enum([
|
|
111
|
+
"low",
|
|
112
|
+
"medium",
|
|
113
|
+
"high"
|
|
114
|
+
]).optional(),
|
|
115
|
+
webSearch: z.boolean().optional()
|
|
116
|
+
}).optional()
|
|
117
|
+
});
|
|
118
|
+
const LABEL = "qgrid.realtime";
|
|
119
|
+
const env = (name) => typeof process === "undefined" ? void 0 : process.env[name];
|
|
120
|
+
const liveCallOutputSchema = z.object({
|
|
121
|
+
sdp: z.string().min(1),
|
|
122
|
+
callId: z.string().min(1),
|
|
123
|
+
requestLogId: z.number().optional()
|
|
124
|
+
});
|
|
125
|
+
const usageSchema = z.object({ audio_duration_ms: z.number() });
|
|
126
|
+
const sessionStartedSchema = z.object({ session: z.object({ id: z.string() }) });
|
|
127
|
+
const sessionClosedSchema = z.object({
|
|
128
|
+
reason: z.string().optional(),
|
|
129
|
+
usage: usageSchema.optional()
|
|
130
|
+
});
|
|
131
|
+
const usageUpdatedSchema = z.object({ usage: usageSchema });
|
|
132
|
+
const transcriptAddedSchema = z.object({
|
|
133
|
+
item: z.object({ text: z.string() }),
|
|
134
|
+
start_ms: z.number(),
|
|
135
|
+
end_ms: z.number()
|
|
136
|
+
});
|
|
137
|
+
const delegationCreatedSchema = z.object({
|
|
138
|
+
item: z.object({ id: z.string() }),
|
|
139
|
+
offset_ms: z.number().optional()
|
|
140
|
+
});
|
|
141
|
+
const errorSchema = z.object({ error: z.object({
|
|
142
|
+
message: z.string(),
|
|
143
|
+
code: z.string().nullish()
|
|
144
|
+
}) });
|
|
145
|
+
const envelopeSchema = z.object({ type: z.string() });
|
|
146
|
+
/**
|
|
147
|
+
* Maps GPT-Live data-channel events (the Codex frameless protocol, not the public API's `session.*`
|
|
148
|
+
* names) to AI SDK events. Spoken text comes from `input_transcript.added` and
|
|
149
|
+
* `output_transcript.added`: `turn.delta` leaves out each turn's first word, which only
|
|
150
|
+
* `turn.created` carries (live session, 2026-11-10). One parser per connection, for the last usage.
|
|
151
|
+
*/
|
|
152
|
+
function createLiveEventParser() {
|
|
153
|
+
let usageMs = 0;
|
|
154
|
+
return (raw) => {
|
|
155
|
+
const type = envelopeSchema.safeParse(raw).data?.type;
|
|
156
|
+
const custom = () => ({
|
|
157
|
+
type: "custom",
|
|
158
|
+
rawType: type ?? "unknown",
|
|
159
|
+
raw
|
|
160
|
+
});
|
|
161
|
+
switch (type) {
|
|
162
|
+
case "session.started": {
|
|
163
|
+
const event = sessionStartedSchema.safeParse(raw);
|
|
164
|
+
return event.success ? {
|
|
165
|
+
type: "session-started",
|
|
166
|
+
sessionId: event.data.session.id,
|
|
167
|
+
raw
|
|
168
|
+
} : custom();
|
|
169
|
+
}
|
|
170
|
+
case "session.usage.updated": {
|
|
171
|
+
const event = usageUpdatedSchema.safeParse(raw);
|
|
172
|
+
if (!event.success) return custom();
|
|
173
|
+
usageMs = event.data.usage.audio_duration_ms;
|
|
174
|
+
return {
|
|
175
|
+
type: "session-usage",
|
|
176
|
+
usage: { seconds: usageMs / 1e3 },
|
|
177
|
+
raw
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
case "session.closed": {
|
|
181
|
+
const event = sessionClosedSchema.safeParse(raw);
|
|
182
|
+
return {
|
|
183
|
+
type: "session-closed",
|
|
184
|
+
usage: { seconds: ((event.success ? event.data.usage?.audio_duration_ms : void 0) ?? usageMs) / 1e3 },
|
|
185
|
+
reason: (event.success ? event.data.reason : void 0) ?? "closed",
|
|
186
|
+
raw
|
|
187
|
+
};
|
|
188
|
+
}
|
|
189
|
+
case "input_transcript.added":
|
|
190
|
+
case "output_transcript.added": {
|
|
191
|
+
const event = transcriptAddedSchema.safeParse(raw);
|
|
192
|
+
return event.success ? {
|
|
193
|
+
type: "transcript-fragment",
|
|
194
|
+
speaker: type === "input_transcript.added" ? "user" : "assistant",
|
|
195
|
+
delta: event.data.item.text,
|
|
196
|
+
startMs: event.data.start_ms,
|
|
197
|
+
endMs: event.data.end_ms,
|
|
198
|
+
raw
|
|
199
|
+
} : custom();
|
|
200
|
+
}
|
|
201
|
+
case "delegation.created": {
|
|
202
|
+
const event = delegationCreatedSchema.safeParse(raw);
|
|
203
|
+
return event.success ? {
|
|
204
|
+
type: "delegation-created",
|
|
205
|
+
delegationId: event.data.item.id,
|
|
206
|
+
target: "provider",
|
|
207
|
+
offsetMs: event.data.offset_ms,
|
|
208
|
+
raw
|
|
209
|
+
} : custom();
|
|
210
|
+
}
|
|
211
|
+
case "error": {
|
|
212
|
+
const event = errorSchema.safeParse(raw);
|
|
213
|
+
return event.success ? {
|
|
214
|
+
type: "error",
|
|
215
|
+
message: event.data.error.message,
|
|
216
|
+
code: event.data.error.code ?? void 0,
|
|
217
|
+
raw
|
|
218
|
+
} : {
|
|
219
|
+
type: "error",
|
|
220
|
+
message: "Invalid GPT-Live error event.",
|
|
221
|
+
raw
|
|
222
|
+
};
|
|
223
|
+
}
|
|
224
|
+
default: return custom();
|
|
225
|
+
}
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
function serializeLiveClientEvent(event) {
|
|
229
|
+
switch (event.type) {
|
|
230
|
+
case "session-close": return { type: "session.close" };
|
|
231
|
+
case "context-append": {
|
|
232
|
+
const content = [{
|
|
233
|
+
type: "input_text",
|
|
234
|
+
text: event.content
|
|
235
|
+
}];
|
|
236
|
+
return event.delegationId === null ? {
|
|
237
|
+
type: "session.context.append",
|
|
238
|
+
content
|
|
239
|
+
} : {
|
|
240
|
+
type: "delegation.context.append",
|
|
241
|
+
delegation_item_id: event.delegationId,
|
|
242
|
+
content
|
|
243
|
+
};
|
|
244
|
+
}
|
|
245
|
+
default: throw new UnsupportedFunctionalityError({ functionality: `${LABEL}: ${event.type}; GPT-Live through qgrid supports session-close and context-append` });
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
/**
|
|
249
|
+
* GPT-Live as an AI SDK realtime model. The browser runtime (`experimental_useRealtime` with
|
|
250
|
+
* `api: { session }`) owns the microphone, playback and the WebRTC peer; your server route calls
|
|
251
|
+
* `doCreateWebRTCSession`, which exchanges the SDP through the qgrid server. qgrid answers the
|
|
252
|
+
* voice's delegations itself and logs the transcript.
|
|
253
|
+
*/
|
|
254
|
+
function createQgridRealtime(modelId, config) {
|
|
255
|
+
if (modelId !== "openai/gpt-live-1") throw new Error(`${LABEL}: unsupported model ${String(modelId)}; expected openai/gpt-live-1`);
|
|
256
|
+
const parse = createLiveEventParser();
|
|
257
|
+
return {
|
|
258
|
+
specificationVersion: "v4",
|
|
259
|
+
provider: QGRID_PROVIDER_NAME,
|
|
260
|
+
modelId,
|
|
261
|
+
capabilities: {
|
|
262
|
+
conversation: "continuous",
|
|
263
|
+
transports: ["webrtc"],
|
|
264
|
+
connections: ["webrtc"],
|
|
265
|
+
startup: "session-start",
|
|
266
|
+
finalization: "session-close"
|
|
267
|
+
},
|
|
268
|
+
getWebRTCConfig: () => ({ dataChannelLabel: "oai-events" }),
|
|
269
|
+
async doCreateWebRTCSession({ sdp, sessionConfig, abortSignal }) {
|
|
270
|
+
const serverUrl = config?.serverUrl ?? env("QGRID_URL") ?? "http://localhost:44900";
|
|
271
|
+
const options = qgridProviderOptionsSchema.safeParse(sessionConfig?.providerOptions?.qgrid ?? {});
|
|
272
|
+
if (!options.success) throw new Error(`${LABEL}: providerOptions.qgrid: ${options.error.issues.map((issue) => `${issue.path.join(".")} ${issue.message}`).join("; ")}`);
|
|
273
|
+
const qgrid = options.data;
|
|
274
|
+
const params = {
|
|
275
|
+
model: modelId,
|
|
276
|
+
sdp,
|
|
277
|
+
voice: sessionConfig?.voice,
|
|
278
|
+
instructions: sessionConfig?.instructions || void 0,
|
|
279
|
+
history: qgrid.history,
|
|
280
|
+
delegation: qgrid.delegation,
|
|
281
|
+
tokenName: qgrid.tokenName,
|
|
282
|
+
logger: qgrid.logger,
|
|
283
|
+
projectName: config?.projectName ?? env("QGRID_PROJECT_NAME")
|
|
284
|
+
};
|
|
285
|
+
const response = await fetch(`${serverUrl}/api/qgrid/liveCall`, {
|
|
286
|
+
method: "POST",
|
|
287
|
+
headers: { "Content-Type": "application/json" },
|
|
288
|
+
body: JSON.stringify({ params }),
|
|
289
|
+
signal: abortSignal
|
|
290
|
+
});
|
|
291
|
+
if (!response.ok) throw new Error(`qgrid liveCall ${response.status}: ${(await response.text()).slice(0, 2e3)}`);
|
|
292
|
+
const output = liveCallOutputSchema.parse(await response.json());
|
|
293
|
+
return {
|
|
294
|
+
sessionId: output.callId,
|
|
295
|
+
sdp: output.sdp,
|
|
296
|
+
requestLogId: output.requestLogId
|
|
297
|
+
};
|
|
298
|
+
},
|
|
299
|
+
parseServerEvent: parse,
|
|
300
|
+
createServerEventParser: createLiveEventParser,
|
|
301
|
+
serializeClientEvent: serializeLiveClientEvent,
|
|
302
|
+
buildSessionConfig: (sessionConfig) => ({
|
|
303
|
+
instructions: sessionConfig.instructions,
|
|
304
|
+
voice: sessionConfig.voice
|
|
305
|
+
})
|
|
306
|
+
};
|
|
307
|
+
}
|
|
308
|
+
//#endregion
|
|
309
|
+
export { DEFAULT_QGRID_SERVER_URL as a, TEXT_STREAM_ID_PREFIX as c, QGRID_OPENAI_SPEECH_VOICES as i, THREAD_COORD_TTL_MS as l, QGRID_GROK_VOICES as n, PENDING_TOOL_RESULTS_WARNING as o, QGRID_LIVE_VOICES as r, QGRID_PROVIDER_NAME as s, createQgridRealtime as t, TOP_LEVEL_SCHEMA_WARNING as u };
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { Experimental_RealtimeModelV4 } from "@ai-sdk/provider";
|
|
2
|
+
//#region src/voices.d.ts
|
|
3
|
+
/**
|
|
4
|
+
* `voice` for `qgrid.speech("grok/...")`: the /tts voices xAI lists for subscription tokens
|
|
5
|
+
* (2026-11-06; default `ara`). The server passes other ids through to xAI unchecked.
|
|
6
|
+
*/
|
|
7
|
+
declare const QGRID_GROK_VOICES: {
|
|
8
|
+
readonly female: readonly ["ara", "aurora", "carina", "celeste", "eve", "iris", "liora", "luna", "ursa"];
|
|
9
|
+
readonly male: readonly ["altair", "atlas", "castor", "cosmo", "helios", "helix", "kepler", "leo", "lumen", "lux", "naksh", "orion", "perseus", "rex", "rigel", "sal", "sirius", "zagan", "zenith"];
|
|
10
|
+
};
|
|
11
|
+
type QgridGrokVoice = (typeof QGRID_GROK_VOICES)[keyof typeof QGRID_GROK_VOICES][number];
|
|
12
|
+
/** `voice` for `qgrid.speech("openai/...")` (default `marin`); the server rejects any other with 400. */
|
|
13
|
+
declare const QGRID_OPENAI_SPEECH_VOICES: readonly ["marin", "cedar", "alloy", "ash", "ballad", "coral", "echo", "sage", "shimmer", "verse"];
|
|
14
|
+
type QgridOpenAISpeechVoice = (typeof QGRID_OPENAI_SPEECH_VOICES)[number];
|
|
15
|
+
/**
|
|
16
|
+
* `sessionConfig.voice` for `qgrid.realtime("openai/gpt-live-1")` and `voice` for the HTTP
|
|
17
|
+
* `liveCall` (default `cove`). The server rejects any other: OpenAI would accept the call and never
|
|
18
|
+
* start the session.
|
|
19
|
+
*/
|
|
20
|
+
declare const QGRID_LIVE_VOICES: readonly ["cove", "juniper", "maple", "spruce", "ember", "vale", "breeze", "arbor", "sol", "aube", "bubbie", "haetsal", "hanul", "himari", "leher", "miko", "neer", "porto", "rindo", "rio", "rivage", "selva", "sonna", "tinta", "viola"];
|
|
21
|
+
type QgridLiveVoice = (typeof QGRID_LIVE_VOICES)[number];
|
|
22
|
+
//#endregion
|
|
23
|
+
//#region src/realtime.d.ts
|
|
24
|
+
/** GPT-Live on a ChatGPT subscription token (server 2.19.2+; `history` needs 2.19.3+). */
|
|
25
|
+
type QgridRealtimeModel = "openai/gpt-live-1";
|
|
26
|
+
type QgridRealtimeConfig = {
|
|
27
|
+
/** Default: process.env.QGRID_URL, then http://localhost:44900. */
|
|
28
|
+
serverUrl?: string;
|
|
29
|
+
/** request_logs.project_name. Default: process.env.QGRID_PROJECT_NAME. */
|
|
30
|
+
projectName?: string;
|
|
31
|
+
};
|
|
32
|
+
/**
|
|
33
|
+
* `sessionConfig.providerOptions.qgrid`, read by `doCreateWebRTCSession` on the server. Build it
|
|
34
|
+
* in your server route; do not forward the browser's object, which the user controls.
|
|
35
|
+
*/
|
|
36
|
+
type QgridRealtimeProviderOptions = {
|
|
37
|
+
/** Exact OpenAI token (`openai/<name>`); omitted, qgrid picks one by quota. */
|
|
38
|
+
tokenName?: string;
|
|
39
|
+
/** false writes no request log for the session. */
|
|
40
|
+
logger?: boolean;
|
|
41
|
+
/** Earlier conversation the session starts from: at most 128 items. */
|
|
42
|
+
history?: Array<{
|
|
43
|
+
role: "user" | "assistant" | "developer";
|
|
44
|
+
text: string;
|
|
45
|
+
}>;
|
|
46
|
+
/** What qgrid answers the voice's hand-offs with (default openai/gpt-6-luna, low effort). */
|
|
47
|
+
delegation?: {
|
|
48
|
+
model?: string;
|
|
49
|
+
effort?: "low" | "medium" | "high";
|
|
50
|
+
webSearch?: boolean;
|
|
51
|
+
};
|
|
52
|
+
};
|
|
53
|
+
type WebRTCSessionOptions = Parameters<NonNullable<Experimental_RealtimeModelV4["doCreateWebRTCSession"]>>[0];
|
|
54
|
+
/** The AI SDK realtime model, with the SDP exchange always present and the request log ID typed. */
|
|
55
|
+
type QgridLiveRealtimeModel = Omit<Experimental_RealtimeModelV4, "doCreateWebRTCSession"> & {
|
|
56
|
+
doCreateWebRTCSession(options: WebRTCSessionOptions): Promise<{
|
|
57
|
+
sessionId: string;
|
|
58
|
+
sdp: string;
|
|
59
|
+
requestLogId?: number;
|
|
60
|
+
}>;
|
|
61
|
+
};
|
|
62
|
+
/**
|
|
63
|
+
* GPT-Live as an AI SDK realtime model. The browser runtime (`experimental_useRealtime` with
|
|
64
|
+
* `api: { session }`) owns the microphone, playback and the WebRTC peer; your server route calls
|
|
65
|
+
* `doCreateWebRTCSession`, which exchanges the SDP through the qgrid server. qgrid answers the
|
|
66
|
+
* voice's delegations itself and logs the transcript.
|
|
67
|
+
*/
|
|
68
|
+
declare function createQgridRealtime(modelId: QgridRealtimeModel, config?: QgridRealtimeConfig): QgridLiveRealtimeModel;
|
|
69
|
+
//#endregion
|
|
70
|
+
export { createQgridRealtime as a, QGRID_OPENAI_SPEECH_VOICES as c, QgridOpenAISpeechVoice as d, QgridRealtimeProviderOptions as i, QgridGrokVoice as l, QgridRealtimeConfig as n, QGRID_GROK_VOICES as o, QgridRealtimeModel as r, QGRID_LIVE_VOICES as s, QgridLiveRealtimeModel as t, QgridLiveVoice as u };
|
|
@@ -0,0 +1,2 @@
|
|
|
1
|
+
import { a as createQgridRealtime, c as QGRID_OPENAI_SPEECH_VOICES, d as QgridOpenAISpeechVoice, i as QgridRealtimeProviderOptions, l as QgridGrokVoice, n as QgridRealtimeConfig, o as QGRID_GROK_VOICES, r as QgridRealtimeModel, s as QGRID_LIVE_VOICES, t as QgridLiveRealtimeModel, u as QgridLiveVoice } from "./realtime-_tFFhM56.js";
|
|
2
|
+
export { QGRID_GROK_VOICES, QGRID_LIVE_VOICES, QGRID_OPENAI_SPEECH_VOICES, type QgridGrokVoice, QgridLiveRealtimeModel, type QgridLiveVoice, type QgridOpenAISpeechVoice, QgridRealtimeConfig, QgridRealtimeModel, QgridRealtimeProviderOptions, createQgridRealtime };
|
package/dist/realtime.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { n as QGRID_GROK_VOICES, r as QGRID_LIVE_VOICES, i as QGRID_OPENAI_SPEECH_VOICES, t as createQgridRealtime } from "./realtime-BgodcUud.js";
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cartanova/qgrid-ai-sdk",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.20.0",
|
|
4
4
|
"description": "AI SDK LanguageModelV3 and ImageModelV3 provider for qgrid",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -17,6 +17,10 @@
|
|
|
17
17
|
".": {
|
|
18
18
|
"import": "./dist/index.js",
|
|
19
19
|
"types": "./dist/index.d.ts"
|
|
20
|
+
},
|
|
21
|
+
"./realtime": {
|
|
22
|
+
"import": "./dist/realtime.js",
|
|
23
|
+
"types": "./dist/realtime.d.ts"
|
|
20
24
|
}
|
|
21
25
|
},
|
|
22
26
|
"dependencies": {
|
|
@@ -31,8 +35,8 @@
|
|
|
31
35
|
"zod": "^4.6.7"
|
|
32
36
|
},
|
|
33
37
|
"peerDependencies": {
|
|
34
|
-
"@ai-sdk/provider": "^4.0.
|
|
35
|
-
"ai": "^7.0.
|
|
38
|
+
"@ai-sdk/provider": "^4.0.15",
|
|
39
|
+
"ai": "^7.0.102",
|
|
36
40
|
"zod": "^3.25.76 || ^4.1.8"
|
|
37
41
|
},
|
|
38
42
|
"engines": {
|