@cartanova/qgrid-ai-sdk 2.16.0 → 2.17.1

package/README.ko.md

@@ -485,9 +485,9 @@ const speech = await generateSpeech({
 speech.audio.uint8Array;
 ```
 
-- `transcribe`는 bytes로 오디오 형식을 감지하고 실패하면 `audio/wav`를 씁니다. 서버는 `audio/*`가 아닌 값을 거부합니다. transcript에는 segment가 없습니다(`segments: []`).
+- `transcribe`는 bytes로 오디오 형식을 감지하고 실패하면 `audio/wav`를 씁니다. 서버는 `audio/*`가 아닌 값을 거부합니다. Grok transcript는 들린 단어마다 segment 하나를 담습니다(`segments: [{ text, startSecond, endSecond }]`, 2.17.0+). OpenAI realtime transcript에는 없습니다.
 - 음성 합성 텍스트는 최대 15,000자입니다. `voice`, `language`, `outputFormat`을 지정하지 않으면 서버 기본값 `ara`, `en`, `mp3`가 쓰이며, live 검증된 조합은 이것뿐입니다. 다른 voice, language(`@ai-sdk/xai`의 `auto` 포함), codec은 그대로 전달되지만 미검증입니다. 서버가 돌려준 오디오 MIME 타입은 `providerMetadata.qgrid.mediaType`에 있고, AI SDK의 `speech.audio.mediaType`은 bytes에서 추정한 값입니다.
-- `providerOptions: { xai: { withTimestamps: true } }`(2.16.0+, Grok 전용)를 주면 글자별 타이밍이 `providerMetadata.qgrid.audioTimestamps`에 담깁니다. `graphChars`는 입력 글자(공백·문장부호·음성 태그 포함), `graphTimes`는 같은 인덱스의 `[start, end]` 초이고, 전체 길이는 `providerMetadata.qgrid.duration`입니다. `@ai-sdk/xai`가 `providerMetadata.xai`에 담는 모양과 같습니다. 텍스트 정규화로 풀린 토큰은 첫 글자가 전체 구간을 가집니다. `qgrid.speech("openai/...")`는 타이밍 데이터가 없어 서버가 `withTimestamps`를 HTTP 400으로 거부합니다.
+- 단어 타이밍: `providerOptions: { xai: { withTimestamps: true } }`(2.16.0+, Grok) 또는 `providerOptions: { qgrid: { withTimestamps: true } }`(2.17.0+, Grok·OpenAI)를 주면 글자별 타이밍이 `providerMetadata.qgrid.audioTimestamps`에 담깁니다. `graphChars`는 입력 글자(공백·문장부호·음성 태그 포함), `graphTimes`는 같은 인덱스의 `[start, end]` 초이고, 전체 길이는 `providerMetadata.qgrid.duration`입니다. `@ai-sdk/xai`가 `providerMetadata.xai`에 담는 모양과 같습니다. `providerMetadata.qgrid.timestampsSource`는 Grok이면 `"provider"`(Grok이 직접 잰 값), OpenAI면 `"aligned"`입니다. OpenAI는 서버가 생성된 오디오를 Grok STT로 받아쓰고 단어 단위로 원문과 짝지어 만듭니다(못 들은 단어는 앞뒤 단어 사이 구간을 나눠 가짐). 타이밍 단계가 실패해도 오디오는 돌려주고, 메타데이터에 `timestampsError`와 AI SDK warning을 남깁니다. 텍스트 정규화로 풀린 토큰은 첫 글자가 전체 구간을 가집니다.
 
 ### OpenAI 음성 인식과 합성 (ChatGPT 구독)
 

package/README.md

@@ -497,9 +497,9 @@ const speech = await generateSpeech({
 speech.audio.uint8Array;
 ```
 
-- `transcribe` detects the audio type from its bytes and falls back to `audio/wav`; the server rejects anything that is not `audio/*`. Transcripts carry no segments (`segments: []`).
+- `transcribe` detects the audio type from its bytes and falls back to `audio/wav`; the server rejects anything that is not `audio/*`. Grok transcripts carry one segment per heard word (`segments: [{ text, startSecond, endSecond }]`, 2.17.0+); OpenAI realtime transcripts carry none.
 - Speech text is limited to 15,000 characters. Unset `voice`, `language` and `outputFormat` use the server defaults `ara`, `en` and `mp3`, the only combination verified live. Other voices, languages (including `@ai-sdk/xai`'s `auto`) and codecs are passed through but unverified. The server's audio MIME type is in `providerMetadata.qgrid.mediaType`; AI SDK's `speech.audio.mediaType` is sniffed from the bytes.
-- `providerOptions: { xai: { withTimestamps: true } }` (2.16.0+, Grok only) returns per-character timing in `providerMetadata.qgrid.audioTimestamps`: `graphChars` (input characters, spaces, punctuation and speech tags included) and the index-aligned `graphTimes` `[start, end]` seconds, plus `providerMetadata.qgrid.duration`. The shape matches what `@ai-sdk/xai` puts under `providerMetadata.xai`. Text normalization gives an expanded token's whole span to its first character. `qgrid.speech("openai/...")` has no timing data and the server rejects `withTimestamps` with HTTP 400.
+- Word timing: `providerOptions: { xai: { withTimestamps: true } }` (2.16.0+, Grok) or `providerOptions: { qgrid: { withTimestamps: true } }` (2.17.0+, Grok or OpenAI) returns per-character timing in `providerMetadata.qgrid.audioTimestamps`: `graphChars` (input characters, spaces, punctuation and speech tags included) and the index-aligned `graphTimes` `[start, end]` seconds, plus `providerMetadata.qgrid.duration`. The shape matches what `@ai-sdk/xai` puts under `providerMetadata.xai`. `providerMetadata.qgrid.timestampsSource` is `"provider"` for Grok, which times its own audio, and `"aligned"` for OpenAI, whose audio the server transcribes with Grok STT and aligns word by word to the text (unheard words share the gap between neighbours). If the timing step fails the audio is still returned, with `timestampsError` in the metadata and an AI SDK warning. Text normalization gives an expanded token's whole span to its first character.
 
 ### OpenAI transcription and speech (ChatGPT subscription)
 

package/dist/index.js

@@ -1531,8 +1531,31 @@ const speakOutputSchema = z.object({
 		graphChars: z.array(z.string()),
 		graphTimes: z.array(z.tuple([z.number(), z.number()]))
 	}).optional(),
-	durationSec: z.number().optional()
+	durationSec: z.number().optional(),
+	timestampsSource: z.enum(["provider", "aligned"]).optional(),
+	timestampsError: z.string().optional()
 });
+/**
+* `providerOptions.qgrid.withTimestamps` works for both providers (OpenAI has no official option
+* for it). `parse*QgridOptions` drop unknown keys, so it is read here directly.
+*/
+const qgridTimestampsOptionSchema = z.looseObject({ withTimestamps: z.boolean().optional() });
+function qgridWantsTimestamps(raw) {
+	return qgridTimestampsOptionSchema.safeParse(raw).data?.withTimestamps === true;
+}
+/** Shared speech metadata: timing fields plus a warning when requested timing could not be made. */
+function speechTimingMetadata(data, metadata, warnings) {
+	if (data.audioTimestamps) metadata.audioTimestamps = data.audioTimestamps;
+	if (data.durationSec !== void 0) metadata.duration = data.durationSec;
+	if (data.timestampsSource) metadata.timestampsSource = data.timestampsSource;
+	if (data.timestampsError) {
+		metadata.timestampsError = data.timestampsError;
+		warnings.push({
+			type: "other",
+			message: `timestamps unavailable: ${data.timestampsError}`
+		});
+	}
+}
 function base64ToBytes(data) {
 	if (typeof Buffer !== "undefined") return new Uint8Array(Buffer.from(data, "base64"));
 	const binary = atob(data);
@@ -1589,7 +1612,7 @@ function createQgridSpeech(modelId, config) {
 			if (options.voice !== void 0) requestParams.voice = options.voice;
 			if (options.language !== void 0) requestParams.language = options.language;
 			if (format !== void 0) requestParams.format = format;
-			if (xai.options?.withTimestamps === true) requestParams.withTimestamps = true;
+			if (xai.options?.withTimestamps === true || qgridWantsTimestamps(options.providerOptions?.qgrid)) requestParams.withTimestamps = true;
 			if (projectName) requestParams.projectName = projectName;
 			const params = {
 				...requestParams,
@@ -1611,8 +1634,7 @@ function createQgridSpeech(modelId, config) {
 				tokenName: data.tokenName
 			};
 			if (data.requestLogId !== void 0) qgridMetadata.requestLogId = data.requestLogId;
-			if (data.audioTimestamps) qgridMetadata.audioTimestamps = data.audioTimestamps;
-			if (data.durationSec !== void 0) qgridMetadata.duration = data.durationSec;
+			speechTimingMetadata(data, qgridMetadata, warnings);
 			return {
 				audio: base64ToBytes(data.audio),
 				warnings,
@@ -1633,6 +1655,8 @@ function createQgridSpeech(modelId, config) {
 * (default `marin`, Codex v2 list), `outputFormat` `wav` (default) or `pcm` — other formats
 * warn and fall back to wav, as the official provider falls back to mp3 — `openai.instructions`
 * as style guidance, `openai.speed` and `language` unsupported with the official warning text.
+* `providerOptions.qgrid.withTimestamps` returns timing the server infers by aligning a Grok STT
+* pass over the generated audio (`providerMetadata.qgrid.timestampsSource: "aligned"`).
 */
 function createOpenAISpeech(modelId, serverUrl, projectName) {
 	return {
@@ -1675,6 +1699,7 @@ function createOpenAISpeech(modelId, serverUrl, projectName) {
 			if (format !== void 0) requestParams.format = format;
 			const instructions = options.instructions ?? openai.options?.instructions ?? void 0;
 			if (instructions !== void 0) requestParams.instructions = instructions;
+			if (qgridWantsTimestamps(options.providerOptions?.qgrid)) requestParams.withTimestamps = true;
 			if (projectName) requestParams.projectName = projectName;
 			const params = {
 				...requestParams,
@@ -1696,6 +1721,7 @@ function createOpenAISpeech(modelId, serverUrl, projectName) {
 				tokenName: data.tokenName
 			};
 			if (data.requestLogId !== void 0) qgridMetadata.requestLogId = data.requestLogId;
+			speechTimingMetadata(data, qgridMetadata, warnings);
 			return {
 				audio: base64ToBytes(data.audio),
 				warnings,
@@ -1784,7 +1810,12 @@ const transcribeOutputSchema = z.object({
 	durationSec: z.number().optional(),
 	model: z.string(),
 	tokenName: z.string(),
-	requestLogId: z.number().optional()
+	requestLogId: z.number().optional(),
+	words: z.array(z.object({
+		text: z.string(),
+		start: z.number(),
+		end: z.number()
+	})).optional()
 });
 function normalizeMediaType(mediaType) {
 	const bare = mediaType.split(";")[0].trim().toLowerCase();
@@ -1840,7 +1871,11 @@ function createQgridTranscription(modelId, config) {
 			if (data.requestLogId !== void 0) qgridMetadata.requestLogId = data.requestLogId;
 			return {
 				text: data.text,
-				segments: [],
+				segments: (data.words ?? []).map((word) => ({
+					text: word.text,
+					startSecond: word.start,
+					endSecond: word.end
+				})),
 				language: data.language,
 				durationInSeconds: data.durationSec,
 				warnings,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cartanova/qgrid-ai-sdk",
-  "version": "2.16.0",
+  "version": "2.17.1",
   "description": "AI SDK LanguageModelV3 and ImageModelV3 provider for qgrid",
   "repository": {
     "type": "git",