@opencode/ai 0.0.0-dev-20039 → 0.0.0-dev-20041
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +86 -1
- package/dist/index.d.ts +4 -0
- package/dist/index.js +3 -0
- package/dist/media-model.d.ts +3 -3
- package/dist/media-model.js +3 -3
- package/dist/promise.d.ts +111 -3
- package/dist/promise.js +9 -1
- package/dist/protocols/bedrock-converse.js +2 -8
- package/dist/protocols/cartesia-speech.d.ts +127 -0
- package/dist/protocols/cartesia-speech.js +126 -0
- package/dist/protocols/deepgram-speech.d.ts +119 -0
- package/dist/protocols/deepgram-speech.js +90 -0
- package/dist/protocols/elevenlabs-speech.d.ts +122 -0
- package/dist/protocols/elevenlabs-speech.js +115 -0
- package/dist/protocols/google-speech.d.ts +160 -0
- package/dist/protocols/google-speech.js +122 -0
- package/dist/protocols/openai-speech.d.ts +116 -0
- package/dist/protocols/openai-speech.js +98 -0
- package/dist/protocols/utils/speech-stream.d.ts +49 -0
- package/dist/protocols/utils/speech-stream.js +67 -0
- package/dist/providers/cartesia.d.ts +24 -0
- package/dist/providers/cartesia.js +22 -0
- package/dist/providers/deepgram.d.ts +24 -0
- package/dist/providers/deepgram.js +28 -0
- package/dist/providers/elevenlabs.d.ts +24 -0
- package/dist/providers/elevenlabs.js +28 -0
- package/dist/providers/google.d.ts +5 -0
- package/dist/providers/google.js +3 -0
- package/dist/providers/index.d.ts +3 -0
- package/dist/providers/index.js +3 -0
- package/dist/providers/openai.d.ts +5 -0
- package/dist/providers/openai.js +6 -1
- package/dist/route/framing.d.ts +5 -1
- package/dist/route/framing.js +9 -0
- package/dist/route/media-protocol.d.ts +35 -2
- package/dist/route/media-protocol.js +22 -1
- package/dist/route/media.d.ts +15 -1
- package/dist/route/media.js +41 -7
- package/dist/speech-client.d.ts +21 -0
- package/dist/speech-client.js +25 -0
- package/dist/speech.d.ts +1150 -0
- package/dist/speech.js +119 -0
- package/dist/utils/bytes.d.ts +1 -0
- package/dist/utils/bytes.js +10 -0
- package/dist/utils/media-type.js +8 -1
- package/package.json +3 -3
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
import { Effect, Schema } from "effect";
|
|
2
|
+
import { Framing } from "../route/framing.js";
|
|
3
|
+
import { MediaProtocol } from "../route/media-protocol.js";
|
|
4
|
+
import { MediaRoute } from "../route/media.js";
|
|
5
|
+
import { AIError, ContentPolicyError, ProviderID, mergeJsonRecords } from "../schema/index.js";
|
|
6
|
+
import { SpeechModel } from "../speech.js";
|
|
7
|
+
import { ProviderShared } from "./shared.js";
|
|
8
|
+
import { SpeechStream } from "./utils/speech-stream.js";
|
|
9
|
+
const ADAPTER = "google-speech";
|
|
10
|
+
const NAME = "Google Speech";
|
|
11
|
+
const PROVIDER = ProviderID.make("google");
|
|
12
|
+
export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta";
|
|
13
|
+
const DEFAULT_SAMPLE_RATE = 24000;
|
|
14
|
+
// ---------------------------------------------------------------------------
|
|
15
|
+
// 3. Streaming event schema
|
|
16
|
+
// ---------------------------------------------------------------------------
|
|
17
|
+
const GenerateContentChunk = Schema.Struct({
|
|
18
|
+
candidates: Schema.optional(Schema.Array(Schema.Struct({
|
|
19
|
+
content: Schema.optional(Schema.Struct({
|
|
20
|
+
parts: Schema.optional(Schema.Array(Schema.Struct({
|
|
21
|
+
text: Schema.optional(Schema.String),
|
|
22
|
+
inlineData: Schema.optional(Schema.Struct({ mimeType: Schema.String, data: Schema.Uint8ArrayFromBase64 })),
|
|
23
|
+
}))),
|
|
24
|
+
})),
|
|
25
|
+
finishReason: Schema.optional(Schema.String),
|
|
26
|
+
}))),
|
|
27
|
+
promptFeedback: Schema.optional(Schema.Struct({ blockReason: Schema.optional(Schema.String), blockReasonMessage: Schema.optional(Schema.String) })),
|
|
28
|
+
usageMetadata: Schema.optional(Schema.Struct({
|
|
29
|
+
promptTokenCount: Schema.optional(Schema.Number),
|
|
30
|
+
candidatesTokenCount: Schema.optional(Schema.Number),
|
|
31
|
+
totalTokenCount: Schema.optional(Schema.Number),
|
|
32
|
+
})),
|
|
33
|
+
modelVersion: Schema.optional(Schema.String),
|
|
34
|
+
responseId: Schema.optional(Schema.String),
|
|
35
|
+
});
|
|
36
|
+
const decodeChunk = MediaProtocol.decodeFrame(ADAPTER, NAME, GenerateContentChunk);
|
|
37
|
+
// ---------------------------------------------------------------------------
|
|
38
|
+
// 5. Request body construction
|
|
39
|
+
// ---------------------------------------------------------------------------
|
|
40
|
+
const fromRequest = Effect.fn("GoogleSpeech.fromRequest")(function* (request) {
|
|
41
|
+
if (request.format !== undefined && request.format !== "pcm")
|
|
42
|
+
return yield* SpeechStream.unsupportedFormat(PROVIDER, ADAPTER, `${NAME} only returns raw PCM; request format "pcm" or omit it, then wrap the samples yourself`);
|
|
43
|
+
const voiceName = SpeechStream.voiceID(request.voice);
|
|
44
|
+
return MediaProtocol.json(mergeJsonRecords({
|
|
45
|
+
contents: [{ role: "user", parts: [{ text: request.text }] }],
|
|
46
|
+
generationConfig: mergeJsonRecords({
|
|
47
|
+
responseModalities: ["AUDIO"],
|
|
48
|
+
speechConfig: {
|
|
49
|
+
voiceConfig: voiceName === undefined ? undefined : { prebuiltVoiceConfig: { voiceName } },
|
|
50
|
+
languageCode: request.language,
|
|
51
|
+
},
|
|
52
|
+
}, request.providerOptions),
|
|
53
|
+
}, request.http?.body) ?? {});
|
|
54
|
+
});
|
|
55
|
+
// Only `gemini-3.1-flash-tts-preview` and later stream; earlier TTS models reject `streamGenerateContent`.
|
|
56
|
+
const path = (request) => request.mode === "stream"
|
|
57
|
+
? `/models/${request.model.id}:streamGenerateContent?alt=sse`
|
|
58
|
+
: `/models/${request.model.id}:generateContent`;
|
|
59
|
+
// ---------------------------------------------------------------------------
|
|
60
|
+
// 6. Stream parsing
|
|
61
|
+
// ---------------------------------------------------------------------------
|
|
62
|
+
const step = Effect.fn("GoogleSpeech.step")(function* (state, frame) {
|
|
63
|
+
const chunk = yield* decodeChunk(frame);
|
|
64
|
+
const blockReason = chunk.promptFeedback?.blockReason;
|
|
65
|
+
if (blockReason !== undefined)
|
|
66
|
+
return yield* new AIError({
|
|
67
|
+
reason: new ContentPolicyError({
|
|
68
|
+
message: `${NAME} blocked the prompt (${blockReason})${chunk.promptFeedback?.blockReasonMessage === undefined ? "" : `: ${chunk.promptFeedback.blockReasonMessage}`}`,
|
|
69
|
+
body: frame,
|
|
70
|
+
}),
|
|
71
|
+
});
|
|
72
|
+
const candidate = chunk.candidates?.[0];
|
|
73
|
+
const audio = (candidate?.content?.parts ?? []).flatMap((part) => part.inlineData === undefined ? [] : [part.inlineData]);
|
|
74
|
+
const next = {
|
|
75
|
+
...state,
|
|
76
|
+
mimeType: state.mimeType ?? audio[0]?.mimeType,
|
|
77
|
+
usage: chunk.usageMetadata ?? state.usage,
|
|
78
|
+
finishReason: candidate?.finishReason ?? state.finishReason,
|
|
79
|
+
modelVersion: chunk.modelVersion ?? state.modelVersion,
|
|
80
|
+
responseId: chunk.responseId ?? state.responseId,
|
|
81
|
+
};
|
|
82
|
+
return [next, audio.flatMap((part) => SpeechStream.delta(next, part.data)[1])];
|
|
83
|
+
});
|
|
84
|
+
const finish = (state) => {
|
|
85
|
+
const usage = state.usage;
|
|
86
|
+
const sampleRate = SpeechStream.sampleRate(state.mimeType) ?? DEFAULT_SAMPLE_RATE;
|
|
87
|
+
return SpeechStream.finish(ADAPTER, state, {
|
|
88
|
+
...SpeechStream.pcm("pcm_s16le", sampleRate, state.mimeType ?? `audio/L16;codec=pcm;rate=${sampleRate}`),
|
|
89
|
+
usage: usage === undefined
|
|
90
|
+
? undefined
|
|
91
|
+
: {
|
|
92
|
+
type: "tokens",
|
|
93
|
+
input: usage.promptTokenCount,
|
|
94
|
+
output: usage.candidatesTokenCount,
|
|
95
|
+
total: ProviderShared.totalTokens(usage.promptTokenCount, usage.candidatesTokenCount, usage.totalTokenCount),
|
|
96
|
+
details: { google: usage },
|
|
97
|
+
},
|
|
98
|
+
providerMetadata: {
|
|
99
|
+
google: { finishReason: state.finishReason, modelVersion: state.modelVersion, responseId: state.responseId },
|
|
100
|
+
},
|
|
101
|
+
detail: state.finishReason === undefined ? undefined : `finish reason: ${state.finishReason}`,
|
|
102
|
+
});
|
|
103
|
+
};
|
|
104
|
+
// ---------------------------------------------------------------------------
|
|
105
|
+
// 7. Protocol and route
|
|
106
|
+
// ---------------------------------------------------------------------------
|
|
107
|
+
export const protocol = MediaProtocol.stream({
|
|
108
|
+
id: ADAPTER,
|
|
109
|
+
name: NAME,
|
|
110
|
+
unsupported: ["instructions", "speed", "timestamps"],
|
|
111
|
+
body: { from: fromRequest },
|
|
112
|
+
// `generateContent` answers with one document shaped exactly like a streamed chunk, so it is a single frame.
|
|
113
|
+
frames: (bytes, context) => context.request.mode === "stream" ? Framing.sse.frame(bytes) : Framing.document.frame(bytes),
|
|
114
|
+
initial: () => ({ chunks: [] }),
|
|
115
|
+
step,
|
|
116
|
+
finish,
|
|
117
|
+
});
|
|
118
|
+
export const model = (input) => SpeechModel.fromRoute({ id: ADAPTER, provider: PROVIDER, protocol, baseURL: DEFAULT_BASE_URL, path: ({ request }) => path(request) }, input);
|
|
119
|
+
export const GoogleSpeech = {
|
|
120
|
+
protocol,
|
|
121
|
+
model,
|
|
122
|
+
};
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { MediaProtocol } from "../route/media-protocol.js";
|
|
2
|
+
import { MediaRoute } from "../route/media.js";
|
|
3
|
+
import { type MediaUsage } from "../schema/index.js";
|
|
4
|
+
import { SpeechModel, type SpeechRequestFor } from "../speech.js";
|
|
5
|
+
import { SpeechStream } from "./utils/speech-stream.js";
|
|
6
|
+
export declare const DEFAULT_BASE_URL = "https://api.openai.com/v1";
|
|
7
|
+
export declare const PATH = "/audio/speech";
|
|
8
|
+
export type OpenAISpeechOptions = Record<string, unknown>;
|
|
9
|
+
export type Request = SpeechRequestFor<OpenAISpeechOptions>;
|
|
10
|
+
interface State extends SpeechStream.Audio {
|
|
11
|
+
readonly done: boolean;
|
|
12
|
+
readonly usage?: MediaUsage;
|
|
13
|
+
}
|
|
14
|
+
export declare const protocol: MediaProtocol.Streamed<Request, {
|
|
15
|
+
readonly type: "audio-delta";
|
|
16
|
+
readonly chunk: Uint8Array<ArrayBufferLike>;
|
|
17
|
+
} | {
|
|
18
|
+
readonly type: "timestamps";
|
|
19
|
+
readonly items: readonly {
|
|
20
|
+
readonly text: string;
|
|
21
|
+
readonly startSeconds: number;
|
|
22
|
+
readonly endSeconds: number;
|
|
23
|
+
}[];
|
|
24
|
+
} | {
|
|
25
|
+
readonly type: "finish";
|
|
26
|
+
readonly audio: import("../media.js").Asset;
|
|
27
|
+
readonly providerMetadata?: {
|
|
28
|
+
readonly [x: string]: {
|
|
29
|
+
readonly [x: string]: unknown;
|
|
30
|
+
};
|
|
31
|
+
} | undefined;
|
|
32
|
+
readonly usage?: {
|
|
33
|
+
readonly type: "tokens";
|
|
34
|
+
readonly input?: number | undefined;
|
|
35
|
+
readonly output?: number | undefined;
|
|
36
|
+
readonly total?: number | undefined;
|
|
37
|
+
readonly details?: {
|
|
38
|
+
readonly [x: string]: unknown;
|
|
39
|
+
} | undefined;
|
|
40
|
+
} | {
|
|
41
|
+
readonly type: "seconds";
|
|
42
|
+
readonly seconds: number;
|
|
43
|
+
} | {
|
|
44
|
+
readonly type: "characters";
|
|
45
|
+
readonly characters: number;
|
|
46
|
+
} | {
|
|
47
|
+
readonly type: "credits";
|
|
48
|
+
readonly credits: number;
|
|
49
|
+
} | {
|
|
50
|
+
readonly type: "compute";
|
|
51
|
+
readonly seconds: number;
|
|
52
|
+
} | undefined;
|
|
53
|
+
readonly notices?: readonly {
|
|
54
|
+
readonly type: "other" | "moderated" | "filtered";
|
|
55
|
+
readonly message: string;
|
|
56
|
+
readonly providerMetadata?: {
|
|
57
|
+
readonly [x: string]: {
|
|
58
|
+
readonly [x: string]: unknown;
|
|
59
|
+
};
|
|
60
|
+
} | undefined;
|
|
61
|
+
}[] | undefined;
|
|
62
|
+
}, string | Uint8Array<ArrayBufferLike>, State>;
|
|
63
|
+
export declare const model: (input: MediaRoute.ModelInput) => SpeechModel<OpenAISpeechOptions>;
|
|
64
|
+
export declare const OpenAISpeech: {
|
|
65
|
+
readonly protocol: MediaProtocol.Streamed<Request, {
|
|
66
|
+
readonly type: "audio-delta";
|
|
67
|
+
readonly chunk: Uint8Array<ArrayBufferLike>;
|
|
68
|
+
} | {
|
|
69
|
+
readonly type: "timestamps";
|
|
70
|
+
readonly items: readonly {
|
|
71
|
+
readonly text: string;
|
|
72
|
+
readonly startSeconds: number;
|
|
73
|
+
readonly endSeconds: number;
|
|
74
|
+
}[];
|
|
75
|
+
} | {
|
|
76
|
+
readonly type: "finish";
|
|
77
|
+
readonly audio: import("../media.js").Asset;
|
|
78
|
+
readonly providerMetadata?: {
|
|
79
|
+
readonly [x: string]: {
|
|
80
|
+
readonly [x: string]: unknown;
|
|
81
|
+
};
|
|
82
|
+
} | undefined;
|
|
83
|
+
readonly usage?: {
|
|
84
|
+
readonly type: "tokens";
|
|
85
|
+
readonly input?: number | undefined;
|
|
86
|
+
readonly output?: number | undefined;
|
|
87
|
+
readonly total?: number | undefined;
|
|
88
|
+
readonly details?: {
|
|
89
|
+
readonly [x: string]: unknown;
|
|
90
|
+
} | undefined;
|
|
91
|
+
} | {
|
|
92
|
+
readonly type: "seconds";
|
|
93
|
+
readonly seconds: number;
|
|
94
|
+
} | {
|
|
95
|
+
readonly type: "characters";
|
|
96
|
+
readonly characters: number;
|
|
97
|
+
} | {
|
|
98
|
+
readonly type: "credits";
|
|
99
|
+
readonly credits: number;
|
|
100
|
+
} | {
|
|
101
|
+
readonly type: "compute";
|
|
102
|
+
readonly seconds: number;
|
|
103
|
+
} | undefined;
|
|
104
|
+
readonly notices?: readonly {
|
|
105
|
+
readonly type: "other" | "moderated" | "filtered";
|
|
106
|
+
readonly message: string;
|
|
107
|
+
readonly providerMetadata?: {
|
|
108
|
+
readonly [x: string]: {
|
|
109
|
+
readonly [x: string]: unknown;
|
|
110
|
+
};
|
|
111
|
+
} | undefined;
|
|
112
|
+
}[] | undefined;
|
|
113
|
+
}, string | Uint8Array<ArrayBufferLike>, State>;
|
|
114
|
+
readonly model: (input: MediaRoute.ModelInput) => SpeechModel<OpenAISpeechOptions>;
|
|
115
|
+
};
|
|
116
|
+
export {};
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import { Effect, Schema } from "effect";
|
|
2
|
+
import { Framing } from "../route/framing.js";
|
|
3
|
+
import { MediaProtocol } from "../route/media-protocol.js";
|
|
4
|
+
import { MediaRoute } from "../route/media.js";
|
|
5
|
+
import { ProviderID, mergeJsonRecords } from "../schema/index.js";
|
|
6
|
+
import { SpeechModel } from "../speech.js";
|
|
7
|
+
import { SpeechStream } from "./utils/speech-stream.js";
|
|
8
|
+
const ADAPTER = "openai-speech";
|
|
9
|
+
const NAME = "OpenAI Speech";
|
|
10
|
+
const PROVIDER = ProviderID.make("openai");
|
|
11
|
+
export const DEFAULT_BASE_URL = "https://api.openai.com/v1";
|
|
12
|
+
export const PATH = "/audio/speech";
|
|
13
|
+
/** `pcm` is raw 24 kHz, 16-bit signed little-endian mono samples without a header. */
|
|
14
|
+
const PCM_SAMPLE_RATE = 24000;
|
|
15
|
+
// ---------------------------------------------------------------------------
|
|
16
|
+
// 3. Streaming event schema
|
|
17
|
+
// ---------------------------------------------------------------------------
|
|
18
|
+
const SpeechStreamEvent = Schema.Union([
|
|
19
|
+
Schema.Struct({ type: Schema.Literal("speech.audio.delta"), audio: Schema.Uint8ArrayFromBase64 }),
|
|
20
|
+
Schema.Struct({
|
|
21
|
+
type: Schema.Literal("speech.audio.done"),
|
|
22
|
+
usage: Schema.optional(Schema.Struct({
|
|
23
|
+
input_tokens: Schema.optional(Schema.Number),
|
|
24
|
+
output_tokens: Schema.optional(Schema.Number),
|
|
25
|
+
total_tokens: Schema.optional(Schema.Number),
|
|
26
|
+
})),
|
|
27
|
+
}),
|
|
28
|
+
]);
|
|
29
|
+
const decodeEvent = MediaProtocol.decodeFrame(ADAPTER, NAME, SpeechStreamEvent);
|
|
30
|
+
// ---------------------------------------------------------------------------
|
|
31
|
+
// 5. Request body construction
|
|
32
|
+
// ---------------------------------------------------------------------------
|
|
33
|
+
// `sse` is not supported for `tts-1` or `tts-1-hd`; those models stream the raw audio body instead.
|
|
34
|
+
const supportsSse = (model) => !/^tts-1(-hd)?(-|$)/.test(model);
|
|
35
|
+
const fromRequest = Effect.fn("OpenAISpeech.fromRequest")(function* (request) {
|
|
36
|
+
return MediaProtocol.json(mergeJsonRecords({
|
|
37
|
+
model: request.model.id,
|
|
38
|
+
input: request.text,
|
|
39
|
+
voice: request.voice,
|
|
40
|
+
instructions: request.instructions,
|
|
41
|
+
response_format: request.format,
|
|
42
|
+
speed: request.speed,
|
|
43
|
+
stream_format: request.mode === "stream" && supportsSse(request.model.id) ? "sse" : undefined,
|
|
44
|
+
}, request.providerOptions, request.http?.body) ?? {});
|
|
45
|
+
});
|
|
46
|
+
// ---------------------------------------------------------------------------
|
|
47
|
+
// 6. Stream parsing
|
|
48
|
+
// ---------------------------------------------------------------------------
|
|
49
|
+
const isSse = (body) => body.type === "json" && body.value.stream_format === "sse";
|
|
50
|
+
const onEvent = Effect.fn("OpenAISpeech.onEvent")(function* (state, frame) {
|
|
51
|
+
const event = yield* decodeEvent(frame);
|
|
52
|
+
if (event.type === "speech.audio.delta")
|
|
53
|
+
return SpeechStream.delta(state, event.audio);
|
|
54
|
+
const usage = event.usage;
|
|
55
|
+
return [
|
|
56
|
+
{
|
|
57
|
+
...state,
|
|
58
|
+
done: true,
|
|
59
|
+
usage: usage === undefined
|
|
60
|
+
? undefined
|
|
61
|
+
: {
|
|
62
|
+
type: "tokens",
|
|
63
|
+
input: usage.input_tokens,
|
|
64
|
+
output: usage.output_tokens,
|
|
65
|
+
total: usage.total_tokens,
|
|
66
|
+
details: { openai: usage },
|
|
67
|
+
},
|
|
68
|
+
},
|
|
69
|
+
[],
|
|
70
|
+
];
|
|
71
|
+
});
|
|
72
|
+
const finish = (state, context) => {
|
|
73
|
+
if (isSse(context.body) && !state.done)
|
|
74
|
+
return Effect.fail(MediaProtocol.incomplete(ADAPTER));
|
|
75
|
+
const format = context.request.format ?? "mp3";
|
|
76
|
+
return SpeechStream.finish(ADAPTER, state, {
|
|
77
|
+
...(format === "pcm" ? SpeechStream.pcm("pcm_s16le", PCM_SAMPLE_RATE) : SpeechStream.container(format)),
|
|
78
|
+
usage: state.usage,
|
|
79
|
+
});
|
|
80
|
+
};
|
|
81
|
+
// ---------------------------------------------------------------------------
|
|
82
|
+
// 7. Protocol and route
|
|
83
|
+
// ---------------------------------------------------------------------------
|
|
84
|
+
export const protocol = MediaProtocol.stream({
|
|
85
|
+
id: ADAPTER,
|
|
86
|
+
name: NAME,
|
|
87
|
+
unsupported: ["language", "timestamps"],
|
|
88
|
+
body: { from: fromRequest },
|
|
89
|
+
frames: (bytes, context) => (isSse(context.body) ? Framing.sse.frame(bytes) : bytes),
|
|
90
|
+
initial: () => ({ chunks: [], done: false }),
|
|
91
|
+
step: SpeechStream.step(onEvent),
|
|
92
|
+
finish,
|
|
93
|
+
});
|
|
94
|
+
export const model = (input) => SpeechModel.fromRoute({ id: ADAPTER, provider: PROVIDER, protocol, baseURL: DEFAULT_BASE_URL, path: PATH }, input);
|
|
95
|
+
export const OpenAISpeech = {
|
|
96
|
+
protocol,
|
|
97
|
+
model,
|
|
98
|
+
};
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { Effect } from "effect";
|
|
2
|
+
import { Media } from "../../media.js";
|
|
3
|
+
import type { AIError, MediaUsage, ProviderID, ProviderMetadata } from "../../schema/index.js";
|
|
4
|
+
import { type SpeechEvent, type SpeechVoice } from "../../speech.js";
|
|
5
|
+
export interface Audio {
|
|
6
|
+
/** Appended in place: the route creates fresh state for each response through `initial`. */
|
|
7
|
+
readonly chunks: Array<Uint8Array>;
|
|
8
|
+
}
|
|
9
|
+
export type StepResult<State> = readonly [State, ReadonlyArray<SpeechEvent>];
|
|
10
|
+
/** Empty chunks (keep-alive records) emit nothing. */
|
|
11
|
+
export declare const delta: <State extends Audio>(state: State, chunk: Uint8Array) => StepResult<State>;
|
|
12
|
+
export declare const step: <State extends Audio>(onRecord: (state: State, frame: string) => Effect.Effect<StepResult<State>, AIError>) => (state: State, frame: string | Uint8Array) => Effect.Effect<StepResult<State>, AIError, never>;
|
|
13
|
+
export declare const timestamps: (texts: ReadonlyArray<string>, starts: ReadonlyArray<number>, ends: ReadonlyArray<number>) => ReadonlyArray<SpeechEvent>;
|
|
14
|
+
export declare const voiceID: (voice: SpeechVoice | undefined) => string | undefined;
|
|
15
|
+
export declare const container: (format: string, sampleRate?: number) => {
|
|
16
|
+
mediaType: string;
|
|
17
|
+
info: {
|
|
18
|
+
format: string;
|
|
19
|
+
sampleRate: number | undefined;
|
|
20
|
+
};
|
|
21
|
+
};
|
|
22
|
+
declare const PCM_MEDIA_TYPES: {
|
|
23
|
+
readonly pcm_s16le: "audio/pcm";
|
|
24
|
+
readonly pcm_f32le: "audio/pcm";
|
|
25
|
+
readonly pcm_mulaw: "audio/mulaw";
|
|
26
|
+
readonly pcm_alaw: "audio/alaw";
|
|
27
|
+
};
|
|
28
|
+
export type PcmEncoding = keyof typeof PCM_MEDIA_TYPES;
|
|
29
|
+
export declare const pcm: (encoding: PcmEncoding, sampleRate: number | undefined, mediaType?: string) => {
|
|
30
|
+
mediaType: string;
|
|
31
|
+
info: {
|
|
32
|
+
format: string;
|
|
33
|
+
encoding: "pcm_s16le" | "pcm_f32le" | "pcm_mulaw" | "pcm_alaw";
|
|
34
|
+
sampleRate: number | undefined;
|
|
35
|
+
channels: number;
|
|
36
|
+
};
|
|
37
|
+
};
|
|
38
|
+
export declare const sampleRate: (mediaType: string | undefined) => number | undefined;
|
|
39
|
+
export declare const unsupportedFormat: (provider: ProviderID, route: string, message: string) => AIError;
|
|
40
|
+
/** A declared `mediaType` wins over sniffing: headerless PCM can start with bytes that look like an MPEG frame sync. */
|
|
41
|
+
export declare const finish: (route: string, state: Audio, output: {
|
|
42
|
+
readonly mediaType: string | undefined;
|
|
43
|
+
readonly info?: Media.Info;
|
|
44
|
+
readonly usage?: MediaUsage;
|
|
45
|
+
readonly providerMetadata?: ProviderMetadata;
|
|
46
|
+
readonly detail?: string;
|
|
47
|
+
}) => Effect.Effect<ReadonlyArray<SpeechEvent>, AIError>;
|
|
48
|
+
export declare const headerUsage: (type: "characters" | "credits", value: string | undefined) => MediaUsage | undefined;
|
|
49
|
+
export * as SpeechStream from "./speech-stream.js";
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { Effect } from "effect";
|
|
2
|
+
import { Media } from "../../media.js";
|
|
3
|
+
import { MediaProtocol } from "../../route/media-protocol.js";
|
|
4
|
+
import { SpeechAudioDeltaEvent, SpeechFinishEvent, SpeechTimestampsEvent, } from "../../speech.js";
|
|
5
|
+
import { concatBytes } from "../../utils/bytes.js";
|
|
6
|
+
import { ProviderShared } from "../shared.js";
|
|
7
|
+
/** Empty chunks (keep-alive records) emit nothing. */
|
|
8
|
+
export const delta = (state, chunk) => {
|
|
9
|
+
if (chunk.length === 0)
|
|
10
|
+
return [state, []];
|
|
11
|
+
state.chunks.push(chunk);
|
|
12
|
+
return [state, [SpeechAudioDeltaEvent.make({ chunk })]];
|
|
13
|
+
};
|
|
14
|
+
export const step = (onRecord) => (state, frame) => typeof frame === "string" ? onRecord(state, frame) : Effect.succeed(delta(state, frame));
|
|
15
|
+
export const timestamps = (texts, starts, ends) => texts.length === 0
|
|
16
|
+
? []
|
|
17
|
+
: [
|
|
18
|
+
SpeechTimestampsEvent.make({
|
|
19
|
+
items: texts.map((text, index) => ({ text, startSeconds: starts[index] ?? 0, endSeconds: ends[index] ?? 0 })),
|
|
20
|
+
}),
|
|
21
|
+
];
|
|
22
|
+
export const voiceID = (voice) => (typeof voice === "object" ? voice.id : voice);
|
|
23
|
+
const CONTAINER_MEDIA_TYPES = {
|
|
24
|
+
mp3: "audio/mpeg",
|
|
25
|
+
wav: "audio/wav",
|
|
26
|
+
opus: "audio/ogg",
|
|
27
|
+
aac: "audio/aac",
|
|
28
|
+
flac: "audio/flac",
|
|
29
|
+
};
|
|
30
|
+
export const container = (format, sampleRate) => ({
|
|
31
|
+
mediaType: CONTAINER_MEDIA_TYPES[format],
|
|
32
|
+
info: { format, sampleRate },
|
|
33
|
+
});
|
|
34
|
+
const PCM_MEDIA_TYPES = {
|
|
35
|
+
pcm_s16le: "audio/pcm",
|
|
36
|
+
pcm_f32le: "audio/pcm",
|
|
37
|
+
pcm_mulaw: "audio/mulaw",
|
|
38
|
+
pcm_alaw: "audio/alaw",
|
|
39
|
+
};
|
|
40
|
+
export const pcm = (encoding, sampleRate, mediaType) => ({
|
|
41
|
+
mediaType: mediaType ?? PCM_MEDIA_TYPES[encoding],
|
|
42
|
+
info: { format: "pcm", encoding, sampleRate, channels: 1 },
|
|
43
|
+
});
|
|
44
|
+
export const sampleRate = (mediaType) => {
|
|
45
|
+
const rate = /rate=(\d+)/i.exec(mediaType ?? "")?.[1];
|
|
46
|
+
return rate === undefined ? undefined : Number(rate);
|
|
47
|
+
};
|
|
48
|
+
export const unsupportedFormat = (provider, route, message) => ProviderShared.unsupportedOperation({ operation: "media.format", provider, route, message });
|
|
49
|
+
/** A declared `mediaType` wins over sniffing: headerless PCM can start with bytes that look like an MPEG frame sync. */
|
|
50
|
+
export const finish = (route, state, output) => {
|
|
51
|
+
if (state.chunks.length === 0)
|
|
52
|
+
return Effect.fail(MediaProtocol.frameError(route, `The provider returned no audio${output.detail === undefined ? "" : ` (${output.detail})`}`));
|
|
53
|
+
return Effect.succeed([
|
|
54
|
+
SpeechFinishEvent.make({
|
|
55
|
+
audio: Media.bytes(concatBytes(state.chunks), output.mediaType, { info: output.info }),
|
|
56
|
+
usage: output.usage,
|
|
57
|
+
providerMetadata: output.providerMetadata,
|
|
58
|
+
}),
|
|
59
|
+
]);
|
|
60
|
+
};
|
|
61
|
+
export const headerUsage = (type, value) => {
|
|
62
|
+
const amount = Number(value);
|
|
63
|
+
if (!Number.isFinite(amount))
|
|
64
|
+
return undefined;
|
|
65
|
+
return type === "credits" ? { type, credits: amount } : { type, characters: amount };
|
|
66
|
+
};
|
|
67
|
+
export * as SpeechStream from "./speech-stream.js";
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { type ProviderAuthOption } from "../route/auth-options.js";
|
|
2
|
+
import { HttpOptions, type ModelID } from "../schema/index.js";
|
|
3
|
+
export type { CartesiaEncoding, CartesiaSpeechOptions } from "../protocols/cartesia-speech.js";
|
|
4
|
+
export declare const id: string & import("effect/Brand").Brand<"AI.ProviderID">;
|
|
5
|
+
export type Config = ProviderAuthOption<"optional"> & {
|
|
6
|
+
readonly baseURL?: string;
|
|
7
|
+
readonly headers?: Record<string, string>;
|
|
8
|
+
readonly http?: HttpOptions.Input;
|
|
9
|
+
};
|
|
10
|
+
export declare const configure: (input?: Config) => {
|
|
11
|
+
id: string & import("effect/Brand").Brand<"AI.ProviderID">;
|
|
12
|
+
speech: (modelID: string | ModelID) => import("../speech.js").SpeechModel<import("../protocols/cartesia-speech.js").CartesiaSpeechOptions>;
|
|
13
|
+
configure: (input?: Config) => /*elided*/ any;
|
|
14
|
+
};
|
|
15
|
+
export declare const provider: {
|
|
16
|
+
id: string & import("effect/Brand").Brand<"AI.ProviderID">;
|
|
17
|
+
speech: (modelID: string | ModelID) => import("../speech.js").SpeechModel<import("../protocols/cartesia-speech.js").CartesiaSpeechOptions>;
|
|
18
|
+
configure: (input?: Config) => {
|
|
19
|
+
id: string & import("effect/Brand").Brand<"AI.ProviderID">;
|
|
20
|
+
speech: (modelID: string | ModelID) => import("../speech.js").SpeechModel<import("../protocols/cartesia-speech.js").CartesiaSpeechOptions>;
|
|
21
|
+
configure: /*elided*/ any;
|
|
22
|
+
};
|
|
23
|
+
};
|
|
24
|
+
export declare const speech: (modelID: string | ModelID) => import("../speech.js").SpeechModel<import("../protocols/cartesia-speech.js").CartesiaSpeechOptions>;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { AuthOptions } from "../route/auth-options.js";
|
|
2
|
+
import { HttpOptions, ProviderID } from "../schema/index.js";
|
|
3
|
+
import { CartesiaSpeech, DEFAULT_BASE_URL } from "../protocols/cartesia-speech.js";
|
|
4
|
+
export const id = ProviderID.make("cartesia");
|
|
5
|
+
const baseURL = DEFAULT_BASE_URL;
|
|
6
|
+
const auth = (options) => AuthOptions.bearer(options, "CARTESIA_API_KEY");
|
|
7
|
+
export const configure = (input = {}) => {
|
|
8
|
+
const speech = (modelID) => CartesiaSpeech.model({
|
|
9
|
+
id: modelID,
|
|
10
|
+
auth: auth(input),
|
|
11
|
+
baseURL: input.baseURL ?? baseURL,
|
|
12
|
+
headers: input.headers,
|
|
13
|
+
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
|
|
14
|
+
});
|
|
15
|
+
return {
|
|
16
|
+
id,
|
|
17
|
+
speech,
|
|
18
|
+
configure,
|
|
19
|
+
};
|
|
20
|
+
};
|
|
21
|
+
export const provider = configure();
|
|
22
|
+
export const speech = provider.speech;
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { ProviderAuthOption } from "../route/auth-options.js";
|
|
2
|
+
import { HttpOptions, type ModelID } from "../schema/index.js";
|
|
3
|
+
export type { DeepgramEncoding, DeepgramSpeechOptions } from "../protocols/deepgram-speech.js";
|
|
4
|
+
export declare const id: string & import("effect/Brand").Brand<"AI.ProviderID">;
|
|
5
|
+
export type Config = ProviderAuthOption<"optional"> & {
|
|
6
|
+
readonly baseURL?: string;
|
|
7
|
+
readonly headers?: Record<string, string>;
|
|
8
|
+
readonly http?: HttpOptions.Input;
|
|
9
|
+
};
|
|
10
|
+
export declare const configure: (input?: Config) => {
|
|
11
|
+
id: string & import("effect/Brand").Brand<"AI.ProviderID">;
|
|
12
|
+
speech: (modelID: string | ModelID) => import("../speech.js").SpeechModel<import("../protocols/deepgram-speech.js").DeepgramSpeechOptions>;
|
|
13
|
+
configure: (input?: Config) => /*elided*/ any;
|
|
14
|
+
};
|
|
15
|
+
export declare const provider: {
|
|
16
|
+
id: string & import("effect/Brand").Brand<"AI.ProviderID">;
|
|
17
|
+
speech: (modelID: string | ModelID) => import("../speech.js").SpeechModel<import("../protocols/deepgram-speech.js").DeepgramSpeechOptions>;
|
|
18
|
+
configure: (input?: Config) => {
|
|
19
|
+
id: string & import("effect/Brand").Brand<"AI.ProviderID">;
|
|
20
|
+
speech: (modelID: string | ModelID) => import("../speech.js").SpeechModel<import("../protocols/deepgram-speech.js").DeepgramSpeechOptions>;
|
|
21
|
+
configure: /*elided*/ any;
|
|
22
|
+
};
|
|
23
|
+
};
|
|
24
|
+
export declare const speech: (modelID: string | ModelID) => import("../speech.js").SpeechModel<import("../protocols/deepgram-speech.js").DeepgramSpeechOptions>;
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { Auth } from "../route/auth.js";
|
|
2
|
+
import { HttpOptions, ProviderID } from "../schema/index.js";
|
|
3
|
+
import { DEFAULT_BASE_URL, DeepgramSpeech } from "../protocols/deepgram-speech.js";
|
|
4
|
+
export const id = ProviderID.make("deepgram");
|
|
5
|
+
const baseURL = DEFAULT_BASE_URL;
|
|
6
|
+
const auth = (options) => {
|
|
7
|
+
if ("auth" in options && options.auth)
|
|
8
|
+
return options.auth;
|
|
9
|
+
return Auth.optional("apiKey" in options ? options.apiKey : undefined, "apiKey")
|
|
10
|
+
.orElse(Auth.config("DEEPGRAM_API_KEY"))
|
|
11
|
+
.pipe(Auth.scheme("Token"));
|
|
12
|
+
};
|
|
13
|
+
export const configure = (input = {}) => {
|
|
14
|
+
const speech = (modelID) => DeepgramSpeech.model({
|
|
15
|
+
id: modelID,
|
|
16
|
+
auth: auth(input),
|
|
17
|
+
baseURL: input.baseURL ?? baseURL,
|
|
18
|
+
headers: input.headers,
|
|
19
|
+
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
|
|
20
|
+
});
|
|
21
|
+
return {
|
|
22
|
+
id,
|
|
23
|
+
speech,
|
|
24
|
+
configure,
|
|
25
|
+
};
|
|
26
|
+
};
|
|
27
|
+
export const provider = configure();
|
|
28
|
+
export const speech = provider.speech;
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { ProviderAuthOption } from "../route/auth-options.js";
|
|
2
|
+
import { HttpOptions, type ModelID } from "../schema/index.js";
|
|
3
|
+
export type { ElevenLabsOutputFormat, ElevenLabsSpeechOptions } from "../protocols/elevenlabs-speech.js";
|
|
4
|
+
export declare const id: string & import("effect/Brand").Brand<"AI.ProviderID">;
|
|
5
|
+
export type Config = ProviderAuthOption<"optional"> & {
|
|
6
|
+
readonly baseURL?: string;
|
|
7
|
+
readonly headers?: Record<string, string>;
|
|
8
|
+
readonly http?: HttpOptions.Input;
|
|
9
|
+
};
|
|
10
|
+
export declare const configure: (input?: Config) => {
|
|
11
|
+
id: string & import("effect/Brand").Brand<"AI.ProviderID">;
|
|
12
|
+
speech: (modelID: string | ModelID) => import("../speech.js").SpeechModel<import("../protocols/elevenlabs-speech.js").ElevenLabsSpeechOptions>;
|
|
13
|
+
configure: (input?: Config) => /*elided*/ any;
|
|
14
|
+
};
|
|
15
|
+
export declare const provider: {
|
|
16
|
+
id: string & import("effect/Brand").Brand<"AI.ProviderID">;
|
|
17
|
+
speech: (modelID: string | ModelID) => import("../speech.js").SpeechModel<import("../protocols/elevenlabs-speech.js").ElevenLabsSpeechOptions>;
|
|
18
|
+
configure: (input?: Config) => {
|
|
19
|
+
id: string & import("effect/Brand").Brand<"AI.ProviderID">;
|
|
20
|
+
speech: (modelID: string | ModelID) => import("../speech.js").SpeechModel<import("../protocols/elevenlabs-speech.js").ElevenLabsSpeechOptions>;
|
|
21
|
+
configure: /*elided*/ any;
|
|
22
|
+
};
|
|
23
|
+
};
|
|
24
|
+
export declare const speech: (modelID: string | ModelID) => import("../speech.js").SpeechModel<import("../protocols/elevenlabs-speech.js").ElevenLabsSpeechOptions>;
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { Auth } from "../route/auth.js";
|
|
2
|
+
import { HttpOptions, ProviderID } from "../schema/index.js";
|
|
3
|
+
import { DEFAULT_BASE_URL, ElevenLabsSpeech } from "../protocols/elevenlabs-speech.js";
|
|
4
|
+
export const id = ProviderID.make("elevenlabs");
|
|
5
|
+
const baseURL = DEFAULT_BASE_URL;
|
|
6
|
+
const auth = (options) => {
|
|
7
|
+
if ("auth" in options && options.auth)
|
|
8
|
+
return options.auth;
|
|
9
|
+
return Auth.optional("apiKey" in options ? options.apiKey : undefined, "apiKey")
|
|
10
|
+
.orElse(Auth.config("ELEVENLABS_API_KEY"))
|
|
11
|
+
.pipe(Auth.header("xi-api-key"));
|
|
12
|
+
};
|
|
13
|
+
export const configure = (input = {}) => {
|
|
14
|
+
const speech = (modelID) => ElevenLabsSpeech.model({
|
|
15
|
+
id: modelID,
|
|
16
|
+
auth: auth(input),
|
|
17
|
+
baseURL: input.baseURL ?? baseURL,
|
|
18
|
+
headers: input.headers,
|
|
19
|
+
http: input.http === undefined ? undefined : HttpOptions.make(input.http),
|
|
20
|
+
});
|
|
21
|
+
return {
|
|
22
|
+
id,
|
|
23
|
+
speech,
|
|
24
|
+
configure,
|
|
25
|
+
};
|
|
26
|
+
};
|
|
27
|
+
export const provider = configure();
|
|
28
|
+
export const speech = provider.speech;
|
|
@@ -4,6 +4,7 @@ import type { ProviderPackage } from "../provider-package.js";
|
|
|
4
4
|
import { type ModelID } from "../schema/index.js";
|
|
5
5
|
import { Gemini } from "../protocols/gemini.js";
|
|
6
6
|
export type { GoogleImageOptions } from "../protocols/google-images.js";
|
|
7
|
+
export type { GoogleSpeechOptions } from "../protocols/google-speech.js";
|
|
7
8
|
export type { GoogleVideoOptions } from "../protocols/google-video.js";
|
|
8
9
|
export type GeminiOptionsInput = Gemini.OptionsInput;
|
|
9
10
|
export type GeminiProviderOptionsInput = Gemini.ProviderOptionsInput;
|
|
@@ -106,6 +107,7 @@ export declare const configure: (input?: Config) => {
|
|
|
106
107
|
model: (modelID: string | ModelID) => import("../schema/options.js").LanguageModel<Gemini.OptionsInput, import("../route/client.js").CompactionOperations | undefined>;
|
|
107
108
|
image: (modelID: string | ModelID) => import("../image.js").ImageModel<import("../protocols/google-images.js").GoogleImageOptions>;
|
|
108
109
|
video: (modelID: string | ModelID) => import("../video.js").VideoModel<import("../protocols/google-video.js").GoogleVideoOptions>;
|
|
110
|
+
speech: (modelID: string | ModelID) => import("../speech.js").SpeechModel<import("../protocols/google-speech.js").GoogleSpeechOptions>;
|
|
109
111
|
configure: (input?: Config) => /*elided*/ any;
|
|
110
112
|
};
|
|
111
113
|
export declare const provider: {
|
|
@@ -113,14 +115,17 @@ export declare const provider: {
|
|
|
113
115
|
model: (modelID: string | ModelID) => import("../schema/options.js").LanguageModel<Gemini.OptionsInput, import("../route/client.js").CompactionOperations | undefined>;
|
|
114
116
|
image: (modelID: string | ModelID) => import("../image.js").ImageModel<import("../protocols/google-images.js").GoogleImageOptions>;
|
|
115
117
|
video: (modelID: string | ModelID) => import("../video.js").VideoModel<import("../protocols/google-video.js").GoogleVideoOptions>;
|
|
118
|
+
speech: (modelID: string | ModelID) => import("../speech.js").SpeechModel<import("../protocols/google-speech.js").GoogleSpeechOptions>;
|
|
116
119
|
configure: (input?: Config) => {
|
|
117
120
|
id: string & import("effect/Brand").Brand<"AI.ProviderID">;
|
|
118
121
|
model: (modelID: string | ModelID) => import("../schema/options.js").LanguageModel<Gemini.OptionsInput, import("../route/client.js").CompactionOperations | undefined>;
|
|
119
122
|
image: (modelID: string | ModelID) => import("../image.js").ImageModel<import("../protocols/google-images.js").GoogleImageOptions>;
|
|
120
123
|
video: (modelID: string | ModelID) => import("../video.js").VideoModel<import("../protocols/google-video.js").GoogleVideoOptions>;
|
|
124
|
+
speech: (modelID: string | ModelID) => import("../speech.js").SpeechModel<import("../protocols/google-speech.js").GoogleSpeechOptions>;
|
|
121
125
|
configure: /*elided*/ any;
|
|
122
126
|
};
|
|
123
127
|
};
|
|
124
128
|
export declare const model: ProviderPackage.Definition<Settings, Gemini.ProviderOptionsInput>["model"];
|
|
125
129
|
export declare const image: (modelID: string | ModelID) => import("../image.js").ImageModel<import("../protocols/google-images.js").GoogleImageOptions>;
|
|
126
130
|
export declare const video: (modelID: string | ModelID) => import("../video.js").VideoModel<import("../protocols/google-video.js").GoogleVideoOptions>;
|
|
131
|
+
export declare const speech: (modelID: string | ModelID) => import("../speech.js").SpeechModel<import("../protocols/google-speech.js").GoogleSpeechOptions>;
|