@opencode/ai 0.0.0-dev-20040 → 0.0.0-dev-20041
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +86 -1
- package/dist/index.d.ts +4 -0
- package/dist/index.js +3 -0
- package/dist/media-model.d.ts +3 -3
- package/dist/media-model.js +3 -3
- package/dist/promise.d.ts +111 -3
- package/dist/promise.js +9 -1
- package/dist/protocols/bedrock-converse.js +2 -8
- package/dist/protocols/cartesia-speech.d.ts +127 -0
- package/dist/protocols/cartesia-speech.js +126 -0
- package/dist/protocols/deepgram-speech.d.ts +119 -0
- package/dist/protocols/deepgram-speech.js +90 -0
- package/dist/protocols/elevenlabs-speech.d.ts +122 -0
- package/dist/protocols/elevenlabs-speech.js +115 -0
- package/dist/protocols/google-speech.d.ts +160 -0
- package/dist/protocols/google-speech.js +122 -0
- package/dist/protocols/openai-speech.d.ts +116 -0
- package/dist/protocols/openai-speech.js +98 -0
- package/dist/protocols/utils/speech-stream.d.ts +49 -0
- package/dist/protocols/utils/speech-stream.js +67 -0
- package/dist/providers/cartesia.d.ts +24 -0
- package/dist/providers/cartesia.js +22 -0
- package/dist/providers/deepgram.d.ts +24 -0
- package/dist/providers/deepgram.js +28 -0
- package/dist/providers/elevenlabs.d.ts +24 -0
- package/dist/providers/elevenlabs.js +28 -0
- package/dist/providers/google.d.ts +5 -0
- package/dist/providers/google.js +3 -0
- package/dist/providers/index.d.ts +3 -0
- package/dist/providers/index.js +3 -0
- package/dist/providers/openai.d.ts +5 -0
- package/dist/providers/openai.js +6 -1
- package/dist/route/framing.d.ts +5 -1
- package/dist/route/framing.js +9 -0
- package/dist/route/media-protocol.d.ts +35 -2
- package/dist/route/media-protocol.js +22 -1
- package/dist/route/media.d.ts +15 -1
- package/dist/route/media.js +41 -7
- package/dist/speech-client.d.ts +21 -0
- package/dist/speech-client.js +25 -0
- package/dist/speech.d.ts +1150 -0
- package/dist/speech.js +119 -0
- package/dist/utils/bytes.d.ts +1 -0
- package/dist/utils/bytes.js +10 -0
- package/dist/utils/media-type.js +8 -1
- package/package.json +3 -3
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import { MediaProtocol } from "../route/media-protocol.js";
|
|
2
|
+
import { MediaRoute } from "../route/media.js";
|
|
3
|
+
import { SpeechModel, type SpeechRequestFor } from "../speech.js";
|
|
4
|
+
import { SpeechStream } from "./utils/speech-stream.js";
|
|
5
|
+
export declare const DEFAULT_BASE_URL = "https://api.deepgram.com";
|
|
6
|
+
export declare const PATH = "/v1/speak";
|
|
7
|
+
export type DeepgramSpeechString<Known extends string> = Known | (string & {});
|
|
8
|
+
export type DeepgramEncoding = DeepgramSpeechString<"linear16" | "mulaw" | "alaw" | "mp3" | "opus" | "flac" | "aac">;
|
|
9
|
+
export type DeepgramSpeechOptions = {
|
|
10
|
+
readonly encoding?: DeepgramEncoding;
|
|
11
|
+
readonly container?: DeepgramSpeechString<"wav" | "ogg" | "none">;
|
|
12
|
+
readonly sampleRate?: number;
|
|
13
|
+
readonly bitRate?: number;
|
|
14
|
+
readonly mip_opt_out?: boolean;
|
|
15
|
+
readonly tag?: string;
|
|
16
|
+
} & Record<string, unknown>;
|
|
17
|
+
export type Request = SpeechRequestFor<DeepgramSpeechOptions>;
|
|
18
|
+
export declare const protocol: MediaProtocol.Streamed<Request, {
|
|
19
|
+
readonly type: "audio-delta";
|
|
20
|
+
readonly chunk: Uint8Array<ArrayBufferLike>;
|
|
21
|
+
} | {
|
|
22
|
+
readonly type: "timestamps";
|
|
23
|
+
readonly items: readonly {
|
|
24
|
+
readonly text: string;
|
|
25
|
+
readonly startSeconds: number;
|
|
26
|
+
readonly endSeconds: number;
|
|
27
|
+
}[];
|
|
28
|
+
} | {
|
|
29
|
+
readonly type: "finish";
|
|
30
|
+
readonly audio: import("../media.js").Asset;
|
|
31
|
+
readonly providerMetadata?: {
|
|
32
|
+
readonly [x: string]: {
|
|
33
|
+
readonly [x: string]: unknown;
|
|
34
|
+
};
|
|
35
|
+
} | undefined;
|
|
36
|
+
readonly usage?: {
|
|
37
|
+
readonly type: "tokens";
|
|
38
|
+
readonly input?: number | undefined;
|
|
39
|
+
readonly output?: number | undefined;
|
|
40
|
+
readonly total?: number | undefined;
|
|
41
|
+
readonly details?: {
|
|
42
|
+
readonly [x: string]: unknown;
|
|
43
|
+
} | undefined;
|
|
44
|
+
} | {
|
|
45
|
+
readonly type: "seconds";
|
|
46
|
+
readonly seconds: number;
|
|
47
|
+
} | {
|
|
48
|
+
readonly type: "characters";
|
|
49
|
+
readonly characters: number;
|
|
50
|
+
} | {
|
|
51
|
+
readonly type: "credits";
|
|
52
|
+
readonly credits: number;
|
|
53
|
+
} | {
|
|
54
|
+
readonly type: "compute";
|
|
55
|
+
readonly seconds: number;
|
|
56
|
+
} | undefined;
|
|
57
|
+
readonly notices?: readonly {
|
|
58
|
+
readonly type: "other" | "moderated" | "filtered";
|
|
59
|
+
readonly message: string;
|
|
60
|
+
readonly providerMetadata?: {
|
|
61
|
+
readonly [x: string]: {
|
|
62
|
+
readonly [x: string]: unknown;
|
|
63
|
+
};
|
|
64
|
+
} | undefined;
|
|
65
|
+
}[] | undefined;
|
|
66
|
+
}, Uint8Array<ArrayBufferLike>, SpeechStream.Audio>;
|
|
67
|
+
export declare const model: (input: MediaRoute.ModelInput) => SpeechModel<DeepgramSpeechOptions>;
|
|
68
|
+
export declare const DeepgramSpeech: {
|
|
69
|
+
readonly protocol: MediaProtocol.Streamed<Request, {
|
|
70
|
+
readonly type: "audio-delta";
|
|
71
|
+
readonly chunk: Uint8Array<ArrayBufferLike>;
|
|
72
|
+
} | {
|
|
73
|
+
readonly type: "timestamps";
|
|
74
|
+
readonly items: readonly {
|
|
75
|
+
readonly text: string;
|
|
76
|
+
readonly startSeconds: number;
|
|
77
|
+
readonly endSeconds: number;
|
|
78
|
+
}[];
|
|
79
|
+
} | {
|
|
80
|
+
readonly type: "finish";
|
|
81
|
+
readonly audio: import("../media.js").Asset;
|
|
82
|
+
readonly providerMetadata?: {
|
|
83
|
+
readonly [x: string]: {
|
|
84
|
+
readonly [x: string]: unknown;
|
|
85
|
+
};
|
|
86
|
+
} | undefined;
|
|
87
|
+
readonly usage?: {
|
|
88
|
+
readonly type: "tokens";
|
|
89
|
+
readonly input?: number | undefined;
|
|
90
|
+
readonly output?: number | undefined;
|
|
91
|
+
readonly total?: number | undefined;
|
|
92
|
+
readonly details?: {
|
|
93
|
+
readonly [x: string]: unknown;
|
|
94
|
+
} | undefined;
|
|
95
|
+
} | {
|
|
96
|
+
readonly type: "seconds";
|
|
97
|
+
readonly seconds: number;
|
|
98
|
+
} | {
|
|
99
|
+
readonly type: "characters";
|
|
100
|
+
readonly characters: number;
|
|
101
|
+
} | {
|
|
102
|
+
readonly type: "credits";
|
|
103
|
+
readonly credits: number;
|
|
104
|
+
} | {
|
|
105
|
+
readonly type: "compute";
|
|
106
|
+
readonly seconds: number;
|
|
107
|
+
} | undefined;
|
|
108
|
+
readonly notices?: readonly {
|
|
109
|
+
readonly type: "other" | "moderated" | "filtered";
|
|
110
|
+
readonly message: string;
|
|
111
|
+
readonly providerMetadata?: {
|
|
112
|
+
readonly [x: string]: {
|
|
113
|
+
readonly [x: string]: unknown;
|
|
114
|
+
};
|
|
115
|
+
} | undefined;
|
|
116
|
+
}[] | undefined;
|
|
117
|
+
}, Uint8Array<ArrayBufferLike>, SpeechStream.Audio>;
|
|
118
|
+
readonly model: (input: MediaRoute.ModelInput) => SpeechModel<DeepgramSpeechOptions>;
|
|
119
|
+
};
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { Effect } from "effect";
|
|
2
|
+
import { MediaProtocol } from "../route/media-protocol.js";
|
|
3
|
+
import { MediaRoute } from "../route/media.js";
|
|
4
|
+
import { ProviderID, mergeJsonRecords } from "../schema/index.js";
|
|
5
|
+
import { SpeechModel } from "../speech.js";
|
|
6
|
+
import { SpeechStream } from "./utils/speech-stream.js";
|
|
7
|
+
const ADAPTER = "deepgram-speech";
|
|
8
|
+
const NAME = "Deepgram";
|
|
9
|
+
const PROVIDER = ProviderID.make("deepgram");
|
|
10
|
+
export const DEFAULT_BASE_URL = "https://api.deepgram.com";
|
|
11
|
+
export const PATH = "/v1/speak";
|
|
12
|
+
// ---------------------------------------------------------------------------
|
|
13
|
+
// 5. Request body construction
|
|
14
|
+
// ---------------------------------------------------------------------------
|
|
15
|
+
const FORMATS = {
|
|
16
|
+
mp3: { encoding: "mp3" },
|
|
17
|
+
wav: { encoding: "linear16", container: "wav" },
|
|
18
|
+
pcm: { encoding: "linear16", container: "none" },
|
|
19
|
+
opus: { encoding: "opus" },
|
|
20
|
+
flac: { encoding: "flac" },
|
|
21
|
+
aac: { encoding: "aac" },
|
|
22
|
+
};
|
|
23
|
+
const queryValue = (value) => typeof value === "string" || typeof value === "number" || typeof value === "boolean" ? String(value) : undefined;
|
|
24
|
+
const queryParameters = (request) => {
|
|
25
|
+
const format = request.format === undefined ? undefined : FORMATS[request.format];
|
|
26
|
+
const { encoding, container, sampleRate, bitRate, ...native } = request.providerOptions ?? {};
|
|
27
|
+
return Object.fromEntries(Object.entries({
|
|
28
|
+
...native,
|
|
29
|
+
model: request.model.id,
|
|
30
|
+
encoding: encoding ?? format?.encoding,
|
|
31
|
+
container: container ?? format?.container,
|
|
32
|
+
sample_rate: sampleRate,
|
|
33
|
+
bit_rate: bitRate,
|
|
34
|
+
speed: request.speed,
|
|
35
|
+
}).flatMap(([key, value]) => {
|
|
36
|
+
const text = queryValue(value);
|
|
37
|
+
return text === undefined ? [] : [[key, text]];
|
|
38
|
+
}));
|
|
39
|
+
};
|
|
40
|
+
const fromRequest = Effect.fn("DeepgramSpeech.fromRequest")(function* (request) {
|
|
41
|
+
if (request.format !== undefined &&
|
|
42
|
+
FORMATS[request.format] === undefined &&
|
|
43
|
+
request.providerOptions?.encoding === undefined)
|
|
44
|
+
return yield* SpeechStream.unsupportedFormat(PROVIDER, ADAPTER, `${NAME} has no encoding for format "${request.format}"; pass providerOptions.encoding`);
|
|
45
|
+
return MediaProtocol.json(mergeJsonRecords({ text: request.text }, request.http?.body) ?? {}, queryParameters(request));
|
|
46
|
+
});
|
|
47
|
+
// ---------------------------------------------------------------------------
|
|
48
|
+
// 6. Stream parsing
|
|
49
|
+
// ---------------------------------------------------------------------------
|
|
50
|
+
const HEADERLESS_ENCODINGS = {
|
|
51
|
+
linear16: "pcm_s16le",
|
|
52
|
+
mulaw: "pcm_mulaw",
|
|
53
|
+
alaw: "pcm_alaw",
|
|
54
|
+
};
|
|
55
|
+
const finish = (state, context) => {
|
|
56
|
+
const headers = context.http.headers;
|
|
57
|
+
const mediaType = headers["content-type"];
|
|
58
|
+
const query = queryParameters(context.request);
|
|
59
|
+
const encoding = HEADERLESS_ENCODINGS[query.encoding ?? ""];
|
|
60
|
+
const requestID = headers["dg-request-id"];
|
|
61
|
+
const modelName = headers["dg-model-name"];
|
|
62
|
+
return SpeechStream.finish(ADAPTER, state, {
|
|
63
|
+
...(query.container === "none" && encoding !== undefined
|
|
64
|
+
? SpeechStream.pcm(encoding, SpeechStream.sampleRate(mediaType), mediaType)
|
|
65
|
+
: // Deepgram's default encoding is MP3; WAV is a container around any encoding.
|
|
66
|
+
{ mediaType, info: { format: query.container === "wav" ? "wav" : (query.encoding ?? "mp3") } }),
|
|
67
|
+
usage: SpeechStream.headerUsage("characters", headers["dg-char-count"]),
|
|
68
|
+
providerMetadata: requestID === undefined && modelName === undefined
|
|
69
|
+
? undefined
|
|
70
|
+
: { deepgram: { requestId: requestID, modelName } },
|
|
71
|
+
});
|
|
72
|
+
};
|
|
73
|
+
// ---------------------------------------------------------------------------
|
|
74
|
+
// 7. Protocol and route
|
|
75
|
+
// ---------------------------------------------------------------------------
|
|
76
|
+
export const protocol = MediaProtocol.stream({
|
|
77
|
+
id: ADAPTER,
|
|
78
|
+
name: NAME,
|
|
79
|
+
unsupported: ["voice", "language", "instructions", "timestamps"],
|
|
80
|
+
body: { from: fromRequest },
|
|
81
|
+
frames: (bytes) => bytes,
|
|
82
|
+
initial: () => ({ chunks: [] }),
|
|
83
|
+
step: (state, frame) => Effect.succeed(SpeechStream.delta(state, frame)),
|
|
84
|
+
finish,
|
|
85
|
+
});
|
|
86
|
+
export const model = (input) => SpeechModel.fromRoute({ id: ADAPTER, provider: PROVIDER, protocol, baseURL: DEFAULT_BASE_URL, path: PATH }, input);
|
|
87
|
+
export const DeepgramSpeech = {
|
|
88
|
+
protocol,
|
|
89
|
+
model,
|
|
90
|
+
};
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
import { MediaProtocol } from "../route/media-protocol.js";
|
|
2
|
+
import { MediaRoute } from "../route/media.js";
|
|
3
|
+
import { SpeechModel, type SpeechRequestFor } from "../speech.js";
|
|
4
|
+
import { SpeechStream } from "./utils/speech-stream.js";
|
|
5
|
+
export declare const DEFAULT_BASE_URL = "https://api.elevenlabs.io";
|
|
6
|
+
export declare const PATH = "/v1/text-to-speech";
|
|
7
|
+
export type ElevenLabsSpeechString<Known extends string> = Known | (string & {});
|
|
8
|
+
export type ElevenLabsOutputFormat = ElevenLabsSpeechString<"mp3_22050_32" | "mp3_24000_48" | "mp3_44100_32" | "mp3_44100_64" | "mp3_44100_96" | "mp3_44100_128" | "mp3_44100_192" | "pcm_8000" | "pcm_16000" | "pcm_22050" | "pcm_24000" | "pcm_32000" | "pcm_44100" | "pcm_48000" | "wav_8000" | "wav_16000" | "wav_22050" | "wav_24000" | "wav_32000" | "wav_44100" | "wav_48000" | "ulaw_8000" | "alaw_8000" | "opus_48000_32" | "opus_48000_64" | "opus_48000_96" | "opus_48000_128" | "opus_48000_192">;
|
|
9
|
+
export type ElevenLabsSpeechOptions = {
|
|
10
|
+
readonly outputFormat?: ElevenLabsOutputFormat;
|
|
11
|
+
readonly voice_settings?: {
|
|
12
|
+
readonly stability?: number;
|
|
13
|
+
readonly similarity_boost?: number;
|
|
14
|
+
readonly style?: number;
|
|
15
|
+
readonly use_speaker_boost?: boolean;
|
|
16
|
+
};
|
|
17
|
+
readonly seed?: number;
|
|
18
|
+
readonly apply_text_normalization?: ElevenLabsSpeechString<"auto" | "on" | "off">;
|
|
19
|
+
} & Record<string, unknown>;
|
|
20
|
+
export type Request = SpeechRequestFor<ElevenLabsSpeechOptions>;
|
|
21
|
+
export declare const protocol: MediaProtocol.Streamed<Request, {
|
|
22
|
+
readonly type: "audio-delta";
|
|
23
|
+
readonly chunk: Uint8Array<ArrayBufferLike>;
|
|
24
|
+
} | {
|
|
25
|
+
readonly type: "timestamps";
|
|
26
|
+
readonly items: readonly {
|
|
27
|
+
readonly text: string;
|
|
28
|
+
readonly startSeconds: number;
|
|
29
|
+
readonly endSeconds: number;
|
|
30
|
+
}[];
|
|
31
|
+
} | {
|
|
32
|
+
readonly type: "finish";
|
|
33
|
+
readonly audio: import("../media.js").Asset;
|
|
34
|
+
readonly providerMetadata?: {
|
|
35
|
+
readonly [x: string]: {
|
|
36
|
+
readonly [x: string]: unknown;
|
|
37
|
+
};
|
|
38
|
+
} | undefined;
|
|
39
|
+
readonly usage?: {
|
|
40
|
+
readonly type: "tokens";
|
|
41
|
+
readonly input?: number | undefined;
|
|
42
|
+
readonly output?: number | undefined;
|
|
43
|
+
readonly total?: number | undefined;
|
|
44
|
+
readonly details?: {
|
|
45
|
+
readonly [x: string]: unknown;
|
|
46
|
+
} | undefined;
|
|
47
|
+
} | {
|
|
48
|
+
readonly type: "seconds";
|
|
49
|
+
readonly seconds: number;
|
|
50
|
+
} | {
|
|
51
|
+
readonly type: "characters";
|
|
52
|
+
readonly characters: number;
|
|
53
|
+
} | {
|
|
54
|
+
readonly type: "credits";
|
|
55
|
+
readonly credits: number;
|
|
56
|
+
} | {
|
|
57
|
+
readonly type: "compute";
|
|
58
|
+
readonly seconds: number;
|
|
59
|
+
} | undefined;
|
|
60
|
+
readonly notices?: readonly {
|
|
61
|
+
readonly type: "other" | "moderated" | "filtered";
|
|
62
|
+
readonly message: string;
|
|
63
|
+
readonly providerMetadata?: {
|
|
64
|
+
readonly [x: string]: {
|
|
65
|
+
readonly [x: string]: unknown;
|
|
66
|
+
};
|
|
67
|
+
} | undefined;
|
|
68
|
+
}[] | undefined;
|
|
69
|
+
}, string | Uint8Array<ArrayBufferLike>, SpeechStream.Audio>;
|
|
70
|
+
export declare const model: (input: MediaRoute.ModelInput) => SpeechModel<ElevenLabsSpeechOptions>;
|
|
71
|
+
export declare const ElevenLabsSpeech: {
|
|
72
|
+
readonly protocol: MediaProtocol.Streamed<Request, {
|
|
73
|
+
readonly type: "audio-delta";
|
|
74
|
+
readonly chunk: Uint8Array<ArrayBufferLike>;
|
|
75
|
+
} | {
|
|
76
|
+
readonly type: "timestamps";
|
|
77
|
+
readonly items: readonly {
|
|
78
|
+
readonly text: string;
|
|
79
|
+
readonly startSeconds: number;
|
|
80
|
+
readonly endSeconds: number;
|
|
81
|
+
}[];
|
|
82
|
+
} | {
|
|
83
|
+
readonly type: "finish";
|
|
84
|
+
readonly audio: import("../media.js").Asset;
|
|
85
|
+
readonly providerMetadata?: {
|
|
86
|
+
readonly [x: string]: {
|
|
87
|
+
readonly [x: string]: unknown;
|
|
88
|
+
};
|
|
89
|
+
} | undefined;
|
|
90
|
+
readonly usage?: {
|
|
91
|
+
readonly type: "tokens";
|
|
92
|
+
readonly input?: number | undefined;
|
|
93
|
+
readonly output?: number | undefined;
|
|
94
|
+
readonly total?: number | undefined;
|
|
95
|
+
readonly details?: {
|
|
96
|
+
readonly [x: string]: unknown;
|
|
97
|
+
} | undefined;
|
|
98
|
+
} | {
|
|
99
|
+
readonly type: "seconds";
|
|
100
|
+
readonly seconds: number;
|
|
101
|
+
} | {
|
|
102
|
+
readonly type: "characters";
|
|
103
|
+
readonly characters: number;
|
|
104
|
+
} | {
|
|
105
|
+
readonly type: "credits";
|
|
106
|
+
readonly credits: number;
|
|
107
|
+
} | {
|
|
108
|
+
readonly type: "compute";
|
|
109
|
+
readonly seconds: number;
|
|
110
|
+
} | undefined;
|
|
111
|
+
readonly notices?: readonly {
|
|
112
|
+
readonly type: "other" | "moderated" | "filtered";
|
|
113
|
+
readonly message: string;
|
|
114
|
+
readonly providerMetadata?: {
|
|
115
|
+
readonly [x: string]: {
|
|
116
|
+
readonly [x: string]: unknown;
|
|
117
|
+
};
|
|
118
|
+
} | undefined;
|
|
119
|
+
}[] | undefined;
|
|
120
|
+
}, string | Uint8Array<ArrayBufferLike>, SpeechStream.Audio>;
|
|
121
|
+
readonly model: (input: MediaRoute.ModelInput) => SpeechModel<ElevenLabsSpeechOptions>;
|
|
122
|
+
};
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { Effect, Schema } from "effect";
|
|
2
|
+
import { Framing } from "../route/framing.js";
|
|
3
|
+
import { MediaProtocol } from "../route/media-protocol.js";
|
|
4
|
+
import { MediaRoute } from "../route/media.js";
|
|
5
|
+
import { ProviderID, mergeJsonRecords } from "../schema/index.js";
|
|
6
|
+
import { SpeechModel } from "../speech.js";
|
|
7
|
+
import { ProviderShared, optionalNull } from "./shared.js";
|
|
8
|
+
import { SpeechStream } from "./utils/speech-stream.js";
|
|
9
|
+
const ADAPTER = "elevenlabs-speech";
|
|
10
|
+
const NAME = "ElevenLabs";
|
|
11
|
+
const PROVIDER = ProviderID.make("elevenlabs");
|
|
12
|
+
export const DEFAULT_BASE_URL = "https://api.elevenlabs.io";
|
|
13
|
+
export const PATH = "/v1/text-to-speech";
|
|
14
|
+
// ---------------------------------------------------------------------------
|
|
15
|
+
// 3. Streaming event schema
|
|
16
|
+
// ---------------------------------------------------------------------------
|
|
17
|
+
const Alignment = Schema.Struct({
|
|
18
|
+
characters: Schema.Array(Schema.String),
|
|
19
|
+
character_start_times_seconds: Schema.Array(Schema.Number),
|
|
20
|
+
character_end_times_seconds: Schema.Array(Schema.Number),
|
|
21
|
+
});
|
|
22
|
+
const TimestampedAudio = Schema.Struct({
|
|
23
|
+
audio_base64: Schema.Uint8ArrayFromBase64,
|
|
24
|
+
alignment: optionalNull(Alignment),
|
|
25
|
+
});
|
|
26
|
+
const decodeRecord = MediaProtocol.decodeFrame(ADAPTER, NAME, TimestampedAudio);
|
|
27
|
+
// ---------------------------------------------------------------------------
|
|
28
|
+
// 5. Request body construction
|
|
29
|
+
// ---------------------------------------------------------------------------
|
|
30
|
+
const OUTPUT_FORMATS = {
|
|
31
|
+
mp3: "mp3_44100_128",
|
|
32
|
+
pcm: "pcm_24000",
|
|
33
|
+
wav: "wav_24000",
|
|
34
|
+
opus: "opus_48000_64",
|
|
35
|
+
};
|
|
36
|
+
/** WAV is served only by the non-streaming endpoints. */
|
|
37
|
+
const outputFormat = Effect.fn("ElevenLabsSpeech.outputFormat")(function* (request) {
|
|
38
|
+
const format = request.providerOptions?.outputFormat ?? OUTPUT_FORMATS[request.format ?? "mp3"];
|
|
39
|
+
if (format === undefined)
|
|
40
|
+
return yield* SpeechStream.unsupportedFormat(PROVIDER, ADAPTER, `${NAME} has no default output format for "${request.format}"; pass providerOptions.outputFormat`);
|
|
41
|
+
if (request.mode === "stream" && format.startsWith("wav_"))
|
|
42
|
+
return yield* SpeechStream.unsupportedFormat(PROVIDER, ADAPTER, `${NAME} streams mp3, pcm, opus, ulaw, and alaw but not "${format}"; use generate for WAV`);
|
|
43
|
+
return format;
|
|
44
|
+
});
|
|
45
|
+
const fromRequest = Effect.fn("ElevenLabsSpeech.fromRequest")(function* (request) {
|
|
46
|
+
if (request.voice === undefined)
|
|
47
|
+
return yield* ProviderShared.invalidRequest(`${NAME} requires a voice id; pass it as \`voice\``);
|
|
48
|
+
const { outputFormat: _outputFormat, ...native } = request.providerOptions ?? {};
|
|
49
|
+
return MediaProtocol.json(mergeJsonRecords({
|
|
50
|
+
text: request.text,
|
|
51
|
+
model_id: request.model.id,
|
|
52
|
+
language_code: request.language,
|
|
53
|
+
voice_settings: request.speed === undefined ? undefined : { speed: request.speed },
|
|
54
|
+
}, native, request.http?.body) ?? {}, { output_format: yield* outputFormat(request) });
|
|
55
|
+
});
|
|
56
|
+
const path = (request) => `${PATH}/${encodeURIComponent(SpeechStream.voiceID(request.voice) ?? "")}${request.mode === "stream" ? "/stream" : ""}${request.timestamps === true ? "/with-timestamps" : ""}`;
|
|
57
|
+
// ---------------------------------------------------------------------------
|
|
58
|
+
// 6. Stream parsing
|
|
59
|
+
// ---------------------------------------------------------------------------
|
|
60
|
+
const onRecord = Effect.fn("ElevenLabsSpeech.onRecord")(function* (state, frame) {
|
|
61
|
+
const record = yield* decodeRecord(frame);
|
|
62
|
+
const [next, events] = SpeechStream.delta(state, record.audio_base64);
|
|
63
|
+
const alignment = record.alignment;
|
|
64
|
+
if (!alignment)
|
|
65
|
+
return [next, events];
|
|
66
|
+
return [
|
|
67
|
+
next,
|
|
68
|
+
[
|
|
69
|
+
...events,
|
|
70
|
+
...SpeechStream.timestamps(alignment.characters, alignment.character_start_times_seconds, alignment.character_end_times_seconds),
|
|
71
|
+
],
|
|
72
|
+
];
|
|
73
|
+
});
|
|
74
|
+
const PCM_CODECS = {
|
|
75
|
+
pcm: "pcm_s16le",
|
|
76
|
+
ulaw: "pcm_mulaw",
|
|
77
|
+
alaw: "pcm_alaw",
|
|
78
|
+
};
|
|
79
|
+
const describeOutput = (format) => {
|
|
80
|
+
const [codec = format, rate] = format.split("_");
|
|
81
|
+
const sampleRate = rate === undefined ? undefined : Number(rate);
|
|
82
|
+
const encoding = PCM_CODECS[codec];
|
|
83
|
+
return encoding === undefined ? SpeechStream.container(codec, sampleRate) : SpeechStream.pcm(encoding, sampleRate);
|
|
84
|
+
};
|
|
85
|
+
const finish = Effect.fn("ElevenLabsSpeech.finish")(function* (state, context) {
|
|
86
|
+
const requestID = context.http.headers["request-id"];
|
|
87
|
+
return yield* SpeechStream.finish(ADAPTER, state, {
|
|
88
|
+
...describeOutput(yield* outputFormat(context.request)),
|
|
89
|
+
// `character-cost` is billed credits, not a character count (3 for 20 characters on `eleven_flash_v2_5`).
|
|
90
|
+
usage: SpeechStream.headerUsage("credits", context.http.headers["character-cost"]),
|
|
91
|
+
providerMetadata: requestID === undefined ? undefined : { elevenlabs: { requestId: requestID } },
|
|
92
|
+
});
|
|
93
|
+
});
|
|
94
|
+
// ---------------------------------------------------------------------------
|
|
95
|
+
// 7. Protocol and route
|
|
96
|
+
// ---------------------------------------------------------------------------
|
|
97
|
+
export const protocol = MediaProtocol.stream({
|
|
98
|
+
id: ADAPTER,
|
|
99
|
+
name: NAME,
|
|
100
|
+
unsupported: ["instructions"],
|
|
101
|
+
body: { from: fromRequest },
|
|
102
|
+
frames: (bytes, context) => {
|
|
103
|
+
if (context.request.timestamps !== true)
|
|
104
|
+
return bytes;
|
|
105
|
+
return context.request.mode === "stream" ? Framing.lines.frame(bytes) : Framing.document.frame(bytes);
|
|
106
|
+
},
|
|
107
|
+
initial: () => ({ chunks: [] }),
|
|
108
|
+
step: SpeechStream.step(onRecord),
|
|
109
|
+
finish,
|
|
110
|
+
});
|
|
111
|
+
export const model = (input) => SpeechModel.fromRoute({ id: ADAPTER, provider: PROVIDER, protocol, baseURL: DEFAULT_BASE_URL, path: ({ request }) => path(request) }, input);
|
|
112
|
+
export const ElevenLabsSpeech = {
|
|
113
|
+
protocol,
|
|
114
|
+
model,
|
|
115
|
+
};
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
import { Schema } from "effect";
|
|
2
|
+
import { MediaProtocol } from "../route/media-protocol.js";
|
|
3
|
+
import { MediaRoute } from "../route/media.js";
|
|
4
|
+
import { SpeechModel, type SpeechRequestFor } from "../speech.js";
|
|
5
|
+
import { SpeechStream } from "./utils/speech-stream.js";
|
|
6
|
+
export declare const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta";
|
|
7
|
+
/** Style is directed in the text itself, and `speechConfig.multiSpeakerVoiceConfig` excludes `voice`. */
|
|
8
|
+
export type GoogleSpeechOptions = {
|
|
9
|
+
readonly temperature?: number;
|
|
10
|
+
readonly seed?: number;
|
|
11
|
+
readonly speechConfig?: {
|
|
12
|
+
readonly multiSpeakerVoiceConfig?: {
|
|
13
|
+
readonly speakerVoiceConfigs: ReadonlyArray<{
|
|
14
|
+
readonly speaker: string;
|
|
15
|
+
readonly voiceConfig: {
|
|
16
|
+
readonly prebuiltVoiceConfig: {
|
|
17
|
+
readonly voiceName: string;
|
|
18
|
+
};
|
|
19
|
+
};
|
|
20
|
+
}>;
|
|
21
|
+
};
|
|
22
|
+
};
|
|
23
|
+
} & Record<string, unknown>;
|
|
24
|
+
export type Request = SpeechRequestFor<GoogleSpeechOptions>;
|
|
25
|
+
declare const GenerateContentChunk: Schema.Struct<{
|
|
26
|
+
readonly candidates: Schema.optional<Schema.$Array<Schema.Struct<{
|
|
27
|
+
readonly content: Schema.optional<Schema.Struct<{
|
|
28
|
+
readonly parts: Schema.optional<Schema.$Array<Schema.Struct<{
|
|
29
|
+
readonly text: Schema.optional<Schema.String>;
|
|
30
|
+
readonly inlineData: Schema.optional<Schema.Struct<{
|
|
31
|
+
readonly mimeType: Schema.String;
|
|
32
|
+
readonly data: Schema.Uint8ArrayFromBase64;
|
|
33
|
+
}>>;
|
|
34
|
+
}>>>;
|
|
35
|
+
}>>;
|
|
36
|
+
readonly finishReason: Schema.optional<Schema.String>;
|
|
37
|
+
}>>>;
|
|
38
|
+
readonly promptFeedback: Schema.optional<Schema.Struct<{
|
|
39
|
+
readonly blockReason: Schema.optional<Schema.String>;
|
|
40
|
+
readonly blockReasonMessage: Schema.optional<Schema.String>;
|
|
41
|
+
}>>;
|
|
42
|
+
readonly usageMetadata: Schema.optional<Schema.Struct<{
|
|
43
|
+
readonly promptTokenCount: Schema.optional<Schema.Number>;
|
|
44
|
+
readonly candidatesTokenCount: Schema.optional<Schema.Number>;
|
|
45
|
+
readonly totalTokenCount: Schema.optional<Schema.Number>;
|
|
46
|
+
}>>;
|
|
47
|
+
readonly modelVersion: Schema.optional<Schema.String>;
|
|
48
|
+
readonly responseId: Schema.optional<Schema.String>;
|
|
49
|
+
}>;
|
|
50
|
+
type GenerateContentChunk = Schema.Schema.Type<typeof GenerateContentChunk>;
|
|
51
|
+
interface State extends SpeechStream.Audio {
|
|
52
|
+
readonly mimeType?: string;
|
|
53
|
+
readonly usage?: GenerateContentChunk["usageMetadata"];
|
|
54
|
+
readonly finishReason?: string;
|
|
55
|
+
readonly modelVersion?: string;
|
|
56
|
+
readonly responseId?: string;
|
|
57
|
+
}
|
|
58
|
+
export declare const protocol: MediaProtocol.Streamed<Request, {
|
|
59
|
+
readonly type: "audio-delta";
|
|
60
|
+
readonly chunk: Uint8Array<ArrayBufferLike>;
|
|
61
|
+
} | {
|
|
62
|
+
readonly type: "timestamps";
|
|
63
|
+
readonly items: readonly {
|
|
64
|
+
readonly text: string;
|
|
65
|
+
readonly startSeconds: number;
|
|
66
|
+
readonly endSeconds: number;
|
|
67
|
+
}[];
|
|
68
|
+
} | {
|
|
69
|
+
readonly type: "finish";
|
|
70
|
+
readonly audio: import("../media.js").Asset;
|
|
71
|
+
readonly providerMetadata?: {
|
|
72
|
+
readonly [x: string]: {
|
|
73
|
+
readonly [x: string]: unknown;
|
|
74
|
+
};
|
|
75
|
+
} | undefined;
|
|
76
|
+
readonly usage?: {
|
|
77
|
+
readonly type: "tokens";
|
|
78
|
+
readonly input?: number | undefined;
|
|
79
|
+
readonly output?: number | undefined;
|
|
80
|
+
readonly total?: number | undefined;
|
|
81
|
+
readonly details?: {
|
|
82
|
+
readonly [x: string]: unknown;
|
|
83
|
+
} | undefined;
|
|
84
|
+
} | {
|
|
85
|
+
readonly type: "seconds";
|
|
86
|
+
readonly seconds: number;
|
|
87
|
+
} | {
|
|
88
|
+
readonly type: "characters";
|
|
89
|
+
readonly characters: number;
|
|
90
|
+
} | {
|
|
91
|
+
readonly type: "credits";
|
|
92
|
+
readonly credits: number;
|
|
93
|
+
} | {
|
|
94
|
+
readonly type: "compute";
|
|
95
|
+
readonly seconds: number;
|
|
96
|
+
} | undefined;
|
|
97
|
+
readonly notices?: readonly {
|
|
98
|
+
readonly type: "other" | "moderated" | "filtered";
|
|
99
|
+
readonly message: string;
|
|
100
|
+
readonly providerMetadata?: {
|
|
101
|
+
readonly [x: string]: {
|
|
102
|
+
readonly [x: string]: unknown;
|
|
103
|
+
};
|
|
104
|
+
} | undefined;
|
|
105
|
+
}[] | undefined;
|
|
106
|
+
}, string, State>;
|
|
107
|
+
export declare const model: (input: MediaRoute.ModelInput) => SpeechModel<GoogleSpeechOptions>;
|
|
108
|
+
export declare const GoogleSpeech: {
|
|
109
|
+
readonly protocol: MediaProtocol.Streamed<Request, {
|
|
110
|
+
readonly type: "audio-delta";
|
|
111
|
+
readonly chunk: Uint8Array<ArrayBufferLike>;
|
|
112
|
+
} | {
|
|
113
|
+
readonly type: "timestamps";
|
|
114
|
+
readonly items: readonly {
|
|
115
|
+
readonly text: string;
|
|
116
|
+
readonly startSeconds: number;
|
|
117
|
+
readonly endSeconds: number;
|
|
118
|
+
}[];
|
|
119
|
+
} | {
|
|
120
|
+
readonly type: "finish";
|
|
121
|
+
readonly audio: import("../media.js").Asset;
|
|
122
|
+
readonly providerMetadata?: {
|
|
123
|
+
readonly [x: string]: {
|
|
124
|
+
readonly [x: string]: unknown;
|
|
125
|
+
};
|
|
126
|
+
} | undefined;
|
|
127
|
+
readonly usage?: {
|
|
128
|
+
readonly type: "tokens";
|
|
129
|
+
readonly input?: number | undefined;
|
|
130
|
+
readonly output?: number | undefined;
|
|
131
|
+
readonly total?: number | undefined;
|
|
132
|
+
readonly details?: {
|
|
133
|
+
readonly [x: string]: unknown;
|
|
134
|
+
} | undefined;
|
|
135
|
+
} | {
|
|
136
|
+
readonly type: "seconds";
|
|
137
|
+
readonly seconds: number;
|
|
138
|
+
} | {
|
|
139
|
+
readonly type: "characters";
|
|
140
|
+
readonly characters: number;
|
|
141
|
+
} | {
|
|
142
|
+
readonly type: "credits";
|
|
143
|
+
readonly credits: number;
|
|
144
|
+
} | {
|
|
145
|
+
readonly type: "compute";
|
|
146
|
+
readonly seconds: number;
|
|
147
|
+
} | undefined;
|
|
148
|
+
readonly notices?: readonly {
|
|
149
|
+
readonly type: "other" | "moderated" | "filtered";
|
|
150
|
+
readonly message: string;
|
|
151
|
+
readonly providerMetadata?: {
|
|
152
|
+
readonly [x: string]: {
|
|
153
|
+
readonly [x: string]: unknown;
|
|
154
|
+
};
|
|
155
|
+
} | undefined;
|
|
156
|
+
}[] | undefined;
|
|
157
|
+
}, string, State>;
|
|
158
|
+
readonly model: (input: MediaRoute.ModelInput) => SpeechModel<GoogleSpeechOptions>;
|
|
159
|
+
};
|
|
160
|
+
export {};
|