@runappsai/sdk 0.0.0-stage → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +274 -2
- package/dist/activity-tracker.d.ts +113 -0
- package/dist/activity-tracker.d.ts.map +1 -0
- package/dist/activity-tracker.js +176 -0
- package/dist/activity-tracker.js.map +1 -0
- package/dist/audio.d.ts +129 -0
- package/dist/audio.d.ts.map +1 -0
- package/dist/audio.js +267 -0
- package/dist/audio.js.map +1 -0
- package/dist/browser-auth.d.ts +179 -0
- package/dist/browser-auth.d.ts.map +1 -0
- package/dist/browser-auth.js +1361 -0
- package/dist/browser-auth.js.map +1 -0
- package/dist/chat.d.ts +221 -0
- package/dist/chat.d.ts.map +1 -0
- package/dist/chat.js +155 -0
- package/dist/chat.js.map +1 -0
- package/dist/client.d.ts +155 -0
- package/dist/client.d.ts.map +1 -0
- package/dist/client.js +157 -0
- package/dist/client.js.map +1 -0
- package/dist/computer.d.ts +57 -0
- package/dist/computer.d.ts.map +1 -0
- package/dist/computer.js +21 -0
- package/dist/computer.js.map +1 -0
- package/dist/embeddings.d.ts +64 -0
- package/dist/embeddings.d.ts.map +1 -0
- package/dist/embeddings.js +30 -0
- package/dist/embeddings.js.map +1 -0
- package/dist/errors.d.ts +24 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +29 -0
- package/dist/errors.js.map +1 -0
- package/dist/event-wrap.d.ts +59 -0
- package/dist/event-wrap.d.ts.map +1 -0
- package/dist/event-wrap.js +142 -0
- package/dist/event-wrap.js.map +1 -0
- package/dist/events.d.ts +119 -0
- package/dist/events.d.ts.map +1 -0
- package/dist/events.js +87 -0
- package/dist/events.js.map +1 -0
- package/dist/files.d.ts +218 -0
- package/dist/files.d.ts.map +1 -0
- package/dist/files.js +212 -0
- package/dist/files.js.map +1 -0
- package/dist/image.d.ts +185 -0
- package/dist/image.d.ts.map +1 -0
- package/dist/image.js +148 -0
- package/dist/image.js.map +1 -0
- package/dist/index.d.ts +30 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +26 -0
- package/dist/index.js.map +1 -0
- package/dist/media.d.ts +56 -0
- package/dist/media.d.ts.map +1 -0
- package/dist/media.js +133 -0
- package/dist/media.js.map +1 -0
- package/dist/model_options.d.ts +165 -0
- package/dist/model_options.d.ts.map +1 -0
- package/dist/model_options.js +59 -0
- package/dist/model_options.js.map +1 -0
- package/dist/models.d.ts +88 -0
- package/dist/models.d.ts.map +1 -0
- package/dist/models.js +55 -0
- package/dist/models.js.map +1 -0
- package/dist/sdk.umd.js +4 -0
- package/dist/sdk.umd.js.map +1 -0
- package/dist/transport.d.ts +120 -0
- package/dist/transport.d.ts.map +1 -0
- package/dist/transport.js +214 -0
- package/dist/transport.js.map +1 -0
- package/dist/types.d.ts +31 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/dist/validate.d.ts +16 -0
- package/dist/validate.d.ts.map +1 -0
- package/dist/validate.js +329 -0
- package/dist/validate.js.map +1 -0
- package/dist/video.d.ts +134 -0
- package/dist/video.d.ts.map +1 -0
- package/dist/video.js +60 -0
- package/dist/video.js.map +1 -0
- package/package.json +54 -4
- package/src/activity-tracker.ts +251 -0
- package/src/audio.ts +440 -0
- package/src/browser-auth.ts +1529 -0
- package/src/chat.ts +368 -0
- package/src/client.ts +250 -0
- package/src/computer.ts +80 -0
- package/src/embeddings.ts +97 -0
- package/src/errors.ts +29 -0
- package/src/event-wrap.ts +168 -0
- package/src/events.ts +172 -0
- package/src/files.ts +347 -0
- package/src/image.ts +339 -0
- package/src/index.ts +151 -0
- package/src/media.ts +147 -0
- package/src/model_options.ts +220 -0
- package/src/models.ts +121 -0
- package/src/transport.ts +292 -0
- package/src/types.ts +31 -0
- package/src/validate.ts +364 -0
- package/src/video.ts +215 -0
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* In-memory session-scoped activity index — turns the raw event
|
|
3
|
+
* stream from `SDKEvents` into the snapshot the badge UI reads each
|
|
4
|
+
* frame: which calls are in flight, what just finished, what the
|
|
5
|
+
* running session cost is.
|
|
6
|
+
*
|
|
7
|
+
* Memory bounded:
|
|
8
|
+
* - `active` map shrinks on every end/error (calls don't accumulate
|
|
9
|
+
* past their settle event).
|
|
10
|
+
* - `recent` is a bounded ring of the last RECENT_MAX completions.
|
|
11
|
+
* - tokens/sec windows hold only `RATE_WINDOW_MS` worth of deltas;
|
|
12
|
+
* stale samples get dropped on every push.
|
|
13
|
+
*
|
|
14
|
+
* Snapshot pattern (vs. emitting another event per change): the
|
|
15
|
+
* badge re-renders on `requestAnimationFrame` while it's open, so
|
|
16
|
+
* a pull-snapshot API matches that cadence — pushing on every
|
|
17
|
+
* delta would over-paint and cost battery on token-heavy streams.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import type {
|
|
21
|
+
RequestEndEvent,
|
|
22
|
+
RequestErrorEvent,
|
|
23
|
+
RequestStartEvent,
|
|
24
|
+
RequestStreamDeltaEvent,
|
|
25
|
+
SDKCapability,
|
|
26
|
+
SDKEvents,
|
|
27
|
+
} from "./events.js";
|
|
28
|
+
|
|
29
|
+
/** Sliding window for tokens/sec — 1.5s gives a snappy display
|
|
30
|
+
* without jittering on every chunk arrival (most providers emit
|
|
31
|
+
* ~5-30 deltas/s). */
|
|
32
|
+
const RATE_WINDOW_MS = 1500;
|
|
33
|
+
|
|
34
|
+
/** Cap on the recent-completion ring buffer. The popover lists the
|
|
35
|
+
* last 5 by default but we keep a few extra so a future "show more"
|
|
36
|
+
* UI doesn't need a refactor. */
|
|
37
|
+
const RECENT_MAX = 20;
|
|
38
|
+
|
|
39
|
+
/** One LLM call currently in flight. */
|
|
40
|
+
export interface ActiveCall {
|
|
41
|
+
id: string;
|
|
42
|
+
model: string;
|
|
43
|
+
capability: SDKCapability;
|
|
44
|
+
startedAt: number;
|
|
45
|
+
/** True for stream:true. Drives the pulsing-ring animation. */
|
|
46
|
+
streaming: boolean;
|
|
47
|
+
/** Cumulative completion tokens (0 until first delta). */
|
|
48
|
+
tokensSoFar: number;
|
|
49
|
+
/** Tokens per second over the last RATE_WINDOW_MS. 0 until the
|
|
50
|
+
* window has 2+ samples. */
|
|
51
|
+
tokensPerSec: number;
|
|
52
|
+
/** Internal sample buffer — exposed for tests; UI ignores. */
|
|
53
|
+
rateSamples: Array<{ t: number; tokens: number }>;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** One LLM call that already finished (success OR failure). */
|
|
57
|
+
export interface CompletedCall {
|
|
58
|
+
id: string;
|
|
59
|
+
model: string;
|
|
60
|
+
capability: SDKCapability;
|
|
61
|
+
latencyMs: number;
|
|
62
|
+
totalTokens: number;
|
|
63
|
+
costUSD?: number;
|
|
64
|
+
finishReason?: string;
|
|
65
|
+
/** false for RequestErrorEvent settles. */
|
|
66
|
+
ok: boolean;
|
|
67
|
+
/** Error message when ok=false. */
|
|
68
|
+
errorMessage?: string;
|
|
69
|
+
/** Wall clock at settle — drives the "30s ago" relative display. */
|
|
70
|
+
endedAt: number;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Running session totals — shown in the popover's footer. */
|
|
74
|
+
export interface SessionStats {
|
|
75
|
+
totalCalls: number;
|
|
76
|
+
totalCostUSD: number;
|
|
77
|
+
errorCount: number;
|
|
78
|
+
/** Wall clock when the tracker first saw an event. Drives the
|
|
79
|
+
* "session lasted N minutes" string. */
|
|
80
|
+
startedAt: number;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** Pull-snapshot of the tracker — taken once per badge re-render. */
|
|
84
|
+
export interface ActivitySnapshot {
|
|
85
|
+
active: ActiveCall[];
|
|
86
|
+
recent: CompletedCall[];
|
|
87
|
+
session: SessionStats;
|
|
88
|
+
/** Highest-priority status colour for the LED dot:
|
|
89
|
+
* "error" → recent failure within ERROR_HIGHLIGHT_MS
|
|
90
|
+
* "active" → at least one in-flight call
|
|
91
|
+
* "idle" → nothing happening
|
|
92
|
+
* The badge's CSS uses this as a class. */
|
|
93
|
+
status: "idle" | "active" | "error";
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** Window during which a recent error keeps the LED red. 30s is
|
|
97
|
+
* long enough to grab attention but short enough that an old
|
|
98
|
+
* failure doesn't dominate when subsequent calls succeed. */
|
|
99
|
+
const ERROR_HIGHLIGHT_MS = 30_000;
|
|
100
|
+
|
|
101
|
+
export class ActivityTracker {
|
|
102
|
+
private active = new Map<string, ActiveCall>();
|
|
103
|
+
private recent: CompletedCall[] = [];
|
|
104
|
+
private session: SessionStats = {
|
|
105
|
+
totalCalls: 0,
|
|
106
|
+
totalCostUSD: 0,
|
|
107
|
+
errorCount: 0,
|
|
108
|
+
startedAt: Date.now(),
|
|
109
|
+
};
|
|
110
|
+
/** Track most recent error wall-clock so the LED stays red for
|
|
111
|
+
* ERROR_HIGHLIGHT_MS even after the error scrolls out of recent. */
|
|
112
|
+
private lastErrorAt = 0;
|
|
113
|
+
/** "Snapshot changed" listeners. Fired AFTER every event handler
|
|
114
|
+
* runs (start / delta / end / error), letting UIs re-render in
|
|
115
|
+
* push-mode instead of polling. Without this hook, the badge's
|
|
116
|
+
* ring/LED stayed idle until the user opened the popover —
|
|
117
|
+
* events landed in the tracker but nobody told the DOM. */
|
|
118
|
+
private changeListeners = new Set<() => void>();
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Subscribe the tracker to a bus. Returns a single disposer that
|
|
122
|
+
* unsubscribes all four handlers — callers (BrowserAuth in our
|
|
123
|
+
* case) attach this on construction and dispose on teardown.
|
|
124
|
+
*/
|
|
125
|
+
attach(events: SDKEvents): () => void {
|
|
126
|
+
const offs = [
|
|
127
|
+
events.on("request:start", (e) => { this.onStart(e); this.notify(); }),
|
|
128
|
+
events.on("request:streamDelta", (e) => { this.onDelta(e); this.notify(); }),
|
|
129
|
+
events.on("request:end", (e) => { this.onEnd(e); this.notify(); }),
|
|
130
|
+
events.on("request:error", (e) => { this.onError(e); this.notify(); }),
|
|
131
|
+
];
|
|
132
|
+
return () => {
|
|
133
|
+
for (const o of offs) o();
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Subscribe to "snapshot may have changed" notifications. Fires
|
|
139
|
+
* synchronously after each event handler runs — typical consumer
|
|
140
|
+
* is the badge's `requestRedraw()` which coalesces multiple
|
|
141
|
+
* notifications into one rAF paint.
|
|
142
|
+
*
|
|
143
|
+
* Returns a disposer; call to unsubscribe.
|
|
144
|
+
*/
|
|
145
|
+
onChange(handler: () => void): () => void {
|
|
146
|
+
this.changeListeners.add(handler);
|
|
147
|
+
return () => this.changeListeners.delete(handler);
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
private notify(): void {
|
|
151
|
+
for (const h of this.changeListeners) {
|
|
152
|
+
try {
|
|
153
|
+
h();
|
|
154
|
+
} catch (err) {
|
|
155
|
+
// eslint-disable-next-line no-console
|
|
156
|
+
console.error("[runjobs sdk] ActivityTracker onChange listener threw:", err);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/** O(active+recent) — small in practice, fine on every frame. */
|
|
162
|
+
snapshot(): ActivitySnapshot {
|
|
163
|
+
const now = Date.now();
|
|
164
|
+
return {
|
|
165
|
+
active: Array.from(this.active.values()),
|
|
166
|
+
recent: this.recent.slice(),
|
|
167
|
+
session: { ...this.session },
|
|
168
|
+
status:
|
|
169
|
+
now - this.lastErrorAt < ERROR_HIGHLIGHT_MS
|
|
170
|
+
? "error"
|
|
171
|
+
: this.active.size > 0
|
|
172
|
+
? "active"
|
|
173
|
+
: "idle",
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
// ─── handlers ─────────────────────────────────────────────────
|
|
178
|
+
|
|
179
|
+
private onStart(e: RequestStartEvent): void {
|
|
180
|
+
this.active.set(e.id, {
|
|
181
|
+
id: e.id,
|
|
182
|
+
model: e.model,
|
|
183
|
+
capability: e.capability,
|
|
184
|
+
startedAt: e.startedAt,
|
|
185
|
+
streaming: e.streaming,
|
|
186
|
+
tokensSoFar: 0,
|
|
187
|
+
tokensPerSec: 0,
|
|
188
|
+
rateSamples: [],
|
|
189
|
+
});
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
private onDelta(e: RequestStreamDeltaEvent): void {
|
|
193
|
+
const call = this.active.get(e.id);
|
|
194
|
+
if (!call) return; // delta after end — drop
|
|
195
|
+
call.tokensSoFar = e.totalTokens;
|
|
196
|
+
const now = Date.now();
|
|
197
|
+
call.rateSamples.push({ t: now, tokens: e.deltaTokens });
|
|
198
|
+
// Drop samples older than the window.
|
|
199
|
+
const cutoff = now - RATE_WINDOW_MS;
|
|
200
|
+
while (call.rateSamples.length > 0 && call.rateSamples[0]!.t < cutoff) {
|
|
201
|
+
call.rateSamples.shift();
|
|
202
|
+
}
|
|
203
|
+
if (call.rateSamples.length >= 2) {
|
|
204
|
+
const sumTokens = call.rateSamples.reduce((s, x) => s + x.tokens, 0);
|
|
205
|
+
const spanMs = now - call.rateSamples[0]!.t;
|
|
206
|
+
call.tokensPerSec = spanMs > 0 ? (sumTokens * 1000) / spanMs : 0;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
private onEnd(e: RequestEndEvent): void {
|
|
211
|
+
const call = this.active.get(e.id);
|
|
212
|
+
this.active.delete(e.id);
|
|
213
|
+
this.pushRecent({
|
|
214
|
+
id: e.id,
|
|
215
|
+
model: e.model,
|
|
216
|
+
capability: e.capability,
|
|
217
|
+
latencyMs: e.latencyMs,
|
|
218
|
+
totalTokens: e.totalTokens || call?.tokensSoFar || 0,
|
|
219
|
+
...(e.costUSD !== undefined && { costUSD: e.costUSD }),
|
|
220
|
+
...(e.finishReason !== undefined && { finishReason: e.finishReason }),
|
|
221
|
+
ok: true,
|
|
222
|
+
endedAt: Date.now(),
|
|
223
|
+
});
|
|
224
|
+
this.session.totalCalls += 1;
|
|
225
|
+
if (e.costUSD !== undefined) this.session.totalCostUSD += e.costUSD;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
private onError(e: RequestErrorEvent): void {
|
|
229
|
+
const call = this.active.get(e.id);
|
|
230
|
+
this.active.delete(e.id);
|
|
231
|
+
this.pushRecent({
|
|
232
|
+
id: e.id,
|
|
233
|
+
model: e.model,
|
|
234
|
+
capability: e.capability,
|
|
235
|
+
latencyMs: e.latencyMs,
|
|
236
|
+
totalTokens: call?.tokensSoFar || 0,
|
|
237
|
+
ok: false,
|
|
238
|
+
errorMessage: e.error.message,
|
|
239
|
+
endedAt: Date.now(),
|
|
240
|
+
});
|
|
241
|
+
this.session.totalCalls += 1;
|
|
242
|
+
this.session.errorCount += 1;
|
|
243
|
+
this.lastErrorAt = Date.now();
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
private pushRecent(c: CompletedCall): void {
|
|
247
|
+
// Newest first — popover renders top-to-bottom newest→oldest.
|
|
248
|
+
this.recent.unshift(c);
|
|
249
|
+
if (this.recent.length > RECENT_MAX) this.recent.length = RECENT_MAX;
|
|
250
|
+
}
|
|
251
|
+
}
|
package/src/audio.ts
ADDED
|
@@ -0,0 +1,440 @@
|
|
|
1
|
+
import type { Transport } from "./transport.js";
|
|
2
|
+
import type { Usage } from "./types.js";
|
|
3
|
+
import type { SDKEvents } from "./events.js";
|
|
4
|
+
import { wrapEvents } from "./event-wrap.js";
|
|
5
|
+
import { decodeMediaUrl } from "./media.js";
|
|
6
|
+
|
|
7
|
+
/* ------------------------------------------------------------------ */
|
|
8
|
+
/* TTS */
|
|
9
|
+
/* ------------------------------------------------------------------ */
|
|
10
|
+
/* */
|
|
11
|
+
/* Voice metadata (id / name / gender / preview_url / supported */
|
|
12
|
+
/* emotions) is no longer fetched through a dedicated endpoint — */
|
|
13
|
+
/* every text_to_speech row on /v1/models carries its full catalog */
|
|
14
|
+
/* under `options.voices` and `options.supported_emotions`. Use */
|
|
15
|
+
/* `client.models.get(name)` (or the entry from `client.models.list`) */
|
|
16
|
+
/* and read those fields off `model.options`. */
|
|
17
|
+
/* ------------------------------------------------------------------ */
|
|
18
|
+
|
|
19
|
+
export interface SpeechParams {
|
|
20
|
+
input: string;
|
|
21
|
+
/** Named voice id (`alloy`, `nova`, …; provider-dependent). Omit
|
|
22
|
+
* when supplying `reference_audio_url` for zero-shot voice
|
|
23
|
+
* cloning — the cloned timbre overrides any named voice. */
|
|
24
|
+
voice?: string;
|
|
25
|
+
response_format?: string;
|
|
26
|
+
speed?: number;
|
|
27
|
+
/** Provider-specific. Inspect the chosen TTS model's
|
|
28
|
+
* `options.supported_emotions` (via `client.models.get(name)`) for
|
|
29
|
+
* the legal enum values. */
|
|
30
|
+
emotion?: string;
|
|
31
|
+
/** -12 … 12 semitones (provider-dependent). */
|
|
32
|
+
pitch?: number;
|
|
33
|
+
/** 0.1 … 10.0 — 1.0 = normal. */
|
|
34
|
+
volume?: number;
|
|
35
|
+
/** -12 … 12 — voice timbre shift (provider-dependent). */
|
|
36
|
+
timber?: number;
|
|
37
|
+
/**
|
|
38
|
+
* Free-form natural-language directive for voiceclone (CosyVoice).
|
|
39
|
+
* e.g. "用四川话快速地说"
|
|
40
|
+
*/
|
|
41
|
+
instruct_text?: string;
|
|
42
|
+
/**
|
|
43
|
+
* Public URL of a reference clip whose timbre to match (zero-shot
|
|
44
|
+
* voice cloning). When set, voiceclone-capable models ignore `voice`
|
|
45
|
+
* and synthesize matching the referenced audio.
|
|
46
|
+
*/
|
|
47
|
+
reference_audio_url?: string;
|
|
48
|
+
/** Transcript of the reference clip — improves CosyVoice quality. */
|
|
49
|
+
reference_text?: string;
|
|
50
|
+
user?: string;
|
|
51
|
+
/**
|
|
52
|
+
* Vendor-specific top-level fields the canonical params don't model.
|
|
53
|
+
* Used for music-generation models on the TTS bucket — ACE-Step needs
|
|
54
|
+
* `tags` (genre, required), `duration`, `seed`, `scheduler`, etc.
|
|
55
|
+
* The SDK spreads these at the request body's TOP LEVEL (not nested
|
|
56
|
+
* under "extra") to match the gateway's extractSpeechExtra contract.
|
|
57
|
+
* Canonical fields above always win over extra keys with the same name.
|
|
58
|
+
*/
|
|
59
|
+
extra?: Record<string, unknown>;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export interface SpeechResponse {
|
|
63
|
+
/** Decoded audio bytes. */
|
|
64
|
+
data: Uint8Array;
|
|
65
|
+
/** MIME type reported by the gateway (e.g. "audio/mpeg"). */
|
|
66
|
+
contentType: string;
|
|
67
|
+
usage: Usage;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/* ------------------------------------------------------------------ */
|
|
71
|
+
/* STT */
|
|
72
|
+
/* ------------------------------------------------------------------ */
|
|
73
|
+
|
|
74
|
+
/** Same shape as image.ImageFileInput — kept independent so callers can import either. */
|
|
75
|
+
export interface AudioFileInput {
|
|
76
|
+
data: Blob | ArrayBuffer | Uint8Array | string;
|
|
77
|
+
filename: string;
|
|
78
|
+
contentType?: string;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export interface TranscribeParams {
|
|
82
|
+
file: AudioFileInput;
|
|
83
|
+
language?: string;
|
|
84
|
+
prompt?: string;
|
|
85
|
+
response_format?: string;
|
|
86
|
+
timestamp_granularities?: ("segment" | "word")[];
|
|
87
|
+
user?: string;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export interface TranscribeResponse {
|
|
91
|
+
text: string;
|
|
92
|
+
usage: Usage;
|
|
93
|
+
/** Raw upstream payload — segment timings, etc., when present. */
|
|
94
|
+
raw?: Record<string, unknown>;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/* ------------------------------------------------------------------ */
|
|
98
|
+
/* Service */
|
|
99
|
+
/* ------------------------------------------------------------------ */
|
|
100
|
+
|
|
101
|
+
export class AudioService {
|
|
102
|
+
constructor(
|
|
103
|
+
private readonly transport: Transport,
|
|
104
|
+
private readonly events: SDKEvents,
|
|
105
|
+
) {}
|
|
106
|
+
|
|
107
|
+
// (Removed: listVoices — fetch voice metadata via client.models.get /
|
|
108
|
+
// client.models.list and read model.options.voices /
|
|
109
|
+
// model.options.supported_emotions.)
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Generate speech from text. Output bytes are base64-decoded by the SDK
|
|
113
|
+
* — `response.data` is a `Uint8Array` you can write directly to disk:
|
|
114
|
+
*
|
|
115
|
+
* ```ts
|
|
116
|
+
* await fs.writeFile("out.mp3", await client.audio.speech("OpenAI/TTS", {
|
|
117
|
+
* input: "Hello!",
|
|
118
|
+
* voice: "nova",
|
|
119
|
+
* }).then(r => r.data));
|
|
120
|
+
* ```
|
|
121
|
+
*/
|
|
122
|
+
async speech(
|
|
123
|
+
model: string,
|
|
124
|
+
params: SpeechParams,
|
|
125
|
+
init?: { signal?: AbortSignal },
|
|
126
|
+
): Promise<SpeechResponse> {
|
|
127
|
+
return wrapEvents(
|
|
128
|
+
this.events,
|
|
129
|
+
{ model, capability: "text_to_speech" },
|
|
130
|
+
async () => {
|
|
131
|
+
// Wire shape: {audio_url: "data:<mime>;base64,...", usage: ...}.
|
|
132
|
+
const raw = await this.transport.postJSON<{
|
|
133
|
+
audio_url: string;
|
|
134
|
+
usage: Usage;
|
|
135
|
+
}>(
|
|
136
|
+
"/v1/audio/speech",
|
|
137
|
+
buildSpeechBody(model, params),
|
|
138
|
+
init,
|
|
139
|
+
);
|
|
140
|
+
const { bytes, contentType } = await decodeMediaUrl(raw.audio_url);
|
|
141
|
+
return { data: bytes, contentType, usage: raw.usage };
|
|
142
|
+
},
|
|
143
|
+
(r) => ({ costUSD: r.usage?.total_cost }),
|
|
144
|
+
);
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Async equivalent of `speech()`. Submits the job to the gateway's
|
|
149
|
+
* async TTS endpoint, polls every ~3s until terminal, then decodes
|
|
150
|
+
* the result audio_url. Use this when a request may exceed the
|
|
151
|
+
* ~100s origin timeout — ACE-Step music generation at high quality,
|
|
152
|
+
* large CosyVoice batches, etc. The sync `speech()` method is the
|
|
153
|
+
* right choice for short-clip TTS that fits inside Cloudflare's
|
|
154
|
+
* timeout (skips the submit + poll round-trips).
|
|
155
|
+
*
|
|
156
|
+
* Returns the same `SpeechResponse` shape as `speech()` — `data` +
|
|
157
|
+
* `usage` — so callers can swap the two methods with no other change.
|
|
158
|
+
*
|
|
159
|
+
* Caller's `signal` deadline bounds the poll wait. Without one,
|
|
160
|
+
* an internal 10-minute cap applies.
|
|
161
|
+
*/
|
|
162
|
+
async speechAsync(
|
|
163
|
+
model: string,
|
|
164
|
+
params: SpeechParams,
|
|
165
|
+
init?: { signal?: AbortSignal; pollIntervalMs?: number },
|
|
166
|
+
): Promise<SpeechResponse> {
|
|
167
|
+
return wrapEvents(
|
|
168
|
+
this.events,
|
|
169
|
+
{ model, capability: "text_to_speech" },
|
|
170
|
+
async () => {
|
|
171
|
+
const submit = await this.transport.postJSON<{ id: string; status: string }>(
|
|
172
|
+
"/v1/async/audio/speech",
|
|
173
|
+
buildSpeechBody(model, params),
|
|
174
|
+
init,
|
|
175
|
+
);
|
|
176
|
+
if (!submit.id) {
|
|
177
|
+
throw new Error("runapps: speech submit response missing job id");
|
|
178
|
+
}
|
|
179
|
+
return waitSpeechJob(this.transport, submit.id, init);
|
|
180
|
+
},
|
|
181
|
+
(r) => ({ costUSD: r.usage?.total_cost }),
|
|
182
|
+
);
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/** Transcribe audio to text via multipart upload. */
|
|
186
|
+
async transcribe(
|
|
187
|
+
model: string,
|
|
188
|
+
params: TranscribeParams,
|
|
189
|
+
init?: { signal?: AbortSignal },
|
|
190
|
+
): Promise<TranscribeResponse> {
|
|
191
|
+
const form = buildTranscribeForm(model, params);
|
|
192
|
+
return wrapEvents(
|
|
193
|
+
this.events,
|
|
194
|
+
{ model, capability: "speech_to_text" },
|
|
195
|
+
() =>
|
|
196
|
+
this.transport.postMultipart<TranscribeResponse>(
|
|
197
|
+
"/v1/audio/transcriptions",
|
|
198
|
+
form,
|
|
199
|
+
init,
|
|
200
|
+
),
|
|
201
|
+
(r) => ({ costUSD: r.usage?.total_cost }),
|
|
202
|
+
);
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* Async equivalent of `transcribe()`. Submits the upload, polls
|
|
207
|
+
* every ~3s until terminal, returns the same `TranscribeResponse`.
|
|
208
|
+
* Use this for long audio (lectures, podcasts, multi-hour
|
|
209
|
+
* recordings) where Whisper can take minutes — well past
|
|
210
|
+
* Cloudflare's ~100s sync ceiling.
|
|
211
|
+
*
|
|
212
|
+
* Caller's `signal` deadline bounds the poll wait. Without one,
|
|
213
|
+
* an internal 10-minute cap applies.
|
|
214
|
+
*/
|
|
215
|
+
async transcribeAsync(
|
|
216
|
+
model: string,
|
|
217
|
+
params: TranscribeParams,
|
|
218
|
+
init?: { signal?: AbortSignal; pollIntervalMs?: number },
|
|
219
|
+
): Promise<TranscribeResponse> {
|
|
220
|
+
return wrapEvents(
|
|
221
|
+
this.events,
|
|
222
|
+
{ model, capability: "speech_to_text" },
|
|
223
|
+
async () => {
|
|
224
|
+
const form = buildTranscribeForm(model, params);
|
|
225
|
+
const submit = await this.transport.postMultipart<{ id: string; status: string }>(
|
|
226
|
+
"/v1/async/audio/transcriptions",
|
|
227
|
+
form,
|
|
228
|
+
init,
|
|
229
|
+
);
|
|
230
|
+
if (!submit.id) {
|
|
231
|
+
throw new Error("runapps: transcribe submit response missing job id");
|
|
232
|
+
}
|
|
233
|
+
return waitTranscribeJob(this.transport, submit.id, init);
|
|
234
|
+
},
|
|
235
|
+
(r) => ({ costUSD: r.usage?.total_cost }),
|
|
236
|
+
);
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
/* ------------------------------------------------------------------ */
|
|
241
|
+
/* Helpers */
|
|
242
|
+
/* ------------------------------------------------------------------ */
|
|
243
|
+
|
|
244
|
+
/**
|
|
245
|
+
* Polls the gateway's async speech-status endpoint until the job
|
|
246
|
+
* reaches a terminal state, then decodes the result `audio_url`
|
|
247
|
+
* data: URI into raw bytes + mime. Used by `AudioService.speechAsync`.
|
|
248
|
+
*
|
|
249
|
+
* Default poll cadence: first poll 2s after submit, then every 3s.
|
|
250
|
+
* Caller can override via `init.pollIntervalMs`. `init.signal`
|
|
251
|
+
* cancels the poll loop.
|
|
252
|
+
*/
|
|
253
|
+
async function waitSpeechJob(
|
|
254
|
+
transport: Transport,
|
|
255
|
+
jobId: string,
|
|
256
|
+
init?: { signal?: AbortSignal; pollIntervalMs?: number },
|
|
257
|
+
): Promise<SpeechResponse> {
|
|
258
|
+
const path = `/v1/async/audio/speech/${encodeURIComponent(jobId)}`;
|
|
259
|
+
const interval = init?.pollIntervalMs ?? 3000;
|
|
260
|
+
// First poll happens fast — typical TTS already done in 2-5s.
|
|
261
|
+
await sleep(2000, init?.signal);
|
|
262
|
+
// 10-minute hard cap if caller didn't supply a signal with a
|
|
263
|
+
// narrower deadline. Long enough for the slowest ACE-Step jobs.
|
|
264
|
+
const startedAt = Date.now();
|
|
265
|
+
const internalDeadline = startedAt + 10 * 60 * 1000;
|
|
266
|
+
|
|
267
|
+
for (;;) {
|
|
268
|
+
if (init?.signal?.aborted) {
|
|
269
|
+
throw new DOMException("speech poll aborted", "AbortError");
|
|
270
|
+
}
|
|
271
|
+
if (!init?.signal && Date.now() > internalDeadline) {
|
|
272
|
+
throw new Error("runapps: speech job timed out (10 min internal cap)");
|
|
273
|
+
}
|
|
274
|
+
const status = await transport.getJSON<{
|
|
275
|
+
id: string;
|
|
276
|
+
status: string;
|
|
277
|
+
audio_url?: string;
|
|
278
|
+
usage?: Usage;
|
|
279
|
+
error?: string;
|
|
280
|
+
}>(path, init);
|
|
281
|
+
if (status.status === "succeeded") {
|
|
282
|
+
if (!status.audio_url) {
|
|
283
|
+
throw new Error("runapps: speech job succeeded but no audio_url returned");
|
|
284
|
+
}
|
|
285
|
+
const { bytes, contentType } = await decodeMediaUrl(status.audio_url);
|
|
286
|
+
return {
|
|
287
|
+
data: bytes,
|
|
288
|
+
contentType,
|
|
289
|
+
usage: status.usage ?? { total_cost: 0 },
|
|
290
|
+
};
|
|
291
|
+
}
|
|
292
|
+
if (status.status === "failed") {
|
|
293
|
+
throw new Error(status.error || "runapps: speech job failed");
|
|
294
|
+
}
|
|
295
|
+
// queued | running → wait + retry.
|
|
296
|
+
await sleep(interval, init?.signal);
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/**
|
|
301
|
+
* Polls the gateway's async transcribe-status endpoint until terminal,
|
|
302
|
+
* then assembles a TranscribeResponse from the final payload. Used by
|
|
303
|
+
* `AudioService.transcribeAsync`.
|
|
304
|
+
*/
|
|
305
|
+
async function waitTranscribeJob(
|
|
306
|
+
transport: Transport,
|
|
307
|
+
jobId: string,
|
|
308
|
+
init?: { signal?: AbortSignal; pollIntervalMs?: number },
|
|
309
|
+
): Promise<TranscribeResponse> {
|
|
310
|
+
const path = `/v1/async/audio/transcriptions/${encodeURIComponent(jobId)}`;
|
|
311
|
+
const interval = init?.pollIntervalMs ?? 3000;
|
|
312
|
+
await sleep(2000, init?.signal);
|
|
313
|
+
const internalDeadline = Date.now() + 10 * 60 * 1000;
|
|
314
|
+
for (;;) {
|
|
315
|
+
if (init?.signal?.aborted) {
|
|
316
|
+
throw new DOMException("transcribe poll aborted", "AbortError");
|
|
317
|
+
}
|
|
318
|
+
if (!init?.signal && Date.now() > internalDeadline) {
|
|
319
|
+
throw new Error("runapps: transcribe job timed out (10 min internal cap)");
|
|
320
|
+
}
|
|
321
|
+
const status = await transport.getJSON<Record<string, unknown>>(path, init);
|
|
322
|
+
switch (status.status) {
|
|
323
|
+
case "succeeded":
|
|
324
|
+
return assembleTranscribeResponse(status);
|
|
325
|
+
case "failed":
|
|
326
|
+
throw new Error((status.error as string) || "runapps: transcribe job failed");
|
|
327
|
+
case "queued":
|
|
328
|
+
case "running":
|
|
329
|
+
await sleep(interval, init?.signal);
|
|
330
|
+
break;
|
|
331
|
+
default:
|
|
332
|
+
throw new Error(`runapps: unknown transcribe job status ${JSON.stringify(status.status)}`);
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
/**
|
|
338
|
+
* Pulls canonical text + usage out of a succeeded transcribe-status
|
|
339
|
+
* payload, leaves the rest under .raw — matches the sync response
|
|
340
|
+
* shape produced by `transcribe()`.
|
|
341
|
+
*/
|
|
342
|
+
function assembleTranscribeResponse(payload: Record<string, unknown>): TranscribeResponse {
|
|
343
|
+
const out: TranscribeResponse = {
|
|
344
|
+
text: (payload.text as string) ?? "",
|
|
345
|
+
usage: (payload.usage as Usage) ?? { total_cost: 0 },
|
|
346
|
+
};
|
|
347
|
+
const raw: Record<string, unknown> = {};
|
|
348
|
+
for (const [k, v] of Object.entries(payload)) {
|
|
349
|
+
if (k === "text" || k === "usage" || k === "id" || k === "status") continue;
|
|
350
|
+
raw[k] = v;
|
|
351
|
+
}
|
|
352
|
+
if (Object.keys(raw).length > 0) {
|
|
353
|
+
out.raw = raw;
|
|
354
|
+
}
|
|
355
|
+
return out;
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
function sleep(ms: number, signal?: AbortSignal): Promise<void> {
|
|
359
|
+
return new Promise((resolve, reject) => {
|
|
360
|
+
if (signal?.aborted) {
|
|
361
|
+
reject(new DOMException("aborted", "AbortError"));
|
|
362
|
+
return;
|
|
363
|
+
}
|
|
364
|
+
const timer = setTimeout(() => {
|
|
365
|
+
signal?.removeEventListener("abort", onAbort);
|
|
366
|
+
resolve();
|
|
367
|
+
}, ms);
|
|
368
|
+
const onAbort = () => {
|
|
369
|
+
clearTimeout(timer);
|
|
370
|
+
reject(new DOMException("aborted", "AbortError"));
|
|
371
|
+
};
|
|
372
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
373
|
+
});
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
/**
|
|
377
|
+
* buildSpeechBody assembles the wire body for /v1/audio/speech (and
|
|
378
|
+
* its async sibling): `{model, ...canonical, ...extra}` flattened at
|
|
379
|
+
* the top level. The gateway's extractSpeechExtra peels non-canonical
|
|
380
|
+
* keys back into req.Extra, so the round-trip is symmetric. Canonical
|
|
381
|
+
* fields always win over `extra` keys of the same name — never let
|
|
382
|
+
* Extra silently override a typed field.
|
|
383
|
+
*/
|
|
384
|
+
function buildSpeechBody(model: string, params: SpeechParams): Record<string, unknown> {
|
|
385
|
+
const { extra, ...canonical } = params;
|
|
386
|
+
const body: Record<string, unknown> = { model, ...canonical };
|
|
387
|
+
if (extra) {
|
|
388
|
+
for (const [k, v] of Object.entries(extra)) {
|
|
389
|
+
if (k in body) continue;
|
|
390
|
+
body[k] = v;
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
return body;
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
function buildTranscribeForm(model: string, params: TranscribeParams): FormData {
|
|
397
|
+
const form = new FormData();
|
|
398
|
+
form.set("model", model);
|
|
399
|
+
if (params.language) form.set("language", params.language);
|
|
400
|
+
if (params.prompt) form.set("prompt", params.prompt);
|
|
401
|
+
if (params.response_format) form.set("response_format", params.response_format);
|
|
402
|
+
if (params.timestamp_granularities) {
|
|
403
|
+
for (const g of params.timestamp_granularities) {
|
|
404
|
+
form.append("timestamp_granularities[]", g);
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
if (params.user) form.set("user", params.user);
|
|
408
|
+
|
|
409
|
+
const ct = params.file.contentType ?? guessAudioMime(params.file.filename);
|
|
410
|
+
const blob =
|
|
411
|
+
params.file.data instanceof Blob
|
|
412
|
+
? params.file.data
|
|
413
|
+
: new Blob([params.file.data as BlobPart], { type: ct });
|
|
414
|
+
form.set("file", blob, params.file.filename);
|
|
415
|
+
return form;
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
function guessAudioMime(filename: string): string {
|
|
419
|
+
const ext = filename.toLowerCase().split(".").pop() ?? "";
|
|
420
|
+
switch (ext) {
|
|
421
|
+
case "mp3":
|
|
422
|
+
return "audio/mpeg";
|
|
423
|
+
case "wav":
|
|
424
|
+
return "audio/wav";
|
|
425
|
+
case "m4a":
|
|
426
|
+
return "audio/mp4";
|
|
427
|
+
case "flac":
|
|
428
|
+
return "audio/flac";
|
|
429
|
+
case "ogg":
|
|
430
|
+
return "audio/ogg";
|
|
431
|
+
case "webm":
|
|
432
|
+
return "audio/webm";
|
|
433
|
+
default:
|
|
434
|
+
return "application/octet-stream";
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
// (base64ToBytes was here — promoted to media.ts as part of
|
|
439
|
+
// decodeMediaUrl. The audio path now goes through that single helper
|
|
440
|
+
// so image and audio decode through one implementation.)
|