@convbased/sdk 0.1.0

package/src/tts.ts

@@ -0,0 +1,364 @@
+// Text-to-speech client for the Convbased IndexTTS2 service. This path is pure
+// GraphQL — it does not touch WebRTC or the signaling socket. Synthesis is
+// asynchronous: you submit a job, then poll until it reaches a terminal state.
+//
+// Typical flow:
+//   const tts = new TtsClient({ apiKey });
+//   const { key } = await tts.uploadReferenceAudio(file); // reference voice
+//   const result = await tts.synthesize({ referenceKey: key, text: "你好" });
+//   audio.src = result.url; // presigned, ~1h
+
+import { DEFAULT_GRAPHQL_URL } from "./endpoints.js";
+import { graphqlRequest } from "./graphql.js";
+import { uploadAudio } from "./upload.js";
+
+/** Optional emotion / sampling controls forwarded verbatim to IndexTTS2. */
+export interface TtsParams {
+	emo_alpha?: number;
+	emo_vector?: number[];
+	use_emo_text?: boolean;
+	emo_text?: string;
+	temperature?: number;
+	top_p?: number;
+	top_k?: number;
+}
+
+export type TtsJobStatus =
+	| "queued"
+	| "warming"
+	| "processing"
+	| "done"
+	| "failed"
+	| "cancelled";
+
+export interface TtsResult {
+	/** COS key of the synthesized audio. */
+	key: string;
+	/** Presigned URL of the synthesized audio (valid ~1h), or null if unsigned. */
+	url: string | null;
+	/** Input-text token count the charge was based on. */
+	tokenCount: number;
+	/** Duration of the synthesized audio, in seconds. */
+	audioDurationSec: number;
+	/** Amount deducted from the wallet for this synthesis. */
+	amountCharged: number;
+	/** Wallet balance after the deduction. */
+	balanceAfter: number;
+}
+
+export interface TtsJob {
+	jobId: string;
+	status: TtsJobStatus;
+	/** 1-based queue position while `queued`; 0 otherwise. */
+	position: number;
+	/** Result, populated once `status === "done"`. */
+	result: TtsResult | null;
+	/** Server-reported error key when `status === "failed"`. */
+	error: string | null;
+}
+
+export interface TtsPricing {
+	/** Price charged per input-text token. */
+	pricePerToken: number;
+	/** Minimum charge applied to any single synthesis. */
+	minCharge: number;
+}
+
+export interface TtsClientOptions {
+	/** Convbased API key. Required unless `accessToken` is provided. */
+	apiKey?: string;
+	/** Short-lived JWT access token (alternative to `apiKey`). */
+	accessToken?: string;
+	/**
+	 * GraphQL endpoint. Defaults to the production Convbased endpoint
+	 * (`https://api.weights.chat/api/v1/graphql`). Override for self-hosted
+	 * deployments.
+	 */
+	graphqlUrl?: string;
+	/** Optional logger; defaults to `console` for warn/error only. */
+	logger?: Partial<Pick<Console, "debug" | "info" | "warn" | "error">>;
+}
+
+export interface SubmitTtsOptions {
+	/** COS key of an already-uploaded reference voice (see `uploadReferenceAudio`). */
+	referenceKey: string;
+	/** Text to synthesize. */
+	text: string;
+	/** Optional emotion / sampling controls. */
+	params?: TtsParams;
+}
+
+export interface SynthesizeOptions {
+	/** COS key of an already-uploaded reference voice. Provide this or `referenceAudio`. */
+	referenceKey?: string;
+	/** A reference-voice `Blob`/`File` to upload first. Provide this or `referenceKey`. */
+	referenceAudio?: Blob;
+	/** Text to synthesize. */
+	text: string;
+	/** Optional emotion / sampling controls. */
+	params?: TtsParams;
+	/** Poll interval while waiting for the job, in ms. Default 1500. */
+	pollIntervalMs?: number;
+	/** Give up waiting after this many ms. Default 300_000 (5 min). */
+	timeoutMs?: number;
+	/** Abort the wait (and cancel the job if still queued). */
+	signal?: AbortSignal;
+	/** Called on every poll with the latest job snapshot (queue position, status…). */
+	onJob?: (job: TtsJob) => void;
+}
+
+interface TtsJobWire {
+	job_id: string;
+	status: TtsJobStatus;
+	position: number;
+	result: {
+		key: string;
+		url: string | null;
+		token_count: number;
+		audio_duration_sec: number;
+		amount_charged: number;
+		balance_after: number;
+	} | null;
+	error: string | null;
+}
+
+const JOB_FIELDS = /* GraphQL */ `
+	job_id
+	status
+	position
+	result {
+		key
+		url
+		token_count
+		audio_duration_sec
+		amount_charged
+		balance_after
+	}
+	error
+`;
+
+export class TtsClient {
+	private readonly graphqlUrl: string;
+	private readonly apiKey?: string;
+	private readonly accessToken?: string;
+	private readonly logger: Pick<Console, "debug" | "info" | "warn" | "error">;
+
+	constructor(options: TtsClientOptions) {
+		if (!options.apiKey && !options.accessToken) {
+			throw new Error("TtsClient requires either `apiKey` or `accessToken`");
+		}
+		this.graphqlUrl = options.graphqlUrl ?? DEFAULT_GRAPHQL_URL;
+		this.apiKey = options.apiKey;
+		this.accessToken = options.accessToken;
+		const provided = options.logger ?? {};
+		this.logger = {
+			debug: provided.debug ?? (() => {}),
+			info: provided.info ?? (() => {}),
+			warn: provided.warn ?? console.warn.bind(console),
+			error: provided.error ?? console.error.bind(console),
+		};
+	}
+
+	/** Upload a reference-voice `Blob`/`File` and resolve its COS key. */
+	async uploadReferenceAudio(
+		file: Blob,
+		opts?: { filename?: string; contentType?: string; signal?: AbortSignal }
+	): Promise<{ key: string }> {
+		return uploadAudio({
+			graphqlUrl: this.graphqlUrl,
+			apiKey: this.apiKey,
+			accessToken: this.accessToken,
+			file,
+			filename: opts?.filename,
+			contentType: opts?.contentType,
+			signal: opts?.signal,
+		});
+	}
+
+	/** Current billing rule: `cost = max(tokens * pricePerToken, minCharge)`. */
+	async getPricing(signal?: AbortSignal): Promise<TtsPricing> {
+		const data = await graphqlRequest<{
+			ttsPricing: { price_per_token: number; min_charge: number };
+		}>({
+			graphqlUrl: this.graphqlUrl,
+			apiKey: this.apiKey,
+			accessToken: this.accessToken,
+			signal,
+			query: /* GraphQL */ `
+				query {
+					ttsPricing {
+						price_per_token
+						min_charge
+					}
+				}
+			`,
+		});
+		return {
+			pricePerToken: data.ttsPricing.price_per_token,
+			minCharge: data.ttsPricing.min_charge,
+		};
+	}
+
+	/** Enqueue a synthesis job; resolves immediately with the queued job. */
+	async submit(opts: SubmitTtsOptions, signal?: AbortSignal): Promise<TtsJob> {
+		const data = await graphqlRequest<{ submitTts: TtsJobWire }>({
+			graphqlUrl: this.graphqlUrl,
+			apiKey: this.apiKey,
+			accessToken: this.accessToken,
+			signal,
+			query: /* GraphQL */ `
+				mutation SubmitTts($input: SynthesizeTtsInput!) {
+					submitTts(input: $input) {
+						${JOB_FIELDS}
+					}
+				}
+			`,
+			variables: {
+				input: {
+					reference_key: opts.referenceKey,
+					text: opts.text,
+					params: opts.params ?? null,
+				},
+			},
+		});
+		return toJob(data.submitTts);
+	}
+
+	/** Read the current status/result of a job. */
+	async getJob(jobId: string, signal?: AbortSignal): Promise<TtsJob> {
+		const data = await graphqlRequest<{ ttsJob: TtsJobWire }>({
+			graphqlUrl: this.graphqlUrl,
+			apiKey: this.apiKey,
+			accessToken: this.accessToken,
+			signal,
+			query: /* GraphQL */ `
+				query TtsJob($jobId: String!) {
+					ttsJob(jobId: $jobId) {
+						${JOB_FIELDS}
+					}
+				}
+			`,
+			variables: { jobId },
+		});
+		return toJob(data.ttsJob);
+	}
+
+	/** Cancel a job. Only effective while it is still `queued`. */
+	async cancel(jobId: string, signal?: AbortSignal): Promise<TtsJob> {
+		const data = await graphqlRequest<{ cancelTtsJob: TtsJobWire }>({
+			graphqlUrl: this.graphqlUrl,
+			apiKey: this.apiKey,
+			accessToken: this.accessToken,
+			signal,
+			query: /* GraphQL */ `
+				mutation CancelTtsJob($jobId: String!) {
+					cancelTtsJob(jobId: $jobId) {
+						${JOB_FIELDS}
+					}
+				}
+			`,
+			variables: { jobId },
+		});
+		return toJob(data.cancelTtsJob);
+	}
+
+	/**
+	 * One-call synthesis: (optionally upload the reference voice,) submit, then
+	 * poll until the job finishes. Resolves with the `TtsResult` on success;
+	 * rejects if the job fails/cancels, times out, or `signal` aborts.
+	 */
+	async synthesize(opts: SynthesizeOptions): Promise<TtsResult> {
+		if (!opts.referenceKey && !opts.referenceAudio) {
+			throw new Error(
+				"synthesize() requires either `referenceKey` or `referenceAudio`"
+			);
+		}
+		const pollIntervalMs = opts.pollIntervalMs ?? 1500;
+		const timeoutMs = opts.timeoutMs ?? 300_000;
+		const deadline = Date.now() + timeoutMs;
+
+		const referenceKey =
+			opts.referenceKey ??
+			(await this.uploadReferenceAudio(opts.referenceAudio!, {
+				signal: opts.signal,
+			})).key;
+
+		const submitted = await this.submit(
+			{ referenceKey, text: opts.text, params: opts.params },
+			opts.signal
+		);
+		opts.onJob?.(submitted);
+
+		let job = submitted;
+		try {
+			while (job.status !== "done") {
+				if (opts.signal?.aborted) {
+					throw new DOMException("Aborted", "AbortError");
+				}
+				if (job.status === "failed") {
+					throw new Error(job.error || "TTS job failed");
+				}
+				if (job.status === "cancelled") {
+					throw new Error("TTS job was cancelled");
+				}
+				if (Date.now() > deadline) {
+					throw new Error(
+						`Timed out waiting for TTS job ${job.jobId} after ${timeoutMs}ms`
+					);
+				}
+				await delay(pollIntervalMs, opts.signal);
+				job = await this.getJob(job.jobId, opts.signal);
+				opts.onJob?.(job);
+			}
+		} catch (err) {
+			// Best-effort: stop a still-queued job so we don't pay for a result
+			// nobody is waiting for.
+			if (opts.signal?.aborted) {
+				this.cancel(job.jobId).catch(() => {});
+			}
+			throw err;
+		}
+
+		if (!job.result) {
+			throw new Error("TTS job is done but carried no result");
+		}
+		return job.result;
+	}
+}
+
+function toJob(wire: TtsJobWire): TtsJob {
+	return {
+		jobId: wire.job_id,
+		status: wire.status,
+		position: wire.position,
+		result: wire.result
+			? {
+					key: wire.result.key,
+					url: wire.result.url,
+					tokenCount: wire.result.token_count,
+					audioDurationSec: wire.result.audio_duration_sec,
+					amountCharged: wire.result.amount_charged,
+					balanceAfter: wire.result.balance_after,
+				}
+			: null,
+		error: wire.error,
+	};
+}
+
+function delay(ms: number, signal?: AbortSignal): Promise<void> {
+	return new Promise<void>((resolve, reject) => {
+		if (signal?.aborted) {
+			reject(new DOMException("Aborted", "AbortError"));
+			return;
+		}
+		const timer = setTimeout(() => {
+			signal?.removeEventListener("abort", onAbort);
+			resolve();
+		}, ms);
+		const onAbort = () => {
+			clearTimeout(timer);
+			reject(new DOMException("Aborted", "AbortError"));
+		};
+		signal?.addEventListener("abort", onAbort, { once: true });
+	});
+}

package/src/types.ts

@@ -0,0 +1,201 @@
+// Wire-protocol types mirrored from ServerAPI / signaling.
+
+export interface RTCServersConfig {
+	urls: string[];
+	username?: string;
+	credential?: string;
+}
+
+export interface RTCPreferences {
+	model_id: string;
+	sample_rate: number;
+	pitch?: number;
+	rms_mix_rate?: number;
+	f0_threshold?: number;
+	block_time?: number;
+	crossfade_time?: number;
+	extra_time?: number;
+	f0_autotune?: boolean;
+	f0_autotune_strength?: number;
+	proposed_pitch?: boolean;
+	proposed_pitch_threshold?: number;
+	enable_limiter?: boolean;
+	limiter_threshold?: number;
+	enable_lookahead?: boolean;
+	lookahead_time?: number;
+	formant?: number;
+	index_rate?: number;
+	protect?: number;
+	threshold?: number;
+	[key: string]: unknown;
+}
+
+/**
+ * Per-task parameters for offline file inference (voice-to-voice). Forwarded
+ * verbatim to the inference node alongside `task_start`. Distinct from the
+ * live `RTCPreferences` — file inference exposes `f0_method`, `use_pv`, etc.
+ */
+export interface FileInferencePreferences {
+	pitch?: number;
+	f0_method?: "rmvpe" | "fcpe";
+	f0_threshold?: number;
+	index_rate?: number;
+	protect?: number;
+	f0_autotune?: boolean;
+	f0_autotune_strength?: number;
+	proposed_pitch?: boolean;
+	proposed_pitch_threshold?: number;
+	sample_rate?: number;
+	formant?: number;
+	block_time?: number;
+	crossfade_time?: number;
+	extra_time?: number;
+	use_pv?: boolean;
+	rms_mix_rate?: number;
+	threshold?: number;
+	enable_limiter?: boolean;
+	limiter_threshold?: number;
+	enable_lookahead?: boolean;
+	lookahead_time?: number;
+	[key: string]: unknown;
+}
+
+export type OutgoingMessage =
+	| { type: "offer"; sdp?: string; preferences: RTCPreferences }
+	| { type: "ice_candidate"; candidate: RTCIceCandidateInit }
+	| { type: "config"; preferences: Partial<RTCPreferences> }
+	| {
+			type: "task_start";
+			task_id: string;
+			audio_key: string;
+			generate_name?: string;
+			format?: string;
+			preferences?: FileInferencePreferences;
+	  }
+	| { type: "task_stop"; task_id?: string }
+	| { type: "exit" }
+	| { type: "ping" }
+	| { type: "pong" };
+
+export enum RTCStatusCode {
+	ERROR = 2000,
+	GPU_INSUFFICIENT = 2001,
+	DUPLICATE_CONNECTION = 2002,
+	MODEL_NOT_FOUND = 2003,
+	UNPAID_SERVICE = 2004,
+	REQUEST_TOO_FAST = 2005,
+
+	CONNECTED = 3000,
+	REQUEST_RECEIVED = 3001,
+	TRACK_READY = 3002,
+	RESPONSE_SENT = 3003,
+	LOADING_MODEL = 3004,
+	SERVICE_READY = 3009,
+
+	// File inference (voice-to-voice) task lifecycle codes.
+	TASK_PROGRESS = 3010,
+	TASK_FINISHED = 3011,
+	TASK_ACK = 3012,
+
+	SHUTDOWN = 4000,
+	SERVER_CLOSED = 5000,
+}
+
+export type TaskStatus = "success" | "failure" | "cancelled";
+
+export type IncomingMessage =
+	| {
+			type: "message" | "shutdown" | "error";
+			message?: string;
+			code?: number;
+	  }
+	| { type: "answer"; sdp: string }
+	| { type: "ice_candidate"; candidate: RTCIceCandidateInit }
+	| {
+			type: "task_ack";
+			task_id: string;
+			status: "queued" | "started";
+			queue_position?: number;
+			code?: number;
+	  }
+	| { type: "task_progress"; task_id: string; progress: number; code?: number }
+	| {
+			type: "task_finished";
+			task_id: string;
+			status: TaskStatus;
+			result_key?: string;
+			download_url?: string;
+			error?: string;
+			code?: number;
+	  }
+	| { type: "ping" }
+	| { type: "pong" }
+	| Record<string, unknown>;
+
+export type ConnectionState =
+	| "idle"
+	| "signaling"
+	| "negotiating"
+	| "connecting"
+	| "connected"
+	| "closing"
+	| "closed"
+	| "error";
+
+export interface ConvbasedClientOptions {
+	/** Convbased API key issued in the Web console. Required unless `accessToken` is provided. */
+	apiKey?: string;
+	/** Optional JWT access token; takes a back seat to `apiKey` if both are provided. */
+	accessToken?: string;
+	/**
+	 * Signaling WebSocket URL. Defaults to the production Convbased endpoint
+	 * (`wss://api.weights.chat/api/signaling/ws`). Override only for
+	 * self-hosted deployments. URLs ending in `/ws` are used as-is; bare
+	 * hosts get `/signaling/ws` appended.
+	 */
+	signalingUrl?: string;
+	/**
+	 * GraphQL endpoint used to fetch TURN credentials. Defaults to the
+	 * production Convbased endpoint (`https://api.weights.chat/api/v1/graphql`).
+	 * Pass `false` to disable the auto-fetch entirely (will fall back to
+	 * `iceServers` if provided, else public STUN).
+	 */
+	graphqlUrl?: string | false;
+	/** Statically-configured ICE servers. If omitted and `graphqlUrl` is set, the SDK fetches `rtcServers`. */
+	iceServers?: RTCServersConfig[];
+	/** `relay` forces TURN-only — useful when STUN is blocked. */
+	iceTransportPolicy?: RTCIceTransportPolicy;
+	/** Opus bitrate in kbps. Default 64. */
+	bitrate?: number;
+	/** Send stereo. Default false. */
+	stereo?: boolean;
+	/** How long to wait for `SERVICE_READY` after sending the offer, in ms. Default 120_000. */
+	signalingTimeoutMs?: number;
+	/** How long to wait for the initial WebSocket open, in ms. Default 20_000. */
+	connectTimeoutMs?: number;
+	/** Optional logger; defaults to `console` for warn/error only. */
+	logger?: Partial<Pick<Console, "debug" | "info" | "warn" | "error">>;
+}
+
+export interface ConnectOptions {
+	/** Model ID to load on the inference node — required. */
+	modelId: string;
+	/** Microphone constraints. Pass an existing `MediaStream` to skip getUserMedia entirely. */
+	audio?: MediaStream | MediaTrackConstraints | boolean;
+	/** Additional RVC preferences forwarded to the node. */
+	preferences?: Partial<Omit<RTCPreferences, "model_id" | "sample_rate">>;
+	/** Sample rate to advertise to the node. Defaults to the AudioContext's `sampleRate`, falling back to 48000. */
+	sampleRate?: number;
+}
+
+export interface ConnectionStats {
+	rttMs: number;
+	jitter: number;
+	packetsLost: number;
+}
+
+export interface ServerMessageEvent {
+	code?: number;
+	message?: string;
+	raw: IncomingMessage;
+}

package/src/upload.ts

@@ -0,0 +1,132 @@
+// Audio upload helpers shared by TTS (reference voice) and file inference
+// (source audio). Two steps mirror Convbased-Web: ask the GraphQL service for
+// a presigned PUT (`requestAudioUpload`), then PUT the bytes straight to object
+// storage. The returned COS `key` is what you hand to `submitTts` /
+// `startTask`.
+
+import { graphqlRequest, type GraphQLAuth } from "./graphql.js";
+
+export interface HeaderKV {
+	name: string;
+	value: string;
+}
+
+export interface PresignedUpload {
+	/** COS object key — pass this back to the service (e.g. as `reference_key` / `audio_key`). */
+	key: string;
+	/** Presigned URL to PUT the bytes to. */
+	upload_url: string;
+	/** HTTP method to use for the upload (always `PUT` today). */
+	method: string;
+	/** Headers the upload request must include (notably `Content-Type`). */
+	headers: HeaderKV[];
+	/** Seconds until the presigned URL expires. */
+	expires_in: number;
+	bucket?: string | null;
+	region?: string | null;
+	/** CDN base URL for the bucket, when configured. */
+	url?: string | null;
+}
+
+const REQUEST_AUDIO_UPLOAD = /* GraphQL */ `
+	mutation RequestAudioUpload($input: RequestUploadInput!) {
+		requestAudioUpload(input: $input) {
+			key
+			upload_url
+			method
+			expires_in
+			headers {
+				name
+				value
+			}
+			bucket
+			region
+			url
+		}
+	}
+`;
+
+/** Ask the service for a presigned PUT for an audio file. */
+export async function requestAudioUpload(
+	args: GraphQLAuth & {
+		graphqlUrl: string;
+		filename: string;
+		contentType: string;
+		size: number;
+		signal?: AbortSignal;
+	}
+): Promise<PresignedUpload> {
+	const data = await graphqlRequest<{ requestAudioUpload: PresignedUpload }>({
+		graphqlUrl: args.graphqlUrl,
+		apiKey: args.apiKey,
+		accessToken: args.accessToken,
+		signal: args.signal,
+		query: REQUEST_AUDIO_UPLOAD,
+		variables: {
+			input: {
+				filename: args.filename,
+				content_type: args.contentType,
+				size: args.size,
+			},
+		},
+	});
+	return data.requestAudioUpload;
+}
+
+/** PUT raw bytes to a presigned upload target. */
+export async function putToPresigned(
+	presigned: PresignedUpload,
+	body: Blob | ArrayBuffer | ArrayBufferView,
+	signal?: AbortSignal
+): Promise<void> {
+	const headers: Record<string, string> = {};
+	for (const h of presigned.headers ?? []) {
+		if (h?.name) headers[h.name] = h.value;
+	}
+	const res = await fetch(presigned.upload_url, {
+		method: presigned.method || "PUT",
+		headers,
+		body: body as BodyInit,
+		signal,
+	});
+	if (!res.ok) {
+		throw new Error(
+			`Audio upload failed: HTTP ${res.status} ${res.statusText}`
+		);
+	}
+}
+
+/**
+ * Upload an audio `Blob`/`File` end-to-end (presign + PUT) and resolve the COS
+ * `key`. Filename and content type are taken from the `File` when available;
+ * override via `opts` when uploading a bare `Blob`.
+ */
+export async function uploadAudio(
+	args: GraphQLAuth & {
+		graphqlUrl: string;
+		file: Blob;
+		filename?: string;
+		contentType?: string;
+		signal?: AbortSignal;
+	}
+): Promise<{ key: string }> {
+	const maybeFile = args.file as File;
+	const filename = args.filename ?? maybeFile.name ?? "audio.wav";
+	const contentType =
+		args.contentType ||
+		args.file.type ||
+		"application/octet-stream";
+
+	const presigned = await requestAudioUpload({
+		graphqlUrl: args.graphqlUrl,
+		apiKey: args.apiKey,
+		accessToken: args.accessToken,
+		signal: args.signal,
+		filename,
+		contentType,
+		size: args.file.size,
+	});
+
+	await putToPresigned(presigned, args.file, args.signal);
+	return { key: presigned.key };
+}