@decartai/sdk 0.0.47 → 0.0.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +9 -2
- package/dist/index.js +7 -2
- package/dist/queue/polling.js +9 -4
- package/dist/queue/types.d.ts +2 -0
- package/dist/realtime/client.d.ts +7 -8
- package/dist/realtime/client.js +128 -27
- package/dist/realtime/diagnostics.d.ts +78 -0
- package/dist/realtime/subscribe-client.d.ts +2 -0
- package/dist/realtime/telemetry-reporter.js +120 -0
- package/dist/realtime/webrtc-connection.js +156 -22
- package/dist/realtime/webrtc-manager.js +39 -5
- package/dist/realtime/webrtc-stats.d.ts +59 -0
- package/dist/realtime/webrtc-stats.js +154 -0
- package/dist/shared/types.d.ts +1 -0
- package/dist/shared/types.js +11 -4
- package/dist/utils/errors.d.ts +5 -1
- package/dist/utils/errors.js +39 -5
- package/dist/utils/logger.d.ts +18 -0
- package/dist/utils/logger.js +37 -0
- package/package.json +5 -2
package/dist/index.d.ts
CHANGED
|
@@ -1,13 +1,16 @@
|
|
|
1
|
+
import { LogLevel, Logger, createConsoleLogger, noopLogger } from "./utils/logger.js";
|
|
1
2
|
import { ImageModelDefinition, ImageModels, Model, ModelDefinition, RealTimeModels, VideoModelDefinition, VideoModels, isImageModel, isRealtimeModel, isVideoModel, models } from "./shared/model.js";
|
|
2
3
|
import { FileInput, ProcessOptions, ReactNativeFile } from "./process/types.js";
|
|
3
4
|
import { ProcessClient } from "./process/client.js";
|
|
4
5
|
import { JobStatus, JobStatusResponse, JobSubmitResponse, QueueJobResult, QueueSubmitAndPollOptions, QueueSubmitOptions } from "./queue/types.js";
|
|
5
6
|
import { QueueClient } from "./queue/client.js";
|
|
6
7
|
import { DecartSDKError, ERROR_CODES } from "./utils/errors.js";
|
|
8
|
+
import { ConnectionPhase, DiagnosticEvent, DiagnosticEventName, DiagnosticEvents, IceCandidateEvent, IceStateEvent, PeerConnectionStateEvent, PhaseTimingEvent, ReconnectEvent, SelectedCandidatePairEvent, SignalingStateEvent, VideoStallEvent } from "./realtime/diagnostics.js";
|
|
7
9
|
import { ConnectionState } from "./realtime/types.js";
|
|
8
10
|
import { SetInput } from "./realtime/methods.js";
|
|
9
11
|
import { RealTimeSubscribeClient, SubscribeEvents, SubscribeOptions } from "./realtime/subscribe-client.js";
|
|
10
|
-
import {
|
|
12
|
+
import { WebRTCStats } from "./realtime/webrtc-stats.js";
|
|
13
|
+
import { Events, RealTimeClient, RealTimeClientConnectOptions, RealTimeClientInitialState } from "./realtime/client.js";
|
|
11
14
|
import { ModelState } from "./shared/types.js";
|
|
12
15
|
import { CreateTokenResponse, TokensClient } from "./tokens/client.js";
|
|
13
16
|
|
|
@@ -17,11 +20,15 @@ type DecartClientOptions = {
|
|
|
17
20
|
apiKey?: never;
|
|
18
21
|
baseUrl?: string;
|
|
19
22
|
integration?: string;
|
|
23
|
+
logger?: Logger;
|
|
24
|
+
telemetry?: boolean;
|
|
20
25
|
} | {
|
|
21
26
|
proxy?: never;
|
|
22
27
|
apiKey?: string;
|
|
23
28
|
baseUrl?: string;
|
|
24
29
|
integration?: string;
|
|
30
|
+
logger?: Logger;
|
|
31
|
+
telemetry?: boolean;
|
|
25
32
|
};
|
|
26
33
|
/**
|
|
27
34
|
* Create a Decart API client.
|
|
@@ -121,4 +128,4 @@ declare const createDecartClient: (options?: DecartClientOptions) => {
|
|
|
121
128
|
tokens: TokensClient;
|
|
122
129
|
};
|
|
123
130
|
//#endregion
|
|
124
|
-
export { type
|
|
131
|
+
export { type ConnectionPhase, type ConnectionState, type CreateTokenResponse, DecartClientOptions, type DecartSDKError, type DiagnosticEvent, type DiagnosticEventName, type DiagnosticEvents, ERROR_CODES, type FileInput, type IceCandidateEvent, type IceStateEvent, type ImageModelDefinition, type ImageModels, type JobStatus, type JobStatusResponse, type JobSubmitResponse, type LogLevel, type Logger, type Model, type ModelDefinition, type ModelState, type PeerConnectionStateEvent, type PhaseTimingEvent, type ProcessClient, type ProcessOptions, type QueueClient, type QueueJobResult, type QueueSubmitAndPollOptions, type QueueSubmitOptions, type ReactNativeFile, type RealTimeClient, type RealTimeClientConnectOptions, type RealTimeClientInitialState, type Events as RealTimeEvents, type RealTimeModels, type RealTimeSubscribeClient, type ReconnectEvent, type SelectedCandidatePairEvent, type SetInput, type SignalingStateEvent, type SubscribeEvents, type SubscribeOptions, type TokensClient, type VideoModelDefinition, type VideoModels, type VideoStallEvent, type WebRTCStats, createConsoleLogger, createDecartClient, isImageModel, isRealtimeModel, isVideoModel, models, noopLogger };
|
package/dist/index.js
CHANGED
|
@@ -5,6 +5,7 @@ import { isImageModel, isRealtimeModel, isVideoModel, models } from "./shared/mo
|
|
|
5
5
|
import { createRealTimeClient } from "./realtime/client.js";
|
|
6
6
|
import { createTokensClient } from "./tokens/client.js";
|
|
7
7
|
import { readEnv } from "./utils/env.js";
|
|
8
|
+
import { createConsoleLogger, noopLogger } from "./utils/logger.js";
|
|
8
9
|
import { z } from "zod";
|
|
9
10
|
|
|
10
11
|
//#region src/index.ts
|
|
@@ -56,11 +57,15 @@ const createDecartClient = (options = {}) => {
|
|
|
56
57
|
if (isProxyMode && "proxy" in parsedOptions.data && parsedOptions.data.proxy) baseUrl = parsedOptions.data.proxy;
|
|
57
58
|
else baseUrl = parsedOptions.data.baseUrl || "https://api.decart.ai";
|
|
58
59
|
const { integration } = parsedOptions.data;
|
|
60
|
+
const logger = "logger" in options && options.logger ? options.logger : noopLogger;
|
|
61
|
+
const telemetryEnabled = "telemetry" in options && options.telemetry === false ? false : true;
|
|
59
62
|
return {
|
|
60
63
|
realtime: createRealTimeClient({
|
|
61
64
|
baseUrl: "wss://api3.decart.ai",
|
|
62
65
|
apiKey: apiKey || "",
|
|
63
|
-
integration
|
|
66
|
+
integration,
|
|
67
|
+
logger,
|
|
68
|
+
telemetryEnabled
|
|
64
69
|
}),
|
|
65
70
|
process: createProcessClient({
|
|
66
71
|
baseUrl,
|
|
@@ -81,4 +86,4 @@ const createDecartClient = (options = {}) => {
|
|
|
81
86
|
};
|
|
82
87
|
|
|
83
88
|
//#endregion
|
|
84
|
-
export { ERROR_CODES, createDecartClient, isImageModel, isRealtimeModel, isVideoModel, models };
|
|
89
|
+
export { ERROR_CODES, createConsoleLogger, createDecartClient, isImageModel, isRealtimeModel, isVideoModel, models, noopLogger };
|
package/dist/queue/polling.js
CHANGED
|
@@ -19,12 +19,17 @@ async function pollUntilComplete({ checkStatus, getContent, onStatusChange, sign
|
|
|
19
19
|
if (signal?.aborted) throw new Error("Polling aborted");
|
|
20
20
|
const status = await checkStatus();
|
|
21
21
|
if (onStatusChange) onStatusChange(status);
|
|
22
|
-
if (status.status === "completed")
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
22
|
+
if (status.status === "completed") {
|
|
23
|
+
const data = await getContent();
|
|
24
|
+
return {
|
|
25
|
+
status: "completed",
|
|
26
|
+
job_id: status.job_id,
|
|
27
|
+
data
|
|
28
|
+
};
|
|
29
|
+
}
|
|
26
30
|
if (status.status === "failed") return {
|
|
27
31
|
status: "failed",
|
|
32
|
+
job_id: status.job_id,
|
|
28
33
|
error: "Job failed"
|
|
29
34
|
};
|
|
30
35
|
await sleep(POLLING_DEFAULTS.interval);
|
package/dist/queue/types.d.ts
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { DecartSDKError } from "../utils/errors.js";
|
|
2
|
+
import { DiagnosticEvent } from "./diagnostics.js";
|
|
2
3
|
import { ConnectionState } from "./types.js";
|
|
3
4
|
import { SetInput } from "./methods.js";
|
|
5
|
+
import { WebRTCStats } from "./webrtc-stats.js";
|
|
4
6
|
import { z } from "zod";
|
|
5
7
|
|
|
6
8
|
//#region src/realtime/client.d.ts
|
|
@@ -10,13 +12,10 @@ declare const realTimeClientInitialStateSchema: z.ZodObject<{
|
|
|
10
12
|
text: z.ZodString;
|
|
11
13
|
enhance: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
|
|
12
14
|
}, z.core.$strip>>;
|
|
15
|
+
image: z.ZodOptional<z.ZodUnion<readonly [z.ZodCustom<Blob, Blob>, z.ZodCustom<File, File>, z.ZodString]>>;
|
|
13
16
|
}, z.core.$strip>;
|
|
14
17
|
type OnRemoteStreamFn = (stream: MediaStream) => void;
|
|
15
18
|
type RealTimeClientInitialState = z.infer<typeof realTimeClientInitialStateSchema>;
|
|
16
|
-
declare const avatarOptionsSchema: z.ZodObject<{
|
|
17
|
-
avatarImage: z.ZodUnion<readonly [z.ZodCustom<Blob, Blob>, z.ZodCustom<File, File>, z.ZodString]>;
|
|
18
|
-
}, z.core.$strip>;
|
|
19
|
-
type AvatarOptions = z.infer<typeof avatarOptionsSchema>;
|
|
20
19
|
declare const realTimeClientConnectOptionsSchema: z.ZodObject<{
|
|
21
20
|
model: z.ZodObject<{
|
|
22
21
|
name: z.ZodUnion<readonly [z.ZodUnion<readonly [z.ZodLiteral<"mirage">, z.ZodLiteral<"mirage_v2">, z.ZodLiteral<"lucy_v2v_720p_rt">, z.ZodLiteral<"lucy_2_rt">, z.ZodLiteral<"live_avatar">]>, z.ZodUnion<readonly [z.ZodLiteral<"lucy-dev-i2v">, z.ZodLiteral<"lucy-fast-v2v">, z.ZodLiteral<"lucy-pro-t2v">, z.ZodLiteral<"lucy-pro-i2v">, z.ZodLiteral<"lucy-pro-v2v">, z.ZodLiteral<"lucy-pro-flf2v">, z.ZodLiteral<"lucy-motion">, z.ZodLiteral<"lucy-restyle-v2v">]>, z.ZodUnion<readonly [z.ZodLiteral<"lucy-pro-t2i">, z.ZodLiteral<"lucy-pro-i2i">]>]>;
|
|
@@ -33,11 +32,9 @@ declare const realTimeClientConnectOptionsSchema: z.ZodObject<{
|
|
|
33
32
|
text: z.ZodString;
|
|
34
33
|
enhance: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
|
|
35
34
|
}, z.core.$strip>>;
|
|
35
|
+
image: z.ZodOptional<z.ZodUnion<readonly [z.ZodCustom<Blob, Blob>, z.ZodCustom<File, File>, z.ZodString]>>;
|
|
36
36
|
}, z.core.$strip>>;
|
|
37
37
|
customizeOffer: z.ZodOptional<z.ZodCustom<z.core.$InferInnerFunctionTypeAsync<z.core.$ZodFunctionArgs, z.core.$ZodFunctionOut>, z.core.$InferInnerFunctionTypeAsync<z.core.$ZodFunctionArgs, z.core.$ZodFunctionOut>>>;
|
|
38
|
-
avatar: z.ZodOptional<z.ZodObject<{
|
|
39
|
-
avatarImage: z.ZodUnion<readonly [z.ZodCustom<Blob, Blob>, z.ZodCustom<File, File>, z.ZodString]>;
|
|
40
|
-
}, z.core.$strip>>;
|
|
41
38
|
}, z.core.$strip>;
|
|
42
39
|
type RealTimeClientConnectOptions = z.infer<typeof realTimeClientConnectOptionsSchema>;
|
|
43
40
|
type Events = {
|
|
@@ -46,6 +43,8 @@ type Events = {
|
|
|
46
43
|
generationTick: {
|
|
47
44
|
seconds: number;
|
|
48
45
|
};
|
|
46
|
+
diagnostic: DiagnosticEvent;
|
|
47
|
+
stats: WebRTCStats;
|
|
49
48
|
};
|
|
50
49
|
type RealTimeClient = {
|
|
51
50
|
set: (input: SetInput) => Promise<void>;
|
|
@@ -69,4 +68,4 @@ type RealTimeClient = {
|
|
|
69
68
|
playAudio?: (audio: Blob | File | ArrayBuffer) => Promise<void>;
|
|
70
69
|
};
|
|
71
70
|
//#endregion
|
|
72
|
-
export {
|
|
71
|
+
export { Events, RealTimeClient, RealTimeClientConnectOptions, RealTimeClientInitialState };
|
package/dist/realtime/client.js
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { classifyWebrtcError } from "../utils/errors.js";
|
|
2
2
|
import { modelDefinitionSchema } from "../shared/model.js";
|
|
3
3
|
import { modelStateSchema } from "../shared/types.js";
|
|
4
4
|
import { AudioStreamManager } from "./audio-stream-manager.js";
|
|
5
5
|
import { createEventBuffer } from "./event-buffer.js";
|
|
6
6
|
import { realtimeMethods } from "./methods.js";
|
|
7
7
|
import { decodeSubscribeToken, encodeSubscribeToken } from "./subscribe-client.js";
|
|
8
|
+
import { NullTelemetryReporter, TelemetryReporter } from "./telemetry-reporter.js";
|
|
8
9
|
import { WebRTCManager } from "./webrtc-manager.js";
|
|
10
|
+
import { WebRTCStatsCollector } from "./webrtc-stats.js";
|
|
9
11
|
import { z } from "zod";
|
|
10
12
|
|
|
11
13
|
//#region src/realtime/client.ts
|
|
@@ -51,25 +53,19 @@ async function imageToBase64(image) {
|
|
|
51
53
|
}
|
|
52
54
|
const realTimeClientInitialStateSchema = modelStateSchema;
|
|
53
55
|
const createAsyncFunctionSchema = (schema) => z.custom((fn) => schema.implementAsync(fn));
|
|
54
|
-
const avatarOptionsSchema = z.object({ avatarImage: z.union([
|
|
55
|
-
z.instanceof(Blob),
|
|
56
|
-
z.instanceof(File),
|
|
57
|
-
z.string()
|
|
58
|
-
]) });
|
|
59
56
|
const realTimeClientConnectOptionsSchema = z.object({
|
|
60
57
|
model: modelDefinitionSchema,
|
|
61
58
|
onRemoteStream: z.custom((val) => typeof val === "function", { message: "onRemoteStream must be a function" }),
|
|
62
59
|
initialState: realTimeClientInitialStateSchema.optional(),
|
|
63
|
-
customizeOffer: createAsyncFunctionSchema(z.function()).optional()
|
|
64
|
-
avatar: avatarOptionsSchema.optional()
|
|
60
|
+
customizeOffer: createAsyncFunctionSchema(z.function()).optional()
|
|
65
61
|
});
|
|
66
62
|
const createRealTimeClient = (opts) => {
|
|
67
|
-
const { baseUrl, apiKey, integration } = opts;
|
|
63
|
+
const { baseUrl, apiKey, integration, logger } = opts;
|
|
68
64
|
const connect = async (stream, options) => {
|
|
69
65
|
const parsedOptions = realTimeClientConnectOptionsSchema.safeParse(options);
|
|
70
66
|
if (!parsedOptions.success) throw parsedOptions.error;
|
|
71
67
|
const isAvatarLive = options.model.name === "live_avatar";
|
|
72
|
-
const { onRemoteStream, initialState
|
|
68
|
+
const { onRemoteStream, initialState } = parsedOptions.data;
|
|
73
69
|
let audioStreamManager;
|
|
74
70
|
let inputStream;
|
|
75
71
|
if (isAvatarLive && !stream) {
|
|
@@ -77,14 +73,11 @@ const createRealTimeClient = (opts) => {
|
|
|
77
73
|
inputStream = audioStreamManager.getStream();
|
|
78
74
|
} else inputStream = stream ?? new MediaStream();
|
|
79
75
|
let webrtcManager;
|
|
76
|
+
let telemetryReporter = new NullTelemetryReporter();
|
|
77
|
+
let handleConnectionStateChange = null;
|
|
80
78
|
try {
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
const response = await fetch(avatar.avatarImage);
|
|
84
|
-
if (!response.ok) throw new Error(`Failed to fetch image: ${response.status} ${response.statusText}`);
|
|
85
|
-
avatarImageBase64 = await blobToBase64(await response.blob());
|
|
86
|
-
} else avatarImageBase64 = await blobToBase64(avatar.avatarImage);
|
|
87
|
-
const initialPrompt = isAvatarLive && initialState?.prompt ? {
|
|
79
|
+
const initialImage = initialState?.image ? await imageToBase64(initialState.image) : void 0;
|
|
80
|
+
const initialPrompt = initialState?.prompt ? {
|
|
88
81
|
text: initialState.prompt.text,
|
|
89
82
|
enhance: initialState.prompt.enhance
|
|
90
83
|
} : void 0;
|
|
@@ -93,27 +86,69 @@ const createRealTimeClient = (opts) => {
|
|
|
93
86
|
webrtcManager = new WebRTCManager({
|
|
94
87
|
webrtcUrl: `${url}?api_key=${encodeURIComponent(apiKey)}&model=${encodeURIComponent(options.model.name)}`,
|
|
95
88
|
integration,
|
|
89
|
+
logger,
|
|
90
|
+
onDiagnostic: (name, data) => {
|
|
91
|
+
emitOrBuffer("diagnostic", {
|
|
92
|
+
name,
|
|
93
|
+
data
|
|
94
|
+
});
|
|
95
|
+
addTelemetryDiagnostic(name, data);
|
|
96
|
+
},
|
|
96
97
|
onRemoteStream,
|
|
97
98
|
onConnectionStateChange: (state) => {
|
|
98
99
|
emitOrBuffer("connectionChange", state);
|
|
100
|
+
handleConnectionStateChange?.(state);
|
|
99
101
|
},
|
|
100
102
|
onError: (error) => {
|
|
101
|
-
|
|
102
|
-
emitOrBuffer("error",
|
|
103
|
+
logger.error("WebRTC error", { error: error.message });
|
|
104
|
+
emitOrBuffer("error", classifyWebrtcError(error));
|
|
103
105
|
},
|
|
104
106
|
customizeOffer: options.customizeOffer,
|
|
105
107
|
vp8MinBitrate: 300,
|
|
106
108
|
vp8StartBitrate: 600,
|
|
107
|
-
|
|
108
|
-
|
|
109
|
+
modelName: options.model.name,
|
|
110
|
+
initialImage,
|
|
109
111
|
initialPrompt
|
|
110
112
|
});
|
|
111
113
|
const manager = webrtcManager;
|
|
112
114
|
let sessionId = null;
|
|
113
115
|
let subscribeToken = null;
|
|
116
|
+
const pendingTelemetryDiagnostics = [];
|
|
117
|
+
let telemetryReporterReady = false;
|
|
118
|
+
const addTelemetryDiagnostic = (name, data, timestamp = Date.now()) => {
|
|
119
|
+
if (!opts.telemetryEnabled) return;
|
|
120
|
+
if (!telemetryReporterReady) {
|
|
121
|
+
pendingTelemetryDiagnostics.push({
|
|
122
|
+
name,
|
|
123
|
+
data,
|
|
124
|
+
timestamp
|
|
125
|
+
});
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
128
|
+
telemetryReporter.addDiagnostic({
|
|
129
|
+
name,
|
|
130
|
+
data,
|
|
131
|
+
timestamp
|
|
132
|
+
});
|
|
133
|
+
};
|
|
114
134
|
const sessionIdListener = (msg) => {
|
|
115
135
|
subscribeToken = encodeSubscribeToken(msg.session_id, msg.server_ip, msg.server_port);
|
|
116
136
|
sessionId = msg.session_id;
|
|
137
|
+
if (opts.telemetryEnabled) {
|
|
138
|
+
if (telemetryReporterReady) telemetryReporter.stop();
|
|
139
|
+
const reporter = new TelemetryReporter({
|
|
140
|
+
apiKey,
|
|
141
|
+
sessionId: msg.session_id,
|
|
142
|
+
model: options.model.name,
|
|
143
|
+
integration,
|
|
144
|
+
logger
|
|
145
|
+
});
|
|
146
|
+
reporter.start();
|
|
147
|
+
telemetryReporter = reporter;
|
|
148
|
+
telemetryReporterReady = true;
|
|
149
|
+
for (const diagnostic of pendingTelemetryDiagnostics) telemetryReporter.addDiagnostic(diagnostic);
|
|
150
|
+
pendingTelemetryDiagnostics.length = 0;
|
|
151
|
+
}
|
|
117
152
|
};
|
|
118
153
|
manager.getWebsocketMessageEmitter().on("sessionId", sessionIdListener);
|
|
119
154
|
const tickListener = (msg) => {
|
|
@@ -122,16 +157,74 @@ const createRealTimeClient = (opts) => {
|
|
|
122
157
|
manager.getWebsocketMessageEmitter().on("generationTick", tickListener);
|
|
123
158
|
await manager.connect(inputStream);
|
|
124
159
|
const methods = realtimeMethods(manager, imageToBase64);
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
160
|
+
let statsCollector = null;
|
|
161
|
+
let statsCollectorPeerConnection = null;
|
|
162
|
+
const STALL_FPS_THRESHOLD = .5;
|
|
163
|
+
let videoStalled = false;
|
|
164
|
+
let stallStartMs = 0;
|
|
165
|
+
const startStatsCollection = () => {
|
|
166
|
+
statsCollector?.stop();
|
|
167
|
+
videoStalled = false;
|
|
168
|
+
stallStartMs = 0;
|
|
169
|
+
statsCollector = new WebRTCStatsCollector();
|
|
170
|
+
const pc = manager.getPeerConnection();
|
|
171
|
+
statsCollectorPeerConnection = pc;
|
|
172
|
+
if (pc) statsCollector.start(pc, (stats) => {
|
|
173
|
+
emitOrBuffer("stats", stats);
|
|
174
|
+
telemetryReporter.addStats(stats);
|
|
175
|
+
const fps = stats.video?.framesPerSecond ?? 0;
|
|
176
|
+
if (!videoStalled && stats.video && fps < STALL_FPS_THRESHOLD) {
|
|
177
|
+
videoStalled = true;
|
|
178
|
+
stallStartMs = Date.now();
|
|
179
|
+
emitOrBuffer("diagnostic", {
|
|
180
|
+
name: "videoStall",
|
|
181
|
+
data: {
|
|
182
|
+
stalled: true,
|
|
183
|
+
durationMs: 0
|
|
184
|
+
}
|
|
185
|
+
});
|
|
186
|
+
addTelemetryDiagnostic("videoStall", {
|
|
187
|
+
stalled: true,
|
|
188
|
+
durationMs: 0
|
|
189
|
+
}, stallStartMs);
|
|
190
|
+
} else if (videoStalled && fps >= STALL_FPS_THRESHOLD) {
|
|
191
|
+
const durationMs = Date.now() - stallStartMs;
|
|
192
|
+
videoStalled = false;
|
|
193
|
+
emitOrBuffer("diagnostic", {
|
|
194
|
+
name: "videoStall",
|
|
195
|
+
data: {
|
|
196
|
+
stalled: false,
|
|
197
|
+
durationMs
|
|
198
|
+
}
|
|
199
|
+
});
|
|
200
|
+
addTelemetryDiagnostic("videoStall", {
|
|
201
|
+
stalled: false,
|
|
202
|
+
durationMs
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
});
|
|
206
|
+
return () => {
|
|
207
|
+
statsCollector?.stop();
|
|
208
|
+
statsCollector = null;
|
|
209
|
+
statsCollectorPeerConnection = null;
|
|
210
|
+
};
|
|
211
|
+
};
|
|
212
|
+
handleConnectionStateChange = (state) => {
|
|
213
|
+
if (!opts.telemetryEnabled) return;
|
|
214
|
+
if (state !== "connected" && state !== "generating") return;
|
|
215
|
+
const peerConnection = manager.getPeerConnection();
|
|
216
|
+
if (!peerConnection || peerConnection === statsCollectorPeerConnection) return;
|
|
217
|
+
startStatsCollection();
|
|
218
|
+
};
|
|
219
|
+
if (opts.telemetryEnabled) startStatsCollection();
|
|
129
220
|
const client = {
|
|
130
221
|
set: methods.set,
|
|
131
222
|
setPrompt: methods.setPrompt,
|
|
132
223
|
isConnected: () => manager.isConnected(),
|
|
133
224
|
getConnectionState: () => manager.getConnectionState(),
|
|
134
225
|
disconnect: () => {
|
|
226
|
+
statsCollector?.stop();
|
|
227
|
+
telemetryReporter.stop();
|
|
135
228
|
stop();
|
|
136
229
|
manager.cleanup();
|
|
137
230
|
audioStreamManager?.cleanup();
|
|
@@ -157,6 +250,7 @@ const createRealTimeClient = (opts) => {
|
|
|
157
250
|
flush();
|
|
158
251
|
return client;
|
|
159
252
|
} catch (error) {
|
|
253
|
+
telemetryReporter.stop();
|
|
160
254
|
webrtcManager?.cleanup();
|
|
161
255
|
audioStreamManager?.cleanup();
|
|
162
256
|
throw error;
|
|
@@ -171,13 +265,20 @@ const createRealTimeClient = (opts) => {
|
|
|
171
265
|
webrtcManager = new WebRTCManager({
|
|
172
266
|
webrtcUrl: subscribeUrl,
|
|
173
267
|
integration,
|
|
268
|
+
logger,
|
|
269
|
+
onDiagnostic: (name, data) => {
|
|
270
|
+
emitOrBuffer("diagnostic", {
|
|
271
|
+
name,
|
|
272
|
+
data
|
|
273
|
+
});
|
|
274
|
+
},
|
|
174
275
|
onRemoteStream: options.onRemoteStream,
|
|
175
276
|
onConnectionStateChange: (state) => {
|
|
176
277
|
emitOrBuffer("connectionChange", state);
|
|
177
278
|
},
|
|
178
279
|
onError: (error) => {
|
|
179
|
-
|
|
180
|
-
emitOrBuffer("error",
|
|
280
|
+
logger.error("WebRTC subscribe error", { error: error.message });
|
|
281
|
+
emitOrBuffer("error", classifyWebrtcError(error));
|
|
181
282
|
}
|
|
182
283
|
});
|
|
183
284
|
const manager = webrtcManager;
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
//#region src/realtime/diagnostics.d.ts
|
|
2
|
+
/** Connection phase names for timing events. */
|
|
3
|
+
type ConnectionPhase = "websocket" | "avatar-image" | "initial-prompt" | "webrtc-handshake" | "total";
|
|
4
|
+
type PhaseTimingEvent = {
|
|
5
|
+
phase: ConnectionPhase;
|
|
6
|
+
durationMs: number;
|
|
7
|
+
success: boolean;
|
|
8
|
+
error?: string;
|
|
9
|
+
};
|
|
10
|
+
type IceCandidateEvent = {
|
|
11
|
+
source: "local" | "remote";
|
|
12
|
+
candidateType: "host" | "srflx" | "prflx" | "relay" | "unknown";
|
|
13
|
+
protocol: "udp" | "tcp" | "unknown";
|
|
14
|
+
address?: string;
|
|
15
|
+
port?: number;
|
|
16
|
+
};
|
|
17
|
+
type IceStateEvent = {
|
|
18
|
+
state: string;
|
|
19
|
+
previousState: string;
|
|
20
|
+
timestampMs: number;
|
|
21
|
+
};
|
|
22
|
+
type PeerConnectionStateEvent = {
|
|
23
|
+
state: string;
|
|
24
|
+
previousState: string;
|
|
25
|
+
timestampMs: number;
|
|
26
|
+
};
|
|
27
|
+
type SignalingStateEvent = {
|
|
28
|
+
state: string;
|
|
29
|
+
previousState: string;
|
|
30
|
+
timestampMs: number;
|
|
31
|
+
};
|
|
32
|
+
type SelectedCandidatePairEvent = {
|
|
33
|
+
local: {
|
|
34
|
+
candidateType: string;
|
|
35
|
+
protocol: string;
|
|
36
|
+
address?: string;
|
|
37
|
+
port?: number;
|
|
38
|
+
};
|
|
39
|
+
remote: {
|
|
40
|
+
candidateType: string;
|
|
41
|
+
protocol: string;
|
|
42
|
+
address?: string;
|
|
43
|
+
port?: number;
|
|
44
|
+
};
|
|
45
|
+
};
|
|
46
|
+
type ReconnectEvent = {
|
|
47
|
+
attempt: number;
|
|
48
|
+
maxAttempts: number;
|
|
49
|
+
durationMs: number;
|
|
50
|
+
success: boolean;
|
|
51
|
+
error?: string;
|
|
52
|
+
};
|
|
53
|
+
type VideoStallEvent = {
|
|
54
|
+
/** True when a stall is detected, false when recovered. */
|
|
55
|
+
stalled: boolean;
|
|
56
|
+
/** Duration of the stall in ms (0 when stall first detected, actual duration on recovery). */
|
|
57
|
+
durationMs: number;
|
|
58
|
+
};
|
|
59
|
+
/** All diagnostic event types keyed by name. */
|
|
60
|
+
type DiagnosticEvents = {
|
|
61
|
+
phaseTiming: PhaseTimingEvent;
|
|
62
|
+
iceCandidate: IceCandidateEvent;
|
|
63
|
+
iceStateChange: IceStateEvent;
|
|
64
|
+
peerConnectionStateChange: PeerConnectionStateEvent;
|
|
65
|
+
signalingStateChange: SignalingStateEvent;
|
|
66
|
+
selectedCandidatePair: SelectedCandidatePairEvent;
|
|
67
|
+
reconnect: ReconnectEvent;
|
|
68
|
+
videoStall: VideoStallEvent;
|
|
69
|
+
};
|
|
70
|
+
type DiagnosticEventName = keyof DiagnosticEvents;
|
|
71
|
+
/** A single diagnostic event with its name and typed data. */
|
|
72
|
+
type DiagnosticEvent = { [K in DiagnosticEventName]: {
|
|
73
|
+
name: K;
|
|
74
|
+
data: DiagnosticEvents[K];
|
|
75
|
+
} }[DiagnosticEventName];
|
|
76
|
+
/** Callback for emitting diagnostic events from the connection/manager layers. */
|
|
77
|
+
//#endregion
|
|
78
|
+
export { ConnectionPhase, DiagnosticEvent, DiagnosticEventName, DiagnosticEvents, IceCandidateEvent, IceStateEvent, PeerConnectionStateEvent, PhaseTimingEvent, ReconnectEvent, SelectedCandidatePairEvent, SignalingStateEvent, VideoStallEvent };
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { DecartSDKError } from "../utils/errors.js";
|
|
2
|
+
import { DiagnosticEvent } from "./diagnostics.js";
|
|
2
3
|
import { ConnectionState } from "./types.js";
|
|
3
4
|
|
|
4
5
|
//#region src/realtime/subscribe-client.d.ts
|
|
@@ -6,6 +7,7 @@ import { ConnectionState } from "./types.js";
|
|
|
6
7
|
type SubscribeEvents = {
|
|
7
8
|
connectionChange: ConnectionState;
|
|
8
9
|
error: DecartSDKError;
|
|
10
|
+
diagnostic: DiagnosticEvent;
|
|
9
11
|
};
|
|
10
12
|
type RealTimeSubscribeClient = {
|
|
11
13
|
isConnected: () => boolean;
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import { VERSION } from "../version.js";
|
|
2
|
+
import { buildAuthHeaders } from "../shared/request.js";
|
|
3
|
+
|
|
4
|
+
//#region src/realtime/telemetry-reporter.ts
|
|
5
|
+
const DEFAULT_REPORT_INTERVAL_MS = 1e4;
|
|
6
|
+
const TELEMETRY_URL = "https://platform.decart.ai/api/v1/telemetry";
|
|
7
|
+
/**
|
|
8
|
+
* Maximum number of items per array (stats / diagnostics) in a single report.
|
|
9
|
+
* Matches the backend Zod schema which enforces `z.array().max(120)`.
|
|
10
|
+
*/
|
|
11
|
+
const MAX_ITEMS_PER_REPORT = 120;
|
|
12
|
+
/** No-op reporter that silently discards all data. Used when telemetry is disabled. */
|
|
13
|
+
var NullTelemetryReporter = class {
|
|
14
|
+
start() {}
|
|
15
|
+
addStats() {}
|
|
16
|
+
addDiagnostic() {}
|
|
17
|
+
flush() {}
|
|
18
|
+
stop() {}
|
|
19
|
+
};
|
|
20
|
+
var TelemetryReporter = class {
|
|
21
|
+
apiKey;
|
|
22
|
+
sessionId;
|
|
23
|
+
model;
|
|
24
|
+
integration;
|
|
25
|
+
logger;
|
|
26
|
+
reportIntervalMs;
|
|
27
|
+
intervalId = null;
|
|
28
|
+
statsBuffer = [];
|
|
29
|
+
diagnosticsBuffer = [];
|
|
30
|
+
constructor(options) {
|
|
31
|
+
this.apiKey = options.apiKey;
|
|
32
|
+
this.sessionId = options.sessionId;
|
|
33
|
+
this.model = options.model;
|
|
34
|
+
this.integration = options.integration;
|
|
35
|
+
this.logger = options.logger;
|
|
36
|
+
this.reportIntervalMs = options.reportIntervalMs ?? DEFAULT_REPORT_INTERVAL_MS;
|
|
37
|
+
}
|
|
38
|
+
/** Start the periodic reporting timer. */
|
|
39
|
+
start() {
|
|
40
|
+
if (this.intervalId !== null) return;
|
|
41
|
+
this.intervalId = setInterval(() => this.flush(), this.reportIntervalMs);
|
|
42
|
+
}
|
|
43
|
+
/** Add a stats snapshot to the buffer. */
|
|
44
|
+
addStats(stats) {
|
|
45
|
+
this.statsBuffer.push(stats);
|
|
46
|
+
}
|
|
47
|
+
/** Add a diagnostic event to the buffer. */
|
|
48
|
+
addDiagnostic(event) {
|
|
49
|
+
this.diagnosticsBuffer.push(event);
|
|
50
|
+
}
|
|
51
|
+
/** Flush buffered data immediately. */
|
|
52
|
+
flush() {
|
|
53
|
+
this.sendReport(false);
|
|
54
|
+
}
|
|
55
|
+
/** Stop the reporter and send a final report with keepalive. */
|
|
56
|
+
stop() {
|
|
57
|
+
if (this.intervalId !== null) {
|
|
58
|
+
clearInterval(this.intervalId);
|
|
59
|
+
this.intervalId = null;
|
|
60
|
+
}
|
|
61
|
+
this.sendReport(true);
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Build a single chunk from the front of the buffers, respecting MAX_ITEMS_PER_REPORT.
|
|
65
|
+
* Returns null when both buffers are empty.
|
|
66
|
+
*/
|
|
67
|
+
createReportChunk() {
|
|
68
|
+
if (this.statsBuffer.length === 0 && this.diagnosticsBuffer.length === 0) return null;
|
|
69
|
+
const tags = {
|
|
70
|
+
session_id: this.sessionId,
|
|
71
|
+
sdk_version: VERSION,
|
|
72
|
+
...this.model ? { model: this.model } : {},
|
|
73
|
+
...this.integration ? { integration: this.integration } : {}
|
|
74
|
+
};
|
|
75
|
+
return {
|
|
76
|
+
sessionId: this.sessionId,
|
|
77
|
+
timestamp: Date.now(),
|
|
78
|
+
sdkVersion: VERSION,
|
|
79
|
+
...this.model ? { model: this.model } : {},
|
|
80
|
+
tags,
|
|
81
|
+
stats: this.statsBuffer.splice(0, MAX_ITEMS_PER_REPORT),
|
|
82
|
+
diagnostics: this.diagnosticsBuffer.splice(0, MAX_ITEMS_PER_REPORT)
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
sendReport(keepalive) {
|
|
86
|
+
if (this.statsBuffer.length === 0 && this.diagnosticsBuffer.length === 0) return;
|
|
87
|
+
try {
|
|
88
|
+
const commonHeaders = {
|
|
89
|
+
...buildAuthHeaders({
|
|
90
|
+
apiKey: this.apiKey,
|
|
91
|
+
integration: this.integration
|
|
92
|
+
}),
|
|
93
|
+
"Content-Type": "application/json"
|
|
94
|
+
};
|
|
95
|
+
let chunk = this.createReportChunk();
|
|
96
|
+
while (chunk !== null) {
|
|
97
|
+
const isLast = this.statsBuffer.length === 0 && this.diagnosticsBuffer.length === 0;
|
|
98
|
+
fetch(TELEMETRY_URL, {
|
|
99
|
+
method: "POST",
|
|
100
|
+
headers: commonHeaders,
|
|
101
|
+
body: JSON.stringify(chunk),
|
|
102
|
+
keepalive: keepalive && isLast
|
|
103
|
+
}).then((response) => {
|
|
104
|
+
if (!response.ok) this.logger.warn("Telemetry report rejected", {
|
|
105
|
+
status: response.status,
|
|
106
|
+
statusText: response.statusText
|
|
107
|
+
});
|
|
108
|
+
}).catch((error) => {
|
|
109
|
+
this.logger.debug("Telemetry report failed", { error: String(error) });
|
|
110
|
+
});
|
|
111
|
+
chunk = this.createReportChunk();
|
|
112
|
+
}
|
|
113
|
+
} catch (error) {
|
|
114
|
+
this.logger.debug("Telemetry report failed", { error: String(error) });
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
//#endregion
|
|
120
|
+
export { NullTelemetryReporter, TelemetryReporter };
|