@decartai/sdk 0.0.66 → 0.0.68
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -0
- package/dist/index.d.ts +2 -2
- package/dist/realtime/client.d.ts +7 -2
- package/dist/realtime/client.js +48 -119
- package/dist/realtime/mirror-stream.js +116 -0
- package/dist/realtime/{diagnostics.d.ts → observability/diagnostics.d.ts} +1 -1
- package/dist/realtime/observability/realtime-observability.js +109 -0
- package/dist/realtime/{telemetry-reporter.js → observability/telemetry-reporter.js} +3 -3
- package/dist/realtime/observability/webrtc-stats.d.ts +147 -0
- package/dist/realtime/observability/webrtc-stats.js +278 -0
- package/dist/realtime/subscribe-client.d.ts +3 -1
- package/dist/realtime/webrtc-connection.js +18 -19
- package/dist/realtime/webrtc-manager.js +24 -3
- package/package.json +1 -1
- package/dist/realtime/webrtc-stats.d.ts +0 -59
- package/dist/realtime/webrtc-stats.js +0 -154
package/README.md
CHANGED
|
@@ -62,6 +62,33 @@ realtimeClient.setPrompt("Cyberpunk city");
|
|
|
62
62
|
realtimeClient.disconnect();
|
|
63
63
|
```
|
|
64
64
|
|
|
65
|
+
#### Front-camera mirroring
|
|
66
|
+
|
|
67
|
+
Pre-flip the input stream:
|
|
68
|
+
|
|
69
|
+
```ts
|
|
70
|
+
const realtimeClient = await client.realtime.connect(stream, {
|
|
71
|
+
model,
|
|
72
|
+
mirror: "auto", // or true to always mirror
|
|
73
|
+
// ...
|
|
74
|
+
});
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Options:
|
|
78
|
+
- `false` (default) — never mirror.
|
|
79
|
+
- `"auto"` — mirror when the input track reports `facingMode: "user"` (mobile front cameras).
|
|
80
|
+
- `true` — always mirror (e.g. desktop webcams).
|
|
81
|
+
|
|
82
|
+
#### Output resolution
|
|
83
|
+
|
|
84
|
+
```ts
|
|
85
|
+
const realtimeClient = await client.realtime.connect(stream, {
|
|
86
|
+
model,
|
|
87
|
+
resolution: "1080p", // default: "720p"
|
|
88
|
+
// ...
|
|
89
|
+
});
|
|
90
|
+
```
|
|
91
|
+
|
|
65
92
|
### Async Processing (Queue API)
|
|
66
93
|
|
|
67
94
|
For video generation jobs, use the queue API to submit jobs and poll for results:
|
package/dist/index.d.ts
CHANGED
|
@@ -5,11 +5,11 @@ import { ProcessClient } from "./process/client.js";
|
|
|
5
5
|
import { JobStatus, JobStatusResponse, JobSubmitResponse, QueueJobResult, QueueSubmitAndPollOptions, QueueSubmitOptions } from "./queue/types.js";
|
|
6
6
|
import { QueueClient } from "./queue/client.js";
|
|
7
7
|
import { DecartSDKError, ERROR_CODES } from "./utils/errors.js";
|
|
8
|
-
import { ConnectionPhase, DiagnosticEvent, DiagnosticEventName, DiagnosticEvents, IceCandidateEvent, IceStateEvent, PeerConnectionStateEvent, PhaseTimingEvent, ReconnectEvent, SelectedCandidatePairEvent, SignalingStateEvent, VideoStallEvent } from "./realtime/diagnostics.js";
|
|
9
8
|
import { ConnectionState } from "./realtime/types.js";
|
|
9
|
+
import { ConnectionPhase, DiagnosticEvent, DiagnosticEventName, DiagnosticEvents, IceCandidateEvent, IceStateEvent, PeerConnectionStateEvent, PhaseTimingEvent, ReconnectEvent, SelectedCandidatePairEvent, SignalingStateEvent, VideoStallEvent } from "./realtime/observability/diagnostics.js";
|
|
10
|
+
import { WebRTCStats } from "./realtime/observability/webrtc-stats.js";
|
|
10
11
|
import { SetInput } from "./realtime/methods.js";
|
|
11
12
|
import { RealTimeSubscribeClient, SubscribeEvents, SubscribeOptions } from "./realtime/subscribe-client.js";
|
|
12
|
-
import { WebRTCStats } from "./realtime/webrtc-stats.js";
|
|
13
13
|
import { Events, RealTimeClient, RealTimeClientConnectOptions, RealTimeClientInitialState } from "./realtime/client.js";
|
|
14
14
|
import { ModelState } from "./shared/types.js";
|
|
15
15
|
import { CreateTokenOptions, CreateTokenResponse, TokensClient } from "./tokens/client.js";
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { CustomModelDefinition, ModelDefinition } from "../shared/model.js";
|
|
2
2
|
import { DecartSDKError } from "../utils/errors.js";
|
|
3
|
-
import { DiagnosticEvent } from "./diagnostics.js";
|
|
4
3
|
import { ConnectionState } from "./types.js";
|
|
4
|
+
import { DiagnosticEvent } from "./observability/diagnostics.js";
|
|
5
|
+
import { WebRTCStats } from "./observability/webrtc-stats.js";
|
|
5
6
|
import { SetInput } from "./methods.js";
|
|
6
|
-
import { WebRTCStats } from "./webrtc-stats.js";
|
|
7
7
|
import { z } from "zod";
|
|
8
8
|
|
|
9
9
|
//#region src/realtime/client.d.ts
|
|
@@ -36,6 +36,11 @@ declare const realTimeClientConnectOptionsSchema: z.ZodObject<{
|
|
|
36
36
|
image: z.ZodOptional<z.ZodUnion<readonly [z.ZodCustom<Blob, Blob>, z.ZodCustom<File, File>, z.ZodString]>>;
|
|
37
37
|
}, z.core.$strip>>;
|
|
38
38
|
customizeOffer: z.ZodOptional<z.ZodCustom<z.core.$InferInnerFunctionTypeAsync<z.core.$ZodFunctionArgs, z.core.$ZodFunctionOut>, z.core.$InferInnerFunctionTypeAsync<z.core.$ZodFunctionArgs, z.core.$ZodFunctionOut>>>;
|
|
39
|
+
mirror: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<"auto">, z.ZodBoolean]>>;
|
|
40
|
+
resolution: z.ZodOptional<z.ZodEnum<{
|
|
41
|
+
"720p": "720p";
|
|
42
|
+
"1080p": "1080p";
|
|
43
|
+
}>>;
|
|
39
44
|
}, z.core.$strip>;
|
|
40
45
|
type RealTimeClientConnectOptions = Omit<z.infer<typeof realTimeClientConnectOptionsSchema>, "model"> & {
|
|
41
46
|
model: ModelDefinition | CustomModelDefinition;
|
package/dist/realtime/client.js
CHANGED
|
@@ -3,10 +3,10 @@ import { modelDefinitionSchema } from "../shared/model.js";
|
|
|
3
3
|
import { modelStateSchema } from "../shared/types.js";
|
|
4
4
|
import { createEventBuffer } from "./event-buffer.js";
|
|
5
5
|
import { realtimeMethods } from "./methods.js";
|
|
6
|
+
import { createMirroredStream, shouldMirrorTrack } from "./mirror-stream.js";
|
|
7
|
+
import { RealtimeObservability } from "./observability/realtime-observability.js";
|
|
6
8
|
import { decodeSubscribeToken, encodeSubscribeToken } from "./subscribe-client.js";
|
|
7
|
-
import { NullTelemetryReporter, TelemetryReporter } from "./telemetry-reporter.js";
|
|
8
9
|
import { WebRTCManager } from "./webrtc-manager.js";
|
|
9
|
-
import { WebRTCStatsCollector } from "./webrtc-stats.js";
|
|
10
10
|
import { z } from "zod";
|
|
11
11
|
|
|
12
12
|
//#region src/realtime/client.ts
|
|
@@ -56,18 +56,39 @@ const realTimeClientConnectOptionsSchema = z.object({
|
|
|
56
56
|
model: modelDefinitionSchema,
|
|
57
57
|
onRemoteStream: z.custom((val) => typeof val === "function", { message: "onRemoteStream must be a function" }),
|
|
58
58
|
initialState: realTimeClientInitialStateSchema.optional(),
|
|
59
|
-
customizeOffer: createAsyncFunctionSchema(z.function()).optional()
|
|
59
|
+
customizeOffer: createAsyncFunctionSchema(z.function()).optional(),
|
|
60
|
+
mirror: z.union([z.literal("auto"), z.boolean()]).optional(),
|
|
61
|
+
resolution: z.enum(["720p", "1080p"]).optional()
|
|
60
62
|
});
|
|
61
63
|
const createRealTimeClient = (opts) => {
|
|
62
64
|
const { baseUrl, apiKey, integration, logger } = opts;
|
|
63
65
|
const connect = async (stream, options) => {
|
|
64
66
|
const parsedOptions = realTimeClientConnectOptionsSchema.safeParse(options);
|
|
65
67
|
if (!parsedOptions.success) throw parsedOptions.error;
|
|
66
|
-
const { onRemoteStream, initialState } = parsedOptions.data;
|
|
67
|
-
const
|
|
68
|
+
const { onRemoteStream, initialState, resolution } = parsedOptions.data;
|
|
69
|
+
const mirror = parsedOptions.data.mirror ?? false;
|
|
70
|
+
let inputStream = stream ?? new MediaStream();
|
|
71
|
+
let mirroredStream;
|
|
72
|
+
if (mirror !== false) try {
|
|
73
|
+
const firstVideoTrack = inputStream.getVideoTracks?.()[0];
|
|
74
|
+
if (firstVideoTrack && (mirror === true || shouldMirrorTrack(firstVideoTrack))) {
|
|
75
|
+
mirroredStream = createMirroredStream(inputStream, { fps: options.model.fps });
|
|
76
|
+
inputStream = mirroredStream.stream;
|
|
77
|
+
} else if (mirror === true && !firstVideoTrack) logger.warn("mirror: true requested but no video track was found on the input stream");
|
|
78
|
+
} catch (error) {
|
|
79
|
+
logger.warn("Failed to mirror input stream; falling back to un-mirrored input", { error: error instanceof Error ? error.message : String(error) });
|
|
80
|
+
}
|
|
68
81
|
let webrtcManager;
|
|
69
|
-
|
|
70
|
-
|
|
82
|
+
const { emitter: eventEmitter, emitOrBuffer, flush, stop } = createEventBuffer();
|
|
83
|
+
const observability = new RealtimeObservability({
|
|
84
|
+
telemetryEnabled: opts.telemetryEnabled,
|
|
85
|
+
apiKey,
|
|
86
|
+
model: options.model.name,
|
|
87
|
+
integration,
|
|
88
|
+
logger,
|
|
89
|
+
onDiagnostic: (event) => emitOrBuffer("diagnostic", event),
|
|
90
|
+
onStats: opts.telemetryEnabled ? (stats) => emitOrBuffer("stats", stats) : void 0
|
|
91
|
+
});
|
|
71
92
|
try {
|
|
72
93
|
const initialImage = initialState?.image ? await imageToBase64(initialState.image) : void 0;
|
|
73
94
|
const initialPrompt = initialState?.prompt ? {
|
|
@@ -75,22 +96,15 @@ const createRealTimeClient = (opts) => {
|
|
|
75
96
|
enhance: initialState.prompt.enhance
|
|
76
97
|
} : void 0;
|
|
77
98
|
const url = `${baseUrl}${options.model.urlPath}`;
|
|
78
|
-
const
|
|
99
|
+
const resolutionQs = resolution ? `&resolution=${encodeURIComponent(resolution)}` : "";
|
|
79
100
|
webrtcManager = new WebRTCManager({
|
|
80
|
-
webrtcUrl: `${url}?api_key=${encodeURIComponent(apiKey)}&model=${encodeURIComponent(options.model.name)}`,
|
|
101
|
+
webrtcUrl: `${url}?api_key=${encodeURIComponent(apiKey)}&model=${encodeURIComponent(options.model.name)}${resolutionQs}`,
|
|
81
102
|
integration,
|
|
82
103
|
logger,
|
|
83
|
-
|
|
84
|
-
emitOrBuffer("diagnostic", {
|
|
85
|
-
name,
|
|
86
|
-
data
|
|
87
|
-
});
|
|
88
|
-
addTelemetryDiagnostic(name, data);
|
|
89
|
-
},
|
|
104
|
+
observability,
|
|
90
105
|
onRemoteStream,
|
|
91
106
|
onConnectionStateChange: (state) => {
|
|
92
107
|
emitOrBuffer("connectionChange", state);
|
|
93
|
-
handleConnectionStateChange?.(state);
|
|
94
108
|
},
|
|
95
109
|
onError: (error) => {
|
|
96
110
|
logger.error("WebRTC error", { error: error.message });
|
|
@@ -105,42 +119,10 @@ const createRealTimeClient = (opts) => {
|
|
|
105
119
|
const manager = webrtcManager;
|
|
106
120
|
let sessionId = null;
|
|
107
121
|
let subscribeToken = null;
|
|
108
|
-
const pendingTelemetryDiagnostics = [];
|
|
109
|
-
let telemetryReporterReady = false;
|
|
110
|
-
const addTelemetryDiagnostic = (name, data, timestamp = Date.now()) => {
|
|
111
|
-
if (!opts.telemetryEnabled) return;
|
|
112
|
-
if (!telemetryReporterReady) {
|
|
113
|
-
pendingTelemetryDiagnostics.push({
|
|
114
|
-
name,
|
|
115
|
-
data,
|
|
116
|
-
timestamp
|
|
117
|
-
});
|
|
118
|
-
return;
|
|
119
|
-
}
|
|
120
|
-
telemetryReporter.addDiagnostic({
|
|
121
|
-
name,
|
|
122
|
-
data,
|
|
123
|
-
timestamp
|
|
124
|
-
});
|
|
125
|
-
};
|
|
126
122
|
const sessionIdListener = (msg) => {
|
|
127
123
|
subscribeToken = encodeSubscribeToken(msg.session_id, msg.server_ip, msg.server_port);
|
|
128
124
|
sessionId = msg.session_id;
|
|
129
|
-
|
|
130
|
-
if (telemetryReporterReady) telemetryReporter.stop();
|
|
131
|
-
const reporter = new TelemetryReporter({
|
|
132
|
-
apiKey,
|
|
133
|
-
sessionId: msg.session_id,
|
|
134
|
-
model: options.model.name,
|
|
135
|
-
integration,
|
|
136
|
-
logger
|
|
137
|
-
});
|
|
138
|
-
reporter.start();
|
|
139
|
-
telemetryReporter = reporter;
|
|
140
|
-
telemetryReporterReady = true;
|
|
141
|
-
for (const diagnostic of pendingTelemetryDiagnostics) telemetryReporter.addDiagnostic(diagnostic);
|
|
142
|
-
pendingTelemetryDiagnostics.length = 0;
|
|
143
|
-
}
|
|
125
|
+
observability.sessionStarted(msg.session_id);
|
|
144
126
|
};
|
|
145
127
|
manager.getWebsocketMessageEmitter().on("sessionId", sessionIdListener);
|
|
146
128
|
const tickListener = (msg) => {
|
|
@@ -149,76 +131,16 @@ const createRealTimeClient = (opts) => {
|
|
|
149
131
|
manager.getWebsocketMessageEmitter().on("generationTick", tickListener);
|
|
150
132
|
await manager.connect(inputStream);
|
|
151
133
|
const methods = realtimeMethods(manager, imageToBase64);
|
|
152
|
-
let statsCollector = null;
|
|
153
|
-
let statsCollectorPeerConnection = null;
|
|
154
|
-
const STALL_FPS_THRESHOLD = .5;
|
|
155
|
-
let videoStalled = false;
|
|
156
|
-
let stallStartMs = 0;
|
|
157
|
-
const startStatsCollection = () => {
|
|
158
|
-
statsCollector?.stop();
|
|
159
|
-
videoStalled = false;
|
|
160
|
-
stallStartMs = 0;
|
|
161
|
-
statsCollector = new WebRTCStatsCollector();
|
|
162
|
-
const pc = manager.getPeerConnection();
|
|
163
|
-
statsCollectorPeerConnection = pc;
|
|
164
|
-
if (pc) statsCollector.start(pc, (stats) => {
|
|
165
|
-
emitOrBuffer("stats", stats);
|
|
166
|
-
telemetryReporter.addStats(stats);
|
|
167
|
-
const fps = stats.video?.framesPerSecond ?? 0;
|
|
168
|
-
if (!videoStalled && stats.video && fps < STALL_FPS_THRESHOLD) {
|
|
169
|
-
videoStalled = true;
|
|
170
|
-
stallStartMs = Date.now();
|
|
171
|
-
emitOrBuffer("diagnostic", {
|
|
172
|
-
name: "videoStall",
|
|
173
|
-
data: {
|
|
174
|
-
stalled: true,
|
|
175
|
-
durationMs: 0
|
|
176
|
-
}
|
|
177
|
-
});
|
|
178
|
-
addTelemetryDiagnostic("videoStall", {
|
|
179
|
-
stalled: true,
|
|
180
|
-
durationMs: 0
|
|
181
|
-
}, stallStartMs);
|
|
182
|
-
} else if (videoStalled && fps >= STALL_FPS_THRESHOLD) {
|
|
183
|
-
const durationMs = Date.now() - stallStartMs;
|
|
184
|
-
videoStalled = false;
|
|
185
|
-
emitOrBuffer("diagnostic", {
|
|
186
|
-
name: "videoStall",
|
|
187
|
-
data: {
|
|
188
|
-
stalled: false,
|
|
189
|
-
durationMs
|
|
190
|
-
}
|
|
191
|
-
});
|
|
192
|
-
addTelemetryDiagnostic("videoStall", {
|
|
193
|
-
stalled: false,
|
|
194
|
-
durationMs
|
|
195
|
-
});
|
|
196
|
-
}
|
|
197
|
-
});
|
|
198
|
-
return () => {
|
|
199
|
-
statsCollector?.stop();
|
|
200
|
-
statsCollector = null;
|
|
201
|
-
statsCollectorPeerConnection = null;
|
|
202
|
-
};
|
|
203
|
-
};
|
|
204
|
-
handleConnectionStateChange = (state) => {
|
|
205
|
-
if (!opts.telemetryEnabled) return;
|
|
206
|
-
if (state !== "connected" && state !== "generating") return;
|
|
207
|
-
const peerConnection = manager.getPeerConnection();
|
|
208
|
-
if (!peerConnection || peerConnection === statsCollectorPeerConnection) return;
|
|
209
|
-
startStatsCollection();
|
|
210
|
-
};
|
|
211
|
-
if (opts.telemetryEnabled) startStatsCollection();
|
|
212
134
|
const client = {
|
|
213
135
|
set: methods.set,
|
|
214
136
|
setPrompt: methods.setPrompt,
|
|
215
137
|
isConnected: () => manager.isConnected(),
|
|
216
138
|
getConnectionState: () => manager.getConnectionState(),
|
|
217
139
|
disconnect: () => {
|
|
218
|
-
|
|
219
|
-
telemetryReporter.stop();
|
|
140
|
+
observability.stop();
|
|
220
141
|
stop();
|
|
221
142
|
manager.cleanup();
|
|
143
|
+
mirroredStream?.dispose();
|
|
222
144
|
},
|
|
223
145
|
on: eventEmitter.on,
|
|
224
146
|
off: eventEmitter.off,
|
|
@@ -237,8 +159,9 @@ const createRealTimeClient = (opts) => {
|
|
|
237
159
|
flush();
|
|
238
160
|
return client;
|
|
239
161
|
} catch (error) {
|
|
240
|
-
|
|
162
|
+
observability.stop();
|
|
241
163
|
webrtcManager?.cleanup();
|
|
164
|
+
mirroredStream?.dispose();
|
|
242
165
|
throw error;
|
|
243
166
|
}
|
|
244
167
|
};
|
|
@@ -247,17 +170,21 @@ const createRealTimeClient = (opts) => {
|
|
|
247
170
|
const subscribeUrl = `${baseUrl}/subscribe/${encodeURIComponent(sid)}?IP=${encodeURIComponent(ip)}&port=${encodeURIComponent(port)}&api_key=${encodeURIComponent(apiKey)}`;
|
|
248
171
|
const { emitter: eventEmitter, emitOrBuffer, flush, stop } = createEventBuffer();
|
|
249
172
|
let webrtcManager;
|
|
173
|
+
const observability = new RealtimeObservability({
|
|
174
|
+
telemetryEnabled: opts.telemetryEnabled,
|
|
175
|
+
apiKey,
|
|
176
|
+
integration,
|
|
177
|
+
logger,
|
|
178
|
+
onDiagnostic: (event) => emitOrBuffer("diagnostic", event),
|
|
179
|
+
onStats: opts.telemetryEnabled ? (stats) => emitOrBuffer("stats", stats) : void 0
|
|
180
|
+
});
|
|
181
|
+
observability.sessionStarted(sid);
|
|
250
182
|
try {
|
|
251
183
|
webrtcManager = new WebRTCManager({
|
|
252
184
|
webrtcUrl: subscribeUrl,
|
|
253
185
|
integration,
|
|
254
186
|
logger,
|
|
255
|
-
|
|
256
|
-
emitOrBuffer("diagnostic", {
|
|
257
|
-
name,
|
|
258
|
-
data
|
|
259
|
-
});
|
|
260
|
-
},
|
|
187
|
+
observability,
|
|
261
188
|
onRemoteStream: options.onRemoteStream,
|
|
262
189
|
onConnectionStateChange: (state) => {
|
|
263
190
|
emitOrBuffer("connectionChange", state);
|
|
@@ -273,6 +200,7 @@ const createRealTimeClient = (opts) => {
|
|
|
273
200
|
isConnected: () => manager.isConnected(),
|
|
274
201
|
getConnectionState: () => manager.getConnectionState(),
|
|
275
202
|
disconnect: () => {
|
|
203
|
+
observability.stop();
|
|
276
204
|
stop();
|
|
277
205
|
manager.cleanup();
|
|
278
206
|
},
|
|
@@ -282,6 +210,7 @@ const createRealTimeClient = (opts) => {
|
|
|
282
210
|
flush();
|
|
283
211
|
return client;
|
|
284
212
|
} catch (error) {
|
|
213
|
+
observability.stop();
|
|
285
214
|
webrtcManager?.cleanup();
|
|
286
215
|
throw error;
|
|
287
216
|
}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
//#region src/realtime/mirror-stream.ts
|
|
2
|
+
function isMediaStreamTrackProcessorSupported() {
|
|
3
|
+
return typeof globalThis !== "undefined" && typeof globalThis.MediaStreamTrackProcessor === "function" && typeof globalThis.MediaStreamTrackGenerator === "function";
|
|
4
|
+
}
|
|
5
|
+
function shouldMirrorTrack(track) {
|
|
6
|
+
if (track.kind !== "video") return false;
|
|
7
|
+
let facingMode;
|
|
8
|
+
try {
|
|
9
|
+
facingMode = track.getSettings?.().facingMode;
|
|
10
|
+
} catch {
|
|
11
|
+
return false;
|
|
12
|
+
}
|
|
13
|
+
return facingMode === "user";
|
|
14
|
+
}
|
|
15
|
+
function createMirroredStream(input, opts) {
|
|
16
|
+
const [sourceVideo] = input.getVideoTracks();
|
|
17
|
+
const audioTracks = input.getAudioTracks();
|
|
18
|
+
if (!sourceVideo) return {
|
|
19
|
+
stream: input,
|
|
20
|
+
dispose: () => {},
|
|
21
|
+
impl: "noop"
|
|
22
|
+
};
|
|
23
|
+
if (isMediaStreamTrackProcessorSupported()) return createWithTrackProcessor(sourceVideo, audioTracks);
|
|
24
|
+
return createWithCanvas(sourceVideo, audioTracks, opts.fps);
|
|
25
|
+
}
|
|
26
|
+
function createWithTrackProcessor(sourceVideo, audioTracks) {
|
|
27
|
+
const Processor = globalThis.MediaStreamTrackProcessor;
|
|
28
|
+
const Generator = globalThis.MediaStreamTrackGenerator;
|
|
29
|
+
if (!new OffscreenCanvas(1, 1).getContext("2d")) throw new Error("createMirroredStream: OffscreenCanvas 2D context unavailable");
|
|
30
|
+
const processor = new Processor({ track: sourceVideo });
|
|
31
|
+
const generator = new Generator({ kind: "video" });
|
|
32
|
+
let canvas = new OffscreenCanvas(1, 1);
|
|
33
|
+
let ctx = canvas.getContext("2d");
|
|
34
|
+
const transform = new TransformStream({ transform(frame, controller) {
|
|
35
|
+
const w = frame.displayWidth;
|
|
36
|
+
const h = frame.displayHeight;
|
|
37
|
+
if (canvas.width !== w || canvas.height !== h) {
|
|
38
|
+
canvas = new OffscreenCanvas(w, h);
|
|
39
|
+
ctx = canvas.getContext("2d");
|
|
40
|
+
}
|
|
41
|
+
let flipped;
|
|
42
|
+
try {
|
|
43
|
+
ctx.save();
|
|
44
|
+
ctx.setTransform(-1, 0, 0, 1, w, 0);
|
|
45
|
+
ctx.drawImage(frame, 0, 0, w, h);
|
|
46
|
+
ctx.restore();
|
|
47
|
+
flipped = new VideoFrame(canvas, {
|
|
48
|
+
timestamp: frame.timestamp,
|
|
49
|
+
alpha: "discard"
|
|
50
|
+
});
|
|
51
|
+
controller.enqueue(flipped);
|
|
52
|
+
flipped = void 0;
|
|
53
|
+
} finally {
|
|
54
|
+
flipped?.close();
|
|
55
|
+
frame.close();
|
|
56
|
+
}
|
|
57
|
+
} });
|
|
58
|
+
processor.readable.pipeThrough(transform).pipeTo(generator.writable).catch(() => {});
|
|
59
|
+
const stream = new MediaStream([generator, ...audioTracks]);
|
|
60
|
+
let disposed = false;
|
|
61
|
+
return {
|
|
62
|
+
stream,
|
|
63
|
+
impl: "track-processor",
|
|
64
|
+
dispose: () => {
|
|
65
|
+
if (disposed) return;
|
|
66
|
+
disposed = true;
|
|
67
|
+
generator.stop();
|
|
68
|
+
}
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
function createWithCanvas(sourceVideo, audioTracks, fps) {
|
|
72
|
+
if (typeof document === "undefined") throw new Error("createMirroredStream requires a DOM environment (document is undefined)");
|
|
73
|
+
const canvas = document.createElement("canvas");
|
|
74
|
+
const ctx = canvas.getContext("2d");
|
|
75
|
+
if (!ctx) throw new Error("createMirroredStream: 2D canvas context unavailable");
|
|
76
|
+
if (typeof canvas.captureStream !== "function") throw new Error("createMirroredStream: canvas.captureStream unavailable");
|
|
77
|
+
const [flippedTrack] = canvas.captureStream(fps).getVideoTracks();
|
|
78
|
+
if (!flippedTrack) throw new Error("createMirroredStream: canvas.captureStream produced no video track");
|
|
79
|
+
const video = document.createElement("video");
|
|
80
|
+
video.muted = true;
|
|
81
|
+
video.playsInline = true;
|
|
82
|
+
video.autoplay = true;
|
|
83
|
+
video.srcObject = new MediaStream([sourceVideo]);
|
|
84
|
+
let disposed = false;
|
|
85
|
+
let rafHandle = null;
|
|
86
|
+
const draw = () => {
|
|
87
|
+
if (disposed) return;
|
|
88
|
+
const w = video.videoWidth;
|
|
89
|
+
const h = video.videoHeight;
|
|
90
|
+
if (w > 0 && h > 0) {
|
|
91
|
+
if (canvas.width !== w) canvas.width = w;
|
|
92
|
+
if (canvas.height !== h) canvas.height = h;
|
|
93
|
+
ctx.save();
|
|
94
|
+
ctx.setTransform(-1, 0, 0, 1, w, 0);
|
|
95
|
+
ctx.drawImage(video, 0, 0, w, h);
|
|
96
|
+
ctx.restore();
|
|
97
|
+
}
|
|
98
|
+
rafHandle = requestAnimationFrame(draw);
|
|
99
|
+
};
|
|
100
|
+
video.play().catch(() => {});
|
|
101
|
+
rafHandle = requestAnimationFrame(draw);
|
|
102
|
+
return {
|
|
103
|
+
stream: new MediaStream([flippedTrack, ...audioTracks]),
|
|
104
|
+
impl: "canvas",
|
|
105
|
+
dispose: () => {
|
|
106
|
+
if (disposed) return;
|
|
107
|
+
disposed = true;
|
|
108
|
+
if (rafHandle !== null) cancelAnimationFrame(rafHandle);
|
|
109
|
+
flippedTrack.stop();
|
|
110
|
+
video.srcObject = null;
|
|
111
|
+
}
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
//#endregion
|
|
116
|
+
export { createMirroredStream, shouldMirrorTrack };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
//#region src/realtime/diagnostics.d.ts
|
|
1
|
+
//#region src/realtime/observability/diagnostics.d.ts
|
|
2
2
|
/** Connection phase names for timing events. */
|
|
3
3
|
type ConnectionPhase = "websocket" | "avatar-image" | "initial-prompt" | "webrtc-handshake" | "total";
|
|
4
4
|
type PhaseTimingEvent = {
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import { NullTelemetryReporter, TelemetryReporter } from "./telemetry-reporter.js";
|
|
2
|
+
import { WebRTCStatsCollector } from "./webrtc-stats.js";
|
|
3
|
+
|
|
4
|
+
//#region src/realtime/observability/realtime-observability.ts
|
|
5
|
+
const STALL_FPS_THRESHOLD = .5;
|
|
6
|
+
var RealtimeObservability = class {
|
|
7
|
+
telemetryReporter = new NullTelemetryReporter();
|
|
8
|
+
telemetryReporterReady = false;
|
|
9
|
+
pendingTelemetryDiagnostics = [];
|
|
10
|
+
statsCollector = null;
|
|
11
|
+
statsCollectorSource = null;
|
|
12
|
+
videoStalled = false;
|
|
13
|
+
stallStartMs = 0;
|
|
14
|
+
constructor(options) {
|
|
15
|
+
this.options = options;
|
|
16
|
+
}
|
|
17
|
+
diagnostic(name, data, timestamp = Date.now()) {
|
|
18
|
+
this.options.logger.debug(name, data);
|
|
19
|
+
this.options.onDiagnostic?.({
|
|
20
|
+
name,
|
|
21
|
+
data
|
|
22
|
+
});
|
|
23
|
+
this.addTelemetryDiagnostic(name, data, timestamp);
|
|
24
|
+
}
|
|
25
|
+
sessionStarted(sessionId) {
|
|
26
|
+
if (!this.options.telemetryEnabled) return;
|
|
27
|
+
if (this.telemetryReporterReady) this.telemetryReporter.stop();
|
|
28
|
+
const reporter = new TelemetryReporter({
|
|
29
|
+
apiKey: this.options.apiKey,
|
|
30
|
+
sessionId,
|
|
31
|
+
model: this.options.model,
|
|
32
|
+
integration: this.options.integration,
|
|
33
|
+
logger: this.options.logger
|
|
34
|
+
});
|
|
35
|
+
reporter.start();
|
|
36
|
+
this.telemetryReporter = reporter;
|
|
37
|
+
this.telemetryReporterReady = true;
|
|
38
|
+
for (const diagnostic of this.pendingTelemetryDiagnostics) this.telemetryReporter.addDiagnostic(diagnostic);
|
|
39
|
+
this.pendingTelemetryDiagnostics.length = 0;
|
|
40
|
+
}
|
|
41
|
+
setStatsProvider(source) {
|
|
42
|
+
if (!source) {
|
|
43
|
+
this.stopStats();
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
if (source === this.statsCollectorSource) return;
|
|
47
|
+
this.stopStats();
|
|
48
|
+
this.statsCollectorSource = source;
|
|
49
|
+
if (!this.options.telemetryEnabled && !this.options.onStats) return;
|
|
50
|
+
this.statsCollector = new WebRTCStatsCollector();
|
|
51
|
+
this.statsCollector.start(source, (stats) => this.handleStats(stats));
|
|
52
|
+
}
|
|
53
|
+
stopStats() {
|
|
54
|
+
this.statsCollector?.stop();
|
|
55
|
+
this.statsCollector = null;
|
|
56
|
+
this.statsCollectorSource = null;
|
|
57
|
+
this.resetStallDetection();
|
|
58
|
+
}
|
|
59
|
+
stop() {
|
|
60
|
+
this.stopStats();
|
|
61
|
+
this.telemetryReporter.stop();
|
|
62
|
+
this.telemetryReporter = new NullTelemetryReporter();
|
|
63
|
+
this.telemetryReporterReady = false;
|
|
64
|
+
this.pendingTelemetryDiagnostics.length = 0;
|
|
65
|
+
}
|
|
66
|
+
handleStats(stats) {
|
|
67
|
+
this.options.onStats?.(stats);
|
|
68
|
+
this.telemetryReporter.addStats(stats);
|
|
69
|
+
this.detectVideoStall(stats);
|
|
70
|
+
}
|
|
71
|
+
detectVideoStall(stats) {
|
|
72
|
+
const fps = stats.video?.framesPerSecond ?? 0;
|
|
73
|
+
if (!this.videoStalled && stats.video && fps < STALL_FPS_THRESHOLD) {
|
|
74
|
+
this.videoStalled = true;
|
|
75
|
+
this.stallStartMs = Date.now();
|
|
76
|
+
this.diagnostic("videoStall", {
|
|
77
|
+
stalled: true,
|
|
78
|
+
durationMs: 0
|
|
79
|
+
}, this.stallStartMs);
|
|
80
|
+
} else if (this.videoStalled && fps >= STALL_FPS_THRESHOLD) {
|
|
81
|
+
const durationMs = Date.now() - this.stallStartMs;
|
|
82
|
+
this.videoStalled = false;
|
|
83
|
+
this.diagnostic("videoStall", {
|
|
84
|
+
stalled: false,
|
|
85
|
+
durationMs
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
addTelemetryDiagnostic(name, data, timestamp) {
|
|
90
|
+
if (!this.options.telemetryEnabled) return;
|
|
91
|
+
const diagnostic = {
|
|
92
|
+
name,
|
|
93
|
+
data,
|
|
94
|
+
timestamp
|
|
95
|
+
};
|
|
96
|
+
if (!this.telemetryReporterReady) {
|
|
97
|
+
this.pendingTelemetryDiagnostics.push(diagnostic);
|
|
98
|
+
return;
|
|
99
|
+
}
|
|
100
|
+
this.telemetryReporter.addDiagnostic(diagnostic);
|
|
101
|
+
}
|
|
102
|
+
resetStallDetection() {
|
|
103
|
+
this.videoStalled = false;
|
|
104
|
+
this.stallStartMs = 0;
|
|
105
|
+
}
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
//#endregion
|
|
109
|
+
export { RealtimeObservability };
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { VERSION } from "
|
|
2
|
-
import { buildAuthHeaders } from "
|
|
1
|
+
import { VERSION } from "../../version.js";
|
|
2
|
+
import { buildAuthHeaders } from "../../shared/request.js";
|
|
3
3
|
|
|
4
|
-
//#region src/realtime/telemetry-reporter.ts
|
|
4
|
+
//#region src/realtime/observability/telemetry-reporter.ts
|
|
5
5
|
const DEFAULT_REPORT_INTERVAL_MS = 1e4;
|
|
6
6
|
const TELEMETRY_URL = "https://platform.decart.ai/api/v1/telemetry";
|
|
7
7
|
/**
|