@livepeer-frameworks/player-core 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -9
- package/dist/player.css +182 -42
- package/package.json +1 -1
- package/src/core/ABRController.ts +38 -36
- package/src/core/CodecUtils.ts +49 -46
- package/src/core/Disposable.ts +4 -4
- package/src/core/EventEmitter.ts +1 -1
- package/src/core/GatewayClient.ts +41 -39
- package/src/core/InteractionController.ts +89 -82
- package/src/core/LiveDurationProxy.ts +14 -15
- package/src/core/MetaTrackManager.ts +73 -65
- package/src/core/MistReporter.ts +72 -45
- package/src/core/MistSignaling.ts +59 -56
- package/src/core/PlayerController.ts +527 -384
- package/src/core/PlayerInterface.ts +83 -59
- package/src/core/PlayerManager.ts +79 -133
- package/src/core/PlayerRegistry.ts +59 -42
- package/src/core/QualityMonitor.ts +38 -31
- package/src/core/ScreenWakeLockManager.ts +8 -9
- package/src/core/SeekingUtils.ts +31 -22
- package/src/core/StreamStateClient.ts +74 -68
- package/src/core/SubtitleManager.ts +24 -22
- package/src/core/TelemetryReporter.ts +34 -31
- package/src/core/TimeFormat.ts +13 -17
- package/src/core/TimerManager.ts +24 -8
- package/src/core/UrlUtils.ts +20 -17
- package/src/core/detector.ts +44 -44
- package/src/core/index.ts +57 -48
- package/src/core/scorer.ts +136 -141
- package/src/core/selector.ts +2 -6
- package/src/global.d.ts +1 -1
- package/src/index.ts +46 -35
- package/src/players/DashJsPlayer.ts +164 -115
- package/src/players/HlsJsPlayer.ts +132 -78
- package/src/players/MewsWsPlayer/SourceBufferManager.ts +41 -36
- package/src/players/MewsWsPlayer/WebSocketManager.ts +9 -9
- package/src/players/MewsWsPlayer/index.ts +192 -152
- package/src/players/MewsWsPlayer/types.ts +21 -21
- package/src/players/MistPlayer.ts +45 -26
- package/src/players/MistWebRTCPlayer/index.ts +175 -129
- package/src/players/NativePlayer.ts +203 -143
- package/src/players/VideoJsPlayer.ts +170 -118
- package/src/players/WebCodecsPlayer/JitterBuffer.ts +6 -7
- package/src/players/WebCodecsPlayer/LatencyProfiles.ts +43 -43
- package/src/players/WebCodecsPlayer/RawChunkParser.ts +10 -10
- package/src/players/WebCodecsPlayer/SyncController.ts +45 -53
- package/src/players/WebCodecsPlayer/WebSocketController.ts +66 -68
- package/src/players/WebCodecsPlayer/index.ts +263 -221
- package/src/players/WebCodecsPlayer/polyfills/MediaStreamTrackGenerator.ts +12 -17
- package/src/players/WebCodecsPlayer/types.ts +56 -56
- package/src/players/WebCodecsPlayer/worker/decoder.worker.ts +238 -182
- package/src/players/WebCodecsPlayer/worker/types.ts +31 -31
- package/src/players/index.ts +8 -8
- package/src/styles/animations.css +2 -1
- package/src/styles/player.css +182 -42
- package/src/styles/tailwind.css +473 -159
- package/src/types.ts +43 -43
- package/src/vanilla/FrameWorksPlayer.ts +29 -14
- package/src/vanilla/index.ts +4 -4
|
@@ -19,15 +19,15 @@ import type {
|
|
|
19
19
|
DecodedFrame,
|
|
20
20
|
VideoDecoderInit,
|
|
21
21
|
AudioDecoderInit,
|
|
22
|
-
} from
|
|
23
|
-
import type { PipelineStats, FrameTrackerStats } from
|
|
22
|
+
} from "./types";
|
|
23
|
+
import type { PipelineStats, FrameTrackerStats } from "../types";
|
|
24
24
|
|
|
25
25
|
// ============================================================================
|
|
26
26
|
// Global State
|
|
27
27
|
// ============================================================================
|
|
28
28
|
|
|
29
29
|
const pipelines = new Map<number, PipelineState>();
|
|
30
|
-
let debugging: boolean |
|
|
30
|
+
let debugging: boolean | "verbose" = false;
|
|
31
31
|
let uidCounter = 0;
|
|
32
32
|
|
|
33
33
|
// Frame timing state (shared across all pipelines)
|
|
@@ -62,7 +62,9 @@ const WARMUP_TIMEOUT_MS = 300; // Reduced from 500ms - start faster to reduce la
|
|
|
62
62
|
function getTrackBaseTime(idx: number, frameTimeMs: number, now: number): number {
|
|
63
63
|
if (!trackBaseTimes.has(idx)) {
|
|
64
64
|
trackBaseTimes.set(idx, now - frameTimeMs / frameTiming.speed.combined);
|
|
65
|
-
log(
|
|
65
|
+
log(
|
|
66
|
+
`Track ${idx} baseTime: ${trackBaseTimes.get(idx)!.toFixed(0)} (first frame @ ${frameTimeMs.toFixed(0)}ms)`
|
|
67
|
+
);
|
|
66
68
|
}
|
|
67
69
|
return trackBaseTimes.get(idx)!;
|
|
68
70
|
}
|
|
@@ -77,7 +79,7 @@ function resetBaseTime(): void {
|
|
|
77
79
|
|
|
78
80
|
function cloneVideoFrame(frame: VideoFrame): VideoFrame | null {
|
|
79
81
|
try {
|
|
80
|
-
if (
|
|
82
|
+
if ("clone" in frame) {
|
|
81
83
|
return (frame as VideoFrame).clone();
|
|
82
84
|
}
|
|
83
85
|
return new VideoFrame(frame);
|
|
@@ -87,7 +89,7 @@ function cloneVideoFrame(frame: VideoFrame): VideoFrame | null {
|
|
|
87
89
|
}
|
|
88
90
|
|
|
89
91
|
function pushFrameHistory(pipeline: PipelineState, frame: VideoFrame, timestamp: number): void {
|
|
90
|
-
if (pipeline.track.type !==
|
|
92
|
+
if (pipeline.track.type !== "video") return;
|
|
91
93
|
if (!pipeline.frameHistory) pipeline.frameHistory = [];
|
|
92
94
|
|
|
93
95
|
const cloned = cloneVideoFrame(frame);
|
|
@@ -99,7 +101,9 @@ function pushFrameHistory(pipeline: PipelineState, frame: VideoFrame, timestamp:
|
|
|
99
101
|
while (pipeline.frameHistory.length > MAX_FRAME_HISTORY) {
|
|
100
102
|
const entry = pipeline.frameHistory.shift();
|
|
101
103
|
if (entry) {
|
|
102
|
-
try {
|
|
104
|
+
try {
|
|
105
|
+
entry.frame.close();
|
|
106
|
+
} catch {}
|
|
103
107
|
}
|
|
104
108
|
}
|
|
105
109
|
|
|
@@ -114,7 +118,7 @@ function alignHistoryCursorToLastOutput(pipeline: PipelineState): void {
|
|
|
114
118
|
return;
|
|
115
119
|
}
|
|
116
120
|
// Find first history entry greater than last output, then step back one
|
|
117
|
-
const idx = pipeline.frameHistory.findIndex(entry => entry.timestamp > lastTs);
|
|
121
|
+
const idx = pipeline.frameHistory.findIndex((entry) => entry.timestamp > lastTs);
|
|
118
122
|
if (idx === -1) {
|
|
119
123
|
pipeline.historyCursor = pipeline.frameHistory.length - 1;
|
|
120
124
|
return;
|
|
@@ -125,7 +129,7 @@ function alignHistoryCursorToLastOutput(pipeline: PipelineState): void {
|
|
|
125
129
|
function getPrimaryVideoPipeline(): PipelineState | null {
|
|
126
130
|
let selected: PipelineState | null = null;
|
|
127
131
|
for (const pipeline of pipelines.values()) {
|
|
128
|
-
if (pipeline.track.type ===
|
|
132
|
+
if (pipeline.track.type === "video") {
|
|
129
133
|
if (!selected || pipeline.idx < selected.idx) {
|
|
130
134
|
selected = pipeline;
|
|
131
135
|
}
|
|
@@ -153,11 +157,11 @@ const MAX_PAUSED_INPUT_QUEUE = 600;
|
|
|
153
157
|
// Logging
|
|
154
158
|
// ============================================================================
|
|
155
159
|
|
|
156
|
-
function log(msg: string, level:
|
|
160
|
+
function log(msg: string, level: "info" | "warn" | "error" = "info"): void {
|
|
157
161
|
if (!debugging) return;
|
|
158
162
|
|
|
159
163
|
const message: WorkerToMainMessage = {
|
|
160
|
-
type:
|
|
164
|
+
type: "log",
|
|
161
165
|
msg,
|
|
162
166
|
level,
|
|
163
167
|
uid: uidCounter++,
|
|
@@ -166,7 +170,7 @@ function log(msg: string, level: 'info' | 'warn' | 'error' = 'info'): void {
|
|
|
166
170
|
}
|
|
167
171
|
|
|
168
172
|
function logVerbose(msg: string): void {
|
|
169
|
-
if (debugging !==
|
|
173
|
+
if (debugging !== "verbose") return;
|
|
170
174
|
log(msg);
|
|
171
175
|
}
|
|
172
176
|
|
|
@@ -178,49 +182,49 @@ self.onmessage = (event: MessageEvent<MainToWorkerMessage>) => {
|
|
|
178
182
|
const msg = event.data;
|
|
179
183
|
|
|
180
184
|
switch (msg.type) {
|
|
181
|
-
case
|
|
185
|
+
case "create":
|
|
182
186
|
handleCreate(msg);
|
|
183
187
|
break;
|
|
184
188
|
|
|
185
|
-
case
|
|
189
|
+
case "configure":
|
|
186
190
|
handleConfigure(msg);
|
|
187
191
|
break;
|
|
188
192
|
|
|
189
|
-
case
|
|
193
|
+
case "receive":
|
|
190
194
|
handleReceive(msg);
|
|
191
195
|
break;
|
|
192
196
|
|
|
193
|
-
case
|
|
197
|
+
case "setwritable":
|
|
194
198
|
handleSetWritable(msg);
|
|
195
199
|
break;
|
|
196
200
|
|
|
197
|
-
case
|
|
201
|
+
case "creategenerator":
|
|
198
202
|
handleCreateGenerator(msg);
|
|
199
203
|
break;
|
|
200
204
|
|
|
201
|
-
case
|
|
205
|
+
case "close":
|
|
202
206
|
handleClose(msg);
|
|
203
207
|
break;
|
|
204
208
|
|
|
205
|
-
case
|
|
209
|
+
case "frametiming":
|
|
206
210
|
handleFrameTiming(msg);
|
|
207
211
|
break;
|
|
208
212
|
|
|
209
|
-
case
|
|
213
|
+
case "seek":
|
|
210
214
|
handleSeek(msg);
|
|
211
215
|
break;
|
|
212
216
|
|
|
213
|
-
case
|
|
217
|
+
case "framestep":
|
|
214
218
|
handleFrameStep(msg);
|
|
215
219
|
break;
|
|
216
220
|
|
|
217
|
-
case
|
|
221
|
+
case "debugging":
|
|
218
222
|
debugging = msg.value;
|
|
219
223
|
log(`Debugging set to: ${msg.value}`);
|
|
220
224
|
break;
|
|
221
225
|
|
|
222
226
|
default:
|
|
223
|
-
log(`Unknown message type: ${(msg as any).type}`,
|
|
227
|
+
log(`Unknown message type: ${(msg as any).type}`, "warn");
|
|
224
228
|
}
|
|
225
229
|
};
|
|
226
230
|
|
|
@@ -228,7 +232,7 @@ self.onmessage = (event: MessageEvent<MainToWorkerMessage>) => {
|
|
|
228
232
|
// Pipeline Management
|
|
229
233
|
// ============================================================================
|
|
230
234
|
|
|
231
|
-
function handleCreate(msg: MainToWorkerMessage & { type:
|
|
235
|
+
function handleCreate(msg: MainToWorkerMessage & { type: "create" }): void {
|
|
232
236
|
const { idx, track, opts, uid } = msg;
|
|
233
237
|
|
|
234
238
|
log(`Creating pipeline for track ${idx} (${track.type} ${track.codec})`);
|
|
@@ -243,8 +247,8 @@ function handleCreate(msg: MainToWorkerMessage & { type: 'create' }): void {
|
|
|
243
247
|
writer: null,
|
|
244
248
|
inputQueue: [],
|
|
245
249
|
outputQueue: [],
|
|
246
|
-
frameHistory: track.type ===
|
|
247
|
-
historyCursor: track.type ===
|
|
250
|
+
frameHistory: track.type === "video" ? [] : undefined,
|
|
251
|
+
historyCursor: track.type === "video" ? null : undefined,
|
|
248
252
|
stats: {
|
|
249
253
|
framesIn: 0,
|
|
250
254
|
framesDecoded: 0,
|
|
@@ -254,12 +258,12 @@ function handleCreate(msg: MainToWorkerMessage & { type: 'create' }): void {
|
|
|
254
258
|
lastOutputTimestamp: 0,
|
|
255
259
|
decoderQueueSize: 0,
|
|
256
260
|
// Debug info for error diagnosis
|
|
257
|
-
lastChunkType:
|
|
261
|
+
lastChunkType: "" as string,
|
|
258
262
|
lastChunkSize: 0,
|
|
259
|
-
lastChunkBytes:
|
|
263
|
+
lastChunkBytes: "" as string,
|
|
260
264
|
},
|
|
261
265
|
optimizeForLatency: opts.optimizeForLatency,
|
|
262
|
-
payloadFormat: opts.payloadFormat ||
|
|
266
|
+
payloadFormat: opts.payloadFormat || "avcc",
|
|
263
267
|
};
|
|
264
268
|
|
|
265
269
|
pipelines.set(idx, pipeline);
|
|
@@ -272,32 +276,32 @@ function handleCreate(msg: MainToWorkerMessage & { type: 'create' }): void {
|
|
|
272
276
|
sendAck(uid, idx);
|
|
273
277
|
}
|
|
274
278
|
|
|
275
|
-
function handleConfigure(msg: MainToWorkerMessage & { type:
|
|
279
|
+
function handleConfigure(msg: MainToWorkerMessage & { type: "configure" }): void {
|
|
276
280
|
const { idx, header, uid } = msg;
|
|
277
281
|
|
|
278
|
-
log(`Received configure for track ${idx}, header length=${header?.byteLength ??
|
|
282
|
+
log(`Received configure for track ${idx}, header length=${header?.byteLength ?? "null"}`);
|
|
279
283
|
|
|
280
284
|
const pipeline = pipelines.get(idx);
|
|
281
285
|
|
|
282
286
|
if (!pipeline) {
|
|
283
|
-
log(`Cannot configure: pipeline ${idx} not found`,
|
|
284
|
-
sendError(uid, idx,
|
|
287
|
+
log(`Cannot configure: pipeline ${idx} not found`, "error");
|
|
288
|
+
sendError(uid, idx, "Pipeline not found");
|
|
285
289
|
return;
|
|
286
290
|
}
|
|
287
291
|
|
|
288
292
|
// Skip if already configured and decoder is ready
|
|
289
293
|
// This prevents duplicate configuration when both WS INIT and HTTP fallback fire
|
|
290
|
-
if (pipeline.configured && pipeline.decoder && pipeline.decoder.state ===
|
|
294
|
+
if (pipeline.configured && pipeline.decoder && pipeline.decoder.state === "configured") {
|
|
291
295
|
log(`Track ${idx} already configured, skipping duplicate configure`);
|
|
292
296
|
sendAck(uid, idx);
|
|
293
297
|
return;
|
|
294
298
|
}
|
|
295
299
|
|
|
296
300
|
try {
|
|
297
|
-
if (pipeline.track.type ===
|
|
301
|
+
if (pipeline.track.type === "video") {
|
|
298
302
|
log(`Configuring video decoder for track ${idx}...`);
|
|
299
303
|
configureVideoDecoder(pipeline, header);
|
|
300
|
-
} else if (pipeline.track.type ===
|
|
304
|
+
} else if (pipeline.track.type === "audio") {
|
|
301
305
|
log(`Configuring audio decoder for track ${idx}...`);
|
|
302
306
|
configureAudioDecoder(pipeline, header);
|
|
303
307
|
}
|
|
@@ -306,7 +310,7 @@ function handleConfigure(msg: MainToWorkerMessage & { type: 'configure' }): void
|
|
|
306
310
|
log(`Successfully configured decoder for track ${idx}`);
|
|
307
311
|
sendAck(uid, idx);
|
|
308
312
|
} catch (err) {
|
|
309
|
-
log(`Failed to configure decoder for track ${idx}: ${err}`,
|
|
313
|
+
log(`Failed to configure decoder for track ${idx}: ${err}`, "error");
|
|
310
314
|
sendError(uid, idx, String(err));
|
|
311
315
|
}
|
|
312
316
|
}
|
|
@@ -315,8 +319,8 @@ function configureVideoDecoder(pipeline: PipelineState, description?: Uint8Array
|
|
|
315
319
|
const track = pipeline.track;
|
|
316
320
|
|
|
317
321
|
// Handle JPEG codec separately via ImageDecoder (Phase 2C)
|
|
318
|
-
if (track.codec ===
|
|
319
|
-
log(
|
|
322
|
+
if (track.codec === "JPEG" || track.codec.toLowerCase() === "jpeg") {
|
|
323
|
+
log("JPEG codec detected - will use ImageDecoder");
|
|
320
324
|
pipeline.configured = true;
|
|
321
325
|
// JPEG doesn't need a persistent decoder - each frame is decoded individually
|
|
322
326
|
return;
|
|
@@ -324,14 +328,14 @@ function configureVideoDecoder(pipeline: PipelineState, description?: Uint8Array
|
|
|
324
328
|
|
|
325
329
|
// Close existing decoder if any (per rawws.js reconfiguration pattern)
|
|
326
330
|
if (pipeline.decoder) {
|
|
327
|
-
if (pipeline.decoder.state ===
|
|
331
|
+
if (pipeline.decoder.state === "configured") {
|
|
328
332
|
try {
|
|
329
333
|
pipeline.decoder.reset();
|
|
330
334
|
} catch {
|
|
331
335
|
// Ignore reset errors
|
|
332
336
|
}
|
|
333
337
|
}
|
|
334
|
-
if (pipeline.decoder.state !==
|
|
338
|
+
if (pipeline.decoder.state !== "closed") {
|
|
335
339
|
try {
|
|
336
340
|
pipeline.decoder.close();
|
|
337
341
|
} catch {
|
|
@@ -346,18 +350,18 @@ function configureVideoDecoder(pipeline: PipelineState, description?: Uint8Array
|
|
|
346
350
|
const config: VideoDecoderInit = {
|
|
347
351
|
codec: track.codecstring || track.codec.toLowerCase(),
|
|
348
352
|
optimizeForLatency: pipeline.optimizeForLatency,
|
|
349
|
-
hardwareAcceleration:
|
|
353
|
+
hardwareAcceleration: "prefer-hardware",
|
|
350
354
|
};
|
|
351
355
|
|
|
352
356
|
// Pass description directly from WebSocket INIT data (per reference rawws.js line 1052)
|
|
353
357
|
// For Annex B format (ws/video/h264), SPS/PPS comes inline in the bitstream - skip description
|
|
354
|
-
if (pipeline.payloadFormat ===
|
|
358
|
+
if (pipeline.payloadFormat === "annexb") {
|
|
355
359
|
log(`Annex B mode - SPS/PPS inline in bitstream, no description needed`);
|
|
356
360
|
} else if (description && description.byteLength > 0) {
|
|
357
361
|
config.description = description;
|
|
358
362
|
log(`Configuring with description (${description.byteLength} bytes)`);
|
|
359
363
|
} else {
|
|
360
|
-
log(`No description provided - decoder may fail on H.264/HEVC`,
|
|
364
|
+
log(`No description provided - decoder may fail on H.264/HEVC`, "warn");
|
|
361
365
|
}
|
|
362
366
|
|
|
363
367
|
log(`Configuring video decoder: ${config.codec}`);
|
|
@@ -379,29 +383,29 @@ function configureVideoDecoder(pipeline: PipelineState, description?: Uint8Array
|
|
|
379
383
|
*/
|
|
380
384
|
function mapAudioCodec(codec: string, codecstring?: string): string {
|
|
381
385
|
// If we have a full codec string like "mp4a.40.2", use it
|
|
382
|
-
if (codecstring && codecstring.startsWith(
|
|
386
|
+
if (codecstring && codecstring.startsWith("mp4a.")) {
|
|
383
387
|
return codecstring;
|
|
384
388
|
}
|
|
385
389
|
|
|
386
390
|
// Map common MistServer codec names to WebCodecs codec strings
|
|
387
391
|
const normalized = codec.toLowerCase();
|
|
388
392
|
switch (normalized) {
|
|
389
|
-
case
|
|
390
|
-
case
|
|
391
|
-
return
|
|
392
|
-
case
|
|
393
|
-
return
|
|
394
|
-
case
|
|
395
|
-
return
|
|
396
|
-
case
|
|
397
|
-
return
|
|
398
|
-
case
|
|
399
|
-
case
|
|
400
|
-
return
|
|
401
|
-
case
|
|
402
|
-
case
|
|
403
|
-
case
|
|
404
|
-
return
|
|
393
|
+
case "aac":
|
|
394
|
+
case "mp4a":
|
|
395
|
+
return "mp4a.40.2"; // AAC-LC
|
|
396
|
+
case "mp3":
|
|
397
|
+
return "mp3";
|
|
398
|
+
case "opus":
|
|
399
|
+
return "opus";
|
|
400
|
+
case "flac":
|
|
401
|
+
return "flac";
|
|
402
|
+
case "ac3":
|
|
403
|
+
case "ac-3":
|
|
404
|
+
return "ac-3";
|
|
405
|
+
case "pcm_s16le":
|
|
406
|
+
case "pcm_s32le":
|
|
407
|
+
case "pcm_f32le":
|
|
408
|
+
return "pcm-" + normalized.replace("pcm_", "").replace("le", "-le");
|
|
405
409
|
default:
|
|
406
410
|
log(`Unknown audio codec: ${codec}, trying as-is`);
|
|
407
411
|
return codecstring || codec;
|
|
@@ -432,7 +436,9 @@ function configureAudioDecoder(pipeline: PipelineState, description?: Uint8Array
|
|
|
432
436
|
decoder.configure(config as AudioDecoderConfig);
|
|
433
437
|
pipeline.decoder = decoder;
|
|
434
438
|
|
|
435
|
-
log(
|
|
439
|
+
log(
|
|
440
|
+
`Audio decoder configured: ${config.codec} ${config.sampleRate}Hz ${config.numberOfChannels}ch`
|
|
441
|
+
);
|
|
436
442
|
}
|
|
437
443
|
|
|
438
444
|
function handleDecodedFrame(pipeline: PipelineState, frame: VideoFrame | AudioData): void {
|
|
@@ -450,10 +456,13 @@ function handleDecodedFrame(pipeline: PipelineState, frame: VideoFrame | AudioDa
|
|
|
450
456
|
// Log first few decoded frames
|
|
451
457
|
if (pipeline.stats.framesDecoded <= 3) {
|
|
452
458
|
const frameType = pipeline.track.type;
|
|
453
|
-
const extraInfo =
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
459
|
+
const extraInfo =
|
|
460
|
+
frameType === "audio"
|
|
461
|
+
? ` (${(frame as AudioData).numberOfFrames} samples, ${(frame as AudioData).sampleRate}Hz)`
|
|
462
|
+
: ` (${(frame as VideoFrame).displayWidth}x${(frame as VideoFrame).displayHeight})`;
|
|
463
|
+
log(
|
|
464
|
+
`Decoded ${frameType} frame ${pipeline.stats.framesDecoded} for track ${pipeline.idx}: ts=${timestamp}μs${extraInfo}`
|
|
465
|
+
);
|
|
457
466
|
}
|
|
458
467
|
|
|
459
468
|
// Add to output queue for scheduled release
|
|
@@ -468,16 +477,19 @@ function handleDecodedFrame(pipeline: PipelineState, frame: VideoFrame | AudioDa
|
|
|
468
477
|
}
|
|
469
478
|
|
|
470
479
|
function handleDecoderError(pipeline: PipelineState, err: DOMException): void {
|
|
471
|
-
log(`Decoder error on track ${pipeline.idx}: ${err.name}: ${err.message}`,
|
|
472
|
-
log(
|
|
480
|
+
log(`Decoder error on track ${pipeline.idx}: ${err.name}: ${err.message}`, "error");
|
|
481
|
+
log(
|
|
482
|
+
` Last chunk info: type=${pipeline.stats.lastChunkType}, size=${pipeline.stats.lastChunkSize}, first bytes=[${pipeline.stats.lastChunkBytes}]`,
|
|
483
|
+
"error"
|
|
484
|
+
);
|
|
473
485
|
|
|
474
486
|
// Per rawws.js: reset the pipeline after decoder error
|
|
475
487
|
// This clears queues and recreates the decoder if needed
|
|
476
488
|
resetPipelineAfterError(pipeline);
|
|
477
489
|
|
|
478
490
|
const message: WorkerToMainMessage = {
|
|
479
|
-
type:
|
|
480
|
-
kind:
|
|
491
|
+
type: "sendevent",
|
|
492
|
+
kind: "error",
|
|
481
493
|
message: `Decoder error: ${err.message}`,
|
|
482
494
|
idx: pipeline.idx,
|
|
483
495
|
uid: uidCounter++,
|
|
@@ -501,16 +513,16 @@ function resetPipelineAfterError(pipeline: PipelineState): void {
|
|
|
501
513
|
pipeline.configured = false;
|
|
502
514
|
|
|
503
515
|
// If decoder is closed, we need to recreate it (can't reset a closed decoder)
|
|
504
|
-
if (pipeline.decoder && pipeline.decoder.state ===
|
|
516
|
+
if (pipeline.decoder && pipeline.decoder.state === "closed") {
|
|
505
517
|
log(`Decoder closed for track ${pipeline.idx}, will recreate on next configure`);
|
|
506
518
|
pipeline.decoder = null;
|
|
507
|
-
} else if (pipeline.decoder && pipeline.decoder.state !==
|
|
519
|
+
} else if (pipeline.decoder && pipeline.decoder.state !== "closed") {
|
|
508
520
|
// Try to reset if not closed
|
|
509
521
|
try {
|
|
510
522
|
pipeline.decoder.reset();
|
|
511
523
|
log(`Reset decoder for track ${pipeline.idx}`);
|
|
512
524
|
} catch (e) {
|
|
513
|
-
log(`Failed to reset decoder for track ${pipeline.idx}: ${e}`,
|
|
525
|
+
log(`Failed to reset decoder for track ${pipeline.idx}: ${e}`, "warn");
|
|
514
526
|
pipeline.decoder = null;
|
|
515
527
|
}
|
|
516
528
|
}
|
|
@@ -520,7 +532,7 @@ function resetPipelineAfterError(pipeline: PipelineState): void {
|
|
|
520
532
|
// Frame Input/Output
|
|
521
533
|
// ============================================================================
|
|
522
534
|
|
|
523
|
-
function handleReceive(msg: MainToWorkerMessage & { type:
|
|
535
|
+
function handleReceive(msg: MainToWorkerMessage & { type: "receive" }): void {
|
|
524
536
|
const { idx, chunk } = msg;
|
|
525
537
|
const pipeline = pipelines.get(idx);
|
|
526
538
|
|
|
@@ -532,7 +544,9 @@ function handleReceive(msg: MainToWorkerMessage & { type: 'receive' }): void {
|
|
|
532
544
|
if (!pipeline.configured || !pipeline.decoder) {
|
|
533
545
|
// Queue for later
|
|
534
546
|
pipeline.inputQueue.push(chunk);
|
|
535
|
-
logVerbose(
|
|
547
|
+
logVerbose(
|
|
548
|
+
`Queued chunk for track ${idx} (configured=${pipeline.configured}, decoder=${!!pipeline.decoder})`
|
|
549
|
+
);
|
|
536
550
|
return;
|
|
537
551
|
}
|
|
538
552
|
|
|
@@ -548,19 +562,23 @@ function handleReceive(msg: MainToWorkerMessage & { type: 'receive' }): void {
|
|
|
548
562
|
|
|
549
563
|
// Log only first 3 chunks per track to confirm receiving
|
|
550
564
|
if (pipeline.stats.framesIn < 3) {
|
|
551
|
-
log(
|
|
565
|
+
log(
|
|
566
|
+
`Received chunk ${pipeline.stats.framesIn} for track ${idx}: type=${chunk.type}, ts=${chunk.timestamp / 1000}ms, size=${chunk.data.byteLength}`
|
|
567
|
+
);
|
|
552
568
|
}
|
|
553
569
|
|
|
554
570
|
// Check if we need to drop frames due to decoder pressure (Phase 2B)
|
|
555
571
|
if (shouldDropFramesDueToDecoderPressure(pipeline)) {
|
|
556
|
-
if (chunk.type ===
|
|
572
|
+
if (chunk.type === "key") {
|
|
557
573
|
// Always accept keyframes - they're needed to resume
|
|
558
574
|
decodeChunk(pipeline, chunk);
|
|
559
575
|
} else {
|
|
560
576
|
// Drop delta frames when decoder is overwhelmed
|
|
561
577
|
pipeline.stats.framesDropped++;
|
|
562
578
|
_totalFramesDropped++;
|
|
563
|
-
logVerbose(
|
|
579
|
+
logVerbose(
|
|
580
|
+
`Dropped delta frame @ ${chunk.timestamp / 1000}ms (decoder queue: ${pipeline.decoder.decodeQueueSize})`
|
|
581
|
+
);
|
|
564
582
|
}
|
|
565
583
|
return;
|
|
566
584
|
}
|
|
@@ -591,7 +609,7 @@ function _dropToNextKeyframe(pipeline: PipelineState): number {
|
|
|
591
609
|
if (pipeline.inputQueue.length === 0) return 0;
|
|
592
610
|
|
|
593
611
|
// Find next keyframe in queue
|
|
594
|
-
const keyframeIdx = pipeline.inputQueue.findIndex(c => c.type ===
|
|
612
|
+
const keyframeIdx = pipeline.inputQueue.findIndex((c) => c.type === "key");
|
|
595
613
|
|
|
596
614
|
if (keyframeIdx <= 0) {
|
|
597
615
|
// No keyframe or keyframe is first - nothing to drop
|
|
@@ -603,14 +621,14 @@ function _dropToNextKeyframe(pipeline: PipelineState): number {
|
|
|
603
621
|
pipeline.stats.framesDropped += dropped.length;
|
|
604
622
|
_totalFramesDropped += dropped.length;
|
|
605
623
|
|
|
606
|
-
log(`Dropped ${dropped.length} frames to next keyframe`,
|
|
624
|
+
log(`Dropped ${dropped.length} frames to next keyframe`, "warn");
|
|
607
625
|
|
|
608
626
|
return dropped.length;
|
|
609
627
|
}
|
|
610
628
|
|
|
611
629
|
function decodeChunk(
|
|
612
630
|
pipeline: PipelineState,
|
|
613
|
-
chunk: { type:
|
|
631
|
+
chunk: { type: "key" | "delta"; timestamp: number; data: Uint8Array }
|
|
614
632
|
): void {
|
|
615
633
|
if (pipeline.closed) return;
|
|
616
634
|
|
|
@@ -622,7 +640,7 @@ function decodeChunk(
|
|
|
622
640
|
try {
|
|
623
641
|
// Handle JPEG via ImageDecoder (Phase 2C)
|
|
624
642
|
const codec = pipeline.track.codec;
|
|
625
|
-
if (codec ===
|
|
643
|
+
if (codec === "JPEG" || codec.toLowerCase() === "jpeg") {
|
|
626
644
|
decodeJpegFrame(pipeline, chunk);
|
|
627
645
|
return;
|
|
628
646
|
}
|
|
@@ -636,10 +654,12 @@ function decodeChunk(
|
|
|
636
654
|
pipeline.stats.lastChunkType = chunk.type;
|
|
637
655
|
pipeline.stats.lastChunkSize = chunk.data.byteLength;
|
|
638
656
|
// Show first 8 bytes to identify format (Annex B starts 0x00 0x00 0x00 0x01, AVCC starts with length)
|
|
639
|
-
const firstBytes = Array.from(chunk.data.slice(0, 8))
|
|
657
|
+
const firstBytes = Array.from(chunk.data.slice(0, 8))
|
|
658
|
+
.map((b) => "0x" + b.toString(16).padStart(2, "0"))
|
|
659
|
+
.join(" ");
|
|
640
660
|
pipeline.stats.lastChunkBytes = firstBytes;
|
|
641
661
|
|
|
642
|
-
if (pipeline.track.type ===
|
|
662
|
+
if (pipeline.track.type === "video") {
|
|
643
663
|
// AVCC mode: frames pass through unchanged (decoder has SPS/PPS from description)
|
|
644
664
|
const encodedChunk = new EncodedVideoChunk({
|
|
645
665
|
type: chunk.type,
|
|
@@ -649,8 +669,12 @@ function decodeChunk(
|
|
|
649
669
|
|
|
650
670
|
const decoder = pipeline.decoder as VideoDecoder;
|
|
651
671
|
if (pipeline.stats.framesIn <= 3) {
|
|
652
|
-
const firstBytes = Array.from(chunk.data.slice(0, 16))
|
|
653
|
-
|
|
672
|
+
const firstBytes = Array.from(chunk.data.slice(0, 16))
|
|
673
|
+
.map((b) => "0x" + b.toString(16).padStart(2, "0"))
|
|
674
|
+
.join(" ");
|
|
675
|
+
log(
|
|
676
|
+
`Calling decode() for track ${pipeline.idx}: state=${decoder.state}, queueSize=${decoder.decodeQueueSize}, chunk type=${chunk.type}, ts=${timestampUs}μs`
|
|
677
|
+
);
|
|
654
678
|
log(` First 16 bytes: ${firstBytes}`);
|
|
655
679
|
}
|
|
656
680
|
|
|
@@ -659,11 +683,11 @@ function decodeChunk(
|
|
|
659
683
|
if (pipeline.stats.framesIn <= 3) {
|
|
660
684
|
log(`After decode() for track ${pipeline.idx}: queueSize=${decoder.decodeQueueSize}`);
|
|
661
685
|
}
|
|
662
|
-
} else if (pipeline.track.type ===
|
|
686
|
+
} else if (pipeline.track.type === "audio") {
|
|
663
687
|
// Audio chunks are always treated as "key" frames - per MistServer rawws.js line 1127
|
|
664
688
|
// Audio codecs don't use inter-frame dependencies like video does
|
|
665
689
|
const encodedChunk = new EncodedAudioChunk({
|
|
666
|
-
type:
|
|
690
|
+
type: "key",
|
|
667
691
|
timestamp: timestampUs,
|
|
668
692
|
data: chunk.data,
|
|
669
693
|
});
|
|
@@ -675,9 +699,11 @@ function decodeChunk(
|
|
|
675
699
|
pipeline.stats.decoderQueueSize = pipeline.decoder.decodeQueueSize;
|
|
676
700
|
}
|
|
677
701
|
|
|
678
|
-
logVerbose(
|
|
702
|
+
logVerbose(
|
|
703
|
+
`Decoded chunk ${chunk.type} @ ${chunk.timestamp / 1000}ms for track ${pipeline.idx}`
|
|
704
|
+
);
|
|
679
705
|
} catch (err) {
|
|
680
|
-
log(`Decode error on track ${pipeline.idx}: ${err}`,
|
|
706
|
+
log(`Decode error on track ${pipeline.idx}: ${err}`, "error");
|
|
681
707
|
}
|
|
682
708
|
}
|
|
683
709
|
|
|
@@ -687,20 +713,20 @@ function decodeChunk(
|
|
|
687
713
|
*/
|
|
688
714
|
async function decodeJpegFrame(
|
|
689
715
|
pipeline: PipelineState,
|
|
690
|
-
chunk: { type:
|
|
716
|
+
chunk: { type: "key" | "delta"; timestamp: number; data: Uint8Array }
|
|
691
717
|
): Promise<void> {
|
|
692
718
|
if (pipeline.closed) return;
|
|
693
719
|
|
|
694
720
|
// Check if ImageDecoder is available
|
|
695
|
-
if (typeof ImageDecoder ===
|
|
696
|
-
log(
|
|
721
|
+
if (typeof ImageDecoder === "undefined") {
|
|
722
|
+
log("ImageDecoder not available - JPEG streams not supported", "error");
|
|
697
723
|
return;
|
|
698
724
|
}
|
|
699
725
|
|
|
700
726
|
try {
|
|
701
727
|
// Create ImageDecoder for this frame
|
|
702
728
|
const decoder = new ImageDecoder({
|
|
703
|
-
type:
|
|
729
|
+
type: "image/jpeg",
|
|
704
730
|
data: chunk.data,
|
|
705
731
|
});
|
|
706
732
|
|
|
@@ -721,7 +747,7 @@ async function decodeJpegFrame(
|
|
|
721
747
|
|
|
722
748
|
logVerbose(`Decoded JPEG frame @ ${chunk.timestamp / 1000}ms for track ${pipeline.idx}`);
|
|
723
749
|
} catch (err) {
|
|
724
|
-
log(`JPEG decode error on track ${pipeline.idx}: ${err}`,
|
|
750
|
+
log(`JPEG decode error on track ${pipeline.idx}: ${err}`, "error");
|
|
725
751
|
}
|
|
726
752
|
}
|
|
727
753
|
|
|
@@ -740,7 +766,10 @@ function processOutputQueue(pipeline: PipelineState): void {
|
|
|
740
766
|
|
|
741
767
|
if (!pipeline.writer || pipeline.outputQueue.length === 0) {
|
|
742
768
|
if (pipeline.outputQueue.length > 0 && !pipeline.writer) {
|
|
743
|
-
log(
|
|
769
|
+
log(
|
|
770
|
+
`Cannot output: no writer for track ${pipeline.idx} (queue has ${pipeline.outputQueue.length} frames)`,
|
|
771
|
+
"warn"
|
|
772
|
+
);
|
|
744
773
|
}
|
|
745
774
|
return;
|
|
746
775
|
}
|
|
@@ -750,8 +779,8 @@ function processOutputQueue(pipeline: PipelineState): void {
|
|
|
750
779
|
// Sort output queue by timestamp - MistServer can send frames out of order
|
|
751
780
|
// This is more robust than just swapping adjacent frames
|
|
752
781
|
if (pipeline.outputQueue.length > 1) {
|
|
753
|
-
const wasSorted = pipeline.outputQueue.every(
|
|
754
|
-
i === 0 || arr[i - 1].timestamp <= entry.timestamp
|
|
782
|
+
const wasSorted = pipeline.outputQueue.every(
|
|
783
|
+
(entry, i, arr) => i === 0 || arr[i - 1].timestamp <= entry.timestamp
|
|
755
784
|
);
|
|
756
785
|
if (!wasSorted) {
|
|
757
786
|
pipeline.outputQueue.sort((a, b) => a.timestamp - b.timestamp);
|
|
@@ -779,7 +808,9 @@ function processOutputQueue(pipeline: PipelineState): void {
|
|
|
779
808
|
// Complete warmup when we have enough buffer OR timeout
|
|
780
809
|
if (bufferMs >= WARMUP_BUFFER_MS || elapsed >= WARMUP_TIMEOUT_MS) {
|
|
781
810
|
warmupComplete = true;
|
|
782
|
-
log(
|
|
811
|
+
log(
|
|
812
|
+
`Buffer warmup complete: ${bufferMs.toFixed(0)}ms buffer, ${pipeline.outputQueue.length} frames queued (track ${pipeline.idx})`
|
|
813
|
+
);
|
|
783
814
|
} else {
|
|
784
815
|
// Not ready yet - schedule another check
|
|
785
816
|
setTimeout(() => processOutputQueue(pipeline), 10);
|
|
@@ -789,7 +820,9 @@ function processOutputQueue(pipeline: PipelineState): void {
|
|
|
789
820
|
// Not enough frames yet - schedule another check
|
|
790
821
|
if (elapsed >= WARMUP_TIMEOUT_MS) {
|
|
791
822
|
warmupComplete = true;
|
|
792
|
-
log(
|
|
823
|
+
log(
|
|
824
|
+
`Buffer warmup timeout - starting with ${pipeline.outputQueue.length} frame(s) (track ${pipeline.idx})`
|
|
825
|
+
);
|
|
793
826
|
} else {
|
|
794
827
|
setTimeout(() => processOutputQueue(pipeline), 10);
|
|
795
828
|
return;
|
|
@@ -845,7 +878,9 @@ function shouldOutputFrame(
|
|
|
845
878
|
// How early/late is this frame? Positive = too early, negative = late
|
|
846
879
|
const delay = targetTime - now;
|
|
847
880
|
|
|
848
|
-
logVerbose(
|
|
881
|
+
logVerbose(
|
|
882
|
+
`Frame timing: track=${trackIdx} frame=${frameTimeMs.toFixed(0)}ms, target=${targetTime.toFixed(0)}, now=${now.toFixed(0)}, delay=${delay.toFixed(1)}ms`
|
|
883
|
+
);
|
|
849
884
|
|
|
850
885
|
// Output immediately if ready or late (per rawws.js line 889: delay <= 2)
|
|
851
886
|
if (delay <= 2) {
|
|
@@ -856,7 +891,11 @@ function shouldOutputFrame(
|
|
|
856
891
|
return { shouldOutput: false, earliness: -delay, checkDelayMs: Math.max(1, Math.floor(delay)) };
|
|
857
892
|
}
|
|
858
893
|
|
|
859
|
-
function outputFrame(
|
|
894
|
+
function outputFrame(
|
|
895
|
+
pipeline: PipelineState,
|
|
896
|
+
entry: DecodedFrame,
|
|
897
|
+
options?: { skipHistory?: boolean }
|
|
898
|
+
): void {
|
|
860
899
|
if (!pipeline.writer || pipeline.closed) {
|
|
861
900
|
entry.frame.close();
|
|
862
901
|
return;
|
|
@@ -869,55 +908,60 @@ function outputFrame(pipeline: PipelineState, entry: DecodedFrame, options?: { s
|
|
|
869
908
|
|
|
870
909
|
// Log first few output frames
|
|
871
910
|
if (pipeline.stats.framesOut <= 3) {
|
|
872
|
-
log(
|
|
911
|
+
log(
|
|
912
|
+
`Output frame ${pipeline.stats.framesOut} for track ${pipeline.idx}: ts=${entry.timestamp}μs`
|
|
913
|
+
);
|
|
873
914
|
}
|
|
874
915
|
|
|
875
916
|
// Store history for frame stepping (video only)
|
|
876
|
-
if (pipeline.track.type ===
|
|
917
|
+
if (pipeline.track.type === "video" && !options?.skipHistory) {
|
|
877
918
|
pushFrameHistory(pipeline, entry.frame as VideoFrame, entry.timestamp);
|
|
878
919
|
}
|
|
879
920
|
|
|
880
921
|
// Write returns a Promise - handle rejection to avoid unhandled promise errors
|
|
881
922
|
// Frame ownership is transferred to the stream, so we don't need to close() on success
|
|
882
|
-
pipeline.writer
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
//
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
923
|
+
pipeline.writer
|
|
924
|
+
.write(entry.frame)
|
|
925
|
+
.then(() => {
|
|
926
|
+
// Send timeupdate event on successful write
|
|
927
|
+
const message: WorkerToMainMessage = {
|
|
928
|
+
type: "sendevent",
|
|
929
|
+
kind: "timeupdate",
|
|
930
|
+
idx: pipeline.idx,
|
|
931
|
+
time: entry.timestamp / 1e6,
|
|
932
|
+
uid: uidCounter++,
|
|
933
|
+
};
|
|
934
|
+
self.postMessage(message);
|
|
935
|
+
})
|
|
936
|
+
.catch((err: Error) => {
|
|
937
|
+
// Check for "stream closed" errors - these are expected during cleanup
|
|
938
|
+
const errStr = String(err);
|
|
939
|
+
if (errStr.includes("Stream closed") || errStr.includes("InvalidStateError")) {
|
|
940
|
+
// Expected during player cleanup - silently mark pipeline as closed
|
|
941
|
+
pipeline.closed = true;
|
|
942
|
+
} else {
|
|
943
|
+
log(`Failed to write frame: ${err}`, "error");
|
|
944
|
+
}
|
|
945
|
+
// Frame may not have been consumed by the stream - try to close it
|
|
946
|
+
try {
|
|
947
|
+
entry.frame.close();
|
|
948
|
+
} catch {
|
|
949
|
+
// Frame may already be detached/closed
|
|
950
|
+
}
|
|
951
|
+
});
|
|
908
952
|
}
|
|
909
953
|
|
|
910
954
|
// ============================================================================
|
|
911
955
|
// Track Generator / Writable Stream
|
|
912
956
|
// ============================================================================
|
|
913
957
|
|
|
914
|
-
function handleSetWritable(msg: MainToWorkerMessage & { type:
|
|
958
|
+
function handleSetWritable(msg: MainToWorkerMessage & { type: "setwritable" }): void {
|
|
915
959
|
const { idx, writable, uid } = msg;
|
|
916
960
|
const pipeline = pipelines.get(idx);
|
|
917
961
|
|
|
918
962
|
if (!pipeline) {
|
|
919
|
-
log(`Cannot set writable: pipeline ${idx} not found`,
|
|
920
|
-
sendError(uid, idx,
|
|
963
|
+
log(`Cannot set writable: pipeline ${idx} not found`, "error");
|
|
964
|
+
sendError(uid, idx, "Pipeline not found");
|
|
921
965
|
return;
|
|
922
966
|
}
|
|
923
967
|
|
|
@@ -931,29 +975,29 @@ function handleSetWritable(msg: MainToWorkerMessage & { type: 'setwritable' }):
|
|
|
931
975
|
|
|
932
976
|
// Notify main thread track is ready
|
|
933
977
|
const message: WorkerToMainMessage = {
|
|
934
|
-
type:
|
|
978
|
+
type: "addtrack",
|
|
935
979
|
idx,
|
|
936
980
|
uid,
|
|
937
|
-
status:
|
|
981
|
+
status: "ok",
|
|
938
982
|
};
|
|
939
983
|
self.postMessage(message);
|
|
940
984
|
}
|
|
941
985
|
|
|
942
|
-
function handleCreateGenerator(msg: MainToWorkerMessage & { type:
|
|
986
|
+
function handleCreateGenerator(msg: MainToWorkerMessage & { type: "creategenerator" }): void {
|
|
943
987
|
const { idx, uid } = msg;
|
|
944
988
|
const pipeline = pipelines.get(idx);
|
|
945
989
|
|
|
946
990
|
if (!pipeline) {
|
|
947
|
-
log(`Cannot create generator: pipeline ${idx} not found`,
|
|
948
|
-
sendError(uid, idx,
|
|
991
|
+
log(`Cannot create generator: pipeline ${idx} not found`, "error");
|
|
992
|
+
sendError(uid, idx, "Pipeline not found");
|
|
949
993
|
return;
|
|
950
994
|
}
|
|
951
995
|
|
|
952
996
|
// Safari: VideoTrackGenerator is available in worker (not MediaStreamTrackGenerator)
|
|
953
997
|
// Reference: webcodecsworker.js line 852-863
|
|
954
998
|
// @ts-ignore - VideoTrackGenerator may not be in types
|
|
955
|
-
if (typeof VideoTrackGenerator !==
|
|
956
|
-
if (pipeline.track.type ===
|
|
999
|
+
if (typeof VideoTrackGenerator !== "undefined") {
|
|
1000
|
+
if (pipeline.track.type === "video") {
|
|
957
1001
|
// Safari video: use VideoTrackGenerator
|
|
958
1002
|
// @ts-ignore
|
|
959
1003
|
const generator = new VideoTrackGenerator();
|
|
@@ -962,16 +1006,16 @@ function handleCreateGenerator(msg: MainToWorkerMessage & { type: 'creategenerat
|
|
|
962
1006
|
|
|
963
1007
|
// Send track back to main thread
|
|
964
1008
|
const message: WorkerToMainMessage = {
|
|
965
|
-
type:
|
|
1009
|
+
type: "addtrack",
|
|
966
1010
|
idx,
|
|
967
1011
|
track: generator.track,
|
|
968
1012
|
uid,
|
|
969
|
-
status:
|
|
1013
|
+
status: "ok",
|
|
970
1014
|
};
|
|
971
1015
|
// @ts-ignore - transferring MediaStreamTrack
|
|
972
1016
|
self.postMessage(message, [generator.track]);
|
|
973
1017
|
log(`Created VideoTrackGenerator for track ${idx} (Safari video)`);
|
|
974
|
-
} else if (pipeline.track.type ===
|
|
1018
|
+
} else if (pipeline.track.type === "audio") {
|
|
975
1019
|
// Safari audio: relay frames to main thread via postMessage
|
|
976
1020
|
// Reference: webcodecsworker.js line 773-800
|
|
977
1021
|
// Main thread creates the audio generator, we just send frames
|
|
@@ -981,26 +1025,26 @@ function handleCreateGenerator(msg: MainToWorkerMessage & { type: 'creategenerat
|
|
|
981
1025
|
const frameUid = uidCounter++;
|
|
982
1026
|
// Set up listener for response
|
|
983
1027
|
const timeoutId = setTimeout(() => {
|
|
984
|
-
reject(new Error(
|
|
1028
|
+
reject(new Error("writeframe timeout"));
|
|
985
1029
|
}, 5000);
|
|
986
1030
|
|
|
987
1031
|
const handler = (e: MessageEvent) => {
|
|
988
1032
|
const msg = e.data;
|
|
989
|
-
if (msg.type ===
|
|
1033
|
+
if (msg.type === "writeframe" && msg.idx === idx && msg.uid === frameUid) {
|
|
990
1034
|
clearTimeout(timeoutId);
|
|
991
|
-
self.removeEventListener(
|
|
992
|
-
if (msg.status ===
|
|
1035
|
+
self.removeEventListener("message", handler);
|
|
1036
|
+
if (msg.status === "ok") {
|
|
993
1037
|
resolve();
|
|
994
1038
|
} else {
|
|
995
|
-
reject(new Error(msg.error ||
|
|
1039
|
+
reject(new Error(msg.error || "writeframe failed"));
|
|
996
1040
|
}
|
|
997
1041
|
}
|
|
998
1042
|
};
|
|
999
|
-
self.addEventListener(
|
|
1043
|
+
self.addEventListener("message", handler);
|
|
1000
1044
|
|
|
1001
1045
|
// Send frame to main thread (transfer AudioData)
|
|
1002
1046
|
const msg = {
|
|
1003
|
-
type:
|
|
1047
|
+
type: "writeframe",
|
|
1004
1048
|
idx,
|
|
1005
1049
|
frame,
|
|
1006
1050
|
uid: frameUid,
|
|
@@ -1013,16 +1057,16 @@ function handleCreateGenerator(msg: MainToWorkerMessage & { type: 'creategenerat
|
|
|
1013
1057
|
|
|
1014
1058
|
// Notify main thread to set up audio generator
|
|
1015
1059
|
const message: WorkerToMainMessage = {
|
|
1016
|
-
type:
|
|
1060
|
+
type: "addtrack",
|
|
1017
1061
|
idx,
|
|
1018
1062
|
uid,
|
|
1019
|
-
status:
|
|
1063
|
+
status: "ok",
|
|
1020
1064
|
};
|
|
1021
1065
|
self.postMessage(message);
|
|
1022
1066
|
log(`Set up frame relay for track ${idx} (Safari audio)`);
|
|
1023
1067
|
}
|
|
1024
1068
|
// @ts-ignore - MediaStreamTrackGenerator may not be in standard types
|
|
1025
|
-
} else if (typeof MediaStreamTrackGenerator !==
|
|
1069
|
+
} else if (typeof MediaStreamTrackGenerator !== "undefined") {
|
|
1026
1070
|
// Chrome/Edge: use MediaStreamTrackGenerator in worker
|
|
1027
1071
|
// @ts-ignore
|
|
1028
1072
|
const generator = new MediaStreamTrackGenerator({ kind: pipeline.track.type });
|
|
@@ -1031,18 +1075,18 @@ function handleCreateGenerator(msg: MainToWorkerMessage & { type: 'creategenerat
|
|
|
1031
1075
|
|
|
1032
1076
|
// Send track back to main thread
|
|
1033
1077
|
const message: WorkerToMainMessage = {
|
|
1034
|
-
type:
|
|
1078
|
+
type: "addtrack",
|
|
1035
1079
|
idx,
|
|
1036
1080
|
track: generator,
|
|
1037
1081
|
uid,
|
|
1038
|
-
status:
|
|
1082
|
+
status: "ok",
|
|
1039
1083
|
};
|
|
1040
1084
|
// @ts-ignore - transferring MediaStreamTrack
|
|
1041
1085
|
self.postMessage(message, [generator]);
|
|
1042
1086
|
log(`Created MediaStreamTrackGenerator for track ${idx}`);
|
|
1043
1087
|
} else {
|
|
1044
|
-
log(
|
|
1045
|
-
sendError(uid, idx,
|
|
1088
|
+
log("Neither VideoTrackGenerator nor MediaStreamTrackGenerator available in worker", "warn");
|
|
1089
|
+
sendError(uid, idx, "No track generator available");
|
|
1046
1090
|
}
|
|
1047
1091
|
}
|
|
1048
1092
|
|
|
@@ -1050,7 +1094,7 @@ function handleCreateGenerator(msg: MainToWorkerMessage & { type: 'creategenerat
|
|
|
1050
1094
|
// Seeking & Timing
|
|
1051
1095
|
// ============================================================================
|
|
1052
1096
|
|
|
1053
|
-
function handleSeek(msg: MainToWorkerMessage & { type:
|
|
1097
|
+
function handleSeek(msg: MainToWorkerMessage & { type: "seek" }): void {
|
|
1054
1098
|
const { seekTime, uid } = msg;
|
|
1055
1099
|
|
|
1056
1100
|
log(`Seek to ${seekTime}ms`);
|
|
@@ -1080,7 +1124,7 @@ function flushPipeline(pipeline: PipelineState): void {
|
|
|
1080
1124
|
pipeline.outputQueue = [];
|
|
1081
1125
|
|
|
1082
1126
|
// Reset decoder if possible
|
|
1083
|
-
if (pipeline.decoder && pipeline.decoder.state !==
|
|
1127
|
+
if (pipeline.decoder && pipeline.decoder.state !== "closed") {
|
|
1084
1128
|
try {
|
|
1085
1129
|
pipeline.decoder.reset();
|
|
1086
1130
|
} catch {
|
|
@@ -1089,26 +1133,28 @@ function flushPipeline(pipeline: PipelineState): void {
|
|
|
1089
1133
|
}
|
|
1090
1134
|
}
|
|
1091
1135
|
|
|
1092
|
-
function handleFrameTiming(msg: MainToWorkerMessage & { type:
|
|
1136
|
+
function handleFrameTiming(msg: MainToWorkerMessage & { type: "frametiming" }): void {
|
|
1093
1137
|
const { action, speed, tweak, uid } = msg;
|
|
1094
1138
|
|
|
1095
|
-
if (action ===
|
|
1139
|
+
if (action === "setSpeed") {
|
|
1096
1140
|
if (speed !== undefined) frameTiming.speed.main = speed;
|
|
1097
1141
|
if (tweak !== undefined) frameTiming.speed.tweak = tweak;
|
|
1098
1142
|
frameTiming.speed.combined = frameTiming.speed.main * frameTiming.speed.tweak;
|
|
1099
|
-
log(
|
|
1100
|
-
|
|
1143
|
+
log(
|
|
1144
|
+
`Speed set to ${frameTiming.speed.combined} (main: ${frameTiming.speed.main}, tweak: ${frameTiming.speed.tweak})`
|
|
1145
|
+
);
|
|
1146
|
+
} else if (action === "setPaused") {
|
|
1101
1147
|
frameTiming.paused = msg.paused === true;
|
|
1102
1148
|
log(`Frame timing paused=${frameTiming.paused}`);
|
|
1103
|
-
} else if (action ===
|
|
1149
|
+
} else if (action === "reset") {
|
|
1104
1150
|
frameTiming.seeking = false;
|
|
1105
|
-
log(
|
|
1151
|
+
log("Frame timing reset (seek complete)");
|
|
1106
1152
|
}
|
|
1107
1153
|
|
|
1108
1154
|
sendAck(uid);
|
|
1109
1155
|
}
|
|
1110
1156
|
|
|
1111
|
-
function handleFrameStep(msg: MainToWorkerMessage & { type:
|
|
1157
|
+
function handleFrameStep(msg: MainToWorkerMessage & { type: "framestep" }): void {
|
|
1112
1158
|
const { direction, uid } = msg;
|
|
1113
1159
|
|
|
1114
1160
|
log(`FrameStep request dir=${direction} paused=${frameTiming.paused}`);
|
|
@@ -1130,7 +1176,9 @@ function handleFrameStep(msg: MainToWorkerMessage & { type: 'framestep' }): void
|
|
|
1130
1176
|
if (pipeline.historyCursor === null || pipeline.historyCursor === undefined) {
|
|
1131
1177
|
alignHistoryCursorToLastOutput(pipeline);
|
|
1132
1178
|
}
|
|
1133
|
-
log(
|
|
1179
|
+
log(
|
|
1180
|
+
`FrameStep pipeline idx=${pipeline.idx} outQueue=${pipeline.outputQueue.length} history=${pipeline.frameHistory.length} cursor=${pipeline.historyCursor}`
|
|
1181
|
+
);
|
|
1134
1182
|
|
|
1135
1183
|
if (direction < 0) {
|
|
1136
1184
|
const nextIndex = (pipeline.historyCursor ?? 0) - 1;
|
|
@@ -1148,7 +1196,11 @@ function handleFrameStep(msg: MainToWorkerMessage & { type: 'framestep' }): void
|
|
|
1148
1196
|
return;
|
|
1149
1197
|
}
|
|
1150
1198
|
log(`FrameStep back: output ts=${entry.timestamp}`);
|
|
1151
|
-
outputFrame(
|
|
1199
|
+
outputFrame(
|
|
1200
|
+
pipeline,
|
|
1201
|
+
{ frame: clone, timestamp: entry.timestamp, decodedAt: performance.now() },
|
|
1202
|
+
{ skipHistory: true }
|
|
1203
|
+
);
|
|
1152
1204
|
sendAck(uid);
|
|
1153
1205
|
return;
|
|
1154
1206
|
}
|
|
@@ -1166,15 +1218,19 @@ function handleFrameStep(msg: MainToWorkerMessage & { type: 'framestep' }): void
|
|
|
1166
1218
|
return;
|
|
1167
1219
|
}
|
|
1168
1220
|
log(`FrameStep forward (history): output ts=${entry.timestamp}`);
|
|
1169
|
-
outputFrame(
|
|
1221
|
+
outputFrame(
|
|
1222
|
+
pipeline,
|
|
1223
|
+
{ frame: clone, timestamp: entry.timestamp, decodedAt: performance.now() },
|
|
1224
|
+
{ skipHistory: true }
|
|
1225
|
+
);
|
|
1170
1226
|
sendAck(uid);
|
|
1171
1227
|
return;
|
|
1172
1228
|
}
|
|
1173
1229
|
|
|
1174
1230
|
// Otherwise, output the next queued frame
|
|
1175
1231
|
if (pipeline.outputQueue.length > 1) {
|
|
1176
|
-
const wasSorted = pipeline.outputQueue.every(
|
|
1177
|
-
i === 0 || arr[i - 1].timestamp <= entry.timestamp
|
|
1232
|
+
const wasSorted = pipeline.outputQueue.every(
|
|
1233
|
+
(entry, i, arr) => i === 0 || arr[i - 1].timestamp <= entry.timestamp
|
|
1178
1234
|
);
|
|
1179
1235
|
if (!wasSorted) {
|
|
1180
1236
|
pipeline.outputQueue.sort((a, b) => a.timestamp - b.timestamp);
|
|
@@ -1182,7 +1238,7 @@ function handleFrameStep(msg: MainToWorkerMessage & { type: 'framestep' }): void
|
|
|
1182
1238
|
}
|
|
1183
1239
|
|
|
1184
1240
|
const lastTs = pipeline.stats.lastOutputTimestamp;
|
|
1185
|
-
let idx = pipeline.outputQueue.findIndex(e => e.timestamp > lastTs);
|
|
1241
|
+
let idx = pipeline.outputQueue.findIndex((e) => e.timestamp > lastTs);
|
|
1186
1242
|
if (idx === -1 && pipeline.outputQueue.length > 0) idx = 0;
|
|
1187
1243
|
if (idx === -1) {
|
|
1188
1244
|
log(`FrameStep forward: no queued frame available`);
|
|
@@ -1204,7 +1260,7 @@ function handleFrameStep(msg: MainToWorkerMessage & { type: 'framestep' }): void
|
|
|
1204
1260
|
// Cleanup
|
|
1205
1261
|
// ============================================================================
|
|
1206
1262
|
|
|
1207
|
-
function handleClose(msg: MainToWorkerMessage & { type:
|
|
1263
|
+
function handleClose(msg: MainToWorkerMessage & { type: "close" }): void {
|
|
1208
1264
|
const { idx, waitEmpty, uid } = msg;
|
|
1209
1265
|
const pipeline = pipelines.get(idx);
|
|
1210
1266
|
|
|
@@ -1232,7 +1288,7 @@ function closePipeline(pipeline: PipelineState, uid: number): void {
|
|
|
1232
1288
|
pipeline.closed = true;
|
|
1233
1289
|
|
|
1234
1290
|
// Close decoder
|
|
1235
|
-
if (pipeline.decoder && pipeline.decoder.state !==
|
|
1291
|
+
if (pipeline.decoder && pipeline.decoder.state !== "closed") {
|
|
1236
1292
|
try {
|
|
1237
1293
|
pipeline.decoder.close();
|
|
1238
1294
|
} catch {
|
|
@@ -1270,10 +1326,10 @@ function closePipeline(pipeline: PipelineState, uid: number): void {
|
|
|
1270
1326
|
}
|
|
1271
1327
|
|
|
1272
1328
|
const message: WorkerToMainMessage = {
|
|
1273
|
-
type:
|
|
1329
|
+
type: "closed",
|
|
1274
1330
|
idx: pipeline.idx,
|
|
1275
1331
|
uid,
|
|
1276
|
-
status:
|
|
1332
|
+
status: "ok",
|
|
1277
1333
|
};
|
|
1278
1334
|
self.postMessage(message);
|
|
1279
1335
|
}
|
|
@@ -1307,7 +1363,7 @@ function sendStats(): void {
|
|
|
1307
1363
|
}
|
|
1308
1364
|
|
|
1309
1365
|
const message: WorkerToMainMessage = {
|
|
1310
|
-
type:
|
|
1366
|
+
type: "stats",
|
|
1311
1367
|
stats: {
|
|
1312
1368
|
frameTiming: {
|
|
1313
1369
|
in: frameTiming.in,
|
|
@@ -1341,20 +1397,20 @@ function createFrameTrackerStats(): FrameTrackerStats {
|
|
|
1341
1397
|
|
|
1342
1398
|
function sendAck(uid: number, idx?: number): void {
|
|
1343
1399
|
const message: WorkerToMainMessage = {
|
|
1344
|
-
type:
|
|
1400
|
+
type: "ack",
|
|
1345
1401
|
uid,
|
|
1346
1402
|
idx,
|
|
1347
|
-
status:
|
|
1403
|
+
status: "ok",
|
|
1348
1404
|
};
|
|
1349
1405
|
self.postMessage(message);
|
|
1350
1406
|
}
|
|
1351
1407
|
|
|
1352
1408
|
function sendError(uid: number, idx: number | undefined, error: string): void {
|
|
1353
1409
|
const message: WorkerToMainMessage = {
|
|
1354
|
-
type:
|
|
1410
|
+
type: "ack",
|
|
1355
1411
|
uid,
|
|
1356
1412
|
idx,
|
|
1357
|
-
status:
|
|
1413
|
+
status: "error",
|
|
1358
1414
|
error,
|
|
1359
1415
|
};
|
|
1360
1416
|
self.postMessage(message);
|
|
@@ -1364,4 +1420,4 @@ function sendError(uid: number, idx: number | undefined, error: string): void {
|
|
|
1364
1420
|
// Worker Initialization
|
|
1365
1421
|
// ============================================================================
|
|
1366
1422
|
|
|
1367
|
-
log(
|
|
1423
|
+
log("WebCodecs decoder worker initialized");
|