@bnhf/prismcast 1.3.4-2026.2.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +7 -0
- package/README.md +347 -0
- package/bin/prismcast +6 -0
- package/dist/app.d.ts +6 -0
- package/dist/app.js +315 -0
- package/dist/app.js.map +1 -0
- package/dist/browser/cdp.d.ts +38 -0
- package/dist/browser/cdp.js +155 -0
- package/dist/browser/cdp.js.map +1 -0
- package/dist/browser/channelSelection.d.ts +65 -0
- package/dist/browser/channelSelection.js +202 -0
- package/dist/browser/channelSelection.js.map +1 -0
- package/dist/browser/display.d.ts +34 -0
- package/dist/browser/display.js +54 -0
- package/dist/browser/display.js.map +1 -0
- package/dist/browser/index.d.ts +205 -0
- package/dist/browser/index.js +1205 -0
- package/dist/browser/index.js.map +1 -0
- package/dist/browser/tuning/fox.d.ts +2 -0
- package/dist/browser/tuning/fox.js +83 -0
- package/dist/browser/tuning/fox.js.map +1 -0
- package/dist/browser/tuning/hbo.d.ts +2 -0
- package/dist/browser/tuning/hbo.js +237 -0
- package/dist/browser/tuning/hbo.js.map +1 -0
- package/dist/browser/tuning/hulu.d.ts +2 -0
- package/dist/browser/tuning/hulu.js +550 -0
- package/dist/browser/tuning/hulu.js.map +1 -0
- package/dist/browser/tuning/sling.d.ts +2 -0
- package/dist/browser/tuning/sling.js +518 -0
- package/dist/browser/tuning/sling.js.map +1 -0
- package/dist/browser/tuning/thumbnailRow.d.ts +2 -0
- package/dist/browser/tuning/thumbnailRow.js +108 -0
- package/dist/browser/tuning/thumbnailRow.js.map +1 -0
- package/dist/browser/tuning/tileClick.d.ts +2 -0
- package/dist/browser/tuning/tileClick.js +103 -0
- package/dist/browser/tuning/tileClick.js.map +1 -0
- package/dist/browser/tuning/youtubeTv.d.ts +2 -0
- package/dist/browser/tuning/youtubeTv.js +182 -0
- package/dist/browser/tuning/youtubeTv.js.map +1 -0
- package/dist/browser/video.d.ts +289 -0
- package/dist/browser/video.js +996 -0
- package/dist/browser/video.js.map +1 -0
- package/dist/channels/index.d.ts +3 -0
- package/dist/channels/index.js +392 -0
- package/dist/channels/index.js.map +1 -0
- package/dist/config/index.d.ts +53 -0
- package/dist/config/index.js +233 -0
- package/dist/config/index.js.map +1 -0
- package/dist/config/presets.d.ts +98 -0
- package/dist/config/presets.js +241 -0
- package/dist/config/presets.js.map +1 -0
- package/dist/config/profiles.d.ts +79 -0
- package/dist/config/profiles.js +245 -0
- package/dist/config/profiles.js.map +1 -0
- package/dist/config/providers.d.ts +120 -0
- package/dist/config/providers.js +450 -0
- package/dist/config/providers.js.map +1 -0
- package/dist/config/sites.d.ts +22 -0
- package/dist/config/sites.js +377 -0
- package/dist/config/sites.js.map +1 -0
- package/dist/config/userChannels.d.ts +178 -0
- package/dist/config/userChannels.js +543 -0
- package/dist/config/userChannels.js.map +1 -0
- package/dist/config/userConfig.d.ts +235 -0
- package/dist/config/userConfig.js +913 -0
- package/dist/config/userConfig.js.map +1 -0
- package/dist/hdhr/channelMap.d.ts +21 -0
- package/dist/hdhr/channelMap.js +82 -0
- package/dist/hdhr/channelMap.js.map +1 -0
- package/dist/hdhr/deviceId.d.ts +11 -0
- package/dist/hdhr/deviceId.js +84 -0
- package/dist/hdhr/deviceId.js.map +1 -0
- package/dist/hdhr/discover.d.ts +6 -0
- package/dist/hdhr/discover.js +155 -0
- package/dist/hdhr/discover.js.map +1 -0
- package/dist/hdhr/index.d.ts +9 -0
- package/dist/hdhr/index.js +87 -0
- package/dist/hdhr/index.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +144 -0
- package/dist/index.js.map +1 -0
- package/dist/routes/assets.d.ts +6 -0
- package/dist/routes/assets.js +79 -0
- package/dist/routes/assets.js.map +1 -0
- package/dist/routes/auth.d.ts +6 -0
- package/dist/routes/auth.js +77 -0
- package/dist/routes/auth.js.map +1 -0
- package/dist/routes/channels.d.ts +6 -0
- package/dist/routes/channels.js +40 -0
- package/dist/routes/channels.js.map +1 -0
- package/dist/routes/components.d.ts +138 -0
- package/dist/routes/components.js +210 -0
- package/dist/routes/components.js.map +1 -0
- package/dist/routes/config.d.ts +72 -0
- package/dist/routes/config.js +1977 -0
- package/dist/routes/config.js.map +1 -0
- package/dist/routes/debug.d.ts +6 -0
- package/dist/routes/debug.js +274 -0
- package/dist/routes/debug.js.map +1 -0
- package/dist/routes/health.d.ts +6 -0
- package/dist/routes/health.js +85 -0
- package/dist/routes/health.js.map +1 -0
- package/dist/routes/hls.d.ts +6 -0
- package/dist/routes/hls.js +25 -0
- package/dist/routes/hls.js.map +1 -0
- package/dist/routes/index.d.ts +19 -0
- package/dist/routes/index.js +49 -0
- package/dist/routes/index.js.map +1 -0
- package/dist/routes/logs.d.ts +6 -0
- package/dist/routes/logs.js +164 -0
- package/dist/routes/logs.js.map +1 -0
- package/dist/routes/mpegts.d.ts +6 -0
- package/dist/routes/mpegts.js +19 -0
- package/dist/routes/mpegts.js.map +1 -0
- package/dist/routes/play.d.ts +6 -0
- package/dist/routes/play.js +18 -0
- package/dist/routes/play.js.map +1 -0
- package/dist/routes/playlist.d.ts +36 -0
- package/dist/routes/playlist.js +134 -0
- package/dist/routes/playlist.js.map +1 -0
- package/dist/routes/root.d.ts +6 -0
- package/dist/routes/root.js +2920 -0
- package/dist/routes/root.js.map +1 -0
- package/dist/routes/streams.d.ts +6 -0
- package/dist/routes/streams.js +88 -0
- package/dist/routes/streams.js.map +1 -0
- package/dist/routes/theme.d.ts +15 -0
- package/dist/routes/theme.js +275 -0
- package/dist/routes/theme.js.map +1 -0
- package/dist/routes/ui.d.ts +56 -0
- package/dist/routes/ui.js +354 -0
- package/dist/routes/ui.js.map +1 -0
- package/dist/service/commands.d.ts +41 -0
- package/dist/service/commands.js +391 -0
- package/dist/service/commands.js.map +1 -0
- package/dist/service/generators.d.ts +33 -0
- package/dist/service/generators.js +432 -0
- package/dist/service/generators.js.map +1 -0
- package/dist/service/index.d.ts +2 -0
- package/dist/service/index.js +7 -0
- package/dist/service/index.js.map +1 -0
- package/dist/streaming/clients.d.ts +48 -0
- package/dist/streaming/clients.js +114 -0
- package/dist/streaming/clients.js.map +1 -0
- package/dist/streaming/fmp4Segmenter.d.ts +61 -0
- package/dist/streaming/fmp4Segmenter.js +461 -0
- package/dist/streaming/fmp4Segmenter.js.map +1 -0
- package/dist/streaming/hls.d.ts +120 -0
- package/dist/streaming/hls.js +722 -0
- package/dist/streaming/hls.js.map +1 -0
- package/dist/streaming/hlsSegments.d.ts +54 -0
- package/dist/streaming/hlsSegments.js +162 -0
- package/dist/streaming/hlsSegments.js.map +1 -0
- package/dist/streaming/lifecycle.d.ts +33 -0
- package/dist/streaming/lifecycle.js +185 -0
- package/dist/streaming/lifecycle.js.map +1 -0
- package/dist/streaming/monitor.d.ts +74 -0
- package/dist/streaming/monitor.js +1310 -0
- package/dist/streaming/monitor.js.map +1 -0
- package/dist/streaming/mp4Parser.d.ts +74 -0
- package/dist/streaming/mp4Parser.js +566 -0
- package/dist/streaming/mp4Parser.js.map +1 -0
- package/dist/streaming/mpegts.d.ts +14 -0
- package/dist/streaming/mpegts.js +248 -0
- package/dist/streaming/mpegts.js.map +1 -0
- package/dist/streaming/registry.d.ts +119 -0
- package/dist/streaming/registry.js +127 -0
- package/dist/streaming/registry.js.map +1 -0
- package/dist/streaming/setup.d.ts +135 -0
- package/dist/streaming/setup.js +670 -0
- package/dist/streaming/setup.js.map +1 -0
- package/dist/streaming/showInfo.d.ts +30 -0
- package/dist/streaming/showInfo.js +362 -0
- package/dist/streaming/showInfo.js.map +1 -0
- package/dist/streaming/statusEmitter.d.ts +125 -0
- package/dist/streaming/statusEmitter.js +139 -0
- package/dist/streaming/statusEmitter.js.map +1 -0
- package/dist/types/index.d.ts +403 -0
- package/dist/types/index.js +6 -0
- package/dist/types/index.js.map +1 -0
- package/dist/utils/debugFilter.d.ts +38 -0
- package/dist/utils/debugFilter.js +157 -0
- package/dist/utils/debugFilter.js.map +1 -0
- package/dist/utils/delay.d.ts +6 -0
- package/dist/utils/delay.js +15 -0
- package/dist/utils/delay.js.map +1 -0
- package/dist/utils/errors.d.ts +15 -0
- package/dist/utils/errors.js +40 -0
- package/dist/utils/errors.js.map +1 -0
- package/dist/utils/evaluate.d.ts +51 -0
- package/dist/utils/evaluate.js +124 -0
- package/dist/utils/evaluate.js.map +1 -0
- package/dist/utils/ffmpeg.d.ts +65 -0
- package/dist/utils/ffmpeg.js +317 -0
- package/dist/utils/ffmpeg.js.map +1 -0
- package/dist/utils/fileLogger.d.ts +25 -0
- package/dist/utils/fileLogger.js +248 -0
- package/dist/utils/fileLogger.js.map +1 -0
- package/dist/utils/format.d.ts +16 -0
- package/dist/utils/format.js +46 -0
- package/dist/utils/format.js.map +1 -0
- package/dist/utils/html.d.ts +6 -0
- package/dist/utils/html.js +24 -0
- package/dist/utils/html.js.map +1 -0
- package/dist/utils/index.d.ts +15 -0
- package/dist/utils/index.js +20 -0
- package/dist/utils/index.js.map +1 -0
- package/dist/utils/logEmitter.d.ts +17 -0
- package/dist/utils/logEmitter.js +30 -0
- package/dist/utils/logEmitter.js.map +1 -0
- package/dist/utils/logger.d.ts +82 -0
- package/dist/utils/logger.js +219 -0
- package/dist/utils/logger.js.map +1 -0
- package/dist/utils/m3u.d.ts +32 -0
- package/dist/utils/m3u.js +148 -0
- package/dist/utils/m3u.js.map +1 -0
- package/dist/utils/morganStream.d.ts +7 -0
- package/dist/utils/morganStream.js +33 -0
- package/dist/utils/morganStream.js.map +1 -0
- package/dist/utils/platform.d.ts +64 -0
- package/dist/utils/platform.js +157 -0
- package/dist/utils/platform.js.map +1 -0
- package/dist/utils/retry.d.ts +15 -0
- package/dist/utils/retry.js +82 -0
- package/dist/utils/retry.js.map +1 -0
- package/dist/utils/streamContext.d.ts +28 -0
- package/dist/utils/streamContext.js +33 -0
- package/dist/utils/streamContext.js.map +1 -0
- package/dist/utils/version.d.ts +37 -0
- package/dist/utils/version.js +228 -0
- package/dist/utils/version.js.map +1 -0
- package/package.json +92 -0
- package/prismcast.png +0 -0
- package/prismcast.svg +74 -0
|
@@ -0,0 +1,1310 @@
|
|
|
1
|
+
/* Copyright(C) 2024-2026, HJD (https://github.com/hjdhjd). All rights reserved.
|
|
2
|
+
*
|
|
3
|
+
* monitor.ts: Playback health monitoring for PrismCast.
|
|
4
|
+
*/
|
|
5
|
+
import { EvaluateTimeoutError, LOG, formatError, getAbortSignal, isSessionClosedError, runWithStreamContext, startTimer } from "../utils/index.js";
|
|
6
|
+
import { applyVideoStyles, buildVideoSelectorType, checkVideoPresence, enforceVideoVolume, ensurePlayback, findVideoContext, getVideoState, tuneToChannel, validateVideoElement, verifyFullscreen } from "../browser/video.js";
|
|
7
|
+
import { getChannelLogo, getShowName } from "./showInfo.js";
|
|
8
|
+
import { getLastSegmentSize, getStream, getStreamMemoryUsage } from "./registry.js";
|
|
9
|
+
import { CONFIG } from "../config/index.js";
|
|
10
|
+
import { emitStreamHealthChanged } from "./statusEmitter.js";
|
|
11
|
+
import { getClientSummary } from "./clients.js";
|
|
12
|
+
import { resizeAndMinimizeWindow } from "../browser/cdp.js";
|
|
13
|
+
// Recovery method names used in logging. Centralized to ensure consistency across start, success, and failure messages.
|
|
14
|
+
const RECOVERY_METHODS = {
|
|
15
|
+
pageNavigation: "page navigation",
|
|
16
|
+
playUnmute: "play/unmute",
|
|
17
|
+
sourceReload: "source reload",
|
|
18
|
+
tabReplacement: "tab replacement"
|
|
19
|
+
};
|
|
20
|
+
/* These mappings connect recovery method names to their corresponding counter fields in RecoveryMetrics. Using a mapping pattern instead of if/else chains reduces
|
|
21
|
+
* code duplication, makes adding new recovery methods trivial (add one entry to each map), ensures consistency between attempt and success counting, and provides
|
|
22
|
+
* type safety via the RecoveryMetrics interface.
|
|
23
|
+
*/
|
|
24
|
+
// Maps recovery method names to their attempt counter field names.
|
|
25
|
+
const ATTEMPT_FIELDS = {
|
|
26
|
+
[RECOVERY_METHODS.pageNavigation]: "pageNavigationAttempts",
|
|
27
|
+
[RECOVERY_METHODS.playUnmute]: "playUnmuteAttempts",
|
|
28
|
+
[RECOVERY_METHODS.sourceReload]: "sourceReloadAttempts",
|
|
29
|
+
[RECOVERY_METHODS.tabReplacement]: "tabReplacementAttempts"
|
|
30
|
+
};
|
|
31
|
+
// Maps recovery method names to their success counter field names.
|
|
32
|
+
const SUCCESS_FIELDS = {
|
|
33
|
+
[RECOVERY_METHODS.pageNavigation]: "pageNavigationSuccesses",
|
|
34
|
+
[RECOVERY_METHODS.playUnmute]: "playUnmuteSuccesses",
|
|
35
|
+
[RECOVERY_METHODS.sourceReload]: "sourceReloadSuccesses",
|
|
36
|
+
[RECOVERY_METHODS.tabReplacement]: "tabReplacementSuccesses"
|
|
37
|
+
};
|
|
38
|
+
/**
|
|
39
|
+
* Creates a new RecoveryMetrics object with all counters initialized to zero.
|
|
40
|
+
* @returns A fresh RecoveryMetrics object.
|
|
41
|
+
*/
|
|
42
|
+
function createRecoveryMetrics() {
|
|
43
|
+
return {
|
|
44
|
+
currentRecoveryMethod: null,
|
|
45
|
+
currentRecoveryStartTime: null,
|
|
46
|
+
pageNavigationAttempts: 0,
|
|
47
|
+
pageNavigationSuccesses: 0,
|
|
48
|
+
playUnmuteAttempts: 0,
|
|
49
|
+
playUnmuteSuccesses: 0,
|
|
50
|
+
sourceReloadAttempts: 0,
|
|
51
|
+
sourceReloadSuccesses: 0,
|
|
52
|
+
tabReplacementAttempts: 0,
|
|
53
|
+
tabReplacementSuccesses: 0,
|
|
54
|
+
totalRecoveryTimeMs: 0
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Gets the total number of recovery attempts across all methods. Iterates over ATTEMPT_FIELDS to sum all attempt counters, ensuring new recovery methods are
|
|
59
|
+
* automatically included without code changes.
|
|
60
|
+
* @param metrics - The recovery metrics object.
|
|
61
|
+
* @returns Total recovery attempts.
|
|
62
|
+
*/
|
|
63
|
+
export function getTotalRecoveryAttempts(metrics) {
|
|
64
|
+
let total = 0;
|
|
65
|
+
for (const fieldName of Object.values(ATTEMPT_FIELDS)) {
|
|
66
|
+
total += metrics[fieldName];
|
|
67
|
+
}
|
|
68
|
+
return total;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Gets the total number of successful recoveries across all methods. Iterates over SUCCESS_FIELDS to sum all success counters, ensuring new recovery methods
|
|
72
|
+
* are automatically included without code changes.
|
|
73
|
+
* @param metrics - The recovery metrics object.
|
|
74
|
+
* @returns Total successful recoveries.
|
|
75
|
+
*/
|
|
76
|
+
function getTotalRecoverySuccesses(metrics) {
|
|
77
|
+
let total = 0;
|
|
78
|
+
for (const fieldName of Object.values(SUCCESS_FIELDS)) {
|
|
79
|
+
total += metrics[fieldName];
|
|
80
|
+
}
|
|
81
|
+
return total;
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Formats recovery duration from start time to now.
|
|
85
|
+
* @param startTime - The timestamp when recovery started.
|
|
86
|
+
* @returns Formatted duration string like "2.1s".
|
|
87
|
+
*/
|
|
88
|
+
function formatRecoveryDuration(startTime) {
|
|
89
|
+
const durationMs = Date.now() - startTime;
|
|
90
|
+
return (durationMs / 1000).toFixed(1) + "s";
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Maps issue category to user-friendly description for logging.
|
|
94
|
+
* @param category - The issue category from getIssueCategory().
|
|
95
|
+
* @returns User-friendly description.
|
|
96
|
+
*/
|
|
97
|
+
function getIssueDescription(category) {
|
|
98
|
+
switch (category) {
|
|
99
|
+
case "paused": {
|
|
100
|
+
return "paused";
|
|
101
|
+
}
|
|
102
|
+
case "buffering": {
|
|
103
|
+
return "buffering";
|
|
104
|
+
}
|
|
105
|
+
default: {
|
|
106
|
+
return "stalled";
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Maps recovery level to method name.
|
|
112
|
+
* @param level - The recovery level (1, 2, or 3).
|
|
113
|
+
* @returns The recovery method name.
|
|
114
|
+
*/
|
|
115
|
+
function getRecoveryMethod(level) {
|
|
116
|
+
switch (level) {
|
|
117
|
+
case 1: {
|
|
118
|
+
return RECOVERY_METHODS.playUnmute;
|
|
119
|
+
}
|
|
120
|
+
case 2: {
|
|
121
|
+
return RECOVERY_METHODS.sourceReload;
|
|
122
|
+
}
|
|
123
|
+
default: {
|
|
124
|
+
return RECOVERY_METHODS.pageNavigation;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Records a recovery attempt in the metrics. Uses the ATTEMPT_FIELDS mapping to find the correct counter field, eliminating the need for if/else chains. This
|
|
130
|
+
* makes adding new recovery methods trivial - just add an entry to ATTEMPT_FIELDS.
|
|
131
|
+
*
|
|
132
|
+
* Note: Tab replacement calls this once per logical attempt even though it may internally retry the onTabReplacement callback. The retry is an implementation
|
|
133
|
+
* detail of executeTabReplacement, not a separate recovery attempt from the monitor's perspective. The circuit breaker likewise records one failure per logical
|
|
134
|
+
* attempt, not per callback invocation.
|
|
135
|
+
* @param metrics - The metrics object to update.
|
|
136
|
+
* @param method - The recovery method being attempted.
|
|
137
|
+
*/
|
|
138
|
+
function recordRecoveryAttempt(metrics, method) {
|
|
139
|
+
// Cast to the specific field type to handle potential unknown methods at runtime. The mapping ensures valid methods resolve to counter field names.
|
|
140
|
+
const field = ATTEMPT_FIELDS[method];
|
|
141
|
+
if (field !== undefined) {
|
|
142
|
+
metrics[field]++;
|
|
143
|
+
}
|
|
144
|
+
metrics.currentRecoveryStartTime = Date.now();
|
|
145
|
+
metrics.currentRecoveryMethod = method;
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Records a successful recovery in the metrics and clears the pending recovery state. Uses the SUCCESS_FIELDS mapping to find the correct counter field,
|
|
149
|
+
* eliminating the need for if/else chains. This makes adding new recovery methods trivial - just add an entry to SUCCESS_FIELDS.
|
|
150
|
+
* @param metrics - The metrics object to update.
|
|
151
|
+
* @param method - The recovery method that succeeded.
|
|
152
|
+
*/
|
|
153
|
+
function recordRecoverySuccess(metrics, method) {
|
|
154
|
+
// Cast to the specific field type to handle potential unknown methods at runtime. The mapping ensures valid methods resolve to counter field names.
|
|
155
|
+
const field = SUCCESS_FIELDS[method];
|
|
156
|
+
if (field !== undefined) {
|
|
157
|
+
metrics[field]++;
|
|
158
|
+
}
|
|
159
|
+
if (metrics.currentRecoveryStartTime !== null) {
|
|
160
|
+
metrics.totalRecoveryTimeMs += Date.now() - metrics.currentRecoveryStartTime;
|
|
161
|
+
}
|
|
162
|
+
metrics.currentRecoveryStartTime = null;
|
|
163
|
+
metrics.currentRecoveryMethod = null;
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* Capitalizes the first letter of a string.
|
|
167
|
+
* @param str - The string to capitalize.
|
|
168
|
+
* @returns The string with the first letter capitalized.
|
|
169
|
+
*/
|
|
170
|
+
function capitalize(str) {
|
|
171
|
+
return str.charAt(0).toUpperCase() + str.slice(1);
|
|
172
|
+
}
|
|
173
|
+
/**
|
|
174
|
+
* Formats the recovery metrics summary for the termination log. Uses the SUCCESS_FIELDS mapping to iterate over all recovery methods, eliminating hardcoded
|
|
175
|
+
* checks for each method type. This ensures new recovery methods are automatically included in the summary.
|
|
176
|
+
* @param metrics - The recovery metrics object.
|
|
177
|
+
* @returns Formatted summary string, or empty string if no recoveries occurred.
|
|
178
|
+
*/
|
|
179
|
+
export function formatRecoveryMetricsSummary(metrics) {
|
|
180
|
+
const totalAttempts = getTotalRecoveryAttempts(metrics);
|
|
181
|
+
if (totalAttempts === 0) {
|
|
182
|
+
return "No recoveries needed.";
|
|
183
|
+
}
|
|
184
|
+
const totalSuccesses = getTotalRecoverySuccesses(metrics);
|
|
185
|
+
// Build the breakdown of recovery methods used by iterating over all methods in SUCCESS_FIELDS. This automatically includes any new recovery methods added to
|
|
186
|
+
// the mapping without requiring code changes here.
|
|
187
|
+
const parts = [];
|
|
188
|
+
for (const [methodName, fieldName] of Object.entries(SUCCESS_FIELDS)) {
|
|
189
|
+
const count = metrics[fieldName];
|
|
190
|
+
if (count > 0) {
|
|
191
|
+
parts.push(String(count) + "× " + methodName);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
// Calculate average recovery time.
|
|
195
|
+
const avgTimeMs = totalSuccesses > 0 ? metrics.totalRecoveryTimeMs / totalSuccesses : 0;
|
|
196
|
+
const avgTimeStr = (avgTimeMs / 1000).toFixed(1) + "s";
|
|
197
|
+
// Format: "Recoveries: 8 (5× source reload, 3× page navigation), avg 4.2s."
|
|
198
|
+
if (parts.length > 0) {
|
|
199
|
+
return "Recoveries: " + String(totalSuccesses) + " (" + parts.join(", ") + "), avg " + avgTimeStr + ".";
|
|
200
|
+
}
|
|
201
|
+
// Edge case: attempts but no successes (stream terminated before recovery completed).
|
|
202
|
+
return "Recoveries: " + String(totalAttempts) + " attempted, 0 succeeded.";
|
|
203
|
+
}
|
|
204
|
+
/**
|
|
205
|
+
* Records a failure and checks whether the circuit breaker should trip. This centralizes the circuit breaker logic that was previously duplicated in multiple
|
|
206
|
+
* recovery paths. The function updates the state in place and returns whether the breaker should trip.
|
|
207
|
+
* @param state - The circuit breaker state to update.
|
|
208
|
+
* @param now - The current timestamp.
|
|
209
|
+
* @returns Result indicating whether the circuit breaker should trip and diagnostic info.
|
|
210
|
+
*/
|
|
211
|
+
function checkCircuitBreaker(state, now) {
|
|
212
|
+
// Record this failure.
|
|
213
|
+
state.totalFailureCount++;
|
|
214
|
+
state.firstFailureTime ??= now;
|
|
215
|
+
// Check if we're within the failure window.
|
|
216
|
+
const withinWindow = (now - state.firstFailureTime) < CONFIG.recovery.circuitBreakerWindow;
|
|
217
|
+
// Determine if we should trip.
|
|
218
|
+
const shouldTrip = withinWindow && (state.totalFailureCount >= CONFIG.recovery.circuitBreakerThreshold);
|
|
219
|
+
// Reset the window if we're outside it (start fresh count).
|
|
220
|
+
if (!withinWindow) {
|
|
221
|
+
state.totalFailureCount = 1;
|
|
222
|
+
state.firstFailureTime = now;
|
|
223
|
+
}
|
|
224
|
+
return { shouldTrip, totalCount: state.totalFailureCount, withinWindow };
|
|
225
|
+
}
|
|
226
|
+
/**
|
|
227
|
+
* Resets the circuit breaker state. Called when sustained healthy playback is achieved.
|
|
228
|
+
* @param state - The circuit breaker state to reset.
|
|
229
|
+
*/
|
|
230
|
+
function resetCircuitBreaker(state) {
|
|
231
|
+
state.firstFailureTime = null;
|
|
232
|
+
state.totalFailureCount = 0;
|
|
233
|
+
}
|
|
234
|
+
/**
|
|
235
|
+
* Formats the issue type for diagnostic logging. Returns a human-readable string describing what triggered the recovery. Multiple issues can occur simultaneously
|
|
236
|
+
* (e.g., "paused, stalled"), so we collect all applicable issues into a comma-separated list.
|
|
237
|
+
* @param state - The video state object containing paused, ended, hasError, etc.
|
|
238
|
+
* @param isStalled - Whether the video is stalled (not progressing).
|
|
239
|
+
* @param isBuffering - Whether the video is actively buffering.
|
|
240
|
+
* @returns A description of the issue.
|
|
241
|
+
*/
|
|
242
|
+
function formatIssueType(state, isStalled, isBuffering) {
|
|
243
|
+
const issues = [];
|
|
244
|
+
if (state.paused) {
|
|
245
|
+
issues.push("paused");
|
|
246
|
+
}
|
|
247
|
+
if (state.ended) {
|
|
248
|
+
issues.push("ended");
|
|
249
|
+
}
|
|
250
|
+
if (state.error) {
|
|
251
|
+
issues.push("error");
|
|
252
|
+
}
|
|
253
|
+
// Distinguish between buffering (temporary, network-related) and stalled (stopped for unknown reason). Both result in no progression, but buffering indicates the
|
|
254
|
+
// player is actively trying to get more data.
|
|
255
|
+
if (isStalled && isBuffering) {
|
|
256
|
+
issues.push("buffering");
|
|
257
|
+
}
|
|
258
|
+
if (isStalled && !isBuffering) {
|
|
259
|
+
issues.push("stalled");
|
|
260
|
+
}
|
|
261
|
+
return issues.length > 0 ? issues.join(", ") : "unknown";
|
|
262
|
+
}
|
|
263
|
+
/**
|
|
264
|
+
* Determines the issue category for recovery path selection. This is separate from formatIssueType (which is for logging) because recovery decisions need a single
|
|
265
|
+
* category, not a list of all issues. The categories are:
|
|
266
|
+
* - "paused": Video is paused but not buffering. L1 (play/unmute) may help.
|
|
267
|
+
* - "buffering": Video is buffering or stalled with low readyState. Skip L1, go to L2 (source reload).
|
|
268
|
+
* - "other": Error, ended, or unknown state. Skip L1, go to L2 (source reload).
|
|
269
|
+
* @param state - The video state object.
|
|
270
|
+
* @param isStalled - Whether the video is stalled (not progressing).
|
|
271
|
+
* @param isBuffering - Whether the video is actively buffering.
|
|
272
|
+
* @returns The issue category for recovery path selection.
|
|
273
|
+
*/
|
|
274
|
+
function getIssueCategory(state, isStalled, isBuffering) {
|
|
275
|
+
// Error and ended states take priority - these need aggressive recovery.
|
|
276
|
+
if (state.error || state.ended) {
|
|
277
|
+
return "other";
|
|
278
|
+
}
|
|
279
|
+
// Buffering (readyState < 3 with active network) needs source reload, not play/unmute.
|
|
280
|
+
if (isBuffering) {
|
|
281
|
+
return "buffering";
|
|
282
|
+
}
|
|
283
|
+
// Stalled with low readyState is effectively buffering.
|
|
284
|
+
if (isStalled && (state.readyState < 3)) {
|
|
285
|
+
return "buffering";
|
|
286
|
+
}
|
|
287
|
+
// Paused state (without buffering) may respond to play/unmute.
|
|
288
|
+
if (state.paused) {
|
|
289
|
+
return "paused";
|
|
290
|
+
}
|
|
291
|
+
// Stalled without low readyState - unknown cause, treat as buffering.
|
|
292
|
+
if (isStalled) {
|
|
293
|
+
return "buffering";
|
|
294
|
+
}
|
|
295
|
+
return "other";
|
|
296
|
+
}
|
|
297
|
+
export function monitorPlaybackHealth(page, context, profile, url, streamId, streamInfo, onCircuitBreak, onTabReplacement) {
|
|
298
|
+
/* Monitor state variables. These track the video's behavior over time and control recovery decisions.
|
|
299
|
+
*/
|
|
300
|
+
// The current page reference. This can change after tab replacement recovery, when the old tab is closed and a new one is created. We use a mutable variable so we
|
|
301
|
+
// can update the reference after replacement.
|
|
302
|
+
let currentPage = page;
|
|
303
|
+
// The video's currentTime from the previous check. Used to detect whether the video is progressing. Null on first check since we have no previous value.
|
|
304
|
+
let lastTime = null;
|
|
305
|
+
// Number of consecutive checks where currentTime did not advance. We require multiple consecutive stalls before triggering recovery to avoid reacting to momentary
|
|
306
|
+
// hiccups. Reset to 0 when progression is detected.
|
|
307
|
+
let stallCount = 0;
|
|
308
|
+
// Number of consecutive checks where the video reports paused state. Like stallCount, we require multiple consecutive paused checks (> stallCountThreshold) before
|
|
309
|
+
// triggering recovery. This filters out transient rebuffer pauses where the player briefly pauses itself to refill its buffer and resumes on its own. Without this
|
|
310
|
+
// hysteresis, every transient rebuffer pause triggers an unnecessary L1 recovery (play/unmute) that logs noise but does nothing useful.
|
|
311
|
+
let pauseCount = 0;
|
|
312
|
+
// Current escalation level (0-4). Level 0 means no recovery needed. Each time recovery is triggered, this increments to try increasingly aggressive actions.
|
|
313
|
+
// Resets to 0 after sustained healthy playback.
|
|
314
|
+
let escalationLevel = 0;
|
|
315
|
+
// Timestamp of the last recovery attempt. Used to calculate healthy playback duration for escalation reset.
|
|
316
|
+
let lastRecoveryTime = 0;
|
|
317
|
+
// Timestamp when buffering started, or null if not currently buffering. Used to apply the buffering grace period - we don't trigger recovery for buffering until
|
|
318
|
+
// it exceeds BUFFERING_GRACE_PERIOD.
|
|
319
|
+
let bufferingStartTime = null;
|
|
320
|
+
// Timestamps of recent page reloads within the PAGE_RELOAD_WINDOW. Used to enforce MAX_PAGE_RELOADS limit. Old entries are pruned on each check.
|
|
321
|
+
let pageReloadTimestamps = [];
|
|
322
|
+
// Counter for consecutive page navigation failures. If navigation fails twice in a row, we fall back to source reload (level 2) instead. This prevents getting
|
|
323
|
+
// stuck in a loop when navigation itself is the problem.
|
|
324
|
+
let consecutiveNavigationFailures = 0;
|
|
325
|
+
// Track whether source reload (L2) has been attempted in the current page session. Log analysis shows the first source reload often works (~58%), but the second
|
|
326
|
+
// always fails and leaves the video at readyState=0. When this flag is true and recovery is needed, we skip L2 and go directly to L3 (page reload).
|
|
327
|
+
let sourceReloadAttempted = false;
|
|
328
|
+
// Segment production monitoring state. After L2/L3 recovery, we track whether segments are actually being produced. If recovery reports success but the capture
|
|
329
|
+
// pipeline is dead (MediaRecorder stopped producing data), we need to escalate to tab replacement.
|
|
330
|
+
let preRecoverySegmentIndex = null; // Segment index when recovery started, used to detect if new segments are produced.
|
|
331
|
+
let segmentWaitStartTime = null; // Timestamp when we started waiting for segment production after recovery grace period.
|
|
332
|
+
let segmentProductionStalled = false; // Flag indicating segment production has stalled after recovery.
|
|
333
|
+
// Continuous segment size monitoring state. Detects spontaneous capture pipeline death (no preceding recovery event) by checking segment sizes. Dead pipelines
|
|
334
|
+
// produce tiny segments (18 bytes observed) while the video element appears healthy. This complements post-recovery index monitoring.
|
|
335
|
+
let lastCheckedSegmentIndex = 0; // Last segment index we inspected (to detect new segments).
|
|
336
|
+
let consecutiveTinySegments = 0; // Count of consecutive tiny segments.
|
|
337
|
+
let wasInTinySegmentState = false; // For detecting spontaneous recovery (tiny→valid transition without explicit recovery).
|
|
338
|
+
// Track whether the browser window needs to be re-minimized after recovery. Recovery actions (especially ensureFullscreen) can cause the window to un-minimize.
|
|
339
|
+
// We set this flag when recovery is triggered and clear it after the recovery grace period passes and we see a healthy check.
|
|
340
|
+
let pendingReMinimize = false;
|
|
341
|
+
// Graduated fullscreen reinforcement counter. Counts consecutive ticks where verifyFullscreen() returns false. On tick 1 we apply basic CSS styles (sufficient
|
|
342
|
+
// for well-behaved sites like Hulu). On tick 2+ we escalate to !important priority to override sites that actively fight style changes. Reset to 0 when the
|
|
343
|
+
// video fills the viewport again.
|
|
344
|
+
let fullscreenReapplyCount = 0;
|
|
345
|
+
// Flag indicating the cleanup function was called. When true, the next interval check will clear itself.
|
|
346
|
+
let intervalCleared = false;
|
|
347
|
+
// Flag indicating a recovery operation is in progress. We skip health checks during recovery to avoid triggering additional recovery while one is running.
|
|
348
|
+
let recoveryInProgress = false;
|
|
349
|
+
// The current video context (page or frame). This can change after a page navigation recovery, when we need to find the new video context.
|
|
350
|
+
let currentContext = context;
|
|
351
|
+
// Circuit breaker state. Tracks total failures within a time window and trips (terminates the stream) when too many failures occur.
|
|
352
|
+
const circuitBreaker = { firstFailureTime: null, totalFailureCount: 0 };
|
|
353
|
+
// Counter for consecutive "video not found" occurrences. We apply a grace period before triggering recovery to handle momentary context invalidation or readyState
|
|
354
|
+
// fluctuations. Reset to 0 when video is found.
|
|
355
|
+
let videoNotFoundCount = 0;
|
|
356
|
+
// Counter for consecutive evaluate timeouts. When the browser tab becomes unresponsive, evaluate() calls will timeout instead of returning data. After 3
|
|
357
|
+
// consecutive timeouts, we trigger tab replacement recovery (if the callback is provided). Reset to 0 on successful getVideoState().
|
|
358
|
+
let consecutiveTimeouts = 0;
|
|
359
|
+
// Total recovery attempts for status reporting.
|
|
360
|
+
let totalRecoveryAttempts = 0;
|
|
361
|
+
// Last known video state for status reporting.
|
|
362
|
+
let lastVideoState = null;
|
|
363
|
+
// Recovery metrics tracked throughout the stream's lifetime.
|
|
364
|
+
const metrics = createRecoveryMetrics();
|
|
365
|
+
// Last issue tracking for UI display. Stores what triggered recovery and when, so users can see stream history.
|
|
366
|
+
let lastIssueType = null;
|
|
367
|
+
let lastIssueTime = null;
|
|
368
|
+
// Recovery grace period. After a recovery action, we wait before checking for new issues to give the action time to take effect. L1 (play/unmute) is a quick
|
|
369
|
+
// action. L2 (source reload) and L3 (page reload) need more time for rebuffering/navigation.
|
|
370
|
+
const recoveryGracePeriods = [0, 3000, 10000, 10000]; // L0, L1, L2, L3 in milliseconds.
|
|
371
|
+
// Segment stall timeout. After L2/L3 recovery completes, if no new segments are produced within this timeout, the capture pipeline is considered dead and we
|
|
372
|
+
// escalate directly to tab replacement. This catches the case where recovery reports success but the MediaRecorder/FFmpeg pipeline has silently died.
|
|
373
|
+
const SEGMENT_STALL_TIMEOUT = 10000; // 10 seconds.
|
|
374
|
+
// Tiny segment detection thresholds. Used for continuous segment size monitoring to detect dead capture pipelines. When video capture dies but audio continues,
|
|
375
|
+
// segments contain only audio data. Audio is transcoded at a controlled bitrate (max 512Kbps), so audio-only segments are at most ~192KB for 3-second segments.
|
|
376
|
+
// The 500KB threshold catches both dead captures (18 bytes) and audio-only captures while staying well below the smallest video preset (480p/3Mbps ≈ 750KB/segment).
|
|
377
|
+
const TINY_SEGMENT_THRESHOLD = 512000; // 500KB - segments below this indicate dead or degraded capture.
|
|
378
|
+
const TINY_SEGMENT_COUNT_TRIGGER = 10; // Trigger recovery after 10 consecutive tiny segments (~20 seconds with 2-second segments).
|
|
379
|
+
// Fixed margin in milliseconds before the maxContinuousPlayback limit at which a proactive reload is triggered. Two minutes provides enough time for page
|
|
380
|
+
// navigation and video reinitialization to complete before the site enforces its cutoff.
|
|
381
|
+
const PROACTIVE_RELOAD_MARGIN_MS = 120000;
|
|
382
|
+
let recoveryGraceUntil = 0;
|
|
383
|
+
// Timestamp of the most recent full page navigation. Used to calculate elapsed continuous playback for proactive reload when maxContinuousPlayback is configured.
|
|
384
|
+
// Initialized to Date.now() because the monitor starts immediately after tuneToChannel() succeeds in stream setup, meaning a page load just completed. Reset
|
|
385
|
+
// after any successful page navigation recovery or tab replacement, but NOT after source reloads (L2) which preserve the page's JavaScript context.
|
|
386
|
+
let lastPageNavigationTime = Date.now();
|
|
387
|
+
// Pre-compute the selector type string for video element selection. This is passed to evaluate() calls.
|
|
388
|
+
const selectorType = buildVideoSelectorType(profile);
|
|
389
|
+
// Capture stream context for re-establishing on each interval tick. AsyncLocalStorage context is lost when entering setInterval callbacks.
|
|
390
|
+
const streamContext = { channelName: streamInfo.channelName ?? undefined, streamId, url };
|
|
391
|
+
// Helper to mark a discontinuity in the HLS playlist after recovery events that disrupt the video source. The segmenter flushes its current fragment buffer and sets
|
|
392
|
+
// a pending discontinuity flag so the next segment boundary includes an #EXT-X-DISCONTINUITY tag. This tells HLS clients to flush their decoder state.
|
|
393
|
+
const markStreamDiscontinuity = () => {
|
|
394
|
+
getStream(streamInfo.numericStreamId)?.segmenter?.markDiscontinuity();
|
|
395
|
+
};
|
|
396
|
+
/**
|
|
397
|
+
* Computes the health status classification based on current monitor state.
|
|
398
|
+
* @returns The health status classification.
|
|
399
|
+
*/
|
|
400
|
+
function computeHealthStatus() {
|
|
401
|
+
// Error state takes precedence.
|
|
402
|
+
if (lastVideoState?.error) {
|
|
403
|
+
return "error";
|
|
404
|
+
}
|
|
405
|
+
// If we're at escalation level 3 (page reload), we're in serious trouble.
|
|
406
|
+
if (escalationLevel >= 3) {
|
|
407
|
+
return "error";
|
|
408
|
+
}
|
|
409
|
+
// If we're actively recovering (levels 1-2).
|
|
410
|
+
if (escalationLevel > 0) {
|
|
411
|
+
return "recovering";
|
|
412
|
+
}
|
|
413
|
+
// If we're buffering (within grace period).
|
|
414
|
+
if (bufferingStartTime !== null) {
|
|
415
|
+
return "buffering";
|
|
416
|
+
}
|
|
417
|
+
// If we have consecutive stalls but not yet triggering recovery.
|
|
418
|
+
if (stallCount > 0) {
|
|
419
|
+
return "stalled";
|
|
420
|
+
}
|
|
421
|
+
return "healthy";
|
|
422
|
+
}
|
|
423
|
+
/**
|
|
424
|
+
* Emits a status update for this stream.
|
|
425
|
+
*/
|
|
426
|
+
function emitStatusUpdate() {
|
|
427
|
+
const now = Date.now();
|
|
428
|
+
// Get current memory usage from the stream's HLS segment buffers.
|
|
429
|
+
const entry = getStream(streamInfo.numericStreamId);
|
|
430
|
+
const memoryBytes = entry ? getStreamMemoryUsage(entry).total : 0;
|
|
431
|
+
// Get the channel key from the registry entry for logo lookup.
|
|
432
|
+
const channelKey = entry?.info.storeKey ?? "";
|
|
433
|
+
// Get current client counts and type breakdown for this stream.
|
|
434
|
+
const clientSummary = getClientSummary(streamInfo.numericStreamId);
|
|
435
|
+
const status = {
|
|
436
|
+
bufferingDuration: bufferingStartTime ? Math.round((now - bufferingStartTime) / 1000) : null,
|
|
437
|
+
channel: streamInfo.channelName,
|
|
438
|
+
clientCount: clientSummary.total,
|
|
439
|
+
clients: clientSummary.clients,
|
|
440
|
+
currentTime: lastVideoState?.time ?? 0,
|
|
441
|
+
duration: Math.round((now - streamInfo.startTime.getTime()) / 1000),
|
|
442
|
+
escalationLevel,
|
|
443
|
+
health: computeHealthStatus(),
|
|
444
|
+
id: streamInfo.numericStreamId,
|
|
445
|
+
lastIssueTime,
|
|
446
|
+
lastIssueType,
|
|
447
|
+
lastRecoveryTime: lastRecoveryTime > 0 ? lastRecoveryTime : null,
|
|
448
|
+
logoUrl: channelKey ? (getChannelLogo(channelKey) ?? "") : "",
|
|
449
|
+
memoryBytes,
|
|
450
|
+
networkState: lastVideoState?.networkState ?? 0,
|
|
451
|
+
pageReloadsInWindow: pageReloadTimestamps.length,
|
|
452
|
+
providerName: streamInfo.providerName,
|
|
453
|
+
readyState: lastVideoState?.readyState ?? 0,
|
|
454
|
+
recoveryAttempts: totalRecoveryAttempts,
|
|
455
|
+
showName: getShowName(streamInfo.numericStreamId),
|
|
456
|
+
startTime: streamInfo.startTime.toISOString(),
|
|
457
|
+
url
|
|
458
|
+
};
|
|
459
|
+
emitStreamHealthChanged(status);
|
|
460
|
+
}
|
|
461
|
+
/**
|
|
462
|
+
* Finalizes tab replacement by clearing the recovery flag and emitting status. This helper ensures consistent cleanup across all tab replacement exit paths (success,
|
|
463
|
+
* failure, and error). The flag must be reset before emitting status to prevent the monitor from getting stuck if emitStatusUpdate() throws.
|
|
464
|
+
*/
|
|
465
|
+
function finalizeTabReplacement() {
|
|
466
|
+
recoveryInProgress = false;
|
|
467
|
+
emitStatusUpdate();
|
|
468
|
+
}
|
|
469
|
+
/**
|
|
470
|
+
* Resets all segment monitoring state variables. Called after successful recovery or sustained healthy playback to clear tracking for both post-recovery index
|
|
471
|
+
* monitoring and continuous size monitoring.
|
|
472
|
+
*/
|
|
473
|
+
function resetSegmentMonitoringState() {
|
|
474
|
+
preRecoverySegmentIndex = null;
|
|
475
|
+
segmentWaitStartTime = null;
|
|
476
|
+
segmentProductionStalled = false;
|
|
477
|
+
consecutiveTinySegments = 0;
|
|
478
|
+
wasInTinySegmentState = false;
|
|
479
|
+
lastCheckedSegmentIndex = getStream(streamInfo.numericStreamId)?.segmenter?.getSegmentIndex() ?? 0;
|
|
480
|
+
}
|
|
481
|
+
/**
|
|
482
|
+
* Resets all failure/retry counters to zero. Called after successful tab replacement or page navigation to give the stream a fresh start.
|
|
483
|
+
*/
|
|
484
|
+
function resetRecoveryCounters() {
|
|
485
|
+
consecutiveTimeouts = 0;
|
|
486
|
+
consecutiveNavigationFailures = 0;
|
|
487
|
+
fullscreenReapplyCount = 0;
|
|
488
|
+
pauseCount = 0;
|
|
489
|
+
videoNotFoundCount = 0;
|
|
490
|
+
stallCount = 0;
|
|
491
|
+
}
|
|
492
|
+
/**
|
|
493
|
+
* Resets escalation level and related flags. Called after successful recovery to allow the stream to start from level 0 on future issues.
|
|
494
|
+
*/
|
|
495
|
+
function resetEscalationState() {
|
|
496
|
+
escalationLevel = 0;
|
|
497
|
+
sourceReloadAttempted = false;
|
|
498
|
+
}
|
|
499
|
+
/**
|
|
500
|
+
* Sets the recovery grace period and re-minimize flag after a recovery action. The grace period prevents the monitor from immediately detecting new issues while the
|
|
501
|
+
* recovery action takes effect.
|
|
502
|
+
* @param level - The recovery level (1-3) to determine grace period duration.
|
|
503
|
+
*/
|
|
504
|
+
function setRecoveryGracePeriod(level) {
|
|
505
|
+
pendingReMinimize = true;
|
|
506
|
+
recoveryGraceUntil = Date.now() + recoveryGracePeriods[level];
|
|
507
|
+
}
|
|
508
|
+
/**
|
|
509
|
+
* Handles tab replacement failure by checking the circuit breaker. If the breaker trips, terminates the stream. Returns the appropriate outcome for the caller.
|
|
510
|
+
* @param context - Description of the failure for logging.
|
|
511
|
+
* @returns The tab replacement outcome (failed or terminated).
|
|
512
|
+
*/
|
|
513
|
+
function handleTabReplacementFailure(context) {
|
|
514
|
+
const cbResult = checkCircuitBreaker(circuitBreaker, Date.now());
|
|
515
|
+
if (cbResult.shouldTrip) {
|
|
516
|
+
LOG.error("Circuit breaker tripped after %s. Stream appears fundamentally broken.", context);
|
|
517
|
+
clearInterval(interval);
|
|
518
|
+
onCircuitBreak();
|
|
519
|
+
return { outcome: "terminated" };
|
|
520
|
+
}
|
|
521
|
+
return { outcome: "failed" };
|
|
522
|
+
}
|
|
523
|
+
/**
|
|
524
|
+
* Applies successful tab replacement state. Updates page and context references, logs recovery duration, records metrics, and resets all failure/escalation state
|
|
525
|
+
* for the fresh tab. Consolidated here so the try and catch paths in executeTabReplacement share a single implementation.
|
|
526
|
+
* @param result - The successful tab replacement result containing the new page and context.
|
|
527
|
+
*/
|
|
528
|
+
function applyTabReplacementSuccess(result) {
|
|
529
|
+
currentPage = result.page;
|
|
530
|
+
currentContext = result.context;
|
|
531
|
+
const duration = formatRecoveryDuration(metrics.currentRecoveryStartTime ?? Date.now());
|
|
532
|
+
LOG.info("Recovered in %s via %s.", duration, RECOVERY_METHODS.tabReplacement);
|
|
533
|
+
recordRecoverySuccess(metrics, RECOVERY_METHODS.tabReplacement);
|
|
534
|
+
// Full state reset for fresh tab.
|
|
535
|
+
lastPageNavigationTime = Date.now();
|
|
536
|
+
resetRecoveryCounters();
|
|
537
|
+
resetEscalationState();
|
|
538
|
+
resetSegmentMonitoringState();
|
|
539
|
+
setRecoveryGracePeriod(3);
|
|
540
|
+
resetCircuitBreaker(circuitBreaker);
|
|
541
|
+
}
|
|
542
|
+
/**
|
|
543
|
+
* Handles tab replacement failure after all retry attempts are exhausted. Clears stale recovery metrics (preventing ghost "Recovered" logs from the
|
|
544
|
+
* deferred-success check), runs the circuit breaker, and detects zombie streams where the old page was destroyed but no replacement was created.
|
|
545
|
+
* @param context - Description of the failure for circuit breaker logging.
|
|
546
|
+
* @returns The tab replacement outcome (failed or terminated).
|
|
547
|
+
*/
|
|
548
|
+
function handleExhaustedTabReplacement(context) {
|
|
549
|
+
// Clear stale recovery metrics so the deferred-success check does not falsely log "Recovered" from leftover state set by recordRecoveryAttempt.
|
|
550
|
+
metrics.currentRecoveryStartTime = null;
|
|
551
|
+
metrics.currentRecoveryMethod = null;
|
|
552
|
+
LOG.warn("Tab replacement unsuccessful after retry.");
|
|
553
|
+
const failureOutcome = handleTabReplacementFailure(context);
|
|
554
|
+
// If the circuit breaker did not trip but the old page is gone (handler destroyed it before createPageWithCapture failed), the stream is unrecoverable. The
|
|
555
|
+
// next monitor tick would silently clear the interval via currentPage.isClosed() with no termination log, no status emission, and no cleanup — creating a
|
|
556
|
+
// zombie entry in the registry. Terminate explicitly instead.
|
|
557
|
+
if ((failureOutcome.outcome === "failed") && currentPage.isClosed()) {
|
|
558
|
+
LOG.error("Tab replacement failed and old page is closed. Stream is unrecoverable. Terminating stream.");
|
|
559
|
+
clearInterval(interval);
|
|
560
|
+
onCircuitBreak();
|
|
561
|
+
return { outcome: "terminated" };
|
|
562
|
+
}
|
|
563
|
+
return failureOutcome;
|
|
564
|
+
}
|
|
565
|
+
/**
|
|
566
|
+
* Executes tab replacement recovery with full error handling. This unified helper handles all tab replacement triggers (tiny segments, stalled capture, unresponsive
|
|
567
|
+
* tab) consistently, including metrics recording, success/failure logging, circuit breaker checks, and state resets.
|
|
568
|
+
*
|
|
569
|
+
* On failure, retries onTabReplacement once before giving up. The handler destroys old resources (capture, segmenter, FFmpeg, page) before calling
|
|
570
|
+
* createPageWithCapture, so a retry is the only chance to save the stream when the first attempt fails. All handler cleanup steps are idempotent on retry:
|
|
571
|
+
* rawCaptureStream.destroyed guard, segmenter stop() checks state.stopped, FFmpeg kill() checks ffmpeg.killed, page close checks !oldPage.isClosed(), and
|
|
572
|
+
* unregisterManagedPage is idempotent.
|
|
573
|
+
* @param issueType - Description of what triggered the replacement (for logging and UI display).
|
|
574
|
+
* @returns The tab replacement outcome.
|
|
575
|
+
*/
|
|
576
|
+
async function executeTabReplacement(issueType) {
|
|
577
|
+
// Guard: caller should ensure onTabReplacement exists, but TypeScript needs explicit narrowing.
|
|
578
|
+
if (!onTabReplacement) {
|
|
579
|
+
return { outcome: "failed" };
|
|
580
|
+
}
|
|
581
|
+
recoveryInProgress = true;
|
|
582
|
+
totalRecoveryAttempts++;
|
|
583
|
+
lastRecoveryTime = Date.now();
|
|
584
|
+
lastIssueType = issueType;
|
|
585
|
+
lastIssueTime = Date.now();
|
|
586
|
+
const tabRecoveryElapsed = startTimer();
|
|
587
|
+
recordRecoveryAttempt(metrics, RECOVERY_METHODS.tabReplacement);
|
|
588
|
+
try {
|
|
589
|
+
let result = await onTabReplacement();
|
|
590
|
+
// First attempt failed — retry once. See idempotency notes in the JSDoc above.
|
|
591
|
+
if (!result) {
|
|
592
|
+
LOG.debug("recovery:tab", "Tab replacement attempt 1/2 failed. Retrying...");
|
|
593
|
+
try {
|
|
594
|
+
result = await onTabReplacement();
|
|
595
|
+
}
|
|
596
|
+
catch (retryError) {
|
|
597
|
+
LOG.debug("recovery:tab", "Tab replacement attempt 2/2 failed: %s.", formatError(retryError));
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
if (result) {
|
|
601
|
+
applyTabReplacementSuccess(result);
|
|
602
|
+
return { outcome: "success" };
|
|
603
|
+
}
|
|
604
|
+
return handleExhaustedTabReplacement("tab replacement unsuccessful");
|
|
605
|
+
}
|
|
606
|
+
catch (error) {
|
|
607
|
+
// Unexpected error (not from onTabReplacement — those are caught internally by the handler in hls.ts and return null). Guard against registry corruption,
|
|
608
|
+
// getStream failures, or other unexpected errors.
|
|
609
|
+
LOG.debug("recovery:tab", "Tab replacement attempt 1/2 failed: %s. Retrying...", formatError(error));
|
|
610
|
+
try {
|
|
611
|
+
const retryResult = await onTabReplacement();
|
|
612
|
+
if (retryResult) {
|
|
613
|
+
applyTabReplacementSuccess(retryResult);
|
|
614
|
+
return { outcome: "success" };
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
catch (retryError) {
|
|
618
|
+
LOG.debug("recovery:tab", "Tab replacement attempt 2/2 failed: %s.", formatError(retryError));
|
|
619
|
+
}
|
|
620
|
+
return handleExhaustedTabReplacement("tab replacement error");
|
|
621
|
+
}
|
|
622
|
+
finally {
|
|
623
|
+
LOG.debug("timing:recovery", "Tab replacement completed. Total: %sms.", tabRecoveryElapsed());
|
|
624
|
+
finalizeTabReplacement();
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
/**
|
|
628
|
+
* Performs page navigation recovery with validation. This is the single recovery function used by both the "video not found" and "escalation level 4" code paths,
|
|
629
|
+
* ensuring consistent behavior. The function:
|
|
630
|
+
* 1. Calls tuneToChannel to reinitialize playback
|
|
631
|
+
* 2. Checks for unexpected new tabs
|
|
632
|
+
* 3. Validates the page URL
|
|
633
|
+
* 4. Validates the video element exists and is accessible
|
|
634
|
+
* 5. Only returns success if all validations pass
|
|
635
|
+
* @returns Recovery result with the new context if successful.
|
|
636
|
+
*/
|
|
637
|
+
async function performPageNavigationRecovery() {
|
|
638
|
+
const navRecoveryElapsed = startTimer();
|
|
639
|
+
// Track page count before navigation to detect unexpected new tabs (popups, ads).
|
|
640
|
+
const browser = currentPage.browser();
|
|
641
|
+
const pageCountBefore = (await browser.pages()).length;
|
|
642
|
+
try {
|
|
643
|
+
// Use tuneToChannel to reinitialize playback. This is the single source of truth for channel initialization, ensuring recovery uses the exact same sequence
|
|
644
|
+
// as initial setup (navigation, channel selection, video detection, click-to-play, playback).
|
|
645
|
+
const { context: newContext } = await tuneToChannel(currentPage, url, profile);
|
|
646
|
+
// Check for unexpected new tabs created during tuning.
|
|
647
|
+
const pageCountAfter = (await browser.pages()).length;
|
|
648
|
+
if (pageCountAfter > pageCountBefore) {
|
|
649
|
+
LOG.debug("recovery:nav", "Detected %s new tab(s) created during navigation.", pageCountAfter - pageCountBefore);
|
|
650
|
+
}
|
|
651
|
+
// Validate that we're on the expected page.
|
|
652
|
+
const currentUrl = currentPage.url();
|
|
653
|
+
const expectedHostname = new URL(url).hostname;
|
|
654
|
+
if (!currentUrl.includes(expectedHostname)) {
|
|
655
|
+
LOG.debug("recovery:nav", "Page URL after navigation (%s) does not match expected hostname.", currentUrl);
|
|
656
|
+
}
|
|
657
|
+
// Validate that the video element is accessible and has reasonable state.
|
|
658
|
+
const validationState = await validateVideoElement(newContext, selectorType);
|
|
659
|
+
if (validationState.found) {
|
|
660
|
+
LOG.debug("timing:recovery", "Page navigation recovery succeeded. Total: %sms.", navRecoveryElapsed());
|
|
661
|
+
return { newContext, success: true };
|
|
662
|
+
}
|
|
663
|
+
LOG.warn("Page navigation completed but video element not found in new context.");
|
|
664
|
+
LOG.debug("timing:recovery", "Page navigation recovery failed (no video). Total: %sms.", navRecoveryElapsed());
|
|
665
|
+
return { success: false };
|
|
666
|
+
}
|
|
667
|
+
catch (error) {
|
|
668
|
+
LOG.warn("Failed to reinitialize video after page navigation: %s.", formatError(error));
|
|
669
|
+
LOG.debug("timing:recovery", "Page navigation recovery failed (error). Total: %sms.", navRecoveryElapsed());
|
|
670
|
+
return { success: false };
|
|
671
|
+
}
|
|
672
|
+
}
|
|
673
|
+
/* Main monitoring interval. This runs every MONITOR_INTERVAL milliseconds to check video state and trigger recovery when needed.
|
|
674
|
+
*
|
|
675
|
+
* IMPORTANT: Early returns must call emitStatusUpdate() before returning (except when the stream is terminating, e.g., page closed or circuit breaker tripped). This
|
|
676
|
+
* ensures SSE clients always have current status data (duration, memory, health) even during recovery, buffering, or video search periods. Without this, the
|
|
677
|
+
* streamStatuses map becomes stale and new SSE connections receive outdated snapshots.
|
|
678
|
+
*
|
|
679
|
+
* CHECK ORDER MATTERS: The recoveryInProgress check must come BEFORE the currentPage.isClosed() check. During tab replacement, the old page is intentionally closed
|
|
680
|
+
* while the handler creates a new page. If we check isClosed() first, we would terminate the interval while recovery is still in progress, causing status updates to
|
|
681
|
+
* stop permanently. The sequence is: (1) intervalCleared for explicit cleanup, (2) recoveryInProgress to continue during recovery, (3) isClosed() for unexpected
|
|
682
|
+
* page termination outside of recovery.
|
|
683
|
+
*/
|
|
684
|
+
const interval = setInterval(() => {
|
|
685
|
+
// Stop monitoring if cleanup was requested.
|
|
686
|
+
if (intervalCleared) {
|
|
687
|
+
clearInterval(interval);
|
|
688
|
+
return;
|
|
689
|
+
}
|
|
690
|
+
// Skip health checks if a recovery operation is in progress. During tab replacement, the old page will be closed but we must keep the interval running until the
|
|
691
|
+
// new page is assigned. Emit status so SSE clients see current state (health, duration, memory) even during recovery.
|
|
692
|
+
if (recoveryInProgress) {
|
|
693
|
+
emitStatusUpdate();
|
|
694
|
+
return;
|
|
695
|
+
}
|
|
696
|
+
// Stop monitoring if the page was closed outside of recovery. This handles cases like browser disconnect or explicit stream termination.
|
|
697
|
+
if (currentPage.isClosed()) {
|
|
698
|
+
clearInterval(interval);
|
|
699
|
+
return;
|
|
700
|
+
}
|
|
701
|
+
// Re-establish stream context for this interval tick. AsyncLocalStorage context is lost when entering setInterval callbacks.
|
|
702
|
+
runWithStreamContext(streamContext, async () => {
|
|
703
|
+
try {
|
|
704
|
+
// Early exit if the stream's abort signal has been triggered. This prevents wasted work when the stream is being terminated.
|
|
705
|
+
const abortSignal = getAbortSignal(streamId);
|
|
706
|
+
if (abortSignal?.aborted) {
|
|
707
|
+
clearInterval(interval);
|
|
708
|
+
return;
|
|
709
|
+
}
|
|
710
|
+
// Capture current timestamp for all timing calculations in this check cycle.
|
|
711
|
+
const now = Date.now();
|
|
712
|
+
// Gather current video state for analysis. The getVideoState helper encapsulates video element selection and returns all properties needed for health analysis.
|
|
713
|
+
// We catch frame detachment errors specifically to handle context invalidation differently from normal "video not found" cases.
|
|
714
|
+
let stateInfo = null;
|
|
715
|
+
let contextInvalidated = false;
|
|
716
|
+
try {
|
|
717
|
+
stateInfo = await getVideoState(currentContext, selectorType);
|
|
718
|
+
}
|
|
719
|
+
catch (stateError) {
|
|
720
|
+
// Check for execution context destroyed errors, which indicate the frame was detached.
|
|
721
|
+
const errorMessage = formatError(stateError);
|
|
722
|
+
const isContextDestroyed = ["context", "destroyed", "detached", "target closed"].some((term) => errorMessage.toLowerCase().includes(term));
|
|
723
|
+
if (isContextDestroyed) {
|
|
724
|
+
LOG.debug("recovery:context", "Video context was invalidated (frame detached). Will re-search for video.");
|
|
725
|
+
contextInvalidated = true;
|
|
726
|
+
}
|
|
727
|
+
else {
|
|
728
|
+
// Other errors should be propagated.
|
|
729
|
+
throw stateError;
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
// Map to the VideoState type used by the monitor (includes 'time' alias for currentTime).
|
|
733
|
+
const state = stateInfo ? { ...stateInfo, time: stateInfo.currentTime } : null;
|
|
734
|
+
// If context was invalidated (frame detached), immediately try to find the video in a new context.
|
|
735
|
+
if (contextInvalidated) {
|
|
736
|
+
LOG.debug("recovery:context", "Re-searching for video after context invalidation.");
|
|
737
|
+
try {
|
|
738
|
+
const newContext = await findVideoContext(currentPage, profile);
|
|
739
|
+
const validationState = await validateVideoElement(newContext, selectorType);
|
|
740
|
+
if (validationState.found) {
|
|
741
|
+
LOG.info("Video found in new context after detachment. readyState=%s.", validationState.readyState);
|
|
742
|
+
currentContext = newContext;
|
|
743
|
+
videoNotFoundCount = 0;
|
|
744
|
+
// Emit status so SSE clients stay current even when returning early after context re-search.
|
|
745
|
+
emitStatusUpdate();
|
|
746
|
+
return;
|
|
747
|
+
}
|
|
748
|
+
}
|
|
749
|
+
catch (searchError) {
|
|
750
|
+
LOG.warn("Context re-search after detachment failed: %s.", formatError(searchError));
|
|
751
|
+
}
|
|
752
|
+
// If re-search failed, let the normal "video not found" logic handle it.
|
|
753
|
+
}
|
|
754
|
+
/* If no video element found, apply a grace period before triggering recovery. The video may be temporarily unavailable due to:
|
|
755
|
+
* - readyState fluctuations (selectReadyVideo finds no video with readyState >= 3)
|
|
756
|
+
* - Frame detachment/reattachment during page updates
|
|
757
|
+
* - Momentary DOM changes during ad transitions
|
|
758
|
+
*
|
|
759
|
+
* We distinguish between "no video element exists" and "video exists but not ready". The latter is treated as buffering and given more time.
|
|
760
|
+
*/
|
|
761
|
+
if (!state) {
|
|
762
|
+
// Determine context type for diagnostic logging.
|
|
763
|
+
const contextType = currentContext === currentPage ? "main page" : "iframe";
|
|
764
|
+
const frameCount = currentPage.frames().length;
|
|
765
|
+
// Check video presence to distinguish between "no video" and "video exists but not ready".
|
|
766
|
+
let presence = null;
|
|
767
|
+
try {
|
|
768
|
+
presence = await checkVideoPresence(currentContext, selectorType);
|
|
769
|
+
}
|
|
770
|
+
catch (_error) {
|
|
771
|
+
// If presence check fails (context destroyed), treat as no video.
|
|
772
|
+
}
|
|
773
|
+
if (presence?.anyVideoExists && !presence.readyVideoFound) {
|
|
774
|
+
// Video element exists but doesn't meet readyState criteria. This is a buffering condition, not a missing video condition.
|
|
775
|
+
// Apply the normal buffering grace period instead of escalating to navigation.
|
|
776
|
+
LOG.info("Video element exists but not ready (count=%s, maxReadyState=%s). Treating as buffering.", presence.videoCount, presence.maxReadyState);
|
|
777
|
+
// Reset video not found counter since video actually exists.
|
|
778
|
+
videoNotFoundCount = 0;
|
|
779
|
+
// Emit status so SSE clients see current state even during this buffering condition.
|
|
780
|
+
emitStatusUpdate();
|
|
781
|
+
// Let the normal buffering detection handle this on subsequent checks.
|
|
782
|
+
return;
|
|
783
|
+
}
|
|
784
|
+
videoNotFoundCount++;
|
|
785
|
+
LOG.warn("Video element not found (attempt %s/3). Context: %s, frames: %s, videoCount: %s.", videoNotFoundCount, contextType, frameCount, presence?.videoCount ?? 0);
|
|
786
|
+
// Grace period: Wait for 2 consecutive failures before attempting context re-search, 3 before full navigation.
|
|
787
|
+
if (videoNotFoundCount < 2) {
|
|
788
|
+
// First failure - just log and wait for next check. Emit status so SSE clients stay current.
|
|
789
|
+
emitStatusUpdate();
|
|
790
|
+
return;
|
|
791
|
+
}
|
|
792
|
+
// After 2+ failures, try re-searching frames to see if video moved to a different context.
|
|
793
|
+
if (videoNotFoundCount === 2) {
|
|
794
|
+
LOG.debug("recovery:context", "Re-searching frames for video element.");
|
|
795
|
+
try {
|
|
796
|
+
const newContext = await findVideoContext(currentPage, profile);
|
|
797
|
+
const validationState = await validateVideoElement(newContext, selectorType);
|
|
798
|
+
if (validationState.found) {
|
|
799
|
+
LOG.info("Video found in different context after re-search. readyState=%s.", validationState.readyState);
|
|
800
|
+
currentContext = newContext;
|
|
801
|
+
videoNotFoundCount = 0;
|
|
802
|
+
// Emit status so SSE clients see current state.
|
|
803
|
+
emitStatusUpdate();
|
|
804
|
+
return;
|
|
805
|
+
}
|
|
806
|
+
LOG.warn("Re-search did not find video in any frame.");
|
|
807
|
+
}
|
|
808
|
+
catch (error) {
|
|
809
|
+
LOG.warn("Frame re-search failed: %s.", formatError(error));
|
|
810
|
+
}
|
|
811
|
+
// Emit status so SSE clients stay current during video search.
|
|
812
|
+
emitStatusUpdate();
|
|
813
|
+
return;
|
|
814
|
+
}
|
|
815
|
+
// After 3+ consecutive failures, escalate to full page navigation recovery.
|
|
816
|
+
LOG.warn("Video element not found. Attempting %s...", RECOVERY_METHODS.pageNavigation);
|
|
817
|
+
// Check circuit breaker for too many failures.
|
|
818
|
+
const cbResult = checkCircuitBreaker(circuitBreaker, now);
|
|
819
|
+
if (cbResult.shouldTrip) {
|
|
820
|
+
LOG.error("Circuit breaker tripped after %s failures. Stream appears fundamentally broken.", cbResult.totalCount);
|
|
821
|
+
clearInterval(interval);
|
|
822
|
+
onCircuitBreak();
|
|
823
|
+
return;
|
|
824
|
+
}
|
|
825
|
+
// Set escalation to level 3 to trigger page navigation. We skip lower levels since they require a video element.
|
|
826
|
+
// Note: Keep state updates in sync with the main recovery path in the needsRecovery block below.
|
|
827
|
+
escalationLevel = 3;
|
|
828
|
+
lastRecoveryTime = now;
|
|
829
|
+
totalRecoveryAttempts++;
|
|
830
|
+
pendingReMinimize = true;
|
|
831
|
+
recoveryInProgress = true;
|
|
832
|
+
recordRecoveryAttempt(metrics, RECOVERY_METHODS.pageNavigation);
|
|
833
|
+
// Check page reload limit before attempting recovery.
|
|
834
|
+
const reloadWindow = now - CONFIG.playback.pageReloadWindow;
|
|
835
|
+
pageReloadTimestamps = pageReloadTimestamps.filter((ts) => ts > reloadWindow);
|
|
836
|
+
if (pageReloadTimestamps.length >= CONFIG.playback.maxPageReloads) {
|
|
837
|
+
LOG.error("Exceeded maximum page navigations (%s in %s minutes). Cannot recover without video element.", CONFIG.playback.maxPageReloads, Math.round(CONFIG.playback.pageReloadWindow / 60000));
|
|
838
|
+
clearInterval(interval);
|
|
839
|
+
onCircuitBreak();
|
|
840
|
+
return;
|
|
841
|
+
}
|
|
842
|
+
pageReloadTimestamps.push(now);
|
|
843
|
+
// Use the unified recovery function with validation.
|
|
844
|
+
const recoveryResult = await performPageNavigationRecovery();
|
|
845
|
+
// Page navigation disrupted the video stream. Mark a discontinuity regardless of navigation success so HLS clients resynchronize their decoders.
|
|
846
|
+
markStreamDiscontinuity();
|
|
847
|
+
// Set grace period to give page navigation time to take effect (L3 = 10 seconds).
|
|
848
|
+
recoveryGraceUntil = now + recoveryGracePeriods[3];
|
|
849
|
+
if (recoveryResult.success && recoveryResult.newContext) {
|
|
850
|
+
// Update the context reference for subsequent monitor checks (only after validation succeeds).
|
|
851
|
+
currentContext = recoveryResult.newContext;
|
|
852
|
+
// Log success with timing.
|
|
853
|
+
const duration = formatRecoveryDuration(metrics.currentRecoveryStartTime ?? now);
|
|
854
|
+
LOG.info("Recovered in %s via %s.", duration, RECOVERY_METHODS.pageNavigation);
|
|
855
|
+
recordRecoverySuccess(metrics, RECOVERY_METHODS.pageNavigation);
|
|
856
|
+
// Reset state after successful page navigation recovery.
|
|
857
|
+
lastPageNavigationTime = Date.now();
|
|
858
|
+
resetRecoveryCounters();
|
|
859
|
+
resetEscalationState();
|
|
860
|
+
resetSegmentMonitoringState();
|
|
861
|
+
}
|
|
862
|
+
else {
|
|
863
|
+
consecutiveNavigationFailures++;
|
|
864
|
+
LOG.warn("Page navigation unsuccessful.");
|
|
865
|
+
}
|
|
866
|
+
recoveryInProgress = false;
|
|
867
|
+
// Emit status so SSE clients see the recovery result.
|
|
868
|
+
emitStatusUpdate();
|
|
869
|
+
return;
|
|
870
|
+
}
|
|
871
|
+
// Video was found - reset the not found counter, timeout counter, and save state for status reporting.
|
|
872
|
+
videoNotFoundCount = 0;
|
|
873
|
+
consecutiveTimeouts = 0;
|
|
874
|
+
lastVideoState = state;
|
|
875
|
+
/* Volume enforcement. Some sites aggressively mute videos (e.g., France24 mutes on page visibility change, some sites mute for ads). We restore volume on
|
|
876
|
+
* every check to ensure audio is captured.
|
|
877
|
+
*/
|
|
878
|
+
if (state.muted || (state.volume < 1)) {
|
|
879
|
+
await enforceVideoVolume(currentContext, selectorType);
|
|
880
|
+
}
|
|
881
|
+
/* Stall detection. We compare currentTime to the previous check to determine if the video is progressing. The STALL_THRESHOLD (0.1 seconds) allows for minor
|
|
882
|
+
* timing variations while still detecting genuinely stalled videos.
|
|
883
|
+
*/
|
|
884
|
+
// Video is progressing if: this is the first check (no previous time), OR currentTime has advanced by at least STALL_THRESHOLD since last check.
|
|
885
|
+
const isProgressing = (lastTime === null) || (Math.abs(state.time - lastTime) >= CONFIG.playback.stallThreshold);
|
|
886
|
+
/* Buffering detection. True buffering occurs when the player needs more data (readyState < 3) AND is actively fetching it (networkState === 2). We use AND
|
|
887
|
+
* rather than OR because networkState === 2 is normal for live streams - data continuously arrives. Only when combined with insufficient data does it indicate
|
|
888
|
+
* actual buffering.
|
|
889
|
+
*/
|
|
890
|
+
const isBuffering = (state.readyState < 3) && (state.networkState === 2);
|
|
891
|
+
/* Buffering grace period tracking. When buffering starts, we record the timestamp. We only trigger recovery if buffering exceeds BUFFERING_GRACE_PERIOD. This
|
|
892
|
+
* allows normal network buffering to resolve without intervention.
|
|
893
|
+
*/
|
|
894
|
+
if (isBuffering && !bufferingStartTime) {
|
|
895
|
+
bufferingStartTime = now;
|
|
896
|
+
}
|
|
897
|
+
else if (!isBuffering) {
|
|
898
|
+
bufferingStartTime = null;
|
|
899
|
+
}
|
|
900
|
+
// Check if we're within the buffering grace period (recently started buffering and haven't exceeded the threshold).
|
|
901
|
+
const withinBufferingGrace = isBuffering && bufferingStartTime && ((now - bufferingStartTime) < CONFIG.playback.bufferingGracePeriod);
|
|
902
|
+
// Check if we're within the recovery grace period (recently performed a recovery action and waiting for it to take effect).
|
|
903
|
+
const withinRecoveryGrace = now < recoveryGraceUntil;
|
|
904
|
+
/* Segment production monitoring. After L2/L3 recovery completes (grace period ends), we verify that segments are actually being produced. If recovery reported
|
|
905
|
+
* success but the capture pipeline is dead (MediaRecorder stopped producing data, FFmpeg stdin idle), segments will stop flowing while the video element
|
|
906
|
+
* appears healthy. This catches the 20+ minute freeze bug where PrismCast reports "Recovered" but Channels DVR receives no data.
|
|
907
|
+
*/
|
|
908
|
+
if ((preRecoverySegmentIndex !== null) && !withinRecoveryGrace) {
|
|
909
|
+
// Start the segment wait timer when recovery grace period ends.
|
|
910
|
+
segmentWaitStartTime ??= now;
|
|
911
|
+
// Check if segments are flowing by comparing current index to pre-recovery index.
|
|
912
|
+
const entry = getStream(streamInfo.numericStreamId);
|
|
913
|
+
const currentSegmentIndex = entry?.segmenter?.getSegmentIndex() ?? null;
|
|
914
|
+
if ((currentSegmentIndex !== null) && (currentSegmentIndex > preRecoverySegmentIndex)) {
|
|
915
|
+
// Segments are flowing - recovery actually succeeded. Clear tracking state.
|
|
916
|
+
preRecoverySegmentIndex = null;
|
|
917
|
+
segmentWaitStartTime = null;
|
|
918
|
+
segmentProductionStalled = false;
|
|
919
|
+
}
|
|
920
|
+
else if ((now - segmentWaitStartTime) > SEGMENT_STALL_TIMEOUT) {
|
|
921
|
+
// No new segments for SEGMENT_STALL_TIMEOUT after recovery grace period. The capture pipeline is dead.
|
|
922
|
+
LOG.warn("No segments produced for %ss after recovery. Capture pipeline appears dead.", SEGMENT_STALL_TIMEOUT / 1000);
|
|
923
|
+
segmentProductionStalled = true;
|
|
924
|
+
}
|
|
925
|
+
}
|
|
926
|
+
/* Continuous segment size monitoring. Runs on every healthy interval to detect spontaneous capture pipeline death (no preceding recovery event). Dead pipelines
|
|
927
|
+
* produce tiny segments (18 bytes observed) while the video element appears healthy. This catches failures that post-recovery index monitoring misses because
|
|
928
|
+
* there's no recovery to trigger monitoring.
|
|
929
|
+
*/
|
|
930
|
+
const sizeCheckEntry = getStream(streamInfo.numericStreamId);
|
|
931
|
+
const currentSegmentIndex = sizeCheckEntry?.segmenter?.getSegmentIndex() ?? 0;
|
|
932
|
+
if ((currentSegmentIndex > lastCheckedSegmentIndex) && sizeCheckEntry) {
|
|
933
|
+
// A new segment was produced. Check its size.
|
|
934
|
+
const segmentSize = getLastSegmentSize(sizeCheckEntry) ?? 0;
|
|
935
|
+
if (segmentSize < TINY_SEGMENT_THRESHOLD) {
|
|
936
|
+
consecutiveTinySegments++;
|
|
937
|
+
wasInTinySegmentState = true;
|
|
938
|
+
if (consecutiveTinySegments >= TINY_SEGMENT_COUNT_TRIGGER) {
|
|
939
|
+
LOG.warn("Detected %d consecutive tiny segments (%d bytes). Capture pipeline appears dead.", consecutiveTinySegments, segmentSize);
|
|
940
|
+
// Trigger tab replacement if available, otherwise let circuit breaker handle it via segmentProductionStalled. Return unconditionally after tab
|
|
941
|
+
// replacement (matching stalled-capture and unresponsive-tab triggers) to avoid falling through the rest of the tick with stale pre-replacement state.
|
|
942
|
+
if (onTabReplacement && !recoveryInProgress) {
|
|
943
|
+
await executeTabReplacement("tiny segments");
|
|
944
|
+
return;
|
|
945
|
+
}
|
|
946
|
+
else if (!onTabReplacement) {
|
|
947
|
+
// No tab replacement callback - set stalled flag for circuit breaker.
|
|
948
|
+
segmentProductionStalled = true;
|
|
949
|
+
}
|
|
950
|
+
}
|
|
951
|
+
}
|
|
952
|
+
else {
|
|
953
|
+
// Valid segment size. Check for spontaneous recovery from tiny segment state. We don't mark discontinuity here - only tab replacement marks discontinuity.
|
|
954
|
+
// Self-healing may be transient and not require decoder reset.
|
|
955
|
+
if (wasInTinySegmentState) {
|
|
956
|
+
LOG.debug("recovery:segments", "Segment production self-healed (%d bytes).", segmentSize);
|
|
957
|
+
}
|
|
958
|
+
// Reset tiny segment tracking.
|
|
959
|
+
consecutiveTinySegments = 0;
|
|
960
|
+
wasInTinySegmentState = false;
|
|
961
|
+
}
|
|
962
|
+
lastCheckedSegmentIndex = currentSegmentIndex;
|
|
963
|
+
}
|
|
964
|
+
/* Re-minimize check. After recovery, the browser window may have been un-minimized by fullscreen actions. As soon as the stream is healthy (progressing without
|
|
965
|
+
* issues), we re-minimize to reduce GPU usage.
|
|
966
|
+
*/
|
|
967
|
+
if (pendingReMinimize && isProgressing && !state.paused && !state.error && !state.ended) {
|
|
968
|
+
LOG.debug("recovery", "Re-minimizing browser window after successful recovery.");
|
|
969
|
+
pendingReMinimize = false;
|
|
970
|
+
await resizeAndMinimizeWindow(currentPage, true);
|
|
971
|
+
}
|
|
972
|
+
/* Fullscreen reinforcement. Some streaming sites (notably Hulu) revert the video to a mini-player or PiP layout in response to browser state changes such as
|
|
973
|
+
* window minimization or visibility events. Because the video continues playing normally in the smaller frame, no existing recovery condition is triggered — the
|
|
974
|
+
* health monitor sees healthy, progressing playback while the captured frame shows a small video in the corner of the viewport. We verify that the video fills
|
|
975
|
+
* the viewport on every healthy tick and re-apply CSS fullscreen styling when it shrinks. The response is graduated: basic CSS first (sufficient for
|
|
976
|
+
* well-behaved sites like Hulu), escalating to !important priority only if basic styles don't hold by the next tick. The readyState guard prevents false
|
|
977
|
+
* positives during momentary readyState dips where verifyFullscreen() cannot find a ready video even though the video layout has not changed. A null return
|
|
978
|
+
* from verifyFullscreen() indicates the check was inconclusive (e.g. context destroyed) and is ignored.
|
|
979
|
+
*/
|
|
980
|
+
if (isProgressing && !state.paused && !state.error && !state.ended && !withinRecoveryGrace && (state.readyState >= 3)) {
|
|
981
|
+
const isFullscreen = await verifyFullscreen(currentContext, selectorType);
|
|
982
|
+
if (isFullscreen === false) {
|
|
983
|
+
fullscreenReapplyCount++;
|
|
984
|
+
// Graduated escalation: first attempt uses basic CSS (sufficient for well-behaved sites like Hulu that only need a nudge). If the basic styles
|
|
985
|
+
// didn't hold by the next tick, escalate to !important priority to override sites that actively fight style changes.
|
|
986
|
+
const useImportant = fullscreenReapplyCount > 1;
|
|
987
|
+
if (fullscreenReapplyCount === 1) {
|
|
988
|
+
LOG.info("Video no longer fills viewport. Re-applying fullscreen styling.");
|
|
989
|
+
}
|
|
990
|
+
else if (fullscreenReapplyCount === 2) {
|
|
991
|
+
LOG.info("Basic fullscreen styling did not hold. Escalating to !important priority.");
|
|
992
|
+
}
|
|
993
|
+
await applyVideoStyles(currentContext, selectorType, useImportant);
|
|
994
|
+
}
|
|
995
|
+
else if (isFullscreen && (fullscreenReapplyCount > 0)) {
|
|
996
|
+
LOG.info("Video fullscreen restored.");
|
|
997
|
+
fullscreenReapplyCount = 0;
|
|
998
|
+
}
|
|
999
|
+
}
|
|
1000
|
+
/* Stall counter management. We increment stallCount when the video is not progressing and not within buffering grace. We reset to 0 when progression resumes.
|
|
1001
|
+
* This hysteresis prevents reacting to single-frame hiccups.
|
|
1002
|
+
*/
|
|
1003
|
+
if (!isProgressing && !withinBufferingGrace) {
|
|
1004
|
+
stallCount++;
|
|
1005
|
+
}
|
|
1006
|
+
else if (isProgressing) {
|
|
1007
|
+
stallCount = 0;
|
|
1008
|
+
}
|
|
1009
|
+
/* Pause counter management. We increment pauseCount when video.paused is true and reset when it clears. This provides the same hysteresis as stall detection,
|
|
1010
|
+
* filtering out transient rebuffer pauses (where the player briefly pauses to refill its buffer) while still catching genuine persistent pauses.
|
|
1011
|
+
*/
|
|
1012
|
+
if (state.paused) {
|
|
1013
|
+
pauseCount++;
|
|
1014
|
+
}
|
|
1015
|
+
else {
|
|
1016
|
+
pauseCount = 0;
|
|
1017
|
+
}
|
|
1018
|
+
/* Recovery decision. We trigger recovery when any of these conditions are met AND we're not within the recovery grace period:
|
|
1019
|
+
* - Video has an error state
|
|
1020
|
+
* - Video ended (live streams shouldn't end)
|
|
1021
|
+
* - Video is paused persistently (pauseCount exceeds threshold and not just buffering)
|
|
1022
|
+
* - Video is stalled for too long (stallCount exceeds threshold and not in buffering grace)
|
|
1023
|
+
* - Segment production has stalled after recovery (capture pipeline dead)
|
|
1024
|
+
*/
|
|
1025
|
+
const needsRecovery = !withinRecoveryGrace && (state.error || state.ended ||
|
|
1026
|
+
(state.paused && !withinBufferingGrace && (pauseCount > CONFIG.playback.stallCountThreshold)) ||
|
|
1027
|
+
(!isProgressing && !withinBufferingGrace && (stallCount > CONFIG.playback.stallCountThreshold)) ||
|
|
1028
|
+
segmentProductionStalled);
|
|
1029
|
+
/* Escalation reset. After sustained healthy playback (SUSTAINED_PLAYBACK_REQUIRED, default 60 seconds), we reset the escalation level and circuit breaker.
|
|
1030
|
+
* This allows a stream that recovered to start fresh, rather than immediately escalating to aggressive recovery on the next issue.
|
|
1031
|
+
*/
|
|
1032
|
+
if (isProgressing && !state.paused && !state.ended && !state.error) {
|
|
1033
|
+
// If a recovery was pending confirmation (L1/L2), log success now that we have healthy playback.
|
|
1034
|
+
if ((metrics.currentRecoveryStartTime !== null) && (metrics.currentRecoveryMethod !== null)) {
|
|
1035
|
+
const duration = formatRecoveryDuration(metrics.currentRecoveryStartTime);
|
|
1036
|
+
LOG.info("Recovered in %s via %s.", duration, metrics.currentRecoveryMethod);
|
|
1037
|
+
recordRecoverySuccess(metrics, metrics.currentRecoveryMethod);
|
|
1038
|
+
}
|
|
1039
|
+
const healthyDuration = now - lastRecoveryTime;
|
|
1040
|
+
if ((escalationLevel > 0) && (healthyDuration > CONFIG.playback.sustainedPlaybackRequired)) {
|
|
1041
|
+
// Clear buffering state. The bufferingStartTime may persist through recovery cycles due to networkState === 2 (NETWORK_LOADING) being true for live streams
|
|
1042
|
+
// even during healthy playback. Since we have confirmed 60 seconds of progression, the stream is definitively not buffering.
|
|
1043
|
+
bufferingStartTime = null;
|
|
1044
|
+
// Reset escalation, segment tracking, and circuit breaker. Sustained healthy playback confirms the stream works.
|
|
1045
|
+
resetEscalationState();
|
|
1046
|
+
resetSegmentMonitoringState();
|
|
1047
|
+
resetCircuitBreaker(circuitBreaker);
|
|
1048
|
+
}
|
|
1049
|
+
}
|
|
1050
|
+
/* Proactive page reload. Some streaming sites enforce a maximum continuous playback duration (e.g., NBC.com cuts streams after 4 hours). When a domain
|
|
1051
|
+
* configures maxContinuousPlayback, we proactively reload the page before the site's limit expires to maintain uninterrupted streaming. The reload triggers
|
|
1052
|
+
* PROACTIVE_RELOAD_MARGIN_MS (2 minutes) before the configured limit, giving enough time for page navigation and video reinitialization.
|
|
1053
|
+
*
|
|
1054
|
+
* This check runs only when playback is healthy (escalationLevel === 0), not within a recovery grace period, and progressing normally. If recovery is already
|
|
1055
|
+
* in progress, the ongoing recovery will eventually perform a page navigation if needed. The page reload rate limit is also checked to avoid consuming reload
|
|
1056
|
+
* budget that error recovery needs. The timer resets after any successful full page navigation (proactive or recovery-triggered).
|
|
1057
|
+
*/
|
|
1058
|
+
if ((profile.maxContinuousPlayback !== null) && (escalationLevel === 0) && !withinRecoveryGrace && isProgressing && !state.paused && !state.error &&
|
|
1059
|
+
!state.ended) {
|
|
1060
|
+
const maxPlaybackMs = profile.maxContinuousPlayback * 3600000;
|
|
1061
|
+
const elapsedMs = now - lastPageNavigationTime;
|
|
1062
|
+
if (elapsedMs >= (maxPlaybackMs - PROACTIVE_RELOAD_MARGIN_MS)) {
|
|
1063
|
+
const elapsedHours = (elapsedMs / 3600000).toFixed(1);
|
|
1064
|
+
LOG.info("Proactive reload after %sh of continuous playback (site limit: %sh). Reloading page to prevent stream cutoff.", elapsedHours, String(profile.maxContinuousPlayback));
|
|
1065
|
+
recoveryInProgress = true;
|
|
1066
|
+
// Check page reload rate limit before attempting. Proactive reload is best-effort maintenance — if the reload budget is exhausted from recent error
|
|
1067
|
+
// recoveries, we gracefully yield. If the site eventually cuts the stream, normal error recovery handles it.
|
|
1068
|
+
const reloadWindow = now - CONFIG.playback.pageReloadWindow;
|
|
1069
|
+
pageReloadTimestamps = pageReloadTimestamps.filter((ts) => ts > reloadWindow);
|
|
1070
|
+
if (pageReloadTimestamps.length >= CONFIG.playback.maxPageReloads) {
|
|
1071
|
+
LOG.warn("Proactive reload deferred — page navigation rate limit reached (%s in %s minutes).", CONFIG.playback.maxPageReloads, Math.round(CONFIG.playback.pageReloadWindow / 60000));
|
|
1072
|
+
// Set a grace period to prevent this deferral from re-triggering every 2 seconds while the rate limit remains in effect. The 10-second L3 grace
|
|
1073
|
+
// period spaces out re-checks, and the rate-limit window (default 15 minutes) will eventually expire old timestamps to allow the proactive reload. We
|
|
1074
|
+
// set recoveryGraceUntil directly rather than calling setRecoveryGracePeriod() because no recovery action was performed — the window state is unchanged
|
|
1075
|
+
// and pendingReMinimize should not be set.
|
|
1076
|
+
recoveryGraceUntil = now + recoveryGracePeriods[3];
|
|
1077
|
+
recoveryInProgress = false;
|
|
1078
|
+
emitStatusUpdate();
|
|
1079
|
+
return;
|
|
1080
|
+
}
|
|
1081
|
+
pageReloadTimestamps.push(now);
|
|
1082
|
+
const recoveryResult = await performPageNavigationRecovery();
|
|
1083
|
+
// Page navigation disrupted the video stream. Mark a discontinuity so HLS clients resynchronize their decoders.
|
|
1084
|
+
markStreamDiscontinuity();
|
|
1085
|
+
setRecoveryGracePeriod(3);
|
|
1086
|
+
if (recoveryResult.success && recoveryResult.newContext) {
|
|
1087
|
+
currentContext = recoveryResult.newContext;
|
|
1088
|
+
lastPageNavigationTime = Date.now();
|
|
1089
|
+
LOG.info("Proactive reload completed successfully.");
|
|
1090
|
+
resetRecoveryCounters();
|
|
1091
|
+
resetSegmentMonitoringState();
|
|
1092
|
+
}
|
|
1093
|
+
else {
|
|
1094
|
+
LOG.warn("Proactive reload unsuccessful. Will retry after recovery grace period.");
|
|
1095
|
+
}
|
|
1096
|
+
recoveryInProgress = false;
|
|
1097
|
+
emitStatusUpdate();
|
|
1098
|
+
return;
|
|
1099
|
+
}
|
|
1100
|
+
}
|
|
1101
|
+
/* Recovery execution. When recovery is needed, we update circuit breaker state, determine the appropriate recovery level based on issue type and history, and
|
|
1102
|
+
* execute the recovery action. The recovery system is issue-aware:
|
|
1103
|
+
* - Paused issues try L1 (play/unmute) first since it works ~50% of the time for paused state
|
|
1104
|
+
* - Buffering issues skip L1 and go directly to L2 (source reload) since L1 never helps buffering
|
|
1105
|
+
* - If L2 has already been attempted, skip to L3 (page reload) since a second L2 always fails
|
|
1106
|
+
*/
|
|
1107
|
+
if (needsRecovery) {
|
|
1108
|
+
/* Segment production stall handling. When we detect that segments stopped flowing after L2/L3 recovery, the capture pipeline is dead and normal recovery
|
|
1109
|
+
* won't help. We skip the escalation ladder and go directly to tab replacement if available.
|
|
1110
|
+
*/
|
|
1111
|
+
if (segmentProductionStalled && onTabReplacement) {
|
|
1112
|
+
LOG.warn("Capture pipeline stalled after recovery. Escalating directly to %s...", RECOVERY_METHODS.tabReplacement);
|
|
1113
|
+
await executeTabReplacement("capture pipeline stalled");
|
|
1114
|
+
return;
|
|
1115
|
+
}
|
|
1116
|
+
// Check circuit breaker for too many failures. The helper handles incrementing, window checks, and resetting if outside the window.
|
|
1117
|
+
const cbResult = checkCircuitBreaker(circuitBreaker, now);
|
|
1118
|
+
if (cbResult.shouldTrip) {
|
|
1119
|
+
const elapsedSeconds = circuitBreaker.firstFailureTime ? Math.round((now - circuitBreaker.firstFailureTime) / 1000) : 0;
|
|
1120
|
+
LOG.error("Circuit breaker tripped after %s failures in %ss. Stream appears fundamentally broken.", cbResult.totalCount, elapsedSeconds);
|
|
1121
|
+
clearInterval(interval);
|
|
1122
|
+
onCircuitBreak();
|
|
1123
|
+
return;
|
|
1124
|
+
}
|
|
1125
|
+
/* Issue-aware escalation. Instead of blindly incrementing the level, we determine the appropriate level based on:
|
|
1126
|
+
* 1. The type of issue (paused vs buffering vs other)
|
|
1127
|
+
* 2. Whether source reload (L2) has already been attempted in this page session
|
|
1128
|
+
* 3. The current escalation level
|
|
1129
|
+
*
|
|
1130
|
+
* Levels:
|
|
1131
|
+
* - Level 1: Basic play/unmute - only for paused issues
|
|
1132
|
+
* - Level 2: Reload video source - for buffering/other issues, or when L1 fails
|
|
1133
|
+
* - Level 3: Page navigation - when L2 fails or has already been attempted
|
|
1134
|
+
*/
|
|
1135
|
+
const issueCategory = getIssueCategory(state, !isProgressing, isBuffering);
|
|
1136
|
+
let nextLevel;
|
|
1137
|
+
if ((issueCategory === "paused") && (escalationLevel === 0)) {
|
|
1138
|
+
// Paused issues: try L1 first (play/unmute works ~50% for paused).
|
|
1139
|
+
nextLevel = 1;
|
|
1140
|
+
}
|
|
1141
|
+
else if (!sourceReloadAttempted) {
|
|
1142
|
+
// First recovery attempt for buffering/other, or L1 didn't fix paused: try L2 (source reload).
|
|
1143
|
+
nextLevel = 2;
|
|
1144
|
+
}
|
|
1145
|
+
else {
|
|
1146
|
+
// Source reload already attempted: go to L3 (page reload).
|
|
1147
|
+
nextLevel = 3;
|
|
1148
|
+
}
|
|
1149
|
+
// Note: Keep state updates in sync with the video-not-found recovery path above.
|
|
1150
|
+
escalationLevel = nextLevel;
|
|
1151
|
+
lastRecoveryTime = now;
|
|
1152
|
+
totalRecoveryAttempts++;
|
|
1153
|
+
pendingReMinimize = true;
|
|
1154
|
+
// Get recovery method name for logging and metrics.
|
|
1155
|
+
const recoveryMethod = getRecoveryMethod(escalationLevel);
|
|
1156
|
+
// Store issue type and time for UI display.
|
|
1157
|
+
const issueType = formatIssueType(state, !isProgressing, isBuffering);
|
|
1158
|
+
lastIssueType = issueType;
|
|
1159
|
+
lastIssueTime = now;
|
|
1160
|
+
// If a previous recovery was pending (L1 or L2 that didn't result in healthy playback), log that it was unsuccessful before starting the new attempt.
|
|
1161
|
+
if (metrics.currentRecoveryMethod !== null) {
|
|
1162
|
+
LOG.warn("%s unsuccessful. Attempting %s...", capitalize(metrics.currentRecoveryMethod), recoveryMethod);
|
|
1163
|
+
}
|
|
1164
|
+
else {
|
|
1165
|
+
// First recovery attempt - log with issue description.
|
|
1166
|
+
const issueDesc = getIssueDescription(issueCategory);
|
|
1167
|
+
LOG.warn("Playback %s. Attempting %s...", issueDesc, recoveryMethod);
|
|
1168
|
+
}
|
|
1169
|
+
// Record this recovery attempt in metrics.
|
|
1170
|
+
recordRecoveryAttempt(metrics, recoveryMethod);
|
|
1171
|
+
// For L2/L3 recovery, record the current segment index so we can verify segments are flowing after recovery completes.
|
|
1172
|
+
if (escalationLevel >= 2) {
|
|
1173
|
+
const entry = getStream(streamInfo.numericStreamId);
|
|
1174
|
+
preRecoverySegmentIndex = entry?.segmenter?.getSegmentIndex() ?? null;
|
|
1175
|
+
segmentWaitStartTime = null; // Will be set after recovery grace period ends.
|
|
1176
|
+
segmentProductionStalled = false;
|
|
1177
|
+
}
|
|
1178
|
+
// Mark recovery in progress to prevent overlapping recovery attempts.
|
|
1179
|
+
recoveryInProgress = true;
|
|
1180
|
+
try {
|
|
1181
|
+
/* Levels 1-2: In-page recovery. These levels are handled by ensurePlayback() which performs recovery actions without navigating the page.
|
|
1182
|
+
*/
|
|
1183
|
+
if (escalationLevel <= 2) {
|
|
1184
|
+
await ensurePlayback(currentPage, currentContext, profile, { recoveryLevel: escalationLevel, skipNativeFullscreen: true });
|
|
1185
|
+
// Track that source reload was attempted so we skip directly to L3 next time.
|
|
1186
|
+
if (escalationLevel === 2) {
|
|
1187
|
+
sourceReloadAttempted = true;
|
|
1188
|
+
// The source reload disrupted the video stream. Mark a discontinuity so HLS clients resynchronize their decoders.
|
|
1189
|
+
markStreamDiscontinuity();
|
|
1190
|
+
}
|
|
1191
|
+
// Set grace period to give this recovery level time to take effect before the next check.
|
|
1192
|
+
recoveryGraceUntil = now + recoveryGracePeriods[escalationLevel];
|
|
1193
|
+
}
|
|
1194
|
+
else {
|
|
1195
|
+
/* Level 3: Page navigation recovery. This is the most aggressive recovery - we navigate to the URL again and reinitialize everything.
|
|
1196
|
+
*/
|
|
1197
|
+
// Safety check: If page navigation has failed twice consecutively, fall back to source reload. This prevents getting stuck in a loop when navigation
|
|
1198
|
+
// itself is broken (e.g., network issues, site blocking).
|
|
1199
|
+
if (consecutiveNavigationFailures >= 2) {
|
|
1200
|
+
LOG.warn("Page navigation has failed %s consecutive times. Falling back to source reload recovery.", consecutiveNavigationFailures);
|
|
1201
|
+
escalationLevel = 2;
|
|
1202
|
+
consecutiveNavigationFailures = 0;
|
|
1203
|
+
// Reset source reload tracking so the fallback L2 gets a fair chance. Without this, the next recovery cycle would skip L2 and try L3 again.
|
|
1204
|
+
sourceReloadAttempted = false;
|
|
1205
|
+
}
|
|
1206
|
+
else {
|
|
1207
|
+
// Check page reload limit to prevent excessive navigations. We allow MAX_PAGE_RELOADS within PAGE_RELOAD_WINDOW.
|
|
1208
|
+
const reloadWindow = now - CONFIG.playback.pageReloadWindow;
|
|
1209
|
+
// Prune old timestamps outside the window.
|
|
1210
|
+
pageReloadTimestamps = pageReloadTimestamps.filter((ts) => {
|
|
1211
|
+
return ts > reloadWindow;
|
|
1212
|
+
});
|
|
1213
|
+
if (pageReloadTimestamps.length >= CONFIG.playback.maxPageReloads) {
|
|
1214
|
+
LOG.warn("Exceeded maximum page navigations (%s in %s minutes). Falling back to source reload.", CONFIG.playback.maxPageReloads, Math.round(CONFIG.playback.pageReloadWindow / 60000));
|
|
1215
|
+
escalationLevel = 2;
|
|
1216
|
+
// Reset source reload tracking so the fallback L2 gets a fair chance.
|
|
1217
|
+
sourceReloadAttempted = false;
|
|
1218
|
+
}
|
|
1219
|
+
else {
|
|
1220
|
+
pageReloadTimestamps.push(now);
|
|
1221
|
+
// Use the unified recovery function with validation.
|
|
1222
|
+
const recoveryResult = await performPageNavigationRecovery();
|
|
1223
|
+
// Page navigation disrupted the video stream. Mark a discontinuity regardless of navigation success so HLS clients resynchronize their decoders.
|
|
1224
|
+
markStreamDiscontinuity();
|
|
1225
|
+
// Set grace period to give page navigation time to take effect (L3 = 10 seconds).
|
|
1226
|
+
recoveryGraceUntil = now + recoveryGracePeriods[3];
|
|
1227
|
+
if (recoveryResult.success && recoveryResult.newContext) {
|
|
1228
|
+
// Update the context reference to the new context (only after validation succeeds).
|
|
1229
|
+
currentContext = recoveryResult.newContext;
|
|
1230
|
+
// Log success with timing.
|
|
1231
|
+
const duration = formatRecoveryDuration(metrics.currentRecoveryStartTime ?? now);
|
|
1232
|
+
LOG.info("Recovered in %s via %s.", duration, RECOVERY_METHODS.pageNavigation);
|
|
1233
|
+
recordRecoverySuccess(metrics, RECOVERY_METHODS.pageNavigation);
|
|
1234
|
+
// Reset state after successful page navigation recovery.
|
|
1235
|
+
lastPageNavigationTime = Date.now();
|
|
1236
|
+
resetRecoveryCounters();
|
|
1237
|
+
resetEscalationState();
|
|
1238
|
+
resetSegmentMonitoringState();
|
|
1239
|
+
}
|
|
1240
|
+
else {
|
|
1241
|
+
consecutiveNavigationFailures++;
|
|
1242
|
+
LOG.warn("Page navigation unsuccessful (attempt %s/2).", consecutiveNavigationFailures);
|
|
1243
|
+
}
|
|
1244
|
+
}
|
|
1245
|
+
}
|
|
1246
|
+
}
|
|
1247
|
+
}
|
|
1248
|
+
catch (error) {
|
|
1249
|
+
LOG.warn("Recovery via %s failed: %s.", getRecoveryMethod(escalationLevel), formatError(error));
|
|
1250
|
+
}
|
|
1251
|
+
recoveryInProgress = false;
|
|
1252
|
+
}
|
|
1253
|
+
// Update lastTime for the next stall check.
|
|
1254
|
+
lastTime = state.time;
|
|
1255
|
+
// Emit status update for SSE subscribers.
|
|
1256
|
+
emitStatusUpdate();
|
|
1257
|
+
}
|
|
1258
|
+
catch (error) {
|
|
1259
|
+
recoveryInProgress = false;
|
|
1260
|
+
// If the session or page was closed, stop monitoring gracefully.
|
|
1261
|
+
if (isSessionClosedError(error) || currentPage.isClosed()) {
|
|
1262
|
+
clearInterval(interval);
|
|
1263
|
+
return;
|
|
1264
|
+
}
|
|
1265
|
+
// Check for evaluate timeout errors, which indicate the browser tab may be unresponsive.
|
|
1266
|
+
if (error instanceof EvaluateTimeoutError) {
|
|
1267
|
+
consecutiveTimeouts++;
|
|
1268
|
+
LOG.warn("Monitor check timed out (%s consecutive). Tab may be unresponsive.", consecutiveTimeouts);
|
|
1269
|
+
// Update issue state so SSE clients can show the degraded state.
|
|
1270
|
+
lastIssueType = "tab timing out";
|
|
1271
|
+
lastIssueTime = Date.now();
|
|
1272
|
+
// After 3 consecutive timeouts, attempt tab replacement if the callback is available.
|
|
1273
|
+
if ((consecutiveTimeouts >= 3) && onTabReplacement) {
|
|
1274
|
+
LOG.warn("Tab unresponsive. Attempting %s...", RECOVERY_METHODS.tabReplacement);
|
|
1275
|
+
await executeTabReplacement("tab unresponsive");
|
|
1276
|
+
return;
|
|
1277
|
+
}
|
|
1278
|
+
// Emit status so SSE clients see current duration/memory even during timeout degradation (when consecutiveTimeouts < 3).
|
|
1279
|
+
emitStatusUpdate();
|
|
1280
|
+
return;
|
|
1281
|
+
}
|
|
1282
|
+
// Log abort errors at debug level since they're expected during stream termination. Log other errors at error level.
|
|
1283
|
+
const errorMessage = formatError(error);
|
|
1284
|
+
if (errorMessage.includes("aborted")) {
|
|
1285
|
+
LOG.debug("recovery", "Monitor check aborted: %s.", errorMessage);
|
|
1286
|
+
}
|
|
1287
|
+
else {
|
|
1288
|
+
LOG.error("Monitor check failed: %s.", errorMessage);
|
|
1289
|
+
}
|
|
1290
|
+
// Emit status for non-abort errors so SSE clients stay current. Abort errors don't need this because termination is already in progress and the next
|
|
1291
|
+
// tick's abort check will clean up.
|
|
1292
|
+
if (!errorMessage.includes("aborted")) {
|
|
1293
|
+
emitStatusUpdate();
|
|
1294
|
+
}
|
|
1295
|
+
}
|
|
1296
|
+
}).catch((outerError) => {
|
|
1297
|
+
// Log errors that escape the inner try/catch. In normal operation we should not reach here - if we do, there's a bug to investigate.
|
|
1298
|
+
LOG.warn("Monitor tick error escaped inner try/catch: %s.", formatError(outerError));
|
|
1299
|
+
});
|
|
1300
|
+
}, CONFIG.playback.monitorInterval);
|
|
1301
|
+
/* Return the cleanup function. The caller (stream handler) should call this when the stream ends to stop monitoring. Returns the recovery metrics for the
|
|
1302
|
+
* termination summary log.
|
|
1303
|
+
*/
|
|
1304
|
+
return function () {
|
|
1305
|
+
intervalCleared = true;
|
|
1306
|
+
clearInterval(interval);
|
|
1307
|
+
return metrics;
|
|
1308
|
+
};
|
|
1309
|
+
}
|
|
1310
|
+
//# sourceMappingURL=monitor.js.map
|