eyevinn-channel-engine 4.0.8 → 4.0.10

package/dist/engine/playhead_state.d.ts

@@ -19,7 +19,7 @@ declare class SharedPlayheadState {
     getState(): Promise<4>;
     getLastM3u8(): Promise<any>;
     getValues(keys: any): Promise<any>;
-    set(key: any, value: any): Promise<any>;
+    set(key: any, value: any, isLeader: any): Promise<any>;
     setState(newState: any): Promise<4>;
     setLastM3u8(m3u8: any): Promise<any>;
 }

package/dist/engine/playhead_state.js

@@ -97,12 +97,16 @@ var SharedPlayheadState = /** @class */ (function () {
             });
         });
     };
-    SharedPlayheadState.prototype.set = function (key, value) {
+    SharedPlayheadState.prototype.set = function (key, value, isLeader) {
         return __awaiter(this, void 0, void 0, function () {
             return __generator(this, function (_a) {
                 switch (_a.label) {
-                    case 0: return [4 /*yield*/, this.store.set(this.sessionId, key, value)];
+                    case 0:
+                        if (!isLeader) return [3 /*break*/, 2];
+                        return [4 /*yield*/, this.store.set(this.sessionId, key, value)];
                     case 1: return [2 /*return*/, _a.sent()];
+                    case 2: return [4 /*yield*/, this.store.get(this.sessionId, key)];
+                    case 3: return [2 /*return*/, _a.sent()];
                 }
             });
         });

package/dist/engine/redis_state_store.d.ts

@@ -2,6 +2,7 @@ export = RedisStateStore;
 declare class RedisStateStore {
     constructor(keyPrefix: any, opts: any);
     keyPrefix: any;
+    volatileKeyTTL: any;
     client: any;
     initAsync(id: any, initData: any): Promise<{}>;
     resetAsync(id: any, initData: any): Promise<void>;

package/dist/engine/redis_state_store.js

@@ -36,7 +36,7 @@ var __generator = (this && this.__generator) || function (thisArg, body) {
 };
 var redis = require("redis");
 var debug = require("debug")("redis-state-store");
-var VOLATILE_KEY_TTL = 5; // Timeout so it should not expire within one normal increment iteration (in seconds)
+var DEFAULT_VOLATILE_KEY_TTL = 5; // Timeout so it should not expire within one normal increment iteration (in seconds)
 var RedisStateStore = /** @class */ (function () {
     function RedisStateStore(keyPrefix, opts) {
         this.keyPrefix = keyPrefix;
@@ -45,6 +45,11 @@ var RedisStateStore = /** @class */ (function () {
             this.keyPrefix = prependPrefix + this.keyPrefix;
             debug("Prepending keyprefix with ".concat(prependPrefix, " => ").concat(this.keyPrefix));
         }
+        this.volatileKeyTTL = DEFAULT_VOLATILE_KEY_TTL;
+        if (opts.volatileKeyTTL) {
+            debug("Overriding default, volatileKeyTTL=".concat(opts.volatileKeyTTL, "s"));
+            this.volatileKeyTTL = opts.volatileKeyTTL;
+        }
         this.client = redis.createClient(opts.redisUrl);
     }
     RedisStateStore.prototype.initAsync = function (id, initData) {
@@ -126,15 +131,17 @@ var RedisStateStore = /** @class */ (function () {
     };
     RedisStateStore.prototype.getAsync = function (id, key) {
         return __awaiter(this, void 0, void 0, function () {
-            var storeKey, getAsync, data;
+            var startMs, storeKey, getAsync, data;
             var _this = this;
             return __generator(this, function (_a) {
                 switch (_a.label) {
                     case 0:
+                        startMs = Date.now();
                         storeKey = "" + this.keyPrefix + id + key;
                         getAsync = new Promise(function (resolve, reject) {
                             _this.client.get(storeKey, function (err, reply) {
-                                debug("REDIS get ".concat(storeKey, ":").concat(reply ? reply.length + " chars" : "null"));
+                                var ioTimeMs = Date.now() - startMs;
+                                debug("REDIS get ".concat(storeKey, ":").concat(reply ? reply.length + " chars" : "null", " (").concat(ioTimeMs, "ms) ").concat(ioTimeMs > 1000 ? 'REDISSLOW!' : ''));
                                 if (!err) {
                                     resolve(JSON.parse(reply));
                                 }
@@ -153,15 +160,17 @@ var RedisStateStore = /** @class */ (function () {
     };
     RedisStateStore.prototype.setAsync = function (id, key, value) {
         return __awaiter(this, void 0, void 0, function () {
-            var storeKey, setAsync;
+            var startMs, storeKey, setAsync;
             var _this = this;
             return __generator(this, function (_a) {
                 switch (_a.label) {
                     case 0:
+                        startMs = Date.now();
                         storeKey = "" + this.keyPrefix + id + key;
                         setAsync = new Promise(function (resolve, reject) {
                             _this.client.set(storeKey, JSON.stringify(value), function (err, res) {
-                                debug("REDIS set ".concat(storeKey, ": ").concat(res));
+                                var ioTimeMs = Date.now() - startMs;
+                                debug("REDIS set ".concat(storeKey, ": ").concat(res, " (").concat(ioTimeMs, "ms) ").concat(ioTimeMs > 1000 ? 'REDISSLOW!' : ''));
                                 if (!err) {
                                     resolve(value);
                                 }
@@ -187,9 +196,9 @@ var RedisStateStore = /** @class */ (function () {
                         data = _a.sent();
                         storeKey = "" + this.keyPrefix + id + key;
                         expireAsync = new Promise(function (resolve, reject) {
-                            _this.client.expire(storeKey, VOLATILE_KEY_TTL, function (err, res) {
+                            _this.client.expire(storeKey, _this.volatileKeyTTL, function (err, res) {
                                 if (!err) {
-                                    debug("REDIS expire ".concat(storeKey, " ").concat(VOLATILE_KEY_TTL, "s: ").concat(res === 1 ? "OK" : "KEY DOES NOT EXIST"));
+                                    debug("REDIS expire ".concat(storeKey, " ").concat(_this.volatileKeyTTL, "s: ").concat(res === 1 ? "OK" : "KEY DOES NOT EXIST"));
                                     resolve();
                                 }
                                 else {
@@ -207,14 +216,17 @@ var RedisStateStore = /** @class */ (function () {
     };
     RedisStateStore.prototype.removeAsync = function (id, key) {
         return __awaiter(this, void 0, void 0, function () {
-            var storeKey, delAsync;
+            var startMs, storeKey, delAsync;
             var _this = this;
             return __generator(this, function (_a) {
                 switch (_a.label) {
                     case 0:
+                        startMs = Date.now();
                         storeKey = "" + this.keyPrefix + id + key;
                         delAsync = new Promise(function (resolve, reject) {
                             _this.client.del(storeKey, function (err, res) {
+                                var ioTimeMs = Date.now() - startMs;
+                                debug("REDIS remove ".concat(storeKey, ": (").concat(ioTimeMs, "ms) ").concat(ioTimeMs > 1000 ? 'REDISSLOW!' : ''));
                                 if (!err) {
                                     resolve();
                                 }

package/dist/engine/server.d.ts

@@ -22,6 +22,7 @@ export interface ChannelEngineOpts {
     adCopyMgrUri?: string;
     adXchangeUri?: string;
     noSessionDataTags?: boolean;
+    volatileKeyTTL?: number;
 }
 export interface VodRequest {
     sessionId: string;

package/dist/engine/server.js

@@ -101,11 +101,13 @@ var ChannelEngine = /** @class */ (function () {
                 redisUrl: options.redisUrl,
                 memcachedUrl: options.memcachedUrl,
                 cacheTTL: options.sharedStoreCacheTTL,
+                volatileKeyTTL: options.volatileKeyTTL,
             }),
             playheadStateStore: new PlayheadStateStore({
                 redisUrl: options.redisUrl,
                 memcachedUrl: options.memcachedUrl,
                 cacheTTL: options.sharedStoreCacheTTL,
+                volatileKeyTTL: options.volatileKeyTTL,
             }),
             instanceId: this.instanceId,
         };
@@ -114,6 +116,7 @@ var ChannelEngine = /** @class */ (function () {
                 redisUrl: options.redisUrl,
                 memcachedUrl: options.memcachedUrl,
                 cacheTTL: options.sharedStoreCacheTTL,
+                volatileKeyTTL: options.volatileKeyTTL,
             }),
             instanceId: this.instanceId,
         };
@@ -326,7 +329,9 @@ var ChannelEngine = /** @class */ (function () {
                 }
             });
         }); };
-        StreamSwitchLoop(this.streamSwitchTimeIntervalMs);
+        if (this.streamSwitchManager) {
+            StreamSwitchLoop(this.streamSwitchTimeIntervalMs);
+        }
     }
     ChannelEngine.prototype.updateStreamSwitchAsync = function () {
         return __awaiter(this, void 0, void 0, function () {
@@ -884,34 +889,39 @@ var ChannelEngine = /** @class */ (function () {
                         debug(req.params);
                         session = sessions[req.params[1]];
                         sessionLive = sessionsLive[req.params[1]];
-                        if (!(session && sessionLive)) return [3 /*break*/, 11];
+                        if (!(session && sessionLive)) return [3 /*break*/, 12];
                         _a.label = 1;
                     case 1:
-                        _a.trys.push([1, 9, , 10]);
-                        _a.label = 2;
+                        _a.trys.push([1, 10, , 11]);
+                        body = null;
+                        if (!!this.streamSwitchManager) return [3 /*break*/, 3];
+                        debug("[".concat(req.params[1], "]: Responding with VOD2Live manifest"));
+                        return [4 /*yield*/, session.getCurrentMediaManifestAsync(req.params[0], req.headers["x-playback-session-id"])];
                     case 2:
-                        if (!(switcherStatus[req.params[1]] === null || switcherStatus[req.params[1]] === undefined)) return [3 /*break*/, 4];
+                        body = _a.sent();
+                        return [3 /*break*/, 9];
+                    case 3:
+                        if (!(switcherStatus[req.params[1]] === null || switcherStatus[req.params[1]] === undefined)) return [3 /*break*/, 5];
                         debug("[".concat(req.params[1], "]: (").concat(switcherStatus[req.params[1]], ") Waiting for streamSwitcher to respond"));
                         return [4 /*yield*/, timer(500)];
-                    case 3:
-                        _a.sent();
-                        return [3 /*break*/, 2];
                     case 4:
-                        body = null;
+                        _a.sent();
+                        return [3 /*break*/, 3];
+                    case 5:
                         debug("switcherStatus[".concat(req.params[1], "]=[").concat(switcherStatus[req.params[1]], "]"));
-                        if (!switcherStatus[req.params[1]]) return [3 /*break*/, 6];
+                        if (!switcherStatus[req.params[1]]) return [3 /*break*/, 7];
                         debug("[".concat(req.params[1], "]: Responding with Live-stream manifest"));
                         return [4 /*yield*/, sessionLive.getCurrentMediaManifestAsync(req.params[0])];
-                    case 5:
-                        body = _a.sent();
-                        return [3 /*break*/, 8];
                     case 6:
+                        body = _a.sent();
+                        return [3 /*break*/, 9];
+                    case 7:
                         debug("[".concat(req.params[1], "]: Responding with VOD2Live manifest"));
                         return [4 /*yield*/, session.getCurrentMediaManifestAsync(req.params[0], req.headers["x-playback-session-id"])];
-                    case 7:
-                        body = _a.sent();
-                        _a.label = 8;
                     case 8:
+                        body = _a.sent();
+                        _a.label = 9;
+                    case 9:
                         //verbose(`[${session.sessionId}] body=`);
                         //verbose(body);
                         res.sendRaw(200, Buffer.from(body, 'utf8'), {
@@ -921,17 +931,17 @@ var ChannelEngine = /** @class */ (function () {
                             "X-Instance-Id": this.instanceId + "<".concat(version, ">"),
                         });
                         next();
-                        return [3 /*break*/, 10];
-                    case 9:
+                        return [3 /*break*/, 11];
+                    case 10:
                         err_6 = _a.sent();
                         next(this._gracefulErrorHandler(err_6));
-                        return [3 /*break*/, 10];
-                    case 10: return [3 /*break*/, 12];
-                    case 11:
+                        return [3 /*break*/, 11];
+                    case 11: return [3 /*break*/, 13];
+                    case 12:
                         err = new errs.NotFoundError('Invalid session(s)');
                         next(err);
-                        _a.label = 12;
-                    case 12: return [2 /*return*/];
+                        _a.label = 13;
+                    case 13: return [2 /*return*/];
                 }
             });
         });

package/dist/engine/session.js

@@ -253,7 +253,7 @@ var Session = /** @class */ (function () {
                         tickInterval = firstDuration < 2 ? 2 : firstDuration;
                         debug("[".concat(this._sessionId, "]: I am the leader and updated tick interval to ").concat(tickInterval, " sec"));
                         cloudWatchLog(!this.cloudWatchLogging, 'engine-session', { event: 'tickIntervalUpdated', channel: this._sessionId, tickIntervalSec: tickInterval });
-                        this._playheadState.set("tickInterval", tickInterval);
+                        this._playheadState.set("tickInterval", tickInterval, isLeader);
                         return [3 /*break*/, 21];
                     case 13:
                         if (!(state == PlayheadState.STOPPED)) return [3 /*break*/, 14];
@@ -326,7 +326,7 @@ var Session = /** @class */ (function () {
                     case 17:
                         _a.sent();
                         tsTickEnd = Date.now();
-                        return [4 /*yield*/, this._playheadState.set("tickMs", (tsTickEnd - tsIncrementBegin))];
+                        return [4 /*yield*/, this._playheadState.set("tickMs", (tsTickEnd - tsIncrementBegin), isLeader)];
                     case 18:
                         _a.sent();
                         cloudWatchLog(!this.cloudWatchLogging, 'engine-session', { event: 'tickInterval', channel: this._sessionId, tickTimeMs: (tsTickEnd - tsIncrementBegin) });
@@ -335,7 +335,7 @@ var Session = /** @class */ (function () {
                     case 19:
                         lastDuration = _a.sent();
                         nextTickInterval = lastDuration < 2 ? 2 : lastDuration;
-                        return [4 /*yield*/, this._playheadState.set("tickInterval", nextTickInterval)];
+                        return [4 /*yield*/, this._playheadState.set("tickInterval", nextTickInterval, isLeader)];
                     case 20:
                         _a.sent();
                         cloudWatchLog(!this.cloudWatchLogging, "engine-session", { event: "tickIntervalUpdated", channel: this._sessionId, tickIntervalSec: nextTickInterval });
@@ -375,12 +375,15 @@ var Session = /** @class */ (function () {
     };
     Session.prototype.stopPlayheadAsync = function () {
         return __awaiter(this, void 0, void 0, function () {
+            var isLeader;
             return __generator(this, function (_a) {
                 switch (_a.label) {
-                    case 0:
-                        debug("[".concat(this._sessionId, "]: Stopping playhead consumer"));
-                        return [4 /*yield*/, this._playheadState.set("state", PlayheadState.STOPPED)];
+                    case 0: return [4 /*yield*/, this._sessionStateStore.isLeader(this._instanceId)];
                     case 1:
+                        isLeader = _a.sent();
+                        debug("[".concat(this._sessionId, "]: Stopping playhead consumer"));
+                        return [4 /*yield*/, this._playheadState.set("state", PlayheadState.STOPPED, isLeader)];
+                    case 2:
                         _a.sent();
                         return [2 /*return*/];
                 }
@@ -1031,6 +1034,7 @@ var Session = /** @class */ (function () {
                         index = 0;
                         audioSeqLastIdx = currentVod_1.getLiveMediaSequencesCount("audio") - 1;
                         thresh = 0.5;
+                        debug("[".concat(this._sessionId, "]: About to determine audio increment"));
                         _o.label = 19;
                     case 19: return [4 /*yield*/, this._getAudioPlayheadPosition(sessionState_1.vodMediaSeqAudio + index)];
                     case 20:
@@ -1058,6 +1062,7 @@ var Session = /** @class */ (function () {
                         audioIncrement = index;
                         _o.label = 23;
                     case 23:
+                        debug("[".concat(this._sessionId, "]: Will increment audio with ").concat(audioIncrement));
                         _d = sessionState;
                         return [4 /*yield*/, this._sessionState.increment("vodMediaSeqAudio", audioIncrement)];
                     case 24:
@@ -1090,20 +1095,23 @@ var Session = /** @class */ (function () {
                         this.isSwitchingBackToV2L = false;
                         _o.label = 32;
                     case 32:
+                        if (isLeader) {
+                            debug("[".concat(this._sessionId, "]: I am the leader, updating PlayheadState values"));
+                        }
                         _j = playheadState;
-                        return [4 /*yield*/, this._playheadState.set("mediaSeq", sessionState.mediaSeq)];
+                        return [4 /*yield*/, this._playheadState.set("mediaSeq", sessionState.mediaSeq, isLeader)];
                     case 33:
                         _j.mediaSeq = _o.sent();
                         _k = playheadState;
-                        return [4 /*yield*/, this._playheadState.set("mediaSeqAudio", sessionState.mediaSeqAudio)];
+                        return [4 /*yield*/, this._playheadState.set("mediaSeqAudio", sessionState.mediaSeqAudio, isLeader)];
                     case 34:
                         _k.mediaSeqAudio = _o.sent();
                         _l = playheadState;
-                        return [4 /*yield*/, this._playheadState.set("vodMediaSeqVideo", sessionState.vodMediaSeqVideo)];
+                        return [4 /*yield*/, this._playheadState.set("vodMediaSeqVideo", sessionState.vodMediaSeqVideo, isLeader)];
                     case 35:
                         _l.vodMediaSeqVideo = _o.sent();
                         _m = playheadState;
-                        return [4 /*yield*/, this._playheadState.set("vodMediaSeqAudio", sessionState.vodMediaSeqAudio)];
+                        return [4 /*yield*/, this._playheadState.set("vodMediaSeqAudio", sessionState.vodMediaSeqAudio, isLeader)];
                     case 36:
                         _m.vodMediaSeqAudio = _o.sent();
                         if (currentVod.sequenceAlwaysContainNewSegments) {
@@ -1481,11 +1489,11 @@ var Session = /** @class */ (function () {
     };
     Session.prototype._insertSlate = function (currentVod) {
         return __awaiter(this, void 0, void 0, function () {
-            var slateVod, sessionState, endValue, endValueAudio, lastDiscontinuity, lastDiscontinuityAudio;
+            var slateVod, sessionState, endValue, endValueAudio, lastDiscontinuity, lastDiscontinuityAudio, isLeader;
             return __generator(this, function (_a) {
                 switch (_a.label) {
                     case 0:
-                        if (!this.slateUri) return [3 /*break*/, 13];
+                        if (!this.slateUri) return [3 /*break*/, 14];
                         console.error("[".concat(this._sessionId, "]: Will insert slate"));
                         return [4 /*yield*/, this._loadSlate(currentVod)];
                     case 1:
@@ -1510,39 +1518,42 @@ var Session = /** @class */ (function () {
                             lastDiscontinuity = currentVod.getLastDiscontinuity();
                             lastDiscontinuityAudio = currentVod.getLastDiscontinuityAudio();
                         }
-                        return [4 /*yield*/, this._sessionState.set("vodMediaSeqVideo", 0)];
+                        return [4 /*yield*/, this._sessionStateStore.isLeader(this._instanceId)];
                     case 3:
-                        _a.sent();
-                        return [4 /*yield*/, this._sessionState.set("vodMediaSeqAudio", 0)];
+                        isLeader = _a.sent();
+                        return [4 /*yield*/, this._sessionState.set("vodMediaSeqVideo", 0)];
                     case 4:
                         _a.sent();
-                        return [4 /*yield*/, this._sessionState.set("state", SessionState.VOD_NEXT_INITIATING)];
+                        return [4 /*yield*/, this._sessionState.set("vodMediaSeqAudio", 0)];
                     case 5:
                         _a.sent();
-                        return [4 /*yield*/, this._sessionState.setCurrentVod(slateVod)];
+                        return [4 /*yield*/, this._sessionState.set("state", SessionState.VOD_NEXT_INITIATING)];
                     case 6:
                         _a.sent();
-                        return [4 /*yield*/, this._sessionState.set("mediaSeq", sessionState.mediaSeq + endValue)];
+                        return [4 /*yield*/, this._sessionState.setCurrentVod(slateVod)];
                     case 7:
                         _a.sent();
-                        return [4 /*yield*/, this._sessionState.set("mediaSeqAudio", sessionState.mediaSeqAudio + endValueAudio)];
+                        return [4 /*yield*/, this._sessionState.set("mediaSeq", sessionState.mediaSeq + endValue)];
                     case 8:
                         _a.sent();
-                        return [4 /*yield*/, this._sessionState.set("discSeq", sessionState.discSeq + lastDiscontinuity)];
+                        return [4 /*yield*/, this._sessionState.set("mediaSeqAudio", sessionState.mediaSeqAudio + endValueAudio)];
                     case 9:
                         _a.sent();
-                        return [4 /*yield*/, this._sessionState.set("discSeqAudio", sessionState.discSeqAudio + lastDiscontinuityAudio)];
+                        return [4 /*yield*/, this._sessionState.set("discSeq", sessionState.discSeq + lastDiscontinuity)];
                     case 10:
                         _a.sent();
-                        return [4 /*yield*/, this._sessionState.set("slateCount", sessionState.slateCount + 1)];
+                        return [4 /*yield*/, this._sessionState.set("discSeqAudio", sessionState.discSeqAudio + lastDiscontinuityAudio)];
                     case 11:
                         _a.sent();
-                        return [4 /*yield*/, this._playheadState.set("playheadRef", Date.now())];
+                        return [4 /*yield*/, this._sessionState.set("slateCount", sessionState.slateCount + 1)];
                     case 12:
+                        _a.sent();
+                        return [4 /*yield*/, this._playheadState.set("playheadRef", Date.now(), isLeader)];
+                    case 13:
                         _a.sent();
                         cloudWatchLog(!this.cloudWatchLogging, 'engine-session', { event: 'slateInserted', channel: this._sessionId });
                         return [2 /*return*/, slateVod];
-                    case 13: return [2 /*return*/, null];
+                    case 14: return [2 /*return*/, null];
                 }
             });
         });
@@ -1636,7 +1647,8 @@ var Session = /** @class */ (function () {
                         cloudWatchLog(!this.cloudWatchLogging, 'engine-session', { event: 'loadVod', channel: this._sessionId, loadTimeMs: Date.now() - loadStart });
                         debug("[".concat(this._sessionId, "]: first VOD loaded"));
                         debug("[".concat(this._sessionId, "]: ").concat(currentVod.getDeltaTimes()));
-                        debug("[".concat(this._sessionId, "]: ").concat(currentVod.getPlayheadPositions()));
+                        debug("[".concat(this._sessionId, "]: playhead positions [V]=").concat(currentVod.getPlayheadPositions("video")));
+                        debug("[".concat(this._sessionId, "]: playhead positions [A]=").concat(currentVod.getPlayheadPositions("audio")));
                         //debug(newVod);
                         _c = sessionState;
                         return [4 /*yield*/, this._sessionState.set("mediaSeq", 0)];
@@ -1663,7 +1675,7 @@ var Session = /** @class */ (function () {
                         return [4 /*yield*/, this._sessionState.set("vodMediaSeqAudio", 0)];
                     case 15:
                         _h.vodMediaSeqAudio = _2.sent();
-                        return [4 /*yield*/, this._playheadState.set("playheadRef", Date.now())];
+                        return [4 /*yield*/, this._playheadState.set("playheadRef", Date.now(), isLeader)];
                     case 16:
                         _2.sent();
                         this.produceEvent({
@@ -1858,7 +1870,8 @@ var Session = /** @class */ (function () {
                         cloudWatchLog(!this.cloudWatchLogging, 'engine-session', { event: 'loadVod', channel: this._sessionId, loadTimeMs: Date.now() - loadStart });
                         this.leaderIsSettingNextVod = true;
                         debug("[".concat(this._sessionId, "]: next VOD loaded (").concat(newVod.getDeltaTimes(), ")"));
-                        debug("[".concat(this._sessionId, "]: ").concat(newVod.getPlayheadPositions()));
+                        debug("[".concat(this._sessionId, "]: playhead positions [V]=").concat(newVod.getPlayheadPositions("video")));
+                        debug("[".concat(this._sessionId, "]: playhead positions [A]=").concat(newVod.getPlayheadPositions("audio")));
                         currentVod = newVod;
                         debug("[".concat(this._sessionId, "]: msequences=").concat(currentVod.getLiveMediaSequencesCount(), "; audio msequences=").concat(currentVod.getLiveMediaSequencesCount("audio")));
                         _r = sessionState;
@@ -1894,10 +1907,10 @@ var Session = /** @class */ (function () {
                     case 50:
                         _x.currentVod = _2.sent();
                         this.leaderIsSettingNextVod = false;
-                        return [4 /*yield*/, this._playheadState.set("playheadRef", Date.now())];
+                        return [4 /*yield*/, this._playheadState.set("playheadRef", Date.now(), isLeader)];
                     case 51:
                         _2.sent();
-                        return [4 /*yield*/, this._playheadState.set("diffCompensation", this.diffCompensation)];
+                        return [4 /*yield*/, this._playheadState.set("diffCompensation", this.diffCompensation, isLeader)];
                     case 52:
                         _2.sent();
                         debug("[".concat(this._sessionId, "]: sharing durrent vods diffCompensation=").concat(this.diffCompensation));
@@ -1995,7 +2008,7 @@ var Session = /** @class */ (function () {
                         return [4 /*yield*/, this._sessionState.set("mediaSeq", mSeq)];
                     case 65:
                         _2.sent();
-                        return [4 /*yield*/, this._playheadState.set("mediaSeq", mSeq)];
+                        return [4 /*yield*/, this._playheadState.set("mediaSeq", mSeq, isLeader)];
                     case 66:
                         _2.sent();
                         return [4 /*yield*/, this._sessionState.set("discSeq", dSeq)];
@@ -2033,13 +2046,13 @@ var Session = /** @class */ (function () {
                         return [4 /*yield*/, this._sessionState.set("vodMediaSeqAudio", 0)];
                     case 73:
                         _2.sent();
-                        return [4 /*yield*/, this._playheadState.set("vodMediaSeqVideo", 0)];
+                        return [4 /*yield*/, this._playheadState.set("vodMediaSeqVideo", 0, isLeader)];
                     case 74:
                         _2.sent();
-                        return [4 /*yield*/, this._playheadState.set("vodMediaSeqAudio", 0)];
+                        return [4 /*yield*/, this._playheadState.set("vodMediaSeqAudio", 0, isLeader)];
                     case 75:
                         _2.sent();
-                        return [4 /*yield*/, this._playheadState.set("playheadRef", Date.now())];
+                        return [4 /*yield*/, this._playheadState.set("playheadRef", Date.now(), isLeader)];
                     case 76:
                         _2.sent();
                         // 4) Log to debug and cloudwatch

package/dist/package.json

@@ -1,6 +1,6 @@
 {
   "name": "eyevinn-channel-engine",
-  "version": "4.0.8",
+  "version": "4.0.10",
   "description": "OTT TV Channel Engine",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/engine/playhead_state.js

@@ -31,8 +31,12 @@ class SharedPlayheadState {
     return await this.store.getValues(this.sessionId, keys);
   }
 
-  async set(key, value) {
-    return await this.store.set(this.sessionId, key, value);
+  async set(key, value, isLeader) {
+    if (isLeader) {
+      return await this.store.set(this.sessionId, key, value);
+    } else {
+      return await this.store.get(this.sessionId, key);
+    }
   }
 
   async setState(newState) {

package/engine/redis_state_store.js

@@ -1,7 +1,7 @@
 const redis = require("redis");
 const debug = require("debug")("redis-state-store");
 
-const VOLATILE_KEY_TTL = 5; // Timeout so it should not expire within one normal increment iteration (in seconds)
+const DEFAULT_VOLATILE_KEY_TTL = 5; // Timeout so it should not expire within one normal increment iteration (in seconds)
 
 class RedisStateStore {
   constructor(keyPrefix, opts) {
@@ -11,6 +11,11 @@ class RedisStateStore {
       this.keyPrefix = prependPrefix + this.keyPrefix;
       debug(`Prepending keyprefix with ${prependPrefix} => ${this.keyPrefix}`);
     }
+    this.volatileKeyTTL = DEFAULT_VOLATILE_KEY_TTL;
+    if (opts.volatileKeyTTL) {
+      debug(`Overriding default, volatileKeyTTL=${opts.volatileKeyTTL}s`);
+      this.volatileKeyTTL = opts.volatileKeyTTL;  
+    }
     this.client = redis.createClient(opts.redisUrl);
   }
 
@@ -48,10 +53,12 @@ class RedisStateStore {
   }
 
   async getAsync(id, key) {
+    const startMs = Date.now();
     const storeKey = "" + this.keyPrefix + id + key;
     const getAsync = new Promise((resolve, reject) => {
       this.client.get(storeKey, (err, reply) => {
-        debug(`REDIS get ${storeKey}:${reply ? reply.length + " chars" : "null"}`);
+        const ioTimeMs = Date.now() - startMs;
+        debug(`REDIS get ${storeKey}:${reply ? reply.length + " chars" : "null"} (${ioTimeMs}ms) ${ioTimeMs > 1000 ? 'REDISSLOW!' : ''}`);
         if (!err) {
           resolve(JSON.parse(reply));
         } else {
@@ -64,10 +71,12 @@ class RedisStateStore {
   }
 
   async setAsync(id, key, value) {
+    const startMs = Date.now();
     const storeKey = "" + this.keyPrefix + id + key;
     const setAsync = new Promise((resolve, reject) => {
       this.client.set(storeKey, JSON.stringify(value), (err, res) => {
-        debug(`REDIS set ${storeKey}: ${res}`);
+        const ioTimeMs = Date.now() - startMs;
+        debug(`REDIS set ${storeKey}: ${res} (${ioTimeMs}ms) ${ioTimeMs > 1000 ? 'REDISSLOW!' : ''}`);
         if (!err) {
           resolve(value);
         } else {
@@ -82,9 +91,9 @@ class RedisStateStore {
     const data = await this.setAsync(id, key, value);
     const storeKey = "" + this.keyPrefix + id + key;
     const expireAsync = new Promise((resolve, reject) => {
-      this.client.expire(storeKey, VOLATILE_KEY_TTL, (err, res) => {
+      this.client.expire(storeKey, this.volatileKeyTTL, (err, res) => {
         if (!err) {
-          debug(`REDIS expire ${storeKey} ${VOLATILE_KEY_TTL}s: ${res === 1 ? "OK" : "KEY DOES NOT EXIST"}`);
+          debug(`REDIS expire ${storeKey} ${this.volatileKeyTTL}s: ${res === 1 ? "OK" : "KEY DOES NOT EXIST"}`);
           resolve();
         } else {
           reject(err);
@@ -96,9 +105,12 @@ class RedisStateStore {
   }
 
   async removeAsync(id, key) {
+    const startMs = Date.now();
     const storeKey = "" + this.keyPrefix + id + key;
     const delAsync = new Promise((resolve, reject) => {
       this.client.del(storeKey, (err, res) => {
+        const ioTimeMs = Date.now() - startMs;
+        debug(`REDIS remove ${storeKey}: (${ioTimeMs}ms) ${ioTimeMs > 1000 ? 'REDISSLOW!' : ''}`);
         if (!err) {
           resolve();
         } else {

package/engine/server.ts

@@ -47,6 +47,7 @@ export interface ChannelEngineOpts {
   adCopyMgrUri?: string; // deprecated
   adXchangeUri?: string; // deprecated
   noSessionDataTags?: boolean;
+  volatileKeyTTL?: number;
 }
 
 interface StreamerOpts {
@@ -206,11 +207,13 @@ export class ChannelEngine {
         redisUrl: options.redisUrl, 
         memcachedUrl: options.memcachedUrl, 
         cacheTTL: options.sharedStoreCacheTTL,
+        volatileKeyTTL: options.volatileKeyTTL,
       }),
       playheadStateStore: new PlayheadStateStore({ 
         redisUrl: options.redisUrl, 
         memcachedUrl: options.memcachedUrl, 
         cacheTTL: options.sharedStoreCacheTTL,
+        volatileKeyTTL: options.volatileKeyTTL,
       }),
       instanceId: this.instanceId,
     };
@@ -220,6 +223,7 @@ export class ChannelEngine {
         redisUrl: options.redisUrl, 
         memcachedUrl: options.memcachedUrl, 
         cacheTTL: options.sharedStoreCacheTTL,
+        volatileKeyTTL: options.volatileKeyTTL,
       }),
       instanceId: this.instanceId,
     };
@@ -353,7 +357,9 @@ export class ChannelEngine {
         }
       }
     }
-    StreamSwitchLoop(this.streamSwitchTimeIntervalMs);
+    if (this.streamSwitchManager) {
+      StreamSwitchLoop(this.streamSwitchTimeIntervalMs);
+    }
   }
 
   async updateStreamSwitchAsync() {
@@ -706,19 +712,25 @@ export class ChannelEngine {
     const sessionLive = sessionsLive[req.params[1]];
     if (session && sessionLive) {
       try {
-        while (switcherStatus[req.params[1]] === null || switcherStatus[req.params[1]] === undefined) {
-          debug(`[${req.params[1]}]: (${switcherStatus[req.params[1]]}) Waiting for streamSwitcher to respond`);
-          await timer(500);
-        }
         let body = null;
-        debug(`switcherStatus[${req.params[1]}]=[${switcherStatus[req.params[1]]}]`);
-        if (switcherStatus[req.params[1]]) {
-          debug(`[${req.params[1]}]: Responding with Live-stream manifest`);
-          body = await sessionLive.getCurrentMediaManifestAsync(req.params[0]);
-        } else {
+        if (!this.streamSwitchManager) {
           debug(`[${req.params[1]}]: Responding with VOD2Live manifest`);
           body = await session.getCurrentMediaManifestAsync(req.params[0], req.headers["x-playback-session-id"]);
+        } else {
+          while (switcherStatus[req.params[1]] === null || switcherStatus[req.params[1]] === undefined) {
+            debug(`[${req.params[1]}]: (${switcherStatus[req.params[1]]}) Waiting for streamSwitcher to respond`);
+            await timer(500);
+          }
+          debug(`switcherStatus[${req.params[1]}]=[${switcherStatus[req.params[1]]}]`);
+          if (switcherStatus[req.params[1]]) {
+            debug(`[${req.params[1]}]: Responding with Live-stream manifest`);
+            body = await sessionLive.getCurrentMediaManifestAsync(req.params[0]);
+          } else {
+            debug(`[${req.params[1]}]: Responding with VOD2Live manifest`);
+            body = await session.getCurrentMediaManifestAsync(req.params[0], req.headers["x-playback-session-id"]);
+          }
         }
+
         //verbose(`[${session.sessionId}] body=`);
         //verbose(body);
         res.sendRaw(200, Buffer.from(body, 'utf8'), {

package/engine/session.js

@@ -185,7 +185,7 @@ class Session {
           debug(`[${this._sessionId}]: I am the leader and updated tick interval to ${tickInterval} sec`);
           cloudWatchLog(!this.cloudWatchLogging, 'engine-session',
             { event: 'tickIntervalUpdated', channel: this._sessionId, tickIntervalSec: tickInterval });
-          this._playheadState.set("tickInterval", tickInterval);
+          this._playheadState.set("tickInterval", tickInterval, isLeader);
         } else if (state == PlayheadState.STOPPED) {
           debug(`[${this._sessionId}]: Stopping playhead`);
           return;
@@ -251,14 +251,14 @@ class Session {
           debug(`[${this._sessionId}]: (${(new Date()).toISOString()}) ${timeSpentInIncrement}sec in increment. Next tick in ${tickInterval} seconds`)
           await timer((tickInterval * 1000) - 50);
           const tsTickEnd = Date.now();
-          await this._playheadState.set("tickMs", (tsTickEnd - tsIncrementBegin));
+          await this._playheadState.set("tickMs", (tsTickEnd - tsIncrementBegin), isLeader);
           cloudWatchLog(!this.cloudWatchLogging, 'engine-session',
             { event: 'tickInterval', channel: this._sessionId, tickTimeMs: (tsTickEnd - tsIncrementBegin) });
           if (this.alwaysNewSegments) {
             // Use dynamic base-tickInterval. Set according to duration of latest segment.
             const lastDuration = await this._getLastDuration(manifest);
             const nextTickInterval = lastDuration < 2 ? 2 : lastDuration;
-            await this._playheadState.set("tickInterval", nextTickInterval);
+            await this._playheadState.set("tickInterval", nextTickInterval, isLeader);
             cloudWatchLog(!this.cloudWatchLogging, "engine-session", { event: "tickIntervalUpdated", channel: this._sessionId, tickIntervalSec: nextTickInterval });
           }
         }
@@ -280,8 +280,9 @@ class Session {
   }
 
   async stopPlayheadAsync() {
+    const isLeader = await this._sessionStateStore.isLeader(this._instanceId);
     debug(`[${this._sessionId}]: Stopping playhead consumer`);
-    await this._playheadState.set("state", PlayheadState.STOPPED);
+    await this._playheadState.set("state", PlayheadState.STOPPED, isLeader);
   }
 
   async getStatusAsync() {
@@ -733,6 +734,7 @@ class Session {
         let index = 0;
         const audioSeqLastIdx = currentVod.getLiveMediaSequencesCount("audio") - 1;
         const thresh = 0.5;
+        debug(`[${this._sessionId}]: About to determine audio increment`);
         do {
           const audioPosition = (await this._getAudioPlayheadPosition(sessionState.vodMediaSeqAudio + index)) * 1000;
           posDiff = position - audioPosition;
@@ -752,6 +754,7 @@ class Session {
         } while (!(-thresh < posDiff && posDiff < thresh));
         audioIncrement = index;
       }
+      debug(`[${this._sessionId}]: Will increment audio with ${audioIncrement}`);
       sessionState.vodMediaSeqAudio = await this._sessionState.increment("vodMediaSeqAudio", audioIncrement);
     }
 
@@ -769,10 +772,13 @@ class Session {
       this.isSwitchingBackToV2L = false;
     }
 
-    playheadState.mediaSeq = await this._playheadState.set("mediaSeq", sessionState.mediaSeq);
-    playheadState.mediaSeqAudio = await this._playheadState.set("mediaSeqAudio", sessionState.mediaSeqAudio);
-    playheadState.vodMediaSeqVideo = await this._playheadState.set("vodMediaSeqVideo", sessionState.vodMediaSeqVideo);
-    playheadState.vodMediaSeqAudio = await this._playheadState.set("vodMediaSeqAudio", sessionState.vodMediaSeqAudio);
+    if (isLeader) {
+      debug(`[${this._sessionId}]: I am the leader, updating PlayheadState values`);
+    }
+    playheadState.mediaSeq = await this._playheadState.set("mediaSeq", sessionState.mediaSeq, isLeader);
+    playheadState.mediaSeqAudio = await this._playheadState.set("mediaSeqAudio", sessionState.mediaSeqAudio, isLeader);
+    playheadState.vodMediaSeqVideo = await this._playheadState.set("vodMediaSeqVideo", sessionState.vodMediaSeqVideo, isLeader);
+    playheadState.vodMediaSeqAudio = await this._playheadState.set("vodMediaSeqAudio", sessionState.vodMediaSeqAudio, isLeader);
 
     if (currentVod.sequenceAlwaysContainNewSegments) {
       const mediaSequenceValue = currentVod.mediaSequenceValues[playheadState.vodMediaSeqVideo];
@@ -1075,6 +1081,7 @@ class Session {
         lastDiscontinuity = currentVod.getLastDiscontinuity();
         lastDiscontinuityAudio = currentVod.getLastDiscontinuityAudio();
       }
+      const isLeader = await this._sessionStateStore.isLeader(this._instanceId);
       await this._sessionState.set("vodMediaSeqVideo", 0);
       await this._sessionState.set("vodMediaSeqAudio", 0);
       await this._sessionState.set("state", SessionState.VOD_NEXT_INITIATING);
@@ -1084,7 +1091,7 @@ class Session {
       await this._sessionState.set("discSeq", sessionState.discSeq + lastDiscontinuity);
       await this._sessionState.set("discSeqAudio", sessionState.discSeqAudio + lastDiscontinuityAudio);
       await this._sessionState.set("slateCount", sessionState.slateCount + 1);
-      await this._playheadState.set("playheadRef", Date.now());
+      await this._playheadState.set("playheadRef", Date.now(), isLeader);
 
       cloudWatchLog(!this.cloudWatchLogging, 'engine-session', { event: 'slateInserted', channel: this._sessionId });
 
@@ -1160,7 +1167,8 @@ class Session {
               { event: 'loadVod', channel: this._sessionId, loadTimeMs: Date.now() - loadStart });
             debug(`[${this._sessionId}]: first VOD loaded`);
             debug(`[${this._sessionId}]: ${currentVod.getDeltaTimes()}`);
-            debug(`[${this._sessionId}]: ${currentVod.getPlayheadPositions()}`);
+            debug(`[${this._sessionId}]: playhead positions [V]=${currentVod.getPlayheadPositions("video")}`);
+            debug(`[${this._sessionId}]: playhead positions [A]=${currentVod.getPlayheadPositions("audio")}`);
             //debug(newVod);
             sessionState.mediaSeq = await this._sessionState.set("mediaSeq", 0);
             sessionState.mediaSeqAudio = await this._sessionState.set("mediaSeqAudio", 0);
@@ -1168,7 +1176,7 @@ class Session {
             sessionState.discSeqAudio = await this._sessionState.set("discSeqAudio", 0);
             sessionState.vodMediaSeqVideo = await this._sessionState.set("vodMediaSeqVideo", 0);
             sessionState.vodMediaSeqAudio = await this._sessionState.set("vodMediaSeqAudio", 0);
-            await this._playheadState.set("playheadRef", Date.now());
+            await this._playheadState.set("playheadRef", Date.now(), isLeader);
             this.produceEvent({
               type: 'NOW_PLAYING',
               data: {
@@ -1323,7 +1331,8 @@ class Session {
               { event: 'loadVod', channel: this._sessionId, loadTimeMs: Date.now() - loadStart });
             this.leaderIsSettingNextVod = true;
             debug(`[${this._sessionId}]: next VOD loaded (${newVod.getDeltaTimes()})`);
-            debug(`[${this._sessionId}]: ${newVod.getPlayheadPositions()}`);
+            debug(`[${this._sessionId}]: playhead positions [V]=${newVod.getPlayheadPositions("video")}`);
+            debug(`[${this._sessionId}]: playhead positions [A]=${newVod.getPlayheadPositions("audio")}`);
             currentVod = newVod;
             debug(`[${this._sessionId}]: msequences=${currentVod.getLiveMediaSequencesCount()}; audio msequences=${currentVod.getLiveMediaSequencesCount("audio")}`);
             sessionState.vodMediaSeqVideo = await this._sessionState.set("vodMediaSeqVideo", 0);
@@ -1336,8 +1345,8 @@ class Session {
             await this._sessionState.remove("nextVod");
             sessionState.currentVod = await this._sessionState.setCurrentVod(currentVod, { ttl: currentVod.getDuration() * 1000 });
             this.leaderIsSettingNextVod = false;
-            await this._playheadState.set("playheadRef", Date.now());
-            await this._playheadState.set("diffCompensation", this.diffCompensation);
+            await this._playheadState.set("playheadRef", Date.now(), isLeader);
+            await this._playheadState.set("diffCompensation", this.diffCompensation, isLeader);
             debug(`[${this._sessionId}]: sharing durrent vods diffCompensation=${this.diffCompensation}`);
             this.produceEvent({
               type: 'NOW_PLAYING',
@@ -1414,7 +1423,7 @@ class Session {
               return;
             }
             await this._sessionState.set("mediaSeq", mSeq);
-            await this._playheadState.set("mediaSeq", mSeq);
+            await this._playheadState.set("mediaSeq", mSeq, isLeader);
             await this._sessionState.set("discSeq", dSeq);
             // TODO: support demux^
             debug(`[${this._sessionId}]: Setting current media and discontinuity count -> [${mSeq}]:[${dSeq}]`);
@@ -1435,9 +1444,9 @@ class Session {
             await this._sessionState.set("vodReloaded", 1);
             await this._sessionState.set("vodMediaSeqVideo", 0);
             await this._sessionState.set("vodMediaSeqAudio", 0);
-            await this._playheadState.set("vodMediaSeqVideo", 0);
-            await this._playheadState.set("vodMediaSeqAudio", 0);
-            await this._playheadState.set("playheadRef", Date.now());
+            await this._playheadState.set("vodMediaSeqVideo", 0, isLeader);
+            await this._playheadState.set("vodMediaSeqAudio", 0, isLeader);
+            await this._playheadState.set("playheadRef", Date.now(), isLeader);
             // 4) Log to debug and cloudwatch
             debug(`[${this._sessionId}]: LEADER: Set new Reloaded VOD and vodMediaSeq counts in store.`);
             debug(`[${this._sessionId}]: next VOD Reloaded (${currentVod.getDeltaTimes()})`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "eyevinn-channel-engine",
-  "version": "4.0.8",
+  "version": "4.0.10",
   "description": "OTT TV Channel Engine",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",