eyevinn-channel-engine 3.1.7 → 3.1.8

package/engine/session.js

@@ -43,9 +43,20 @@ class Session {
     this.playheadDiffThreshold = DEFAULT_PLAYHEAD_DIFF_THRESHOLD;
     this.maxTickInterval = DEFAULT_MAX_TICK_INTERVAL;
     this.diffCompensation = null;
+    this.prevVodMediaSeq = {
+      video: null,
+      audio: null
+    }
     this.waitingForNextVod = false;
     this.leaderIsSettingNextVod = false;
-    this.isSwitchingBackFromLive = false;
+    this.isSwitchingBackToV2L = false;
+    this.switchDataForSession = {
+      mediaSeq: null,
+      discSeq: null,
+      mediaSeqOffset: null,
+      transitionSegments: null,
+      reloadBehind: null,
+    }
 
     if (config) {
       if (config.sessionId) {
@@ -139,7 +150,14 @@ class Session {
         state = await this._playheadState.getState();
 
         const isLeader = await this._sessionStateStore.isLeader(this._instanceId);
-        if (isLeader && [SessionState.VOD_NEXT_INIT, SessionState.VOD_NEXT_INITIATING].indexOf(sessionState.state) !== -1) {
+        if (isLeader && 
+          [
+            SessionState.VOD_NEXT_INIT,
+            SessionState.VOD_NEXT_INITIATING,
+            SessionState.VOD_RELOAD_INIT,
+            SessionState.VOD_RELOAD_INITIATING
+          ].indexOf(sessionState.state) !== -1) {
+
           const firstDuration = await this._getFirstDuration(manifest);
           const tickInterval = firstDuration < 2 ? 2 : firstDuration;
           debug(`[${this._sessionId}]: I am the leader and updated tick interval to ${tickInterval} sec`);
@@ -152,13 +170,13 @@ class Session {
         } else {
           const reqTickInterval = playheadState.tickInterval;
           const timeSpentInIncrement = (tsIncrementEnd - tsIncrementBegin) / 1000;
-          let tickInterval = (reqTickInterval) - timeSpentInIncrement;
+          let tickInterval = reqTickInterval - timeSpentInIncrement;
           const delta = await this._getCurrentDeltaTime();
           if (delta != 0) {
             debug(`[${this._sessionId}]: Delta time is != 0 need will adjust ${delta}sec to tick interval`);
             tickInterval += delta;
           }
-          const position = await this._getCurrentPlayheadPosition() * 1000;
+          const position = (await this._getCurrentPlayheadPosition()) * 1000;
           const timePosition = Date.now() - playheadState.playheadRef;
           const diff = position - timePosition;
           debug(`[${this._sessionId}]: ${timePosition}:${position}:${diff > 0 ? '+' : ''}${diff}ms`);
@@ -214,7 +232,7 @@ class Session {
     if (this.disabledPlayhead) {
       return {
         sessionId: this._sessionId,
-      }
+      };
     } else {
       const playheadState = await this._playheadState.getValues(["tickMs", "mediaSeq", "vodMediaSeqVideo"]);
       const state = await this._playheadState.getState();
@@ -243,11 +261,16 @@ class Session {
     await this._playheadStateStore.reset(this._sessionId);
   }
 
+  async getSessionState() {
+    const state = await this._sessionState.get("state");
+    return state;
+  }
+
   async getTruncatedVodSegments(vodUri, duration) {
     try {
       const hlsVod = await this._truncateSlate(null, duration, vodUri);
       let vodSegments = hlsVod.getMediaSegments();
-      Object.keys(vodSegments).forEach(bw => vodSegments[bw].unshift({ discontinuity: true, cue: { in: true } }));
+      Object.keys(vodSegments).forEach((bw) => vodSegments[bw].unshift({ discontinuity: true, cue: { in: true } }));
       return vodSegments;
     } catch (exc) {
       debug(`[${this._sessionId}]: Failed to generate truncated VOD!`);
@@ -257,20 +280,25 @@ class Session {
 
   async setCurrentMediaSequenceSegments(segments, mSeqOffset, reloadBehind) {
     if (!this._sessionState) {
-      throw new Error('Session not ready');
+      throw new Error("Session not ready");
     }
+    this.isSwitchingBackToV2L = true;
 
-    this.isSwitchingBackFromLive = true;
+    this.switchDataForSession.reloadBehind = reloadBehind;
+    this.switchDataForSession.transitionSegments = segments;
+    this.switchDataForSession.mediaSeqOffset = mSeqOffset;
 
     let isLeader = await this._sessionStateStore.isLeader(this._instanceId);
     if (!isLeader) {
       debug(`[${this._sessionId}]: FOLLOWER: Invalidate cache to ensure having the correct VOD!`);
       await this._sessionState.clearCurrentVodCache();
+
       let vodReloaded = await this._sessionState.get("vodReloaded");
       let attempts = 9;
       while (!isLeader && !vodReloaded && attempts > 0) {
         debug(`[${this._sessionId}]: FOLLOWER: I arrived before LEADER. Waiting (1000ms) for LEADER to reload currentVod in store! (tries left=${attempts})`);
         await timer(1000);
+        await this._sessionStateStore.clearLeaderCache();
         isLeader = await this._sessionStateStore.isLeader(this._instanceId);
         vodReloaded = await this._sessionState.get("vodReloaded");
         attempts--;
@@ -282,43 +310,26 @@ class Session {
       if (!isLeader || vodReloaded) {
         debug(`[${this._sessionId}]: FOLLOWER: leader is alive, and has presumably updated currentVod. Clearing the cache now`);
         await this._sessionState.clearCurrentVodCache();
-        this.isSwitchingBackFromLive = false;
         return;
       }
-    }
+      debug(`[${this._sessionId}]: NEW LEADER: Setting state=VOD_RELOAD_INIT`);
+      this.isSwitchingBackToV2L = true;
+      await this._sessionState.set("state", SessionState.VOD_RELOAD_INIT)
 
-    if (isLeader) {
-      let state = await this._sessionState.get("state");
-      let ATTEMPTS = 5;
-      // Do Not want to reload when session is loading next Vod
-      while (state !== SessionState.VOD_PLAYING && ATTEMPTS > 0) {
-        debug(`[${this._sessionId}]: LEADER: Wait for session to initialize next VOD. (tries left=${ATTEMPTS})`);
+    } else {
+      let vodReloaded = await this._sessionState.get("vodReloaded");
+      let attempts = 9;
+      while (!vodReloaded && attempts > 0) {
+        debug(`[${this._sessionId}]: LEADER: Waiting (1000ms) to buy some time reloading vod and adding it to store! (tries left=${attempts})`);
         await timer(1000);
-        state = await this._sessionState.get("state");
-        ATTEMPTS--;
-      }
-      if (ATTEMPTS === 0) {
-        throw new Error("Failed to reload currentVod! Need a new LEADER!");
+        vodReloaded = await this._sessionState.get("vodReloaded");
+        attempts--;
       }
-
-      debug(`[${this._sessionId}]: LEADER: making changes to current VOD. I will also update currentVod in store.`);
-      const playheadState = await this._playheadState.getValues(["vodMediaSeqVideo"]);
-      let currentVod = await this._sessionState.getCurrentVod();
-      let currentMseq = playheadState.vodMediaSeqVideo + mSeqOffset;
-      if (currentMseq > currentVod.getLiveMediaSequencesCount() - 1) {
-        currentMseq = currentVod.getLiveMediaSequencesCount() - 1;
+      if (attempts === 0) {
+        debug(`[${this._sessionId}]: LEADER: WARNING! Vod was never Reloaded!`);
+        return; 
       }
-      // TODO: Support reloading with audioSegments as well
-      await currentVod.reload(currentMseq, segments, null, reloadBehind);
-      await this._sessionState.setCurrentVod(currentVod, { ttl: (currentVod.getDuration() * 1000) });
-      await this._sessionState.set("vodReloaded", 1);
-      await this._sessionState.set("vodMediaSeqVideo", 0);
-      await this._sessionState.set("vodMediaSeqAudio", 0);
-      await this._playheadState.set("vodMediaSeqVideo", 0);
-      await this._playheadState.set("vodMediaSeqAudio", 0);
-      await this._playheadState.set("playheadRef", Date.now());
     }
-    this.isSwitchingBackFromLive = false;
   }
 
   async getCurrentMediaSequenceSegments() {
@@ -329,6 +340,18 @@ class Session {
     if (isLeader) {
       await this._sessionState.set("vodReloaded", 0);
     }
+
+    // Only read data from store if state is VOD_PLAYING
+    let state = await this.getSessionState();
+    let tries = 12;
+    while (state !== SessionState.VOD_PLAYING && tries > 0) {
+    const waitTimeMs = 500; 
+    debug(`[${this._sessionId}]: state=${state} - Waiting ${waitTimeMs}ms_${tries} until Leader has finished loading next vod.`);
+      await timer(waitTimeMs);
+      tries--;
+      state = await this.getSessionState();
+    }
+
     const playheadState = await this._playheadState.getValues(["mediaSeq", "vodMediaSeqVideo"]);
     if (playheadState.vodMediaSeqVideo === 0) {
       if (!isLeader) {
@@ -340,7 +363,7 @@ class Session {
     if (currentVod) {
       try {
         const mediaSegments = currentVod.getLiveMediaSequenceSegments(playheadState.vodMediaSeqVideo);
-        debug(`[${this._sessionId}]: Requesting all segments from Media Sequence: ${playheadState.vodMediaSeqVideo}`);
+        debug(`[${this._sessionId}]: Requesting all segments from Media Sequence: ${playheadState.vodMediaSeqVideo}_${currentVod.getLiveMediaSequencesCount()}`);
         return mediaSegments;
       } catch (err) {
         logerror(this._sessionId, err);
@@ -354,25 +377,31 @@ class Session {
 
   async setCurrentMediaAndDiscSequenceCount(_mediaSeq, _discSeq) {
     if (!this._sessionState) {
-      throw new Error('Session not ready');
+      throw new Error("Session not ready");
     }
-    
-    this.isSwitchingBackFromLive = true;
 
-    const isLeader = await this._sessionStateStore.isLeader(this._instanceId);
-    if (isLeader) {
-      await this._sessionState.set("mediaSeq", _mediaSeq);
-      await this._playheadState.set("mediaSeq", _mediaSeq);
-      await this._sessionState.set("discSeq", _discSeq);
-      debug(`[${this._sessionId}]: Setting current media and discontinuity count -> [${_mediaSeq}]:[${_discSeq}]`);
-    }
-    this.isSwitchingBackFromLive = false;
+    this.isSwitchingBackToV2L = true;
+
+    this.switchDataForSession.mediaSeq = _mediaSeq;
+    this.switchDataForSession.discSeq = _discSeq;
   }
 
   async getCurrentMediaAndDiscSequenceCount() {
     if (!this._sessionState) {
       throw new Error('Session not ready');
     }
+
+    // Only read data from store if state is VOD_PLAYING
+    let state = await this.getSessionState();
+    let tries = 12;
+    while (state !== SessionState.VOD_PLAYING && tries > 0) {
+      const waitTimeMs = 500; 
+      debug(`[${this._sessionId}]: state=${state} - Waiting ${waitTimeMs}ms_${tries} until Leader has finished loading next vod.`);
+      await timer(waitTimeMs);
+      tries--;
+      state = await this.getSessionState();
+    }
+    
     const playheadState = await this._playheadState.getValues(["mediaSeq", "vodMediaSeqVideo"]);
     const discSeqOffset = await this._sessionState.get("discSeq");
     if (playheadState.vodMediaSeqVideo === 0) {
@@ -407,8 +436,8 @@ class Session {
     }
     // Be sure that the leader is not in the middle of setting new vod data in store.
     // Followers will never run this part...
-    let tries = 4;
-    while(tries > 0 && this.leaderIsSettingNextVod) {
+    let tries = 12;
+    while (tries > 0 && this.leaderIsSettingNextVod) {
       debug(`[${this._sessionId}]: Leader is setting the next vod. Waiting 500ms_${tries}`);
       await timer(500);
       tries--;
@@ -417,17 +446,23 @@ class Session {
     const sessionState = await this._sessionState.getValues(["vodMediaSeqVideo", "discSeq"]);
     const playheadState = await this._playheadState.getValues(["mediaSeq", "vodMediaSeqVideo"]);
 
+    if (this.prevVodMediaSeq.video !== null) {
+      this.prevVodMediaSeq.video = playheadState.vodMediaSeqVideo;
+    }
+
     if (playheadState.vodMediaSeqVideo > sessionState.vodMediaSeqVideo) {
       debug(`[${this._sessionId}]: Recently Loaded Next Vod. PlayheadState not up-to-date. Return the last generated m3u8`);
       const m3u8 = await this._playheadState.getLastM3u8();
       if (m3u8) {
+        this.prevVodMediaSeq.video = playheadState.vodMediaSeqVideo;
         return m3u8;
       } else {
         debug(`[${this._sessionId}]: We don't have any previously generated m3u8`);
       }
     }
 
-    if (playheadState.vodMediaSeqVideo < 2) { 
+    if (playheadState.vodMediaSeqVideo < 2 || playheadState.vodMediaSeqVideo < this.prevVodMediaSeq.video) { 
+      debug(`[${this._sessionId}]: current[${playheadState.vodMediaSeqVideo}]_prev[${this.prevVodMediaSeq.video}]`);
       const isLeader = await this._sessionStateStore.isLeader(this._instanceId);
       if (!isLeader) {
         debug(`[${this._sessionId}]: Not a leader and first|second media sequence in a VOD is requested. Invalidate cache to ensure having the correct VOD.`);
@@ -441,6 +476,7 @@ class Session {
         debug(`[${this._sessionId}]: [${playheadState.vodMediaSeqVideo}]_[${currentVod.getLiveMediaSequencesCount()}]`);
         const m3u8 = currentVod.getLiveMediaSequences(playheadState.mediaSeq, bw, playheadState.vodMediaSeqVideo, sessionState.discSeq, this.targetDurationPadding, this.forceTargetDuration);
         debug(`[${this._sessionId}]: [${playheadState.mediaSeq + playheadState.vodMediaSeqVideo}][${sessionState.discSeq}][+${this.targetDurationPadding || 0}] Current media manifest for ${bw} requested`);
+        this.prevVodMediaSeq.video = playheadState.vodMediaSeqVideo;
         return m3u8;
       } catch (err) {
         logerror(this._sessionId, err);
@@ -482,13 +518,6 @@ class Session {
   async incrementAsync() {
     await this._tickAsync(); 
     const isLeader = await this._sessionStateStore.isLeader(this._instanceId);
-    // Avoid reading old values if new ones are on their way...
-    let tries = 10;
-    while (tries > 0 && this.isSwitchingBackFromLive) {
-      debug(`[${this._sessionId}]: Leader is currently setting new data in store. Waiting 500ms_${tries}`);
-      await timer(500);
-      tries--;
-    }
 
     let sessionState = await this._sessionState.getValues(
       ["state", "mediaSeq", "discSeq", "vodMediaSeqVideo", "vodMediaSeqAudio"]);
@@ -502,15 +531,17 @@ class Session {
         sessionState.discSeq === null) {
       debug(`[${this._sessionId}]: Session is not ready yet`);
       debug(sessionState);
+      await this._sessionState.clearCurrentVodCache();
       if (isLeader) {
         debug(`[${this._sessionId}]: I am the leader, trying to initiate the session`);
         sessionState.state = await this._sessionState.set("state", SessionState.VOD_INIT);
       }
       return null;
     }
-    if (sessionState.state === SessionState.VOD_NEXT_INITIATING) {
+    if (sessionState.state === SessionState.VOD_NEXT_INITIATING || sessionState.state === SessionState.VOD_RELOAD_INITIATING) {
       if (isLeader) {
-        debug(`[${this._sessionId}]: I am the leader and have just initiated next VOD, let's move to VOD_PLAYING`);
+        const leaderAction = sessionState.state === SessionState.VOD_NEXT_INITIATING ? "initiated" : "reloaded";
+        debug(`[${this._sessionId}]: I am the leader and have just ${leaderAction} next VOD, let's move to VOD_PLAYING`);
         sessionState.state = await this._sessionState.set("state", SessionState.VOD_PLAYING);
       } else {
         debug(`[${this._sessionId}]: Return the last generated m3u8 to give the leader some time`);
@@ -532,6 +563,11 @@ class Session {
       sessionState.state = await this._sessionState.set("state", SessionState.VOD_NEXT_INIT);
     }
 
+    if (this.isSwitchingBackToV2L) {
+      sessionState.state = await this._sessionState.set("state", SessionState.VOD_RELOAD_INIT);
+      this.isSwitchingBackToV2L = false;
+    }
+
     playheadState.mediaSeq = await this._playheadState.set("mediaSeq", sessionState.mediaSeq);
     playheadState.vodMediaSeqVideo = await this._playheadState.set("vodMediaSeqVideo", sessionState.vodMediaSeqVideo);
     playheadState.vodMediaSeqAudio = await this._playheadState.set("vodMediaSeqAudio", sessionState.vodMediaSeqAudio);
@@ -809,6 +845,10 @@ class Session {
       case SessionState.VOD_INIT:
       case SessionState.VOD_INIT_BY_ID:
         try {
+          // Needed if store was reset
+          await this._sessionStateStore.clearLeaderCache();
+          isLeader = await this._sessionStateStore.isLeader(this._instanceId);
+
           let nextVodPromise;
           if (sessionState.state === SessionState.VOD_INIT) {
             debug(`[${this._sessionId}]: state=VOD_INIT`);
@@ -898,6 +938,17 @@ class Session {
             currentVod = await this._sessionState.getCurrentVod();
             this.waitingForNextVod = false;
           }
+        } else {
+          // Handle edge case where store has been reset, but leader has not cleared cache.
+          if (this.prevVodMediaSeq.video !== null) {
+            this.prevVodMediaSeq.video = sessionState.vodMediaSeqVideo;
+          }
+          if (this.prevVodMediaSeq.video < sessionState.vodMediaSeqVideo) {
+            debug(`[${this._sessionId}]: current[${sessionState.vodMediaSeqVideo}], prev[${this.prevVodMediaSeq.video}], total[${currentVod.getLiveMediaSequencesCount()}]`);
+            await this._sessionState.clearCurrentVodCache();
+            currentVod = await this._sessionState.getCurrentVod();
+            this.prevVodMediaSeq.video = sessionState.vodMediaSeqVideo;
+          }
         }
         debug(`[${this._sessionId}]: state=VOD_PLAYING (${sessionState.vodMediaSeqVideo}_${sessionState.vodMediaSeqAudio}, ${currentVod.getLiveMediaSequencesCount()})`);
         return;
@@ -1014,6 +1065,77 @@ class Session {
           }
         }
         break;
+        case SessionState.VOD_RELOAD_INIT:
+          try {
+            debug(`[${this._sessionId}]: state=VOD_RELOAD_INIT`);
+            if (isLeader) {
+
+              const startTS = Date.now();
+
+              // 1) To tell Follower that, Leader is working on it!
+              sessionState.state = await this._sessionState.set("state", SessionState.VOD_RELOAD_INITIATING);
+
+              // 2) Set new 'offset' sequences, to carry on the continuity from session-live 
+              const mSeq = this.switchDataForSession.mediaSeq;
+              const dSeq = this.switchDataForSession.discSeq;
+              const mSeqOffset = this.switchDataForSession.mediaSeqOffset;
+              const reloadBehind = this.switchDataForSession.reloadBehind;
+              const segments = this.switchDataForSession.transitionSegments;
+
+              if ([mSeq,dSeq,mSeqOffset,reloadBehind,segments].includes(null)) {
+                debug(`[${this._sessionId}]: LEADER: Cannot Reload VOD, missing switch-back data`);
+                return;
+              }
+
+              await this._sessionState.set("mediaSeq", mSeq);
+              await this._playheadState.set("mediaSeq", mSeq);
+              await this._sessionState.set("discSeq", dSeq);
+              debug(`[${this._sessionId}]: Setting current media and discontinuity count -> [${mSeq}]:[${dSeq}]`);
+
+              // 3) Set new media segments/currentVod, to carry on the continuity from session-live
+              debug(`[${this._sessionId}]: LEADER: making changes to current VOD. I will also update currentVod in store.`);
+              const playheadState = await this._playheadState.getValues(["vodMediaSeqVideo"]);
+              let currentVod = await this._sessionState.getCurrentVod();
+              let currentMseq = playheadState.vodMediaSeqVideo + mSeqOffset;
+              if (currentMseq > currentVod.getLiveMediaSequencesCount() - 1) {
+                currentMseq = currentVod.getLiveMediaSequencesCount() - 1;
+              }
+
+              // TODO: Support reloading with audioSegments as well
+
+              await currentVod.reload(currentMseq, segments, null, reloadBehind);
+              await this._sessionState.setCurrentVod(currentVod, { ttl: currentVod.getDuration() * 1000 });
+              await this._sessionState.set("vodReloaded", 1);
+              await this._sessionState.set("vodMediaSeqVideo", 0);
+              await this._sessionState.set("vodMediaSeqAudio", 0);
+              await this._playheadState.set("vodMediaSeqVideo", 0);
+              await this._playheadState.set("vodMediaSeqAudio", 0);
+              await this._playheadState.set("playheadRef", Date.now());
+              debug(`[${this._sessionId}]: LEADER: Set new Reloaded VOD and vodMediaSeq counts in store.`);
+              // 4) emit cloudwatch event object
+              cloudWatchLog(!this.cloudWatchLogging, "engine-session", { event: "switchback", channel: this._sessionId, reqTimeMs: Date.now() - startTS });
+
+              return;
+            } else {
+              debug(`[${this._sessionId}]: not a leader so will go directly to state VOD_RELOAD_INITIATING`);
+              sessionState.state = await this._sessionState.set("state", SessionState.VOD_RELOAD_INITIATING);
+            }
+          } catch (err) {
+            debug("Failed to init reload vod");
+            throw err;
+          } 
+        break;
+        case SessionState.VOD_RELOAD_INITIATING:
+          debug(`[${this._sessionId}]: state=VOD_RELOAD_INITIATING (${sessionState.vodMediaSeqVideo}_${sessionState.vodMediaSeqAudio}, ${currentVod.getLiveMediaSequencesCount()})`);
+          if (!isLeader) {
+            debug(`[${this._sessionId}]: not the leader so just waiting for the VOD to be reloaded`);
+            if (sessionState.vodMediaSeqVideo === 0 || this.waitingForNextVod) {
+              debug(`[${this._sessionId}]: First mediasequence in VOD and I am not the leader so invalidate current VOD cache and fetch the new one from the leader`);
+              await this._sessionState.clearCurrentVodCache();
+            }
+            this.waitingForNextVod = true;
+          }
+          return;
       default:
         throw new Error("Invalid state: " + sessionState.state);
     }

package/engine/session_live.js

@@ -48,6 +48,7 @@ class SessionLive {
     this.firstTime = true;
     this.allowedToSet = false;
     this.pushAmount = 0;
+    this.restAmount = 0;
     this.waitForPlayhead = true;
     this.blockGenerateManifest = false;
 
@@ -502,7 +503,7 @@ class SessionLive {
         // Do not push duplicates
         const liveSegURIs = this.liveSegQueue[liveBw].filter(seg => seg.uri).map(seg => seg.uri);
         if (liveSegFromLeader.uri && liveSegURIs.includes(liveSegFromLeader.uri)) {
-          debug(`[${this.sessionId}]: FOLLOWER: Found duplicate live segment. Skip push!`);
+          debug(`[${this.sessionId}]: FOLLOWER: Found duplicate live segment. Skip push! (${liveBw})`);
         } else {
           this.liveSegQueue[liveBw].push(liveSegFromLeader);
           debug(`[${this.sessionId}]: FOLLOWER: Pushed segment (${liveSegFromLeader.uri ? liveSegFromLeader.uri : "Disc-tag"}) to 'liveSegQueue' (${liveBw})`);
@@ -671,28 +672,40 @@ class SessionLive {
         let leadersFirstSeqCounts = await this.sessionLiveState.get("firstCounts");
         let tries = 20;
 
-        while ((!leadersFirstSeqCounts.liveSourceMseqCount && tries > 0) || leadersFirstSeqCounts.liveSourceMseqCount === 0) {
+        while (!isLeader && (!leadersFirstSeqCounts.liveSourceMseqCount && tries > 0) || leadersFirstSeqCounts.liveSourceMseqCount === 0) {
           debug(`[${this.sessionId}]: NEW FOLLOWER: Waiting for LEADER to add 'firstCounts' in store! Will look again after 1000ms (tries left=${tries})`);
           await timer(1000);
           leadersFirstSeqCounts = await this.sessionLiveState.get("firstCounts");
           tries--;
+          // Might take over as Leader if Leader is not setting data due to being down.
+          isLeader = await this.sessionLiveStateStore.isLeader(this.instanceId);
+          if (isLeader) {
+            debug(`[${this.sessionId}][${this.instanceId}]: I'm the new leader, and now I am going to add 'firstCounts' in store`);
+          }
         }
 
         if (tries === 0) {
           isLeader = await this.sessionLiveStateStore.isLeader(this.instanceId);
           if (isLeader) {
-            debug(`[${this.instanceId}]: I'm the new leader, and now I am going to add 'firstCounts' in store`);
+            debug(`[${this.sessionId}][${this.instanceId}]: I'm the new leader, and now I am going to add 'firstCounts' in store`);
             break;
           } else {
-            debug(`[${this.instanceId}]: The leader is still alive`);
+            debug(`[${this.sessionId}][${this.instanceId}]: The leader is still alive`);
             leadersFirstSeqCounts = await this.sessionLiveState.get("firstCounts");
             if (!leadersFirstSeqCounts.liveSourceMseqCount) {
-              debug(`[${this.instanceId}]: Could not find 'firstCounts' in store. Abort Executing Promises II & Returning to Playhead.`);
+              debug(`[${this.sessionId}][${this.instanceId}]: Could not find 'firstCounts' in store. Abort Executing Promises II & Returning to Playhead.`);
               return;
             }
           }
         }
 
+        if (isLeader) {
+          debug(`[${this.sessionId}]: NEW LEADER: Original Leader went missing, I am retrying live source fetch...`);
+          await this.sessionLiveState.set("transitSegs", this.vodSegments);
+          debug(`[${this.sessionId}]: NEW LEADER: I am adding 'transitSegs' to Store for future followers`);
+          continue;
+        }
+
         // Respawners never do this, only starter followers.
         // Edge Case: FOLLOWER transitioned from session with different segments from LEADER
         if (leadersFirstSeqCounts.discSeqCount !== this.discSeqCount) {
@@ -713,6 +726,10 @@ class SessionLive {
         if (currentMseqRaw === leadersFirstSeqCounts.liveSourceMseqCount) {
           this.pushAmount = 1; // Follower from start
         } else {
+
+          // TODO: To support and account for past discontinuity tags in the Live Source stream,
+          // we will need to get the real 'current' discontinuity-sequence count from Leader somehow.
+
           // RESPAWNED NODES
           this.pushAmount = currentMseqRaw - leadersFirstSeqCounts.liveSourceMseqCount + 1;
 
@@ -772,7 +789,7 @@ class SessionLive {
       for (let i = 0; i < Object.keys(this.mediaManifestURIs).length; i++) {
         let bw = Object.keys(this.mediaManifestURIs)[i];
         // will add new segments to live seg queue
-        pushPromises.push(this._parseMediaManifest(this.liveSourceM3Us[bw].M3U, this.mediaManifestURIs[bw], bw));
+        pushPromises.push(this._parseMediaManifest(this.liveSourceM3Us[bw].M3U, this.mediaManifestURIs[bw], bw, isLeader));
         debug(`[${this.sessionId}]: Pushed pushPromise for bw=${bw}`);
       }
       // Segment Pushing
@@ -961,6 +978,12 @@ class SessionLive {
     // Update Session Live Media Sequence Count...
     this.prevMediaSeqCount = this.mediaSeqCount;
     this.mediaSeqCount += removedSegments;
+    if (this.restAmount) {
+      this.mediaSeqCount += this.restAmount;
+      debug(`[${this.sessionId}]: ${instanceName}: Added restAmount=[${this.restAmount}] to 'mediaSeqCount'`);
+      this.restAmount = 0;
+    }
+
     if (this.discSeqCount !== this.prevDiscSeqCount) {
       debug(`[${this.sessionId}]: ${instanceName}: Incrementing Dseq Count from {${this.prevDiscSeqCount}} -> {${this.discSeqCount}}`);
     }
@@ -1015,7 +1038,7 @@ class SessionLive {
     });
   }
 
-  _parseMediaManifest(m3u, mediaManifestUri, liveTargetBandwidth) {
+  _parseMediaManifest(m3u, mediaManifestUri, liveTargetBandwidth, isLeader) {
     return new Promise(async (resolve, reject) => {
       try {
         if (!this.liveSegQueue[liveTargetBandwidth]) {
@@ -1038,10 +1061,13 @@ class SessionLive {
         }
         this.targetDuration = m3u.get("targetDuration");
         let startIdx = m3u.items.PlaylistItem.length - this.pushAmount;
-        startIdx = startIdx < 0 ? 0 : startIdx;
+        if (startIdx < 0) {
+          this.restAmount = startIdx * -1;
+          startIdx = 0;
+        } 
         if (mediaManifestUri) {
           // push segments
-          this._addLiveSegmentsToQueue(startIdx, m3u.items.PlaylistItem, baseUrl, liveTargetBandwidth);
+          this._addLiveSegmentsToQueue(startIdx, m3u.items.PlaylistItem, baseUrl, liveTargetBandwidth, isLeader);
         }
         resolve();
       } catch (exc) {
@@ -1059,7 +1085,9 @@ class SessionLive {
    * @param {string} baseUrl
    * @param {string} liveTargetBandwidth
    */
-  _addLiveSegmentsToQueue(startIdx, playlistItems, baseUrl, liveTargetBandwidth) {
+  _addLiveSegmentsToQueue(startIdx, playlistItems, baseUrl, liveTargetBandwidth, isLeader) {
+    const leaderOrFollower = isLeader ? "LEADER": "NEW FOLLOWER";
+
     for (let i = startIdx; i < playlistItems.length; i++) {
       let seg = {};
       let playlistItem = playlistItems[i];
@@ -1127,10 +1155,15 @@ class SessionLive {
         if (daterangeData) {
           seg["daterange"] = daterangeData;
         }
-        // Push new Live Segments!
-        debug(`[${this.sessionId}]: Adding Live Segment to Queue (for bw=${liveTargetBandwidth})\n${seg.uri ? seg.uri : ''}`);
-        this.liveSegQueue[liveTargetBandwidth].push(seg);
-        this.liveSegsForFollowers[liveTargetBandwidth].push(seg);
+        // Push new Live Segments! But do not push duplicates
+        const liveSegURIs = this.liveSegQueue[liveTargetBandwidth].filter(seg => seg.uri).map(seg => seg.uri);
+        if (seg.uri && liveSegURIs.includes(seg.uri)) {
+          debug(`[${this.sessionId}]: ${leaderOrFollower}: Found duplicate live segment. Skip push! (${liveTargetBandwidth})`);
+        } else {
+          this.liveSegQueue[liveTargetBandwidth].push(seg);
+          this.liveSegsForFollowers[liveTargetBandwidth].push(seg);
+          debug(`[${this.sessionId}]: ${leaderOrFollower}: Pushed segment (${seg.uri ? seg.uri : "Disc-tag"}) to 'liveSegQueue' (${liveTargetBandwidth})`);
+        }
       }
     }
   }

package/engine/session_state.js

@@ -7,6 +7,8 @@ const SessionState = Object.freeze({
   VOD_PLAYING: 2,
   VOD_NEXT_INIT: 3,
   VOD_NEXT_INITIATING: 4,
+  VOD_RELOAD_INIT: 5,
+  VOD_RELOAD_INITIATING: 6
 });
 
 const CURRENTVOD_CACHE_TTL = 5 * 60 * 1000; // 5 minutes
@@ -157,6 +159,11 @@ class SessionStateStore extends SharedStateStore {
     await this.setVolatile("", instanceId, t);
   }
 
+  async clearLeaderCache() {
+    debug(`[${this.instanceId}]: clearing 'leader' cache`);
+    this.cache.leader.value = null;
+  }
+
   async isLeader(instanceId) {
     if (!instanceId) {
       throw new Error("Cannot determine leader without instance id");
@@ -164,7 +171,7 @@ class SessionStateStore extends SharedStateStore {
     let leader;
     if (this.cache.leader.value && Date.now() < this.cache.leader.ts + this.cacheTTL) {
       leader = this.cache.leader.value;
-      debug(`[${instanceId}]: reading 'leader' from cache: I am ${leader === instanceId ? "" : "NOT"} the leader!`);
+      debug(`[${instanceId}]: reading 'leader' from cache: I am${leader === instanceId ? "" : " NOT"} the leader!`);
       return leader === instanceId;
     }
     leader = await this.get("", "leader");
@@ -189,7 +196,7 @@ class SessionStateStore extends SharedStateStore {
         }
       }
     }
-    debug(`[${instanceId}]: I am ${leader === instanceId ? "" : "NOT"} the leader!`);
+    debug(`[${instanceId}]: I am${leader === instanceId ? "" : " NOT"} the leader!`);
     this.cache.leader.ts = Date.now();
     this.cache.leader.value = leader;
     return leader === instanceId;

package/engine/stream_switcher.js

@@ -2,6 +2,7 @@ const debug = require("debug")("engine-stream-switcher");
 const crypto = require("crypto");
 const fetch = require("node-fetch");
 const { AbortController } = require("abort-controller");
+const { SessionState } = require('./session_state');
 
 const SwitcherState = Object.freeze({
   V2L_TO_LIVE: 1,
@@ -88,8 +89,8 @@ class StreamSwitcher {
       return null;
     }
     // Handle Complete Storage Reset
-    let sessionState = await session._sessionState.get("state");
-    if (sessionState === 1 || !sessionState) {
+    let sessionState = await session.getSessionState();
+    if (sessionState === SessionState.VOD_INIT || !sessionState) {
       this.working = true;
       sessionLive.waitForPlayhead = false;
       sessionLive.allowedToSet = false; // only have effect on leader
@@ -341,14 +342,14 @@ class StreamSwitcher {
             return false;
           }
 
-          await session.setCurrentMediaSequenceSegments(
-            liveSegments.currMseqSegs,
-            liveSegments.segCount
-          );
           await session.setCurrentMediaAndDiscSequenceCount(
             liveCounts.mediaSeq,
             liveCounts.discSeq
           );
+          await session.setCurrentMediaSequenceSegments(
+            liveSegments.currMseqSegs,
+            liveSegments.segCount
+          );
 
           this.working = false;
           this.streamTypeLive = false;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "eyevinn-channel-engine",
-  "version": "3.1.7",
+  "version": "3.1.8",
   "description": "OTT TV Channel Engine",
   "main": "index.js",
   "scripts": {