eyevinn-channel-engine 3.1.7 → 3.2.0

package/engine/session_live.js

@@ -48,6 +48,7 @@ class SessionLive {
     this.firstTime = true;
     this.allowedToSet = false;
     this.pushAmount = 0;
+    this.restAmount = 0;
     this.waitForPlayhead = true;
     this.blockGenerateManifest = false;
 
@@ -158,16 +159,16 @@ class SessionLive {
         this.waitForPlayhead = false;
 
         // Let the playhead move at an interval set according to live segment duration
-        let liveSegmentDurationMs = DEFAULT_PLAYHEAD_INTERVAL_MS;
-        let liveBws = Object.keys(this.liveSegQueue);
-        if (liveBws.length !== 0 && this.liveSegQueue[liveBws[0]].length !== 0 && this.liveSegQueue[liveBws[0]][0].duration) {
-          liveSegmentDurationMs = this.liveSegQueue[liveBws[0]][0].duration * 1000;
-        }
+        const liveSegmentDurationMs = this._getAnyFirstSegmentDurationMs() || DEFAULT_PLAYHEAD_INTERVAL_MS;
 
         // Set the timer
         let timerValueMs = 0;
         if (this.timerCompensation) {
-          if (liveSegmentDurationMs - (tsIncrementEnd - tsIncrementBegin) > 0) {
+          const isLeader = await this.sessionLiveStateStore.isLeader(this.instanceId);
+          const incrementDuration = tsIncrementEnd - tsIncrementBegin;
+          if (incrementDuration >= liveSegmentDurationMs * 0.5 && isLeader) {
+            timerValueMs = liveSegmentDurationMs;
+          } else {
             timerValueMs = liveSegmentDurationMs - (tsIncrementEnd - tsIncrementBegin);
           }
         } else {
@@ -502,7 +503,7 @@ class SessionLive {
         // Do not push duplicates
         const liveSegURIs = this.liveSegQueue[liveBw].filter(seg => seg.uri).map(seg => seg.uri);
         if (liveSegFromLeader.uri && liveSegURIs.includes(liveSegFromLeader.uri)) {
-          debug(`[${this.sessionId}]: FOLLOWER: Found duplicate live segment. Skip push!`);
+          debug(`[${this.sessionId}]: FOLLOWER: Found duplicate live segment. Skip push! (${liveBw})`);
         } else {
           this.liveSegQueue[liveBw].push(liveSegFromLeader);
           debug(`[${this.sessionId}]: FOLLOWER: Pushed segment (${liveSegFromLeader.uri ? liveSegFromLeader.uri : "Disc-tag"}) to 'liveSegQueue' (${liveBw})`);
@@ -540,12 +541,14 @@ class SessionLive {
       if (!leadersMediaSeqRaw < this.lastRequestedMediaSeqRaw && this.blockGenerateManifest) {
         this.blockGenerateManifest = false;
       }
-      let attempts = 5;
 
+      let attempts = 5;
       //  CHECK AGAIN CASE 1: Store Empty
       while (!leadersMediaSeqRaw && attempts > 0) {
-        debug(`[${this.sessionId}]: FOLLOWER: Leader has not put anything in store... Will check again in 2000ms (Tries left=[${attempts}])`);
-        await timer(2000);
+        const segDur = this._getAnyFirstSegmentDurationMs() || DEFAULT_PLAYHEAD_INTERVAL_MS;
+        const waitTimeMs = parseInt(segDur / 3, 10);
+        debug(`[${this.sessionId}]: FOLLOWER: Leader has not put anything in store... Will check again in ${waitTimeMs}ms (Tries left=[${attempts}])`);
+        await timer(waitTimeMs);
         this.timerCompensation = false;
         leadersMediaSeqRaw = await this.sessionLiveState.get("lastRequestedMediaSeqRaw");
         attempts--;
@@ -565,8 +568,10 @@ class SessionLive {
       attempts = 5;
       //  CHECK AGAIN CASE 2: Store Old
       while (leadersMediaSeqRaw <= this.lastRequestedMediaSeqRaw && attempts > 0) {
-        debug(`[${this.sessionId}]: FOLLOWER: Cannot find anything NEW in store... Will check again in 2000ms (Tries left=[${attempts}])`);
-        await timer(2000);
+        const segDur = this._getAnyFirstSegmentDurationMs() || DEFAULT_PLAYHEAD_INTERVAL_MS;
+        const waitTimeMs = parseInt(segDur / 3, 10);
+        debug(`[${this.sessionId}]: FOLLOWER: Cannot find anything NEW in store... Will check again in ${waitTimeMs}ms (Tries left=[${attempts}])`);
+        await timer(waitTimeMs);
         this.timerCompensation = false;
         leadersMediaSeqRaw = await this.sessionLiveState.get("lastRequestedMediaSeqRaw");
         attempts--;
@@ -631,7 +636,9 @@ class SessionLive {
 
       // Handle if any promise got rejected
       if (manifestList.some((result) => result.status === "rejected")) {
-        debug(`[${this.sessionId}]: ALERT! Promises I: Failed, Rejection Found! Trying again...`);
+        FETCH_ATTEMPTS--;
+        debug(`[${this.sessionId}]: ALERT! Promises I: Failed, Rejection Found! Trying again in 1000ms...`);
+        await timer(1000)
         continue;
       }
 
@@ -671,28 +678,40 @@ class SessionLive {
         let leadersFirstSeqCounts = await this.sessionLiveState.get("firstCounts");
         let tries = 20;
 
-        while ((!leadersFirstSeqCounts.liveSourceMseqCount && tries > 0) || leadersFirstSeqCounts.liveSourceMseqCount === 0) {
+        while (!isLeader && (!leadersFirstSeqCounts.liveSourceMseqCount && tries > 0) || leadersFirstSeqCounts.liveSourceMseqCount === 0) {
           debug(`[${this.sessionId}]: NEW FOLLOWER: Waiting for LEADER to add 'firstCounts' in store! Will look again after 1000ms (tries left=${tries})`);
           await timer(1000);
           leadersFirstSeqCounts = await this.sessionLiveState.get("firstCounts");
           tries--;
+          // Might take over as Leader if Leader is not setting data due to being down.
+          isLeader = await this.sessionLiveStateStore.isLeader(this.instanceId);
+          if (isLeader) {
+            debug(`[${this.sessionId}][${this.instanceId}]: I'm the new leader, and now I am going to add 'firstCounts' in store`);
+          }
         }
 
         if (tries === 0) {
           isLeader = await this.sessionLiveStateStore.isLeader(this.instanceId);
           if (isLeader) {
-            debug(`[${this.instanceId}]: I'm the new leader, and now I am going to add 'firstCounts' in store`);
+            debug(`[${this.sessionId}][${this.instanceId}]: I'm the new leader, and now I am going to add 'firstCounts' in store`);
             break;
           } else {
-            debug(`[${this.instanceId}]: The leader is still alive`);
+            debug(`[${this.sessionId}][${this.instanceId}]: The leader is still alive`);
             leadersFirstSeqCounts = await this.sessionLiveState.get("firstCounts");
             if (!leadersFirstSeqCounts.liveSourceMseqCount) {
-              debug(`[${this.instanceId}]: Could not find 'firstCounts' in store. Abort Executing Promises II & Returning to Playhead.`);
+              debug(`[${this.sessionId}][${this.instanceId}]: Could not find 'firstCounts' in store. Abort Executing Promises II & Returning to Playhead.`);
               return;
             }
           }
         }
 
+        if (isLeader) {
+          debug(`[${this.sessionId}]: NEW LEADER: Original Leader went missing, I am retrying live source fetch...`);
+          await this.sessionLiveState.set("transitSegs", this.vodSegments);
+          debug(`[${this.sessionId}]: NEW LEADER: I am adding 'transitSegs' to Store for future followers`);
+          continue;
+        }
+
         // Respawners never do this, only starter followers.
         // Edge Case: FOLLOWER transitioned from session with different segments from LEADER
         if (leadersFirstSeqCounts.discSeqCount !== this.discSeqCount) {
@@ -713,6 +732,10 @@ class SessionLive {
         if (currentMseqRaw === leadersFirstSeqCounts.liveSourceMseqCount) {
           this.pushAmount = 1; // Follower from start
         } else {
+
+          // TODO: To support and account for past discontinuity tags in the Live Source stream,
+          // we will need to get the real 'current' discontinuity-sequence count from Leader somehow.
+
           // RESPAWNED NODES
           this.pushAmount = currentMseqRaw - leadersFirstSeqCounts.liveSourceMseqCount + 1;
 
@@ -772,7 +795,7 @@ class SessionLive {
       for (let i = 0; i < Object.keys(this.mediaManifestURIs).length; i++) {
         let bw = Object.keys(this.mediaManifestURIs)[i];
         // will add new segments to live seg queue
-        pushPromises.push(this._parseMediaManifest(this.liveSourceM3Us[bw].M3U, this.mediaManifestURIs[bw], bw));
+        pushPromises.push(this._parseMediaManifest(this.liveSourceM3Us[bw].M3U, this.mediaManifestURIs[bw], bw, isLeader));
         debug(`[${this.sessionId}]: Pushed pushPromise for bw=${bw}`);
       }
       // Segment Pushing
@@ -961,6 +984,12 @@ class SessionLive {
     // Update Session Live Media Sequence Count...
     this.prevMediaSeqCount = this.mediaSeqCount;
     this.mediaSeqCount += removedSegments;
+    if (this.restAmount) {
+      this.mediaSeqCount += this.restAmount;
+      debug(`[${this.sessionId}]: ${instanceName}: Added restAmount=[${this.restAmount}] to 'mediaSeqCount'`);
+      this.restAmount = 0;
+    }
+
     if (this.discSeqCount !== this.prevDiscSeqCount) {
       debug(`[${this.sessionId}]: ${instanceName}: Incrementing Dseq Count from {${this.prevDiscSeqCount}} -> {${this.discSeqCount}}`);
     }
@@ -1015,7 +1044,7 @@ class SessionLive {
     });
   }
 
-  _parseMediaManifest(m3u, mediaManifestUri, liveTargetBandwidth) {
+  _parseMediaManifest(m3u, mediaManifestUri, liveTargetBandwidth, isLeader) {
     return new Promise(async (resolve, reject) => {
       try {
         if (!this.liveSegQueue[liveTargetBandwidth]) {
@@ -1038,10 +1067,13 @@ class SessionLive {
         }
         this.targetDuration = m3u.get("targetDuration");
         let startIdx = m3u.items.PlaylistItem.length - this.pushAmount;
-        startIdx = startIdx < 0 ? 0 : startIdx;
+        if (startIdx < 0) {
+          this.restAmount = startIdx * -1;
+          startIdx = 0;
+        } 
         if (mediaManifestUri) {
           // push segments
-          this._addLiveSegmentsToQueue(startIdx, m3u.items.PlaylistItem, baseUrl, liveTargetBandwidth);
+          this._addLiveSegmentsToQueue(startIdx, m3u.items.PlaylistItem, baseUrl, liveTargetBandwidth, isLeader);
         }
         resolve();
       } catch (exc) {
@@ -1059,7 +1091,9 @@ class SessionLive {
    * @param {string} baseUrl
    * @param {string} liveTargetBandwidth
    */
-  _addLiveSegmentsToQueue(startIdx, playlistItems, baseUrl, liveTargetBandwidth) {
+  _addLiveSegmentsToQueue(startIdx, playlistItems, baseUrl, liveTargetBandwidth, isLeader) {
+    const leaderOrFollower = isLeader ? "LEADER": "NEW FOLLOWER";
+
     for (let i = startIdx; i < playlistItems.length; i++) {
       let seg = {};
       let playlistItem = playlistItems[i];
@@ -1127,10 +1161,15 @@ class SessionLive {
         if (daterangeData) {
           seg["daterange"] = daterangeData;
         }
-        // Push new Live Segments!
-        debug(`[${this.sessionId}]: Adding Live Segment to Queue (for bw=${liveTargetBandwidth})\n${seg.uri ? seg.uri : ''}`);
-        this.liveSegQueue[liveTargetBandwidth].push(seg);
-        this.liveSegsForFollowers[liveTargetBandwidth].push(seg);
+        // Push new Live Segments! But do not push duplicates
+        const liveSegURIs = this.liveSegQueue[liveTargetBandwidth].filter(seg => seg.uri).map(seg => seg.uri);
+        if (seg.uri && liveSegURIs.includes(seg.uri)) {
+          debug(`[${this.sessionId}]: ${leaderOrFollower}: Found duplicate live segment. Skip push! (${liveTargetBandwidth})`);
+        } else {
+          this.liveSegQueue[liveTargetBandwidth].push(seg);
+          this.liveSegsForFollowers[liveTargetBandwidth].push(seg);
+          debug(`[${this.sessionId}]: ${leaderOrFollower}: Pushed segment (${seg.uri ? seg.uri : "Disc-tag"}) to 'liveSegQueue' (${liveTargetBandwidth})`);
+        }
       }
     }
   }
@@ -1329,6 +1368,27 @@ class SessionLive {
     this.mediaManifestURIs = newItem;
   }
 
+  _getAnyFirstSegmentDurationMs() {
+    if (this._isEmpty(this.liveSegQueue)) {
+      return null;
+    }
+  
+    const bw0 = Object.keys(this.liveSegQueue)[0]; 
+    if (this.liveSegQueue[bw0].length === 0) {
+      return null;
+    }
+    
+    for (let i = 0; i < this.liveSegQueue[bw0].length; i++) {
+      const segment = this.liveSegQueue[bw0][i];
+      if (!segment.duration) {
+        continue;
+      }
+      return segment.duration * 1000;
+    }
+
+    return null;
+  }
+
   _isEmpty(obj) {
     if (!obj) {
       return true;

package/engine/session_state.js

@@ -7,6 +7,8 @@ const SessionState = Object.freeze({
   VOD_PLAYING: 2,
   VOD_NEXT_INIT: 3,
   VOD_NEXT_INITIATING: 4,
+  VOD_RELOAD_INIT: 5,
+  VOD_RELOAD_INITIATING: 6
 });
 
 const CURRENTVOD_CACHE_TTL = 5 * 60 * 1000; // 5 minutes
@@ -157,6 +159,11 @@ class SessionStateStore extends SharedStateStore {
     await this.setVolatile("", instanceId, t);
   }
 
+  async clearLeaderCache() {
+    debug(`[${this.instanceId}]: clearing 'leader' cache`);
+    this.cache.leader.value = null;
+  }
+
   async isLeader(instanceId) {
     if (!instanceId) {
       throw new Error("Cannot determine leader without instance id");
@@ -164,7 +171,7 @@ class SessionStateStore extends SharedStateStore {
     let leader;
     if (this.cache.leader.value && Date.now() < this.cache.leader.ts + this.cacheTTL) {
       leader = this.cache.leader.value;
-      debug(`[${instanceId}]: reading 'leader' from cache: I am ${leader === instanceId ? "" : "NOT"} the leader!`);
+      debug(`[${instanceId}]: reading 'leader' from cache: I am${leader === instanceId ? "" : " NOT"} the leader!`);
       return leader === instanceId;
     }
     leader = await this.get("", "leader");
@@ -189,7 +196,7 @@ class SessionStateStore extends SharedStateStore {
         }
       }
     }
-    debug(`[${instanceId}]: I am ${leader === instanceId ? "" : "NOT"} the leader!`);
+    debug(`[${instanceId}]: I am${leader === instanceId ? "" : " NOT"} the leader!`);
     this.cache.leader.ts = Date.now();
     this.cache.leader.value = leader;
     return leader === instanceId;

package/engine/stream_switcher.js

@@ -2,6 +2,8 @@ const debug = require("debug")("engine-stream-switcher");
 const crypto = require("crypto");
 const fetch = require("node-fetch");
 const { AbortController } = require("abort-controller");
+const { SessionState } = require('./session_state');
+const { timer } = require("./util")
 
 const SwitcherState = Object.freeze({
   V2L_TO_LIVE: 1,
@@ -88,8 +90,8 @@ class StreamSwitcher {
       return null;
     }
     // Handle Complete Storage Reset
-    let sessionState = await session._sessionState.get("state");
-    if (sessionState === 1 || !sessionState) {
+    let sessionState = await session.getSessionState();
+    if (sessionState === SessionState.VOD_INIT || !sessionState) {
       this.working = true;
       sessionLive.waitForPlayhead = false;
       sessionLive.allowedToSet = false; // only have effect on leader
@@ -150,6 +152,11 @@ class StreamSwitcher {
       );
       validURI = await this._validURI(scheduleObj.uri);
       tries++;
+      if (!validURI) {
+        const delayMs = tries * 500;
+        debug(`[${this.sessionId}]: Going to try validating Master URI again in ${delayMs}ms`);
+        await timer(delayMs);
+      }
     }
     if (!validURI) {
       debug(`[${this.sessionId}]: Unreachable URI: [${scheduleObj.uri}]`);
@@ -341,14 +348,14 @@ class StreamSwitcher {
             return false;
           }
 
-          await session.setCurrentMediaSequenceSegments(
-            liveSegments.currMseqSegs,
-            liveSegments.segCount
-          );
           await session.setCurrentMediaAndDiscSequenceCount(
             liveCounts.mediaSeq,
             liveCounts.discSeq
           );
+          await session.setCurrentMediaSequenceSegments(
+            liveSegments.currMseqSegs,
+            liveSegments.segCount
+          );
 
           this.working = false;
           this.streamTypeLive = false;

package/engine/util.js

@@ -122,6 +122,34 @@ const logerror = (sessionId, err) => {
   console.error(err);
 };
 
+const timer = (ms) => new Promise((res) => setTimeout(res, ms));
+
+class WaitTimeGenerator {
+  constructor(defaultIntervalMs, minValue) {
+    (this.timestamp = null), (this.prevWaitTime = null), (this.defaultIntervalMs = defaultIntervalMs || 3000), (this.minValue = minValue);
+  }
+  _getWaitTimeFromTimestamp() {
+    if (!this.timestamp) {
+      this.timestamp = new Date();
+    }
+    const sec = this.timestamp.getSeconds();
+    const defaultSec = this.defaultIntervalMs / 1000;
+    const d = parseInt(sec / defaultSec);
+    const waitSec = defaultSec * (d + 1) - sec;
+    return waitSec * 1000;
+  }
+  async getWaitTime(plannedTime) {
+    if (!this.timestamp || (this.prevWaitTime === this.minValue && plannedTime !== this.minValue)) {
+      this.timestamp = new Date();
+      const waitMs = this._getWaitTimeFromTimestamp();
+      this.prevWaitTime = waitMs;
+      return waitMs;
+    }
+    this.prevWaitTime = plannedTime;
+    return plannedTime;
+  }
+}
+
 module.exports = {
   filterQueryParser,
   applyFilter,
@@ -129,4 +157,6 @@ module.exports = {
   m3u8Header,
   toHHMMSS,
   logerror,
+  timer,
+  WaitTimeGenerator,
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "eyevinn-channel-engine",
-  "version": "3.1.7",
+  "version": "3.2.0",
   "description": "OTT TV Channel Engine",
   "main": "index.js",
   "scripts": {