@warren-bank/hls-proxy 3.3.0 → 3.4.0

package/README.md

@@ -128,6 +128,8 @@ options:
 --max-segments <number>
 --cache-timeout <number>
 --cache-key <number>
+--cache-storage <adapter>
+--cache-storage-fs-dirpath <dirpath>
 -v <number>
 --acl-whitelist <ip_address_list>
 --http-proxy <http[s]://[user:pass@]hostname:port>
@@ -328,6 +330,15 @@ options:
     * full filename of .ts file
   * `2`:
     * full URL of .ts file
+* _--cache-storage_ selects a storage adapter that is used to hold the cache of prefetched video segments
+  * this option is only meaningful when _--prefetch_ is enabled
+  * `memory` (default):
+    * uses RAM
+  * `filesystem`:
+    * each video segment is written to a new file within a specified directory
+    * filenames are random and unique
+* _--cache-storage-fs-dirpath_ specifies the directory in which to save video segments when using a filesystem-based cache storage adapter
+  * this option is only meaningful when _--prefetch_ is enabled and _--cache-storage_ is `filesystem`
 * _-v_ sets logging verbosity level:
   * `-1`:
     * silent

package/hls-proxy/bin/hlsd.js

@@ -32,18 +32,20 @@ const server = (use_tls)
     })
 
 const middleware = require('../proxy')({
-  is_secure:      use_tls,
-  host:           normalize_host(argv_vals["--host"], argv_vals["--port"]),
-  req_headers:    argv_vals["--req-headers"],
-  req_options:    argv_vals["--req-options"],
-  hooks:          argv_vals["--hooks"],
-  cache_segments: argv_vals["--prefetch"],
-  max_segments:   argv_vals["--max-segments"],
-  cache_timeout:  argv_vals["--cache-timeout"],
-  cache_key:      argv_vals["--cache-key"],
-  debug_level:    argv_vals["-v"],
-  acl_whitelist:  argv_vals["--acl-whitelist"],
-  http_proxy:     argv_vals["--http-proxy"]
+  is_secure:                            use_tls,
+  host:                                 normalize_host(argv_vals["--host"], argv_vals["--port"]),
+  req_headers:                          argv_vals["--req-headers"],
+  req_options:                          argv_vals["--req-options"],
+  hooks:                                argv_vals["--hooks"],
+  cache_segments:                       argv_vals["--prefetch"],
+  max_segments:                         argv_vals["--max-segments"],
+  cache_timeout:                        argv_vals["--cache-timeout"],
+  cache_key:                            argv_vals["--cache-key"],
+  cache_storage:                        argv_vals["--cache-storage"],
+  cache_storage_fs_dirpath:             argv_vals["--cache-storage-fs-dirpath"],
+  debug_level:                          argv_vals["-v"],
+  acl_whitelist:                        argv_vals["--acl-whitelist"],
+  http_proxy:                           argv_vals["--http-proxy"]
 })
 
 if (middleware.connection)

package/hls-proxy/bin/lib/help.js

@@ -24,6 +24,8 @@ options:
 --max-segments <number>
 --cache-timeout <number>
 --cache-key <number>
+--cache-storage <adapter>
+--cache-storage-fs-dirpath <dirpath>
 -v <number>
 --acl-whitelist <ip_address_list>
 --http-proxy <http[s]://[user:pass@]hostname:port>

package/hls-proxy/bin/lib/process_argv.js

@@ -29,6 +29,8 @@ const argv_flags = {
   "--max-segments":                         {num:  "int"},
   "--cache-timeout":                        {num:  "int"},
   "--cache-key":                            {num:  "int"},
+  "--cache-storage":                        {enum: ["memory", "filesystem"]},
+  "--cache-storage-fs-dirpath":             {file: "path-exists"},
 
   "-v":                                     {num:  "int"},
   "--acl-whitelist":                        {},

package/hls-proxy/manifest_parser.js

@@ -121,10 +121,12 @@ const parse_manifest = function(m3u8_content, m3u8_url, referer_url, hooks, cach
   if (embedded_urls && Array.isArray(embedded_urls) && embedded_urls.length) {
     embedded_urls.forEach(embedded_url => {
       redirect_embedded_url(embedded_url, hooks, m3u8_url, debug)
-      finalize_embedded_url(embedded_url, vod_start_at_ms, debug)
-      encode_embedded_url(embedded_url, redirected_base_url, debug)
-      get_prefetch_url(embedded_url, should_prefetch_url, prefetch_urls)
-      modify_m3u8_line(embedded_url, m3u8_lines)
+      if (validate_embedded_url(embedded_url)) {
+        finalize_embedded_url(embedded_url, vod_start_at_ms, debug)
+        encode_embedded_url(embedded_url, redirected_base_url, debug)
+        get_prefetch_url(embedded_url, should_prefetch_url, prefetch_urls)
+        modify_m3u8_line(embedded_url, m3u8_lines)
+      }
     })
   }
 
@@ -291,6 +293,21 @@ const redirect_embedded_url = function(embedded_url, hooks, m3u8_url, debug) {
   }
 }
 
+const validate_embedded_url = function(embedded_url) {
+  if (embedded_url.redirected_url === '') {
+    return true
+  }
+  else {
+    const url = new URL(
+      embedded_url.redirected_url || embedded_url.resolved_match_url
+    )
+
+    const supported_protocols = ['http:','https:']
+
+    return (url.protocol && (supported_protocols.indexOf(url.protocol.toLowerCase()) >= 0))
+  }
+}
+
 const finalize_embedded_url = function(embedded_url, vod_start_at_ms, debug) {
   if (embedded_url.redirected_url === '') {
     embedded_url.unencoded_url = ''

package/hls-proxy/proxy.js

@@ -35,7 +35,7 @@ const get_middleware = function(params) {
   }
 
   // Create an HTTP tunneling proxy
-  middleware.request = (req, res) => {
+  middleware.request = async (req, res) => {
     debug(3, 'proxying (raw):', req.url)
 
     utils.add_CORS_headers(res)
@@ -56,13 +56,13 @@ const get_middleware = function(params) {
     }
 
     if (cache_segments && !is_m3u8) {
-      let segment = get_segment(url, url_type)  // Buffer (cached segment data), false (prefetch is pending: add callback), undefined (no prefetch is pending)
+      let segment = await get_segment(url, url_type)  // Buffer (cached segment data), false (prefetch is pending: add callback), undefined (no prefetch is pending)
 
-      if (segment && segment.length) {          // Buffer (cached segment data)
+      if (segment && segment.length) {                // Buffer (cached segment data)
         send_cache_segment(segment)
         return
       }
-      else if (segment === false) {             // false (prefetch is pending: add callback)
+      else if (segment === false) {                   // false (prefetch is pending: add callback)
         add_listener(url, url_type, send_cache_segment)
         return
       }

package/hls-proxy/segment_cache.js

@@ -7,9 +7,10 @@ module.exports = function(params) {
 
   if (!cache_segments) return {}
 
-  const debug               = utils.debug.bind(null, params)
-  const get_request_options = utils.get_request_options.bind(null, params)
-  const should_prefetch_url = utils.should_prefetch_url.bind(null, params)
+  const debug                 = utils.debug.bind(null, params)
+  const get_request_options   = utils.get_request_options.bind(null, params)
+  const should_prefetch_url   = utils.should_prefetch_url.bind(null, params)
+  const cache_storage_adapter = require('./segment_cache_storage')(params)
 
   // maps: "m3u8_url" => {access: timestamp, ts: []}
   const cache = {}
@@ -35,10 +36,10 @@ module.exports = function(params) {
   }
 
   const clear_ts = function(m3u8_url) {
-    const data = get_cache(m3u8_url)
-    if (!data) return
+    const ts = get_ts(m3u8_url)
+    if (!ts || !ts.length) return
 
-    data.ts = []
+    ts_garbage_collect(m3u8_url, 0, ts.length - 1)
   }
 
   const get_timestamp = function() {
@@ -78,13 +79,20 @@ module.exports = function(params) {
 
   const ts_garbage_collect = function(m3u8_url, start, count) {
     const ts = get_ts(m3u8_url)
-    if (!ts) return
+    if (!ts || !ts.length || (start >= ts.length)) return
 
     for (let i=start; i < (start + count); i++) {
       if (i >= ts.length) break
 
-      ts[i].databuffer = undefined
+      const segment = ts[i]
+
+      if (segment.has)
+        cache_storage_adapter.remove(segment.state)
+
+      segment.has = false
+      segment.cb  = []
     }
+
     ts.splice(start, count)
   }
 
@@ -132,7 +140,7 @@ module.exports = function(params) {
     let i, segment
 
     for (i=(ts.length - 1); i>=0; i--) {
-      segment = ts[i]  // {key, databuffer}
+      segment = ts[i]  // {key, has, cb, state}
       if (segment && (segment.key === key)) {
         index = i
         break
@@ -173,7 +181,7 @@ module.exports = function(params) {
 
       // placeholder to prevent multiple download requests
       index = ts.length
-      ts[index] = {key: get_privatekey_from_url(url), databuffer: false}
+      ts[index] = {key: get_privatekey_from_url(url), has: false, cb: [], state: {}}
 
       let options = get_request_options(url, /* is_m3u8= */ false, referer_url)
       promise = request(options, '', {binary: true, stream: false, cookieJar: cookies.getCookieJar()})
@@ -188,15 +196,17 @@ module.exports = function(params) {
         if (!dont_touch_access)
           touch_access(m3u8_url)
 
-        let segment = ts[index].databuffer
-        if (segment && (segment instanceof Array)) {
-          segment.forEach((cb) => {
+        const segment = ts[index]
+        segment.has = true
+        cache_storage_adapter.set(segment.state, response)
+        if (segment.cb.length) {
+          segment.cb.forEach((cb) => {
             cb(response)
 
             debug(1, 'cache (callback complete):', debug_url)
           })
+          segment.cb = []
         }
-        ts[index].databuffer = response
 
         // cleanup: prune length of ts[] so it contains no more than "max_segments"
         if (ts.length > max_segments) {
@@ -217,34 +227,38 @@ module.exports = function(params) {
     return promise
   }
 
-  const get_segment = function(url, url_type) {
+  const get_segment = async function(url, url_type) {
     if (! should_prefetch_url(url, url_type)) return undefined
 
     let debug_url = (debug_level >= 3) ? url : get_publickey_from_url(url)
+    let segment   = find_segment(url)
 
-    let segment = find_segment(url)
     if (segment !== undefined) {
       const {m3u8_url, index} = segment
       const ts = get_ts(m3u8_url)
       touch_access(m3u8_url)
 
-      segment = ts[index].databuffer
+      segment = ts[index]
+
+      if (segment.has) {
+        debug(1, 'cache (hit):', debug_url)
+
+        segment = await cache_storage_adapter.get(segment.state)
 
-      if ((segment === false) || (segment instanceof Array)) {
+        // cleanup: remove all previous segments
+        // =====================================
+        // todo:
+        //   - why does this sometimes cause the video player to get stuck.. repeatedly request the .m3u8 file, but stop requesting any .ts segments?
+        //   - is it a coincidence that commenting this line appears to stop such behavior?
+        //   - could it possibly be a race condition? cleanup also occurs asynchronously when prefetch responses are received, but javascript (node) is single threaded.. and this code doesn't yield or use a timer.
+        // =====================================
+        // ts_garbage_collect(m3u8_url, 0, (index + 1))
+      }
+      else {
         debug(1, 'cache (pending prefetch):', debug_url)
 
-        return false
+        segment = false
       }
-      debug(1, 'cache (hit):', debug_url)
-
-      // cleanup: remove all previous segments
-      // =====================================
-      // todo:
-      //   - why does this sometimes cause the video player to get stuck.. repeatedly request the .m3u8 file, but stop requesting any .ts segments?
-      //   - is it a coincidence that commenting this line appears to stop such behavior?
-      //   - could it possibly be a race condition? cleanup also occurs asynchronously when prefetch responses are received, but javascript (node) is single threaded.. and this code doesn't yield or use a timer.
-      // =====================================
-      // ts_garbage_collect(m3u8_url, 0, (index + 1))
     }
     else {
       debug(1, 'cache (miss):', debug_url)
@@ -263,22 +277,17 @@ module.exports = function(params) {
       const ts = get_ts(m3u8_url)
       touch_access(m3u8_url)
 
-      segment = ts[index].databuffer
-
-      if (segment === false) {
-        ts[index].databuffer = [cb]
+      segment = ts[index]
 
-        debug(1, 'cache (callback added):', debug_url)
-      }
-      else if (segment instanceof Array) {
-        ts[index].databuffer.push(cb)
+      if (segment.has) {
+        cb(segment)
 
-        debug(1, 'cache (callback added):', debug_url)
+        debug(1, 'cache (callback complete):', debug_url)        
       }
       else {
-        cb(segment)
+        segment.cb.push(cb)
 
-        debug(1, 'cache (callback complete):', debug_url)
+        debug(1, 'cache (callback added):', debug_url)
       }
     }
     return true

package/hls-proxy/segment_cache_storage/filesystem/index.js

@@ -0,0 +1,56 @@
+const crypto = require('crypto')
+const fs     = require('fs')
+const path   = require('path')
+
+const {denodeify} = require('@warren-bank/node-request')
+
+const $fs = {
+  writeFile: denodeify(fs.writeFile),
+  readFile:  denodeify(fs.readFile),
+  rm:        denodeify(fs.rm)
+}
+
+module.exports = function(dirpath) {
+
+  // synchronous (private)
+  const get_random_filename = (state) => {
+    let random_bytes, fname, fpath
+
+    while (true) {
+      random_bytes = get_random_bytes()
+      fname        = convert_random_bytes_to_filename(random_bytes)
+      fpath        = path.join(dirpath, fname)
+
+      if (!fs.existsSync(fpath)) {
+        state.fpath = fpath
+        return
+      }
+    }
+  }
+
+  // synchronous (private)
+  const get_random_bytes = () => crypto.randomBytes(30)
+
+  // synchronous (private)
+  const convert_random_bytes_to_filename = (buffer) => buffer.toString('base64').replaceAll('/', '_')
+
+  // async
+  const set = async (state, blob) => {
+    get_random_filename(state)
+
+    await $fs.writeFile(state.fpath, blob)
+  }
+
+  // async
+  const get = async (state) => {
+    return await $fs.readFile(state.fpath, {encoding: null})
+  }
+
+  // async
+  const remove = async (state) => {
+    await $fs.rm(state.fpath, {force: true})
+    delete state.fpath
+  }
+
+  return {set, get, remove}
+}

package/hls-proxy/segment_cache_storage/index.js

@@ -0,0 +1,14 @@
+module.exports = function(params) {
+  const {cache_storage, cache_storage_fs_dirpath} = params
+
+  if (cache_storage) {
+    if (cache_storage === 'memory')
+      return require('./memory')()
+
+    if ((cache_storage === 'filesystem') && cache_storage_fs_dirpath)
+      return require('./filesystem')(cache_storage_fs_dirpath)
+  }
+
+  // default
+  return require('./memory')()
+}

package/hls-proxy/segment_cache_storage/memory/index.js

@@ -0,0 +1,17 @@
+module.exports = function() {
+
+  // async
+  const set = async (state, blob) => {
+    state.databuffer = blob
+  }
+
+  // async
+  const get = async (state) => state.databuffer
+
+  // async
+  const remove = async (state) => {
+    delete state.databuffer
+  }
+
+  return {set, get, remove}
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@warren-bank/hls-proxy",
   "description": "Node.js server to proxy HLS video streams",
-  "version":     "3.3.0",
+  "version":     "3.4.0",
   "scripts": {
     "start":     "node hls-proxy/bin/hlsd.js",
     "sudo": "sudo node hls-proxy/bin/hlsd.js"
@@ -12,7 +12,7 @@
   "dependencies": {
     "@warren-bank/node-process-argv": "^1.2.1",
     "@warren-bank/node-request": "^2.0.12",
-    "@warren-bank/url": "^3.1.0",
+    "@warren-bank/url": "^3.1.2",
     "hpagent": "^1.2.0",
     "tough-cookie": "^3.0.1"
   },