@warren-bank/hls-proxy 3.5.3 → 3.6.0

package/.gitattributes

@@ -1,3 +1,4 @@
+# ------------------------------------------------
 # https://github.com/github/linguist
 # https://github.com/github/linguist#vendored-code
 # https://github.com/github/linguist/blob/master/lib/linguist/languages.yml
@@ -6,3 +7,13 @@
 .related/* linguist-vendored=true
 
 * linguist-language=JavaScript
+
+# ------------------------------------------------
+
+* text=auto
+
+*.cmd text eol=crlf
+*.bat text eol=crlf
+*.sh  text eol=lf
+
+# ------------------------------------------------

package/README.md

@@ -57,6 +57,19 @@
       const hls_proxy_url  = `${proxy_url}/${ btoa(`${video_url}|${referer_url}`) }${file_extension}`
     }
   ```
+* [example Javascript]: construction of URL to _HLS Proxy_ for video stream w/ "Referer" and additional request headers
+  ```javascript
+    {
+      const proxy_url      = 'http://127.0.0.1:8080'
+      const video_url      = 'https://example.com/video/master.m3u8'
+      const referer_url    = 'https://example.com/videos.html'
+      const file_extension = '.m3u8'
+      const headers        = {"Cookie": "foo=bar"}
+
+      const querystring    = `?headers=${ encodeURIComponent( btoa( JSON.stringify(headers) ) ) }`
+      const hls_proxy_url  = `${proxy_url}/${ btoa(`${video_url}|${referer_url}`) }${file_extension}${querystring}`
+    }
+  ```
 * [example Bash]: construction of URL to _HLS Proxy_ for video stream
   ```bash
     proxy_url='http://127.0.0.1:8080'
@@ -74,6 +87,17 @@
 
     hls_proxy_url="${proxy_url}/"$(echo -n "${video_url}|${referer_url}" | base64 --wrap=0)"$file_extension"
   ```
+* [example Bash]: construction of URL to _HLS Proxy_ for video stream w/ "Referer" and additional request headers
+  ```bash
+    proxy_url='http://127.0.0.1:8080'
+    video_url='https://example.com/video/master.m3u8'
+    referer_url='https://example.com/videos.html'
+    file_extension='.m3u8'
+    headers_json='{"Cookie": "foo=bar"}'
+
+    querystring='?headers='$(echo -n "$headers_json" | base64 --wrap=0)
+    hls_proxy_url="${proxy_url}/"$(echo -n "${video_url}|${referer_url}" | base64 --wrap=0)"${file_extension}${querystring}"
+  ```
 
 ##### notes:
 

package/hls-proxy/expressjs_utils.js

@@ -1,3 +1,5 @@
+const {URL} = require('./url')
+
 const get_full_req_url = function(req) {
   return req.originalUrl || req.url
 }
@@ -6,6 +8,10 @@ const has_req_param = function(req, key) {
   return (req.params && (typeof req.params === 'object') && req.params[key])
 }
 
+const has_req_query = function(req, key) {
+  return (req.query && (typeof req.query === 'object') && req.query[key])
+}
+
 const get_proxy_req_url = function(req) {
   const key = "0"
   return has_req_param(req, key)
@@ -13,6 +19,18 @@ const get_proxy_req_url = function(req) {
     : req.url
 }
 
+const get_proxy_req_query = function(req, key) {
+  if (has_req_param(req, key))
+    return req.params[key]
+
+  if (has_req_query(req, key))
+    return req.query[key]
+
+  const url = new URL(req.url)
+  const qs  = url.searchParams
+  return qs.get(key)
+}
+
 const get_base_req_url = function(req) {
   let base_url = ''
   const key = "0"
@@ -28,5 +46,6 @@ const get_base_req_url = function(req) {
 module.exports = {
   get_full_req_url,
   get_proxy_req_url,
+  get_proxy_req_query,
   get_base_req_url
 }

package/hls-proxy/manifest_parser.js

@@ -112,12 +112,13 @@ const parse_HHMMSS_to_seconds = function(str) {
 //   prefetch_urls: [],
 //   modified_m3u8: ''
 // }
-const parse_manifest = function(m3u8_content, m3u8_url, referer_url, hooks, cache_segments, debug, vod_start_at_ms, redirected_base_url, should_prefetch_url, manifest_extension, segment_extension, qs_password) {
+const parse_manifest = function(m3u8_content, m3u8_url, referer_url, querystring_req_headers, hooks, cache_segments, debug, vod_start_at_ms, redirected_base_url, should_prefetch_url, manifest_extension, segment_extension, qs_password) {
   const m3u8_lines = m3u8_content.split(regexs.m3u8_line_separator)
   m3u8_content = null
 
   const meta_data     = {}
   const embedded_urls = extract_embedded_urls(m3u8_lines, m3u8_url, referer_url, (cache_segments ? meta_data : null))
+  const qs_headers    = !!querystring_req_headers ? utils.base64_encode(JSON.stringify(querystring_req_headers)) : null
   const prefetch_urls = []
 
   if (embedded_urls && Array.isArray(embedded_urls) && embedded_urls.length) {
@@ -125,7 +126,7 @@ const parse_manifest = function(m3u8_content, m3u8_url, referer_url, hooks, cach
       redirect_embedded_url(embedded_url, hooks, m3u8_url, debug)
       if (validate_embedded_url(embedded_url)) {
         finalize_embedded_url(embedded_url, vod_start_at_ms, debug)
-        encode_embedded_url(embedded_url, hooks, redirected_base_url, debug, manifest_extension, segment_extension, qs_password)
+        encode_embedded_url(embedded_url, hooks, redirected_base_url, debug, manifest_extension, segment_extension, qs_headers, qs_password)
         get_prefetch_url(embedded_url, should_prefetch_url, prefetch_urls)
         modify_m3u8_line(embedded_url, m3u8_lines)
       }
@@ -332,7 +333,7 @@ const finalize_embedded_url = function(embedded_url, vod_start_at_ms, debug) {
   }
 }
 
-const encode_embedded_url = function(embedded_url, hooks, redirected_base_url, debug, manifest_extension, segment_extension, qs_password) {
+const encode_embedded_url = function(embedded_url, hooks, redirected_base_url, debug, manifest_extension, segment_extension, qs_headers, qs_password) {
   if (embedded_url.unencoded_url) {
     let file_extension = embedded_url.url_type
     if (file_extension) {
@@ -344,9 +345,6 @@ const encode_embedded_url = function(embedded_url, hooks, redirected_base_url, d
 
     embedded_url.encoded_url = `${redirected_base_url}/${ utils.base64_encode(embedded_url.unencoded_url) }.${file_extension || 'other'}`
 
-    if (qs_password)
-      embedded_url.encoded_url += `?password=${qs_password}`
-
     debug(3, 'redirecting (proxied):', embedded_url.encoded_url)
 
     if (hooks && (hooks instanceof Object) && hooks.redirect_final && (typeof hooks.redirect_final === 'function')) {
@@ -354,6 +352,18 @@ const encode_embedded_url = function(embedded_url, hooks, redirected_base_url, d
 
       debug(3, 'redirecting (proxied, post-hook):', embedded_url.encoded_url)
     }
+
+    let qs_pairs = []
+    if (qs_headers)
+      qs_pairs.push(['headers', qs_headers])
+    if (qs_password)
+      qs_pairs.push(['password', qs_password])
+    if (qs_pairs.length) {
+      qs_pairs = qs_pairs.map(pair => `${pair[0]}=${encodeURIComponent(pair[1])}`)
+
+      embedded_url.encoded_url += '?' + qs_pairs.join('&')
+      debug(3, 'redirecting (proxied, with querystring):', embedded_url.encoded_url)
+    }
   }
   else {
     embedded_url.encoded_url = ''
@@ -382,7 +392,7 @@ const modify_m3u8_line = function(embedded_url, m3u8_lines) {
   }
 }
 
-const modify_m3u8_content = function(params, segment_cache, m3u8_content, m3u8_url, referer_url, inbound_req_headers, redirected_base_url, qs_password) {
+const modify_m3u8_content = function(params, segment_cache, m3u8_content, m3u8_url, referer_url, querystring_req_headers, inbound_req_headers, redirected_base_url, qs_password) {
   const {hooks, cache_segments, max_segments, debug_level, manifest_extension, segment_extension} = params
 
   const {has_cache, get_time_since_last_access, is_expired, prefetch_segment} = segment_cache
@@ -424,13 +434,13 @@ const modify_m3u8_content = function(params, segment_cache, m3u8_content, m3u8_u
           const matching_url = urls[0]
           urls[0] = undefined
 
-          promise = prefetch_segment(m3u8_url, matching_url, referer_url, inbound_req_headers, dont_touch_access)
+          promise = prefetch_segment(m3u8_url, matching_url, referer_url, querystring_req_headers, inbound_req_headers, dont_touch_access)
         }
 
         promise.then(() => {
           urls.forEach((matching_url, index) => {
             if (matching_url) {
-              prefetch_segment(m3u8_url, matching_url, referer_url, inbound_req_headers, dont_touch_access)
+              prefetch_segment(m3u8_url, matching_url, referer_url, querystring_req_headers, inbound_req_headers, dont_touch_access)
 
               urls[index] = undefined
             }
@@ -440,7 +450,7 @@ const modify_m3u8_content = function(params, segment_cache, m3u8_content, m3u8_u
     : null
 
   {
-    const parsed_manifest = parse_manifest(m3u8_content, m3u8_url, referer_url, hooks, cache_segments, debug, vod_start_at_ms, redirected_base_url, should_prefetch_url, manifest_extension, segment_extension, qs_password)
+    const parsed_manifest = parse_manifest(m3u8_content, m3u8_url, referer_url, querystring_req_headers, hooks, cache_segments, debug, vod_start_at_ms, redirected_base_url, should_prefetch_url, manifest_extension, segment_extension, qs_password)
     is_vod          = !!parsed_manifest.meta_data.is_vod                  // default: false => hls live stream
     seg_duration_ms = parsed_manifest.meta_data.seg_duration_ms || 10000  // default: 10 seconds in ms
     prefetch_urls   = parsed_manifest.prefetch_urls

package/hls-proxy/proxy.js

@@ -47,7 +47,7 @@ const get_middleware = function(params) {
 
     utils.add_CORS_headers(res)
 
-    const {redirected_base_url, url_type, url, referer_url} = parse_req_url(req)
+    const {redirected_base_url, url_type, url, referer_url, querystring_req_headers} = parse_req_url(req)
 
     if (!url) {
       res.writeHead(400)
@@ -79,7 +79,7 @@ const get_middleware = function(params) {
       }
     }
 
-    const options = get_request_options(url, is_m3u8, referer_url, req.headers)
+    const options = get_request_options(url, is_m3u8, referer_url, querystring_req_headers, req.headers)
     debug(1, 'proxying:', url)
     debug(3, 'm3u8:', (is_m3u8 ? 'true' : 'false'))
 
@@ -107,7 +107,7 @@ const get_middleware = function(params) {
           : url
 
         res.writeHead(200, { "content-type": "application/x-mpegURL" })
-        res.end( modify_m3u8_content(response.toString().trim(), m3u8_url, referer_url, req.headers, redirected_base_url, qs_password) )
+        res.end( modify_m3u8_content(response.toString().trim(), m3u8_url, referer_url, querystring_req_headers, req.headers, redirected_base_url, qs_password) )
       }
     })
     .catch((e) => {

package/hls-proxy/segment_cache.js

@@ -161,7 +161,7 @@ module.exports = function(params) {
     }
   }
 
-  const prefetch_segment = function(m3u8_url, url, referer_url, inbound_req_headers, dont_touch_access) {
+  const prefetch_segment = function(m3u8_url, url, referer_url, querystring_req_headers, inbound_req_headers, dont_touch_access) {
     let promise = Promise.resolve()
 
     if (cache[m3u8_url] === undefined) {
@@ -184,7 +184,7 @@ module.exports = function(params) {
       index = ts.length
       ts[index] = {key: get_privatekey_from_url(url), has: false, cb: [], type: null, state: {}}
 
-      let options = get_request_options(url, /* is_m3u8= */ false, referer_url, inbound_req_headers)
+      let options = get_request_options(url, /* is_m3u8= */ false, referer_url, querystring_req_headers, inbound_req_headers)
       promise = request(options, '', {binary: true, stream: false, cookieJar: cookies.getCookieJar()})
       .then(({redirects, response}) => {
         debug(1, `prefetch (complete, ${response.length} bytes):`, debug_url)

package/hls-proxy/timers.js

@@ -11,7 +11,7 @@ const initialize_timers = function(params) {
     const get_request_options = utils.get_request_options.bind(null, params)
 
     const request_wrapper = function(url, POST_data, user_config) {
-      const options = get_request_options(url, /* is_m3u8= */ false, /* referer_url= */ null, /* inbound_req_headers= */ null)
+      const options = get_request_options(url, /* is_m3u8= */ false, /* referer_url= */ null, /* querystring_req_headers= */ null, /* inbound_req_headers= */ null)
       const config  = Object.assign(
         {},
         (user_config || {}),

package/hls-proxy/utils.js

@@ -19,7 +19,7 @@ const base64_decode = function(str) {
 const parse_req_url = function(params, req) {
   const {is_secure, host, manifest_extension, segment_extension, hooks} = params
 
-  const result = {redirected_base_url: '', url_type: '', url: '', referer_url: ''}
+  const result = {redirected_base_url: '', url_type: '', url: '', referer_url: '', querystring_req_headers: null}
 
   const matches = regexs.req_url.exec( expressjs.get_proxy_req_url(req) )
 
@@ -56,6 +56,18 @@ const parse_req_url = function(params, req) {
 
       result.url = url
     }
+
+    let qs_headers = expressjs.get_proxy_req_query(req, 'headers')
+    if (qs_headers) {
+      try {
+        qs_headers = base64_decode( decodeURIComponent( qs_headers ) ).trim()
+        qs_headers = JSON.parse(qs_headers)
+
+        if (qs_headers && (qs_headers instanceof Object))
+          result.querystring_req_headers = qs_headers
+      }
+      catch(e) {}
+    }
   }
 
   return result
@@ -120,21 +132,31 @@ const debug = function() {
   }
 }
 
-const normalize_req_headers = function(req_headers) {
+const normalize_req_headers = function(req_headers, blacklist) {
   const normalized = {}
 
+  if (blacklist && !Array.isArray(blacklist))
+    blacklist = null
+  if (blacklist)
+    blacklist = blacklist.filter(val => val && (typeof val === 'string')).map(val => val.toLowerCase())
+  if (blacklist && !blacklist.length)
+    blacklist = null
+
   for (let name in req_headers) {
-    normalized[ name.toLowerCase() ] = req_headers[name]
+    const lc_name = name.toLowerCase()
+
+    if (!blacklist || (blacklist.indexOf(lc_name) === -1))
+      normalized[lc_name] = req_headers[name]
   }
 
   return normalized
 }
 
-const get_request_options = function(params, url, is_m3u8, referer_url, inbound_req_headers) {
+const get_request_options = function(params, url, is_m3u8, referer_url, querystring_req_headers, inbound_req_headers) {
   const {copy_req_headers, req_headers, req_options, hooks, http_proxy} = params
 
   const copied_req_headers = (copy_req_headers && inbound_req_headers && (inbound_req_headers instanceof Object))
-    ? normalize_req_headers(inbound_req_headers)
+    ? normalize_req_headers(inbound_req_headers, ['host'])
     : null
 
   const additional_req_options = (hooks && (hooks instanceof Object) && hooks.add_request_options && (typeof hooks.add_request_options === 'function'))
@@ -145,7 +167,7 @@ const get_request_options = function(params, url, is_m3u8, referer_url, inbound_
     ? hooks.add_request_headers(url, is_m3u8)
     : null
 
-  if (!req_options && !http_proxy && !additional_req_options && !copied_req_headers && !req_headers && !additional_req_headers && !referer_url) return url
+  if (!req_options && !http_proxy && !additional_req_options && !copied_req_headers && !req_headers && !additional_req_headers && !referer_url && !querystring_req_headers) return url
 
   const request_options = Object.assign(
     {},
@@ -161,7 +183,8 @@ const get_request_options = function(params, url, is_m3u8, referer_url, inbound_
     ((additional_req_options && additional_req_options.headers) ? additional_req_options.headers : {}),
     (req_headers             || {}),
     (additional_req_headers  || {}),
-    (referer_url ? {"referer": referer_url, "origin": referer_url.replace(regexs.origin, '$1')} : {})
+    (referer_url ? {"referer": referer_url, "origin": referer_url.replace(regexs.origin, '$1')} : {}),
+    (querystring_req_headers || {})
   )
 
   // normalize

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@warren-bank/hls-proxy",
   "description": "Node.js server to proxy HLS video streams",
-  "version":     "3.5.3",
+  "version":     "3.6.0",
   "scripts": {
     "start":     "node hls-proxy/bin/hlsd.js",
     "sudo": "sudo node hls-proxy/bin/hlsd.js"