@warren-bank/hls-proxy 3.2.4 → 3.3.0

package/README.md

@@ -110,7 +110,7 @@ options:
 --help
 --version
 --tls
---host <ip_address>
+--host <host>
 --port <number>
 --req-headers <filepath>
 --origin <header>
@@ -130,6 +130,7 @@ options:
 --cache-key <number>
 -v <number>
 --acl-whitelist <ip_address_list>
+--http-proxy <http[s]://[user:pass@]hostname:port>
 --tls-cert <filepath>
 --tls-key <filepath>
 --tls-pass <filepath>
@@ -342,6 +343,9 @@ options:
     * show the content of .m3u8 files (both before and after URLs are modified)
 * _--acl-whitelist_ restricts proxy server access to clients at IP addresses in whitelist
   * ex: `"192.168.1.100,192.168.1.101,192.168.1.102"`
+* --http-proxy enables all outbound HTTP and HTTPS requests from HLS-Proxy to be tunnelled through an additional external web proxy server
+  * SOCKS proxies are not supported
+  * ex: `http://myusername:mypassword@myproxy.example.com:1234`
 * _--tls-cert_ is the filepath to a security certificate to use for HTTPS
 * _--tls-key_ is the filepath to the private key for the _--tls-cert_ security certificate
 * _--tls-pass_ is the filepath to a text file containing the security passphrase for the _--tls-key_ private key
@@ -600,7 +604,8 @@ curl --silent --insecure "$URL"
     * internal `proxy` module exports an Object containing event listeners to process requests that can be either:
       - added to an instance of [`http.Server`](https://nodejs.org/api/http.html#class-httpserver)
       - added to an [`Express.js`](https://github.com/expressjs/express) application as middleware to handle a custom route
-        * important limitation: since `/` is a valid character in a base64 encoded URL, the path for a custom route needs to end with a character that is not allowed in base64 encoding (ex: `'/proxy_/*'`)
+        * important requirement: the path for a custom route needs to include exactly one unnamed [parameter](https://expressjs.com/en/guide/routing.html#route-parameters) that matches the base64 encoded URL and (optionally) a file extension (ex: `'/proxy/*'`)
+        * the use of nested routers is supported
   - system requirements:
     * Node.js v16.0.0 and higher
       - required features: [`Proxy` constructor](https://node.green/#ES2015-built-ins-Proxy-constructor-requires-new), [`Proxy` 'apply' handler](https://node.green/#ES2015-built-ins-Proxy--apply--handler), [`Reflect.apply`](https://node.green/#ES2015-built-ins-Reflect-Reflect-apply), [`RegExp` 'd' flag](https://node.green/#ES2022-features-RegExp-Match-Indices---hasIndices-----d--flag-)

package/hls-proxy/bin/hlsd.js

@@ -42,7 +42,8 @@ const middleware = require('../proxy')({
   cache_timeout:  argv_vals["--cache-timeout"],
   cache_key:      argv_vals["--cache-key"],
   debug_level:    argv_vals["-v"],
-  acl_whitelist:  argv_vals["--acl-whitelist"]
+  acl_whitelist:  argv_vals["--acl-whitelist"],
+  http_proxy:     argv_vals["--http-proxy"]
 })
 
 if (middleware.connection)

package/hls-proxy/bin/lib/help.js

@@ -6,7 +6,7 @@ options:
 --help
 --version
 --tls
---host <ip_address>
+--host <host>
 --port <number>
 --req-headers <filepath>
 --origin <header>
@@ -26,6 +26,7 @@ options:
 --cache-key <number>
 -v <number>
 --acl-whitelist <ip_address_list>
+--http-proxy <http[s]://[user:pass@]hostname:port>
 --tls-cert <filepath>
 --tls-key <filepath>
 --tls-pass <filepath>

package/hls-proxy/bin/lib/process_argv.js

@@ -1,5 +1,7 @@
 const process_argv = require('@warren-bank/node-process-argv')
 
+const {HttpProxyAgent, HttpsProxyAgent} = require('hpagent')
+
 const argv_flags = {
   "--help":                                 {bool: true},
   "--version":                              {bool: true},
@@ -30,6 +32,7 @@ const argv_flags = {
 
   "-v":                                     {num:  "int"},
   "--acl-whitelist":                        {},
+  "--http-proxy":                           {},
 
   "--tls-cert":                             {file: "path-exists"},
   "--tls-key":                              {file: "path-exists"},
@@ -37,7 +40,8 @@ const argv_flags = {
 }
 
 const argv_flag_aliases = {
-  "--help":                                 ["-h"]
+  "--help":                                 ["-h"],
+  "--http-proxy":                           ["--https-proxy", "--proxy"]
 }
 
 let argv_vals = {}
@@ -160,4 +164,19 @@ if (typeof argv_vals["--cache-key"] !== 'number')
 if (typeof argv_vals["-v"] !== 'number')
   argv_vals["-v"] = 0
 
+if (argv_vals["--http-proxy"]) {
+  const proxy_options = {
+    keepAlive:      true,
+    keepAliveMsecs: 1000,
+    maxSockets:     256,
+    maxFreeSockets: 256,
+    proxy:          argv_vals["--http-proxy"]
+  }
+
+  argv_vals["--http-proxy"] = {
+    "http:":  new HttpProxyAgent( proxy_options),
+    "https:": new HttpsProxyAgent(proxy_options)
+  }
+}
+
 module.exports = argv_vals

package/hls-proxy/expressjs_utils.js

@@ -1,32 +1,32 @@
-const get_full_req_url = function(req) {
-  return req.originalUrl || req.url
-}
-
-const has_req_param = function(req, key) {
-  return (req.params && (typeof req.params === 'object') && req.params[key])
-}
-
-const get_proxy_req_url = function(req) {
-  const key = "0"
-  return has_req_param(req, key)
-    ? `/${req.params[key]}`
-    : req.url
-}
-
-const get_base_req_url = function(req) {
-  let base_url = ''
-  const key = "0"
-
-  if (req.path && has_req_param(req, key)) {
-    base_url  = req.baseUrl || ''
-    base_url += req.path.substring(0, (req.path.length - req.params[key].length - 1))
-  }
-
-  return base_url
-}
-
-module.exports = {
-  get_full_req_url,
-  get_proxy_req_url,
-  get_base_req_url
-}
+const get_full_req_url = function(req) {
+  return req.originalUrl || req.url
+}
+
+const has_req_param = function(req, key) {
+  return (req.params && (typeof req.params === 'object') && req.params[key])
+}
+
+const get_proxy_req_url = function(req) {
+  const key = "0"
+  return has_req_param(req, key)
+    ? `/${req.params[key]}`
+    : req.url
+}
+
+const get_base_req_url = function(req) {
+  let base_url = ''
+  const key = "0"
+
+  if (req.path && has_req_param(req, key)) {
+    base_url  = req.baseUrl || ''
+    base_url += req.path.substring(0, (req.path.length - req.params[key].length - 1))
+  }
+
+  return base_url
+}
+
+module.exports = {
+  get_full_req_url,
+  get_proxy_req_url,
+  get_base_req_url
+}

package/hls-proxy/manifest_parser.js

@@ -16,9 +16,6 @@ const url_location_landmarks = {
       '#EXT-X-RENDITION-REPORT:',
       '#EXT-X-DATERANGE:',
       '#EXT-X-CONTENT-STEERING:'
-    ],
-    next_line: [
-      '#EXT-X-STREAM-INF:'
     ]
   },
   ts: {
@@ -26,26 +23,20 @@ const url_location_landmarks = {
       '#EXT-X-MAP:',
       '#EXT-X-PART:',
       '#EXT-X-PRELOAD-HINT:'
-    ],
-    next_line: [
-      '#EXTINF:'
     ]
   },
   json: {
     same_line: [
       '#EXT-X-SESSION-DATA:'
-    ],
-    next_line: []
+    ]
   },
   key: {
     same_line: [
       '#EXT-X-KEY:'
-    ],
-    next_line: []
+    ]
   },
   other: {
-    same_line: [],
-    next_line: []
+    same_line: []
   }
 }
 
@@ -148,63 +139,82 @@ const parse_manifest = function(m3u8_content, m3u8_url, referer_url, hooks, cach
 const extract_embedded_urls = function(m3u8_lines, m3u8_url, referer_url, meta_data) {
   const embedded_urls = []
 
-  let m3u8_line, has_next_m3u8_line, next_m3u8_line, matches, matching_landmark, matching_url
+  // one of: ['master','media']
+  // determined by the detection of either: ['#EXT-X-STREAM-INF','#EXTINF'], respectively
+  // until the type is determined, URI lines are ignored (as per the HLS spec)
+  let manifest_type = null
+
+  let m3u8_line, matches, matching_landmark, matching_url
 
   for (let i=0; i < m3u8_lines.length; i++) {
-    m3u8_line          = m3u8_lines[i]
-    has_next_m3u8_line = ((i+1) < m3u8_lines.length)
-
-    matches = regexs.m3u8_line_landmark.exec(m3u8_line)
-    if (!matches) continue
-    matching_landmark = matches[1]
-
-    matches = regexs.m3u8_line_url.exec(m3u8_line)
-    matching_url = matches
-      ? matches[1]
-      : null
-
-    if (meta_data !== null)
-      extract_meta_data(meta_data, m3u8_line, matching_landmark)
-
-    for (let url_type in url_location_landmarks) {
-      if (matching_url && (url_location_landmarks[url_type]['same_line'].indexOf(matching_landmark) >= 0)) {
-        embedded_urls.push({
-          line_index:         i,
-          url_indices:        matches.indices[1],
-          url_type:           url_type,
-          original_match_url: matching_url,
-          resolved_match_url: (new URL(matching_url, m3u8_url)).href,
-          redirected_url:     null,
-          referer_url:        referer_url,
-          encoded_url:        null
-        })
-        break
+    m3u8_line = m3u8_lines[i]
+
+    if (is_m3u8_line_a_blank(m3u8_line) || is_m3u8_line_a_comment(m3u8_line))
+      continue
+
+    if (is_m3u8_line_a_tag(m3u8_line)) {
+      matches = regexs.m3u8_line_landmark.exec(m3u8_line)
+      if (!matches) continue
+      matching_landmark = matches[1]
+
+      if (manifest_type === null) {
+        if (matching_landmark === '#EXT-X-STREAM-INF:')
+          manifest_type = 'master'
+        else if (matching_landmark === '#EXTINF:')
+          manifest_type = 'media'
       }
-      if (has_next_m3u8_line && (url_location_landmarks[url_type]['next_line'].indexOf(matching_landmark) >= 0)) {
-        next_m3u8_line = m3u8_lines[i+1].trim()
 
-        if (next_m3u8_line && (next_m3u8_line[0] !== '#')) {
-          i++
+      if (meta_data !== null)
+        extract_meta_data(meta_data, m3u8_line, matching_landmark)
 
+      matches = regexs.m3u8_line_url.exec(m3u8_line)
+      if (!matches) continue
+      matching_url = matches[1]
+
+      for (let url_type in url_location_landmarks) {
+        if (url_location_landmarks[url_type]['same_line'].indexOf(matching_landmark) >= 0) {
           embedded_urls.push({
             line_index:         i,
-            url_indices:        null,
+            url_indices:        matches.indices[1],
             url_type:           url_type,
-            original_match_url: next_m3u8_line,
-            resolved_match_url: (new URL(next_m3u8_line, m3u8_url)).href,
+            original_match_url: matching_url,
+            resolved_match_url: (new URL(matching_url, m3u8_url)).href,
             redirected_url:     null,
             referer_url:        referer_url,
             encoded_url:        null
           })
+          break
         }
-        break
       }
     }
+    else {
+      // line is a URI
+      if (manifest_type === null) continue
+
+      const url_type = (manifest_type === 'master') ? 'm3u8' : 'ts'
+
+      embedded_urls.push({
+        line_index:         i,
+        url_indices:        null,
+        url_type:           url_type,
+        original_match_url: m3u8_line,
+        resolved_match_url: (new URL(m3u8_line, m3u8_url)).href,
+        redirected_url:     null,
+        referer_url:        referer_url,
+        encoded_url:        null
+      })
+    }
   }
 
   return embedded_urls
 }
 
+const is_m3u8_line_a_blank = (line) => (!line || !line.trim())
+
+const is_m3u8_line_a_comment = (line) => ((line.indexOf('#') === 0) && !is_m3u8_line_a_tag(line))
+
+const is_m3u8_line_a_tag = (line) => (line.indexOf('#EXT') === 0)
+
 const extract_meta_data = function(meta_data, m3u8_line, matching_landmark) {
   for (let meta_data_key in meta_data_location_landmarks) {
     if (meta_data_location_landmarks[meta_data_key]['same_line'].indexOf(matching_landmark) >= 0) {

package/hls-proxy/utils.js

@@ -97,7 +97,7 @@ const debug = function() {
 }
 
 const get_request_options = function(params, url, is_m3u8, referer_url) {
-  const {req_headers, req_options, hooks} = params
+  const {req_headers, req_options, hooks, http_proxy} = params
 
   const additional_req_options = (hooks && (hooks instanceof Object) && hooks.add_request_options && (typeof hooks.add_request_options === 'function'))
     ? hooks.add_request_options(url, is_m3u8)
@@ -107,7 +107,7 @@ const get_request_options = function(params, url, is_m3u8, referer_url) {
     ? hooks.add_request_headers(url, is_m3u8)
     : null
 
-  if (!req_options && !additional_req_options && !req_headers && !additional_req_headers && !referer_url) return url
+  if (!req_options && !http_proxy && !additional_req_options && !req_headers && !additional_req_headers && !referer_url) return url
 
   const request_options = Object.assign(
     {},
@@ -125,6 +125,13 @@ const get_request_options = function(params, url, is_m3u8, referer_url) {
     (referer_url ? {"referer": referer_url, "origin": referer_url.replace(regexs.origin, '$1')} : {})
   )
 
+  // normalize
+  if (request_options.protocol)
+    request_options.protocol = request_options.protocol.toLowerCase()
+
+  if (!request_options.agent && http_proxy && request_options.protocol && http_proxy[request_options.protocol])
+    request_options.agent = http_proxy[request_options.protocol]
+
   return request_options
 }
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@warren-bank/hls-proxy",
   "description": "Node.js server to proxy HLS video streams",
-  "version":     "3.2.4",
+  "version":     "3.3.0",
   "scripts": {
     "start":     "node hls-proxy/bin/hlsd.js",
     "sudo": "sudo node hls-proxy/bin/hlsd.js"
@@ -11,8 +11,9 @@
   },
   "dependencies": {
     "@warren-bank/node-process-argv": "^1.2.1",
-    "@warren-bank/node-request": "^2.0.10",
+    "@warren-bank/node-request": "^2.0.12",
     "@warren-bank/url": "^3.1.0",
+    "hpagent": "^1.2.0",
     "tough-cookie": "^3.0.1"
   },
   "license": "GPL-2.0",