epg-grabber 0.30.2 → 0.32.0

package/README.md

@@ -75,11 +75,12 @@ Arguments:
 
 - `-c, --config`: path to config file
 - `-o, --output`: path to output file or path template (example: `guides/{site}.{lang}.xml`; default: `guide.xml`)
-- `--channels`: path to list of channels
+- `--channels`: path to list of channels; you can also use wildcard to specify the path to multiple files at once (example: `example.com_*.channels.xml`)
 - `--lang`: set default language for all programs (default: `en`)
 - `--days`: number of days for which to grab the program (default: `1`)
 - `--delay`: delay between requests in milliseconds (default: `3000`)
 - `--timeout`: set a timeout for each request in milliseconds (default: `5000`)
+- `--max-connections`: set a limit on the number of concurrent requests per site (default: `1`)
 - `--cache-ttl`: maximum time for storing each request in milliseconds (default: `0`)
 - `--gzip`: compress the output (default: `false`)
 - `--debug`: enable debug mode (default: `false`)
@@ -93,10 +94,11 @@ Arguments:
 module.exports = {
   site: 'example.com', // site domain name (required)
   output: 'example.com.guide.xml', // path to output file or path template (example: 'guides/{site}.{lang}.xml'; default: 'guide.xml')
-  channels: 'example.com.channels.xml', // path to channels.xml file (required)
+  channels: 'example.com.channels.xml', // path to list of channels; you can also use an array to specify the path to multiple files at once (example: ['channels1.xml', 'channels2.xml']; required)
   lang: 'fr', // default language for all programs (default: 'en')
   days: 3, // number of days for which to grab the program (default: 1)
   delay: 5000, // delay between requests (default: 3000)
+  maxConnections: 200, // limit on the number of concurrent requests (default: 1)
 
   request: { // request options (details: https://github.com/axios/axios#request-config)
 

package/bin/epg-grabber.js

@@ -12,6 +12,7 @@ const { name, version, description } = require('../package.json')
 const _ = require('lodash')
 const dayjs = require('dayjs')
 const utc = require('dayjs/plugin/utc')
+const { TaskQueue } = require('cwait')
 
 dayjs.extend(utc)
 
@@ -21,11 +22,16 @@ program
   .description(description)
   .requiredOption('-c, --config <config>', 'Path to [site].config.js file')
   .option('-o, --output <output>', 'Path to output file')
-  .option('--channels <channels>', 'Path to channels.xml file')
+  .option('--channels <channels>', 'Path to list of channels')
   .option('--lang <lang>', 'Set default language for all programs')
   .option('--days <days>', 'Number of days for which to grab the program', parseNumber)
   .option('--delay <delay>', 'Delay between requests (in milliseconds)', parseNumber)
   .option('--timeout <timeout>', 'Set a timeout for each request (in milliseconds)', parseNumber)
+  .option(
+    '--max-connections <maxConnections>',
+    'Set a limit on the number of concurrent requests per site',
+    parseNumber
+  )
   .option(
     '--cache-ttl <cacheTtl>',
     'Maximum time for storing each request (in milliseconds)',
@@ -53,22 +59,37 @@ async function main() {
     curl: options.curl,
     lang: options.lang,
     delay: options.delay,
+    maxConnections: options.maxConnections,
     request: {}
   })
 
   if (options.timeout) config.request.timeout = options.timeout
   if (options.cacheTtl) config.request.cache.ttl = options.cacheTtl
+
   if (options.channels) config.channels = options.channels
-  else if (config.channels)
-    config.channels = file.join(file.dirname(options.config), config.channels)
-  else throw new Error("The required 'channels' property is missing")
 
-  if (!config.channels) return logger.error('Path to [site].channels.xml is missing')
-  logger.info(`Loading '${config.channels}'...`)
-  const grabber = new EPGGrabber(config)
+  let parsedChannels = []
+  if (config.channels) {
+    const dir = file.dirname(options.config)
 
-  const channelsXML = file.read(config.channels)
-  const { channels: parsedChannels } = parseChannels(channelsXML)
+    let files = []
+    if (Array.isArray(config.channels)) {
+      files = config.channels.map(path => file.join(dir, path))
+    } else if (typeof config.channels === 'string') {
+      files = await file.list(config.channels)
+    } else {
+      throw new Error('The "channels" attribute must be of type array or string')
+    }
+
+    for (let filepath of files) {
+      logger.info(`Loading '${filepath}'...`)
+      const channelsXML = file.read(filepath)
+      const { channels } = parseChannels(channelsXML)
+      parsedChannels = parsedChannels.concat(channels)
+    }
+  } else throw new Error('Path to "channels" is missing')
+
+  const grabber = new EPGGrabber(config)
 
   let template = options.output || config.output
   const variables = file.templateVariables(template)
@@ -89,33 +110,45 @@ async function main() {
     let programs = []
     let i = 1
     let days = config.days || 1
+    const maxConnections = config.maxConnections || 1
     const total = channels.length * days
     const utcDate = getUTCDate()
     const dates = Array.from({ length: days }, (_, i) => utcDate.add(i, 'd'))
+    const taskQueue = new TaskQueue(Promise, maxConnections)
+
+    let queue = []
     for (let channel of channels) {
       if (!channel.logo && config.logo) {
         channel.logo = await grabber.loadLogo(channel)
       }
 
       for (let date of dates) {
-        await grabber
-          .grab(channel, date, (data, err) => {
-            logger.info(
-              `[${i}/${total}] ${config.site} - ${data.channel.xmltv_id} - ${dayjs
-                .utc(data.date)
-                .format('MMM D, YYYY')} (${data.programs.length} programs)`
-            )
-
-            if (err) logger.error(err.message)
-
-            if (i < total) i++
-          })
-          .then(results => {
-            programs = programs.concat(results)
-          })
+        queue.push({ channel, date })
       }
     }
 
+    await Promise.all(
+      queue.map(
+        taskQueue.wrap(async ({ channel, date }) => {
+          await grabber
+            .grab(channel, date, (data, err) => {
+              logger.info(
+                `[${i}/${total}] ${config.site} - ${data.channel.xmltv_id} - ${dayjs
+                  .utc(data.date)
+                  .format('MMM D, YYYY')} (${data.programs.length} programs)`
+              )
+
+              if (err) logger.error(err.message)
+
+              if (i < total) i++
+            })
+            .then(results => {
+              programs = programs.concat(results)
+            })
+        })
+      )
+    )
+
     programs = _.uniqBy(programs, p => p.start + p.channel)
 
     const xml = generateXMLTV({ channels, programs })

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "epg-grabber",
-  "version": "0.30.2",
+  "version": "0.32.0",
   "description": "Node.js CLI tool for grabbing EPG from different sites",
   "main": "src/index.js",
   "preferGlobal": true,
@@ -34,6 +34,7 @@
     "axios-mock-adapter": "^1.20.0",
     "commander": "^7.1.0",
     "curl-generator": "^0.2.0",
+    "cwait": "^1.1.2",
     "dayjs": "^1.10.4",
     "epg-parser": "^0.1.6",
     "fs-extra": "^11.1.1",

package/src/client.js

@@ -13,125 +13,125 @@ module.exports.parseResponse = parseResponse
 let timeout
 
 function create(config) {
-	const client = setupCache(
-		axios.create({
-			headers: {
-				'User-Agent':
-					'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36 Edg/79.0.309.71'
-			}
-		})
-	)
-
-	client.interceptors.request.use(
-		function (request) {
-			if (config.debug) {
-				console.log('Request:', JSON.stringify(request, null, 2))
-			}
-			return request
-		},
-		function (error) {
-			return Promise.reject(error)
-		}
-	)
-
-	client.interceptors.response.use(
-		function (response) {
-			if (config.debug) {
-				const data =
-					isObject(response.data) || Array.isArray(response.data)
-						? JSON.stringify(response.data)
-						: response.data.toString()
-				console.log(
-					'Response:',
-					JSON.stringify(
-						{
-							headers: response.headers,
-							data,
-							cached: response.cached
-						},
-						null,
-						2
-					)
-				)
-			}
-
-			clearTimeout(timeout)
-			return response
-		},
-		function (error) {
-			clearTimeout(timeout)
-			return Promise.reject(error)
-		}
-	)
-
-	return client
+  const client = setupCache(
+    axios.create({
+      headers: {
+        'User-Agent':
+          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36 Edg/79.0.309.71'
+      }
+    })
+  )
+
+  client.interceptors.request.use(
+    function (request) {
+      if (config.debug) {
+        console.log('Request:', JSON.stringify(request, null, 2))
+      }
+      return request
+    },
+    function (error) {
+      return Promise.reject(error)
+    }
+  )
+
+  client.interceptors.response.use(
+    function (response) {
+      if (config.debug) {
+        const data =
+          isObject(response.data) || Array.isArray(response.data)
+            ? JSON.stringify(response.data)
+            : response.data.toString()
+        console.log(
+          'Response:',
+          JSON.stringify(
+            {
+              headers: response.headers,
+              data,
+              cached: response.cached
+            },
+            null,
+            2
+          )
+        )
+      }
+
+      clearTimeout(timeout)
+      return response
+    },
+    function (error) {
+      clearTimeout(timeout)
+      return Promise.reject(error)
+    }
+  )
+
+  return client
 }
 
 async function buildRequest({ channel, date, config }) {
-	const CancelToken = axios.CancelToken
-	const source = CancelToken.source()
-	const request = { ...config.request }
-	timeout = setTimeout(() => {
-		source.cancel('Connection timeout')
-	}, request.timeout)
-	request.headers = await getRequestHeaders({ channel, date, config })
-	request.url = await getRequestUrl({ channel, date, config })
-	request.data = await getRequestData({ channel, date, config })
-	request.cancelToken = source.token
-
-	if (config.curl) {
-		const curl = CurlGenerator({
-			url: request.url,
-			method: request.method,
-			headers: request.headers,
-			body: request.data
-		})
-		console.log(curl)
-	}
-
-	return request
+  const CancelToken = axios.CancelToken
+  const source = CancelToken.source()
+  const request = { ...config.request }
+  timeout = setTimeout(() => {
+    source.cancel('Connection timeout')
+  }, request.timeout)
+  request.headers = await getRequestHeaders({ channel, date, config })
+  request.url = await getRequestUrl({ channel, date, config })
+  request.data = await getRequestData({ channel, date, config })
+  request.cancelToken = source.token
+
+  if (config.curl) {
+    const curl = CurlGenerator({
+      url: request.url,
+      method: request.method,
+      headers: request.headers,
+      body: request.data
+    })
+    console.log(curl)
+  }
+
+  return request
 }
 
 function parseResponse(response) {
-	return {
-		content: response.data.toString(),
-		buffer: response.data,
-		headers: response.headers,
-		request: response.request,
-		cached: response.cached
-	}
+  return {
+    content: response.data.toString(),
+    buffer: response.data,
+    headers: response.headers,
+    request: response.request,
+    cached: response.cached
+  }
 }
 
 async function getRequestHeaders({ channel, date, config }) {
-	if (typeof config.request.headers === 'function') {
-		const headers = config.request.headers({ channel, date })
-		if (isPromise(headers)) {
-			return await headers
-		}
-		return headers
-	}
-
-	return config.request.headers || null
+  if (typeof config.request.headers === 'function') {
+    const headers = config.request.headers({ channel, date })
+    if (isPromise(headers)) {
+      return await headers
+    }
+    return headers
+  }
+
+  return config.request.headers || null
 }
 
 async function getRequestData({ channel, date, config }) {
-	if (typeof config.request.data === 'function') {
-		const data = config.request.data({ channel, date })
-		if (isPromise(data)) {
-			return await data
-		}
-		return data
-	}
-	return config.request.data || null
+  if (typeof config.request.data === 'function') {
+    const data = config.request.data({ channel, date })
+    if (isPromise(data)) {
+      return await data
+    }
+    return data
+  }
+  return config.request.data || null
 }
 
 async function getRequestUrl({ channel, date, config }) {
-	if (typeof config.url === 'function') {
-		const url = config.url({ channel, date })
-		if (isPromise(url)) {
-			return await url
-		}
-		return url
-	}
-	return config.url
+  if (typeof config.url === 'function') {
+    const url = config.url({ channel, date })
+    if (isPromise(url)) {
+      return await url
+    }
+    return url
+  }
+  return config.url
 }

package/src/file.js

@@ -1,6 +1,8 @@
 const fs = require('fs')
 const path = require('path')
+const glob = require('glob')
 
+module.exports.list = list
 module.exports.read = read
 module.exports.write = write
 module.exports.resolve = resolve
@@ -9,6 +11,14 @@ module.exports.dirname = dirname
 module.exports.templateVariables = templateVariables
 module.exports.templateFormat = templateFormat
 
+function list(pattern) {
+  return new Promise(resolve => {
+    glob(pattern, function (err, files) {
+      resolve(files)
+    })
+  })
+}
+
 function read(filepath) {
   return fs.readFileSync(path.resolve(filepath), { encoding: 'utf-8' })
 }

package/tests/__data__/input/example_2.channels.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<site site="example2.com">
+  <channels>
+    <channel xmltv_id="3TV.com" site_id="3">3 TV</channel>
+    <channel xmltv_id="4TV.com" site_id="4">4 TV</channel>
+  </channels>
+</site>

package/tests/__data__/input/example_channels.config.js

@@ -0,0 +1,32 @@
+const dayjs = require('dayjs')
+const utc = require('dayjs/plugin/utc')
+
+dayjs.extend(utc)
+
+module.exports = {
+  site: 'example.com',
+  days: 2,
+  channels: ['example.channels.xml', 'example_2.channels.xml'],
+  output: 'tests/__data__/output/guide.xml',
+  url: () => 'http://example.com/20210319/1tv.json',
+  request: {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      Cookie: 'abc=123'
+    },
+    data() {
+      return { accountID: '123' }
+    }
+  },
+  parser: () => {
+    return [
+      {
+        title: 'Program1',
+        start: 1640995200000,
+        stop: 1640998800000
+      }
+    ]
+  },
+  logo: () => 'http://example.com/logos/1TV.png?x=шеллы&sid=777'
+}

package/tests/__data__/output/guide.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8" ?><tv date="20230709">
+<channel id="1TV.com"><display-name>1 TV</display-name><icon src="https://example.com/logos/1TV.png"/><url>https://example.com</url></channel>
+<channel id="2TV.com"><display-name>2 TV</display-name><icon src="http://example.com/logos/1TV.png?x=шеллы&amp;sid=777"/><url>https://example.com</url></channel>
+<channel id="3TV.com"><display-name>3 TV</display-name><icon src="http://example.com/logos/1TV.png?x=шеллы&amp;sid=777"/><url>https://example2.com</url></channel>
+<channel id="4TV.com"><display-name>4 TV</display-name><icon src="http://example.com/logos/1TV.png?x=шеллы&amp;sid=777"/><url>https://example2.com</url></channel>
+</tv>

package/tests/bin.test.js

@@ -105,3 +105,25 @@ it('removes duplicates of the program', () => {
 
   expect(output.programs).toEqual(expected.programs)
 })
+
+it('can load multiple "channels.xml" files at once', () => {
+  const stdout = execSync(
+    `node ${pwd}/bin/epg-grabber.js --config=tests/__data__/input/example.config.js --channels=tests/__data__/input/example*.channels.xml --timeout=1`,
+    {
+      encoding: 'utf8'
+    }
+  )
+
+  expect(stdoutResultTester(stdout)).toBe(true)
+})
+
+it('can parse list of "channels.xml" from array', () => {
+  const stdout = execSync(
+    `node ${pwd}/bin/epg-grabber.js --config=tests/__data__/input/example_channels.config.js --timeout=1`,
+    {
+      encoding: 'utf8'
+    }
+  )
+
+  expect(stdoutResultTester(stdout)).toBe(true)
+})

package/tests/__data__/output/duplicates.guide.xml

@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?><tv date="20230511">
-<channel id="1TV.com"><display-name>1 TV</display-name><icon src="https://example.com/logos/1TV.png"/><url>https://example.com</url></channel>
-<channel id="2TV.com"><display-name>2 TV</display-name><url>https://example.com</url></channel>
-<programme start="20220101000000 +0000" stop="20220101010000 +0000" channel="1TV.com"><title lang="fr">Program1</title></programme>
-<programme start="20220101000000 +0000" stop="20220101010000 +0000" channel="2TV.com"><title>Program1</title></programme>
-</tv>