epg-grabber 0.30.1 → 0.31.0

package/README.md

@@ -80,6 +80,7 @@ Arguments:
 - `--days`: number of days for which to grab the program (default: `1`)
 - `--delay`: delay between requests in milliseconds (default: `3000`)
 - `--timeout`: set a timeout for each request in milliseconds (default: `5000`)
+- `--max-connections`: set a limit on the number of concurrent requests per site (default: `1`)
 - `--cache-ttl`: maximum time for storing each request in milliseconds (default: `0`)
 - `--gzip`: compress the output (default: `false`)
 - `--debug`: enable debug mode (default: `false`)
@@ -97,6 +98,7 @@ module.exports = {
   lang: 'fr', // default language for all programs (default: 'en')
   days: 3, // number of days for which to grab the program (default: 1)
   delay: 5000, // delay between requests (default: 3000)
+  maxConnections: 200, // limit on the number of concurrent requests (default: 1)
 
   request: { // request options (details: https://github.com/axios/axios#request-config)
 

package/bin/epg-grabber.js

@@ -12,6 +12,7 @@ const { name, version, description } = require('../package.json')
 const _ = require('lodash')
 const dayjs = require('dayjs')
 const utc = require('dayjs/plugin/utc')
+const { TaskQueue } = require('cwait')
 
 dayjs.extend(utc)
 
@@ -26,6 +27,11 @@ program
   .option('--days <days>', 'Number of days for which to grab the program', parseNumber)
   .option('--delay <delay>', 'Delay between requests (in milliseconds)', parseNumber)
   .option('--timeout <timeout>', 'Set a timeout for each request (in milliseconds)', parseNumber)
+  .option(
+    '--max-connections <maxConnections>',
+    'Set a limit on the number of concurrent requests per site',
+    parseNumber
+  )
   .option(
     '--cache-ttl <cacheTtl>',
     'Maximum time for storing each request (in milliseconds)',
@@ -53,6 +59,7 @@ async function main() {
     curl: options.curl,
     lang: options.lang,
     delay: options.delay,
+    maxConnections: options.maxConnections,
     request: {}
   })
 
@@ -89,33 +96,45 @@ async function main() {
     let programs = []
     let i = 1
     let days = config.days || 1
+    const maxConnections = config.maxConnections || 1
     const total = channels.length * days
     const utcDate = getUTCDate()
     const dates = Array.from({ length: days }, (_, i) => utcDate.add(i, 'd'))
+    const taskQueue = new TaskQueue(Promise, maxConnections)
+
+    let queue = []
     for (let channel of channels) {
       if (!channel.logo && config.logo) {
         channel.logo = await grabber.loadLogo(channel)
       }
 
       for (let date of dates) {
-        await grabber
-          .grab(channel, date, (data, err) => {
-            logger.info(
-              `[${i}/${total}] ${config.site} - ${data.channel.id} - ${dayjs
-                .utc(data.date)
-                .format('MMM D, YYYY')} (${data.programs.length} programs)`
-            )
-
-            if (err) logger.error(err.message)
-
-            if (i < total) i++
-          })
-          .then(results => {
-            programs = programs.concat(results)
-          })
+        queue.push({ channel, date })
       }
     }
 
+    await Promise.all(
+      queue.map(
+        taskQueue.wrap(async ({ channel, date }) => {
+          await grabber
+            .grab(channel, date, (data, err) => {
+              logger.info(
+                `[${i}/${total}] ${config.site} - ${data.channel.xmltv_id} - ${dayjs
+                  .utc(data.date)
+                  .format('MMM D, YYYY')} (${data.programs.length} programs)`
+              )
+
+              if (err) logger.error(err.message)
+
+              if (i < total) i++
+            })
+            .then(results => {
+              programs = programs.concat(results)
+            })
+        })
+      )
+    )
+
     programs = _.uniqBy(programs, p => p.start + p.channel)
 
     const xml = generateXMLTV({ channels, programs })

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "epg-grabber",
-  "version": "0.30.1",
+  "version": "0.31.0",
   "description": "Node.js CLI tool for grabbing EPG from different sites",
   "main": "src/index.js",
   "preferGlobal": true,
@@ -34,6 +34,7 @@
     "axios-mock-adapter": "^1.20.0",
     "commander": "^7.1.0",
     "curl-generator": "^0.2.0",
+    "cwait": "^1.1.2",
     "dayjs": "^1.10.4",
     "epg-parser": "^0.1.6",
     "fs-extra": "^11.1.1",

package/src/client.js

@@ -13,125 +13,125 @@ module.exports.parseResponse = parseResponse
 let timeout
 
 function create(config) {
-	const client = setupCache(
-		axios.create({
-			headers: {
-				'User-Agent':
-					'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36 Edg/79.0.309.71'
-			}
-		})
-	)
-
-	client.interceptors.request.use(
-		function (request) {
-			if (config.debug) {
-				console.log('Request:', JSON.stringify(request, null, 2))
-			}
-			return request
-		},
-		function (error) {
-			return Promise.reject(error)
-		}
-	)
-
-	client.interceptors.response.use(
-		function (response) {
-			if (config.debug) {
-				const data =
-					isObject(response.data) || Array.isArray(response.data)
-						? JSON.stringify(response.data)
-						: response.data.toString()
-				console.log(
-					'Response:',
-					JSON.stringify(
-						{
-							headers: response.headers,
-							data,
-							cached: response.cached
-						},
-						null,
-						2
-					)
-				)
-			}
-
-			clearTimeout(timeout)
-			return response
-		},
-		function (error) {
-			clearTimeout(timeout)
-			return Promise.reject(error)
-		}
-	)
-
-	return client
+  const client = setupCache(
+    axios.create({
+      headers: {
+        'User-Agent':
+          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36 Edg/79.0.309.71'
+      }
+    })
+  )
+
+  client.interceptors.request.use(
+    function (request) {
+      if (config.debug) {
+        console.log('Request:', JSON.stringify(request, null, 2))
+      }
+      return request
+    },
+    function (error) {
+      return Promise.reject(error)
+    }
+  )
+
+  client.interceptors.response.use(
+    function (response) {
+      if (config.debug) {
+        const data =
+          isObject(response.data) || Array.isArray(response.data)
+            ? JSON.stringify(response.data)
+            : response.data.toString()
+        console.log(
+          'Response:',
+          JSON.stringify(
+            {
+              headers: response.headers,
+              data,
+              cached: response.cached
+            },
+            null,
+            2
+          )
+        )
+      }
+
+      clearTimeout(timeout)
+      return response
+    },
+    function (error) {
+      clearTimeout(timeout)
+      return Promise.reject(error)
+    }
+  )
+
+  return client
 }
 
 async function buildRequest({ channel, date, config }) {
-	const CancelToken = axios.CancelToken
-	const source = CancelToken.source()
-	const request = { ...config.request }
-	timeout = setTimeout(() => {
-		source.cancel('Connection timeout')
-	}, request.timeout)
-	request.headers = await getRequestHeaders({ channel, date, config })
-	request.url = await getRequestUrl({ channel, date, config })
-	request.data = await getRequestData({ channel, date, config })
-	request.cancelToken = source.token
-
-	if (config.curl) {
-		const curl = CurlGenerator({
-			url: request.url,
-			method: request.method,
-			headers: request.headers,
-			body: request.data
-		})
-		console.log(curl)
-	}
-
-	return request
+  const CancelToken = axios.CancelToken
+  const source = CancelToken.source()
+  const request = { ...config.request }
+  timeout = setTimeout(() => {
+    source.cancel('Connection timeout')
+  }, request.timeout)
+  request.headers = await getRequestHeaders({ channel, date, config })
+  request.url = await getRequestUrl({ channel, date, config })
+  request.data = await getRequestData({ channel, date, config })
+  request.cancelToken = source.token
+
+  if (config.curl) {
+    const curl = CurlGenerator({
+      url: request.url,
+      method: request.method,
+      headers: request.headers,
+      body: request.data
+    })
+    console.log(curl)
+  }
+
+  return request
 }
 
 function parseResponse(response) {
-	return {
-		content: response.data.toString(),
-		buffer: response.data,
-		headers: response.headers,
-		request: response.request,
-		cached: response.cached
-	}
+  return {
+    content: response.data.toString(),
+    buffer: response.data,
+    headers: response.headers,
+    request: response.request,
+    cached: response.cached
+  }
 }
 
 async function getRequestHeaders({ channel, date, config }) {
-	if (typeof config.request.headers === 'function') {
-		const headers = config.request.headers({ channel, date })
-		if (isPromise(headers)) {
-			return await headers
-		}
-		return headers
-	}
-
-	return config.request.headers || null
+  if (typeof config.request.headers === 'function') {
+    const headers = config.request.headers({ channel, date })
+    if (isPromise(headers)) {
+      return await headers
+    }
+    return headers
+  }
+
+  return config.request.headers || null
 }
 
 async function getRequestData({ channel, date, config }) {
-	if (typeof config.request.data === 'function') {
-		const data = config.request.data({ channel, date })
-		if (isPromise(data)) {
-			return await data
-		}
-		return data
-	}
-	return config.request.data || null
+  if (typeof config.request.data === 'function') {
+    const data = config.request.data({ channel, date })
+    if (isPromise(data)) {
+      return await data
+    }
+    return data
+  }
+  return config.request.data || null
 }
 
 async function getRequestUrl({ channel, date, config }) {
-	if (typeof config.url === 'function') {
-		const url = config.url({ channel, date })
-		if (isPromise(url)) {
-			return await url
-		}
-		return url
-	}
-	return config.url
+  if (typeof config.url === 'function') {
+    const url = config.url({ channel, date })
+    if (isPromise(url)) {
+      return await url
+    }
+    return url
+  }
+  return config.url
 }

package/tests/__data__/output/duplicates.guide.xml

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8" ?><tv date="20230511">
+<?xml version="1.0" encoding="UTF-8" ?><tv date="20230521">
 <channel id="1TV.com"><display-name>1 TV</display-name><icon src="https://example.com/logos/1TV.png"/><url>https://example.com</url></channel>
 <channel id="2TV.com"><display-name>2 TV</display-name><url>https://example.com</url></channel>
 <programme start="20220101000000 +0000" stop="20220101010000 +0000" channel="1TV.com"><title lang="fr">Program1</title></programme>