epg-grabber 0.29.7 → 0.30.1

package/.github/workflows/test.yml

@@ -0,0 +1,15 @@
+name: test
+on:
+  push:
+    branches:
+      - master
+  workflow_dispatch:
+  pull_request:
+    types: [opened, synchronize, reopened, edited]
+jobs:
+  main:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - run: npm install
+      - run: npm test

package/README.md

@@ -1,4 +1,4 @@
-# EPG Grabber [![Build Status](https://app.travis-ci.com/freearhey/epg-grabber.svg?branch=master)](https://app.travis-ci.com/freearhey/epg-grabber)
+# EPG Grabber [![test](https://github.com/freearhey/epg-grabber/actions/workflows/test.yml/badge.svg)](https://github.com/freearhey/epg-grabber/actions/workflows/test.yml)
 
 Node.js CLI tool for grabbing EPG from different websites.
 
@@ -74,25 +74,25 @@ epg-grabber --config=example.com.config.js
 Arguments:
 
 - `-c, --config`: path to config file
-- `-o, --output`: path to output file (default: 'guide.xml')
-- `--channels`: path to list of channels (can be specified via config file)
-- `--lang`: set default language for all programs (default: 'en')
-- `--days`: number of days for which to grab the program (default: 1)
-- `--delay`: delay between requests (default: 3000)
-- `--timeout`: set a timeout for each request (default: 5000)
-- `--cache-ttl`: maximum time for storing each request in milliseconds (default: 0)
-- `--gzip`: compress the output (default: false)
-- `--debug`: enable debug mode (default: false)
-- `--curl`: display current request as CURL (default: false)
+- `-o, --output`: path to output file or path template (example: `guides/{site}.{lang}.xml`; default: `guide.xml`)
+- `--channels`: path to list of channels
+- `--lang`: set default language for all programs (default: `en`)
+- `--days`: number of days for which to grab the program (default: `1`)
+- `--delay`: delay between requests in milliseconds (default: `3000`)
+- `--timeout`: set a timeout for each request in milliseconds (default: `5000`)
+- `--cache-ttl`: maximum time for storing each request in milliseconds (default: `0`)
+- `--gzip`: compress the output (default: `false`)
+- `--debug`: enable debug mode (default: `false`)
+- `--curl`: display current request as CURL (default: `false`)
 - `--log`: path to log file (optional)
-- `--log-level`: set the log level (default: 'info')
+- `--log-level`: set the log level (default: `info`)
 
 ## Site Config
 
 ```js
 module.exports = {
   site: 'example.com', // site domain name (required)
-  output: 'example.com.guide.xml', // path to output file (default: 'guide.xml')
+  output: 'example.com.guide.xml', // path to output file or path template (example: 'guides/{site}.{lang}.xml'; default: 'guide.xml')
   channels: 'example.com.channels.xml', // path to channels.xml file (required)
   lang: 'fr', // default language for all programs (default: 'en')
   days: 3, // number of days for which to grab the program (default: 1)
@@ -235,6 +235,34 @@ You can also specify the language and logo for each channel individually, like s
 >France 24</channel>
 ```
 
+## How to use SOCKS proxy?
+
+First, you need to install [socks-proxy-agent](https://www.npmjs.com/package/socks-proxy-agent):
+
+```sh
+npm install socks-proxy-agent
+```
+
+Then you can use it to create an agent that acts as a SOCKS proxy. Here is an example of how to do it with the Tor SOCKS proxy:
+
+```js
+const { SocksProxyAgent } = require('socks-proxy-agent')
+
+const torProxyAgent = new SocksProxyAgent('socks://localhost:9050')
+
+module.exports = {
+  site: 'example.com',
+  url: 'https://example.com/epg.json',
+  request: {
+    httpsAgent: torProxyAgent,
+    httpAgent: torProxyAgent
+  },
+  parser(context) {
+    // ...
+  }
+}
+```
+
 ## Contribution
 
 If you find a bug or want to contribute to the code or documentation, you can help by submitting an [issue](https://github.com/freearhey/epg-grabber/issues) or a [pull request](https://github.com/freearhey/epg-grabber/pulls).

package/bin/epg-grabber.js

@@ -24,8 +24,8 @@ program
   .option('--channels <channels>', 'Path to channels.xml file')
   .option('--lang <lang>', 'Set default language for all programs')
   .option('--days <days>', 'Number of days for which to grab the program', parseNumber)
-  .option('--delay <delay>', 'Delay between requests (in mileseconds)', parseNumber)
-  .option('--timeout <timeout>', 'Set a timeout for each request (in mileseconds)', parseNumber)
+  .option('--delay <delay>', 'Delay between requests (in milliseconds)', parseNumber)
+  .option('--timeout <timeout>', 'Set a timeout for each request (in milliseconds)', parseNumber)
   .option(
     '--cache-ttl <cacheTtl>',
     'Maximum time for storing each request (in milliseconds)',
@@ -68,52 +68,70 @@ async function main() {
   const grabber = new EPGGrabber(config)
 
   const channelsXML = file.read(config.channels)
-  const { channels } = parseChannels(channelsXML)
-
-  let programs = []
-  let i = 1
-  let days = config.days || 1
-  const total = channels.length * days
-  const utcDate = getUTCDate()
-  const dates = Array.from({ length: days }, (_, i) => utcDate.add(i, 'd'))
-  for (let channel of channels) {
-    if (!channel.logo && config.logo) {
-      channel.logo = await grabber.loadLogo(channel)
+  const { channels: parsedChannels } = parseChannels(channelsXML)
+
+  let template = options.output || config.output
+  const variables = file.templateVariables(template)
+
+  const groups = _.groupBy(parsedChannels, channel => {
+    let groupId = ''
+    for (let key in channel) {
+      if (variables.includes(key)) {
+        groupId += channel[key]
+      }
     }
 
-    for (let date of dates) {
-      await grabber
-        .grab(channel, date, (data, err) => {
-          logger.info(
-            `[${i}/${total}] ${config.site} - ${data.channel.id} - ${dayjs
-              .utc(data.date)
-              .format('MMM D, YYYY')} (${data.programs.length} programs)`
-          )
-
-          if (err) logger.error(err.message)
-
-          if (i < total) i++
-        })
-        .then(results => {
-          programs = programs.concat(results)
-        })
+    return groupId
+  })
+
+  for (let groupId in groups) {
+    const channels = groups[groupId]
+    let programs = []
+    let i = 1
+    let days = config.days || 1
+    const total = channels.length * days
+    const utcDate = getUTCDate()
+    const dates = Array.from({ length: days }, (_, i) => utcDate.add(i, 'd'))
+    for (let channel of channels) {
+      if (!channel.logo && config.logo) {
+        channel.logo = await grabber.loadLogo(channel)
+      }
+
+      for (let date of dates) {
+        await grabber
+          .grab(channel, date, (data, err) => {
+            logger.info(
+              `[${i}/${total}] ${config.site} - ${data.channel.id} - ${dayjs
+                .utc(data.date)
+                .format('MMM D, YYYY')} (${data.programs.length} programs)`
+            )
+
+            if (err) logger.error(err.message)
+
+            if (i < total) i++
+          })
+          .then(results => {
+            programs = programs.concat(results)
+          })
+      }
+    }
+
+    programs = _.uniqBy(programs, p => p.start + p.channel)
+
+    const xml = generateXMLTV({ channels, programs })
+    let outputPath = file.templateFormat(template, channels[0])
+    if (options.gzip) {
+      outputPath = outputPath || 'guide.xml.gz'
+      const compressed = await gzip(xml)
+      file.write(outputPath, compressed)
+    } else {
+      outputPath = outputPath || 'guide.xml'
+      file.write(outputPath, xml)
     }
-  }
 
-  programs = _.uniqBy(programs, p => p.start + p.channel)
-
-  const xml = generateXMLTV({ channels, programs })
-  let outputPath = options.output || config.output
-  if (options.gzip) {
-    outputPath = outputPath || 'guide.xml.gz'
-    const compressed = await gzip(xml)
-    file.write(outputPath, compressed)
-  } else {
-    outputPath = outputPath || 'guide.xml'
-    file.write(outputPath, xml)
+    logger.info(`File '${outputPath}' successfully saved`)
   }
 
-  logger.info(`File '${outputPath}' successfully saved`)
   logger.info('Finish')
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "epg-grabber",
-  "version": "0.29.7",
+  "version": "0.30.1",
   "description": "Node.js CLI tool for grabbing EPG from different sites",
   "main": "src/index.js",
   "preferGlobal": true,
@@ -36,6 +36,7 @@
     "curl-generator": "^0.2.0",
     "dayjs": "^1.10.4",
     "epg-parser": "^0.1.6",
+    "fs-extra": "^11.1.1",
     "glob": "^7.1.6",
     "lodash": "^4.17.21",
     "node-gzip": "^1.1.2",

package/src/Channel.js

@@ -1,23 +1,23 @@
 class Channel {
-	constructor(c) {
-		const data = {
-			id: c.id || c.xmltv_id,
-			name: c.name,
-			site: c.site || '',
-			site_id: c.site_id,
-			lang: c.lang || '',
-			logo: c.logo || '',
-			url: c.url || toURL(c.site)
-		}
+  constructor(c) {
+    const data = {
+      xmltv_id: c.xmltv_id,
+      name: c.name,
+      site: c.site || '',
+      site_id: c.site_id,
+      lang: c.lang || '',
+      logo: c.logo || '',
+      url: c.url || toURL(c.site)
+    }
 
-		for (let key in data) {
-			this[key] = data[key]
-		}
-	}
+    for (let key in data) {
+      this[key] = data[key]
+    }
+  }
 }
 
 module.exports = Channel
 
 function toURL(site) {
-	return site ? `https://${site}` : ''
+  return site ? `https://${site}` : ''
 }

package/src/Program.js

@@ -3,140 +3,140 @@ const { toArray, toUnix, parseNumber } = require('./utils')
 const Channel = require('./Channel')
 
 class Program {
-	constructor(p, c) {
-		if (!(c instanceof Channel)) {
-			throw new Error('The second argument in the constructor must be the "Channel" class')
-		}
-
-		const data = {
-			site: c.site || '',
-			channel: c.id || '',
-			titles: toArray(p.titles || p.title).map(text => toTextObject(text, c.lang)),
-			sub_titles: toArray(p.sub_titles || p.sub_title).map(text => toTextObject(text, c.lang)),
-			descriptions: toArray(p.descriptions || p.description || p.desc).map(text =>
-				toTextObject(text, c.lang)
-			),
-			icon: toIconObject(p.icon),
-			episodeNumbers: p.episodeNum || p.episodeNumbers || getEpisodeNumbers(p.season, p.episode),
-			date: p.date ? toUnix(p.date) : null,
-			start: p.start ? toUnix(p.start) : null,
-			stop: p.stop ? toUnix(p.stop) : null,
-			urls: toArray(p.urls || p.url).map(toUrlObject),
-			ratings: toArray(p.ratings || p.rating).map(toRatingObject),
-			categories: toArray(p.categories || p.category).map(text => toTextObject(text, c.lang)),
-			directors: toArray(p.directors || p.director).map(toPersonObject),
-			actors: toArray(p.actors || p.actor).map(toPersonObject),
-			writers: toArray(p.writers || p.writer).map(toPersonObject),
-			adapters: toArray(p.adapters || p.adapter).map(toPersonObject),
-			producers: toArray(p.producers || p.producer).map(toPersonObject),
-			composers: toArray(p.composers || p.composer).map(toPersonObject),
-			editors: toArray(p.editors || p.editor).map(toPersonObject),
-			presenters: toArray(p.presenters || p.presenter).map(toPersonObject),
-			commentators: toArray(p.commentators || p.commentator).map(toPersonObject),
-			guests: toArray(p.guests || p.guest).map(toPersonObject)
-		}
-
-		for (let key in data) {
-			this[key] = data[key]
-		}
-	}
+  constructor(p, c) {
+    if (!(c instanceof Channel)) {
+      throw new Error('The second argument in the constructor must be the "Channel" class')
+    }
+
+    const data = {
+      site: c.site || '',
+      channel: c.xmltv_id || '',
+      titles: toArray(p.titles || p.title).map(text => toTextObject(text, c.lang)),
+      sub_titles: toArray(p.sub_titles || p.sub_title).map(text => toTextObject(text, c.lang)),
+      descriptions: toArray(p.descriptions || p.description || p.desc).map(text =>
+        toTextObject(text, c.lang)
+      ),
+      icon: toIconObject(p.icon),
+      episodeNumbers: p.episodeNum || p.episodeNumbers || getEpisodeNumbers(p.season, p.episode),
+      date: p.date ? toUnix(p.date) : null,
+      start: p.start ? toUnix(p.start) : null,
+      stop: p.stop ? toUnix(p.stop) : null,
+      urls: toArray(p.urls || p.url).map(toUrlObject),
+      ratings: toArray(p.ratings || p.rating).map(toRatingObject),
+      categories: toArray(p.categories || p.category).map(text => toTextObject(text, c.lang)),
+      directors: toArray(p.directors || p.director).map(toPersonObject),
+      actors: toArray(p.actors || p.actor).map(toPersonObject),
+      writers: toArray(p.writers || p.writer).map(toPersonObject),
+      adapters: toArray(p.adapters || p.adapter).map(toPersonObject),
+      producers: toArray(p.producers || p.producer).map(toPersonObject),
+      composers: toArray(p.composers || p.composer).map(toPersonObject),
+      editors: toArray(p.editors || p.editor).map(toPersonObject),
+      presenters: toArray(p.presenters || p.presenter).map(toPersonObject),
+      commentators: toArray(p.commentators || p.commentator).map(toPersonObject),
+      guests: toArray(p.guests || p.guest).map(toPersonObject)
+    }
+
+    for (let key in data) {
+      this[key] = data[key]
+    }
+  }
 }
 
 module.exports = Program
 
 function toTextObject(text, lang) {
-	if (typeof text === 'string') {
-		return { value: text, lang }
-	}
-
-	return {
-		value: text.value,
-		lang: text.lang
-	}
+  if (typeof text === 'string') {
+    return { value: text, lang }
+  }
+
+  return {
+    value: text.value,
+    lang: text.lang
+  }
 }
 
 function toPersonObject(person) {
-	if (typeof person === 'string') {
-		return {
-			value: person,
-			url: [],
-			image: []
-		}
-	}
-
-	return {
-		value: person.value,
-		url: toArray(person.url).map(toUrlObject),
-		image: toArray(person.image).map(toImageObject)
-	}
+  if (typeof person === 'string') {
+    return {
+      value: person,
+      url: [],
+      image: []
+    }
+  }
+
+  return {
+    value: person.value,
+    url: toArray(person.url).map(toUrlObject),
+    image: toArray(person.image).map(toImageObject)
+  }
 }
 
 function toImageObject(image) {
-	if (typeof image === 'string') return { type: '', size: '', orient: '', system: '', value: image }
-
-	return {
-		type: image.type || '',
-		size: image.size || '',
-		orient: image.orient || '',
-		system: image.system || '',
-		value: image.value
-	}
+  if (typeof image === 'string') return { type: '', size: '', orient: '', system: '', value: image }
+
+  return {
+    type: image.type || '',
+    size: image.size || '',
+    orient: image.orient || '',
+    system: image.system || '',
+    value: image.value
+  }
 }
 
 function toRatingObject(rating) {
-	if (typeof rating === 'string') return { system: '', icon: '', value: rating }
+  if (typeof rating === 'string') return { system: '', icon: '', value: rating }
 
-	return {
-		system: rating.system || '',
-		icon: rating.icon || '',
-		value: rating.value || ''
-	}
+  return {
+    system: rating.system || '',
+    icon: rating.icon || '',
+    value: rating.value || ''
+  }
 }
 
 function toUrlObject(url) {
-	if (typeof url === 'string') return { system: '', value: url }
+  if (typeof url === 'string') return { system: '', value: url }
 
-	return {
-		system: url.system || '',
-		value: url.value || ''
-	}
+  return {
+    system: url.system || '',
+    value: url.value || ''
+  }
 }
 
 function toIconObject(icon) {
-	if (!icon) return { src: '' }
-	if (typeof icon === 'string') return { src: icon }
+  if (!icon) return { src: '' }
+  if (typeof icon === 'string') return { src: icon }
 
-	return {
-		src: icon.src || ''
-	}
+  return {
+    src: icon.src || ''
+  }
 }
 
 function getEpisodeNumbers(s, e) {
-	s = parseNumber(s)
-	e = parseNumber(e)
+  s = parseNumber(s)
+  e = parseNumber(e)
 
-	return [createXMLTVNS(s, e), createOnScreen(s, e)].filter(Boolean)
+  return [createXMLTVNS(s, e), createOnScreen(s, e)].filter(Boolean)
 }
 
 function createXMLTVNS(s, e) {
-	if (!e) return null
-	s = s || 1
+  if (!e) return null
+  s = s || 1
 
-	return {
-		system: 'xmltv_ns',
-		value: `${s - 1}.${e - 1}.0/1`
-	}
+  return {
+    system: 'xmltv_ns',
+    value: `${s - 1}.${e - 1}.0/1`
+  }
 }
 
 function createOnScreen(s, e) {
-	if (!e) return null
-	s = s || 1
+  if (!e) return null
+  s = s || 1
 
-	s = padStart(s, 2, '0')
-	e = padStart(e, 2, '0')
+  s = padStart(s, 2, '0')
+  e = padStart(e, 2, '0')
 
-	return {
-		system: 'onscreen',
-		value: `S${s}E${e}`
-	}
+  return {
+    system: 'onscreen',
+    value: `S${s}E${e}`
+  }
 }

package/src/file.js

@@ -6,28 +6,47 @@ module.exports.write = write
 module.exports.resolve = resolve
 module.exports.join = join
 module.exports.dirname = dirname
+module.exports.templateVariables = templateVariables
+module.exports.templateFormat = templateFormat
 
 function read(filepath) {
-	return fs.readFileSync(path.resolve(filepath), { encoding: 'utf-8' })
+  return fs.readFileSync(path.resolve(filepath), { encoding: 'utf-8' })
 }
 
 function write(filepath, data) {
-	const dir = path.resolve(path.dirname(filepath))
-	if (!fs.existsSync(dir)) {
-		fs.mkdirSync(dir, { recursive: true })
-	}
+  const dir = path.resolve(path.dirname(filepath))
+  if (!fs.existsSync(dir)) {
+    fs.mkdirSync(dir, { recursive: true })
+  }
 
-	fs.writeFileSync(path.resolve(filepath), data)
+  fs.writeFileSync(path.resolve(filepath), data)
 }
 
 function resolve(filepath) {
-	return path.resolve(filepath)
+  return path.resolve(filepath)
 }
 
 function join(path1, path2) {
-	return path.join(path1, path2)
+  return path.join(path1, path2)
 }
 
 function dirname(filepath) {
-	return path.dirname(filepath)
+  return path.dirname(filepath)
+}
+
+function templateVariables(template) {
+  const match = template.match(/{[^}]+}/g)
+
+  return Array.isArray(match) ? match.map(s => s.substring(1, s.length - 1)) : []
+}
+
+function templateFormat(template, obj) {
+  let output = template
+  for (let key in obj) {
+    const regex = new RegExp(`{${key}}`, 'g')
+    const value = obj[key] || undefined
+    output = output.replace(regex, value)
+  }
+
+  return output
 }

package/src/xmltv.js

@@ -29,7 +29,7 @@ function createElements(channels, programs, date) {
     ...channels.map(channel => {
       return (
         '\r\n' +
-        el('channel', { id: channel.id }, [
+        el('channel', { id: channel.xmltv_id }, [
           el('display-name', {}, [escapeString(channel.name)]),
           el('icon', { src: channel.logo }),
           el('url', {}, [channel.url])

package/tests/Channel.test.js

@@ -12,28 +12,7 @@ it('can create new Channel', () => {
 
   expect(channel).toMatchObject({
     name: '1 TV',
-    id: '1TV.com',
-    site_id: '1',
-    site: 'example.com',
-    url: 'https://example.com',
-    lang: 'fr',
-    logo: 'https://example.com/logos/1TV.png'
-  })
-})
-
-it('can create channel from exist object', () => {
-  const channel = new Channel({
-    name: '1 TV',
-    id: '1TV.com',
-    site_id: '1',
-    site: 'example.com',
-    lang: 'fr',
-    logo: 'https://example.com/logos/1TV.png'
-  })
-
-  expect(channel).toMatchObject({
-    name: '1 TV',
-    id: '1TV.com',
+    xmltv_id: '1TV.com',
     site_id: '1',
     site: 'example.com',
     url: 'https://example.com',

package/tests/__data__/input/example.config.js

@@ -7,7 +7,7 @@ module.exports = {
   site: 'example.com',
   days: 2,
   channels: 'example.channels.xml',
-  output: 'tests/output/guide.xml',
+  output: 'tests/__data__/output/guide.xml',
   url: () => 'http://example.com/20210319/1tv.json',
   request: {
     method: 'POST',

package/tests/__data__/output/duplicates.guide.xml

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="UTF-8" ?><tv date="20230110">
+<?xml version="1.0" encoding="UTF-8" ?><tv date="20230511">
 <channel id="1TV.com"><display-name>1 TV</display-name><icon src="https://example.com/logos/1TV.png"/><url>https://example.com</url></channel>
 <channel id="2TV.com"><display-name>2 TV</display-name><url>https://example.com</url></channel>
 <programme start="20220101000000 +0000" stop="20220101010000 +0000" channel="1TV.com"><title lang="fr">Program1</title></programme>

package/tests/bin.test.js

@@ -1,5 +1,5 @@
 const { execSync } = require('child_process')
-const fs = require('fs')
+const fs = require('fs-extra')
 const path = require('path')
 const epgParser = require('epg-parser')
 
@@ -11,6 +11,10 @@ function stdoutResultTester(stdout) {
   })
 }
 
+beforeEach(() => {
+  fs.emptyDirSync('tests/__data__/output')
+})
+
 it('can load config', () => {
   const stdout = execSync(
     `node ${pwd}/bin/epg-grabber.js --config=tests/__data__/input/example.config.js --delay=0`,
@@ -62,6 +66,26 @@ it('can generate gzip version', () => {
   )
 })
 
+it('can produce multiple outputs', () => {
+  const stdout = execSync(
+    `node ${pwd}/bin/epg-grabber.js \
+      --config=tests/__data__/input/mini.config.js \
+      --channels=tests/__data__/input/example.channels.xml \
+      --output=tests/__data__/output/{lang}/{xmltv_id}.xml`,
+    {
+      encoding: 'utf8'
+    }
+  )
+
+  expect(stdoutResultTester(stdout)).toBe(true)
+  expect(stdout.includes("File 'tests/__data__/output/fr/1TV.com.xml' successfully saved")).toBe(
+    true
+  )
+  expect(
+    stdout.includes("File 'tests/__data__/output/undefined/2TV.com.xml' successfully saved")
+  ).toBe(true)
+})
+
 it('removes duplicates of the program', () => {
   const stdout = execSync(
     `node ${pwd}/bin/epg-grabber.js \

package/.travis.yml

@@ -1,8 +0,0 @@
-language: node_js
-
-node_js:
-  - '10'
-
-script:
-  - npm run lint
-  - npm run test

package/tests/__data__/output/mini.guide.xml

@@ -1,4 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?><tv date="20230110">
-<channel id="1TV.com"><display-name>1 TV</display-name><icon src="https://example.com/logos/1TV.png"/><url>https://example.com</url></channel>
-<channel id="2TV.com"><display-name>2 TV</display-name><url>https://example.com</url></channel>
-</tv>