epg-grabber 0.12.1 → 0.15.1

package/README.md

@@ -8,7 +8,64 @@ Node.js CLI tool for grabbing EPG from different websites.
 npm install -g epg-grabber
 ```
 
-## Usage
+## Quick Start
+
+```sh
+epg-grabber --config=example.com.config.js
+```
+
+#### example.com.config.js
+
+```js
+module.exports = {
+  site: 'example.com',
+  channels: 'example.com.channels.xml',
+  url: function (context) {
+    const { date, channel } = context
+
+    return `https://api.example.com/${date.format('YYYY-MM-DD')}/channel/${channel.site_id}`
+  },
+  parser: function (context) {
+    const programs = JSON.parse(context.content)
+
+    return programs.map(program => {
+      return {
+        title: program.title,
+        start: program.start,
+        stop: program.stop
+      }
+    })
+  }
+}
+```
+
+#### example.com.channels.xml
+
+```xml
+<?xml version="1.0" ?>
+<site site="example.com">
+  <channels>
+    <channel site_id="cnn-23" xmltv_id="CNN.us">CNN</channel>
+  </channels>
+</site>
+```
+
+## Example Output
+
+```xml
+<tv>
+  <channel id="CNN.us">
+    <display-name>CNN</display-name>
+    <url>https://example.com</url>
+  </channel>
+  <programme start="20211116040000 +0000" stop="20211116050000 +0000" channel="CNN.us">
+    <title lang="en">News at 10PM</title>
+  </programme>
+  // ...
+</tv>
+```
+
+## CLI
 
 ```sh
 epg-grabber --config=example.com.config.js
@@ -22,9 +79,12 @@ Arguments:
 - `--lang`: set default language for all programs (default: 'en')
 - `--days`: number of days for which to grab the program (default: 1)
 - `--delay`: delay between requests (default: 3000)
+- `--timeout`: set a timeout for each request (default: 5000)
 - `--debug`: enable debug mode (default: false)
+- `--log`: path to log file (optional)
+- `--log-level`: set the log level (default: 'info')
 
-#### example.com.config.js
+## Site Config
 
 ```js
 module.exports = {
@@ -41,12 +101,11 @@ module.exports = {
     timeout: 5000,
 
     /**
-     * @param {object} date The 'dayjs' instance with the requested date
-     * @param {object} channel Data about the requested channel
+     * @param {object} context
      *
      * @return {string} The function should return headers for each request (optional)
      */
-    headers: function({ date, channel }) {
+    headers: function(context) {
       return {
         'User-Agent':
           'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36 Edg/79.0.309.71'
@@ -54,12 +113,13 @@ module.exports = {
     },
 
     /**
-     * @param {object} date The 'dayjs' instance with the requested date
-     * @param {object} channel Data about the requested channel
+     * @param {object} context
      *
      * @return {string} The function should return data for each request (optional)
      */
-    data: function({ date, channel }) {
+    data: function(context) {
+      const { channel, date } = context
+
       return {
         channels: [channel.site_id],
         dateStart: date.format('YYYY-MM-DDT00:00:00-00:00'),
@@ -69,32 +129,29 @@ module.exports = {
   },
 
   /**
-   * @param {object} date The 'dayjs' instance with the requested date
-   * @param {object} channel Data about the requested channel
+   * @param {object} context
    *
    * @return {string} The function should return URL of the program page for the channel
    */
-  url: function ({ date, channel }) {
-    return `https://example.com/${date.format('YYYY-MM-DD')}/channel/${channel.site_id}.html`
+  url: function (context) {
+    return `https://example.com/${context.date.format('YYYY-MM-DD')}/channel/${context.channel.site_id}.html`
   },
 
   /**
-   * @param {object} channel Data about the requested channel
-   * @param {string} content The response received after the request at the above url
+   * @param {object} context
    *
    * @return {string} The function should return URL of the channel logo (optional)
    */
-  logo: function ({ channel, content }) {
-    return `https://example.com/logos/${channel.site_id}.png`
+  logo: function (context) {
+    return `https://example.com/logos/${context.channel.site_id}.png`
   },
 
   /**
-   * @param {object} date The 'dayjs' instance with the requested date
-   * @param {string} content The response received after the request at the above url
+   * @param {object} context
    *
    * @return {array} The function should return an array of programs with their descriptions
    */
-  parser: function ({ date, content }) {
+  parser: function (context) {
 
     // content parsing...
 
@@ -114,7 +171,16 @@ module.exports = {
 }
 ```
 
-#### example.com.channels.xml
+## Context Object
+
+From each function in `config.js` you can access a `context` object containing the following data:
+
+- `channel`: The object describing the current channel (xmltv_id, site_id, name, lang)
+- `date`: The 'dayjs' instance with the requested date
+- `content`: The response data as a String
+- `buffer`: The response data as an ArrayBuffer
+
+## Channels List
 
 ```xml
 <?xml version="1.0" ?>

package/bin/epg-grabber.js

@@ -8,6 +8,8 @@ const grabber = require('../src/index')
 const utils = require('../src/utils')
 const { name, version, description } = require('../package.json')
 const merge = require('lodash.merge')
+const { createLogger, format, transports } = require('winston')
+const { combine, timestamp, printf } = format
 
 program
   .name(name)
@@ -17,44 +19,84 @@ program
   .option('-o, --output <output>', 'Path to output file')
   .option('--channels <channels>', 'Path to channels.xml file')
   .option('--lang <lang>', 'Set default language for all programs')
-  .option('--days <days>', 'Number of days for which to grab the program', parseInteger)
+  .option('--days <days>', 'Number of days for which to grab the program', parseInteger, 1)
   .option('--delay <delay>', 'Delay between requests (in mileseconds)', parseInteger)
+  .option('--timeout <timeout>', 'Set a timeout for each request (in mileseconds)', parseInteger)
   .option('--debug', 'Enable debug mode', false)
+  .option('--log <log>', 'Path to log file')
+  .option('--log-level <level>', 'Set log level', 'info')
   .parse(process.argv)
 
 const options = program.opts()
 
+const fileFormat = printf(({ level, message, timestamp }) => {
+  return `[${timestamp}] ${level.toUpperCase()}: ${message}`
+})
+
+const consoleFormat = printf(({ level, message, timestamp }) => {
+  if (level === 'error') return `  Error: ${message}`
+
+  return message
+})
+
+const t = [new transports.Console({ format: consoleFormat })]
+
+if (options.log) {
+  t.push(
+    new transports.File({
+      filename: path.resolve(options.log),
+      format: combine(timestamp(), fileFormat),
+      options: { flags: 'w' }
+    })
+  )
+}
+
+const logger = createLogger({
+  level: options.logLevel,
+  transports: t
+})
+
 async function main() {
-  console.log('\r\nStarting...')
+  logger.info('Starting...')
 
-  console.log(`Loading '${options.config}'...`)
+  logger.info(`Loading '${options.config}'...`)
   let config = require(path.resolve(options.config))
-  config = merge(config, options)
+  config = merge(config, {
+    days: options.days,
+    debug: options.debug,
+    lang: options.lang,
+    delay: options.delay,
+    request: {
+      timeout: options.timeout
+    }
+  })
 
   if (options.channels) config.channels = options.channels
   else if (config.channels)
     config.channels = path.join(path.dirname(options.config), config.channels)
   else throw new Error("The required 'channels' property is missing")
 
-  if (!config.channels) throw new Error('Path to [site].channels.xml is missing')
-  console.log(`Loading '${config.channels}'...`)
+  if (!config.channels) return logger.error('Path to [site].channels.xml is missing')
+  logger.info(`Loading '${config.channels}'...`)
   const channelsXML = fs.readFileSync(path.resolve(config.channels), { encoding: 'utf-8' })
-  const parsed = utils.parseChannels(channelsXML)
-  const channels = parsed.channels || []
+  const channels = utils.parseChannels(channelsXML)
 
   let programs = []
+  let i = 1
+  let days = options.days || 1
+  const total = channels.length * days
   for (let channel of channels) {
     await grabber
       .grab(channel, config, (data, err) => {
-        console.log(
-          `  ${config.site} - ${data.channel.xmltv_id} - ${data.date.format('MMM D, YYYY')} (${
-            data.programs.length
-          } programs)`
+        logger.info(
+          `[${i}/${total}] ${config.site} - ${data.channel.xmltv_id} - ${data.date.format(
+            'MMM D, YYYY'
+          )} (${data.programs.length} programs)`
         )
 
-        if (err) {
-          console.log(`    Error: ${err.message}`)
-        }
+        if (err) logger.error(err.message)
+
+        if (i < total) i++
       })
       .then(results => {
         programs = programs.concat(results)
@@ -62,11 +104,11 @@ async function main() {
   }
 
   const xml = utils.convertToXMLTV({ config, channels, programs })
-  const outputPath = config.output || 'guide.xml'
+  const outputPath = options.output || config.output || 'guide.xml'
   utils.writeToFile(outputPath, xml)
 
-  console.log(`File '${outputPath}' successfully saved`)
-  console.log('Finish')
+  logger.info(`File '${outputPath}' successfully saved`)
+  logger.info('Finish')
 }
 
 main()

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "epg-grabber",
-  "version": "0.12.1",
+  "version": "0.15.1",
   "description": "Node.js CLI tool for grabbing EPG from different sites",
   "main": "src/index.js",
   "preferGlobal": true,
@@ -36,6 +36,7 @@
     "glob": "^7.1.6",
     "lodash.merge": "^4.6.2",
     "tough-cookie": "^4.0.0",
+    "winston": "^3.3.3",
     "xml-js": "^1.6.11"
   },
   "devDependencies": {

package/src/utils.js

@@ -59,11 +59,12 @@ utils.parseChannels = function (xml) {
       const channel = el.attributes
       if (!el.elements) throw new Error(`Channel '${channel.xmltv_id}' has no valid name`)
       channel.name = el.elements.find(el => el.type === 'text').text
+      channel.site = channel.site || site
 
       return channel
     })
 
-  return { site, channels }
+  return channels
 }
 
 utils.sleep = function (ms) {
@@ -122,8 +123,8 @@ utils.convertToXMLTV = function ({ channels, programs }) {
     const title = utils.escapeString(program.title)
     const description = utils.escapeString(program.description)
     const categories = Array.isArray(program.category) ? program.category : [program.category]
-    const start = program.start ? dayjs.utc(program.start).format('YYYYMMDDHHmmss ZZ') : ''
-    const stop = program.stop ? dayjs.utc(program.stop).format('YYYYMMDDHHmmss ZZ') : ''
+    const start = program.start ? dayjs.unix(program.start).utc().format('YYYYMMDDHHmmss ZZ') : ''
+    const stop = program.stop ? dayjs.unix(program.stop).utc().format('YYYYMMDDHHmmss ZZ') : ''
     const lang = program.lang || 'en'
     const icon = utils.escapeString(program.icon)
 
@@ -264,9 +265,16 @@ utils.parsePrograms = async function (data, config) {
   return programs
     .filter(i => i)
     .map(program => {
-      program.channel = channel.xmltv_id
-      program.lang = program.lang || channel.lang || config.lang || 'en'
-      return program
+      return {
+        title: program.title,
+        description: program.description || null,
+        category: program.category || null,
+        icon: program.icon || null,
+        channel: channel.xmltv_id,
+        lang: program.lang || channel.lang || config.lang || 'en',
+        start: program.start ? dayjs(program.start).unix() : null,
+        stop: program.stop ? dayjs(program.stop).unix() : null
+      }
     })
 }
 

package/tests/bin.test.js

@@ -1,5 +1,8 @@
 const { execSync } = require('child_process')
 const pwd = `${__dirname}/..`
+import axios from 'axios'
+
+jest.mock('axios')
 
 function stdoutResultTester(stdout) {
   return [`Finish`].every(val => {
@@ -19,6 +22,8 @@ it('can load config', () => {
 })
 
 it('can load mini config', () => {
+  axios.mockImplementation(() => Promise.resolve({ data: '' }))
+
   const result = execSync(
     `node ${pwd}/bin/epg-grabber.js \
       --config=tests/input/mini.config.js \
@@ -26,7 +31,8 @@ it('can load mini config', () => {
       --output=tests/output/mini.guide.xml \
       --lang=fr \
       --days=3 \
-      --delay=0`,
+      --delay=0 \
+      --timeout=10000`,
     {
       encoding: 'utf8'
     }

package/tests/input/example.com.channels.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <site site="example.com">
   <channels>
-    <channel xmltv_id="1TV.com" site="example.com" site_id="1" lang="fr" logo="https://example.com/logos/1TV.png">1 TV</channel>
+    <channel xmltv_id="1TV.com" site_id="1" lang="fr" logo="https://example.com/logos/1TV.png">1 TV</channel>
     <channel xmltv_id="2TV.com" site="example.com" site_id="2">2 TV</channel>
   </channels>
 </site>

package/tests/utils.test.js

@@ -28,9 +28,8 @@ it('can load valid config.js', () => {
 
 it('can parse valid channels.xml', () => {
   const file = fs.readFileSync('./tests/input/example.com.channels.xml', { encoding: 'utf-8' })
-  const parsed = utils.parseChannels(file)
-  expect(parsed.site).toBe('example.com')
-  expect(parsed.channels).toEqual([
+  const channels = utils.parseChannels(file)
+  expect(channels).toEqual([
     {
       name: '1 TV',
       xmltv_id: '1TV.com',
@@ -52,22 +51,20 @@ it('can parse valid channels.xml', () => {
 
 it('can convert object to xmltv string', () => {
   const file = fs.readFileSync('./tests/input/example.com.channels.xml', { encoding: 'utf-8' })
-  const parsed = utils.parseChannels(file)
-  const channels = parsed.channels
+  const channels = utils.parseChannels(file)
   const programs = [
     {
       title: 'Program 1',
       description: 'Description for Program 1',
-      start: '2021-03-19 06:00:00 +0000',
-      stop: '2021-03-19 06:30:00 +0000',
+      start: 1616133600,
+      stop: 1616135400,
       category: 'Test',
       icon: 'https://example.com/images/Program1.png?x=шеллы&sid=777',
       channel: '1TV.com',
       lang: 'it'
     }
   ]
-  const config = { site: 'example.com' }
-  const output = utils.convertToXMLTV({ config, channels, programs })
+  const output = utils.convertToXMLTV({ channels, programs })
   expect(output).toBe(
     '<?xml version="1.0" encoding="UTF-8" ?><tv>\r\n<channel id="1TV.com"><display-name>1 TV</display-name><icon src="https://example.com/logos/1TV.png"/><url>https://example.com</url></channel>\r\n<channel id="2TV.com"><display-name>2 TV</display-name><url>https://example.com</url></channel>\r\n<programme start="20210319060000 +0000" stop="20210319063000 +0000" channel="1TV.com"><title lang="it">Program 1</title><desc lang="it">Description for Program 1</desc><category lang="it">Test</category><icon src="https://example.com/images/Program1.png?x=шеллы&amp;sid=777"/></programme>\r\n</tv>'
   )
@@ -87,8 +84,8 @@ it('can convert object to xmltv string without categories', () => {
   const programs = [
     {
       title: 'Program 1',
-      start: '2021-03-19 06:00:00 +0000',
-      stop: '2021-03-19 06:30:00 +0000',
+      start: 1616133600,
+      stop: 1616135400,
       channel: '1TV.com',
       lang: 'it'
     }
@@ -102,22 +99,20 @@ it('can convert object to xmltv string without categories', () => {
 
 it('can convert object to xmltv string with multiple categories', () => {
   const file = fs.readFileSync('./tests/input/example.com.channels.xml', { encoding: 'utf-8' })
-  const parsed = utils.parseChannels(file)
-  const channels = parsed.channels
+  const channels = utils.parseChannels(file)
   const programs = [
     {
       title: 'Program 1',
       description: 'Description for Program 1',
-      start: '2021-03-19 06:00:00 +0000',
-      stop: '2021-03-19 06:30:00 +0000',
+      start: 1616133600,
+      stop: 1616135400,
       category: ['Test1', 'Test2'],
       icon: 'https://example.com/images/Program1.png?x=шеллы&sid=777',
       channel: '1TV.com',
       lang: 'it'
     }
   ]
-  const config = { site: 'example.com' }
-  const output = utils.convertToXMLTV({ config, channels, programs })
+  const output = utils.convertToXMLTV({ channels, programs })
   expect(output).toBe(
     '<?xml version="1.0" encoding="UTF-8" ?><tv>\r\n<channel id="1TV.com"><display-name>1 TV</display-name><icon src="https://example.com/logos/1TV.png"/><url>https://example.com</url></channel>\r\n<channel id="2TV.com"><display-name>2 TV</display-name><url>https://example.com</url></channel>\r\n<programme start="20210319060000 +0000" stop="20210319063000 +0000" channel="1TV.com"><title lang="it">Program 1</title><desc lang="it">Description for Program 1</desc><category lang="it">Test1</category><category lang="it">Test2</category><icon src="https://example.com/images/Program1.png?x=шеллы&amp;sid=777"/></programme>\r\n</tv>'
   )