epg-grabber 0.9.1 → 0.10.0

package/bin/epg-grabber.js

@@ -0,0 +1,70 @@
+#! /usr/bin/env node
+
+const { Command } = require('commander')
+const program = new Command()
+const path = require('path')
+const grabber = require('../src/index')
+const utils = require('../src/utils')
+const { name, version, description } = require('../package.json')
+const merge = require('lodash.merge')
+
+program
+  .name(name)
+  .version(version, '-v, --version')
+  .description(description)
+  .requiredOption('-c, --config <config>', 'Path to [site].config.js file')
+  .option('-o, --output <output>', 'Path to output file')
+  .option('--channels <channels>', 'Path to channels.xml file')
+  .option('--lang <lang>', 'Set default language for all programs')
+  .option('--days <days>', 'Number of days for which to grab the program', parseInteger)
+  .option('--delay <delay>', 'Delay between requests (in mileseconds)', parseInteger)
+  .option('--debug', 'Enable debug mode', false)
+  .parse(process.argv)
+
+const options = program.opts()
+
+async function main() {
+  console.log('\r\nStarting...')
+
+  console.log(`Loading '${options.config}'...`)
+  let config = require(path.resolve(options.config))
+  config = merge(config, options)
+
+  if (options.channels) config.channels = options.channels
+  else if (config.channels)
+    config.channels = path.join(path.dirname(options.config), config.channels)
+  else throw new Error("The required 'channels' property is missing")
+
+  console.log('Parsing:')
+  let programs = []
+  const channels = utils.parseChannels(config.channels)
+  for (let channel of channels) {
+    await grabber
+      .grab(channel, config, (data, err) => {
+        console.log(
+          `  ${config.site} - ${data.channel.xmltv_id} - ${data.date.format('MMM D, YYYY')} (${
+            data.programs.length
+          } programs)`
+        )
+
+        if (err) {
+          console.log(`    Error: ${err.message}`)
+        }
+      })
+      .then(results => {
+        programs = programs.concat(results)
+      })
+  }
+
+  const xml = utils.convertToXMLTV({ config, channels, programs })
+  utils.writeToFile(config.output, xml)
+
+  console.log(`File '${config.output}' successfully saved`)
+  console.log('Finish')
+}
+
+main()
+
+function parseInteger(val) {
+  return val ? parseInt(val) : null
+}

package/package.json

@@ -1,11 +1,11 @@
 {
   "name": "epg-grabber",
-  "version": "0.9.1",
+  "version": "0.10.0",
   "description": "Node.js CLI tool for grabbing EPG from different sites",
   "main": "src/index.js",
   "preferGlobal": true,
   "bin": {
-    "epg-grabber": "src/index.js"
+    "epg-grabber": "bin/epg-grabber.js"
   },
   "scripts": {
     "lint": "npx eslint ./src/**/*.js",
@@ -30,6 +30,7 @@
   "dependencies": {
     "axios": "^0.21.1",
     "axios-cookiejar-support": "^1.0.1",
+    "axios-mock-adapter": "^1.20.0",
     "commander": "^7.1.0",
     "dayjs": "^1.10.4",
     "glob": "^7.1.6",
@@ -43,6 +44,9 @@
     "babel-jest": "^26.6.3",
     "eslint": "^7.22.0",
     "jest": "^26.6.3",
-    "jest-mock-axios": "^4.3.0"
+    "jest-mock-axios": "^4.4.1"
+  },
+  "jest": {
+    "testEnvironment": "node"
   }
 }

package/src/index.js

@@ -1,74 +1,37 @@
-#! /usr/bin/env node
-
-const { Command } = require('commander')
-const program = new Command()
 const utils = require('./utils')
-const { name, version, description } = require('../package.json')
-
-program
-  .name(name)
-  .version(version, '-v, --version')
-  .description(description)
-  .option('-c, --config <config>', 'Path to [site].config.js file')
-  .option('-o, --output <output>', 'Path to output file', 'guide.xml')
-  .option('--channels <channels>', 'Path to channels.xml file')
-  .option('--lang <lang>', 'Set default language for all programs', 'en')
-  .option('--days <days>', 'Number of days for which to grab the program', 1)
-  .option('--delay <delay>', 'Delay between requests (in mileseconds)', 3000)
-  .option('--debug', 'Enable debug mode', false)
-  .parse(process.argv)
-
-const options = program.opts()
-const config = utils.loadConfig(options)
-
-async function main() {
-  console.log('\r\nStarting...')
 
-  const channels = utils.parseChannels(config.channels)
-  const utcDate = utils.getUTCDate()
-  const dates = Array.from({ length: config.days }, (_, i) => utcDate.add(i, 'd'))
+module.exports = {
+  grab: async function (channel, config, cb) {
+    config = utils.loadConfig(config)
+    channel.lang = channel.lang || config.lang || null
 
-  const queue = []
-  channels.forEach(channel => {
+    const utcDate = utils.getUTCDate()
+    const dates = Array.from({ length: config.days }, (_, i) => utcDate.add(i, 'd'))
+    const queue = []
     dates.forEach(date => {
       queue.push({ date, channel })
     })
-  })
 
-  let programs = []
-  console.log('Parsing:')
-  for (let item of queue) {
-    if (options.debug) console.time('    Response Time')
-    await utils
-      .buildRequest(item, config)
-      .then(utils.fetchData)
-      .then(async response => {
-        if (options.debug) console.timeEnd('    Response Time')
-        if (options.debug) console.time('    Parsing Time')
-        const results = await utils.parseResponse(item, response, config)
-        if (options.debug) console.timeEnd('    Parsing Time')
-        programs = programs.concat(results)
-      })
-      .then(utils.sleep(config.delay))
-      .catch(err => {
-        console.log(
-          `  ${config.site} - ${item.channel.xmltv_id} - ${item.date.format(
-            'MMM D, YYYY'
-          )} (0 programs)`
-        )
-        console.log(`    Error: ${err.message}`)
-        if (options.debug) {
-          console.timeEnd('    Response Time')
-          console.timeEnd('    Parsing Time')
-        }
-      })
-  }
-
-  const xml = utils.convertToXMLTV({ config, channels, programs })
-  utils.writeToFile(config.output, xml)
-
-  console.log(`File '${config.output}' successfully saved`)
-  console.log('Finish')
+    let programs = []
+    for (let item of queue) {
+      await utils
+        .buildRequest(item, config)
+        .then(request => utils.fetchData(request))
+        .then(response => utils.parseResponse(item, response, config))
+        .then(results => {
+          item.programs = results
+          cb(item, null)
+          programs = programs.concat(results)
+        })
+        .catch(err => {
+          item.programs = []
+          cb(item, err)
+        })
+
+      await utils.sleep(config.delay)
+    }
+
+    return programs
+  },
+  convertToXMLTV: utils.convertToXMLTV
 }
-
-main()

package/src/utils.js

@@ -14,18 +14,7 @@ const utils = {}
 const defaultUserAgent =
   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36 Edg/79.0.309.71'
 
-utils.loadConfig = function (options) {
-  const file = options.config
-  if (!file) throw new Error('Path to [site].config.js is missing')
-  console.log(`Loading '${file}'...`)
-
-  const configPath = path.resolve(file)
-  const config = require(configPath)
-
-  if (options.channels) config.channels = options.channels
-  else if (config.channels) config.channels = path.join(path.dirname(file), config.channels)
-  else throw new Error("The required 'channels' property is missing")
-
+utils.loadConfig = function (config) {
   if (!config.site) throw new Error("The required 'site' property is missing")
   if (!config.url) throw new Error("The required 'url' property is missing")
   if (typeof config.url !== 'function' && typeof config.url !== 'string')
@@ -37,10 +26,10 @@ utils.loadConfig = function (options) {
     throw new Error("The 'logo' property should return the function")
 
   const defaultConfig = {
-    days: options.days ? parseInt(options.days) : 1,
-    lang: options.lang || 'en',
-    delay: options.delay ? parseInt(options.delay) : 3000,
-    output: options.output || 'guide.xml',
+    days: 1,
+    lang: 'en',
+    delay: 3000,
+    output: 'guide.xml',
     request: {
       method: 'GET',
       maxContentLength: 5 * 1024 * 1024,
@@ -60,8 +49,7 @@ utils.parseChannels = function (filename) {
 
   const xml = fs.readFileSync(path.resolve(filename), { encoding: 'utf-8' })
   const result = convert.xml2js(xml)
-  const site = result.elements.find(el => el.name === 'site')
-  const channels = site.elements.find(el => el.name === 'channels')
+  const channels = result.elements.find(el => el.name === 'channels')
 
   return channels.elements
     .filter(el => el.name === 'channel')
@@ -69,16 +57,13 @@ utils.parseChannels = function (filename) {
       const channel = el.attributes
       if (!el.elements) throw new Error(`Channel '${channel.xmltv_id}' has no valid name`)
       channel.name = el.elements.find(el => el.type === 'text').text
-      channel.site = channel.site || site.attributes.site
 
       return channel
     })
 }
 
 utils.sleep = function (ms) {
-  return function (x) {
-    return new Promise(resolve => setTimeout(() => resolve(x), ms))
-  }
+  return new Promise(resolve => setTimeout(resolve, ms))
 }
 
 utils.escapeString = function (string, defaultValue = '') {
@@ -109,18 +94,18 @@ utils.escapeString = function (string, defaultValue = '') {
     .trim()
 }
 
-utils.convertToXMLTV = function ({ config, channels, programs }) {
-  const url = config.site ? 'https://' + config.site : null
+utils.convertToXMLTV = function ({ channels, programs }) {
   let output = `<?xml version="1.0" encoding="UTF-8" ?><tv>\r\n`
   for (let channel of channels) {
-    const id = this.escapeString(channel['xmltv_id'])
-    const displayName = this.escapeString(channel.name)
+    const id = utils.escapeString(channel['xmltv_id'])
+    const displayName = utils.escapeString(channel.name)
     output += `<channel id="${id}"><display-name>${displayName}</display-name>`
     if (channel.logo) {
-      const logo = this.escapeString(channel.logo)
+      const logo = utils.escapeString(channel.logo)
       output += `<icon src="${logo}"/>`
     }
-    if (url) {
+    if (channel.site) {
+      const url = channel.site ? 'https://' + channel.site : null
       output += `<url>${url}</url>`
     }
     output += `</channel>\r\n`
@@ -129,14 +114,14 @@ utils.convertToXMLTV = function ({ config, channels, programs }) {
   for (let program of programs) {
     if (!program) continue
 
-    const channel = this.escapeString(program.channel)
-    const title = this.escapeString(program.title)
-    const description = this.escapeString(program.description)
-    const category = this.escapeString(program.category)
+    const channel = utils.escapeString(program.channel)
+    const title = utils.escapeString(program.title)
+    const description = utils.escapeString(program.description)
+    const category = utils.escapeString(program.category)
     const start = program.start ? dayjs.utc(program.start).format('YYYYMMDDHHmmss ZZ') : ''
     const stop = program.stop ? dayjs.utc(program.stop).format('YYYYMMDDHHmmss ZZ') : ''
-    const lang = program.lang || config.lang
-    const icon = this.escapeString(program.icon)
+    const lang = program.lang || 'en'
+    const icon = utils.escapeString(program.icon)
 
     if (start && title) {
       output += `<programme start="${start}"`
@@ -184,6 +169,10 @@ utils.buildRequest = async function (item, config) {
   request.url = await utils.getRequestUrl(item, config)
   request.data = await utils.getRequestData(item, config)
 
+  if (config.debug) {
+    console.log('Request:', JSON.stringify(request, null, 2))
+  }
+
   return request
 }
 
@@ -199,7 +188,7 @@ utils.getRequestHeaders = async function (item, config) {
     }
     return headers
   }
-  return config.request.headers
+  return config.request.headers || null
 }
 
 utils.getRequestData = async function (item, config) {
@@ -210,7 +199,7 @@ utils.getRequestData = async function (item, config) {
     }
     return data
   }
-  return config.request.data
+  return config.request.data || null
 }
 
 utils.getRequestUrl = async function (item, config) {
@@ -229,28 +218,20 @@ utils.getUTCDate = function () {
 }
 
 utils.parseResponse = async (item, response, config) => {
-  const options = merge(item, config, {
+  const data = merge(item, config, {
     content: response.data.toString(),
     buffer: response.data
   })
 
   if (!item.channel.logo && config.logo) {
-    item.channel.logo = await utils.loadLogo(options, config)
+    item.channel.logo = await utils.loadLogo(data, config)
   }
 
-  const parsed = await utils.parsePrograms(options, config)
-
-  console.log(
-    `  ${config.site} - ${item.channel.xmltv_id} - ${item.date.format('MMM D, YYYY')} (${
-      parsed.length
-    } programs)`
-  )
-
-  return parsed
+  return await utils.parsePrograms(data, config)
 }
 
-utils.parsePrograms = async function (options, config) {
-  let programs = config.parser(options)
+utils.parsePrograms = async function (data, config) {
+  let programs = config.parser(data)
 
   if (this.isPromise(programs)) {
     programs = await programs
@@ -260,12 +241,12 @@ utils.parsePrograms = async function (options, config) {
     throw new Error('Parser should return an array')
   }
 
-  const channel = options.channel
+  const channel = data.channel
   return programs
     .filter(i => i)
     .map(program => {
       program.channel = channel.xmltv_id
-      program.lang = program.lang || channel.lang || undefined
+      program.lang = program.lang || channel.lang || config.lang || 'en'
       return program
     })
 }

package/tests/bin.test.js

@@ -0,0 +1,37 @@
+const { execSync } = require('child_process')
+const pwd = `${__dirname}/..`
+
+function stdoutResultTester(stdout) {
+  return [`Finish`].every(val => {
+    return RegExp(val).test(stdout)
+  })
+}
+
+it('can load config', () => {
+  const result = execSync(
+    `node ${pwd}/bin/epg-grabber.js --config=tests/input/example.com.config.js`,
+    {
+      encoding: 'utf8'
+    }
+  )
+
+  expect(stdoutResultTester(result)).toBe(true)
+})
+
+it('can load mini config', () => {
+  const result = execSync(
+    `node ${pwd}/bin/epg-grabber.js \
+      --config=tests/input/mini.config.js \
+      --channels=tests/input/example.com.channels.xml \
+      --output=tests/output/mini.guide.xml \
+      --lang=fr \
+      --days=3 \
+      --delay=5000`,
+    {
+      encoding: 'utf8'
+    }
+  )
+
+  expect(stdoutResultTester(result)).toBe(true)
+  expect(result.includes("File 'tests/output/mini.guide.xml' successfully saved")).toBe(true)
+})

package/tests/index.test.js

@@ -1,34 +1,45 @@
-const { execSync } = require('child_process')
-const pwd = `${__dirname}/..`
+/**
+ * @jest-environment node
+ */
 
-function stdoutResultTester(stdout) {
-  return [`Finish`].every(val => {
-    return RegExp(val).test(stdout)
-  })
-}
+import grabber from '../src/index'
+import utils from '../src/utils'
+import axios from 'axios'
+import path from 'path'
 
-it('can load config', () => {
-  const result = execSync(`node ${pwd}/src/index.js --config=tests/input/example.com.config.js`, {
-    encoding: 'utf8'
-  })
+jest.mock('axios')
 
-  expect(stdoutResultTester(result)).toBe(true)
-})
-
-it('can load mini config', () => {
-  const result = execSync(
-    `node ${pwd}/src/index.js \
-      --config=tests/input/mini.config.js \
-      --channels=tests/input/example.com.channels.xml \
-      --output=tests/output/mini.guide.xml \
-      --lang=fr \
-      --days=3 \
-      --delay=5000`,
-    {
-      encoding: 'utf8'
+it('can grab single channel programs', done => {
+  const data = {
+    data: {
+      toString: () => 'string'
     }
-  )
+  }
+  axios.mockImplementation(() => Promise.resolve(data))
 
-  expect(stdoutResultTester(result)).toBe(true)
-  expect(result.includes("File 'tests/output/mini.guide.xml' successfully saved")).toBe(true)
+  const config = utils.loadConfig(require(path.resolve('./tests/input/mini.config.js')))
+  const channel = {
+    site: 'example.com',
+    site_id: '1',
+    xmltv_id: '1TV.fr',
+    lang: 'fr',
+    name: '1TV'
+  }
+  grabber
+    .grab(channel, config, (data, err) => {
+      if (err) {
+        console.log(`    Error: ${err.message}`)
+        done()
+      } else {
+        console.log(
+          `  ${data.channel.site} - ${data.channel.xmltv_id} - ${data.date.format(
+            'MMM D, YYYY'
+          )} (${data.programs.length} programs)`
+        )
+      }
+    })
+    .then(programs => {
+      expect(programs.length).toBe(0)
+      done()
+    })
 })

package/tests/input/example.com.channels.xml

@@ -1,7 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<site site="example.com">
-  <channels>
-    <channel site_id="1" xmltv_id="1TV.com" lang="fr" logo="https://example.com/logos/1TV.png">1 TV</channel>
-    <channel site_id="2" xmltv_id="2TV.com">2 TV</channel>
-  </channels>
-</site>
+<channels>
+  <channel xmltv_id="1TV.com" site="example.com" site_id="1" lang="fr" logo="https://example.com/logos/1TV.png">1 TV</channel>
+  <channel xmltv_id="2TV.com" site="example.com" site_id="2">2 TV</channel>
+</channels>

package/tests/input/mini.config.js

@@ -1,5 +1,5 @@
 module.exports = {
   site: 'example.com',
-  url: () => 'http://example.com/20210319/1tv.json',
+  url: 'http://example.com/20210319/1tv.json',
   parser: () => []
 }

package/tests/utils.test.js

@@ -1,14 +1,13 @@
 import mockAxios from 'jest-mock-axios'
 import utils from '../src/utils'
+import path from 'path'
 
 it('can load valid config.js', () => {
-  const config = utils.loadConfig({ config: './tests/input/example.com.config.js' })
+  const config = utils.loadConfig(require(path.resolve('./tests/input/example.com.config.js')))
   expect(config).toMatchObject({
-    channels: 'tests/input/example.com.channels.xml',
     days: 1,
     delay: 3000,
     lang: 'en',
-    output: 'tests/output/guide.xml',
     site: 'example.com'
   })
   expect(config.request).toMatchObject({
@@ -116,7 +115,7 @@ it('can fetch data', async () => {
 })
 
 it('can build request async', async () => {
-  const config = utils.loadConfig({ config: './tests/input/async.config.js' })
+  const config = utils.loadConfig(require(path.resolve('./tests/input/async.config.js')))
   return utils.buildRequest({}, config).then(request => {
     expect(request).toMatchObject({
       data: { accountID: '123' },
@@ -137,14 +136,14 @@ it('can build request async', async () => {
 })
 
 it('can load logo async', async () => {
-  const config = utils.loadConfig({ config: './tests/input/async.config.js' })
+  const config = utils.loadConfig(require(path.resolve('./tests/input/async.config.js')))
   return utils.loadLogo({}, config).then(logo => {
     expect(logo).toBe('http://example.com/logos/1TV.png?x=шеллы&sid=777')
   })
 })
 
 it('can parse programs async', async () => {
-  const config = utils.loadConfig({ config: './tests/input/async.config.js' })
+  const config = utils.loadConfig(require(path.resolve('./tests/input/async.config.js')))
   return utils
     .parsePrograms({ channel: { xmltv_id: '1tv', lang: 'en' } }, config)
     .then(programs => {