epg-grabber 0.31.0 → 0.33.0

package/README.md

@@ -43,11 +43,9 @@ module.exports = {
 
 ```xml
 <?xml version="1.0" ?>
-<site site="example.com">
-  <channels>
-    <channel site_id="cnn-23" xmltv_id="CNN.us">CNN</channel>
-  </channels>
-</site>
+<channels site="example.com">
+  <channel site_id="cnn-23" xmltv_id="CNN.us">CNN</channel>
+</channels>
 ```
 
 ## Example Output
@@ -75,7 +73,7 @@ Arguments:
 
 - `-c, --config`: path to config file
 - `-o, --output`: path to output file or path template (example: `guides/{site}.{lang}.xml`; default: `guide.xml`)
-- `--channels`: path to list of channels
+- `--channels`: path to list of channels; you can also use wildcard to specify the path to multiple files at once (example: `example.com_*.channels.xml`)
 - `--lang`: set default language for all programs (default: `en`)
 - `--days`: number of days for which to grab the program (default: `1`)
 - `--delay`: delay between requests in milliseconds (default: `3000`)
@@ -94,7 +92,7 @@ Arguments:
 module.exports = {
   site: 'example.com', // site domain name (required)
   output: 'example.com.guide.xml', // path to output file or path template (example: 'guides/{site}.{lang}.xml'; default: 'guide.xml')
-  channels: 'example.com.channels.xml', // path to channels.xml file (required)
+  channels: 'example.com.channels.xml', // path to list of channels; you can also use an array to specify the path to multiple files at once (example: ['channels1.xml', 'channels2.xml']; required)
   lang: 'fr', // default language for all programs (default: 'en')
   days: 3, // number of days for which to grab the program (default: 1)
   delay: 5000, // delay between requests (default: 3000)
@@ -218,18 +216,17 @@ From each function in `config.js` you can access a `context` object containing t
 
 ```xml
 <?xml version="1.0" ?>
-<site site="example.com">
-  <channels>
-    <channel site_id="cnn-23" xmltv_id="CNN.us">CNN</channel>
-    ...
-  </channels>
-</site>
+<channels site="example.com">
+  <channel site_id="cnn-23" xmltv_id="CNN.us">CNN</channel>
+  ...
+</channels>
 ```
 
-You can also specify the language and logo for each channel individually, like so:
+You can also specify the language, site and logo for each channel individually, like so:
 
 ```xml
 <channel
+  site="example.com"
   site_id="france-24"
   xmltv_id="France24.fr"
   lang="fr"

package/bin/epg-grabber.js

@@ -22,7 +22,7 @@ program
   .description(description)
   .requiredOption('-c, --config <config>', 'Path to [site].config.js file')
   .option('-o, --output <output>', 'Path to output file')
-  .option('--channels <channels>', 'Path to channels.xml file')
+  .option('--channels <channels>', 'Path to list of channels')
   .option('--lang <lang>', 'Set default language for all programs')
   .option('--days <days>', 'Number of days for which to grab the program', parseNumber)
   .option('--delay <delay>', 'Delay between requests (in milliseconds)', parseNumber)
@@ -65,17 +65,31 @@ async function main() {
 
   if (options.timeout) config.request.timeout = options.timeout
   if (options.cacheTtl) config.request.cache.ttl = options.cacheTtl
+
   if (options.channels) config.channels = options.channels
-  else if (config.channels)
-    config.channels = file.join(file.dirname(options.config), config.channels)
-  else throw new Error("The required 'channels' property is missing")
 
-  if (!config.channels) return logger.error('Path to [site].channels.xml is missing')
-  logger.info(`Loading '${config.channels}'...`)
-  const grabber = new EPGGrabber(config)
+  let parsedChannels = []
+  if (config.channels) {
+    const dir = file.dirname(options.config)
+
+    let files = []
+    if (Array.isArray(config.channels)) {
+      files = config.channels.map(path => file.join(dir, path))
+    } else if (typeof config.channels === 'string') {
+      files = await file.list(config.channels)
+    } else {
+      throw new Error('The "channels" attribute must be of type array or string')
+    }
+
+    for (let filepath of files) {
+      logger.info(`Loading '${filepath}'...`)
+      const channelsXML = file.read(filepath)
+      const channels = parseChannels(channelsXML)
+      parsedChannels = parsedChannels.concat(channels)
+    }
+  } else throw new Error('Path to "channels" is missing')
 
-  const channelsXML = file.read(config.channels)
-  const { channels: parsedChannels } = parseChannels(channelsXML)
+  const grabber = new EPGGrabber(config)
 
   let template = options.output || config.output
   const variables = file.templateVariables(template)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "epg-grabber",
-  "version": "0.31.0",
+  "version": "0.33.0",
   "description": "Node.js CLI tool for grabbing EPG from different sites",
   "main": "src/index.js",
   "preferGlobal": true,
@@ -8,7 +8,7 @@
     "epg-grabber": "bin/epg-grabber.js"
   },
   "scripts": {
-    "lint": "npx eslint ./src/**/*.js",
+    "lint": "npx eslint ./src/**/*.js ./tests/**/*.js",
     "test": "npx jest"
   },
   "publishConfig": {

package/src/client.js

@@ -15,6 +15,7 @@ let timeout
 function create(config) {
   const client = setupCache(
     axios.create({
+      ignoreCookieErrors: true,
       headers: {
         'User-Agent':
           'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36 Edg/79.0.309.71'

package/src/file.js

@@ -1,6 +1,8 @@
 const fs = require('fs')
 const path = require('path')
+const glob = require('glob')
 
+module.exports.list = list
 module.exports.read = read
 module.exports.write = write
 module.exports.resolve = resolve
@@ -9,6 +11,14 @@ module.exports.dirname = dirname
 module.exports.templateVariables = templateVariables
 module.exports.templateFormat = templateFormat
 
+function list(pattern) {
+  return new Promise(resolve => {
+    glob(pattern, function (err, files) {
+      resolve(files)
+    })
+  })
+}
+
 function read(filepath) {
   return fs.readFileSync(path.resolve(filepath), { encoding: 'utf-8' })
 }

package/src/index.d.ts

@@ -0,0 +1,110 @@
+import dayjs from 'dayjs'
+
+export declare class Channel {
+  xmltv_id?: string
+  name: string
+  site?: string
+  site_id: string
+  lang?: string
+  logo?: string
+}
+
+type TextObject = {
+  lang: string
+  value: string
+}
+
+type IconObject = {
+  src: string
+}
+
+type UrlObject = {
+  system: string
+  value: string
+}
+
+type RatingObject = {
+  system: string
+  icon: string
+  value: string
+}
+
+type ImageObject = {
+  type: string
+  size: string
+  orient: string
+  system: string
+  value: string
+}
+
+type PersonObject = {
+  value: string
+  url: UrlObject[]
+  image: ImageObject[]
+}
+
+export declare class Program {
+  site?: string
+  channel?: string
+  titles: TextObject[]
+  sub_titles?: TextObject[]
+  descriptions?: TextObject[]
+  icon?: IconObject
+  episodeNumbers?: string[]
+  date?: number
+  start: number
+  stop?: number
+  urls?: UrlObject[]
+  ratings?: RatingObject[]
+  categories?: TextObject[]
+  directors?: PersonObject[]
+  actors?: PersonObject[]
+  writers?: PersonObject[]
+  adapters?: PersonObject[]
+  producers?: PersonObject[]
+  composers?: PersonObject[]
+  editors?: PersonObject[]
+  presenters?: PersonObject[]
+  commentators?: PersonObject[]
+  guests?: PersonObject[]
+}
+
+export declare type SiteConfig = {
+  site: string
+  days?: number
+  output?: string
+  channels?: () => object[] | string | Promise<object[]>
+  delay?: number
+  maxConnections?: number
+  request?: object
+  url: () => string | string | Promise<string>
+  logo?: () => string | string | Promise<string>
+  parser: () => object[] | Promise<object[]>
+}
+
+export declare function generateXMLTV({
+  channels,
+  programs,
+  date
+}: {
+  channels: Channel[]
+  programs: Program[]
+  date?: string | null
+}): string
+
+export declare function parseChannels(xml: string): { site: string; channels: Channel[] }
+
+export type GrabCallbackData = {
+  channel: Channel
+  programs: Program[]
+  date: dayjs.Dayjs
+}
+
+export declare class EPGGrabber {
+  constructor(config: object)
+  grab(
+    channel: Channel,
+    date: string | dayjs.Dayjs,
+    cb: (data: GrabCallbackData, err: Error) => void
+  ): Promise<Program[]>
+}

package/src/parser.js

@@ -9,11 +9,19 @@ module.exports.parsePrograms = parsePrograms
 function parseChannels(xml) {
   const result = convert.xml2js(xml)
   const siteTag = result.elements.find(el => el.name === 'site') || {}
-  if (!siteTag.elements) return []
-  const rootSite = siteTag.attributes.site
 
-  const channelsTag = siteTag.elements.find(el => el.name === 'channels')
-  if (!channelsTag.elements) return []
+  const channelsTag =
+    siteTag && Array.isArray(siteTag.elements)
+      ? siteTag.elements.find(el => el.name === 'channels')
+      : result.elements.find(el => el.name === 'channels')
+  if (!channelsTag || !channelsTag.elements) return []
+
+  let rootSite = ''
+  if (siteTag && siteTag.attributes && siteTag.attributes.site) {
+    rootSite = siteTag.attributes.site
+  } else if (channelsTag && channelsTag.attributes && channelsTag.attributes.site) {
+    rootSite = channelsTag.attributes.site
+  }
 
   const channels = channelsTag.elements
     .filter(el => el.name === 'channel')
@@ -26,7 +34,7 @@ function parseChannels(xml) {
       return new Channel(c)
     })
 
-  return { site: rootSite, channels }
+  return channels
 }
 
 async function parsePrograms(data) {

package/tests/__data__/input/example.channels.xml

@@ -1,7 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<site site="example.com">
-  <channels>
-    <channel xmltv_id="1TV.com" site_id="1" lang="fr" logo="https://example.com/logos/1TV.png">1 TV</channel>
-    <channel xmltv_id="2TV.com" site="example.com" site_id="2">2 TV</channel>
-  </channels>
-</site>
+<channels site="example.com">
+  <channel xmltv_id="1TV.com" site_id="1" lang="fr" logo="https://example.com/logos/1TV.png">1 TV</channel>
+  <channel xmltv_id="2TV.com" site_id="2">2 TV</channel>
+</channels>

package/tests/__data__/input/example_2.channels.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<site site="example2.com">
+  <channels>
+    <channel xmltv_id="3TV.com" site_id="3">3 TV</channel>
+    <channel xmltv_id="4TV.com" site_id="4">4 TV</channel>
+  </channels>
+</site>

package/tests/__data__/input/example_3.channels.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<channels>
+  <channel site="example.com" xmltv_id="1TV.com" site_id="1" lang="fr" logo="https://example.com/logos/1TV.png">1 TV</channel>
+  <channel site="example.com" xmltv_id="2TV.com" site_id="2">2 TV</channel>
+</channels>

package/tests/__data__/input/example_channels.config.js

@@ -0,0 +1,32 @@
+const dayjs = require('dayjs')
+const utc = require('dayjs/plugin/utc')
+
+dayjs.extend(utc)
+
+module.exports = {
+  site: 'example.com',
+  days: 2,
+  channels: ['example.channels.xml', 'example_2.channels.xml'],
+  output: 'tests/__data__/output/guide.xml',
+  url: () => 'http://example.com/20210319/1tv.json',
+  request: {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      Cookie: 'abc=123'
+    },
+    data() {
+      return { accountID: '123' }
+    }
+  },
+  parser: () => {
+    return [
+      {
+        title: 'Program1',
+        start: 1640995200000,
+        stop: 1640998800000
+      }
+    ]
+  },
+  logo: () => 'http://example.com/logos/1TV.png?x=шеллы&sid=777'
+}

package/tests/__data__/input/legacy.channels.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<site site="example.com">
+  <channels>
+    <channel xmltv_id="1TV.com" site_id="1" lang="fr" logo="https://example.com/logos/1TV.png">1 TV</channel>
+    <channel xmltv_id="2TV.com" site_id="2">2 TV</channel>
+  </channels>
+</site>

package/tests/__data__/output/guide.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8" ?><tv date="20230927">
+<channel id="1TV.com"><display-name>1 TV</display-name><icon src="https://example.com/logos/1TV.png"/><url>https://example.com</url></channel>
+<channel id="2TV.com"><display-name>2 TV</display-name><icon src="http://example.com/logos/1TV.png?x=шеллы&amp;sid=777"/><url>https://example.com</url></channel>
+<channel id="3TV.com"><display-name>3 TV</display-name><icon src="http://example.com/logos/1TV.png?x=шеллы&amp;sid=777"/><url>https://example2.com</url></channel>
+<channel id="4TV.com"><display-name>4 TV</display-name><icon src="http://example.com/logos/1TV.png?x=шеллы&amp;sid=777"/><url>https://example2.com</url></channel>
+</tv>

package/tests/__mocks__/axios.js

@@ -1,2 +1,3 @@
-import mockAxios from 'jest-mock-axios'
-export default mockAxios
+const mockAxios = require('jest-mock-axios')
+
+module.exports = mockAxios

package/tests/bin.test.js

@@ -87,7 +87,7 @@ it('can produce multiple outputs', () => {
 })
 
 it('removes duplicates of the program', () => {
-  const stdout = execSync(
+  execSync(
     `node ${pwd}/bin/epg-grabber.js \
       --config=tests/__data__/input/duplicates.config.js \
       --channels=tests/__data__/input/example.channels.xml \
@@ -105,3 +105,25 @@ it('removes duplicates of the program', () => {
 
   expect(output.programs).toEqual(expected.programs)
 })
+
+it('can load multiple "channels.xml" files at once', () => {
+  const stdout = execSync(
+    `node ${pwd}/bin/epg-grabber.js --config=tests/__data__/input/example.config.js --channels=tests/__data__/input/example*.channels.xml --timeout=1`,
+    {
+      encoding: 'utf8'
+    }
+  )
+
+  expect(stdoutResultTester(stdout)).toBe(true)
+})
+
+it('can parse list of "channels.xml" from array', () => {
+  const stdout = execSync(
+    `node ${pwd}/bin/epg-grabber.js --config=tests/__data__/input/example_channels.config.js --timeout=1`,
+    {
+      encoding: 'utf8'
+    }
+  )
+
+  expect(stdoutResultTester(stdout)).toBe(true)
+})

package/tests/parser.test.js

@@ -3,14 +3,83 @@ import Channel from '../src/Channel'
 import Program from '../src/Program'
 import fs from 'fs'
 
-it('can parse valid channels.xml', () => {
-  const file = fs.readFileSync('./tests/__data__/input/example.channels.xml', { encoding: 'utf-8' })
-  const { channels, site } = parseChannels(file)
+it('can parse channels.xml', () => {
+  const file = fs.readFileSync('./tests/__data__/input/example.channels.xml', {
+    encoding: 'utf-8'
+  })
+  const channels = parseChannels(file)
 
-  expect(typeof site).toBe('string')
   expect(channels.length).toBe(2)
   expect(channels[0]).toBeInstanceOf(Channel)
   expect(channels[1]).toBeInstanceOf(Channel)
+  expect(channels[0]).toMatchObject({
+    site: 'example.com',
+    site_id: '1',
+    xmltv_id: '1TV.com',
+    lang: 'fr',
+    logo: 'https://example.com/logos/1TV.png',
+    name: '1 TV'
+  })
+  expect(channels[1]).toMatchObject({
+    site: 'example.com',
+    site_id: '2',
+    lang: '',
+    logo: '',
+    xmltv_id: '2TV.com',
+    name: '2 TV'
+  })
+})
+
+it('can parse channels.xml with inline site attribute', () => {
+  const file = fs.readFileSync('./tests/__data__/input/example_3.channels.xml', {
+    encoding: 'utf-8'
+  })
+  const channels = parseChannels(file)
+
+  expect(channels.length).toBe(2)
+  expect(channels[0]).toBeInstanceOf(Channel)
+  expect(channels[1]).toBeInstanceOf(Channel)
+  expect(channels[0]).toMatchObject({
+    site: 'example.com',
+    site_id: '1',
+    xmltv_id: '1TV.com',
+    lang: 'fr',
+    logo: 'https://example.com/logos/1TV.png',
+    name: '1 TV'
+  })
+  expect(channels[1]).toMatchObject({
+    site: 'example.com',
+    site_id: '2',
+    lang: '',
+    logo: '',
+    xmltv_id: '2TV.com',
+    name: '2 TV'
+  })
+})
+
+it('can parse legacy channels.xml', () => {
+  const file = fs.readFileSync('./tests/__data__/input/legacy.channels.xml', { encoding: 'utf-8' })
+  const channels = parseChannels(file)
+
+  expect(channels.length).toBe(2)
+  expect(channels[0]).toBeInstanceOf(Channel)
+  expect(channels[1]).toBeInstanceOf(Channel)
+  expect(channels[0]).toMatchObject({
+    site: 'example.com',
+    site_id: '1',
+    xmltv_id: '1TV.com',
+    lang: 'fr',
+    logo: 'https://example.com/logos/1TV.png',
+    name: '1 TV'
+  })
+  expect(channels[1]).toMatchObject({
+    site: 'example.com',
+    site_id: '2',
+    lang: '',
+    logo: '',
+    xmltv_id: '2TV.com',
+    name: '2 TV'
+  })
 })
 
 it('can parse programs', done => {

package/tests/__data__/output/duplicates.guide.xml

@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?><tv date="20230521">
-<channel id="1TV.com"><display-name>1 TV</display-name><icon src="https://example.com/logos/1TV.png"/><url>https://example.com</url></channel>
-<channel id="2TV.com"><display-name>2 TV</display-name><url>https://example.com</url></channel>
-<programme start="20220101000000 +0000" stop="20220101010000 +0000" channel="1TV.com"><title lang="fr">Program1</title></programme>
-<programme start="20220101000000 +0000" stop="20220101010000 +0000" channel="2TV.com"><title>Program1</title></programme>
-</tv>