epg-grabber 0.28.6 → 0.29.2

package/bin/epg-grabber.js

@@ -9,6 +9,7 @@ const { EPGGrabber, parseChannels, generateXMLTV } = require('../src/index')
 const { create: createLogger } = require('../src/logger')
 const { parseNumber, getUTCDate } = require('../src/utils')
 const { name, version, description } = require('../package.json')
+const _ = require('lodash')
 const dayjs = require('dayjs')
 const utc = require('dayjs/plugin/utc')
 
@@ -99,6 +100,8 @@ async function main() {
     }
   }
 
+  programs = _.uniqBy(programs, p => p.start + p.channel)
+
   const xml = generateXMLTV({ channels, programs })
   let outputPath = options.output || config.output
   if (options.gzip) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "epg-grabber",
-  "version": "0.28.6",
+  "version": "0.29.2",
   "description": "Node.js CLI tool for grabbing EPG from different sites",
   "main": "src/index.js",
   "preferGlobal": true,
@@ -35,6 +35,7 @@
     "commander": "^7.1.0",
     "curl-generator": "^0.2.0",
     "dayjs": "^1.10.4",
+    "epg-parser": "^0.1.6",
     "glob": "^7.1.6",
     "lodash": "^4.17.21",
     "node-gzip": "^1.1.2",

package/src/Program.js

@@ -1,14 +1,21 @@
 const { padStart } = require('lodash')
 const { toArray, toUnix, parseNumber } = require('./utils')
+const Channel = require('./Channel')
 
 class Program {
-	constructor(p) {
+	constructor(p, c) {
+		if (!(c instanceof Channel)) {
+			throw new Error('The second argument in the constructor must be the "Channel" class')
+		}
+
 		const data = {
-			site: p.site || '',
-			channel: p.channel || '',
-			title: p.title || '',
-			sub_title: p.sub_title || '',
-			description: [p.description, p.desc].find(i => i) || '',
+			site: p.site || c.site || '',
+			channel: p.channel || c.id || '',
+			titles: toArray(p.titles || p.title).map(text => toTextObject(text, c.lang)),
+			sub_titles: toArray(p.sub_titles || p.sub_title).map(text => toTextObject(text, c.lang)),
+			descriptions: toArray(p.descriptions || p.description || p.desc).map(text =>
+				toTextObject(text, c.lang)
+			),
 			icon: toIconObject(p.icon),
 			episodeNumbers: p.episodeNumbers || getEpisodeNumbers(p.season, p.episode),
 			date: p.date ? toUnix(p.date) : null,
@@ -16,7 +23,7 @@ class Program {
 			stop: p.stop ? toUnix(p.stop) : null,
 			urls: toArray(p.urls || p.url).map(toUrlObject),
 			ratings: toArray(p.ratings || p.rating).map(toRatingObject),
-			categories: toArray(p.categories || p.category),
+			categories: toArray(p.categories || p.category).map(text => toTextObject(text, c.lang)),
 			directors: toArray(p.directors || p.director).map(toPersonObject),
 			actors: toArray(p.actors || p.actor).map(toPersonObject),
 			writers: toArray(p.writers || p.writer).map(toPersonObject),
@@ -37,6 +44,17 @@ class Program {
 
 module.exports = Program
 
+function toTextObject(text, lang) {
+	if (typeof text === 'string') {
+		return { value: text, lang }
+	}
+
+	return {
+		value: text.value,
+		lang: text.lang
+	}
+}
+
 function toPersonObject(person) {
 	if (typeof person === 'string') {
 		return {

package/src/index.js

@@ -27,6 +27,10 @@ class EPGGrabber {
   }
 
   async grab(channel, date, cb = () => {}) {
+    if (!(channel instanceof Channel)) {
+      throw new Error('The first argument must be the "Channel" class')
+    }
+
     await sleep(this.config.delay)
 
     date = typeof date === 'string' ? getUTCDate(date) : date

package/src/parser.js

@@ -41,12 +41,5 @@ async function parsePrograms(data) {
     throw new Error('Parser should return an array')
   }
 
-  return programs
-    .filter(i => i)
-    .map(p => {
-      p.site = channel.site
-      p.channel = p.channel || channel.id
-
-      return new Program(p)
-    })
+  return programs.filter(i => i).map(p => new Program(p, channel))
 }

package/src/xmltv.js

@@ -47,9 +47,15 @@ function createElements(channels, programs, date) {
 						channel: program.channel
 					},
 					[
-						el('title', {}, [escapeString(program.title)]),
-						el('sub-title', {}, [escapeString(program.sub_title)]),
-						el('desc', {}, [escapeString(program.description)]),
+						...program.titles.map(title =>
+							el('title', { lang: title.lang }, [escapeString(title.value)])
+						),
+						...program.sub_titles.map(sub_title =>
+							el('sub-title', { lang: sub_title.lang }, [escapeString(sub_title.value)])
+						),
+						...program.descriptions.map(desc =>
+							el('desc', { lang: desc.lang }, [escapeString(desc.value)])
+						),
 						el('credits', {}, [
 							...program.directors.map(data => createCastMember('director', data)),
 							...program.actors.map(data => createCastMember('actor', data)),
@@ -63,7 +69,9 @@ function createElements(channels, programs, date) {
 							...program.guests.map(data => createCastMember('guest', data))
 						]),
 						el('date', {}, [formatDate(program.date, 'YYYYMMDD')]),
-						...program.categories.map(category => el('category', {}, [escapeString(category)])),
+						...program.categories.map(category =>
+							el('category', { lang: category.lang }, [escapeString(category.value)])
+						),
 						el('icon', { src: program.icon.src }),
 						...program.urls.map(createURL),
 						...program.episodeNumbers.map(episode =>

package/tests/Program.test.js

@@ -1,61 +1,65 @@
 import Channel from '../src/Channel'
 import Program from '../src/Program'
 
-const channel = new Channel({ xmltv_id: '1tv', lang: 'en', site: 'example.com' })
+const channel = new Channel({ xmltv_id: '1tv', lang: 'fr', site: 'example.com' })
 
 it('can create new Program', () => {
-  const program = new Program({
-    site: channel.site,
-    channel: channel.id,
-    title: 'Title',
-    sub_title: 'Subtitle',
-    description: 'Description',
-    icon: 'https://example.com/image.jpg',
-    season: 9,
-    episode: 238,
-    date: '20220506',
-    start: 1616133600000,
-    stop: '2021-03-19T06:30:00.000Z',
-    url: 'http://example.com/title.html',
-    category: ['Category1', 'Category2'],
-    rating: {
-      system: 'MPAA',
-      value: 'PG',
-      icon: 'http://example.com/pg_symbol.png'
-    },
-    directors: 'Director1',
-    actors: [
-      'Actor1',
-      { value: 'Actor2', url: 'http://actor2.com', image: 'http://actor2.com/image.png' }
-    ],
-    writer: {
-      value: 'Writer1',
-      url: { system: 'imdb', value: 'http://imdb.com/p/writer1' },
-      image: {
-        value: 'https://example.com/image.jpg',
-        type: 'person',
-        size: '2',
-        system: 'TestSystem',
-        orient: 'P'
-      }
+  const program = new Program(
+    {
+      title: 'Title',
+      sub_title: 'Subtitle',
+      description: 'Description',
+      icon: 'https://example.com/image.jpg',
+      season: 9,
+      episode: 238,
+      date: '20220506',
+      start: 1616133600000,
+      stop: '2021-03-19T06:30:00.000Z',
+      url: 'http://example.com/title.html',
+      category: ['Category1', 'Category2'],
+      rating: {
+        system: 'MPAA',
+        value: 'PG',
+        icon: 'http://example.com/pg_symbol.png'
+      },
+      directors: 'Director1',
+      actors: [
+        'Actor1',
+        { value: 'Actor2', url: 'http://actor2.com', image: 'http://actor2.com/image.png' }
+      ],
+      writer: {
+        value: 'Writer1',
+        url: { system: 'imdb', value: 'http://imdb.com/p/writer1' },
+        image: {
+          value: 'https://example.com/image.jpg',
+          type: 'person',
+          size: '2',
+          system: 'TestSystem',
+          orient: 'P'
+        }
+      },
+      adapters: [
+        {
+          value: 'Adapter1',
+          url: ['http://imdb.com/p/adapter1', 'http://imdb.com/p/adapter2'],
+          image: ['https://example.com/image1.jpg', 'https://example.com/image2.jpg']
+        }
+      ]
     },
-    adapters: [
-      {
-        value: 'Adapter1',
-        url: ['http://imdb.com/p/adapter1', 'http://imdb.com/p/adapter2'],
-        image: ['https://example.com/image1.jpg', 'https://example.com/image2.jpg']
-      }
-    ]
-  })
+    channel
+  )
 
   expect(program).toMatchObject({
     site: 'example.com',
     channel: '1tv',
-    title: 'Title',
-    sub_title: 'Subtitle',
-    description: 'Description',
+    titles: [{ value: 'Title', lang: 'fr' }],
+    sub_titles: [{ value: 'Subtitle', lang: 'fr' }],
+    descriptions: [{ value: 'Description', lang: 'fr' }],
     urls: [{ system: '', value: 'http://example.com/title.html' }],
-    categories: ['Category1', 'Category2'],
+    categories: [
+      { value: 'Category1', lang: 'fr' },
+      { value: 'Category2', lang: 'fr' }
+    ],
     icon: { src: 'https://example.com/image.jpg' },
     episodeNumbers: [
       { system: 'xmltv_ns', value: '8.237.0/1' },
@@ -132,24 +136,26 @@ it('can create new Program', () => {
 })
 
 it('can create program from exist object', () => {
-  const program = new Program({
-    channel: channel.id,
-    title: 'Program 1',
-    start: '2021-03-19T06:00:00.000Z',
-    stop: '2021-03-19T06:30:00.000Z',
-    ratings: {
-      system: 'MPAA',
-      value: 'PG',
-      icon: 'http://example.com/pg_symbol.png'
+  const program = new Program(
+    {
+      titles: [{ value: 'Program 1', lang: 'de' }],
+      start: '2021-03-19T06:00:00.000Z',
+      stop: '2021-03-19T06:30:00.000Z',
+      ratings: {
+        system: 'MPAA',
+        value: 'PG',
+        icon: 'http://example.com/pg_symbol.png'
+      },
+      actors: [{ value: 'Actor1', url: [], image: [] }]
     },
-    actors: [{ value: 'Actor1', url: [], image: [] }]
-  })
+    channel
+  )
 
   expect(program).toMatchObject({
     channel: '1tv',
-    title: 'Program 1',
-    sub_title: '',
-    description: '',
+    titles: [{ value: 'Program 1', lang: 'de' }],
+    sub_titles: [],
+    descriptions: [],
     urls: [],
     categories: [],
     icon: {},
@@ -178,13 +184,15 @@ it('can create program from exist object', () => {
 })
 
 it('can create program without season number', () => {
-  const program = new Program({
-    channel: channel.id,
-    title: 'Program 1',
-    start: '2021-03-19T06:00:00.000Z',
-    stop: '2021-03-19T06:30:00.000Z',
-    episode: 238
-  })
+  const program = new Program(
+    {
+      title: 'Program 1',
+      start: '2021-03-19T06:00:00.000Z',
+      stop: '2021-03-19T06:30:00.000Z',
+      episode: 238
+    },
+    channel
+  )
 
   expect(program.episodeNumbers).toMatchObject([
     { system: 'xmltv_ns', value: '0.237.0/1' },
@@ -193,13 +201,16 @@ it('can create program without season number', () => {
 })
 
 it('can create program without episode number', () => {
-  const program = new Program({
-    channel: channel.id,
-    title: 'Program 1',
-    start: '2021-03-19T06:00:00.000Z',
-    stop: '2021-03-19T06:30:00.000Z',
-    season: 3
-  })
+  const program = new Program(
+    {
+      channel: channel.id,
+      title: 'Program 1',
+      start: '2021-03-19T06:00:00.000Z',
+      stop: '2021-03-19T06:30:00.000Z',
+      season: 3
+    },
+    channel
+  )
 
   expect(program.episodeNumbers).toMatchObject([])
 })

package/tests/__data__/expected/duplicates.guide.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8" ?><tv date="20220828">
+<channel id="1TV.com"><display-name>1 TV</display-name><icon src="https://example.com/logos/1TV.png"/><url>https://example.com</url></channel>
+<channel id="2TV.com"><display-name>2 TV</display-name><url>https://example.com</url></channel>
+<programme start="20220101000000 +0000" stop="20220101010000 +0000" channel="1TV.com"><title lang="fr">Program1</title></programme>
+<programme start="20220101000000 +0000" stop="20220101010000 +0000" channel="2TV.com"><title>Program1</title></programme>
+</tv>

package/tests/__data__/input/duplicates.config.js

@@ -0,0 +1,18 @@
+module.exports = {
+  site: 'example.com',
+  url: 'https://google.com',
+  parser() {
+    return [
+      {
+        title: 'Program1',
+        start: 1640995200000,
+        stop: 1640998800000
+      },
+      {
+        title: 'Program1',
+        start: 1640995200000,
+        stop: 1640998900000
+      }
+    ]
+  }
+}

package/tests/__data__/output/duplicates.guide.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8" ?><tv date="20220829">
+<channel id="1TV.com"><display-name>1 TV</display-name><icon src="https://example.com/logos/1TV.png"/><url>https://example.com</url></channel>
+<channel id="2TV.com"><display-name>2 TV</display-name><url>https://example.com</url></channel>
+<programme start="20220101000000 +0000" stop="20220101010000 +0000" channel="1TV.com"><title lang="fr">Program1</title></programme>
+<programme start="20220101000000 +0000" stop="20220101010000 +0000" channel="2TV.com"><title>Program1</title></programme>
+</tv>

package/tests/__data__/output/mini.guide.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8" ?><tv date="20220829">
+<channel id="1TV.com"><display-name>1 TV</display-name><icon src="https://example.com/logos/1TV.png"/><url>https://example.com</url></channel>
+<channel id="2TV.com"><display-name>2 TV</display-name><url>https://example.com</url></channel>
+</tv>

package/tests/bin.test.js

@@ -1,4 +1,7 @@
 const { execSync } = require('child_process')
+const fs = require('fs')
+const path = require('path')
+const epgParser = require('epg-parser')
 
 const pwd = `${__dirname}/..`
 
@@ -10,7 +13,7 @@ function stdoutResultTester(stdout) {
 
 it('can load config', () => {
   const result = execSync(
-    `node ${pwd}/bin/epg-grabber.js --config=tests/input/example.config.js --delay=0`,
+    `node ${pwd}/bin/epg-grabber.js --config=tests/__data__/input/example.config.js --delay=0`,
     {
       encoding: 'utf8'
     }
@@ -22,9 +25,9 @@ it('can load config', () => {
 it('can load mini config', () => {
   const result = execSync(
     `node ${pwd}/bin/epg-grabber.js \
-      --config=tests/input/mini.config.js \
-      --channels=tests/input/example.channels.xml \
-      --output=tests/output/mini.guide.xml \
+      --config=tests/__data__/input/mini.config.js \
+      --channels=tests/__data__/input/example.channels.xml \
+      --output=tests/__data__/output/mini.guide.xml \
       --lang=fr \
       --days=3 \
       --delay=0 \
@@ -36,15 +39,17 @@ it('can load mini config', () => {
   )
 
   expect(stdoutResultTester(result)).toBe(true)
-  expect(result.includes("File 'tests/output/mini.guide.xml' successfully saved")).toBe(true)
+  expect(result.includes("File 'tests/__data__/output/mini.guide.xml' successfully saved")).toBe(
+    true
+  )
 })
 
 it('can generate gzip version', () => {
   const result = execSync(
     `node ${pwd}/bin/epg-grabber.js \
-      --config=tests/input/mini.config.js \
-      --channels=tests/input/example.channels.xml \
-      --output=tests/output/mini.guide.xml.gz \
+      --config=tests/__data__/input/mini.config.js \
+      --channels=tests/__data__/input/example.channels.xml \
+      --output=tests/__data__/output/mini.guide.xml.gz \
       --gzip`,
     {
       encoding: 'utf8'
@@ -52,5 +57,27 @@ it('can generate gzip version', () => {
   )
 
   expect(stdoutResultTester(result)).toBe(true)
-  expect(result.includes("File 'tests/output/mini.guide.xml.gz' successfully saved")).toBe(true)
+  expect(result.includes("File 'tests/__data__/output/mini.guide.xml.gz' successfully saved")).toBe(
+    true
+  )
+})
+
+it('removes duplicates of the program', () => {
+  const result = execSync(
+    `node ${pwd}/bin/epg-grabber.js \
+      --config=tests/__data__/input/duplicates.config.js \
+      --channels=tests/__data__/input/example.channels.xml \
+      --output=tests/__data__/output/duplicates.guide.xml`,
+    {
+      encoding: 'utf8'
+    }
+  )
+
+  let output = fs.readFileSync(path.resolve(__dirname, '__data__/output/duplicates.guide.xml'))
+  let expected = fs.readFileSync(path.resolve(__dirname, '__data__/expected/duplicates.guide.xml'))
+
+  output = epgParser.parse(output)
+  expected = epgParser.parse(expected)
+
+  expect(output.programs).toEqual(expected.programs)
 })

package/tests/config.test.js

@@ -3,7 +3,7 @@ import path from 'path'
 import fs from 'fs'
 
 it('can load config', () => {
-  const config = loadConfig(require(path.resolve('./tests/input/example.config.js')))
+  const config = loadConfig(require(path.resolve('./tests/__data__/input/example.config.js')))
   expect(config).toMatchObject({
     days: 1,
     delay: 3000,

package/tests/index.test.js

@@ -2,7 +2,7 @@
  * @jest-environment node
  */
 
-import { EPGGrabber } from '../src/index'
+import { EPGGrabber, Channel } from '../src/index'
 import axios from 'axios'
 
 jest.mock('axios')
@@ -20,13 +20,13 @@ it('return "Connection timeout" error if server does not response', done => {
     },
     parser: () => []
   }
-  const channel = {
+  const channel = new Channel({
     site: 'example.com',
     site_id: 'cnn',
     xmltv_id: 'CNN.us',
     lang: 'en',
     name: 'CNN'
-  }
+  })
   const grabber = new EPGGrabber(config)
   grabber.grab(channel, '2022-01-01', (data, err) => {
     expect(err.message).toBe('Connection timeout')
@@ -47,13 +47,13 @@ it('can grab single channel programs', done => {
     url: 'http://example.com/20210319/1tv.json',
     parser: () => []
   }
-  const channel = {
+  const channel = new Channel({
     site: 'example.com',
     site_id: '1',
     xmltv_id: '1TV.fr',
     lang: 'fr',
     name: '1TV'
-  }
+  })
   const grabber = new EPGGrabber(config)
   grabber
     .grab(channel, '2022-01-01', (data, err) => {

package/tests/parser.test.js

@@ -4,7 +4,7 @@ import Program from '../src/Program'
 import fs from 'fs'
 
 it('can parse valid channels.xml', () => {
-  const file = fs.readFileSync('./tests/input/example.channels.xml', { encoding: 'utf-8' })
+  const file = fs.readFileSync('./tests/__data__/input/example.channels.xml', { encoding: 'utf-8' })
   const { channels, site } = parseChannels(file)
 
   expect(typeof site).toBe('string')
@@ -14,8 +14,8 @@ it('can parse valid channels.xml', () => {
 })
 
 it('can parse programs', done => {
-  const channel = { xmltv_id: '1tv' }
-  const config = require('./input/example.config.js')
+  const channel = new Channel({ xmltv_id: '1tv' })
+  const config = require('./__data__/input/example.config.js')
 
   parsePrograms({ channel, config })
     .then(programs => {
@@ -27,8 +27,8 @@ it('can parse programs', done => {
 })
 
 it('can parse programs async', done => {
-  const channel = { xmltv_id: '1tv' }
-  const config = require('./input/async.config.js')
+  const channel = new Channel({ xmltv_id: '1tv' })
+  const config = require('./__data__/input/async.config.js')
 
   parsePrograms({ channel, config })
     .then(programs => {

package/tests/xmltv.test.js

@@ -14,11 +14,12 @@ const channels = [
   new Channel({
     xmltv_id: '2TV.co',
     name: '2 TV',
-    site: 'example.com'
+    site: 'example.com',
+    lang: 'es'
   })
 ]
 
-fit('can generate xmltv', () => {
+it('can generate xmltv', () => {
   const programs = [
     new Program(
       {
@@ -33,7 +34,6 @@ fit('can generate xmltv', () => {
         season: 9,
         episode: 239,
         icon: 'https://example.com/images/Program1.png?x=шеллы&sid=777',
-        channel: '1TV.co',
         rating: {
           system: 'MPAA',
           value: 'PG',
@@ -60,12 +60,20 @@ fit('can generate xmltv', () => {
         writer: 'Writer 1'
       },
       channels[0]
+    ),
+    new Program(
+      {
+        title: 'Program 2',
+        start: '2021-03-19T06:00:00.000Z',
+        stop: '2021-03-19T06:30:00.000Z'
+      },
+      channels[1]
     )
   ]
 
   const output = xmltv.generate({ channels, programs })
 
   expect(output).toBe(
-    '<?xml version="1.0" encoding="UTF-8" ?><tv date="20220505">\r\n<channel id="1TV.co"><display-name>1 TV</display-name><icon src="https://example.com/logos/1TV.png"/><url>https://example.com</url></channel>\r\n<channel id="2TV.co"><display-name>2 TV</display-name><url>https://example.com</url></channel>\r\n<programme start="20210319060000 +0000" stop="20210319063000 +0000" channel="1TV.co"><title>Program 1</title><sub-title>Sub-title &amp; 1</sub-title><desc>Description for Program 1</desc><credits><director>Director 1<url system="TestSystem">http://example.com/director1.html</url><image>https://example.com/image1.jpg</image><image type="person" size="2" orient="P" system="TestSystem">https://example.com/image2.jpg</image></director><director>Director 2</director><actor>Actor 1</actor><actor>Actor 2</actor><writer>Writer 1</writer></credits><date>20220506</date><category>Test</category><icon src="https://example.com/images/Program1.png?x=шеллы&amp;sid=777"/><url>http://example.com/title.html</url><episode-num system="xmltv_ns">8.238.0/1</episode-num><episode-num system="onscreen">S09E239</episode-num><rating system="MPAA"><value>PG</value><icon src="http://example.com/pg_symbol.png"/></rating></programme>\r\n</tv>'
+    '<?xml version="1.0" encoding="UTF-8" ?><tv date="20220505">\r\n<channel id="1TV.co"><display-name>1 TV</display-name><icon src="https://example.com/logos/1TV.png"/><url>https://example.com</url></channel>\r\n<channel id="2TV.co"><display-name>2 TV</display-name><url>https://example.com</url></channel>\r\n<programme start="20210319060000 +0000" stop="20210319063000 +0000" channel="1TV.co"><title>Program 1</title><sub-title>Sub-title &amp; 1</sub-title><desc>Description for Program 1</desc><credits><director>Director 1<url system="TestSystem">http://example.com/director1.html</url><image>https://example.com/image1.jpg</image><image type="person" size="2" orient="P" system="TestSystem">https://example.com/image2.jpg</image></director><director>Director 2</director><actor>Actor 1</actor><actor>Actor 2</actor><writer>Writer 1</writer></credits><date>20220506</date><category>Test</category><icon src="https://example.com/images/Program1.png?x=шеллы&amp;sid=777"/><url>http://example.com/title.html</url><episode-num system="xmltv_ns">8.238.0/1</episode-num><episode-num system="onscreen">S09E239</episode-num><rating system="MPAA"><value>PG</value><icon src="http://example.com/pg_symbol.png"/></rating></programme>\r\n<programme start="20210319060000 +0000" stop="20210319063000 +0000" channel="2TV.co"><title lang="es">Program 2</title></programme>\r\n</tv>'
   )
 })