epg-grabber 0.27.2 → 0.28.2

package/README.md

@@ -166,12 +166,24 @@ module.exports = {
         title, // program title (required)
         start, // start time of the program (required)
         stop, // end time of the program (required)
+        sub_title, // program sub-title (optional)
         description, // description of the program (optional)
-        category, // program type (optional)
+        category, // type of program (optional)
         season, // season number (optional)
         episode, // episode number (optional)
+        date, // the date the programme or film was finished (optional)
         icon, // image associated with the program (optional)
-        lang // language of the description (default: 'en')
+        rating, // program rating (optional)
+        director, // the name of director (optional)
+        actor, // the name of actor (optional)
+        writer, // the name of writer (optional)
+        adapter, // the name of adapter (optional)
+        producer, // the name of producer (optional)
+        composer, // the name of composer (optional)
+        editor, // the name of editor (optional)
+        presenter, // the name of presenter (optional)
+        commentator, // the name of commentator (optional)
+        guest // the name of guest (optional)
       },
       ...
     ]

package/bin/epg-grabber.js

@@ -2,15 +2,17 @@
 
 const { Command } = require('commander')
 const program = new Command()
-const fs = require('fs')
-const path = require('path')
-const EPGGrabber = require('../src/index')
-const utils = require('../src/utils')
-const { name, version, description } = require('../package.json')
 const { merge } = require('lodash')
 const { gzip } = require('node-gzip')
-const { createLogger, format, transports } = require('winston')
-const { combine, timestamp, printf } = format
+const file = require('../src/file')
+const { EPGGrabber, parseChannels, generateXMLTV } = require('../src/index')
+const { create: createLogger } = require('../src/logger')
+const { parseNumber, getUTCDate } = require('../src/utils')
+const { name, version, description } = require('../package.json')
+const dayjs = require('dayjs')
+const utc = require('dayjs/plugin/utc')
+
+dayjs.extend(utc)
 
 program
   .name(name)
@@ -20,13 +22,13 @@ program
   .option('-o, --output <output>', 'Path to output file')
   .option('--channels <channels>', 'Path to channels.xml file')
   .option('--lang <lang>', 'Set default language for all programs')
-  .option('--days <days>', 'Number of days for which to grab the program', parseInteger, 1)
-  .option('--delay <delay>', 'Delay between requests (in mileseconds)', parseInteger)
-  .option('--timeout <timeout>', 'Set a timeout for each request (in mileseconds)', parseInteger)
+  .option('--days <days>', 'Number of days for which to grab the program', parseNumber, 1)
+  .option('--delay <delay>', 'Delay between requests (in mileseconds)', parseNumber)
+  .option('--timeout <timeout>', 'Set a timeout for each request (in mileseconds)', parseNumber)
   .option(
     '--cache-ttl <cacheTtl>',
     'Maximum time for storing each request (in milliseconds)',
-    parseInteger
+    parseNumber
   )
   .option('--gzip', 'Compress the output', false)
   .option('--debug', 'Enable debug mode', false)
@@ -36,39 +38,13 @@ program
   .parse(process.argv)
 
 const options = program.opts()
-
-const fileFormat = printf(({ level, message, timestamp }) => {
-  return `[${timestamp}] ${level.toUpperCase()}: ${message}`
-})
-
-const consoleFormat = printf(({ level, message, timestamp }) => {
-  if (level === 'error') return `  Error: ${message}`
-
-  return message
-})
-
-const t = [new transports.Console({ format: consoleFormat })]
-
-if (options.log) {
-  t.push(
-    new transports.File({
-      filename: path.resolve(options.log),
-      format: combine(timestamp(), fileFormat),
-      options: { flags: 'w' }
-    })
-  )
-}
-
-const logger = createLogger({
-  level: options.logLevel,
-  transports: t
-})
+const logger = createLogger(options)
 
 async function main() {
   logger.info('Starting...')
 
   logger.info(`Loading '${options.config}'...`)
-  let config = require(path.resolve(options.config))
+  let config = require(file.resolve(options.config))
   config = merge(config, {
     days: options.days,
     debug: options.debug,
@@ -83,29 +59,34 @@ async function main() {
   if (options.cacheTtl) config.request.cache.ttl = options.cacheTtl
   if (options.channels) config.channels = options.channels
   else if (config.channels)
-    config.channels = path.join(path.dirname(options.config), config.channels)
+    config.channels = file.join(file.dirname(options.config), config.channels)
   else throw new Error("The required 'channels' property is missing")
 
   if (!config.channels) return logger.error('Path to [site].channels.xml is missing')
   logger.info(`Loading '${config.channels}'...`)
-  const channelsXML = fs.readFileSync(path.resolve(config.channels), { encoding: 'utf-8' })
-  const { channels } = utils.parseChannels(channelsXML)
+  const grabber = new EPGGrabber(config)
+
+  const channelsXML = file.read(config.channels)
+  const { channels } = parseChannels(channelsXML)
 
   let programs = []
   let i = 1
   let days = options.days || 1
   const total = channels.length * days
-  const utcDate = utils.getUTCDate()
+  const utcDate = getUTCDate()
   const dates = Array.from({ length: config.days }, (_, i) => utcDate.add(i, 'd'))
-  const grabber = new EPGGrabber(config)
   for (let channel of channels) {
+    if (!channel.logo && config.logo) {
+      channel.logo = await grabber.loadLogo(channel)
+    }
+
     for (let date of dates) {
       await grabber
         .grab(channel, date, (data, err) => {
           logger.info(
-            `[${i}/${total}] ${config.site} - ${data.channel.xmltv_id} - ${data.date.format(
-              'MMM D, YYYY'
-            )} (${data.programs.length} programs)`
+            `[${i}/${total}] ${config.site} - ${data.channel.id} - ${dayjs
+              .utc(data.date)
+              .format('MMM D, YYYY')} (${data.programs.length} programs)`
           )
 
           if (err) logger.error(err.message)
@@ -118,15 +99,15 @@ async function main() {
     }
   }
 
-  const xml = utils.convertToXMLTV({ config, channels, programs })
+  const xml = generateXMLTV({ channels, programs })
   let outputPath = options.output || config.output
   if (options.gzip) {
     outputPath = outputPath || 'guide.xml.gz'
     const compressed = await gzip(xml)
-    utils.writeToFile(outputPath, compressed)
+    file.write(outputPath, compressed)
   } else {
     outputPath = outputPath || 'guide.xml'
-    utils.writeToFile(outputPath, xml)
+    file.write(outputPath, xml)
   }
 
   logger.info(`File '${outputPath}' successfully saved`)
@@ -134,7 +115,3 @@ async function main() {
 }
 
 main()
-
-function parseInteger(val) {
-  return val ? parseInt(val) : null
-}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "epg-grabber",
-  "version": "0.27.2",
+  "version": "0.28.2",
   "description": "Node.js CLI tool for grabbing EPG from different sites",
   "main": "src/index.js",
   "preferGlobal": true,

package/src/Channel.js

@@ -0,0 +1,19 @@
+class Channel {
+	constructor(c) {
+		const data = {
+			id: c.xmltv_id,
+			name: c.name,
+			site: c.site || '',
+			site_id: c.site_id,
+			lang: c.lang || '',
+			logo: c.logo || '',
+			url: c.site ? `https://${c.site}` : ''
+		}
+
+		for (let key in data) {
+			this[key] = data[key]
+		}
+	}
+}
+
+module.exports = Channel

package/src/Program.js

@@ -0,0 +1,122 @@
+const { padStart } = require('lodash')
+const { toArray, toUnix, parseNumber } = require('./utils')
+
+class Program {
+	constructor(p, c) {
+		const data = {
+			channel: c.id,
+			title: p.title,
+			sub_title: p.sub_title || '',
+			description: p.description || p.desc,
+			icon: toIconObject(p.icon),
+			episodeNumbers: getEpisodeNumbers(p.season, p.episode),
+			date: p.date ? toUnix(p.date) : null,
+			start: toUnix(p.start),
+			stop: toUnix(p.stop),
+			urls: toArray(p.url).map(toUrlObject),
+			ratings: toArray(p.ratings || p.rating).map(toRatingObject),
+			categories: toArray(p.categories || p.category),
+			directors: toArray(p.directors || p.director).map(toPersonObject),
+			actors: toArray(p.actors || p.actor).map(toPersonObject),
+			writers: toArray(p.writers || p.writer).map(toPersonObject),
+			adapters: toArray(p.adapters || p.adapter).map(toPersonObject),
+			producers: toArray(p.producers || p.producer).map(toPersonObject),
+			composers: toArray(p.composers || p.composer).map(toPersonObject),
+			editors: toArray(p.editors || p.editor).map(toPersonObject),
+			presenters: toArray(p.presenters || p.presenter).map(toPersonObject),
+			commentators: toArray(p.commentators || p.commentator).map(toPersonObject),
+			guests: toArray(p.guests || p.guest).map(toPersonObject)
+		}
+
+		for (let key in data) {
+			this[key] = data[key]
+		}
+	}
+}
+
+module.exports = Program
+
+function toPersonObject(person) {
+	if (typeof person === 'string') {
+		return {
+			value: person,
+			url: [],
+			image: []
+		}
+	}
+
+	return {
+		value: person.value,
+		url: toArray(person.url).map(toUrlObject),
+		image: toArray(person.image).map(toImageObject)
+	}
+}
+
+function toImageObject(image) {
+	if (typeof image === 'string') return { type: '', size: '', orient: '', system: '', value: image }
+
+	return {
+		type: image.type || '',
+		size: image.size || '',
+		orient: image.orient || '',
+		system: image.system || '',
+		value: image.value
+	}
+}
+
+function toRatingObject(rating) {
+	if (typeof rating === 'string') return { system: '', icon: '', value: rating }
+
+	return {
+		system: rating.system || '',
+		icon: rating.icon || '',
+		value: rating.value || ''
+	}
+}
+
+function toUrlObject(url) {
+	if (typeof url === 'string') return { system: '', value: url }
+
+	return {
+		system: url.system || '',
+		value: url.value || ''
+	}
+}
+
+function toIconObject(icon) {
+	if (!icon || typeof icon === 'string') return { src: icon }
+
+	return {
+		src: icon.src || ''
+	}
+}
+
+function getEpisodeNumbers(s, e) {
+	s = parseNumber(s)
+	e = parseNumber(e)
+
+	return [createXMLTVNS(s, e), createOnScreen(s, e)].filter(Boolean)
+}
+
+function createXMLTVNS(s, e) {
+	if (!e) return null
+	s = s || 1
+
+	return {
+		system: 'xmltv_ns',
+		value: `${s - 1}.${e - 1}.0/1`
+	}
+}
+
+function createOnScreen(s, e) {
+	if (!e) return null
+	s = s || 1
+
+	s = padStart(s, 2, '0')
+	e = padStart(e, 2, '0')
+
+	return {
+		system: 'onscreen',
+		value: `S${s}E${e}`
+	}
+}

package/src/client.js

@@ -0,0 +1,138 @@
+const { CurlGenerator } = require('curl-generator')
+const axios = require('axios').default
+const axiosCookieJarSupport = require('axios-cookiejar-support').default
+const { setupCache } = require('axios-cache-interceptor')
+const { isObject, isPromise, getUTCDate } = require('./utils')
+
+axiosCookieJarSupport(axios)
+
+module.exports.create = create
+module.exports.buildRequest = buildRequest
+module.exports.parseResponse = parseResponse
+
+let timeout
+
+function create(config) {
+	const client = setupCache(
+		axios.create({
+			headers: {
+				'User-Agent':
+					'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36 Edg/79.0.309.71'
+			}
+		})
+	)
+
+	client.interceptors.request.use(
+		function (request) {
+			if (config.debug) {
+				console.log('Request:', JSON.stringify(request, null, 2))
+			}
+			return request
+		},
+		function (error) {
+			return Promise.reject(error)
+		}
+	)
+
+	client.interceptors.response.use(
+		function (response) {
+			if (config.debug) {
+				const data =
+					isObject(response.data) || Array.isArray(response.data)
+						? JSON.stringify(response.data)
+						: response.data.toString()
+				console.log(
+					'Response:',
+					JSON.stringify(
+						{
+							headers: response.headers,
+							data,
+							cached: response.cached
+						},
+						null,
+						2
+					)
+				)
+			}
+
+			clearTimeout(timeout)
+			return response
+		},
+		function (error) {
+			clearTimeout(timeout)
+			return Promise.reject(error)
+		}
+	)
+
+	return client
+}
+
+async function buildRequest({ channel, date, config }) {
+	date = typeof date === 'string' ? getUTCDate(date) : date
+	const CancelToken = axios.CancelToken
+	const source = CancelToken.source()
+	const request = { ...config.request }
+	timeout = setTimeout(() => {
+		source.cancel('Connection timeout')
+	}, request.timeout)
+	request.headers = await getRequestHeaders({ channel, date, config })
+	request.url = await getRequestUrl({ channel, date, config })
+	request.data = await getRequestData({ channel, date, config })
+	request.cancelToken = source.token
+
+	if (config.curl) {
+		const curl = CurlGenerator({
+			url: request.url,
+			method: request.method,
+			headers: request.headers,
+			body: request.data
+		})
+		console.log(curl)
+	}
+
+	return request
+}
+
+function parseResponse(response) {
+	return {
+		content: response.data.toString(),
+		buffer: response.data,
+		headers: response.headers,
+		request: response.request,
+		cached: response.cached
+	}
+}
+
+async function getRequestHeaders({ channel, date, config }) {
+	if (typeof config.request.headers === 'function') {
+		const headers = config.request.headers({ channel, date })
+		if (isPromise(headers)) {
+			return await headers
+		}
+		return headers
+	}
+
+	return config.request.headers || null
+}
+
+async function getRequestData({ channel, date, config }) {
+	if (typeof config.request.data === 'function') {
+		const data = config.request.data({ channel, date })
+		if (isPromise(data)) {
+			return await data
+		}
+		return data
+	}
+	return config.request.data || null
+}
+
+async function getRequestUrl({ channel, date, config }) {
+	if (typeof config.url === 'function') {
+		const url = config.url({ channel, date })
+		if (isPromise(url)) {
+			return await url
+		}
+		return url
+	}
+	return config.url
+}

package/src/config.js

@@ -0,0 +1,34 @@
+const tough = require('tough-cookie')
+const { merge } = require('lodash')
+
+module.exports.load = load
+
+function load(config) {
+	if (!config.site) throw new Error("The required 'site' property is missing")
+	if (!config.url) throw new Error("The required 'url' property is missing")
+	if (typeof config.url !== 'function' && typeof config.url !== 'string')
+		throw new Error("The 'url' property should return the function or string")
+	if (!config.parser) throw new Error("The required 'parser' function is missing")
+	if (typeof config.parser !== 'function')
+		throw new Error("The 'parser' property should return the function")
+	if (config.logo && typeof config.logo !== 'function')
+		throw new Error("The 'logo' property should return the function")
+
+	const defaultConfig = {
+		days: 1,
+		lang: 'en',
+		delay: 3000,
+		output: 'guide.xml',
+		request: {
+			method: 'GET',
+			maxContentLength: 5 * 1024 * 1024,
+			timeout: 5000,
+			withCredentials: true,
+			jar: new tough.CookieJar(),
+			responseType: 'arraybuffer',
+			cache: false
+		}
+	}
+
+	return merge(defaultConfig, config)
+}

package/src/file.js

@@ -0,0 +1,33 @@
+const fs = require('fs')
+const path = require('path')
+
+module.exports.read = read
+module.exports.write = write
+module.exports.resolve = resolve
+module.exports.join = join
+module.exports.dirname = dirname
+
+function read(filepath) {
+	return fs.readFileSync(path.resolve(filepath), { encoding: 'utf-8' })
+}
+
+function write(filepath, data) {
+	const dir = path.resolve(path.dirname(filepath))
+	if (!fs.existsSync(dir)) {
+		fs.mkdirSync(dir, { recursive: true })
+	}
+
+	fs.writeFileSync(path.resolve(filepath), data)
+}
+
+function resolve(filepath) {
+	return path.resolve(filepath)
+}
+
+function join(path1, path2) {
+	return path.join(path1, path2)
+}
+
+function dirname(filepath) {
+	return path.dirname(filepath)
+}

package/src/index.js

@@ -1,41 +1,47 @@
-const utils = require('./utils')
+const { merge } = require('lodash')
+const { create: createClient, buildRequest, parseResponse } = require('./client')
+const { parseChannels, parsePrograms } = require('./parser')
+const { generate: generateXMLTV } = require('./xmltv')
+const { load: loadConfig } = require('./config')
+const { sleep, isPromise } = require('./utils')
+
+module.exports.generateXMLTV = generateXMLTV
+module.exports.parseChannels = parseChannels
 
 class EPGGrabber {
   constructor(config = {}) {
-    this.config = utils.loadConfig(config)
-    this.client = utils.createClient(config)
+    this.config = loadConfig(config)
+    this.client = createClient(config)
+  }
+
+  async loadLogo(channel) {
+    const logo = this.config.logo({ channel })
+    if (isPromise(logo)) {
+      return await logo
+    }
+    return logo
   }
 
   async grab(channel, date, cb = () => {}) {
-    date = typeof date === 'string' ? utils.getUTCDate(date) : date
-    channel.lang = channel.lang || this.config.lang || null
-
-    let programs = []
-    const item = { date, channel }
-    await utils
-      .buildRequest(item, this.config)
-      .then(request => utils.fetchData(this.client, request))
-      .then(response => utils.parseResponse(item, response, this.config))
-      .then(results => {
-        item.programs = results
-        cb(item, null)
-        programs = programs.concat(results)
-      })
-      .catch(error => {
-        item.programs = []
-        if (this.config.debug) {
-          console.log('Error:', JSON.stringify(error, null, 2))
-        }
-        cb(item, error)
-      })
+    await sleep(this.config.delay)
 
-    await utils.sleep(this.config.delay)
+    return buildRequest({ channel, date, config: this.config })
+      .then(this.client)
+      .then(parseResponse)
+      .then(data => merge({ channel, date, config: this.config }, data))
+      .then(parsePrograms)
+      .then(programs => {
+        cb({ channel, date, programs })
 
-    return programs
+        return programs
+      })
+      .catch(err => {
+        if (this.config.debug) console.log('Error:', JSON.stringify(err, null, 2))
+        cb({ channel, date, programs: [] }, err)
+
+        return []
+      })
   }
 }
 
-EPGGrabber.convertToXMLTV = utils.convertToXMLTV
-EPGGrabber.parseChannels = utils.parseChannels
-
-module.exports = EPGGrabber
+module.exports.EPGGrabber = EPGGrabber

package/src/logger.js

@@ -0,0 +1,34 @@
+const { createLogger, format, transports } = require('winston')
+const { combine, timestamp, printf } = format
+const path = require('path')
+
+module.exports.create = create
+
+function create(options) {
+  const fileFormat = printf(({ level, message, timestamp }) => {
+    return `[${timestamp}] ${level.toUpperCase()}: ${message}`
+  })
+
+  const consoleFormat = printf(({ level, message }) => {
+    if (level === 'error') return `  Error: ${message}`
+
+    return message
+  })
+
+  const t = [new transports.Console({ format: consoleFormat })]
+
+  if (options.log) {
+    t.push(
+      new transports.File({
+        filename: path.resolve(options.log),
+        format: combine(timestamp(), fileFormat),
+        options: { flags: 'w' }
+      })
+    )
+  }
+
+  return createLogger({
+    level: options.logLevel,
+    transports: t
+  })
+}

package/src/parser.js

@@ -0,0 +1,45 @@
+const convert = require('xml-js')
+const Channel = require('./Channel')
+const Program = require('./Program')
+const { isPromise } = require('./utils')
+
+module.exports.parseChannels = parseChannels
+module.exports.parsePrograms = parsePrograms
+
+function parseChannels(xml) {
+  const result = convert.xml2js(xml)
+  const siteTag = result.elements.find(el => el.name === 'site') || {}
+  if (!siteTag.elements) return []
+  const rootSite = siteTag.attributes.site
+
+  const channelsTag = siteTag.elements.find(el => el.name === 'channels')
+  if (!channelsTag.elements) return []
+
+  const channels = channelsTag.elements
+    .filter(el => el.name === 'channel')
+    .map(el => {
+      let { xmltv_id, site, logo } = el.attributes
+      const name = el.elements.find(el => el.type === 'text').text
+      if (!name) throw new Error(`Channel '${xmltv_id}' has no valid name`)
+      site = site || rootSite
+
+      return new Channel({ name, xmltv_id, logo, site })
+    })
+
+  return { site: rootSite, channels }
+}
+
+async function parsePrograms(data) {
+  const { config, channel } = data
+  let programs = config.parser(data)
+
+  if (isPromise(programs)) {
+    programs = await programs
+  }
+
+  if (!Array.isArray(programs)) {
+    throw new Error('Parser should return an array')
+  }
+
+  return programs.filter(i => i).map(p => new Program(p, channel))
+}