epg-grabber 0.27.0 → 0.28.0

package/README.md

@@ -166,12 +166,24 @@ module.exports = {
         title, // program title (required)
         start, // start time of the program (required)
         stop, // end time of the program (required)
+        sub_title, // program sub-title (optional)
         description, // description of the program (optional)
-        category, // program type (optional)
+        category, // type of program (optional)
         season, // season number (optional)
         episode, // episode number (optional)
+        date, // the date the programme or film was finished (optional)
         icon, // image associated with the program (optional)
-        lang // language of the description (default: 'en')
+        rating, // program rating (optional)
+        director, // the name of director (optional)
+        actor, // the name of actor (optional)
+        writer, // the name of writer (optional)
+        adapter, // the name of adapter (optional)
+        producer, // the name of producer (optional)
+        composer, // the name of composer (optional)
+        editor, // the name of editor (optional)
+        presenter, // the name of presenter (optional)
+        commentator, // the name of commentator (optional)
+        guest // the name of guest (optional)
       },
       ...
     ]

package/bin/epg-grabber.js

@@ -2,15 +2,13 @@
 
 const { Command } = require('commander')
 const program = new Command()
-const fs = require('fs')
-const path = require('path')
-const EPGGrabber = require('../src/index')
-const utils = require('../src/utils')
-const { name, version, description } = require('../package.json')
 const { merge } = require('lodash')
 const { gzip } = require('node-gzip')
-const { createLogger, format, transports } = require('winston')
-const { combine, timestamp, printf } = format
+const file = require('../src/file')
+const EPGGrabber = require('../src/index')
+const { create: createLogger } = require('../src/logger')
+const { parseInteger, getUTCDate } = require('../src/utils')
+const { name, version, description } = require('../package.json')
 
 program
   .name(name)
@@ -36,39 +34,13 @@ program
   .parse(process.argv)
 
 const options = program.opts()
-
-const fileFormat = printf(({ level, message, timestamp }) => {
-  return `[${timestamp}] ${level.toUpperCase()}: ${message}`
-})
-
-const consoleFormat = printf(({ level, message, timestamp }) => {
-  if (level === 'error') return `  Error: ${message}`
-
-  return message
-})
-
-const t = [new transports.Console({ format: consoleFormat })]
-
-if (options.log) {
-  t.push(
-    new transports.File({
-      filename: path.resolve(options.log),
-      format: combine(timestamp(), fileFormat),
-      options: { flags: 'w' }
-    })
-  )
-}
-
-const logger = createLogger({
-  level: options.logLevel,
-  transports: t
-})
+const logger = createLogger(options)
 
 async function main() {
   logger.info('Starting...')
 
   logger.info(`Loading '${options.config}'...`)
-  let config = require(path.resolve(options.config))
+  let config = require(file.resolve(options.config))
   config = merge(config, {
     days: options.days,
     debug: options.debug,
@@ -83,22 +55,27 @@ async function main() {
   if (options.cacheTtl) config.request.cache.ttl = options.cacheTtl
   if (options.channels) config.channels = options.channels
   else if (config.channels)
-    config.channels = path.join(path.dirname(options.config), config.channels)
+    config.channels = file.join(file.dirname(options.config), config.channels)
   else throw new Error("The required 'channels' property is missing")
 
   if (!config.channels) return logger.error('Path to [site].channels.xml is missing')
   logger.info(`Loading '${config.channels}'...`)
-  const channelsXML = fs.readFileSync(path.resolve(config.channels), { encoding: 'utf-8' })
-  const { channels } = utils.parseChannels(channelsXML)
+  const grabber = new EPGGrabber(config)
+
+  const channelsXML = file.read(config.channels)
+  const { channels } = grabber.parseChannels(channelsXML)
 
   let programs = []
   let i = 1
   let days = options.days || 1
   const total = channels.length * days
-  const utcDate = utils.getUTCDate()
+  const utcDate = getUTCDate()
   const dates = Array.from({ length: config.days }, (_, i) => utcDate.add(i, 'd'))
-  const grabber = new EPGGrabber(config)
   for (let channel of channels) {
+    if (!channel.logo && config.logo) {
+      channel.logo = await grabber.loadLogo(channel)
+    }
+
     for (let date of dates) {
       await grabber
         .grab(channel, date, (data, err) => {
@@ -118,15 +95,15 @@ async function main() {
     }
   }
 
-  const xml = utils.convertToXMLTV({ config, channels, programs })
+  const xml = grabber.generateXMLTV({ channels, programs })
   let outputPath = options.output || config.output
   if (options.gzip) {
     outputPath = outputPath || 'guide.xml.gz'
     const compressed = await gzip(xml)
-    utils.writeToFile(outputPath, compressed)
+    file.write(outputPath, compressed)
   } else {
     outputPath = outputPath || 'guide.xml'
-    utils.writeToFile(outputPath, xml)
+    file.write(outputPath, xml)
   }
 
   logger.info(`File '${outputPath}' successfully saved`)
@@ -134,7 +111,3 @@ async function main() {
 }
 
 main()
-
-function parseInteger(val) {
-  return val ? parseInt(val) : null
-}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "epg-grabber",
-  "version": "0.27.0",
+  "version": "0.28.0",
   "description": "Node.js CLI tool for grabbing EPG from different sites",
   "main": "src/index.js",
   "preferGlobal": true,

package/src/channels.js

@@ -0,0 +1,26 @@
+const convert = require('xml-js')
+
+module.exports.parse = parse
+
+function parse(xml) {
+  const result = convert.xml2js(xml)
+  const siteTag = result.elements.find(el => el.name === 'site') || {}
+  if (!siteTag.elements) return []
+  const site = siteTag.attributes.site
+
+  const channelsTag = siteTag.elements.find(el => el.name === 'channels')
+  if (!channelsTag.elements) return []
+
+  const channels = channelsTag.elements
+    .filter(el => el.name === 'channel')
+    .map(el => {
+      const channel = el.attributes
+      if (!el.elements) throw new Error(`Channel '${channel.xmltv_id}' has no valid name`)
+      channel.name = el.elements.find(el => el.type === 'text').text
+      channel.site = channel.site || site
+
+      return channel
+    })
+
+  return { site, channels }
+}

package/src/client.js

@@ -0,0 +1,138 @@
+const { CurlGenerator } = require('curl-generator')
+const axios = require('axios').default
+const axiosCookieJarSupport = require('axios-cookiejar-support').default
+const { setupCache } = require('axios-cache-interceptor')
+const { isObject, isPromise, getUTCDate } = require('./utils')
+
+axiosCookieJarSupport(axios)
+
+module.exports.create = create
+module.exports.buildRequest = buildRequest
+module.exports.parseResponse = parseResponse
+
+let timeout
+
+function create(config) {
+	const client = setupCache(
+		axios.create({
+			headers: {
+				'User-Agent':
+					'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36 Edg/79.0.309.71'
+			}
+		})
+	)
+
+	client.interceptors.request.use(
+		function (request) {
+			if (config.debug) {
+				console.log('Request:', JSON.stringify(request, null, 2))
+			}
+			return request
+		},
+		function (error) {
+			return Promise.reject(error)
+		}
+	)
+
+	client.interceptors.response.use(
+		function (response) {
+			if (config.debug) {
+				const data =
+					isObject(response.data) || Array.isArray(response.data)
+						? JSON.stringify(response.data)
+						: response.data.toString()
+				console.log(
+					'Response:',
+					JSON.stringify(
+						{
+							headers: response.headers,
+							data,
+							cached: response.cached
+						},
+						null,
+						2
+					)
+				)
+			}
+
+			clearTimeout(timeout)
+			return response
+		},
+		function (error) {
+			clearTimeout(timeout)
+			return Promise.reject(error)
+		}
+	)
+
+	return client
+}
+
+async function buildRequest({ channel, date, config }) {
+	date = typeof date === 'string' ? getUTCDate(date) : date
+	const CancelToken = axios.CancelToken
+	const source = CancelToken.source()
+	const request = { ...config.request }
+	timeout = setTimeout(() => {
+		source.cancel('Connection timeout')
+	}, request.timeout)
+	request.headers = await getRequestHeaders({ channel, date, config })
+	request.url = await getRequestUrl({ channel, date, config })
+	request.data = await getRequestData({ channel, date, config })
+	request.cancelToken = source.token
+
+	if (config.curl) {
+		const curl = CurlGenerator({
+			url: request.url,
+			method: request.method,
+			headers: request.headers,
+			body: request.data
+		})
+		console.log(curl)
+	}
+
+	return request
+}
+
+function parseResponse(response) {
+	return {
+		content: response.data.toString(),
+		buffer: response.data,
+		headers: response.headers,
+		request: response.request,
+		cached: response.cached
+	}
+}
+
+async function getRequestHeaders({ channel, date, config }) {
+	if (typeof config.request.headers === 'function') {
+		const headers = config.request.headers({ channel, date })
+		if (isPromise(headers)) {
+			return await headers
+		}
+		return headers
+	}
+
+	return config.request.headers || null
+}
+
+async function getRequestData({ channel, date, config }) {
+	if (typeof config.request.data === 'function') {
+		const data = config.request.data({ channel, date })
+		if (isPromise(data)) {
+			return await data
+		}
+		return data
+	}
+	return config.request.data || null
+}
+
+async function getRequestUrl({ channel, date, config }) {
+	if (typeof config.url === 'function') {
+		const url = config.url({ channel, date })
+		if (isPromise(url)) {
+			return await url
+		}
+		return url
+	}
+	return config.url
+}

package/src/config.js

@@ -0,0 +1,34 @@
+const tough = require('tough-cookie')
+const { merge } = require('lodash')
+
+module.exports.load = load
+
+function load(config) {
+	if (!config.site) throw new Error("The required 'site' property is missing")
+	if (!config.url) throw new Error("The required 'url' property is missing")
+	if (typeof config.url !== 'function' && typeof config.url !== 'string')
+		throw new Error("The 'url' property should return the function or string")
+	if (!config.parser) throw new Error("The required 'parser' function is missing")
+	if (typeof config.parser !== 'function')
+		throw new Error("The 'parser' property should return the function")
+	if (config.logo && typeof config.logo !== 'function')
+		throw new Error("The 'logo' property should return the function")
+
+	const defaultConfig = {
+		days: 1,
+		lang: 'en',
+		delay: 3000,
+		output: 'guide.xml',
+		request: {
+			method: 'GET',
+			maxContentLength: 5 * 1024 * 1024,
+			timeout: 5000,
+			withCredentials: true,
+			jar: new tough.CookieJar(),
+			responseType: 'arraybuffer',
+			cache: false
+		}
+	}
+
+	return merge(defaultConfig, config)
+}

package/src/file.js

@@ -0,0 +1,33 @@
+const fs = require('fs')
+const path = require('path')
+
+module.exports.read = read
+module.exports.write = write
+module.exports.resolve = resolve
+module.exports.join = join
+module.exports.dirname = dirname
+
+function read(filepath) {
+	return fs.readFileSync(path.resolve(filepath), { encoding: 'utf-8' })
+}
+
+function write(filepath, data) {
+	const dir = path.resolve(path.dirname(filepath))
+	if (!fs.existsSync(dir)) {
+		fs.mkdirSync(dir, { recursive: true })
+	}
+
+	fs.writeFileSync(path.resolve(filepath), data)
+}
+
+function resolve(filepath) {
+	return path.resolve(filepath)
+}
+
+function join(path1, path2) {
+	return path.join(path1, path2)
+}
+
+function dirname(filepath) {
+	return path.dirname(filepath)
+}

package/src/index.js

@@ -1,41 +1,48 @@
-const utils = require('./utils')
+const { merge } = require('lodash')
+const { create: createClient, buildRequest, parseResponse } = require('./client')
+const { generate: generateXMLTV } = require('./xmltv')
+const { parse: parseChannels } = require('./channels')
+const { parse: parsePrograms } = require('./programs')
+const { load: loadConfig } = require('./config')
+const { sleep, isPromise } = require('./utils')
 
 class EPGGrabber {
   constructor(config = {}) {
-    this.config = utils.loadConfig(config)
-    this.client = utils.createClient(config)
+    this.config = loadConfig(config)
+    this.client = createClient(config)
+  }
+
+  async loadLogo(channel) {
+    const logo = this.config.logo({ channel })
+    if (isPromise(logo)) {
+      return await logo
+    }
+    return logo
   }
 
   async grab(channel, date, cb = () => {}) {
-    date = typeof date === 'string' ? utils.getUTCDate(date) : date
-    channel.lang = channel.lang || this.config.lang || null
-
-    let programs = []
-    const item = { date, channel }
-    await utils
-      .buildRequest(item, this.config)
-      .then(request => utils.fetchData(this.client, request))
-      .then(response => utils.parseResponse(item, response, this.config))
-      .then(results => {
-        item.programs = results
-        cb(item, null)
-        programs = programs.concat(results)
-      })
-      .catch(error => {
-        item.programs = []
-        if (this.config.debug) {
-          console.log('Error:', JSON.stringify(error, null, 2))
-        }
-        cb(item, error)
-      })
+    await sleep(this.config.delay)
 
-    await utils.sleep(this.config.delay)
+    return buildRequest({ channel, date, config: this.config })
+      .then(this.client)
+      .then(parseResponse)
+      .then(data => merge({ channel, date, config: this.config }, data))
+      .then(parsePrograms)
+      .then(programs => {
+        cb({ channel, date, programs })
 
-    return programs
+        return programs
+      })
+      .catch(err => {
+        if (this.config.debug) console.log('Error:', JSON.stringify(err, null, 2))
+        cb({ channel, date, programs: [] }, err)
+
+        return []
+      })
   }
 }
 
-EPGGrabber.convertToXMLTV = utils.convertToXMLTV
-EPGGrabber.parseChannels = utils.parseChannels
+EPGGrabber.prototype.generateXMLTV = generateXMLTV
+EPGGrabber.prototype.parseChannels = parseChannels
 
 module.exports = EPGGrabber

package/src/logger.js

@@ -0,0 +1,34 @@
+const { createLogger, format, transports } = require('winston')
+const { combine, timestamp, printf } = format
+const path = require('path')
+
+module.exports.create = create
+
+function create(options) {
+  const fileFormat = printf(({ level, message, timestamp }) => {
+    return `[${timestamp}] ${level.toUpperCase()}: ${message}`
+  })
+
+  const consoleFormat = printf(({ level, message }) => {
+    if (level === 'error') return `  Error: ${message}`
+
+    return message
+  })
+
+  const t = [new transports.Console({ format: consoleFormat })]
+
+  if (options.log) {
+    t.push(
+      new transports.File({
+        filename: path.resolve(options.log),
+        format: combine(timestamp(), fileFormat),
+        options: { flags: 'w' }
+      })
+    )
+  }
+
+  return createLogger({
+    level: options.logLevel,
+    transports: t
+  })
+}

package/src/programs.js

@@ -0,0 +1,24 @@
+const { isPromise } = require('./utils')
+
+module.exports.parse = parse
+
+async function parse(data) {
+	const { config, channel } = data
+	let programs = config.parser(data)
+
+	if (isPromise(programs)) {
+		programs = await programs
+	}
+
+	if (!Array.isArray(programs)) {
+		throw new Error('Parser should return an array')
+	}
+
+	return programs
+		.filter(i => i)
+		.map(program => {
+			program.channel = channel.xmltv_id
+
+			return program
+		})
+}