epg-grabber 0.22.0 → 0.25.0

package/README.md

@@ -80,7 +80,10 @@ Arguments:
 - `--days`: number of days for which to grab the program (default: 1)
 - `--delay`: delay between requests (default: 3000)
 - `--timeout`: set a timeout for each request (default: 5000)
+- `--cache-max-age`: maximum time for storing each request in milliseconds (default: 0)
+- `--gzip`: compress the output (default: false)
 - `--debug`: enable debug mode (default: false)
+- `--curl`: display current request as CURL (default: false)
 - `--log`: path to log file (optional)
 - `--log-level`: set the log level (default: 'info')
 
@@ -99,6 +102,13 @@ module.exports = {
 
     method: 'GET',
     timeout: 5000,
+    cache: { // cache options (details: https://github.com/RasCarlito/axios-cache-adapter#options)
+      maxAge: 0,
+      readHeaders: false,
+      exclude: {
+        query: false
+      }
+    },
 
     /**
      * @param {object} context

package/bin/epg-grabber.js

@@ -4,10 +4,11 @@ const { Command } = require('commander')
 const program = new Command()
 const fs = require('fs')
 const path = require('path')
-const grabber = require('../src/index')
+const EPGGrabber = require('../src/index')
 const utils = require('../src/utils')
 const { name, version, description } = require('../package.json')
 const { merge } = require('lodash')
+const { gzip } = require('node-gzip')
 const { createLogger, format, transports } = require('winston')
 const { combine, timestamp, printf } = format
 
@@ -22,7 +23,14 @@ program
   .option('--days <days>', 'Number of days for which to grab the program', parseInteger, 1)
   .option('--delay <delay>', 'Delay between requests (in mileseconds)', parseInteger)
   .option('--timeout <timeout>', 'Set a timeout for each request (in mileseconds)', parseInteger)
+  .option(
+    '--cache-max-age <cacheMaxAge>',
+    'Maximum time for storing each request (in milliseconds)',
+    parseInteger
+  )
+  .option('--gzip', 'Compress the output', false)
   .option('--debug', 'Enable debug mode', false)
+  .option('--curl', 'Display request as CURL', false)
   .option('--log <log>', 'Path to log file')
   .option('--log-level <level>', 'Set log level', 'info')
   .parse(process.argv)
@@ -64,10 +72,15 @@ async function main() {
   config = merge(config, {
     days: options.days,
     debug: options.debug,
+    gzip: options.gzip,
+    curl: options.curl,
     lang: options.lang,
     delay: options.delay,
     request: {
-      timeout: options.timeout
+      timeout: options.timeout,
+      cache: {
+        maxAge: options.cacheMaxAge
+      }
     }
   })
 
@@ -87,10 +100,11 @@ async function main() {
   const total = channels.length * days
   const utcDate = utils.getUTCDate()
   const dates = Array.from({ length: config.days }, (_, i) => utcDate.add(i, 'd'))
+  const grabber = new EPGGrabber(config)
   for (let channel of channels) {
     for (let date of dates) {
       await grabber
-        .grab(channel, date, config, (data, err) => {
+        .grab(channel, date, (data, err) => {
           logger.info(
             `[${i}/${total}] ${config.site} - ${data.channel.xmltv_id} - ${data.date.format(
               'MMM D, YYYY'
@@ -108,8 +122,15 @@ async function main() {
   }
 
   const xml = utils.convertToXMLTV({ config, channels, programs })
-  const outputPath = options.output || config.output || 'guide.xml'
-  utils.writeToFile(outputPath, xml)
+  let outputPath = options.output || config.output
+  if (options.gzip) {
+    outputPath = outputPath || 'guide.xml.gz'
+    const compressed = await gzip(xml)
+    utils.writeToFile(outputPath, compressed)
+  } else {
+    outputPath = outputPath || 'guide.xml'
+    utils.writeToFile(outputPath, xml)
+  }
 
   logger.info(`File '${outputPath}' successfully saved`)
   logger.info('Finish')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "epg-grabber",
-  "version": "0.22.0",
+  "version": "0.25.0",
   "description": "Node.js CLI tool for grabbing EPG from different sites",
   "main": "src/index.js",
   "preferGlobal": true,
@@ -29,12 +29,15 @@
   },
   "dependencies": {
     "axios": "^0.21.1",
+    "axios-cache-adapter": "^2.7.3",
     "axios-cookiejar-support": "^1.0.1",
     "axios-mock-adapter": "^1.20.0",
     "commander": "^7.1.0",
+    "curl-generator": "^0.2.0",
     "dayjs": "^1.10.4",
     "glob": "^7.1.6",
     "lodash": "^4.17.21",
+    "node-gzip": "^1.1.2",
     "tough-cookie": "^4.0.0",
     "winston": "^3.3.3",
     "xml-js": "^1.6.11"

package/src/index.js

@@ -1,18 +1,21 @@
 const utils = require('./utils')
 
-module.exports = {
-  grab: async function (channel, date, config, cb) {
+class EPGGrabber {
+  constructor(config = {}) {
+    this.config = utils.loadConfig(config)
+    this.client = utils.createClient(config)
+  }
+
+  async grab(channel, date, cb = () => {}) {
     date = typeof date === 'string' ? utils.getUTCDate(date) : date
-    config = utils.loadConfig(config)
-    channel.lang = channel.lang || config.lang || null
+    channel.lang = channel.lang || this.config.lang || null
 
     let programs = []
-
     const item = { date, channel }
     await utils
-      .buildRequest(item, config)
-      .then(request => utils.fetchData(request))
-      .then(response => utils.parseResponse(item, response, config))
+      .buildRequest(item, this.config)
+      .then(request => utils.fetchData(this.client, request))
+      .then(response => utils.parseResponse(item, response, this.config))
       .then(results => {
         item.programs = results
         cb(item, null)
@@ -20,16 +23,19 @@ module.exports = {
       })
       .catch(error => {
         item.programs = []
-        if (config.debug) {
+        if (this.config.debug) {
           console.log('Error:', JSON.stringify(error, null, 2))
         }
         cb(item, error)
       })
 
-    await utils.sleep(config.delay)
+    await utils.sleep(this.config.delay)
 
     return programs
-  },
-  convertToXMLTV: utils.convertToXMLTV,
-  parseChannels: utils.parseChannels
+  }
 }
+
+EPGGrabber.convertToXMLTV = utils.convertToXMLTV
+EPGGrabber.parseChannels = utils.parseChannels
+
+module.exports = EPGGrabber

package/src/utils.js

@@ -3,11 +3,13 @@ const { padStart } = require('lodash')
 const path = require('path')
 const axios = require('axios').default
 const axiosCookieJarSupport = require('axios-cookiejar-support').default
+const axiosCacheAdapter = require('axios-cache-adapter')
 const tough = require('tough-cookie')
 const convert = require('xml-js')
 const { merge } = require('lodash')
 const dayjs = require('dayjs')
 const utc = require('dayjs/plugin/utc')
+const { CurlGenerator } = require('curl-generator')
 dayjs.extend(utc)
 axiosCookieJarSupport(axios)
 
@@ -38,13 +40,65 @@ utils.loadConfig = function (config) {
       timeout: 5000,
       withCredentials: true,
       jar: new tough.CookieJar(),
-      responseType: 'arraybuffer'
+      responseType: 'arraybuffer',
+      cache: {
+        readHeaders: false,
+        exclude: {
+          query: false
+        },
+        maxAge: 0
+      }
     }
   }
 
   return merge(defaultConfig, config)
 }
 
+utils.createClient = function (config) {
+  const client = axiosCacheAdapter.setup()
+  client.interceptors.request.use(
+    function (request) {
+      if (config.debug) {
+        console.log('Request:', JSON.stringify(request, null, 2))
+      }
+      return request
+    },
+    function (error) {
+      return Promise.reject(error)
+    }
+  )
+  client.interceptors.response.use(
+    function (response) {
+      if (config.debug) {
+        const data = utils.isObject(response.data)
+          ? JSON.stringify(response.data)
+          : response.data.toString()
+        console.log(
+          'Response:',
+          JSON.stringify(
+            {
+              headers: response.headers,
+              data,
+              fromCache: response.request.fromCache === true
+            },
+            null,
+            2
+          )
+        )
+      }
+
+      clearTimeout(timeout)
+      return response
+    },
+    function (error) {
+      clearTimeout(timeout)
+      return Promise.reject(error)
+    }
+  )
+
+  return client
+}
+
 utils.parseChannels = function (xml) {
   const result = convert.xml2js(xml)
   const siteTag = result.elements.find(el => el.name === 'site') || {}
@@ -204,26 +258,21 @@ utils.buildRequest = async function (item, config) {
   request.data = await utils.getRequestData(item, config)
   request.cancelToken = source.token
 
-  if (config.debug) {
-    console.log('Request:', JSON.stringify(request, null, 2))
+  if (config.curl) {
+    const curl = CurlGenerator({
+      url: request.url,
+      method: request.method,
+      headers: request.headers,
+      body: request.data
+    })
+    console.log(curl)
   }
 
   return request
 }
 
-utils.fetchData = function (request) {
-  axios.interceptors.response.use(
-    function (response) {
-      clearTimeout(timeout)
-      return response
-    },
-    function (error) {
-      clearTimeout(timeout)
-      return Promise.reject(error)
-    }
-  )
-
-  return axios(request)
+utils.fetchData = function (client, request) {
+  return client(request)
 }
 
 utils.getRequestHeaders = async function (item, config) {
@@ -266,24 +315,15 @@ utils.getUTCDate = function (d = null) {
 }
 
 utils.parseResponse = async (item, response, config) => {
-  if (config.debug) {
-    console.log(
-      'Response:',
-      JSON.stringify(
-        {
-          headers: response.headers,
-          data: response.data.toString()
-        },
-        null,
-        2
-      )
-    )
-  }
-
+  const content = utils.isObject(response.data)
+    ? JSON.stringify(response.data)
+    : response.data.toString()
+  const buffer = Buffer.from(content, 'utf8')
   const data = merge(item, config, {
-    content: response.data.toString(),
-    buffer: response.data,
-    headers: response.headers
+    content,
+    buffer,
+    headers: response.headers,
+    request: response.request
   })
 
   if (!item.channel.logo && config.logo) {
@@ -335,4 +375,8 @@ utils.isPromise = function (promise) {
   return !!promise && typeof promise.then === 'function'
 }
 
+utils.isObject = function (a) {
+  return !!a && a.constructor === Object
+}
+
 module.exports = utils

package/tests/bin.test.js

@@ -38,3 +38,19 @@ it('can load mini config', () => {
   expect(stdoutResultTester(result)).toBe(true)
   expect(result.includes("File 'tests/output/mini.guide.xml' successfully saved")).toBe(true)
 })
+
+it('can generate gzip version', () => {
+  const result = execSync(
+    `node ${pwd}/bin/epg-grabber.js \
+      --config=tests/input/mini.config.js \
+      --channels=tests/input/example.com.channels.xml \
+      --output=tests/output/mini.guide.xml.gz \
+      --gzip`,
+    {
+      encoding: 'utf8'
+    }
+  )
+
+  expect(stdoutResultTester(result)).toBe(true)
+  expect(result.includes("File 'tests/output/mini.guide.xml.gz' successfully saved")).toBe(true)
+})

package/tests/index.test.js

@@ -2,7 +2,7 @@
  * @jest-environment node
  */
 
-import grabber from '../src/index'
+import EPGGrabber from '../src/index'
 import axios from 'axios'
 
 jest.mock('axios')
@@ -27,7 +27,8 @@ it('return "Connection timeout" error if server does not response', done => {
     lang: 'en',
     name: 'CNN'
   }
-  grabber.grab(channel, '2022-01-01', config, (data, err) => {
+  const grabber = new EPGGrabber(config)
+  grabber.grab(channel, '2022-01-01', (data, err) => {
     expect(err.message).toBe('Connection timeout')
     done()
   })
@@ -53,8 +54,9 @@ it('can grab single channel programs', done => {
     lang: 'fr',
     name: '1TV'
   }
+  const grabber = new EPGGrabber(config)
   grabber
-    .grab(channel, '2022-01-01', config, (data, err) => {
+    .grab(channel, '2022-01-01', (data, err) => {
       if (err) {
         console.log(`    Error: ${err.message}`)
         done()

package/tests/utils.test.js

@@ -1,5 +1,6 @@
 import mockAxios from 'jest-mock-axios'
 import utils from '../src/utils'
+import axios from 'axios'
 import path from 'path'
 import fs from 'fs'
 
@@ -148,7 +149,7 @@ it('can fetch data', () => {
     url: 'http://example.com/20210319/1tv.json',
     withCredentials: true
   }
-  utils.fetchData(request).then(jest.fn).catch(jest.fn)
+  utils.fetchData(axios, request).then(jest.fn).catch(jest.fn)
   expect(mockAxios).toHaveBeenCalledWith(
     expect.objectContaining({
       data: { accountID: '123' },