epg-grabber 0.45.0 → 0.46.0

package/dist/cli.js

@@ -1,12 +1,11 @@
 #!/usr/bin/env node
-import { n as name, v as version, d as description, p as parseNumber, l as loadJs, a as parseProxy, E as EPGGrabberMock, b as EPGGrabber, i as isObject, g as getAbsPath, c as getUTCDate } from './index-CHTSs3QX.js';
+import { n as name, v as version, d as description, p as parseNumber, l as loadJs, a as parseProxy, E as EPGGrabberMock, b as EPGGrabber, c as defaultConfig, i as isObject, g as getAbsPath, e as getUTCDate } from './index-DUJ5SdIU.js';
 import { Collection, Template } from '@freearhey/core';
 import { Command, Option } from 'commander';
 import { SocksProxyAgent } from 'socks-proxy-agent';
 import { CurlGenerator } from 'curl-generator';
 import winston from 'winston';
 import path from 'path';
-import { AxiosHeaders } from 'axios';
 import { TaskQueue } from 'cwait';
 import merge from 'lodash.merge';
 import Promise$1 from 'bluebird';
@@ -19,6 +18,7 @@ import 'dayjs';
 import 'node:url';
 import 'axios-mock-adapter';
 import 'lodash.padstart';
+import 'axios';
 import 'axios-cache-interceptor';
 import 'xml-js';
 
@@ -121,22 +121,19 @@ async function main() {
   if (typeof options.debug === "boolean") config.debug = options.debug;
   if (typeof options.curl === "boolean") config.curl = options.curl;
   if (typeof options.gzip === "boolean") config.gzip = options.gzip;
+  logger.debug(`Config: ${JSON.stringify(config, null, 2)}`);
   const grabber = process.env.NODE_ENV === "test" ? new EPGGrabberMock(config) : new EPGGrabber(config);
-  const globalConfig = grabber.globalConfig;
-  logger.debug(`Config: ${JSON.stringify(globalConfig, null, 2)}`);
   grabber.client.instance.interceptors.request.use(
     (request) => {
       logger.debug(`Request: ${JSON.stringify(request, null, 2)}`);
-      if (globalConfig.curl) {
-        const headers = request.headers instanceof AxiosHeaders ? request.headers : {};
-        const method = request.method || "GET";
-        const curl = CurlGenerator({
-          url: request.url || "",
-          method,
-          headers,
-          body: request.data
-        });
-        logger.info(curl);
+      const curl = config.curl || defaultConfig.curl;
+      if (curl) {
+        const url = request.url || "";
+        const method = request.method ? request.method : "GET";
+        const headers = request.headers ? request.headers.toJSON() : void 0;
+        const body = request.data ? request.data : void 0;
+        const curl2 = CurlGenerator({ url, method, headers, body });
+        logger.info(curl2);
       }
       return request;
     },
@@ -160,11 +157,11 @@ async function main() {
     },
     (error) => Promise$1.reject(error)
   );
-  if (!Array.isArray(globalConfig.channels) || !globalConfig.channels.length)
+  if (!Array.isArray(config.channels) || !config.channels.length)
     throw new Error('Path to "*.channels.xml" is missing');
   const channels = new Collection();
   const rootDir = options.channels ? process.cwd() : path$1.dirname(options.config);
-  globalConfig.channels.forEach((filepath) => {
+  config.channels.forEach((filepath) => {
     const absFilepath = getAbsPath(filepath, rootDir);
     logger.debug(`Loading "${absFilepath}"...`);
     const channelsXML = fs.readFileSync(absFilepath, "utf8");
@@ -172,9 +169,12 @@ async function main() {
     channels.concat(new Collection(channelsFromXML));
   });
   if (channels.isEmpty()) throw new Error("No channels found");
-  if (typeof globalConfig.output !== "string")
-    throw new Error('The "output" property should return the string');
-  const template = new Template(globalConfig.output);
+  const days = config.days || defaultConfig.days;
+  const maxConnections = config.maxConnections || defaultConfig.maxConnections;
+  const gzip = config.gzip || defaultConfig.gzip;
+  const defaultOutput = gzip ? defaultConfig.output + ".gz" : defaultConfig.output;
+  const output = config.output || defaultOutput;
+  const template = new Template(output);
   const variables = template.variables();
   const groups = channels.groupBy((channel) => {
     let groupId = "";
@@ -187,27 +187,21 @@ async function main() {
     return groupId;
   });
   logger.info("Processing...");
-  if (typeof globalConfig.days !== "number")
-    throw new Error('The "days" property should return the number');
-  if (typeof globalConfig.maxConnections !== "number")
-    throw new Error('The "maxConnections" property should return the number');
-  if (typeof globalConfig.gzip !== "boolean")
-    throw new Error('The "gzip" property should return the boolean');
   for (const groupId of groups.keys()) {
     const group = groups.get(groupId);
     const groupChannels = new Collection(group);
     let programs = new Collection();
     let index = 1;
-    const total = groupChannels.count() * globalConfig.days;
+    const total = groupChannels.count() * days;
     const utcDate = getUTCDate(process.env.CURR_DATE);
-    const dates = Array.from({ length: globalConfig.days }, (_, i) => utcDate.add(i, "d"));
+    const dates = Array.from({ length: days }, (_, i) => utcDate.add(i, "d"));
     let queue = new Collection();
     groupChannels.forEach((channel) => {
       for (let date of dates) {
         queue.add({ channel, date });
       }
     });
-    const taskQueue = new TaskQueue(Promise$1, globalConfig.maxConnections);
+    const taskQueue = new TaskQueue(Promise$1, maxConnections);
     const requests = queue.map(
       taskQueue.wrap(async (queueItem) => {
         const { channel, date } = queueItem;
@@ -231,12 +225,10 @@ async function main() {
     let outputPath = template.format(channelSample.toObject());
     const outputDir = path$1.dirname(outputPath);
     fs.mkdirSync(outputDir, { recursive: true });
-    if (globalConfig.gzip) {
+    if (gzip) {
       const compressed = pako.gzip(xml);
-      outputPath = outputPath || "guide.xml.gz";
       fs.writeFileSync(outputPath, compressed);
     } else {
-      outputPath = outputPath || "guide.xml";
       fs.writeFileSync(outputPath, xml);
     }
     logger.info(`File '${outputPath}' successfully saved`);

package/dist/{index-CHTSs3QX.js → index-DUJ5SdIU.js}

@@ -102,7 +102,7 @@ function toArray(value) {
 }
 
 var name = "epg-grabber";
-var version = "0.45.0";
+var version = "0.46.0";
 var description = "Node.js CLI tool for grabbing EPG from different sites";
 var homepage = "https://github.com/freearhey/epg-grabber";
 
@@ -725,13 +725,13 @@ class EPGGrabber {
   globalConfig = {};
   client;
   constructor(config = {}) {
-    this.globalConfig = merge(defaultConfig, config);
+    this.globalConfig = config;
     this.client = new Client();
   }
   async loadLogo(channel, date, config = {}) {
     if (!(channel instanceof Channel))
       throw new Error('The first argument must be the "Channel" class');
-    config = merge(config, this.globalConfig);
+    config = merge(defaultConfig, config, this.globalConfig);
     if (typeof config.logo !== "function") return null;
     const requestContext = { channel, date: getUTCDate(date), config };
     const logo = config.logo(requestContext);
@@ -749,8 +749,7 @@ class EPGGrabber {
       config = {};
     }
     const utcDate = getUTCDate(date);
-    const requestContext = { channel, date: utcDate, config };
-    config = merge(config, this.globalConfig);
+    config = merge(defaultConfig, config, this.globalConfig);
     if (!config.parser) throw new Error("Could not find parser() in the config file");
     if (!config.site) throw new Error("The required 'site' property is missing");
     if (!config.url) throw new Error("The required 'url' property is missing");
@@ -763,6 +762,7 @@ class EPGGrabber {
       throw new Error("The 'logo' property should return the function");
     try {
       if (typeof config.delay === "number") await sleep(config.delay);
+      const requestContext = { channel, date: utcDate, config };
       const request = await Client.buildRequest(requestContext);
       const response = await this.client.sendRequest(request);
       const parserContext = {
@@ -826,10 +826,10 @@ class EPGGrabberMock extends EPGGrabber {
       config = {};
     }
     const utcDate = getUTCDate(date);
-    config = merge(config, this.globalConfig);
+    config = merge(defaultConfig, config, this.globalConfig);
     if (!config.parser) throw new Error("Could not find parser() in the config file");
-    const requestContext = { channel, date: utcDate, config };
     try {
+      const requestContext = { channel, date: utcDate, config };
       const request = await Client.buildRequest(requestContext);
       const mock = new AxiosMockAdapter(this.client.instance);
       mock.onAny().reply(200, Buffer.from(JSON.stringify([]), "utf8"));
@@ -854,4 +854,4 @@ class EPGGrabberMock extends EPGGrabber {
   }
 }
 
-export { Channel as C, EPGGrabberMock as E, Program as P, parseProxy as a, EPGGrabber as b, getUTCDate as c, description as d, getAbsPath as g, isObject as i, loadJs as l, name as n, parseNumber as p, version as v };
+export { Channel as C, EPGGrabberMock as E, Program as P, parseProxy as a, EPGGrabber as b, defaultConfig as c, description as d, getUTCDate as e, getAbsPath as g, isObject as i, loadJs as l, name as n, parseNumber as p, version as v };

package/dist/index.js

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-export { C as Channel, b as EPGGrabber, E as EPGGrabberMock, P as Program } from './index-CHTSs3QX.js';
+export { C as Channel, b as EPGGrabber, E as EPGGrabberMock, P as Program } from './index-DUJ5SdIU.js';
 import 'axios-mock-adapter';
 import 'lodash.merge';
 import 'xml-js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "epg-grabber",
-  "version": "0.45.0",
+  "version": "0.46.0",
   "description": "Node.js CLI tool for grabbing EPG from different sites",
   "homepage": "https://github.com/freearhey/epg-grabber",
   "preferGlobal": true,

package/src/cli.ts

@@ -8,6 +8,7 @@ import { Command, OptionValues, Option } from 'commander'
 import { EPGGrabber, EPGGrabberMock } from './index'
 import { SocksProxyAgent } from 'socks-proxy-agent'
 import { CurlGenerator } from 'curl-generator'
+import defaultConfig from './default.config'
 import { Program, Channel } from './models'
 import { Logger } from './core/logger'
 import { SiteConfig } from './types'
@@ -116,17 +117,17 @@ async function main() {
   if (typeof options.curl === 'boolean') config.curl = options.curl
   if (typeof options.gzip === 'boolean') config.gzip = options.gzip
 
+  logger.debug(`Config: ${JSON.stringify(config, null, 2)}`)
+
   const grabber =
     process.env.NODE_ENV === 'test' ? new EPGGrabberMock(config) : new EPGGrabber(config)
 
-  const globalConfig = grabber.globalConfig
-
-  logger.debug(`Config: ${JSON.stringify(globalConfig, null, 2)}`)
-
   grabber.client.instance.interceptors.request.use(
     request => {
       logger.debug(`Request: ${JSON.stringify(request, null, 2)}`)
-      if (globalConfig.curl) {
+
+      const curl = config.curl || defaultConfig.curl
+      if (curl) {
         type AllowedMethods =
           | 'GET'
           | 'get'
@@ -139,15 +140,14 @@ async function main() {
           | 'DELETE'
           | 'delete'
 
-        const headers = request.headers instanceof AxiosHeaders ? request.headers : {}
-        const method = (request.method as AllowedMethods) || 'GET'
+        const url = request.url || ''
+        const method = request.method ? (request.method as AllowedMethods) : 'GET'
+        const headers = request.headers
+          ? (request.headers.toJSON() as Record<string, string>)
+          : undefined
+        const body = request.data ? (request.data as CurlBody) : undefined
 
-        const curl = CurlGenerator({
-          url: request.url || '',
-          method,
-          headers,
-          body: request.data as CurlBody
-        })
+        const curl = CurlGenerator({ url, method, headers, body })
 
         logger.info(curl)
       }
@@ -182,12 +182,12 @@ async function main() {
     error => Promise.reject(error)
   )
 
-  if (!Array.isArray(globalConfig.channels) || !globalConfig.channels.length)
+  if (!Array.isArray(config.channels) || !config.channels.length)
     throw new Error('Path to "*.channels.xml" is missing')
 
   const channels = new Collection<Channel>()
   const rootDir = options.channels ? process.cwd() : path.dirname(options.config)
-  globalConfig.channels.forEach((filepath: string) => {
+  config.channels.forEach((filepath: string) => {
     const absFilepath = getAbsPath(filepath, rootDir)
 
     logger.debug(`Loading "${absFilepath}"...`)
@@ -199,10 +199,12 @@ async function main() {
 
   if (channels.isEmpty()) throw new Error('No channels found')
 
-  if (typeof globalConfig.output !== 'string')
-    throw new Error('The "output" property should return the string')
-
-  const template = new Template(globalConfig.output)
+  const days = config.days || defaultConfig.days
+  const maxConnections = config.maxConnections || defaultConfig.maxConnections
+  const gzip = config.gzip || defaultConfig.gzip
+  const defaultOutput = gzip ? defaultConfig.output + '.gz' : defaultConfig.output
+  const output = config.output || defaultOutput
+  const template = new Template(output)
   const variables = template.variables()
   const groups: Dictionary<Channel[]> = channels.groupBy((channel: Channel) => {
     let groupId = ''
@@ -217,22 +219,15 @@ async function main() {
   })
 
   logger.info('Processing...')
-  if (typeof globalConfig.days !== 'number')
-    throw new Error('The "days" property should return the number')
-  if (typeof globalConfig.maxConnections !== 'number')
-    throw new Error('The "maxConnections" property should return the number')
-  if (typeof globalConfig.gzip !== 'boolean')
-    throw new Error('The "gzip" property should return the boolean')
-
   for (const groupId of groups.keys()) {
     const group = groups.get(groupId)
     const groupChannels = new Collection<Channel>(group)
     let programs = new Collection<Program>()
     let index = 1
 
-    const total = groupChannels.count() * globalConfig.days
+    const total = groupChannels.count() * days
     const utcDate = getUTCDate(process.env.CURR_DATE)
-    const dates = Array.from({ length: globalConfig.days }, (_, i) => utcDate.add(i, 'd'))
+    const dates = Array.from({ length: days }, (_, i) => utcDate.add(i, 'd'))
 
     let queue = new Collection<QueueItem>()
     groupChannels.forEach((channel: Channel) => {
@@ -241,7 +236,7 @@ async function main() {
       }
     })
 
-    const taskQueue = new TaskQueue(Promise, globalConfig.maxConnections)
+    const taskQueue = new TaskQueue(Promise, maxConnections)
     const requests = queue.map(
       taskQueue.wrap(async (queueItem: QueueItem) => {
         const { channel, date } = queueItem
@@ -277,12 +272,10 @@ async function main() {
 
     fs.mkdirSync(outputDir, { recursive: true })
 
-    if (globalConfig.gzip) {
+    if (gzip) {
       const compressed = pako.gzip(xml)
-      outputPath = outputPath || 'guide.xml.gz'
       fs.writeFileSync(outputPath, compressed)
     } else {
-      outputPath = outputPath || 'guide.xml'
       fs.writeFileSync(outputPath, xml)
     }
 

package/src/index.ts

@@ -18,7 +18,7 @@ export class EPGGrabber {
   client: Client
 
   constructor(config: SiteConfig = {}) {
-    this.globalConfig = merge(defaultConfig, config)
+    this.globalConfig = config
     this.client = new Client()
   }
 
@@ -30,7 +30,7 @@ export class EPGGrabber {
     if (!(channel instanceof Channel))
       throw new Error('The first argument must be the "Channel" class')
 
-    config = merge(config, this.globalConfig)
+    config = merge(defaultConfig, config, this.globalConfig)
 
     if (typeof config.logo !== 'function') return null
 
@@ -58,9 +58,8 @@ export class EPGGrabber {
     }
 
     const utcDate = getUTCDate(date)
-    const requestContext = { channel, date: utcDate, config }
 
-    config = merge(config, this.globalConfig)
+    config = merge(defaultConfig, config, this.globalConfig)
 
     if (!config.parser) throw new Error('Could not find parser() in the config file')
     if (!config.site) throw new Error("The required 'site' property is missing")
@@ -76,6 +75,8 @@ export class EPGGrabber {
     try {
       if (typeof config.delay === 'number') await sleep(config.delay)
 
+      const requestContext = { channel, date: utcDate, config }
+
       const request = await Client.buildRequest(requestContext)
 
       const response = await this.client.sendRequest(request)
@@ -174,13 +175,13 @@ export class EPGGrabberMock extends EPGGrabber {
 
     const utcDate = getUTCDate(date)
 
-    config = merge(config, this.globalConfig)
+    config = merge(defaultConfig, config, this.globalConfig)
 
     if (!config.parser) throw new Error('Could not find parser() in the config file')
 
-    const requestContext = { channel, date: utcDate, config }
-
     try {
+      const requestContext = { channel, date: utcDate, config }
+
       const request = await Client.buildRequest(requestContext)
 
       const mock = new AxiosMockAdapter(this.client.instance)

package/tests/cli.test.ts

@@ -18,30 +18,22 @@ it('can load config', () => {
   )
 
   expect(stdout).contains(`{
+  "site": "example.com",
   "days": 2,
-  "delay": 0,
-  "output": "tests/__data__/output/example.guide.xml",
   "channels": [
     "example.channels.xml"
   ],
+  "output": "tests/__data__/output/example.guide.xml",
   "request": {
     "method": "POST",
-    "maxContentLength": 5242880,
-    "timeout": 1,
-    "withCredentials": true,
-    "responseType": "arraybuffer",
-    "cache": false,
     "headers": {
-      "User-Agent": "EPGGrabber/0.44.0 (https://github.com/freearhey/epg-grabber)",
       "Content-Type": "application/json",
       "Cookie": "abc=123"
-    }
+    },
+    "timeout": 1
   },
-  "maxConnections": 1,
-  "curl": false,
-  "debug": true,
-  "gzip": false,
-  "site": "example.com"
+  "delay": 0,
+  "debug": true
 }`)
   expect(stdout).contains("File 'tests/__data__/output/example.guide.xml' successfully saved")
   expect(content('tests/__data__/output/example.guide.xml')).toEqual(
@@ -66,22 +58,13 @@ it('can load mini config', () => {
   )
 
   expect(stdout).contains(`{
-  "days": 3,
-  "delay": 0,
-  "output": "tests/__data__/output/mini.guide.xml",
+  "site": "example.com",
+  "url": "http://example.com/20210319/1tv.json",
   "channels": [
     "tests/__data__/input/example.channels.xml"
   ],
   "request": {
-    "method": "GET",
-    "maxContentLength": 5242880,
     "timeout": 1,
-    "withCredentials": true,
-    "responseType": "arraybuffer",
-    "cache": false,
-    "headers": {
-      "User-Agent": "EPGGrabber/0.44.0 (https://github.com/freearhey/epg-grabber)"
-    },
     "httpAgent": {
       "_events": {},
       "_eventsCount": 2,
@@ -135,12 +118,10 @@ it('can load mini config', () => {
       "socketOptions": null
     }
   },
-  "maxConnections": 1,
-  "curl": false,
-  "debug": true,
-  "gzip": false,
-  "site": "example.com",
-  "url": "http://example.com/20210319/1tv.json"
+  "output": "tests/__data__/output/mini.guide.xml",
+  "days": 3,
+  "delay": 0,
+  "debug": true
 }`)
   expect(stdout).contains("File 'tests/__data__/output/mini.guide.xml' successfully saved")
   expect(content('tests/__data__/output/mini.guide.xml')).toEqual(
@@ -164,29 +145,18 @@ it('can generate gzip version', () => {
   )
 
   expect(stdout).contains(`{
-  "days": 1,
-  "delay": 0,
-  "output": "tests/__data__/output/mini.guide.xml.gz",
+  "site": "example.com",
+  "url": "http://example.com/20210319/1tv.json",
   "channels": [
     "tests/__data__/input/example.channels.xml"
   ],
   "request": {
-    "method": "GET",
-    "maxContentLength": 5242880,
-    "timeout": 1,
-    "withCredentials": true,
-    "responseType": "arraybuffer",
-    "cache": false,
-    "headers": {
-      "User-Agent": "EPGGrabber/0.44.0 (https://github.com/freearhey/epg-grabber)"
-    }
+    "timeout": 1
   },
-  "maxConnections": 1,
-  "curl": false,
+  "output": "tests/__data__/output/mini.guide.xml.gz",
+  "delay": 0,
   "debug": true,
-  "gzip": true,
-  "site": "example.com",
-  "url": "http://example.com/20210319/1tv.json"
+  "gzip": true
 }`)
   expect(stdout).contains("File 'tests/__data__/output/mini.guide.xml.gz' successfully saved")
   expect(fs.readFileSync('tests/__data__/output/mini.guide.xml.gz')).toEqual(
@@ -209,29 +179,17 @@ it('can produce multiple outputs', () => {
   )
 
   expect(stdout).contains(`{
-  "days": 1,
-  "delay": 0,
-  "output": "tests/__data__/output/{lang}/{xmltv_id}.xml",
+  "site": "example.com",
+  "url": "http://example.com/20210319/1tv.json",
   "channels": [
     "tests/__data__/input/example.channels.xml"
   ],
   "request": {
-    "method": "GET",
-    "maxContentLength": 5242880,
-    "timeout": 1,
-    "withCredentials": true,
-    "responseType": "arraybuffer",
-    "cache": false,
-    "headers": {
-      "User-Agent": "EPGGrabber/0.44.0 (https://github.com/freearhey/epg-grabber)"
-    }
+    "timeout": 1
   },
-  "maxConnections": 1,
-  "curl": false,
-  "debug": true,
-  "gzip": false,
-  "site": "example.com",
-  "url": "http://example.com/20210319/1tv.json"
+  "output": "tests/__data__/output/{lang}/{xmltv_id}.xml",
+  "delay": 0,
+  "debug": true
 }`)
   expect(stdout).contains("File 'tests/__data__/output/fr/1TV.com.xml' successfully saved")
   expect(stdout).contains("File 'tests/__data__/output/undefined/2TV.com.xml' successfully saved")
@@ -252,31 +210,23 @@ it('can load multiple "*.channels.xml" files at once', () => {
   )
 
   expect(stdout).contains(`{
+  "site": "example.com",
   "days": 2,
-  "delay": 0,
-  "output": "tests/__data__/output/wildcard.guide.xml",
   "channels": [
     "tests/__data__/input/example_3.channels.xml",
     "tests/__data__/input/example_2.channels.xml"
   ],
+  "output": "tests/__data__/output/wildcard.guide.xml",
   "request": {
     "method": "POST",
-    "maxContentLength": 5242880,
-    "timeout": 1,
-    "withCredentials": true,
-    "responseType": "arraybuffer",
-    "cache": false,
     "headers": {
-      "User-Agent": "EPGGrabber/0.44.0 (https://github.com/freearhey/epg-grabber)",
       "Content-Type": "application/json",
       "Cookie": "abc=123"
-    }
+    },
+    "timeout": 1
   },
-  "maxConnections": 1,
-  "curl": false,
-  "debug": true,
-  "gzip": false,
-  "site": "example.com"
+  "delay": 0,
+  "debug": true
 }`)
   expect(stdout).contains("File 'tests/__data__/output/wildcard.guide.xml' successfully saved")
   expect(content('tests/__data__/output/wildcard.guide.xml')).toEqual(
@@ -293,31 +243,23 @@ it('can parse list of "*.channels.xml" from array', () => {
   )
 
   expect(stdout).contains(`{
+  "site": "example.com",
   "days": 2,
-  "delay": 0,
-  "output": "tests/__data__/output/channels_array.guide.xml",
   "channels": [
     "example_2.channels.xml",
     "example_3.channels.xml"
   ],
+  "output": "tests/__data__/output/channels_array.guide.xml",
   "request": {
     "method": "POST",
-    "maxContentLength": 5242880,
-    "timeout": 1,
-    "withCredentials": true,
-    "responseType": "arraybuffer",
-    "cache": false,
     "headers": {
-      "User-Agent": "EPGGrabber/0.44.0 (https://github.com/freearhey/epg-grabber)",
       "Content-Type": "application/json",
       "Cookie": "abc=123"
-    }
+    },
+    "timeout": 1
   },
-  "maxConnections": 1,
-  "curl": false,
-  "debug": true,
-  "gzip": false,
-  "site": "example.com"
+  "delay": 0,
+  "debug": true
 }`)
   expect(stdout).contains(
     "File 'tests/__data__/output/channels_array.guide.xml' successfully saved"