epg-grabber 0.44.0 → 0.46.0

package/README.md

@@ -75,7 +75,6 @@ Arguments:
 - `-o, --output`: path to output file or path template (example: `guides/{site}.{lang}.xml`; default: `guide.xml`)
 - `-x, --proxy`: use the specified proxy (example: `socks5://username:password@127.0.0.1:1234`)
 - `--channels`: path to list of channels; you can also use wildcard to specify the path to multiple files at once (example: `example.com_*.channels.xml`)
-- `--lang`: set default language for all programs (default: `en`)
 - `--days`: number of days for which to grab the program (default: `1`)
 - `--delay`: delay between requests in milliseconds (default: `3000`)
 - `--timeout`: set a timeout for each request in milliseconds (default: `5000`)
@@ -94,7 +93,6 @@ module.exports = {
   site: 'example.com', // site domain name (required)
   output: 'example.com.guide.xml', // path to output file or path template (example: 'guides/{site}.{lang}.xml'; default: 'guide.xml')
   channels: 'example.com.channels.xml', // path to list of channels; you can also use an array to specify the path to multiple files at once (example: ['channels1.xml', 'channels2.xml']; required)
-  lang: 'fr', // default language for all programs (default: 'en')
   days: 3, // number of days for which to grab the program (default: 1)
   delay: 5000, // delay between requests (default: 3000)
   maxConnections: 200, // limit on the number of concurrent requests (default: 1)

package/dist/cli.js

@@ -1,93 +1,182 @@
 #!/usr/bin/env node
-import { n as name, v as version, d as description, p as parseNumber, L as Logger, l as loadJs, a as parseProxy, E as EPGGrabber, g as getUTCDate } from './index-DQUOgwl_.js';
-import { Template, Collection } from '@freearhey/core';
+import { n as name, v as version, d as description, p as parseNumber, l as loadJs, a as parseProxy, E as EPGGrabberMock, b as EPGGrabber, c as defaultConfig, i as isObject, g as getAbsPath, e as getUTCDate } from './index-DUJ5SdIU.js';
+import { Collection, Template } from '@freearhey/core';
 import { Command, Option } from 'commander';
 import { SocksProxyAgent } from 'socks-proxy-agent';
+import { CurlGenerator } from 'curl-generator';
+import winston from 'winston';
+import path from 'path';
 import { TaskQueue } from 'cwait';
+import merge from 'lodash.merge';
 import Promise$1 from 'bluebird';
-import path from 'node:path';
+import path$1 from 'node:path';
+import { glob } from 'glob';
 import fs from 'fs-extra';
 import pako from 'pako';
-import _ from 'lodash';
-import 'dayjs';
 import 'dayjs/plugin/utc.js';
+import 'dayjs';
 import 'node:url';
-import 'glob';
+import 'axios-mock-adapter';
+import 'lodash.padstart';
 import 'axios';
-import 'curl-generator';
-import 'winston';
-import 'path';
 import 'axios-cache-interceptor';
 import 'xml-js';
 
+const { combine, timestamp, printf } = winston.format;
+class Logger {
+  #logger;
+  constructor(options) {
+    options = options || {};
+    const fileFormat = printf(({ level, message, timestamp: timestamp2 }) => {
+      return `[${timestamp2}] ${level.toUpperCase()}: ${message}`;
+    });
+    const templateFunction = (info) => {
+      if (info.level === "error") return `  Error: ${info.message}`;
+      if (typeof info.message === "string") return info.message;
+      return "";
+    };
+    const consoleFormat = printf(templateFunction);
+    const transports = [
+      new winston.transports.Console({ format: consoleFormat })
+    ];
+    if (options.log) {
+      transports.push(
+        new winston.transports.File({
+          filename: path.resolve(options.log),
+          format: combine(timestamp(), fileFormat),
+          options: { flags: "w" }
+        })
+      );
+    }
+    this.#logger = winston.createLogger({
+      level: options.logLevel,
+      transports
+    });
+  }
+  info(message) {
+    this.#logger.info(message);
+  }
+  debug(message) {
+    this.#logger.debug(message);
+  }
+  error(message) {
+    this.#logger.error(message);
+  }
+}
+
 const program = new Command();
 program.name(name).version(version, "-v, --version").description(description).addOption(
   new Option("-c, --config <config>", "Path to [site].config.js file").makeOptionMandatory()
-).addOption(new Option("-o, --output <output>", "Path to output file").default("guide.xml")).addOption(new Option("-x, --proxy <url>", "Use the specified proxy")).addOption(new Option("--channels <channels>", "Path to list of channels")).addOption(new Option("--lang <lang>", "Set default language for all programs")).addOption(
-  new Option("--days <days>", "Number of days for which to grab the program").argParser(parseNumber).default(1)
+).addOption(new Option("-o, --output <output>", "Path to output file")).addOption(new Option("-x, --proxy <url>", "Use the specified proxy")).addOption(new Option("--channels <channels>", "Path to list of channels")).addOption(
+  new Option("--days <days>", "Number of days for which to grab the program").argParser(
+    parseNumber
+  )
 ).addOption(
-  new Option("--delay <delay>", "Delay between requests (in milliseconds)").argParser(parseNumber).default(3e3)
+  new Option("--delay <delay>", "Delay between requests (in milliseconds)").argParser(parseNumber)
 ).addOption(
-  new Option("--timeout <timeout>", "Set a timeout for each request (in milliseconds)").argParser(parseNumber).default(5e3)
+  new Option("--timeout <timeout>", "Set a timeout for each request (in milliseconds)").argParser(
+    parseNumber
+  )
 ).addOption(
   new Option(
     "--max-connections <maxConnections>",
     "Set a limit on the number of concurrent requests per site"
-  ).argParser(parseNumber).default(1)
+  ).argParser(parseNumber)
 ).addOption(
   new Option(
     "--cache-ttl <cacheTtl>",
     "Maximum time for storing each request (in milliseconds)"
   ).argParser(parseNumber)
-).addOption(new Option("--gzip", "Compress the output").default(false)).addOption(new Option("--debug", "Enable debug mode").default(false)).addOption(new Option("--curl", "Display request as CURL").default(false)).addOption(new Option("--log <log>", "Path to log file")).addOption(new Option("--log-level <level>", "Set log level").default("info")).parse(process.argv);
+).addOption(new Option("--gzip", "Compress the output")).addOption(new Option("--debug", "Enable debug mode")).addOption(new Option("--curl", "Display request as CURL")).addOption(new Option("--log <log>", "Path to log file")).addOption(new Option("--log-level <level>", "Set log level")).parse(process.argv);
 const options = program.opts();
 const logger = new Logger({
   log: options.log,
-  logLevel: options.debug ? "debug" : options.logLevel
+  logLevel: options.debug === true ? "debug" : options.logLevel
 });
 async function main() {
   logger.info("Starting...");
-  logger.debug(`Options: ${JSON.stringify(options, null, 2)}`);
   logger.info(`Loading '${options.config}'...`);
-  let configObject = await loadJs(options.config);
-  configObject = _.merge(configObject, {
-    filepath: path.resolve(options.config),
-    channels: typeof options.channels === "string" ? path.resolve(options.channels) : void 0,
-    request: {}
-  });
-  if (configObject.output === void 0) configObject.output = options.output;
-  if (configObject.days === void 0) configObject.days = options.days;
-  if (configObject.delay === void 0) configObject.delay = options.delay;
-  if (configObject.curl === void 0) configObject.curl = options.curl;
-  if (configObject.debug === void 0) configObject.debug = options.debug;
-  if (configObject.gzip === void 0) configObject.gzip = options.gzip;
-  if (configObject.maxConnections === void 0)
-    configObject.maxConnections = options.maxConnections;
-  if (configObject.request.timeout === void 0) configObject.request.timeout = options.timeout;
-  if (options.cacheTtl !== void 0) configObject.request.cache = { ttl: options.cacheTtl };
+  let config = await loadJs(options.config);
+  config.channels = Array.isArray(config.channels) ? config.channels : typeof config.channels === "string" ? [config.channels] : [];
+  if (typeof options.cacheTtl === "number")
+    config = merge(config, { request: { cache: { ttl: options.cacheTtl } } });
+  if (typeof options.timeout === "number")
+    config = merge(config, { request: { timeout: options.timeout } });
   if (options.proxy !== void 0) {
     const proxy = parseProxy(options.proxy);
     if (proxy.protocol && ["socks", "socks5", "socks5h", "socks4", "socks4a"].includes(String(proxy.protocol))) {
       const socksProxyAgent = new SocksProxyAgent(options.proxy);
-      configObject.request = {
-        ...configObject.request,
-        ...{ httpAgent: socksProxyAgent, httpsAgent: socksProxyAgent }
-      };
+      config = merge(config, {
+        request: { httpAgent: socksProxyAgent, httpsAgent: socksProxyAgent }
+      });
     } else {
-      configObject.request = { ...configObject.request, ...{ proxy } };
+      config = merge(config, { request: { proxy } });
     }
   }
-  const grabber = new EPGGrabber(configObject, { logger });
-  logger.info("Loading channels...");
-  const channels = await grabber.loadChannels();
-  const template = new Template(configObject.output);
+  if (typeof options.channels === "string") config.channels = await glob(options.channels);
+  if (typeof options.output === "string") config.output = options.output;
+  if (typeof options.days === "number") config.days = options.days;
+  if (typeof options.delay === "number") config.delay = options.delay;
+  if (typeof options.maxConnections === "number") config.maxConnections = options.maxConnections;
+  if (typeof options.debug === "boolean") config.debug = options.debug;
+  if (typeof options.curl === "boolean") config.curl = options.curl;
+  if (typeof options.gzip === "boolean") config.gzip = options.gzip;
+  logger.debug(`Config: ${JSON.stringify(config, null, 2)}`);
+  const grabber = process.env.NODE_ENV === "test" ? new EPGGrabberMock(config) : new EPGGrabber(config);
+  grabber.client.instance.interceptors.request.use(
+    (request) => {
+      logger.debug(`Request: ${JSON.stringify(request, null, 2)}`);
+      const curl = config.curl || defaultConfig.curl;
+      if (curl) {
+        const url = request.url || "";
+        const method = request.method ? request.method : "GET";
+        const headers = request.headers ? request.headers.toJSON() : void 0;
+        const body = request.data ? request.data : void 0;
+        const curl2 = CurlGenerator({ url, method, headers, body });
+        logger.info(curl2);
+      }
+      return request;
+    },
+    (error) => Promise$1.reject(error)
+  );
+  grabber.client.instance.interceptors.response.use(
+    (response) => {
+      const data = response.data ? isObject(response.data) || Array.isArray(response.data) ? JSON.stringify(response.data) : response.data.toString() : void 0;
+      logger.debug(
+        `Response: ${JSON.stringify(
+          {
+            headers: response.headers,
+            data,
+            cached: response.cached
+          },
+          null,
+          2
+        )}`
+      );
+      return response;
+    },
+    (error) => Promise$1.reject(error)
+  );
+  if (!Array.isArray(config.channels) || !config.channels.length)
+    throw new Error('Path to "*.channels.xml" is missing');
+  const channels = new Collection();
+  const rootDir = options.channels ? process.cwd() : path$1.dirname(options.config);
+  config.channels.forEach((filepath) => {
+    const absFilepath = getAbsPath(filepath, rootDir);
+    logger.debug(`Loading "${absFilepath}"...`);
+    const channelsXML = fs.readFileSync(absFilepath, "utf8");
+    const channelsFromXML = EPGGrabber.parseChannelsXML(channelsXML);
+    channels.concat(new Collection(channelsFromXML));
+  });
+  if (channels.isEmpty()) throw new Error("No channels found");
+  const days = config.days || defaultConfig.days;
+  const maxConnections = config.maxConnections || defaultConfig.maxConnections;
+  const gzip = config.gzip || defaultConfig.gzip;
+  const defaultOutput = gzip ? defaultConfig.output + ".gz" : defaultConfig.output;
+  const output = config.output || defaultOutput;
+  const template = new Template(output);
   const variables = template.variables();
-  if (!channels.length) {
-    logger.info("No channels found");
-    logger.info("Exit");
-    return;
-  }
-  const groups = new Collection(channels).groupBy((channel) => {
+  const groups = channels.groupBy((channel) => {
     let groupId = "";
     for (const key in channel) {
       if (variables.includes(key)) {
@@ -98,16 +187,14 @@ async function main() {
     return groupId;
   });
   logger.info("Processing...");
-  for (let groupId of groups.keys()) {
+  for (const groupId of groups.keys()) {
     const group = groups.get(groupId);
     const groupChannels = new Collection(group);
     let programs = new Collection();
     let index = 1;
-    let days = configObject.days;
-    const maxConnections = configObject.maxConnections;
     const total = groupChannels.count() * days;
     const utcDate = getUTCDate(process.env.CURR_DATE);
-    const dates = Array.from({ length: days }, (_2, i) => utcDate.add(i, "d"));
+    const dates = Array.from({ length: days }, (_, i) => utcDate.add(i, "d"));
     let queue = new Collection();
     groupChannels.forEach((channel) => {
       for (let date of dates) {
@@ -118,13 +205,13 @@ async function main() {
     const requests = queue.map(
       taskQueue.wrap(async (queueItem) => {
         const { channel, date } = queueItem;
-        if (!channel.logo && configObject.logo) {
+        if (!channel.logo) {
           channel.logo = await grabber.loadLogo(channel, date);
         }
         const _programs = await grabber.grab(channel, date, (context, error) => {
           const { channel: channel2, date: date2, programs: programs2 } = context;
           logger.info(
-            `[${index}/${total}] ${configObject.site} - ${channel2.xmltv_id || channel2.site_id} - ${date2.format("MMM D, YYYY")} (${programs2.length} programs)`
+            `[${index}/${total}] ${channel2.site} - ${channel2.xmltv_id || channel2.site_id} - ${date2.format("MMM D, YYYY")} (${programs2.length} programs)`
           );
           if (error) logger.error(error.message);
           if (index < total) index++;
@@ -133,18 +220,15 @@ async function main() {
       })
     );
     await Promise$1.all(requests.all());
-    programs = programs.uniqBy((program2) => program2.start + program2.channel);
     const xml = EPGGrabber.generateXMLTV(groupChannels.all(), programs.all(), utcDate);
     const channelSample = groupChannels.sample();
     let outputPath = template.format(channelSample.toObject());
-    const outputDir = path.dirname(outputPath);
+    const outputDir = path$1.dirname(outputPath);
     fs.mkdirSync(outputDir, { recursive: true });
-    if (options.gzip) {
+    if (gzip) {
       const compressed = pako.gzip(xml);
-      outputPath = outputPath || "guide.xml.gz";
       fs.writeFileSync(outputPath, compressed);
     } else {
-      outputPath = outputPath || "guide.xml";
       fs.writeFileSync(outputPath, xml);
     }
     logger.info(`File '${outputPath}' successfully saved`);