epg-grabber 0.40.4 → 0.42.0

package/README.md

@@ -118,9 +118,9 @@ module.exports = {
     },
 
     /**
-     * @param {object} context
+     * @param {object} Request context
      *
-     * @return {string} The function should return headers for each request (optional)
+     * @return {object} The function should return headers for each request (optional)
      */
     headers: function(context) {
       return {
@@ -130,9 +130,9 @@ module.exports = {
     },
 
     /**
-     * @param {object} context
+     * @param {object} Request context
      *
-     * @return {string} The function should return data for each request (optional)
+     * @return {any} The function should return data for each request (optional)
      */
     data: function(context) {
       const { channel, date } = context
@@ -146,7 +146,7 @@ module.exports = {
   },
 
   /**
-   * @param {object} context
+   * @param {object} Request context
    *
    * @return {string} The function should return URL of the program page for the channel
    */
@@ -155,7 +155,7 @@ module.exports = {
   },
 
   /**
-   * @param {object} context
+   * @param {object} Request context
    *
    * @return {string} The function should return URL of the channel logo (optional)
    */
@@ -164,7 +164,7 @@ module.exports = {
   },
 
   /**
-   * @param {object} context
+   * @param {object} Parser context
    *
    * @return {array} The function should return an array of programs with their descriptions
    */
@@ -184,9 +184,16 @@ module.exports = {
 }
 ```
 
-## Context Object
+## Request Context Object
 
-From each function in `config.js` you can access a `context` object containing the following data:
+Inside `url()`, `logo()`, `request.data()`, `request.headers()` functions in `config.js` you can access a `context` object containing the following data:
+
+- `channel`: The object describing the current channel (xmltv_id, site_id, name, lang)
+- `date`: The 'dayjs' instance with the requested date
+
+## Parser Context Object
+
+Inside `parser()` function in `config.js` you can access a `context` object containing the following data:
 
 - `channel`: The object describing the current channel (xmltv_id, site_id, name, lang)
 - `date`: The 'dayjs' instance with the requested date

package/dist/cli.js

@@ -0,0 +1,168 @@
+#!/usr/bin/env node
+import { n as name, v as version, d as description, p as parseNumber, L as Logger, E as EPGGrabber, g as getUTCDate, a as generateXMLTV } from './index-Bcnk0U8G.js';
+import { Template, Collection } from '@freearhey/core';
+import { Command, Option } from 'commander';
+import { SocksProxyAgent } from 'socks-proxy-agent';
+import { URL } from 'node:url';
+import path from 'node:path';
+import { TaskQueue } from 'cwait';
+import Promise$1 from 'bluebird';
+import fs from 'fs-extra';
+import pako from 'pako';
+import _ from 'lodash';
+import 'dayjs';
+import 'dayjs/plugin/utc.js';
+import 'xml-js';
+import 'glob';
+import 'axios';
+import 'curl-generator';
+import 'axios-cache-interceptor';
+import 'winston';
+import 'path';
+
+class ProxyParser {
+  static parse(proxy) {
+    const parsed = new URL(proxy);
+    const result = {
+      protocol: parsed.protocol.replace(":", "") || null,
+      host: parsed.hostname,
+      port: parsed.port ? parseInt(parsed.port) : null
+    };
+    if (parsed.username || parsed.password) {
+      result.auth = {};
+      if (parsed.username) result.auth.username = parsed.username;
+      if (parsed.password) result.auth.password = parsed.password;
+    }
+    return result;
+  }
+}
+
+class Storage {
+  static async loadJs(filepath) {
+    const absPath = path.resolve(filepath);
+    return (await import(absPath)).default;
+  }
+}
+
+const program = new Command();
+program.name(name).version(version, "-v, --version").description(description).addOption(
+  new Option("-c, --config <config>", "Path to [site].config.js file").makeOptionMandatory()
+).addOption(new Option("-o, --output <output>", "Path to output file").default("guide.xml")).addOption(new Option("-x, --proxy <url>", "Use the specified proxy")).addOption(new Option("--channels <channels>", "Path to list of channels")).addOption(new Option("--lang <lang>", "Set default language for all programs")).addOption(
+  new Option("--days <days>", "Number of days for which to grab the program").argParser(parseNumber).default(1)
+).addOption(
+  new Option("--delay <delay>", "Delay between requests (in milliseconds)").argParser(parseNumber).default(3e3)
+).addOption(
+  new Option("--timeout <timeout>", "Set a timeout for each request (in milliseconds)").argParser(parseNumber).default(5e3)
+).addOption(
+  new Option(
+    "--max-connections <maxConnections>",
+    "Set a limit on the number of concurrent requests per site"
+  ).argParser(parseNumber).default(1)
+).addOption(
+  new Option(
+    "--cache-ttl <cacheTtl>",
+    "Maximum time for storing each request (in milliseconds)"
+  ).argParser(parseNumber)
+).addOption(new Option("--gzip", "Compress the output").default(false)).addOption(new Option("--debug", "Enable debug mode").default(false)).addOption(new Option("--curl", "Display request as CURL").default(false)).addOption(new Option("--log <log>", "Path to log file")).addOption(new Option("--log-level <level>", "Set log level").default("info")).parse(process.argv);
+const options = program.opts();
+const logger = new Logger({
+  log: options.log,
+  logLevel: options.debug ? "debug" : options.logLevel
+});
+async function main() {
+  logger.info("Starting...");
+  logger.debug(`Options: ${JSON.stringify(options, null, 2)}`);
+  logger.info(`Loading '${options.config}'...`);
+  let configObject = await Storage.loadJs(options.config);
+  configObject = _.merge(configObject, {
+    filepath: path.resolve(options.config),
+    channels: typeof options.channels === "string" ? path.resolve(options.channels) : void 0,
+    request: {}
+  });
+  if (configObject.output === void 0) configObject.output = options.output;
+  if (configObject.days === void 0) configObject.days = options.days;
+  if (configObject.delay === void 0) configObject.delay = options.delay;
+  if (configObject.curl === void 0) configObject.curl = options.curl;
+  if (configObject.debug === void 0) configObject.debug = options.debug;
+  if (configObject.gzip === void 0) configObject.gzip = options.gzip;
+  if (configObject.maxConnections === void 0)
+    configObject.maxConnections = options.maxConnections;
+  if (configObject.request.timeout === void 0) configObject.request.timeout = options.timeout;
+  if (options.cacheTtl !== void 0) configObject.request.cache = { ttl: options.cacheTtl };
+  if (options.proxy !== void 0) {
+    const proxy = ProxyParser.parse(options.proxy);
+    if (proxy.protocol && ["socks", "socks5", "socks5h", "socks4", "socks4a"].includes(String(proxy.protocol))) {
+      const socksProxyAgent = new SocksProxyAgent(options.proxy);
+      configObject.request = {
+        ...configObject.request,
+        ...{ httpAgent: socksProxyAgent, httpsAgent: socksProxyAgent }
+      };
+    } else {
+      configObject.request = { ...configObject.request, ...{ proxy } };
+    }
+  }
+  const grabber = new EPGGrabber(configObject, { logger });
+  logger.info("Loading channel list...");
+  const channelList = await grabber.loadChannels();
+  const template = new Template(configObject.output);
+  const groups = channelList.getGroups({ template });
+  if (channelList.getChannels().isEmpty()) {
+    logger.info("No channels found");
+    logger.info("Exit");
+    return;
+  }
+  logger.info("Processing...");
+  for (let groupId of groups.keys()) {
+    const group = groups.get(groupId);
+    const channels = new Collection(group);
+    let programs = new Collection();
+    let index = 1;
+    let days = configObject.days;
+    const maxConnections = configObject.maxConnections;
+    const total = channels.count() * days;
+    const utcDate = getUTCDate(process.env.CURR_DATE);
+    const dates = Array.from({ length: days }, (_2, i) => utcDate.add(i, "d"));
+    let queue = new Collection();
+    channels.forEach((channel) => {
+      for (let date of dates) {
+        queue.add({ channel, date });
+      }
+    });
+    const taskQueue = new TaskQueue(Promise$1, maxConnections);
+    const requests = queue.map(
+      taskQueue.wrap(async (queueItem) => {
+        const { channel, date } = queueItem;
+        if (!channel.logo && configObject.logo) {
+          channel.logo = await grabber.loadLogo({ channel, date });
+        }
+        const _programs = await grabber.grab(channel, date, (context, error) => {
+          const { channel: channel2, date: date2, programs: programs2 } = context;
+          logger.info(
+            `[${index}/${total}] ${configObject.site} - ${channel2.xmltv_id || channel2.site_id} - ${date2.format("MMM D, YYYY")} (${programs2.length} programs)`
+          );
+          if (error) logger.error(error.message);
+          if (index < total) index++;
+        });
+        programs.concat(new Collection(_programs));
+      })
+    );
+    await Promise$1.all(requests.all());
+    programs = programs.uniqBy((program2) => program2.start + program2.channel);
+    const xml = generateXMLTV(channels.all(), programs.all(), utcDate);
+    const channelSample = channels.sample();
+    let outputPath = template.format(channelSample.toObject());
+    const outputDir = path.dirname(outputPath);
+    fs.mkdirSync(outputDir, { recursive: true });
+    if (options.gzip) {
+      const compressed = pako.gzip(xml);
+      outputPath = outputPath || "guide.xml.gz";
+      fs.writeFileSync(outputPath, compressed);
+    } else {
+      outputPath = outputPath || "guide.xml";
+      fs.writeFileSync(outputPath, xml);
+    }
+    logger.info(`File '${outputPath}' successfully saved`);
+  }
+  logger.info("Finished");
+}
+main();