epg-grabber 0.37.4 → 0.38.0

package/README.md

@@ -73,6 +73,7 @@ Arguments:
 
 - `-c, --config`: path to config file
 - `-o, --output`: path to output file or path template (example: `guides/{site}.{lang}.xml`; default: `guide.xml`)
+- `-x, --proxy`: use the specified proxy
 - `--channels`: path to list of channels; you can also use wildcard to specify the path to multiple files at once (example: `example.com_*.channels.xml`)
 - `--lang`: set default language for all programs (default: `en`)
 - `--days`: number of days for which to grab the program (default: `1`)
@@ -196,7 +197,7 @@ From each function in `config.js` you can access a `context` object containing t
 
 ## Program Object
 
-| Property            | Aliases                          | Type                                             | Required |
+| Property        | Aliases                          | Type                                             | Required |
 | --------------- | -------------------------------- | ------------------------------------------------ | -------- |
 | start           |                                  | `String` or `Number` or `Date()`                 | true     |
 | stop            |                                  | `String` or `Number` or `Date()`                 | true     |
@@ -390,30 +391,8 @@ You can also specify the language, site, url, logo and LCN (Logical Channel Numb
 
 ## How to use SOCKS proxy?
 
-First, you need to install [socks-proxy-agent](https://www.npmjs.com/package/socks-proxy-agent):
-
-```sh
-npm install socks-proxy-agent
 ```
-
-Then you can use it to create an agent that acts as a SOCKS proxy. Here is an example of how to do it with the Tor SOCKS proxy:
-
-```js
-const { SocksProxyAgent } = require('socks-proxy-agent')
-
-const torProxyAgent = new SocksProxyAgent('socks://localhost:9050')
-
-module.exports = {
-  site: 'example.com',
-  url: 'https://example.com/epg.json',
-  request: {
-    httpsAgent: torProxyAgent,
-    httpAgent: torProxyAgent
-  },
-  parser(context) {
-    // ...
-  }
-}
+epg-grabber --config=example.com.config.js --proxy=socks://localhost:9050
 ```
 
 ## Contribution

package/bin/epg-grabber.js

@@ -5,14 +5,15 @@ const program = new Command()
 const { merge } = require('lodash')
 const { gzip } = require('node-gzip')
 const file = require('../src/file')
-const { EPGGrabber, parseChannels, generateXMLTV } = require('../src/index')
+const { EPGGrabber, EPGGrabberMock, parseChannels, generateXMLTV } = require('../src/index')
 const { create: createLogger } = require('../src/logger')
-const { parseNumber, getUTCDate } = require('../src/utils')
+const { parseNumber, getUTCDate, parseProxy } = require('../src/utils')
 const { name, version, description } = require('../package.json')
 const _ = require('lodash')
 const dayjs = require('dayjs')
 const utc = require('dayjs/plugin/utc')
 const { TaskQueue } = require('cwait')
+const { SocksProxyAgent } = require('socks-proxy-agent')
 
 dayjs.extend(utc)
 
@@ -22,6 +23,7 @@ program
   .description(description)
   .requiredOption('-c, --config <config>', 'Path to [site].config.js file')
   .option('-o, --output <output>', 'Path to output file')
+  .option('-x, --proxy <url>', 'Use the specified proxy')
   .option('--channels <channels>', 'Path to list of channels')
   .option('--lang <lang>', 'Set default language for all programs')
   .option('--days <days>', 'Number of days for which to grab the program', parseNumber)
@@ -63,10 +65,26 @@ async function main() {
     request: {}
   })
 
-  if (options.timeout) config.request.timeout = options.timeout
-  if (options.cacheTtl) config.request.cache.ttl = options.cacheTtl
-
-  if (options.channels) config.channels = options.channels
+  if (options.timeout !== undefined) config.request.timeout = options.timeout
+  if (options.cacheTtl !== undefined) config.request.cache.ttl = options.cacheTtl
+  if (options.channels !== undefined) config.channels = options.channels
+  if (options.proxy !== undefined) {
+    const proxy = parseProxy(options.proxy)
+
+    if (
+      proxy.protocol &&
+      ['socks', 'socks5', 'socks5h', 'socks4', 'socks4a'].includes(String(proxy.protocol))
+    ) {
+      const socksProxyAgent = new SocksProxyAgent(options.proxy)
+
+      config.request = {
+        ...config.request,
+        ...{ httpAgent: socksProxyAgent, httpsAgent: socksProxyAgent }
+      }
+    } else {
+      config.request = { ...config.request, ...{ proxy } }
+    }
+  }
 
   let parsedChannels = []
   if (config.channels) {
@@ -89,9 +107,10 @@ async function main() {
     }
   } else throw new Error('Path to "channels" is missing')
 
-  const grabber = new EPGGrabber(config)
+  const grabber =
+    process.env.NODE_ENV === 'test' ? new EPGGrabberMock(config) : new EPGGrabber(config)
 
-  let template = options.output || config.output
+  let template = options.output || config.output || 'guide.xml'
   const variables = file.templateVariables(template)
 
   const groups = _.groupBy(parsedChannels, channel => {
@@ -133,9 +152,11 @@ async function main() {
           await grabber
             .grab(channel, date, (data, err) => {
               logger.info(
-                `[${i}/${total}] ${config.site} - ${data.channel.xmltv_id} - ${dayjs
-                  .utc(data.date)
-                  .format('MMM D, YYYY')} (${data.programs.length} programs)`
+                `[${i}/${total}] ${config.site} - ${
+                  data.channel.xmltv_id || data.channel.site_id
+                } - ${dayjs.utc(data.date).format('MMM D, YYYY')} (${
+                  data.programs.length
+                } programs)`
               )
 
               if (err) logger.error(err.message)

package/eslint.config.mjs

@@ -0,0 +1,25 @@
+import globals from "globals";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import js from "@eslint/js";
+import { FlatCompat } from "@eslint/eslintrc";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const compat = new FlatCompat({
+    baseDirectory: __dirname,
+    recommendedConfig: js.configs.recommended,
+    allConfig: js.configs.all
+});
+
+export default [...compat.extends("eslint:recommended"), {
+    languageOptions: {
+        globals: {
+            ...Object.fromEntries(Object.entries(globals.browser).map(([key]) => [key, "off"])),
+            ...globals.node,
+        },
+
+        ecmaVersion: 12,
+        sourceType: "commonjs",
+    },
+}];

package/package.json

@@ -1,9 +1,10 @@
 {
   "name": "epg-grabber",
-  "version": "0.37.4",
+  "version": "0.38.0",
   "description": "Node.js CLI tool for grabbing EPG from different sites",
   "main": "src/index.js",
   "preferGlobal": true,
+  "homepage": "https://github.com/freearhey/epg-grabber",
   "bin": {
     "epg-grabber": "bin/epg-grabber.js"
   },
@@ -41,6 +42,7 @@
     "http-cookie-agent": "^6.0.8",
     "lodash": "^4.17.21",
     "node-gzip": "^1.1.2",
+    "socks-proxy-agent": "^8.0.5",
     "tough-cookie": "^5.0.0",
     "winston": "^3.3.3",
     "xml-js": "^1.6.11"
@@ -48,8 +50,11 @@
   "devDependencies": {
     "@babel/core": "^7.13.14",
     "@babel/preset-env": "^7.13.12",
+    "@eslint/eslintrc": "^3.2.0",
+    "@eslint/js": "^9.18.0",
     "babel-jest": "^29.7.0",
     "eslint": "^9.17.0",
+    "globals": "^15.14.0",
     "jest": "^29.7.0",
     "jest-mock-axios": "^4.4.1"
   },

package/src/Channel.js

@@ -1,5 +1,3 @@
-const { toArray } = require('./utils')
-
 class Channel {
   constructor(c) {
     const data = {

package/src/client.js

@@ -1,11 +1,7 @@
 const { CurlGenerator } = require('curl-generator')
 const { default: axios } = require('axios')
-const { CookieJar } = require('tough-cookie')
 const { setupCache } = require('axios-cache-interceptor')
 const { isObject, isPromise } = require('./utils')
-const { HttpCookieAgent, HttpsCookieAgent } = require('http-cookie-agent/http')
-
-const jar = new CookieJar()
 
 module.exports.create = create
 module.exports.buildRequest = buildRequest
@@ -16,17 +12,10 @@ let timeout
 function create(config) {
   const client = axios.defaults.cache
     ? axios
-    : setupCookie(
-        setupCache(
-          axios.create({
-            jar,
-            ignoreCookieErrors: true,
-            headers: {
-              'User-Agent':
-                'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36 Edg/79.0.309.71'
-            }
-          })
-        )
+    : setupCache(
+        axios.create({
+          ignoreCookieErrors: true
+        })
       )
 
   client.interceptors.request.use(
@@ -142,44 +131,3 @@ async function getRequestUrl({ channel, date, config }) {
   }
   return config.url
 }
-
-const AGENT_CREATED_BY_AXIOS_COOKIEJAR_SUPPORT = Symbol('AGENT_CREATED_BY_AXIOS_COOKIEJAR_SUPPORT')
-
-function requestInterceptor(config) {
-  if (!config.jar) {
-    return config
-  }
-
-  config.httpAgent = new HttpCookieAgent({ cookies: { jar: config.jar } })
-  Object.defineProperty(config.httpAgent, AGENT_CREATED_BY_AXIOS_COOKIEJAR_SUPPORT, {
-    configurable: false,
-    enumerable: false,
-    value: true,
-    writable: false
-  })
-
-  config.httpsAgent = new HttpsCookieAgent({ cookies: { jar: config.jar } })
-  Object.defineProperty(config.httpsAgent, AGENT_CREATED_BY_AXIOS_COOKIEJAR_SUPPORT, {
-    configurable: false,
-    enumerable: false,
-    value: true,
-    writable: false
-  })
-
-  return config
-}
-
-function setupCookie(axios) {
-  axios.interceptors.request.use(requestInterceptor)
-
-  if ('create' in axios) {
-    const create = axios.create
-    axios.create = (...args) => {
-      const instance = create.apply(axios, args)
-      instance.interceptors.request.use(requestInterceptor)
-      return instance
-    }
-  }
-
-  return axios
-}

package/src/config.js

@@ -1,5 +1,5 @@
-const tough = require('tough-cookie')
 const { merge } = require('lodash')
+const { version, homepage } = require('../package.json')
 
 module.exports.parse = parse
 
@@ -24,9 +24,11 @@ function parse(config) {
       maxContentLength: 5 * 1024 * 1024,
       timeout: 5000,
       withCredentials: true,
-      jar: new tough.CookieJar(),
       responseType: 'arraybuffer',
-      cache: false
+      cache: false,
+      headers: {
+        'User-Agent': `EPGGrabber/${version} (${homepage})`
+      }
     }
   }
 

package/src/index.d.ts

@@ -101,19 +101,21 @@ export type GrabCallbackData = {
 }
 
 export declare class EPGGrabber {
-  constructor(config: SiteConfig)
+  constructor(config?: SiteConfig)
   grab(
     channel: Channel,
     date: string | dayjs.Dayjs,
-    cb: (data: GrabCallbackData, err: Error | null) => void
+    config?: SiteConfig,
+    cb?: (data: GrabCallbackData, err: Error | null) => void
   ): Promise<Program[]>
 }
 
 export declare class EPGGrabberMock {
-  constructor(config: SiteConfig)
+  constructor(config?: SiteConfig)
   grab(
     channel: Channel,
     date: string | dayjs.Dayjs,
-    cb: (data: GrabCallbackData, err: Error | null) => void
+    config?: SiteConfig,
+    cb?: (data: GrabCallbackData, err: Error | null) => void
   ): Promise<Program[]>
 }

package/src/index.js

@@ -60,10 +60,18 @@ class EPGGrabber {
 }
 
 class EPGGrabberMock {
-  constructor(config) {
+  constructor(config = {}) {
     this.config = config
   }
 
+  async loadLogo(channel) {
+    const logo = this.config.logo({ channel })
+    if (isPromise(logo)) {
+      return await logo
+    }
+    return logo
+  }
+
   async grab(channel, date, config = {}, cb = () => {}) {
     let _date = getUTCDate(date)
     if (typeof config == 'function') {

package/src/utils.js

@@ -1,3 +1,4 @@
+const { URL } = require('node:url')
 const dayjs = require('dayjs')
 const utc = require('dayjs/plugin/utc')
 
@@ -13,6 +14,21 @@ module.exports.formatDate = formatDate
 module.exports.toArray = toArray
 module.exports.toUnix = toUnix
 module.exports.isDate = isDate
+module.exports.parseProxy = parseProxy
+
+function parseProxy(_url) {
+  const parsed = new URL(_url)
+
+  return {
+    protocol: parsed.protocol.replace(':', '') || null,
+    auth: {
+      username: parsed.username || null,
+      password: parsed.password || null
+    },
+    host: parsed.hostname,
+    port: parsed.port ? parseInt(parsed.port) : null
+  }
+}
 
 function sleep(ms) {
   return new Promise(resolve => setTimeout(resolve, ms))

package/tests/__data__/output/guide.xml

@@ -1,6 +1,10 @@
-<?xml version="1.0" encoding="UTF-8" ?><tv date="20241215">
+<?xml version="1.0" encoding="UTF-8" ?><tv date="20250131">
 <channel id="1TV.com"><display-name>1 TV</display-name><icon src="https://example.com/logos/1TV.png"/><url>https://example.com</url></channel>
 <channel id="2TV.com"><display-name>2 TV</display-name><url>https://example.com</url></channel>
 <channel id="3TV.com"><display-name>3 TV</display-name><url>https://example2.com</url></channel>
 <channel id="4TV.com"><display-name>4 TV</display-name><url>https://example2.com</url></channel>
+<programme start="20220101000000 +0000" stop="20220101010000 +0000" channel="1TV.com"><title lang="fr">Program1</title></programme>
+<programme start="20220101000000 +0000" stop="20220101010000 +0000" channel="4TV.com"><title>Program1</title></programme>
+<programme start="20220101000000 +0000" stop="20220101010000 +0000" channel="3TV.com"><title>Program1</title></programme>
+<programme start="20220101000000 +0000" stop="20220101010000 +0000" channel="2TV.com"><title>Program1</title></programme>
 </tv>

package/tests/bin.test.js

@@ -36,6 +36,7 @@ it('can load mini config', () => {
       --days=3 \
       --delay=0 \
       --debug \
+      --proxy=socks://127.0.0.1:1086 \
       --timeout=1`,
     {
       encoding: 'utf8'

package/tests/utils.test.js

@@ -1,4 +1,4 @@
-import { escapeString } from '../src/utils'
+import { escapeString, parseProxy } from '../src/utils'
 
 it('can escape string', () => {
   const string = 'Música тест dun.  &<>"\'\r\n'
@@ -9,3 +9,10 @@ it('can escape url', () => {
   const string = 'http://example.com/logos/1TV.png?param1=val&param2=val'
   expect(escapeString(string)).toBe('http://example.com/logos/1TV.png?param1=val&amp;param2=val')
 })
+
+it('can parse proxy', () => {
+  const string = 'socks://127.0.0.1:1234'
+  expect(parseProxy(string)).toMatchObject({
+    protocol: 'socks'
+  })
+})

package/.eslintrc.js

@@ -1,11 +0,0 @@
-module.exports = {
-  env: {
-    browser: false,
-    node: true,
-    es6: true
-  },
-  extends: 'eslint:recommended',
-  parserOptions: {
-    ecmaVersion: 12
-  }
-}