epg-grabber 0.37.5 → 0.39.0

package/README.md

@@ -73,6 +73,7 @@ Arguments:
 
 - `-c, --config`: path to config file
 - `-o, --output`: path to output file or path template (example: `guides/{site}.{lang}.xml`; default: `guide.xml`)
+- `-x, --proxy`: use the specified proxy (example: `socks5://username:password@127.0.0.1:1234`)
 - `--channels`: path to list of channels; you can also use wildcard to specify the path to multiple files at once (example: `example.com_*.channels.xml`)
 - `--lang`: set default language for all programs (default: `en`)
 - `--days`: number of days for which to grab the program (default: `1`)
@@ -97,6 +98,7 @@ module.exports = {
   days: 3, // number of days for which to grab the program (default: 1)
   delay: 5000, // delay between requests (default: 3000)
   maxConnections: 200, // limit on the number of concurrent requests (default: 1)
+  curl: true, // display each request as CURL (default: false)
 
   request: { // request options (details: https://github.com/axios/axios#request-config)
 
@@ -196,7 +198,7 @@ From each function in `config.js` you can access a `context` object containing t
 
 ## Program Object
 
-| Property            | Aliases                          | Type                                             | Required |
+| Property        | Aliases                          | Type                                             | Required |
 | --------------- | -------------------------------- | ------------------------------------------------ | -------- |
 | start           |                                  | `String` or `Number` or `Date()`                 | true     |
 | stop            |                                  | `String` or `Number` or `Date()`                 | true     |
@@ -390,30 +392,8 @@ You can also specify the language, site, url, logo and LCN (Logical Channel Numb
 
 ## How to use SOCKS proxy?
 
-First, you need to install [socks-proxy-agent](https://www.npmjs.com/package/socks-proxy-agent):
-
-```sh
-npm install socks-proxy-agent
 ```
-
-Then you can use it to create an agent that acts as a SOCKS proxy. Here is an example of how to do it with the Tor SOCKS proxy:
-
-```js
-const { SocksProxyAgent } = require('socks-proxy-agent')
-
-const torProxyAgent = new SocksProxyAgent('socks://localhost:9050')
-
-module.exports = {
-  site: 'example.com',
-  url: 'https://example.com/epg.json',
-  request: {
-    httpsAgent: torProxyAgent,
-    httpAgent: torProxyAgent
-  },
-  parser(context) {
-    // ...
-  }
-}
+epg-grabber --config=example.com.config.js --proxy=socks://localhost:9050
 ```
 
 ## Contribution

package/bin/epg-grabber.js

@@ -5,14 +5,15 @@ const program = new Command()
 const { merge } = require('lodash')
 const { gzip } = require('node-gzip')
 const file = require('../src/file')
-const { EPGGrabber, parseChannels, generateXMLTV } = require('../src/index')
+const { EPGGrabber, EPGGrabberMock, parseChannels, generateXMLTV } = require('../src/index')
 const { create: createLogger } = require('../src/logger')
-const { parseNumber, getUTCDate } = require('../src/utils')
+const { parseNumber, getUTCDate, parseProxy } = require('../src/utils')
 const { name, version, description } = require('../package.json')
 const _ = require('lodash')
 const dayjs = require('dayjs')
 const utc = require('dayjs/plugin/utc')
 const { TaskQueue } = require('cwait')
+const { SocksProxyAgent } = require('socks-proxy-agent')
 
 dayjs.extend(utc)
 
@@ -22,6 +23,7 @@ program
   .description(description)
   .requiredOption('-c, --config <config>', 'Path to [site].config.js file')
   .option('-o, --output <output>', 'Path to output file')
+  .option('-x, --proxy <url>', 'Use the specified proxy')
   .option('--channels <channels>', 'Path to list of channels')
   .option('--lang <lang>', 'Set default language for all programs')
   .option('--days <days>', 'Number of days for which to grab the program', parseNumber)
@@ -63,10 +65,26 @@ async function main() {
     request: {}
   })
 
-  if (options.timeout) config.request.timeout = options.timeout
-  if (options.cacheTtl) config.request.cache.ttl = options.cacheTtl
-
-  if (options.channels) config.channels = options.channels
+  if (options.timeout !== undefined) config.request.timeout = options.timeout
+  if (options.cacheTtl !== undefined) config.request.cache.ttl = options.cacheTtl
+  if (options.channels !== undefined) config.channels = options.channels
+  if (options.proxy !== undefined) {
+    const proxy = parseProxy(options.proxy)
+
+    if (
+      proxy.protocol &&
+      ['socks', 'socks5', 'socks5h', 'socks4', 'socks4a'].includes(String(proxy.protocol))
+    ) {
+      const socksProxyAgent = new SocksProxyAgent(options.proxy)
+
+      config.request = {
+        ...config.request,
+        ...{ httpAgent: socksProxyAgent, httpsAgent: socksProxyAgent }
+      }
+    } else {
+      config.request = { ...config.request, ...{ proxy } }
+    }
+  }
 
   let parsedChannels = []
   if (config.channels) {
@@ -89,9 +107,10 @@ async function main() {
     }
   } else throw new Error('Path to "channels" is missing')
 
-  const grabber = new EPGGrabber(config)
+  const grabber =
+    process.env.NODE_ENV === 'test' ? new EPGGrabberMock(config) : new EPGGrabber(config)
 
-  let template = options.output || config.output
+  let template = options.output || config.output || 'guide.xml'
   const variables = file.templateVariables(template)
 
   const groups = _.groupBy(parsedChannels, channel => {
@@ -133,9 +152,11 @@ async function main() {
           await grabber
             .grab(channel, date, (data, err) => {
               logger.info(
-                `[${i}/${total}] ${config.site} - ${data.channel.xmltv_id} - ${dayjs
-                  .utc(data.date)
-                  .format('MMM D, YYYY')} (${data.programs.length} programs)`
+                `[${i}/${total}] ${config.site} - ${
+                  data.channel.xmltv_id || data.channel.site_id
+                } - ${dayjs.utc(data.date).format('MMM D, YYYY')} (${
+                  data.programs.length
+                } programs)`
               )
 
               if (err) logger.error(err.message)

package/package.json

@@ -1,9 +1,10 @@
 {
   "name": "epg-grabber",
-  "version": "0.37.5",
+  "version": "0.39.0",
   "description": "Node.js CLI tool for grabbing EPG from different sites",
   "main": "src/index.js",
   "preferGlobal": true,
+  "homepage": "https://github.com/freearhey/epg-grabber",
   "bin": {
     "epg-grabber": "bin/epg-grabber.js"
   },
@@ -41,6 +42,7 @@
     "http-cookie-agent": "^6.0.8",
     "lodash": "^4.17.21",
     "node-gzip": "^1.1.2",
+    "socks-proxy-agent": "^8.0.5",
     "tough-cookie": "^5.0.0",
     "winston": "^3.3.3",
     "xml-js": "^1.6.11"

package/src/client.js

@@ -1,11 +1,7 @@
 const { CurlGenerator } = require('curl-generator')
 const { default: axios } = require('axios')
-const { CookieJar } = require('tough-cookie')
 const { setupCache } = require('axios-cache-interceptor')
 const { isObject, isPromise } = require('./utils')
-const { HttpCookieAgent, HttpsCookieAgent } = require('http-cookie-agent/http')
-
-const jar = new CookieJar()
 
 module.exports.create = create
 module.exports.buildRequest = buildRequest
@@ -16,17 +12,10 @@ let timeout
 function create(config) {
   const client = axios.defaults.cache
     ? axios
-    : setupCookie(
-        setupCache(
-          axios.create({
-            jar,
-            ignoreCookieErrors: true,
-            headers: {
-              'User-Agent':
-                'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36 Edg/79.0.309.71'
-            }
-          })
-        )
+    : setupCache(
+        axios.create({
+          ignoreCookieErrors: true
+        })
       )
 
   client.interceptors.request.use(
@@ -142,44 +131,3 @@ async function getRequestUrl({ channel, date, config }) {
   }
   return config.url
 }
-
-const AGENT_CREATED_BY_AXIOS_COOKIEJAR_SUPPORT = Symbol('AGENT_CREATED_BY_AXIOS_COOKIEJAR_SUPPORT')
-
-function requestInterceptor(config) {
-  if (!config.jar) {
-    return config
-  }
-
-  config.httpAgent = new HttpCookieAgent({ cookies: { jar: config.jar } })
-  Object.defineProperty(config.httpAgent, AGENT_CREATED_BY_AXIOS_COOKIEJAR_SUPPORT, {
-    configurable: false,
-    enumerable: false,
-    value: true,
-    writable: false
-  })
-
-  config.httpsAgent = new HttpsCookieAgent({ cookies: { jar: config.jar } })
-  Object.defineProperty(config.httpsAgent, AGENT_CREATED_BY_AXIOS_COOKIEJAR_SUPPORT, {
-    configurable: false,
-    enumerable: false,
-    value: true,
-    writable: false
-  })
-
-  return config
-}
-
-function setupCookie(axios) {
-  axios.interceptors.request.use(requestInterceptor)
-
-  if ('create' in axios) {
-    const create = axios.create
-    axios.create = (...args) => {
-      const instance = create.apply(axios, args)
-      instance.interceptors.request.use(requestInterceptor)
-      return instance
-    }
-  }
-
-  return axios
-}

package/src/config.js

@@ -1,5 +1,5 @@
-const tough = require('tough-cookie')
 const { merge } = require('lodash')
+const { version, homepage } = require('../package.json')
 
 module.exports.parse = parse
 
@@ -24,9 +24,11 @@ function parse(config) {
       maxContentLength: 5 * 1024 * 1024,
       timeout: 5000,
       withCredentials: true,
-      jar: new tough.CookieJar(),
       responseType: 'arraybuffer',
-      cache: false
+      cache: false,
+      headers: {
+        'User-Agent': `EPGGrabber/${version} (${homepage})`
+      }
     }
   }
 

package/src/index.d.ts

@@ -80,6 +80,7 @@ export declare type SiteConfig = {
   url: () => string | string | Promise<string>
   logo?: () => string | string | Promise<string>
   parser: () => object[] | Promise<object[]>
+  curl?: boolean
 }
 
 export declare function generateXMLTV({

package/src/index.js

@@ -64,6 +64,14 @@ class EPGGrabberMock {
     this.config = config
   }
 
+  async loadLogo(channel) {
+    const logo = this.config.logo({ channel })
+    if (isPromise(logo)) {
+      return await logo
+    }
+    return logo
+  }
+
   async grab(channel, date, config = {}, cb = () => {}) {
     let _date = getUTCDate(date)
     if (typeof config == 'function') {
@@ -72,6 +80,9 @@ class EPGGrabberMock {
     }
     config = merge(this.config, config)
     config = parseConfig(config)
+
+    await buildRequest({ channel, date, config })
+
     let _programs = await config.parser({ channel, date: _date })
     let programs = _programs.map(data => new Program(data, channel))
 

package/src/utils.js

@@ -1,3 +1,4 @@
+const { URL } = require('node:url')
 const dayjs = require('dayjs')
 const utc = require('dayjs/plugin/utc')
 
@@ -13,6 +14,21 @@ module.exports.formatDate = formatDate
 module.exports.toArray = toArray
 module.exports.toUnix = toUnix
 module.exports.isDate = isDate
+module.exports.parseProxy = parseProxy
+
+function parseProxy(_url) {
+  const parsed = new URL(_url)
+
+  return {
+    protocol: parsed.protocol.replace(':', '') || null,
+    auth: {
+      username: parsed.username || null,
+      password: parsed.password || null
+    },
+    host: parsed.hostname,
+    port: parsed.port ? parseInt(parsed.port) : null
+  }
+}
 
 function sleep(ms) {
   return new Promise(resolve => setTimeout(resolve, ms))

package/tests/__data__/output/guide.xml

@@ -1,6 +1,10 @@
-<?xml version="1.0" encoding="UTF-8" ?><tv date="20250124">
+<?xml version="1.0" encoding="UTF-8" ?><tv date="20250715">
 <channel id="1TV.com"><display-name>1 TV</display-name><icon src="https://example.com/logos/1TV.png"/><url>https://example.com</url></channel>
 <channel id="2TV.com"><display-name>2 TV</display-name><url>https://example.com</url></channel>
 <channel id="3TV.com"><display-name>3 TV</display-name><url>https://example2.com</url></channel>
 <channel id="4TV.com"><display-name>4 TV</display-name><url>https://example2.com</url></channel>
+<programme start="20220101000000 +0000" stop="20220101010000 +0000" channel="1TV.com"><title lang="fr">Program1</title></programme>
+<programme start="20220101000000 +0000" stop="20220101010000 +0000" channel="2TV.com"><title>Program1</title></programme>
+<programme start="20220101000000 +0000" stop="20220101010000 +0000" channel="4TV.com"><title>Program1</title></programme>
+<programme start="20220101000000 +0000" stop="20220101010000 +0000" channel="3TV.com"><title>Program1</title></programme>
 </tv>

package/tests/bin.test.js

@@ -36,6 +36,7 @@ it('can load mini config', () => {
       --days=3 \
       --delay=0 \
       --debug \
+      --proxy=socks://127.0.0.1:1086 \
       --timeout=1`,
     {
       encoding: 'utf8'

package/tests/utils.test.js

@@ -1,4 +1,4 @@
-import { escapeString } from '../src/utils'
+import { escapeString, parseProxy } from '../src/utils'
 
 it('can escape string', () => {
   const string = 'Música тест dun.  &<>"\'\r\n'
@@ -9,3 +9,10 @@ it('can escape url', () => {
   const string = 'http://example.com/logos/1TV.png?param1=val&param2=val'
   expect(escapeString(string)).toBe('http://example.com/logos/1TV.png?param1=val&amp;param2=val')
 })
+
+it('can parse proxy', () => {
+  const string = 'socks://127.0.0.1:1234'
+  expect(parseProxy(string)).toMatchObject({
+    protocol: 'socks'
+  })
+})