fakefilter 1.1.13 → 1.1.1582

package/events.js

@@ -1,59 +0,0 @@
-
-const debug = require('debug')('fetch')
-const winston = require('winston')
-const { getConfig, Sentry } = require('./config')
-const { promiseTimeout, getDomainPart,domainFromEmail, eventsOfProvider } = require('./shared')
-const Table = require('easy-table')
-const chalk = require('chalk')
-const vars = require('./vars')
-const argv = require('minimist')(process.argv.slice(2))
-const { parseDomain, ParseResultType } = require("parse-domain")
-const { CoadminService } = require('coadmin-lib')
-const coadmin_service = new CoadminService('fakefilter/fetch')
-const MAX_RUN_HOURS = 2
-
-
-async function start() {
-    try {
-       
-        let config = await getConfig()
-        let selectedProvider = false
-        if (argv._.length > 0) 
-            selectedProvider=argv._[0]
-        if (!selectedProvider) {
-            console.log(chalk.blue('Usage:'))
-            console.log(`node events.js <provider>`)
-            process.exit(0)
-        }
-
-        let events = await eventsOfProvider(selectedProvider,7)
-        if (events) {
-            console.log(chalk.yellow(`${events.length} events found for ${selectedProvider}`))
-            var t = new Table()
-            let stats = {
-                success:0,
-                other:0,
-                failed:0
-            }
-            for(let event of events) {
-                // console.log(event)
-                t.cell("t",new Date(event.t).toISOString())
-                t.cell("event",event.event)
-                t.cell("result",event.result==='success' ? '✅ success' : '❌ '+event.result)
-                t.newRow()
-                if (event.result==='success') stats.success++;  else
-                if (event.result==='failed') stats.failed++; 
-                else  stats.other++
-            }
-            console.log(t.toString())
-            console.log(`Total success: ${chalk.green(stats.success)} failed: ${chalk.red(stats.failed)} other: ${stats.other}`)
-            console.log(`This is only from last 7 days`)
-        }
-        process.exit(0)
-    } catch (err) {
-        console.log(err)
-        process.exit(0)
-    }
-}
-
-start()

package/fetch.js

@@ -1,180 +0,0 @@
-const { tryLock } = require('tcp-mutex')
-const debug = require('debug')('fetch')
-const winston = require('winston')
-const { getConfig } = require('./config')
-const { promiseTimeout, getDomainPart,domainFromEmail, eventsOfProvider } = require('./shared')
-const chalk = require('chalk')
-const vars = require('./vars')
-const argv = require('minimist')(process.argv.slice(2))
-const { parseDomain, ParseResultType } = require("parse-domain")
-const { CoadminService } = require('coadmin-lib')
-const coadmin_service = new CoadminService('fakefilter/fetch')
-const MAX_RUN_HOURS = 2
-
-let succeed = 0
-let failed = []
-
-
-const logger = winston.createLogger({
-    level: 'info',
-    format: winston.format.combine(
-        winston.format.timestamp(),
-        winston.format.json(),  
-    ),
-    defaultMeta: { script: 'fetch.js' },
-
-    transports: [
-        //
-        // - Write all logs with importance level of `error` or less to `error.log`
-        // - Write all logs with importance level of `info` or less to `combined.log`
-        //
-        new winston.transports.File({ filename: 'logs/error.log', level: 'error' }),
-        new winston.transports.File({ filename: 'logs/combined.log' }),
-        // log to Console too
-        //   new winston.transports.Console()
-    ],
-})
-
-// load all providers/*.js
-var glob = require('glob'), path = require('path')
-const { array_shuffle } = require('mybase')
-glob.sync('./providers/*.js').forEach(function (file) {
-    require(path.resolve(file));
-})
-
-// shuffle providers to randomize
-vars.providers = array_shuffle(vars.providers)
-
-async function processProvider(p, headless = true) {
-    return new Promise(async function (resolve, reject) {
-        logger.info(`Processing ${p.name}`)
-        debug(`Processing ${p.name}`)
-
-        if (p.hasOwnProperty('min_hours') && parseInt(p['min_hours'])>0) {
-            let min_hours = parseInt(p['min_hours'])
-            // if min_hours specified we rate limit the provider
-            let events = await eventsOfProvider(p.name,1)
-            if (events && events.length>0) {
-                let last_event = events[events.length-1]
-                let since_last_event = Date.now()-last_event.t
-                let since_last_event_hours = since_last_event/(60*60*1000)
-                if (since_last_event_hours<min_hours) {
-                    // need to skip processing this provider
-                    console.log(chalk.yellow(`Provider ${p.name} skipped due to min_hours(${min_hours} hours) rule`))
-                    return resolve(true)
-                } 
-                console.log(last_event,since_last_event_hours)
-            }
-        }
-
-        // console.log(chalk.yellow(p.name))
-        let initResult = await p.init()
-        debug(`Init done, result = ${initResult ? true : false}`)
-        // console.log(`init=`,initResult)
-        if (initResult === true) {
-            debug(`Calling start`)
-            let res = await p.start(headless)
-            debug(`Start result`, res)
-            if (res && res.hasOwnProperty('email')) {
-                await vars.models.Events.create({
-                    provider: p.name,
-                    t: Date.now(),
-                    event: 'fetch',
-                    result: 'success'
-                })
-                // console.log(res)
-                succeed++
-                // success
-                let emails = []
-                if (typeof res.email === 'string') emails.push(res.email);
-                else emails = res.email
-                for (let e of emails) {
-                    let thedomain = p.randomSubdomain ? getDomainPart(domainFromEmail(e)) : domainFromEmail(e)
-                    if (vars.config.ignore_domains.includes(thedomain)) {
-                        console.log(chalk.red(`Ignored domain: ${thedomain} ignored`))
-                        continue
-                    }
-                    console.log(chalk.green(`Provider ${p.name} - ${thedomain}`))
-
-                    logger.info(`Provider ${p.name} - domain: ${thedomain}`)
-                    await vars.models.Harvested.create({
-                        provider: p.name,
-                        t: Date.now(),
-                        domain: thedomain
-                    })
-                }
-                return resolve(true)
-            }
-            await vars.models.Events.create({
-                provider: p.name,
-                t:Date.now(),
-                event:'fetch',
-                result:'failed'
-            })
-            failed.push(p.name)
-            console.log(chalk.red(`Provider ${p.name} - failed`))
-        }
-        logger.error(`Provider ${p.name} - failed - initResult:${initResult ? 'true' : 'false'}`)
-        resolve(false)
-    })
-}
-
-async function start() {
-    try {
-        // make sure we never stuck
-        setTimeout(() => {
-            logger.error('MAX RUNTIME reached')
-            process.exit(0)
-        }, MAX_RUN_HOURS * 60 * 60 * 1000)
-
-        let config = await getConfig()
-        
-        
-        let selectedProvider = []
-        if (argv._.length > 0) {
-            selectedProvider.push(...argv._)
-            console.log(`Selected providers:`, selectedProvider.join(', '))
-        } else await tryLock(config.lockPort)
-
-        coadmin_service.report('started')
-
-        for (var p of vars.providers) {
-            if (selectedProvider.length > 0 && !selectedProvider.includes(p.name)) continue
-            if (p.disabled) {
-                console.log(chalk.yellow(`▪️ Provider ${p.name} is disabled, skipping`))
-                continue
-            }
-            try {
-                await promiseTimeout(120000, processProvider(p, argv.v ? false : true))
-                coadmin_service.report_every(1, 'fetching')
-            } catch (err) {
-                console.log(err)
-            }
-
-            // make sure to close the browser - attempt to protect the memory
-            if (vars.browsers.length > 0)
-                try {
-                    let browser = vars.browsers.shift()
-                    await browser.close()
-                    coadmin_service.report_every(1, 'closing browsers')
-                } catch (e) {
-                    console.log(e)
-                }
-        }
-
-        if (failed.length > 0)
-            console.log(chalk.bold(`Finished..`), chalk.green(`${succeed} succeed`), chalk.red(`${failed.length} failed`))
-        else
-            console.log(chalk.bold(`Finished..`), chalk.green(`${succeed} succeed`))
-        if (failed.length > 0)
-            console.log(chalk.red(`Failed providers:\n`), chalk.red(failed.join('\n')))
-
-        coadmin_service.report('stopped')
-        process.exit(0)
-    } catch (err) {
-        console.log(chalk.red(err))
-        process.exit(0)
-    }
-}
-
-start()

package/models/events.js

@@ -1,37 +0,0 @@
-const { Sequelize, DataTypes, Model } = require('sequelize');
-const vars = require('./../vars')
-
-class Events extends Model { }
-
-function initEvents() {
-    Events.init({
-        t: {
-            type: DataTypes.BIGINT,
-            allowNull: true
-        },
-        provider: {
-            type: DataTypes.STRING(255),
-            allowNull: false,
-            comment: 'service hostname, domainname'
-        },
-        event: {
-            type: DataTypes.STRING(32),
-            allowNull: false,
-        },
-        result: {
-            type: DataTypes.STRING(32),
-            allowNull: false,
-        },
-    }, {
-        // Other model options go here
-        sequelize:vars.sequelize, // We need to pass the connection instance
-        modelName: 'Events',// We need to choose the model name
-        tableName: 'events'
-    })
-}
-
-
-module.exports = {
-    Events,
-    initEvents
-}

package/models/harvested.js

@@ -1,45 +0,0 @@
-const { Sequelize, DataTypes, Model } = require('sequelize');
-const vars = require('./../vars')
-
-class Harvested extends Model { }
-
-function initHarvested() {
-    Harvested.init({
-        t: {
-            type: DataTypes.BIGINT,
-            allowNull: true
-        },
-        provider: {
-            type: DataTypes.STRING(255),
-            allowNull: false,
-            comment: 'service hostname, domainname'
-        },
-        domain: {
-            type: DataTypes.STRING(255),
-            allowNull: false,
-            comment: 'domainname'
-        },
-    }, {
-        // Other model options go here
-        sequelize:vars.sequelize, // We need to pass the connection instance
-        modelName: 'Harvested',// We need to choose the model name
-        tableName: 'harvested',
-        indexes:[
-            {
-                fields:['provider'],
-            },
-            {
-                fields:['domain']
-            },
-            {
-                fields:['t']
-            },
-        ]
-    })
-}
-
-
-module.exports = {
-    Harvested,
-    initHarvested
-}

package/models/harvested_clone.js

@@ -1,44 +0,0 @@
-const { Sequelize, DataTypes, Model } = require('sequelize');
-const vars = require('./../vars')
-
-class HarvestedClone extends Model { }
-
-function initHarvestedClone() {
-    HarvestedClone.init({
-        t: {
-            type: DataTypes.BIGINT,
-            allowNull: true
-        },
-        provider: {
-            type: DataTypes.STRING(255),
-            allowNull: false,
-            comment: 'service hostname, domainname'
-        },
-        domain: {
-            type: DataTypes.STRING(255),
-            allowNull: false,
-            comment: 'domainname'
-        },
-    }, {
-        // Other model options go here
-        sequelize:vars.sequelize, // We need to pass the connection instance
-        modelName: 'HarvestedClone',// We need to choose the model name
-        tableName: 'harvested_clone',
-        indexes:[
-            {
-                fields:['provider'],
-            },
-            {
-                fields:['domain']
-            },
-            {
-                fields:['t']
-            },
-        ]
-    })
-}
-
-module.exports = {
-    HarvestedClone,
-    initHarvestedClone
-}

package/providers/10-minute-mail.com.js

@@ -1,39 +0,0 @@
-const { validEmail } = require('mybase')
-const { Sentry } = require('../config')
-const { initiateEmpty, launch, emailBy_id_innerText, emailBy_querySelector_innerHTML, emailBy_id_value, emailBy_querySelector_value } = require('../shared')
-const vars = require('./../vars')
-const URL = 'https://10-minute-mail.com'
-
-function start(headless=true,waitTime=3) {
-  return new Promise(async function (resolve,reject) {
-    try {
-      let {page,browser} = await launch(headless,URL,waitTime,'#openRandomButton')
-      let email = await emailBy_querySelector_innerHTML(page,'strong')
-    
-      if (browser) await browser.close()
-      if (email) return resolve({
-        email,
-        URL,
-        t:Date.now()
-      })
-      return resolve(null)
-    } catch (err) {
-      console.log(err)
-      Sentry.captureException(err)
-      resolve(false)
-    }
-  })
-}
-
-// for cli usage - development
-if (require.main==module) {
-  start(false).then(res=>console.log(res))
-}
-
-// pending
-vars.providers.push({
-  name:'10-minute-mail.com',
-  init:initiateEmpty,
-  start:start,
-  disabled:true
-})

package/providers/10-minuten-mail.de.js

@@ -1,39 +0,0 @@
-const { validEmail } = require('mybase')
-const { Sentry } = require('../config')
-const { initiateEmpty, launch, emailBy_id_innerText, emailBy_querySelector_innerHTML, emailBy_id_value, emailBy_querySelector_value } = require('../shared')
-const vars = require('./../vars')
-const URL = 'https://10-minuten-mail.de'
-
-function start(headless=true,waitTime=3) {
-  return new Promise(async function (resolve,reject) {
-    try {
-      let {page,browser} = await launch(headless,URL,waitTime,'#openRandomButton')
-      let email = await emailBy_querySelector_innerHTML(page,'strong')
-    
-      if (browser) await browser.close()
-      if (email) return resolve({
-        email,
-        URL,
-        t:Date.now()
-      })
-      return resolve(null)
-    } catch (err) {
-      console.log(err)
-      Sentry.captureException(err)
-      resolve(false)
-    }
-  })
-}
-
-// for cli usage - development
-if (require.main==module) {
-  start(false).then(res=>console.log(res))
-}
-
-// pending
-vars.providers.push({
-  name:'10-minuten-mail.de',
-  init:initiateEmpty,
-  start:start,
-  disabled:true
-})

package/providers/10mails.net.js

@@ -1,41 +0,0 @@
-const { validEmail, wait } = require('mybase')
-const { Sentry } = require('../config')
-const { initiateEmpty, launch, emailBy_id_innerText, emailBy_querySelector_innerHTML, emailBy_id_value, emailFrom_combobox_id, emailBy_querySelector_value, emailFrom_combobox_selector } = require('../shared')
-const vars = require('./../vars')
-const URL = 'http://10mails.net'
-
-function start(headless=true,waitTime=3) {
-  return new Promise(async function (resolve,reject) {
-    try {
-      let {page,browser} = await launch(headless,URL,waitTime)
-      
-      
-      let email = await emailBy_querySelector_innerHTML(page,'span#tma')
-      
-      if (browser) await browser.close()
-      if (email) return resolve({
-        email,
-        URL,
-        t:Date.now()
-      })
-      return resolve(null)
-    } catch (err) {
-      console.log(err)
-      Sentry.captureException(err)
-      resolve(false)
-    }
-  })
-}
-
-// for cli usage - development
-if (require.main==module) {
-  start(false).then(res=>console.log(res))
-}
-
-
-vars.providers.push({
-  name:'10mails.net',
-  init:initiateEmpty,
-  start:start,
-  disabled:true
-})

package/providers/10minut.com.pl.js

@@ -1,39 +0,0 @@
-const { validEmail, wait } = require('mybase')
-const { Sentry } = require('../config')
-const { initiateEmpty, launch, emailBy_id_innerText, emailBy_querySelector_innerHTML, emailBy_id_value, emailFrom_combobox_id, emailBy_querySelector_value, emailFrom_combobox_selector, emailBy_querySelector_innerText, emailBy_querySelector_placeholder } = require('../shared')
-const vars = require('./../vars')
-const URL = 'https://10minut.com.pl'
-
-function start(headless=true,waitTime=3) {
-  return new Promise(async function (resolve,reject) {
-    try {
-      let {page,browser} = await launch(headless,URL,waitTime)
-      
-      let email = await emailBy_querySelector_value(page,'input[type="text"]')
-      if (browser) await browser.close()
-      if (email) return resolve({
-        email,
-        URL,
-        t:Date.now()
-      })
-      return resolve(null)
-    } catch (err) {
-      console.log(err)
-      Sentry.captureException(err)
-      resolve(false)
-    }
-  })
-}
-
-// for cli usage - development
-if (require.main==module) {
-  start(false).then(res=>console.log(res))
-}
-
-
-vars.providers.push({
-  name:'10minut.com.pl',
-  init:initiateEmpty,
-  start:start,
-  disabled:true
-})

package/providers/10minut.xyz.js

@@ -1,40 +0,0 @@
-const { validEmail, wait } = require('mybase')
-const { Sentry } = require('../config')
-const { initiateEmpty, launch, emailBy_id_innerText, emailBy_querySelector_innerHTML, emailBy_id_value, emailFrom_combobox_id, emailBy_querySelector_value, emailFrom_combobox_selector, emailBy_querySelector_innerText, email_from_raw } = require('../shared')
-const vars = require('./../vars')
-const URL = 'https://10minut.xyz'
-
-function start(headless=true,waitTime=3) {
-  return new Promise(async function (resolve,reject) {
-    try {
-      let {page,browser} = await launch(headless,URL,waitTime)
-      
-      let raw = await emailBy_querySelector_innerText(page,'span.input-group-text',true)
-      let email = email_from_raw(raw)
-      // console.log(email)
-      if (browser) await browser.close()
-      if (email.length>0) return resolve({
-        email,
-        URL,
-        t:Date.now()
-      })
-      return resolve(null)
-    } catch (err) {
-      console.log(err)
-      Sentry.captureException(err)
-      resolve(false)
-    }
-  })
-}
-
-// for cli usage - development
-if (require.main==module) {
-  start(false).then(res=>console.log(res))
-}
-
-
-vars.providers.push({
-  name:'10minut.xyz',
-  init:initiateEmpty,
-  start:start
-})

package/providers/10minuteemails.com.js

@@ -1,39 +0,0 @@
-const { validEmail } = require('mybase')
-const { Sentry } = require('../config')
-const { initiateEmpty, launch, emailBy_id_innerText, emailBy_querySelector_innerHTML, emailBy_id_value, emailBy_querySelector_value } = require('../shared')
-const vars = require('./../vars')
-const URL = 'https://10minuteemails.com'
-
-function start(headless=true,waitTime=3) {
-  return new Promise(async function (resolve,reject) {
-    try {
-      let {page,browser} = await launch(headless,URL,waitTime,'input[value="Create a Random Email"]')
-      // document.querySelectorAll('form.flex input')[0].click()
-      let email = await emailBy_querySelector_innerHTML(page,'div.block')
-      
-      if (browser) await browser.close()
-      if (email) return resolve({
-        email,
-        URL,
-        t:Date.now()
-      })
-      return resolve(null)
-    } catch (err) {
-      console.log(err)
-      Sentry.captureException(err)
-      resolve(false)
-    }
-  })
-}
-
-// for cli usage - development
-if (require.main==module) {
-  start(false).then(res=>console.log(res))
-}
-
-// disabled in favor of extention
-// vars.providers.push({
-//   name:'10minuteemails.com',
-//   init:initiateEmpty,
-//   start:start
-// })

package/providers/10minutemail.co.za.js

@@ -1,39 +0,0 @@
-const { validEmail } = require('mybase')
-const { Sentry } = require('../config')
-const { initiateEmpty, launch, emailBy_id_innerText, emailBy_querySelector_innerHTML, emailBy_id_value, emailBy_querySelector_value } = require('../shared')
-const vars = require('./../vars')
-const URL = 'https://10minutemail.co.za'
-
-function start(headless=true,waitTime=3) {
-  return new Promise(async function (resolve,reject) {
-    try {
-      let {page,browser} = await launch(headless,URL,waitTime,'input[name="getemail"]')
-      let email = await emailBy_querySelector_value(page,'input[name="mail"]')
-    
-      if (browser) await browser.close()
-      if (email) return resolve({
-        email,
-        URL,
-        t:Date.now()
-      })
-      return resolve(null)
-    } catch (err) {
-      console.log(err)
-      Sentry.captureException(err)
-      resolve(false)
-    }
-  })
-}
-
-// for cli usage - development
-if (require.main==module) {
-  start(false).then(res=>console.log(res))
-}
-
-// pending
-vars.providers.push({
-  name:'10minutemail.co.za',
-  init:initiateEmpty,
-  start:start,
-  disabled:true
-})