fakefilter 0.1.656 → 1.1.10

package/chart-update.js

@@ -0,0 +1,179 @@
+const path = require('path')
+const fs = require('fs')
+var Table = require('easy-table')
+const winston = require('winston')
+const { getConfig, Sentry } = require('./config')
+const chalk = require('chalk')
+const vars = require('./vars')
+const { softexit, wait } = require('mybase')
+const { pullDomains, pullEventsProviders, pullEventsByProvider } = require('./shared')
+const QuickChart = require('quickchart-js');
+const argv = require('minimist')(process.argv.slice(2))
+const md5 = require('md5')
+
+function writeToFile(fn,content) {
+    // let fn = 'repo/markdown/README.md'
+    fs.writeFileSync(fn,content)
+    console.log(chalk.green(`+ file ${fn}`))
+    return true
+}
+
+function dailyEventGraphic(providerName,events) {
+    return new Promise(async function (resolve,reject) {
+        const myChart = new QuickChart()
+        let data={}
+        let fn = `success-rate-${md5(providerName)}.png`
+        let successrate={}
+        let labels=[]
+        let chartFile = path.join(__dirname,`/apiserver/public/img/dynamic/${fn}`)
+        if (fs.existsSync(chartFile)) {
+            // lets look if the file was recently rendered
+            let stats = fs.statSync(chartFile)
+            let mtimeMinutes = (new Date()-stats.mtime)/1000/60 // modified time in minutes
+            if (mtimeMinutes<200) {
+                console.log(`Chart for ${providerName} - taken from cache ${chartFile}`)
+                return resolve(fn)
+            }
+        }
+        await wait(2) // rate limit
+
+        for(let row of events) {
+            let dayString = new Date(row.t).toISOString().substring(0,10)
+            if (!data.hasOwnProperty(dayString))
+                {
+                    data[dayString]={success:0,failed:0}
+                    successrate[dayString]=0
+                    labels.push(dayString)
+                }
+            if (row.result==='success') data[dayString].success++; else
+                data[dayString].failed++;
+        }
+        for(let date of Object.keys(data)) {
+            successrate[date]=((data[date].success/(data[date].failed+data[date].success)))*100
+        }
+
+        let chartConfig1 = {
+            type: 'bar',
+            data: { labels, 
+                datasets: [{ label: `Success Rate - ${providerName} - %`, 
+                backgroundColor: QuickChart.getGradientFillHelper('vertical', ['green', 'red']),
+                    data: Object.keys(successrate).map(dayString=>successrate[dayString]) }
+                ] },
+        }
+        myChart.setConfig(chartConfig1)
+        
+        
+        await myChart.toFile(chartFile)
+        console.log(`Creating success-rate chart for ${providerName}`)
+        return resolve(fn)
+
+        // calcualte success rate
+        
+    })
+}
+function dailyCoverageGraph(rows) {
+    return new Promise(async function (resolve,reject) {
+        const myChart = new QuickChart();
+        const myChart2 = new QuickChart();
+        let data={}
+        let labels=[]
+        let uniqueDomains=[]
+        let dataUniqueDomains={}
+        // let cnt = 10
+        for(let row of rows) {
+            let dayString = new Date(row.t).toISOString().substring(0,10)
+            if (!uniqueDomains.includes(row.domain)) uniqueDomains.push(row.domain)
+            if (!data.hasOwnProperty(dayString))
+                {
+                    data[dayString]=[]
+                    dataUniqueDomains[dayString]=uniqueDomains.length
+                    labels.push(dayString)
+                }
+            
+            if (!data[dayString].includes(row.domain)) data[dayString].push(row.domain)
+            // console.log(row)
+            // if (cnt<10) break
+            // cnt--
+        }
+        // console.log(dataUniqueDomains)
+        labels = labels.reverse()
+        let chartConfig2 = {
+            type: 'bar',
+            data: { labels, 
+                datasets: [{ label: 'Unique Domains', data: Object.keys(dataUniqueDomains).map(dayString=>dataUniqueDomains[dayString]) }] },
+        }
+
+        let chartConfig1 = {
+            type: 'bar',
+            data: { labels, 
+                datasets: [{ label: 'Daily Coverage', data: Object.keys(data).map(dayString=>data[dayString].length) }] },
+        }
+        
+        // console.log(chartConfig1)
+        myChart.setConfig(chartConfig1)
+        myChart2.setConfig(chartConfig2)
+        await myChart.toFile(path.join(__dirname,'/apiserver/public/img/dynamic/chart1.png'))
+        
+        await myChart2.toFile(path.join(__dirname,'/apiserver/public/img/dynamic/chart2.png'))
+        
+        return resolve()
+    })
+}
+
+async function start() {
+    try {
+        await getConfig()
+        let events_providers = await pullEventsProviders(false,60)
+        let markdown3=``
+        for(let provider of events_providers) {
+            // break
+            // if (provider==='temporarymail.com') 
+            {
+                try {
+                    let r = await pullEventsByProvider(provider,60)
+                    let chartfn = await dailyEventGraphic(provider,r)
+                    
+                    markdown3+=`### ${provider}
+![Success Rate for ${provider}](https://fakefilter.net/public/img/dynamic/${chartfn})
+
+`
+                }catch(err) {
+                    Sentry.captureException(err)
+                    console.log(err)
+                }
+            }
+        }
+
+        let domain_rows = await pullDomains(200,false)
+        let urls = await dailyCoverageGraph(domain_rows)  
+        
+let markdown1=`
+# Charts
+## All Providers
+### Daily Coverage
+Amount of coverage, actions against all known Disposable Email Providers per Day
+![Daily Coverage](https://fakefilter.net/public/img/dynamic/chart1.png)
+
+### Daily Unique Disposable Domains
+Number of unique known validated Disposable Domains in Total at given Day
+![Daily Disposable Domains](https://fakefilter.net/public/img/dynamic/chart1.png)
+
+## Success Rate
+Per Provider Success Rate per Day - A lower Success rate might mean different things; counter measures from provider to be detected like capchas or even service downtime.
+
+${markdown3}
+`
+        writeToFile(path.join(__dirname, 'repo/CHARTS.md'),markdown1)
+
+        console.log(urls)
+        process.exit(0)
+
+    }
+    catch(err) {
+        console.log(err)
+        Sentry.captureException(err)
+        softexit("",1)
+    }
+}
+
+start()

package/cleanup.js

@@ -0,0 +1,58 @@
+const { Op } = require("sequelize")
+const path = require('path')
+const fs = require('fs')
+const winston = require('winston')
+const { getConfig, Sentry } = require('./config')
+const chalk = require('chalk')
+const vars = require('./vars')
+const { softexit, wait } = require('mybase')
+const { pullDomains, pullEventsProviders, pullEventsByProvider } = require('./shared')
+const QuickChart = require('quickchart-js');
+const argv = require('minimist')(process.argv.slice(2))
+
+/*
+    we need to vacuum Harvested table to avoid excessive load times of domains
+*/
+
+async function start() {
+    try {
+        await getConfig()
+        let rows = await pullDomains(10000)
+        // const progress = require('progressbar').create().step('combining')
+        // progress.setTotal(rows.length)
+        let memory = {}
+        let keep_provider_domains_per_day = 1 // never put this to 0 or lower!!! never
+        let deleted = 0
+        if (keep_provider_domains_per_day > 0)
+            for (let row of rows) {
+                // await wait(1 / 1000)
+                // progress.addTick()
+                // console.log(row,provider)
+                let day = new Date(row.t).toISOString().substring(0, 10)
+                let provider = row.provider.toLowerCase()
+                let domain = row.domain.toLowerCase()
+
+                if (!memory.hasOwnProperty(day)) memory[day] = {}
+                if (!memory[day].hasOwnProperty(provider)) memory[day][provider] = {}
+                if (!memory[day][provider].hasOwnProperty(domain)) memory[day][provider][domain] = 0
+                memory[day][provider][domain]++
+                if (memory[day][provider][domain] > keep_provider_domains_per_day) {
+                    console.log(`day:${day} domain:${chalk.bold(domain)} provider:${chalk.yellow(provider)} found ${chalk.blue(memory[day][provider][domain])} times`)
+                    await vars.models.Harvested.destroy({ where: { id: row.id } })
+                    deleted++
+                }
+            }
+        console.log(`Deleted ${deleted} rows from Harvested to vacuum`)
+        process.exit(0)
+
+
+    }
+    catch (err) {
+        console.log(err)
+        Sentry.captureException(err)
+        await wait(5)
+        process.exit(0)
+    }
+}
+
+start()

package/config.js

@@ -0,0 +1,126 @@
+const fs = require('fs')
+const path = require('path')
+const argv = require('minimist')(process.argv.slice(2))
+const vars = require('./vars')
+const { Sequelize } = require('sequelize')
+var chalk = require('chalk')
+var debug = require('debug')('fakefilter')
+var { vaultFill,isLocal,canReadAndWrite } = require('mybase')
+const { Harvested,initHarvested } = require('./models/harvested')
+const { HarvestedClone,initHarvestedClone } = require('./models/harvested_clone')
+const { Events,initEvents } = require('./models/events')
+
+
+var vault = require("node-vault")({
+  apiVersion: 'v1', // default
+  endpoint: 'https://vault2.vpn1.com:8200',
+  token: 'd7130f39-8466-4444-b2b7-49c93da2c597'
+})
+
+var Sentry = require('@sentry/node');
+Sentry.init({
+    dsn:'https://1074c423d608484e951455dc3c0d69bb@sentry.def24.com/25',
+    captureUnhandledRejections: true,
+    name: require('os').hostname()
+})
+
+function syncModel(modelName) {
+    return new Promise(async function (resolve, reject) {
+        console.log(`Synching model: ${modelName.name}`)
+        try {
+            await modelName.sync({ alter: true })
+            return resolve(true)
+        } catch (err) {
+            console.log(err)
+            console.log(chalk.red(`Exception by synching model: ${modelName.name}`))
+            process.exit(0)
+        }
+    })
+
+}
+
+function getConfig()
+{
+    return new Promise(async function (resolve,reject) {
+        try {
+            
+            await vaultFill(vault,config)
+            await canReadAndWrite(config.cache_folder,true)
+
+            var sequelize = new Sequelize(
+                config.mysqlserver.db,
+                config.mysqlserver.login,
+                config.mysqlserver.password, {
+                        host: config.mysqlserver.host,
+                        dialect: 'mysql',
+                        logging: false
+                }
+            )
+            vars.sequelize = sequelize
+            await sequelize.authenticate()
+
+            // initialize
+            await initHarvested()
+            await initHarvestedClone()
+            await initEvents()
+
+
+
+            vars.models.Harvested = Harvested
+            vars.models.HarvestedClone = HarvestedClone
+            vars.models.Events = Events
+            vars.config = config
+
+            if (argv.sync) {
+                await syncModel(Harvested)
+                await syncModel(HarvestedClone)
+                await syncModel(Events)
+                process.exit(0)
+            }
+
+            resolve(config)
+        }
+        catch(err) {
+            reject(err)
+        }
+        
+    })
+}
+
+var config = {
+    lockPort: 9994,
+    captcha_resolver_api:'7a1f7a34d2c261393528e62088a75515',
+    mysqlserver:{
+        db:'fakefilter',
+        login:'fakefilter',
+        password:'Q3p5hfxyrZLs',
+        host:'mix.def24.com'
+    },
+    apiserver: {
+        ip: '127.0.0.1',
+        port: 5520
+    },
+    cache_folder:path.join(__dirname,"cache"),
+    main_folder:__dirname,
+    ignore_domains : ['nytimes.com','yahoo.gr','outlook.sa','live.nl','yahoo.it','comcast.net','mail.ua','yahoo.com.br','outlook.es','aliyun.com','cps.edu','sina.cn','yahoo.ca',
+    't-online.de','yahoo.co.jp','hotmail.es','yahoo.in','vip.qq.com','yandex.ua','139.com','hotmail.it','orange.fr','hotmail.de','yahoo.com.hk','gmx.at','live.co.uk','gmx.net','yahoo.co.in','yahoo.de','yahoo.es','me.com','o2.pl','gmx.fr','abv.bg','yahoo.co.id','wp.pl','i.ua','msn.com','live.fr','googlemail.com','gmx.com','yahoo.co.uk','yahoo.fr','naver.com','sina.com','yandex.com','gmx.de','aol.com','hotmail.co.uk','yahoo.com.tw','hotmail.fr','bk.ru','mail.com','web.de','outlook.fr','protonmail.com','ukr.net','live.com','icloud.com','126.com','yandex.ru','outlook.com','163.com','mail.ru','hotmail.com','yahoo.com','qq.com','gmail.com','zoho.com','cloudflare.com']
+    // vaultKey: 'vault@/secret/vault'
+}
+
+if (require.main===module) {
+    getConfig().then(c=>{
+        console.log('11')
+        if (!argv.sync) console.log(c)
+        console.log('22')
+        process.exit(0)
+    })
+}
+
+
+
+
+module.exports = {
+    Sentry,
+    getConfig,
+    config
+}

package/events.js

@@ -0,0 +1,59 @@
+
+const debug = require('debug')('fetch')
+const winston = require('winston')
+const { getConfig, Sentry } = require('./config')
+const { promiseTimeout, getDomainPart,domainFromEmail, eventsOfProvider } = require('./shared')
+const Table = require('easy-table')
+const chalk = require('chalk')
+const vars = require('./vars')
+const argv = require('minimist')(process.argv.slice(2))
+const { parseDomain, ParseResultType } = require("parse-domain")
+const { CoadminService } = require('coadmin-lib')
+const coadmin_service = new CoadminService('fakefilter/fetch')
+const MAX_RUN_HOURS = 2
+
+
+async function start() {
+    try {
+       
+        let config = await getConfig()
+        let selectedProvider = false
+        if (argv._.length > 0) 
+            selectedProvider=argv._[0]
+        if (!selectedProvider) {
+            console.log(chalk.blue('Usage:'))
+            console.log(`node events.js <provider>`)
+            process.exit(0)
+        }
+
+        let events = await eventsOfProvider(selectedProvider,7)
+        if (events) {
+            console.log(chalk.yellow(`${events.length} events found for ${selectedProvider}`))
+            var t = new Table()
+            let stats = {
+                success:0,
+                other:0,
+                failed:0
+            }
+            for(let event of events) {
+                // console.log(event)
+                t.cell("t",new Date(event.t).toISOString())
+                t.cell("event",event.event)
+                t.cell("result",event.result==='success' ? '✅ success' : '❌ '+event.result)
+                t.newRow()
+                if (event.result==='success') stats.success++;  else
+                if (event.result==='failed') stats.failed++; 
+                else  stats.other++
+            }
+            console.log(t.toString())
+            console.log(`Total success: ${chalk.green(stats.success)} failed: ${chalk.red(stats.failed)} other: ${stats.other}`)
+            console.log(`This is only from last 7 days`)
+        }
+        process.exit(0)
+    } catch (err) {
+        console.log(err)
+        process.exit(0)
+    }
+}
+
+start()

package/fetch.js

@@ -0,0 +1,177 @@
+const { tryLock } = require('tcp-mutex')
+const debug = require('debug')('fetch')
+const winston = require('winston')
+const { getConfig } = require('./config')
+const { promiseTimeout, getDomainPart,domainFromEmail, eventsOfProvider } = require('./shared')
+const chalk = require('chalk')
+const vars = require('./vars')
+const argv = require('minimist')(process.argv.slice(2))
+const { parseDomain, ParseResultType } = require("parse-domain")
+const { CoadminService } = require('coadmin-lib')
+const coadmin_service = new CoadminService('fakefilter/fetch')
+const MAX_RUN_HOURS = 2
+
+let succeed = 0
+let failed = []
+
+
+const logger = winston.createLogger({
+    level: 'info',
+    format: winston.format.combine(
+        winston.format.timestamp(),
+        winston.format.json(),  
+    ),
+    defaultMeta: { script: 'fetch.js' },
+
+    transports: [
+        //
+        // - Write all logs with importance level of `error` or less to `error.log`
+        // - Write all logs with importance level of `info` or less to `combined.log`
+        //
+        new winston.transports.File({ filename: 'logs/error.log', level: 'error' }),
+        new winston.transports.File({ filename: 'logs/combined.log' }),
+        // log to Console too
+        //   new winston.transports.Console()
+    ],
+})
+
+// load all providers/*.js
+var glob = require('glob'), path = require('path')
+const { array_shuffle } = require('mybase')
+glob.sync('./providers/*.js').forEach(function (file) {
+    require(path.resolve(file));
+})
+
+// shuffle providers to randomize
+vars.providers = array_shuffle(vars.providers)
+
+async function processProvider(p, headless = true) {
+    return new Promise(async function (resolve, reject) {
+        logger.info(`Processing ${p.name}`)
+        debug(`Processing ${p.name}`)
+
+        if (p.hasOwnProperty('min_hours') && parseInt(p['min_hours'])>0) {
+            let min_hours = parseInt(p['min_hours'])
+            // if min_hours specified we rate limit the provider
+            let events = await eventsOfProvider(p.name,1)
+            if (events && events.length>0) {
+                let last_event = events[events.length-1]
+                let since_last_event = Date.now()-last_event.t
+                let since_last_event_hours = since_last_event/(60*60*1000)
+                if (since_last_event_hours<min_hours) {
+                    // need to skip processing this provider
+                    console.log(chalk.yellow(`Provider ${p.name} skipped due to min_hours(${min_hours} hours) rule`))
+                    return resolve(true)
+                } 
+                console.log(last_event,since_last_event_hours)
+            }
+        }
+
+        // console.log(chalk.yellow(p.name))
+        let initResult = await p.init()
+        debug(`Init done, result = ${initResult ? true : false}`)
+        // console.log(`init=`,initResult)
+        if (initResult === true) {
+            debug(`Calling start`)
+            let res = await p.start(headless)
+            debug(`Start result`, res)
+            if (res && res.hasOwnProperty('email')) {
+                await vars.models.Events.create({
+                    provider: p.name,
+                    t: Date.now(),
+                    event: 'fetch',
+                    result: 'success'
+                })
+                // console.log(res)
+                succeed++
+                // success
+                let emails = []
+                if (typeof res.email === 'string') emails.push(res.email);
+                else emails = res.email
+                for (let e of emails) {
+                    let thedomain = p.randomSubdomain ? getDomainPart(domainFromEmail(e)) : domainFromEmail(e)
+                    if (vars.config.ignore_domains.includes(thedomain)) {
+                        console.log(chalk.red(`Ignored domain: ${thedomain} ignored`))
+                        continue
+                    }
+                    console.log(chalk.green(`Provider ${p.name} - ${thedomain}`))
+
+                    logger.info(`Provider ${p.name} - domain: ${thedomain}`)
+                    await vars.models.Harvested.create({
+                        provider: p.name,
+                        t: Date.now(),
+                        domain: thedomain
+                    })
+                }
+                return resolve(true)
+            }
+            await vars.models.Events.create({
+                provider: p.name,
+                t:Date.now(),
+                event:'fetch',
+                result:'failed'
+            })
+            failed.push(p.name)
+            console.log(chalk.red(`Provider ${p.name} - failed`))
+        }
+        logger.error(`Provider ${p.name} - failed - initResult:${initResult ? 'true' : 'false'}`)
+        resolve(false)
+    })
+}
+
+async function start() {
+    try {
+        // make sure we never stuck
+        setTimeout(() => {
+            logger.error('MAX RUNTIME reached')
+            process.exit(0)
+        }, MAX_RUN_HOURS * 60 * 60 * 1000)
+
+        let config = await getConfig()
+        
+        
+        let selectedProvider = []
+        if (argv._.length > 0) {
+            selectedProvider.push(...argv._)
+            console.log(`Selected providers:`, selectedProvider.join(', '))
+        } else await tryLock(config.lockPort)
+
+        coadmin_service.report('started')
+
+        for (var p of vars.providers) {
+            if (selectedProvider.length > 0 && !selectedProvider.includes(p.name)) continue
+
+            try {
+                await promiseTimeout(120000, processProvider(p, argv.v ? false : true))
+                coadmin_service.report_every(1, 'fetching')
+            } catch (err) {
+                console.log(err)
+            }
+
+            // make sure to close the browser - attempt to protect the memory
+            if (vars.browsers.length > 0)
+                try {
+                    let browser = vars.browsers.shift()
+                    await browser.close()
+                    coadmin_service.report_every(1, 'closing browsers')
+                } catch (e) {
+                    console.log(e)
+                }
+        }
+
+        if (failed.length > 0)
+            console.log(chalk.bold(`Finished..`), chalk.green(`${succeed} succeed`), chalk.red(`${failed.length} failed`))
+        else
+            console.log(chalk.bold(`Finished..`), chalk.green(`${succeed} succeed`))
+        if (failed.length > 0)
+            console.log(chalk.red(`Failed providers:\n`), chalk.red(failed.join('\n')))
+
+        coadmin_service.report('stopped')
+        process.exit(0)
+    } catch (err) {
+        console.log(chalk.red(err))
+        process.exit(0)
+    }
+}
+
+start()

package/models/events.js

@@ -0,0 +1,37 @@
+const { Sequelize, DataTypes, Model } = require('sequelize');
+const vars = require('./../vars')
+
+class Events extends Model { }
+
+function initEvents() {
+    Events.init({
+        t: {
+            type: DataTypes.BIGINT,
+            allowNull: true
+        },
+        provider: {
+            type: DataTypes.STRING(255),
+            allowNull: false,
+            comment: 'service hostname, domainname'
+        },
+        event: {
+            type: DataTypes.STRING(32),
+            allowNull: false,
+        },
+        result: {
+            type: DataTypes.STRING(32),
+            allowNull: false,
+        },
+    }, {
+        // Other model options go here
+        sequelize:vars.sequelize, // We need to pass the connection instance
+        modelName: 'Events',// We need to choose the model name
+        tableName: 'events'
+    })
+}
+
+
+module.exports = {
+    Events,
+    initEvents
+}

package/models/harvested.js

@@ -0,0 +1,45 @@
+const { Sequelize, DataTypes, Model } = require('sequelize');
+const vars = require('./../vars')
+
+class Harvested extends Model { }
+
+function initHarvested() {
+    Harvested.init({
+        t: {
+            type: DataTypes.BIGINT,
+            allowNull: true
+        },
+        provider: {
+            type: DataTypes.STRING(255),
+            allowNull: false,
+            comment: 'service hostname, domainname'
+        },
+        domain: {
+            type: DataTypes.STRING(255),
+            allowNull: false,
+            comment: 'domainname'
+        },
+    }, {
+        // Other model options go here
+        sequelize:vars.sequelize, // We need to pass the connection instance
+        modelName: 'Harvested',// We need to choose the model name
+        tableName: 'harvested',
+        indexes:[
+            {
+                fields:['provider'],
+            },
+            {
+                fields:['domain']
+            },
+            {
+                fields:['t']
+            },
+        ]
+    })
+}
+
+
+module.exports = {
+    Harvested,
+    initHarvested
+}