fakefilter 1.1.13 → 1.1.1581

package/chart-update.js

@@ -1,261 +0,0 @@
-const path = require('path')
-const fs = require('fs')
-var Table = require('easy-table')
-const winston = require('winston')
-const { getConfig, Sentry } = require('./config')
-const chalk = require('chalk')
-const vars = require('./vars')
-const { softexit, wait } = require('mybase')
-const { pullDomains, pullEventsProviders, pullEventsByProvider } = require('./shared')
-const QuickChart = require('quickchart-js');
-const argv = require('minimist')(process.argv.slice(2))
-const md5 = require('md5')
-
-function writeToFile(fn, content) {
-    // let fn = 'repo/markdown/README.md'
-    fs.writeFileSync(fn, content)
-    console.log(chalk.green(`+ file ${fn}`))
-    return true
-}
-
-// function isEventGraphicCached(providerName) {
-//     let fn = `success-rate-${md5(providerName)}.png`
-//     let chartFile = path.join(__dirname, `/apiserver/public/img/dynamic/${fn}`)
-//     if (fs.existsSync(chartFile)) {
-//         // lets look if the file was recently rendered
-//         let stats = fs.statSync(chartFile)
-//         let mtimeMinutes = (new Date() - stats.mtime) / 1000 / 60 // modified time in minutes
-//         if (mtimeMinutes < 200) {
-//             console.log(`Chart for ${providerName} - taken from cache ${chartFile}`)
-//             return chartFile
-//         }
-//     }
-//     return false
-// }
-
-function dailyEventGraphic(providerName,daysAgo=60) {
-    return new Promise(async function (resolve, reject) {
-        const myChart = new QuickChart()
-        let data = {}
-
-        let successrate = {}
-        let labels = []
-        let fn = `success-rate-${md5(providerName)}.png`
-        let chartFile = path.join(__dirname, `/apiserver/public/img/dynamic/${fn}`)
-        if (fs.existsSync(chartFile)) {
-            // lets look if the file was recently rendered
-            let stats = fs.statSync(chartFile)
-            let mtimeMinutes = (new Date() - stats.mtime) / 1000 / 60 // modified time in minutes
-            if (mtimeMinutes < 200) {
-                console.log(chalk.gray(`chart for ${providerName} - taken from cache ${chartFile}`))
-                return resolve(fn)
-            }
-        }
-        await wait(2) // rate limit
-        let events = await pullEventsByProvider(providerName, daysAgo)
-        for (let row of events) {
-            let dayString = new Date(row.t).toISOString().substring(0, 10)
-            if (!data.hasOwnProperty(dayString)) {
-                data[dayString] = { success: 0, failed: 0 }
-                successrate[dayString] = 0
-                labels.push(dayString)
-            }
-            if (row.result === 'success') data[dayString].success++; else
-                data[dayString].failed++;
-        }
-        for (let date of Object.keys(data)) {
-            successrate[date] = ((data[date].success / (data[date].failed + data[date].success))) * 100
-        }
-
-        let chartConfig1 = {
-            type: 'bar',
-            data: {
-                labels,
-                datasets: [{
-                    label: `Success Rate - ${providerName} - %`,
-                    backgroundColor: QuickChart.getGradientFillHelper('vertical', ['green', 'red']),
-                    data: Object.keys(successrate).map(dayString => successrate[dayString])
-                }
-                ]
-            },
-        }
-        myChart.setConfig(chartConfig1)
-
-
-        await myChart.toFile(chartFile)
-        console.log(`Creating success-rate chart for ${providerName}`)
-        return resolve(fn)
-
-        // calcualte success rate
-
-    })
-}
-function dailyCoverageGraph(rows) {
-    return new Promise(async function (resolve, reject) {
-        const myChart = new QuickChart();
-        const myChart2 = new QuickChart();
-        const myChart3 = new QuickChart();
-        let data = {}
-        let labels = []
-        let uniqueDomains = []
-        let dataUniqueDomains = {}
-        let dataUniqueDomainsMonthly = {}
-
-        rows = rows.sort((a, b) => a.t - b.t)
-        for (let row of rows) {
-            let dayString = new Date(row.t).toISOString().substring(0, 10)
-            let monthString = new Date(row.t).toISOString().substring(0, 7)
-            if (!uniqueDomains.includes(row.domain)) uniqueDomains.push(row.domain)
-            if (!data.hasOwnProperty(dayString)) {
-                data[dayString] = []
-                dataUniqueDomains[dayString] = uniqueDomains.length
-                labels.push(dayString)
-            }
-            dataUniqueDomainsMonthly[monthString] = uniqueDomains.length
-
-            if (!data[dayString].includes(row.domain)) data[dayString].push(row.domain)
-            // console.log(row)
-            // if (cnt<10) break
-            // cnt--
-        }
-        // console.log(dataUniqueDomains)
-        labels = labels.reverse().slice(0, 200).reverse()
-        let labelsM = Object.keys(dataUniqueDomainsMonthly).reverse().slice(0, 200).reverse()
-        let chartConfig2 = {
-            type: 'line',
-            data: {
-                labels,
-                datasets: [{
-                    label: 'Unique Domains in FakeFilter',
-                    data: labels.map(dayString => dataUniqueDomains[dayString]),
-                    fill: 'start',
-                    borderWidth: 0,
-                    pointRadius: 0,
-                    borderColor: 'rgba(255, 255, 0, 0.5)',
-                    backgroundColor: 'rgba(255, 255, 0, 0.5)',
-                }]
-            },
-        }
-
-        let chartConfig3 = {
-            type: 'line',
-            data: {
-                labels: labelsM,
-                datasets: [{
-                    label: 'Unique Domains in FakeFilter Per Month',
-                    data: labelsM.map(monthString => dataUniqueDomainsMonthly[monthString]),
-                    fill: 'start',
-                    borderWidth: 0,
-                    pointRadius: 0,
-                    borderColor: 'rgba(255, 120, 0, 0.5)',
-                    backgroundColor: 'rgba(255, 120, 0, 0.5)',
-                }]
-            },
-        }
-
-        let chartConfig1 = {
-            // type: 'bar',
-            type: 'line',
-            data: {
-                labels,
-                datasets: [{
-                    label: 'Daily Coverage',
-                    data: labels.map(dayString => data[dayString].length),
-                    fill: 'start',
-                    borderWidth: 0,
-                    pointRadius: 0,
-                    borderColor: 'rgba(0, 255, 0, 0.5)',
-                    backgroundColor: 'rgba(0, 255, 0, 0.5)',
-                }]
-            },
-            options: {
-                scales: {
-                    yAxes: [{
-                        stacked: true,
-                        ticks: {
-                            min: -10,
-                        }
-                    }]
-                },
-                legend: {
-                    display: true,
-                },
-            },
-        }
-
-        // console.log(chartConfig1)
-        myChart.setConfig(chartConfig1)
-        myChart2.setConfig(chartConfig2)
-        myChart3.setConfig(chartConfig3)
-        await myChart.toFile(path.join(__dirname, '/apiserver/public/img/dynamic/chart1.png'))
-
-        await myChart2.toFile(path.join(__dirname, '/apiserver/public/img/dynamic/chart2.png'))
-        await myChart3.toFile(path.join(__dirname, '/apiserver/public/img/dynamic/chart3.png'))
-
-        return resolve()
-    })
-}
-
-async function start() {
-    try {
-        await getConfig()
-        let events_providers = await pullEventsProviders(false, 60)
-        let markdown3 = ``
-        // if (false)
-        for (let provider of events_providers) {
-            // break
-            // if (provider==='temporarymail.com') 
-            {
-                try {
-                    
-                    let chartfn = await dailyEventGraphic(provider,60)
-
-
-                    markdown3 += `### ${provider}
-![Success Rate for ${provider}](https://fakefilter.net/public/img/dynamic/${chartfn})
-
-`
-                } catch (err) {
-                    Sentry.captureException(err)
-                    console.log(err)
-                }
-            }
-        }
-
-        let domain_rows = await pullDomains(1000, false)
-        let urls = await dailyCoverageGraph(domain_rows)
-
-        let markdown1 = `
-# Charts
-## All Providers
-### Daily Coverage
-Amount of coverage, actions against all known Disposable Email Providers per Day
-![Daily Coverage](https://fakefilter.net/public/img/dynamic/chart1.png)
-
-### Daily Unique Disposable Domains
-Number of unique known validated Disposable Domains in Total at given Day.
-![Daily Disposable Domains](https://fakefilter.net/public/img/dynamic/chart2.png)
-
-### Monthly Unique Disposable Domains
-Number of unique known validated Disposable Domains in Total at given Month.
-![Daily Disposable Domains](https://fakefilter.net/public/img/dynamic/chart3.png)
-
-## Success Rate
-Per Provider Success Rate per Day - A lower Success rate might mean different things; counter measures from provider to be detected like capchas or even service downtime.
-
-${markdown3}
-`
-        writeToFile(path.join(__dirname, 'repo/CHARTS.md'), markdown1)
-
-        console.log(urls)
-        process.exit(0)
-
-    }
-    catch (err) {
-        console.log(err)
-        Sentry.captureException(err)
-        softexit("", 1)
-    }
-}
-
-start()

package/cleanup.js

@@ -1,58 +0,0 @@
-const { Op } = require("sequelize")
-const path = require('path')
-const fs = require('fs')
-const winston = require('winston')
-const { getConfig, Sentry } = require('./config')
-const chalk = require('chalk')
-const vars = require('./vars')
-const { softexit, wait } = require('mybase')
-const { pullDomains, pullEventsProviders, pullEventsByProvider } = require('./shared')
-const QuickChart = require('quickchart-js');
-const argv = require('minimist')(process.argv.slice(2))
-
-/*
-    we need to vacuum Harvested table to avoid excessive load times of domains
-*/
-
-async function start() {
-    try {
-        await getConfig()
-        let rows = await pullDomains(10000)
-        // const progress = require('progressbar').create().step('combining')
-        // progress.setTotal(rows.length)
-        let memory = {}
-        let keep_provider_domains_per_day = 1 // never put this to 0 or lower!!! never
-        let deleted = 0
-        if (keep_provider_domains_per_day > 0)
-            for (let row of rows) {
-                // await wait(1 / 1000)
-                // progress.addTick()
-                // console.log(row,provider)
-                let day = new Date(row.t).toISOString().substring(0, 10)
-                let provider = row.provider.toLowerCase()
-                let domain = row.domain.toLowerCase()
-
-                if (!memory.hasOwnProperty(day)) memory[day] = {}
-                if (!memory[day].hasOwnProperty(provider)) memory[day][provider] = {}
-                if (!memory[day][provider].hasOwnProperty(domain)) memory[day][provider][domain] = 0
-                memory[day][provider][domain]++
-                if (memory[day][provider][domain] > keep_provider_domains_per_day) {
-                    console.log(`day:${day} domain:${chalk.bold(domain)} provider:${chalk.yellow(provider)} found ${chalk.blue(memory[day][provider][domain])} times`)
-                    await vars.models.Harvested.destroy({ where: { id: row.id } })
-                    deleted++
-                }
-            }
-        console.log(`Deleted ${deleted} rows from Harvested to vacuum`)
-        process.exit(0)
-
-
-    }
-    catch (err) {
-        console.log(err)
-        Sentry.captureException(err)
-        await wait(5)
-        process.exit(0)
-    }
-}
-
-start()

package/config.js

@@ -1,126 +0,0 @@
-const fs = require('fs')
-const path = require('path')
-const argv = require('minimist')(process.argv.slice(2))
-const vars = require('./vars')
-const { Sequelize } = require('sequelize')
-var chalk = require('chalk')
-var debug = require('debug')('fakefilter')
-var { vaultFill,isLocal,canReadAndWrite } = require('mybase')
-const { Harvested,initHarvested } = require('./models/harvested')
-const { HarvestedClone,initHarvestedClone } = require('./models/harvested_clone')
-const { Events,initEvents } = require('./models/events')
-
-
-var vault = require("node-vault")({
-  apiVersion: 'v1', // default
-  endpoint: 'https://vault3.vpn1.org',
-  token: 'd7130f39-8466-4444-b2b7-49c93da2c597'
-})
-
-var Sentry = require('@sentry/node');
-Sentry.init({
-    dsn:'https://1074c423d608484e951455dc3c0d69bb@sentry.def24.com/25',
-    captureUnhandledRejections: true,
-    name: require('os').hostname()
-})
-
-function syncModel(modelName) {
-    return new Promise(async function (resolve, reject) {
-        console.log(`Synching model: ${modelName.name}`)
-        try {
-            await modelName.sync({ alter: true })
-            return resolve(true)
-        } catch (err) {
-            console.log(err)
-            console.log(chalk.red(`Exception by synching model: ${modelName.name}`))
-            process.exit(0)
-        }
-    })
-
-}
-
-function getConfig()
-{
-    return new Promise(async function (resolve,reject) {
-        try {
-            
-            await vaultFill(vault,config)
-            await canReadAndWrite(config.cache_folder,true)
-
-            var sequelize = new Sequelize(
-                config.mysqlserver.db,
-                config.mysqlserver.login,
-                config.mysqlserver.password, {
-                        host: config.mysqlserver.host,
-                        dialect: 'mysql',
-                        logging: false
-                }
-            )
-            vars.sequelize = sequelize
-            await sequelize.authenticate()
-
-            // initialize
-            await initHarvested()
-            await initHarvestedClone()
-            await initEvents()
-
-
-
-            vars.models.Harvested = Harvested
-            vars.models.HarvestedClone = HarvestedClone
-            vars.models.Events = Events
-            vars.config = config
-
-            if (argv.sync) {
-                await syncModel(Harvested)
-                await syncModel(HarvestedClone)
-                await syncModel(Events)
-                process.exit(0)
-            }
-
-            resolve(config)
-        }
-        catch(err) {
-            reject(err)
-        }
-        
-    })
-}
-
-var config = {
-    lockPort: 9994,
-    captcha_resolver_api:'7a1f7a34d2c261393528e62088a75515',
-    mysqlserver:{
-        db:'fakefilter',
-        login:'fakefilter',
-        password:'Q3p5hfxyrZLs',
-        host:'mix.def24.com'
-    },
-    apiserver: {
-        ip: '127.0.0.1',
-        port: 5520
-    },
-    cache_folder:path.join(__dirname,"cache"),
-    main_folder:__dirname,
-    ignore_domains : ['samsung.com','nytimes.com','yahoo.gr','outlook.sa','live.nl','yahoo.it','comcast.net','mail.ua','yahoo.com.br','outlook.es','aliyun.com','cps.edu','sina.cn','yahoo.ca',
-    't-online.de','yahoo.co.jp','hotmail.es','yahoo.in','vip.qq.com','yandex.ua','139.com','hotmail.it','orange.fr','hotmail.de','yahoo.com.hk','gmx.at','live.co.uk','gmx.net','yahoo.co.in','yahoo.de','yahoo.es','me.com','o2.pl','gmx.fr','abv.bg','yahoo.co.id','wp.pl','i.ua','msn.com','live.fr','googlemail.com','gmx.com','yahoo.co.uk','yahoo.fr','naver.com','sina.com','yandex.com','gmx.de','aol.com','hotmail.co.uk','yahoo.com.tw','hotmail.fr','bk.ru','mail.com','web.de','outlook.fr','protonmail.com','ukr.net','live.com','icloud.com','126.com','yandex.ru','outlook.com','163.com','mail.ru','hotmail.com','yahoo.com','qq.com','gmail.com','zoho.com','cloudflare.com']
-    // vaultKey: 'vault@/secret/vault'
-}
-
-if (require.main===module) {
-    getConfig().then(c=>{
-        console.log('11')
-        if (!argv.sync) console.log(c)
-        console.log('22')
-        process.exit(0)
-    })
-}
-
-
-
-
-module.exports = {
-    Sentry,
-    getConfig,
-    config
-}

package/demos/visitor0/index.js

@@ -1,41 +0,0 @@
-const fs = require('fs')
-const { Sentry } = require('../../config')
-const { launch } = require('../../shared')
-const { wait } = require('mybase');
-
-// const URL = 'https://fakefilter.net/public/img/test.html'
-// const URL = 'https://sdnz.jdjy.sh.cn/'
-const URL = `https://visitor0.com/static/demo/basic.html?2`
-// const URL = `https://infosimples.github.io/detect-headless/`
-// const URL = `https://leaky.page/timer.html`
-const lookingFor = 'bitrix24'.toLowerCase();
-
-
-function start(pageWaitTime = 1, matchWaitTime = 5) {
-    return new Promise(async function (resolve, reject) {
-        try {
-            let { browser, page } = await launch(true, URL, pageWaitTime, false, false, 'domcontentloaded')
-            await wait(5)
-
-
-            console.log(`[${Date.now()}] searching for our match ${lookingFor}`)
-            const renderedDOM = await page.content();
-            fs.writeFileSync('renderedDOM.html', renderedDOM)
-
-
-            if (browser) await browser.close()
-
-            return resolve()
-        } catch (err) {
-            console.log(err)
-            Sentry.captureException(err)
-            resolve(false)
-        }
-    })
-}
-
-// for cli usage - development
-if (require.main == module) {
-    start(false).then(res => { process.exit(0) })
-}
-

package/demos/visitor0/renderedDOM.html

@@ -1,8 +0,0 @@
-<html><head>
-        
-    </head>
-    <body>
-        <h1>Hello World</h1>
-        <script src="/tag"></script>
-    
-</body></html>

package/elements/demo.js

@@ -1,78 +0,0 @@
-const chalk = require('chalk')
-const fs = require('fs')
-const { Sentry } = require('../config')
-const { launch } = require('../shared')
-// const URL = 'https://www.blackdesertfoundry.com/all-recipes/'
-const cheerio = require('cheerio');
-const { wait } = require('mybase');
-
-// const URL = 'https://fakefilter.net/public/img/test.html'
-// const URL = 'https://sdnz.jdjy.sh.cn/'
-const URL = `https://fords.com.ua/nasos_vakuumnyy_ford_ranger_1997_2007_hmpx.html`
-const lookingFor = 'bitrix24'.toLowerCase();
-
-function searchFor(renderedDOM) {
-    const $ = cheerio.load(renderedDOM);
-    // console.log($('body').html())
-
-    const scriptSrcUrls = [];
-    const foundItems = []
-    $('script[src]').each((index, element) => {
-        const wholeTag = $(element).toString().toLowerCase(); 
-
-        const srcUrl = $(element).attr('src');
-        scriptSrcUrls.push(srcUrl);
-        console.log(wholeTag)
-        if (wholeTag.includes(lookingFor)) {
-            foundItems.push({
-                srcUrl,
-                wholeTag
-            })
-        }
-    });
-
-    console.log(scriptSrcUrls)
-    console.log(foundItems)
-    return foundItems
-}
-
-function start(pageWaitTime = 1, matchWaitTime = 5) {
-    return new Promise(async function (resolve, reject) {
-        try {
-            let { browser, page } = await launch(true, URL, pageWaitTime, false, false, 'domcontentloaded')
-            await wait(5)
-            const started = Date.now()
-            while(true) {
-                console.log(`[${Date.now()}] searching for our match ${lookingFor}`)
-                const renderedDOM = await page.content();
-                fs.writeFileSync('renderedDOM.html', renderedDOM)
-                const foundItems = searchFor(renderedDOM)
-                if (foundItems.length > 0) {
-                    console.log(chalk.green('found'))
-                    break
-                }
-                await wait(1)
-                if (Date.now() - started > matchWaitTime*1000) {
-                    console.log(chalk.red('not found'))
-                    break
-                }
-            }
-
-
-            
-            if (browser) await browser.close()
-
-            return resolve()
-        } catch (err) {
-            console.log(err)
-            Sentry.captureException(err)
-            resolve(false)
-        }
-    })
-}
-
-// for cli usage - development
-if (require.main == module) {
-    start(false).then(res => { process.exit(0) })
-}
-