fakefilter 0.1.1401 → 1.1.12

package/events.js

@@ -0,0 +1,59 @@
+
+const debug = require('debug')('fetch')
+const winston = require('winston')
+const { getConfig, Sentry } = require('./config')
+const { promiseTimeout, getDomainPart,domainFromEmail, eventsOfProvider } = require('./shared')
+const Table = require('easy-table')
+const chalk = require('chalk')
+const vars = require('./vars')
+const argv = require('minimist')(process.argv.slice(2))
+const { parseDomain, ParseResultType } = require("parse-domain")
+const { CoadminService } = require('coadmin-lib')
+const coadmin_service = new CoadminService('fakefilter/fetch')
+const MAX_RUN_HOURS = 2
+
+
+async function start() {
+    try {
+       
+        let config = await getConfig()
+        let selectedProvider = false
+        if (argv._.length > 0) 
+            selectedProvider=argv._[0]
+        if (!selectedProvider) {
+            console.log(chalk.blue('Usage:'))
+            console.log(`node events.js <provider>`)
+            process.exit(0)
+        }
+
+        let events = await eventsOfProvider(selectedProvider,7)
+        if (events) {
+            console.log(chalk.yellow(`${events.length} events found for ${selectedProvider}`))
+            var t = new Table()
+            let stats = {
+                success:0,
+                other:0,
+                failed:0
+            }
+            for(let event of events) {
+                // console.log(event)
+                t.cell("t",new Date(event.t).toISOString())
+                t.cell("event",event.event)
+                t.cell("result",event.result==='success' ? '✅ success' : '❌ '+event.result)
+                t.newRow()
+                if (event.result==='success') stats.success++;  else
+                if (event.result==='failed') stats.failed++; 
+                else  stats.other++
+            }
+            console.log(t.toString())
+            console.log(`Total success: ${chalk.green(stats.success)} failed: ${chalk.red(stats.failed)} other: ${stats.other}`)
+            console.log(`This is only from last 7 days`)
+        }
+        process.exit(0)
+    } catch (err) {
+        console.log(err)
+        process.exit(0)
+    }
+}
+
+start()

package/fetch.js

@@ -0,0 +1,180 @@
+const { tryLock } = require('tcp-mutex')
+const debug = require('debug')('fetch')
+const winston = require('winston')
+const { getConfig } = require('./config')
+const { promiseTimeout, getDomainPart,domainFromEmail, eventsOfProvider } = require('./shared')
+const chalk = require('chalk')
+const vars = require('./vars')
+const argv = require('minimist')(process.argv.slice(2))
+const { parseDomain, ParseResultType } = require("parse-domain")
+const { CoadminService } = require('coadmin-lib')
+const coadmin_service = new CoadminService('fakefilter/fetch')
+const MAX_RUN_HOURS = 2
+
+let succeed = 0
+let failed = []
+
+
+const logger = winston.createLogger({
+    level: 'info',
+    format: winston.format.combine(
+        winston.format.timestamp(),
+        winston.format.json(),  
+    ),
+    defaultMeta: { script: 'fetch.js' },
+
+    transports: [
+        //
+        // - Write all logs with importance level of `error` or less to `error.log`
+        // - Write all logs with importance level of `info` or less to `combined.log`
+        //
+        new winston.transports.File({ filename: 'logs/error.log', level: 'error' }),
+        new winston.transports.File({ filename: 'logs/combined.log' }),
+        // log to Console too
+        //   new winston.transports.Console()
+    ],
+})
+
+// load all providers/*.js
+var glob = require('glob'), path = require('path')
+const { array_shuffle } = require('mybase')
+glob.sync('./providers/*.js').forEach(function (file) {
+    require(path.resolve(file));
+})
+
+// shuffle providers to randomize
+vars.providers = array_shuffle(vars.providers)
+
+async function processProvider(p, headless = true) {
+    return new Promise(async function (resolve, reject) {
+        logger.info(`Processing ${p.name}`)
+        debug(`Processing ${p.name}`)
+
+        if (p.hasOwnProperty('min_hours') && parseInt(p['min_hours'])>0) {
+            let min_hours = parseInt(p['min_hours'])
+            // if min_hours specified we rate limit the provider
+            let events = await eventsOfProvider(p.name,1)
+            if (events && events.length>0) {
+                let last_event = events[events.length-1]
+                let since_last_event = Date.now()-last_event.t
+                let since_last_event_hours = since_last_event/(60*60*1000)
+                if (since_last_event_hours<min_hours) {
+                    // need to skip processing this provider
+                    console.log(chalk.yellow(`Provider ${p.name} skipped due to min_hours(${min_hours} hours) rule`))
+                    return resolve(true)
+                } 
+                console.log(last_event,since_last_event_hours)
+            }
+        }
+
+        // console.log(chalk.yellow(p.name))
+        let initResult = await p.init()
+        debug(`Init done, result = ${initResult ? true : false}`)
+        // console.log(`init=`,initResult)
+        if (initResult === true) {
+            debug(`Calling start`)
+            let res = await p.start(headless)
+            debug(`Start result`, res)
+            if (res && res.hasOwnProperty('email')) {
+                await vars.models.Events.create({
+                    provider: p.name,
+                    t: Date.now(),
+                    event: 'fetch',
+                    result: 'success'
+                })
+                // console.log(res)
+                succeed++
+                // success
+                let emails = []
+                if (typeof res.email === 'string') emails.push(res.email);
+                else emails = res.email
+                for (let e of emails) {
+                    let thedomain = p.randomSubdomain ? getDomainPart(domainFromEmail(e)) : domainFromEmail(e)
+                    if (vars.config.ignore_domains.includes(thedomain)) {
+                        console.log(chalk.red(`Ignored domain: ${thedomain} ignored`))
+                        continue
+                    }
+                    console.log(chalk.green(`Provider ${p.name} - ${thedomain}`))
+
+                    logger.info(`Provider ${p.name} - domain: ${thedomain}`)
+                    await vars.models.Harvested.create({
+                        provider: p.name,
+                        t: Date.now(),
+                        domain: thedomain
+                    })
+                }
+                return resolve(true)
+            }
+            await vars.models.Events.create({
+                provider: p.name,
+                t:Date.now(),
+                event:'fetch',
+                result:'failed'
+            })
+            failed.push(p.name)
+            console.log(chalk.red(`Provider ${p.name} - failed`))
+        }
+        logger.error(`Provider ${p.name} - failed - initResult:${initResult ? 'true' : 'false'}`)
+        resolve(false)
+    })
+}
+
+async function start() {
+    try {
+        // make sure we never stuck
+        setTimeout(() => {
+            logger.error('MAX RUNTIME reached')
+            process.exit(0)
+        }, MAX_RUN_HOURS * 60 * 60 * 1000)
+
+        let config = await getConfig()
+        
+        
+        let selectedProvider = []
+        if (argv._.length > 0) {
+            selectedProvider.push(...argv._)
+            console.log(`Selected providers:`, selectedProvider.join(', '))
+        } else await tryLock(config.lockPort)
+
+        coadmin_service.report('started')
+
+        for (var p of vars.providers) {
+            if (selectedProvider.length > 0 && !selectedProvider.includes(p.name)) continue
+            if (p.disabled) {
+                console.log(chalk.yellow(`▪️ Provider ${p.name} is disabled, skipping`))
+                continue
+            }
+            try {
+                await promiseTimeout(120000, processProvider(p, argv.v ? false : true))
+                coadmin_service.report_every(1, 'fetching')
+            } catch (err) {
+                console.log(err)
+            }
+
+            // make sure to close the browser - attempt to protect the memory
+            if (vars.browsers.length > 0)
+                try {
+                    let browser = vars.browsers.shift()
+                    await browser.close()
+                    coadmin_service.report_every(1, 'closing browsers')
+                } catch (e) {
+                    console.log(e)
+                }
+        }
+
+        if (failed.length > 0)
+            console.log(chalk.bold(`Finished..`), chalk.green(`${succeed} succeed`), chalk.red(`${failed.length} failed`))
+        else
+            console.log(chalk.bold(`Finished..`), chalk.green(`${succeed} succeed`))
+        if (failed.length > 0)
+            console.log(chalk.red(`Failed providers:\n`), chalk.red(failed.join('\n')))
+
+        coadmin_service.report('stopped')
+        process.exit(0)
+    } catch (err) {
+        console.log(chalk.red(err))
+        process.exit(0)
+    }
+}
+
+start()

package/models/events.js

@@ -0,0 +1,37 @@
+const { Sequelize, DataTypes, Model } = require('sequelize');
+const vars = require('./../vars')
+
+class Events extends Model { }
+
+function initEvents() {
+    Events.init({
+        t: {
+            type: DataTypes.BIGINT,
+            allowNull: true
+        },
+        provider: {
+            type: DataTypes.STRING(255),
+            allowNull: false,
+            comment: 'service hostname, domainname'
+        },
+        event: {
+            type: DataTypes.STRING(32),
+            allowNull: false,
+        },
+        result: {
+            type: DataTypes.STRING(32),
+            allowNull: false,
+        },
+    }, {
+        // Other model options go here
+        sequelize:vars.sequelize, // We need to pass the connection instance
+        modelName: 'Events',// We need to choose the model name
+        tableName: 'events'
+    })
+}
+
+
+module.exports = {
+    Events,
+    initEvents
+}

package/models/harvested.js

@@ -0,0 +1,45 @@
+const { Sequelize, DataTypes, Model } = require('sequelize');
+const vars = require('./../vars')
+
+class Harvested extends Model { }
+
+function initHarvested() {
+    Harvested.init({
+        t: {
+            type: DataTypes.BIGINT,
+            allowNull: true
+        },
+        provider: {
+            type: DataTypes.STRING(255),
+            allowNull: false,
+            comment: 'service hostname, domainname'
+        },
+        domain: {
+            type: DataTypes.STRING(255),
+            allowNull: false,
+            comment: 'domainname'
+        },
+    }, {
+        // Other model options go here
+        sequelize:vars.sequelize, // We need to pass the connection instance
+        modelName: 'Harvested',// We need to choose the model name
+        tableName: 'harvested',
+        indexes:[
+            {
+                fields:['provider'],
+            },
+            {
+                fields:['domain']
+            },
+            {
+                fields:['t']
+            },
+        ]
+    })
+}
+
+
+module.exports = {
+    Harvested,
+    initHarvested
+}

package/models/harvested_clone.js

@@ -0,0 +1,44 @@
+const { Sequelize, DataTypes, Model } = require('sequelize');
+const vars = require('./../vars')
+
+class HarvestedClone extends Model { }
+
+function initHarvestedClone() {
+    HarvestedClone.init({
+        t: {
+            type: DataTypes.BIGINT,
+            allowNull: true
+        },
+        provider: {
+            type: DataTypes.STRING(255),
+            allowNull: false,
+            comment: 'service hostname, domainname'
+        },
+        domain: {
+            type: DataTypes.STRING(255),
+            allowNull: false,
+            comment: 'domainname'
+        },
+    }, {
+        // Other model options go here
+        sequelize:vars.sequelize, // We need to pass the connection instance
+        modelName: 'HarvestedClone',// We need to choose the model name
+        tableName: 'harvested_clone',
+        indexes:[
+            {
+                fields:['provider'],
+            },
+            {
+                fields:['domain']
+            },
+            {
+                fields:['t']
+            },
+        ]
+    })
+}
+
+module.exports = {
+    HarvestedClone,
+    initHarvestedClone
+}

package/package.json

@@ -1,28 +1,49 @@
 {
   "name": "fakefilter",
-  "version": "0.1.1401",
-  "description": "reliable fake and temp email filter solution for site operators",
-  "license": "BSD-3-Clause",
-  "main": "index.js",
-  "types": "index.d.ts",
-  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
-  },
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/7c/fakefilter.git"
+  "version": "1.1.12",
+  "description": "",
+  "main": "fetch.js",
+  "dependencies": {
+    "@7c/autovalidate": "^1.0.3",
+    "@sentry/node": "^6.18.1",
+    "axios": "^1.5.0",
+    "body-parser": "^1.19.2",
+    "chalk": "^4.0.0",
+    "cheerio": "^1.0.0",
+    "coadmin-lib": "^1.0.4",
+    "cors": "^2.8.5",
+    "debug": "^4.3.4",
+    "easy-table": "^1.2.0",
+    "express": "^4.17.3",
+    "md5": "^2.3.0",
+    "minimist": "^1.2.5",
+    "mybase": "^1.0.48",
+    "mysql2": "^2.3.3",
+    "node-vault": "^0.9.22",
+    "parse-domain": "^5.0.0",
+    "playwright": "^1.49.0",
+    "playwright-extra": "^4.3.6",
+    "progressbar": "^1.3.0",
+    "punycode": "^2.3.0",
+    "puppeteer": "^13.4.1",
+    "puppeteer-extra": "^3.2.3",
+    "puppeteer-extra-plugin-recaptcha": "^3.5.0",
+    "puppeteer-extra-plugin-stealth": "^2.9.0",
+    "quickchart-js": "^3.0.0",
+    "sequelize": "^6.17.0",
+    "tcp-mutex": "^1.0.1",
+    "useragentsdata": "^1.0.23",
+    "validator": "^13.7.0",
+    "winston": "^3.6.0"
   },
-  "keywords": [
-    "fakeemail",
-    "tempemail",
-    "tempmail",
-    "email",
-    "filter"
-  ],
-  "author": "7c",
-  "bugs": {
-    "url": "https://github.com/7c/fakefilter/issues"
+  "scripts": {
+    "test": "npx mocha --exit"
   },
-  "homepage": "https://github.com/7c/fakefilter#readme",
-  "dependencies": {}
+  "author": "",
+  "license": "ISC",
+  "devDependencies": {
+    "chai": "^4.3.7",
+    "mocha": "^10.2.0",
+    "should": "^13.2.3"
+  }
 }

package/providers/10-minute-mail.com.js

@@ -0,0 +1,39 @@
+const { validEmail } = require('mybase')
+const { Sentry } = require('../config')
+const { initiateEmpty, launch, emailBy_id_innerText, emailBy_querySelector_innerHTML, emailBy_id_value, emailBy_querySelector_value } = require('../shared')
+const vars = require('./../vars')
+const URL = 'https://10-minute-mail.com'
+
+function start(headless=true,waitTime=3) {
+  return new Promise(async function (resolve,reject) {
+    try {
+      let {page,browser} = await launch(headless,URL,waitTime,'#openRandomButton')
+      let email = await emailBy_querySelector_innerHTML(page,'strong')
+    
+      if (browser) await browser.close()
+      if (email) return resolve({
+        email,
+        URL,
+        t:Date.now()
+      })
+      return resolve(null)
+    } catch (err) {
+      console.log(err)
+      Sentry.captureException(err)
+      resolve(false)
+    }
+  })
+}
+
+// for cli usage - development
+if (require.main==module) {
+  start(false).then(res=>console.log(res))
+}
+
+// pending
+vars.providers.push({
+  name:'10-minute-mail.com',
+  init:initiateEmpty,
+  start:start,
+  disabled:true
+})

package/providers/10-minuten-mail.de.js

@@ -0,0 +1,39 @@
+const { validEmail } = require('mybase')
+const { Sentry } = require('../config')
+const { initiateEmpty, launch, emailBy_id_innerText, emailBy_querySelector_innerHTML, emailBy_id_value, emailBy_querySelector_value } = require('../shared')
+const vars = require('./../vars')
+const URL = 'https://10-minuten-mail.de'
+
+function start(headless=true,waitTime=3) {
+  return new Promise(async function (resolve,reject) {
+    try {
+      let {page,browser} = await launch(headless,URL,waitTime,'#openRandomButton')
+      let email = await emailBy_querySelector_innerHTML(page,'strong')
+    
+      if (browser) await browser.close()
+      if (email) return resolve({
+        email,
+        URL,
+        t:Date.now()
+      })
+      return resolve(null)
+    } catch (err) {
+      console.log(err)
+      Sentry.captureException(err)
+      resolve(false)
+    }
+  })
+}
+
+// for cli usage - development
+if (require.main==module) {
+  start(false).then(res=>console.log(res))
+}
+
+// pending
+vars.providers.push({
+  name:'10-minuten-mail.de',
+  init:initiateEmpty,
+  start:start,
+  disabled:true
+})

package/providers/10mails.net.js

@@ -0,0 +1,41 @@
+const { validEmail, wait } = require('mybase')
+const { Sentry } = require('../config')
+const { initiateEmpty, launch, emailBy_id_innerText, emailBy_querySelector_innerHTML, emailBy_id_value, emailFrom_combobox_id, emailBy_querySelector_value, emailFrom_combobox_selector } = require('../shared')
+const vars = require('./../vars')
+const URL = 'http://10mails.net'
+
+function start(headless=true,waitTime=3) {
+  return new Promise(async function (resolve,reject) {
+    try {
+      let {page,browser} = await launch(headless,URL,waitTime)
+      
+      
+      let email = await emailBy_querySelector_innerHTML(page,'span#tma')
+      
+      if (browser) await browser.close()
+      if (email) return resolve({
+        email,
+        URL,
+        t:Date.now()
+      })
+      return resolve(null)
+    } catch (err) {
+      console.log(err)
+      Sentry.captureException(err)
+      resolve(false)
+    }
+  })
+}
+
+// for cli usage - development
+if (require.main==module) {
+  start(false).then(res=>console.log(res))
+}
+
+
+vars.providers.push({
+  name:'10mails.net',
+  init:initiateEmpty,
+  start:start,
+  disabled:true
+})

package/providers/10minut.com.pl.js

@@ -0,0 +1,39 @@
+const { validEmail, wait } = require('mybase')
+const { Sentry } = require('../config')
+const { initiateEmpty, launch, emailBy_id_innerText, emailBy_querySelector_innerHTML, emailBy_id_value, emailFrom_combobox_id, emailBy_querySelector_value, emailFrom_combobox_selector, emailBy_querySelector_innerText, emailBy_querySelector_placeholder } = require('../shared')
+const vars = require('./../vars')
+const URL = 'https://10minut.com.pl'
+
+function start(headless=true,waitTime=3) {
+  return new Promise(async function (resolve,reject) {
+    try {
+      let {page,browser} = await launch(headless,URL,waitTime)
+      
+      let email = await emailBy_querySelector_value(page,'input[type="text"]')
+      if (browser) await browser.close()
+      if (email) return resolve({
+        email,
+        URL,
+        t:Date.now()
+      })
+      return resolve(null)
+    } catch (err) {
+      console.log(err)
+      Sentry.captureException(err)
+      resolve(false)
+    }
+  })
+}
+
+// for cli usage - development
+if (require.main==module) {
+  start(false).then(res=>console.log(res))
+}
+
+
+vars.providers.push({
+  name:'10minut.com.pl',
+  init:initiateEmpty,
+  start:start,
+  disabled:true
+})

package/providers/10minut.xyz.js

@@ -0,0 +1,40 @@
+const { validEmail, wait } = require('mybase')
+const { Sentry } = require('../config')
+const { initiateEmpty, launch, emailBy_id_innerText, emailBy_querySelector_innerHTML, emailBy_id_value, emailFrom_combobox_id, emailBy_querySelector_value, emailFrom_combobox_selector, emailBy_querySelector_innerText, email_from_raw } = require('../shared')
+const vars = require('./../vars')
+const URL = 'https://10minut.xyz'
+
+function start(headless=true,waitTime=3) {
+  return new Promise(async function (resolve,reject) {
+    try {
+      let {page,browser} = await launch(headless,URL,waitTime)
+      
+      let raw = await emailBy_querySelector_innerText(page,'span.input-group-text',true)
+      let email = email_from_raw(raw)
+      // console.log(email)
+      if (browser) await browser.close()
+      if (email.length>0) return resolve({
+        email,
+        URL,
+        t:Date.now()
+      })
+      return resolve(null)
+    } catch (err) {
+      console.log(err)
+      Sentry.captureException(err)
+      resolve(false)
+    }
+  })
+}
+
+// for cli usage - development
+if (require.main==module) {
+  start(false).then(res=>console.log(res))
+}
+
+
+vars.providers.push({
+  name:'10minut.xyz',
+  init:initiateEmpty,
+  start:start
+})

package/providers/10minuteemails.com.js

@@ -0,0 +1,39 @@
+const { validEmail } = require('mybase')
+const { Sentry } = require('../config')
+const { initiateEmpty, launch, emailBy_id_innerText, emailBy_querySelector_innerHTML, emailBy_id_value, emailBy_querySelector_value } = require('../shared')
+const vars = require('./../vars')
+const URL = 'https://10minuteemails.com'
+
+function start(headless=true,waitTime=3) {
+  return new Promise(async function (resolve,reject) {
+    try {
+      let {page,browser} = await launch(headless,URL,waitTime,'input[value="Create a Random Email"]')
+      // document.querySelectorAll('form.flex input')[0].click()
+      let email = await emailBy_querySelector_innerHTML(page,'div.block')
+      
+      if (browser) await browser.close()
+      if (email) return resolve({
+        email,
+        URL,
+        t:Date.now()
+      })
+      return resolve(null)
+    } catch (err) {
+      console.log(err)
+      Sentry.captureException(err)
+      resolve(false)
+    }
+  })
+}
+
+// for cli usage - development
+if (require.main==module) {
+  start(false).then(res=>console.log(res))
+}
+
+// disabled in favor of extention
+// vars.providers.push({
+//   name:'10minuteemails.com',
+//   init:initiateEmpty,
+//   start:start
+// })