fakefilter 0.1.650 → 1.1.10

package/repo-update.js

@@ -0,0 +1,121 @@
+const fs = require('fs')
+var Table = require('easy-table')
+const winston = require('winston')
+const { getConfig, Sentry } = require('./config')
+const chalk = require('chalk')
+const vars = require('./vars')
+const argv = require('minimist')(process.argv.slice(2))
+
+// load all providers/*.js
+var glob = require('glob'), path = require('path')
+const { pullDomains, combineDomains, getProviderDetails, obfuscatedTime } = require('./shared')
+const { isLocal } = require('mybase')
+const { perDomain } = require('./apiserver/inc/shared')
+glob.sync('./providers/*.js').forEach(function (file) {
+    require(path.resolve(file));
+});
+
+/*
+this file is designed to deploy repo/ folder with data taken from mysql
+*/
+
+
+const logger = winston.createLogger({
+    level: 'info',
+    format: winston.format.json(),
+    defaultMeta: { service: 'user-service' },
+    transports: [
+        //
+        // - Write all logs with importance level of `error` or less to `error.log`
+        // - Write all logs with importance level of `info` or less to `combined.log`
+        //
+        new winston.transports.File({ filename: 'logs/repo-markdown-error.log', level: 'error' }),
+        new winston.transports.File({ filename: 'logs/repo-markdown-combined.log' }),
+        // log to Console too
+        new winston.transports.Console()
+    ],
+})
+
+
+
+function writeToFile(fn, content) {
+    // let fn = 'repo/markdown/README.md'
+    fs.writeFileSync(fn, content)
+    console.log(chalk.green(`+ file ${fn}`))
+    return true
+}
+
+async function start() {
+    try {
+        await getConfig()
+
+
+        let rows = await pullDomains(isLocal() ? 10 : 1000, logger)
+        let combined = await combineDomains(rows, true)
+        let json2domains = perDomain(combined)
+        let markdown = `# Providers\n\n`
+        let now = Math.round(Date.now() / 1000)
+        let json = {
+            version: 1,
+            t: now,
+            domains: {},
+            // flat:[]
+        }
+        let json2 = {
+            version: 2,
+            t: now,
+            domains: json2domains,
+        }
+        let updated = `Last Update: ${new Date().toISOString()}`
+        let txt = `# Fake and Temp Mail Providers\n# https://github.com/7c/fakefilter\n# ${updated}\n\n\n`
+
+        for (let provider of Object.keys(combined)) {
+            let domains = combined[provider]
+            var t = new Table()
+            console.log(chalk.bold.yellow(provider))
+            markdown += `## ${provider}\n`
+            txt += `## ${provider}\n`
+
+            for (let domain of Object.keys(domains)) {
+                let data = domains[domain]
+                domain = domain.toLowerCase()
+
+                for (let host of Object.keys(data.hosts)) {
+                    // json v1 format - which is host based format
+                    json.domains[host] = {
+                        provider,
+                        firstseen: data.hosts[host].firstseen,
+                        lastseen: data.hosts[host].lastseen,
+                    }
+                    t.cell('domain', host)
+                    markdown += `- ${host}\n`
+                    txt += `${host}\n`
+                    t.newRow()
+                }
+            }
+            markdown += `\n`
+            txt += `\n`
+            // console.log(t.toString())
+        }
+        markdown += `\n\n${updated}\n`
+        // deploy to file
+        if (!argv.dry) {
+            // markdown    
+            writeToFile(path.join(__dirname, 'repo/markdown/README.md'), markdown)
+            // json v1
+            writeToFile(path.join(__dirname, 'repo/json/data.json'), JSON.stringify(json))
+            // json v2
+            writeToFile(path.join(__dirname, 'repo/json/data_version2.json'), JSON.stringify(json2))
+            // txt
+            writeToFile(path.join(__dirname, 'repo/txt/data.txt'), txt)
+        } else console.log(chalk.yellow(`dry flag: did not deploy to files`))
+        // console.log(json)
+
+        // console.log(markdown)
+        process.exit(0)
+    } catch (err) {
+        console.log(chalk.red(err))
+    }
+}
+
+start()

package/shared.js

@@ -0,0 +1,559 @@
+const punycode = require('punycode/')
+const { Op } = require("sequelize")
+const { parseDomain, ParseResultType } = require("parse-domain")
+const argv = require('minimist')(process.argv.slice(2))
+const chalk = require('chalk')
+var dns = require('dns')
+const vars = require('./vars')
+const { wait, validEmail } = require('mybase');
+const puppeteer = require('puppeteer-extra')
+const RecaptchaPlugin = require('puppeteer-extra-plugin-recaptcha')
+var { randomUseragent, UseragentByFamily } = require('useragentsdata');
+const { Sentry, config } = require('./config');
+const StealthPlugin = require('puppeteer-extra-plugin-stealth');
+const { default: isEmail } = require("validator/lib/isEmail")
+const UA = UseragentByFamily('chrome', 'Mac OS X').ua
+const psl = require('psl')
+const { hostnameFromEmailAddress } = require("./apiserver/inc/shared")
+// console.log(chalk.bold(`Useragent:${UA}`))
+
+function getDomainPart(hostname) {
+  const parseResult = parseDomain(hostname)
+  const domain = parseResult.domain + '.' + parseResult.topLevelDomains.join('.')
+  if (hostname !== domain) {
+    // console.log(chalk.red(`Is this a subdomain ? `),parseResult,domain,hostname)
+    // collect+=`${chalk.yellow(domain)} <> ${chalk.green(hostname)}\n`
+    // process.exit(0)
+    return domain
+  }
+  return hostname
+}
+
+
+async function initiateEmpty() {
+  return new Promise(async function (resolve, reject) {
+    return resolve(true)
+  })
+}
+
+const promiseTimeout = function (ms, promise) {
+  // Create a promise that rejects in <ms> milliseconds
+  let timeout = new Promise((resolve, reject) => {
+    let id = setTimeout(() => {
+      clearTimeout(id);
+      reject('TIMEDOUT')
+    }, ms)
+  })
+
+  // Returns a race between our timeout and the passed in promise
+  return Promise.race([
+    promise,
+    timeout
+  ])
+}
+
+function launch(headless, url, waitTime = 3, CLICK_SELECTOR = false, captchaPlugin = false, waitUntil = 'load') {
+  return new Promise(async function (resolve, reject) {
+    try {
+      if (captchaPlugin) puppeteer.use(
+        RecaptchaPlugin({
+          provider: {
+            id: '2captcha',
+            token: config.captcha_resolver_api
+          },
+          visualFeedback: true // colorize reCAPTCHAs (violet = detected, green = solved)
+        })
+      )
+      puppeteer.use(StealthPlugin())
+      const browser = await puppeteer.launch({
+        args: ['--no-sandbox'],
+        // args: [
+        //   // '--disable-features=IsolateOrigins,site-per-process',
+        //   // '--flag-switches-begin --disable-site-isolation-trials --flag-switches-end',
+        //   '--disable-web-security',
+        //   '--no-sandbox'
+        // ],
+        defaultViewport: null,
+        headless: headless ? true : false
+      })
+      vars.browsers.push(browser)
+      const page = await browser.newPage()
+      await page.setUserAgent(UA)
+      await page.goto(url, { waitUntil });
+      if (argv.ss) await page.screenshot({ path: 'screenshots/goto.png', fullPage: true })
+
+      await wait(waitTime)
+      if (CLICK_SELECTOR) {
+
+        // await page.waitForNavigation()
+        try {
+          // we use puppeeter click function
+          await page.click(CLICK_SELECTOR)
+        } catch (err3) {
+          // if it fails we click via javascript
+          await page.evaluate((CLICK_SELECTOR) => {
+            let btn = document.querySelector(CLICK_SELECTOR)
+            if (btn) {
+              btn.click()
+              return true
+            }
+            return false
+          }, CLICK_SELECTOR)
+        }
+
+        // await page.waitForNavigation()
+        await wait(waitTime)
+        if (argv.ss) await page.screenshot({ path: 'screenshots/afterclick.png', fullPage: true })
+      }
+      return resolve({ page, browser })
+    } catch (err) {
+      console.log(err)
+      console.log(`Browser handels:${vars.browsers.length} UA:${UA}`)
+      Sentry.captureException(err)
+      resolve(false)
+    }
+  })
+}
+
+function resolveMx(host) {
+  return new Promise((resolve, reject) => {
+    dns.resolveMx(host, (err, addresses) => {
+      if (err) return resolve(false)
+      resolve(addresses)
+    })
+  })
+}
+
+
+async function emailBy_querySelector_innerText(page, Selector, raw = false) {
+  if (!page) return false
+  let got = await page.evaluate((Selector) => {
+    let response = []
+    for (let item of document.querySelectorAll(Selector))
+      response.push(item.innerText)
+    return response
+  }, Selector)
+  // await page.screenshot({path: 'screenshots/after.png', fullPage: true})
+  if (raw) return got
+  // console.log(got)
+  for (let item of got)
+    if (validEmail(item.trim())) return item.toLowerCase().trim()
+  return false
+}
+
+async function emailBy_querySelector_innerHTML(page, Selector, raw = false) {
+  if (!page) return false
+  let got = await page.evaluate((Selector) => {
+    let response = []
+    for (let item of document.querySelectorAll(Selector))
+      response.push(item.innerHTML)
+    return response
+  }, Selector)
+  // await page.screenshot({path: 'screenshots/after.png', fullPage: true})
+  if (raw) return got
+  // console.log(got)
+  for (let item of got)
+    if (validEmail(item.trim())) return item.toLowerCase().trim()
+  return false
+}
+
+
+async function emailBy_querySelector_placeholder(page, Selector, raw = false) {
+  if (!page) return false
+  let got = await page.evaluate((Selector) => {
+    let response = []
+    for (let item of document.querySelectorAll(Selector))
+      response.push(item.placeholder)
+    return response
+  }, Selector)
+  if (raw) return got
+  for (let item of got)
+    if (validEmail(item.trim())) return item.toLowerCase().trim()
+  return false
+}
+
+async function emailBy_querySelector_value(page, Selector, raw = false) {
+  if (!page) return false
+  let got = await page.evaluate((Selector) => {
+    let response = []
+    for (let item of document.querySelectorAll(Selector))
+      response.push(item.value)
+    return response
+  }, Selector)
+  if (raw) return got
+  for (let item of got)
+    if (validEmail(item.trim())) return item.toLowerCase().trim()
+  return false
+}
+
+async function emailBy_id_value(page, tagId) {
+  if (!page) return false
+  let got = await page.evaluate((tagId) => {
+    return document.getElementById(tagId).value
+  }, tagId)
+
+  if (validEmail(got.trim())) return got.toLowerCase().trim()
+  return false
+}
+
+async function emailBy_id_innerText(page, tagId, raw = false) {
+  if (!page) return false
+  let got = await page.evaluate((tagId) => {
+    console.log(`>`, tagId)
+    return document.getElementById(tagId).innerText
+  }, tagId)
+  // console.log(`emailBy_id_innerText`,got)
+  if (raw) return got
+  if (validEmail(got.trim())) return got.toLowerCase().trim()
+  return false
+}
+
+
+function findByIdValue(headless, url, tagId, waitTime = 3) {
+  async function pull(page, theid) {
+
+    let got = await page.evaluate((theid) => {
+
+      return document.getElementById(theid).value
+    }, tagId)
+
+    if (validEmail(got.trim())) return got.trim()
+
+    return false
+  }
+
+  return new Promise(async function (resolve, reject) {
+    try {
+      let { page, browser } = await launch(headless, url, waitTime)
+      let emailAddr = await pull(page, tagId)
+      await browser.close()
+      if (emailAddr) return resolve({
+        email: emailAddr,
+        url,
+        t: Date.now()
+      })
+      return resolve(null)
+    } catch (err) {
+      console.log(err)
+      Sentry.captureException(err)
+      resolve(false)
+    }
+  })
+}
+
+function emailFrom_combobox_selector(page, Selector, replaceRX = false) {
+  return new Promise(async function (resolve, reject) {
+    if (!page) return resolve(false)
+    let got = await page.evaluate((Selector) => {
+      let response = []
+      // console.log(`>>`,Selector)
+      // console.log(document.querySelectorAll(Selector))
+      for (let option of document.querySelectorAll(Selector)[0].children) {
+        console.log(`option`, option)
+        let txt = option.innerText.toLowerCase().trim()
+        response.push(`any@` + txt)
+      }
+      return response
+    }, Selector)
+    let email = []
+    for (let item of got) {
+      if (replaceRX) item = item.replace(replaceRX, '').toLowerCase().trim()
+      if (item && validEmail(item.toLowerCase().trim())) email.push(item.toLowerCase().trim())
+    }
+
+    if (email.length > 0) return resolve(email)
+    return resolve(false)
+  })
+}
+
+function emailFrom_combobox_id(page, elementID, replaceRX = false) {
+  return new Promise(async function (resolve, reject) {
+    if (!page) return resolve(false) // in case page could not be loaded
+    let got = await page.evaluate((elementID) => {
+      // console.log(`emailendID`, elementID)
+
+      let response = []
+      for (let option of document.getElementById(elementID).children) {
+        // console.log(`option`,option)
+        let txt = option.innerText.toLowerCase().trim()
+        if (txt.search(/^@/) == 0) response.push(`any` + txt); else
+          response.push(`any@` + txt)
+      }
+      return response
+    }, elementID)
+    // console.log(got)
+    let email = []
+    for (let item of got) {
+      if (validEmail(item)) {
+        email.push(item)
+        continue
+      }
+      if (replaceRX) item = item.replace(replaceRX, '').toLowerCase().trim()
+      if (item && validEmail(item.toLowerCase().trim())) email.push(item.toLowerCase().trim())
+    }
+
+    if (email.length > 0) return resolve(email)
+    return resolve(false)
+  })
+}
+
+
+
+function email_from_raw(rawArray) {
+  let email = []
+  if (Array.isArray(rawArray))
+    for (let i of rawArray) {
+      i = i.trim()
+      if (validEmail(`any@${i}`) && !email.includes(`any@${i}`)) email.push(`any@${i}`);
+      else
+        if (validEmail(`any${i}`) && !email.includes(`any${i}`)) email.push(`any${i}`);
+    }
+
+  if (email.length > 0) return email
+  return null
+}
+
+
+
+function pullEventsByProvider(providerName, daysago = 60) {
+  return new Promise(async function (resolve, reject) {
+    let since = Date.now() - daysago * 24 * 60 * 60 * 1000
+    try {
+      let rows = await vars.models.Events.findAll({
+        // attributes:['provider'],
+        where: { t: { [Op.gt]: since }, provider: providerName },
+        raw: true
+      })
+
+      return resolve(rows)
+    } catch (err) {
+      console.log(err)
+      if (logger) logger.error(err)
+      Sentry.captureException(err)
+    }
+    return resolve(false)
+  })
+}
+
+
+function pullEventsProviders(logger, daysago = 60) {
+  return new Promise(async function (resolve, reject) {
+    let since = Date.now() - daysago * 24 * 60 * 60 * 1000
+    try {
+      let rows = await vars.models.Events.findAll({
+        attributes: ['provider'],
+        where: { t: { [Op.gt]: since } },
+        raw: true
+      })
+      let providers = []
+      rows.map(r => !providers.includes(r.provider) && providers.push(r.provider))
+
+      return resolve(providers.sort())
+    } catch (err) {
+      console.log(err)
+      if (logger) logger.error(err)
+      Sentry.captureException(err)
+    }
+    return resolve(false)
+  })
+}
+
+
+function pullDomains(days = 600, logger) {
+  return new Promise(async function (resolve, reject) {
+    let yearago = Date.now() - days * 24 * 60 * 60 * 1000
+    try {
+      let rows = await vars.models.Harvested.findAll({
+        where: { t: { [Op.gt]: yearago } },
+        order: [["t", "DESC"]],
+        raw: true
+      })
+
+      return resolve(rows)
+    } catch (err) {
+      console.log(err)
+      if (logger) logger.error(err)
+      Sentry.captureException(err)
+    }
+    return resolve(false)
+  })
+}
+
+function getProviderDetails(name) {
+  for (let provider of vars.providers) {
+    if (provider.name === name) return provider
+  }
+  return null
+}
+
+function combineDomains(rows,obfuscateTime=true) {
+  return new Promise(async function (resolve, reject) {
+    let result = {} // provider/domain
+    const progress = require('progressbar').create().step('combining')
+    progress.setTotal(rows.length)
+    for (let row of rows) {
+      // detect IDN domains
+      let pc = punycode.toASCII(row.domain)
+      if (pc!==row.domain) {
+        console.log(`IDN Domain found: ${row.domain} provider: ${row.provider}`)
+        row.unicodeDomain = row.domain
+        row.domain = pc
+      }
+
+      await wait(1 / 1000)
+      progress.addTick()
+      let hostname = row.domain.toLowerCase()
+      // parse domains with psl
+      let domainparse = psl.parse(hostname)
+      if (!domainparse) {
+        console.log(`Not parseable hostname : ${hostname} - detected id: ${row.id} - provider: ${row.provider}`)
+        continue
+      }
+      // lets take the domain of the hostname
+      let domain = domainparse.domain
+      let domain_pc = punycode.toASCII(domain)
+
+      // not listed tld found, not listed tlds are made up ones
+      if (!domainparse.listed) {
+        console.log(`Not listed TLD: ${domain} - detected id: ${row.id} - provider: ${row.provider} `)
+        // what to do with those
+      }
+      // ignore whitelisted domains
+      if (vars.config.ignore_domains.includes(domain)) {
+        try { await vars.models.Harvested.destroy({ where: { id: row.id } }) } catch (_) { } // DELETE
+        console.log(chalk.red(`Ignoring whitelisted domain:${domain} at row: ${row.id} provider:${row.provider} - DELETED`))
+        continue
+      }
+
+      // make sure email address is valid by validator library
+      if (!isEmail(`any@${domain}`)) {
+        try { await vars.models.Harvested.destroy({ where: { id: row.id } }) } catch (_) { } // DELETE
+        console.log(`Invalid domain: ${domain} - detected id: ${row.id} - DELETED`)
+        continue
+      }
+
+      // create the provider
+      if (!result.hasOwnProperty(row.provider)) result[row.provider] = {}
+      
+      // create the domain inside provider
+      if (!result[row.provider].hasOwnProperty(domain))
+        result[row.provider][domain] = {
+          hosts:{[hostname]:{
+            firstseen: 5645821813648,
+            lastseen: 0,
+          }}
+        }
+        else {
+          if (!result[row.provider][domain].hosts.hasOwnProperty(hostname))
+            result[row.provider][domain].hosts[hostname]={
+              firstseen: 5645821813648,
+              lastseen: 0,
+            }
+        }
+
+        if (row.unicodeDomain) 
+          result[row.provider][domain].unicodeDomain = punycode.toUnicode(domain)
+      
+      // result[row.provider][domain].isSubdomain = domainparse.domain !== hostname ? true : false
+      // if (result[row.provider][domain].isSubdomain) {
+      //   result[row.provider][domain].domain = domainparse.domain
+      //   result[row.provider][domain].tld = domainparse.tld
+      //   // console.log(`provider: ${row.provider} hostname:${hostname} - domain:${domainparse.domain}`)
+      // }
+
+      // let providerDetails = getProviderDetails(row.provider)
+      // if (providerDetails && providerDetails.randomSubdomain === true) result[row.provider][domain].randomSubdomain = true
+      let t = obfuscateTime ? obfuscatedTime(row.t) : row.t
+      // firstseen of host
+      if (t < result[row.provider][domain].hosts[hostname].firstseen) result[row.provider][domain].hosts[hostname].firstseen = t
+      // lastseen
+      if (t > result[row.provider][domain].hosts[hostname].lastseen) result[row.provider][domain].hosts[hostname].lastseen = t
+    }
+    return resolve(result)
+  })
+}
+
+function obfuscatedTime(dt) {
+  return Math.round(dt / 10000000) * 10000
+}
+
+function domainFromEmail(email) {
+  let domPart = email.split(/@/)[1]
+  return domPart
+}
+
+
+function eventsOfProvider(providerName, days = 7) {
+  const { Op } = require("sequelize")
+  return new Promise(async function (resolve, reject) {
+    try {
+      let got = await vars.models.Events.findAll({
+        where: {
+          provider: providerName,
+          createdAt: {
+            [Op.gt]: new Date(new Date() - days * 24 * 60 * 60 * 1000)
+          }
+        }, raw: true
+      })
+      return resolve(got)
+    } catch (err) {
+      console.log(err)
+      Sentry.captureException(err)
+      return resolve(false)
+    }
+
+  })
+}
+
+function is_domain_banned(hostname_or_email, current_filter = {}) {
+  // determins if the main domain of this hostname is banned
+  let hostname
+  // lets parse the hostname from email address
+  if (hostname_or_email && typeof hostname_or_email === 'string' && hostname_or_email.search(/^.*@.+$/) === 0)
+    hostname = hostnameFromEmailAddress(hostname_or_email);
+  else hostname = hostname_or_email
+
+  // lets work with the hostname
+  if (hostname && typeof hostname === 'string') {
+    hostname = hostname.toLowerCase().trim()
+    let domainparse = psl.parse(hostname)
+    if (domainparse && typeof domainparse.domain === 'string' && domainparse.domain.length > 0) {
+      return current_filter.hasOwnProperty(domainparse.domain) ? current_filter[domainparse.domain] : false
+    }
+  }
+  return null
+}
+
+
+module.exports = {
+  findByIdValue,
+  initiateEmpty,
+  launch,
+
+
+  emailBy_id_innerText,
+  emailBy_querySelector_innerHTML,
+  emailBy_querySelector_innerText,
+  emailBy_querySelector_placeholder,
+  emailBy_querySelector_value,
+  emailBy_id_value,
+
+  emailFrom_combobox_id,
+  emailFrom_combobox_selector,
+
+  email_from_raw,
+
+  resolveMx,
+  promiseTimeout,
+  getDomainPart,
+
+  combineDomains,
+  getProviderDetails,
+  pullDomains,
+  pullEventsProviders,
+  pullEventsByProvider,
+  obfuscatedTime,
+  domainFromEmail,
+  eventsOfProvider,
+  is_domain_banned
+}