fakefilter 1.1.10 → 1.1.12

package/providers/temporary-email.org.js

@@ -0,0 +1,36 @@
+
+const { Sentry } = require('../config')
+const { initiateEmpty, launch, emailBy_id_innerText, emailBy_querySelector_innerHTML, emailBy_id_value } = require('../shared')
+const vars = require('./../vars')
+const URL = 'https://temporary-email.org'
+
+function start(headless=true,waitTime=3) {
+  return new Promise(async function (resolve,reject) {
+    try {
+      let {page,browser} = await launch(headless,URL,waitTime)
+      let email = await emailBy_id_value(page,'trsh_mail')
+      if (browser) await browser.close()
+      if (email) return resolve({
+        email,
+        URL,
+        t:Date.now()
+      })
+      return resolve(null)
+    } catch (err) {
+      console.log(err)
+      Sentry.captureException(err)
+      resolve(false)
+    }
+  })
+}
+
+// for cli usage - development
+if (require.main==module) {
+  start(false).then(res=>console.log(res))
+}
+
+vars.providers.push({
+  name:'temporary-email.org',
+  init:initiateEmpty,
+  start:start
+})

package/providers/tuamaeaquelaursa.com.js

@@ -0,0 +1,25 @@
+const { findByIdValue, initiateEmpty } = require('../shared')
+const vars = require('./../vars')
+const URL = 'https://tuamaeaquelaursa.com/'
+
+function start(headless = true, waitTime = 3) {
+  return new Promise(async function (resolve, reject) {
+    return resolve({
+      email: [`any@tuamaeaquelaursa.com`],
+      URL,
+      t: Date.now()
+    })
+  })
+}
+
+
+// for cli usage - development
+if (require.main == module) {
+  start(false).then(res => console.log(res))
+}
+
+vars.providers.push({
+  name: 'tuamaeaquelaursa.com',
+  init: initiateEmpty,
+  start: start
+})

package/providers/youxiang.dev.js

@@ -0,0 +1,25 @@
+const { findByIdValue, initiateEmpty } = require('../shared')
+const vars = require('./../vars')
+const URL = 'https://youxiang.dev/'
+
+function start(headless = true, waitTime = 3) {
+  return new Promise(async function (resolve, reject) {
+    return resolve({
+      email: [`any@youxiang.dev`],
+      URL,
+      t: Date.now()
+    })
+  })
+}
+
+
+// for cli usage - development
+if (require.main == module) {
+  start(false).then(res => console.log(res))
+}
+
+vars.providers.push({
+  name: 'youxiang.dev',
+  init: initiateEmpty,
+  start: start
+})

package/repo-update.js

@@ -49,11 +49,13 @@ async function start() {
     try {
         await getConfig()
 
+        // we only pull domains harvested last 365 days, so domains
+        let rows = await pullDomains(isLocal() ? 10 : 600, logger)
+        let [ combined,expiredDomains ] = await combineDomains(rows, true, 365)
 
-        let rows = await pullDomains(isLocal() ? 10 : 1000, logger)
-        let combined = await combineDomains(rows, true)
-        let json2domains = perDomain(combined)
+        let json2domains = perDomain(combined,true)
         let markdown = `# Providers\n\n`
+        let markdownExpired = `# Providers Expired Domains\nOur service removes domains from fakefilter after a year of inactivity. If these domains reappear, they are reinstated in our database for a minimum of 365 days.\n\n`
         let now = Math.round(Date.now() / 1000)
         let json = {
             version: 1,
@@ -69,39 +71,50 @@ async function start() {
         let updated = `Last Update: ${new Date().toISOString()}`
         let txt = `# Fake and Temp Mail Providers\n# https://github.com/7c/fakefilter\n# ${updated}\n\n\n`
 
+        // generate json v1 & markdown
         for (let provider of Object.keys(combined)) {
             let domains = combined[provider]
-            var t = new Table()
+            // var t = new Table()
             console.log(chalk.bold.yellow(provider))
             markdown += `## ${provider}\n`
+            markdownExpired += `## ${provider}\n`
             txt += `## ${provider}\n`
 
             for (let domain of Object.keys(domains)) {
                 let data = domains[domain]
                 domain = domain.toLowerCase()
+                let lastSeenAgo = Math.floor((now - data.lastseen)/60/60/24) // days
 
                 for (let host of Object.keys(data.hosts)) {
                     // json v1 format - which is host based format
-                    json.domains[host] = {
-                        provider,
-                        firstseen: data.hosts[host].firstseen,
-                        lastseen: data.hosts[host].lastseen,
+                    if (!data.expired) {
+                        json.domains[host] = {
+                            provider,
+                            firstseen: data.hosts[host].firstseen,
+                            lastseen: data.hosts[host].lastseen,
+                        }
+                        markdown += `- ${host} (lastseen ${lastSeenAgo} days ago)\n`
+                        txt += `${host}\n`
+                    } else {
+                        markdownExpired += `- ${host} (expired ${lastSeenAgo} days ago)\n`
                     }
-                    t.cell('domain', host)
-                    markdown += `- ${host}\n`
-                    txt += `${host}\n`
-                    t.newRow()
+                    
+                    
+                    // t.newRow()
                 }
             }
             markdown += `\n`
+            markdownExpired += `\n`
             txt += `\n`
             // console.log(t.toString())
         }
         markdown += `\n\n${updated}\n`
         // deploy to file
         if (!argv.dry) {
-            // markdown    
+            // markdown not expired
             writeToFile(path.join(__dirname, 'repo/markdown/README.md'), markdown)
+            // markdown expired
+            writeToFile(path.join(__dirname, 'repo/markdown/EXPIRED.md'),markdownExpired)
             // json v1
             writeToFile(path.join(__dirname, 'repo/json/data.json'), JSON.stringify(json))
             // json v2

package/shared.js

@@ -1,3 +1,4 @@
+const debug = require('debug')('_shared')
 const punycode = require('punycode/')
 const { Op } = require("sequelize")
 const { parseDomain, ParseResultType } = require("parse-domain")
@@ -5,7 +6,7 @@ const argv = require('minimist')(process.argv.slice(2))
 const chalk = require('chalk')
 var dns = require('dns')
 const vars = require('./vars')
-const { wait, validEmail } = require('mybase');
+const { wait, validEmail, isLocal, utcnow } = require('mybase');
 const puppeteer = require('puppeteer-extra')
 const RecaptchaPlugin = require('puppeteer-extra-plugin-recaptcha')
 var { randomUseragent, UseragentByFamily } = require('useragentsdata');
@@ -52,7 +53,8 @@ const promiseTimeout = function (ms, promise) {
   ])
 }
 
-function launch(headless, url, waitTime = 3, CLICK_SELECTOR = false, captchaPlugin = false, waitUntil = 'load') {
+function launch(headless, url, waitTime = 3, CLICK_SELECTOR = false, captchaPlugin = false, waitUntil = 'load', socks5 = '') {
+  debug(`launching ${url}`)
   return new Promise(async function (resolve, reject) {
     try {
       if (captchaPlugin) puppeteer.use(
@@ -65,8 +67,18 @@ function launch(headless, url, waitTime = 3, CLICK_SELECTOR = false, captchaPlug
         })
       )
       puppeteer.use(StealthPlugin())
+      debug(`launching browser`)
+
+      const args = ['--no-sandbox']
+      if (argv.socks5 || socks5 !== '') {
+        argv.socks5 = argv.socks5 || socks5
+        debug(`using socks5 proxy ${argv.socks5}`)
+        args.push(`--proxy-server=socks5://${argv.socks5}`)
+      }
+
       const browser = await puppeteer.launch({
-        args: ['--no-sandbox'],
+        args,
+        devtools: isLocal(), // This line opens the DevTools
         // args: [
         //   // '--disable-features=IsolateOrigins,site-per-process',
         //   // '--flag-switches-begin --disable-site-isolation-trials --flag-switches-end',
@@ -77,14 +89,23 @@ function launch(headless, url, waitTime = 3, CLICK_SELECTOR = false, captchaPlug
         headless: headless ? true : false
       })
       vars.browsers.push(browser)
+      debug(`opening new page`)
       const page = await browser.newPage()
+      debug(`setting useragent`)
       await page.setUserAgent(UA)
+      debug(`going to ${url} waituntil: ${waitUntil}`)
       await page.goto(url, { waitUntil });
-      if (argv.ss) await page.screenshot({ path: 'screenshots/goto.png', fullPage: true })
 
+      if (argv.ss) {
+        debug(`taking screenshot as goto.png`)
+        await page.screenshot({ path: 'screenshots/goto.png', fullPage: true })
+      }
+
+      debug(`waiting ${waitTime} seconds`)
       await wait(waitTime)
+      debug(`waiting ended`)
       if (CLICK_SELECTOR) {
-
+        debug(`clicking ${CLICK_SELECTOR}`)
         // await page.waitForNavigation()
         try {
           // we use puppeeter click function
@@ -103,8 +124,13 @@ function launch(headless, url, waitTime = 3, CLICK_SELECTOR = false, captchaPlug
 
         // await page.waitForNavigation()
         await wait(waitTime)
-        if (argv.ss) await page.screenshot({ path: 'screenshots/afterclick.png', fullPage: true })
+
+      }
+      if (argv.ss) {
+        debug(`taking screenshot as afterclick.png`)
+        await page.screenshot({ path: 'screenshots/afterclick.png', fullPage: true })
       }
+      debug(`finished launching`)
       return resolve({ page, browser })
     } catch (err) {
       console.log(err)
@@ -126,6 +152,7 @@ function resolveMx(host) {
 
 
 async function emailBy_querySelector_innerText(page, Selector, raw = false) {
+  debug(`emailBy_querySelector_innerText '${Selector}'`)
   if (!page) return false
   let got = await page.evaluate((Selector) => {
     let response = []
@@ -133,6 +160,7 @@ async function emailBy_querySelector_innerText(page, Selector, raw = false) {
       response.push(item.innerText)
     return response
   }, Selector)
+  debug(`got ${got}`)
   // await page.screenshot({path: 'screenshots/after.png', fullPage: true})
   if (raw) return got
   // console.log(got)
@@ -141,7 +169,21 @@ async function emailBy_querySelector_innerText(page, Selector, raw = false) {
   return false
 }
 
+async function clickto_element_withText(page, element, text) {
+  debug(`clickto_element_withText ${element} ${text}`)
+  if (argv.ss) await page.screenshot({ path: 'screenshots/clickto_element_withText.png', fullPage: true })
+  const got = await page.evaluate((element, text) => {
+    const spanElement = Array.from(document.querySelectorAll(element))
+      .find(span => span.textContent.trim() === text);
+
+    if (spanElement) spanElement.click()
+    return spanElement
+  }, element, text)
+  debug(`clickto_element_withText ${got}`)
+}
+
 async function emailBy_querySelector_innerHTML(page, Selector, raw = false) {
+  debug(`emailBy_querySelector_innerHTML '${Selector}'`)
   if (!page) return false
   let got = await page.evaluate((Selector) => {
     let response = []
@@ -173,6 +215,7 @@ async function emailBy_querySelector_placeholder(page, Selector, raw = false) {
 }
 
 async function emailBy_querySelector_value(page, Selector, raw = false) {
+  debug(`emailBy_querySelector_value '${Selector}'`)
   if (!page) return false
   let got = await page.evaluate((Selector) => {
     let response = []
@@ -180,22 +223,58 @@ async function emailBy_querySelector_value(page, Selector, raw = false) {
       response.push(item.value)
     return response
   }, Selector)
+  debug(`got '${got}'`)
   if (raw) return got
   for (let item of got)
-    if (validEmail(item.trim())) return item.toLowerCase().trim()
+    if (item && validEmail(item.trim())) return item.toLowerCase().trim()
   return false
 }
 
 async function emailBy_id_value(page, tagId) {
-  if (!page) return false
+  debug(`emailBy_id_value ${tagId} ${page}`)
+  if (!page) {
+    debug(`no page found`)
+    return false
+  }
+  debug(`page.evaluate '${tagId}'`)
   let got = await page.evaluate((tagId) => {
-    return document.getElementById(tagId).value
+    return document.getElementById(tagId)?.value
   }, tagId)
-
-  if (validEmail(got.trim())) return got.toLowerCase().trim()
+  debug(`got ${got}`)
+  if (validEmail(got?.trim())) return got.toLowerCase().trim()
   return false
 }
 
+async function email_recursive_search(page, selector) {
+  if (!page) return false
+  let emails = await page.evaluate((selector) => {
+    function findAllEmails(element, regex, emails = []) {
+      // Check the current element
+      if (regex.test(element.innerText)) {
+        if (!emails.includes(element.innerText)) emails.push(element.innerText);
+      }
+
+      // Recursively search in each child
+      for (let child of element.children) {
+        findAllEmails(child, regex, emails);
+      }
+
+      return emails;
+    }
+
+    let emailRegex = /^((([a-z]|\d|[!#\$%&'\*\+\-\/=\?\^_`{\|}~]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])+(\.([a-z]|\d|[!#\$%&'\*\+\-\/=\?\^_`{\|}~]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])+)*)|((\x22)((((\x20|\x09)*(\x0d\x0a))?(\x20|\x09)+)?(([\x01-\x08\x0b\x0c\x0e-\x1f\x7f]|\x21|[\x23-\x5b]|[\x5d-\x7e]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(\\([\x01-\x09\x0b\x0c\x0d-\x7f]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF]))))*(((\x20|\x09)*(\x0d\x0a))?(\x20|\x09)+)?(\x22)))@((([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])*([a-z]|\d|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])))\.)+(([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])|(([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])([a-z]|\d|-|\.|_|~|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])*([a-z]|[\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF])))$/i;
+    // https://stackoverflow.com/questions/46155/how-can-i-validate-an-email-address-in-javascript
+    let emails = []
+    for (let item of document.querySelectorAll(selector)) {
+      findAllEmails(item, emailRegex, emails);
+    }
+    return emails
+  }, selector)
+  emails = emails.filter(e => validEmail(e))
+  return emails
+}
+
+
 async function emailBy_id_innerText(page, tagId, raw = false) {
   if (!page) return false
   let got = await page.evaluate((tagId) => {
@@ -241,7 +320,8 @@ function findByIdValue(headless, url, tagId, waitTime = 3) {
   })
 }
 
-function emailFrom_combobox_selector(page, Selector, replaceRX = false) {
+function emailFrom_combobox_selector(page, Selector, replaceRX = false, raw = false) {
+  debug(`emailFrom_combobox_selector ${Selector}`)
   return new Promise(async function (resolve, reject) {
     if (!page) return resolve(false)
     let got = await page.evaluate((Selector) => {
@@ -249,12 +329,15 @@ function emailFrom_combobox_selector(page, Selector, replaceRX = false) {
       // console.log(`>>`,Selector)
       // console.log(document.querySelectorAll(Selector))
       for (let option of document.querySelectorAll(Selector)[0].children) {
-        console.log(`option`, option)
         let txt = option.innerText.toLowerCase().trim()
-        response.push(`any@` + txt)
+        if (txt.search(/^@/) == 0) response.push(`any` + txt); else
+          response.push(`any@` + txt)
+        console.log(`option`, option, response)
       }
       return response
     }, Selector)
+    debug(`got ${got}`)
+    // if (raw) return got
     let email = []
     for (let item of got) {
       if (replaceRX) item = item.replace(replaceRX, '').toLowerCase().trim()
@@ -386,90 +469,180 @@ function getProviderDetails(name) {
   return null
 }
 
-function combineDomains(rows,obfuscateTime=true) {
+function combineDomains(rows, obfuscateTime = true, expiration_days = 2 * 365) {
+  let profile = {
+    punycode: 0,
+    domainparse: 0,
+    progressbar: 0,
+    whitelistcheck: 0,
+    emailvalidation: 0,
+    total: 0
+  }
+  let domainparse_cache = {
+
+  }
   return new Promise(async function (resolve, reject) {
     let result = {} // provider/domain
     const progress = require('progressbar').create().step('combining')
     progress.setTotal(rows.length)
+    const started = Date.now()
+    let cnt = 0
     for (let row of rows) {
+      // if (row.domain!=='maitrimony.com') continue
+      cnt++
+
+
       // detect IDN domains
+      let time0 = Date.now()
       let pc = punycode.toASCII(row.domain)
-      if (pc!==row.domain) {
+      if (pc !== row.domain) {
         console.log(`IDN Domain found: ${row.domain} provider: ${row.provider}`)
         row.unicodeDomain = row.domain
         row.domain = pc
       }
+      profile.punycode += Date.now() - time0
 
-      await wait(1 / 1000)
-      progress.addTick()
+      time0 = Date.now()
+      if (Math.random() * 500 < 1) {
+        await wait(1 / 10000)
+        progress.setTick(cnt)
+        // if (cnt>100000) break
+      }
+      profile.progressbar += Date.now() - time0
+
+
+      // parse domains with psl
+      time0 = Date.now()
       let hostname = row.domain.toLowerCase()
+      let domainparse
       // parse domains with psl
-      let domainparse = psl.parse(hostname)
-      if (!domainparse) {
-        console.log(`Not parseable hostname : ${hostname} - detected id: ${row.id} - provider: ${row.provider}`)
+      if (domainparse_cache.hasOwnProperty(hostname)) {
+        domainparse = domainparse_cache[hostname]
+      } else {
+        domainparse = psl.parse(hostname)
+      }
+
+      if (!domainparse || !domainparse?.domain) {
+        console.log(`Not parseable hostname : ${hostname} - detected id: ${row.id} - provider: ${row.provider} - ${chalk.red('deleting')}`)
+        try { await vars.models.Harvested.destroy({ where: { id: row.id } }) } catch (_) { } // DELETE
         continue
       }
+      domainparse_cache[hostname] = domainparse
+
+
+
       // lets take the domain of the hostname
       let domain = domainparse.domain
-      let domain_pc = punycode.toASCII(domain)
+      // let domain_pc = punycode.toASCII(domain)
+      profile.domainparse += Date.now() - time0
 
       // not listed tld found, not listed tlds are made up ones
       if (!domainparse.listed) {
         console.log(`Not listed TLD: ${domain} - detected id: ${row.id} - provider: ${row.provider} `)
         // what to do with those
       }
+
+
+
       // ignore whitelisted domains
+      time0 = Date.now()
       if (vars.config.ignore_domains.includes(domain)) {
         try { await vars.models.Harvested.destroy({ where: { id: row.id } }) } catch (_) { } // DELETE
         console.log(chalk.red(`Ignoring whitelisted domain:${domain} at row: ${row.id} provider:${row.provider} - DELETED`))
         continue
       }
+      profile.whitelistcheck += Date.now() - time0
+
 
       // make sure email address is valid by validator library
+      time0 = Date.now()
       if (!isEmail(`any@${domain}`)) {
         try { await vars.models.Harvested.destroy({ where: { id: row.id } }) } catch (_) { } // DELETE
         console.log(`Invalid domain: ${domain} - detected id: ${row.id} - DELETED`)
         continue
       }
+      profile.emailvalidation += Date.now() - time0
 
       // create the provider
       if (!result.hasOwnProperty(row.provider)) result[row.provider] = {}
-      
+
       // create the domain inside provider
       if (!result[row.provider].hasOwnProperty(domain))
         result[row.provider][domain] = {
-          hosts:{[hostname]:{
-            firstseen: 5645821813648,
-            lastseen: 0,
-          }}
-        }
-        else {
-          if (!result[row.provider][domain].hosts.hasOwnProperty(hostname))
-            result[row.provider][domain].hosts[hostname]={
+          // domain level firstseen & lastseen
+          firstseen: 5645821813648,
+          lastseen: 0,
+
+          hosts: {
+            [hostname]: {
               firstseen: 5645821813648,
               lastseen: 0,
             }
+          }
         }
+      else {
+        if (!result[row.provider][domain].hosts.hasOwnProperty(hostname))
+          result[row.provider][domain].hosts[hostname] = {
+            firstseen: 5645821813648,
+            lastseen: 0,
+          }
+      }
+
+      if (row.unicodeDomain)
+        result[row.provider][domain].unicodeDomain = punycode.toUnicode(domain)
 
-        if (row.unicodeDomain) 
-          result[row.provider][domain].unicodeDomain = punycode.toUnicode(domain)
-      
-      // result[row.provider][domain].isSubdomain = domainparse.domain !== hostname ? true : false
-      // if (result[row.provider][domain].isSubdomain) {
-      //   result[row.provider][domain].domain = domainparse.domain
-      //   result[row.provider][domain].tld = domainparse.tld
-      //   // console.log(`provider: ${row.provider} hostname:${hostname} - domain:${domainparse.domain}`)
-      // }
-
-      // let providerDetails = getProviderDetails(row.provider)
-      // if (providerDetails && providerDetails.randomSubdomain === true) result[row.provider][domain].randomSubdomain = true
       let t = obfuscateTime ? obfuscatedTime(row.t) : row.t
-      // firstseen of host
+
+      // firstseen of domain
+      if (t < result[row.provider][domain].firstseen) result[row.provider][domain].firstseen = t
+      // lastseen update of domain
+      if (t > result[row.provider][domain].lastseen) result[row.provider][domain].lastseen = t
+
+      // firstseen of hostname
       if (t < result[row.provider][domain].hosts[hostname].firstseen) result[row.provider][domain].hosts[hostname].firstseen = t
-      // lastseen
+
+      // lastseen update of hostname
       if (t > result[row.provider][domain].hosts[hostname].lastseen) result[row.provider][domain].hosts[hostname].lastseen = t
     }
-    return resolve(result)
+    profile.total = Date.now() - started
+    console.log(profile)
+
+    // routine to detect and remove expired domains from the results
+    // based on lastseen property
+    let expiredDomains = {
+
+    }
+    if (expiration_days > 0) {
+      for (let provider of Object.keys(result)) {
+        let inner = result[provider]
+        for (let domain of Object.keys(inner)) {
+          let inner2 = inner[domain]
+          if (!inner2?.lastseen || !inner2?.firstseen) {
+            console.log(chalk.red(`📌 domain without lastseen or firstseen: ${domain} - provider: ${provider}`))
+            continue
+          }
+          // we check domain level expiration
+          // for now expiration implementation rudimentary
+          let lastseenDaysAgo = Math.round((utcnow() - inner2.lastseen) / 60 / 60 / 24)
+          if (lastseenDaysAgo > expiration_days) {
+            if (!expiredDomains.hasOwnProperty(domain)) expiredDomains[domain] = {}
+            if (!expiredDomains[domain].hasOwnProperty(provider)) expiredDomains[domain][provider] = {
+              firstseen: inner2.firstseen,
+              lastseen: inner2.lastseen,
+            }
+            result[provider][domain].expired = true
+
+            // console.log(`expired domain: ${chalk.yellow(domain)} @ ${chalk.blue(provider)} - firstseen: ${new Date(inner2.firstseen * 1000).toISOString()} - lastseen: ${new Date(inner2.lastseen * 1000).toISOString()} (${lastseenDaysAgo} days ago)`)
+          }
+
+        }
+      }
+    }
+
+    progress.finish()
+    console.log(`✅ finished ${rows.length} rows, took ${(Math.floor(Date.now() - started) / 1000)} seconds`)
+    console.log(`✅ expired domains: ${Object.keys(expiredDomains).length}`)
+    return resolve([result, expiredDomains])
   })
 }
 
@@ -532,11 +705,13 @@ module.exports = {
 
 
   emailBy_id_innerText,
+  emailBy_id_value,
+
   emailBy_querySelector_innerHTML,
   emailBy_querySelector_innerText,
   emailBy_querySelector_placeholder,
   emailBy_querySelector_value,
-  emailBy_id_value,
+
 
   emailFrom_combobox_id,
   emailFrom_combobox_selector,
@@ -555,5 +730,7 @@ module.exports = {
   obfuscatedTime,
   domainFromEmail,
   eventsOfProvider,
-  is_domain_banned
+  is_domain_banned,
+  email_recursive_search,
+  clickto_element_withText
 }

package/update-repo.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+source /opt/coadmin-peer/inc/shared.sh
 optstring=":v"
 verbose=
 while getopts ${optstring} arg; do
@@ -7,25 +8,11 @@ while getopts ${optstring} arg; do
   esac
 done
 
-report_test_ok() {
-    local id="$1"
-    local details="$2"
-    test -z $2 && details="-"
-    test -e /opt/coadmin-peer/tools/report_test.js && {
-        test -z $verbose || echo "✅ $id"
-        node /opt/coadmin-peer/tools/report_test.js --project 'fakefilter' --group 'update-repo.sh' --id "$id" --result "ok" --details "$details"
-    }
-}
+pkill -f repo-update.js
+pkill -f chart-update.js
+pkill -f repo-update.js
+pkill -f chart-update.js
 
-report_test_error() {
-    local id="$1"
-    local details="$2"
-    test -z $2 && details="-"
-    test -e /opt/coadmin-peer/tools/report_test.js && {
-        test -z $verbose || echo "❌ $id"
-        node /opt/coadmin-peer/tools/report_test.js --project 'fakefilter' --group 'update-repo.sh' --id "$id" --result "error" --details "$details"
-    }
-}
 
 cd /opt/fakefilter && {
     node chart-update.js
@@ -33,7 +20,7 @@ cd /opt/fakefilter && {
     if cd repo/; then 
       # git add assets/img/*.png
         report_test_ok "repo folder exists"
-        if ./release.sh 'auto update' && npm publish; then       
+        if ./release.sh 'auto update'; then       
             report_test_ok "npm published"
         else
             report_test_error "npm published"