fakefilter 1.1.13 → 1.1.1582

package/repo-update.js

@@ -1,134 +0,0 @@
-const fs = require('fs')
-var Table = require('easy-table')
-const winston = require('winston')
-const { getConfig, Sentry } = require('./config')
-const chalk = require('chalk')
-const vars = require('./vars')
-const argv = require('minimist')(process.argv.slice(2))
-
-// load all providers/*.js
-var glob = require('glob'), path = require('path')
-const { pullDomains, combineDomains, getProviderDetails, obfuscatedTime } = require('./shared')
-const { isLocal } = require('mybase')
-const { perDomain } = require('./apiserver/inc/shared')
-glob.sync('./providers/*.js').forEach(function (file) {
-    require(path.resolve(file));
-});
-
-/*
-this file is designed to deploy repo/ folder with data taken from mysql
-*/
-
-
-const logger = winston.createLogger({
-    level: 'info',
-    format: winston.format.json(),
-    defaultMeta: { service: 'user-service' },
-    transports: [
-        //
-        // - Write all logs with importance level of `error` or less to `error.log`
-        // - Write all logs with importance level of `info` or less to `combined.log`
-        //
-        new winston.transports.File({ filename: 'logs/repo-markdown-error.log', level: 'error' }),
-        new winston.transports.File({ filename: 'logs/repo-markdown-combined.log' }),
-        // log to Console too
-        new winston.transports.Console()
-    ],
-})
-
-
-
-function writeToFile(fn, content) {
-    // let fn = 'repo/markdown/README.md'
-    fs.writeFileSync(fn, content)
-    console.log(chalk.green(`+ file ${fn}`))
-    return true
-}
-
-async function start() {
-    try {
-        await getConfig()
-
-        // we only pull domains harvested last 365 days, so domains
-        let rows = await pullDomains(isLocal() ? 10 : 600, logger)
-        let [ combined,expiredDomains ] = await combineDomains(rows, true, 365)
-
-        let json2domains = perDomain(combined,true)
-        let markdown = `# Providers\n\n`
-        let markdownExpired = `# Providers Expired Domains\nOur service removes domains from fakefilter after a year of inactivity. If these domains reappear, they are reinstated in our database for a minimum of 365 days.\n\n`
-        let now = Math.round(Date.now() / 1000)
-        let json = {
-            version: 1,
-            t: now,
-            domains: {},
-            // flat:[]
-        }
-        let json2 = {
-            version: 2,
-            t: now,
-            domains: json2domains,
-        }
-        let updated = `Last Update: ${new Date().toISOString()}`
-        let txt = `# Fake and Temp Mail Providers\n# https://github.com/7c/fakefilter\n# ${updated}\n\n\n`
-
-        // generate json v1 & markdown
-        for (let provider of Object.keys(combined)) {
-            let domains = combined[provider]
-            // var t = new Table()
-            console.log(chalk.bold.yellow(provider))
-            markdown += `## ${provider}\n`
-            markdownExpired += `## ${provider}\n`
-            txt += `## ${provider}\n`
-
-            for (let domain of Object.keys(domains)) {
-                let data = domains[domain]
-                domain = domain.toLowerCase()
-                let lastSeenAgo = Math.floor((now - data.lastseen)/60/60/24) // days
-
-                for (let host of Object.keys(data.hosts)) {
-                    // json v1 format - which is host based format
-                    if (!data.expired) {
-                        json.domains[host] = {
-                            provider,
-                            firstseen: data.hosts[host].firstseen,
-                            lastseen: data.hosts[host].lastseen,
-                        }
-                        markdown += `- ${host} (lastseen ${lastSeenAgo} days ago)\n`
-                        txt += `${host}\n`
-                    } else {
-                        markdownExpired += `- ${host} (expired ${lastSeenAgo} days ago)\n`
-                    }
-                    
-                    
-                    // t.newRow()
-                }
-            }
-            markdown += `\n`
-            markdownExpired += `\n`
-            txt += `\n`
-            // console.log(t.toString())
-        }
-        markdown += `\n\n${updated}\n`
-        // deploy to file
-        if (!argv.dry) {
-            // markdown not expired
-            writeToFile(path.join(__dirname, 'repo/markdown/README.md'), markdown)
-            // markdown expired
-            writeToFile(path.join(__dirname, 'repo/markdown/EXPIRED.md'),markdownExpired)
-            // json v1
-            writeToFile(path.join(__dirname, 'repo/json/data.json'), JSON.stringify(json))
-            // json v2
-            writeToFile(path.join(__dirname, 'repo/json/data_version2.json'), JSON.stringify(json2))
-            // txt
-            writeToFile(path.join(__dirname, 'repo/txt/data.txt'), txt)
-        } else console.log(chalk.yellow(`dry flag: did not deploy to files`))
-        // console.log(json)
-
-        // console.log(markdown)
-        process.exit(0)
-    } catch (err) {
-        console.log(chalk.red(err))
-    }
-}
-
-start()