fakefilter 1.1.10 → 1.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/apiserver/inc/shared.js +3 -1
- package/apiserver/inc/worker_fakefilter.js +3 -3
- package/apiserver/public/img/test.html +48 -0
- package/chart-update.js +156 -74
- package/config.js +2 -2
- package/demos/visitor0/index.js +41 -0
- package/demos/visitor0/renderedDOM.html +8 -0
- package/elements/demo.js +78 -0
- package/elements/renderedDOM.html +6459 -0
- package/elements/screenshots/afterclick.png +0 -0
- package/elements/screenshots/goto.png +0 -0
- package/fetch.js +4 -1
- package/package.json +6 -3
- package/providers/10-minute-mail.com.js +2 -1
- package/providers/10-minuten-mail.de.js +2 -1
- package/providers/10mails.net.js +2 -1
- package/providers/10minut.com.pl.js +2 -1
- package/providers/10minutemail.co.za.js +2 -1
- package/providers/10minutesemail.net.js +2 -2
- package/providers/10minutesmail.us.js +2 -1
- package/providers/22.do.js +25 -0
- package/providers/abandonmail.com.js +2 -1
- package/providers/akmail.in.js +3 -3
- package/providers/altaddress.org.js +50 -0
- package/providers/anonymmail.net.js +47 -0
- package/providers/awgarstone.com.js +15 -0
- package/providers/brodilla.email.js +3 -2
- package/providers/chacuo.net.js +38 -0
- package/providers/dispemail.com.js +3 -2
- package/providers/disposableemail.us.js +2 -1
- package/providers/dispostable.com.js +2 -1
- package/providers/emailnator.com.js +37 -0
- package/providers/etempmail.com.js +36 -0
- package/providers/fakemail.io.js +2 -1
- package/providers/faxmail.co.js +47 -0
- package/providers/findtempmail.com.js +3 -2
- package/providers/foxiomail.com.js +3 -2
- package/providers/getnada.cc.js +37 -0
- package/providers/gmailcity.com.js +37 -0
- package/providers/hottempmail.com.js +3 -2
- package/providers/instantemailaddress.com.js +3 -2
- package/providers/internxt.com.js +47 -0
- package/providers/jooko.info.js +2 -1
- package/providers/kopeechka.store.js +0 -10
- package/providers/kuku.lu.js +37 -0
- package/providers/linshiyou.com.js +25 -0
- package/providers/linshiyouxiang.net.js +36 -0
- package/providers/mail.td.js +13 -13
- package/providers/mail.tm.js +1 -1
- package/providers/mail1.js +44 -0
- package/providers/mail4qa.com.js +37 -0
- package/providers/muellmail.com.js +7 -8
- package/providers/snapmail.cc.js +42 -0
- package/providers/static.js +32 -0
- package/providers/temp-inbox.me.js +47 -0
- package/providers/temp-mail.org.2.js +60 -0
- package/providers/temp-mail.org.js +36 -0
- package/providers/temp-mailbox.com.js +35 -0
- package/providers/tempmail.adguard.com.js +31 -0
- package/providers/tempmail.io.js +38 -0
- package/providers/tempmail.quest.js +45 -0
- package/providers/temporary-email.org.js +36 -0
- package/providers/tuamaeaquelaursa.com.js +25 -0
- package/providers/youxiang.dev.js +25 -0
- package/repo-update.js +26 -13
- package/shared.js +225 -48
- package/update-repo.sh +14 -21
package/apiserver/inc/shared.js
CHANGED
|
@@ -13,7 +13,7 @@ function hostnameFromEmailAddress(email) {
|
|
|
13
13
|
|
|
14
14
|
|
|
15
15
|
|
|
16
|
-
function perDomain(combined) {
|
|
16
|
+
function perDomain(combined, withoutExpired=false) {
|
|
17
17
|
let json = {
|
|
18
18
|
v:2,
|
|
19
19
|
domains: {},
|
|
@@ -22,6 +22,8 @@ function perDomain(combined) {
|
|
|
22
22
|
let domains = combined[provider]
|
|
23
23
|
for (let domain of Object.keys(domains)) {
|
|
24
24
|
let data = domains[domain]
|
|
25
|
+
if (withoutExpired && data.expired) continue
|
|
26
|
+
|
|
25
27
|
if (!json.domains.hasOwnProperty(domain)) json.domains[domain] = {
|
|
26
28
|
providers: [provider],
|
|
27
29
|
hosts: data.hosts
|
|
@@ -26,10 +26,10 @@ async function worker(config) {
|
|
|
26
26
|
try {
|
|
27
27
|
let rows = await pullDomains(isLocal() ? 30 : 10000)
|
|
28
28
|
if (rows) {
|
|
29
|
-
let combined = await combineDomains(rows)
|
|
29
|
+
let [combined,expiredDomains] = await combineDomains(rows,true,365)
|
|
30
30
|
if (combined) {
|
|
31
|
-
vars.current_filter = perDomain(combined)
|
|
32
|
-
console.log(chalk.green(`\
|
|
31
|
+
vars.current_filter = perDomain(combined,true)
|
|
32
|
+
console.log(chalk.green(`\ncurrent_filter updated with ${Object.keys(vars.current_filter).length} domains`))
|
|
33
33
|
// lets write this into the cache file
|
|
34
34
|
try {
|
|
35
35
|
fs.writeFileSync(cacheFile, JSON.stringify(vars.current_filter))
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="UTF-8">
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
6
|
+
<title>5x5 Matrix</title>
|
|
7
|
+
<style>
|
|
8
|
+
.matrix {
|
|
9
|
+
display: grid;
|
|
10
|
+
grid-template-columns: repeat(5, 50px);
|
|
11
|
+
grid-template-rows: repeat(5, 50px);
|
|
12
|
+
gap: 5px;
|
|
13
|
+
}
|
|
14
|
+
.cell {
|
|
15
|
+
width: 50px;
|
|
16
|
+
height: 50px;
|
|
17
|
+
display: flex;
|
|
18
|
+
align-items: center;
|
|
19
|
+
justify-content: center;
|
|
20
|
+
border: 1px solid #ccc;
|
|
21
|
+
}
|
|
22
|
+
</style>
|
|
23
|
+
</head>
|
|
24
|
+
<body>
|
|
25
|
+
<script>
|
|
26
|
+
// Create the matrix container
|
|
27
|
+
const matrix = document.createElement('div');
|
|
28
|
+
matrix.className = 'matrix';
|
|
29
|
+
|
|
30
|
+
// Generate 5x5 cells with random colors
|
|
31
|
+
for (let i = 0; i < 25; i++) {
|
|
32
|
+
const cell = document.createElement('div');
|
|
33
|
+
cell.className = 'cell';
|
|
34
|
+
|
|
35
|
+
// Set a random background color
|
|
36
|
+
cell.style.backgroundColor = `#${Math.floor(Math.random() * 16777215).toString(16)}`;
|
|
37
|
+
|
|
38
|
+
// Optional: Add some text (like the cell number)
|
|
39
|
+
cell.textContent = i + 1;
|
|
40
|
+
|
|
41
|
+
matrix.appendChild(cell);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// Append the matrix to the body
|
|
45
|
+
document.body.appendChild(matrix);
|
|
46
|
+
</script>
|
|
47
|
+
</body>
|
|
48
|
+
</html>
|
package/chart-update.js
CHANGED
|
@@ -11,111 +11,187 @@ const QuickChart = require('quickchart-js');
|
|
|
11
11
|
const argv = require('minimist')(process.argv.slice(2))
|
|
12
12
|
const md5 = require('md5')
|
|
13
13
|
|
|
14
|
-
function writeToFile(fn,content) {
|
|
14
|
+
function writeToFile(fn, content) {
|
|
15
15
|
// let fn = 'repo/markdown/README.md'
|
|
16
|
-
fs.writeFileSync(fn,content)
|
|
16
|
+
fs.writeFileSync(fn, content)
|
|
17
17
|
console.log(chalk.green(`+ file ${fn}`))
|
|
18
18
|
return true
|
|
19
19
|
}
|
|
20
20
|
|
|
21
|
-
function
|
|
22
|
-
|
|
21
|
+
// function isEventGraphicCached(providerName) {
|
|
22
|
+
// let fn = `success-rate-${md5(providerName)}.png`
|
|
23
|
+
// let chartFile = path.join(__dirname, `/apiserver/public/img/dynamic/${fn}`)
|
|
24
|
+
// if (fs.existsSync(chartFile)) {
|
|
25
|
+
// // lets look if the file was recently rendered
|
|
26
|
+
// let stats = fs.statSync(chartFile)
|
|
27
|
+
// let mtimeMinutes = (new Date() - stats.mtime) / 1000 / 60 // modified time in minutes
|
|
28
|
+
// if (mtimeMinutes < 200) {
|
|
29
|
+
// console.log(`Chart for ${providerName} - taken from cache ${chartFile}`)
|
|
30
|
+
// return chartFile
|
|
31
|
+
// }
|
|
32
|
+
// }
|
|
33
|
+
// return false
|
|
34
|
+
// }
|
|
35
|
+
|
|
36
|
+
function dailyEventGraphic(providerName,daysAgo=60) {
|
|
37
|
+
return new Promise(async function (resolve, reject) {
|
|
23
38
|
const myChart = new QuickChart()
|
|
24
|
-
let data={}
|
|
39
|
+
let data = {}
|
|
40
|
+
|
|
41
|
+
let successrate = {}
|
|
42
|
+
let labels = []
|
|
25
43
|
let fn = `success-rate-${md5(providerName)}.png`
|
|
26
|
-
let
|
|
27
|
-
let labels=[]
|
|
28
|
-
let chartFile = path.join(__dirname,`/apiserver/public/img/dynamic/${fn}`)
|
|
44
|
+
let chartFile = path.join(__dirname, `/apiserver/public/img/dynamic/${fn}`)
|
|
29
45
|
if (fs.existsSync(chartFile)) {
|
|
30
46
|
// lets look if the file was recently rendered
|
|
31
47
|
let stats = fs.statSync(chartFile)
|
|
32
|
-
let mtimeMinutes = (new Date()-stats.mtime)/1000/60 // modified time in minutes
|
|
33
|
-
if (mtimeMinutes<200) {
|
|
34
|
-
console.log(`
|
|
48
|
+
let mtimeMinutes = (new Date() - stats.mtime) / 1000 / 60 // modified time in minutes
|
|
49
|
+
if (mtimeMinutes < 200) {
|
|
50
|
+
console.log(chalk.gray(`chart for ${providerName} - taken from cache ${chartFile}`))
|
|
35
51
|
return resolve(fn)
|
|
36
52
|
}
|
|
37
53
|
}
|
|
38
54
|
await wait(2) // rate limit
|
|
39
|
-
|
|
40
|
-
for(let row of events) {
|
|
41
|
-
let dayString = new Date(row.t).toISOString().substring(0,10)
|
|
42
|
-
if (!data.hasOwnProperty(dayString))
|
|
43
|
-
{
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
if (row.result==='success') data[dayString].success++; else
|
|
55
|
+
let events = await pullEventsByProvider(providerName, daysAgo)
|
|
56
|
+
for (let row of events) {
|
|
57
|
+
let dayString = new Date(row.t).toISOString().substring(0, 10)
|
|
58
|
+
if (!data.hasOwnProperty(dayString)) {
|
|
59
|
+
data[dayString] = { success: 0, failed: 0 }
|
|
60
|
+
successrate[dayString] = 0
|
|
61
|
+
labels.push(dayString)
|
|
62
|
+
}
|
|
63
|
+
if (row.result === 'success') data[dayString].success++; else
|
|
49
64
|
data[dayString].failed++;
|
|
50
65
|
}
|
|
51
|
-
for(let date of Object.keys(data)) {
|
|
52
|
-
successrate[date]=((data[date].success/(data[date].failed+data[date].success)))*100
|
|
66
|
+
for (let date of Object.keys(data)) {
|
|
67
|
+
successrate[date] = ((data[date].success / (data[date].failed + data[date].success))) * 100
|
|
53
68
|
}
|
|
54
69
|
|
|
55
70
|
let chartConfig1 = {
|
|
56
71
|
type: 'bar',
|
|
57
|
-
data: {
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
72
|
+
data: {
|
|
73
|
+
labels,
|
|
74
|
+
datasets: [{
|
|
75
|
+
label: `Success Rate - ${providerName} - %`,
|
|
76
|
+
backgroundColor: QuickChart.getGradientFillHelper('vertical', ['green', 'red']),
|
|
77
|
+
data: Object.keys(successrate).map(dayString => successrate[dayString])
|
|
78
|
+
}
|
|
79
|
+
]
|
|
80
|
+
},
|
|
62
81
|
}
|
|
63
82
|
myChart.setConfig(chartConfig1)
|
|
64
|
-
|
|
65
|
-
|
|
83
|
+
|
|
84
|
+
|
|
66
85
|
await myChart.toFile(chartFile)
|
|
67
86
|
console.log(`Creating success-rate chart for ${providerName}`)
|
|
68
87
|
return resolve(fn)
|
|
69
88
|
|
|
70
89
|
// calcualte success rate
|
|
71
|
-
|
|
90
|
+
|
|
72
91
|
})
|
|
73
92
|
}
|
|
74
93
|
function dailyCoverageGraph(rows) {
|
|
75
|
-
return new Promise(async function (resolve,reject) {
|
|
94
|
+
return new Promise(async function (resolve, reject) {
|
|
76
95
|
const myChart = new QuickChart();
|
|
77
96
|
const myChart2 = new QuickChart();
|
|
78
|
-
|
|
79
|
-
let
|
|
80
|
-
let
|
|
81
|
-
let
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
97
|
+
const myChart3 = new QuickChart();
|
|
98
|
+
let data = {}
|
|
99
|
+
let labels = []
|
|
100
|
+
let uniqueDomains = []
|
|
101
|
+
let dataUniqueDomains = {}
|
|
102
|
+
let dataUniqueDomainsMonthly = {}
|
|
103
|
+
|
|
104
|
+
rows = rows.sort((a, b) => a.t - b.t)
|
|
105
|
+
for (let row of rows) {
|
|
106
|
+
let dayString = new Date(row.t).toISOString().substring(0, 10)
|
|
107
|
+
let monthString = new Date(row.t).toISOString().substring(0, 7)
|
|
85
108
|
if (!uniqueDomains.includes(row.domain)) uniqueDomains.push(row.domain)
|
|
86
|
-
if (!data.hasOwnProperty(dayString))
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
109
|
+
if (!data.hasOwnProperty(dayString)) {
|
|
110
|
+
data[dayString] = []
|
|
111
|
+
dataUniqueDomains[dayString] = uniqueDomains.length
|
|
112
|
+
labels.push(dayString)
|
|
113
|
+
}
|
|
114
|
+
dataUniqueDomainsMonthly[monthString] = uniqueDomains.length
|
|
115
|
+
|
|
93
116
|
if (!data[dayString].includes(row.domain)) data[dayString].push(row.domain)
|
|
94
117
|
// console.log(row)
|
|
95
118
|
// if (cnt<10) break
|
|
96
119
|
// cnt--
|
|
97
120
|
}
|
|
98
121
|
// console.log(dataUniqueDomains)
|
|
99
|
-
labels = labels.reverse()
|
|
122
|
+
labels = labels.reverse().slice(0, 200).reverse()
|
|
123
|
+
let labelsM = Object.keys(dataUniqueDomainsMonthly).reverse().slice(0, 200).reverse()
|
|
100
124
|
let chartConfig2 = {
|
|
101
|
-
type: '
|
|
102
|
-
data: {
|
|
103
|
-
|
|
125
|
+
type: 'line',
|
|
126
|
+
data: {
|
|
127
|
+
labels,
|
|
128
|
+
datasets: [{
|
|
129
|
+
label: 'Unique Domains in FakeFilter',
|
|
130
|
+
data: labels.map(dayString => dataUniqueDomains[dayString]),
|
|
131
|
+
fill: 'start',
|
|
132
|
+
borderWidth: 0,
|
|
133
|
+
pointRadius: 0,
|
|
134
|
+
borderColor: 'rgba(255, 255, 0, 0.5)',
|
|
135
|
+
backgroundColor: 'rgba(255, 255, 0, 0.5)',
|
|
136
|
+
}]
|
|
137
|
+
},
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
let chartConfig3 = {
|
|
141
|
+
type: 'line',
|
|
142
|
+
data: {
|
|
143
|
+
labels: labelsM,
|
|
144
|
+
datasets: [{
|
|
145
|
+
label: 'Unique Domains in FakeFilter Per Month',
|
|
146
|
+
data: labelsM.map(monthString => dataUniqueDomainsMonthly[monthString]),
|
|
147
|
+
fill: 'start',
|
|
148
|
+
borderWidth: 0,
|
|
149
|
+
pointRadius: 0,
|
|
150
|
+
borderColor: 'rgba(255, 120, 0, 0.5)',
|
|
151
|
+
backgroundColor: 'rgba(255, 120, 0, 0.5)',
|
|
152
|
+
}]
|
|
153
|
+
},
|
|
104
154
|
}
|
|
105
155
|
|
|
106
156
|
let chartConfig1 = {
|
|
107
|
-
type: 'bar',
|
|
108
|
-
|
|
109
|
-
|
|
157
|
+
// type: 'bar',
|
|
158
|
+
type: 'line',
|
|
159
|
+
data: {
|
|
160
|
+
labels,
|
|
161
|
+
datasets: [{
|
|
162
|
+
label: 'Daily Coverage',
|
|
163
|
+
data: labels.map(dayString => data[dayString].length),
|
|
164
|
+
fill: 'start',
|
|
165
|
+
borderWidth: 0,
|
|
166
|
+
pointRadius: 0,
|
|
167
|
+
borderColor: 'rgba(0, 255, 0, 0.5)',
|
|
168
|
+
backgroundColor: 'rgba(0, 255, 0, 0.5)',
|
|
169
|
+
}]
|
|
170
|
+
},
|
|
171
|
+
options: {
|
|
172
|
+
scales: {
|
|
173
|
+
yAxes: [{
|
|
174
|
+
stacked: true,
|
|
175
|
+
ticks: {
|
|
176
|
+
min: -10,
|
|
177
|
+
}
|
|
178
|
+
}]
|
|
179
|
+
},
|
|
180
|
+
legend: {
|
|
181
|
+
display: true,
|
|
182
|
+
},
|
|
183
|
+
},
|
|
110
184
|
}
|
|
111
|
-
|
|
185
|
+
|
|
112
186
|
// console.log(chartConfig1)
|
|
113
187
|
myChart.setConfig(chartConfig1)
|
|
114
188
|
myChart2.setConfig(chartConfig2)
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
189
|
+
myChart3.setConfig(chartConfig3)
|
|
190
|
+
await myChart.toFile(path.join(__dirname, '/apiserver/public/img/dynamic/chart1.png'))
|
|
191
|
+
|
|
192
|
+
await myChart2.toFile(path.join(__dirname, '/apiserver/public/img/dynamic/chart2.png'))
|
|
193
|
+
await myChart3.toFile(path.join(__dirname, '/apiserver/public/img/dynamic/chart3.png'))
|
|
194
|
+
|
|
119
195
|
return resolve()
|
|
120
196
|
})
|
|
121
197
|
}
|
|
@@ -123,31 +199,33 @@ function dailyCoverageGraph(rows) {
|
|
|
123
199
|
async function start() {
|
|
124
200
|
try {
|
|
125
201
|
await getConfig()
|
|
126
|
-
let events_providers = await pullEventsProviders(false,60)
|
|
127
|
-
let markdown3
|
|
128
|
-
|
|
202
|
+
let events_providers = await pullEventsProviders(false, 60)
|
|
203
|
+
let markdown3 = ``
|
|
204
|
+
// if (false)
|
|
205
|
+
for (let provider of events_providers) {
|
|
129
206
|
// break
|
|
130
207
|
// if (provider==='temporarymail.com')
|
|
131
208
|
{
|
|
132
209
|
try {
|
|
133
|
-
let r = await pullEventsByProvider(provider,60)
|
|
134
|
-
let chartfn = await dailyEventGraphic(provider,r)
|
|
135
210
|
|
|
136
|
-
|
|
211
|
+
let chartfn = await dailyEventGraphic(provider,60)
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
markdown3 += `### ${provider}
|
|
137
215
|
|
|
138
216
|
|
|
139
217
|
`
|
|
140
|
-
}catch(err) {
|
|
218
|
+
} catch (err) {
|
|
141
219
|
Sentry.captureException(err)
|
|
142
220
|
console.log(err)
|
|
143
221
|
}
|
|
144
222
|
}
|
|
145
223
|
}
|
|
146
224
|
|
|
147
|
-
let domain_rows = await pullDomains(
|
|
148
|
-
let urls = await dailyCoverageGraph(domain_rows)
|
|
149
|
-
|
|
150
|
-
let markdown1
|
|
225
|
+
let domain_rows = await pullDomains(1000, false)
|
|
226
|
+
let urls = await dailyCoverageGraph(domain_rows)
|
|
227
|
+
|
|
228
|
+
let markdown1 = `
|
|
151
229
|
# Charts
|
|
152
230
|
## All Providers
|
|
153
231
|
### Daily Coverage
|
|
@@ -155,24 +233,28 @@ Amount of coverage, actions against all known Disposable Email Providers per Day
|
|
|
155
233
|
|
|
156
234
|
|
|
157
235
|
### Daily Unique Disposable Domains
|
|
158
|
-
Number of unique known validated Disposable Domains in Total at given Day
|
|
159
|
-
|
|
238
|
+
|
|
239
|
+
### Monthly Unique Disposable Domains
|
|
240
|
+
Number of unique known validated Disposable Domains in Total at given Month.
|
|
241
|
+
|
|
160
242
|
|
|
161
243
|
## Success Rate
|
|
162
244
|
Per Provider Success Rate per Day - A lower Success rate might mean different things; counter measures from provider to be detected like capchas or even service downtime.
|
|
163
245
|
|
|
164
246
|
${markdown3}
|
|
165
247
|
`
|
|
166
|
-
writeToFile(path.join(__dirname, 'repo/CHARTS.md'),markdown1)
|
|
248
|
+
writeToFile(path.join(__dirname, 'repo/CHARTS.md'), markdown1)
|
|
167
249
|
|
|
168
250
|
console.log(urls)
|
|
169
251
|
process.exit(0)
|
|
170
252
|
|
|
171
253
|
}
|
|
172
|
-
catch(err) {
|
|
254
|
+
catch (err) {
|
|
173
255
|
console.log(err)
|
|
174
256
|
Sentry.captureException(err)
|
|
175
|
-
softexit("",1)
|
|
257
|
+
softexit("", 1)
|
|
176
258
|
}
|
|
177
259
|
}
|
|
178
260
|
|
package/config.js
CHANGED
|
@@ -13,7 +13,7 @@ const { Events,initEvents } = require('./models/events')
|
|
|
13
13
|
|
|
14
14
|
var vault = require("node-vault")({
|
|
15
15
|
apiVersion: 'v1', // default
|
|
16
|
-
endpoint: 'https://
|
|
16
|
+
endpoint: 'https://vault3.vpn1.org',
|
|
17
17
|
token: 'd7130f39-8466-4444-b2b7-49c93da2c597'
|
|
18
18
|
})
|
|
19
19
|
|
|
@@ -102,7 +102,7 @@ var config = {
|
|
|
102
102
|
},
|
|
103
103
|
cache_folder:path.join(__dirname,"cache"),
|
|
104
104
|
main_folder:__dirname,
|
|
105
|
-
ignore_domains : ['nytimes.com','yahoo.gr','outlook.sa','live.nl','yahoo.it','comcast.net','mail.ua','yahoo.com.br','outlook.es','aliyun.com','cps.edu','sina.cn','yahoo.ca',
|
|
105
|
+
ignore_domains : ['samsung.com','nytimes.com','yahoo.gr','outlook.sa','live.nl','yahoo.it','comcast.net','mail.ua','yahoo.com.br','outlook.es','aliyun.com','cps.edu','sina.cn','yahoo.ca',
|
|
106
106
|
't-online.de','yahoo.co.jp','hotmail.es','yahoo.in','vip.qq.com','yandex.ua','139.com','hotmail.it','orange.fr','hotmail.de','yahoo.com.hk','gmx.at','live.co.uk','gmx.net','yahoo.co.in','yahoo.de','yahoo.es','me.com','o2.pl','gmx.fr','abv.bg','yahoo.co.id','wp.pl','i.ua','msn.com','live.fr','googlemail.com','gmx.com','yahoo.co.uk','yahoo.fr','naver.com','sina.com','yandex.com','gmx.de','aol.com','hotmail.co.uk','yahoo.com.tw','hotmail.fr','bk.ru','mail.com','web.de','outlook.fr','protonmail.com','ukr.net','live.com','icloud.com','126.com','yandex.ru','outlook.com','163.com','mail.ru','hotmail.com','yahoo.com','qq.com','gmail.com','zoho.com','cloudflare.com']
|
|
107
107
|
// vaultKey: 'vault@/secret/vault'
|
|
108
108
|
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
const fs = require('fs')
|
|
2
|
+
const { Sentry } = require('../../config')
|
|
3
|
+
const { launch } = require('../../shared')
|
|
4
|
+
const { wait } = require('mybase');
|
|
5
|
+
|
|
6
|
+
// const URL = 'https://fakefilter.net/public/img/test.html'
|
|
7
|
+
// const URL = 'https://sdnz.jdjy.sh.cn/'
|
|
8
|
+
const URL = `https://visitor0.com/static/demo/basic.html?2`
|
|
9
|
+
// const URL = `https://infosimples.github.io/detect-headless/`
|
|
10
|
+
// const URL = `https://leaky.page/timer.html`
|
|
11
|
+
const lookingFor = 'bitrix24'.toLowerCase();
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
function start(pageWaitTime = 1, matchWaitTime = 5) {
|
|
15
|
+
return new Promise(async function (resolve, reject) {
|
|
16
|
+
try {
|
|
17
|
+
let { browser, page } = await launch(true, URL, pageWaitTime, false, false, 'domcontentloaded')
|
|
18
|
+
await wait(5)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
console.log(`[${Date.now()}] searching for our match ${lookingFor}`)
|
|
22
|
+
const renderedDOM = await page.content();
|
|
23
|
+
fs.writeFileSync('renderedDOM.html', renderedDOM)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
if (browser) await browser.close()
|
|
27
|
+
|
|
28
|
+
return resolve()
|
|
29
|
+
} catch (err) {
|
|
30
|
+
console.log(err)
|
|
31
|
+
Sentry.captureException(err)
|
|
32
|
+
resolve(false)
|
|
33
|
+
}
|
|
34
|
+
})
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// for cli usage - development
|
|
38
|
+
if (require.main == module) {
|
|
39
|
+
start(false).then(res => { process.exit(0) })
|
|
40
|
+
}
|
|
41
|
+
|
package/elements/demo.js
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
const chalk = require('chalk')
|
|
2
|
+
const fs = require('fs')
|
|
3
|
+
const { Sentry } = require('../config')
|
|
4
|
+
const { launch } = require('../shared')
|
|
5
|
+
// const URL = 'https://www.blackdesertfoundry.com/all-recipes/'
|
|
6
|
+
const cheerio = require('cheerio');
|
|
7
|
+
const { wait } = require('mybase');
|
|
8
|
+
|
|
9
|
+
// const URL = 'https://fakefilter.net/public/img/test.html'
|
|
10
|
+
// const URL = 'https://sdnz.jdjy.sh.cn/'
|
|
11
|
+
const URL = `https://fords.com.ua/nasos_vakuumnyy_ford_ranger_1997_2007_hmpx.html`
|
|
12
|
+
const lookingFor = 'bitrix24'.toLowerCase();
|
|
13
|
+
|
|
14
|
+
function searchFor(renderedDOM) {
|
|
15
|
+
const $ = cheerio.load(renderedDOM);
|
|
16
|
+
// console.log($('body').html())
|
|
17
|
+
|
|
18
|
+
const scriptSrcUrls = [];
|
|
19
|
+
const foundItems = []
|
|
20
|
+
$('script[src]').each((index, element) => {
|
|
21
|
+
const wholeTag = $(element).toString().toLowerCase();
|
|
22
|
+
|
|
23
|
+
const srcUrl = $(element).attr('src');
|
|
24
|
+
scriptSrcUrls.push(srcUrl);
|
|
25
|
+
console.log(wholeTag)
|
|
26
|
+
if (wholeTag.includes(lookingFor)) {
|
|
27
|
+
foundItems.push({
|
|
28
|
+
srcUrl,
|
|
29
|
+
wholeTag
|
|
30
|
+
})
|
|
31
|
+
}
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
console.log(scriptSrcUrls)
|
|
35
|
+
console.log(foundItems)
|
|
36
|
+
return foundItems
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function start(pageWaitTime = 1, matchWaitTime = 5) {
|
|
40
|
+
return new Promise(async function (resolve, reject) {
|
|
41
|
+
try {
|
|
42
|
+
let { browser, page } = await launch(true, URL, pageWaitTime, false, false, 'domcontentloaded')
|
|
43
|
+
await wait(5)
|
|
44
|
+
const started = Date.now()
|
|
45
|
+
while(true) {
|
|
46
|
+
console.log(`[${Date.now()}] searching for our match ${lookingFor}`)
|
|
47
|
+
const renderedDOM = await page.content();
|
|
48
|
+
fs.writeFileSync('renderedDOM.html', renderedDOM)
|
|
49
|
+
const foundItems = searchFor(renderedDOM)
|
|
50
|
+
if (foundItems.length > 0) {
|
|
51
|
+
console.log(chalk.green('found'))
|
|
52
|
+
break
|
|
53
|
+
}
|
|
54
|
+
await wait(1)
|
|
55
|
+
if (Date.now() - started > matchWaitTime*1000) {
|
|
56
|
+
console.log(chalk.red('not found'))
|
|
57
|
+
break
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
if (browser) await browser.close()
|
|
64
|
+
|
|
65
|
+
return resolve()
|
|
66
|
+
} catch (err) {
|
|
67
|
+
console.log(err)
|
|
68
|
+
Sentry.captureException(err)
|
|
69
|
+
resolve(false)
|
|
70
|
+
}
|
|
71
|
+
})
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// for cli usage - development
|
|
75
|
+
if (require.main == module) {
|
|
76
|
+
start(false).then(res => { process.exit(0) })
|
|
77
|
+
}
|
|
78
|
+
|