figranium 0.12.0 → 0.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +674 -674
- package/README.md +336 -336
- package/agent.js +1 -1
- package/bin/cli.js +149 -149
- package/common-utils.js +211 -211
- package/dist/assets/{favicon-DmUMR1rm.svg → favicon-DXDXzv5K.svg} +290 -290
- package/dist/assets/index-BaVlGc48.js +18 -0
- package/dist/assets/index-T2xxnq_A.css +1 -0
- package/dist/favicon.svg +290 -290
- package/dist/figranium_icon.svg +290 -290
- package/dist/figranium_logo.svg +60 -60
- package/dist/index.html +26 -26
- package/dist/novnc.html +108 -108
- package/dist/styles.css +86 -86
- package/extraction-worker.js +211 -204
- package/headful.js +584 -569
- package/html-utils.js +24 -24
- package/package.json +82 -82
- package/proxy-rotation.js +261 -261
- package/proxy-utils.js +84 -84
- package/public/favicon.svg +290 -290
- package/public/figranium_icon.svg +290 -290
- package/public/figranium_logo.svg +60 -60
- package/public/novnc.html +108 -108
- package/public/styles.css +86 -86
- package/scrape.js +389 -389
- package/scripts/postinstall.js +21 -21
- package/server.js +626 -625
- package/src/server/cron-parser.js +325 -316
- package/src/server/routes/schedules.js +171 -171
- package/src/server/scheduler.js +379 -381
- package/url-utils.js +339 -295
- package/user-agent-settings.js +76 -76
- package/dist/assets/index-B1CypY6C.css +0 -1
- package/dist/assets/index-B295GWry.js +0 -18
package/common-utils.js
CHANGED
|
@@ -1,211 +1,211 @@
|
|
|
1
|
-
const parseBooleanFlag = (value) => {
|
|
2
|
-
if (typeof value === 'boolean') return value;
|
|
3
|
-
if (value === undefined || value === null) return false;
|
|
4
|
-
const normalized = String(value).toLowerCase();
|
|
5
|
-
return normalized === 'true' || normalized === '1';
|
|
6
|
-
};
|
|
7
|
-
|
|
8
|
-
const sanitizeRunId = (runId) => {
|
|
9
|
-
if (!runId) return null;
|
|
10
|
-
return String(runId).replace(/[^a-zA-Z0-9_-]/g, '');
|
|
11
|
-
};
|
|
12
|
-
|
|
13
|
-
const parseValue = (value) => {
|
|
14
|
-
if (typeof value !== 'string') return value;
|
|
15
|
-
const trimmed = value.trim();
|
|
16
|
-
if (!trimmed) return '';
|
|
17
|
-
if (trimmed === 'true') return true;
|
|
18
|
-
if (trimmed === 'false') return false;
|
|
19
|
-
if (/^-?\d+(\.\d+)?$/.test(trimmed)) return Number(trimmed);
|
|
20
|
-
if ((trimmed.startsWith('{') && trimmed.endsWith('}')) || (trimmed.startsWith('[') && trimmed.endsWith(']'))) {
|
|
21
|
-
try {
|
|
22
|
-
return JSON.parse(trimmed);
|
|
23
|
-
} catch {
|
|
24
|
-
return value;
|
|
25
|
-
}
|
|
26
|
-
}
|
|
27
|
-
return value;
|
|
28
|
-
};
|
|
29
|
-
|
|
30
|
-
const parseCsv = (input) => {
|
|
31
|
-
const text = typeof input === 'string' ? input : String(input || '');
|
|
32
|
-
const len = text.length;
|
|
33
|
-
const rows = [];
|
|
34
|
-
let row = [];
|
|
35
|
-
let current = '';
|
|
36
|
-
let inQuotes = false;
|
|
37
|
-
const specialChar = /[",\n\r]/g;
|
|
38
|
-
|
|
39
|
-
let i = 0;
|
|
40
|
-
while (i < len) {
|
|
41
|
-
if (inQuotes) {
|
|
42
|
-
const nextQuote = text.indexOf('"', i);
|
|
43
|
-
if (nextQuote === -1) {
|
|
44
|
-
current += text.slice(i);
|
|
45
|
-
i = len;
|
|
46
|
-
break;
|
|
47
|
-
}
|
|
48
|
-
current += text.slice(i, nextQuote);
|
|
49
|
-
i = nextQuote;
|
|
50
|
-
if (i + 1 < len && text[i + 1] === '"') {
|
|
51
|
-
current += '"';
|
|
52
|
-
i += 2;
|
|
53
|
-
} else {
|
|
54
|
-
inQuotes = false;
|
|
55
|
-
i += 1;
|
|
56
|
-
}
|
|
57
|
-
} else {
|
|
58
|
-
specialChar.lastIndex = i;
|
|
59
|
-
const match = specialChar.exec(text);
|
|
60
|
-
if (!match) {
|
|
61
|
-
current += text.slice(i);
|
|
62
|
-
i = len;
|
|
63
|
-
break;
|
|
64
|
-
}
|
|
65
|
-
const idx = match.index;
|
|
66
|
-
const char = match[0];
|
|
67
|
-
current += text.slice(i, idx);
|
|
68
|
-
i = idx;
|
|
69
|
-
if (char === '"') {
|
|
70
|
-
inQuotes = true;
|
|
71
|
-
i += 1;
|
|
72
|
-
} else if (char === ',') {
|
|
73
|
-
row.push(current);
|
|
74
|
-
current = '';
|
|
75
|
-
i += 1;
|
|
76
|
-
} else if (char === '\n') {
|
|
77
|
-
row.push(current);
|
|
78
|
-
rows.push(row);
|
|
79
|
-
row = [];
|
|
80
|
-
current = '';
|
|
81
|
-
i += 1;
|
|
82
|
-
} else if (char === '\r') {
|
|
83
|
-
i += 1;
|
|
84
|
-
}
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
row.push(current);
|
|
88
|
-
if (row.length > 1 || row[0] !== '' || rows.length > 0) rows.push(row);
|
|
89
|
-
|
|
90
|
-
if (rows.length === 0) return [];
|
|
91
|
-
const header = rows[0].map((cell, idx) => {
|
|
92
|
-
const trimmed = String(cell || '').trim();
|
|
93
|
-
return trimmed || `column_${idx + 1}`;
|
|
94
|
-
});
|
|
95
|
-
const dataRows = rows.slice(1);
|
|
96
|
-
return dataRows.map((cells) => {
|
|
97
|
-
const obj = {};
|
|
98
|
-
header.forEach((key, idx) => {
|
|
99
|
-
obj[key] = cells[idx] ?? '';
|
|
100
|
-
});
|
|
101
|
-
return obj;
|
|
102
|
-
});
|
|
103
|
-
};
|
|
104
|
-
|
|
105
|
-
const csvEscape = (value) => {
|
|
106
|
-
if (value === undefined || value === null || value === '') return '';
|
|
107
|
-
const text = String(value);
|
|
108
|
-
// ⚡ Bolt: Fast-path for simple values that don't need escaping
|
|
109
|
-
if (/[",\n\r]/.test(text) || /^\s|\s$/.test(text)) {
|
|
110
|
-
return `"${text.replace(/"/g, '""')}"`;
|
|
111
|
-
}
|
|
112
|
-
return text;
|
|
113
|
-
};
|
|
114
|
-
|
|
115
|
-
const toCsvString = (raw) => {
|
|
116
|
-
if (raw === undefined || raw === null) return '';
|
|
117
|
-
if (typeof raw === 'string') {
|
|
118
|
-
const trimmed = raw.trim();
|
|
119
|
-
if (trimmed.startsWith('{') || trimmed.startsWith('[')) {
|
|
120
|
-
try {
|
|
121
|
-
return toCsvString(JSON.parse(trimmed));
|
|
122
|
-
} catch {
|
|
123
|
-
return raw;
|
|
124
|
-
}
|
|
125
|
-
}
|
|
126
|
-
return raw;
|
|
127
|
-
}
|
|
128
|
-
const rows = Array.isArray(raw) ? raw : [raw];
|
|
129
|
-
if (rows.length === 0) return '';
|
|
130
|
-
|
|
131
|
-
// ⚡ Bolt: Use a Set for unique key collection to reduce complexity from O(N * K^2) to O(N * K)
|
|
132
|
-
const allKeysSet = new Set();
|
|
133
|
-
rows.forEach((row) => {
|
|
134
|
-
if (row && typeof row === 'object' && !Array.isArray(row)) {
|
|
135
|
-
Object.keys(row).forEach((key) => {
|
|
136
|
-
allKeysSet.add(key);
|
|
137
|
-
});
|
|
138
|
-
}
|
|
139
|
-
});
|
|
140
|
-
const allKeys = Array.from(allKeysSet);
|
|
141
|
-
|
|
142
|
-
if (allKeys.length === 0) {
|
|
143
|
-
const lines = rows.map((row) => {
|
|
144
|
-
if (Array.isArray(row)) return row.map(csvEscape).join(',');
|
|
145
|
-
return csvEscape(row);
|
|
146
|
-
});
|
|
147
|
-
return lines.join('\n');
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
const headerLine = allKeys.map(csvEscape).join(',');
|
|
151
|
-
const lines = rows.map((row) => {
|
|
152
|
-
const obj = row && typeof row === 'object' ? row : {};
|
|
153
|
-
return allKeys.map((key) => csvEscape(obj[key])).join(',');
|
|
154
|
-
});
|
|
155
|
-
return [headerLine, ...lines].join('\n');
|
|
156
|
-
};
|
|
157
|
-
|
|
158
|
-
const parseCoords = (value) => {
|
|
159
|
-
if (typeof value !== 'string') return null;
|
|
160
|
-
const trimmed = value.trim();
|
|
161
|
-
if (/^\d+(\.\d+)?,\s*\d+(\.\d+)?$/.test(trimmed)) {
|
|
162
|
-
const [x, y] = trimmed.split(',').map((s) => parseFloat(s.trim()));
|
|
163
|
-
return { x, y };
|
|
164
|
-
}
|
|
165
|
-
return null;
|
|
166
|
-
};
|
|
167
|
-
|
|
168
|
-
const cookieMatches = (cookie, requestUrlOrObj) => {
|
|
169
|
-
try {
|
|
170
|
-
const url = (typeof requestUrlOrObj === 'string') ? new URL(requestUrlOrObj) : requestUrlOrObj;
|
|
171
|
-
const host = url.hostname.toLowerCase();
|
|
172
|
-
const path = url.pathname || '/';
|
|
173
|
-
|
|
174
|
-
// Domain matching (RFC 6265)
|
|
175
|
-
let cookieDomain = (cookie.domain || '').toLowerCase();
|
|
176
|
-
if (cookieDomain.startsWith('.')) {
|
|
177
|
-
cookieDomain = cookieDomain.slice(1);
|
|
178
|
-
}
|
|
179
|
-
|
|
180
|
-
const domainMatches = host === cookieDomain || host.endsWith('.' + cookieDomain);
|
|
181
|
-
if (!domainMatches) return false;
|
|
182
|
-
|
|
183
|
-
// Path matching
|
|
184
|
-
const cookiePath = cookie.path || '/';
|
|
185
|
-
const pathMatches = path === cookiePath ||
|
|
186
|
-
(path.startsWith(cookiePath) && (cookiePath.endsWith('/') || path[cookiePath.length] === '/'));
|
|
187
|
-
|
|
188
|
-
if (!pathMatches) return false;
|
|
189
|
-
|
|
190
|
-
// Secure matching
|
|
191
|
-
if (cookie.secure && url.protocol !== 'https:') return false;
|
|
192
|
-
|
|
193
|
-
// Expiry matching
|
|
194
|
-
if (cookie.expires && cookie.expires < Date.now() / 1000) return false;
|
|
195
|
-
|
|
196
|
-
return true;
|
|
197
|
-
} catch (e) {
|
|
198
|
-
return false;
|
|
199
|
-
}
|
|
200
|
-
};
|
|
201
|
-
|
|
202
|
-
module.exports = {
|
|
203
|
-
parseBooleanFlag,
|
|
204
|
-
sanitizeRunId,
|
|
205
|
-
parseCoords,
|
|
206
|
-
parseValue,
|
|
207
|
-
parseCsv,
|
|
208
|
-
csvEscape,
|
|
209
|
-
toCsvString,
|
|
210
|
-
cookieMatches
|
|
211
|
-
};
|
|
1
|
+
const parseBooleanFlag = (value) => {
|
|
2
|
+
if (typeof value === 'boolean') return value;
|
|
3
|
+
if (value === undefined || value === null) return false;
|
|
4
|
+
const normalized = String(value).toLowerCase();
|
|
5
|
+
return normalized === 'true' || normalized === '1';
|
|
6
|
+
};
|
|
7
|
+
|
|
8
|
+
const sanitizeRunId = (runId) => {
|
|
9
|
+
if (!runId) return null;
|
|
10
|
+
return String(runId).replace(/[^a-zA-Z0-9_-]/g, '');
|
|
11
|
+
};
|
|
12
|
+
|
|
13
|
+
const parseValue = (value) => {
|
|
14
|
+
if (typeof value !== 'string') return value;
|
|
15
|
+
const trimmed = value.trim();
|
|
16
|
+
if (!trimmed) return '';
|
|
17
|
+
if (trimmed === 'true') return true;
|
|
18
|
+
if (trimmed === 'false') return false;
|
|
19
|
+
if (/^-?\d+(\.\d+)?$/.test(trimmed)) return Number(trimmed);
|
|
20
|
+
if ((trimmed.startsWith('{') && trimmed.endsWith('}')) || (trimmed.startsWith('[') && trimmed.endsWith(']'))) {
|
|
21
|
+
try {
|
|
22
|
+
return JSON.parse(trimmed);
|
|
23
|
+
} catch {
|
|
24
|
+
return value;
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
return value;
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
const parseCsv = (input) => {
|
|
31
|
+
const text = typeof input === 'string' ? input : String(input || '');
|
|
32
|
+
const len = text.length;
|
|
33
|
+
const rows = [];
|
|
34
|
+
let row = [];
|
|
35
|
+
let current = '';
|
|
36
|
+
let inQuotes = false;
|
|
37
|
+
const specialChar = /[",\n\r]/g;
|
|
38
|
+
|
|
39
|
+
let i = 0;
|
|
40
|
+
while (i < len) {
|
|
41
|
+
if (inQuotes) {
|
|
42
|
+
const nextQuote = text.indexOf('"', i);
|
|
43
|
+
if (nextQuote === -1) {
|
|
44
|
+
current += text.slice(i);
|
|
45
|
+
i = len;
|
|
46
|
+
break;
|
|
47
|
+
}
|
|
48
|
+
current += text.slice(i, nextQuote);
|
|
49
|
+
i = nextQuote;
|
|
50
|
+
if (i + 1 < len && text[i + 1] === '"') {
|
|
51
|
+
current += '"';
|
|
52
|
+
i += 2;
|
|
53
|
+
} else {
|
|
54
|
+
inQuotes = false;
|
|
55
|
+
i += 1;
|
|
56
|
+
}
|
|
57
|
+
} else {
|
|
58
|
+
specialChar.lastIndex = i;
|
|
59
|
+
const match = specialChar.exec(text);
|
|
60
|
+
if (!match) {
|
|
61
|
+
current += text.slice(i);
|
|
62
|
+
i = len;
|
|
63
|
+
break;
|
|
64
|
+
}
|
|
65
|
+
const idx = match.index;
|
|
66
|
+
const char = match[0];
|
|
67
|
+
current += text.slice(i, idx);
|
|
68
|
+
i = idx;
|
|
69
|
+
if (char === '"') {
|
|
70
|
+
inQuotes = true;
|
|
71
|
+
i += 1;
|
|
72
|
+
} else if (char === ',') {
|
|
73
|
+
row.push(current);
|
|
74
|
+
current = '';
|
|
75
|
+
i += 1;
|
|
76
|
+
} else if (char === '\n') {
|
|
77
|
+
row.push(current);
|
|
78
|
+
rows.push(row);
|
|
79
|
+
row = [];
|
|
80
|
+
current = '';
|
|
81
|
+
i += 1;
|
|
82
|
+
} else if (char === '\r') {
|
|
83
|
+
i += 1;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
row.push(current);
|
|
88
|
+
if (row.length > 1 || row[0] !== '' || rows.length > 0) rows.push(row);
|
|
89
|
+
|
|
90
|
+
if (rows.length === 0) return [];
|
|
91
|
+
const header = rows[0].map((cell, idx) => {
|
|
92
|
+
const trimmed = String(cell || '').trim();
|
|
93
|
+
return trimmed || `column_${idx + 1}`;
|
|
94
|
+
});
|
|
95
|
+
const dataRows = rows.slice(1);
|
|
96
|
+
return dataRows.map((cells) => {
|
|
97
|
+
const obj = {};
|
|
98
|
+
header.forEach((key, idx) => {
|
|
99
|
+
obj[key] = cells[idx] ?? '';
|
|
100
|
+
});
|
|
101
|
+
return obj;
|
|
102
|
+
});
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
const csvEscape = (value) => {
|
|
106
|
+
if (value === undefined || value === null || value === '') return '';
|
|
107
|
+
const text = String(value);
|
|
108
|
+
// ⚡ Bolt: Fast-path for simple values that don't need escaping
|
|
109
|
+
if (/[",\n\r]/.test(text) || /^\s|\s$/.test(text)) {
|
|
110
|
+
return `"${text.replace(/"/g, '""')}"`;
|
|
111
|
+
}
|
|
112
|
+
return text;
|
|
113
|
+
};
|
|
114
|
+
|
|
115
|
+
const toCsvString = (raw) => {
|
|
116
|
+
if (raw === undefined || raw === null) return '';
|
|
117
|
+
if (typeof raw === 'string') {
|
|
118
|
+
const trimmed = raw.trim();
|
|
119
|
+
if (trimmed.startsWith('{') || trimmed.startsWith('[')) {
|
|
120
|
+
try {
|
|
121
|
+
return toCsvString(JSON.parse(trimmed));
|
|
122
|
+
} catch {
|
|
123
|
+
return raw;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
return raw;
|
|
127
|
+
}
|
|
128
|
+
const rows = Array.isArray(raw) ? raw : [raw];
|
|
129
|
+
if (rows.length === 0) return '';
|
|
130
|
+
|
|
131
|
+
// ⚡ Bolt: Use a Set for unique key collection to reduce complexity from O(N * K^2) to O(N * K)
|
|
132
|
+
const allKeysSet = new Set();
|
|
133
|
+
rows.forEach((row) => {
|
|
134
|
+
if (row && typeof row === 'object' && !Array.isArray(row)) {
|
|
135
|
+
Object.keys(row).forEach((key) => {
|
|
136
|
+
allKeysSet.add(key);
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
});
|
|
140
|
+
const allKeys = Array.from(allKeysSet);
|
|
141
|
+
|
|
142
|
+
if (allKeys.length === 0) {
|
|
143
|
+
const lines = rows.map((row) => {
|
|
144
|
+
if (Array.isArray(row)) return row.map(csvEscape).join(',');
|
|
145
|
+
return csvEscape(row);
|
|
146
|
+
});
|
|
147
|
+
return lines.join('\n');
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
const headerLine = allKeys.map(csvEscape).join(',');
|
|
151
|
+
const lines = rows.map((row) => {
|
|
152
|
+
const obj = row && typeof row === 'object' ? row : {};
|
|
153
|
+
return allKeys.map((key) => csvEscape(obj[key])).join(',');
|
|
154
|
+
});
|
|
155
|
+
return [headerLine, ...lines].join('\n');
|
|
156
|
+
};
|
|
157
|
+
|
|
158
|
+
const parseCoords = (value) => {
|
|
159
|
+
if (typeof value !== 'string') return null;
|
|
160
|
+
const trimmed = value.trim();
|
|
161
|
+
if (/^\d+(\.\d+)?,\s*\d+(\.\d+)?$/.test(trimmed)) {
|
|
162
|
+
const [x, y] = trimmed.split(',').map((s) => parseFloat(s.trim()));
|
|
163
|
+
return { x, y };
|
|
164
|
+
}
|
|
165
|
+
return null;
|
|
166
|
+
};
|
|
167
|
+
|
|
168
|
+
const cookieMatches = (cookie, requestUrlOrObj) => {
|
|
169
|
+
try {
|
|
170
|
+
const url = (typeof requestUrlOrObj === 'string') ? new URL(requestUrlOrObj) : requestUrlOrObj;
|
|
171
|
+
const host = url.hostname.toLowerCase();
|
|
172
|
+
const path = url.pathname || '/';
|
|
173
|
+
|
|
174
|
+
// Domain matching (RFC 6265)
|
|
175
|
+
let cookieDomain = (cookie.domain || '').toLowerCase();
|
|
176
|
+
if (cookieDomain.startsWith('.')) {
|
|
177
|
+
cookieDomain = cookieDomain.slice(1);
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
const domainMatches = host === cookieDomain || host.endsWith('.' + cookieDomain);
|
|
181
|
+
if (!domainMatches) return false;
|
|
182
|
+
|
|
183
|
+
// Path matching
|
|
184
|
+
const cookiePath = cookie.path || '/';
|
|
185
|
+
const pathMatches = path === cookiePath ||
|
|
186
|
+
(path.startsWith(cookiePath) && (cookiePath.endsWith('/') || path[cookiePath.length] === '/'));
|
|
187
|
+
|
|
188
|
+
if (!pathMatches) return false;
|
|
189
|
+
|
|
190
|
+
// Secure matching
|
|
191
|
+
if (cookie.secure && url.protocol !== 'https:') return false;
|
|
192
|
+
|
|
193
|
+
// Expiry matching
|
|
194
|
+
if (cookie.expires && cookie.expires < Date.now() / 1000) return false;
|
|
195
|
+
|
|
196
|
+
return true;
|
|
197
|
+
} catch (e) {
|
|
198
|
+
return false;
|
|
199
|
+
}
|
|
200
|
+
};
|
|
201
|
+
|
|
202
|
+
module.exports = {
|
|
203
|
+
parseBooleanFlag,
|
|
204
|
+
sanitizeRunId,
|
|
205
|
+
parseCoords,
|
|
206
|
+
parseValue,
|
|
207
|
+
parseCsv,
|
|
208
|
+
csvEscape,
|
|
209
|
+
toCsvString,
|
|
210
|
+
cookieMatches
|
|
211
|
+
};
|