@apmantza/greedysearch-pi 2.1.3 → 2.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +48 -1
- package/README.md +52 -175
- package/bin/search.mjs +2 -0
- package/docs/analysis.md +233 -0
- package/docs/banner.svg +86 -0
- package/docs/development.md +63 -0
- package/docs/inspiration2.md +190 -0
- package/docs/releases.md +88 -0
- package/docs/research.md +82 -0
- package/docs/runtime.md +65 -0
- package/docs/source-fetching.md +33 -0
- package/docs/stealthbrowsermcp.md +807 -0
- package/docs/usage.md +69 -0
- package/extractors/chatgpt.mjs +127 -23
- package/extractors/common.mjs +116 -28
- package/extractors/gemini.mjs +63 -38
- package/package.json +13 -7
- package/scripts/backfill-github-releases.mjs +139 -0
- package/scripts/changelog-extract.mjs +48 -0
- package/scripts/changelog-release.mjs +65 -0
- package/scripts/check-lockfile.mjs +45 -0
- package/scripts/lib/changelog.mjs +168 -0
- package/scripts/lint.mjs +62 -0
- package/scripts/run-tests.ps1 +179 -0
- package/scripts/stealth-check.mjs +663 -0
- package/src/fetcher.mjs +9 -5
- package/src/search/constants.mjs +1 -1
- package/src/search/research.mjs +57 -11
- package/src/search/simple-research.mjs +4 -1
- package/test.mjs +16 -3
- package/skills/greedy-search/skill.md +0 -20
package/docs/usage.md
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
# Usage Guide
|
|
2
|
+
|
|
3
|
+
GreedySearch registers one Pi tool: `greedy_search`.
|
|
4
|
+
|
|
5
|
+
## Common Calls
|
|
6
|
+
|
|
7
|
+
```js
|
|
8
|
+
greedy_search({ query: "React 19 changes" });
|
|
9
|
+
greedy_search({ query: "React 19 changes", synthesize: true });
|
|
10
|
+
greedy_search({ query: "Prisma vs Drizzle", engine: "perplexity" });
|
|
11
|
+
greedy_search({ query: "What is Redis?", depth: "research", maxSources: 5 });
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
## Parameters
|
|
15
|
+
|
|
16
|
+
### Common
|
|
17
|
+
|
|
18
|
+
- `query` — required search/research question.
|
|
19
|
+
- `fullAnswer` — return complete engine output instead of a preview.
|
|
20
|
+
- `headless` — defaults to `true`; set to `false` to show Chrome.
|
|
21
|
+
- `visible` / `alwaysVisible` — force visible Chrome for this run.
|
|
22
|
+
|
|
23
|
+
### Normal Search
|
|
24
|
+
|
|
25
|
+
- `engine` — `all` by default. Supported individual engines:
|
|
26
|
+
`perplexity`, `google`, `chatgpt`, `gemini`, `semantic-scholar`,
|
|
27
|
+
`logically`, and `bing`.
|
|
28
|
+
- `synthesize` — for `engine: "all"`, synthesize fetched sources with the
|
|
29
|
+
configured synthesizer.
|
|
30
|
+
- `synthesizer` — `gemini` or `chatgpt`.
|
|
31
|
+
- `depth` — legacy `fast` / `standard` / `deep` aliases are still accepted;
|
|
32
|
+
prefer `synthesize` for normal search.
|
|
33
|
+
|
|
34
|
+
### Research
|
|
35
|
+
|
|
36
|
+
- `depth: "research"` — run the iterative research workflow.
|
|
37
|
+
- `breadth` — actions per round, 1-5.
|
|
38
|
+
- `iterations` — rounds, 1-3.
|
|
39
|
+
- `maxSources` — fetched source cap, 3-12.
|
|
40
|
+
- `researchOutDir` — custom bundle directory.
|
|
41
|
+
- `writeResearchBundle` — write the bundle to disk; default `true`.
|
|
42
|
+
|
|
43
|
+
## Search Modes
|
|
44
|
+
|
|
45
|
+
- **Individual engine** — calls one engine and returns its answer/sources.
|
|
46
|
+
- **Grounded all-engine search** — fans out to configured engines, ranks
|
|
47
|
+
sources, fetches top source content, and returns confidence metadata.
|
|
48
|
+
- **All + synthesis** — adds a synthesis pass over engine answers and fetched
|
|
49
|
+
source evidence.
|
|
50
|
+
- **Research** — runs planning, searches, source fetching, learning extraction,
|
|
51
|
+
citation audit, and bundle writing.
|
|
52
|
+
|
|
53
|
+
## Configuration
|
|
54
|
+
|
|
55
|
+
Configure all-engine fan-out and synthesis in `~/.pi/greedyconfig`:
|
|
56
|
+
|
|
57
|
+
```json
|
|
58
|
+
{
|
|
59
|
+
"engines": ["perplexity", "google", "chatgpt", "gemini"],
|
|
60
|
+
"synthesizer": "gemini"
|
|
61
|
+
}
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
`semantic-scholar` and `logically` are opt-in research engines. They remain
|
|
65
|
+
available as individual engines and can be added to `engines` when you want them
|
|
66
|
+
in normal `engine: "all"` fan-out.
|
|
67
|
+
|
|
68
|
+
Deep research child searches reuse the configured `engines` list and pass query
|
|
69
|
+
text through stdin to avoid leaking prompts in process arguments.
|
package/extractors/chatgpt.mjs
CHANGED
|
@@ -25,7 +25,6 @@ import {
|
|
|
25
25
|
prepareArgs,
|
|
26
26
|
validateQuery,
|
|
27
27
|
waitForSelector,
|
|
28
|
-
waitForStreamComplete,
|
|
29
28
|
} from "./common.mjs";
|
|
30
29
|
import { dismissConsent, handleVerification } from "./consent.mjs";
|
|
31
30
|
|
|
@@ -131,14 +130,96 @@ const CHATGPT_RESPONSE_SELECTOR = String.raw`(() => {
|
|
|
131
130
|
* and resolve quickly because the entire response
|
|
132
131
|
* actually has finished.
|
|
133
132
|
*/
|
|
134
|
-
async function
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
133
|
+
async function bringToFront(tab, { required = false } = {}) {
|
|
134
|
+
try {
|
|
135
|
+
await cdp(["evalraw", tab, "Page.bringToFront", "{}"], 5000);
|
|
136
|
+
return true;
|
|
137
|
+
} catch (error) {
|
|
138
|
+
console.error(`[chatgpt] bringToFront failed: ${error.message}`);
|
|
139
|
+
if (required) throw error;
|
|
140
|
+
return false;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
async function waitForResponse(tab, timeoutMs = 35000) {
|
|
145
|
+
await bringToFront(tab, { required: true });
|
|
146
|
+
let foregroundAttempts = 0;
|
|
147
|
+
let foregroundInFlight = false;
|
|
148
|
+
const keepForeground = setInterval(() => {
|
|
149
|
+
if (foregroundInFlight) return;
|
|
150
|
+
if (foregroundAttempts >= 8) {
|
|
151
|
+
console.error("[chatgpt] stopping foreground keepalive after 8 attempts");
|
|
152
|
+
clearInterval(keepForeground);
|
|
153
|
+
return;
|
|
154
|
+
}
|
|
155
|
+
foregroundAttempts++;
|
|
156
|
+
foregroundInFlight = true;
|
|
157
|
+
bringToFront(tab).finally(() => {
|
|
158
|
+
foregroundInFlight = false;
|
|
159
|
+
});
|
|
160
|
+
}, 4000);
|
|
161
|
+
try {
|
|
162
|
+
const code = String.raw`
|
|
163
|
+
new Promise((resolve, reject) => {
|
|
164
|
+
const _deadline = Date.now() + ${timeoutMs};
|
|
165
|
+
const _baseInterval = 700;
|
|
166
|
+
const _stableRounds = 5;
|
|
167
|
+
let _lastLen = -1;
|
|
168
|
+
let _stableCount = 0;
|
|
169
|
+
|
|
170
|
+
function _jitter(ms) {
|
|
171
|
+
return Math.max(50, ms + (Math.random() * ms * 0.4 - ms * 0.2));
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
function _assistantAfterLastUser() {
|
|
175
|
+
const all = document.querySelectorAll('[data-message-author-role]');
|
|
176
|
+
let lastUserIdx = -1;
|
|
177
|
+
for (let i = 0; i < all.length; i++) {
|
|
178
|
+
if (all[i].getAttribute('data-message-author-role') === 'user') lastUserIdx = i;
|
|
179
|
+
}
|
|
180
|
+
if (lastUserIdx < 0) return null;
|
|
181
|
+
let assistant = null;
|
|
182
|
+
for (let i = lastUserIdx + 1; i < all.length; i++) {
|
|
183
|
+
if (all[i].getAttribute('data-message-author-role') === 'assistant') assistant = all[i];
|
|
184
|
+
}
|
|
185
|
+
return assistant;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
function _poll() {
|
|
189
|
+
try {
|
|
190
|
+
const el = _assistantAfterLastUser();
|
|
191
|
+
const text = (el?.innerText || '').trim();
|
|
192
|
+
const len = text.length;
|
|
193
|
+
const streaming = !!el?.querySelector('.streaming-animation,[data-is-streaming="true"]') ||
|
|
194
|
+
!!document.querySelector('button[data-testid="stop-button"], button[aria-label="Stop generating"], button[aria-label*="Stop"]');
|
|
195
|
+
if (len >= 1 && !streaming) {
|
|
196
|
+
if (len === _lastLen) {
|
|
197
|
+
_stableCount++;
|
|
198
|
+
if (_stableCount >= _stableRounds) { resolve(len); return; }
|
|
199
|
+
} else {
|
|
200
|
+
_lastLen = len;
|
|
201
|
+
_stableCount = 0;
|
|
202
|
+
}
|
|
203
|
+
} else if (len !== _lastLen) {
|
|
204
|
+
_lastLen = len;
|
|
205
|
+
_stableCount = 0;
|
|
206
|
+
}
|
|
207
|
+
if (Date.now() < _deadline) setTimeout(_poll, _jitter(_baseInterval));
|
|
208
|
+
else if (_lastLen >= 1 && !streaming) resolve(_lastLen);
|
|
209
|
+
else reject(new Error('ChatGPT response did not finish streaming within ${timeoutMs}ms'));
|
|
210
|
+
} catch (e) { reject(e); }
|
|
211
|
+
}
|
|
212
|
+
_poll();
|
|
213
|
+
})`;
|
|
214
|
+
const lenStr = await cdp(["eval", tab, code], timeoutMs + 10000);
|
|
215
|
+
const len = parseInt(lenStr, 10) || 0;
|
|
216
|
+
if (len >= 1) return len;
|
|
217
|
+
throw new Error(
|
|
218
|
+
`ChatGPT response did not finish streaming within ${timeoutMs}ms`,
|
|
219
|
+
);
|
|
220
|
+
} finally {
|
|
221
|
+
clearInterval(keepForeground);
|
|
222
|
+
}
|
|
142
223
|
}
|
|
143
224
|
|
|
144
225
|
/**
|
|
@@ -211,6 +292,8 @@ async function extractAnswerFromDom(tab) {
|
|
|
211
292
|
skipped: 'no-assistant-response',
|
|
212
293
|
});
|
|
213
294
|
}
|
|
295
|
+
const streaming = !!assistant.querySelector('.streaming-animation,[data-is-streaming="true"]') ||
|
|
296
|
+
!!document.querySelector('button[data-testid="stop-button"], button[aria-label="Stop generating"], button[aria-label*="Stop"]');
|
|
214
297
|
const answer = (assistant.innerText || assistant.textContent || '').trim();
|
|
215
298
|
const seen = new Set();
|
|
216
299
|
const sources = [];
|
|
@@ -222,13 +305,16 @@ async function extractAnswerFromDom(tab) {
|
|
|
222
305
|
sources.push({ title, url });
|
|
223
306
|
if (sources.length >= 10) break;
|
|
224
307
|
}
|
|
225
|
-
return JSON.stringify({ answer, sources });
|
|
308
|
+
return JSON.stringify({ answer, sources, streaming });
|
|
226
309
|
})()
|
|
227
310
|
`,
|
|
228
311
|
]);
|
|
229
312
|
try {
|
|
230
313
|
return JSON.parse(raw);
|
|
231
|
-
} catch {
|
|
314
|
+
} catch (error) {
|
|
315
|
+
console.error(
|
|
316
|
+
`[chatgpt] DOM fallback JSON parse failed (${raw.length} chars): ${error.message}`,
|
|
317
|
+
);
|
|
232
318
|
return { answer: "", sources: [], skipped: "parse-error" };
|
|
233
319
|
}
|
|
234
320
|
}
|
|
@@ -307,6 +393,9 @@ async function extractAnswer(tab, env) {
|
|
|
307
393
|
let domFallback = null;
|
|
308
394
|
if (!answer) {
|
|
309
395
|
domFallback = await extractAnswerFromDom(tab);
|
|
396
|
+
if (domFallback.streaming) {
|
|
397
|
+
return { answer: "", sources: [], skipped: "still-streaming" };
|
|
398
|
+
}
|
|
310
399
|
answer = domFallback.answer;
|
|
311
400
|
env.fallbackUsed = answer ? "dom" : null;
|
|
312
401
|
}
|
|
@@ -335,15 +424,25 @@ async function extractAnswer(tab, env) {
|
|
|
335
424
|
trimmed.length <= 50 &&
|
|
336
425
|
/\s|[.,!?;:]/.test(trimmed);
|
|
337
426
|
const looksLikeLongAnswer = trimmed.length > 50;
|
|
338
|
-
|
|
427
|
+
const words = trimmed.split(/\s+/).filter(Boolean);
|
|
428
|
+
const domainRepeats = (
|
|
429
|
+
trimmed.match(/\b[a-z0-9-]+\.(?:com|org|net|dev|io)\b/gi) || []
|
|
430
|
+
).length;
|
|
431
|
+
const looksLikeCitationStub =
|
|
432
|
+
domainRepeats >= 4 &&
|
|
433
|
+
domainRepeats >= Math.max(3, Math.floor(words.length / 3));
|
|
434
|
+
if (
|
|
435
|
+
(!looksLikeShortAnswer && !looksLikeLongAnswer) ||
|
|
436
|
+
looksLikeCitationStub
|
|
437
|
+
) {
|
|
339
438
|
console.error(
|
|
340
|
-
`[chatgpt] DOM fallback answer
|
|
439
|
+
`[chatgpt] DOM fallback answer suspicious (${trimmed.length} chars, domainRepeats=${domainRepeats}: ${JSON.stringify(trimmed.slice(0, 80))}) — returning empty for caller to retry`,
|
|
341
440
|
);
|
|
342
441
|
env.fallbackUsed = null;
|
|
343
442
|
return {
|
|
344
443
|
answer: "",
|
|
345
444
|
sources: [],
|
|
346
|
-
skipped: "header-stub",
|
|
445
|
+
skipped: looksLikeCitationStub ? "citation-stub" : "header-stub",
|
|
347
446
|
};
|
|
348
447
|
}
|
|
349
448
|
}
|
|
@@ -518,15 +617,20 @@ async function main() {
|
|
|
518
617
|
if (!answer) {
|
|
519
618
|
env.blockedBy = "no-response";
|
|
520
619
|
env.skipped = skipped || null;
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
620
|
+
let message = "ChatGPT returned no answer — assistant never responded";
|
|
621
|
+
if (skipped === "no-user-message") {
|
|
622
|
+
message = "ChatGPT still on homepage — query was not submitted";
|
|
623
|
+
} else if (skipped === "no-assistant-response") {
|
|
624
|
+
message = "ChatGPT did not return an assistant response after submit";
|
|
625
|
+
} else if (
|
|
626
|
+
skipped === "header-stub" ||
|
|
627
|
+
skipped === "citation-stub" ||
|
|
628
|
+
skipped === "still-streaming"
|
|
629
|
+
) {
|
|
630
|
+
message =
|
|
631
|
+
"ChatGPT response did not finish rendering after 3 retries — assistant never rendered the body";
|
|
632
|
+
}
|
|
633
|
+
throw new Error(message);
|
|
530
634
|
}
|
|
531
635
|
logStage(env, "done", startTime);
|
|
532
636
|
|
package/extractors/common.mjs
CHANGED
|
@@ -129,7 +129,18 @@ export async function getOrOpenTab(tabPrefix) {
|
|
|
129
129
|
"Target.createTarget",
|
|
130
130
|
'{"url":"about:blank"}',
|
|
131
131
|
]);
|
|
132
|
-
|
|
132
|
+
let createdTarget;
|
|
133
|
+
try {
|
|
134
|
+
createdTarget = JSON.parse(raw);
|
|
135
|
+
} catch (error) {
|
|
136
|
+
throw new Error(
|
|
137
|
+
`Target.createTarget returned invalid JSON: ${error.message}`,
|
|
138
|
+
);
|
|
139
|
+
}
|
|
140
|
+
const { targetId } = createdTarget;
|
|
141
|
+
if (!targetId) {
|
|
142
|
+
throw new Error("Target.createTarget did not return a targetId");
|
|
143
|
+
}
|
|
133
144
|
await cdp(["list"]); // refresh cache
|
|
134
145
|
const tid = targetId.slice(0, 8);
|
|
135
146
|
// Inject stealth patches for anti-detection coverage (both headless + visible).
|
|
@@ -221,40 +232,62 @@ export async function injectHeadlessStealth(tab) {
|
|
|
221
232
|
try { delete window._phantom; } catch(_) {}
|
|
222
233
|
try { delete window.Buffer; } catch(_) {}
|
|
223
234
|
|
|
224
|
-
// Real Chrome without automation
|
|
225
|
-
// A literal false
|
|
226
|
-
//
|
|
227
|
-
|
|
235
|
+
// Real Chrome without automation should not expose navigator.webdriver at all.
|
|
236
|
+
// A literal false or an own-property getter returning undefined is itself a
|
|
237
|
+
// common stealth tell; remove both instance and prototype properties when the
|
|
238
|
+
// descriptor is configurable (as it is with --disable-blink-features).
|
|
239
|
+
try { delete navigator.webdriver; } catch(_) {}
|
|
240
|
+
try { delete Navigator.prototype.webdriver; } catch(_) {}
|
|
228
241
|
Object.defineProperty(navigator, 'vendor', { get: () => 'Google Inc.', configurable: true });
|
|
229
242
|
Object.defineProperty(navigator, 'platform', { get: () => 'Win32', configurable: true });
|
|
230
243
|
Object.defineProperty(navigator, 'maxTouchPoints', { get: () => 0, configurable: true });
|
|
231
244
|
Object.defineProperty(navigator, 'pdfViewerEnabled', { get: () => true, configurable: true });
|
|
245
|
+
Object.defineProperty(navigator, 'productSub', { get: () => '20030107', configurable: true });
|
|
246
|
+
Object.defineProperty(navigator, 'product', { get: () => 'Gecko', configurable: true });
|
|
247
|
+
var __greedyMimeTypes = null;
|
|
248
|
+
function __makeMimeTypes() {
|
|
249
|
+
var pdf = { type: 'application/pdf', suffixes: 'pdf', description: 'Portable Document Format', enabledPlugin: null };
|
|
250
|
+
var textPdf = { type: 'text/pdf', suffixes: 'pdf', description: 'Portable Document Format', enabledPlugin: null };
|
|
251
|
+
try { Object.setPrototypeOf(pdf, MimeType.prototype); } catch(_) {}
|
|
252
|
+
try { Object.setPrototypeOf(textPdf, MimeType.prototype); } catch(_) {}
|
|
253
|
+
var m = [pdf, textPdf];
|
|
254
|
+
try { Object.setPrototypeOf(m, MimeTypeArray.prototype); } catch(_) {}
|
|
255
|
+
m.item = function item(i) { return this[i] || null; };
|
|
256
|
+
m.namedItem = function namedItem(name) { return Array.prototype.find.call(this, function(x) { return x && x.type === name; }) || null; };
|
|
257
|
+
return m;
|
|
258
|
+
}
|
|
232
259
|
Object.defineProperty(navigator, 'plugins', {
|
|
233
260
|
get: () => {
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
261
|
+
__greedyMimeTypes = __greedyMimeTypes || __makeMimeTypes();
|
|
262
|
+
var plugin0 = { name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format' };
|
|
263
|
+
var plugin1 = { name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '' };
|
|
264
|
+
var plugin2 = { name: 'Native Client', filename: 'internal-nacl-plugin', description: '' };
|
|
265
|
+
try { Object.setPrototypeOf(plugin0, Plugin.prototype); } catch(_) {}
|
|
266
|
+
try { Object.setPrototypeOf(plugin1, Plugin.prototype); } catch(_) {}
|
|
267
|
+
try { Object.setPrototypeOf(plugin2, Plugin.prototype); } catch(_) {}
|
|
268
|
+
var p = [plugin0, plugin1, plugin2];
|
|
269
|
+
p.item = function item(i) { return this[i] || null; };
|
|
270
|
+
p.namedItem = function namedItem(name) { return Array.prototype.find.call(this, function(x) { return x && x.name === name; }) || null; };
|
|
271
|
+
p.refresh = function refresh() {};
|
|
272
|
+
try { Object.setPrototypeOf(p, PluginArray.prototype); } catch(_) {}
|
|
273
|
+
try {
|
|
274
|
+
__greedyMimeTypes[0].enabledPlugin = p[0];
|
|
275
|
+
__greedyMimeTypes[1].enabledPlugin = p[0];
|
|
276
|
+
} catch(_) {}
|
|
240
277
|
return p;
|
|
241
278
|
},
|
|
279
|
+
configurable: true,
|
|
242
280
|
});
|
|
243
281
|
Object.defineProperty(navigator, 'mimeTypes', {
|
|
244
282
|
get: () => {
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
{ type: 'text/pdf', suffixes: 'pdf', description: 'Portable Document Format', enabledPlugin: null },
|
|
248
|
-
];
|
|
249
|
-
m.item = function(i) { return m[i] || null; };
|
|
250
|
-
m.namedItem = function(name) { return m.find(function(x) { return x.type === name; }) || null; };
|
|
251
|
-
return m;
|
|
283
|
+
__greedyMimeTypes = __greedyMimeTypes || __makeMimeTypes();
|
|
284
|
+
return __greedyMimeTypes;
|
|
252
285
|
},
|
|
253
286
|
configurable: true,
|
|
254
287
|
});
|
|
255
288
|
Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'], configurable: true });
|
|
256
289
|
try {
|
|
257
|
-
Object.defineProperty(navigator, 'connection', { get: () => ({ effectiveType: '4g', rtt: 50, downlink: 10, saveData: false }), configurable: true });
|
|
290
|
+
Object.defineProperty(navigator, 'connection', { get: () => ({ effectiveType: '4g', rtt: 50, downlink: 10, downlinkMax: Infinity, saveData: false }), configurable: true });
|
|
258
291
|
} catch(_) {}
|
|
259
292
|
if (!navigator.mediaDevices) {
|
|
260
293
|
Object.defineProperty(navigator, 'mediaDevices', {
|
|
@@ -270,6 +303,18 @@ export async function injectHeadlessStealth(tab) {
|
|
|
270
303
|
configurable: true,
|
|
271
304
|
});
|
|
272
305
|
}
|
|
306
|
+
// ── Missing platform APIs (headless often lacks these) ─
|
|
307
|
+
try {
|
|
308
|
+
if (!navigator.share) {
|
|
309
|
+
navigator.share = function() { return Promise.reject(new Error('NotAllowedError')); };
|
|
310
|
+
}
|
|
311
|
+
} catch(_) {}
|
|
312
|
+
try {
|
|
313
|
+
if (!navigator.contentIndex) {
|
|
314
|
+
Object.defineProperty(navigator, 'contentIndex', { get: () => ({ add: function() {}, delete: function() {}, getAll: function() { return Promise.resolve([]); } }), configurable: true });
|
|
315
|
+
}
|
|
316
|
+
} catch(_) {}
|
|
317
|
+
|
|
273
318
|
if (!window.chrome) {
|
|
274
319
|
window.chrome = {
|
|
275
320
|
app: { isInstalled: false, InstallState: {}, RunningState: {} },
|
|
@@ -277,8 +322,8 @@ export async function injectHeadlessStealth(tab) {
|
|
|
277
322
|
OnInstalledReason: {}, OnRestartRequiredReason: {}, PlatformArch: {}, PlatformNaclArch: {}, PlatformOs: {}, RequestUpdateCheckStatus: {},
|
|
278
323
|
connect: () => ({}), sendMessage: () => {}, onMessage: { addListener: () => {} }
|
|
279
324
|
},
|
|
280
|
-
loadTimes: ()
|
|
281
|
-
csi: ()
|
|
325
|
+
loadTimes: function() { return { requestTime: 0, startLoadTime: Date.now() - 5000, commitLoadTime: Date.now() - 3000, finishDocumentLoadTime: Date.now() - 2000, finishLoadTime: Date.now() - 1000, firstPaintTime: Date.now() - 800, navigationType: 'Other', wasFetchedViaSpdy: true, wasNpnNegotiated: true, npnNegotiatedProtocol: 'h2', wasAlternateProtocolAvailable: false, connectionInfo: 'http/2' }; },
|
|
326
|
+
csi: function() { var t = Date.now(); return { onloadT: t - 2000, startE: t - 5000, pageT: 'back', tran: 2 }; },
|
|
282
327
|
};
|
|
283
328
|
}
|
|
284
329
|
var __greedyNativeFns = [];
|
|
@@ -299,6 +344,19 @@ export async function injectHeadlessStealth(tab) {
|
|
|
299
344
|
return getParam.call(this, p);
|
|
300
345
|
});
|
|
301
346
|
} catch(_) {}
|
|
347
|
+
// ── WebGL readPixels noise ──────────────────────────
|
|
348
|
+
// CreepJS and other fingerprinters draw content with WebGL and read back the
|
|
349
|
+
// rendered pixels. Adding subtle noise breaks rendering-based fingerprinting.
|
|
350
|
+
try {
|
|
351
|
+
var origReadPixels = WebGLRenderingContext.prototype.readPixels;
|
|
352
|
+
WebGLRenderingContext.prototype.readPixels = __markNative(function readPixels(x, y, width, height, format, type, pixels) {
|
|
353
|
+
var result = origReadPixels.call(this, x, y, width, height, format, type, pixels);
|
|
354
|
+
if (pixels && pixels.length > 0) {
|
|
355
|
+
pixels[0] ^= 1;
|
|
356
|
+
}
|
|
357
|
+
return result;
|
|
358
|
+
});
|
|
359
|
+
} catch(_) {}
|
|
302
360
|
Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 8, configurable: true });
|
|
303
361
|
Object.defineProperty(navigator, 'deviceMemory', { get: () => 8, configurable: true });
|
|
304
362
|
|
|
@@ -306,7 +364,7 @@ export async function injectHeadlessStealth(tab) {
|
|
|
306
364
|
// Headless rendering engines produce slightly different canvas output
|
|
307
365
|
// than headed Chrome. Subtle noise breaks hash-based fingerprinting.
|
|
308
366
|
try {
|
|
309
|
-
var __canvasNoise = ((Date.now()
|
|
367
|
+
var __canvasNoise = ((Date.now() & 0xFF) | 1);
|
|
310
368
|
var origFill = CanvasRenderingContext2D.prototype.fillText;
|
|
311
369
|
CanvasRenderingContext2D.prototype.fillText = __markNative(function fillText() {
|
|
312
370
|
this.globalAlpha = 0.9995;
|
|
@@ -325,15 +383,39 @@ export async function injectHeadlessStealth(tab) {
|
|
|
325
383
|
HTMLCanvasElement.prototype.toDataURL = __markNative(function toDataURL() {
|
|
326
384
|
var ctx = this.getContext('2d');
|
|
327
385
|
if (ctx) {
|
|
328
|
-
//
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
386
|
+
// Spread noise across canvas to break hash-based fingerprinting.
|
|
387
|
+
// Uses a deterministic pattern so it's consistent per page load
|
|
388
|
+
// but varies between sessions.
|
|
389
|
+
var w = this.width, h = this.height;
|
|
390
|
+
if (w > 0 && h > 0) {
|
|
391
|
+
var imgData = ctx.getImageData(0, 0, Math.min(w, 4), Math.min(h, 4));
|
|
392
|
+
if (imgData && imgData.data) {
|
|
393
|
+
for (var __i = 0; __i < imgData.data.length; __i += 4) {
|
|
394
|
+
imgData.data[__i] ^= (__canvasNoise + __i) & 0xFF;
|
|
395
|
+
}
|
|
396
|
+
ctx.putImageData(imgData, 0, 0);
|
|
397
|
+
}
|
|
398
|
+
}
|
|
332
399
|
}
|
|
333
400
|
return origToDataURL.apply(this, arguments);
|
|
334
401
|
});
|
|
335
402
|
} catch(_) {}
|
|
336
403
|
|
|
404
|
+
// ── AudioContext fingerprint noise ────────────────────
|
|
405
|
+
// Headless Chrome's AudioContext produces slightly different output.
|
|
406
|
+
// Subtle noise breaks audio-based fingerprinting.
|
|
407
|
+
try {
|
|
408
|
+
var __audioSeed = ((Date.now() & 0x1F) | 1);
|
|
409
|
+
var origGetChannelData = AudioBuffer.prototype.getChannelData;
|
|
410
|
+
AudioBuffer.prototype.getChannelData = __markNative(function getChannelData(channel) {
|
|
411
|
+
var data = origGetChannelData.call(this, channel);
|
|
412
|
+
for (var __i = 0; __i < data.length; __i += 64) {
|
|
413
|
+
data[__i] *= 0.99999;
|
|
414
|
+
}
|
|
415
|
+
return data;
|
|
416
|
+
});
|
|
417
|
+
} catch(_) {}
|
|
418
|
+
|
|
337
419
|
// ── window outer dimensions ──────────────────────────
|
|
338
420
|
// outerWidth/Height = 0 in headless — a well-known bot signal.
|
|
339
421
|
// Mirror innerWidth/Height (set by --window-size flag) so the ratio is sane.
|
|
@@ -343,9 +425,15 @@ export async function injectHeadlessStealth(tab) {
|
|
|
343
425
|
} catch(_) {}
|
|
344
426
|
|
|
345
427
|
// ── screen properties ─────────────────────────────────
|
|
428
|
+
// Headless Chrome often reports an 800x600 screen even when the viewport is
|
|
429
|
+
// 1920x1080. Keep screen metrics internally consistent with our launch flags.
|
|
346
430
|
try {
|
|
347
|
-
|
|
348
|
-
|
|
431
|
+
Object.defineProperty(screen, 'width', { get: () => 1920, configurable: true });
|
|
432
|
+
Object.defineProperty(screen, 'height', { get: () => 1080, configurable: true });
|
|
433
|
+
Object.defineProperty(screen, 'availWidth', { get: () => 1920, configurable: true });
|
|
434
|
+
Object.defineProperty(screen, 'availHeight', { get: () => 1040, configurable: true });
|
|
435
|
+
Object.defineProperty(screen, 'colorDepth', { get: () => 24, configurable: true });
|
|
436
|
+
Object.defineProperty(screen, 'pixelDepth', { get: () => 24, configurable: true });
|
|
349
437
|
} catch(_) {}
|
|
350
438
|
|
|
351
439
|
// ── navigator.userAgentData (UA Client Hints) ─────────
|
package/extractors/gemini.mjs
CHANGED
|
@@ -66,6 +66,17 @@ async function typeIntoGemini(tab, text) {
|
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
68
|
|
|
69
|
+
async function bringToFront(tab, { required = false } = {}) {
|
|
70
|
+
try {
|
|
71
|
+
await cdp(["evalraw", tab, "Page.bringToFront", "{}"], 5000);
|
|
72
|
+
return true;
|
|
73
|
+
} catch (error) {
|
|
74
|
+
console.error(`[gemini] bringToFront failed: ${error.message}`);
|
|
75
|
+
if (required) throw error;
|
|
76
|
+
return false;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
69
80
|
async function scrollToBottom(tab) {
|
|
70
81
|
await cdp([
|
|
71
82
|
"eval",
|
|
@@ -93,7 +104,8 @@ async function extractAnswerFromDom(tab) {
|
|
|
93
104
|
new Promise((resolve) => {
|
|
94
105
|
const _deadline = Date.now() + 6000;
|
|
95
106
|
function _tryExtract() {
|
|
96
|
-
const
|
|
107
|
+
const responses = Array.from(document.querySelectorAll('model-response'));
|
|
108
|
+
const resp = responses[responses.length - 1];
|
|
97
109
|
if (resp) {
|
|
98
110
|
const text = (resp.innerText || resp.textContent || '').trim();
|
|
99
111
|
const idx = text.indexOf('\n');
|
|
@@ -126,11 +138,32 @@ async function extractAnswerFromDom(tab) {
|
|
|
126
138
|
);
|
|
127
139
|
try {
|
|
128
140
|
return JSON.parse(raw);
|
|
129
|
-
} catch {
|
|
141
|
+
} catch (error) {
|
|
142
|
+
console.error(
|
|
143
|
+
`[gemini] DOM fallback JSON parse failed (${raw.length} chars): ${error.message}`,
|
|
144
|
+
);
|
|
130
145
|
return { answer: "", sources: [] };
|
|
131
146
|
}
|
|
132
147
|
}
|
|
133
148
|
|
|
149
|
+
async function clickLatestModelResponseCopy(tab) {
|
|
150
|
+
return cdp([
|
|
151
|
+
"eval",
|
|
152
|
+
tab,
|
|
153
|
+
`(() => {
|
|
154
|
+
const responses = Array.from(document.querySelectorAll('model-response'));
|
|
155
|
+
const resp = responses[responses.length - 1];
|
|
156
|
+
if (!resp) return 'no-model-response';
|
|
157
|
+
const buttons = Array.from(resp.querySelectorAll('${S.copyButton}'));
|
|
158
|
+
const nonCodeButtons = buttons.filter((btn) => !btn.closest('pre, code, code-block, .code-block, .code-container'));
|
|
159
|
+
const btn = nonCodeButtons[nonCodeButtons.length - 1] || buttons[buttons.length - 1];
|
|
160
|
+
if (!btn) return 'no-copy-button';
|
|
161
|
+
btn.click();
|
|
162
|
+
return 'clicked';
|
|
163
|
+
})()`,
|
|
164
|
+
]);
|
|
165
|
+
}
|
|
166
|
+
|
|
134
167
|
async function extractAnswer(tab, query = "") {
|
|
135
168
|
const queryNorm = query.toLowerCase().trim();
|
|
136
169
|
|
|
@@ -151,7 +184,8 @@ async function extractAnswer(tab, query = "") {
|
|
|
151
184
|
"eval",
|
|
152
185
|
tab,
|
|
153
186
|
String.raw`(() => {
|
|
154
|
-
const
|
|
187
|
+
const responses = Array.from(document.querySelectorAll('model-response'));
|
|
188
|
+
const r = responses[responses.length - 1];
|
|
155
189
|
if (!r) return false;
|
|
156
190
|
const t = (r.innerText || '').trim();
|
|
157
191
|
// Must have content beyond the locale-specific label
|
|
@@ -174,18 +208,7 @@ async function extractAnswer(tab, query = "") {
|
|
|
174
208
|
// not the absolute last copy button on the page. The page has many
|
|
175
209
|
// copy icons (copy link, copy code, etc.) and the last one is not
|
|
176
210
|
// always the assistant's response copy button.
|
|
177
|
-
await
|
|
178
|
-
"eval",
|
|
179
|
-
tab,
|
|
180
|
-
`(() => {
|
|
181
|
-
const resp = document.querySelector('model-response');
|
|
182
|
-
if (!resp) return 'no-model-response';
|
|
183
|
-
const btn = resp.querySelector('${S.copyButton}');
|
|
184
|
-
if (!btn) return 'no-copy-button';
|
|
185
|
-
btn.click();
|
|
186
|
-
return 'clicked';
|
|
187
|
-
})()`,
|
|
188
|
-
]);
|
|
211
|
+
await clickLatestModelResponseCopy(tab);
|
|
189
212
|
await new Promise((r) => setTimeout(r, 600));
|
|
190
213
|
|
|
191
214
|
let answer = await cdp(["eval", tab, `window.${GLOBAL_VAR} || ''`]);
|
|
@@ -200,35 +223,32 @@ async function extractAnswer(tab, query = "") {
|
|
|
200
223
|
) {
|
|
201
224
|
console.error("[gemini] Clipboard echoed query, retrying in 2s...");
|
|
202
225
|
await new Promise((r) => setTimeout(r, 2000));
|
|
203
|
-
await
|
|
204
|
-
"eval",
|
|
205
|
-
tab,
|
|
206
|
-
`(() => {
|
|
207
|
-
const resp = document.querySelector('model-response');
|
|
208
|
-
if (!resp) return 'no-model-response';
|
|
209
|
-
const btn = resp.querySelector('${S.copyButton}');
|
|
210
|
-
if (!btn) return 'no-copy-button';
|
|
211
|
-
btn.click();
|
|
212
|
-
return 'clicked';
|
|
213
|
-
})()`,
|
|
214
|
-
]);
|
|
226
|
+
await clickLatestModelResponseCopy(tab);
|
|
215
227
|
await new Promise((r) => setTimeout(r, 600));
|
|
216
228
|
answer = await cdp(["eval", tab, `window.${GLOBAL_VAR} || ''`]);
|
|
217
229
|
}
|
|
218
230
|
|
|
219
|
-
// DOM fallback: if
|
|
220
|
-
// read the model-response innerText directly.
|
|
221
|
-
//
|
|
222
|
-
// the click didn't fire.
|
|
231
|
+
// DOM fallback: if clipboard is empty, echoes the query, or looks like a
|
|
232
|
+
// short snippet, read the model-response innerText directly. Skip the
|
|
233
|
+
// fallback when native copy already returned a substantial full answer.
|
|
223
234
|
let domFallback = null;
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
235
|
+
const answerText = answer?.trim() || "";
|
|
236
|
+
const clipboardLooksEchoed =
|
|
237
|
+
queryNorm &&
|
|
238
|
+
(answerText.toLowerCase() === queryNorm ||
|
|
239
|
+
answerText.length < queryNorm.length);
|
|
240
|
+
const clipboardLooksComplete =
|
|
241
|
+
answerText.length > 500 && !clipboardLooksEchoed;
|
|
242
|
+
if (!clipboardLooksComplete) {
|
|
230
243
|
domFallback = await extractAnswerFromDom(tab);
|
|
231
|
-
|
|
244
|
+
const domExtendsClipboard =
|
|
245
|
+
answerText &&
|
|
246
|
+
domFallback.answer.includes(answerText) &&
|
|
247
|
+
domFallback.answer.length > answerText.length * 2;
|
|
248
|
+
if (
|
|
249
|
+
domFallback.answer &&
|
|
250
|
+
(!answerText || clipboardLooksEchoed || domExtendsClipboard)
|
|
251
|
+
) {
|
|
232
252
|
answer = domFallback.answer;
|
|
233
253
|
}
|
|
234
254
|
}
|
|
@@ -304,6 +324,7 @@ async function main() {
|
|
|
304
324
|
await waitForSelector(tab, S.input, 8000, TIMING.inputPoll);
|
|
305
325
|
await new Promise((r) => setTimeout(r, jitter(TIMING.postClick)));
|
|
306
326
|
|
|
327
|
+
await bringToFront(tab, { required: true });
|
|
307
328
|
await injectClipboardInterceptor(tab, GLOBAL_VAR);
|
|
308
329
|
await typeIntoGemini(tab, query);
|
|
309
330
|
await new Promise((r) => setTimeout(r, jitter(TIMING.postType)));
|
|
@@ -316,8 +337,10 @@ async function main() {
|
|
|
316
337
|
|
|
317
338
|
// Wait for Gemini's response to finish streaming before extracting.
|
|
318
339
|
// Periodic scrolling keeps lazy-loaded content triggered in the viewport.
|
|
340
|
+
await bringToFront(tab);
|
|
319
341
|
let pollTick = 0;
|
|
320
342
|
const scrollInterval = setInterval(() => {
|
|
343
|
+
bringToFront(tab).catch(() => null);
|
|
321
344
|
if (++pollTick % 10 === 0) scrollToBottom(tab).catch(() => null);
|
|
322
345
|
}, 6000);
|
|
323
346
|
try {
|
|
@@ -325,6 +348,8 @@ async function main() {
|
|
|
325
348
|
timeout: 45000,
|
|
326
349
|
stableRounds: 5,
|
|
327
350
|
minLength: 60,
|
|
351
|
+
selector:
|
|
352
|
+
"(() => { const responses = Array.from(document.querySelectorAll('model-response')); return responses[responses.length - 1]; })()",
|
|
328
353
|
});
|
|
329
354
|
} finally {
|
|
330
355
|
clearInterval(scrollInterval);
|