@duckmind/dm-darwin-x64 0.52.6 → 0.53.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dm +0 -0
- package/extensions/.dm-extensions.json +14 -10
- package/extensions/dm-cua/bin/browser-cua.mjs +6 -6
- package/extensions/dm-cua/index.js +5 -5
- package/extensions/dm-cua/src/browser-cua-lib.mjs +2 -2
- package/extensions/greedysearch-dm/bin/cdp-greedy.mjs +8 -1
- package/extensions/greedysearch-dm/bin/cdp-headless.mjs +1 -1
- package/extensions/greedysearch-dm/bin/cdp-visible.mjs +1 -1
- package/extensions/greedysearch-dm/bin/cdp.mjs +29 -20
- package/extensions/greedysearch-dm/bin/gschrome.mjs +1 -8
- package/extensions/greedysearch-dm/bin/kill-visible.mjs +1 -1
- package/extensions/greedysearch-dm/bin/launch-visible.mjs +1 -2
- package/extensions/greedysearch-dm/bin/launch.mjs +9 -11
- package/extensions/greedysearch-dm/bin/mcp.mjs +375 -0
- package/extensions/greedysearch-dm/bin/search.mjs +309 -157
- package/extensions/greedysearch-dm/bin/visible.mjs +1 -1
- package/extensions/greedysearch-dm/extractors/bing-copilot.mjs +205 -68
- package/extensions/greedysearch-dm/extractors/chatgpt.mjs +229 -140
- package/extensions/greedysearch-dm/extractors/common.mjs +160 -50
- package/extensions/greedysearch-dm/extractors/consensus.mjs +226 -93
- package/extensions/greedysearch-dm/extractors/consent.mjs +44 -17
- package/extensions/greedysearch-dm/extractors/gemini.mjs +277 -138
- package/extensions/greedysearch-dm/extractors/google-ai.mjs +172 -64
- package/extensions/greedysearch-dm/extractors/logically.mjs +183 -69
- package/extensions/greedysearch-dm/extractors/perplexity.mjs +331 -65
- package/extensions/greedysearch-dm/extractors/semantic-scholar.mjs +118 -41
- package/extensions/greedysearch-dm/index.js +23 -20
- package/extensions/greedysearch-dm/package.json +8 -3
- package/extensions/greedysearch-dm/skills/greedy-search/skill.md +8 -8
- package/extensions/greedysearch-dm/src/fetcher.mjs +2 -2
- package/extensions/greedysearch-dm/src/formatters/results.js +8 -6
- package/extensions/greedysearch-dm/src/github.mjs +7 -7
- package/extensions/greedysearch-dm/src/reddit.mjs +1 -1
- package/extensions/greedysearch-dm/src/search/browser-lifecycle.mjs +10 -8
- package/extensions/greedysearch-dm/src/search/challenge-detect.mjs +48 -0
- package/extensions/greedysearch-dm/src/search/chrome.mjs +130 -54
- package/extensions/greedysearch-dm/src/search/constants.mjs +6 -6
- package/extensions/greedysearch-dm/src/search/engines.mjs +9 -9
- package/extensions/greedysearch-dm/src/search/fetch-source.mjs +160 -83
- package/extensions/greedysearch-dm/src/search/minimize.mjs +1 -0
- package/extensions/greedysearch-dm/src/search/paths.mjs +1 -1
- package/extensions/greedysearch-dm/src/search/pdf.mjs +1 -1
- package/extensions/greedysearch-dm/src/search/port-pid.mjs +1 -1
- package/extensions/greedysearch-dm/src/search/progress.mjs +2 -0
- package/extensions/greedysearch-dm/src/search/recovery.mjs +1 -1
- package/extensions/greedysearch-dm/src/search/research.mjs +262 -167
- package/extensions/greedysearch-dm/src/search/scale-aware.mjs +11 -0
- package/extensions/greedysearch-dm/src/search/simple-research.mjs +827 -0
- package/extensions/greedysearch-dm/src/search/synthesis-runner.mjs +9 -9
- package/extensions/greedysearch-dm/src/search/synthesis.mjs +10 -10
- package/extensions/greedysearch-dm/src/tools/greedy-search-handler.js +23 -20
- package/extensions/greedysearch-dm/src/tools/shared.js +10 -9
- package/extensions/greedysearch-dm/src/utils/node-runtime.mjs +1 -1
- package/package.json +1 -1
- package/extensions/greedysearch-dm/src/search/cdp-endpoint.mjs +0 -3
- package/extensions/greedysearch-dm/src/search/launcher-paths.mjs +0 -3
- package/extensions/greedysearch-dm/src/search/partial-output.mjs +0 -1
- package/extensions/greedysearch-dm/src/utils/browser-executable.mjs +0 -1
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import{randomInt as
|
|
3
|
-
`)[0]?.slice(0,8);if(!
|
|
4
|
-
`)}return
|
|
2
|
+
import{randomInt as J}from"node:crypto";import{spawn as W}from"node:child_process";import{basename as q}from"node:path";import{dirname as B,join as V}from"node:path";import{fileURLToPath as U}from"node:url";function H(e=process.env,t=process.execPath){let r=e.GREEDY_SEARCH_NODE||e.NODE_BINARY||e.NODE;if(r?.trim())return r.trim();let n=q(t||"").toLowerCase();if(n==="node"||n==="node.exe")return t;return"node"}var Y=B(U(import.meta.url)),X=V(Y,"..","bin","cdp.mjs"),Z=new Set(["list","snap","eval","shot","html","nav","net","click","clickxy","type","loadall","evalraw","browse","stop","--tab"]);function G(e){if(!Array.isArray(e)||e.length===0)throw Error("cdp: args must be a non-empty array");if(e[0]==="test")return e.map((t,r)=>O(t,r));if(!Z.has(e[0]))throw Error(`cdp: unknown subcommand '${e[0]}'`);return e.map((t,r)=>O(t,r))}function O(e,t){if(typeof e!=="string")throw Error(`cdp: argv[${t}] must be a string (got ${typeof e})`);if(e.includes("\x00"))throw Error(`cdp: argv[${t}] contains a null byte`);return e}function m(e,t=30000){return z(e,null,t)}function z(e,t=null,r=30000){let n=G(e);return new Promise((i,o)=>{let a=W(H(),[X,...n],{stdio:[t==null?"ignore":"pipe","pipe","pipe"]});if(t!=null)a.stdin.write(t),a.stdin.end();let s="",c="";a.stdout.on("data",(l)=>s+=l),a.stderr.on("data",(l)=>c+=l);let u=setTimeout(()=>{a.kill(),o(Error(`cdp timeout: ${e[0]}`))},r);a.on("close",(l)=>{if(clearTimeout(u),l===0)i(s.trim());else o(Error(c.trim()||`cdp exit ${l}`))})})}async function T(e){if(e)return e;let t=(await m(["list"])).split(`
|
|
3
|
+
`)[0]?.slice(0,8);if(!t)throw Error("No Chrome tabs found. Is Chrome running with --remote-debugging-port=9222?");let r=await m(["evalraw",t,"Target.createTarget",'{"url":"about:blank"}']),n;try{n=JSON.parse(r)}catch(a){throw Error(`Target.createTarget returned invalid JSON: ${a.message}`)}let{targetId:i}=n;if(!i)throw Error("Target.createTarget did not return a targetId");await m(["list"]);let o=i.slice(0,8);try{await Q(o)}catch(a){process.stderr.write(`[getOrOpenTab] stealth injection failed: ${a.message}
|
|
4
|
+
`)}return o}async function Q(e){await m(["evalraw",e,"Page.addScriptToEvaluateOnNewDocument",JSON.stringify({source:`
|
|
5
5
|
(function() {
|
|
6
6
|
// ── Runtime.enable / CDP detection masking ──────────────
|
|
7
7
|
try { delete window.__REBROWSER_RUNTIME_ENABLE; } catch(_) {}
|
|
@@ -12,40 +12,62 @@ import{randomInt as M}from"node:crypto";import{spawn as j}from"node:child_proces
|
|
|
12
12
|
try { delete window._phantom; } catch(_) {}
|
|
13
13
|
try { delete window.Buffer; } catch(_) {}
|
|
14
14
|
|
|
15
|
-
// Real Chrome without automation
|
|
16
|
-
// A literal false
|
|
17
|
-
//
|
|
18
|
-
|
|
15
|
+
// Real Chrome without automation should not expose navigator.webdriver at all.
|
|
16
|
+
// A literal false or an own-property getter returning undefined is itself a
|
|
17
|
+
// common stealth tell; remove both instance and prototype properties when the
|
|
18
|
+
// descriptor is configurable (as it is with --disable-blink-features).
|
|
19
|
+
try { delete navigator.webdriver; } catch(_) {}
|
|
20
|
+
try { delete Navigator.prototype.webdriver; } catch(_) {}
|
|
19
21
|
Object.defineProperty(navigator, 'vendor', { get: () => 'Google Inc.', configurable: true });
|
|
20
22
|
Object.defineProperty(navigator, 'platform', { get: () => 'Win32', configurable: true });
|
|
21
23
|
Object.defineProperty(navigator, 'maxTouchPoints', { get: () => 0, configurable: true });
|
|
22
24
|
Object.defineProperty(navigator, 'pdfViewerEnabled', { get: () => true, configurable: true });
|
|
25
|
+
Object.defineProperty(navigator, 'productSub', { get: () => '20030107', configurable: true });
|
|
26
|
+
Object.defineProperty(navigator, 'product', { get: () => 'Gecko', configurable: true });
|
|
27
|
+
var __greedyMimeTypes = null;
|
|
28
|
+
function __makeMimeTypes() {
|
|
29
|
+
var pdf = { type: 'application/pdf', suffixes: 'pdf', description: 'Portable Document Format', enabledPlugin: null };
|
|
30
|
+
var textPdf = { type: 'text/pdf', suffixes: 'pdf', description: 'Portable Document Format', enabledPlugin: null };
|
|
31
|
+
try { Object.setPrototypeOf(pdf, MimeType.prototype); } catch(_) {}
|
|
32
|
+
try { Object.setPrototypeOf(textPdf, MimeType.prototype); } catch(_) {}
|
|
33
|
+
var m = [pdf, textPdf];
|
|
34
|
+
try { Object.setPrototypeOf(m, MimeTypeArray.prototype); } catch(_) {}
|
|
35
|
+
m.item = function item(i) { return this[i] || null; };
|
|
36
|
+
m.namedItem = function namedItem(name) { return Array.prototype.find.call(this, function(x) { return x && x.type === name; }) || null; };
|
|
37
|
+
return m;
|
|
38
|
+
}
|
|
23
39
|
Object.defineProperty(navigator, 'plugins', {
|
|
24
40
|
get: () => {
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
41
|
+
__greedyMimeTypes = __greedyMimeTypes || __makeMimeTypes();
|
|
42
|
+
var plugin0 = { name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format' };
|
|
43
|
+
var plugin1 = { name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '' };
|
|
44
|
+
var plugin2 = { name: 'Native Client', filename: 'internal-nacl-plugin', description: '' };
|
|
45
|
+
try { Object.setPrototypeOf(plugin0, Plugin.prototype); } catch(_) {}
|
|
46
|
+
try { Object.setPrototypeOf(plugin1, Plugin.prototype); } catch(_) {}
|
|
47
|
+
try { Object.setPrototypeOf(plugin2, Plugin.prototype); } catch(_) {}
|
|
48
|
+
var p = [plugin0, plugin1, plugin2];
|
|
49
|
+
p.item = function item(i) { return this[i] || null; };
|
|
50
|
+
p.namedItem = function namedItem(name) { return Array.prototype.find.call(this, function(x) { return x && x.name === name; }) || null; };
|
|
51
|
+
p.refresh = function refresh() {};
|
|
52
|
+
try { Object.setPrototypeOf(p, PluginArray.prototype); } catch(_) {}
|
|
53
|
+
try {
|
|
54
|
+
__greedyMimeTypes[0].enabledPlugin = p[0];
|
|
55
|
+
__greedyMimeTypes[1].enabledPlugin = p[0];
|
|
56
|
+
} catch(_) {}
|
|
31
57
|
return p;
|
|
32
58
|
},
|
|
59
|
+
configurable: true,
|
|
33
60
|
});
|
|
34
61
|
Object.defineProperty(navigator, 'mimeTypes', {
|
|
35
62
|
get: () => {
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
{ type: 'text/pdf', suffixes: 'pdf', description: 'Portable Document Format', enabledPlugin: null },
|
|
39
|
-
];
|
|
40
|
-
m.item = function(i) { return m[i] || null; };
|
|
41
|
-
m.namedItem = function(name) { return m.find(function(x) { return x.type === name; }) || null; };
|
|
42
|
-
return m;
|
|
63
|
+
__greedyMimeTypes = __greedyMimeTypes || __makeMimeTypes();
|
|
64
|
+
return __greedyMimeTypes;
|
|
43
65
|
},
|
|
44
66
|
configurable: true,
|
|
45
67
|
});
|
|
46
68
|
Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'], configurable: true });
|
|
47
69
|
try {
|
|
48
|
-
Object.defineProperty(navigator, 'connection', { get: () => ({ effectiveType: '4g', rtt: 50, downlink: 10, saveData: false }), configurable: true });
|
|
70
|
+
Object.defineProperty(navigator, 'connection', { get: () => ({ effectiveType: '4g', rtt: 50, downlink: 10, downlinkMax: Infinity, saveData: false }), configurable: true });
|
|
49
71
|
} catch(_) {}
|
|
50
72
|
if (!navigator.mediaDevices) {
|
|
51
73
|
Object.defineProperty(navigator, 'mediaDevices', {
|
|
@@ -61,6 +83,18 @@ import{randomInt as M}from"node:crypto";import{spawn as j}from"node:child_proces
|
|
|
61
83
|
configurable: true,
|
|
62
84
|
});
|
|
63
85
|
}
|
|
86
|
+
// ── Missing platform APIs (headless often lacks these) ─
|
|
87
|
+
try {
|
|
88
|
+
if (!navigator.share) {
|
|
89
|
+
navigator.share = function() { return Promise.reject(new Error('NotAllowedError')); };
|
|
90
|
+
}
|
|
91
|
+
} catch(_) {}
|
|
92
|
+
try {
|
|
93
|
+
if (!navigator.contentIndex) {
|
|
94
|
+
Object.defineProperty(navigator, 'contentIndex', { get: () => ({ add: function() {}, delete: function() {}, getAll: function() { return Promise.resolve([]); } }), configurable: true });
|
|
95
|
+
}
|
|
96
|
+
} catch(_) {}
|
|
97
|
+
|
|
64
98
|
if (!window.chrome) {
|
|
65
99
|
window.chrome = {
|
|
66
100
|
app: { isInstalled: false, InstallState: {}, RunningState: {} },
|
|
@@ -68,8 +102,8 @@ import{randomInt as M}from"node:crypto";import{spawn as j}from"node:child_proces
|
|
|
68
102
|
OnInstalledReason: {}, OnRestartRequiredReason: {}, PlatformArch: {}, PlatformNaclArch: {}, PlatformOs: {}, RequestUpdateCheckStatus: {},
|
|
69
103
|
connect: () => ({}), sendMessage: () => {}, onMessage: { addListener: () => {} }
|
|
70
104
|
},
|
|
71
|
-
loadTimes: ()
|
|
72
|
-
csi: ()
|
|
105
|
+
loadTimes: function() { return { requestTime: 0, startLoadTime: Date.now() - 5000, commitLoadTime: Date.now() - 3000, finishDocumentLoadTime: Date.now() - 2000, finishLoadTime: Date.now() - 1000, firstPaintTime: Date.now() - 800, navigationType: 'Other', wasFetchedViaSpdy: true, wasNpnNegotiated: true, npnNegotiatedProtocol: 'h2', wasAlternateProtocolAvailable: false, connectionInfo: 'http/2' }; },
|
|
106
|
+
csi: function() { var t = Date.now(); return { onloadT: t - 2000, startE: t - 5000, pageT: 'back', tran: 2 }; },
|
|
73
107
|
};
|
|
74
108
|
}
|
|
75
109
|
var __greedyNativeFns = [];
|
|
@@ -90,6 +124,19 @@ import{randomInt as M}from"node:crypto";import{spawn as j}from"node:child_proces
|
|
|
90
124
|
return getParam.call(this, p);
|
|
91
125
|
});
|
|
92
126
|
} catch(_) {}
|
|
127
|
+
// ── WebGL readPixels noise ──────────────────────────
|
|
128
|
+
// CreepJS and other fingerprinters draw content with WebGL and read back the
|
|
129
|
+
// rendered pixels. Adding subtle noise breaks rendering-based fingerprinting.
|
|
130
|
+
try {
|
|
131
|
+
var origReadPixels = WebGLRenderingContext.prototype.readPixels;
|
|
132
|
+
WebGLRenderingContext.prototype.readPixels = __markNative(function readPixels(x, y, width, height, format, type, pixels) {
|
|
133
|
+
var result = origReadPixels.call(this, x, y, width, height, format, type, pixels);
|
|
134
|
+
if (pixels && pixels.length > 0) {
|
|
135
|
+
pixels[0] ^= 1;
|
|
136
|
+
}
|
|
137
|
+
return result;
|
|
138
|
+
});
|
|
139
|
+
} catch(_) {}
|
|
93
140
|
Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 8, configurable: true });
|
|
94
141
|
Object.defineProperty(navigator, 'deviceMemory', { get: () => 8, configurable: true });
|
|
95
142
|
|
|
@@ -97,7 +144,7 @@ import{randomInt as M}from"node:crypto";import{spawn as j}from"node:child_proces
|
|
|
97
144
|
// Headless rendering engines produce slightly different canvas output
|
|
98
145
|
// than headed Chrome. Subtle noise breaks hash-based fingerprinting.
|
|
99
146
|
try {
|
|
100
|
-
var __canvasNoise = ((Date.now()
|
|
147
|
+
var __canvasNoise = ((Date.now() & 0xFF) | 1);
|
|
101
148
|
var origFill = CanvasRenderingContext2D.prototype.fillText;
|
|
102
149
|
CanvasRenderingContext2D.prototype.fillText = __markNative(function fillText() {
|
|
103
150
|
this.globalAlpha = 0.9995;
|
|
@@ -116,15 +163,39 @@ import{randomInt as M}from"node:crypto";import{spawn as j}from"node:child_proces
|
|
|
116
163
|
HTMLCanvasElement.prototype.toDataURL = __markNative(function toDataURL() {
|
|
117
164
|
var ctx = this.getContext('2d');
|
|
118
165
|
if (ctx) {
|
|
119
|
-
//
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
166
|
+
// Spread noise across canvas to break hash-based fingerprinting.
|
|
167
|
+
// Uses a deterministic pattern so it's consistent per page load
|
|
168
|
+
// but varies between sessions.
|
|
169
|
+
var w = this.width, h = this.height;
|
|
170
|
+
if (w > 0 && h > 0) {
|
|
171
|
+
var imgData = ctx.getImageData(0, 0, Math.min(w, 4), Math.min(h, 4));
|
|
172
|
+
if (imgData && imgData.data) {
|
|
173
|
+
for (var __i = 0; __i < imgData.data.length; __i += 4) {
|
|
174
|
+
imgData.data[__i] ^= (__canvasNoise + __i) & 0xFF;
|
|
175
|
+
}
|
|
176
|
+
ctx.putImageData(imgData, 0, 0);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
123
179
|
}
|
|
124
180
|
return origToDataURL.apply(this, arguments);
|
|
125
181
|
});
|
|
126
182
|
} catch(_) {}
|
|
127
183
|
|
|
184
|
+
// ── AudioContext fingerprint noise ────────────────────
|
|
185
|
+
// Headless Chrome's AudioContext produces slightly different output.
|
|
186
|
+
// Subtle noise breaks audio-based fingerprinting.
|
|
187
|
+
try {
|
|
188
|
+
var __audioSeed = ((Date.now() & 0x1F) | 1);
|
|
189
|
+
var origGetChannelData = AudioBuffer.prototype.getChannelData;
|
|
190
|
+
AudioBuffer.prototype.getChannelData = __markNative(function getChannelData(channel) {
|
|
191
|
+
var data = origGetChannelData.call(this, channel);
|
|
192
|
+
for (var __i = 0; __i < data.length; __i += 64) {
|
|
193
|
+
data[__i] *= 0.99999;
|
|
194
|
+
}
|
|
195
|
+
return data;
|
|
196
|
+
});
|
|
197
|
+
} catch(_) {}
|
|
198
|
+
|
|
128
199
|
// ── window outer dimensions ──────────────────────────
|
|
129
200
|
// outerWidth/Height = 0 in headless — a well-known bot signal.
|
|
130
201
|
// Mirror innerWidth/Height (set by --window-size flag) so the ratio is sane.
|
|
@@ -134,9 +205,15 @@ import{randomInt as M}from"node:crypto";import{spawn as j}from"node:child_proces
|
|
|
134
205
|
} catch(_) {}
|
|
135
206
|
|
|
136
207
|
// ── screen properties ─────────────────────────────────
|
|
208
|
+
// Headless Chrome often reports an 800x600 screen even when the viewport is
|
|
209
|
+
// 1920x1080. Keep screen metrics internally consistent with our launch flags.
|
|
137
210
|
try {
|
|
138
|
-
|
|
139
|
-
|
|
211
|
+
Object.defineProperty(screen, 'width', { get: () => 1920, configurable: true });
|
|
212
|
+
Object.defineProperty(screen, 'height', { get: () => 1080, configurable: true });
|
|
213
|
+
Object.defineProperty(screen, 'availWidth', { get: () => 1920, configurable: true });
|
|
214
|
+
Object.defineProperty(screen, 'availHeight', { get: () => 1040, configurable: true });
|
|
215
|
+
Object.defineProperty(screen, 'colorDepth', { get: () => 24, configurable: true });
|
|
216
|
+
Object.defineProperty(screen, 'pixelDepth', { get: () => 24, configurable: true });
|
|
140
217
|
} catch(_) {}
|
|
141
218
|
|
|
142
219
|
// ── navigator.userAgentData (UA Client Hints) ─────────
|
|
@@ -209,12 +286,12 @@ import{randomInt as M}from"node:crypto";import{spawn as j}from"node:child_proces
|
|
|
209
286
|
};
|
|
210
287
|
} catch(_) {}
|
|
211
288
|
})();
|
|
212
|
-
`})])}var
|
|
289
|
+
`})])}var v={postNav:800,postNavSlow:1200,postClick:300,postType:300,inputPoll:400,copyPoll:600,afterVerify:1500};function _(e){let t=e*0.2,r=J(-Math.floor(t),Math.floor(t)+1);return Math.max(50,Math.round(e+r))}async function S(e,t={}){let{timeout:r=20000,interval:n=600,stableRounds:i=3,selector:o="document.body",minLength:a=0,isStreamingExpr:s="false"}=t,c=String.raw`
|
|
213
290
|
new Promise((resolve, reject) => {
|
|
214
|
-
const _deadline = Date.now() + ${
|
|
291
|
+
const _deadline = Date.now() + ${r};
|
|
215
292
|
const _baseInterval = ${n};
|
|
216
|
-
const _stableRounds = ${
|
|
217
|
-
const _minLength = ${
|
|
293
|
+
const _stableRounds = ${i};
|
|
294
|
+
const _minLength = ${a};
|
|
218
295
|
let _lastLen = -1;
|
|
219
296
|
let _stableCount = 0;
|
|
220
297
|
|
|
@@ -225,9 +302,13 @@ import{randomInt as M}from"node:crypto";import{spawn as j}from"node:child_proces
|
|
|
225
302
|
function _poll() {
|
|
226
303
|
try {
|
|
227
304
|
// Re-query DOM each tick — element may not exist at eval start
|
|
228
|
-
const el = ${
|
|
229
|
-
const cur = el?.
|
|
230
|
-
|
|
305
|
+
const el = ${o};
|
|
306
|
+
const cur = el?.textContent?.length ?? 0;
|
|
307
|
+
const streaming = ${s};
|
|
308
|
+
if (streaming) {
|
|
309
|
+
if (cur !== _lastLen) _lastLen = cur;
|
|
310
|
+
_stableCount = 0;
|
|
311
|
+
} else if (cur >= _minLength) {
|
|
231
312
|
if (cur === _lastLen) {
|
|
232
313
|
_stableCount++;
|
|
233
314
|
if (_stableCount >= _stableRounds) { resolve(cur); return; }
|
|
@@ -239,18 +320,18 @@ import{randomInt as M}from"node:crypto";import{spawn as j}from"node:child_proces
|
|
|
239
320
|
if (Date.now() < _deadline) {
|
|
240
321
|
setTimeout(_poll, _jitter(_baseInterval));
|
|
241
322
|
} else {
|
|
242
|
-
if (_lastLen >= _minLength) { resolve(_lastLen); }
|
|
243
|
-
else { reject(new Error('Generation did not stabilise within ${
|
|
323
|
+
if (_lastLen >= _minLength && !streaming) { resolve(_lastLen); }
|
|
324
|
+
else { reject(new Error('Generation did not stabilise within ${r}ms')); }
|
|
244
325
|
}
|
|
245
326
|
} catch(e) { reject(e); }
|
|
246
327
|
}
|
|
247
328
|
|
|
248
329
|
_poll();
|
|
249
330
|
})
|
|
250
|
-
`,
|
|
251
|
-
`)}function
|
|
331
|
+
`,u=await m(["eval",e,c],r+1e4),l=parseInt(u,10)||0;if(l>=a)return l;throw Error(`Generation did not stabilise within ${r}ms`)}async function C(e){let t=e.indexOf("--stdin");if(t===-1)return e;let r=await new Promise((i)=>{let o="";process.stdin.setEncoding("utf8"),process.stdin.on("data",(a)=>o+=a),process.stdin.on("end",()=>i(o.trim()))}),n=[...e];return n[t]=r,n}function D(e){let t=e.includes("--short"),r=e.filter((s)=>s!=="--short"),n=r.indexOf("--tab"),i=n===-1?null:r[n+1];if(n!==-1)r=r.filter((s,c)=>c!==n&&c!==n+1);let o=r.indexOf("--locale"),a=o===-1?null:r[o+1];if(o!==-1)r=r.filter((s,c)=>c!==o&&c!==o+1);return{query:r.join(" "),tabPrefix:i,short:t,locale:a}}function N(e,t){if(!e.length||e[0]==="--help")process.stderr.write(t),process.exit(1)}function $(e,t,r=300){if(!t||e.length<=r)return e;let n=e.slice(0,r),i=n.lastIndexOf(" ");return i>0?`${n.slice(0,i)}…`:`${n}…`}function A(e){process.stdout.write(`${JSON.stringify(e,null,2)}
|
|
332
|
+
`)}function M(e,t=null){if(t){let r=JSON.stringify({_envelope:t,error:e.message});process.stdout.write(`${r}
|
|
252
333
|
`)}process.stderr.write(`Error: ${e.message}
|
|
253
|
-
`),process.exit(1)}import{randomInt as
|
|
334
|
+
`),process.exit(1)}import{randomInt as K}from"node:crypto";import{existsSync as ee,readFileSync as te}from"node:fs";import re from"node:http";var ne=`
|
|
254
335
|
(function() {
|
|
255
336
|
// Google consent page (consent.google.com)
|
|
256
337
|
var g = document.querySelector('#L2AGLb, button[jsname="b3VHJd"], .tHlp8d');
|
|
@@ -267,7 +348,7 @@ import{randomInt as M}from"node:crypto";import{spawn as j}from"node:child_proces
|
|
|
267
348
|
|
|
268
349
|
return null;
|
|
269
350
|
})()
|
|
270
|
-
`,
|
|
351
|
+
`,ie=`
|
|
271
352
|
(function() {
|
|
272
353
|
var url = document.location.href;
|
|
273
354
|
|
|
@@ -298,12 +379,24 @@ import{randomInt as M}from"node:crypto";import{spawn as j}from"node:child_proces
|
|
|
298
379
|
|
|
299
380
|
// --- Cloudflare Turnstile widget inside closed shadow DOM (Copilot, etc.) ---
|
|
300
381
|
// The iframe is not queryable from main document, but the host container
|
|
301
|
-
// (#cf-turnstile) and the hidden response input are.
|
|
302
|
-
|
|
382
|
+
// (#cf-turnstile) and the hidden response input are. When only the
|
|
383
|
+
// hidden response input matches (no #cf-turnstile host and no visible
|
|
384
|
+
// iframe), the actual challenge widget is rendered inside a closed
|
|
385
|
+
// shadow DOM and cannot be auto-clicked. Return a sentinel so callers
|
|
386
|
+
// know to surface this as needs-human verification instead of wasting
|
|
387
|
+
// time on a doomed waitForSelector.
|
|
388
|
+
var cfTurnstileHost = document.querySelector('#cf-turnstile');
|
|
303
389
|
if (cfTurnstileHost) {
|
|
304
390
|
var r2 = cfTurnstileHost.getBoundingClientRect();
|
|
305
391
|
return JSON.stringify({t:'xy',x:r2.left+r2.width/2,y:r2.top+r2.height/2});
|
|
306
392
|
}
|
|
393
|
+
// Hidden cf-chl-widget-*_response input present but no visible host:
|
|
394
|
+
// the widget is in closed shadow DOM. Signal this so handleVerification
|
|
395
|
+
// can return 'needs-human' rather than 'clear'.
|
|
396
|
+
var cfResponseInput = document.querySelector('input[name="cf-turnstile-response"], [id^="cf-chl-widget-"][id$="_response"]');
|
|
397
|
+
if (cfResponseInput && cfResponseInput.value === '') {
|
|
398
|
+
return 'cf-closed-shadow-dom';
|
|
399
|
+
}
|
|
307
400
|
|
|
308
401
|
// --- Cloudflare challenge page ---
|
|
309
402
|
var cfCheckbox = document.querySelector('#cf-stage input[type="checkbox"], .ctp-checkbox-container input');
|
|
@@ -318,15 +411,28 @@ import{randomInt as M}from"node:crypto";import{spawn as j}from"node:child_proces
|
|
|
318
411
|
}
|
|
319
412
|
|
|
320
413
|
// --- Generic verify/continue/proceed buttons (catch-all) ---
|
|
321
|
-
// IMPORTANT: exclude sign-in / OAuth buttons (e.g. "Continue with Google"
|
|
414
|
+
// IMPORTANT: exclude sign-in / OAuth buttons (e.g. "Continue with Google",
|
|
415
|
+
// "Continue with email", "Login or sign up for free"). These appear on
|
|
416
|
+
// many sites (Perplexity, ChatGPT, etc.) when the user isn't logged in,
|
|
417
|
+
// and clicking them triggers a sign-in flow that takes us to a login
|
|
418
|
+
// wall — a much worse outcome than the original search failure we were
|
|
419
|
+
// trying to recover from. The exclusion list must cover both OAuth
|
|
420
|
+
// providers AND generic "sign in / log in / with email" patterns.
|
|
322
421
|
var btns = Array.from(document.querySelectorAll('button, input[type=submit], a[role=button]'));
|
|
323
422
|
var verify = btns.find(b => {
|
|
324
423
|
var t = (b.innerText?.trim() || b.value || '').toLowerCase();
|
|
325
|
-
var isVerifyLike = (t
|
|
424
|
+
var isVerifyLike = (t === 'continue' || t === 'proceed' || t === 'next' ||
|
|
425
|
+
t.startsWith('verify ') || t.startsWith('human ') || t === 'i am human' || t.includes('robot check')) &&
|
|
326
426
|
!t.includes('verified') && !document.querySelector('iframe[src*="recaptcha"]');
|
|
327
427
|
if (!isVerifyLike) return false;
|
|
328
428
|
// Exclude OAuth / sign-in buttons to prevent accidental login flows
|
|
329
|
-
|
|
429
|
+
// — covers "Continue with Google", "Continue with Apple", "Continue
|
|
430
|
+
// with email", "Login or sign up", "Log in", "Sign in", "Sign up",
|
|
431
|
+
// "Single sign-on", and the visible panel "Login or sign up for free"
|
|
432
|
+
// text. The previous list missed "email" and "sso" which let the
|
|
433
|
+
// auto-click land on the email/SSO sign-in buttons on Perplexity's
|
|
434
|
+
// anonymous-mode homepage, navigating us into a login flow.
|
|
435
|
+
var isSignIn = new RegExp("sign.?in|log.?in|sign.?up|with\\s+(google|apple|email|github|facebook|microsoft|sso)|sso|auth", "i").test(t);
|
|
330
436
|
return !isSignIn;
|
|
331
437
|
});
|
|
332
438
|
if (verify) { verify.setAttribute('data-gs-verify','1'); return JSON.stringify({t:'sel',s:'[data-gs-verify="1"]',txt:verify.innerText?.trim()||verify.value}); }
|
|
@@ -337,7 +443,7 @@ import{randomInt as M}from"node:crypto";import{spawn as j}from"node:child_proces
|
|
|
337
443
|
|
|
338
444
|
return null;
|
|
339
445
|
})()
|
|
340
|
-
`,
|
|
446
|
+
`,ae=`
|
|
341
447
|
(function() {
|
|
342
448
|
var url = document.location.href;
|
|
343
449
|
var isVerifyPage = url.includes('/sorry/') ||
|
|
@@ -371,29 +477,31 @@ import{randomInt as M}from"node:crypto";import{spawn as j}from"node:child_proces
|
|
|
371
477
|
|
|
372
478
|
return 'still-verifying';
|
|
373
479
|
})()
|
|
374
|
-
`;async function
|
|
375
|
-
`)[0],
|
|
376
|
-
var el = document.querySelector('${
|
|
480
|
+
`;async function I(e,t){let r=await t(["eval",e,ne]).catch(()=>null);if(r&&r!=="null")await new Promise((n)=>setTimeout(n,1500))}function p(e,t){return K(e*1000,t*1000)/1000}async function oe(e,t){if(!globalThis.WebSocket)return;let r=process.env.CDP_PROFILE_DIR;if(!r)return;let n=`${r.replaceAll("\\","/")}/DevToolsActivePort`;if(!ee(n))return;let i=te(n,"utf8").trim().split(`
|
|
481
|
+
`)[0],o=await new Promise((c,u)=>{let l=re.get(`http://localhost:${i}/json/version`,(f)=>{let g="";f.on("data",(d)=>g+=d),f.on("end",()=>{try{c(JSON.parse(g))}catch{u(Error("bad JSON"))}})});l.on("error",u),l.setTimeout(1000,()=>{l.destroy(),u(Error("timeout"))})}),a=new globalThis.WebSocket(o.webSocketDebuggerUrl),s=0;await new Promise((c)=>{a.onopen=async()=>{let u=(g,d)=>new Promise((x)=>{let b=++s,h=(y)=>{if(JSON.parse(y.data).id===b)a.removeEventListener("message",h),x()};a.addEventListener("message",h),a.send(JSON.stringify({id:b,method:g,params:d}))}),l=e+p(-2,2),f=t+p(-2,2);await u("Input.dispatchMouseEvent",{type:"mouseMoved",x:l,y:f,button:"none"}),await new Promise((g)=>setTimeout(g,p(80,160))),await u("Input.dispatchMouseEvent",{type:"mousePressed",x:l,y:f,button:"left",clickCount:1}),await new Promise((g)=>setTimeout(g,p(30,80))),await u("Input.dispatchMouseEvent",{type:"mouseReleased",x:l+p(-1,1),y:f+p(-1,1),button:"left",clickCount:1}),setTimeout(()=>{a.close(),c()},200)},a.onerror=()=>c(),setTimeout(c,3000)})}async function R(e,t,r,n){let i=Number.parseFloat(r),o=Number.parseFloat(n);if(Number.isNaN(i)||Number.isNaN(o))throw Error(`humanClickXY: invalid coordinates (${r}, ${n})`);let a={button:"left",clickCount:1,modifiers:0},s=i+p(-3,3),c=o+p(-3,3);await t(["evalraw",e,"Input.dispatchMouseEvent",JSON.stringify({...a,type:"mouseMoved",x:s,y:c})]),await new Promise((d)=>setTimeout(d,p(80,180)));let u=i+p(-2,2),l=o+p(-2,2);await t(["evalraw",e,"Input.dispatchMouseEvent",JSON.stringify({...a,type:"mousePressed",x:u,y:l})]),await new Promise((d)=>setTimeout(d,p(30,90)));let f=u+p(-1,1),g=l+p(-1,1);return await t(["evalraw",e,"Input.dispatchMouseEvent",JSON.stringify({...a,type:"mouseReleased",x:f,y:g})]),await oe(i,o).catch(()=>{}),await new Promise((d)=>setTimeout(d,p(100,300))),`human-clicked at (${i.toFixed(0)}, ${o.toFixed(0)})`}async function le(e,t,r){let n=await t(["eval",e,`(function() {
|
|
482
|
+
var el = document.querySelector('${r.replace(/\\/g,"\\\\").replace(/'/g,"\\'")}');
|
|
377
483
|
if (!el) return 'null';
|
|
378
484
|
var r = el.getBoundingClientRect();
|
|
379
485
|
return JSON.stringify({x: r.left + r.width / 2, y: r.top + r.height / 2, w: r.width, h: r.height});
|
|
380
|
-
})()`]).catch(()=>"null");if(!n||n==="null")return null;let
|
|
381
|
-
`),
|
|
382
|
-
`),
|
|
383
|
-
`)
|
|
384
|
-
`),"
|
|
385
|
-
`),"
|
|
486
|
+
})()`]).catch(()=>"null");if(!n||n==="null")return null;let i=JSON.parse(n);if(i.w===0||i.h===0||i.x===0&&i.y===0)return null;let{x:o,y:a}=i;return R(e,t,o,a)}function E(e,t,r){if(!r||r==="null"||r==="cleared"||r==="still-verifying"||r==="cf-closed-shadow-dom")return Promise.resolve("no-challenge");try{let n=JSON.parse(r);if(n.t==="sel"&&n.s)return process.stderr.write(`[greedysearch] Human-clicking "${n.txt}" via CDP...
|
|
487
|
+
`),le(e,t,n.s).then((i)=>i!==null?"clicked":"cant-click");if(n.t==="xy"){if(!n.x&&!n.y)return Promise.resolve("cant-click");return process.stderr.write(`[greedysearch] Human-clicking at (${n.x.toFixed(0)}, ${n.y.toFixed(0)})...
|
|
488
|
+
`),R(e,t,n.x,n.y).then(()=>"clicked")}}catch{}return Promise.resolve("no-challenge")}async function se(e,t){let r=await t(["eval",e,ie]).catch(()=>null);if(r==="cf-closed-shadow-dom"){let n=await ce(e,t).catch(()=>null);if(n)return n;return r}if(r&&r!=="null")return r;return null}async function ce(e,t){if(typeof t!=="function")return null;await t(["evalraw",e,"DOM.enable","{}"]).catch(()=>{});let r=await t(["evalraw",e,"DOM.getDocument",JSON.stringify({depth:-1,pierce:!0})]).catch(()=>null);if(!r)return null;let n;try{n=JSON.parse(r)}catch{return null}if(n.error||!n.root)return null;let i=n.root;return await j(i,e,t)}async function j(e,t,r){if(!e)return null;let n=[];if(e.shadowRoots&&e.shadowRoots.length>0)for(let i of e.shadowRoots)n.push(i);if(e.children)for(let i of e.children)n.push(i);for(let i of n){if(i.nodeName==="IFRAME"){let a=i.attributes||[],s=a.indexOf("src"),c=s>=0?a[s+1]:"";if(c&&/challenges\.cloudflare\.com|turnstile/i.test(c)&&i.backendNodeId){let u=await r(["evalraw",t,"DOM.getBoxModel",JSON.stringify({backendNodeId:i.backendNodeId})]).catch(()=>null);if(!u)continue;let l;try{l=JSON.parse(u)}catch{continue}let f=l?.model?.content||l?.result?.model?.content;if(!f||f.length<8)continue;let g=f[0],d=f[1],x=f[4],b=f[5],h=x-g,y=b-d;if(h<50||y<20)continue;let k=g+h*0.25,P=d+y*0.5;return process.stderr.write(`[greedysearch] Found CF iframe via CDP pierce at (${g.toFixed(0)}, ${d.toFixed(0)}) ${h.toFixed(0)}x${y.toFixed(0)}, clicking checkbox at (${k.toFixed(0)}, ${P.toFixed(0)})
|
|
489
|
+
`),JSON.stringify({t:"xy",x:k,y:P})}}let o=await j(i,t,r);if(o)return o}return null}async function L(e,t,r=30000){let n=await se(e,t);if(!n)return"clear";if(n==="sorry-page"){process.stderr.write(`[greedysearch] Google CAPTCHA detected — please solve it in the browser window (waiting up to ${Math.floor(r/1000)}s)...
|
|
490
|
+
`);let o=Date.now()+r;while(Date.now()<o)if(await new Promise((a)=>setTimeout(a,2000)),!(await t(["eval",e,"document.location.href"]).catch(()=>"")).includes("/sorry/"))return"cleared-by-user";return"needs-human"}let i=await E(e,t,n);if(i==="clicked"){await new Promise((a)=>setTimeout(a,2000));let o=Date.now()+r;while(Date.now()<o){let a=await t(["eval",e,ae]).catch(()=>null);if(a==="cleared"||!a||a==="null")return process.stderr.write(`[greedysearch] Verification cleared.
|
|
491
|
+
`),"clicked";if(a!=="still-verifying")await E(e,t,a),await new Promise((s)=>setTimeout(s,2000));else await new Promise((s)=>setTimeout(s,1500))}return process.stderr.write(`[greedysearch] Verification may require manual intervention.
|
|
492
|
+
`),"needs-human"}if(i==="cant-click")return process.stderr.write(`[greedysearch] Verification challenge detected but cannot be auto-clicked — please solve it manually in the visible browser window.
|
|
493
|
+
`),"needs-human";return"clear"}var F={perplexity:{input:"#ask-input",copyButton:null,sourceItem:"[data-pplx-citation-url]",sourceLink:"a",consent:"#onetrust-accept-btn-handler"},bing:{input:"#userInput",copyButton:'button[data-testid="copy-ai-message-button"]',sourceLink:'a[href^="http"][target="_blank"]',sourceExclude:"copilot.microsoft.com",consent:"#onetrust-accept-btn-handler"},google:{answerContainer:".pWvJNd",sourceLink:'a[href^="http"]',sourceExclude:["google.","gstatic","googleapis"],sourceHeadingParent:"[data-snhf]",consent:'#L2AGLb, button[jsname="b3VHJd"], .tHlp8d'},gemini:{input:"rich-textarea .ql-editor",copyButton:'button:has(mat-icon[data-mat-icon-name="copy"])',sendButton:'button:has(mat-icon[data-mat-icon-name="arrow_upward"]), [data-test-id="send-button"], .send-button',sourcesSidebarButton:"button.legacy-sources-sidebar-button",sourcesExclude:["gemini.google","gstatic","google.com/search"],citationButtonPattern:'button[aria-label*="citation from"]',citationNameRegex:/from\s{1,20}([^.]{1,200})\.\s/}};var w=F.google,ue=50;async function de(e){let t=w.sourceExclude.map((n)=>`!a.href.includes('${n}')`).join(" && "),r=await m(["eval",e,String.raw`
|
|
386
494
|
(function() {
|
|
387
|
-
var el = document.querySelector('${
|
|
495
|
+
var el = document.querySelector('${w.answerContainer}');
|
|
388
496
|
if (!el) return JSON.stringify({ answer: '', sources: [] });
|
|
389
497
|
var answer = el.innerText.trim();
|
|
390
|
-
var sources = Array.from(document.querySelectorAll('${
|
|
391
|
-
.filter(a => ${
|
|
392
|
-
.map(a => ({ url: a.href.split('#')[0], title: (a.closest('${
|
|
498
|
+
var sources = Array.from(document.querySelectorAll('${w.sourceLink}'))
|
|
499
|
+
.filter(a => ${t})
|
|
500
|
+
.map(a => ({ url: a.href.split('#')[0], title: (a.closest('${w.sourceHeadingParent}')?.querySelector('h3, [role=heading]')?.innerText || a.innerText?.trim().split('\n')[0] || '').slice(0, 100) }))
|
|
393
501
|
.filter(s => s.url && s.url.length > 10)
|
|
394
502
|
.filter((v, i, arr) => arr.findIndex(x => x.url === v.url) === i)
|
|
395
503
|
.slice(0, 10);
|
|
396
504
|
return JSON.stringify({ answer, sources });
|
|
397
505
|
})()
|
|
398
|
-
`]);return JSON.parse(
|
|
399
|
-
`;async function
|
|
506
|
+
`]);return JSON.parse(r)}var fe=`Usage: node extractors/google-ai.mjs "<query>" [--tab <prefix>]
|
|
507
|
+
`;async function ge(){let e=await C(process.argv.slice(2));N(e,fe);let{query:t,tabPrefix:r,short:n,locale:i}=D(e);try{if(!r)await m(["list"]);let o=await T(r),a=i?`&hl=${encodeURIComponent(i)}`:"&hl=en",s=`https://www.google.com/search?q=${encodeURIComponent(t)}&udm=50${a}`;if(await new Promise((d)=>setTimeout(d,_(v.postNav))),await I(o,m),!(await m(["eval",o,"document.location.href"]).catch(()=>"")).includes("google.com/search"))await m(["nav",o,s],20000),await new Promise((d)=>setTimeout(d,_(v.postNav)));let u=await L(o,m,1e4);if(u==="needs-human")throw Error("Google verification required — could not be completed automatically");if(u==="clicked"||u==="cleared-by-user")await m(["nav",o,s],20000),await new Promise((d)=>setTimeout(d,_(v.postNav)));await S(o,{timeout:30000,stableRounds:5,selector:`document.querySelector('${w.answerContainer}')`,minLength:ue});let{answer:l,sources:f}=await de(o);if(!l)throw Error("No answer extracted — Google AI Mode may not have responded");let g=await m(["eval",o,"document.location.href"]).catch(()=>s);A({query:t,url:g,answer:$(l,n),sources:f})}catch(o){M(o)}}ge();
|