@duckmind/dm-windows-x64 0.62.3 → 0.63.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dm.exe +0 -0
- package/extensions/.dm-extensions.json +43 -183
- package/extensions/dm-context/src/context.js +1 -2
- package/extensions/dm-context/src/index.js +1 -2
- package/extensions/{greedysearch-dm/bin/cdp.mjs → dm-cua/bin/browser-cdp.mjs} +3 -3
- package/extensions/dm-cua/bin/browser-cua.mjs +1 -1
- package/extensions/{greedysearch-dm/bin/launch.mjs → dm-cua/bin/browser-launch.mjs} +12 -11
- package/extensions/dm-cua/index.js +1 -1
- package/extensions/dm-cua/src/browser-cua-lib.mjs +17 -12
- package/extensions/{greedysearch-dm/src/search → dm-cua/src/browser-helpers}/constants.mjs +11 -11
- package/extensions/{greedysearch-dm/src/utils → dm-cua/src/browser-helpers}/node-runtime.mjs +1 -1
- package/extensions/{greedysearch-dm/src/search → dm-cua/src/browser-helpers}/port-pid.mjs +1 -1
- package/extensions/dm-ponytail/extensions/ponytail.mjs +186 -0
- package/extensions/dm-ponytail/hooks/claude-codex-hooks.json +41 -0
- package/extensions/dm-ponytail/hooks/copilot-hooks.json +21 -0
- package/extensions/dm-ponytail/hooks/ponytail-activate.js +59 -0
- package/extensions/dm-ponytail/hooks/ponytail-config.js +116 -0
- package/extensions/dm-ponytail/hooks/ponytail-instructions.js +81 -0
- package/extensions/dm-ponytail/hooks/ponytail-mode-tracker.js +96 -0
- package/extensions/dm-ponytail/hooks/ponytail-runtime.js +79 -0
- package/extensions/dm-ponytail/hooks/ponytail-statusline.ps1 +24 -0
- package/extensions/dm-ponytail/hooks/ponytail-statusline.sh +18 -0
- package/extensions/dm-ponytail/hooks/ponytail-subagent.js +50 -0
- package/extensions/dm-ponytail/hooks/qoder-hooks.json +26 -0
- package/extensions/dm-ponytail/package.json +19 -0
- package/extensions/dm-ponytail/pi-extension/index.js +186 -0
- package/extensions/dm-ponytail/pi-extension/package.json +8 -0
- package/extensions/dm-ponytail/skills/ponytail/SKILL.md +120 -0
- package/extensions/dm-ponytail/skills/ponytail-audit/SKILL.md +41 -0
- package/extensions/dm-ponytail/skills/ponytail-debt/SKILL.md +44 -0
- package/extensions/dm-ponytail/skills/ponytail-gain/SKILL.md +50 -0
- package/extensions/dm-ponytail/skills/ponytail-help/SKILL.md +71 -0
- package/extensions/dm-ponytail/skills/ponytail-review/SKILL.md +57 -0
- package/package.json +2 -6
- package/extensions/dm-caveman/extensions/caveman.js +0 -300
- package/extensions/dm-caveman/package.json +0 -20
- package/extensions/dm-goal/package.json +0 -23
- package/extensions/dm-goal/src/goal.js +0 -914
- package/extensions/greedysearch-dm/bin/cdp-greedy.mjs +0 -40
- package/extensions/greedysearch-dm/bin/cdp-headless.mjs +0 -5
- package/extensions/greedysearch-dm/bin/cdp-visible.mjs +0 -5
- package/extensions/greedysearch-dm/bin/gschrome.mjs +0 -30
- package/extensions/greedysearch-dm/bin/kill-visible.mjs +0 -7
- package/extensions/greedysearch-dm/bin/launch-visible.mjs +0 -13
- package/extensions/greedysearch-dm/bin/mcp.mjs +0 -400
- package/extensions/greedysearch-dm/bin/search.mjs +0 -635
- package/extensions/greedysearch-dm/bin/visible.mjs +0 -22
- package/extensions/greedysearch-dm/extractors/bing-copilot.mjs +0 -344
- package/extensions/greedysearch-dm/extractors/chatgpt.mjs +0 -354
- package/extensions/greedysearch-dm/extractors/common.mjs +0 -794
- package/extensions/greedysearch-dm/extractors/consensus.mjs +0 -527
- package/extensions/greedysearch-dm/extractors/consent.mjs +0 -449
- package/extensions/greedysearch-dm/extractors/gemini.mjs +0 -291
- package/extensions/greedysearch-dm/extractors/google-ai.mjs +0 -86
- package/extensions/greedysearch-dm/extractors/logically.mjs +0 -589
- package/extensions/greedysearch-dm/extractors/perplexity.mjs +0 -370
- package/extensions/greedysearch-dm/extractors/selectors.mjs +0 -32
- package/extensions/greedysearch-dm/extractors/semantic-scholar.mjs +0 -199
- package/extensions/greedysearch-dm/index.js +0 -123
- package/extensions/greedysearch-dm/package.json +0 -27
- package/extensions/greedysearch-dm/skills/greedy-search/skill.md +0 -18
- package/extensions/greedysearch-dm/src/fetcher.mjs +0 -577
- package/extensions/greedysearch-dm/src/formatters/results.js +0 -95
- package/extensions/greedysearch-dm/src/formatters/sources.js +0 -57
- package/extensions/greedysearch-dm/src/formatters/synthesis.js +0 -49
- package/extensions/greedysearch-dm/src/github.mjs +0 -228
- package/extensions/greedysearch-dm/src/reddit.mjs +0 -154
- package/extensions/greedysearch-dm/src/search/browser-lifecycle.mjs +0 -340
- package/extensions/greedysearch-dm/src/search/challenge-detect.mjs +0 -156
- package/extensions/greedysearch-dm/src/search/chrome.mjs +0 -510
- package/extensions/greedysearch-dm/src/search/defaults.mjs +0 -10
- package/extensions/greedysearch-dm/src/search/engines.mjs +0 -79
- package/extensions/greedysearch-dm/src/search/fetch-source.mjs +0 -470
- package/extensions/greedysearch-dm/src/search/file-sources.mjs +0 -31
- package/extensions/greedysearch-dm/src/search/output.mjs +0 -51
- package/extensions/greedysearch-dm/src/search/paths.mjs +0 -48
- package/extensions/greedysearch-dm/src/search/pdf.mjs +0 -64
- package/extensions/greedysearch-dm/src/search/progress.mjs +0 -109
- package/extensions/greedysearch-dm/src/search/query.mjs +0 -21
- package/extensions/greedysearch-dm/src/search/recovery.mjs +0 -49
- package/extensions/greedysearch-dm/src/search/research.mjs +0 -2255
- package/extensions/greedysearch-dm/src/search/scale-aware.mjs +0 -61
- package/extensions/greedysearch-dm/src/search/simple-research.mjs +0 -418
- package/extensions/greedysearch-dm/src/search/sources.mjs +0 -412
- package/extensions/greedysearch-dm/src/search/synthesis-runner.mjs +0 -127
- package/extensions/greedysearch-dm/src/search/synthesis.mjs +0 -202
- package/extensions/greedysearch-dm/src/tools/greedy-search-handler.js +0 -209
- package/extensions/greedysearch-dm/src/tools/shared.js +0 -226
- package/extensions/greedysearch-dm/src/types.js +0 -0
- package/extensions/greedysearch-dm/src/utils/content.mjs +0 -38
- package/extensions/greedysearch-dm/src/utils/helpers.js +0 -22
- /package/extensions/{greedysearch-dm/src/search → dm-cua/src/browser-helpers}/minimize.mjs +0 -0
- /package/extensions/{greedysearch-dm/src/utils → dm-cua/src/browser-helpers}/system-cmds.mjs +0 -0
|
@@ -1,635 +0,0 @@
|
|
|
1
|
-
import { appendFileSync, existsSync, readFileSync } from "node:fs";
|
|
2
|
-
import { homedir } from "node:os";
|
|
3
|
-
import { join } from "node:path";
|
|
4
|
-
import {
|
|
5
|
-
cdp,
|
|
6
|
-
closeTab,
|
|
7
|
-
closeTabs,
|
|
8
|
-
ensureChrome,
|
|
9
|
-
killHeadlessChrome,
|
|
10
|
-
openNewTab,
|
|
11
|
-
touchActivity
|
|
12
|
-
} from "../src/search/chrome.mjs";
|
|
13
|
-
import {
|
|
14
|
-
ALL_ENGINES,
|
|
15
|
-
ENGINES,
|
|
16
|
-
SYNTHESIZER,
|
|
17
|
-
VISIBLE_RECOVERY_LOG,
|
|
18
|
-
GREEDY_PORT
|
|
19
|
-
} from "../src/search/constants.mjs";
|
|
20
|
-
import { runExtractor } from "../src/search/engines.mjs";
|
|
21
|
-
import {
|
|
22
|
-
fetchMultipleSources,
|
|
23
|
-
fetchTopSource
|
|
24
|
-
} from "../src/search/fetch-source.mjs";
|
|
25
|
-
import { waitForChallengeCleared } from "../src/search/challenge-detect.mjs";
|
|
26
|
-
import { writeSourcesToFiles } from "../src/search/file-sources.mjs";
|
|
27
|
-
import { writeOutput } from "../src/search/output.mjs";
|
|
28
|
-
import {
|
|
29
|
-
findHeadlessBlockedEngines,
|
|
30
|
-
isHeadlessBlockedResult,
|
|
31
|
-
isManualVerificationError
|
|
32
|
-
} from "../src/search/recovery.mjs";
|
|
33
|
-
import {
|
|
34
|
-
buildSourceRegistry,
|
|
35
|
-
mergeFetchDataIntoSources
|
|
36
|
-
} from "../src/search/sources.mjs";
|
|
37
|
-
import { buildConfidence } from "../src/search/synthesis.mjs";
|
|
38
|
-
import {
|
|
39
|
-
getSynthesisStartUrl,
|
|
40
|
-
normalizeSynthesizer,
|
|
41
|
-
synthesizeResults
|
|
42
|
-
} from "../src/search/synthesis-runner.mjs";
|
|
43
|
-
import { normalizeQuery } from "../src/search/query.mjs";
|
|
44
|
-
import { runResearchMode } from "../src/search/research.mjs";
|
|
45
|
-
import { minimizeViaCDP } from "../src/search/minimize.mjs";
|
|
46
|
-
import {
|
|
47
|
-
moduleDirectory,
|
|
48
|
-
resolveGreedySearchExtractorScript
|
|
49
|
-
} from "../src/search/paths.mjs";
|
|
50
|
-
const CONFIG_DIR = join(homedir(), ".config", "greedysearch");
|
|
51
|
-
const CONFIG_FILE = join(CONFIG_DIR, "config.json");
|
|
52
|
-
function loadUserConfig() {
|
|
53
|
-
try {
|
|
54
|
-
if (existsSync(CONFIG_FILE)) {
|
|
55
|
-
return JSON.parse(readFileSync(CONFIG_FILE, "utf8"));
|
|
56
|
-
}
|
|
57
|
-
} catch {}
|
|
58
|
-
return {};
|
|
59
|
-
}
|
|
60
|
-
function logVisibleRecovery(event) {
|
|
61
|
-
try {
|
|
62
|
-
appendFileSync(VISIBLE_RECOVERY_LOG, `${JSON.stringify({ at: new Date().toISOString(), ...event })}
|
|
63
|
-
`, "utf8");
|
|
64
|
-
} catch {}
|
|
65
|
-
}
|
|
66
|
-
async function readStdin() {
|
|
67
|
-
return new Promise((resolve) => {
|
|
68
|
-
let data = "";
|
|
69
|
-
process.stdin.setEncoding("utf8");
|
|
70
|
-
process.stdin.on("data", (chunk) => data += chunk);
|
|
71
|
-
process.stdin.on("end", () => resolve(data.trim()));
|
|
72
|
-
if (process.stdin.isTTY)
|
|
73
|
-
resolve("");
|
|
74
|
-
});
|
|
75
|
-
}
|
|
76
|
-
async function main() {
|
|
77
|
-
const args = process.argv.slice(2);
|
|
78
|
-
if (args[0] === "--dm-print-helper-paths") {
|
|
79
|
-
const binDir = moduleDirectory(import.meta.url);
|
|
80
|
-
console.log(JSON.stringify({
|
|
81
|
-
launchScript: join(binDir, "launch.mjs"),
|
|
82
|
-
searchScript: join(binDir, "search.mjs"),
|
|
83
|
-
perplexityExtractor: resolveGreedySearchExtractorScript("perplexity.mjs", { moduleDir: binDir })
|
|
84
|
-
}));
|
|
85
|
-
return;
|
|
86
|
-
}
|
|
87
|
-
if (args.length < 2 || args[0] === "--help") {
|
|
88
|
-
process.stderr.write(`${[
|
|
89
|
-
'Usage: node search.mjs <engine> "<query>"',
|
|
90
|
-
"",
|
|
91
|
-
"Engines: all, perplexity (p), google (g), chatgpt (gpt), gemini (gem), semantic-scholar (s2), logically (log), bing (b)",
|
|
92
|
-
"",
|
|
93
|
-
"Flags:",
|
|
94
|
-
" --synthesize For engine=all: synthesize fetched sources",
|
|
95
|
-
" --synthesizer <engine> Synthesis engine (default from ~/.dm/greedyconfig)",
|
|
96
|
-
" --fast Legacy quick mode: no source fetching or synthesis",
|
|
97
|
-
" --depth <mode> Legacy: fast|standard|deep aliases, or research",
|
|
98
|
-
" --deep-research Deprecated alias for --research",
|
|
99
|
-
" --research Iterative query/learnings loop (alias: --depth research)",
|
|
100
|
-
" --breadth <n> Research mode query breadth, 1-5 (default: 3)",
|
|
101
|
-
" --iterations <n> Research mode rounds, 1-3 (default: 2)",
|
|
102
|
-
" --max-sources <n> Research mode fetched source cap, 3-12",
|
|
103
|
-
" --research-out-dir <dir> Write research bundle to a specific directory",
|
|
104
|
-
" --no-research-bundle Disable the default .dm/greedysearch-research bundle",
|
|
105
|
-
" --fetch-top-source Fetch content from top source",
|
|
106
|
-
" --inline Output JSON to stdout (for piping)",
|
|
107
|
-
" --locale <lang> Force results language (en, de, fr, etc.)",
|
|
108
|
-
" --visible Always use visible Chrome for this search",
|
|
109
|
-
" --always-visible Alias for --visible",
|
|
110
|
-
" --stdin Read query from stdin (avoids command-line leakage)",
|
|
111
|
-
"",
|
|
112
|
-
"Environment:",
|
|
113
|
-
" GREEDY_SEARCH_VISIBLE Set to 1 to show Chrome window (disables headless)",
|
|
114
|
-
" GREEDY_SEARCH_ALWAYS_VISIBLE Set to 1 to force visible mode for all runs",
|
|
115
|
-
" GREEDY_SEARCH_LOCALE Default locale (default: en)",
|
|
116
|
-
"",
|
|
117
|
-
"Examples:",
|
|
118
|
-
' node search.mjs all "Node.js streams" # Grounded: engines + fetched sources',
|
|
119
|
-
' node search.mjs all "Node.js streams" --synthesize # Add Gemini synthesis',
|
|
120
|
-
' node search.mjs all "quick check" --fast # Legacy fast: no sources/synthesis',
|
|
121
|
-
' node search.mjs all "browser automation" --research --breadth 3 --iterations 2',
|
|
122
|
-
' node search.mjs p "what is memoization" # Single engine search'
|
|
123
|
-
].join(`
|
|
124
|
-
`)}
|
|
125
|
-
`);
|
|
126
|
-
process.exit(1);
|
|
127
|
-
}
|
|
128
|
-
const alwaysVisible = args.includes("--visible") || args.includes("--always-visible") || process.env.GREEDY_SEARCH_ALWAYS_VISIBLE === "1";
|
|
129
|
-
if (alwaysVisible) {
|
|
130
|
-
process.env.GREEDY_SEARCH_VISIBLE = "1";
|
|
131
|
-
process.env.GREEDY_SEARCH_ALWAYS_VISIBLE = "1";
|
|
132
|
-
delete process.env.GREEDY_SEARCH_HEADLESS;
|
|
133
|
-
} else if (process.env.GREEDY_SEARCH_VISIBLE !== "1") {
|
|
134
|
-
process.env.GREEDY_SEARCH_HEADLESS = "1";
|
|
135
|
-
}
|
|
136
|
-
await ensureChrome();
|
|
137
|
-
touchActivity();
|
|
138
|
-
const depthIdx = args.indexOf("--depth");
|
|
139
|
-
const legacyDepth = depthIdx !== -1 && args[depthIdx + 1] ? args[depthIdx + 1].toLowerCase() : null;
|
|
140
|
-
const engineArg = args.find((a) => !a.startsWith("--"))?.toLowerCase();
|
|
141
|
-
const researchMode = args.includes("--research") || args.includes("--deep-research") || legacyDepth === "research";
|
|
142
|
-
const legacyFast = args.includes("--fast") || legacyDepth === "fast";
|
|
143
|
-
const legacySynthesisDepth = legacyDepth === "standard" || legacyDepth === "deep" || args.includes("--deep");
|
|
144
|
-
const shouldFetchSources = engineArg === "all" && !legacyFast;
|
|
145
|
-
const shouldSynthesize = engineArg === "all" && !legacyFast && (args.includes("--synthesize") || legacySynthesisDepth);
|
|
146
|
-
const groundedSynthesis = legacyDepth === "deep" || args.includes("--deep");
|
|
147
|
-
if (args.includes("--deep-research")) {
|
|
148
|
-
process.stderr.write(`[greedysearch] --deep-research is deprecated; use --research or --depth research
|
|
149
|
-
`);
|
|
150
|
-
}
|
|
151
|
-
if (legacySynthesisDepth) {
|
|
152
|
-
process.stderr.write(`[greedysearch] depth fast|standard|deep is deprecated; use default grounded search plus --synthesize when needed
|
|
153
|
-
`);
|
|
154
|
-
}
|
|
155
|
-
const synthesizerIdx = args.indexOf("--synthesizer");
|
|
156
|
-
const synthesizer = normalizeSynthesizer(synthesizerIdx === -1 ? SYNTHESIZER : args[synthesizerIdx + 1]);
|
|
157
|
-
const full = args.includes("--full");
|
|
158
|
-
const short = !full;
|
|
159
|
-
const fetchSource = args.includes("--fetch-top-source");
|
|
160
|
-
const inline = args.includes("--inline");
|
|
161
|
-
const breadthIdx = args.indexOf("--breadth");
|
|
162
|
-
const iterationsIdx = args.indexOf("--iterations");
|
|
163
|
-
const maxSourcesIdx = args.indexOf("--max-sources");
|
|
164
|
-
const researchBreadth = breadthIdx === -1 ? undefined : args[breadthIdx + 1];
|
|
165
|
-
const researchIterations = iterationsIdx === -1 ? undefined : args[iterationsIdx + 1];
|
|
166
|
-
const researchMaxSources = maxSourcesIdx === -1 ? undefined : args[maxSourcesIdx + 1];
|
|
167
|
-
const researchOutDirIdx = args.indexOf("--research-out-dir");
|
|
168
|
-
const researchOutDir = researchOutDirIdx === -1 ? undefined : args[researchOutDirIdx + 1];
|
|
169
|
-
const writeResearchBundle = !args.includes("--no-research-bundle");
|
|
170
|
-
const outIdx = args.indexOf("--out");
|
|
171
|
-
const outFile = outIdx === -1 ? null : args[outIdx + 1];
|
|
172
|
-
const localeIdx = args.indexOf("--locale");
|
|
173
|
-
const envLocale = process.env.GREEDY_SEARCH_LOCALE;
|
|
174
|
-
const userConfig = loadUserConfig();
|
|
175
|
-
let locale = "en";
|
|
176
|
-
if (localeIdx !== -1 && args[localeIdx + 1]) {
|
|
177
|
-
locale = args[localeIdx + 1];
|
|
178
|
-
} else if (envLocale) {
|
|
179
|
-
locale = envLocale;
|
|
180
|
-
} else if (userConfig.locale) {
|
|
181
|
-
locale = userConfig.locale;
|
|
182
|
-
}
|
|
183
|
-
const rest = args.filter((a, i) => a !== "--full" && a !== "--short" && a !== "--fast" && a !== "--fetch-top-source" && a !== "--synthesize" && a !== "--deep-research" && a !== "--deep" && a !== "--research" && a !== "--inline" && a !== "--stdin" && a !== "--headless" && a !== "--visible" && a !== "--always-visible" && a !== "--depth" && a !== "--synthesizer" && a !== "--out" && a !== "--locale" && a !== "--breadth" && a !== "--iterations" && a !== "--max-sources" && a !== "--research-out-dir" && a !== "--no-research-bundle" && a !== "--help" && (depthIdx === -1 || i !== depthIdx + 1) && (synthesizerIdx === -1 || i !== synthesizerIdx + 1) && (outIdx === -1 || i !== outIdx + 1) && (localeIdx === -1 || i !== localeIdx + 1) && (breadthIdx === -1 || i !== breadthIdx + 1) && (iterationsIdx === -1 || i !== iterationsIdx + 1) && (maxSourcesIdx === -1 || i !== maxSourcesIdx + 1) && (researchOutDirIdx === -1 || i !== researchOutDirIdx + 1));
|
|
184
|
-
const engine = rest[0]?.toLowerCase();
|
|
185
|
-
const useStdin = args.includes("--stdin");
|
|
186
|
-
let query;
|
|
187
|
-
if (useStdin) {
|
|
188
|
-
query = await readStdin();
|
|
189
|
-
} else {
|
|
190
|
-
query = rest.slice(1).join(" ");
|
|
191
|
-
}
|
|
192
|
-
if (researchMode) {
|
|
193
|
-
if (engine !== "all") {
|
|
194
|
-
process.stderr.write(`[greedysearch] Research mode uses all engines; ignoring engine "${engine}".
|
|
195
|
-
`);
|
|
196
|
-
}
|
|
197
|
-
const out = await runResearchMode({
|
|
198
|
-
query: normalizeQuery(query),
|
|
199
|
-
breadth: researchBreadth,
|
|
200
|
-
iterations: researchIterations,
|
|
201
|
-
maxSources: researchMaxSources,
|
|
202
|
-
locale,
|
|
203
|
-
short,
|
|
204
|
-
writeBundle: writeResearchBundle,
|
|
205
|
-
researchOutDir
|
|
206
|
-
});
|
|
207
|
-
writeOutput(out, outFile, {
|
|
208
|
-
inline,
|
|
209
|
-
synthesize: true,
|
|
210
|
-
query
|
|
211
|
-
});
|
|
212
|
-
return;
|
|
213
|
-
}
|
|
214
|
-
if (engine === "all") {
|
|
215
|
-
await cdp(["list"]);
|
|
216
|
-
const ENGINE_START_URLS = {
|
|
217
|
-
perplexity: "https://www.perplexity.ai/",
|
|
218
|
-
google: "https://www.google.com/",
|
|
219
|
-
chatgpt: "https://chatgpt.com/",
|
|
220
|
-
gemini: "https://gemini.google.com/app",
|
|
221
|
-
"semantic-scholar": "https://www.semanticscholar.org/",
|
|
222
|
-
semanticscholar: "https://www.semanticscholar.org/",
|
|
223
|
-
s2: "https://www.semanticscholar.org/",
|
|
224
|
-
logically: "https://logically.app/research-assistant/"
|
|
225
|
-
};
|
|
226
|
-
const engineTabs = await Promise.all(ALL_ENGINES.map((e) => openNewTab(ENGINE_START_URLS[e])));
|
|
227
|
-
await cdp(["list"]);
|
|
228
|
-
const engineTimeoutFor = (engineName) => {
|
|
229
|
-
if (!legacyFast)
|
|
230
|
-
return 70000;
|
|
231
|
-
return engineName === "chatgpt" ? 60000 : 35000;
|
|
232
|
-
};
|
|
233
|
-
try {
|
|
234
|
-
const results = await Promise.allSettled(ALL_ENGINES.map((e, i) => runExtractor(ENGINES[e], normalizeQuery(query), engineTabs[i], short, engineTimeoutFor(e), locale).then((r) => {
|
|
235
|
-
process.stderr.write(`PROGRESS:${e}:done
|
|
236
|
-
`);
|
|
237
|
-
return { engine: e, ...r };
|
|
238
|
-
}).catch((err) => {
|
|
239
|
-
throw err;
|
|
240
|
-
})));
|
|
241
|
-
const out = {};
|
|
242
|
-
for (let i = 0;i < results.length; i++) {
|
|
243
|
-
const r = results[i];
|
|
244
|
-
if (r.status === "fulfilled") {
|
|
245
|
-
out[r.value.engine] = r.value;
|
|
246
|
-
} else {
|
|
247
|
-
const err = r.reason;
|
|
248
|
-
const msg = err?.message || "unknown error";
|
|
249
|
-
out[ALL_ENGINES[i]] = { error: msg };
|
|
250
|
-
if (err?.lastStage) {
|
|
251
|
-
process.stderr.write(`[greedysearch] ${ALL_ENGINES[i]} failed at stage '${err.lastStage}': ${msg}
|
|
252
|
-
`);
|
|
253
|
-
}
|
|
254
|
-
if (err?.partialErr) {
|
|
255
|
-
process.stderr.write(`[greedysearch] ${ALL_ENGINES[i]} tail stderr:
|
|
256
|
-
${err.partialErr}
|
|
257
|
-
`);
|
|
258
|
-
}
|
|
259
|
-
}
|
|
260
|
-
}
|
|
261
|
-
const recoveryCandidates = findHeadlessBlockedEngines(out);
|
|
262
|
-
if (recoveryCandidates.length > 0 && process.env.GREEDY_SEARCH_VISIBLE !== "1") {
|
|
263
|
-
logVisibleRecovery({
|
|
264
|
-
scope: "all",
|
|
265
|
-
phase: "start",
|
|
266
|
-
engines: recoveryCandidates,
|
|
267
|
-
reasons: Object.fromEntries(recoveryCandidates.map((engineName) => [
|
|
268
|
-
engineName,
|
|
269
|
-
{
|
|
270
|
-
error: out[engineName]?.error || null,
|
|
271
|
-
envelope: out[engineName]?._envelope || null
|
|
272
|
-
}
|
|
273
|
-
]))
|
|
274
|
-
});
|
|
275
|
-
process.stderr.write(`[greedysearch] \uD83D\uDD13 Headless ${recoveryCandidates.join(", ")} search hit timeout/verification/antibot signals — retrying visible to establish cookies...
|
|
276
|
-
`);
|
|
277
|
-
for (const blockedEngine of recoveryCandidates) {
|
|
278
|
-
process.stderr.write(`[greedysearch] ${blockedEngine} recovery starting in visible mode...
|
|
279
|
-
`);
|
|
280
|
-
}
|
|
281
|
-
await closeTabs(engineTabs);
|
|
282
|
-
await killHeadlessChrome();
|
|
283
|
-
process.env.GREEDY_SEARCH_VISIBLE = "1";
|
|
284
|
-
delete process.env.GREEDY_SEARCH_HEADLESS;
|
|
285
|
-
await ensureChrome();
|
|
286
|
-
await cdp(["list"]);
|
|
287
|
-
const retryTabs = [];
|
|
288
|
-
let keepVisibleForHuman = false;
|
|
289
|
-
let recovered = 0;
|
|
290
|
-
for (let i = 0;i < recoveryCandidates.length; i++) {
|
|
291
|
-
const tab = await openNewTab();
|
|
292
|
-
retryTabs.push(tab);
|
|
293
|
-
}
|
|
294
|
-
try {
|
|
295
|
-
const retries = await Promise.allSettled(recoveryCandidates.map((e, i) => runExtractor(ENGINES[e], query, retryTabs[i], short, null, locale).then((r) => ({ engine: e, ...r })).catch((err) => ({ engine: e, error: err.message }))));
|
|
296
|
-
const stillBlocked = [];
|
|
297
|
-
const manualVerification = [];
|
|
298
|
-
for (const r of retries) {
|
|
299
|
-
if (r.status === "fulfilled" && !r.value.error) {
|
|
300
|
-
out[r.value.engine] = r.value;
|
|
301
|
-
recovered++;
|
|
302
|
-
process.stderr.write(`PROGRESS:${r.value.engine}:done
|
|
303
|
-
`);
|
|
304
|
-
} else if (r.status === "fulfilled") {
|
|
305
|
-
out[r.value.engine] = r.value;
|
|
306
|
-
stillBlocked.push(r.value.engine);
|
|
307
|
-
if (isManualVerificationError(r.value.error)) {
|
|
308
|
-
manualVerification.push(r.value.engine);
|
|
309
|
-
}
|
|
310
|
-
}
|
|
311
|
-
}
|
|
312
|
-
if (recovered > 0) {
|
|
313
|
-
process.stderr.write(`[greedysearch] ✅ ${recovered}/${recoveryCandidates.length} engine(s) recovered — cookies cached for future headless runs.
|
|
314
|
-
`);
|
|
315
|
-
} else {
|
|
316
|
-
process.stderr.write(`[greedysearch] ⚠️ Recovery attempt did not extract an answer — ${recoveryCandidates.join(", ")} may still need manual verification or a DOM fallback.
|
|
317
|
-
`);
|
|
318
|
-
}
|
|
319
|
-
if (stillBlocked.length > 0) {
|
|
320
|
-
process.stderr.write(`[greedysearch] Second visible retry for ${stillBlocked.join(", ")} — Turnstile may have resolved on first attempt...
|
|
321
|
-
`);
|
|
322
|
-
const secondRetries = await Promise.allSettled(stillBlocked.map((e) => {
|
|
323
|
-
const idx = recoveryCandidates.indexOf(e);
|
|
324
|
-
return runExtractor(ENGINES[e], query, retryTabs[idx], short, null, locale).then((r) => ({ engine: e, ...r })).catch((err) => ({ engine: e, error: err.message }));
|
|
325
|
-
}));
|
|
326
|
-
const secondStillBlocked = [];
|
|
327
|
-
for (const r of secondRetries) {
|
|
328
|
-
if (r.status === "fulfilled" && !r.value.error) {
|
|
329
|
-
out[r.value.engine] = r.value;
|
|
330
|
-
recovered++;
|
|
331
|
-
process.stderr.write(`PROGRESS:${r.value.engine}:done
|
|
332
|
-
`);
|
|
333
|
-
process.stderr.write(`[greedysearch] ✅ ${r.value.engine} recovered on second visible retry.
|
|
334
|
-
`);
|
|
335
|
-
} else {
|
|
336
|
-
secondStillBlocked.push(r.value?.engine || "unknown");
|
|
337
|
-
}
|
|
338
|
-
}
|
|
339
|
-
stillBlocked.length = 0;
|
|
340
|
-
stillBlocked.push(...secondStillBlocked);
|
|
341
|
-
}
|
|
342
|
-
logVisibleRecovery({
|
|
343
|
-
scope: "all",
|
|
344
|
-
phase: stillBlocked.length > 0 ? "needs-human" : "success",
|
|
345
|
-
engines: recoveryCandidates,
|
|
346
|
-
results: Object.fromEntries(recoveryCandidates.map((engineName) => [
|
|
347
|
-
engineName,
|
|
348
|
-
{
|
|
349
|
-
mode: out[engineName]?._envelope?.mode || null,
|
|
350
|
-
durationMs: out[engineName]?._envelope?.durationMs || null,
|
|
351
|
-
lastStage: out[engineName]?._envelope?.lastStage || null,
|
|
352
|
-
error: out[engineName]?.error || null
|
|
353
|
-
}
|
|
354
|
-
]))
|
|
355
|
-
});
|
|
356
|
-
if (stillBlocked.length > 0) {
|
|
357
|
-
for (const blockedEngine of stillBlocked) {
|
|
358
|
-
process.stderr.write(`PROGRESS:${blockedEngine}:needs-human
|
|
359
|
-
`);
|
|
360
|
-
}
|
|
361
|
-
const allPollResults = await Promise.all(stillBlocked.map(async (blockedEngine) => {
|
|
362
|
-
const tab = retryTabs[recoveryCandidates.indexOf(blockedEngine)];
|
|
363
|
-
const result = await waitForChallengeCleared({
|
|
364
|
-
tab,
|
|
365
|
-
engine: blockedEngine
|
|
366
|
-
}).catch((pollErr) => ({
|
|
367
|
-
cleared: false,
|
|
368
|
-
reason: pollErr.message || String(pollErr)
|
|
369
|
-
}));
|
|
370
|
-
return { engine: blockedEngine, tab, ...result };
|
|
371
|
-
}));
|
|
372
|
-
const clearedEngines = allPollResults.filter((p) => p.cleared);
|
|
373
|
-
if (clearedEngines.length > 0) {
|
|
374
|
-
process.stderr.write(`[greedysearch] \uD83D\uDD04 Auto-resuming ${clearedEngines.map((p) => p.engine).join(", ")} on cleared tabs...
|
|
375
|
-
`);
|
|
376
|
-
await Promise.allSettled(clearedEngines.map(async (p) => {
|
|
377
|
-
const script = ENGINES[p.engine];
|
|
378
|
-
try {
|
|
379
|
-
const result = await runExtractor(script, query, p.tab, short, null, locale);
|
|
380
|
-
out[p.engine] = result;
|
|
381
|
-
process.stderr.write(`PROGRESS:${p.engine}:done
|
|
382
|
-
`);
|
|
383
|
-
} catch (resumeErr) {
|
|
384
|
-
process.stderr.write(`[greedysearch] ⚠️ Resume extraction failed for ${p.engine}: ${resumeErr.message}
|
|
385
|
-
`);
|
|
386
|
-
}
|
|
387
|
-
}));
|
|
388
|
-
}
|
|
389
|
-
const stillStillBlocked = stillBlocked.filter((e) => !clearedEngines.find((p) => p.engine === e));
|
|
390
|
-
if (stillStillBlocked.length === 0) {
|
|
391
|
-
keepVisibleForHuman = false;
|
|
392
|
-
} else {
|
|
393
|
-
keepVisibleForHuman = true;
|
|
394
|
-
out._needsHumanVerification = {
|
|
395
|
-
engines: stillStillBlocked,
|
|
396
|
-
message: "Visible Chrome is open with the engine page loaded. Solve the Turnstile checkbox or other challenge in the visible window to store cookies. Cookies persist for future runs."
|
|
397
|
-
};
|
|
398
|
-
process.stderr.write(`[greedysearch] \uD83D\uDD13 ${stillStillBlocked.join(", ")} still blocked — keeping visible Chrome open. Solve the challenge in the window to store cookies, then rerun.
|
|
399
|
-
`);
|
|
400
|
-
}
|
|
401
|
-
}
|
|
402
|
-
} finally {
|
|
403
|
-
if (keepVisibleForHuman) {
|
|
404
|
-
minimizeChrome().catch(() => {});
|
|
405
|
-
} else {
|
|
406
|
-
await closeTabs(retryTabs);
|
|
407
|
-
process.stderr.write(`[greedysearch] Switching back to headless Chrome...
|
|
408
|
-
`);
|
|
409
|
-
await killHeadlessChrome();
|
|
410
|
-
delete process.env.GREEDY_SEARCH_VISIBLE;
|
|
411
|
-
process.env.GREEDY_SEARCH_HEADLESS = "1";
|
|
412
|
-
await ensureChrome();
|
|
413
|
-
await cdp(["list"]);
|
|
414
|
-
}
|
|
415
|
-
}
|
|
416
|
-
engineTabs.length = 0;
|
|
417
|
-
}
|
|
418
|
-
for (const engineName of ALL_ENGINES) {
|
|
419
|
-
if (!out[engineName]?.error)
|
|
420
|
-
continue;
|
|
421
|
-
if (recoveryCandidates.includes(engineName)) {
|
|
422
|
-
if (process.env.GREEDY_SEARCH_VISIBLE === "1") {
|
|
423
|
-
process.stderr.write(`PROGRESS:${engineName}:${isManualVerificationError(out[engineName].error) ? "needs-human" : "error"}
|
|
424
|
-
`);
|
|
425
|
-
}
|
|
426
|
-
continue;
|
|
427
|
-
}
|
|
428
|
-
process.stderr.write(`PROGRESS:${engineName}:error
|
|
429
|
-
`);
|
|
430
|
-
}
|
|
431
|
-
out._sources = buildSourceRegistry(out, query);
|
|
432
|
-
if (shouldFetchSources && out._sources.length > 0) {
|
|
433
|
-
process.stderr.write(`PROGRESS:source-fetch:start
|
|
434
|
-
`);
|
|
435
|
-
const fetchedSources = await fetchMultipleSources(out._sources, 5, 8000);
|
|
436
|
-
out._sources = mergeFetchDataIntoSources(out._sources, fetchedSources);
|
|
437
|
-
out._fetchedSources = writeSourcesToFiles(fetchedSources);
|
|
438
|
-
process.stderr.write(`PROGRESS:source-fetch:done
|
|
439
|
-
`);
|
|
440
|
-
}
|
|
441
|
-
if (shouldSynthesize) {
|
|
442
|
-
process.stderr.write(`PROGRESS:synthesis:start
|
|
443
|
-
`);
|
|
444
|
-
process.stderr.write(`[greedysearch] Synthesizing results with ${synthesizer}...
|
|
445
|
-
`);
|
|
446
|
-
let synthesisTab = null;
|
|
447
|
-
try {
|
|
448
|
-
synthesisTab = await openNewTab(getSynthesisStartUrl(synthesizer));
|
|
449
|
-
const synthesis = await synthesizeResults(query, out, {
|
|
450
|
-
grounded: groundedSynthesis,
|
|
451
|
-
tabPrefix: synthesisTab,
|
|
452
|
-
visible: process.env.GREEDY_SEARCH_VISIBLE === "1",
|
|
453
|
-
synthesizer
|
|
454
|
-
});
|
|
455
|
-
out._synthesis = {
|
|
456
|
-
...synthesis,
|
|
457
|
-
synthesized: true
|
|
458
|
-
};
|
|
459
|
-
process.stderr.write(`PROGRESS:synthesis:done
|
|
460
|
-
`);
|
|
461
|
-
} catch (e) {
|
|
462
|
-
process.stderr.write(`[greedysearch] Synthesis failed: ${e.message}
|
|
463
|
-
`);
|
|
464
|
-
out._synthesis = {
|
|
465
|
-
error: e.message,
|
|
466
|
-
synthesized: false,
|
|
467
|
-
synthesizedBy: synthesizer
|
|
468
|
-
};
|
|
469
|
-
} finally {
|
|
470
|
-
if (synthesisTab)
|
|
471
|
-
await closeTab(synthesisTab);
|
|
472
|
-
}
|
|
473
|
-
}
|
|
474
|
-
if (fetchSource) {
|
|
475
|
-
const top = pickTopSource(out);
|
|
476
|
-
if (top)
|
|
477
|
-
out._topSource = await fetchTopSource(top.canonicalUrl || top.url);
|
|
478
|
-
}
|
|
479
|
-
if (!legacyFast)
|
|
480
|
-
out._confidence = buildConfidence(out);
|
|
481
|
-
writeOutput(out, outFile, {
|
|
482
|
-
inline,
|
|
483
|
-
synthesize: shouldSynthesize,
|
|
484
|
-
query
|
|
485
|
-
});
|
|
486
|
-
return;
|
|
487
|
-
} finally {
|
|
488
|
-
await closeTabs(engineTabs);
|
|
489
|
-
}
|
|
490
|
-
}
|
|
491
|
-
const script = ENGINES[engine];
|
|
492
|
-
if (!script) {
|
|
493
|
-
process.stderr.write(`Unknown engine: "${engine}"
|
|
494
|
-
Available: ${Object.keys(ENGINES).join(", ")}
|
|
495
|
-
`);
|
|
496
|
-
process.exit(1);
|
|
497
|
-
}
|
|
498
|
-
try {
|
|
499
|
-
const result = await runExtractor(script, normalizeQuery(query), null, short, null, locale);
|
|
500
|
-
if (fetchSource && result.sources?.length > 0) {
|
|
501
|
-
result.topSource = await fetchTopSource(result.sources[0].url);
|
|
502
|
-
}
|
|
503
|
-
writeOutput(result, outFile, { inline, synthesize: false, query });
|
|
504
|
-
} catch (e) {
|
|
505
|
-
const recoveryEngine = script.includes("bing") ? "bing" : script.includes("perplexity") ? "perplexity" : script.includes("chatgpt") ? "chatgpt" : script.includes("semantic-scholar") ? "semantic-scholar" : script.includes("logically") ? "logically" : null;
|
|
506
|
-
const canRetryVisible = recoveryEngine && process.env.GREEDY_SEARCH_VISIBLE !== "1" && isHeadlessBlockedResult(e);
|
|
507
|
-
if (canRetryVisible) {
|
|
508
|
-
logVisibleRecovery({
|
|
509
|
-
scope: "single",
|
|
510
|
-
phase: "start",
|
|
511
|
-
engines: [recoveryEngine],
|
|
512
|
-
reasons: {
|
|
513
|
-
[recoveryEngine]: {
|
|
514
|
-
error: e.message || null,
|
|
515
|
-
envelope: e.envelope || null,
|
|
516
|
-
lastStage: e.lastStage || null
|
|
517
|
-
}
|
|
518
|
-
}
|
|
519
|
-
});
|
|
520
|
-
process.stderr.write(`[greedysearch] \uD83D\uDD13 ${recoveryEngine} blocked in headless — retrying visible to establish cookies...
|
|
521
|
-
`);
|
|
522
|
-
await killHeadlessChrome();
|
|
523
|
-
process.env.GREEDY_SEARCH_VISIBLE = "1";
|
|
524
|
-
delete process.env.GREEDY_SEARCH_HEADLESS;
|
|
525
|
-
await ensureChrome();
|
|
526
|
-
await cdp(["list"]);
|
|
527
|
-
const retryTab = await openNewTab();
|
|
528
|
-
let keepVisibleForHuman = false;
|
|
529
|
-
try {
|
|
530
|
-
const result = await runExtractor(script, query, retryTab, short, null, locale);
|
|
531
|
-
logVisibleRecovery({
|
|
532
|
-
scope: "single",
|
|
533
|
-
phase: "success",
|
|
534
|
-
engines: [recoveryEngine],
|
|
535
|
-
result: {
|
|
536
|
-
engine: recoveryEngine,
|
|
537
|
-
mode: result._envelope?.mode || null,
|
|
538
|
-
durationMs: result._envelope?.durationMs || null,
|
|
539
|
-
lastStage: result._envelope?.lastStage || null
|
|
540
|
-
}
|
|
541
|
-
});
|
|
542
|
-
if (fetchSource && result.sources?.length > 0) {
|
|
543
|
-
result.topSource = await fetchTopSource(result.sources[0].url);
|
|
544
|
-
}
|
|
545
|
-
writeOutput(result, outFile, { inline, synthesize: false, query });
|
|
546
|
-
return;
|
|
547
|
-
} catch (retryErr) {
|
|
548
|
-
logVisibleRecovery({
|
|
549
|
-
scope: "single",
|
|
550
|
-
phase: "needs-human",
|
|
551
|
-
engines: [recoveryEngine],
|
|
552
|
-
result: {
|
|
553
|
-
engine: recoveryEngine,
|
|
554
|
-
error: retryErr.message || String(retryErr),
|
|
555
|
-
envelope: retryErr.envelope || null
|
|
556
|
-
}
|
|
557
|
-
});
|
|
558
|
-
const pollResult = await waitForChallengeCleared({
|
|
559
|
-
tab: retryTab,
|
|
560
|
-
engine: recoveryEngine
|
|
561
|
-
}).catch((pollErr) => ({
|
|
562
|
-
cleared: false,
|
|
563
|
-
reason: pollErr.message || String(pollErr)
|
|
564
|
-
}));
|
|
565
|
-
if (pollResult.cleared) {
|
|
566
|
-
process.stderr.write(`[greedysearch] \uD83D\uDD04 Auto-resuming ${recoveryEngine} extraction on the now-cleared tab...
|
|
567
|
-
`);
|
|
568
|
-
try {
|
|
569
|
-
const result = await runExtractor(script, query, retryTab, short, null, locale);
|
|
570
|
-
logVisibleRecovery({
|
|
571
|
-
scope: "single",
|
|
572
|
-
phase: "success-after-poll",
|
|
573
|
-
engines: [recoveryEngine],
|
|
574
|
-
result: {
|
|
575
|
-
engine: recoveryEngine,
|
|
576
|
-
mode: result._envelope?.mode || null,
|
|
577
|
-
durationMs: result._envelope?.durationMs || null,
|
|
578
|
-
lastStage: result._envelope?.lastStage || null
|
|
579
|
-
}
|
|
580
|
-
});
|
|
581
|
-
if (fetchSource && result.sources?.length > 0) {
|
|
582
|
-
result.topSource = await fetchTopSource(result.sources[0].url);
|
|
583
|
-
}
|
|
584
|
-
writeOutput(result, outFile, { inline, synthesize: false, query });
|
|
585
|
-
return;
|
|
586
|
-
} catch (resumeErr) {
|
|
587
|
-
process.stderr.write(`[greedysearch] ⚠️ Resume extraction failed: ${resumeErr.message}
|
|
588
|
-
`);
|
|
589
|
-
}
|
|
590
|
-
}
|
|
591
|
-
keepVisibleForHuman = true;
|
|
592
|
-
writeOutput({
|
|
593
|
-
query,
|
|
594
|
-
error: retryErr.message,
|
|
595
|
-
_needsHumanVerification: {
|
|
596
|
-
engines: [recoveryEngine],
|
|
597
|
-
message: "Visible Chrome is open with the engine page loaded. Solve the Turnstile checkbox or other challenge to store cookies. Cookies persist for future runs."
|
|
598
|
-
}
|
|
599
|
-
}, outFile, { inline, synthesize: false, query });
|
|
600
|
-
return;
|
|
601
|
-
} finally {
|
|
602
|
-
if (!keepVisibleForHuman) {
|
|
603
|
-
await closeTab(retryTab);
|
|
604
|
-
await killHeadlessChrome();
|
|
605
|
-
delete process.env.GREEDY_SEARCH_VISIBLE;
|
|
606
|
-
process.env.GREEDY_SEARCH_HEADLESS = "1";
|
|
607
|
-
} else {
|
|
608
|
-
minimizeChrome().catch(() => {});
|
|
609
|
-
}
|
|
610
|
-
}
|
|
611
|
-
}
|
|
612
|
-
process.stderr.write(`Error: ${e.message}
|
|
613
|
-
`);
|
|
614
|
-
process.exit(1);
|
|
615
|
-
}
|
|
616
|
-
}
|
|
617
|
-
function pickTopSource(out) {
|
|
618
|
-
if (Array.isArray(out._sources) && out._sources.length > 0)
|
|
619
|
-
return out._sources[0];
|
|
620
|
-
for (const engine of ["perplexity", "google", "bing"]) {
|
|
621
|
-
const r = out[engine];
|
|
622
|
-
if (r?.sources?.length > 0)
|
|
623
|
-
return r.sources[0];
|
|
624
|
-
}
|
|
625
|
-
return null;
|
|
626
|
-
}
|
|
627
|
-
async function minimizeChrome() {
|
|
628
|
-
if (process.env.GREEDY_SEARCH_HEADLESS === "1")
|
|
629
|
-
return;
|
|
630
|
-
await minimizeViaCDP(GREEDY_PORT);
|
|
631
|
-
}
|
|
632
|
-
main().finally(async () => {
|
|
633
|
-
touchActivity();
|
|
634
|
-
minimizeChrome().catch(() => {});
|
|
635
|
-
});
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
import { spawn } from "node:child_process";
|
|
2
|
-
import { nodeRuntimeCommand } from "../src/utils/node-runtime.mjs";
|
|
3
|
-
const launchVisibleBin = new URL("./launch-visible.mjs", import.meta.url).pathname.replace(/^\/([A-Z]:)/, "$1");
|
|
4
|
-
const flag = process.argv[2] || "";
|
|
5
|
-
if (flag === "--kill") {
|
|
6
|
-
const proc = spawn(nodeRuntimeCommand(), [launchVisibleBin, "--kill"], {
|
|
7
|
-
stdio: "inherit"
|
|
8
|
-
});
|
|
9
|
-
proc.on("close", (code) => process.exit(code ?? 0));
|
|
10
|
-
} else if (flag === "--status") {
|
|
11
|
-
const proc = spawn(nodeRuntimeCommand(), [launchVisibleBin, "--status"], {
|
|
12
|
-
stdio: "inherit"
|
|
13
|
-
});
|
|
14
|
-
proc.on("close", (code) => process.exit(code ?? 0));
|
|
15
|
-
} else {
|
|
16
|
-
console.log("\uD83D\uDE80 Launching visible GreedySearch Chrome...");
|
|
17
|
-
const proc = spawn(nodeRuntimeCommand(), [launchVisibleBin], {
|
|
18
|
-
stdio: "inherit",
|
|
19
|
-
env: { ...process.env, GREEDY_SEARCH_VISIBLE: "1" }
|
|
20
|
-
});
|
|
21
|
-
proc.on("close", (code) => process.exit(code ?? 0));
|
|
22
|
-
}
|