@duckmind/dm-windows-x64 0.60.4 → 0.60.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dm.exe +0 -0
- package/extensions/.dm-extensions.json +70 -123
- package/extensions/dm-9router-ext/src/index.js +410 -5
- package/extensions/dm-caveman/extensions/caveman.js +283 -12
- package/extensions/dm-cliproxy/index.js +182 -2
- package/extensions/dm-cliproxy/scripts/check-config-migration.js +66 -8
- package/extensions/dm-cliproxy/src/apply.js +228 -1
- package/extensions/dm-cliproxy/src/cache.js +42 -1
- package/extensions/dm-cliproxy/src/commands.js +50 -2
- package/extensions/dm-cliproxy/src/compat.js +81 -1
- package/extensions/dm-cliproxy/src/config.js +192 -2
- package/extensions/dm-cliproxy/src/conflicts.js +46 -1
- package/extensions/dm-cliproxy/src/fetch-models.js +190 -1
- package/extensions/dm-cliproxy/src/fetch-usage.js +41 -1
- package/extensions/dm-cliproxy/src/log.js +23 -1
- package/extensions/dm-cliproxy/src/status-quota.js +77 -1
- package/extensions/dm-cliproxy/src/ui-frame.js +50 -1
- package/extensions/dm-cliproxy/src/ui-hub/hub.js +199 -2
- package/extensions/dm-cliproxy/src/ui-hub/index.js +26 -2
- package/extensions/dm-cliproxy/src/ui-hub/shell.js +46 -1
- package/extensions/dm-cliproxy/src/ui-hub/view-diagnostics.js +69 -1
- package/extensions/dm-cliproxy/src/ui-hub/view-models.js +379 -1
- package/extensions/dm-cliproxy/src/ui-hub/view-usage.js +106 -1
- package/extensions/dm-cliproxy/src/ui-picker/catalog.js +41 -1
- package/extensions/dm-cliproxy/src/ui-picker/mutate.js +115 -1
- package/extensions/dm-cliproxy/src/ui-picker/prompt-confirm.js +35 -1
- package/extensions/dm-cliproxy/src/ui-picker/prompt-name.js +62 -1
- package/extensions/dm-cliproxy/src/ui-picker/providers.js +45 -1
- package/extensions/dm-cliproxy/src/ui-picker/render-text.js +34 -1
- package/extensions/dm-cliproxy/src/ui-picker/rows.js +57 -1
- package/extensions/dm-cliproxy/src/ui-setup.js +260 -2
- package/extensions/dm-cliproxy/src/ui-usage.js +151 -1
- package/extensions/dm-cliproxy/src/usage-shared-cache.js +97 -1
- package/extensions/dm-context/src/context.js +144 -1
- package/extensions/dm-context/src/index.js +339 -7
- package/extensions/dm-context/src/utils.js +9 -1
- package/extensions/dm-cua/bin/browser-cua.mjs +73 -8
- package/extensions/dm-cua/index.js +75 -6
- package/extensions/dm-cua/src/browser-cua-lib.mjs +490 -6
- package/extensions/dm-cua/src/browser-install.mjs +331 -2
- package/extensions/dm-fff/src/index.js +688 -12
- package/extensions/dm-fff/src/query.js +60 -1
- package/extensions/dm-goal/src/goal.js +894 -23
- package/extensions/dm-image2/index.js +103 -9
- package/extensions/dm-image2/src/image-lib.mjs +1275 -8
- package/extensions/dm-subagents/install.mjs +62 -8
- package/extensions/dm-subagents/src/agents/agent-management.js +1190 -36
- package/extensions/dm-subagents/src/agents/agent-memory.js +216 -6
- package/extensions/dm-subagents/src/agents/agent-scope.js +5 -1
- package/extensions/dm-subagents/src/agents/agent-selection.js +20 -1
- package/extensions/dm-subagents/src/agents/agent-serializer.js +120 -5
- package/extensions/dm-subagents/src/agents/agents.js +1139 -11
- package/extensions/dm-subagents/src/agents/chain-serializer.js +299 -11
- package/extensions/dm-subagents/src/agents/frontmatter.js +65 -6
- package/extensions/dm-subagents/src/agents/identity.js +29 -1
- package/extensions/dm-subagents/src/agents/proactive-skills.js +141 -1
- package/extensions/dm-subagents/src/agents/skills.js +614 -6
- package/extensions/dm-subagents/src/extension/config.js +35 -2
- package/extensions/dm-subagents/src/extension/control-notices.js +69 -4
- package/extensions/dm-subagents/src/extension/doctor.js +172 -15
- package/extensions/dm-subagents/src/extension/fanout-child.js +158 -231
- package/extensions/dm-subagents/src/extension/index.js +521 -359
- package/extensions/dm-subagents/src/extension/rpc.js +266 -7
- package/extensions/dm-subagents/src/extension/schemas.js +275 -1
- package/extensions/dm-subagents/src/extension/tool-description.js +111 -6
- package/extensions/dm-subagents/src/intercom/intercom-bridge.js +126 -4
- package/extensions/dm-subagents/src/intercom/native-supervisor-channel.js +452 -5
- package/extensions/dm-subagents/src/intercom/result-intercom.js +319 -3
- package/extensions/dm-subagents/src/profiles/profiles.js +458 -3
- package/extensions/dm-subagents/src/runs/background/async-execution.js +834 -40
- package/extensions/dm-subagents/src/runs/background/async-job-tracker.js +435 -14
- package/extensions/dm-subagents/src/runs/background/async-resume.js +334 -8
- package/extensions/dm-subagents/src/runs/background/async-status.js +313 -12
- package/extensions/dm-subagents/src/runs/background/chain-append.js +245 -2
- package/extensions/dm-subagents/src/runs/background/chain-root-attachment.js +136 -1
- package/extensions/dm-subagents/src/runs/background/completion-batcher.js +94 -1
- package/extensions/dm-subagents/src/runs/background/completion-dedupe.js +54 -1
- package/extensions/dm-subagents/src/runs/background/control-channel.js +190 -1
- package/extensions/dm-subagents/src/runs/background/fleet-view.js +483 -17
- package/extensions/dm-subagents/src/runs/background/notify.js +129 -3
- package/extensions/dm-subagents/src/runs/background/parallel-groups.js +34 -1
- package/extensions/dm-subagents/src/runs/background/result-watcher.js +236 -6
- package/extensions/dm-subagents/src/runs/background/run-id-resolver.js +76 -4
- package/extensions/dm-subagents/src/runs/background/run-status.js +427 -23
- package/extensions/dm-subagents/src/runs/background/scheduled-runs.js +487 -4
- package/extensions/dm-subagents/src/runs/background/stale-run-reconciler.js +306 -9
- package/extensions/dm-subagents/src/runs/background/subagent-runner.js +2849 -73
- package/extensions/dm-subagents/src/runs/background/top-level-async.js +5 -1
- package/extensions/dm-subagents/src/runs/background/wait.js +206 -11
- package/extensions/dm-subagents/src/runs/foreground/chain-clarify.js +1013 -12
- package/extensions/dm-subagents/src/runs/foreground/chain-execution.js +980 -101
- package/extensions/dm-subagents/src/runs/foreground/execution.js +1165 -45
- package/extensions/dm-subagents/src/runs/foreground/subagent-executor.js +3157 -222
- package/extensions/dm-subagents/src/runs/shared/acceptance.js +835 -3
- package/extensions/dm-subagents/src/runs/shared/chain-outputs.js +104 -1
- package/extensions/dm-subagents/src/runs/shared/completion-guard.js +116 -3
- package/extensions/dm-subagents/src/runs/shared/dm-args.js +208 -1
- package/extensions/dm-subagents/src/runs/shared/dm-spawn.js +90 -1
- package/extensions/dm-subagents/src/runs/shared/dynamic-fanout.js +282 -1
- package/extensions/dm-subagents/src/runs/shared/long-running-guard.js +148 -1
- package/extensions/dm-subagents/src/runs/shared/mcp-direct-tool-allowlist.js +305 -1
- package/extensions/dm-subagents/src/runs/shared/model-fallback.js +194 -1
- package/extensions/dm-subagents/src/runs/shared/model-scope.js +65 -1
- package/extensions/dm-subagents/src/runs/shared/nested-events.js +851 -8
- package/extensions/dm-subagents/src/runs/shared/nested-path.js +41 -1
- package/extensions/dm-subagents/src/runs/shared/nested-render.js +105 -1
- package/extensions/dm-subagents/src/runs/shared/parallel-utils.js +81 -4
- package/extensions/dm-subagents/src/runs/shared/run-history.js +51 -4
- package/extensions/dm-subagents/src/runs/shared/single-output.js +149 -8
- package/extensions/dm-subagents/src/runs/shared/structured-output.js +58 -1
- package/extensions/dm-subagents/src/runs/shared/subagent-control.js +166 -5
- package/extensions/dm-subagents/src/runs/shared/subagent-prompt-runtime.js +329 -13
- package/extensions/dm-subagents/src/runs/shared/tool-budget.js +73 -1
- package/extensions/dm-subagents/src/runs/shared/turn-budget.js +47 -4
- package/extensions/dm-subagents/src/runs/shared/workflow-graph.js +196 -1
- package/extensions/dm-subagents/src/runs/shared/worktree.js +435 -3
- package/extensions/dm-subagents/src/shared/artifacts.js +92 -2
- package/extensions/dm-subagents/src/shared/atomic-json.js +55 -1
- package/extensions/dm-subagents/src/shared/child-transcript.js +167 -5
- package/extensions/dm-subagents/src/shared/file-coalescer.js +25 -1
- package/extensions/dm-subagents/src/shared/fork-context.js +147 -4
- package/extensions/dm-subagents/src/shared/formatters.js +98 -7
- package/extensions/dm-subagents/src/shared/jsonl-writer.js +56 -2
- package/extensions/dm-subagents/src/shared/model-info.js +62 -1
- package/extensions/dm-subagents/src/shared/post-exit-stdio-guard.js +68 -1
- package/extensions/dm-subagents/src/shared/session-identity.js +6 -1
- package/extensions/dm-subagents/src/shared/session-tokens.js +39 -2
- package/extensions/dm-subagents/src/shared/settings.js +198 -11
- package/extensions/dm-subagents/src/shared/status-format.js +53 -1
- package/extensions/dm-subagents/src/shared/types.js +184 -6
- package/extensions/dm-subagents/src/shared/utils.js +462 -2
- package/extensions/dm-subagents/src/slash/prompt-template-bridge.js +288 -1
- package/extensions/dm-subagents/src/slash/prompt-workflows.js +297 -7
- package/extensions/dm-subagents/src/slash/slash-bridge.js +118 -1
- package/extensions/dm-subagents/src/slash/slash-commands.js +1287 -31
- package/extensions/dm-subagents/src/slash/slash-live-state.js +240 -4
- package/extensions/dm-subagents/src/tui/render-helpers.js +64 -1
- package/extensions/dm-subagents/src/tui/render.js +1542 -4
- package/extensions/dm-usage/index.js +1294 -9
- package/extensions/greedysearch-dm/bin/cdp-greedy.mjs +40 -9
- package/extensions/greedysearch-dm/bin/cdp-headless.mjs +5 -2
- package/extensions/greedysearch-dm/bin/cdp-visible.mjs +5 -2
- package/extensions/greedysearch-dm/bin/cdp.mjs +896 -30
- package/extensions/greedysearch-dm/bin/gschrome.mjs +30 -2
- package/extensions/greedysearch-dm/bin/kill-visible.mjs +7 -2
- package/extensions/greedysearch-dm/bin/launch-visible.mjs +13 -2
- package/extensions/greedysearch-dm/bin/launch.mjs +282 -10
- package/extensions/greedysearch-dm/bin/mcp.mjs +386 -361
- package/extensions/greedysearch-dm/bin/search.mjs +620 -540
- package/extensions/greedysearch-dm/bin/visible.mjs +22 -2
- package/extensions/greedysearch-dm/extractors/bing-copilot.mjs +329 -579
- package/extensions/greedysearch-dm/extractors/chatgpt.mjs +301 -583
- package/extensions/greedysearch-dm/extractors/common.mjs +408 -32
- package/extensions/greedysearch-dm/extractors/consensus.mjs +376 -365
- package/extensions/greedysearch-dm/extractors/consent.mjs +303 -14
- package/extensions/greedysearch-dm/extractors/gemini.mjs +228 -592
- package/extensions/greedysearch-dm/extractors/google-ai.mjs +78 -499
- package/extensions/greedysearch-dm/extractors/logically.mjs +270 -347
- package/extensions/greedysearch-dm/extractors/perplexity.mjs +243 -581
- package/extensions/greedysearch-dm/extractors/selectors.mjs +32 -1
- package/extensions/greedysearch-dm/extractors/semantic-scholar.mjs +130 -317
- package/extensions/greedysearch-dm/index.js +123 -23
- package/extensions/greedysearch-dm/src/fetcher.mjs +576 -2
- package/extensions/greedysearch-dm/src/formatters/results.js +95 -10
- package/extensions/greedysearch-dm/src/formatters/sources.js +57 -1
- package/extensions/greedysearch-dm/src/formatters/synthesis.js +49 -1
- package/extensions/greedysearch-dm/src/github.mjs +222 -7
- package/extensions/greedysearch-dm/src/reddit.mjs +145 -14
- package/extensions/greedysearch-dm/src/search/browser-lifecycle.mjs +340 -9
- package/extensions/greedysearch-dm/src/search/challenge-detect.mjs +112 -4
- package/extensions/greedysearch-dm/src/search/chrome.mjs +486 -285
- package/extensions/greedysearch-dm/src/search/constants.mjs +109 -8
- package/extensions/greedysearch-dm/src/search/defaults.mjs +10 -1
- package/extensions/greedysearch-dm/src/search/engines.mjs +79 -9
- package/extensions/greedysearch-dm/src/search/fetch-source.mjs +441 -349
- package/extensions/greedysearch-dm/src/search/file-sources.mjs +29 -7
- package/extensions/greedysearch-dm/src/search/minimize.mjs +86 -1
- package/extensions/greedysearch-dm/src/search/output.mjs +51 -5
- package/extensions/greedysearch-dm/src/search/paths.mjs +48 -1
- package/extensions/greedysearch-dm/src/search/pdf.mjs +63 -2
- package/extensions/greedysearch-dm/src/search/port-pid.mjs +69 -1
- package/extensions/greedysearch-dm/src/search/progress.mjs +109 -2
- package/extensions/greedysearch-dm/src/search/query.mjs +21 -1
- package/extensions/greedysearch-dm/src/search/recovery.mjs +49 -1
- package/extensions/greedysearch-dm/src/search/research.mjs +2227 -458
- package/extensions/greedysearch-dm/src/search/scale-aware.mjs +61 -11
- package/extensions/greedysearch-dm/src/search/simple-research.mjs +396 -805
- package/extensions/greedysearch-dm/src/search/sources.mjs +412 -1
- package/extensions/greedysearch-dm/src/search/synthesis-runner.mjs +127 -12
- package/extensions/greedysearch-dm/src/search/synthesis.mjs +202 -12
- package/extensions/greedysearch-dm/src/tools/greedy-search-handler.js +209 -23
- package/extensions/greedysearch-dm/src/tools/shared.js +226 -10
- package/extensions/greedysearch-dm/src/utils/content.mjs +35 -4
- package/extensions/greedysearch-dm/src/utils/helpers.js +22 -1
- package/extensions/greedysearch-dm/src/utils/node-runtime.mjs +10 -1
- package/extensions/greedysearch-dm/src/utils/system-cmds.mjs +61 -1
- package/package.json +1 -1
|
@@ -1,9 +1,31 @@
|
|
|
1
|
-
import{mkdirSync
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
1
|
+
import { mkdirSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
const DEFAULT_DIR = join(process.cwd(), ".dm", "greedysearch-sources");
|
|
4
|
+
export function writeSourcesToFiles(fetchedSources, dir = DEFAULT_DIR) {
|
|
5
|
+
mkdirSync(dir, { recursive: true });
|
|
6
|
+
return fetchedSources.map((source) => {
|
|
7
|
+
if (!source.content || source.content.length < 10) {
|
|
8
|
+
return source;
|
|
9
|
+
}
|
|
10
|
+
const safeId = String(source.id || "unknown").replace(/[^a-zA-Z0-9_-]/g, "");
|
|
11
|
+
const urlSlug = (source.canonicalUrl || source.url || "").replace(/^https?:\/\//, "").replace(/[^a-zA-Z0-9]/g, "-").slice(0, 40);
|
|
12
|
+
const filename = `${safeId}-${urlSlug}.md`;
|
|
13
|
+
const filepath = join(dir, filename);
|
|
14
|
+
const header = `---
|
|
15
|
+
url: ${source.finalUrl || source.url}
|
|
16
|
+
title: ${source.title || ""}
|
|
17
|
+
source: ${source.source || "unknown"}
|
|
18
|
+
status: ${source.status || ""}
|
|
19
|
+
chars: ${source.contentChars || source.content.length}
|
|
7
20
|
---
|
|
8
21
|
|
|
9
|
-
`;
|
|
22
|
+
`;
|
|
23
|
+
writeFileSync(filepath, header + source.content, "utf8");
|
|
24
|
+
const { content, ...rest } = source;
|
|
25
|
+
return {
|
|
26
|
+
...rest,
|
|
27
|
+
contentPath: filepath,
|
|
28
|
+
contentChars: source.contentChars || content.length
|
|
29
|
+
};
|
|
30
|
+
});
|
|
31
|
+
}
|
|
@@ -1 +1,86 @@
|
|
|
1
|
-
import
|
|
1
|
+
import http from "node:http";
|
|
2
|
+
export function httpGet(url, timeoutMs = 1000) {
|
|
3
|
+
return new Promise((resolve) => {
|
|
4
|
+
const req = http.get(url, (res) => {
|
|
5
|
+
let body = "";
|
|
6
|
+
res.on("data", (d) => body += d);
|
|
7
|
+
res.on("end", () => resolve({ ok: res.statusCode === 200, body }));
|
|
8
|
+
});
|
|
9
|
+
req.on("error", () => resolve({ ok: false }));
|
|
10
|
+
req.setTimeout(timeoutMs, () => {
|
|
11
|
+
req.destroy();
|
|
12
|
+
resolve({ ok: false });
|
|
13
|
+
});
|
|
14
|
+
});
|
|
15
|
+
}
|
|
16
|
+
export async function minimizeViaCDP(port) {
|
|
17
|
+
try {
|
|
18
|
+
const versionResponse = await httpGet(`http://localhost:${port}/json/version`);
|
|
19
|
+
if (!versionResponse.ok)
|
|
20
|
+
return;
|
|
21
|
+
const version = JSON.parse(versionResponse.body);
|
|
22
|
+
const targetsResponse = await httpGet(`http://localhost:${port}/json/list`);
|
|
23
|
+
if (!targetsResponse.ok)
|
|
24
|
+
return;
|
|
25
|
+
const targets = JSON.parse(targetsResponse.body);
|
|
26
|
+
const targetId = targets.find((t) => t.type === "page")?.id;
|
|
27
|
+
if (!targetId)
|
|
28
|
+
return;
|
|
29
|
+
const wsUrlStr = version.webSocketDebuggerUrl;
|
|
30
|
+
if (typeof wsUrlStr !== "string")
|
|
31
|
+
return;
|
|
32
|
+
const wsUrl = new URL(wsUrlStr);
|
|
33
|
+
if (wsUrl.hostname !== "localhost" && wsUrl.hostname !== "127.0.0.1")
|
|
34
|
+
return;
|
|
35
|
+
if (!/^ws:\/\/localhost:\d+/.test(`ws://${wsUrl.host}`))
|
|
36
|
+
return;
|
|
37
|
+
const ws = new WebSocket(`ws://localhost:${port}${wsUrl.pathname}`);
|
|
38
|
+
await new Promise((resolve) => {
|
|
39
|
+
let settled = false;
|
|
40
|
+
const timeout = setTimeout(() => finish(), 5000);
|
|
41
|
+
const finish = () => {
|
|
42
|
+
if (settled)
|
|
43
|
+
return;
|
|
44
|
+
settled = true;
|
|
45
|
+
clearTimeout(timeout);
|
|
46
|
+
try {
|
|
47
|
+
ws.close();
|
|
48
|
+
} catch {}
|
|
49
|
+
resolve();
|
|
50
|
+
};
|
|
51
|
+
ws.onopen = () => {
|
|
52
|
+
try {
|
|
53
|
+
ws.send(JSON.stringify({
|
|
54
|
+
id: 1,
|
|
55
|
+
method: "Browser.getWindowForTarget",
|
|
56
|
+
params: { targetId }
|
|
57
|
+
}));
|
|
58
|
+
} catch {
|
|
59
|
+
finish();
|
|
60
|
+
}
|
|
61
|
+
};
|
|
62
|
+
ws.onmessage = (ev) => {
|
|
63
|
+
try {
|
|
64
|
+
const msg = JSON.parse(ev.data);
|
|
65
|
+
if (msg.id === 1 && msg.result?.windowId) {
|
|
66
|
+
ws.send(JSON.stringify({
|
|
67
|
+
id: 2,
|
|
68
|
+
method: "Browser.setWindowBounds",
|
|
69
|
+
params: {
|
|
70
|
+
windowId: msg.result.windowId,
|
|
71
|
+
bounds: { windowState: "minimized" }
|
|
72
|
+
}
|
|
73
|
+
}));
|
|
74
|
+
} else if (msg.id === 2) {
|
|
75
|
+
finish();
|
|
76
|
+
} else if (msg.id === 1) {
|
|
77
|
+
finish();
|
|
78
|
+
}
|
|
79
|
+
} catch {
|
|
80
|
+
finish();
|
|
81
|
+
}
|
|
82
|
+
};
|
|
83
|
+
ws.onerror = finish;
|
|
84
|
+
});
|
|
85
|
+
} catch {}
|
|
86
|
+
}
|
|
@@ -1,5 +1,51 @@
|
|
|
1
|
-
import{mkdirSync
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
1
|
+
import { mkdirSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
const __dir = import.meta.dirname || new URL(".", import.meta.url).pathname.replace(/^\/([A-Z]:)/, "$1");
|
|
4
|
+
export function slugify(query) {
|
|
5
|
+
return query.toLowerCase().replaceAll(/[^a-z0-9]+/g, "-").replaceAll(/^-|-$/g, "").slice(0, 60);
|
|
6
|
+
}
|
|
7
|
+
const RESULTS_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000;
|
|
8
|
+
const RESULTS_MIN_KEEP = 10;
|
|
9
|
+
function purgeOldResults(dir) {
|
|
10
|
+
try {
|
|
11
|
+
const files = readdirSync(dir).filter((f) => f.endsWith(".json") || f.endsWith(".md")).map((f) => ({ f, mtime: statSync(join(dir, f)).mtimeMs })).sort((a, b) => b.mtime - a.mtime);
|
|
12
|
+
const cutoff = Date.now() - RESULTS_MAX_AGE_MS;
|
|
13
|
+
for (let i = RESULTS_MIN_KEEP;i < files.length; i++) {
|
|
14
|
+
if (files[i].mtime < cutoff) {
|
|
15
|
+
rmSync(join(dir, files[i].f), { force: true });
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
} catch {}
|
|
19
|
+
}
|
|
20
|
+
export function resultsDir() {
|
|
21
|
+
const dir = join(__dir, "..", "..", "results");
|
|
22
|
+
mkdirSync(dir, { recursive: true });
|
|
23
|
+
purgeOldResults(dir);
|
|
24
|
+
return dir;
|
|
25
|
+
}
|
|
26
|
+
export function writeOutput(data, outFile, { inline = false, synthesize = false, query = "" } = {}) {
|
|
27
|
+
const json = `${JSON.stringify(data, null, 2)}
|
|
28
|
+
`;
|
|
29
|
+
if (outFile) {
|
|
30
|
+
writeFileSync(outFile, json, "utf8");
|
|
31
|
+
process.stderr.write(`Results written to ${outFile}
|
|
32
|
+
`);
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
if (inline) {
|
|
36
|
+
process.stdout.write(json);
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
const ts = new Date().toISOString().replaceAll("T", "_").replaceAll(/[:.]/g, "-").slice(0, 19);
|
|
40
|
+
const slug = slugify(query);
|
|
41
|
+
const base = join(resultsDir(), `${ts}_${slug}`);
|
|
42
|
+
writeFileSync(`${base}.json`, json, "utf8");
|
|
43
|
+
if (synthesize && data._synthesis?.answer) {
|
|
44
|
+
writeFileSync(`${base}-synthesis.md`, data._synthesis.answer, "utf8");
|
|
45
|
+
process.stdout.write(`${base}-synthesis.md
|
|
46
|
+
`);
|
|
47
|
+
} else {
|
|
48
|
+
process.stdout.write(`${base}.json
|
|
49
|
+
`);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
@@ -1 +1,48 @@
|
|
|
1
|
-
import{existsSync
|
|
1
|
+
import { existsSync } from "node:fs";
|
|
2
|
+
import { dirname, join } from "node:path";
|
|
3
|
+
import { fileURLToPath } from "node:url";
|
|
4
|
+
export function moduleDirectory(metaUrl) {
|
|
5
|
+
return fileURLToPath(new URL(".", metaUrl)).replace(/^\/([A-Z]:)/, "$1");
|
|
6
|
+
}
|
|
7
|
+
function uniqueExisting(candidates, exists = existsSync) {
|
|
8
|
+
const seen = new Set;
|
|
9
|
+
for (const candidate of candidates.filter(Boolean)) {
|
|
10
|
+
if (seen.has(candidate))
|
|
11
|
+
continue;
|
|
12
|
+
seen.add(candidate);
|
|
13
|
+
if (exists(candidate))
|
|
14
|
+
return candidate;
|
|
15
|
+
}
|
|
16
|
+
return candidates.filter(Boolean).at(-1);
|
|
17
|
+
}
|
|
18
|
+
export function resolveGreedySearchExtractorScript(script, {
|
|
19
|
+
moduleDir,
|
|
20
|
+
entrypoint = process.argv[1],
|
|
21
|
+
env = process.env,
|
|
22
|
+
exists = existsSync
|
|
23
|
+
} = {}) {
|
|
24
|
+
const entryDir = entrypoint ? dirname(entrypoint) : null;
|
|
25
|
+
const extensionDir = env.GREEDY_SEARCH_EXTENSION_DIR?.trim() || null;
|
|
26
|
+
return uniqueExisting([
|
|
27
|
+
moduleDir ? join(moduleDir, "..", "..", "extractors", script) : null,
|
|
28
|
+
moduleDir ? join(moduleDir, "..", "extractors", script) : null,
|
|
29
|
+
extensionDir ? join(extensionDir, "extractors", script) : null,
|
|
30
|
+
entryDir ? join(entryDir, "..", "extractors", script) : null
|
|
31
|
+
], exists);
|
|
32
|
+
}
|
|
33
|
+
export function resolveGreedySearchLaunchScript({
|
|
34
|
+
moduleDir,
|
|
35
|
+
entrypoint = process.argv[1],
|
|
36
|
+
env = process.env,
|
|
37
|
+
exists = existsSync
|
|
38
|
+
} = {}) {
|
|
39
|
+
const entryDir = entrypoint ? dirname(entrypoint) : null;
|
|
40
|
+
const extensionDir = env.GREEDY_SEARCH_EXTENSION_DIR?.trim() || null;
|
|
41
|
+
return uniqueExisting([
|
|
42
|
+
moduleDir ? join(moduleDir, "..", "..", "bin", "launch.mjs") : null,
|
|
43
|
+
moduleDir ? join(moduleDir, "launch.mjs") : null,
|
|
44
|
+
extensionDir ? join(extensionDir, "bin", "launch.mjs") : null,
|
|
45
|
+
entryDir ? join(entryDir, "launch.mjs") : null,
|
|
46
|
+
entryDir ? join(entryDir, "..", "bin", "launch.mjs") : null
|
|
47
|
+
], exists);
|
|
48
|
+
}
|
|
@@ -1,3 +1,64 @@
|
|
|
1
|
-
|
|
1
|
+
function ensurePdfDomPolyfills() {
|
|
2
|
+
if (typeof globalThis.DOMMatrix === "undefined") {
|
|
3
|
+
globalThis.DOMMatrix = class DOMMatrix {
|
|
4
|
+
constructor(_init = undefined) {}
|
|
5
|
+
multiplySelf() {
|
|
6
|
+
return this;
|
|
7
|
+
}
|
|
8
|
+
preMultiplySelf() {
|
|
9
|
+
return this;
|
|
10
|
+
}
|
|
11
|
+
translateSelf() {
|
|
12
|
+
return this;
|
|
13
|
+
}
|
|
14
|
+
scaleSelf() {
|
|
15
|
+
return this;
|
|
16
|
+
}
|
|
17
|
+
rotateSelf() {
|
|
18
|
+
return this;
|
|
19
|
+
}
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
if (typeof globalThis.ImageData === "undefined") {
|
|
23
|
+
globalThis.ImageData = class ImageData {
|
|
24
|
+
constructor(data = undefined, width = 0, height = 0) {
|
|
25
|
+
this.data = data;
|
|
26
|
+
this.width = width;
|
|
27
|
+
this.height = height;
|
|
28
|
+
}
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
if (typeof globalThis.Path2D === "undefined") {
|
|
32
|
+
globalThis.Path2D = class Path2D {
|
|
33
|
+
constructor(_path = undefined) {}
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
async function loadPdfParseCtor() {
|
|
38
|
+
ensurePdfDomPolyfills();
|
|
39
|
+
const mod = await import("pdf-parse");
|
|
40
|
+
const ctor = mod.PDFParse ?? mod.default;
|
|
41
|
+
if (!ctor)
|
|
42
|
+
throw new Error("pdf-parse did not export PDFParse");
|
|
43
|
+
return ctor;
|
|
44
|
+
}
|
|
45
|
+
export async function extractPdfMarkdown(buffer, url) {
|
|
46
|
+
try {
|
|
47
|
+
const PDFParseCtor = await loadPdfParseCtor();
|
|
48
|
+
const parser = new PDFParseCtor({ data: new Uint8Array(buffer) });
|
|
49
|
+
await parser.load();
|
|
50
|
+
const data = await parser.getText();
|
|
51
|
+
const text = data.text?.trim();
|
|
52
|
+
if (!text)
|
|
53
|
+
return null;
|
|
54
|
+
return {
|
|
55
|
+
title: new URL(url).pathname.split("/").pop() || "Document.pdf",
|
|
56
|
+
content: `## PDF Content (${data.total} pages)
|
|
2
57
|
|
|
3
|
-
${
|
|
58
|
+
${text}`,
|
|
59
|
+
pages: data.total
|
|
60
|
+
};
|
|
61
|
+
} catch (error) {
|
|
62
|
+
return { error: error.message || String(error) };
|
|
63
|
+
}
|
|
64
|
+
}
|
|
@@ -1 +1,69 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { execFileSync } from "node:child_process";
|
|
2
|
+
import { platform } from "node:os";
|
|
3
|
+
import { resolveSystemCmd } from "../utils/system-cmds.mjs";
|
|
4
|
+
function defaultRun(command, args) {
|
|
5
|
+
return execFileSync(resolveSystemCmd(command), args, {
|
|
6
|
+
encoding: "utf8",
|
|
7
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
8
|
+
timeout: 5000
|
|
9
|
+
});
|
|
10
|
+
}
|
|
11
|
+
function firstPid(text) {
|
|
12
|
+
const match = String(text || "").match(/\b(\d+)\b/);
|
|
13
|
+
return match ? Number.parseInt(match[1], 10) : null;
|
|
14
|
+
}
|
|
15
|
+
export function parseLsofPid(output) {
|
|
16
|
+
return firstPid(output);
|
|
17
|
+
}
|
|
18
|
+
export function parseSsPid(output, port) {
|
|
19
|
+
for (const line of String(output || "").split(/\r?\n/)) {
|
|
20
|
+
if (!/\bLISTEN\b/i.test(line))
|
|
21
|
+
continue;
|
|
22
|
+
const localFields = line.trim().split(/\s+/).slice(0, 6);
|
|
23
|
+
if (!localFields.some((field) => field.endsWith(`:${port}`)))
|
|
24
|
+
continue;
|
|
25
|
+
const match = line.match(/\bpid=(\d+)\b/);
|
|
26
|
+
if (match)
|
|
27
|
+
return Number.parseInt(match[1], 10);
|
|
28
|
+
}
|
|
29
|
+
return null;
|
|
30
|
+
}
|
|
31
|
+
export function parseNetstatPid(output, port) {
|
|
32
|
+
for (const line of String(output || "").split(/\r?\n/)) {
|
|
33
|
+
const fields = line.trim().split(/\s+/);
|
|
34
|
+
if (fields.length < 5 || fields[0].toUpperCase() !== "TCP")
|
|
35
|
+
continue;
|
|
36
|
+
if (!fields.at(-2)?.toUpperCase().startsWith("LISTEN"))
|
|
37
|
+
continue;
|
|
38
|
+
if (!fields[1].endsWith(`:${port}`))
|
|
39
|
+
continue;
|
|
40
|
+
const pid = Number.parseInt(fields.at(-1), 10);
|
|
41
|
+
if (Number.isInteger(pid) && pid > 0)
|
|
42
|
+
return pid;
|
|
43
|
+
}
|
|
44
|
+
return null;
|
|
45
|
+
}
|
|
46
|
+
function runQuiet(run, command, args) {
|
|
47
|
+
try {
|
|
48
|
+
return run(command, args);
|
|
49
|
+
} catch {
|
|
50
|
+
return null;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
export function findListeningProcessPid(port, { platformName = platform(), run = defaultRun } = {}) {
|
|
54
|
+
if (platformName === "win32") {
|
|
55
|
+
return parseNetstatPid(runQuiet(run, "netstat", ["-ano", "-p", "TCP"]), port);
|
|
56
|
+
}
|
|
57
|
+
const lsof = runQuiet(run, "lsof", [
|
|
58
|
+
"-nP",
|
|
59
|
+
`-iTCP:${port}`,
|
|
60
|
+
"-sTCP:LISTEN",
|
|
61
|
+
"-t"
|
|
62
|
+
]);
|
|
63
|
+
const lsofPid = parseLsofPid(lsof);
|
|
64
|
+
if (lsofPid)
|
|
65
|
+
return lsofPid;
|
|
66
|
+
if (platformName !== "linux")
|
|
67
|
+
return null;
|
|
68
|
+
return parseSsPid(runQuiet(run, "ss", ["-ltnp"]), port);
|
|
69
|
+
}
|
|
@@ -1,2 +1,109 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
const BAR_WIDTH = 20;
|
|
2
|
+
function formatDuration(ms) {
|
|
3
|
+
if (ms < 1000)
|
|
4
|
+
return "0s";
|
|
5
|
+
const totalSeconds = Math.round(ms / 1000);
|
|
6
|
+
if (totalSeconds < 60)
|
|
7
|
+
return `${totalSeconds}s`;
|
|
8
|
+
const minutes = Math.floor(totalSeconds / 60);
|
|
9
|
+
const seconds = totalSeconds % 60;
|
|
10
|
+
return `${minutes}m ${seconds}s`;
|
|
11
|
+
}
|
|
12
|
+
function renderBar(progress, width = BAR_WIDTH) {
|
|
13
|
+
const filled = Math.round(progress * width);
|
|
14
|
+
const empty = width - filled;
|
|
15
|
+
return "[" + "█".repeat(filled) + "░".repeat(empty) + "]";
|
|
16
|
+
}
|
|
17
|
+
export function createProgressTracker({
|
|
18
|
+
totalActions = 0,
|
|
19
|
+
totalRounds = 0,
|
|
20
|
+
totalFetches = 0,
|
|
21
|
+
silent = false
|
|
22
|
+
} = {}) {
|
|
23
|
+
const startedAt = Date.now();
|
|
24
|
+
let completedActions = 0;
|
|
25
|
+
let completedRounds = 0;
|
|
26
|
+
let completedFetches = 0;
|
|
27
|
+
const actionTimings = [];
|
|
28
|
+
let currentActionStart = null;
|
|
29
|
+
let currentActionLabel = null;
|
|
30
|
+
let lastPrintAt = 0;
|
|
31
|
+
const MIN_PRINT_INTERVAL_MS = 500;
|
|
32
|
+
function recordAction(durationMs) {
|
|
33
|
+
actionTimings.push(durationMs);
|
|
34
|
+
if (actionTimings.length > 5)
|
|
35
|
+
actionTimings.shift();
|
|
36
|
+
}
|
|
37
|
+
function avgActionMs() {
|
|
38
|
+
if (actionTimings.length === 0)
|
|
39
|
+
return null;
|
|
40
|
+
return actionTimings.reduce((a, b) => a + b, 0) / actionTimings.length;
|
|
41
|
+
}
|
|
42
|
+
function buildStatus(phase) {
|
|
43
|
+
const elapsed = Date.now() - startedAt;
|
|
44
|
+
const total = totalActions + totalFetches + totalRounds;
|
|
45
|
+
const done = completedActions + completedFetches + completedRounds;
|
|
46
|
+
const progress = total > 0 ? Math.min(1, done / total) : 0;
|
|
47
|
+
const bar = renderBar(progress);
|
|
48
|
+
const avg = avgActionMs();
|
|
49
|
+
const remaining = Math.max(0, total - done);
|
|
50
|
+
const etaMs = avg ? avg * remaining : null;
|
|
51
|
+
const eta = etaMs ? formatDuration(etaMs) : "—";
|
|
52
|
+
const label = currentActionLabel ? ` ${currentActionLabel}` : "";
|
|
53
|
+
return `${bar} ${done}/${total} (${phase}${label}, ETA ${eta})`;
|
|
54
|
+
}
|
|
55
|
+
function print(phase) {
|
|
56
|
+
if (silent)
|
|
57
|
+
return;
|
|
58
|
+
const now = Date.now();
|
|
59
|
+
if (now - lastPrintAt < MIN_PRINT_INTERVAL_MS && phase !== "done")
|
|
60
|
+
return;
|
|
61
|
+
lastPrintAt = now;
|
|
62
|
+
process.stderr.write(`[greedysearch] ${buildStatus(phase)}
|
|
63
|
+
`);
|
|
64
|
+
}
|
|
65
|
+
return {
|
|
66
|
+
startRound(n) {
|
|
67
|
+
completedRounds = n - 1;
|
|
68
|
+
},
|
|
69
|
+
endRound() {
|
|
70
|
+
completedRounds++;
|
|
71
|
+
print("round");
|
|
72
|
+
},
|
|
73
|
+
startAction(type, label) {
|
|
74
|
+
currentActionStart = Date.now();
|
|
75
|
+
currentActionLabel = `${type}:${(label || "").slice(0, 40)}`;
|
|
76
|
+
print(type);
|
|
77
|
+
},
|
|
78
|
+
endAction() {
|
|
79
|
+
if (currentActionStart) {
|
|
80
|
+
recordAction(Date.now() - currentActionStart);
|
|
81
|
+
currentActionStart = null;
|
|
82
|
+
}
|
|
83
|
+
completedActions++;
|
|
84
|
+
print("action");
|
|
85
|
+
},
|
|
86
|
+
startFetch(label) {
|
|
87
|
+
currentActionStart = Date.now();
|
|
88
|
+
currentActionLabel = `fetch:${(label || "").slice(0, 40)}`;
|
|
89
|
+
print("fetch");
|
|
90
|
+
},
|
|
91
|
+
endFetch(ok = true) {
|
|
92
|
+
if (currentActionStart) {
|
|
93
|
+
recordAction(Date.now() - currentActionStart);
|
|
94
|
+
currentActionStart = null;
|
|
95
|
+
}
|
|
96
|
+
completedFetches++;
|
|
97
|
+
print(ok ? "fetch" : "fetch-failed");
|
|
98
|
+
},
|
|
99
|
+
print() {
|
|
100
|
+
print("progress");
|
|
101
|
+
},
|
|
102
|
+
finish() {
|
|
103
|
+
print("done");
|
|
104
|
+
},
|
|
105
|
+
getElapsedMs() {
|
|
106
|
+
return Date.now() - startedAt;
|
|
107
|
+
}
|
|
108
|
+
};
|
|
109
|
+
}
|
|
@@ -1 +1,21 @@
|
|
|
1
|
-
|
|
1
|
+
const PREAMBLE_RX = /^(can you |could you |please |would you mind |i need to (know|understand) |i want to (know|understand) |i('m| am) (looking for|wondering about|curious about) |i need (information|info) (about|on) |tell me )?(about |explain |describe |give me |help me understand |search for |look up |find |research )?(about |regarding |on |for )?(it|this|the following)?\s*/i;
|
|
2
|
+
const TEMPORAL_RX = /\b(latest|newest|current|recent|up-to-date|up to date)\b/i;
|
|
3
|
+
const VERSION_RX = /\b\d+\.\d+|\bv\d+\b|\b20(2[0-9]|[3-9]\d)\b/i;
|
|
4
|
+
export function stripPreamble(query) {
|
|
5
|
+
const stripped = query.trim().replace(PREAMBLE_RX, "").trim();
|
|
6
|
+
return stripped.length > 4 ? stripped : query.trim();
|
|
7
|
+
}
|
|
8
|
+
export function addRecencyHint(query, year = new Date().getFullYear()) {
|
|
9
|
+
if (!TEMPORAL_RX.test(query))
|
|
10
|
+
return query;
|
|
11
|
+
if (VERSION_RX.test(query))
|
|
12
|
+
return query;
|
|
13
|
+
return `${query.trimEnd()} ${year}`;
|
|
14
|
+
}
|
|
15
|
+
export function normalizeQuery(query) {
|
|
16
|
+
if (!query?.trim())
|
|
17
|
+
return query;
|
|
18
|
+
let q = stripPreamble(query);
|
|
19
|
+
q = addRecencyHint(q);
|
|
20
|
+
return q || query;
|
|
21
|
+
}
|
|
@@ -1 +1,49 @@
|
|
|
1
|
-
|
|
1
|
+
export const HEADLESS_RECOVERY_ENGINES = [
|
|
2
|
+
"perplexity",
|
|
3
|
+
"bing",
|
|
4
|
+
"chatgpt",
|
|
5
|
+
"semantic-scholar",
|
|
6
|
+
"logically"
|
|
7
|
+
];
|
|
8
|
+
const NON_RECOVERABLE_BLOCKED_BY = new Set(["rate-limit"]);
|
|
9
|
+
const HEADLESS_BLOCKED_PATTERN = /timed out|timeout|verification|captcha|cloudflare|turnstile|input not found|ask-input|copy button hidden|sign.in|login required/i;
|
|
10
|
+
const MANUAL_VERIFICATION_PATTERN = /needs-human|verification required|please solve|captcha|cloudflare|turnstile|could not be completed automatically|manual intervention|sign.in|login required/i;
|
|
11
|
+
export function isHeadlessBlockedError(error) {
|
|
12
|
+
return HEADLESS_BLOCKED_PATTERN.test(String(error || ""));
|
|
13
|
+
}
|
|
14
|
+
export function isManualVerificationError(error) {
|
|
15
|
+
return MANUAL_VERIFICATION_PATTERN.test(String(error || ""));
|
|
16
|
+
}
|
|
17
|
+
export function isNonRecoverableBlockedBy(blockedBy) {
|
|
18
|
+
return NON_RECOVERABLE_BLOCKED_BY.has(blockedBy);
|
|
19
|
+
}
|
|
20
|
+
export function findHeadlessBlockedEngines(resultsByEngine) {
|
|
21
|
+
return HEADLESS_RECOVERY_ENGINES.filter((engine) => {
|
|
22
|
+
const result = resultsByEngine?.[engine];
|
|
23
|
+
if (!result)
|
|
24
|
+
return false;
|
|
25
|
+
const blockedBy = result._envelope?.blockedBy;
|
|
26
|
+
if (blockedBy) {
|
|
27
|
+
if (NON_RECOVERABLE_BLOCKED_BY.has(blockedBy))
|
|
28
|
+
return false;
|
|
29
|
+
return true;
|
|
30
|
+
}
|
|
31
|
+
if (result._envelope?.verificationResult === "needs-human")
|
|
32
|
+
return true;
|
|
33
|
+
const error = result.error;
|
|
34
|
+
return error && isHeadlessBlockedError(error);
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
export function isHeadlessBlockedResult(error) {
|
|
38
|
+
if (!error)
|
|
39
|
+
return false;
|
|
40
|
+
const env = error.envelope;
|
|
41
|
+
if (env?.blockedBy) {
|
|
42
|
+
if (NON_RECOVERABLE_BLOCKED_BY.has(env.blockedBy))
|
|
43
|
+
return false;
|
|
44
|
+
return true;
|
|
45
|
+
}
|
|
46
|
+
if (env?.verificationResult === "needs-human")
|
|
47
|
+
return true;
|
|
48
|
+
return isHeadlessBlockedError(error.message);
|
|
49
|
+
}
|