@duckmind/dm-windows-x64 0.63.0 → 0.63.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dm.exe +0 -0
- package/extensions/.dm-extensions.json +138 -2
- package/extensions/dm-web-access/SECURITY.md +5 -0
- package/extensions/dm-web-access/activity.js +65 -0
- package/extensions/dm-web-access/anysearch.js +158 -0
- package/extensions/dm-web-access/auth-fetch.js +131 -0
- package/extensions/dm-web-access/bocha.js +214 -0
- package/extensions/dm-web-access/brave.js +196 -0
- package/extensions/dm-web-access/brightdata-unlocker.js +202 -0
- package/extensions/dm-web-access/brightdata.js +334 -0
- package/extensions/dm-web-access/chrome-cookies.js +627 -0
- package/extensions/dm-web-access/content-find.js +114 -0
- package/extensions/dm-web-access/credential-source.js +150 -0
- package/extensions/dm-web-access/curator-page.js +3559 -0
- package/extensions/dm-web-access/curator-server.js +691 -0
- package/extensions/dm-web-access/data-uri-sanitize.js +312 -0
- package/extensions/dm-web-access/datalab-pdf-extract.js +346 -0
- package/extensions/dm-web-access/declared-web-links.js +167 -0
- package/extensions/dm-web-access/dm-web-fetch-demo.mp4 +0 -0
- package/extensions/dm-web-access/duckduckgo.js +118 -0
- package/extensions/dm-web-access/exa.js +401 -0
- package/extensions/dm-web-access/extract.js +1220 -0
- package/extensions/dm-web-access/feature-config.js +24 -0
- package/extensions/dm-web-access/fetch-params.js +81 -0
- package/extensions/dm-web-access/firecrawl.js +378 -0
- package/extensions/dm-web-access/gemini-adc.js +241 -0
- package/extensions/dm-web-access/gemini-api.js +258 -0
- package/extensions/dm-web-access/gemini-pdf-extract.js +74 -0
- package/extensions/dm-web-access/gemini-search.js +889 -0
- package/extensions/dm-web-access/gemini-url-context.js +97 -0
- package/extensions/dm-web-access/gemini-web-config.js +84 -0
- package/extensions/dm-web-access/gemini-web.js +351 -0
- package/extensions/dm-web-access/github-api.js +166 -0
- package/extensions/dm-web-access/github-extract.js +991 -0
- package/extensions/dm-web-access/github-issue-pr.js +750 -0
- package/extensions/dm-web-access/index.js +3117 -0
- package/extensions/dm-web-access/jina-search.js +242 -0
- package/extensions/dm-web-access/kagi.js +255 -0
- package/extensions/dm-web-access/kimi-search.js +214 -0
- package/extensions/dm-web-access/ollama.js +209 -0
- package/extensions/dm-web-access/openai-search.js +510 -0
- package/extensions/dm-web-access/package.json +34 -0
- package/extensions/dm-web-access/page-query.js +124 -0
- package/extensions/dm-web-access/parallel-mcp.js +223 -0
- package/extensions/dm-web-access/parallel.js +345 -0
- package/extensions/dm-web-access/pdf-extract.js +257 -0
- package/extensions/dm-web-access/perplexity.js +151 -0
- package/extensions/dm-web-access/querit.js +327 -0
- package/extensions/dm-web-access/query-rewrite.js +40 -0
- package/extensions/dm-web-access/render-search-error.js +80 -0
- package/extensions/dm-web-access/rsc-extract.js +347 -0
- package/extensions/dm-web-access/search1api.js +245 -0
- package/extensions/dm-web-access/searchinfinity.js +221 -0
- package/extensions/dm-web-access/searxng.js +223 -0
- package/extensions/dm-web-access/serpbase.js +205 -0
- package/extensions/dm-web-access/serpdive.js +238 -0
- package/extensions/dm-web-access/serper.js +200 -0
- package/extensions/dm-web-access/source-check.js +198 -0
- package/extensions/dm-web-access/ssrf-protection.js +436 -0
- package/extensions/dm-web-access/storage.js +451 -0
- package/extensions/dm-web-access/summary-model-scope.js +83 -0
- package/extensions/dm-web-access/summary-review.js +364 -0
- package/extensions/dm-web-access/tavily.js +199 -0
- package/extensions/dm-web-access/tinyfish.js +325 -0
- package/extensions/dm-web-access/utils.js +476 -0
- package/extensions/dm-web-access/valyu.js +189 -0
- package/extensions/dm-web-access/video-extract.js +336 -0
- package/extensions/dm-web-access/xai-search.js +285 -0
- package/extensions/dm-web-access/xcrawl.js +221 -0
- package/extensions/dm-web-access/youtube-extract.js +279 -0
- package/package.json +9 -1
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
import { activityMonitor } from "./activity.js";
|
|
2
|
+
import { CredentialResolutionError } from "./credential-source.js";
|
|
3
|
+
import { getApiKey, getVersionedApiBase, fetchGeminiApi, isGatewayConfigured, DEFAULT_MODEL } from "./gemini-api.js";
|
|
4
|
+
import { isGeminiAdcAvailable } from "./gemini-adc.js";
|
|
5
|
+
import { isGeminiWebAvailable, queryWithCookies } from "./gemini-web.js";
|
|
6
|
+
import { extractHeadingTitle } from "./extract.js";
|
|
7
|
+
const EXTRACTION_PROMPT = `Extract the complete readable content from this URL as clean markdown.
|
|
8
|
+
Include the page title, all text content, code blocks, and tables.
|
|
9
|
+
Do not summarize — extract the full content.
|
|
10
|
+
|
|
11
|
+
URL: `;
|
|
12
|
+
function shouldRethrow(err) {
|
|
13
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
14
|
+
return err instanceof CredentialResolutionError || message.startsWith("Failed to parse ");
|
|
15
|
+
}
|
|
16
|
+
export async function extractWithUrlContext(url, signal) {
|
|
17
|
+
const requestSignal = AbortSignal.any([
|
|
18
|
+
AbortSignal.timeout(60000),
|
|
19
|
+
...signal ? [signal] : []
|
|
20
|
+
]);
|
|
21
|
+
const apiKey = isGeminiAdcAvailable() ? null : await getApiKey(requestSignal);
|
|
22
|
+
if (!apiKey && !isGatewayConfigured() && !isGeminiAdcAvailable())
|
|
23
|
+
return null;
|
|
24
|
+
const activityId = activityMonitor.logStart({ type: "api", query: `url_context: ${url}` });
|
|
25
|
+
try {
|
|
26
|
+
const model = DEFAULT_MODEL;
|
|
27
|
+
const body = {
|
|
28
|
+
contents: [{ role: "user", parts: [{ text: EXTRACTION_PROMPT + url }] }],
|
|
29
|
+
tools: [{ url_context: {} }]
|
|
30
|
+
};
|
|
31
|
+
const res = await fetchGeminiApi(`${getVersionedApiBase()}/models/${model}:generateContent`, {
|
|
32
|
+
method: "POST",
|
|
33
|
+
headers: { "Content-Type": "application/json" },
|
|
34
|
+
body: JSON.stringify(body),
|
|
35
|
+
signal: requestSignal
|
|
36
|
+
}, apiKey);
|
|
37
|
+
if (!res.ok) {
|
|
38
|
+
activityMonitor.logComplete(activityId, res.status);
|
|
39
|
+
return null;
|
|
40
|
+
}
|
|
41
|
+
const data = await res.json();
|
|
42
|
+
activityMonitor.logComplete(activityId, res.status);
|
|
43
|
+
const metadata = data.candidates?.[0]?.url_context_metadata;
|
|
44
|
+
if (metadata?.url_metadata?.length) {
|
|
45
|
+
const status = metadata.url_metadata[0].url_retrieval_status;
|
|
46
|
+
if (status === "URL_RETRIEVAL_STATUS_UNSAFE" || status === "URL_RETRIEVAL_STATUS_ERROR") {
|
|
47
|
+
return null;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
const content = data.candidates?.[0]?.content?.parts?.map((p) => p.text).filter(Boolean).join(`
|
|
51
|
+
`) ?? "";
|
|
52
|
+
if (!content || content.length < 50)
|
|
53
|
+
return null;
|
|
54
|
+
const title = extractTitleFromContent(content, url);
|
|
55
|
+
return { url, title, content, error: null };
|
|
56
|
+
} catch (err) {
|
|
57
|
+
if (shouldRethrow(err))
|
|
58
|
+
throw err;
|
|
59
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
60
|
+
if (message.toLowerCase().includes("abort")) {
|
|
61
|
+
activityMonitor.logComplete(activityId, 0);
|
|
62
|
+
} else {
|
|
63
|
+
activityMonitor.logError(activityId, message);
|
|
64
|
+
}
|
|
65
|
+
return null;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
export async function extractWithGeminiWeb(url, signal) {
|
|
69
|
+
const cookies = await isGeminiWebAvailable();
|
|
70
|
+
if (!cookies)
|
|
71
|
+
return null;
|
|
72
|
+
const activityId = activityMonitor.logStart({ type: "api", query: `gemini_web: ${url}` });
|
|
73
|
+
try {
|
|
74
|
+
const text = await queryWithCookies(EXTRACTION_PROMPT + url, cookies, {
|
|
75
|
+
signal,
|
|
76
|
+
timeoutMs: 60000
|
|
77
|
+
});
|
|
78
|
+
activityMonitor.logComplete(activityId, 200);
|
|
79
|
+
if (!text || text.length < 50)
|
|
80
|
+
return null;
|
|
81
|
+
const title = extractTitleFromContent(text, url);
|
|
82
|
+
return { url, title, content: text, error: null };
|
|
83
|
+
} catch (err) {
|
|
84
|
+
if (shouldRethrow(err))
|
|
85
|
+
throw err;
|
|
86
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
87
|
+
if (message.toLowerCase().includes("abort")) {
|
|
88
|
+
activityMonitor.logComplete(activityId, 0);
|
|
89
|
+
} else {
|
|
90
|
+
activityMonitor.logError(activityId, message);
|
|
91
|
+
}
|
|
92
|
+
return null;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
function extractTitleFromContent(text, url) {
|
|
96
|
+
return extractHeadingTitle(text) ?? (new URL(url).pathname.split("/").pop() || url);
|
|
97
|
+
}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
2
|
+
import { getWebSearchConfigPath } from "./utils.js";
|
|
3
|
+
const CONFIG_PATH = getWebSearchConfigPath();
|
|
4
|
+
const BROWSER_COOKIE_PRESETS = ["helium", "chrome", "brave", "arc", "chromium", "edge"];
|
|
5
|
+
let cachedConfig = null;
|
|
6
|
+
export function normalizeChromeProfile(value) {
|
|
7
|
+
if (typeof value !== "string")
|
|
8
|
+
return;
|
|
9
|
+
const normalized = value.trim();
|
|
10
|
+
return normalized.length > 0 ? normalized : undefined;
|
|
11
|
+
}
|
|
12
|
+
function parseBrowserCookies(value) {
|
|
13
|
+
if (value === undefined)
|
|
14
|
+
return;
|
|
15
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
16
|
+
throw new Error(`browserCookies in ${CONFIG_PATH} must be an object`);
|
|
17
|
+
}
|
|
18
|
+
const raw = value;
|
|
19
|
+
if ("profilePath" in raw) {
|
|
20
|
+
throw new Error(`browserCookies.profilePath is not supported; use a browser preset and profile directory name in ${CONFIG_PATH}`);
|
|
21
|
+
}
|
|
22
|
+
let browser;
|
|
23
|
+
if (raw.browser !== undefined) {
|
|
24
|
+
if (typeof raw.browser !== "string")
|
|
25
|
+
throw new Error(`browserCookies.browser in ${CONFIG_PATH} must be a supported browser name`);
|
|
26
|
+
const normalized = raw.browser.trim().toLowerCase();
|
|
27
|
+
const preset = BROWSER_COOKIE_PRESETS.find((candidate) => candidate === normalized);
|
|
28
|
+
if (!preset) {
|
|
29
|
+
throw new Error(`Unsupported browserCookies.browser ${JSON.stringify(raw.browser)} in ${CONFIG_PATH}`);
|
|
30
|
+
}
|
|
31
|
+
browser = preset;
|
|
32
|
+
}
|
|
33
|
+
if (raw.profile !== undefined && typeof raw.profile !== "string") {
|
|
34
|
+
throw new Error(`browserCookies.profile in ${CONFIG_PATH} must be a profile directory name`);
|
|
35
|
+
}
|
|
36
|
+
const profile = normalizeChromeProfile(raw.profile);
|
|
37
|
+
if (profile && (profile === "." || profile === ".." || profile.includes("/") || profile.includes("\\"))) {
|
|
38
|
+
throw new Error(`browserCookies.profile in ${CONFIG_PATH} must be a profile directory name, not a path`);
|
|
39
|
+
}
|
|
40
|
+
return { ...browser ? { browser } : {}, ...profile ? { profile } : {} };
|
|
41
|
+
}
|
|
42
|
+
function loadConfig() {
|
|
43
|
+
if (cachedConfig)
|
|
44
|
+
return cachedConfig;
|
|
45
|
+
if (!existsSync(CONFIG_PATH)) {
|
|
46
|
+
cachedConfig = {};
|
|
47
|
+
return cachedConfig;
|
|
48
|
+
}
|
|
49
|
+
const rawText = readFileSync(CONFIG_PATH, "utf-8");
|
|
50
|
+
let raw;
|
|
51
|
+
try {
|
|
52
|
+
raw = JSON.parse(rawText);
|
|
53
|
+
} catch (err) {
|
|
54
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
55
|
+
throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
|
|
56
|
+
}
|
|
57
|
+
if (raw.chromeProfile !== undefined) {
|
|
58
|
+
throw new Error(`chromeProfile in ${CONFIG_PATH} is no longer supported; use browserCookies.profile`);
|
|
59
|
+
}
|
|
60
|
+
cachedConfig = {
|
|
61
|
+
browserCookies: parseBrowserCookies(raw.browserCookies),
|
|
62
|
+
allowBrowserCookies: raw.allowBrowserCookies === true
|
|
63
|
+
};
|
|
64
|
+
return cachedConfig;
|
|
65
|
+
}
|
|
66
|
+
export function getBrowserCookieSelectionFromConfig() {
|
|
67
|
+
return { ...loadConfig().browserCookies };
|
|
68
|
+
}
|
|
69
|
+
export function isBrowserCookieAccessAllowed() {
|
|
70
|
+
if (process.env.DM_ALLOW_BROWSER_COOKIES === "1" || process.env.FEYNMAN_ALLOW_BROWSER_COOKIES === "1") {
|
|
71
|
+
return true;
|
|
72
|
+
}
|
|
73
|
+
if (!existsSync(CONFIG_PATH))
|
|
74
|
+
return false;
|
|
75
|
+
const rawText = readFileSync(CONFIG_PATH, "utf-8");
|
|
76
|
+
let raw;
|
|
77
|
+
try {
|
|
78
|
+
raw = JSON.parse(rawText);
|
|
79
|
+
} catch (err) {
|
|
80
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
81
|
+
throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
|
|
82
|
+
}
|
|
83
|
+
return raw.allowBrowserCookies === true;
|
|
84
|
+
}
|
|
@@ -0,0 +1,351 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
import { basename } from "node:path";
|
|
3
|
+
import { getLastGoogleCookieDiagnostic, getLastGoogleCookieDiagnosticDetails, getGoogleCookies } from "./chrome-cookies.js";
|
|
4
|
+
import { getBrowserCookieSelectionFromConfig, isBrowserCookieAccessAllowed, normalizeChromeProfile } from "./gemini-web-config.js";
|
|
5
|
+
const GEMINI_APP_URL = "https://gemini.google.com/app";
|
|
6
|
+
const GEMINI_STREAM_GENERATE_URL = "https://gemini.google.com/_/BardChatUi/data/assistant.lamda.BardFrontendService/StreamGenerate";
|
|
7
|
+
const GEMINI_UPLOAD_URL = "https://content-push.googleapis.com/upload";
|
|
8
|
+
const GEMINI_UPLOAD_PUSH_ID = "feeds/mcudyrk2a4khkz";
|
|
9
|
+
const GOOGLE_LIST_ACCOUNTS_URL = "https://accounts.google.com/ListAccounts?gpsia=1&source=ChromiumBrowser&laf=b64bin&json=standard";
|
|
10
|
+
const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36";
|
|
11
|
+
const MODEL_HEADER_NAME = "x-goog-ext-525001261-jspb";
|
|
12
|
+
export const DEFAULT_GEMINI_WEB_MODEL = "gemini-3.1-pro";
|
|
13
|
+
const MODEL_HEADERS = {
|
|
14
|
+
[DEFAULT_GEMINI_WEB_MODEL]: '[1,null,null,null,"9d8ca3786ebdfbea",null,null,0,[4]]',
|
|
15
|
+
"gemini-2.5-pro": '[1,null,null,null,"4af6c7f5da75d65d",null,null,0,[4]]',
|
|
16
|
+
"gemini-2.5-flash": '[1,null,null,null,"9ec249fc9ad08861",null,null,0,[4]]'
|
|
17
|
+
};
|
|
18
|
+
const REQUIRED_COOKIES = ["__Secure-1PSID", "__Secure-1PSIDTS"];
|
|
19
|
+
let geminiFetchImpl = null;
|
|
20
|
+
export function setGeminiFetchOverrideForTests(fetchImpl) {
|
|
21
|
+
geminiFetchOverride = fetchImpl;
|
|
22
|
+
}
|
|
23
|
+
let geminiFetchOverride = null;
|
|
24
|
+
export const GEMINI_MAX_HEADER_SIZE = 4 * 1024 * 1024;
|
|
25
|
+
export function createGeminiFetch(undiciImpl) {
|
|
26
|
+
const agent = new undiciImpl.EnvHttpProxyAgent({
|
|
27
|
+
allowH2: false,
|
|
28
|
+
connectTimeout: 30000,
|
|
29
|
+
maxHeaderSize: GEMINI_MAX_HEADER_SIZE,
|
|
30
|
+
pipelining: 1
|
|
31
|
+
});
|
|
32
|
+
return (input, init) => undiciImpl.fetch(input, { ...init, dispatcher: agent });
|
|
33
|
+
}
|
|
34
|
+
export async function resolveGeminiFetch() {
|
|
35
|
+
if (geminiFetchOverride)
|
|
36
|
+
return geminiFetchOverride;
|
|
37
|
+
if (geminiFetchImpl)
|
|
38
|
+
return geminiFetchImpl;
|
|
39
|
+
let undici;
|
|
40
|
+
try {
|
|
41
|
+
undici = await import("undici");
|
|
42
|
+
} catch {
|
|
43
|
+
geminiFetchImpl = fetch;
|
|
44
|
+
return geminiFetchImpl;
|
|
45
|
+
}
|
|
46
|
+
geminiFetchImpl = createGeminiFetch(undici);
|
|
47
|
+
return geminiFetchImpl;
|
|
48
|
+
}
|
|
49
|
+
export async function isGeminiWebAvailable(chromeProfile) {
|
|
50
|
+
if (!isBrowserCookieAccessAllowed())
|
|
51
|
+
return null;
|
|
52
|
+
const configured = getBrowserCookieSelectionFromConfig();
|
|
53
|
+
const result = await getGoogleCookies({
|
|
54
|
+
browser: configured.browser,
|
|
55
|
+
profile: normalizeChromeProfile(chromeProfile) ?? configured.profile,
|
|
56
|
+
requiredCookies: REQUIRED_COOKIES
|
|
57
|
+
});
|
|
58
|
+
if (!result)
|
|
59
|
+
return null;
|
|
60
|
+
return result.cookies;
|
|
61
|
+
}
|
|
62
|
+
export function getGeminiWebAvailabilityDiagnostic() {
|
|
63
|
+
return isBrowserCookieAccessAllowed() ? getLastGoogleCookieDiagnostic() : null;
|
|
64
|
+
}
|
|
65
|
+
export function getGeminiWebAvailabilityDiagnosticDetails() {
|
|
66
|
+
return isBrowserCookieAccessAllowed() ? getLastGoogleCookieDiagnosticDetails() : null;
|
|
67
|
+
}
|
|
68
|
+
export async function getActiveGoogleEmail(cookies) {
|
|
69
|
+
const cookieHeader = buildCookieHeader(cookies);
|
|
70
|
+
if (!cookieHeader)
|
|
71
|
+
return null;
|
|
72
|
+
try {
|
|
73
|
+
const html = await fetchWithCookieRedirects(GEMINI_APP_URL, cookieHeader, 10, AbortSignal.timeout(1e4));
|
|
74
|
+
const email = extractEmailFromGeminiHtml(html);
|
|
75
|
+
if (email)
|
|
76
|
+
return email;
|
|
77
|
+
} catch {}
|
|
78
|
+
try {
|
|
79
|
+
const response = await fetchWithCookieRedirects(GOOGLE_LIST_ACCOUNTS_URL, cookieHeader, 10, AbortSignal.timeout(1e4));
|
|
80
|
+
return extractEmailFromListAccounts(response);
|
|
81
|
+
} catch {
|
|
82
|
+
return null;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
export async function queryWithCookies(prompt, cookieMap, options = {}) {
|
|
86
|
+
const model = options.model ?? DEFAULT_GEMINI_WEB_MODEL;
|
|
87
|
+
if (!MODEL_HEADERS[model]) {
|
|
88
|
+
throw new Error(`Gemini Web does not support model ${model}; configure Gemini API or choose a supported Gemini Web model.`);
|
|
89
|
+
}
|
|
90
|
+
const timeoutMs = options.timeoutMs ?? 120000;
|
|
91
|
+
let fullPrompt = prompt;
|
|
92
|
+
if (options.youtubeUrl) {
|
|
93
|
+
fullPrompt = `${fullPrompt}
|
|
94
|
+
|
|
95
|
+
YouTube video: ${options.youtubeUrl}`;
|
|
96
|
+
}
|
|
97
|
+
const result = await runGeminiWebOnce(fullPrompt, cookieMap, model, options.files, timeoutMs, options.signal);
|
|
98
|
+
if (result.errorMessage)
|
|
99
|
+
throw new Error(result.errorMessage);
|
|
100
|
+
if (!result.text)
|
|
101
|
+
throw new Error("Gemini Web returned empty response");
|
|
102
|
+
return result.text;
|
|
103
|
+
}
|
|
104
|
+
async function runGeminiWebOnce(prompt, cookieMap, model, files, timeoutMs, signal) {
|
|
105
|
+
const effectiveSignal = withTimeout(signal, timeoutMs);
|
|
106
|
+
const cookieHeader = buildCookieHeader(cookieMap);
|
|
107
|
+
const accessToken = await fetchAccessToken(cookieHeader, effectiveSignal);
|
|
108
|
+
const uploaded = [];
|
|
109
|
+
if (files) {
|
|
110
|
+
for (const filePath of files) {
|
|
111
|
+
uploaded.push(await uploadFile(filePath, cookieHeader, effectiveSignal));
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
const fReq = buildFReqPayload(prompt, uploaded);
|
|
115
|
+
const params = new URLSearchParams;
|
|
116
|
+
params.set("at", accessToken);
|
|
117
|
+
params.set("f.req", fReq);
|
|
118
|
+
const res = await (await resolveGeminiFetch())(GEMINI_STREAM_GENERATE_URL, {
|
|
119
|
+
method: "POST",
|
|
120
|
+
redirect: "error",
|
|
121
|
+
headers: {
|
|
122
|
+
"content-type": "application/x-www-form-urlencoded;charset=utf-8",
|
|
123
|
+
host: "gemini.google.com",
|
|
124
|
+
origin: "https://gemini.google.com",
|
|
125
|
+
referer: "https://gemini.google.com/",
|
|
126
|
+
"x-same-domain": "1",
|
|
127
|
+
"user-agent": USER_AGENT,
|
|
128
|
+
cookie: cookieHeader,
|
|
129
|
+
[MODEL_HEADER_NAME]: MODEL_HEADERS[model]
|
|
130
|
+
},
|
|
131
|
+
body: params.toString(),
|
|
132
|
+
signal: effectiveSignal
|
|
133
|
+
});
|
|
134
|
+
const rawText = await res.text();
|
|
135
|
+
if (!res.ok) {
|
|
136
|
+
return { text: "", errorMessage: `Gemini request failed: ${res.status}` };
|
|
137
|
+
}
|
|
138
|
+
try {
|
|
139
|
+
return parseStreamGenerateResponse(rawText);
|
|
140
|
+
} catch (err) {
|
|
141
|
+
let errorCode;
|
|
142
|
+
try {
|
|
143
|
+
const json = JSON.parse(trimJsonEnvelope(rawText));
|
|
144
|
+
errorCode = extractErrorCode(json);
|
|
145
|
+
} catch {}
|
|
146
|
+
return {
|
|
147
|
+
text: "",
|
|
148
|
+
errorCode,
|
|
149
|
+
errorMessage: err instanceof Error ? err.message : String(err)
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
async function fetchAccessToken(cookieHeader, signal) {
|
|
154
|
+
const html = await fetchWithCookieRedirects(GEMINI_APP_URL, cookieHeader, 10, signal);
|
|
155
|
+
for (const key of ["SNlM0e", "thykhd"]) {
|
|
156
|
+
const match = html.match(new RegExp(`"${key}":"(.*?)"`));
|
|
157
|
+
if (match?.[1])
|
|
158
|
+
return match[1];
|
|
159
|
+
}
|
|
160
|
+
throw new Error("Unable to authenticate with Gemini. Make sure you're signed into gemini.google.com in a supported Chromium-based browser.");
|
|
161
|
+
}
|
|
162
|
+
async function fetchWithCookieRedirects(url, cookieHeader, maxRedirects, signal) {
|
|
163
|
+
let current = url;
|
|
164
|
+
const allowedOrigin = new URL(url).origin;
|
|
165
|
+
for (let i = 0;i <= maxRedirects; i++) {
|
|
166
|
+
const res = await (await resolveGeminiFetch())(current, {
|
|
167
|
+
headers: { "user-agent": USER_AGENT, cookie: cookieHeader },
|
|
168
|
+
redirect: "manual",
|
|
169
|
+
signal
|
|
170
|
+
});
|
|
171
|
+
if (res.status >= 300 && res.status < 400) {
|
|
172
|
+
const location = res.headers.get("location");
|
|
173
|
+
if (location) {
|
|
174
|
+
const next = new URL(location, current);
|
|
175
|
+
if (next.origin !== allowedOrigin) {
|
|
176
|
+
throw new Error(`Refusing to send Google cookies across origins: ${allowedOrigin} -> ${next.origin}`);
|
|
177
|
+
}
|
|
178
|
+
current = next.toString();
|
|
179
|
+
continue;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
return await res.text();
|
|
183
|
+
}
|
|
184
|
+
throw new Error(`Too many redirects (>${maxRedirects})`);
|
|
185
|
+
}
|
|
186
|
+
function extractEmailFromGeminiHtml(html) {
|
|
187
|
+
const patterns = [
|
|
188
|
+
/"oPEP7c"\s*:\s*"([^"]+)"/,
|
|
189
|
+
/aria-label="Google Account:[^"]*?\(([^)]+)\)"/,
|
|
190
|
+
/"displayEmail"\s*:\s*"([^"]+)"/,
|
|
191
|
+
/"defaultEmail"\s*:\s*"([^"]+)"/,
|
|
192
|
+
/"email"\s*:\s*"([^"]+)"/,
|
|
193
|
+
/"identifier"\s*:\s*"([^"]+)"/,
|
|
194
|
+
/"gaiaIdentifier"\s*:\s*"([^"]+)"/
|
|
195
|
+
];
|
|
196
|
+
for (const pattern of patterns) {
|
|
197
|
+
const match = html.match(pattern);
|
|
198
|
+
const email = normalizeEmail(match?.[1]);
|
|
199
|
+
if (email)
|
|
200
|
+
return email;
|
|
201
|
+
}
|
|
202
|
+
return findFirstEmail(html);
|
|
203
|
+
}
|
|
204
|
+
function extractEmailFromListAccounts(text) {
|
|
205
|
+
const trimmed = text.replace(/^\)\]\}'\s*/, "");
|
|
206
|
+
try {
|
|
207
|
+
return findEmailInValue(JSON.parse(trimmed)) ?? findFirstEmail(trimmed);
|
|
208
|
+
} catch {
|
|
209
|
+
return findFirstEmail(trimmed);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
function findEmailInValue(value) {
|
|
213
|
+
if (typeof value === "string")
|
|
214
|
+
return normalizeEmail(value);
|
|
215
|
+
if (Array.isArray(value)) {
|
|
216
|
+
for (const item of value) {
|
|
217
|
+
const email = findEmailInValue(item);
|
|
218
|
+
if (email)
|
|
219
|
+
return email;
|
|
220
|
+
}
|
|
221
|
+
return null;
|
|
222
|
+
}
|
|
223
|
+
if (value && typeof value === "object") {
|
|
224
|
+
for (const item of Object.values(value)) {
|
|
225
|
+
const email = findEmailInValue(item);
|
|
226
|
+
if (email)
|
|
227
|
+
return email;
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
return null;
|
|
231
|
+
}
|
|
232
|
+
function findFirstEmail(text) {
|
|
233
|
+
const normalized = decodeEmailEscapes(text);
|
|
234
|
+
const match = normalized.match(/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/i);
|
|
235
|
+
return match?.[0] ?? null;
|
|
236
|
+
}
|
|
237
|
+
function normalizeEmail(value) {
|
|
238
|
+
if (!value)
|
|
239
|
+
return null;
|
|
240
|
+
const normalized = decodeEmailEscapes(value.trim());
|
|
241
|
+
return /^[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}$/i.test(normalized) ? normalized : null;
|
|
242
|
+
}
|
|
243
|
+
function decodeEmailEscapes(value) {
|
|
244
|
+
return value.replace(/\\u0040/gi, "@").replace(/\\x40/gi, "@").replace(/@/gi, "@").replace(/@/gi, "@").replace(/\\"/g, '"').replace(/\\\\/g, "\\");
|
|
245
|
+
}
|
|
246
|
+
async function uploadFile(filePath, cookieHeader, signal) {
|
|
247
|
+
const data = readFileSync(filePath);
|
|
248
|
+
const fileName = basename(filePath);
|
|
249
|
+
const boundary = "----FormBoundary" + Math.random().toString(36).slice(2);
|
|
250
|
+
const header = `--${boundary}\r
|
|
251
|
+
Content-Disposition: form-data; name="file"; filename="${fileName}"\r
|
|
252
|
+
Content-Type: application/octet-stream\r
|
|
253
|
+
\r
|
|
254
|
+
`;
|
|
255
|
+
const footer = `\r
|
|
256
|
+
--${boundary}--\r
|
|
257
|
+
`;
|
|
258
|
+
const body = Buffer.concat([
|
|
259
|
+
Buffer.from(header, "utf-8"),
|
|
260
|
+
data,
|
|
261
|
+
Buffer.from(footer, "utf-8")
|
|
262
|
+
]);
|
|
263
|
+
const res = await (await resolveGeminiFetch())(GEMINI_UPLOAD_URL, {
|
|
264
|
+
method: "POST",
|
|
265
|
+
redirect: "error",
|
|
266
|
+
headers: {
|
|
267
|
+
"content-type": `multipart/form-data; boundary=${boundary}`,
|
|
268
|
+
"push-id": GEMINI_UPLOAD_PUSH_ID,
|
|
269
|
+
"user-agent": USER_AGENT,
|
|
270
|
+
cookie: cookieHeader
|
|
271
|
+
},
|
|
272
|
+
body,
|
|
273
|
+
signal
|
|
274
|
+
});
|
|
275
|
+
if (!res.ok) {
|
|
276
|
+
const text = await res.text();
|
|
277
|
+
throw new Error(`File upload failed: ${res.status} (${text.slice(0, 200)})`);
|
|
278
|
+
}
|
|
279
|
+
return { id: await res.text(), name: fileName };
|
|
280
|
+
}
|
|
281
|
+
function buildFReqPayload(prompt, uploaded) {
|
|
282
|
+
const promptPayload = uploaded.length > 0 ? [prompt, 0, null, uploaded.map((file) => [[file.id, 1]])] : [prompt];
|
|
283
|
+
const innerList = [promptPayload, null, null];
|
|
284
|
+
return JSON.stringify([null, JSON.stringify(innerList)]);
|
|
285
|
+
}
|
|
286
|
+
function withTimeout(signal, timeoutMs) {
|
|
287
|
+
const timeout = AbortSignal.timeout(timeoutMs);
|
|
288
|
+
return signal ? AbortSignal.any([signal, timeout]) : timeout;
|
|
289
|
+
}
|
|
290
|
+
function buildCookieHeader(cookieMap) {
|
|
291
|
+
return Object.entries(cookieMap).filter(([, value]) => typeof value === "string" && value.length > 0).map(([name, value]) => `${name}=${value}`).join("; ");
|
|
292
|
+
}
|
|
293
|
+
function getNestedValue(value, pathParts) {
|
|
294
|
+
let current = value;
|
|
295
|
+
for (const part of pathParts) {
|
|
296
|
+
if (current == null)
|
|
297
|
+
return;
|
|
298
|
+
if (!Array.isArray(current))
|
|
299
|
+
return;
|
|
300
|
+
current = current[part];
|
|
301
|
+
}
|
|
302
|
+
return current;
|
|
303
|
+
}
|
|
304
|
+
function trimJsonEnvelope(text) {
|
|
305
|
+
const start = text.indexOf("[");
|
|
306
|
+
const end = text.lastIndexOf("]");
|
|
307
|
+
if (start === -1 || end === -1 || end <= start) {
|
|
308
|
+
throw new Error("Gemini response did not contain a JSON payload.");
|
|
309
|
+
}
|
|
310
|
+
return text.slice(start, end + 1);
|
|
311
|
+
}
|
|
312
|
+
function extractErrorCode(responseJson) {
|
|
313
|
+
const code = getNestedValue(responseJson, [0, 5, 2, 0, 1, 0]);
|
|
314
|
+
return typeof code === "number" && code >= 0 ? code : undefined;
|
|
315
|
+
}
|
|
316
|
+
function extractCandidateText(candidate) {
|
|
317
|
+
const textRaw = getNestedValue(candidate, [1, 0]);
|
|
318
|
+
let text = typeof textRaw === "string" ? textRaw : "";
|
|
319
|
+
if (/^http:\/\/googleusercontent\.com\/card_content\/\d+/.test(text)) {
|
|
320
|
+
const alt = getNestedValue(candidate, [22, 0]);
|
|
321
|
+
if (typeof alt === "string" && alt.length > 0)
|
|
322
|
+
text = alt;
|
|
323
|
+
}
|
|
324
|
+
return text;
|
|
325
|
+
}
|
|
326
|
+
function parseStreamGenerateResponse(rawText) {
|
|
327
|
+
const responseJson = JSON.parse(trimJsonEnvelope(rawText));
|
|
328
|
+
const errorCode = extractErrorCode(responseJson);
|
|
329
|
+
const parts = Array.isArray(responseJson) ? responseJson : [];
|
|
330
|
+
let firstCandidateSeen = undefined;
|
|
331
|
+
let latestNonEmptyText = "";
|
|
332
|
+
for (let i = 0;i < parts.length; i++) {
|
|
333
|
+
const partBody = getNestedValue(parts[i], [2]);
|
|
334
|
+
if (!partBody || typeof partBody !== "string")
|
|
335
|
+
continue;
|
|
336
|
+
try {
|
|
337
|
+
const parsed = JSON.parse(partBody);
|
|
338
|
+
const candidateList = getNestedValue(parsed, [4]);
|
|
339
|
+
if (!Array.isArray(candidateList) || candidateList.length === 0)
|
|
340
|
+
continue;
|
|
341
|
+
const firstCandidate = candidateList[0];
|
|
342
|
+
if (firstCandidateSeen === undefined)
|
|
343
|
+
firstCandidateSeen = firstCandidate;
|
|
344
|
+
const text = extractCandidateText(firstCandidate);
|
|
345
|
+
if (text.length > 0)
|
|
346
|
+
latestNonEmptyText = text;
|
|
347
|
+
} catch {}
|
|
348
|
+
}
|
|
349
|
+
const text = latestNonEmptyText.length > 0 ? latestNonEmptyText : extractCandidateText(firstCandidateSeen);
|
|
350
|
+
return { text, errorCode };
|
|
351
|
+
}
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
import { execFile } from "node:child_process";
|
|
2
|
+
const MAX_TREE_ENTRIES = 200;
|
|
3
|
+
const MAX_INLINE_FILE_CHARS = 1e5;
|
|
4
|
+
let ghAvailable = null;
|
|
5
|
+
let ghHintShown = false;
|
|
6
|
+
export async function checkGhAvailable(signal) {
|
|
7
|
+
if (ghAvailable !== null)
|
|
8
|
+
return ghAvailable;
|
|
9
|
+
return new Promise((resolve) => {
|
|
10
|
+
execFile("gh", ["--version"], { timeout: 5000, ...signal ? { signal } : {} }, (err) => {
|
|
11
|
+
const available = !err;
|
|
12
|
+
if (!signal?.aborted)
|
|
13
|
+
ghAvailable = available;
|
|
14
|
+
resolve(available);
|
|
15
|
+
});
|
|
16
|
+
});
|
|
17
|
+
}
|
|
18
|
+
export function showGhHint() {
|
|
19
|
+
if (!ghHintShown) {
|
|
20
|
+
ghHintShown = true;
|
|
21
|
+
console.error("[dm-web-access] Install `gh` CLI for better GitHub repo access including private repos.");
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
export async function checkRepoSize(owner, repo) {
|
|
25
|
+
if (!await checkGhAvailable())
|
|
26
|
+
return null;
|
|
27
|
+
return new Promise((resolve) => {
|
|
28
|
+
execFile("gh", ["api", `repos/${owner}/${repo}`, "--jq", ".size"], { timeout: 1e4 }, (err, stdout) => {
|
|
29
|
+
if (err) {
|
|
30
|
+
resolve(null);
|
|
31
|
+
return;
|
|
32
|
+
}
|
|
33
|
+
const kb = parseInt(stdout.trim(), 10);
|
|
34
|
+
resolve(Number.isNaN(kb) ? null : kb);
|
|
35
|
+
});
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
async function getDefaultBranch(owner, repo) {
|
|
39
|
+
if (!await checkGhAvailable())
|
|
40
|
+
return null;
|
|
41
|
+
return new Promise((resolve) => {
|
|
42
|
+
execFile("gh", ["api", `repos/${owner}/${repo}`, "--jq", ".default_branch"], { timeout: 1e4 }, (err, stdout) => {
|
|
43
|
+
if (err) {
|
|
44
|
+
resolve(null);
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
const branch = stdout.trim();
|
|
48
|
+
resolve(branch || null);
|
|
49
|
+
});
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
async function fetchTreeViaApi(owner, repo, ref) {
|
|
53
|
+
if (!await checkGhAvailable())
|
|
54
|
+
return null;
|
|
55
|
+
return new Promise((resolve) => {
|
|
56
|
+
execFile("gh", ["api", `repos/${owner}/${repo}/git/trees/${ref}?recursive=1`, "--jq", ".tree[].path"], { timeout: 15000, maxBuffer: 5 * 1024 * 1024 }, (err, stdout) => {
|
|
57
|
+
if (err) {
|
|
58
|
+
resolve(null);
|
|
59
|
+
return;
|
|
60
|
+
}
|
|
61
|
+
const paths = stdout.trim().split(`
|
|
62
|
+
`).filter(Boolean);
|
|
63
|
+
if (paths.length === 0) {
|
|
64
|
+
resolve(null);
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
const truncated = paths.length > MAX_TREE_ENTRIES;
|
|
68
|
+
const display = paths.slice(0, MAX_TREE_ENTRIES).join(`
|
|
69
|
+
`);
|
|
70
|
+
resolve(truncated ? display + `
|
|
71
|
+
... (${paths.length} total entries)` : display);
|
|
72
|
+
});
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
async function fetchReadmeViaApi(owner, repo, ref) {
|
|
76
|
+
if (!await checkGhAvailable())
|
|
77
|
+
return null;
|
|
78
|
+
return new Promise((resolve) => {
|
|
79
|
+
execFile("gh", ["api", `repos/${owner}/${repo}/readme?ref=${ref}`, "--jq", ".content"], { timeout: 1e4 }, (err, stdout) => {
|
|
80
|
+
if (err) {
|
|
81
|
+
resolve(null);
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
try {
|
|
85
|
+
const decoded = Buffer.from(stdout.trim(), "base64").toString("utf-8");
|
|
86
|
+
resolve(decoded.length > 8192 ? decoded.slice(0, 8192) + `
|
|
87
|
+
|
|
88
|
+
[README truncated at 8K chars]` : decoded);
|
|
89
|
+
} catch {
|
|
90
|
+
resolve(null);
|
|
91
|
+
}
|
|
92
|
+
});
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
async function fetchFileViaApi(owner, repo, path, ref) {
|
|
96
|
+
if (!await checkGhAvailable())
|
|
97
|
+
return null;
|
|
98
|
+
return new Promise((resolve) => {
|
|
99
|
+
execFile("gh", ["api", `repos/${owner}/${repo}/contents/${path}?ref=${ref}`, "--jq", ".content"], { timeout: 1e4, maxBuffer: 2 * 1024 * 1024 }, (err, stdout) => {
|
|
100
|
+
if (err) {
|
|
101
|
+
resolve(null);
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
try {
|
|
105
|
+
resolve(Buffer.from(stdout.trim(), "base64").toString("utf-8"));
|
|
106
|
+
} catch {
|
|
107
|
+
resolve(null);
|
|
108
|
+
}
|
|
109
|
+
});
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
export async function fetchViaApi(url, owner, repo, info, sizeNote) {
|
|
113
|
+
const ref = info.ref || await getDefaultBranch(owner, repo);
|
|
114
|
+
if (!ref)
|
|
115
|
+
return null;
|
|
116
|
+
const lines = [];
|
|
117
|
+
if (sizeNote) {
|
|
118
|
+
lines.push(sizeNote);
|
|
119
|
+
lines.push("");
|
|
120
|
+
}
|
|
121
|
+
if (info.type === "blob" && info.path) {
|
|
122
|
+
const content = await fetchFileViaApi(owner, repo, info.path, ref);
|
|
123
|
+
if (!content)
|
|
124
|
+
return null;
|
|
125
|
+
lines.push(`## ${info.path}`);
|
|
126
|
+
if (content.length > MAX_INLINE_FILE_CHARS) {
|
|
127
|
+
lines.push(content.slice(0, MAX_INLINE_FILE_CHARS));
|
|
128
|
+
lines.push(`
|
|
129
|
+
[File truncated at 100K chars]`);
|
|
130
|
+
} else {
|
|
131
|
+
lines.push(content);
|
|
132
|
+
}
|
|
133
|
+
return {
|
|
134
|
+
url,
|
|
135
|
+
title: `${owner}/${repo} - ${info.path}`,
|
|
136
|
+
content: lines.join(`
|
|
137
|
+
`),
|
|
138
|
+
error: null
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
const [tree, readme] = await Promise.all([
|
|
142
|
+
fetchTreeViaApi(owner, repo, ref),
|
|
143
|
+
fetchReadmeViaApi(owner, repo, ref)
|
|
144
|
+
]);
|
|
145
|
+
if (!tree && !readme)
|
|
146
|
+
return null;
|
|
147
|
+
if (tree) {
|
|
148
|
+
lines.push("## Structure");
|
|
149
|
+
lines.push(tree);
|
|
150
|
+
lines.push("");
|
|
151
|
+
}
|
|
152
|
+
if (readme) {
|
|
153
|
+
lines.push("## README.md");
|
|
154
|
+
lines.push(readme);
|
|
155
|
+
lines.push("");
|
|
156
|
+
}
|
|
157
|
+
lines.push("This is an API-only view. Clone the repo or use `read`/`bash` for deeper exploration.");
|
|
158
|
+
const title = info.path ? `${owner}/${repo} - ${info.path}` : `${owner}/${repo}`;
|
|
159
|
+
return {
|
|
160
|
+
url,
|
|
161
|
+
title,
|
|
162
|
+
content: lines.join(`
|
|
163
|
+
`),
|
|
164
|
+
error: null
|
|
165
|
+
};
|
|
166
|
+
}
|