@duckmind/dm-windows-x64 0.63.0 → 0.63.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. package/dm.exe +0 -0
  2. package/extensions/.dm-extensions.json +200 -4
  3. package/extensions/dm-context/package.json +1 -1
  4. package/extensions/dm-context/skills/context-management/SKILL.md +173 -223
  5. package/extensions/dm-context/skills/context-management/references/development-and-troubleshooting.md +75 -0
  6. package/extensions/dm-context/skills/context-management/references/interleaved-async-work.md +143 -0
  7. package/extensions/dm-context/skills/context-management/references/planning-and-execution.md +81 -0
  8. package/extensions/dm-context/skills/context-management/references/repeated-items-and-batch-work.md +69 -0
  9. package/extensions/dm-context/skills/context-management/references/retry-branch-and-pivot.md +80 -0
  10. package/extensions/dm-context/skills/context-management/references/search-research-and-reading.md +103 -0
  11. package/extensions/dm-context/skills/context-management/references/task-switching-and-cleanup.md +73 -0
  12. package/extensions/dm-context/src/context.js +3 -2
  13. package/extensions/dm-context/src/index.js +168 -84
  14. package/extensions/dm-skills-manager/THIRD_PARTY_NOTICES.md +27 -0
  15. package/extensions/dm-skills-manager/extensions/skills-manager/components.js +265 -0
  16. package/extensions/dm-skills-manager/extensions/skills-manager/constants.js +31 -0
  17. package/extensions/dm-skills-manager/extensions/skills-manager/creation-fallback.js +20 -0
  18. package/extensions/dm-skills-manager/extensions/skills-manager/creation.js +145 -0
  19. package/extensions/dm-skills-manager/extensions/skills-manager/dialog.js +738 -0
  20. package/extensions/dm-skills-manager/extensions/skills-manager/dm-ai-compat.js +15 -0
  21. package/extensions/dm-skills-manager/extensions/skills-manager/format.js +125 -0
  22. package/extensions/dm-skills-manager/extensions/skills-manager/glyphs.js +142 -0
  23. package/extensions/dm-skills-manager/extensions/skills-manager/layout.js +93 -0
  24. package/extensions/dm-skills-manager/extensions/skills-manager/paths.js +76 -0
  25. package/extensions/dm-skills-manager/extensions/skills-manager/registry.js +95 -0
  26. package/extensions/dm-skills-manager/extensions/skills-manager/settings.js +93 -0
  27. package/extensions/dm-skills-manager/extensions/skills-manager/startup.js +32 -0
  28. package/extensions/dm-skills-manager/extensions/skills-manager/toggle.js +54 -0
  29. package/extensions/dm-skills-manager/extensions/skills-manager/types.js +14 -0
  30. package/extensions/dm-skills-manager/extensions/skills-manager/ui.js +121 -0
  31. package/extensions/dm-skills-manager/extensions/skills-manager.js +111 -0
  32. package/extensions/dm-skills-manager/package.json +121 -0
  33. package/extensions/dm-web-access/SECURITY.md +5 -0
  34. package/extensions/dm-web-access/activity.js +65 -0
  35. package/extensions/dm-web-access/anysearch.js +158 -0
  36. package/extensions/dm-web-access/auth-fetch.js +131 -0
  37. package/extensions/dm-web-access/bocha.js +214 -0
  38. package/extensions/dm-web-access/brave.js +196 -0
  39. package/extensions/dm-web-access/brightdata-unlocker.js +202 -0
  40. package/extensions/dm-web-access/brightdata.js +334 -0
  41. package/extensions/dm-web-access/chrome-cookies.js +627 -0
  42. package/extensions/dm-web-access/content-find.js +114 -0
  43. package/extensions/dm-web-access/credential-source.js +150 -0
  44. package/extensions/dm-web-access/curator-page.js +3559 -0
  45. package/extensions/dm-web-access/curator-server.js +691 -0
  46. package/extensions/dm-web-access/data-uri-sanitize.js +312 -0
  47. package/extensions/dm-web-access/datalab-pdf-extract.js +346 -0
  48. package/extensions/dm-web-access/declared-web-links.js +167 -0
  49. package/extensions/dm-web-access/dm-web-fetch-demo.mp4 +0 -0
  50. package/extensions/dm-web-access/duckduckgo.js +118 -0
  51. package/extensions/dm-web-access/exa.js +401 -0
  52. package/extensions/dm-web-access/extract.js +1220 -0
  53. package/extensions/dm-web-access/feature-config.js +24 -0
  54. package/extensions/dm-web-access/fetch-params.js +81 -0
  55. package/extensions/dm-web-access/firecrawl.js +378 -0
  56. package/extensions/dm-web-access/gemini-adc.js +241 -0
  57. package/extensions/dm-web-access/gemini-api.js +258 -0
  58. package/extensions/dm-web-access/gemini-pdf-extract.js +74 -0
  59. package/extensions/dm-web-access/gemini-search.js +889 -0
  60. package/extensions/dm-web-access/gemini-url-context.js +97 -0
  61. package/extensions/dm-web-access/gemini-web-config.js +84 -0
  62. package/extensions/dm-web-access/gemini-web.js +351 -0
  63. package/extensions/dm-web-access/github-api.js +166 -0
  64. package/extensions/dm-web-access/github-extract.js +991 -0
  65. package/extensions/dm-web-access/github-issue-pr.js +750 -0
  66. package/extensions/dm-web-access/index.js +3117 -0
  67. package/extensions/dm-web-access/jina-search.js +242 -0
  68. package/extensions/dm-web-access/kagi.js +255 -0
  69. package/extensions/dm-web-access/kimi-search.js +214 -0
  70. package/extensions/dm-web-access/ollama.js +209 -0
  71. package/extensions/dm-web-access/openai-search.js +510 -0
  72. package/extensions/dm-web-access/package.json +34 -0
  73. package/extensions/dm-web-access/page-query.js +124 -0
  74. package/extensions/dm-web-access/parallel-mcp.js +223 -0
  75. package/extensions/dm-web-access/parallel.js +345 -0
  76. package/extensions/dm-web-access/pdf-extract.js +257 -0
  77. package/extensions/dm-web-access/perplexity.js +151 -0
  78. package/extensions/dm-web-access/querit.js +327 -0
  79. package/extensions/dm-web-access/query-rewrite.js +40 -0
  80. package/extensions/dm-web-access/render-search-error.js +80 -0
  81. package/extensions/dm-web-access/rsc-extract.js +347 -0
  82. package/extensions/dm-web-access/search1api.js +245 -0
  83. package/extensions/dm-web-access/searchinfinity.js +221 -0
  84. package/extensions/dm-web-access/searxng.js +223 -0
  85. package/extensions/dm-web-access/serpbase.js +205 -0
  86. package/extensions/dm-web-access/serpdive.js +238 -0
  87. package/extensions/dm-web-access/serper.js +200 -0
  88. package/extensions/dm-web-access/source-check.js +198 -0
  89. package/extensions/dm-web-access/ssrf-protection.js +436 -0
  90. package/extensions/dm-web-access/storage.js +451 -0
  91. package/extensions/dm-web-access/summary-model-scope.js +83 -0
  92. package/extensions/dm-web-access/summary-review.js +364 -0
  93. package/extensions/dm-web-access/tavily.js +199 -0
  94. package/extensions/dm-web-access/tinyfish.js +325 -0
  95. package/extensions/dm-web-access/utils.js +476 -0
  96. package/extensions/dm-web-access/valyu.js +189 -0
  97. package/extensions/dm-web-access/video-extract.js +336 -0
  98. package/extensions/dm-web-access/xai-search.js +285 -0
  99. package/extensions/dm-web-access/xcrawl.js +221 -0
  100. package/extensions/dm-web-access/youtube-extract.js +279 -0
  101. package/package.json +9 -1
@@ -0,0 +1,312 @@
1
+ import { Buffer } from "node:buffer";
2
+ import { createHash } from "node:crypto";
3
+ const MAX_DATA_URI_HEADER_CHARS = 1024;
4
+ const MAX_MIME_CHARS = 127;
5
+ const MAX_MARKER_SOURCE_CHARS = 180;
6
+ function sha256Bytes(value) {
7
+ return createHash("sha256").update(value).digest("hex");
8
+ }
9
+ function asciiLowerCode(code) {
10
+ return code >= 65 && code <= 90 ? code + 32 : code;
11
+ }
12
+ function matchesDataScheme(text, index) {
13
+ return index + 5 <= text.length && asciiLowerCode(text.charCodeAt(index)) === 100 && asciiLowerCode(text.charCodeAt(index + 1)) === 97 && asciiLowerCode(text.charCodeAt(index + 2)) === 116 && asciiLowerCode(text.charCodeAt(index + 3)) === 97 && text.charCodeAt(index + 4) === 58;
14
+ }
15
+ function isSchemeBoundary(text, index) {
16
+ if (index === 0)
17
+ return true;
18
+ const code = text.charCodeAt(index - 1);
19
+ const alphaNumeric = code >= 48 && code <= 57 || code >= 65 && code <= 90 || code >= 97 && code <= 122;
20
+ return !alphaNumeric && code !== 43 && code !== 45 && code !== 46;
21
+ }
22
+ function findNextDataScheme(text, from) {
23
+ for (let i = from;i + 5 <= text.length; i++) {
24
+ if (matchesDataScheme(text, i) && isSchemeBoundary(text, i))
25
+ return i;
26
+ }
27
+ return -1;
28
+ }
29
+ function dataUriEnclosure(text, start) {
30
+ if (start === 0)
31
+ return null;
32
+ const immediateIndex = start - 1;
33
+ const immediate = text[immediateIndex];
34
+ if (immediate === '"' || immediate === "'" || immediate === "`") {
35
+ const beforeQuote = immediateIndex > 0 ? text[immediateIndex - 1] : "";
36
+ const likelyOpeningQuote = immediateIndex === 0 || beforeQuote === "=" || beforeQuote === "(" || beforeQuote === "[" || beforeQuote === "{" || beforeQuote === ":" || beforeQuote === "," || /\s/.test(beforeQuote);
37
+ if (likelyOpeningQuote)
38
+ return { kind: "quote", close: immediate };
39
+ }
40
+ let openerIndex = immediateIndex;
41
+ while (openerIndex >= 0 && (text[openerIndex] === " " || text[openerIndex] === "\t"))
42
+ openerIndex--;
43
+ const opener = text[openerIndex];
44
+ if (opener === "(")
45
+ return { kind: "parentheses" };
46
+ if (opener === "<")
47
+ return { kind: "angle" };
48
+ return null;
49
+ }
50
+ function isBareDataUriTerminator(ch) {
51
+ const code = ch.charCodeAt(0);
52
+ return code <= 32 || code === 127 || ch === '"' || ch === "`" || ch === "<" || ch === ">";
53
+ }
54
+ function scanDataUriCandidate(text, start) {
55
+ const enclosure = dataUriEnclosure(text, start);
56
+ let comma = -1;
57
+ let depth = enclosure?.kind === "parentheses" ? 1 : 0;
58
+ let index = start + 5;
59
+ for (;index < text.length; index++) {
60
+ const ch = text[index];
61
+ if (enclosure?.kind === "quote") {
62
+ if (ch === enclosure.close)
63
+ break;
64
+ } else if (enclosure?.kind === "parentheses") {
65
+ if (ch === "(") {
66
+ depth++;
67
+ } else if (ch === ")") {
68
+ depth--;
69
+ if (depth === 0)
70
+ break;
71
+ }
72
+ } else if (enclosure?.kind === "angle") {
73
+ if (ch === ">")
74
+ break;
75
+ } else if (isBareDataUriTerminator(ch)) {
76
+ break;
77
+ }
78
+ if (comma < 0 && ch === ",")
79
+ comma = index;
80
+ }
81
+ return { end: index, comma, enclosure };
82
+ }
83
+ function headerEndsWithBase64(text, headerStart, comma) {
84
+ let end = comma;
85
+ while (end > headerStart && (text[end - 1] === " " || text[end - 1] === "\t"))
86
+ end--;
87
+ const token = "base64";
88
+ if (end - headerStart < token.length + 1)
89
+ return false;
90
+ const tokenStart = end - token.length;
91
+ for (let i = 0;i < token.length; i++) {
92
+ if (asciiLowerCode(text.charCodeAt(tokenStart + i)) !== token.charCodeAt(i))
93
+ return false;
94
+ }
95
+ return tokenStart > headerStart && text[tokenStart - 1] === ";";
96
+ }
97
+ function isMimeTokenChar(code) {
98
+ if (code < 33 || code > 126)
99
+ return false;
100
+ switch (code) {
101
+ case 34:
102
+ case 40:
103
+ case 41:
104
+ case 44:
105
+ case 47:
106
+ case 58:
107
+ case 59:
108
+ case 60:
109
+ case 61:
110
+ case 62:
111
+ case 63:
112
+ case 64:
113
+ case 91:
114
+ case 92:
115
+ case 93:
116
+ return false;
117
+ default:
118
+ return true;
119
+ }
120
+ }
121
+ function normalizeMimeType(text, headerStart, comma) {
122
+ let end = headerStart;
123
+ while (end < comma && text[end] !== ";")
124
+ end++;
125
+ if (end === headerStart)
126
+ return "text/plain";
127
+ if (end - headerStart > MAX_MIME_CHARS)
128
+ return "application/octet-stream";
129
+ let slashCount = 0;
130
+ for (let i = headerStart;i < end; i++) {
131
+ const code = text.charCodeAt(i);
132
+ if (code === 47) {
133
+ slashCount++;
134
+ if (i === headerStart || i === end - 1)
135
+ return "application/octet-stream";
136
+ continue;
137
+ }
138
+ if (!isMimeTokenChar(code))
139
+ return "application/octet-stream";
140
+ }
141
+ if (slashCount !== 1)
142
+ return "application/octet-stream";
143
+ return text.slice(headerStart, end).toLowerCase();
144
+ }
145
+ function hexValue(code) {
146
+ if (code >= 48 && code <= 57)
147
+ return code - 48;
148
+ if (code >= 65 && code <= 70)
149
+ return code - 55;
150
+ if (code >= 97 && code <= 102)
151
+ return code - 87;
152
+ return -1;
153
+ }
154
+ function decodePercentEncoded(payload) {
155
+ const output = Buffer.allocUnsafe(Buffer.byteLength(payload, "utf8"));
156
+ let outputOffset = 0;
157
+ let cursor = 0;
158
+ while (cursor < payload.length) {
159
+ const percent = payload.indexOf("%", cursor);
160
+ const runEnd = percent < 0 ? payload.length : percent;
161
+ if (runEnd > cursor) {
162
+ const run = Buffer.from(payload.slice(cursor, runEnd), "utf8");
163
+ run.copy(output, outputOffset);
164
+ outputOffset += run.length;
165
+ }
166
+ if (percent < 0)
167
+ break;
168
+ if (percent + 2 >= payload.length)
169
+ return { error: "invalid-percent-escape" };
170
+ const high = hexValue(payload.charCodeAt(percent + 1));
171
+ const low = hexValue(payload.charCodeAt(percent + 2));
172
+ if (high < 0 || low < 0)
173
+ return { error: "invalid-percent-escape" };
174
+ output[outputOffset++] = high << 4 | low;
175
+ cursor = percent + 3;
176
+ }
177
+ return { bytes: output.subarray(0, outputOffset) };
178
+ }
179
+ function isBase64AlphabetByte(byte) {
180
+ return byte >= 65 && byte <= 90 || byte >= 97 && byte <= 122 || byte >= 48 && byte <= 57 || byte === 43 || byte === 47;
181
+ }
182
+ function decodeBase64Payload(payload) {
183
+ const percentDecoded = decodePercentEncoded(payload);
184
+ if ("error" in percentDecoded)
185
+ return percentDecoded;
186
+ const encoded = percentDecoded.bytes;
187
+ let paddingStart = encoded.length;
188
+ let paddingCount = 0;
189
+ for (let i = 0;i < encoded.length; i++) {
190
+ const byte = encoded[i];
191
+ if (byte > 127)
192
+ return { error: "non-ascii-base64" };
193
+ if (byte === 61) {
194
+ if (paddingStart === encoded.length)
195
+ paddingStart = i;
196
+ paddingCount++;
197
+ continue;
198
+ }
199
+ if (paddingStart !== encoded.length)
200
+ return { error: "invalid-base64-padding" };
201
+ if (!isBase64AlphabetByte(byte))
202
+ return { error: "invalid-base64-character" };
203
+ }
204
+ if (paddingCount > 2)
205
+ return { error: "invalid-base64-padding" };
206
+ if (paddingCount > 0 && encoded.length % 4 !== 0)
207
+ return { error: "invalid-base64-padding" };
208
+ if (paddingCount === 0 && encoded.length % 4 === 1)
209
+ return { error: "invalid-base64-length" };
210
+ if (paddingCount === 1 && paddingStart % 4 !== 3)
211
+ return { error: "invalid-base64-padding" };
212
+ if (paddingCount === 2 && paddingStart % 4 !== 2)
213
+ return { error: "invalid-base64-padding" };
214
+ return { bytes: Buffer.from(encoded.toString("ascii"), "base64") };
215
+ }
216
+ function safeMarkerSource(sourcePath) {
217
+ const source = sourcePath || "value";
218
+ let safe = "";
219
+ for (const ch of source) {
220
+ const code = ch.charCodeAt(0);
221
+ const allowed = code >= 48 && code <= 57 || code >= 65 && code <= 90 || code >= 97 && code <= 122 || ch === "." || ch === "_" || ch === "-" || ch === "[" || ch === "]" || ch === "*" || ch === "$";
222
+ safe += allowed ? ch : "_";
223
+ }
224
+ if (safe.length <= MAX_MARKER_SOURCE_CHARS)
225
+ return safe;
226
+ const suffix = sha256Bytes(source).slice(0, 12);
227
+ return `${safe.slice(0, MAX_MARKER_SOURCE_CHARS - suffix.length - 1)}~${suffix}`;
228
+ }
229
+ function buildDataUriMarker(omission) {
230
+ const decoded = omission.decodedBytes === null ? "unknown" : String(omission.decodedBytes);
231
+ const decodeError = omission.decodeError ? `; decodeError=${omission.decodeError}` : "";
232
+ return `[dm-web-access inline data URI omitted; ordinal=${omission.ordinal}; source=${omission.sourcePath}; mime=${omission.mimeType}; encoding=${omission.encoding}; encodedBytes=${omission.encodedBytes}; decodedBytes=${decoded}${decodeError}; sha256=${omission.sha256}; digestBasis=${omission.digestBasis}; retrieval=not-retained]`;
233
+ }
234
+ function sanitizeTextInternal(text, sourcePath, state) {
235
+ let scanFrom = 0;
236
+ let retainedFrom = 0;
237
+ let outputLength = 0;
238
+ const pieces = [];
239
+ while (scanFrom < text.length) {
240
+ const start = findNextDataScheme(text, scanFrom);
241
+ if (start < 0)
242
+ break;
243
+ const candidate = scanDataUriCandidate(text, start);
244
+ const end = candidate.end;
245
+ const headerStart = start + 5;
246
+ const missingComma = candidate.comma < 0;
247
+ if (missingComma && end === headerStart && candidate.enclosure === null) {
248
+ scanFrom = end;
249
+ continue;
250
+ }
251
+ const headerEnd = missingComma ? end : candidate.comma;
252
+ const headerLength = headerEnd - headerStart;
253
+ const encoding = headerEndsWithBase64(text, headerStart, headerEnd) ? "base64" : "percent-encoded";
254
+ const mimeType = normalizeMimeType(text, headerStart, headerEnd);
255
+ const payload = missingComma ? text.slice(headerStart, end) : text.slice(candidate.comma + 1, end);
256
+ const encodedBytes = Buffer.byteLength(payload, "utf8");
257
+ let decodedBytes = null;
258
+ let decodeError;
259
+ let digestBasis = "encoded";
260
+ let digest = "";
261
+ if (missingComma) {
262
+ decodeError = "missing-comma";
263
+ digest = sha256Bytes(payload);
264
+ } else if (headerLength > MAX_DATA_URI_HEADER_CHARS) {
265
+ decodeError = "header-too-long";
266
+ digest = sha256Bytes(payload);
267
+ } else {
268
+ const decoded = encoding === "base64" ? decodeBase64Payload(payload) : decodePercentEncoded(payload);
269
+ if ("error" in decoded) {
270
+ decodeError = decoded.error;
271
+ digest = sha256Bytes(payload);
272
+ } else {
273
+ decodedBytes = decoded.bytes.length;
274
+ digestBasis = "decoded";
275
+ digest = sha256Bytes(decoded.bytes);
276
+ }
277
+ }
278
+ const omission = {
279
+ ordinal: state.nextOrdinal++,
280
+ sourcePath: safeMarkerSource(sourcePath),
281
+ mimeType,
282
+ encoding,
283
+ encodedBytes,
284
+ decodedBytes,
285
+ ...decodeError ? { decodeError } : {},
286
+ sha256: digest,
287
+ digestBasis,
288
+ retrieval: "not-retained"
289
+ };
290
+ const marker = buildDataUriMarker(omission);
291
+ const prefix = text.slice(retainedFrom, start);
292
+ pieces.push(prefix, marker);
293
+ outputLength += prefix.length;
294
+ const markerStart = outputLength;
295
+ outputLength += marker.length;
296
+ state.omissions.push({ ...omission, markerStart, markerEnd: outputLength });
297
+ retainedFrom = end;
298
+ scanFrom = end;
299
+ }
300
+ if (state.omissions.length === 0 && retainedFrom === 0)
301
+ return text;
302
+ pieces.push(text.slice(retainedFrom));
303
+ return pieces.join("");
304
+ }
305
+ export function sanitizeInlineDataUris(text, sourcePath) {
306
+ const state = { nextOrdinal: 1, omissions: [] };
307
+ const sanitized = sanitizeTextInternal(text, sourcePath, state);
308
+ return {
309
+ text: sanitized,
310
+ omissions: state.omissions.map(({ markerStart: _start, markerEnd: _end, ...omission }) => omission)
311
+ };
312
+ }
@@ -0,0 +1,346 @@
1
+ import { existsSync, readFileSync } from "node:fs";
2
+ import {
3
+ hasCredentialSource,
4
+ resolveCredential,
5
+ redactCredential
6
+ } from "./credential-source.js";
7
+ import { getWebSearchConfigPath } from "./utils.js";
8
+ const DEFAULT_API_HOST = "https://www.datalab.to";
9
+ const API_PREFIX = "/api/v1";
10
+ const DEFAULT_TIMEOUT_MS = 120000;
11
+ const DEFAULT_POLL_INTERVAL_MS = 1500;
12
+ const MAX_RESPONSE_BYTES = 4 * 1024 * 1024;
13
+ const MAX_ERROR_BODY_BYTES = 300;
14
+ const CLEANUP_TIMEOUT_MS = 5000;
15
+ const CONFIG_PATH = getWebSearchConfigPath();
16
+ export const DATALAB_MODE_VALUES = new Set([
17
+ "fast",
18
+ "balanced",
19
+ "accurate"
20
+ ]);
21
+ export const DATALAB_LOCATION_VALUES = new Set([
22
+ "eu",
23
+ "us"
24
+ ]);
25
+ export const DEFAULT_DATALAB_MODE = "balanced";
26
+ export const DEFAULT_PROCESSING_LOCATION = "us";
27
+ export const DEFAULT_DATALAB_TIMEOUT_MS = DEFAULT_TIMEOUT_MS;
28
+ let cachedConfig = null;
29
+ function loadConfig() {
30
+ if (cachedConfig)
31
+ return cachedConfig;
32
+ if (!existsSync(CONFIG_PATH)) {
33
+ cachedConfig = {};
34
+ return cachedConfig;
35
+ }
36
+ const rawText = readFileSync(CONFIG_PATH, "utf-8");
37
+ try {
38
+ cachedConfig = JSON.parse(rawText);
39
+ return cachedConfig;
40
+ } catch (err) {
41
+ const message = err instanceof Error ? err.message : String(err);
42
+ throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
43
+ }
44
+ }
45
+ function normalizeString(value) {
46
+ if (typeof value !== "string")
47
+ return null;
48
+ const normalized = value.trim();
49
+ return normalized.length > 0 ? normalized : null;
50
+ }
51
+ function normalizeFileId(value) {
52
+ if (typeof value === "number" && Number.isFinite(value))
53
+ return String(value);
54
+ return normalizeString(value);
55
+ }
56
+ export function isDatalabApiAvailable() {
57
+ return hasCredentialSource({
58
+ provider: "Datalab",
59
+ configuredValue: loadConfig().datalabApiKey,
60
+ environmentValue: process.env.DATALAB_API_KEY
61
+ });
62
+ }
63
+ export async function getDatalabApiKey(signal) {
64
+ return resolveCredential({
65
+ provider: "Datalab",
66
+ configuredValue: loadConfig().datalabApiKey,
67
+ environmentValue: process.env.DATALAB_API_KEY,
68
+ signal
69
+ });
70
+ }
71
+ export function getDatalabApiBase() {
72
+ const normalized = normalizeString(process.env.DATALAB_API_BASE)?.replace(/\/+$/, "");
73
+ return normalized || `${DEFAULT_API_HOST}${API_PREFIX}`;
74
+ }
75
+ export function getDatalabProcessingLocation() {
76
+ const value = normalizeString(process.env.DATALAB_PROCESSING_LOCATION) ?? DEFAULT_PROCESSING_LOCATION;
77
+ const normalized = value.toLowerCase();
78
+ if (!DATALAB_LOCATION_VALUES.has(normalized)) {
79
+ throw new Error(`Failed to parse DATALAB_PROCESSING_LOCATION: expected "eu" or "us", got "${value}"`);
80
+ }
81
+ return normalized;
82
+ }
83
+ export function normalizeDatalabMode(value) {
84
+ const raw = normalizeString(value);
85
+ if (!raw)
86
+ return DEFAULT_DATALAB_MODE;
87
+ const normalized = raw.toLowerCase();
88
+ if (DATALAB_MODE_VALUES.has(normalized))
89
+ return normalized;
90
+ throw new Error(`Failed to parse datalab mode: expected "fast", "balanced", or "accurate", got "${value}"`);
91
+ }
92
+ export async function extractPDFViaDatalab(buffer, options) {
93
+ options.signal?.throwIfAborted();
94
+ const apiKey = await getDatalabApiKey(options.signal);
95
+ options.signal?.throwIfAborted();
96
+ if (!apiKey) {
97
+ throw new Error("Datalab PDF conversion requires a configured Datalab API key");
98
+ }
99
+ const mode = options.mode ?? normalizeDatalabMode(process.env.DATALAB_MODE);
100
+ const processingLocation = options.processingLocation ?? getDatalabProcessingLocation();
101
+ const timeoutMs = typeof options.timeoutMs === "number" && Number.isFinite(options.timeoutMs) && options.timeoutMs > 0 ? Math.floor(options.timeoutMs) : DEFAULT_TIMEOUT_MS;
102
+ const deadline = Date.now() + timeoutMs;
103
+ const upload = await requestUploadUrl({
104
+ apiKey,
105
+ title: options.title,
106
+ processingLocation,
107
+ signal: options.signal,
108
+ deadline
109
+ });
110
+ if (typeof upload.upload_url !== "string" || !upload.upload_url) {
111
+ throw new Error("Datalab PDF conversion failed: missing upload_url");
112
+ }
113
+ const uploadFileId = normalizeFileId(upload.file_id);
114
+ if (!uploadFileId) {
115
+ throw new Error("Datalab PDF conversion failed: missing file_id");
116
+ }
117
+ let fileId = uploadFileId;
118
+ let reference = normalizeString(upload.reference);
119
+ try {
120
+ const put = await fetchDatalab(upload.upload_url, {
121
+ method: "PUT",
122
+ headers: { "content-type": "application/pdf" },
123
+ body: new Blob([buffer], { type: "application/pdf" }),
124
+ signal: withTimeout(options.signal, remaining(deadline))
125
+ }, apiKey);
126
+ if (!put.ok) {
127
+ throw new Error(`Datalab PDF upload failed: HTTP ${put.status} ${put.statusText}`);
128
+ }
129
+ const confirmed = await confirmUpload(apiKey, String(fileId), options.signal, deadline);
130
+ fileId = normalizeFileId(confirmed.file_id) ?? fileId;
131
+ reference = normalizeString(confirmed.reference) ?? reference;
132
+ if (!reference) {
133
+ throw new Error("Datalab PDF conversion failed: missing file reference");
134
+ }
135
+ const form = new FormData;
136
+ form.append("file_url", reference);
137
+ form.append("output_format", "markdown");
138
+ form.append("mode", mode);
139
+ form.append("max_pages", String(options.maxPages));
140
+ form.append("paginate", "true");
141
+ const submit = await fetchDatalab(`${getDatalabApiBase()}/convert`, {
142
+ method: "POST",
143
+ headers: { "x-api-key": apiKey },
144
+ body: form,
145
+ signal: withTimeout(options.signal, remaining(deadline))
146
+ }, apiKey);
147
+ const state = await readJsonResponse(submit, apiKey);
148
+ if (state.status === "complete")
149
+ return toResult(state);
150
+ if (state.status === "failed") {
151
+ throw new Error(`Datalab PDF conversion failed: ${state.error ?? "unknown error"}`);
152
+ }
153
+ const checkUrl = normalizeCheckUrl(state.request_check_url);
154
+ while (Date.now() < deadline) {
155
+ await sleep(Math.min(DEFAULT_POLL_INTERVAL_MS, remaining(deadline)), options.signal);
156
+ if (Date.now() >= deadline)
157
+ break;
158
+ const poll = await fetchDatalab(checkUrl, {
159
+ method: "GET",
160
+ headers: { "x-api-key": apiKey },
161
+ signal: withTimeout(options.signal, remaining(deadline))
162
+ }, apiKey);
163
+ const next = await readJsonResponse(poll, apiKey);
164
+ if (next.status === "complete")
165
+ return toResult(next);
166
+ if (next.status === "failed") {
167
+ throw new Error(`Datalab PDF conversion failed: ${next.error ?? "unknown error"}`);
168
+ }
169
+ }
170
+ throw new Error("Datalab PDF conversion timed out");
171
+ } finally {
172
+ deleteDatalabFile(apiKey, fileId);
173
+ }
174
+ }
175
+ async function requestUploadUrl(options) {
176
+ const { apiKey, title, processingLocation, signal, deadline } = options;
177
+ const response = await fetchDatalab(`${getDatalabApiBase()}/files/upload`, {
178
+ method: "POST",
179
+ headers: {
180
+ "x-api-key": apiKey,
181
+ "content-type": "application/json"
182
+ },
183
+ body: JSON.stringify({
184
+ filename: `${datalabFilename(title)}.pdf`,
185
+ content_type: "application/pdf",
186
+ processing_location: processingLocation
187
+ }),
188
+ signal: withTimeout(signal, remaining(deadline))
189
+ }, apiKey);
190
+ return readJsonResponse(response, apiKey);
191
+ }
192
+ async function confirmUpload(apiKey, fileId, signal, deadline) {
193
+ const response = await fetchDatalab(`${getDatalabApiBase()}/files/${encodeURIComponent(fileId)}/confirm`, {
194
+ method: "GET",
195
+ headers: { "x-api-key": apiKey },
196
+ signal: withTimeout(signal, remaining(deadline))
197
+ }, apiKey);
198
+ return readJsonResponse(response, apiKey);
199
+ }
200
+ async function deleteDatalabFile(apiKey, fileId) {
201
+ try {
202
+ await fetchDatalab(`${getDatalabApiBase()}/files/${encodeURIComponent(fileId)}`, {
203
+ method: "DELETE",
204
+ headers: { "x-api-key": apiKey },
205
+ signal: AbortSignal.timeout(CLEANUP_TIMEOUT_MS)
206
+ }, apiKey);
207
+ } catch {}
208
+ }
209
+ function toResult(state) {
210
+ const markdown = normalizeString(state.markdown);
211
+ if (!markdown) {
212
+ throw new Error("Datalab PDF conversion returned empty markdown");
213
+ }
214
+ const quality = state.parse_quality_score;
215
+ return {
216
+ markdown,
217
+ pages: numericField(state.page_count) ?? countPageMarkers(markdown),
218
+ ...typeof quality === "number" && Number.isFinite(quality) ? { parseQualityScore: quality } : {}
219
+ };
220
+ }
221
+ function numericField(value) {
222
+ return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.floor(value) : null;
223
+ }
224
+ function countPageMarkers(markdown) {
225
+ return [...markdown.matchAll(/^<!-- Page (\d+) -->$/gm)].length;
226
+ }
227
+ function normalizeCheckUrl(value) {
228
+ if (typeof value !== "string" || !value.trim()) {
229
+ throw new Error("Datalab PDF conversion failed: missing request_check_url");
230
+ }
231
+ const apiBase = getDatalabApiUrl();
232
+ let checkUrl;
233
+ try {
234
+ checkUrl = new URL(value.trim(), apiBase);
235
+ } catch {
236
+ throw new Error("Datalab PDF conversion failed: invalid request_check_url");
237
+ }
238
+ if (checkUrl.origin !== apiBase.origin) {
239
+ throw new Error("Datalab PDF conversion failed: request_check_url has an unexpected origin");
240
+ }
241
+ return checkUrl.toString();
242
+ }
243
+ function getDatalabApiUrl() {
244
+ try {
245
+ return new URL(getDatalabApiBase());
246
+ } catch {
247
+ throw new Error("Failed to parse DATALAB_API_BASE: expected an absolute URL");
248
+ }
249
+ }
250
+ function remaining(deadline) {
251
+ return Math.max(1, deadline - Date.now());
252
+ }
253
+ function withTimeout(signal, timeoutMs) {
254
+ const timeout = AbortSignal.timeout(timeoutMs);
255
+ return signal ? AbortSignal.any([signal, timeout]) : timeout;
256
+ }
257
+ function sleep(ms, signal) {
258
+ return new Promise((resolve, reject) => {
259
+ let timer;
260
+ const cleanup = () => signal?.removeEventListener("abort", onAbort);
261
+ const onTimeout = () => {
262
+ cleanup();
263
+ resolve();
264
+ };
265
+ const onAbort = () => {
266
+ clearTimeout(timer);
267
+ cleanup();
268
+ reject(signal?.reason);
269
+ };
270
+ if (signal?.aborted) {
271
+ onAbort();
272
+ return;
273
+ }
274
+ timer = setTimeout(onTimeout, ms);
275
+ signal?.addEventListener("abort", onAbort, { once: true });
276
+ });
277
+ }
278
+ async function fetchDatalab(url, init, apiKey) {
279
+ try {
280
+ return await fetch(url, { ...init, redirect: init.redirect ?? "error" });
281
+ } catch (error) {
282
+ const message = error instanceof Error ? error.message : String(error);
283
+ const redacted = redactCredential(message, apiKey);
284
+ if (redacted === message)
285
+ throw error;
286
+ const redactedError = new Error(redacted);
287
+ if (error instanceof Error)
288
+ redactedError.name = error.name;
289
+ throw redactedError;
290
+ }
291
+ }
292
+ async function readJsonResponse(response, apiKey) {
293
+ const text = await readResponseText(response);
294
+ if (!response.ok) {
295
+ const detail = text.trim().slice(0, MAX_ERROR_BODY_BYTES);
296
+ const message = detail ? `Datalab PDF conversion failed: HTTP ${response.status} ${response.statusText}: ${detail}` : `Datalab PDF conversion failed: HTTP ${response.status} ${response.statusText}`;
297
+ throw new Error(redactCredential(message, apiKey));
298
+ }
299
+ return parseJsonRecord(text);
300
+ }
301
+ function parseJsonRecord(text) {
302
+ let parsed;
303
+ try {
304
+ parsed = JSON.parse(text);
305
+ } catch (error) {
306
+ throw new Error("Datalab PDF conversion returned invalid JSON", { cause: error });
307
+ }
308
+ if (Object.prototype.toString.call(parsed) !== "[object Object]") {
309
+ throw new Error("Datalab PDF conversion returned invalid JSON object");
310
+ }
311
+ return parsed;
312
+ }
313
+ async function readResponseText(response) {
314
+ const contentLength = Number(response.headers.get("content-length"));
315
+ if (Number.isFinite(contentLength) && contentLength > MAX_RESPONSE_BYTES) {
316
+ throw new Error("Datalab PDF conversion response too large");
317
+ }
318
+ if (!response.body)
319
+ return "";
320
+ const reader = response.body.getReader();
321
+ const decoder = new TextDecoder;
322
+ const chunks = [];
323
+ let bytesRead = 0;
324
+ try {
325
+ while (true) {
326
+ const { done, value } = await reader.read();
327
+ if (done)
328
+ break;
329
+ bytesRead += value.byteLength;
330
+ if (bytesRead > MAX_RESPONSE_BYTES) {
331
+ try {
332
+ await reader.cancel();
333
+ } catch {}
334
+ throw new Error("Datalab PDF conversion response too large");
335
+ }
336
+ chunks.push(decoder.decode(value, { stream: true }));
337
+ }
338
+ chunks.push(decoder.decode());
339
+ return chunks.join("");
340
+ } finally {
341
+ reader.releaseLock();
342
+ }
343
+ }
344
+ function datalabFilename(title) {
345
+ return title.toLowerCase().replace(/[^a-z0-9\s-]/g, "").replace(/\s+/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "").slice(0, 60) || "document";
346
+ }