@herbertgao/pi-extensions 2026.8.7 → 2026.8.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/README.md +7 -5
  2. package/node_modules/@herbertgao/pi-cc-extensions/README.en.md +1 -1
  3. package/node_modules/@herbertgao/pi-cc-extensions/README.md +1 -1
  4. package/node_modules/@herbertgao/pi-cc-extensions/extensions/feature/compact-thinking.ts +63 -15
  5. package/node_modules/@herbertgao/pi-cc-extensions/extensions/renderer/message-display.ts +78 -16
  6. package/node_modules/@herbertgao/pi-cc-extensions/extensions/renderer/mouse/hover.ts +13 -0
  7. package/node_modules/@herbertgao/pi-cc-extensions/extensions/renderer/mouse/interaction.ts +17 -6
  8. package/node_modules/@herbertgao/pi-cc-extensions/package.json +3 -3
  9. package/node_modules/@juicesharp/rpiv-ask-user-question/README.md +4 -4
  10. package/node_modules/@juicesharp/rpiv-ask-user-question/ask-user-question.ts +154 -96
  11. package/node_modules/@juicesharp/rpiv-ask-user-question/docs/hosts.md +3 -0
  12. package/node_modules/@juicesharp/rpiv-ask-user-question/docs/keyboard.md +10 -2
  13. package/node_modules/@juicesharp/rpiv-ask-user-question/docs/localization.md +7 -5
  14. package/node_modules/@juicesharp/rpiv-ask-user-question/docs/tool-schema.md +11 -5
  15. package/node_modules/@juicesharp/rpiv-ask-user-question/locales/en.json +3 -0
  16. package/node_modules/@juicesharp/rpiv-ask-user-question/package.json +2 -2
  17. package/node_modules/@juicesharp/rpiv-ask-user-question/state/key-router.ts +137 -96
  18. package/node_modules/@juicesharp/rpiv-ask-user-question/state/questionnaire-session.ts +44 -32
  19. package/node_modules/@juicesharp/rpiv-ask-user-question/state/selectors/projections.ts +22 -15
  20. package/node_modules/@juicesharp/rpiv-ask-user-question/state/state-reducer.ts +11 -1
  21. package/node_modules/@juicesharp/rpiv-ask-user-question/tool/response-envelope.ts +11 -0
  22. package/node_modules/@juicesharp/rpiv-ask-user-question/tool/types.ts +11 -0
  23. package/node_modules/@juicesharp/rpiv-ask-user-question/view/components/multi-select-view.ts +31 -17
  24. package/node_modules/@juicesharp/rpiv-ask-user-question/view/components/preview/preview-block-renderer.ts +15 -6
  25. package/node_modules/@juicesharp/rpiv-ask-user-question/view/components/wrapping-select.ts +15 -6
  26. package/node_modules/@juicesharp/rpiv-ask-user-question/view/dialog-builder.ts +56 -34
  27. package/node_modules/@juicesharp/rpiv-ask-user-question/view/tab-content-strategy.ts +50 -4
  28. package/node_modules/@narumitw/pi-btw/package.json +2 -2
  29. package/node_modules/@pi-plugins/fast-mode/dist/index.mjs +44 -1856
  30. package/node_modules/@pi-plugins/fast-mode/dist/index.mjs.map +1 -1
  31. package/node_modules/@pi-plugins/fast-mode/package.json +4 -6
  32. package/node_modules/pi-lens/CHANGELOG.md +90 -0
  33. package/node_modules/pi-lens/README.md +106 -0
  34. package/node_modules/pi-lens/dist/clients/bash-file-access.js +97 -6
  35. package/node_modules/pi-lens/dist/clients/blocker-past-eof.js +16 -6
  36. package/node_modules/pi-lens/dist/clients/bounded-telemetry.js +14 -0
  37. package/node_modules/pi-lens/dist/clients/cache-observability.js +353 -76
  38. package/node_modules/pi-lens/dist/clients/degradation-ledger.js +22 -1
  39. package/node_modules/pi-lens/dist/clients/demoted-finding-render.js +155 -0
  40. package/node_modules/pi-lens/dist/clients/deps/pi-tui.js +8 -2
  41. package/node_modules/pi-lens/dist/clients/deps/typebox.js +8 -3
  42. package/node_modules/pi-lens/dist/clients/dispatch/runners/detekt.js +9 -6
  43. package/node_modules/pi-lens/dist/clients/dispatch/runners/hadolint.js +9 -7
  44. package/node_modules/pi-lens/dist/clients/dispatch/runners/htmlhint.js +8 -6
  45. package/node_modules/pi-lens/dist/clients/dispatch/runners/markdownlint.js +8 -10
  46. package/node_modules/pi-lens/dist/clients/dispatch/runners/mypy.js +10 -10
  47. package/node_modules/pi-lens/dist/clients/dispatch/runners/oxlint.js +20 -5
  48. package/node_modules/pi-lens/dist/clients/dispatch/runners/phpstan.js +44 -11
  49. package/node_modules/pi-lens/dist/clients/dispatch/runners/spellcheck.js +10 -15
  50. package/node_modules/pi-lens/dist/clients/dispatch/runners/sqlfluff.js +19 -11
  51. package/node_modules/pi-lens/dist/clients/dispatch/runners/stylelint.js +35 -13
  52. package/node_modules/pi-lens/dist/clients/dispatch/runners/swiftlint.js +8 -8
  53. package/node_modules/pi-lens/dist/clients/dispatch/runners/taplo.js +110 -23
  54. package/node_modules/pi-lens/dist/clients/dispatch/runners/tflint.js +12 -7
  55. package/node_modules/pi-lens/dist/clients/dispatch/runners/utils/tool-failure.js +117 -1
  56. package/node_modules/pi-lens/dist/clients/dispatch/runners/vale.js +16 -14
  57. package/node_modules/pi-lens/dist/clients/dispatch/runners/yamllint.js +8 -9
  58. package/node_modules/pi-lens/dist/clients/finding-delivery-gate.js +22 -0
  59. package/node_modules/pi-lens/dist/clients/jscpd-client.js +5 -2
  60. package/node_modules/pi-lens/dist/clients/language-profile.js +7 -2
  61. package/node_modules/pi-lens/dist/clients/lsp/cascade-tier.js +27 -2
  62. package/node_modules/pi-lens/dist/clients/lsp/client.js +144 -5
  63. package/node_modules/pi-lens/dist/clients/lsp/index.js +151 -6
  64. package/node_modules/pi-lens/dist/clients/module-report.js +14 -1
  65. package/node_modules/pi-lens/dist/clients/ndjson-logger.js +114 -23
  66. package/node_modules/pi-lens/dist/clients/project-diagnostics/scanner.js +18 -9
  67. package/node_modules/pi-lens/dist/clients/project-report.js +62 -27
  68. package/node_modules/pi-lens/dist/clients/read-guard-logger.js +7 -2
  69. package/node_modules/pi-lens/dist/clients/read-guard.js +88 -19
  70. package/node_modules/pi-lens/dist/clients/review-graph/builder.js +194 -31
  71. package/node_modules/pi-lens/dist/clients/review-graph/revision-drift.js +21 -0
  72. package/node_modules/pi-lens/dist/clients/review-graph/workspace-modules.js +6 -3
  73. package/node_modules/pi-lens/dist/clients/runtime-coordinator.js +44 -0
  74. package/node_modules/pi-lens/dist/clients/runtime-session.js +41 -0
  75. package/node_modules/pi-lens/dist/clients/runtime-turn.js +47 -2
  76. package/node_modules/pi-lens/dist/clients/session-event-guard.js +184 -0
  77. package/node_modules/pi-lens/dist/clients/source-filter.js +6 -2
  78. package/node_modules/pi-lens/dist/clients/startup-scan.js +5 -2
  79. package/node_modules/pi-lens/dist/clients/tree-sitter-logger.js +1 -0
  80. package/node_modules/pi-lens/dist/index.js +1224 -393
  81. package/node_modules/pi-lens/dist/tools/lens-diagnostics.js +10 -3
  82. package/node_modules/pi-lens/package.json +18 -5
  83. package/node_modules/pi-lens/scripts/install-selftest.mjs +99 -11
  84. package/node_modules/pi-lens/scripts/lib/host-provided-deps.mjs +75 -0
  85. package/node_modules/pi-lens/scripts/lib/warm-loader-cache.mjs +285 -0
  86. package/node_modules/pi-lens/scripts/warm-loader-cache.mjs +233 -0
  87. package/node_modules/pi-web-access/CHANGELOG.md +14 -0
  88. package/node_modules/pi-web-access/README.md +6 -5
  89. package/node_modules/pi-web-access/chrome-cookies.ts +86 -15
  90. package/node_modules/pi-web-access/data-uri-sanitize.ts +406 -0
  91. package/node_modules/pi-web-access/extract.ts +12 -2
  92. package/node_modules/pi-web-access/firecrawl.ts +18 -2
  93. package/node_modules/pi-web-access/github-extract.ts +65 -36
  94. package/node_modules/pi-web-access/index.ts +19 -4
  95. package/node_modules/pi-web-access/openai-search.ts +17 -6
  96. package/node_modules/pi-web-access/package.json +1 -1
  97. package/node_modules/pi-web-access/pdf-extract.ts +18 -5
  98. package/node_modules/pi-web-access/ssrf-protection.ts +12 -2
  99. package/package.json +9 -9
@@ -0,0 +1,406 @@
1
+ import { Buffer } from "node:buffer";
2
+ import { createHash } from "node:crypto";
3
+
4
+ // Replaces inline RFC 2397 data: URIs in extracted text with explicit bounded
5
+ // omission markers so base64 payloads never reach model-visible tool results,
6
+ // the fetch cache, or session persistence. Carved out of the original
7
+ // persistence-safety layer; only the text-sanitization path is retained.
8
+
9
+ const MAX_DATA_URI_HEADER_CHARS = 1024;
10
+ const MAX_MIME_CHARS = 127;
11
+ const MAX_MARKER_SOURCE_CHARS = 180;
12
+
13
+ export type DataUriEncoding = "base64" | "percent-encoded";
14
+ export type DigestBasis = "decoded" | "encoded";
15
+
16
+ export interface DataUriOmission {
17
+ ordinal: number;
18
+ sourcePath: string;
19
+ mimeType: string;
20
+ encoding: DataUriEncoding;
21
+ encodedBytes: number;
22
+ decodedBytes: number | null;
23
+ decodeError?: string;
24
+ sha256: string;
25
+ digestBasis: DigestBasis;
26
+ retrieval: "not-retained";
27
+ }
28
+
29
+
30
+ interface InternalDataUriOmission extends DataUriOmission {
31
+ markerStart: number;
32
+ markerEnd: number;
33
+ }
34
+
35
+ interface SanitizationState {
36
+ nextOrdinal: number;
37
+ omissions: InternalDataUriOmission[];
38
+ }
39
+
40
+
41
+ function sha256Bytes(value: string | Uint8Array): string {
42
+ return createHash("sha256").update(value).digest("hex");
43
+ }
44
+
45
+
46
+ function asciiLowerCode(code: number): number {
47
+ return code >= 65 && code <= 90 ? code + 32 : code;
48
+ }
49
+
50
+ function matchesDataScheme(text: string, index: number): boolean {
51
+ return index + 5 <= text.length
52
+ && asciiLowerCode(text.charCodeAt(index)) === 100
53
+ && asciiLowerCode(text.charCodeAt(index + 1)) === 97
54
+ && asciiLowerCode(text.charCodeAt(index + 2)) === 116
55
+ && asciiLowerCode(text.charCodeAt(index + 3)) === 97
56
+ && text.charCodeAt(index + 4) === 58;
57
+ }
58
+
59
+ function isSchemeBoundary(text: string, index: number): boolean {
60
+ if (index === 0) return true;
61
+ const code = text.charCodeAt(index - 1);
62
+ const alphaNumeric = (code >= 48 && code <= 57)
63
+ || (code >= 65 && code <= 90)
64
+ || (code >= 97 && code <= 122);
65
+ return !alphaNumeric && code !== 43 && code !== 45 && code !== 46;
66
+ }
67
+
68
+ function findNextDataScheme(text: string, from: number): number {
69
+ for (let i = from; i + 5 <= text.length; i++) {
70
+ if (matchesDataScheme(text, i) && isSchemeBoundary(text, i)) return i;
71
+ }
72
+ return -1;
73
+ }
74
+
75
+ type DataUriEnclosure =
76
+ | { kind: "quote"; close: "\"" | "'" | "`" }
77
+ | { kind: "parentheses" }
78
+ | { kind: "angle" }
79
+ | null;
80
+
81
+ interface ScannedDataUriCandidate {
82
+ end: number;
83
+ comma: number;
84
+ enclosure: DataUriEnclosure;
85
+ }
86
+
87
+ function dataUriEnclosure(text: string, start: number): DataUriEnclosure {
88
+ if (start === 0) return null;
89
+ const immediateIndex = start - 1;
90
+ const immediate = text[immediateIndex];
91
+ if (immediate === "\"" || immediate === "'" || immediate === "`") {
92
+ const beforeQuote = immediateIndex > 0 ? text[immediateIndex - 1] : "";
93
+ const likelyOpeningQuote = immediateIndex === 0
94
+ || beforeQuote === "="
95
+ || beforeQuote === "("
96
+ || beforeQuote === "["
97
+ || beforeQuote === "{"
98
+ || beforeQuote === ":"
99
+ || beforeQuote === ","
100
+ || /\s/.test(beforeQuote);
101
+ if (likelyOpeningQuote) return { kind: "quote", close: immediate };
102
+ }
103
+
104
+ let openerIndex = immediateIndex;
105
+ while (openerIndex >= 0 && (text[openerIndex] === " " || text[openerIndex] === "\t")) openerIndex--;
106
+ const opener = text[openerIndex];
107
+ if (opener === "(") return { kind: "parentheses" };
108
+ if (opener === "<") return { kind: "angle" };
109
+ return null;
110
+ }
111
+
112
+ function isBareDataUriTerminator(ch: string): boolean {
113
+ const code = ch.charCodeAt(0);
114
+ return code <= 32
115
+ || code === 127
116
+ || ch === "\""
117
+ || ch === "`"
118
+ || ch === "<"
119
+ || ch === ">";
120
+ }
121
+
122
+ /** Scan one candidate exactly once. Enclosures consume malformed whitespace and
123
+ * balanced inner parentheses so invalid payload suffixes cannot escape. */
124
+ function scanDataUriCandidate(text: string, start: number): ScannedDataUriCandidate {
125
+ const enclosure = dataUriEnclosure(text, start);
126
+ let comma = -1;
127
+ let depth = enclosure?.kind === "parentheses" ? 1 : 0;
128
+ let index = start + 5;
129
+
130
+ for (; index < text.length; index++) {
131
+ const ch = text[index];
132
+ if (enclosure?.kind === "quote") {
133
+ if (ch === enclosure.close) break;
134
+ } else if (enclosure?.kind === "parentheses") {
135
+ if (ch === "(") {
136
+ depth++;
137
+ } else if (ch === ")") {
138
+ depth--;
139
+ if (depth === 0) break;
140
+ }
141
+ } else if (enclosure?.kind === "angle") {
142
+ if (ch === ">") break;
143
+ } else if (isBareDataUriTerminator(ch)) {
144
+ break;
145
+ }
146
+ if (comma < 0 && ch === ",") comma = index;
147
+ }
148
+
149
+ return { end: index, comma, enclosure };
150
+ }
151
+
152
+ function headerEndsWithBase64(text: string, headerStart: number, comma: number): boolean {
153
+ let end = comma;
154
+ while (end > headerStart && (text[end - 1] === " " || text[end - 1] === "\t")) end--;
155
+ const token = "base64";
156
+ if (end - headerStart < token.length + 1) return false;
157
+ const tokenStart = end - token.length;
158
+ for (let i = 0; i < token.length; i++) {
159
+ if (asciiLowerCode(text.charCodeAt(tokenStart + i)) !== token.charCodeAt(i)) return false;
160
+ }
161
+ return tokenStart > headerStart && text[tokenStart - 1] === ";";
162
+ }
163
+
164
+ function isMimeTokenChar(code: number): boolean {
165
+ if (code < 33 || code > 126) return false;
166
+ // RFC 2045 token = printable ASCII excluding tspecials. Slash is handled
167
+ // separately as the single type/subtype separator.
168
+ switch (code) {
169
+ case 34: // "
170
+ case 40: // (
171
+ case 41: // )
172
+ case 44: // ,
173
+ case 47: // /
174
+ case 58: // :
175
+ case 59: // ;
176
+ case 60: // <
177
+ case 61: // =
178
+ case 62: // >
179
+ case 63: // ?
180
+ case 64: // @
181
+ case 91: // [
182
+ case 92: // backslash
183
+ case 93: // ]
184
+ return false;
185
+ default:
186
+ return true;
187
+ }
188
+ }
189
+
190
+ function normalizeMimeType(text: string, headerStart: number, comma: number): string {
191
+ let end = headerStart;
192
+ while (end < comma && text[end] !== ";") end++;
193
+ if (end === headerStart) return "text/plain";
194
+ if (end - headerStart > MAX_MIME_CHARS) return "application/octet-stream";
195
+
196
+ let slashCount = 0;
197
+ for (let i = headerStart; i < end; i++) {
198
+ const code = text.charCodeAt(i);
199
+ if (code === 47) {
200
+ slashCount++;
201
+ if (i === headerStart || i === end - 1) return "application/octet-stream";
202
+ continue;
203
+ }
204
+ if (!isMimeTokenChar(code)) return "application/octet-stream";
205
+ }
206
+ if (slashCount !== 1) return "application/octet-stream";
207
+ return text.slice(headerStart, end).toLowerCase();
208
+ }
209
+
210
+ function hexValue(code: number): number {
211
+ if (code >= 48 && code <= 57) return code - 48;
212
+ if (code >= 65 && code <= 70) return code - 55;
213
+ if (code >= 97 && code <= 102) return code - 87;
214
+ return -1;
215
+ }
216
+
217
+ function decodePercentEncoded(payload: string): { bytes: Buffer } | { error: string } {
218
+ const output = Buffer.allocUnsafe(Buffer.byteLength(payload, "utf8"));
219
+ let outputOffset = 0;
220
+ let cursor = 0;
221
+
222
+ while (cursor < payload.length) {
223
+ const percent = payload.indexOf("%", cursor);
224
+ const runEnd = percent < 0 ? payload.length : percent;
225
+ if (runEnd > cursor) {
226
+ const run = Buffer.from(payload.slice(cursor, runEnd), "utf8");
227
+ run.copy(output, outputOffset);
228
+ outputOffset += run.length;
229
+ }
230
+ if (percent < 0) break;
231
+ if (percent + 2 >= payload.length) return { error: "invalid-percent-escape" };
232
+ const high = hexValue(payload.charCodeAt(percent + 1));
233
+ const low = hexValue(payload.charCodeAt(percent + 2));
234
+ if (high < 0 || low < 0) return { error: "invalid-percent-escape" };
235
+ output[outputOffset++] = (high << 4) | low;
236
+ cursor = percent + 3;
237
+ }
238
+
239
+ return { bytes: output.subarray(0, outputOffset) };
240
+ }
241
+
242
+ function isBase64AlphabetByte(byte: number): boolean {
243
+ return (byte >= 65 && byte <= 90)
244
+ || (byte >= 97 && byte <= 122)
245
+ || (byte >= 48 && byte <= 57)
246
+ || byte === 43
247
+ || byte === 47;
248
+ }
249
+
250
+ function decodeBase64Payload(payload: string): { bytes: Buffer } | { error: string } {
251
+ const percentDecoded = decodePercentEncoded(payload);
252
+ if ("error" in percentDecoded) return percentDecoded;
253
+ const encoded = percentDecoded.bytes;
254
+ let paddingStart = encoded.length;
255
+ let paddingCount = 0;
256
+
257
+ for (let i = 0; i < encoded.length; i++) {
258
+ const byte = encoded[i];
259
+ if (byte > 127) return { error: "non-ascii-base64" };
260
+ if (byte === 61) {
261
+ if (paddingStart === encoded.length) paddingStart = i;
262
+ paddingCount++;
263
+ continue;
264
+ }
265
+ if (paddingStart !== encoded.length) return { error: "invalid-base64-padding" };
266
+ if (!isBase64AlphabetByte(byte)) return { error: "invalid-base64-character" };
267
+ }
268
+
269
+ if (paddingCount > 2) return { error: "invalid-base64-padding" };
270
+ if (paddingCount > 0 && encoded.length % 4 !== 0) return { error: "invalid-base64-padding" };
271
+ if (paddingCount === 0 && encoded.length % 4 === 1) return { error: "invalid-base64-length" };
272
+ if (paddingCount === 1 && paddingStart % 4 !== 3) return { error: "invalid-base64-padding" };
273
+ if (paddingCount === 2 && paddingStart % 4 !== 2) return { error: "invalid-base64-padding" };
274
+
275
+ return { bytes: Buffer.from(encoded.toString("ascii"), "base64") };
276
+ }
277
+
278
+ function safeMarkerSource(sourcePath: string): string {
279
+ const source = sourcePath || "value";
280
+ let safe = "";
281
+ for (const ch of source) {
282
+ const code = ch.charCodeAt(0);
283
+ const allowed = (code >= 48 && code <= 57)
284
+ || (code >= 65 && code <= 90)
285
+ || (code >= 97 && code <= 122)
286
+ || ch === "."
287
+ || ch === "_"
288
+ || ch === "-"
289
+ || ch === "["
290
+ || ch === "]"
291
+ || ch === "*"
292
+ || ch === "$";
293
+ safe += allowed ? ch : "_";
294
+ }
295
+ if (safe.length <= MAX_MARKER_SOURCE_CHARS) return safe;
296
+ const suffix = sha256Bytes(source).slice(0, 12);
297
+ return `${safe.slice(0, MAX_MARKER_SOURCE_CHARS - suffix.length - 1)}~${suffix}`;
298
+ }
299
+
300
+ function buildDataUriMarker(omission: DataUriOmission): string {
301
+ const decoded = omission.decodedBytes === null ? "unknown" : String(omission.decodedBytes);
302
+ const decodeError = omission.decodeError ? `; decodeError=${omission.decodeError}` : "";
303
+ return `[pi-web-access inline data URI omitted; ordinal=${omission.ordinal}; source=${omission.sourcePath}; mime=${omission.mimeType}; encoding=${omission.encoding}; encodedBytes=${omission.encodedBytes}; decodedBytes=${decoded}${decodeError}; sha256=${omission.sha256}; digestBasis=${omission.digestBasis}; retrieval=not-retained]`;
304
+ }
305
+
306
+ function sanitizeTextInternal(text: string, sourcePath: string, state: SanitizationState): string {
307
+ let scanFrom = 0;
308
+ let retainedFrom = 0;
309
+ let outputLength = 0;
310
+ const pieces: string[] = [];
311
+
312
+ while (scanFrom < text.length) {
313
+ const start = findNextDataScheme(text, scanFrom);
314
+ if (start < 0) break;
315
+ const candidate = scanDataUriCandidate(text, start);
316
+ const end = candidate.end;
317
+ const headerStart = start + 5;
318
+ const missingComma = candidate.comma < 0;
319
+ // A bare prose label such as "data: value" is not an inline URI. Enclosed
320
+ // or non-empty comma-less candidates are malformed URIs and are removed.
321
+ if (missingComma && end === headerStart && candidate.enclosure === null) {
322
+ scanFrom = end;
323
+ continue;
324
+ }
325
+ const headerEnd = missingComma ? end : candidate.comma;
326
+ const headerLength = headerEnd - headerStart;
327
+ const encoding: DataUriEncoding = headerEndsWithBase64(text, headerStart, headerEnd)
328
+ ? "base64"
329
+ : "percent-encoded";
330
+ const mimeType = normalizeMimeType(text, headerStart, headerEnd);
331
+ // Without the required comma, the post-scheme bytes are ambiguous. Count
332
+ // and digest the whole omitted segment on the encoded basis rather than
333
+ // retaining any header/payload-looking suffix.
334
+ const payload = missingComma
335
+ ? text.slice(headerStart, end)
336
+ : text.slice(candidate.comma + 1, end);
337
+ const encodedBytes = Buffer.byteLength(payload, "utf8");
338
+
339
+ let decodedBytes: number | null = null;
340
+ let decodeError: string | undefined;
341
+ let digestBasis: DigestBasis = "encoded";
342
+ let digest = "";
343
+
344
+ if (missingComma) {
345
+ decodeError = "missing-comma";
346
+ digest = sha256Bytes(payload);
347
+ } else if (headerLength > MAX_DATA_URI_HEADER_CHARS) {
348
+ decodeError = "header-too-long";
349
+ digest = sha256Bytes(payload);
350
+ } else {
351
+ const decoded = encoding === "base64"
352
+ ? decodeBase64Payload(payload)
353
+ : decodePercentEncoded(payload);
354
+ if ("error" in decoded) {
355
+ decodeError = decoded.error;
356
+ digest = sha256Bytes(payload);
357
+ } else {
358
+ decodedBytes = decoded.bytes.length;
359
+ digestBasis = "decoded";
360
+ digest = sha256Bytes(decoded.bytes);
361
+ }
362
+ }
363
+
364
+ const omission: DataUriOmission = {
365
+ ordinal: state.nextOrdinal++,
366
+ sourcePath: safeMarkerSource(sourcePath),
367
+ mimeType,
368
+ encoding,
369
+ encodedBytes,
370
+ decodedBytes,
371
+ ...(decodeError ? { decodeError } : {}),
372
+ sha256: digest,
373
+ digestBasis,
374
+ retrieval: "not-retained",
375
+ };
376
+ const marker = buildDataUriMarker(omission);
377
+ const prefix = text.slice(retainedFrom, start);
378
+ pieces.push(prefix, marker);
379
+ outputLength += prefix.length;
380
+ const markerStart = outputLength;
381
+ outputLength += marker.length;
382
+ state.omissions.push({ ...omission, markerStart, markerEnd: outputLength });
383
+ retainedFrom = end;
384
+ scanFrom = end;
385
+ }
386
+
387
+ if (state.omissions.length === 0 && retainedFrom === 0) return text;
388
+ pieces.push(text.slice(retainedFrom));
389
+ return pieces.join("");
390
+ }
391
+
392
+
393
+ /** Replace every recognized RFC 2397-style inline data URI in one string. */
394
+ export function sanitizeInlineDataUris(
395
+ text: string,
396
+ sourcePath: string,
397
+ ): { text: string; omissions: DataUriOmission[] } {
398
+ const state: SanitizationState = { nextOrdinal: 1, omissions: [] };
399
+ const sanitized = sanitizeTextInternal(text, sourcePath, state);
400
+ return {
401
+ text: sanitized,
402
+ omissions: state.omissions.map(({ markerStart: _start, markerEnd: _end, ...omission }) => omission),
403
+ };
404
+ }
405
+
406
+ /** Sanitize every string leaf in a JSON-compatible value with one ordinal sequence. */
@@ -27,6 +27,7 @@ import { formatSeconds, getWebSearchConfigPath } from "./utils.ts";
27
27
  import { isImageEnabled } from "./feature-config.ts";
28
28
  import { assertAuthFetchUrl, authFetchRedirectGuard, type AuthFetchProfile } from "./auth-fetch.ts";
29
29
  import { getBrowserCookiesForHosts, getLastBrowserCookieDiagnostic } from "./chrome-cookies.ts";
30
+ import { sanitizeInlineDataUris } from "./data-uri-sanitize.ts";
30
31
 
31
32
  const DEFAULT_TIMEOUT_MS = 30000;
32
33
  const CONCURRENT_LIMIT = 3;
@@ -1001,7 +1002,7 @@ async function extractViaHttp(
1001
1002
  const requestInit = {
1002
1003
  signal: controller.signal,
1003
1004
  headers: {
1004
- "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
1005
+ "User-Agent": "OpenAI File Downloader, XaiImageApiFetch/1.0",
1005
1006
  "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
1006
1007
  "Accept-Language": "en-US,en;q=0.9",
1007
1008
  "Cache-Control": "no-cache",
@@ -1250,5 +1251,14 @@ export async function fetchAllContent(
1250
1251
  signal?: AbortSignal,
1251
1252
  options?: ExtractOptions,
1252
1253
  ): Promise<ExtractedContent[]> {
1253
- return Promise.all(urls.map((url) => fetchLimit(() => extractContent(url, signal, options))));
1254
+ const results = await Promise.all(urls.map((url) => fetchLimit(() => extractContent(url, signal, options))));
1255
+ if (options?.mode === "raw") return results;
1256
+ // Inline data: URIs in extracted markdown would otherwise flow into tool
1257
+ // results and the fetch cache as opaque base64; typed thumbnail/frame image
1258
+ // blocks are deliberate outputs and are left untouched.
1259
+ return results.map((result, index) => {
1260
+ if (!result.content) return result;
1261
+ const sanitized = sanitizeInlineDataUris(result.content, `urls[${index}].content`);
1262
+ return sanitized.omissions.length > 0 ? { ...result, content: sanitized.text } : result;
1263
+ });
1254
1264
  }
@@ -1,4 +1,5 @@
1
1
  import { existsSync, readFileSync } from "node:fs";
2
+ import net from "node:net";
2
3
  import { activityMonitor } from "./activity.ts";
3
4
  import { redactCredential, resolveCredential } from "./credential-source.ts";
4
5
  import type { ExtractedContent, ExtractOptions } from "./extract.ts";
@@ -184,6 +185,20 @@ function ssrfOptions(options?: FirecrawlExtractOptions | FirecrawlSearchOptions)
184
185
  };
185
186
  }
186
187
 
188
+ function isLoopbackApiUrl(url: URL): boolean {
189
+ const hostname = url.hostname.toLowerCase().replace(/^\[|\]$/g, "").replace(/\.$/, "");
190
+ if (hostname === "localhost" || hostname === "::1") return true;
191
+ if (net.isIP(hostname) !== 4) return false;
192
+ return hostname.split(".")[0] === "127";
193
+ }
194
+
195
+ function firecrawlApiSsrfOptions(
196
+ options: FirecrawlExtractOptions | FirecrawlSearchOptions | undefined,
197
+ allowLoopback: boolean,
198
+ ): ReturnType<typeof ssrfOptions> & { allowLoopback: boolean } {
199
+ return { ...ssrfOptions(options), allowLoopback };
200
+ }
201
+
187
202
  function withoutSensitiveHeaders(headers: Record<string, string>): Record<string, string> {
188
203
  const next = { ...headers };
189
204
  delete next.Authorization;
@@ -200,7 +215,8 @@ async function fetchFirecrawlApi(
200
215
  init: { method: string; headers: Record<string, string>; body: string; signal: AbortSignal },
201
216
  options: FirecrawlExtractOptions | FirecrawlSearchOptions | undefined,
202
217
  ): Promise<Response> {
203
- let current = await validateRemoteUrl(url, ssrfOptions(options));
218
+ const allowLoopback = isLoopbackApiUrl(new URL(url));
219
+ let current = await validateRemoteUrl(url, firecrawlApiSsrfOptions(options, allowLoopback));
204
220
  let headers = init.headers;
205
221
  for (let redirects = 0; redirects <= DEFAULT_MAX_REDIRECTS; redirects++) {
206
222
  const response = await fetch(current, { ...init, headers, redirect: "manual" });
@@ -210,7 +226,7 @@ async function fetchFirecrawlApi(
210
226
  if (!location) return response;
211
227
  if (redirects === DEFAULT_MAX_REDIRECTS) throw new Error(`Too many redirects fetching ${current.toString()}`);
212
228
 
213
- const next = await validateRemoteUrl(new URL(location, current), ssrfOptions(options));
229
+ const next = await validateRemoteUrl(new URL(location, current), firecrawlApiSsrfOptions(options, allowLoopback));
214
230
  if (next.origin !== current.origin) headers = withoutSensitiveHeaders(headers);
215
231
  current = next;
216
232
  }
@@ -1,6 +1,7 @@
1
- import { existsSync, readFileSync, rmSync, statSync, readdirSync, openSync, readSync, closeSync, realpathSync } from "node:fs";
1
+ import { closeSync, existsSync, lstatSync, mkdirSync, openSync, readFileSync, readSync, readdirSync, realpathSync, rmSync, statSync, unlinkSync } from "node:fs";
2
2
  import { execFile, spawn, type ChildProcess } from "node:child_process";
3
- import { extname, join, resolve as resolvePath, sep as pathSep } from "node:path";
3
+ import { createHash } from "node:crypto";
4
+ import { basename, dirname, extname, join, resolve as resolvePath, sep as pathSep } from "node:path";
4
5
  import { activityMonitor } from "./activity.ts";
5
6
  import type { ExtractedContent } from "./extract.ts";
6
7
  import { checkGhAvailable, checkRepoSize, fetchViaApi, showGhHint } from "./github-api.ts";
@@ -39,10 +40,15 @@ export interface GitHubUrlInfo {
39
40
  }
40
41
 
41
42
  interface CachedClone {
42
- localPath: string;
43
+ destination: CloneDestination;
43
44
  clonePromise: Promise<string | null>;
44
45
  }
45
46
 
47
+ interface CloneDestination {
48
+ rootPath: string;
49
+ localPath: string;
50
+ }
51
+
46
52
  interface GitHubCloneConfig {
47
53
  enabled: boolean;
48
54
  maxRepoSizeMB: number;
@@ -137,20 +143,21 @@ export function parseGitHubUrl(url: string): GitHubUrlInfo | null {
137
143
  const host = parsed.hostname.toLowerCase();
138
144
  if (host !== "github.com" && host !== "www.github.com") return null;
139
145
 
140
- const segments = parsed.pathname
141
- .split("/")
142
- .filter(Boolean)
143
- .map((segment) => {
144
- try {
145
- return decodeURIComponent(segment);
146
- } catch {
147
- return segment;
148
- }
149
- });
146
+ const segments: string[] = [];
147
+ for (const segment of parsed.pathname.split("/").filter(Boolean)) {
148
+ try {
149
+ segments.push(decodeURIComponent(segment));
150
+ } catch {
151
+ return null;
152
+ }
153
+ }
150
154
  if (segments.length < 2) return null;
151
155
 
152
156
  const owner = segments[0];
153
157
  const repo = segments[1].replace(/\.git$/, "");
158
+ if (!/^[A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?$/.test(owner)) return null;
159
+ if (owner.includes("--")) return null;
160
+ if (!/^[A-Za-z0-9._-]{1,100}$/.test(repo) || repo === "." || repo === "..") return null;
154
161
 
155
162
  if (NON_CODE_SEGMENTS.has(segments[2]?.toLowerCase())) return null;
156
163
 
@@ -163,6 +170,7 @@ export function parseGitHubUrl(url: string): GitHubUrlInfo | null {
163
170
  if (segments.length < 4) return null;
164
171
 
165
172
  const ref = segments[3];
173
+ if (ref.length === 0 || ref.length > 1024 || /[\0-\x1f\x7f]/.test(ref)) return null;
166
174
  const refIsFullSha = /^[0-9a-f]{40}$/.test(ref);
167
175
  const pathParts = segments.slice(4);
168
176
  const path = pathParts.length > 0 ? pathParts.join("/") : "";
@@ -181,9 +189,32 @@ function cacheKey(owner: string, repo: string, ref?: string): string {
181
189
  return ref ? `${owner}/${repo}@${ref}` : `${owner}/${repo}`;
182
190
  }
183
191
 
184
- function cloneDir(config: GitHubCloneConfig, owner: string, repo: string, ref?: string): string {
185
- const dirName = ref ? `${repo}@${ref}` : repo;
186
- return join(config.clonePath, owner, dirName);
192
+ function cloneDestination(config: GitHubCloneConfig, owner: string, repo: string, ref?: string): CloneDestination | null {
193
+ try {
194
+ mkdirSync(resolvePath(config.clonePath), { recursive: true });
195
+ const rootPath = realpathSync(resolvePath(config.clonePath));
196
+ const digest = createHash("sha256").update(JSON.stringify([owner, repo, ref ?? null])).digest("hex");
197
+ const localPath = resolvePath(rootPath, digest);
198
+ if (dirname(localPath) !== rootPath) return null;
199
+ return { rootPath, localPath };
200
+ } catch {
201
+ return null;
202
+ }
203
+ }
204
+
205
+ function removeCloneDestination(destination: CloneDestination): boolean {
206
+ const rootPath = resolvePath(destination.rootPath);
207
+ const localPath = resolvePath(destination.localPath);
208
+ if (dirname(localPath) !== rootPath || !/^[0-9a-f]{64}$/.test(basename(localPath))) return false;
209
+ try {
210
+ const entry = lstatSync(localPath);
211
+ if (entry.isSymbolicLink()) unlinkSync(localPath);
212
+ else rmSync(localPath, { recursive: true, force: true });
213
+ return true;
214
+ } catch (err) {
215
+ if (err && typeof err === "object" && "code" in err && err.code === "ENOENT") return true;
216
+ return false;
217
+ }
187
218
  }
188
219
 
189
220
  const PROCESS_KILL_GRACE_MS = 3000;
@@ -225,8 +256,9 @@ function terminateProcessTree(child: ChildProcess): void {
225
256
  forceKill.unref();
226
257
  }
227
258
 
228
- function execClone(args: string[], localPath: string, timeoutMs: number, signal?: AbortSignal): Promise<string | null> {
259
+ function execClone(args: string[], destination: CloneDestination, timeoutMs: number, signal?: AbortSignal): Promise<string | null> {
229
260
  return new Promise((resolve) => {
261
+ const { localPath } = destination;
230
262
  let settled = false;
231
263
  let timeout: ReturnType<typeof setTimeout> | undefined;
232
264
  let onAbort: (() => void) | undefined;
@@ -238,10 +270,7 @@ function execClone(args: string[], localPath: string, timeoutMs: number, signal?
238
270
  if (signal && onAbort) signal.removeEventListener("abort", onAbort);
239
271
 
240
272
  if (!success) {
241
- try {
242
- rmSync(localPath, { recursive: true, force: true });
243
- } catch {
244
- }
273
+ removeCloneDestination(destination);
245
274
  resolve(null);
246
275
  return;
247
276
  }
@@ -282,14 +311,11 @@ async function cloneRepo(
282
311
  repo: string,
283
312
  ref: string | undefined,
284
313
  config: GitHubCloneConfig,
314
+ destination: CloneDestination,
285
315
  signal?: AbortSignal,
286
316
  ): Promise<string | null> {
287
- const localPath = cloneDir(config, owner, repo, ref);
288
-
289
- try {
290
- rmSync(localPath, { recursive: true, force: true });
291
- } catch {
292
- }
317
+ const { localPath } = destination;
318
+ if (!removeCloneDestination(destination)) return null;
293
319
 
294
320
  const timeoutMs = config.cloneTimeoutSeconds * 1000;
295
321
  const hasGh = await checkGhAvailable();
@@ -297,7 +323,7 @@ async function cloneRepo(
297
323
  if (hasGh) {
298
324
  const args = ["gh", "repo", "clone", `${owner}/${repo}`, localPath, "--", "--depth", "1", "--single-branch"];
299
325
  if (ref) args.push("--branch", ref);
300
- return execClone(args, localPath, timeoutMs, signal);
326
+ return execClone(args, destination, timeoutMs, signal);
301
327
  }
302
328
 
303
329
  showGhHint();
@@ -306,7 +332,7 @@ async function cloneRepo(
306
332
  const args = ["git", "clone", "--depth", "1", "--single-branch"];
307
333
  if (ref) args.push("--branch", ref);
308
334
  args.push(gitUrl, localPath);
309
- return execClone(args, localPath, timeoutMs, signal);
335
+ return execClone(args, destination, timeoutMs, signal);
310
336
  }
311
337
 
312
338
  function isBinaryFile(filePath: string): boolean {
@@ -671,9 +697,15 @@ export async function extractGitHub(
671
697
  return cachedResult;
672
698
  }
673
699
 
674
- const clonePromise = cloneRepo(owner, repo, info.ref, config, signal);
675
- const localPath = cloneDir(config, owner, repo, info.ref);
676
- cloneCache.set(key, { localPath, clonePromise });
700
+ const destination = cloneDestination(config, owner, repo, info.ref);
701
+ if (!destination) {
702
+ const apiFallback = await fetchViaApi(url, owner, repo, info);
703
+ if (apiFallback) activityMonitor.logComplete(activityId, 200);
704
+ else activityMonitor.logError(activityId, "invalid clone destination");
705
+ return apiFallback;
706
+ }
707
+ const clonePromise = cloneRepo(owner, repo, info.ref, config, destination, signal);
708
+ cloneCache.set(key, { destination, clonePromise });
677
709
 
678
710
  const result = await clonePromise;
679
711
  if (signal?.aborted) {
@@ -707,10 +739,7 @@ export async function extractGitHub(
707
739
 
708
740
  export function clearCloneCache(): void {
709
741
  for (const entry of cloneCache.values()) {
710
- try {
711
- rmSync(entry.localPath, { recursive: true, force: true });
712
- } catch {
713
- }
742
+ removeCloneDestination(entry.destination);
714
743
  }
715
744
  cloneCache.clear();
716
745
  cachedConfig = null;