@herbertgao/pi-extensions 2026.8.7 → 2026.8.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -5
- package/node_modules/@herbertgao/pi-cc-extensions/README.en.md +1 -1
- package/node_modules/@herbertgao/pi-cc-extensions/README.md +1 -1
- package/node_modules/@herbertgao/pi-cc-extensions/extensions/feature/compact-thinking.ts +63 -15
- package/node_modules/@herbertgao/pi-cc-extensions/extensions/renderer/message-display.ts +78 -16
- package/node_modules/@herbertgao/pi-cc-extensions/extensions/renderer/mouse/hover.ts +13 -0
- package/node_modules/@herbertgao/pi-cc-extensions/extensions/renderer/mouse/interaction.ts +17 -6
- package/node_modules/@herbertgao/pi-cc-extensions/package.json +3 -3
- package/node_modules/@juicesharp/rpiv-ask-user-question/README.md +4 -4
- package/node_modules/@juicesharp/rpiv-ask-user-question/ask-user-question.ts +154 -96
- package/node_modules/@juicesharp/rpiv-ask-user-question/docs/hosts.md +3 -0
- package/node_modules/@juicesharp/rpiv-ask-user-question/docs/keyboard.md +10 -2
- package/node_modules/@juicesharp/rpiv-ask-user-question/docs/localization.md +7 -5
- package/node_modules/@juicesharp/rpiv-ask-user-question/docs/tool-schema.md +11 -5
- package/node_modules/@juicesharp/rpiv-ask-user-question/locales/en.json +3 -0
- package/node_modules/@juicesharp/rpiv-ask-user-question/package.json +2 -2
- package/node_modules/@juicesharp/rpiv-ask-user-question/state/key-router.ts +137 -96
- package/node_modules/@juicesharp/rpiv-ask-user-question/state/questionnaire-session.ts +44 -32
- package/node_modules/@juicesharp/rpiv-ask-user-question/state/selectors/projections.ts +22 -15
- package/node_modules/@juicesharp/rpiv-ask-user-question/state/state-reducer.ts +11 -1
- package/node_modules/@juicesharp/rpiv-ask-user-question/tool/response-envelope.ts +11 -0
- package/node_modules/@juicesharp/rpiv-ask-user-question/tool/types.ts +11 -0
- package/node_modules/@juicesharp/rpiv-ask-user-question/view/components/multi-select-view.ts +31 -17
- package/node_modules/@juicesharp/rpiv-ask-user-question/view/components/preview/preview-block-renderer.ts +15 -6
- package/node_modules/@juicesharp/rpiv-ask-user-question/view/components/wrapping-select.ts +15 -6
- package/node_modules/@juicesharp/rpiv-ask-user-question/view/dialog-builder.ts +56 -34
- package/node_modules/@juicesharp/rpiv-ask-user-question/view/tab-content-strategy.ts +50 -4
- package/node_modules/@narumitw/pi-btw/package.json +2 -2
- package/node_modules/@pi-plugins/fast-mode/dist/index.mjs +44 -1856
- package/node_modules/@pi-plugins/fast-mode/dist/index.mjs.map +1 -1
- package/node_modules/@pi-plugins/fast-mode/package.json +4 -6
- package/node_modules/pi-lens/CHANGELOG.md +90 -0
- package/node_modules/pi-lens/README.md +106 -0
- package/node_modules/pi-lens/dist/clients/bash-file-access.js +97 -6
- package/node_modules/pi-lens/dist/clients/blocker-past-eof.js +16 -6
- package/node_modules/pi-lens/dist/clients/bounded-telemetry.js +14 -0
- package/node_modules/pi-lens/dist/clients/cache-observability.js +353 -76
- package/node_modules/pi-lens/dist/clients/degradation-ledger.js +22 -1
- package/node_modules/pi-lens/dist/clients/demoted-finding-render.js +155 -0
- package/node_modules/pi-lens/dist/clients/deps/pi-tui.js +8 -2
- package/node_modules/pi-lens/dist/clients/deps/typebox.js +8 -3
- package/node_modules/pi-lens/dist/clients/dispatch/runners/detekt.js +9 -6
- package/node_modules/pi-lens/dist/clients/dispatch/runners/hadolint.js +9 -7
- package/node_modules/pi-lens/dist/clients/dispatch/runners/htmlhint.js +8 -6
- package/node_modules/pi-lens/dist/clients/dispatch/runners/markdownlint.js +8 -10
- package/node_modules/pi-lens/dist/clients/dispatch/runners/mypy.js +10 -10
- package/node_modules/pi-lens/dist/clients/dispatch/runners/oxlint.js +20 -5
- package/node_modules/pi-lens/dist/clients/dispatch/runners/phpstan.js +44 -11
- package/node_modules/pi-lens/dist/clients/dispatch/runners/spellcheck.js +10 -15
- package/node_modules/pi-lens/dist/clients/dispatch/runners/sqlfluff.js +19 -11
- package/node_modules/pi-lens/dist/clients/dispatch/runners/stylelint.js +35 -13
- package/node_modules/pi-lens/dist/clients/dispatch/runners/swiftlint.js +8 -8
- package/node_modules/pi-lens/dist/clients/dispatch/runners/taplo.js +110 -23
- package/node_modules/pi-lens/dist/clients/dispatch/runners/tflint.js +12 -7
- package/node_modules/pi-lens/dist/clients/dispatch/runners/utils/tool-failure.js +117 -1
- package/node_modules/pi-lens/dist/clients/dispatch/runners/vale.js +16 -14
- package/node_modules/pi-lens/dist/clients/dispatch/runners/yamllint.js +8 -9
- package/node_modules/pi-lens/dist/clients/finding-delivery-gate.js +22 -0
- package/node_modules/pi-lens/dist/clients/jscpd-client.js +5 -2
- package/node_modules/pi-lens/dist/clients/language-profile.js +7 -2
- package/node_modules/pi-lens/dist/clients/lsp/cascade-tier.js +27 -2
- package/node_modules/pi-lens/dist/clients/lsp/client.js +144 -5
- package/node_modules/pi-lens/dist/clients/lsp/index.js +151 -6
- package/node_modules/pi-lens/dist/clients/module-report.js +14 -1
- package/node_modules/pi-lens/dist/clients/ndjson-logger.js +114 -23
- package/node_modules/pi-lens/dist/clients/project-diagnostics/scanner.js +18 -9
- package/node_modules/pi-lens/dist/clients/project-report.js +62 -27
- package/node_modules/pi-lens/dist/clients/read-guard-logger.js +7 -2
- package/node_modules/pi-lens/dist/clients/read-guard.js +88 -19
- package/node_modules/pi-lens/dist/clients/review-graph/builder.js +194 -31
- package/node_modules/pi-lens/dist/clients/review-graph/revision-drift.js +21 -0
- package/node_modules/pi-lens/dist/clients/review-graph/workspace-modules.js +6 -3
- package/node_modules/pi-lens/dist/clients/runtime-coordinator.js +44 -0
- package/node_modules/pi-lens/dist/clients/runtime-session.js +41 -0
- package/node_modules/pi-lens/dist/clients/runtime-turn.js +47 -2
- package/node_modules/pi-lens/dist/clients/session-event-guard.js +184 -0
- package/node_modules/pi-lens/dist/clients/source-filter.js +6 -2
- package/node_modules/pi-lens/dist/clients/startup-scan.js +5 -2
- package/node_modules/pi-lens/dist/clients/tree-sitter-logger.js +1 -0
- package/node_modules/pi-lens/dist/index.js +1224 -393
- package/node_modules/pi-lens/dist/tools/lens-diagnostics.js +10 -3
- package/node_modules/pi-lens/package.json +18 -5
- package/node_modules/pi-lens/scripts/install-selftest.mjs +99 -11
- package/node_modules/pi-lens/scripts/lib/host-provided-deps.mjs +75 -0
- package/node_modules/pi-lens/scripts/lib/warm-loader-cache.mjs +285 -0
- package/node_modules/pi-lens/scripts/warm-loader-cache.mjs +233 -0
- package/node_modules/pi-web-access/CHANGELOG.md +14 -0
- package/node_modules/pi-web-access/README.md +6 -5
- package/node_modules/pi-web-access/chrome-cookies.ts +86 -15
- package/node_modules/pi-web-access/data-uri-sanitize.ts +406 -0
- package/node_modules/pi-web-access/extract.ts +12 -2
- package/node_modules/pi-web-access/firecrawl.ts +18 -2
- package/node_modules/pi-web-access/github-extract.ts +65 -36
- package/node_modules/pi-web-access/index.ts +19 -4
- package/node_modules/pi-web-access/openai-search.ts +17 -6
- package/node_modules/pi-web-access/package.json +1 -1
- package/node_modules/pi-web-access/pdf-extract.ts +18 -5
- package/node_modules/pi-web-access/ssrf-protection.ts +12 -2
- package/package.json +9 -9
|
@@ -0,0 +1,406 @@
|
|
|
1
|
+
import { Buffer } from "node:buffer";
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
3
|
+
|
|
4
|
+
// Replaces inline RFC 2397 data: URIs in extracted text with explicit bounded
|
|
5
|
+
// omission markers so base64 payloads never reach model-visible tool results,
|
|
6
|
+
// the fetch cache, or session persistence. Carved out of the original
|
|
7
|
+
// persistence-safety layer; only the text-sanitization path is retained.
|
|
8
|
+
|
|
9
|
+
const MAX_DATA_URI_HEADER_CHARS = 1024;
|
|
10
|
+
const MAX_MIME_CHARS = 127;
|
|
11
|
+
const MAX_MARKER_SOURCE_CHARS = 180;
|
|
12
|
+
|
|
13
|
+
export type DataUriEncoding = "base64" | "percent-encoded";
|
|
14
|
+
export type DigestBasis = "decoded" | "encoded";
|
|
15
|
+
|
|
16
|
+
export interface DataUriOmission {
|
|
17
|
+
ordinal: number;
|
|
18
|
+
sourcePath: string;
|
|
19
|
+
mimeType: string;
|
|
20
|
+
encoding: DataUriEncoding;
|
|
21
|
+
encodedBytes: number;
|
|
22
|
+
decodedBytes: number | null;
|
|
23
|
+
decodeError?: string;
|
|
24
|
+
sha256: string;
|
|
25
|
+
digestBasis: DigestBasis;
|
|
26
|
+
retrieval: "not-retained";
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
interface InternalDataUriOmission extends DataUriOmission {
|
|
31
|
+
markerStart: number;
|
|
32
|
+
markerEnd: number;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
interface SanitizationState {
|
|
36
|
+
nextOrdinal: number;
|
|
37
|
+
omissions: InternalDataUriOmission[];
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
function sha256Bytes(value: string | Uint8Array): string {
|
|
42
|
+
return createHash("sha256").update(value).digest("hex");
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
function asciiLowerCode(code: number): number {
|
|
47
|
+
return code >= 65 && code <= 90 ? code + 32 : code;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function matchesDataScheme(text: string, index: number): boolean {
|
|
51
|
+
return index + 5 <= text.length
|
|
52
|
+
&& asciiLowerCode(text.charCodeAt(index)) === 100
|
|
53
|
+
&& asciiLowerCode(text.charCodeAt(index + 1)) === 97
|
|
54
|
+
&& asciiLowerCode(text.charCodeAt(index + 2)) === 116
|
|
55
|
+
&& asciiLowerCode(text.charCodeAt(index + 3)) === 97
|
|
56
|
+
&& text.charCodeAt(index + 4) === 58;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function isSchemeBoundary(text: string, index: number): boolean {
|
|
60
|
+
if (index === 0) return true;
|
|
61
|
+
const code = text.charCodeAt(index - 1);
|
|
62
|
+
const alphaNumeric = (code >= 48 && code <= 57)
|
|
63
|
+
|| (code >= 65 && code <= 90)
|
|
64
|
+
|| (code >= 97 && code <= 122);
|
|
65
|
+
return !alphaNumeric && code !== 43 && code !== 45 && code !== 46;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function findNextDataScheme(text: string, from: number): number {
|
|
69
|
+
for (let i = from; i + 5 <= text.length; i++) {
|
|
70
|
+
if (matchesDataScheme(text, i) && isSchemeBoundary(text, i)) return i;
|
|
71
|
+
}
|
|
72
|
+
return -1;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
type DataUriEnclosure =
|
|
76
|
+
| { kind: "quote"; close: "\"" | "'" | "`" }
|
|
77
|
+
| { kind: "parentheses" }
|
|
78
|
+
| { kind: "angle" }
|
|
79
|
+
| null;
|
|
80
|
+
|
|
81
|
+
interface ScannedDataUriCandidate {
|
|
82
|
+
end: number;
|
|
83
|
+
comma: number;
|
|
84
|
+
enclosure: DataUriEnclosure;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function dataUriEnclosure(text: string, start: number): DataUriEnclosure {
|
|
88
|
+
if (start === 0) return null;
|
|
89
|
+
const immediateIndex = start - 1;
|
|
90
|
+
const immediate = text[immediateIndex];
|
|
91
|
+
if (immediate === "\"" || immediate === "'" || immediate === "`") {
|
|
92
|
+
const beforeQuote = immediateIndex > 0 ? text[immediateIndex - 1] : "";
|
|
93
|
+
const likelyOpeningQuote = immediateIndex === 0
|
|
94
|
+
|| beforeQuote === "="
|
|
95
|
+
|| beforeQuote === "("
|
|
96
|
+
|| beforeQuote === "["
|
|
97
|
+
|| beforeQuote === "{"
|
|
98
|
+
|| beforeQuote === ":"
|
|
99
|
+
|| beforeQuote === ","
|
|
100
|
+
|| /\s/.test(beforeQuote);
|
|
101
|
+
if (likelyOpeningQuote) return { kind: "quote", close: immediate };
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
let openerIndex = immediateIndex;
|
|
105
|
+
while (openerIndex >= 0 && (text[openerIndex] === " " || text[openerIndex] === "\t")) openerIndex--;
|
|
106
|
+
const opener = text[openerIndex];
|
|
107
|
+
if (opener === "(") return { kind: "parentheses" };
|
|
108
|
+
if (opener === "<") return { kind: "angle" };
|
|
109
|
+
return null;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function isBareDataUriTerminator(ch: string): boolean {
|
|
113
|
+
const code = ch.charCodeAt(0);
|
|
114
|
+
return code <= 32
|
|
115
|
+
|| code === 127
|
|
116
|
+
|| ch === "\""
|
|
117
|
+
|| ch === "`"
|
|
118
|
+
|| ch === "<"
|
|
119
|
+
|| ch === ">";
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** Scan one candidate exactly once. Enclosures consume malformed whitespace and
|
|
123
|
+
* balanced inner parentheses so invalid payload suffixes cannot escape. */
|
|
124
|
+
function scanDataUriCandidate(text: string, start: number): ScannedDataUriCandidate {
|
|
125
|
+
const enclosure = dataUriEnclosure(text, start);
|
|
126
|
+
let comma = -1;
|
|
127
|
+
let depth = enclosure?.kind === "parentheses" ? 1 : 0;
|
|
128
|
+
let index = start + 5;
|
|
129
|
+
|
|
130
|
+
for (; index < text.length; index++) {
|
|
131
|
+
const ch = text[index];
|
|
132
|
+
if (enclosure?.kind === "quote") {
|
|
133
|
+
if (ch === enclosure.close) break;
|
|
134
|
+
} else if (enclosure?.kind === "parentheses") {
|
|
135
|
+
if (ch === "(") {
|
|
136
|
+
depth++;
|
|
137
|
+
} else if (ch === ")") {
|
|
138
|
+
depth--;
|
|
139
|
+
if (depth === 0) break;
|
|
140
|
+
}
|
|
141
|
+
} else if (enclosure?.kind === "angle") {
|
|
142
|
+
if (ch === ">") break;
|
|
143
|
+
} else if (isBareDataUriTerminator(ch)) {
|
|
144
|
+
break;
|
|
145
|
+
}
|
|
146
|
+
if (comma < 0 && ch === ",") comma = index;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
return { end: index, comma, enclosure };
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function headerEndsWithBase64(text: string, headerStart: number, comma: number): boolean {
|
|
153
|
+
let end = comma;
|
|
154
|
+
while (end > headerStart && (text[end - 1] === " " || text[end - 1] === "\t")) end--;
|
|
155
|
+
const token = "base64";
|
|
156
|
+
if (end - headerStart < token.length + 1) return false;
|
|
157
|
+
const tokenStart = end - token.length;
|
|
158
|
+
for (let i = 0; i < token.length; i++) {
|
|
159
|
+
if (asciiLowerCode(text.charCodeAt(tokenStart + i)) !== token.charCodeAt(i)) return false;
|
|
160
|
+
}
|
|
161
|
+
return tokenStart > headerStart && text[tokenStart - 1] === ";";
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function isMimeTokenChar(code: number): boolean {
|
|
165
|
+
if (code < 33 || code > 126) return false;
|
|
166
|
+
// RFC 2045 token = printable ASCII excluding tspecials. Slash is handled
|
|
167
|
+
// separately as the single type/subtype separator.
|
|
168
|
+
switch (code) {
|
|
169
|
+
case 34: // "
|
|
170
|
+
case 40: // (
|
|
171
|
+
case 41: // )
|
|
172
|
+
case 44: // ,
|
|
173
|
+
case 47: // /
|
|
174
|
+
case 58: // :
|
|
175
|
+
case 59: // ;
|
|
176
|
+
case 60: // <
|
|
177
|
+
case 61: // =
|
|
178
|
+
case 62: // >
|
|
179
|
+
case 63: // ?
|
|
180
|
+
case 64: // @
|
|
181
|
+
case 91: // [
|
|
182
|
+
case 92: // backslash
|
|
183
|
+
case 93: // ]
|
|
184
|
+
return false;
|
|
185
|
+
default:
|
|
186
|
+
return true;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
function normalizeMimeType(text: string, headerStart: number, comma: number): string {
|
|
191
|
+
let end = headerStart;
|
|
192
|
+
while (end < comma && text[end] !== ";") end++;
|
|
193
|
+
if (end === headerStart) return "text/plain";
|
|
194
|
+
if (end - headerStart > MAX_MIME_CHARS) return "application/octet-stream";
|
|
195
|
+
|
|
196
|
+
let slashCount = 0;
|
|
197
|
+
for (let i = headerStart; i < end; i++) {
|
|
198
|
+
const code = text.charCodeAt(i);
|
|
199
|
+
if (code === 47) {
|
|
200
|
+
slashCount++;
|
|
201
|
+
if (i === headerStart || i === end - 1) return "application/octet-stream";
|
|
202
|
+
continue;
|
|
203
|
+
}
|
|
204
|
+
if (!isMimeTokenChar(code)) return "application/octet-stream";
|
|
205
|
+
}
|
|
206
|
+
if (slashCount !== 1) return "application/octet-stream";
|
|
207
|
+
return text.slice(headerStart, end).toLowerCase();
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
function hexValue(code: number): number {
|
|
211
|
+
if (code >= 48 && code <= 57) return code - 48;
|
|
212
|
+
if (code >= 65 && code <= 70) return code - 55;
|
|
213
|
+
if (code >= 97 && code <= 102) return code - 87;
|
|
214
|
+
return -1;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
function decodePercentEncoded(payload: string): { bytes: Buffer } | { error: string } {
|
|
218
|
+
const output = Buffer.allocUnsafe(Buffer.byteLength(payload, "utf8"));
|
|
219
|
+
let outputOffset = 0;
|
|
220
|
+
let cursor = 0;
|
|
221
|
+
|
|
222
|
+
while (cursor < payload.length) {
|
|
223
|
+
const percent = payload.indexOf("%", cursor);
|
|
224
|
+
const runEnd = percent < 0 ? payload.length : percent;
|
|
225
|
+
if (runEnd > cursor) {
|
|
226
|
+
const run = Buffer.from(payload.slice(cursor, runEnd), "utf8");
|
|
227
|
+
run.copy(output, outputOffset);
|
|
228
|
+
outputOffset += run.length;
|
|
229
|
+
}
|
|
230
|
+
if (percent < 0) break;
|
|
231
|
+
if (percent + 2 >= payload.length) return { error: "invalid-percent-escape" };
|
|
232
|
+
const high = hexValue(payload.charCodeAt(percent + 1));
|
|
233
|
+
const low = hexValue(payload.charCodeAt(percent + 2));
|
|
234
|
+
if (high < 0 || low < 0) return { error: "invalid-percent-escape" };
|
|
235
|
+
output[outputOffset++] = (high << 4) | low;
|
|
236
|
+
cursor = percent + 3;
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
return { bytes: output.subarray(0, outputOffset) };
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
function isBase64AlphabetByte(byte: number): boolean {
|
|
243
|
+
return (byte >= 65 && byte <= 90)
|
|
244
|
+
|| (byte >= 97 && byte <= 122)
|
|
245
|
+
|| (byte >= 48 && byte <= 57)
|
|
246
|
+
|| byte === 43
|
|
247
|
+
|| byte === 47;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
function decodeBase64Payload(payload: string): { bytes: Buffer } | { error: string } {
|
|
251
|
+
const percentDecoded = decodePercentEncoded(payload);
|
|
252
|
+
if ("error" in percentDecoded) return percentDecoded;
|
|
253
|
+
const encoded = percentDecoded.bytes;
|
|
254
|
+
let paddingStart = encoded.length;
|
|
255
|
+
let paddingCount = 0;
|
|
256
|
+
|
|
257
|
+
for (let i = 0; i < encoded.length; i++) {
|
|
258
|
+
const byte = encoded[i];
|
|
259
|
+
if (byte > 127) return { error: "non-ascii-base64" };
|
|
260
|
+
if (byte === 61) {
|
|
261
|
+
if (paddingStart === encoded.length) paddingStart = i;
|
|
262
|
+
paddingCount++;
|
|
263
|
+
continue;
|
|
264
|
+
}
|
|
265
|
+
if (paddingStart !== encoded.length) return { error: "invalid-base64-padding" };
|
|
266
|
+
if (!isBase64AlphabetByte(byte)) return { error: "invalid-base64-character" };
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
if (paddingCount > 2) return { error: "invalid-base64-padding" };
|
|
270
|
+
if (paddingCount > 0 && encoded.length % 4 !== 0) return { error: "invalid-base64-padding" };
|
|
271
|
+
if (paddingCount === 0 && encoded.length % 4 === 1) return { error: "invalid-base64-length" };
|
|
272
|
+
if (paddingCount === 1 && paddingStart % 4 !== 3) return { error: "invalid-base64-padding" };
|
|
273
|
+
if (paddingCount === 2 && paddingStart % 4 !== 2) return { error: "invalid-base64-padding" };
|
|
274
|
+
|
|
275
|
+
return { bytes: Buffer.from(encoded.toString("ascii"), "base64") };
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
function safeMarkerSource(sourcePath: string): string {
|
|
279
|
+
const source = sourcePath || "value";
|
|
280
|
+
let safe = "";
|
|
281
|
+
for (const ch of source) {
|
|
282
|
+
const code = ch.charCodeAt(0);
|
|
283
|
+
const allowed = (code >= 48 && code <= 57)
|
|
284
|
+
|| (code >= 65 && code <= 90)
|
|
285
|
+
|| (code >= 97 && code <= 122)
|
|
286
|
+
|| ch === "."
|
|
287
|
+
|| ch === "_"
|
|
288
|
+
|| ch === "-"
|
|
289
|
+
|| ch === "["
|
|
290
|
+
|| ch === "]"
|
|
291
|
+
|| ch === "*"
|
|
292
|
+
|| ch === "$";
|
|
293
|
+
safe += allowed ? ch : "_";
|
|
294
|
+
}
|
|
295
|
+
if (safe.length <= MAX_MARKER_SOURCE_CHARS) return safe;
|
|
296
|
+
const suffix = sha256Bytes(source).slice(0, 12);
|
|
297
|
+
return `${safe.slice(0, MAX_MARKER_SOURCE_CHARS - suffix.length - 1)}~${suffix}`;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
function buildDataUriMarker(omission: DataUriOmission): string {
|
|
301
|
+
const decoded = omission.decodedBytes === null ? "unknown" : String(omission.decodedBytes);
|
|
302
|
+
const decodeError = omission.decodeError ? `; decodeError=${omission.decodeError}` : "";
|
|
303
|
+
return `[pi-web-access inline data URI omitted; ordinal=${omission.ordinal}; source=${omission.sourcePath}; mime=${omission.mimeType}; encoding=${omission.encoding}; encodedBytes=${omission.encodedBytes}; decodedBytes=${decoded}${decodeError}; sha256=${omission.sha256}; digestBasis=${omission.digestBasis}; retrieval=not-retained]`;
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
function sanitizeTextInternal(text: string, sourcePath: string, state: SanitizationState): string {
|
|
307
|
+
let scanFrom = 0;
|
|
308
|
+
let retainedFrom = 0;
|
|
309
|
+
let outputLength = 0;
|
|
310
|
+
const pieces: string[] = [];
|
|
311
|
+
|
|
312
|
+
while (scanFrom < text.length) {
|
|
313
|
+
const start = findNextDataScheme(text, scanFrom);
|
|
314
|
+
if (start < 0) break;
|
|
315
|
+
const candidate = scanDataUriCandidate(text, start);
|
|
316
|
+
const end = candidate.end;
|
|
317
|
+
const headerStart = start + 5;
|
|
318
|
+
const missingComma = candidate.comma < 0;
|
|
319
|
+
// A bare prose label such as "data: value" is not an inline URI. Enclosed
|
|
320
|
+
// or non-empty comma-less candidates are malformed URIs and are removed.
|
|
321
|
+
if (missingComma && end === headerStart && candidate.enclosure === null) {
|
|
322
|
+
scanFrom = end;
|
|
323
|
+
continue;
|
|
324
|
+
}
|
|
325
|
+
const headerEnd = missingComma ? end : candidate.comma;
|
|
326
|
+
const headerLength = headerEnd - headerStart;
|
|
327
|
+
const encoding: DataUriEncoding = headerEndsWithBase64(text, headerStart, headerEnd)
|
|
328
|
+
? "base64"
|
|
329
|
+
: "percent-encoded";
|
|
330
|
+
const mimeType = normalizeMimeType(text, headerStart, headerEnd);
|
|
331
|
+
// Without the required comma, the post-scheme bytes are ambiguous. Count
|
|
332
|
+
// and digest the whole omitted segment on the encoded basis rather than
|
|
333
|
+
// retaining any header/payload-looking suffix.
|
|
334
|
+
const payload = missingComma
|
|
335
|
+
? text.slice(headerStart, end)
|
|
336
|
+
: text.slice(candidate.comma + 1, end);
|
|
337
|
+
const encodedBytes = Buffer.byteLength(payload, "utf8");
|
|
338
|
+
|
|
339
|
+
let decodedBytes: number | null = null;
|
|
340
|
+
let decodeError: string | undefined;
|
|
341
|
+
let digestBasis: DigestBasis = "encoded";
|
|
342
|
+
let digest = "";
|
|
343
|
+
|
|
344
|
+
if (missingComma) {
|
|
345
|
+
decodeError = "missing-comma";
|
|
346
|
+
digest = sha256Bytes(payload);
|
|
347
|
+
} else if (headerLength > MAX_DATA_URI_HEADER_CHARS) {
|
|
348
|
+
decodeError = "header-too-long";
|
|
349
|
+
digest = sha256Bytes(payload);
|
|
350
|
+
} else {
|
|
351
|
+
const decoded = encoding === "base64"
|
|
352
|
+
? decodeBase64Payload(payload)
|
|
353
|
+
: decodePercentEncoded(payload);
|
|
354
|
+
if ("error" in decoded) {
|
|
355
|
+
decodeError = decoded.error;
|
|
356
|
+
digest = sha256Bytes(payload);
|
|
357
|
+
} else {
|
|
358
|
+
decodedBytes = decoded.bytes.length;
|
|
359
|
+
digestBasis = "decoded";
|
|
360
|
+
digest = sha256Bytes(decoded.bytes);
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
const omission: DataUriOmission = {
|
|
365
|
+
ordinal: state.nextOrdinal++,
|
|
366
|
+
sourcePath: safeMarkerSource(sourcePath),
|
|
367
|
+
mimeType,
|
|
368
|
+
encoding,
|
|
369
|
+
encodedBytes,
|
|
370
|
+
decodedBytes,
|
|
371
|
+
...(decodeError ? { decodeError } : {}),
|
|
372
|
+
sha256: digest,
|
|
373
|
+
digestBasis,
|
|
374
|
+
retrieval: "not-retained",
|
|
375
|
+
};
|
|
376
|
+
const marker = buildDataUriMarker(omission);
|
|
377
|
+
const prefix = text.slice(retainedFrom, start);
|
|
378
|
+
pieces.push(prefix, marker);
|
|
379
|
+
outputLength += prefix.length;
|
|
380
|
+
const markerStart = outputLength;
|
|
381
|
+
outputLength += marker.length;
|
|
382
|
+
state.omissions.push({ ...omission, markerStart, markerEnd: outputLength });
|
|
383
|
+
retainedFrom = end;
|
|
384
|
+
scanFrom = end;
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
if (state.omissions.length === 0 && retainedFrom === 0) return text;
|
|
388
|
+
pieces.push(text.slice(retainedFrom));
|
|
389
|
+
return pieces.join("");
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
/** Replace every recognized RFC 2397-style inline data URI in one string. */
|
|
394
|
+
export function sanitizeInlineDataUris(
|
|
395
|
+
text: string,
|
|
396
|
+
sourcePath: string,
|
|
397
|
+
): { text: string; omissions: DataUriOmission[] } {
|
|
398
|
+
const state: SanitizationState = { nextOrdinal: 1, omissions: [] };
|
|
399
|
+
const sanitized = sanitizeTextInternal(text, sourcePath, state);
|
|
400
|
+
return {
|
|
401
|
+
text: sanitized,
|
|
402
|
+
omissions: state.omissions.map(({ markerStart: _start, markerEnd: _end, ...omission }) => omission),
|
|
403
|
+
};
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
/** Sanitize every string leaf in a JSON-compatible value with one ordinal sequence. */
|
|
@@ -27,6 +27,7 @@ import { formatSeconds, getWebSearchConfigPath } from "./utils.ts";
|
|
|
27
27
|
import { isImageEnabled } from "./feature-config.ts";
|
|
28
28
|
import { assertAuthFetchUrl, authFetchRedirectGuard, type AuthFetchProfile } from "./auth-fetch.ts";
|
|
29
29
|
import { getBrowserCookiesForHosts, getLastBrowserCookieDiagnostic } from "./chrome-cookies.ts";
|
|
30
|
+
import { sanitizeInlineDataUris } from "./data-uri-sanitize.ts";
|
|
30
31
|
|
|
31
32
|
const DEFAULT_TIMEOUT_MS = 30000;
|
|
32
33
|
const CONCURRENT_LIMIT = 3;
|
|
@@ -1001,7 +1002,7 @@ async function extractViaHttp(
|
|
|
1001
1002
|
const requestInit = {
|
|
1002
1003
|
signal: controller.signal,
|
|
1003
1004
|
headers: {
|
|
1004
|
-
"User-Agent": "
|
|
1005
|
+
"User-Agent": "OpenAI File Downloader, XaiImageApiFetch/1.0",
|
|
1005
1006
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
|
|
1006
1007
|
"Accept-Language": "en-US,en;q=0.9",
|
|
1007
1008
|
"Cache-Control": "no-cache",
|
|
@@ -1250,5 +1251,14 @@ export async function fetchAllContent(
|
|
|
1250
1251
|
signal?: AbortSignal,
|
|
1251
1252
|
options?: ExtractOptions,
|
|
1252
1253
|
): Promise<ExtractedContent[]> {
|
|
1253
|
-
|
|
1254
|
+
const results = await Promise.all(urls.map((url) => fetchLimit(() => extractContent(url, signal, options))));
|
|
1255
|
+
if (options?.mode === "raw") return results;
|
|
1256
|
+
// Inline data: URIs in extracted markdown would otherwise flow into tool
|
|
1257
|
+
// results and the fetch cache as opaque base64; typed thumbnail/frame image
|
|
1258
|
+
// blocks are deliberate outputs and are left untouched.
|
|
1259
|
+
return results.map((result, index) => {
|
|
1260
|
+
if (!result.content) return result;
|
|
1261
|
+
const sanitized = sanitizeInlineDataUris(result.content, `urls[${index}].content`);
|
|
1262
|
+
return sanitized.omissions.length > 0 ? { ...result, content: sanitized.text } : result;
|
|
1263
|
+
});
|
|
1254
1264
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { existsSync, readFileSync } from "node:fs";
|
|
2
|
+
import net from "node:net";
|
|
2
3
|
import { activityMonitor } from "./activity.ts";
|
|
3
4
|
import { redactCredential, resolveCredential } from "./credential-source.ts";
|
|
4
5
|
import type { ExtractedContent, ExtractOptions } from "./extract.ts";
|
|
@@ -184,6 +185,20 @@ function ssrfOptions(options?: FirecrawlExtractOptions | FirecrawlSearchOptions)
|
|
|
184
185
|
};
|
|
185
186
|
}
|
|
186
187
|
|
|
188
|
+
function isLoopbackApiUrl(url: URL): boolean {
|
|
189
|
+
const hostname = url.hostname.toLowerCase().replace(/^\[|\]$/g, "").replace(/\.$/, "");
|
|
190
|
+
if (hostname === "localhost" || hostname === "::1") return true;
|
|
191
|
+
if (net.isIP(hostname) !== 4) return false;
|
|
192
|
+
return hostname.split(".")[0] === "127";
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
function firecrawlApiSsrfOptions(
|
|
196
|
+
options: FirecrawlExtractOptions | FirecrawlSearchOptions | undefined,
|
|
197
|
+
allowLoopback: boolean,
|
|
198
|
+
): ReturnType<typeof ssrfOptions> & { allowLoopback: boolean } {
|
|
199
|
+
return { ...ssrfOptions(options), allowLoopback };
|
|
200
|
+
}
|
|
201
|
+
|
|
187
202
|
function withoutSensitiveHeaders(headers: Record<string, string>): Record<string, string> {
|
|
188
203
|
const next = { ...headers };
|
|
189
204
|
delete next.Authorization;
|
|
@@ -200,7 +215,8 @@ async function fetchFirecrawlApi(
|
|
|
200
215
|
init: { method: string; headers: Record<string, string>; body: string; signal: AbortSignal },
|
|
201
216
|
options: FirecrawlExtractOptions | FirecrawlSearchOptions | undefined,
|
|
202
217
|
): Promise<Response> {
|
|
203
|
-
|
|
218
|
+
const allowLoopback = isLoopbackApiUrl(new URL(url));
|
|
219
|
+
let current = await validateRemoteUrl(url, firecrawlApiSsrfOptions(options, allowLoopback));
|
|
204
220
|
let headers = init.headers;
|
|
205
221
|
for (let redirects = 0; redirects <= DEFAULT_MAX_REDIRECTS; redirects++) {
|
|
206
222
|
const response = await fetch(current, { ...init, headers, redirect: "manual" });
|
|
@@ -210,7 +226,7 @@ async function fetchFirecrawlApi(
|
|
|
210
226
|
if (!location) return response;
|
|
211
227
|
if (redirects === DEFAULT_MAX_REDIRECTS) throw new Error(`Too many redirects fetching ${current.toString()}`);
|
|
212
228
|
|
|
213
|
-
const next = await validateRemoteUrl(new URL(location, current),
|
|
229
|
+
const next = await validateRemoteUrl(new URL(location, current), firecrawlApiSsrfOptions(options, allowLoopback));
|
|
214
230
|
if (next.origin !== current.origin) headers = withoutSensitiveHeaders(headers);
|
|
215
231
|
current = next;
|
|
216
232
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { closeSync, existsSync, lstatSync, mkdirSync, openSync, readFileSync, readSync, readdirSync, realpathSync, rmSync, statSync, unlinkSync } from "node:fs";
|
|
2
2
|
import { execFile, spawn, type ChildProcess } from "node:child_process";
|
|
3
|
-
import {
|
|
3
|
+
import { createHash } from "node:crypto";
|
|
4
|
+
import { basename, dirname, extname, join, resolve as resolvePath, sep as pathSep } from "node:path";
|
|
4
5
|
import { activityMonitor } from "./activity.ts";
|
|
5
6
|
import type { ExtractedContent } from "./extract.ts";
|
|
6
7
|
import { checkGhAvailable, checkRepoSize, fetchViaApi, showGhHint } from "./github-api.ts";
|
|
@@ -39,10 +40,15 @@ export interface GitHubUrlInfo {
|
|
|
39
40
|
}
|
|
40
41
|
|
|
41
42
|
interface CachedClone {
|
|
42
|
-
|
|
43
|
+
destination: CloneDestination;
|
|
43
44
|
clonePromise: Promise<string | null>;
|
|
44
45
|
}
|
|
45
46
|
|
|
47
|
+
interface CloneDestination {
|
|
48
|
+
rootPath: string;
|
|
49
|
+
localPath: string;
|
|
50
|
+
}
|
|
51
|
+
|
|
46
52
|
interface GitHubCloneConfig {
|
|
47
53
|
enabled: boolean;
|
|
48
54
|
maxRepoSizeMB: number;
|
|
@@ -137,20 +143,21 @@ export function parseGitHubUrl(url: string): GitHubUrlInfo | null {
|
|
|
137
143
|
const host = parsed.hostname.toLowerCase();
|
|
138
144
|
if (host !== "github.com" && host !== "www.github.com") return null;
|
|
139
145
|
|
|
140
|
-
const segments =
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
}
|
|
149
|
-
});
|
|
146
|
+
const segments: string[] = [];
|
|
147
|
+
for (const segment of parsed.pathname.split("/").filter(Boolean)) {
|
|
148
|
+
try {
|
|
149
|
+
segments.push(decodeURIComponent(segment));
|
|
150
|
+
} catch {
|
|
151
|
+
return null;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
150
154
|
if (segments.length < 2) return null;
|
|
151
155
|
|
|
152
156
|
const owner = segments[0];
|
|
153
157
|
const repo = segments[1].replace(/\.git$/, "");
|
|
158
|
+
if (!/^[A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?$/.test(owner)) return null;
|
|
159
|
+
if (owner.includes("--")) return null;
|
|
160
|
+
if (!/^[A-Za-z0-9._-]{1,100}$/.test(repo) || repo === "." || repo === "..") return null;
|
|
154
161
|
|
|
155
162
|
if (NON_CODE_SEGMENTS.has(segments[2]?.toLowerCase())) return null;
|
|
156
163
|
|
|
@@ -163,6 +170,7 @@ export function parseGitHubUrl(url: string): GitHubUrlInfo | null {
|
|
|
163
170
|
if (segments.length < 4) return null;
|
|
164
171
|
|
|
165
172
|
const ref = segments[3];
|
|
173
|
+
if (ref.length === 0 || ref.length > 1024 || /[\0-\x1f\x7f]/.test(ref)) return null;
|
|
166
174
|
const refIsFullSha = /^[0-9a-f]{40}$/.test(ref);
|
|
167
175
|
const pathParts = segments.slice(4);
|
|
168
176
|
const path = pathParts.length > 0 ? pathParts.join("/") : "";
|
|
@@ -181,9 +189,32 @@ function cacheKey(owner: string, repo: string, ref?: string): string {
|
|
|
181
189
|
return ref ? `${owner}/${repo}@${ref}` : `${owner}/${repo}`;
|
|
182
190
|
}
|
|
183
191
|
|
|
184
|
-
function
|
|
185
|
-
|
|
186
|
-
|
|
192
|
+
function cloneDestination(config: GitHubCloneConfig, owner: string, repo: string, ref?: string): CloneDestination | null {
|
|
193
|
+
try {
|
|
194
|
+
mkdirSync(resolvePath(config.clonePath), { recursive: true });
|
|
195
|
+
const rootPath = realpathSync(resolvePath(config.clonePath));
|
|
196
|
+
const digest = createHash("sha256").update(JSON.stringify([owner, repo, ref ?? null])).digest("hex");
|
|
197
|
+
const localPath = resolvePath(rootPath, digest);
|
|
198
|
+
if (dirname(localPath) !== rootPath) return null;
|
|
199
|
+
return { rootPath, localPath };
|
|
200
|
+
} catch {
|
|
201
|
+
return null;
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
function removeCloneDestination(destination: CloneDestination): boolean {
|
|
206
|
+
const rootPath = resolvePath(destination.rootPath);
|
|
207
|
+
const localPath = resolvePath(destination.localPath);
|
|
208
|
+
if (dirname(localPath) !== rootPath || !/^[0-9a-f]{64}$/.test(basename(localPath))) return false;
|
|
209
|
+
try {
|
|
210
|
+
const entry = lstatSync(localPath);
|
|
211
|
+
if (entry.isSymbolicLink()) unlinkSync(localPath);
|
|
212
|
+
else rmSync(localPath, { recursive: true, force: true });
|
|
213
|
+
return true;
|
|
214
|
+
} catch (err) {
|
|
215
|
+
if (err && typeof err === "object" && "code" in err && err.code === "ENOENT") return true;
|
|
216
|
+
return false;
|
|
217
|
+
}
|
|
187
218
|
}
|
|
188
219
|
|
|
189
220
|
const PROCESS_KILL_GRACE_MS = 3000;
|
|
@@ -225,8 +256,9 @@ function terminateProcessTree(child: ChildProcess): void {
|
|
|
225
256
|
forceKill.unref();
|
|
226
257
|
}
|
|
227
258
|
|
|
228
|
-
function execClone(args: string[],
|
|
259
|
+
function execClone(args: string[], destination: CloneDestination, timeoutMs: number, signal?: AbortSignal): Promise<string | null> {
|
|
229
260
|
return new Promise((resolve) => {
|
|
261
|
+
const { localPath } = destination;
|
|
230
262
|
let settled = false;
|
|
231
263
|
let timeout: ReturnType<typeof setTimeout> | undefined;
|
|
232
264
|
let onAbort: (() => void) | undefined;
|
|
@@ -238,10 +270,7 @@ function execClone(args: string[], localPath: string, timeoutMs: number, signal?
|
|
|
238
270
|
if (signal && onAbort) signal.removeEventListener("abort", onAbort);
|
|
239
271
|
|
|
240
272
|
if (!success) {
|
|
241
|
-
|
|
242
|
-
rmSync(localPath, { recursive: true, force: true });
|
|
243
|
-
} catch {
|
|
244
|
-
}
|
|
273
|
+
removeCloneDestination(destination);
|
|
245
274
|
resolve(null);
|
|
246
275
|
return;
|
|
247
276
|
}
|
|
@@ -282,14 +311,11 @@ async function cloneRepo(
|
|
|
282
311
|
repo: string,
|
|
283
312
|
ref: string | undefined,
|
|
284
313
|
config: GitHubCloneConfig,
|
|
314
|
+
destination: CloneDestination,
|
|
285
315
|
signal?: AbortSignal,
|
|
286
316
|
): Promise<string | null> {
|
|
287
|
-
const localPath =
|
|
288
|
-
|
|
289
|
-
try {
|
|
290
|
-
rmSync(localPath, { recursive: true, force: true });
|
|
291
|
-
} catch {
|
|
292
|
-
}
|
|
317
|
+
const { localPath } = destination;
|
|
318
|
+
if (!removeCloneDestination(destination)) return null;
|
|
293
319
|
|
|
294
320
|
const timeoutMs = config.cloneTimeoutSeconds * 1000;
|
|
295
321
|
const hasGh = await checkGhAvailable();
|
|
@@ -297,7 +323,7 @@ async function cloneRepo(
|
|
|
297
323
|
if (hasGh) {
|
|
298
324
|
const args = ["gh", "repo", "clone", `${owner}/${repo}`, localPath, "--", "--depth", "1", "--single-branch"];
|
|
299
325
|
if (ref) args.push("--branch", ref);
|
|
300
|
-
return execClone(args,
|
|
326
|
+
return execClone(args, destination, timeoutMs, signal);
|
|
301
327
|
}
|
|
302
328
|
|
|
303
329
|
showGhHint();
|
|
@@ -306,7 +332,7 @@ async function cloneRepo(
|
|
|
306
332
|
const args = ["git", "clone", "--depth", "1", "--single-branch"];
|
|
307
333
|
if (ref) args.push("--branch", ref);
|
|
308
334
|
args.push(gitUrl, localPath);
|
|
309
|
-
return execClone(args,
|
|
335
|
+
return execClone(args, destination, timeoutMs, signal);
|
|
310
336
|
}
|
|
311
337
|
|
|
312
338
|
function isBinaryFile(filePath: string): boolean {
|
|
@@ -671,9 +697,15 @@ export async function extractGitHub(
|
|
|
671
697
|
return cachedResult;
|
|
672
698
|
}
|
|
673
699
|
|
|
674
|
-
const
|
|
675
|
-
|
|
676
|
-
|
|
700
|
+
const destination = cloneDestination(config, owner, repo, info.ref);
|
|
701
|
+
if (!destination) {
|
|
702
|
+
const apiFallback = await fetchViaApi(url, owner, repo, info);
|
|
703
|
+
if (apiFallback) activityMonitor.logComplete(activityId, 200);
|
|
704
|
+
else activityMonitor.logError(activityId, "invalid clone destination");
|
|
705
|
+
return apiFallback;
|
|
706
|
+
}
|
|
707
|
+
const clonePromise = cloneRepo(owner, repo, info.ref, config, destination, signal);
|
|
708
|
+
cloneCache.set(key, { destination, clonePromise });
|
|
677
709
|
|
|
678
710
|
const result = await clonePromise;
|
|
679
711
|
if (signal?.aborted) {
|
|
@@ -707,10 +739,7 @@ export async function extractGitHub(
|
|
|
707
739
|
|
|
708
740
|
export function clearCloneCache(): void {
|
|
709
741
|
for (const entry of cloneCache.values()) {
|
|
710
|
-
|
|
711
|
-
rmSync(entry.localPath, { recursive: true, force: true });
|
|
712
|
-
} catch {
|
|
713
|
-
}
|
|
742
|
+
removeCloneDestination(entry.destination);
|
|
714
743
|
}
|
|
715
744
|
cloneCache.clear();
|
|
716
745
|
cachedConfig = null;
|