@juspay/neurolink 9.94.4 → 9.94.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/browser/neurolink.min.js +401 -401
- package/dist/core/modules/MessageBuilder.js +32 -3
- package/dist/lib/core/modules/MessageBuilder.js +32 -3
- package/dist/lib/types/generate.d.ts +1 -0
- package/dist/lib/types/multimodal.d.ts +34 -3
- package/dist/lib/types/stream.d.ts +1 -0
- package/dist/lib/utils/errorHandling.d.ts +22 -0
- package/dist/lib/utils/errorHandling.js +55 -0
- package/dist/lib/utils/fileDetector.js +210 -31
- package/dist/lib/utils/imageCache.js +11 -8
- package/dist/lib/utils/imageProcessor.d.ts +37 -0
- package/dist/lib/utils/imageProcessor.js +222 -16
- package/dist/lib/utils/logSanitize.d.ts +100 -1
- package/dist/lib/utils/logSanitize.js +152 -2
- package/dist/lib/utils/messageBuilder.js +80 -9
- package/dist/types/generate.d.ts +1 -0
- package/dist/types/multimodal.d.ts +34 -3
- package/dist/types/stream.d.ts +1 -0
- package/dist/utils/errorHandling.d.ts +22 -0
- package/dist/utils/errorHandling.js +55 -0
- package/dist/utils/fileDetector.js +210 -31
- package/dist/utils/imageCache.js +11 -8
- package/dist/utils/imageProcessor.d.ts +37 -0
- package/dist/utils/imageProcessor.js +222 -16
- package/dist/utils/logSanitize.d.ts +100 -1
- package/dist/utils/logSanitize.js +152 -2
- package/dist/utils/messageBuilder.js +80 -9
- package/package.json +1 -1
|
@@ -2,13 +2,32 @@
|
|
|
2
2
|
* Image processing utilities for multimodal support
|
|
3
3
|
* Handles format conversion for different AI providers
|
|
4
4
|
*/
|
|
5
|
+
import { basename } from "path";
|
|
5
6
|
import { logger } from "./logger.js";
|
|
7
|
+
import { redactPathFromMessage, redactUrlForError, redactUrlsInText, sanitizeErrorCause, } from "./logSanitize.js";
|
|
6
8
|
import { urlDownloadRateLimiter } from "./rateLimiter.js";
|
|
7
9
|
import { withRetry } from "./retryHandler.js";
|
|
8
10
|
import { SYSTEM_LIMITS } from "../core/constants.js";
|
|
9
11
|
import { SIZE_LIMITS_BYTES } from "../processors/config/sizeLimits.js";
|
|
10
12
|
import { getImageCache } from "./imageCache.js";
|
|
11
13
|
import { ErrorFactory, NeuroLinkError } from "./errorHandling.js";
|
|
14
|
+
/**
|
|
15
|
+
* Minimum bytes required just to detect an image format from magic bytes (#293).
|
|
16
|
+
*/
|
|
17
|
+
export const MIN_IMAGE_SIZE = 12;
|
|
18
|
+
/**
|
|
19
|
+
* Minimum plausible byte size for a non-empty image of each format (#293). A
|
|
20
|
+
* buffer that carries a format's magic bytes but is smaller than this is
|
|
21
|
+
* truncated/corrupt and would fail (or render blank) downstream — reject early
|
|
22
|
+
* with a clear message instead.
|
|
23
|
+
*/
|
|
24
|
+
export const MIN_VALID_IMAGE_SIZE = {
|
|
25
|
+
"image/png": 67,
|
|
26
|
+
"image/jpeg": 125,
|
|
27
|
+
"image/gif": 43,
|
|
28
|
+
"image/webp": 20,
|
|
29
|
+
"image/avif": 100,
|
|
30
|
+
};
|
|
12
31
|
/**
|
|
13
32
|
* Network error codes that should trigger a retry
|
|
14
33
|
*/
|
|
@@ -58,6 +77,25 @@ function isRetryableDownloadError(error) {
|
|
|
58
77
|
}
|
|
59
78
|
return false;
|
|
60
79
|
}
|
|
80
|
+
/**
|
|
81
|
+
* Reject `detectImageType()`'s `application/octet-stream` sentinel — the
|
|
82
|
+
* honest "no known image signature (PNG/JPEG/GIF/WebP/BMP/TIFF/SVG/AVIF)
|
|
83
|
+
* matched" fallback (#261/#286). Packaging that sentinel into a data URI
|
|
84
|
+
* hands vision providers (OpenAI/Anthropic/Google) a MIME type they reject
|
|
85
|
+
* outright with an HTTP 400, so every public conversion path that turns
|
|
86
|
+
* unknown bytes into image output must fail loud here first instead of
|
|
87
|
+
* letting the sentinel slip through to a caller.
|
|
88
|
+
*
|
|
89
|
+
* @param mediaType - The MIME type returned by `detectImageType()`.
|
|
90
|
+
* @throws Error if `mediaType` is the octet-stream sentinel.
|
|
91
|
+
*/
|
|
92
|
+
function assertKnownImageType(mediaType) {
|
|
93
|
+
if (mediaType === "application/octet-stream") {
|
|
94
|
+
logger.error("Unable to detect a supported image format from file content");
|
|
95
|
+
throw new Error("Unsupported or corrupted image: no known image signature " +
|
|
96
|
+
"(PNG/JPEG/GIF/WebP/BMP/TIFF/SVG/AVIF) was found in the file content.");
|
|
97
|
+
}
|
|
98
|
+
}
|
|
61
99
|
/**
|
|
62
100
|
* Image processor class for handling provider-specific image formatting
|
|
63
101
|
*/
|
|
@@ -71,12 +109,12 @@ export class ImageProcessor {
|
|
|
71
109
|
* @returns Processed image as data URI
|
|
72
110
|
*/
|
|
73
111
|
static async process(content, _options) {
|
|
74
|
-
//
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
112
|
+
// #293: reject empty AND small/truncated buffers (not just 0 bytes) before
|
|
113
|
+
// processing; returns the detected media type.
|
|
114
|
+
const mediaType = this.validateBufferNotEmpty(content);
|
|
115
|
+
// #261/#286 follow-up: fail loud instead of packaging the octet-stream
|
|
116
|
+
// sentinel as a "valid" image (see assertKnownImageType() above).
|
|
117
|
+
assertKnownImageType(mediaType);
|
|
80
118
|
const base64 = ImageProcessor.safeBase64Convert(content, "image processing");
|
|
81
119
|
const dataUri = `data:${mediaType};base64,${base64}`;
|
|
82
120
|
// Validate output before returning
|
|
@@ -91,6 +129,73 @@ export class ImageProcessor {
|
|
|
91
129
|
},
|
|
92
130
|
};
|
|
93
131
|
}
|
|
132
|
+
/**
|
|
133
|
+
* Strict magic-byte match: does `content` actually begin with `mediaType`'s
|
|
134
|
+
* signature? `detectImageType` returns `image/jpeg` as a *fallback* for
|
|
135
|
+
* unrecognized data, so a per-format minimum must only be enforced when the
|
|
136
|
+
* bytes genuinely match — otherwise valid BMP/TIFF/unknown buffers would be
|
|
137
|
+
* wrongly rejected as "truncated jpeg" (#293 review).
|
|
138
|
+
*/
|
|
139
|
+
static hasStrictImageMagic(content, mediaType) {
|
|
140
|
+
const b = content;
|
|
141
|
+
switch (mediaType) {
|
|
142
|
+
case "image/png":
|
|
143
|
+
return (b.length >= 4 &&
|
|
144
|
+
b[0] === 0x89 &&
|
|
145
|
+
b[1] === 0x50 &&
|
|
146
|
+
b[2] === 0x4e &&
|
|
147
|
+
b[3] === 0x47);
|
|
148
|
+
case "image/jpeg":
|
|
149
|
+
return b.length >= 3 && b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff;
|
|
150
|
+
case "image/gif":
|
|
151
|
+
return b.length >= 3 && b[0] === 0x47 && b[1] === 0x49 && b[2] === 0x46;
|
|
152
|
+
case "image/webp":
|
|
153
|
+
return (b.length >= 12 &&
|
|
154
|
+
b.subarray(0, 4).toString("latin1") === "RIFF" &&
|
|
155
|
+
b.subarray(8, 12).toString("latin1") === "WEBP");
|
|
156
|
+
case "image/avif":
|
|
157
|
+
return (b.length >= 12 &&
|
|
158
|
+
b.subarray(4, 8).toString("latin1") === "ftyp" &&
|
|
159
|
+
b.subarray(8, 12).toString("latin1").startsWith("avif"));
|
|
160
|
+
default:
|
|
161
|
+
return false;
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* Validate that a buffer is a plausibly-complete image (#293): non-empty and,
|
|
166
|
+
* for a strictly-identified raster format, at least the minimum plausible byte
|
|
167
|
+
* size. Returns the detected media type.
|
|
168
|
+
*
|
|
169
|
+
* @throws Error when the buffer is empty or a recognized format is truncated.
|
|
170
|
+
*/
|
|
171
|
+
static validateBufferNotEmpty(content) {
|
|
172
|
+
if (content.length === 0) {
|
|
173
|
+
logger.error("Empty buffer provided");
|
|
174
|
+
throw ErrorFactory.imageBufferInvalid("Invalid image processing: buffer is empty");
|
|
175
|
+
}
|
|
176
|
+
const mediaType = this.detectImageType(content);
|
|
177
|
+
// SVG is text and can be legitimately tiny (e.g. `<svg/>`), so the raster
|
|
178
|
+
// size floors don't apply to it (#293 review).
|
|
179
|
+
if (mediaType === "image/svg+xml") {
|
|
180
|
+
return mediaType;
|
|
181
|
+
}
|
|
182
|
+
// A buffer too small to carry any raster header is not a usable image.
|
|
183
|
+
if (content.length < MIN_IMAGE_SIZE) {
|
|
184
|
+
throw ErrorFactory.imageBufferInvalid(`Invalid image processing: buffer too small (${content.length} bytes) — ` +
|
|
185
|
+
`a minimum of ${MIN_IMAGE_SIZE} bytes is required for a valid image.`);
|
|
186
|
+
}
|
|
187
|
+
// Only enforce a per-format minimum when the format was strictly identified
|
|
188
|
+
// from magic bytes (never from detectImageType's jpeg fallback), so valid
|
|
189
|
+
// BMP/TIFF/unknown buffers are not mis-rejected as "truncated jpeg".
|
|
190
|
+
const minForFormat = MIN_VALID_IMAGE_SIZE[mediaType];
|
|
191
|
+
if (minForFormat !== undefined &&
|
|
192
|
+
content.length < minForFormat &&
|
|
193
|
+
this.hasStrictImageMagic(content, mediaType)) {
|
|
194
|
+
throw ErrorFactory.imageBufferInvalid(`Invalid image processing: ${mediaType} buffer is truncated ` +
|
|
195
|
+
`(${content.length} bytes, minimum ${minForFormat} for a valid ${mediaType}).`);
|
|
196
|
+
}
|
|
197
|
+
return mediaType;
|
|
198
|
+
}
|
|
94
199
|
/**
|
|
95
200
|
* Validate processed output meets required format
|
|
96
201
|
* Checks:
|
|
@@ -327,20 +432,50 @@ export class ImageProcessor {
|
|
|
327
432
|
return "image/svg+xml";
|
|
328
433
|
}
|
|
329
434
|
}
|
|
330
|
-
// AVIF:
|
|
435
|
+
// AVIF (ISOBMFF): "ftyp" box at offset 4, major brand at offset 8.
|
|
436
|
+
// Accept the AVIF-spec brand variants — 'avif' (still image), 'avis'
|
|
437
|
+
// (image sequence), and 'avio' (intra-only AV1 image/sequence — the
|
|
438
|
+
// spec lists it under compatible_brands rather than major_brand, but
|
|
439
|
+
// real-world encoders emit it as major_brand too, so it's accepted
|
|
440
|
+
// here) — and compare bytes directly rather than via Buffer.toString()
|
|
441
|
+
// (which would utf-8-decode non-ASCII bytes) so the check is exact
|
|
442
|
+
// (#286).
|
|
331
443
|
if (input.length >= 12) {
|
|
332
|
-
const
|
|
333
|
-
|
|
334
|
-
|
|
444
|
+
const isFtyp = input[4] === 0x66 && // 'f'
|
|
445
|
+
input[5] === 0x74 && // 't'
|
|
446
|
+
input[6] === 0x79 && // 'y'
|
|
447
|
+
input[7] === 0x70; // 'p'
|
|
448
|
+
const brand = String.fromCharCode(input[8], input[9], input[10], input[11]);
|
|
449
|
+
if (isFtyp &&
|
|
450
|
+
(brand === "avif" || brand === "avis" || brand === "avio")) {
|
|
335
451
|
return "image/avif";
|
|
336
452
|
}
|
|
337
453
|
}
|
|
454
|
+
// BMP: "BM" magic (0x42 0x4D)
|
|
455
|
+
if (input[0] === 0x42 && input[1] === 0x4d) {
|
|
456
|
+
return "image/bmp";
|
|
457
|
+
}
|
|
458
|
+
// TIFF: little-endian "II*\0" or big-endian "MM\0*"
|
|
459
|
+
if ((input[0] === 0x49 &&
|
|
460
|
+
input[1] === 0x49 &&
|
|
461
|
+
input[2] === 0x2a &&
|
|
462
|
+
input[3] === 0x00) ||
|
|
463
|
+
(input[0] === 0x4d &&
|
|
464
|
+
input[1] === 0x4d &&
|
|
465
|
+
input[2] === 0x00 &&
|
|
466
|
+
input[3] === 0x2a)) {
|
|
467
|
+
return "image/tiff";
|
|
468
|
+
}
|
|
338
469
|
}
|
|
339
|
-
|
|
470
|
+
// No known image signature matched. Return a safe, honest sentinel
|
|
471
|
+
// rather than mislabeling arbitrary bytes as JPEG (#261): a wrong
|
|
472
|
+
// image/jpeg label silently corrupts the downstream provider request.
|
|
473
|
+
logger.warn("Could not detect image type from magic bytes; using application/octet-stream");
|
|
474
|
+
return "application/octet-stream";
|
|
340
475
|
}
|
|
341
476
|
catch (error) {
|
|
342
477
|
logger.warn("Failed to detect image type, using default:", error);
|
|
343
|
-
return "
|
|
478
|
+
return "application/octet-stream";
|
|
344
479
|
}
|
|
345
480
|
}
|
|
346
481
|
/**
|
|
@@ -415,6 +550,12 @@ export class ImageProcessor {
|
|
|
415
550
|
"image/tiff",
|
|
416
551
|
"image/svg+xml",
|
|
417
552
|
"image/avif",
|
|
553
|
+
// Deliberately excludes "application/octet-stream": that's
|
|
554
|
+
// detectImageType()'s honest sentinel for bytes matching no known
|
|
555
|
+
// image signature (#261), not a real image format. Vision providers
|
|
556
|
+
// (OpenAI/Anthropic/Google) reject it outright with an HTTP 400, so
|
|
557
|
+
// process() must fail loud instead of packaging it as a valid image
|
|
558
|
+
// (see the octet-stream guard in process() below).
|
|
418
559
|
];
|
|
419
560
|
return supportedFormats.includes(mediaType.toLowerCase());
|
|
420
561
|
}
|
|
@@ -495,7 +636,19 @@ export class ImageProcessor {
|
|
|
495
636
|
*/
|
|
496
637
|
static processImage(image, provider, model) {
|
|
497
638
|
try {
|
|
639
|
+
// #257: reject an oversized buffer before any format-detection or
|
|
640
|
+
// conversion work runs on it — mirrors the guard every
|
|
641
|
+
// `processImageForX()` branch below already applies via
|
|
642
|
+
// `safeBase64Convert()`, so the typed IMAGE_TOO_LARGE error surfaces
|
|
643
|
+
// here too instead of being pre-empted by the octet-stream check next.
|
|
644
|
+
if (Buffer.isBuffer(image)) {
|
|
645
|
+
ImageProcessor.validateBufferSize(image, "image processing");
|
|
646
|
+
}
|
|
498
647
|
const mediaType = ImageProcessor.detectImageType(image);
|
|
648
|
+
// #261/#286 follow-up: reject the octet-stream sentinel here too — the
|
|
649
|
+
// returned ProcessedImage.mediaType is what callers use to build a
|
|
650
|
+
// data URI, so this is a separate public path that must not emit it.
|
|
651
|
+
assertKnownImageType(mediaType);
|
|
499
652
|
const size = typeof image === "string"
|
|
500
653
|
? Buffer.byteLength(image, "base64")
|
|
501
654
|
: image.length;
|
|
@@ -687,6 +840,15 @@ export const imageUtils = {
|
|
|
687
840
|
const cleanBase64 = base64.includes(",") ? base64.split(",")[1] : base64;
|
|
688
841
|
return Buffer.from(cleanBase64, "base64");
|
|
689
842
|
},
|
|
843
|
+
/**
|
|
844
|
+
* Redact `filePath` (both as given and its `path.resolve()`'d form) from
|
|
845
|
+
* an error message. Exposed directly (not just used internally by
|
|
846
|
+
* {@link fileToBase64DataUri}) so the resolved-path branch can be verified
|
|
847
|
+
* by a deterministic unit test — real `fs` errors only ever embed the
|
|
848
|
+
* literal path as passed, never a resolved variant, so that branch isn't
|
|
849
|
+
* otherwise observable through the async file-reading API.
|
|
850
|
+
*/
|
|
851
|
+
redactPathFromMessage,
|
|
690
852
|
/**
|
|
691
853
|
* Convert file path to base64 data URI
|
|
692
854
|
*/
|
|
@@ -706,11 +868,38 @@ export const imageUtils = {
|
|
|
706
868
|
// Enhanced MIME detection: try buffer content first, fallback to filename
|
|
707
869
|
const mimeType = ImageProcessor.detectImageType(buffer) ||
|
|
708
870
|
ImageProcessor.detectImageType(filePath);
|
|
871
|
+
// #261/#286 follow-up: reject the octet-stream sentinel here too —
|
|
872
|
+
// this is a third public path (alongside process() and
|
|
873
|
+
// processImage()) that can otherwise package unknown bytes into a
|
|
874
|
+
// `data:application/octet-stream;base64,...` URI a vision provider
|
|
875
|
+
// rejects outright.
|
|
876
|
+
assertKnownImageType(mimeType);
|
|
709
877
|
const base64 = buffer.toString("base64");
|
|
710
878
|
return `data:${mimeType};base64,${base64}`;
|
|
711
879
|
}
|
|
712
880
|
catch (error) {
|
|
713
|
-
|
|
881
|
+
// Full path stays in the debug log; the thrown (potentially
|
|
882
|
+
// client-facing) error only gets the basename. The underlying fs
|
|
883
|
+
// error's own message is scrubbed too — Node's ENOENT/EACCES text
|
|
884
|
+
// embeds the full path verbatim (e.g. "no such file or directory,
|
|
885
|
+
// stat '/full/path'"), which would otherwise leak the host's
|
|
886
|
+
// directory layout right back through the "reason" suffix. The raw
|
|
887
|
+
// `error` is never attached as `cause` either — that would leave the
|
|
888
|
+
// unredacted path reachable via `error.cause.message` for anything
|
|
889
|
+
// that walks the cause chain (cause-aware logging, telemetry); a
|
|
890
|
+
// sanitized copy (via the shared `sanitizeErrorCause`) is attached
|
|
891
|
+
// instead, mirroring `urlToBase64DataUri` below.
|
|
892
|
+
logger.debug("fileToBase64DataUri failed", { filePath, error });
|
|
893
|
+
const safeName = basename(filePath);
|
|
894
|
+
const cause = sanitizeErrorCause(error, { filePath });
|
|
895
|
+
const reason = error instanceof Error ? cause.message : "Unknown error";
|
|
896
|
+
// Assigned to a variable before throwing (rather than `throw new
|
|
897
|
+
// Error(...)` inline) so the sanitized `cause` — a deliberately NEW,
|
|
898
|
+
// redacted Error — isn't mistaken by tooling for a dropped/altered
|
|
899
|
+
// reference to the originally caught `error`; the raw error is never
|
|
900
|
+
// reachable from the thrown result at all, by design.
|
|
901
|
+
const wrapped = new Error(`Failed to convert file to base64 (${safeName}): ${reason}`, { cause });
|
|
902
|
+
throw wrapped;
|
|
714
903
|
}
|
|
715
904
|
},
|
|
716
905
|
/**
|
|
@@ -796,14 +985,31 @@ export const imageUtils = {
|
|
|
796
985
|
maxDelay: SYSTEM_LIMITS.DEFAULT_MAX_DELAY,
|
|
797
986
|
retryCondition: isRetryableDownloadError,
|
|
798
987
|
onRetry: (attempt, error) => {
|
|
799
|
-
|
|
988
|
+
// Some fetch/DNS/TLS errors embed the full request URL (including
|
|
989
|
+
// a presigned query token) in `error.message` — redact that text
|
|
990
|
+
// too, not just the explicit `url` interpolation below. The
|
|
991
|
+
// non-Error branch is sanitized too: String(error) on a thrown
|
|
992
|
+
// non-Error value (e.g. a raw string or object) can just as
|
|
993
|
+
// easily carry the same signed URL.
|
|
994
|
+
const message = redactUrlsInText(error instanceof Error ? error.message : String(error));
|
|
800
995
|
const attemptsLeft = maxAttempts - attempt;
|
|
801
|
-
logger.warn(`⚠️ Image download attempt ${attempt} failed for ${url}: ${message}. ${attemptsLeft} ${attemptsLeft === 1 ? "attempt" : "attempts"} remaining...`);
|
|
996
|
+
logger.warn(`⚠️ Image download attempt ${attempt} failed for ${redactUrlForError(url)}: ${message}. ${attemptsLeft} ${attemptsLeft === 1 ? "attempt" : "attempts"} remaining...`);
|
|
802
997
|
},
|
|
803
998
|
});
|
|
804
999
|
}
|
|
805
1000
|
catch (error) {
|
|
806
|
-
|
|
1001
|
+
// Query string / fragment stripped — a presigned S3/GCS URL or SAS
|
|
1002
|
+
// token embedded there must not be echoed into a thrown error. The
|
|
1003
|
+
// underlying error's own message is scrubbed too, since fetch/DNS/TLS
|
|
1004
|
+
// errors often embed the full URL (query string and all) themselves.
|
|
1005
|
+
// The raw `error` is never attached as `cause` — that would leave the
|
|
1006
|
+
// unredacted URL reachable via `error.cause.message` for anything
|
|
1007
|
+
// that walks the cause chain (cause-aware logging, telemetry); a
|
|
1008
|
+
// sanitized copy is attached instead.
|
|
1009
|
+
const sourceUrl = redactUrlForError(url);
|
|
1010
|
+
const cause = sanitizeErrorCause(error);
|
|
1011
|
+
const wrapped = new Error(`Failed to download and convert URL to base64 (${sourceUrl}): ${cause.message}`, { cause });
|
|
1012
|
+
throw wrapped;
|
|
807
1013
|
}
|
|
808
1014
|
},
|
|
809
1015
|
/**
|
|
@@ -61,11 +61,110 @@ export declare function safeDebugSerialize(value: unknown, maxLen?: number): str
|
|
|
61
61
|
* for diagnostics. Use this for any `baseURL`/endpoint a caller may have
|
|
62
62
|
* supplied with inline credentials. The match is global, so every `//…@`
|
|
63
63
|
* authority in the string is redacted (e.g. a proxy chain or a URL embedded
|
|
64
|
-
* in a query parameter), not just the first.
|
|
64
|
+
* in a query parameter), not just the first. IPv6 literal hosts (`//u:p@[::1]/x`)
|
|
65
|
+
* and percent-encoded userinfo (`//u:p%40x@host/x`) are handled naturally —
|
|
66
|
+
* neither `[`/`]` nor `%` stop either pass below, so the bracketed host or
|
|
67
|
+
* encoded byte just rides along as part of the (non-redacted) remainder.
|
|
68
|
+
*
|
|
69
|
+
* This is best-effort defense-in-depth for free-form logging, deliberately
|
|
70
|
+
* regex-based so it can scrub *multiple* embedded authorities out of
|
|
71
|
+
* arbitrary text (a real URL parser only ever parses one URL at a time).
|
|
72
|
+
* It is NOT the primary protection for user-facing errors — that's
|
|
73
|
+
* {@link redactUrlForError}, which parses the single input via `new URL()`
|
|
74
|
+
* and reconstructs from protocol+host+pathname, never touching userinfo at
|
|
75
|
+
* all. Prefer that function whenever the input is known to be exactly one
|
|
76
|
+
* URL.
|
|
77
|
+
*
|
|
78
|
+
* Two passes, deliberately:
|
|
79
|
+
* 1. Well-formed authorities — credentials with no raw `/` — bounded at the
|
|
80
|
+
* authority's first `/` (or string end). Greedy backtracking naturally
|
|
81
|
+
* absorbs multiple `@` segments within one authority (e.g. `//a@b@host`
|
|
82
|
+
* → `//***@host`) while leaving a second, separate `//...@` authority
|
|
83
|
+
* elsewhere in the string (e.g. one embedded in a query parameter)
|
|
84
|
+
* untouched, because pass 1's own exclusion of `/` stops it at the first
|
|
85
|
+
* `/`, before it can ever reach a nested authority.
|
|
86
|
+
* 2. Malformed authorities whose credentials contain an unescaped `/`
|
|
87
|
+
* (e.g. `//user:sec/ret@host/path`) — pass 1 can't find an `@` before
|
|
88
|
+
* hitting that `/`, so this pass allows `/` and redacts up to the last
|
|
89
|
+
* remaining `@`. It stops before any *nested* `//` (via a negative
|
|
90
|
+
* lookahead) rather than excluding a specific character like `*` —
|
|
91
|
+
* excluding a literal character is what let a password containing both
|
|
92
|
+
* `/` and `*` (e.g. `//user:pa*ss/word@host/path`, legal under RFC 3986)
|
|
93
|
+
* slip through both passes entirely in an earlier version of this
|
|
94
|
+
* function.
|
|
65
95
|
*
|
|
66
96
|
* @param url - The URL (or URL-shaped string) to redact.
|
|
67
97
|
*/
|
|
68
98
|
export declare function redactUrlCredentials(url: string): string;
|
|
99
|
+
/**
|
|
100
|
+
* Reduce a URL to its origin + pathname for safe inclusion in error messages
|
|
101
|
+
* and logs, dropping the query string and fragment — where presigned-URL
|
|
102
|
+
* signatures, SAS tokens, and other secrets commonly live — while keeping
|
|
103
|
+
* enough of the URL (scheme, host, path) for the error to stay diagnostic.
|
|
104
|
+
*
|
|
105
|
+
* Falls back to a best-effort redacted string (query/fragment and
|
|
106
|
+
* `user:pass@` stripped) when the input isn't a parseable absolute URL.
|
|
107
|
+
* Either way the result is capped at 200 chars so a very long host/path
|
|
108
|
+
* can't blow up the error message.
|
|
109
|
+
*
|
|
110
|
+
* @param url - The URL (or URL-shaped string) to redact.
|
|
111
|
+
*/
|
|
112
|
+
export declare function redactUrlForError(url: string): string;
|
|
113
|
+
/**
|
|
114
|
+
* Scrub any embedded absolute URLs out of free-form text — typically the
|
|
115
|
+
* `.message` of an underlying network/DNS/TLS error thrown by `fetch`,
|
|
116
|
+
* undici's `request()`, or Node's resolver, all of which frequently embed
|
|
117
|
+
* the full request URL (query string, credentials, and all) verbatim.
|
|
118
|
+
* Interpolating that message raw into a thrown or logged error would leak a
|
|
119
|
+
* presigned URL's token even after the explicit URL argument was already
|
|
120
|
+
* redacted via {@link redactUrlForError}. Finds each `scheme://…` run in the
|
|
121
|
+
* text and replaces it with its redacted form.
|
|
122
|
+
*
|
|
123
|
+
* @param text - Free-form text (typically `Error.message`) to scrub.
|
|
124
|
+
*/
|
|
125
|
+
export declare function redactUrlsInText(text: string): string;
|
|
126
|
+
/**
|
|
127
|
+
* Redact every occurrence of `filePath` from an error message, checking both
|
|
128
|
+
* the path exactly as given and its `path.resolve()`'d form.
|
|
129
|
+
*
|
|
130
|
+
* Node's own fs errors (ENOENT/EACCES) embed the exact string passed to the
|
|
131
|
+
* fs call, so a plain substring match usually suffices — but wrapped or
|
|
132
|
+
* lower-level errors sometimes normalize the path first (relative→absolute,
|
|
133
|
+
* trailing-slash variations), which an exact-match-only redaction would miss.
|
|
134
|
+
* This is defense-in-depth, not a guarantee: a `realpath`-resolved (symlink-
|
|
135
|
+
* following) variant can still differ from both forms and would slip through.
|
|
136
|
+
*
|
|
137
|
+
* @param message - The error message to scrub.
|
|
138
|
+
* @param filePath - The path to redact (both as given and resolved).
|
|
139
|
+
*/
|
|
140
|
+
export declare function redactPathFromMessage(message: string, filePath: string): string;
|
|
141
|
+
/**
|
|
142
|
+
* Build a sanitized copy of a network/URL/file error, safe to attach as
|
|
143
|
+
* `Error.cause` on a rethrow.
|
|
144
|
+
*
|
|
145
|
+
* Redacting only the OUTER thrown error's message isn't enough: if the RAW
|
|
146
|
+
* underlying error is attached as `cause`, anything that walks the cause
|
|
147
|
+
* chain (cause-aware logging, telemetry spans) can still recover an
|
|
148
|
+
* unredacted URL/path/secret via `error.cause.message`. This returns a NEW
|
|
149
|
+
* `Error` whose message has been through {@link redactUrlsInText} — and,
|
|
150
|
+
* when `options.filePath` is supplied, {@link redactPathFromMessage} too, so
|
|
151
|
+
* a filesystem error's ENOENT/EACCES text doesn't leak the host's directory
|
|
152
|
+
* layout via the cause chain the way the outer message already avoids.
|
|
153
|
+
* `.name` and `.code` (`NodeJS.ErrnoException`) are copied over so retry
|
|
154
|
+
* classification (e.g. `isRetryableNetworkError`) still works when it
|
|
155
|
+
* inspects the cause. The raw original is never referenced by the result.
|
|
156
|
+
*
|
|
157
|
+
* Handles non-`Error` thrown values too (a raw string/object can just as
|
|
158
|
+
* easily carry an unredacted URL or path) by running their `String()` form
|
|
159
|
+
* through the same redaction before wrapping.
|
|
160
|
+
*
|
|
161
|
+
* @param error - The raw underlying error (or thrown value) to sanitize.
|
|
162
|
+
* @param options - `filePath`: a known filesystem path to redact to its
|
|
163
|
+
* basename, in addition to URL redaction.
|
|
164
|
+
*/
|
|
165
|
+
export declare function sanitizeErrorCause(error: unknown, options?: {
|
|
166
|
+
filePath?: string;
|
|
167
|
+
}): Error;
|
|
69
168
|
/**
|
|
70
169
|
* Recursively sanitize a record/array, returning a structurally identical
|
|
71
170
|
* value with sensitive keys redacted and string values run through
|
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
* jina_ (Jina), fish- (Fish Audio)
|
|
16
16
|
* - Generic key=value pairs: api_key=…, access_token: …, secret_key=…
|
|
17
17
|
*/
|
|
18
|
+
import { basename, resolve as resolvePath } from "path";
|
|
18
19
|
const TOKEN_PREFIXES = [
|
|
19
20
|
"sk",
|
|
20
21
|
"pk",
|
|
@@ -156,12 +157,161 @@ export function safeDebugSerialize(value, maxLen = 10_000) {
|
|
|
156
157
|
* for diagnostics. Use this for any `baseURL`/endpoint a caller may have
|
|
157
158
|
* supplied with inline credentials. The match is global, so every `//…@`
|
|
158
159
|
* authority in the string is redacted (e.g. a proxy chain or a URL embedded
|
|
159
|
-
* in a query parameter), not just the first.
|
|
160
|
+
* in a query parameter), not just the first. IPv6 literal hosts (`//u:p@[::1]/x`)
|
|
161
|
+
* and percent-encoded userinfo (`//u:p%40x@host/x`) are handled naturally —
|
|
162
|
+
* neither `[`/`]` nor `%` stop either pass below, so the bracketed host or
|
|
163
|
+
* encoded byte just rides along as part of the (non-redacted) remainder.
|
|
164
|
+
*
|
|
165
|
+
* This is best-effort defense-in-depth for free-form logging, deliberately
|
|
166
|
+
* regex-based so it can scrub *multiple* embedded authorities out of
|
|
167
|
+
* arbitrary text (a real URL parser only ever parses one URL at a time).
|
|
168
|
+
* It is NOT the primary protection for user-facing errors — that's
|
|
169
|
+
* {@link redactUrlForError}, which parses the single input via `new URL()`
|
|
170
|
+
* and reconstructs from protocol+host+pathname, never touching userinfo at
|
|
171
|
+
* all. Prefer that function whenever the input is known to be exactly one
|
|
172
|
+
* URL.
|
|
173
|
+
*
|
|
174
|
+
* Two passes, deliberately:
|
|
175
|
+
* 1. Well-formed authorities — credentials with no raw `/` — bounded at the
|
|
176
|
+
* authority's first `/` (or string end). Greedy backtracking naturally
|
|
177
|
+
* absorbs multiple `@` segments within one authority (e.g. `//a@b@host`
|
|
178
|
+
* → `//***@host`) while leaving a second, separate `//...@` authority
|
|
179
|
+
* elsewhere in the string (e.g. one embedded in a query parameter)
|
|
180
|
+
* untouched, because pass 1's own exclusion of `/` stops it at the first
|
|
181
|
+
* `/`, before it can ever reach a nested authority.
|
|
182
|
+
* 2. Malformed authorities whose credentials contain an unescaped `/`
|
|
183
|
+
* (e.g. `//user:sec/ret@host/path`) — pass 1 can't find an `@` before
|
|
184
|
+
* hitting that `/`, so this pass allows `/` and redacts up to the last
|
|
185
|
+
* remaining `@`. It stops before any *nested* `//` (via a negative
|
|
186
|
+
* lookahead) rather than excluding a specific character like `*` —
|
|
187
|
+
* excluding a literal character is what let a password containing both
|
|
188
|
+
* `/` and `*` (e.g. `//user:pa*ss/word@host/path`, legal under RFC 3986)
|
|
189
|
+
* slip through both passes entirely in an earlier version of this
|
|
190
|
+
* function.
|
|
160
191
|
*
|
|
161
192
|
* @param url - The URL (or URL-shaped string) to redact.
|
|
162
193
|
*/
|
|
163
194
|
export function redactUrlCredentials(url) {
|
|
164
|
-
|
|
195
|
+
const wellFormed = url.replace(/\/\/[^/\s"'<>]*@/g, "//***@");
|
|
196
|
+
return wellFormed.replace(/\/\/(?:(?!\/\/)[^\s"'<>])*@/g, "//***@");
|
|
197
|
+
}
|
|
198
|
+
/**
|
|
199
|
+
* Reduce a URL to its origin + pathname for safe inclusion in error messages
|
|
200
|
+
* and logs, dropping the query string and fragment — where presigned-URL
|
|
201
|
+
* signatures, SAS tokens, and other secrets commonly live — while keeping
|
|
202
|
+
* enough of the URL (scheme, host, path) for the error to stay diagnostic.
|
|
203
|
+
*
|
|
204
|
+
* Falls back to a best-effort redacted string (query/fragment and
|
|
205
|
+
* `user:pass@` stripped) when the input isn't a parseable absolute URL.
|
|
206
|
+
* Either way the result is capped at 200 chars so a very long host/path
|
|
207
|
+
* can't blow up the error message.
|
|
208
|
+
*
|
|
209
|
+
* @param url - The URL (or URL-shaped string) to redact.
|
|
210
|
+
*/
|
|
211
|
+
export function redactUrlForError(url) {
|
|
212
|
+
const truncate = (s) => s.length > 200 ? `${s.slice(0, 200)}…` : s;
|
|
213
|
+
try {
|
|
214
|
+
const parsed = new URL(url);
|
|
215
|
+
// Reconstruct explicitly from protocol + host + pathname — never
|
|
216
|
+
// parsed.username/parsed.password, and never parsed.href (which
|
|
217
|
+
// re-serializes any embedded userinfo) — so a `user:pass@host` in the
|
|
218
|
+
// input can't leak into a diagnostic error message.
|
|
219
|
+
return truncate(`${parsed.protocol}//${parsed.host}${parsed.pathname}`);
|
|
220
|
+
}
|
|
221
|
+
catch {
|
|
222
|
+
// Unparseable input: `new URL()` gave us nothing to work with, so
|
|
223
|
+
// best-effort drop the query/fragment (where secrets like tokens live)
|
|
224
|
+
// and strip any `//user:pass@` segment before falling back to the raw
|
|
225
|
+
// string — never echo the input completely unredacted.
|
|
226
|
+
const withoutQueryOrFragment = url.split(/[?#]/)[0];
|
|
227
|
+
return truncate(redactUrlCredentials(withoutQueryOrFragment));
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
/**
|
|
231
|
+
* Scrub any embedded absolute URLs out of free-form text — typically the
|
|
232
|
+
* `.message` of an underlying network/DNS/TLS error thrown by `fetch`,
|
|
233
|
+
* undici's `request()`, or Node's resolver, all of which frequently embed
|
|
234
|
+
* the full request URL (query string, credentials, and all) verbatim.
|
|
235
|
+
* Interpolating that message raw into a thrown or logged error would leak a
|
|
236
|
+
* presigned URL's token even after the explicit URL argument was already
|
|
237
|
+
* redacted via {@link redactUrlForError}. Finds each `scheme://…` run in the
|
|
238
|
+
* text and replaces it with its redacted form.
|
|
239
|
+
*
|
|
240
|
+
* @param text - Free-form text (typically `Error.message`) to scrub.
|
|
241
|
+
*/
|
|
242
|
+
export function redactUrlsInText(text) {
|
|
243
|
+
return text.replace(/[a-zA-Z][a-zA-Z0-9+.-]*:\/\/[^\s"'<>)]+/g, (match) => redactUrlForError(match));
|
|
244
|
+
}
|
|
245
|
+
/**
|
|
246
|
+
* Redact every occurrence of `filePath` from an error message, checking both
|
|
247
|
+
* the path exactly as given and its `path.resolve()`'d form.
|
|
248
|
+
*
|
|
249
|
+
* Node's own fs errors (ENOENT/EACCES) embed the exact string passed to the
|
|
250
|
+
* fs call, so a plain substring match usually suffices — but wrapped or
|
|
251
|
+
* lower-level errors sometimes normalize the path first (relative→absolute,
|
|
252
|
+
* trailing-slash variations), which an exact-match-only redaction would miss.
|
|
253
|
+
* This is defense-in-depth, not a guarantee: a `realpath`-resolved (symlink-
|
|
254
|
+
* following) variant can still differ from both forms and would slip through.
|
|
255
|
+
*
|
|
256
|
+
* @param message - The error message to scrub.
|
|
257
|
+
* @param filePath - The path to redact (both as given and resolved).
|
|
258
|
+
*/
|
|
259
|
+
export function redactPathFromMessage(message, filePath) {
|
|
260
|
+
const safeName = basename(filePath);
|
|
261
|
+
// Longest-first: the resolved (absolute) form always contains the relative
|
|
262
|
+
// form as a substring, so redacting the relative form first would only
|
|
263
|
+
// strip the trailing segment of an absolute path and leave its parent
|
|
264
|
+
// directory behind (e.g. "/Users/me/project/foo/bar.png" redacting
|
|
265
|
+
// "foo/bar.png" first leaves "/Users/me/project/bar.png"). Redacting the
|
|
266
|
+
// longer resolved variant first removes the whole absolute path in one
|
|
267
|
+
// pass, so the shorter relative pass below has nothing left to match.
|
|
268
|
+
return message
|
|
269
|
+
.split(resolvePath(filePath))
|
|
270
|
+
.join(safeName)
|
|
271
|
+
.split(filePath)
|
|
272
|
+
.join(safeName);
|
|
273
|
+
}
|
|
274
|
+
/**
|
|
275
|
+
* Build a sanitized copy of a network/URL/file error, safe to attach as
|
|
276
|
+
* `Error.cause` on a rethrow.
|
|
277
|
+
*
|
|
278
|
+
* Redacting only the OUTER thrown error's message isn't enough: if the RAW
|
|
279
|
+
* underlying error is attached as `cause`, anything that walks the cause
|
|
280
|
+
* chain (cause-aware logging, telemetry spans) can still recover an
|
|
281
|
+
* unredacted URL/path/secret via `error.cause.message`. This returns a NEW
|
|
282
|
+
* `Error` whose message has been through {@link redactUrlsInText} — and,
|
|
283
|
+
* when `options.filePath` is supplied, {@link redactPathFromMessage} too, so
|
|
284
|
+
* a filesystem error's ENOENT/EACCES text doesn't leak the host's directory
|
|
285
|
+
* layout via the cause chain the way the outer message already avoids.
|
|
286
|
+
* `.name` and `.code` (`NodeJS.ErrnoException`) are copied over so retry
|
|
287
|
+
* classification (e.g. `isRetryableNetworkError`) still works when it
|
|
288
|
+
* inspects the cause. The raw original is never referenced by the result.
|
|
289
|
+
*
|
|
290
|
+
* Handles non-`Error` thrown values too (a raw string/object can just as
|
|
291
|
+
* easily carry an unredacted URL or path) by running their `String()` form
|
|
292
|
+
* through the same redaction before wrapping.
|
|
293
|
+
*
|
|
294
|
+
* @param error - The raw underlying error (or thrown value) to sanitize.
|
|
295
|
+
* @param options - `filePath`: a known filesystem path to redact to its
|
|
296
|
+
* basename, in addition to URL redaction.
|
|
297
|
+
*/
|
|
298
|
+
export function sanitizeErrorCause(error, options) {
|
|
299
|
+
const redact = (text) => {
|
|
300
|
+
const urlSafe = redactUrlsInText(text);
|
|
301
|
+
return options?.filePath
|
|
302
|
+
? redactPathFromMessage(urlSafe, options.filePath)
|
|
303
|
+
: urlSafe;
|
|
304
|
+
};
|
|
305
|
+
if (!(error instanceof Error)) {
|
|
306
|
+
return new Error(redact(String(error)));
|
|
307
|
+
}
|
|
308
|
+
const sanitized = new Error(redact(error.message));
|
|
309
|
+
sanitized.name = error.name;
|
|
310
|
+
const code = error.code;
|
|
311
|
+
if (code !== undefined) {
|
|
312
|
+
sanitized.code = code;
|
|
313
|
+
}
|
|
314
|
+
return sanitized;
|
|
165
315
|
}
|
|
166
316
|
/**
|
|
167
317
|
* Recursively sanitize a record/array, returning a structurally identical
|