@juspay/neurolink 9.95.2 → 10.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/CHANGELOG.md +23 -0
  2. package/dist/browser/neurolink.min.js +380 -380
  3. package/dist/cli/commands/proxy.d.ts +26 -0
  4. package/dist/cli/commands/proxy.js +132 -13
  5. package/dist/cli/factories/commandFactory.d.ts +7 -0
  6. package/dist/cli/factories/commandFactory.js +29 -21
  7. package/dist/cli/utils/inputValidation.d.ts +1 -0
  8. package/dist/cli/utils/inputValidation.js +6 -1
  9. package/dist/core/constants.d.ts +3 -0
  10. package/dist/core/constants.js +10 -0
  11. package/dist/lib/core/constants.d.ts +3 -0
  12. package/dist/lib/core/constants.js +10 -0
  13. package/dist/lib/types/file.d.ts +38 -1
  14. package/dist/lib/utils/fileDetector.js +69 -6
  15. package/dist/lib/utils/imageCache.d.ts +10 -2
  16. package/dist/lib/utils/imageCache.js +12 -23
  17. package/dist/lib/utils/imageProcessor.d.ts +25 -0
  18. package/dist/lib/utils/imageProcessor.js +28 -1
  19. package/dist/lib/utils/logSanitize.d.ts +97 -0
  20. package/dist/lib/utils/logSanitize.js +163 -0
  21. package/dist/lib/utils/messageBuilder.js +36 -0
  22. package/dist/lib/utils/pdfProcessor.d.ts +30 -1
  23. package/dist/lib/utils/pdfProcessor.js +215 -52
  24. package/dist/types/file.d.ts +38 -1
  25. package/dist/utils/fileDetector.js +69 -6
  26. package/dist/utils/imageCache.d.ts +10 -2
  27. package/dist/utils/imageCache.js +12 -23
  28. package/dist/utils/imageProcessor.d.ts +25 -0
  29. package/dist/utils/imageProcessor.js +28 -1
  30. package/dist/utils/logSanitize.d.ts +97 -0
  31. package/dist/utils/logSanitize.js +163 -0
  32. package/dist/utils/messageBuilder.js +36 -0
  33. package/dist/utils/pdfProcessor.d.ts +30 -1
  34. package/dist/utils/pdfProcessor.js +215 -52
  35. package/package.json +1 -1
@@ -14,7 +14,7 @@
14
14
  */
15
15
  import { createHash } from "crypto";
16
16
  import { logger } from "./logger.js";
17
- import { redactUrlForError } from "./logSanitize.js";
17
+ import { normalizeUrlForCache, redactUrlForError } from "./logSanitize.js";
18
18
  /**
19
19
  * LRU Cache for downloaded images
20
20
  *
@@ -85,30 +85,19 @@ export class ImageCache {
85
85
  return value;
86
86
  }
87
87
  /**
88
- * Normalize URL for consistent cache key generation
89
- * Removes tracking parameters and normalizes the URL
88
+ * Normalize URL for consistent cache key generation.
89
+ *
90
+ * Delegates to the shared {@link normalizeUrlForCache} — this normalized
91
+ * URL is used as the in-memory Map key (see `get`/`set` below), which lives
92
+ * for the process lifetime, so both tracking-param noise and presigned-URL
93
+ * auth/signature secrets must be stripped consistently with
94
+ * `fileDetector.cacheKeyForUrl`'s identical requirement. See
95
+ * `stripSensitiveUrlParamsForCacheKey`'s doc comment for why a naive
96
+ * strip-only approach (no hash suffix) would be a correctness bug, not just
97
+ * a hygiene one.
90
98
  */
91
99
  normalizeUrl(url) {
92
- try {
93
- const parsed = new URL(url);
94
- // Remove common tracking parameters that don't affect content
95
- const trackingParams = [
96
- "utm_source",
97
- "utm_medium",
98
- "utm_campaign",
99
- "utm_term",
100
- "utm_content",
101
- "fbclid",
102
- "gclid",
103
- "_ga",
104
- ];
105
- trackingParams.forEach((param) => parsed.searchParams.delete(param));
106
- return parsed.toString();
107
- }
108
- catch {
109
- // If URL parsing fails, use the original URL
110
- return url;
111
- }
100
+ return normalizeUrlForCache(url);
112
101
  }
113
102
  /**
114
103
  * Generate content hash from image data
@@ -13,6 +13,31 @@ export declare const MIN_IMAGE_SIZE = 12;
13
13
  * buffer that carries a format's magic bytes but is smaller than this is
14
14
  * truncated/corrupt and would fail (or render blank) downstream — reject early
15
15
  * with a clear message instead.
16
+ *
17
+ * Each threshold approximates the smallest byte count that format's
18
+ * container can structurally be while still holding the mandatory
19
+ * header/chunk skeleton for a minimal (e.g. 1×1 pixel) image — not an exact
20
+ * spec minimum, but close enough that anything smaller is unambiguously
21
+ * truncated:
22
+ * - **PNG (67)**: 8-byte signature + IHDR chunk (4 len + 4 type + 13 data +
23
+ * 4 CRC = 25) + a minimal IDAT chunk carrying one zlib-compressed
24
+ * scanline (~22 bytes) + IEND chunk (4 len + 4 type + 0 data + 4 CRC =
25
+ * 12) — matches the commonly-cited size of the smallest possible valid
26
+ * PNG (a 1×1 transparent pixel).
27
+ * - **JPEG (125)**: SOI + APP0/JFIF marker (18) + at least one DQT
28
+ * quantization table (~69 for one 8-bit luma table) + SOF0 + DHT + SOS
29
+ * headers + a minimal entropy-coded scan + EOI — the smallest legal
30
+ * baseline-JPEG skeleton for a 1×1 image.
31
+ * - **GIF (43)**: 6-byte GIF87a/89a header + 7-byte Logical Screen
32
+ * Descriptor + a 2-color (6-byte) color table + 10-byte Image
33
+ * Descriptor + minimal LZW-coded image sub-blocks + 1-byte trailer.
34
+ * - **WEBP (20)**: RIFF container header ("RIFF" + size + "WEBP" = 12
35
+ * bytes) + the first chunk's fourCC + size (8 bytes) — enough to confirm
36
+ * a well-formed RIFF/WEBP container and its first chunk, before any
37
+ * actual VP8/VP8L bitstream payload.
38
+ * - **AVIF (100)**: ISOBMFF `ftyp` box plus the mandatory `meta` box
39
+ * skeleton (`hdlr`/`pitm`/`iloc`/`iinf` sub-boxes) — roughly the minimum
40
+ * to hold AVIF's required box structure before any AV1 image item data.
16
41
  */
17
42
  export declare const MIN_VALID_IMAGE_SIZE: Record<string, number>;
18
43
  /**
@@ -20,6 +20,31 @@ export const MIN_IMAGE_SIZE = 12;
20
20
  * buffer that carries a format's magic bytes but is smaller than this is
21
21
  * truncated/corrupt and would fail (or render blank) downstream — reject early
22
22
  * with a clear message instead.
23
+ *
24
+ * Each threshold approximates the smallest byte count that format's
25
+ * container can structurally be while still holding the mandatory
26
+ * header/chunk skeleton for a minimal (e.g. 1×1 pixel) image — not an exact
27
+ * spec minimum, but close enough that anything smaller is unambiguously
28
+ * truncated:
29
+ * - **PNG (67)**: 8-byte signature + IHDR chunk (4 len + 4 type + 13 data +
30
+ * 4 CRC = 25) + a minimal IDAT chunk carrying one zlib-compressed
31
+ * scanline (~22 bytes) + IEND chunk (4 len + 4 type + 0 data + 4 CRC =
32
+ * 12) — matches the commonly-cited size of the smallest possible valid
33
+ * PNG (a 1×1 transparent pixel).
34
+ * - **JPEG (125)**: SOI + APP0/JFIF marker (18) + at least one DQT
35
+ * quantization table (~69 for one 8-bit luma table) + SOF0 + DHT + SOS
36
+ * headers + a minimal entropy-coded scan + EOI — the smallest legal
37
+ * baseline-JPEG skeleton for a 1×1 image.
38
+ * - **GIF (43)**: 6-byte GIF87a/89a header + 7-byte Logical Screen
39
+ * Descriptor + a 2-color (6-byte) color table + 10-byte Image
40
+ * Descriptor + minimal LZW-coded image sub-blocks + 1-byte trailer.
41
+ * - **WEBP (20)**: RIFF container header ("RIFF" + size + "WEBP" = 12
42
+ * bytes) + the first chunk's fourCC + size (8 bytes) — enough to confirm
43
+ * a well-formed RIFF/WEBP container and its first chunk, before any
44
+ * actual VP8/VP8L bitstream payload.
45
+ * - **AVIF (100)**: ISOBMFF `ftyp` box plus the mandatory `meta` box
46
+ * skeleton (`hdlr`/`pitm`/`iloc`/`iinf` sub-boxes) — roughly the minimum
47
+ * to hold AVIF's required box structure before any AV1 image item data.
23
48
  */
24
49
  export const MIN_VALID_IMAGE_SIZE = {
25
50
  "image/png": 67,
@@ -932,7 +957,9 @@ export const imageUtils = {
932
957
  const cache = getImageCache();
933
958
  const cached = cache.get(url);
934
959
  if (cached) {
935
- logger.debug("Using cached image for URL", { url: url.substring(0, 50) });
960
+ logger.debug("Using cached image for URL", {
961
+ url: redactUrlForError(url),
962
+ });
936
963
  return cached.dataUri;
937
964
  }
938
965
  // Apply rate limiting before download
@@ -110,6 +110,103 @@ export declare function redactUrlCredentials(url: string): string;
110
110
  * @param url - The URL (or URL-shaped string) to redact.
111
111
  */
112
112
  export declare function redactUrlForError(url: string): string;
113
+ /**
114
+ * Query-parameter names that commonly carry authentication/signature
115
+ * secrets in presigned URLs, matched case-insensitively: generic
116
+ * `token`/`signature`, AWS SigV4 (`X-Amz-Signature`, `X-Amz-Credential`,
117
+ * `X-Amz-Security-Token`, and the legacy `AWSAccessKeyId`/`Signature`/
118
+ * `Expires` query-auth params), Azure SAS (`sig`/`se`/`sp`/`sr`/`sv`), and
119
+ * GCS V4 signed URLs (`X-Goog-Signature`, `X-Goog-Credential`,
120
+ * `X-Goog-Algorithm`, `X-Goog-Date`, `X-Goog-Expires`,
121
+ * `X-Goog-SignedHeaders`).
122
+ *
123
+ * Unlike {@link redactUrlForError} (which drops the *entire* query string
124
+ * for a one-off error message), this is used where a URL is normalized
125
+ * into a long-lived in-memory cache key — content-varying query params
126
+ * must survive so distinct resources still get distinct keys, but a
127
+ * presigned URL's secret must never persist as a Map key for the process
128
+ * lifetime.
129
+ */
130
+ export declare const SENSITIVE_URL_QUERY_PARAM_DENYLIST: readonly string[];
131
+ /**
132
+ * Strip common tracking/analytics query params (`utm_*`, `fbclid`, `gclid`,
133
+ * `_ga`) from `parsed` in place, so callers that build a cache key from a
134
+ * URL don't miss a cache hit solely because two otherwise-identical URLs
135
+ * carry different tracking noise.
136
+ *
137
+ * Shared by `ImageCache.normalizeUrl` and `fileDetector.cacheKeyForUrl` —
138
+ * both build a long-lived in-memory cache key from a URL and should
139
+ * normalize tracking params the same way. Call this before
140
+ * {@link stripSensitiveUrlParamsForCacheKey} so both normalizations apply
141
+ * consistently regardless of caller.
142
+ *
143
+ * @param parsed - A `URL` instance to strip tracking params from in place.
144
+ */
145
+ export declare function stripTrackingParams(parsed: URL): void;
146
+ /**
147
+ * Build a secret-free, but still auth-context-distinct, cache key from a
148
+ * URL that may carry presigned-URL secrets in its query string.
149
+ *
150
+ * Naively stripping {@link SENSITIVE_URL_QUERY_PARAM_DENYLIST} params before
151
+ * using a URL as a long-lived cache key (an earlier version of this code did
152
+ * exactly that) creates a correctness bug, not just a hygiene one: two
153
+ * *different* presigned URLs for the same underlying object — e.g. one
154
+ * whose signature has since expired or been revoked, and a fresh one —
155
+ * strip down to the *same* key and collide. A cache populated under the
156
+ * first would silently serve its bytes back for the second, without ever
157
+ * revalidating against the origin.
158
+ *
159
+ * Fix: still remove the sensitive params from `parsed` in place (so the
160
+ * plaintext key never retains a raw secret — the original goal), but fold a
161
+ * short, non-reversible hash of the *removed* key=value pairs into the
162
+ * returned suffix whenever any were present, so distinct auth contexts keep
163
+ * distinct cache keys. Two requests presenting the exact same secret value
164
+ * still collide (correct — that's genuinely the same authorized request);
165
+ * two different secret values for the same object no longer do.
166
+ *
167
+ * Callers combine the return value with the now-stripped `parsed.toString()`
168
+ * to form the final cache key (see `ImageCache.normalizeUrl` and
169
+ * `fileDetector.cacheKeyForUrl`), rather than this function returning the
170
+ * full key itself, so each caller can apply its own additional
171
+ * normalization (e.g. tracking-param removal) in between.
172
+ *
173
+ * Matching against {@link SENSITIVE_URL_QUERY_PARAM_DENYLIST} is
174
+ * case-insensitive (both the denylist lookup and the value folded into the
175
+ * hash lower-case the param name), so the original casing of a sensitive
176
+ * param's name is not preserved anywhere in the returned key. That's fine
177
+ * here — this produces a cache key, not a user-facing URL, and the same
178
+ * lower-cased name always hashes to the same suffix for the same value
179
+ * regardless of how the caller happened to case it (`Token=` vs `token=`).
180
+ * Non-sensitive params left on `parsed` keep their original casing untouched.
181
+ *
182
+ * @param parsed - A `URL` instance to strip sensitive params from in place.
183
+ * @returns "" when no sensitive params were present; otherwise a short hex
184
+ * suffix — prefixed with `\0` (a byte `URL#toString()` never emits, since
185
+ * every URL component is percent-encoded, so it can't collide with real
186
+ * URL content) — to append to the caller's cache key.
187
+ */
188
+ export declare function stripSensitiveUrlParamsForCacheKey(parsed: URL): string;
189
+ /**
190
+ * Build a secret-free, auth-context-distinct cache key from a URL, for any
191
+ * caller that needs a long-lived in-memory Map key derived from a URL.
192
+ *
193
+ * Centralizes the exact sequence both existing callers used independently:
194
+ * parse via `new URL()`, strip tracking noise ({@link stripTrackingParams}),
195
+ * then strip and hash-suffix sensitive auth params
196
+ * ({@link stripSensitiveUrlParamsForCacheKey}) so distinct presigned URLs for
197
+ * the same object don't collide (see that function's doc comment for why a
198
+ * naive strip-only approach is a correctness bug, not just a hygiene one).
199
+ *
200
+ * Shared by `ImageCache.normalizeUrl` and `fileDetector.cacheKeyForUrl` — both
201
+ * build a cache key from a URL and must normalize it identically, or the same
202
+ * URL could hash to two different keys depending on which cache looked it up.
203
+ *
204
+ * Falls back to the raw `url` string unchanged when it isn't a parseable
205
+ * absolute URL (mirrors both callers' pre-existing catch behavior).
206
+ *
207
+ * @param url - The URL (or URL-shaped string) to normalize into a cache key.
208
+ */
209
+ export declare function normalizeUrlForCache(url: string): string;
113
210
  /**
114
211
  * Scrub any embedded absolute URLs out of free-form text — typically the
115
212
  * `.message` of an underlying network/DNS/TLS error thrown by `fetch`,
@@ -15,6 +15,7 @@
15
15
  * jina_ (Jina), fish- (Fish Audio)
16
16
  * - Generic key=value pairs: api_key=…, access_token: …, secret_key=…
17
17
  */
18
+ import { createHash } from "crypto";
18
19
  import { basename, resolve as resolvePath } from "path";
19
20
  const TOKEN_PREFIXES = [
20
21
  "sk",
@@ -227,6 +228,168 @@ export function redactUrlForError(url) {
227
228
  return truncate(redactUrlCredentials(withoutQueryOrFragment));
228
229
  }
229
230
  }
231
+ /**
232
+ * Query-parameter names that commonly carry authentication/signature
233
+ * secrets in presigned URLs, matched case-insensitively: generic
234
+ * `token`/`signature`, AWS SigV4 (`X-Amz-Signature`, `X-Amz-Credential`,
235
+ * `X-Amz-Security-Token`, and the legacy `AWSAccessKeyId`/`Signature`/
236
+ * `Expires` query-auth params), Azure SAS (`sig`/`se`/`sp`/`sr`/`sv`), and
237
+ * GCS V4 signed URLs (`X-Goog-Signature`, `X-Goog-Credential`,
238
+ * `X-Goog-Algorithm`, `X-Goog-Date`, `X-Goog-Expires`,
239
+ * `X-Goog-SignedHeaders`).
240
+ *
241
+ * Unlike {@link redactUrlForError} (which drops the *entire* query string
242
+ * for a one-off error message), this is used where a URL is normalized
243
+ * into a long-lived in-memory cache key — content-varying query params
244
+ * must survive so distinct resources still get distinct keys, but a
245
+ * presigned URL's secret must never persist as a Map key for the process
246
+ * lifetime.
247
+ */
248
+ export const SENSITIVE_URL_QUERY_PARAM_DENYLIST = [
249
+ "token",
250
+ "signature",
251
+ "x-amz-signature",
252
+ "x-amz-credential",
253
+ "x-amz-security-token",
254
+ "awsaccesskeyid",
255
+ "expires",
256
+ "sig",
257
+ "se",
258
+ "sp",
259
+ "sr",
260
+ "sv",
261
+ "x-goog-signature",
262
+ "x-goog-credential",
263
+ "x-goog-algorithm",
264
+ "x-goog-date",
265
+ "x-goog-expires",
266
+ "x-goog-signedheaders",
267
+ ];
268
+ /** Common tracking/analytics query params that don't affect the fetched
269
+ * content, so two URLs differing only by these should map to the same
270
+ * cache key rather than missing each other. */
271
+ const TRACKING_URL_QUERY_PARAMS = [
272
+ "utm_source",
273
+ "utm_medium",
274
+ "utm_campaign",
275
+ "utm_term",
276
+ "utm_content",
277
+ "fbclid",
278
+ "gclid",
279
+ "_ga",
280
+ ];
281
+ /**
282
+ * Strip common tracking/analytics query params (`utm_*`, `fbclid`, `gclid`,
283
+ * `_ga`) from `parsed` in place, so callers that build a cache key from a
284
+ * URL don't miss a cache hit solely because two otherwise-identical URLs
285
+ * carry different tracking noise.
286
+ *
287
+ * Shared by `ImageCache.normalizeUrl` and `fileDetector.cacheKeyForUrl` —
288
+ * both build a long-lived in-memory cache key from a URL and should
289
+ * normalize tracking params the same way. Call this before
290
+ * {@link stripSensitiveUrlParamsForCacheKey} so both normalizations apply
291
+ * consistently regardless of caller.
292
+ *
293
+ * @param parsed - A `URL` instance to strip tracking params from in place.
294
+ */
295
+ export function stripTrackingParams(parsed) {
296
+ TRACKING_URL_QUERY_PARAMS.forEach((param) => parsed.searchParams.delete(param));
297
+ }
298
+ /**
299
+ * Build a secret-free, but still auth-context-distinct, cache key from a
300
+ * URL that may carry presigned-URL secrets in its query string.
301
+ *
302
+ * Naively stripping {@link SENSITIVE_URL_QUERY_PARAM_DENYLIST} params before
303
+ * using a URL as a long-lived cache key (an earlier version of this code did
304
+ * exactly that) creates a correctness bug, not just a hygiene one: two
305
+ * *different* presigned URLs for the same underlying object — e.g. one
306
+ * whose signature has since expired or been revoked, and a fresh one —
307
+ * strip down to the *same* key and collide. A cache populated under the
308
+ * first would silently serve its bytes back for the second, without ever
309
+ * revalidating against the origin.
310
+ *
311
+ * Fix: still remove the sensitive params from `parsed` in place (so the
312
+ * plaintext key never retains a raw secret — the original goal), but fold a
313
+ * short, non-reversible hash of the *removed* key=value pairs into the
314
+ * returned suffix whenever any were present, so distinct auth contexts keep
315
+ * distinct cache keys. Two requests presenting the exact same secret value
316
+ * still collide (correct — that's genuinely the same authorized request);
317
+ * two different secret values for the same object no longer do.
318
+ *
319
+ * Callers combine the return value with the now-stripped `parsed.toString()`
320
+ * to form the final cache key (see `ImageCache.normalizeUrl` and
321
+ * `fileDetector.cacheKeyForUrl`), rather than this function returning the
322
+ * full key itself, so each caller can apply its own additional
323
+ * normalization (e.g. tracking-param removal) in between.
324
+ *
325
+ * Matching against {@link SENSITIVE_URL_QUERY_PARAM_DENYLIST} is
326
+ * case-insensitive (both the denylist lookup and the value folded into the
327
+ * hash lower-case the param name), so the original casing of a sensitive
328
+ * param's name is not preserved anywhere in the returned key. That's fine
329
+ * here — this produces a cache key, not a user-facing URL, and the same
330
+ * lower-cased name always hashes to the same suffix for the same value
331
+ * regardless of how the caller happened to case it (`Token=` vs `token=`).
332
+ * Non-sensitive params left on `parsed` keep their original casing untouched.
333
+ *
334
+ * @param parsed - A `URL` instance to strip sensitive params from in place.
335
+ * @returns "" when no sensitive params were present; otherwise a short hex
336
+ * suffix — prefixed with `\0` (a byte `URL#toString()` never emits, since
337
+ * every URL component is percent-encoded, so it can't collide with real
338
+ * URL content) — to append to the caller's cache key.
339
+ */
340
+ export function stripSensitiveUrlParamsForCacheKey(parsed) {
341
+ const strippedPairs = [];
342
+ const keysToDelete = [];
343
+ for (const [key, value] of parsed.searchParams.entries()) {
344
+ if (SENSITIVE_URL_QUERY_PARAM_DENYLIST.includes(key.toLowerCase())) {
345
+ strippedPairs.push(`${key.toLowerCase()}=${value}`);
346
+ keysToDelete.push(key);
347
+ }
348
+ }
349
+ if (strippedPairs.length === 0) {
350
+ return "";
351
+ }
352
+ keysToDelete.forEach((key) => parsed.searchParams.delete(key));
353
+ // Sort so key order in the original URL doesn't affect the hash — the
354
+ // same *set* of secret values must always hash to the same suffix.
355
+ strippedPairs.sort();
356
+ const hash = createHash("sha256")
357
+ .update(strippedPairs.join("&"))
358
+ .digest("hex")
359
+ .slice(0, 16);
360
+ return `\0auth=${hash}`;
361
+ }
362
+ /**
363
+ * Build a secret-free, auth-context-distinct cache key from a URL, for any
364
+ * caller that needs a long-lived in-memory Map key derived from a URL.
365
+ *
366
+ * Centralizes the exact sequence both existing callers used independently:
367
+ * parse via `new URL()`, strip tracking noise ({@link stripTrackingParams}),
368
+ * then strip and hash-suffix sensitive auth params
369
+ * ({@link stripSensitiveUrlParamsForCacheKey}) so distinct presigned URLs for
370
+ * the same object don't collide (see that function's doc comment for why a
371
+ * naive strip-only approach is a correctness bug, not just a hygiene one).
372
+ *
373
+ * Shared by `ImageCache.normalizeUrl` and `fileDetector.cacheKeyForUrl` — both
374
+ * build a cache key from a URL and must normalize it identically, or the same
375
+ * URL could hash to two different keys depending on which cache looked it up.
376
+ *
377
+ * Falls back to the raw `url` string unchanged when it isn't a parseable
378
+ * absolute URL (mirrors both callers' pre-existing catch behavior).
379
+ *
380
+ * @param url - The URL (or URL-shaped string) to normalize into a cache key.
381
+ */
382
+ export function normalizeUrlForCache(url) {
383
+ try {
384
+ const parsed = new URL(url);
385
+ stripTrackingParams(parsed);
386
+ const authSuffix = stripSensitiveUrlParamsForCacheKey(parsed);
387
+ return parsed.toString() + authSuffix;
388
+ }
389
+ catch {
390
+ return url;
391
+ }
392
+ }
230
393
  /**
231
394
  * Scrub any embedded absolute URLs out of free-form text — typically the
232
395
  * `.message` of an underlying network/DNS/TLS error thrown by `fetch`,
@@ -1,5 +1,6 @@
1
1
  import { existsSync, readFileSync, statSync } from "fs";
2
2
  import { readFile as readFileAsync, stat as statAsync } from "fs/promises";
3
+ import { basename } from "path";
3
4
  import { getGlobalDispatcher, interceptors, request } from "undici";
4
5
  import { MultimodalLogger, ProviderImageAdapter, } from "../adapters/providerImageAdapter.js";
5
6
  import { CONVERSATION_INSTRUCTIONS, STRUCTURED_OUTPUT_INSTRUCTIONS, } from "../config/conversationMemory.js";
@@ -1026,6 +1027,41 @@ async function processExplicitPdfFiles(options, maxSize, provider) {
1026
1027
  throw error;
1027
1028
  }
1028
1029
  }
1030
+ // #309: enforce the provider's page/size ceilings across ALL PDFs, not just
1031
+ // per-file. N files each just under the single-file limit can still blow past
1032
+ // it in aggregate (e.g. three 40-page PDFs → 120 pages for a 100-page API).
1033
+ const aggregateConfig = PDFProcessor.getProviderConfig(provider);
1034
+ if (aggregateConfig && pdfFiles.length > 1) {
1035
+ // A null pageCount (accurate count unavailable — see
1036
+ // PDFProcessor.getAccuratePageCount) is treated as 0 in the sum below,
1037
+ // which can undercount the aggregate and let a combined request over
1038
+ // the provider's page limit slip through silently. Enforcement still
1039
+ // runs against the known sum — a PDF with an unknown count must not
1040
+ // fail the request outright — but the gap itself must not be silent.
1041
+ const unknownPageCountFiles = pdfFiles.filter((f) => f.pageCount === null || f.pageCount === undefined);
1042
+ const totalPages = pdfFiles.reduce((sum, f) => sum + (f.pageCount ?? 0), 0);
1043
+ const totalMB = pdfFiles.reduce((sum, f) => sum + f.buffer.length, 0) / (1024 * 1024);
1044
+ if (unknownPageCountFiles.length > 0) {
1045
+ // Filenames are caller-controlled and may be full paths carrying
1046
+ // PII/internal directory segments — log only the basename.
1047
+ const unknownFileNames = unknownPageCountFiles
1048
+ .map((f) => (f.filename ? basename(f.filename) : "<unnamed file>"))
1049
+ .join(", ");
1050
+ logger.warn(`[PDF] Aggregate page-limit check across ${pdfFiles.length} PDFs could only be ` +
1051
+ `partially verified: ${unknownPageCountFiles.length} file(s) have an unknown ` +
1052
+ `page count (${unknownFileNames}), so the known total (${totalPages}) may ` +
1053
+ `undercount the true combined page count.`);
1054
+ }
1055
+ if (totalPages > aggregateConfig.maxPages) {
1056
+ throw new Error(`[PDF] Combined page count across ${pdfFiles.length} PDFs (${totalPages}) exceeds the ` +
1057
+ `${aggregateConfig.maxPages}-page limit for ${provider}. ` +
1058
+ `Split the request or reduce the number of PDFs.`);
1059
+ }
1060
+ if (totalMB > aggregateConfig.maxSizeMB) {
1061
+ throw new Error(`[PDF] Combined size across ${pdfFiles.length} PDFs (${totalMB.toFixed(2)}MB) exceeds the ` +
1062
+ `${aggregateConfig.maxSizeMB}MB limit for ${provider}.`);
1063
+ }
1064
+ }
1029
1065
  return pdfFiles;
1030
1066
  }
1031
1067
  /**
@@ -7,7 +7,7 @@
7
7
  *
8
8
  * The conversion uses pdf-to-img package (MuPDF-based) for high-quality conversion.
9
9
  */
10
- import type { FileProcessingResult, PDFProcessorOptions, PDFProviderConfig, PDFImageConversionOptions, PDFImageConversionResult } from "../types/index.js";
10
+ import type { FileProcessingResult, PDFProcessorOptions, PDFProviderConfig, PDFImageConversionOptions, PDFImageConversionResult, PDFImagePage } from "../types/index.js";
11
11
  export declare class PDFProcessor {
12
12
  private static readonly PDF_SIGNATURE;
13
13
  static process(content: Buffer, options?: PDFProcessorOptions): Promise<FileProcessingResult>;
@@ -20,6 +20,15 @@ export declare class PDFProcessor {
20
20
  static getProviderConfig(provider: string): PDFProviderConfig | null;
21
21
  private static isValidPDF;
22
22
  private static extractBasicMetadata;
23
+ /**
24
+ * Accurate page count via pdf-parse (pdfjs) (#287). The header regex in
25
+ * `extractBasicMetadata` only sees plaintext `/Type /Page` markers and misses
26
+ * compressed/object-stream PDFs (and miscounts when a page dict spans a chunk
27
+ * boundary). This parses the document properly, bounded by a timeout so a
28
+ * pathological PDF can't block; returns null on timeout/failure so the caller
29
+ * falls back to the regex estimate.
30
+ */
31
+ private static getAccuratePageCount;
23
32
  static estimateTokens(pageCount: number, mode?: "text-only" | "visual"): number;
24
33
  /**
25
34
  * Estimate the largest page's rendered pixel count from the PDF's MediaBox
@@ -61,6 +70,26 @@ export declare class PDFProcessor {
61
70
  * ```
62
71
  */
63
72
  static convertToImages(pdfBuffer: Buffer, options?: PDFImageConversionOptions): Promise<PDFImageConversionResult>;
73
+ /**
74
+ * Shared input validation for the image-conversion paths. Returns the PDF
75
+ * size in MB (needed by the memory-estimate log). Throws on any invalid input
76
+ * with the same messages the batch path has always used.
77
+ */
78
+ private static validateImageConversionInput;
79
+ /**
80
+ * Streaming variant of {@link convertToImages} (#302): yields each page's
81
+ * base64 PNG as soon as it renders instead of buffering the whole document,
82
+ * and reports progress via `options.onProgress`. A page that fails to render
83
+ * is yielded with `error` set (per-page isolation, #294) rather than aborting
84
+ * the stream.
85
+ *
86
+ * NOT a wrapper of/over {@link convertToImages}, despite the similar
87
+ * contract — the two are independent, parallel implementations (each does
88
+ * its own `pdf-to-img` import, downscale calculation, and page loop)
89
+ * rather than one delegating to the other. Keep behavior changes (page
90
+ * isolation, downscale, password handling) in sync across both by hand.
91
+ */
92
+ static convertToImagesStream(pdfBuffer: Buffer, options?: PDFImageConversionOptions): AsyncGenerator<PDFImagePage, void, void>;
64
93
  /**
65
94
  * Convert a PDF file path to an array of base64 PNG images
66
95
  *