@juspay/neurolink 9.95.2 → 10.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/CHANGELOG.md +23 -0
  2. package/dist/browser/neurolink.min.js +380 -380
  3. package/dist/cli/commands/proxy.d.ts +26 -0
  4. package/dist/cli/commands/proxy.js +132 -13
  5. package/dist/cli/factories/commandFactory.d.ts +7 -0
  6. package/dist/cli/factories/commandFactory.js +29 -21
  7. package/dist/cli/utils/inputValidation.d.ts +1 -0
  8. package/dist/cli/utils/inputValidation.js +6 -1
  9. package/dist/core/constants.d.ts +3 -0
  10. package/dist/core/constants.js +10 -0
  11. package/dist/lib/core/constants.d.ts +3 -0
  12. package/dist/lib/core/constants.js +10 -0
  13. package/dist/lib/types/file.d.ts +38 -1
  14. package/dist/lib/utils/fileDetector.js +69 -6
  15. package/dist/lib/utils/imageCache.d.ts +10 -2
  16. package/dist/lib/utils/imageCache.js +12 -23
  17. package/dist/lib/utils/imageProcessor.d.ts +25 -0
  18. package/dist/lib/utils/imageProcessor.js +28 -1
  19. package/dist/lib/utils/logSanitize.d.ts +97 -0
  20. package/dist/lib/utils/logSanitize.js +163 -0
  21. package/dist/lib/utils/messageBuilder.js +36 -0
  22. package/dist/lib/utils/pdfProcessor.d.ts +30 -1
  23. package/dist/lib/utils/pdfProcessor.js +215 -52
  24. package/dist/types/file.d.ts +38 -1
  25. package/dist/utils/fileDetector.js +69 -6
  26. package/dist/utils/imageCache.d.ts +10 -2
  27. package/dist/utils/imageCache.js +12 -23
  28. package/dist/utils/imageProcessor.d.ts +25 -0
  29. package/dist/utils/imageProcessor.js +28 -1
  30. package/dist/utils/logSanitize.d.ts +97 -0
  31. package/dist/utils/logSanitize.js +163 -0
  32. package/dist/utils/messageBuilder.js +36 -0
  33. package/dist/utils/pdfProcessor.d.ts +30 -1
  34. package/dist/utils/pdfProcessor.js +215 -52
  35. package/package.json +1 -1
@@ -13,6 +13,32 @@ import type { CommandModule } from "yargs";
13
13
  import type { Hono } from "hono";
14
14
  import type { AccountAllowlist, LoadedProxyConfig, ModelRouterInterface, ProxyGuardArgs, ProxyNeurolinkRuntime, ProxyStartArgs, ProxyStartStrategy, ProxyStatusArgs, ProxyTelemetryArgs, ProxyReadinessState } from "../../lib/types/index.js";
15
15
  import { ProxyRuntimeConfigStore } from "../../lib/proxy/runtimeConfig.js";
16
+ /**
17
+ * Best-effort check that a pid actually belongs to a neurolink proxy process,
18
+ * so a stale/recycled supervisor pid is never mistaken for a live supervisor
19
+ * and signalled. Returns false when the process cannot be confirmed as ours.
20
+ *
21
+ * @param expectedStartTimeIso - The `ProxySupervisorState.startTime` (ISO
22
+ * string) recorded when the supervisor we expect at `pid` was launched.
23
+ * When provided, this is cross-checked against `pid`'s actual OS-reported
24
+ * start time (see {@link getProcessStartTime}) — the args match alone
25
+ * ("neurolink" + "proxy" both present) is not airtight, since an
26
+ * unrelated process (e.g. a shell running this very test suite, or a
27
+ * coincidentally-named script) could match it too. A pid recycled by such
28
+ * a process would have a start time far from the recorded supervisor
29
+ * startTime, which this catches.
30
+ *
31
+ * Deliberately fail-open on anything unparseable: `ps -o lstart=` output
32
+ * parsing is inherently a little fragile (locale/format quirks), and the
33
+ * cost of a false "can't confirm" is silently regressing to pre-hardening
34
+ * behavior, while the cost of a false "confirmed mismatch" is leaving a
35
+ * real supervisor running and orphaning its socket during uninstall — the
36
+ * former is the safer failure mode. So this only ever returns false for
37
+ * the startTime check when BOTH timestamps parsed successfully AND their
38
+ * drift exceeds the tolerance; any parse failure is treated as "can't
39
+ * confirm a mismatch" and falls through to the args-only result.
40
+ */
41
+ export declare function processLooksLikeProxySupervisor(pid: number, expectedStartTimeIso?: string): Promise<boolean>;
16
42
  export declare function mapClaudeErrorTypeToStatus(errorType?: string): number;
17
43
  export declare function createProxyStartApp(params: {
18
44
  neurolink: ProxyNeurolinkRuntime["neurolink"];
@@ -41,6 +41,14 @@ const _require = createRequire(import.meta.url);
41
41
  const PROXY_VERSION = packageJson.version;
42
42
  const PROXY_TELEMETRY_SCRIPT_PATH = fileURLToPath(new URL("../../../scripts/observability/manage-local-openobserve.sh", import.meta.url));
43
43
  const PROXY_LIFECYCLE_SHUTDOWN_TIMEOUT_MS = 5_000;
44
+ // Allowed drift between a pid's OS-reported start time and the persisted
45
+ // ProxySupervisorState.startTime before processLooksLikeProxySupervisor
46
+ // treats it as a confident mismatch (recycled pid). Generous on purpose:
47
+ // `supervisorStartedAt` is captured inside runLaunchdProxySupervisor, which
48
+ // runs after Node has already booted and done setup, so it always lags the
49
+ // OS-level fork/exec by a little — a tight tolerance would produce false
50
+ // mismatches on a slow/cold-started machine.
51
+ const PROXY_SUPERVISOR_START_TIME_TOLERANCE_MS = 10_000;
44
52
  const PROXY_UPDATE_CONTROL_TOKEN = process.env.NEUROLINK_PROXY_UPDATE_CONTROL_TOKEN?.trim() ||
45
53
  crypto.randomUUID();
46
54
  // =============================================================================
@@ -98,23 +106,131 @@ function getProcessStatus(pid) {
98
106
  }
99
107
  }
100
108
  /**
101
- * Best-effort check that a pid actually belongs to a neurolink proxy process,
102
- * so a stale/recycled supervisor pid is never mistaken for a live supervisor
103
- * and signalled. Returns false when the process cannot be confirmed as ours.
109
+ * Read a single `ps -o <field>=` field for `pid`, off the event loop (async
110
+ * `execFile`, not `execFileSync`) and bounded by a short timeout, so a
111
+ * hung/zombie `ps` invocation can never block the event loop or hang the
112
+ * (often uninstall-path) async caller. Shared by {@link getProcessStartTime}
113
+ * and {@link processLooksLikeProxySupervisor}, which both read a single
114
+ * `ps -o <field>=` column for the same pid.
115
+ *
116
+ * Returns null — never throws — on any error, timeout, or empty output, so
117
+ * callers uniformly treat "couldn't read" as "can't confirm either way"
118
+ * rather than distinguishing the failure mode.
104
119
  */
105
- async function processLooksLikeProxySupervisor(pid) {
120
+ async function readPsField(pid, field) {
106
121
  try {
107
- const { execFileSync } = await import("node:child_process");
108
- const args = execFileSync("ps", ["-p", String(pid), "-o", "args="], {
122
+ const { execFile } = await import("node:child_process");
123
+ const { promisify } = await import("node:util");
124
+ const execFileAsync = promisify(execFile);
125
+ const { stdout } = await withTimeout(execFileAsync("ps", ["-p", String(pid), "-o", `${field}=`], {
109
126
  encoding: "utf-8",
110
- stdio: ["ignore", "pipe", "ignore"],
111
- });
112
- return /neurolink/i.test(args);
127
+ }), 2000, `ps -o ${field}= timed out for pid ${pid}`);
128
+ const trimmed = stdout.trim();
129
+ return trimmed || null;
113
130
  }
114
131
  catch {
115
- // `ps` unavailable or the pid vanished — do not signal an unverified pid.
132
+ return null;
133
+ }
134
+ }
135
+ /**
136
+ * Best-effort, macOS-only (uninstall is macOS-only) read of `pid`'s
137
+ * OS-reported start time via `ps -o lstart=`, which prints a fixed-width
138
+ * `Www Mmm dd hh:mm:ss yyyy` timestamp — the same shape `Date`'s parser
139
+ * already understands (it's the format `Date#toString()` itself produces
140
+ * modulo the trailing timezone). Returns null (never throws) when `ps`
141
+ * fails or its output doesn't parse, so callers can tell "confirmed
142
+ * mismatch" apart from "couldn't confirm either way".
143
+ */
144
+ async function getProcessStartTime(pid) {
145
+ const out = await readPsField(pid, "lstart");
146
+ if (!out) {
147
+ return null;
148
+ }
149
+ // `ps -o lstart=` prints the OS start time in the machine's LOCAL timezone
150
+ // with no offset, e.g. "Sun Jul 19 20:12:41 2026". Parse the components and
151
+ // build the Date via the local-time constructor rather than relying on the
152
+ // engine's implementation-defined parsing of non-ISO date strings. The
153
+ // result is the same absolute instant the supervisor persisted as an
154
+ // ISO-UTC string, so the drift comparison against it is timezone-safe.
155
+ const m = out.match(/^\w{3}\s+(\w{3})\s+(\d{1,2})\s+(\d{2}):(\d{2}):(\d{2})\s+(\d{4})$/);
156
+ if (!m) {
157
+ return null;
158
+ }
159
+ const monthIndex = [
160
+ "Jan",
161
+ "Feb",
162
+ "Mar",
163
+ "Apr",
164
+ "May",
165
+ "Jun",
166
+ "Jul",
167
+ "Aug",
168
+ "Sep",
169
+ "Oct",
170
+ "Nov",
171
+ "Dec",
172
+ ].indexOf(m[1]);
173
+ if (monthIndex < 0) {
174
+ return null;
175
+ }
176
+ const parsed = new Date(Number(m[6]), monthIndex, Number(m[2]), Number(m[3]), Number(m[4]), Number(m[5]));
177
+ return Number.isNaN(parsed.getTime()) ? null : parsed;
178
+ }
179
+ /**
180
+ * Best-effort check that a pid actually belongs to a neurolink proxy process,
181
+ * so a stale/recycled supervisor pid is never mistaken for a live supervisor
182
+ * and signalled. Returns false when the process cannot be confirmed as ours.
183
+ *
184
+ * @param expectedStartTimeIso - The `ProxySupervisorState.startTime` (ISO
185
+ * string) recorded when the supervisor we expect at `pid` was launched.
186
+ * When provided, this is cross-checked against `pid`'s actual OS-reported
187
+ * start time (see {@link getProcessStartTime}) — the args match alone
188
+ * ("neurolink" + "proxy" both present) is not airtight, since an
189
+ * unrelated process (e.g. a shell running this very test suite, or a
190
+ * coincidentally-named script) could match it too. A pid recycled by such
191
+ * a process would have a start time far from the recorded supervisor
192
+ * startTime, which this catches.
193
+ *
194
+ * Deliberately fail-open on anything unparseable: `ps -o lstart=` output
195
+ * parsing is inherently a little fragile (locale/format quirks), and the
196
+ * cost of a false "can't confirm" is silently regressing to pre-hardening
197
+ * behavior, while the cost of a false "confirmed mismatch" is leaving a
198
+ * real supervisor running and orphaning its socket during uninstall — the
199
+ * former is the safer failure mode. So this only ever returns false for
200
+ * the startTime check when BOTH timestamps parsed successfully AND their
201
+ * drift exceeds the tolerance; any parse failure is treated as "can't
202
+ * confirm a mismatch" and falls through to the args-only result.
203
+ */
204
+ export async function processLooksLikeProxySupervisor(pid, expectedStartTimeIso) {
205
+ const args = await readPsField(pid, "args");
206
+ if (args === null) {
207
+ // `ps` unavailable, timed out, or the pid vanished — do not signal an
208
+ // unverified pid.
209
+ return false;
210
+ }
211
+ // `/neurolink/i` alone is too broad: any process whose args merely
212
+ // *mention* "neurolink" (e.g. a shell running in a directory named
213
+ // "neurolink", an editor with a neurolink file open) would match, and a
214
+ // stale/recycled pid running such a process could then be SIGTERM'd/
215
+ // SIGKILL'd by `ensureSupervisorStoppedBeforeClear` during uninstall.
216
+ // Require the actual proxy-supervisor invocation — both "neurolink" AND
217
+ // the "proxy" subcommand present in args.
218
+ if (!(/neurolink/i.test(args) && /\bproxy\b/i.test(args))) {
116
219
  return false;
117
220
  }
221
+ if (!expectedStartTimeIso) {
222
+ return true;
223
+ }
224
+ const expected = new Date(expectedStartTimeIso);
225
+ if (Number.isNaN(expected.getTime())) {
226
+ return true; // recorded startTime itself is unparseable — can't confirm a mismatch
227
+ }
228
+ const actual = await getProcessStartTime(pid);
229
+ if (!actual) {
230
+ return true; // ps -o lstart= unavailable/unparseable — can't confirm a mismatch
231
+ }
232
+ const driftMs = Math.abs(actual.getTime() - expected.getTime());
233
+ return driftMs <= PROXY_SUPERVISOR_START_TIME_TOLERANCE_MS;
118
234
  }
119
235
  /**
120
236
  * Ensure any supervisor recorded in state is actually stopped before its state
@@ -124,14 +240,17 @@ async function processLooksLikeProxySupervisor(pid) {
124
240
  * the uninstall instead of silently discarding the record.
125
241
  */
126
242
  async function ensureSupervisorStoppedBeforeClear() {
127
- const pid = loadProxySupervisorState()?.pid;
243
+ const supervisorState = loadProxySupervisorState();
244
+ const pid = supervisorState?.pid;
128
245
  if (!pid || getProcessStatus(pid) === "not_running") {
129
246
  return true;
130
247
  }
131
248
  // Guard against a stale/recycled pid: only signal a process that still looks
132
249
  // like a neurolink proxy supervisor, so uninstall never terminates an
133
- // unrelated, same-user process that inherited the recorded pid.
134
- if (!(await processLooksLikeProxySupervisor(pid))) {
250
+ // unrelated, same-user process that inherited the recorded pid. Also cross-
251
+ // checks the pid's actual OS start time against the recorded
252
+ // supervisorState.startTime (see processLooksLikeProxySupervisor).
253
+ if (!(await processLooksLikeProxySupervisor(pid, supervisorState?.startTime))) {
135
254
  return true;
136
255
  }
137
256
  try {
@@ -192,6 +192,13 @@ export declare class CLICommandFactory {
192
192
  * objects independently.
193
193
  */
194
194
  private static buildCsvOptionsFromArgv;
195
+ /**
196
+ * Build video processor options from CLI argv. Shared by executeGenerate,
197
+ * executeStream and executeBatch, which previously built byte-for-byte
198
+ * identical `videoOptions` objects independently (mirrors
199
+ * {@link buildCsvOptionsFromArgv}).
200
+ */
201
+ private static buildVideoOptionsFromArgv;
195
202
  /**
196
203
  * Build output configuration for generate request
197
204
  */
@@ -2438,6 +2438,20 @@ export class CLICommandFactory {
2438
2438
  parseTimeoutMs: argv.csvParseTimeoutMs,
2439
2439
  };
2440
2440
  }
2441
+ /**
2442
+ * Build video processor options from CLI argv. Shared by executeGenerate,
2443
+ * executeStream and executeBatch, which previously built byte-for-byte
2444
+ * identical `videoOptions` objects independently (mirrors
2445
+ * {@link buildCsvOptionsFromArgv}).
2446
+ */
2447
+ static buildVideoOptionsFromArgv(argv) {
2448
+ return {
2449
+ frames: argv.videoFrames,
2450
+ quality: argv.videoQuality,
2451
+ format: argv.videoFormat,
2452
+ transcribeAudio: argv.transcribeAudio,
2453
+ };
2454
+ }
2441
2455
  /**
2442
2456
  * Build output configuration for generate request
2443
2457
  */
@@ -2677,12 +2691,7 @@ export class CLICommandFactory {
2677
2691
  password: CLICommandFactory.resolvePdfPassword(argv),
2678
2692
  },
2679
2693
  csvOptions: CLICommandFactory.buildCsvOptionsFromArgv(argv),
2680
- videoOptions: {
2681
- frames: argv.videoFrames,
2682
- quality: argv.videoQuality,
2683
- format: argv.videoFormat,
2684
- transcribeAudio: argv.transcribeAudio,
2685
- },
2694
+ videoOptions: CLICommandFactory.buildVideoOptionsFromArgv(argv),
2686
2695
  output: outputConfig,
2687
2696
  provider: enhancedOptions.provider,
2688
2697
  model: enhancedOptions.model,
@@ -2932,12 +2941,7 @@ export class CLICommandFactory {
2932
2941
  password: CLICommandFactory.resolvePdfPassword(argv),
2933
2942
  },
2934
2943
  csvOptions: CLICommandFactory.buildCsvOptionsFromArgv(argv),
2935
- videoOptions: {
2936
- frames: argv.videoFrames,
2937
- quality: argv.videoQuality,
2938
- format: argv.videoFormat,
2939
- transcribeAudio: argv.transcribeAudio,
2940
- },
2944
+ videoOptions: CLICommandFactory.buildVideoOptionsFromArgv(argv),
2941
2945
  provider: enhancedOptions.provider,
2942
2946
  model: enhancedOptions.model,
2943
2947
  temperature: enhancedOptions.temperature,
@@ -3411,6 +3415,7 @@ export class CLICommandFactory {
3411
3415
  // see createBatchCommand — so `argv.file` can never be set here; no
3412
3416
  // exclusion needed to protect the (differently-named) positional.
3413
3417
  validateCliInputFiles(argv);
3418
+ validateCsvMaxRows(argv);
3414
3419
  const batchImages = CLICommandFactory.processCliImages(argv.image);
3415
3420
  const batchCsvFiles = CLICommandFactory.processCliCSVFiles(argv.csv);
3416
3421
  const batchPdfFiles = CLICommandFactory.processCliPDFFiles(argv.pdf);
@@ -3469,6 +3474,12 @@ export class CLICommandFactory {
3469
3474
  logger.alwaysStderr(chalk.yellow("⚠️ Multimodal files (--image/--csv/--pdf/--video) are attached to ALL prompts in this batch."));
3470
3475
  spinner?.start();
3471
3476
  }
3477
+ // Resolve the PDF password once for the whole batch. resolvePdfPassword
3478
+ // emits a "visible in shell history" stderr warning when the flag is set;
3479
+ // it must fire once per run (like generate/stream), not once per prompt.
3480
+ const batchPdfPassword = batchPdfFiles?.length
3481
+ ? CLICommandFactory.resolvePdfPassword(argv)
3482
+ : undefined;
3472
3483
  for (let i = 0; i < prompts.length; i++) {
3473
3484
  if (spinner) {
3474
3485
  spinner.text = `Processing ${i + 1}/${prompts.length}: ${prompts[i].substring(0, 30)}...`;
@@ -3518,18 +3529,15 @@ export class CLICommandFactory {
3518
3529
  // unlike generate/stream, so an empty options object here is
3519
3530
  // pure noise on every prompt of every batch run.
3520
3531
  ...(batchCsvFiles?.length && {
3521
- csvOptions: {
3522
- maxRows: argv.csvMaxRows,
3523
- formatStyle: argv.csvFormat,
3532
+ csvOptions: CLICommandFactory.buildCsvOptionsFromArgv(argv),
3533
+ }),
3534
+ ...(batchPdfFiles?.length && {
3535
+ pdfOptions: {
3536
+ password: batchPdfPassword,
3524
3537
  },
3525
3538
  }),
3526
3539
  ...(batchVideoFiles?.length && {
3527
- videoOptions: {
3528
- frames: argv.videoFrames,
3529
- quality: argv.videoQuality,
3530
- format: argv.videoFormat,
3531
- transcribeAudio: argv.transcribeAudio,
3532
- },
3540
+ videoOptions: CLICommandFactory.buildVideoOptionsFromArgv(argv),
3533
3541
  }),
3534
3542
  provider: enhancedOptions.provider,
3535
3543
  model: enhancedOptions.model,
@@ -21,6 +21,7 @@ export declare const CLI_SOFT_LIMITS_MB: {
21
21
  readonly IMAGE_MAX_MB: 10;
22
22
  readonly CSV_MAX_MB: 50;
23
23
  readonly PDF_MAX_MB: 100;
24
+ readonly PDF_SOFT_MAX_MB: 50;
24
25
  readonly VIDEO_MAX_MB: 500;
25
26
  };
26
27
  /**
@@ -27,6 +27,11 @@ export const CLI_SOFT_LIMITS_MB = {
27
27
  IMAGE_MAX_MB: 10,
28
28
  CSV_MAX_MB: 50,
29
29
  PDF_MAX_MB: 100,
30
+ // Distinct from PDF_MAX_MB above (which mirrors the processor's true
31
+ // 100MB hard cap in lib/processors/config/sizeLimits.ts and is kept here
32
+ // for backward compatibility). Docs promise a 50MB *warning* threshold
33
+ // specifically for --pdf — wire warnAtMB to this value, not the hard cap.
34
+ PDF_SOFT_MAX_MB: 50,
30
35
  VIDEO_MAX_MB: 500,
31
36
  };
32
37
  // `file://` is intentionally excluded here: it addresses a local path, so it
@@ -165,7 +170,7 @@ export function validateCliInputFiles(argv) {
165
170
  {
166
171
  option: "--pdf",
167
172
  value: argv.pdf,
168
- warnAtMB: CLI_SOFT_LIMITS_MB.PDF_MAX_MB,
173
+ warnAtMB: CLI_SOFT_LIMITS_MB.PDF_SOFT_MAX_MB,
169
174
  },
170
175
  {
171
176
  option: "--video",
@@ -122,6 +122,9 @@ export declare const PDF_LIMITS: {
122
122
  MAX_SIZE_MB: number;
123
123
  DEFAULT_MAX_PAGES: number;
124
124
  MAX_SCALE: number;
125
+ MIN_SCALE: number;
126
+ DEFAULT_SCALE: number;
127
+ PAGE_COUNT_TIMEOUT_MS: number;
125
128
  DEFAULT_MAX_CANVAS_PIXELS: number;
126
129
  MIN_EFFECTIVE_SCALE: number;
127
130
  };
@@ -220,6 +220,16 @@ export const PDF_LIMITS = {
220
220
  // Upper bound for the render scale factor. Above this, a single page can
221
221
  // allocate hundreds of MB; scale <= 0 produces a degenerate viewport.
222
222
  MAX_SCALE: 10,
223
+ // Lower bound for the render scale factor (#297). Below this the render is
224
+ // effectively unreadable; enforcing it makes the documented 0.1–10 range real
225
+ // (previously only scale <= 0 was rejected).
226
+ MIN_SCALE: 0.1,
227
+ // Default render scale (#297). Lowered from 2 → 1.5 to roughly halve the
228
+ // per-page canvas memory while staying legible for OCR/vision.
229
+ DEFAULT_SCALE: 1.5,
230
+ // Timeout (ms) for the accurate pdf-parse page-count probe (#287); on timeout
231
+ // the processor falls back to the regex estimate rather than blocking.
232
+ PAGE_COUNT_TIMEOUT_MS: 5000,
223
233
  // Default per-page pixel ceiling for image conversion (#260). 16.7M px ×
224
234
  // 4 bytes RGBA ≈ 64 MB per page — safely bounded. A larger page is
225
235
  // uniformly downscaled to stay under this instead of allocating gigabytes.
@@ -122,6 +122,9 @@ export declare const PDF_LIMITS: {
122
122
  MAX_SIZE_MB: number;
123
123
  DEFAULT_MAX_PAGES: number;
124
124
  MAX_SCALE: number;
125
+ MIN_SCALE: number;
126
+ DEFAULT_SCALE: number;
127
+ PAGE_COUNT_TIMEOUT_MS: number;
125
128
  DEFAULT_MAX_CANVAS_PIXELS: number;
126
129
  MIN_EFFECTIVE_SCALE: number;
127
130
  };
@@ -220,6 +220,16 @@ export const PDF_LIMITS = {
220
220
  // Upper bound for the render scale factor. Above this, a single page can
221
221
  // allocate hundreds of MB; scale <= 0 produces a degenerate viewport.
222
222
  MAX_SCALE: 10,
223
+ // Lower bound for the render scale factor (#297). Below this the render is
224
+ // effectively unreadable; enforcing it makes the documented 0.1–10 range real
225
+ // (previously only scale <= 0 was rejected).
226
+ MIN_SCALE: 0.1,
227
+ // Default render scale (#297). Lowered from 2 → 1.5 to roughly halve the
228
+ // per-page canvas memory while staying legible for OCR/vision.
229
+ DEFAULT_SCALE: 1.5,
230
+ // Timeout (ms) for the accurate pdf-parse page-count probe (#287); on timeout
231
+ // the processor falls back to the regex estimate rather than blocking.
232
+ PAGE_COUNT_TIMEOUT_MS: 5000,
223
233
  // Default per-page pixel ceiling for image conversion (#260). 16.7M px ×
224
234
  // 4 bytes RGBA ≈ 64 MB per page — safely bounded. A larger page is
225
235
  // uniformly downscaled to stay under this instead of allocating gigabytes.
@@ -95,6 +95,8 @@ export type FileProcessingResult = {
95
95
  estimatedPages?: number | null;
96
96
  provider?: string;
97
97
  apiType?: PDFAPIType;
98
+ /** Provider's citations requirement for visual PDF analysis (#349). */
99
+ requiresCitations?: boolean | "auto";
98
100
  officeFormat?: OfficeDocumentType;
99
101
  pageCount?: number;
100
102
  slideCount?: number;
@@ -210,6 +212,14 @@ export type PDFProviderConfig = {
210
212
  maxSizeMB: number;
211
213
  maxPages: number;
212
214
  supportsNative: boolean;
215
+ /**
216
+ * Whether this provider needs source citations enabled for visual PDF
217
+ * analysis (#349). `"auto"` = enable when the request requires visual
218
+ * grounding (currently Bedrock's Converse document blocks); `false` = the
219
+ * provider handles PDFs without an explicit citations flag. Surfaced on
220
+ * `FileProcessingResult.metadata.requiresCitations` so downstream provider
221
+ * adapters can act on it instead of the value being dead config.
222
+ */
213
223
  requiresCitations: boolean | "auto";
214
224
  apiType: PDFAPIType;
215
225
  };
@@ -405,10 +415,32 @@ export type PDFImageConversionOptions = {
405
415
  maxCanvasPixels?: number;
406
416
  /** Password for an encrypted PDF (passed to the underlying renderer) (#258). */
407
417
  password?: string;
418
+ /** Per-page progress callback invoked as each page is rendered (#302). */
419
+ onProgress?: (progress: PDFImageConversionProgress) => void | Promise<void>;
420
+ };
421
+ /** Progress reported per page during streaming conversion (#302). */
422
+ export type PDFImageConversionProgress = {
423
+ /** Number of pages successfully converted so far. */
424
+ pagesConverted: number;
425
+ /** Total pages in the document (known up-front from the renderer). */
426
+ totalPages: number;
427
+ /** Elapsed time since conversion started, in milliseconds. */
428
+ elapsedMs: number;
429
+ };
430
+ /** A single streamed page result (#302). `error` is set when that page failed. */
431
+ export type PDFImagePage = {
432
+ /** 1-based page index. */
433
+ pageIndex: number;
434
+ /** Base64-encoded PNG for the page (empty string when `error` is set). */
435
+ image: string;
436
+ /** Byte size of the rendered PNG (0 when `error` is set). */
437
+ imageSizeBytes: number;
438
+ /** Populated when this page failed to render (#294). */
439
+ error?: string;
408
440
  };
409
441
  /** Result of PDF to image conversion. */
410
442
  export type PDFImageConversionResult = {
411
- /** Array of base64-encoded PNG images (one per page) */
443
+ /** Array of base64-encoded PNG images (one per successfully converted page) */
412
444
  images: string[];
413
445
  /** Number of pages converted */
414
446
  pageCount: number;
@@ -416,6 +448,11 @@ export type PDFImageConversionResult = {
416
448
  conversionTimeMs: number;
417
449
  /** Any warnings during conversion */
418
450
  warnings?: string[];
451
+ /** Per-page failures — present only when some pages failed to render (#294). */
452
+ errors?: Array<{
453
+ page: number;
454
+ error: string;
455
+ }>;
419
456
  };
420
457
  /** Options for filename sanitization. */
421
458
  export type SanitizeFileNameOptions = {
@@ -25,7 +25,7 @@ import { CSVProcessor } from "./csvProcessor.js";
25
25
  import { ImageProcessor } from "./imageProcessor.js";
26
26
  import { logger } from "./logger.js";
27
27
  import { withTimeout } from "./errorHandling.js";
28
- import { redactUrlForError, sanitizeErrorCause } from "./logSanitize.js";
28
+ import { normalizeUrlForCache, redactUrlForError, sanitizeErrorCause, } from "./logSanitize.js";
29
29
  import { mimeHintToExtension, mimeHintToFileType, normalizeMimeHint, } from "./mimeTypeHints.js";
30
30
  import { PDFProcessor } from "./pdfProcessor.js";
31
31
  /**
@@ -54,22 +54,40 @@ const DEFAULT_RETRY_DELAY = 1000; // milliseconds
54
54
  const URL_CONTENT_TYPE_TTL_MS = 60_000;
55
55
  const URL_CONTENT_TYPE_CACHE_MAX_SIZE = 512;
56
56
  const urlContentTypeCache = new Map();
57
+ /**
58
+ * Build the Map key for `urlContentTypeCache`. Delegates to the shared
59
+ * {@link normalizeUrlForCache} — this is a module-level, process-lifetime
60
+ * cache, so it must strip presigned-URL auth/signature query params (see
61
+ * `SENSITIVE_URL_QUERY_PARAM_DENYLIST`) the same way `ImageCache.normalizeUrl`
62
+ * does, folding a short hash of the stripped params into the key whenever any
63
+ * were present so two different presigned URLs for the same path don't
64
+ * collide and serve each other's cached content-type across auth contexts.
65
+ * Also strips tracking/analytics params first, so two URLs differing only by
66
+ * tracking noise (e.g. `utm_source`) still hit the same cache entry instead
67
+ * of missing each other. Falls back to the raw URL if it isn't a parseable
68
+ * absolute URL. Shared by both `getCachedUrlContentType` and
69
+ * `setCachedUrlContentType` so lookups and writes always agree on the key.
70
+ */
71
+ function cacheKeyForUrl(url) {
72
+ return normalizeUrlForCache(url);
73
+ }
57
74
  function getCachedUrlContentType(url, now) {
58
- const hit = urlContentTypeCache.get(url);
75
+ const key = cacheKeyForUrl(url);
76
+ const hit = urlContentTypeCache.get(key);
59
77
  if (hit && hit.expiresAt > now) {
60
78
  // Bump recency: Map iteration order follows insertion order, and the
61
79
  // eviction below deletes the *first* key, so a plain `get` on a hot
62
80
  // entry would leave it first in line for eviction despite being the
63
81
  // most recently used. Re-inserting turns the size-bounded FIFO below
64
82
  // into an actual LRU.
65
- urlContentTypeCache.delete(url);
66
- urlContentTypeCache.set(url, hit);
83
+ urlContentTypeCache.delete(key);
84
+ urlContentTypeCache.set(key, hit);
67
85
  return hit.contentType;
68
86
  }
69
87
  if (hit) {
70
88
  // Entry exists but its TTL has passed — treat as a miss and evict it
71
89
  // immediately rather than serving (or retaining) stale data.
72
- urlContentTypeCache.delete(url);
90
+ urlContentTypeCache.delete(key);
73
91
  }
74
92
  return undefined;
75
93
  }
@@ -89,7 +107,8 @@ function setCachedUrlContentType(url, contentType, now) {
89
107
  if (!contentType) {
90
108
  return;
91
109
  }
92
- urlContentTypeCache.set(url, {
110
+ const key = cacheKeyForUrl(url);
111
+ urlContentTypeCache.set(key, {
93
112
  contentType,
94
113
  expiresAt: now + URL_CONTENT_TYPE_TTL_MS,
95
114
  });
@@ -1413,6 +1432,50 @@ export class FileDetector {
1413
1432
  const timeout = options?.timeout || FileDetector.DEFAULT_NETWORK_TIMEOUT;
1414
1433
  const maxRetries = options?.maxRetries ?? DEFAULT_MAX_RETRIES;
1415
1434
  const retryDelay = options?.retryDelay ?? DEFAULT_RETRY_DELAY;
1435
+ // #317: pre-flight HEAD to reject an oversized file BEFORE downloading any
1436
+ // body. content-length is advisory (chunked responses omit it), so a
1437
+ // missing/invalid header — or a server that refuses HEAD — falls through to
1438
+ // the streaming byte guard below; only a genuine oversize rejection stops
1439
+ // the GET from ever running.
1440
+ //
1441
+ // #323: skip the pre-flight entirely when this exact URL was recently seen
1442
+ // (its Content-Type is still cached, whether from a prior loadFromURL GET
1443
+ // or a MimeTypeStrategy HEAD) — issuing a fresh HEAD here would defeat the
1444
+ // whole point of that cache. The streaming byte guard in the GET below
1445
+ // still enforces maxSize even without a pre-flight, so this doesn't remove
1446
+ // the oversize protection — it only skips the redundant round-trip for a
1447
+ // URL we've already been talking to within the last 60s.
1448
+ if (getCachedUrlContentType(url, Date.now()) === undefined) {
1449
+ try {
1450
+ const head = await request(url, {
1451
+ dispatcher: getGlobalDispatcher().compose(interceptors.redirect({ maxRedirections: 5 })),
1452
+ method: "HEAD",
1453
+ headersTimeout: FileDetector.DEFAULT_HEAD_TIMEOUT,
1454
+ bodyTimeout: FileDetector.DEFAULT_HEAD_TIMEOUT,
1455
+ });
1456
+ // Drain/close the (empty) HEAD body so the connection can be reused.
1457
+ await head.body.dump();
1458
+ // Only trust `content-length` on a genuine 2xx response. A non-2xx
1459
+ // HEAD (redirect the dispatcher didn't follow, 403/404/405 "HEAD not
1460
+ // allowed", 5xx, …) can still carry a stale/irrelevant
1461
+ // `content-length` header — enforcing size off of that would reject
1462
+ // (or silently pass) based on the wrong body. Treat any non-2xx HEAD
1463
+ // as if the header were missing and fall through to the streaming
1464
+ // GET guard below, which enforces maxSize independently either way.
1465
+ if (head.statusCode >= 200 && head.statusCode < 300) {
1466
+ const declaredLength = Number(head.headers["content-length"]);
1467
+ if (Number.isFinite(declaredLength) && declaredLength > maxSize) {
1468
+ throw new Error(`File too large: ${formatFileSize(declaredLength)} (max: ${formatFileSize(maxSize)})`);
1469
+ }
1470
+ }
1471
+ }
1472
+ catch (error) {
1473
+ if (error instanceof Error && /File too large/.test(error.message)) {
1474
+ throw error;
1475
+ }
1476
+ logger.debug(`[FileDetector] HEAD pre-flight skipped for ${redactUrlForError(url)}: ${sanitizeErrorCause(error).message}`);
1477
+ }
1478
+ }
1416
1479
  return withRetry(async () => {
1417
1480
  try {
1418
1481
  const response = await request(url, {
@@ -33,8 +33,16 @@ export declare class ImageCache {
33
33
  */
34
34
  private parseConfigValue;
35
35
  /**
36
- * Normalize URL for consistent cache key generation
37
- * Removes tracking parameters and normalizes the URL
36
+ * Normalize URL for consistent cache key generation.
37
+ *
38
+ * Delegates to the shared {@link normalizeUrlForCache} — this normalized
39
+ * URL is used as the in-memory Map key (see `get`/`set` below), which lives
40
+ * for the process lifetime, so both tracking-param noise and presigned-URL
41
+ * auth/signature secrets must be stripped consistently with
42
+ * `fileDetector.cacheKeyForUrl`'s identical requirement. See
43
+ * `stripSensitiveUrlParamsForCacheKey`'s doc comment for why a naive
44
+ * strip-only approach (no hash suffix) would be a correctness bug, not just
45
+ * a hygiene one.
38
46
  */
39
47
  private normalizeUrl;
40
48
  /**