@juspay/neurolink 9.95.2 → 10.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/dist/browser/neurolink.min.js +380 -380
- package/dist/cli/commands/proxy.d.ts +26 -0
- package/dist/cli/commands/proxy.js +132 -13
- package/dist/cli/factories/commandFactory.d.ts +7 -0
- package/dist/cli/factories/commandFactory.js +29 -21
- package/dist/cli/utils/inputValidation.d.ts +1 -0
- package/dist/cli/utils/inputValidation.js +6 -1
- package/dist/core/constants.d.ts +3 -0
- package/dist/core/constants.js +10 -0
- package/dist/lib/core/constants.d.ts +3 -0
- package/dist/lib/core/constants.js +10 -0
- package/dist/lib/types/file.d.ts +38 -1
- package/dist/lib/utils/fileDetector.js +69 -6
- package/dist/lib/utils/imageCache.d.ts +10 -2
- package/dist/lib/utils/imageCache.js +12 -23
- package/dist/lib/utils/imageProcessor.d.ts +25 -0
- package/dist/lib/utils/imageProcessor.js +28 -1
- package/dist/lib/utils/logSanitize.d.ts +97 -0
- package/dist/lib/utils/logSanitize.js +163 -0
- package/dist/lib/utils/messageBuilder.js +36 -0
- package/dist/lib/utils/pdfProcessor.d.ts +30 -1
- package/dist/lib/utils/pdfProcessor.js +215 -52
- package/dist/types/file.d.ts +38 -1
- package/dist/utils/fileDetector.js +69 -6
- package/dist/utils/imageCache.d.ts +10 -2
- package/dist/utils/imageCache.js +12 -23
- package/dist/utils/imageProcessor.d.ts +25 -0
- package/dist/utils/imageProcessor.js +28 -1
- package/dist/utils/logSanitize.d.ts +97 -0
- package/dist/utils/logSanitize.js +163 -0
- package/dist/utils/messageBuilder.js +36 -0
- package/dist/utils/pdfProcessor.d.ts +30 -1
- package/dist/utils/pdfProcessor.js +215 -52
- package/package.json +1 -1
|
@@ -13,6 +13,32 @@ import type { CommandModule } from "yargs";
|
|
|
13
13
|
import type { Hono } from "hono";
|
|
14
14
|
import type { AccountAllowlist, LoadedProxyConfig, ModelRouterInterface, ProxyGuardArgs, ProxyNeurolinkRuntime, ProxyStartArgs, ProxyStartStrategy, ProxyStatusArgs, ProxyTelemetryArgs, ProxyReadinessState } from "../../lib/types/index.js";
|
|
15
15
|
import { ProxyRuntimeConfigStore } from "../../lib/proxy/runtimeConfig.js";
|
|
16
|
+
/**
|
|
17
|
+
* Best-effort check that a pid actually belongs to a neurolink proxy process,
|
|
18
|
+
* so a stale/recycled supervisor pid is never mistaken for a live supervisor
|
|
19
|
+
* and signalled. Returns false when the process cannot be confirmed as ours.
|
|
20
|
+
*
|
|
21
|
+
* @param expectedStartTimeIso - The `ProxySupervisorState.startTime` (ISO
|
|
22
|
+
* string) recorded when the supervisor we expect at `pid` was launched.
|
|
23
|
+
* When provided, this is cross-checked against `pid`'s actual OS-reported
|
|
24
|
+
* start time (see {@link getProcessStartTime}) — the args match alone
|
|
25
|
+
* ("neurolink" + "proxy" both present) is not airtight, since an
|
|
26
|
+
* unrelated process (e.g. a shell running this very test suite, or a
|
|
27
|
+
* coincidentally-named script) could match it too. A pid recycled by such
|
|
28
|
+
* a process would have a start time far from the recorded supervisor
|
|
29
|
+
* startTime, which this catches.
|
|
30
|
+
*
|
|
31
|
+
* Deliberately fail-open on anything unparseable: `ps -o lstart=` output
|
|
32
|
+
* parsing is inherently a little fragile (locale/format quirks), and the
|
|
33
|
+
* cost of a false "can't confirm" is silently regressing to pre-hardening
|
|
34
|
+
* behavior, while the cost of a false "confirmed mismatch" is leaving a
|
|
35
|
+
* real supervisor running and orphaning its socket during uninstall — the
|
|
36
|
+
* former is the safer failure mode. So this only ever returns false for
|
|
37
|
+
* the startTime check when BOTH timestamps parsed successfully AND their
|
|
38
|
+
* drift exceeds the tolerance; any parse failure is treated as "can't
|
|
39
|
+
* confirm a mismatch" and falls through to the args-only result.
|
|
40
|
+
*/
|
|
41
|
+
export declare function processLooksLikeProxySupervisor(pid: number, expectedStartTimeIso?: string): Promise<boolean>;
|
|
16
42
|
export declare function mapClaudeErrorTypeToStatus(errorType?: string): number;
|
|
17
43
|
export declare function createProxyStartApp(params: {
|
|
18
44
|
neurolink: ProxyNeurolinkRuntime["neurolink"];
|
|
@@ -41,6 +41,14 @@ const _require = createRequire(import.meta.url);
|
|
|
41
41
|
const PROXY_VERSION = packageJson.version;
|
|
42
42
|
const PROXY_TELEMETRY_SCRIPT_PATH = fileURLToPath(new URL("../../../scripts/observability/manage-local-openobserve.sh", import.meta.url));
|
|
43
43
|
const PROXY_LIFECYCLE_SHUTDOWN_TIMEOUT_MS = 5_000;
|
|
44
|
+
// Allowed drift between a pid's OS-reported start time and the persisted
|
|
45
|
+
// ProxySupervisorState.startTime before processLooksLikeProxySupervisor
|
|
46
|
+
// treats it as a confident mismatch (recycled pid). Generous on purpose:
|
|
47
|
+
// `supervisorStartedAt` is captured inside runLaunchdProxySupervisor, which
|
|
48
|
+
// runs after Node has already booted and done setup, so it always lags the
|
|
49
|
+
// OS-level fork/exec by a little — a tight tolerance would produce false
|
|
50
|
+
// mismatches on a slow/cold-started machine.
|
|
51
|
+
const PROXY_SUPERVISOR_START_TIME_TOLERANCE_MS = 10_000;
|
|
44
52
|
const PROXY_UPDATE_CONTROL_TOKEN = process.env.NEUROLINK_PROXY_UPDATE_CONTROL_TOKEN?.trim() ||
|
|
45
53
|
crypto.randomUUID();
|
|
46
54
|
// =============================================================================
|
|
@@ -98,23 +106,131 @@ function getProcessStatus(pid) {
|
|
|
98
106
|
}
|
|
99
107
|
}
|
|
100
108
|
/**
|
|
101
|
-
*
|
|
102
|
-
*
|
|
103
|
-
*
|
|
109
|
+
* Read a single `ps -o <field>=` field for `pid`, off the event loop (async
|
|
110
|
+
* `execFile`, not `execFileSync`) and bounded by a short timeout, so a
|
|
111
|
+
* hung/zombie `ps` invocation can never block the event loop or hang the
|
|
112
|
+
* (often uninstall-path) async caller. Shared by {@link getProcessStartTime}
|
|
113
|
+
* and {@link processLooksLikeProxySupervisor}, which both read a single
|
|
114
|
+
* `ps -o <field>=` column for the same pid.
|
|
115
|
+
*
|
|
116
|
+
* Returns null — never throws — on any error, timeout, or empty output, so
|
|
117
|
+
* callers uniformly treat "couldn't read" as "can't confirm either way"
|
|
118
|
+
* rather than distinguishing the failure mode.
|
|
104
119
|
*/
|
|
105
|
-
async function
|
|
120
|
+
async function readPsField(pid, field) {
|
|
106
121
|
try {
|
|
107
|
-
const {
|
|
108
|
-
const
|
|
122
|
+
const { execFile } = await import("node:child_process");
|
|
123
|
+
const { promisify } = await import("node:util");
|
|
124
|
+
const execFileAsync = promisify(execFile);
|
|
125
|
+
const { stdout } = await withTimeout(execFileAsync("ps", ["-p", String(pid), "-o", `${field}=`], {
|
|
109
126
|
encoding: "utf-8",
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
return
|
|
127
|
+
}), 2000, `ps -o ${field}= timed out for pid ${pid}`);
|
|
128
|
+
const trimmed = stdout.trim();
|
|
129
|
+
return trimmed || null;
|
|
113
130
|
}
|
|
114
131
|
catch {
|
|
115
|
-
|
|
132
|
+
return null;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Best-effort, macOS-only (uninstall is macOS-only) read of `pid`'s
|
|
137
|
+
* OS-reported start time via `ps -o lstart=`, which prints a fixed-width
|
|
138
|
+
* `Www Mmm dd hh:mm:ss yyyy` timestamp — the same shape `Date`'s parser
|
|
139
|
+
* already understands (it's the format `Date#toString()` itself produces
|
|
140
|
+
* modulo the trailing timezone). Returns null (never throws) when `ps`
|
|
141
|
+
* fails or its output doesn't parse, so callers can tell "confirmed
|
|
142
|
+
* mismatch" apart from "couldn't confirm either way".
|
|
143
|
+
*/
|
|
144
|
+
async function getProcessStartTime(pid) {
|
|
145
|
+
const out = await readPsField(pid, "lstart");
|
|
146
|
+
if (!out) {
|
|
147
|
+
return null;
|
|
148
|
+
}
|
|
149
|
+
// `ps -o lstart=` prints the OS start time in the machine's LOCAL timezone
|
|
150
|
+
// with no offset, e.g. "Sun Jul 19 20:12:41 2026". Parse the components and
|
|
151
|
+
// build the Date via the local-time constructor rather than relying on the
|
|
152
|
+
// engine's implementation-defined parsing of non-ISO date strings. The
|
|
153
|
+
// result is the same absolute instant the supervisor persisted as an
|
|
154
|
+
// ISO-UTC string, so the drift comparison against it is timezone-safe.
|
|
155
|
+
const m = out.match(/^\w{3}\s+(\w{3})\s+(\d{1,2})\s+(\d{2}):(\d{2}):(\d{2})\s+(\d{4})$/);
|
|
156
|
+
if (!m) {
|
|
157
|
+
return null;
|
|
158
|
+
}
|
|
159
|
+
const monthIndex = [
|
|
160
|
+
"Jan",
|
|
161
|
+
"Feb",
|
|
162
|
+
"Mar",
|
|
163
|
+
"Apr",
|
|
164
|
+
"May",
|
|
165
|
+
"Jun",
|
|
166
|
+
"Jul",
|
|
167
|
+
"Aug",
|
|
168
|
+
"Sep",
|
|
169
|
+
"Oct",
|
|
170
|
+
"Nov",
|
|
171
|
+
"Dec",
|
|
172
|
+
].indexOf(m[1]);
|
|
173
|
+
if (monthIndex < 0) {
|
|
174
|
+
return null;
|
|
175
|
+
}
|
|
176
|
+
const parsed = new Date(Number(m[6]), monthIndex, Number(m[2]), Number(m[3]), Number(m[4]), Number(m[5]));
|
|
177
|
+
return Number.isNaN(parsed.getTime()) ? null : parsed;
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
180
|
+
* Best-effort check that a pid actually belongs to a neurolink proxy process,
|
|
181
|
+
* so a stale/recycled supervisor pid is never mistaken for a live supervisor
|
|
182
|
+
* and signalled. Returns false when the process cannot be confirmed as ours.
|
|
183
|
+
*
|
|
184
|
+
* @param expectedStartTimeIso - The `ProxySupervisorState.startTime` (ISO
|
|
185
|
+
* string) recorded when the supervisor we expect at `pid` was launched.
|
|
186
|
+
* When provided, this is cross-checked against `pid`'s actual OS-reported
|
|
187
|
+
* start time (see {@link getProcessStartTime}) — the args match alone
|
|
188
|
+
* ("neurolink" + "proxy" both present) is not airtight, since an
|
|
189
|
+
* unrelated process (e.g. a shell running this very test suite, or a
|
|
190
|
+
* coincidentally-named script) could match it too. A pid recycled by such
|
|
191
|
+
* a process would have a start time far from the recorded supervisor
|
|
192
|
+
* startTime, which this catches.
|
|
193
|
+
*
|
|
194
|
+
* Deliberately fail-open on anything unparseable: `ps -o lstart=` output
|
|
195
|
+
* parsing is inherently a little fragile (locale/format quirks), and the
|
|
196
|
+
* cost of a false "can't confirm" is silently regressing to pre-hardening
|
|
197
|
+
* behavior, while the cost of a false "confirmed mismatch" is leaving a
|
|
198
|
+
* real supervisor running and orphaning its socket during uninstall — the
|
|
199
|
+
* former is the safer failure mode. So this only ever returns false for
|
|
200
|
+
* the startTime check when BOTH timestamps parsed successfully AND their
|
|
201
|
+
* drift exceeds the tolerance; any parse failure is treated as "can't
|
|
202
|
+
* confirm a mismatch" and falls through to the args-only result.
|
|
203
|
+
*/
|
|
204
|
+
export async function processLooksLikeProxySupervisor(pid, expectedStartTimeIso) {
|
|
205
|
+
const args = await readPsField(pid, "args");
|
|
206
|
+
if (args === null) {
|
|
207
|
+
// `ps` unavailable, timed out, or the pid vanished — do not signal an
|
|
208
|
+
// unverified pid.
|
|
209
|
+
return false;
|
|
210
|
+
}
|
|
211
|
+
// `/neurolink/i` alone is too broad: any process whose args merely
|
|
212
|
+
// *mention* "neurolink" (e.g. a shell running in a directory named
|
|
213
|
+
// "neurolink", an editor with a neurolink file open) would match, and a
|
|
214
|
+
// stale/recycled pid running such a process could then be SIGTERM'd/
|
|
215
|
+
// SIGKILL'd by `ensureSupervisorStoppedBeforeClear` during uninstall.
|
|
216
|
+
// Require the actual proxy-supervisor invocation — both "neurolink" AND
|
|
217
|
+
// the "proxy" subcommand present in args.
|
|
218
|
+
if (!(/neurolink/i.test(args) && /\bproxy\b/i.test(args))) {
|
|
116
219
|
return false;
|
|
117
220
|
}
|
|
221
|
+
if (!expectedStartTimeIso) {
|
|
222
|
+
return true;
|
|
223
|
+
}
|
|
224
|
+
const expected = new Date(expectedStartTimeIso);
|
|
225
|
+
if (Number.isNaN(expected.getTime())) {
|
|
226
|
+
return true; // recorded startTime itself is unparseable — can't confirm a mismatch
|
|
227
|
+
}
|
|
228
|
+
const actual = await getProcessStartTime(pid);
|
|
229
|
+
if (!actual) {
|
|
230
|
+
return true; // ps -o lstart= unavailable/unparseable — can't confirm a mismatch
|
|
231
|
+
}
|
|
232
|
+
const driftMs = Math.abs(actual.getTime() - expected.getTime());
|
|
233
|
+
return driftMs <= PROXY_SUPERVISOR_START_TIME_TOLERANCE_MS;
|
|
118
234
|
}
|
|
119
235
|
/**
|
|
120
236
|
* Ensure any supervisor recorded in state is actually stopped before its state
|
|
@@ -124,14 +240,17 @@ async function processLooksLikeProxySupervisor(pid) {
|
|
|
124
240
|
* the uninstall instead of silently discarding the record.
|
|
125
241
|
*/
|
|
126
242
|
async function ensureSupervisorStoppedBeforeClear() {
|
|
127
|
-
const
|
|
243
|
+
const supervisorState = loadProxySupervisorState();
|
|
244
|
+
const pid = supervisorState?.pid;
|
|
128
245
|
if (!pid || getProcessStatus(pid) === "not_running") {
|
|
129
246
|
return true;
|
|
130
247
|
}
|
|
131
248
|
// Guard against a stale/recycled pid: only signal a process that still looks
|
|
132
249
|
// like a neurolink proxy supervisor, so uninstall never terminates an
|
|
133
|
-
// unrelated, same-user process that inherited the recorded pid.
|
|
134
|
-
|
|
250
|
+
// unrelated, same-user process that inherited the recorded pid. Also cross-
|
|
251
|
+
// checks the pid's actual OS start time against the recorded
|
|
252
|
+
// supervisorState.startTime (see processLooksLikeProxySupervisor).
|
|
253
|
+
if (!(await processLooksLikeProxySupervisor(pid, supervisorState?.startTime))) {
|
|
135
254
|
return true;
|
|
136
255
|
}
|
|
137
256
|
try {
|
|
@@ -192,6 +192,13 @@ export declare class CLICommandFactory {
|
|
|
192
192
|
* objects independently.
|
|
193
193
|
*/
|
|
194
194
|
private static buildCsvOptionsFromArgv;
|
|
195
|
+
/**
|
|
196
|
+
* Build video processor options from CLI argv. Shared by executeGenerate,
|
|
197
|
+
* executeStream and executeBatch, which previously built byte-for-byte
|
|
198
|
+
* identical `videoOptions` objects independently (mirrors
|
|
199
|
+
* {@link buildCsvOptionsFromArgv}).
|
|
200
|
+
*/
|
|
201
|
+
private static buildVideoOptionsFromArgv;
|
|
195
202
|
/**
|
|
196
203
|
* Build output configuration for generate request
|
|
197
204
|
*/
|
|
@@ -2438,6 +2438,20 @@ export class CLICommandFactory {
|
|
|
2438
2438
|
parseTimeoutMs: argv.csvParseTimeoutMs,
|
|
2439
2439
|
};
|
|
2440
2440
|
}
|
|
2441
|
+
/**
|
|
2442
|
+
* Build video processor options from CLI argv. Shared by executeGenerate,
|
|
2443
|
+
* executeStream and executeBatch, which previously built byte-for-byte
|
|
2444
|
+
* identical `videoOptions` objects independently (mirrors
|
|
2445
|
+
* {@link buildCsvOptionsFromArgv}).
|
|
2446
|
+
*/
|
|
2447
|
+
static buildVideoOptionsFromArgv(argv) {
|
|
2448
|
+
return {
|
|
2449
|
+
frames: argv.videoFrames,
|
|
2450
|
+
quality: argv.videoQuality,
|
|
2451
|
+
format: argv.videoFormat,
|
|
2452
|
+
transcribeAudio: argv.transcribeAudio,
|
|
2453
|
+
};
|
|
2454
|
+
}
|
|
2441
2455
|
/**
|
|
2442
2456
|
* Build output configuration for generate request
|
|
2443
2457
|
*/
|
|
@@ -2677,12 +2691,7 @@ export class CLICommandFactory {
|
|
|
2677
2691
|
password: CLICommandFactory.resolvePdfPassword(argv),
|
|
2678
2692
|
},
|
|
2679
2693
|
csvOptions: CLICommandFactory.buildCsvOptionsFromArgv(argv),
|
|
2680
|
-
videoOptions:
|
|
2681
|
-
frames: argv.videoFrames,
|
|
2682
|
-
quality: argv.videoQuality,
|
|
2683
|
-
format: argv.videoFormat,
|
|
2684
|
-
transcribeAudio: argv.transcribeAudio,
|
|
2685
|
-
},
|
|
2694
|
+
videoOptions: CLICommandFactory.buildVideoOptionsFromArgv(argv),
|
|
2686
2695
|
output: outputConfig,
|
|
2687
2696
|
provider: enhancedOptions.provider,
|
|
2688
2697
|
model: enhancedOptions.model,
|
|
@@ -2932,12 +2941,7 @@ export class CLICommandFactory {
|
|
|
2932
2941
|
password: CLICommandFactory.resolvePdfPassword(argv),
|
|
2933
2942
|
},
|
|
2934
2943
|
csvOptions: CLICommandFactory.buildCsvOptionsFromArgv(argv),
|
|
2935
|
-
videoOptions:
|
|
2936
|
-
frames: argv.videoFrames,
|
|
2937
|
-
quality: argv.videoQuality,
|
|
2938
|
-
format: argv.videoFormat,
|
|
2939
|
-
transcribeAudio: argv.transcribeAudio,
|
|
2940
|
-
},
|
|
2944
|
+
videoOptions: CLICommandFactory.buildVideoOptionsFromArgv(argv),
|
|
2941
2945
|
provider: enhancedOptions.provider,
|
|
2942
2946
|
model: enhancedOptions.model,
|
|
2943
2947
|
temperature: enhancedOptions.temperature,
|
|
@@ -3411,6 +3415,7 @@ export class CLICommandFactory {
|
|
|
3411
3415
|
// see createBatchCommand — so `argv.file` can never be set here; no
|
|
3412
3416
|
// exclusion needed to protect the (differently-named) positional.
|
|
3413
3417
|
validateCliInputFiles(argv);
|
|
3418
|
+
validateCsvMaxRows(argv);
|
|
3414
3419
|
const batchImages = CLICommandFactory.processCliImages(argv.image);
|
|
3415
3420
|
const batchCsvFiles = CLICommandFactory.processCliCSVFiles(argv.csv);
|
|
3416
3421
|
const batchPdfFiles = CLICommandFactory.processCliPDFFiles(argv.pdf);
|
|
@@ -3469,6 +3474,12 @@ export class CLICommandFactory {
|
|
|
3469
3474
|
logger.alwaysStderr(chalk.yellow("⚠️ Multimodal files (--image/--csv/--pdf/--video) are attached to ALL prompts in this batch."));
|
|
3470
3475
|
spinner?.start();
|
|
3471
3476
|
}
|
|
3477
|
+
// Resolve the PDF password once for the whole batch. resolvePdfPassword
|
|
3478
|
+
// emits a "visible in shell history" stderr warning when the flag is set;
|
|
3479
|
+
// it must fire once per run (like generate/stream), not once per prompt.
|
|
3480
|
+
const batchPdfPassword = batchPdfFiles?.length
|
|
3481
|
+
? CLICommandFactory.resolvePdfPassword(argv)
|
|
3482
|
+
: undefined;
|
|
3472
3483
|
for (let i = 0; i < prompts.length; i++) {
|
|
3473
3484
|
if (spinner) {
|
|
3474
3485
|
spinner.text = `Processing ${i + 1}/${prompts.length}: ${prompts[i].substring(0, 30)}...`;
|
|
@@ -3518,18 +3529,15 @@ export class CLICommandFactory {
|
|
|
3518
3529
|
// unlike generate/stream, so an empty options object here is
|
|
3519
3530
|
// pure noise on every prompt of every batch run.
|
|
3520
3531
|
...(batchCsvFiles?.length && {
|
|
3521
|
-
csvOptions:
|
|
3522
|
-
|
|
3523
|
-
|
|
3532
|
+
csvOptions: CLICommandFactory.buildCsvOptionsFromArgv(argv),
|
|
3533
|
+
}),
|
|
3534
|
+
...(batchPdfFiles?.length && {
|
|
3535
|
+
pdfOptions: {
|
|
3536
|
+
password: batchPdfPassword,
|
|
3524
3537
|
},
|
|
3525
3538
|
}),
|
|
3526
3539
|
...(batchVideoFiles?.length && {
|
|
3527
|
-
videoOptions:
|
|
3528
|
-
frames: argv.videoFrames,
|
|
3529
|
-
quality: argv.videoQuality,
|
|
3530
|
-
format: argv.videoFormat,
|
|
3531
|
-
transcribeAudio: argv.transcribeAudio,
|
|
3532
|
-
},
|
|
3540
|
+
videoOptions: CLICommandFactory.buildVideoOptionsFromArgv(argv),
|
|
3533
3541
|
}),
|
|
3534
3542
|
provider: enhancedOptions.provider,
|
|
3535
3543
|
model: enhancedOptions.model,
|
|
@@ -27,6 +27,11 @@ export const CLI_SOFT_LIMITS_MB = {
|
|
|
27
27
|
IMAGE_MAX_MB: 10,
|
|
28
28
|
CSV_MAX_MB: 50,
|
|
29
29
|
PDF_MAX_MB: 100,
|
|
30
|
+
// Distinct from PDF_MAX_MB above (which mirrors the processor's true
|
|
31
|
+
// 100MB hard cap in lib/processors/config/sizeLimits.ts and is kept here
|
|
32
|
+
// for backward compatibility). Docs promise a 50MB *warning* threshold
|
|
33
|
+
// specifically for --pdf — wire warnAtMB to this value, not the hard cap.
|
|
34
|
+
PDF_SOFT_MAX_MB: 50,
|
|
30
35
|
VIDEO_MAX_MB: 500,
|
|
31
36
|
};
|
|
32
37
|
// `file://` is intentionally excluded here: it addresses a local path, so it
|
|
@@ -165,7 +170,7 @@ export function validateCliInputFiles(argv) {
|
|
|
165
170
|
{
|
|
166
171
|
option: "--pdf",
|
|
167
172
|
value: argv.pdf,
|
|
168
|
-
warnAtMB: CLI_SOFT_LIMITS_MB.
|
|
173
|
+
warnAtMB: CLI_SOFT_LIMITS_MB.PDF_SOFT_MAX_MB,
|
|
169
174
|
},
|
|
170
175
|
{
|
|
171
176
|
option: "--video",
|
package/dist/core/constants.d.ts
CHANGED
|
@@ -122,6 +122,9 @@ export declare const PDF_LIMITS: {
|
|
|
122
122
|
MAX_SIZE_MB: number;
|
|
123
123
|
DEFAULT_MAX_PAGES: number;
|
|
124
124
|
MAX_SCALE: number;
|
|
125
|
+
MIN_SCALE: number;
|
|
126
|
+
DEFAULT_SCALE: number;
|
|
127
|
+
PAGE_COUNT_TIMEOUT_MS: number;
|
|
125
128
|
DEFAULT_MAX_CANVAS_PIXELS: number;
|
|
126
129
|
MIN_EFFECTIVE_SCALE: number;
|
|
127
130
|
};
|
package/dist/core/constants.js
CHANGED
|
@@ -220,6 +220,16 @@ export const PDF_LIMITS = {
|
|
|
220
220
|
// Upper bound for the render scale factor. Above this, a single page can
|
|
221
221
|
// allocate hundreds of MB; scale <= 0 produces a degenerate viewport.
|
|
222
222
|
MAX_SCALE: 10,
|
|
223
|
+
// Lower bound for the render scale factor (#297). Below this the render is
|
|
224
|
+
// effectively unreadable; enforcing it makes the documented 0.1–10 range real
|
|
225
|
+
// (previously only scale <= 0 was rejected).
|
|
226
|
+
MIN_SCALE: 0.1,
|
|
227
|
+
// Default render scale (#297). Lowered from 2 → 1.5 to roughly halve the
|
|
228
|
+
// per-page canvas memory while staying legible for OCR/vision.
|
|
229
|
+
DEFAULT_SCALE: 1.5,
|
|
230
|
+
// Timeout (ms) for the accurate pdf-parse page-count probe (#287); on timeout
|
|
231
|
+
// the processor falls back to the regex estimate rather than blocking.
|
|
232
|
+
PAGE_COUNT_TIMEOUT_MS: 5000,
|
|
223
233
|
// Default per-page pixel ceiling for image conversion (#260). 16.7M px ×
|
|
224
234
|
// 4 bytes RGBA ≈ 64 MB per page — safely bounded. A larger page is
|
|
225
235
|
// uniformly downscaled to stay under this instead of allocating gigabytes.
|
|
@@ -122,6 +122,9 @@ export declare const PDF_LIMITS: {
|
|
|
122
122
|
MAX_SIZE_MB: number;
|
|
123
123
|
DEFAULT_MAX_PAGES: number;
|
|
124
124
|
MAX_SCALE: number;
|
|
125
|
+
MIN_SCALE: number;
|
|
126
|
+
DEFAULT_SCALE: number;
|
|
127
|
+
PAGE_COUNT_TIMEOUT_MS: number;
|
|
125
128
|
DEFAULT_MAX_CANVAS_PIXELS: number;
|
|
126
129
|
MIN_EFFECTIVE_SCALE: number;
|
|
127
130
|
};
|
|
@@ -220,6 +220,16 @@ export const PDF_LIMITS = {
|
|
|
220
220
|
// Upper bound for the render scale factor. Above this, a single page can
|
|
221
221
|
// allocate hundreds of MB; scale <= 0 produces a degenerate viewport.
|
|
222
222
|
MAX_SCALE: 10,
|
|
223
|
+
// Lower bound for the render scale factor (#297). Below this the render is
|
|
224
|
+
// effectively unreadable; enforcing it makes the documented 0.1–10 range real
|
|
225
|
+
// (previously only scale <= 0 was rejected).
|
|
226
|
+
MIN_SCALE: 0.1,
|
|
227
|
+
// Default render scale (#297). Lowered from 2 → 1.5 to roughly halve the
|
|
228
|
+
// per-page canvas memory while staying legible for OCR/vision.
|
|
229
|
+
DEFAULT_SCALE: 1.5,
|
|
230
|
+
// Timeout (ms) for the accurate pdf-parse page-count probe (#287); on timeout
|
|
231
|
+
// the processor falls back to the regex estimate rather than blocking.
|
|
232
|
+
PAGE_COUNT_TIMEOUT_MS: 5000,
|
|
223
233
|
// Default per-page pixel ceiling for image conversion (#260). 16.7M px ×
|
|
224
234
|
// 4 bytes RGBA ≈ 64 MB per page — safely bounded. A larger page is
|
|
225
235
|
// uniformly downscaled to stay under this instead of allocating gigabytes.
|
package/dist/lib/types/file.d.ts
CHANGED
|
@@ -95,6 +95,8 @@ export type FileProcessingResult = {
|
|
|
95
95
|
estimatedPages?: number | null;
|
|
96
96
|
provider?: string;
|
|
97
97
|
apiType?: PDFAPIType;
|
|
98
|
+
/** Provider's citations requirement for visual PDF analysis (#349). */
|
|
99
|
+
requiresCitations?: boolean | "auto";
|
|
98
100
|
officeFormat?: OfficeDocumentType;
|
|
99
101
|
pageCount?: number;
|
|
100
102
|
slideCount?: number;
|
|
@@ -210,6 +212,14 @@ export type PDFProviderConfig = {
|
|
|
210
212
|
maxSizeMB: number;
|
|
211
213
|
maxPages: number;
|
|
212
214
|
supportsNative: boolean;
|
|
215
|
+
/**
|
|
216
|
+
* Whether this provider needs source citations enabled for visual PDF
|
|
217
|
+
* analysis (#349). `"auto"` = enable when the request requires visual
|
|
218
|
+
* grounding (currently Bedrock's Converse document blocks); `false` = the
|
|
219
|
+
* provider handles PDFs without an explicit citations flag. Surfaced on
|
|
220
|
+
* `FileProcessingResult.metadata.requiresCitations` so downstream provider
|
|
221
|
+
* adapters can act on it instead of the value being dead config.
|
|
222
|
+
*/
|
|
213
223
|
requiresCitations: boolean | "auto";
|
|
214
224
|
apiType: PDFAPIType;
|
|
215
225
|
};
|
|
@@ -405,10 +415,32 @@ export type PDFImageConversionOptions = {
|
|
|
405
415
|
maxCanvasPixels?: number;
|
|
406
416
|
/** Password for an encrypted PDF (passed to the underlying renderer) (#258). */
|
|
407
417
|
password?: string;
|
|
418
|
+
/** Per-page progress callback invoked as each page is rendered (#302). */
|
|
419
|
+
onProgress?: (progress: PDFImageConversionProgress) => void | Promise<void>;
|
|
420
|
+
};
|
|
421
|
+
/** Progress reported per page during streaming conversion (#302). */
|
|
422
|
+
export type PDFImageConversionProgress = {
|
|
423
|
+
/** Number of pages successfully converted so far. */
|
|
424
|
+
pagesConverted: number;
|
|
425
|
+
/** Total pages in the document (known up-front from the renderer). */
|
|
426
|
+
totalPages: number;
|
|
427
|
+
/** Elapsed time since conversion started, in milliseconds. */
|
|
428
|
+
elapsedMs: number;
|
|
429
|
+
};
|
|
430
|
+
/** A single streamed page result (#302). `error` is set when that page failed. */
|
|
431
|
+
export type PDFImagePage = {
|
|
432
|
+
/** 1-based page index. */
|
|
433
|
+
pageIndex: number;
|
|
434
|
+
/** Base64-encoded PNG for the page (empty string when `error` is set). */
|
|
435
|
+
image: string;
|
|
436
|
+
/** Byte size of the rendered PNG (0 when `error` is set). */
|
|
437
|
+
imageSizeBytes: number;
|
|
438
|
+
/** Populated when this page failed to render (#294). */
|
|
439
|
+
error?: string;
|
|
408
440
|
};
|
|
409
441
|
/** Result of PDF to image conversion. */
|
|
410
442
|
export type PDFImageConversionResult = {
|
|
411
|
-
/** Array of base64-encoded PNG images (one per page) */
|
|
443
|
+
/** Array of base64-encoded PNG images (one per successfully converted page) */
|
|
412
444
|
images: string[];
|
|
413
445
|
/** Number of pages converted */
|
|
414
446
|
pageCount: number;
|
|
@@ -416,6 +448,11 @@ export type PDFImageConversionResult = {
|
|
|
416
448
|
conversionTimeMs: number;
|
|
417
449
|
/** Any warnings during conversion */
|
|
418
450
|
warnings?: string[];
|
|
451
|
+
/** Per-page failures — present only when some pages failed to render (#294). */
|
|
452
|
+
errors?: Array<{
|
|
453
|
+
page: number;
|
|
454
|
+
error: string;
|
|
455
|
+
}>;
|
|
419
456
|
};
|
|
420
457
|
/** Options for filename sanitization. */
|
|
421
458
|
export type SanitizeFileNameOptions = {
|
|
@@ -25,7 +25,7 @@ import { CSVProcessor } from "./csvProcessor.js";
|
|
|
25
25
|
import { ImageProcessor } from "./imageProcessor.js";
|
|
26
26
|
import { logger } from "./logger.js";
|
|
27
27
|
import { withTimeout } from "./errorHandling.js";
|
|
28
|
-
import { redactUrlForError, sanitizeErrorCause } from "./logSanitize.js";
|
|
28
|
+
import { normalizeUrlForCache, redactUrlForError, sanitizeErrorCause, } from "./logSanitize.js";
|
|
29
29
|
import { mimeHintToExtension, mimeHintToFileType, normalizeMimeHint, } from "./mimeTypeHints.js";
|
|
30
30
|
import { PDFProcessor } from "./pdfProcessor.js";
|
|
31
31
|
/**
|
|
@@ -54,22 +54,40 @@ const DEFAULT_RETRY_DELAY = 1000; // milliseconds
|
|
|
54
54
|
const URL_CONTENT_TYPE_TTL_MS = 60_000;
|
|
55
55
|
const URL_CONTENT_TYPE_CACHE_MAX_SIZE = 512;
|
|
56
56
|
const urlContentTypeCache = new Map();
|
|
57
|
+
/**
|
|
58
|
+
* Build the Map key for `urlContentTypeCache`. Delegates to the shared
|
|
59
|
+
* {@link normalizeUrlForCache} — this is a module-level, process-lifetime
|
|
60
|
+
* cache, so it must strip presigned-URL auth/signature query params (see
|
|
61
|
+
* `SENSITIVE_URL_QUERY_PARAM_DENYLIST`) the same way `ImageCache.normalizeUrl`
|
|
62
|
+
* does, folding a short hash of the stripped params into the key whenever any
|
|
63
|
+
* were present so two different presigned URLs for the same path don't
|
|
64
|
+
* collide and serve each other's cached content-type across auth contexts.
|
|
65
|
+
* Also strips tracking/analytics params first, so two URLs differing only by
|
|
66
|
+
* tracking noise (e.g. `utm_source`) still hit the same cache entry instead
|
|
67
|
+
* of missing each other. Falls back to the raw URL if it isn't a parseable
|
|
68
|
+
* absolute URL. Shared by both `getCachedUrlContentType` and
|
|
69
|
+
* `setCachedUrlContentType` so lookups and writes always agree on the key.
|
|
70
|
+
*/
|
|
71
|
+
function cacheKeyForUrl(url) {
|
|
72
|
+
return normalizeUrlForCache(url);
|
|
73
|
+
}
|
|
57
74
|
function getCachedUrlContentType(url, now) {
|
|
58
|
-
const
|
|
75
|
+
const key = cacheKeyForUrl(url);
|
|
76
|
+
const hit = urlContentTypeCache.get(key);
|
|
59
77
|
if (hit && hit.expiresAt > now) {
|
|
60
78
|
// Bump recency: Map iteration order follows insertion order, and the
|
|
61
79
|
// eviction below deletes the *first* key, so a plain `get` on a hot
|
|
62
80
|
// entry would leave it first in line for eviction despite being the
|
|
63
81
|
// most recently used. Re-inserting turns the size-bounded FIFO below
|
|
64
82
|
// into an actual LRU.
|
|
65
|
-
urlContentTypeCache.delete(
|
|
66
|
-
urlContentTypeCache.set(
|
|
83
|
+
urlContentTypeCache.delete(key);
|
|
84
|
+
urlContentTypeCache.set(key, hit);
|
|
67
85
|
return hit.contentType;
|
|
68
86
|
}
|
|
69
87
|
if (hit) {
|
|
70
88
|
// Entry exists but its TTL has passed — treat as a miss and evict it
|
|
71
89
|
// immediately rather than serving (or retaining) stale data.
|
|
72
|
-
urlContentTypeCache.delete(
|
|
90
|
+
urlContentTypeCache.delete(key);
|
|
73
91
|
}
|
|
74
92
|
return undefined;
|
|
75
93
|
}
|
|
@@ -89,7 +107,8 @@ function setCachedUrlContentType(url, contentType, now) {
|
|
|
89
107
|
if (!contentType) {
|
|
90
108
|
return;
|
|
91
109
|
}
|
|
92
|
-
|
|
110
|
+
const key = cacheKeyForUrl(url);
|
|
111
|
+
urlContentTypeCache.set(key, {
|
|
93
112
|
contentType,
|
|
94
113
|
expiresAt: now + URL_CONTENT_TYPE_TTL_MS,
|
|
95
114
|
});
|
|
@@ -1413,6 +1432,50 @@ export class FileDetector {
|
|
|
1413
1432
|
const timeout = options?.timeout || FileDetector.DEFAULT_NETWORK_TIMEOUT;
|
|
1414
1433
|
const maxRetries = options?.maxRetries ?? DEFAULT_MAX_RETRIES;
|
|
1415
1434
|
const retryDelay = options?.retryDelay ?? DEFAULT_RETRY_DELAY;
|
|
1435
|
+
// #317: pre-flight HEAD to reject an oversized file BEFORE downloading any
|
|
1436
|
+
// body. content-length is advisory (chunked responses omit it), so a
|
|
1437
|
+
// missing/invalid header — or a server that refuses HEAD — falls through to
|
|
1438
|
+
// the streaming byte guard below; only a genuine oversize rejection stops
|
|
1439
|
+
// the GET from ever running.
|
|
1440
|
+
//
|
|
1441
|
+
// #323: skip the pre-flight entirely when this exact URL was recently seen
|
|
1442
|
+
// (its Content-Type is still cached, whether from a prior loadFromURL GET
|
|
1443
|
+
// or a MimeTypeStrategy HEAD) — issuing a fresh HEAD here would defeat the
|
|
1444
|
+
// whole point of that cache. The streaming byte guard in the GET below
|
|
1445
|
+
// still enforces maxSize even without a pre-flight, so this doesn't remove
|
|
1446
|
+
// the oversize protection — it only skips the redundant round-trip for a
|
|
1447
|
+
// URL we've already been talking to within the last 60s.
|
|
1448
|
+
if (getCachedUrlContentType(url, Date.now()) === undefined) {
|
|
1449
|
+
try {
|
|
1450
|
+
const head = await request(url, {
|
|
1451
|
+
dispatcher: getGlobalDispatcher().compose(interceptors.redirect({ maxRedirections: 5 })),
|
|
1452
|
+
method: "HEAD",
|
|
1453
|
+
headersTimeout: FileDetector.DEFAULT_HEAD_TIMEOUT,
|
|
1454
|
+
bodyTimeout: FileDetector.DEFAULT_HEAD_TIMEOUT,
|
|
1455
|
+
});
|
|
1456
|
+
// Drain/close the (empty) HEAD body so the connection can be reused.
|
|
1457
|
+
await head.body.dump();
|
|
1458
|
+
// Only trust `content-length` on a genuine 2xx response. A non-2xx
|
|
1459
|
+
// HEAD (redirect the dispatcher didn't follow, 403/404/405 "HEAD not
|
|
1460
|
+
// allowed", 5xx, …) can still carry a stale/irrelevant
|
|
1461
|
+
// `content-length` header — enforcing size off of that would reject
|
|
1462
|
+
// (or silently pass) based on the wrong body. Treat any non-2xx HEAD
|
|
1463
|
+
// as if the header were missing and fall through to the streaming
|
|
1464
|
+
// GET guard below, which enforces maxSize independently either way.
|
|
1465
|
+
if (head.statusCode >= 200 && head.statusCode < 300) {
|
|
1466
|
+
const declaredLength = Number(head.headers["content-length"]);
|
|
1467
|
+
if (Number.isFinite(declaredLength) && declaredLength > maxSize) {
|
|
1468
|
+
throw new Error(`File too large: ${formatFileSize(declaredLength)} (max: ${formatFileSize(maxSize)})`);
|
|
1469
|
+
}
|
|
1470
|
+
}
|
|
1471
|
+
}
|
|
1472
|
+
catch (error) {
|
|
1473
|
+
if (error instanceof Error && /File too large/.test(error.message)) {
|
|
1474
|
+
throw error;
|
|
1475
|
+
}
|
|
1476
|
+
logger.debug(`[FileDetector] HEAD pre-flight skipped for ${redactUrlForError(url)}: ${sanitizeErrorCause(error).message}`);
|
|
1477
|
+
}
|
|
1478
|
+
}
|
|
1416
1479
|
return withRetry(async () => {
|
|
1417
1480
|
try {
|
|
1418
1481
|
const response = await request(url, {
|
|
@@ -33,8 +33,16 @@ export declare class ImageCache {
|
|
|
33
33
|
*/
|
|
34
34
|
private parseConfigValue;
|
|
35
35
|
/**
|
|
36
|
-
* Normalize URL for consistent cache key generation
|
|
37
|
-
*
|
|
36
|
+
* Normalize URL for consistent cache key generation.
|
|
37
|
+
*
|
|
38
|
+
* Delegates to the shared {@link normalizeUrlForCache} — this normalized
|
|
39
|
+
* URL is used as the in-memory Map key (see `get`/`set` below), which lives
|
|
40
|
+
* for the process lifetime, so both tracking-param noise and presigned-URL
|
|
41
|
+
* auth/signature secrets must be stripped consistently with
|
|
42
|
+
* `fileDetector.cacheKeyForUrl`'s identical requirement. See
|
|
43
|
+
* `stripSensitiveUrlParamsForCacheKey`'s doc comment for why a naive
|
|
44
|
+
* strip-only approach (no hash suffix) would be a correctness bug, not just
|
|
45
|
+
* a hygiene one.
|
|
38
46
|
*/
|
|
39
47
|
private normalizeUrl;
|
|
40
48
|
/**
|