@juspay/neurolink 9.95.3 → 10.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/dist/browser/neurolink.min.js +356 -356
- package/dist/cli/commands/proxy.d.ts +26 -0
- package/dist/cli/commands/proxy.js +132 -13
- package/dist/cli/factories/commandFactory.d.ts +7 -0
- package/dist/cli/factories/commandFactory.js +29 -21
- package/dist/cli/utils/inputValidation.d.ts +1 -0
- package/dist/cli/utils/inputValidation.js +6 -1
- package/dist/lib/utils/fileDetector.js +38 -10
- package/dist/lib/utils/imageCache.d.ts +10 -2
- package/dist/lib/utils/imageCache.js +12 -23
- package/dist/lib/utils/imageProcessor.d.ts +25 -0
- package/dist/lib/utils/imageProcessor.js +28 -1
- package/dist/lib/utils/logSanitize.d.ts +97 -0
- package/dist/lib/utils/logSanitize.js +163 -0
- package/dist/lib/utils/messageBuilder.js +19 -0
- package/dist/lib/utils/pdfProcessor.d.ts +7 -1
- package/dist/lib/utils/pdfProcessor.js +27 -3
- package/dist/utils/fileDetector.js +38 -10
- package/dist/utils/imageCache.d.ts +10 -2
- package/dist/utils/imageCache.js +12 -23
- package/dist/utils/imageProcessor.d.ts +25 -0
- package/dist/utils/imageProcessor.js +28 -1
- package/dist/utils/logSanitize.d.ts +97 -0
- package/dist/utils/logSanitize.js +163 -0
- package/dist/utils/messageBuilder.js +19 -0
- package/dist/utils/pdfProcessor.d.ts +7 -1
- package/dist/utils/pdfProcessor.js +27 -3
- package/package.json +1 -1
|
@@ -13,6 +13,32 @@ import type { CommandModule } from "yargs";
|
|
|
13
13
|
import type { Hono } from "hono";
|
|
14
14
|
import type { AccountAllowlist, LoadedProxyConfig, ModelRouterInterface, ProxyGuardArgs, ProxyNeurolinkRuntime, ProxyStartArgs, ProxyStartStrategy, ProxyStatusArgs, ProxyTelemetryArgs, ProxyReadinessState } from "../../lib/types/index.js";
|
|
15
15
|
import { ProxyRuntimeConfigStore } from "../../lib/proxy/runtimeConfig.js";
|
|
16
|
+
/**
|
|
17
|
+
* Best-effort check that a pid actually belongs to a neurolink proxy process,
|
|
18
|
+
* so a stale/recycled supervisor pid is never mistaken for a live supervisor
|
|
19
|
+
* and signalled. Returns false when the process cannot be confirmed as ours.
|
|
20
|
+
*
|
|
21
|
+
* @param expectedStartTimeIso - The `ProxySupervisorState.startTime` (ISO
|
|
22
|
+
* string) recorded when the supervisor we expect at `pid` was launched.
|
|
23
|
+
* When provided, this is cross-checked against `pid`'s actual OS-reported
|
|
24
|
+
* start time (see {@link getProcessStartTime}) — the args match alone
|
|
25
|
+
* ("neurolink" + "proxy" both present) is not airtight, since an
|
|
26
|
+
* unrelated process (e.g. a shell running this very test suite, or a
|
|
27
|
+
* coincidentally-named script) could match it too. A pid recycled by such
|
|
28
|
+
* a process would have a start time far from the recorded supervisor
|
|
29
|
+
* startTime, which this catches.
|
|
30
|
+
*
|
|
31
|
+
* Deliberately fail-open on anything unparseable: `ps -o lstart=` output
|
|
32
|
+
* parsing is inherently a little fragile (locale/format quirks), and the
|
|
33
|
+
* cost of a false "can't confirm" is silently regressing to pre-hardening
|
|
34
|
+
* behavior, while the cost of a false "confirmed mismatch" is leaving a
|
|
35
|
+
* real supervisor running and orphaning its socket during uninstall — the
|
|
36
|
+
* former is the safer failure mode. So this only ever returns false for
|
|
37
|
+
* the startTime check when BOTH timestamps parsed successfully AND their
|
|
38
|
+
* drift exceeds the tolerance; any parse failure is treated as "can't
|
|
39
|
+
* confirm a mismatch" and falls through to the args-only result.
|
|
40
|
+
*/
|
|
41
|
+
export declare function processLooksLikeProxySupervisor(pid: number, expectedStartTimeIso?: string): Promise<boolean>;
|
|
16
42
|
export declare function mapClaudeErrorTypeToStatus(errorType?: string): number;
|
|
17
43
|
export declare function createProxyStartApp(params: {
|
|
18
44
|
neurolink: ProxyNeurolinkRuntime["neurolink"];
|
|
@@ -41,6 +41,14 @@ const _require = createRequire(import.meta.url);
|
|
|
41
41
|
const PROXY_VERSION = packageJson.version;
|
|
42
42
|
const PROXY_TELEMETRY_SCRIPT_PATH = fileURLToPath(new URL("../../../scripts/observability/manage-local-openobserve.sh", import.meta.url));
|
|
43
43
|
const PROXY_LIFECYCLE_SHUTDOWN_TIMEOUT_MS = 5_000;
|
|
44
|
+
// Allowed drift between a pid's OS-reported start time and the persisted
|
|
45
|
+
// ProxySupervisorState.startTime before processLooksLikeProxySupervisor
|
|
46
|
+
// treats it as a confident mismatch (recycled pid). Generous on purpose:
|
|
47
|
+
// `supervisorStartedAt` is captured inside runLaunchdProxySupervisor, which
|
|
48
|
+
// runs after Node has already booted and done setup, so it always lags the
|
|
49
|
+
// OS-level fork/exec by a little — a tight tolerance would produce false
|
|
50
|
+
// mismatches on a slow/cold-started machine.
|
|
51
|
+
const PROXY_SUPERVISOR_START_TIME_TOLERANCE_MS = 10_000;
|
|
44
52
|
const PROXY_UPDATE_CONTROL_TOKEN = process.env.NEUROLINK_PROXY_UPDATE_CONTROL_TOKEN?.trim() ||
|
|
45
53
|
crypto.randomUUID();
|
|
46
54
|
// =============================================================================
|
|
@@ -98,23 +106,131 @@ function getProcessStatus(pid) {
|
|
|
98
106
|
}
|
|
99
107
|
}
|
|
100
108
|
/**
|
|
101
|
-
*
|
|
102
|
-
*
|
|
103
|
-
*
|
|
109
|
+
* Read a single `ps -o <field>=` field for `pid`, off the event loop (async
|
|
110
|
+
* `execFile`, not `execFileSync`) and bounded by a short timeout, so a
|
|
111
|
+
* hung/zombie `ps` invocation can never block the event loop or hang the
|
|
112
|
+
* (often uninstall-path) async caller. Shared by {@link getProcessStartTime}
|
|
113
|
+
* and {@link processLooksLikeProxySupervisor}, which both read a single
|
|
114
|
+
* `ps -o <field>=` column for the same pid.
|
|
115
|
+
*
|
|
116
|
+
* Returns null — never throws — on any error, timeout, or empty output, so
|
|
117
|
+
* callers uniformly treat "couldn't read" as "can't confirm either way"
|
|
118
|
+
* rather than distinguishing the failure mode.
|
|
104
119
|
*/
|
|
105
|
-
async function
|
|
120
|
+
async function readPsField(pid, field) {
|
|
106
121
|
try {
|
|
107
|
-
const {
|
|
108
|
-
const
|
|
122
|
+
const { execFile } = await import("node:child_process");
|
|
123
|
+
const { promisify } = await import("node:util");
|
|
124
|
+
const execFileAsync = promisify(execFile);
|
|
125
|
+
const { stdout } = await withTimeout(execFileAsync("ps", ["-p", String(pid), "-o", `${field}=`], {
|
|
109
126
|
encoding: "utf-8",
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
return
|
|
127
|
+
}), 2000, `ps -o ${field}= timed out for pid ${pid}`);
|
|
128
|
+
const trimmed = stdout.trim();
|
|
129
|
+
return trimmed || null;
|
|
113
130
|
}
|
|
114
131
|
catch {
|
|
115
|
-
|
|
132
|
+
return null;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Best-effort, macOS-only (uninstall is macOS-only) read of `pid`'s
|
|
137
|
+
* OS-reported start time via `ps -o lstart=`, which prints a fixed-width
|
|
138
|
+
* `Www Mmm dd hh:mm:ss yyyy` timestamp — the same shape `Date`'s parser
|
|
139
|
+
* already understands (it's the format `Date#toString()` itself produces
|
|
140
|
+
* modulo the trailing timezone). Returns null (never throws) when `ps`
|
|
141
|
+
* fails or its output doesn't parse, so callers can tell "confirmed
|
|
142
|
+
* mismatch" apart from "couldn't confirm either way".
|
|
143
|
+
*/
|
|
144
|
+
async function getProcessStartTime(pid) {
|
|
145
|
+
const out = await readPsField(pid, "lstart");
|
|
146
|
+
if (!out) {
|
|
147
|
+
return null;
|
|
148
|
+
}
|
|
149
|
+
// `ps -o lstart=` prints the OS start time in the machine's LOCAL timezone
|
|
150
|
+
// with no offset, e.g. "Sun Jul 19 20:12:41 2026". Parse the components and
|
|
151
|
+
// build the Date via the local-time constructor rather than relying on the
|
|
152
|
+
// engine's implementation-defined parsing of non-ISO date strings. The
|
|
153
|
+
// result is the same absolute instant the supervisor persisted as an
|
|
154
|
+
// ISO-UTC string, so the drift comparison against it is timezone-safe.
|
|
155
|
+
const m = out.match(/^\w{3}\s+(\w{3})\s+(\d{1,2})\s+(\d{2}):(\d{2}):(\d{2})\s+(\d{4})$/);
|
|
156
|
+
if (!m) {
|
|
157
|
+
return null;
|
|
158
|
+
}
|
|
159
|
+
const monthIndex = [
|
|
160
|
+
"Jan",
|
|
161
|
+
"Feb",
|
|
162
|
+
"Mar",
|
|
163
|
+
"Apr",
|
|
164
|
+
"May",
|
|
165
|
+
"Jun",
|
|
166
|
+
"Jul",
|
|
167
|
+
"Aug",
|
|
168
|
+
"Sep",
|
|
169
|
+
"Oct",
|
|
170
|
+
"Nov",
|
|
171
|
+
"Dec",
|
|
172
|
+
].indexOf(m[1]);
|
|
173
|
+
if (monthIndex < 0) {
|
|
174
|
+
return null;
|
|
175
|
+
}
|
|
176
|
+
const parsed = new Date(Number(m[6]), monthIndex, Number(m[2]), Number(m[3]), Number(m[4]), Number(m[5]));
|
|
177
|
+
return Number.isNaN(parsed.getTime()) ? null : parsed;
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
180
|
+
* Best-effort check that a pid actually belongs to a neurolink proxy process,
|
|
181
|
+
* so a stale/recycled supervisor pid is never mistaken for a live supervisor
|
|
182
|
+
* and signalled. Returns false when the process cannot be confirmed as ours.
|
|
183
|
+
*
|
|
184
|
+
* @param expectedStartTimeIso - The `ProxySupervisorState.startTime` (ISO
|
|
185
|
+
* string) recorded when the supervisor we expect at `pid` was launched.
|
|
186
|
+
* When provided, this is cross-checked against `pid`'s actual OS-reported
|
|
187
|
+
* start time (see {@link getProcessStartTime}) — the args match alone
|
|
188
|
+
* ("neurolink" + "proxy" both present) is not airtight, since an
|
|
189
|
+
* unrelated process (e.g. a shell running this very test suite, or a
|
|
190
|
+
* coincidentally-named script) could match it too. A pid recycled by such
|
|
191
|
+
* a process would have a start time far from the recorded supervisor
|
|
192
|
+
* startTime, which this catches.
|
|
193
|
+
*
|
|
194
|
+
* Deliberately fail-open on anything unparseable: `ps -o lstart=` output
|
|
195
|
+
* parsing is inherently a little fragile (locale/format quirks), and the
|
|
196
|
+
* cost of a false "can't confirm" is silently regressing to pre-hardening
|
|
197
|
+
* behavior, while the cost of a false "confirmed mismatch" is leaving a
|
|
198
|
+
* real supervisor running and orphaning its socket during uninstall — the
|
|
199
|
+
* former is the safer failure mode. So this only ever returns false for
|
|
200
|
+
* the startTime check when BOTH timestamps parsed successfully AND their
|
|
201
|
+
* drift exceeds the tolerance; any parse failure is treated as "can't
|
|
202
|
+
* confirm a mismatch" and falls through to the args-only result.
|
|
203
|
+
*/
|
|
204
|
+
export async function processLooksLikeProxySupervisor(pid, expectedStartTimeIso) {
|
|
205
|
+
const args = await readPsField(pid, "args");
|
|
206
|
+
if (args === null) {
|
|
207
|
+
// `ps` unavailable, timed out, or the pid vanished — do not signal an
|
|
208
|
+
// unverified pid.
|
|
209
|
+
return false;
|
|
210
|
+
}
|
|
211
|
+
// `/neurolink/i` alone is too broad: any process whose args merely
|
|
212
|
+
// *mention* "neurolink" (e.g. a shell running in a directory named
|
|
213
|
+
// "neurolink", an editor with a neurolink file open) would match, and a
|
|
214
|
+
// stale/recycled pid running such a process could then be SIGTERM'd/
|
|
215
|
+
// SIGKILL'd by `ensureSupervisorStoppedBeforeClear` during uninstall.
|
|
216
|
+
// Require the actual proxy-supervisor invocation — both "neurolink" AND
|
|
217
|
+
// the "proxy" subcommand present in args.
|
|
218
|
+
if (!(/neurolink/i.test(args) && /\bproxy\b/i.test(args))) {
|
|
116
219
|
return false;
|
|
117
220
|
}
|
|
221
|
+
if (!expectedStartTimeIso) {
|
|
222
|
+
return true;
|
|
223
|
+
}
|
|
224
|
+
const expected = new Date(expectedStartTimeIso);
|
|
225
|
+
if (Number.isNaN(expected.getTime())) {
|
|
226
|
+
return true; // recorded startTime itself is unparseable — can't confirm a mismatch
|
|
227
|
+
}
|
|
228
|
+
const actual = await getProcessStartTime(pid);
|
|
229
|
+
if (!actual) {
|
|
230
|
+
return true; // ps -o lstart= unavailable/unparseable — can't confirm a mismatch
|
|
231
|
+
}
|
|
232
|
+
const driftMs = Math.abs(actual.getTime() - expected.getTime());
|
|
233
|
+
return driftMs <= PROXY_SUPERVISOR_START_TIME_TOLERANCE_MS;
|
|
118
234
|
}
|
|
119
235
|
/**
|
|
120
236
|
* Ensure any supervisor recorded in state is actually stopped before its state
|
|
@@ -124,14 +240,17 @@ async function processLooksLikeProxySupervisor(pid) {
|
|
|
124
240
|
* the uninstall instead of silently discarding the record.
|
|
125
241
|
*/
|
|
126
242
|
async function ensureSupervisorStoppedBeforeClear() {
|
|
127
|
-
const
|
|
243
|
+
const supervisorState = loadProxySupervisorState();
|
|
244
|
+
const pid = supervisorState?.pid;
|
|
128
245
|
if (!pid || getProcessStatus(pid) === "not_running") {
|
|
129
246
|
return true;
|
|
130
247
|
}
|
|
131
248
|
// Guard against a stale/recycled pid: only signal a process that still looks
|
|
132
249
|
// like a neurolink proxy supervisor, so uninstall never terminates an
|
|
133
|
-
// unrelated, same-user process that inherited the recorded pid.
|
|
134
|
-
|
|
250
|
+
// unrelated, same-user process that inherited the recorded pid. Also cross-
|
|
251
|
+
// checks the pid's actual OS start time against the recorded
|
|
252
|
+
// supervisorState.startTime (see processLooksLikeProxySupervisor).
|
|
253
|
+
if (!(await processLooksLikeProxySupervisor(pid, supervisorState?.startTime))) {
|
|
135
254
|
return true;
|
|
136
255
|
}
|
|
137
256
|
try {
|
|
@@ -192,6 +192,13 @@ export declare class CLICommandFactory {
|
|
|
192
192
|
* objects independently.
|
|
193
193
|
*/
|
|
194
194
|
private static buildCsvOptionsFromArgv;
|
|
195
|
+
/**
|
|
196
|
+
* Build video processor options from CLI argv. Shared by executeGenerate,
|
|
197
|
+
* executeStream and executeBatch, which previously built byte-for-byte
|
|
198
|
+
* identical `videoOptions` objects independently (mirrors
|
|
199
|
+
* {@link buildCsvOptionsFromArgv}).
|
|
200
|
+
*/
|
|
201
|
+
private static buildVideoOptionsFromArgv;
|
|
195
202
|
/**
|
|
196
203
|
* Build output configuration for generate request
|
|
197
204
|
*/
|
|
@@ -2438,6 +2438,20 @@ export class CLICommandFactory {
|
|
|
2438
2438
|
parseTimeoutMs: argv.csvParseTimeoutMs,
|
|
2439
2439
|
};
|
|
2440
2440
|
}
|
|
2441
|
+
/**
|
|
2442
|
+
* Build video processor options from CLI argv. Shared by executeGenerate,
|
|
2443
|
+
* executeStream and executeBatch, which previously built byte-for-byte
|
|
2444
|
+
* identical `videoOptions` objects independently (mirrors
|
|
2445
|
+
* {@link buildCsvOptionsFromArgv}).
|
|
2446
|
+
*/
|
|
2447
|
+
static buildVideoOptionsFromArgv(argv) {
|
|
2448
|
+
return {
|
|
2449
|
+
frames: argv.videoFrames,
|
|
2450
|
+
quality: argv.videoQuality,
|
|
2451
|
+
format: argv.videoFormat,
|
|
2452
|
+
transcribeAudio: argv.transcribeAudio,
|
|
2453
|
+
};
|
|
2454
|
+
}
|
|
2441
2455
|
/**
|
|
2442
2456
|
* Build output configuration for generate request
|
|
2443
2457
|
*/
|
|
@@ -2677,12 +2691,7 @@ export class CLICommandFactory {
|
|
|
2677
2691
|
password: CLICommandFactory.resolvePdfPassword(argv),
|
|
2678
2692
|
},
|
|
2679
2693
|
csvOptions: CLICommandFactory.buildCsvOptionsFromArgv(argv),
|
|
2680
|
-
videoOptions:
|
|
2681
|
-
frames: argv.videoFrames,
|
|
2682
|
-
quality: argv.videoQuality,
|
|
2683
|
-
format: argv.videoFormat,
|
|
2684
|
-
transcribeAudio: argv.transcribeAudio,
|
|
2685
|
-
},
|
|
2694
|
+
videoOptions: CLICommandFactory.buildVideoOptionsFromArgv(argv),
|
|
2686
2695
|
output: outputConfig,
|
|
2687
2696
|
provider: enhancedOptions.provider,
|
|
2688
2697
|
model: enhancedOptions.model,
|
|
@@ -2932,12 +2941,7 @@ export class CLICommandFactory {
|
|
|
2932
2941
|
password: CLICommandFactory.resolvePdfPassword(argv),
|
|
2933
2942
|
},
|
|
2934
2943
|
csvOptions: CLICommandFactory.buildCsvOptionsFromArgv(argv),
|
|
2935
|
-
videoOptions:
|
|
2936
|
-
frames: argv.videoFrames,
|
|
2937
|
-
quality: argv.videoQuality,
|
|
2938
|
-
format: argv.videoFormat,
|
|
2939
|
-
transcribeAudio: argv.transcribeAudio,
|
|
2940
|
-
},
|
|
2944
|
+
videoOptions: CLICommandFactory.buildVideoOptionsFromArgv(argv),
|
|
2941
2945
|
provider: enhancedOptions.provider,
|
|
2942
2946
|
model: enhancedOptions.model,
|
|
2943
2947
|
temperature: enhancedOptions.temperature,
|
|
@@ -3411,6 +3415,7 @@ export class CLICommandFactory {
|
|
|
3411
3415
|
// see createBatchCommand — so `argv.file` can never be set here; no
|
|
3412
3416
|
// exclusion needed to protect the (differently-named) positional.
|
|
3413
3417
|
validateCliInputFiles(argv);
|
|
3418
|
+
validateCsvMaxRows(argv);
|
|
3414
3419
|
const batchImages = CLICommandFactory.processCliImages(argv.image);
|
|
3415
3420
|
const batchCsvFiles = CLICommandFactory.processCliCSVFiles(argv.csv);
|
|
3416
3421
|
const batchPdfFiles = CLICommandFactory.processCliPDFFiles(argv.pdf);
|
|
@@ -3469,6 +3474,12 @@ export class CLICommandFactory {
|
|
|
3469
3474
|
logger.alwaysStderr(chalk.yellow("⚠️ Multimodal files (--image/--csv/--pdf/--video) are attached to ALL prompts in this batch."));
|
|
3470
3475
|
spinner?.start();
|
|
3471
3476
|
}
|
|
3477
|
+
// Resolve the PDF password once for the whole batch. resolvePdfPassword
|
|
3478
|
+
// emits a "visible in shell history" stderr warning when the flag is set;
|
|
3479
|
+
// it must fire once per run (like generate/stream), not once per prompt.
|
|
3480
|
+
const batchPdfPassword = batchPdfFiles?.length
|
|
3481
|
+
? CLICommandFactory.resolvePdfPassword(argv)
|
|
3482
|
+
: undefined;
|
|
3472
3483
|
for (let i = 0; i < prompts.length; i++) {
|
|
3473
3484
|
if (spinner) {
|
|
3474
3485
|
spinner.text = `Processing ${i + 1}/${prompts.length}: ${prompts[i].substring(0, 30)}...`;
|
|
@@ -3518,18 +3529,15 @@ export class CLICommandFactory {
|
|
|
3518
3529
|
// unlike generate/stream, so an empty options object here is
|
|
3519
3530
|
// pure noise on every prompt of every batch run.
|
|
3520
3531
|
...(batchCsvFiles?.length && {
|
|
3521
|
-
csvOptions:
|
|
3522
|
-
|
|
3523
|
-
|
|
3532
|
+
csvOptions: CLICommandFactory.buildCsvOptionsFromArgv(argv),
|
|
3533
|
+
}),
|
|
3534
|
+
...(batchPdfFiles?.length && {
|
|
3535
|
+
pdfOptions: {
|
|
3536
|
+
password: batchPdfPassword,
|
|
3524
3537
|
},
|
|
3525
3538
|
}),
|
|
3526
3539
|
...(batchVideoFiles?.length && {
|
|
3527
|
-
videoOptions:
|
|
3528
|
-
frames: argv.videoFrames,
|
|
3529
|
-
quality: argv.videoQuality,
|
|
3530
|
-
format: argv.videoFormat,
|
|
3531
|
-
transcribeAudio: argv.transcribeAudio,
|
|
3532
|
-
},
|
|
3540
|
+
videoOptions: CLICommandFactory.buildVideoOptionsFromArgv(argv),
|
|
3533
3541
|
}),
|
|
3534
3542
|
provider: enhancedOptions.provider,
|
|
3535
3543
|
model: enhancedOptions.model,
|
|
@@ -27,6 +27,11 @@ export const CLI_SOFT_LIMITS_MB = {
|
|
|
27
27
|
IMAGE_MAX_MB: 10,
|
|
28
28
|
CSV_MAX_MB: 50,
|
|
29
29
|
PDF_MAX_MB: 100,
|
|
30
|
+
// Distinct from PDF_MAX_MB above (which mirrors the processor's true
|
|
31
|
+
// 100MB hard cap in lib/processors/config/sizeLimits.ts and is kept here
|
|
32
|
+
// for backward compatibility). Docs promise a 50MB *warning* threshold
|
|
33
|
+
// specifically for --pdf — wire warnAtMB to this value, not the hard cap.
|
|
34
|
+
PDF_SOFT_MAX_MB: 50,
|
|
30
35
|
VIDEO_MAX_MB: 500,
|
|
31
36
|
};
|
|
32
37
|
// `file://` is intentionally excluded here: it addresses a local path, so it
|
|
@@ -165,7 +170,7 @@ export function validateCliInputFiles(argv) {
|
|
|
165
170
|
{
|
|
166
171
|
option: "--pdf",
|
|
167
172
|
value: argv.pdf,
|
|
168
|
-
warnAtMB: CLI_SOFT_LIMITS_MB.
|
|
173
|
+
warnAtMB: CLI_SOFT_LIMITS_MB.PDF_SOFT_MAX_MB,
|
|
169
174
|
},
|
|
170
175
|
{
|
|
171
176
|
option: "--video",
|
|
@@ -25,7 +25,7 @@ import { CSVProcessor } from "./csvProcessor.js";
|
|
|
25
25
|
import { ImageProcessor } from "./imageProcessor.js";
|
|
26
26
|
import { logger } from "./logger.js";
|
|
27
27
|
import { withTimeout } from "./errorHandling.js";
|
|
28
|
-
import { redactUrlForError, sanitizeErrorCause } from "./logSanitize.js";
|
|
28
|
+
import { normalizeUrlForCache, redactUrlForError, sanitizeErrorCause, } from "./logSanitize.js";
|
|
29
29
|
import { mimeHintToExtension, mimeHintToFileType, normalizeMimeHint, } from "./mimeTypeHints.js";
|
|
30
30
|
import { PDFProcessor } from "./pdfProcessor.js";
|
|
31
31
|
/**
|
|
@@ -54,22 +54,40 @@ const DEFAULT_RETRY_DELAY = 1000; // milliseconds
|
|
|
54
54
|
const URL_CONTENT_TYPE_TTL_MS = 60_000;
|
|
55
55
|
const URL_CONTENT_TYPE_CACHE_MAX_SIZE = 512;
|
|
56
56
|
const urlContentTypeCache = new Map();
|
|
57
|
+
/**
|
|
58
|
+
* Build the Map key for `urlContentTypeCache`. Delegates to the shared
|
|
59
|
+
* {@link normalizeUrlForCache} — this is a module-level, process-lifetime
|
|
60
|
+
* cache, so it must strip presigned-URL auth/signature query params (see
|
|
61
|
+
* `SENSITIVE_URL_QUERY_PARAM_DENYLIST`) the same way `ImageCache.normalizeUrl`
|
|
62
|
+
* does, folding a short hash of the stripped params into the key whenever any
|
|
63
|
+
* were present so two different presigned URLs for the same path don't
|
|
64
|
+
* collide and serve each other's cached content-type across auth contexts.
|
|
65
|
+
* Also strips tracking/analytics params first, so two URLs differing only by
|
|
66
|
+
* tracking noise (e.g. `utm_source`) still hit the same cache entry instead
|
|
67
|
+
* of missing each other. Falls back to the raw URL if it isn't a parseable
|
|
68
|
+
* absolute URL. Shared by both `getCachedUrlContentType` and
|
|
69
|
+
* `setCachedUrlContentType` so lookups and writes always agree on the key.
|
|
70
|
+
*/
|
|
71
|
+
function cacheKeyForUrl(url) {
|
|
72
|
+
return normalizeUrlForCache(url);
|
|
73
|
+
}
|
|
57
74
|
function getCachedUrlContentType(url, now) {
|
|
58
|
-
const
|
|
75
|
+
const key = cacheKeyForUrl(url);
|
|
76
|
+
const hit = urlContentTypeCache.get(key);
|
|
59
77
|
if (hit && hit.expiresAt > now) {
|
|
60
78
|
// Bump recency: Map iteration order follows insertion order, and the
|
|
61
79
|
// eviction below deletes the *first* key, so a plain `get` on a hot
|
|
62
80
|
// entry would leave it first in line for eviction despite being the
|
|
63
81
|
// most recently used. Re-inserting turns the size-bounded FIFO below
|
|
64
82
|
// into an actual LRU.
|
|
65
|
-
urlContentTypeCache.delete(
|
|
66
|
-
urlContentTypeCache.set(
|
|
83
|
+
urlContentTypeCache.delete(key);
|
|
84
|
+
urlContentTypeCache.set(key, hit);
|
|
67
85
|
return hit.contentType;
|
|
68
86
|
}
|
|
69
87
|
if (hit) {
|
|
70
88
|
// Entry exists but its TTL has passed — treat as a miss and evict it
|
|
71
89
|
// immediately rather than serving (or retaining) stale data.
|
|
72
|
-
urlContentTypeCache.delete(
|
|
90
|
+
urlContentTypeCache.delete(key);
|
|
73
91
|
}
|
|
74
92
|
return undefined;
|
|
75
93
|
}
|
|
@@ -89,7 +107,8 @@ function setCachedUrlContentType(url, contentType, now) {
|
|
|
89
107
|
if (!contentType) {
|
|
90
108
|
return;
|
|
91
109
|
}
|
|
92
|
-
|
|
110
|
+
const key = cacheKeyForUrl(url);
|
|
111
|
+
urlContentTypeCache.set(key, {
|
|
93
112
|
contentType,
|
|
94
113
|
expiresAt: now + URL_CONTENT_TYPE_TTL_MS,
|
|
95
114
|
});
|
|
@@ -1436,16 +1455,25 @@ export class FileDetector {
|
|
|
1436
1455
|
});
|
|
1437
1456
|
// Drain/close the (empty) HEAD body so the connection can be reused.
|
|
1438
1457
|
await head.body.dump();
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1458
|
+
// Only trust `content-length` on a genuine 2xx response. A non-2xx
|
|
1459
|
+
// HEAD (redirect the dispatcher didn't follow, 403/404/405 "HEAD not
|
|
1460
|
+
// allowed", 5xx, …) can still carry a stale/irrelevant
|
|
1461
|
+
// `content-length` header — enforcing size off of that would reject
|
|
1462
|
+
// (or silently pass) based on the wrong body. Treat any non-2xx HEAD
|
|
1463
|
+
// as if the header were missing and fall through to the streaming
|
|
1464
|
+
// GET guard below, which enforces maxSize independently either way.
|
|
1465
|
+
if (head.statusCode >= 200 && head.statusCode < 300) {
|
|
1466
|
+
const declaredLength = Number(head.headers["content-length"]);
|
|
1467
|
+
if (Number.isFinite(declaredLength) && declaredLength > maxSize) {
|
|
1468
|
+
throw new Error(`File too large: ${formatFileSize(declaredLength)} (max: ${formatFileSize(maxSize)})`);
|
|
1469
|
+
}
|
|
1442
1470
|
}
|
|
1443
1471
|
}
|
|
1444
1472
|
catch (error) {
|
|
1445
1473
|
if (error instanceof Error && /File too large/.test(error.message)) {
|
|
1446
1474
|
throw error;
|
|
1447
1475
|
}
|
|
1448
|
-
logger.debug(`[FileDetector] HEAD pre-flight skipped for ${url}: ${error
|
|
1476
|
+
logger.debug(`[FileDetector] HEAD pre-flight skipped for ${redactUrlForError(url)}: ${sanitizeErrorCause(error).message}`);
|
|
1449
1477
|
}
|
|
1450
1478
|
}
|
|
1451
1479
|
return withRetry(async () => {
|
|
@@ -33,8 +33,16 @@ export declare class ImageCache {
|
|
|
33
33
|
*/
|
|
34
34
|
private parseConfigValue;
|
|
35
35
|
/**
|
|
36
|
-
* Normalize URL for consistent cache key generation
|
|
37
|
-
*
|
|
36
|
+
* Normalize URL for consistent cache key generation.
|
|
37
|
+
*
|
|
38
|
+
* Delegates to the shared {@link normalizeUrlForCache} — this normalized
|
|
39
|
+
* URL is used as the in-memory Map key (see `get`/`set` below), which lives
|
|
40
|
+
* for the process lifetime, so both tracking-param noise and presigned-URL
|
|
41
|
+
* auth/signature secrets must be stripped consistently with
|
|
42
|
+
* `fileDetector.cacheKeyForUrl`'s identical requirement. See
|
|
43
|
+
* `stripSensitiveUrlParamsForCacheKey`'s doc comment for why a naive
|
|
44
|
+
* strip-only approach (no hash suffix) would be a correctness bug, not just
|
|
45
|
+
* a hygiene one.
|
|
38
46
|
*/
|
|
39
47
|
private normalizeUrl;
|
|
40
48
|
/**
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
*/
|
|
15
15
|
import { createHash } from "crypto";
|
|
16
16
|
import { logger } from "./logger.js";
|
|
17
|
-
import { redactUrlForError } from "./logSanitize.js";
|
|
17
|
+
import { normalizeUrlForCache, redactUrlForError } from "./logSanitize.js";
|
|
18
18
|
/**
|
|
19
19
|
* LRU Cache for downloaded images
|
|
20
20
|
*
|
|
@@ -85,30 +85,19 @@ export class ImageCache {
|
|
|
85
85
|
return value;
|
|
86
86
|
}
|
|
87
87
|
/**
|
|
88
|
-
* Normalize URL for consistent cache key generation
|
|
89
|
-
*
|
|
88
|
+
* Normalize URL for consistent cache key generation.
|
|
89
|
+
*
|
|
90
|
+
* Delegates to the shared {@link normalizeUrlForCache} — this normalized
|
|
91
|
+
* URL is used as the in-memory Map key (see `get`/`set` below), which lives
|
|
92
|
+
* for the process lifetime, so both tracking-param noise and presigned-URL
|
|
93
|
+
* auth/signature secrets must be stripped consistently with
|
|
94
|
+
* `fileDetector.cacheKeyForUrl`'s identical requirement. See
|
|
95
|
+
* `stripSensitiveUrlParamsForCacheKey`'s doc comment for why a naive
|
|
96
|
+
* strip-only approach (no hash suffix) would be a correctness bug, not just
|
|
97
|
+
* a hygiene one.
|
|
90
98
|
*/
|
|
91
99
|
normalizeUrl(url) {
|
|
92
|
-
|
|
93
|
-
const parsed = new URL(url);
|
|
94
|
-
// Remove common tracking parameters that don't affect content
|
|
95
|
-
const trackingParams = [
|
|
96
|
-
"utm_source",
|
|
97
|
-
"utm_medium",
|
|
98
|
-
"utm_campaign",
|
|
99
|
-
"utm_term",
|
|
100
|
-
"utm_content",
|
|
101
|
-
"fbclid",
|
|
102
|
-
"gclid",
|
|
103
|
-
"_ga",
|
|
104
|
-
];
|
|
105
|
-
trackingParams.forEach((param) => parsed.searchParams.delete(param));
|
|
106
|
-
return parsed.toString();
|
|
107
|
-
}
|
|
108
|
-
catch {
|
|
109
|
-
// If URL parsing fails, use the original URL
|
|
110
|
-
return url;
|
|
111
|
-
}
|
|
100
|
+
return normalizeUrlForCache(url);
|
|
112
101
|
}
|
|
113
102
|
/**
|
|
114
103
|
* Generate content hash from image data
|
|
@@ -13,6 +13,31 @@ export declare const MIN_IMAGE_SIZE = 12;
|
|
|
13
13
|
* buffer that carries a format's magic bytes but is smaller than this is
|
|
14
14
|
* truncated/corrupt and would fail (or render blank) downstream — reject early
|
|
15
15
|
* with a clear message instead.
|
|
16
|
+
*
|
|
17
|
+
* Each threshold approximates the smallest byte count that format's
|
|
18
|
+
* container can structurally be while still holding the mandatory
|
|
19
|
+
* header/chunk skeleton for a minimal (e.g. 1×1 pixel) image — not an exact
|
|
20
|
+
* spec minimum, but close enough that anything smaller is unambiguously
|
|
21
|
+
* truncated:
|
|
22
|
+
* - **PNG (67)**: 8-byte signature + IHDR chunk (4 len + 4 type + 13 data +
|
|
23
|
+
* 4 CRC = 25) + a minimal IDAT chunk carrying one zlib-compressed
|
|
24
|
+
* scanline (~22 bytes) + IEND chunk (4 len + 4 type + 0 data + 4 CRC =
|
|
25
|
+
* 12) — matches the commonly-cited size of the smallest possible valid
|
|
26
|
+
* PNG (a 1×1 transparent pixel).
|
|
27
|
+
* - **JPEG (125)**: SOI + APP0/JFIF marker (18) + at least one DQT
|
|
28
|
+
* quantization table (~69 for one 8-bit luma table) + SOF0 + DHT + SOS
|
|
29
|
+
* headers + a minimal entropy-coded scan + EOI — the smallest legal
|
|
30
|
+
* baseline-JPEG skeleton for a 1×1 image.
|
|
31
|
+
* - **GIF (43)**: 6-byte GIF87a/89a header + 7-byte Logical Screen
|
|
32
|
+
* Descriptor + a 2-color (6-byte) color table + 10-byte Image
|
|
33
|
+
* Descriptor + minimal LZW-coded image sub-blocks + 1-byte trailer.
|
|
34
|
+
* - **WEBP (20)**: RIFF container header ("RIFF" + size + "WEBP" = 12
|
|
35
|
+
* bytes) + the first chunk's fourCC + size (8 bytes) — enough to confirm
|
|
36
|
+
* a well-formed RIFF/WEBP container and its first chunk, before any
|
|
37
|
+
* actual VP8/VP8L bitstream payload.
|
|
38
|
+
* - **AVIF (100)**: ISOBMFF `ftyp` box plus the mandatory `meta` box
|
|
39
|
+
* skeleton (`hdlr`/`pitm`/`iloc`/`iinf` sub-boxes) — roughly the minimum
|
|
40
|
+
* to hold AVIF's required box structure before any AV1 image item data.
|
|
16
41
|
*/
|
|
17
42
|
export declare const MIN_VALID_IMAGE_SIZE: Record<string, number>;
|
|
18
43
|
/**
|
|
@@ -20,6 +20,31 @@ export const MIN_IMAGE_SIZE = 12;
|
|
|
20
20
|
* buffer that carries a format's magic bytes but is smaller than this is
|
|
21
21
|
* truncated/corrupt and would fail (or render blank) downstream — reject early
|
|
22
22
|
* with a clear message instead.
|
|
23
|
+
*
|
|
24
|
+
* Each threshold approximates the smallest byte count that format's
|
|
25
|
+
* container can structurally be while still holding the mandatory
|
|
26
|
+
* header/chunk skeleton for a minimal (e.g. 1×1 pixel) image — not an exact
|
|
27
|
+
* spec minimum, but close enough that anything smaller is unambiguously
|
|
28
|
+
* truncated:
|
|
29
|
+
* - **PNG (67)**: 8-byte signature + IHDR chunk (4 len + 4 type + 13 data +
|
|
30
|
+
* 4 CRC = 25) + a minimal IDAT chunk carrying one zlib-compressed
|
|
31
|
+
* scanline (~22 bytes) + IEND chunk (4 len + 4 type + 0 data + 4 CRC =
|
|
32
|
+
* 12) — matches the commonly-cited size of the smallest possible valid
|
|
33
|
+
* PNG (a 1×1 transparent pixel).
|
|
34
|
+
* - **JPEG (125)**: SOI + APP0/JFIF marker (18) + at least one DQT
|
|
35
|
+
* quantization table (~69 for one 8-bit luma table) + SOF0 + DHT + SOS
|
|
36
|
+
* headers + a minimal entropy-coded scan + EOI — the smallest legal
|
|
37
|
+
* baseline-JPEG skeleton for a 1×1 image.
|
|
38
|
+
* - **GIF (43)**: 6-byte GIF87a/89a header + 7-byte Logical Screen
|
|
39
|
+
* Descriptor + a 2-color (6-byte) color table + 10-byte Image
|
|
40
|
+
* Descriptor + minimal LZW-coded image sub-blocks + 1-byte trailer.
|
|
41
|
+
* - **WEBP (20)**: RIFF container header ("RIFF" + size + "WEBP" = 12
|
|
42
|
+
* bytes) + the first chunk's fourCC + size (8 bytes) — enough to confirm
|
|
43
|
+
* a well-formed RIFF/WEBP container and its first chunk, before any
|
|
44
|
+
* actual VP8/VP8L bitstream payload.
|
|
45
|
+
* - **AVIF (100)**: ISOBMFF `ftyp` box plus the mandatory `meta` box
|
|
46
|
+
* skeleton (`hdlr`/`pitm`/`iloc`/`iinf` sub-boxes) — roughly the minimum
|
|
47
|
+
* to hold AVIF's required box structure before any AV1 image item data.
|
|
23
48
|
*/
|
|
24
49
|
export const MIN_VALID_IMAGE_SIZE = {
|
|
25
50
|
"image/png": 67,
|
|
@@ -932,7 +957,9 @@ export const imageUtils = {
|
|
|
932
957
|
const cache = getImageCache();
|
|
933
958
|
const cached = cache.get(url);
|
|
934
959
|
if (cached) {
|
|
935
|
-
logger.debug("Using cached image for URL", {
|
|
960
|
+
logger.debug("Using cached image for URL", {
|
|
961
|
+
url: redactUrlForError(url),
|
|
962
|
+
});
|
|
936
963
|
return cached.dataUri;
|
|
937
964
|
}
|
|
938
965
|
// Apply rate limiting before download
|