@juspay/neurolink 10.10.7 → 10.10.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/README.md +37 -8
- package/dist/browser/neurolink.min.js +343 -343
- package/dist/cli/factories/commandFactory.js +8 -4
- package/dist/lib/processors/media/VideoProcessor.d.ts +13 -3
- package/dist/lib/processors/media/VideoProcessor.js +53 -12
- package/dist/lib/skills/skillsManager.js +24 -5
- package/dist/lib/types/file.d.ts +41 -0
- package/dist/lib/types/generate.d.ts +19 -0
- package/dist/lib/types/stream.d.ts +15 -0
- package/dist/lib/utils/errorHandling.d.ts +21 -0
- package/dist/lib/utils/errorHandling.js +53 -0
- package/dist/lib/utils/fileDetector.js +9 -6
- package/dist/lib/utils/messageBuilder.js +151 -39
- package/dist/lib/utils/multimodalOptionsBuilder.d.ts +2 -0
- package/dist/lib/utils/pdfProcessor.d.ts +11 -0
- package/dist/lib/utils/pdfProcessor.js +27 -0
- package/dist/processors/media/VideoProcessor.d.ts +13 -3
- package/dist/processors/media/VideoProcessor.js +53 -12
- package/dist/skills/skillsManager.js +24 -5
- package/dist/types/file.d.ts +41 -0
- package/dist/types/generate.d.ts +19 -0
- package/dist/types/stream.d.ts +15 -0
- package/dist/utils/errorHandling.d.ts +21 -0
- package/dist/utils/errorHandling.js +53 -0
- package/dist/utils/fileDetector.js +9 -6
- package/dist/utils/messageBuilder.js +151 -39
- package/dist/utils/multimodalOptionsBuilder.d.ts +2 -0
- package/dist/utils/pdfProcessor.d.ts +11 -0
- package/dist/utils/pdfProcessor.js +27 -0
- package/package.json +1 -1
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { existsSync, readFileSync, statSync } from "fs";
|
|
2
2
|
import { readFile as readFileAsync, stat as statAsync } from "fs/promises";
|
|
3
|
-
import { basename } from "path";
|
|
4
3
|
import { getGlobalDispatcher, interceptors, request } from "undici";
|
|
5
4
|
import { MultimodalLogger, ProviderImageAdapter, } from "../adapters/providerImageAdapter.js";
|
|
6
5
|
import { CONVERSATION_INSTRUCTIONS, STRUCTURED_OUTPUT_INSTRUCTIONS, } from "../config/conversationMemory.js";
|
|
7
6
|
import { getAvailableInputTokens } from "../constants/contextWindows.js";
|
|
7
|
+
import { PDF_LIMITS } from "../core/constants.js";
|
|
8
8
|
import { enforceAggregateFileBudget, FILE_READ_BUDGET_PERCENT, } from "../context/fileTokenBudget.js";
|
|
9
9
|
import { isCSVContent, SIZE_TIER_THRESHOLDS } from "../types/index.js";
|
|
10
10
|
import { tracers, ATTR, withSpan } from "../telemetry/index.js";
|
|
@@ -809,6 +809,20 @@ export function mergeMediaFileAliases(input) {
|
|
|
809
809
|
input.audioFiles = undefined;
|
|
810
810
|
input.videoFiles = undefined;
|
|
811
811
|
}
|
|
812
|
+
/**
|
|
813
|
+
* #478: `transcribeAudio` (CLI `--transcribe-audio`) is accepted by the options
|
|
814
|
+
* surface but no video-audio transcription exists yet — VideoProcessor extracts
|
|
815
|
+
* keyframes and embedded subtitle tracks only, and the transcription step is
|
|
816
|
+
* still open as #433. Say so once per request rather than letting the caller
|
|
817
|
+
* believe a transcript was produced and silently omitted.
|
|
818
|
+
*/
|
|
819
|
+
function warnIfVideoTranscriptionRequested(videoOptions) {
|
|
820
|
+
if (videoOptions?.transcribeAudio) {
|
|
821
|
+
logger.warn("[NEUROLINK] Video audio transcription was requested but is not implemented yet " +
|
|
822
|
+
"(tracked as #433). Keyframes and any embedded subtitle tracks are still extracted; " +
|
|
823
|
+
"spoken audio will not be transcribed.");
|
|
824
|
+
}
|
|
825
|
+
}
|
|
812
826
|
/**
|
|
813
827
|
* Process the unified files array with auto-detection.
|
|
814
828
|
* Handles lazy file registration, full processing, and preview injection.
|
|
@@ -825,6 +839,7 @@ export async function processUnifiedFilesArray(options, maxSize, provider) {
|
|
|
825
839
|
}
|
|
826
840
|
const totalFiles = options.input.files.length;
|
|
827
841
|
const files = options.input.files;
|
|
842
|
+
warnIfVideoTranscriptionRequested(options.videoOptions);
|
|
828
843
|
return withSpan({
|
|
829
844
|
name: "neurolink.file.process_all",
|
|
830
845
|
tracer: tracers.file,
|
|
@@ -882,6 +897,15 @@ export async function processUnifiedFilesArray(options, maxSize, provider) {
|
|
|
882
897
|
"unknown",
|
|
883
898
|
],
|
|
884
899
|
csvOptions: options.csvOptions,
|
|
900
|
+
// #478: videos arrive through this unified `files` path, so this is
|
|
901
|
+
// where the CLI's frame/quality/format request has to be handed on.
|
|
902
|
+
videoOptions: options.videoOptions
|
|
903
|
+
? {
|
|
904
|
+
frames: options.videoOptions.frames,
|
|
905
|
+
quality: options.videoOptions.quality,
|
|
906
|
+
format: options.videoOptions.format,
|
|
907
|
+
}
|
|
908
|
+
: undefined,
|
|
885
909
|
provider: provider,
|
|
886
910
|
mimetypeHint: fileMimetypeHint,
|
|
887
911
|
});
|
|
@@ -1020,6 +1044,83 @@ function enforcePostProcessingBudget(options, provider, model) {
|
|
|
1020
1044
|
`budget=${contextWindow.toLocaleString()} tokens, ` +
|
|
1021
1045
|
`utilization=${contextWindow > 0 ? ((totalContentTokens / contextWindow) * 100).toFixed(1) : "N/A"}%`);
|
|
1022
1046
|
}
|
|
1047
|
+
/**
|
|
1048
|
+
* #309: enforce the provider's page/size ceilings across ALL PDFs in a request,
|
|
1049
|
+
* not just per-file. N files each just under the single-file limit can still
|
|
1050
|
+
* blow past it in aggregate (e.g. three 40-page PDFs → 120 pages for a
|
|
1051
|
+
* 100-page API).
|
|
1052
|
+
*
|
|
1053
|
+
* Shared by both PDF submission surfaces: `input.pdfFiles` (via
|
|
1054
|
+
* `processExplicitPdfFiles`) and `input.content` with `type: "pdf"` (via
|
|
1055
|
+
* `convertContentToProviderFormat`). The latter previously built its own
|
|
1056
|
+
* `pdfFiles` array and reached the provider without ever calling this guard,
|
|
1057
|
+
* so the limit was bypassable by moving the same payload to `input.content`.
|
|
1058
|
+
*/
|
|
1059
|
+
/**
|
|
1060
|
+
* Basename that strips BOTH separators regardless of host platform.
|
|
1061
|
+
*
|
|
1062
|
+
* `path.basename` only understands the host's separator, so on a POSIX server
|
|
1063
|
+
* a Windows-style filename (`C:\Users\alice\q3-merger.pdf`) comes back
|
|
1064
|
+
* completely unchanged — defeating the point of trimming it before it reaches
|
|
1065
|
+
* a log line, since caller-controlled paths can carry usernames and internal
|
|
1066
|
+
* directory structure.
|
|
1067
|
+
*/
|
|
1068
|
+
function safeBasename(filename) {
|
|
1069
|
+
const lastSeparator = Math.max(filename.lastIndexOf("/"), filename.lastIndexOf("\\"));
|
|
1070
|
+
const trimmed = lastSeparator === -1 ? filename : filename.slice(lastSeparator + 1);
|
|
1071
|
+
return trimmed || "<unnamed file>";
|
|
1072
|
+
}
|
|
1073
|
+
async function enforceAggregatePdfLimits(pdfFiles, provider, { trustSuppliedPageCounts }) {
|
|
1074
|
+
const aggregateConfig = PDFProcessor.getProviderConfig(provider);
|
|
1075
|
+
// Only an empty set is exempt. A single PDF must still be checked: on the
|
|
1076
|
+
// `input.content` path nothing else validates it (that path never goes
|
|
1077
|
+
// through FileDetector.detectAndProcess / PDFProcessor.process), so bailing
|
|
1078
|
+
// at length <= 1 let one 200-page document through untouched.
|
|
1079
|
+
if (!aggregateConfig || pdfFiles.length === 0) {
|
|
1080
|
+
return;
|
|
1081
|
+
}
|
|
1082
|
+
// Byte total is free to compute — enforce it BEFORE parsing anything, so an
|
|
1083
|
+
// oversized request is rejected without first spending parser CPU/memory on
|
|
1084
|
+
// every document in it.
|
|
1085
|
+
const totalMB = pdfFiles.reduce((sum, f) => sum + f.buffer.length, 0) / (1024 * 1024);
|
|
1086
|
+
if (totalMB > aggregateConfig.maxSizeMB) {
|
|
1087
|
+
throw ErrorFactory.pdfAggregateSizeLimitExceeded(pdfFiles.length, totalMB, aggregateConfig.maxSizeMB, provider);
|
|
1088
|
+
}
|
|
1089
|
+
// `trustSuppliedPageCounts` is the difference between the two surfaces.
|
|
1090
|
+
// On `input.pdfFiles` the count comes from FileDetector's own detection, so
|
|
1091
|
+
// it is authoritative. On `input.content` it is `metadata.pages` — plain
|
|
1092
|
+
// caller input — and trusting it lets a request declare `pages: 1` for each
|
|
1093
|
+
// of three 40-page PDFs and sail past the ceiling. There, the count is
|
|
1094
|
+
// always re-derived from the bytes and the supplied value is ignored.
|
|
1095
|
+
const pageCounts = await Promise.all(pdfFiles.map(async (f) => trustSuppliedPageCounts && typeof f.pageCount === "number"
|
|
1096
|
+
? f.pageCount
|
|
1097
|
+
: await PDFProcessor.resolvePageCount(f.buffer)));
|
|
1098
|
+
// Filenames are caller-controlled and may be full paths carrying
|
|
1099
|
+
// PII/internal directory segments — surface only the basename, stripping
|
|
1100
|
+
// both separators so a Windows path is trimmed on a POSIX host too.
|
|
1101
|
+
const unknownFileNames = pdfFiles
|
|
1102
|
+
.filter((_, i) => typeof pageCounts[i] !== "number")
|
|
1103
|
+
.map((f) => (f.filename ? safeBasename(f.filename) : "<unnamed file>"));
|
|
1104
|
+
const totalPages = pageCounts.reduce((sum, p) => sum + (typeof p === "number" ? p : 0), 0);
|
|
1105
|
+
if (unknownFileNames.length > 0) {
|
|
1106
|
+
if (!trustSuppliedPageCounts) {
|
|
1107
|
+
// Untrusted surface: an unreadable count is indistinguishable from an
|
|
1108
|
+
// evasion attempt, and counting it as zero is precisely the hole. Fail
|
|
1109
|
+
// closed rather than admit an unverifiable document.
|
|
1110
|
+
throw ErrorFactory.pdfPageCountUnverifiable(unknownFileNames, provider);
|
|
1111
|
+
}
|
|
1112
|
+
// Trusted surface: detection already vetted these, so one unreadable
|
|
1113
|
+
// count must not fail an otherwise valid request — but it must not be
|
|
1114
|
+
// silent either, since the known sum may undercount the true total.
|
|
1115
|
+
logger.warn(`[PDF] Aggregate page-limit check across ${pdfFiles.length} PDFs could only be ` +
|
|
1116
|
+
`partially verified: ${unknownFileNames.length} file(s) have an unknown ` +
|
|
1117
|
+
`page count (${unknownFileNames.join(", ")}), so the known total (${totalPages}) may ` +
|
|
1118
|
+
`undercount the true combined page count.`);
|
|
1119
|
+
}
|
|
1120
|
+
if (totalPages > aggregateConfig.maxPages) {
|
|
1121
|
+
throw ErrorFactory.pdfAggregatePageLimitExceeded(pdfFiles.length, totalPages, aggregateConfig.maxPages, provider);
|
|
1122
|
+
}
|
|
1123
|
+
}
|
|
1023
1124
|
/**
|
|
1024
1125
|
* Process explicit PDF files and return structured PDF entries for multimodal processing.
|
|
1025
1126
|
*/
|
|
@@ -1050,6 +1151,10 @@ async function processExplicitPdfFiles(options, maxSize, provider) {
|
|
|
1050
1151
|
// #260: carry the per-page canvas-pixel ceiling so the caller can
|
|
1051
1152
|
// raise (or lower) the memory guard for the image-fallback render.
|
|
1052
1153
|
maxCanvasPixels: options.pdfOptions?.maxCanvasPixels,
|
|
1154
|
+
// #297: render scale / page ceiling, so the lowered default is
|
|
1155
|
+
// actually reachable and callers can trade sharpness for memory.
|
|
1156
|
+
scale: options.pdfOptions?.scale,
|
|
1157
|
+
maxPages: options.pdfOptions?.maxPages,
|
|
1053
1158
|
});
|
|
1054
1159
|
logger.info(`[PDF] ✅ Queued for multimodal: ${filename} (${result.metadata?.estimatedPages ?? "unknown"} pages)`);
|
|
1055
1160
|
}
|
|
@@ -1059,41 +1164,10 @@ async function processExplicitPdfFiles(options, maxSize, provider) {
|
|
|
1059
1164
|
throw error;
|
|
1060
1165
|
}
|
|
1061
1166
|
}
|
|
1062
|
-
//
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
if (aggregateConfig && pdfFiles.length > 1) {
|
|
1067
|
-
// A null pageCount (accurate count unavailable — see
|
|
1068
|
-
// PDFProcessor.getAccuratePageCount) is treated as 0 in the sum below,
|
|
1069
|
-
// which can undercount the aggregate and let a combined request over
|
|
1070
|
-
// the provider's page limit slip through silently. Enforcement still
|
|
1071
|
-
// runs against the known sum — a PDF with an unknown count must not
|
|
1072
|
-
// fail the request outright — but the gap itself must not be silent.
|
|
1073
|
-
const unknownPageCountFiles = pdfFiles.filter((f) => f.pageCount === null || f.pageCount === undefined);
|
|
1074
|
-
const totalPages = pdfFiles.reduce((sum, f) => sum + (f.pageCount ?? 0), 0);
|
|
1075
|
-
const totalMB = pdfFiles.reduce((sum, f) => sum + f.buffer.length, 0) / (1024 * 1024);
|
|
1076
|
-
if (unknownPageCountFiles.length > 0) {
|
|
1077
|
-
// Filenames are caller-controlled and may be full paths carrying
|
|
1078
|
-
// PII/internal directory segments — log only the basename.
|
|
1079
|
-
const unknownFileNames = unknownPageCountFiles
|
|
1080
|
-
.map((f) => (f.filename ? basename(f.filename) : "<unnamed file>"))
|
|
1081
|
-
.join(", ");
|
|
1082
|
-
logger.warn(`[PDF] Aggregate page-limit check across ${pdfFiles.length} PDFs could only be ` +
|
|
1083
|
-
`partially verified: ${unknownPageCountFiles.length} file(s) have an unknown ` +
|
|
1084
|
-
`page count (${unknownFileNames}), so the known total (${totalPages}) may ` +
|
|
1085
|
-
`undercount the true combined page count.`);
|
|
1086
|
-
}
|
|
1087
|
-
if (totalPages > aggregateConfig.maxPages) {
|
|
1088
|
-
throw new Error(`[PDF] Combined page count across ${pdfFiles.length} PDFs (${totalPages}) exceeds the ` +
|
|
1089
|
-
`${aggregateConfig.maxPages}-page limit for ${provider}. ` +
|
|
1090
|
-
`Split the request or reduce the number of PDFs.`);
|
|
1091
|
-
}
|
|
1092
|
-
if (totalMB > aggregateConfig.maxSizeMB) {
|
|
1093
|
-
throw new Error(`[PDF] Combined size across ${pdfFiles.length} PDFs (${totalMB.toFixed(2)}MB) exceeds the ` +
|
|
1094
|
-
`${aggregateConfig.maxSizeMB}MB limit for ${provider}.`);
|
|
1095
|
-
}
|
|
1096
|
-
}
|
|
1167
|
+
// Counts here come from FileDetector's detection, so they are authoritative.
|
|
1168
|
+
await enforceAggregatePdfLimits(pdfFiles, provider, {
|
|
1169
|
+
trustSuppliedPageCounts: true,
|
|
1170
|
+
});
|
|
1097
1171
|
return pdfFiles;
|
|
1098
1172
|
}
|
|
1099
1173
|
/**
|
|
@@ -1389,7 +1463,15 @@ async function convertContentToProviderFormat(content, provider, _model, pdfOpti
|
|
|
1389
1463
|
// guard as the `input.pdfFiles` path (see `processExplicitPdfFiles`).
|
|
1390
1464
|
password: pdfOptions?.password,
|
|
1391
1465
|
maxCanvasPixels: pdfOptions?.maxCanvasPixels,
|
|
1466
|
+
scale: pdfOptions?.scale,
|
|
1467
|
+
maxPages: pdfOptions?.maxPages,
|
|
1392
1468
|
}));
|
|
1469
|
+
// #309: same aggregate ceiling as `input.pdfFiles`. Without this, moving an
|
|
1470
|
+
// over-limit payload from `input.pdfFiles` to `input.content` skipped the
|
|
1471
|
+
// check entirely and the request went straight to the provider.
|
|
1472
|
+
await enforceAggregatePdfLimits(pdfFiles, provider, {
|
|
1473
|
+
trustSuppliedPageCounts: false,
|
|
1474
|
+
});
|
|
1393
1475
|
return await convertMultimodalToProviderFormat(text, images, pdfFiles, provider, _model);
|
|
1394
1476
|
}
|
|
1395
1477
|
/**
|
|
@@ -1734,7 +1816,10 @@ async function convertSimpleImagesToProviderFormat(text, images, provider, _mode
|
|
|
1734
1816
|
/**
|
|
1735
1817
|
* Convert multimodal content (images + PDFs) to provider format
|
|
1736
1818
|
*/
|
|
1737
|
-
async function convertMultimodalToProviderFormat(text, images,
|
|
1819
|
+
async function convertMultimodalToProviderFormat(text, images,
|
|
1820
|
+
// The canonical entry shape (#309) rather than a fourth copy of it inline —
|
|
1821
|
+
// which is what let the render knobs stop short of this function.
|
|
1822
|
+
pdfFiles, provider, model) {
|
|
1738
1823
|
const content = [
|
|
1739
1824
|
{ type: "text", text },
|
|
1740
1825
|
];
|
|
@@ -1768,14 +1853,41 @@ async function convertMultimodalToProviderFormat(text, images, pdfFiles, provide
|
|
|
1768
1853
|
logger.info(`[PDF→Image] Provider ${provider} doesn't support native PDF. Converting ${pdfFiles.length} PDF(s) to images...`);
|
|
1769
1854
|
for (const pdf of pdfFiles) {
|
|
1770
1855
|
try {
|
|
1856
|
+
const effectiveMaxPages = pdf.maxPages ?? PDF_LIMITS.DEFAULT_MAX_PAGES;
|
|
1771
1857
|
const conversionResult = await PDFImageConverter.convertToImages(pdf.buffer, {
|
|
1772
|
-
|
|
1773
|
-
|
|
1858
|
+
// #297: this is the only PDF→image call the product actually makes,
|
|
1859
|
+
// and it used to hardcode scale 2.0 — silently overriding the
|
|
1860
|
+
// lowered PDF_LIMITS.DEFAULT_SCALE and keeping the memory cost the
|
|
1861
|
+
// issue reports (a 100-page render at 2.0 is ~776MB; 1.5 is ~44%
|
|
1862
|
+
// fewer pixels per page). Callers can raise it back per request.
|
|
1863
|
+
scale: pdf.scale ?? PDF_LIMITS.DEFAULT_SCALE,
|
|
1864
|
+
// Page ceiling guards token overflow; also now caller-adjustable
|
|
1865
|
+
// rather than a constant nothing could reach.
|
|
1866
|
+
maxPages: effectiveMaxPages,
|
|
1774
1867
|
...(pdf.password ? { password: pdf.password } : {}), // #258
|
|
1775
1868
|
...(pdf.maxCanvasPixels
|
|
1776
1869
|
? { maxCanvasPixels: pdf.maxCanvasPixels }
|
|
1777
1870
|
: {}), // #260
|
|
1778
1871
|
});
|
|
1872
|
+
// The renderer stops at maxPages, so a longer document is silently
|
|
1873
|
+
// truncated — say so rather than letting the model answer from a
|
|
1874
|
+
// partial document as though it had the whole thing.
|
|
1875
|
+
//
|
|
1876
|
+
// Keyed on the cap being reached, not on pdf.pageCount: that field is
|
|
1877
|
+
// null whenever `input.content` omits `metadata.pages`, which is the
|
|
1878
|
+
// common case, so a page-count comparison would simply never fire
|
|
1879
|
+
// there. Reaching the cap is also unambiguous — a short count caused by
|
|
1880
|
+
// per-page render failures (#294 isolates those into `errors`) would
|
|
1881
|
+
// otherwise be misreported as a maxPages truncation.
|
|
1882
|
+
if (conversionResult.pageCount >= effectiveMaxPages) {
|
|
1883
|
+
logger.warn(`[PDF→Image] ${safeBasename(pdf.filename)} hit the ${effectiveMaxPages}-page ` +
|
|
1884
|
+
`conversion limit. Any pages beyond that were not sent — the model may be ` +
|
|
1885
|
+
`answering from a partial document. Raise pdfOptions.maxPages or split the file.`);
|
|
1886
|
+
}
|
|
1887
|
+
if (conversionResult.errors && conversionResult.errors.length > 0) {
|
|
1888
|
+
logger.warn(`[PDF→Image] ${safeBasename(pdf.filename)}: ${conversionResult.errors.length} page(s) ` +
|
|
1889
|
+
`failed to render and were omitted (page ${conversionResult.errors.map((e) => e.page).join(", ")}).`);
|
|
1890
|
+
}
|
|
1779
1891
|
logger.info(`[PDF→Image] ✅ Converted ${pdf.filename}: ${conversionResult.pageCount} page(s) → images`);
|
|
1780
1892
|
// Add each page as an ImagePart (raw base64, not data: URI — see SSRF note above)
|
|
1781
1893
|
conversionResult.images.forEach((base64Image, pageIndex) => {
|
|
@@ -58,6 +58,8 @@ export declare function buildMultimodalOptions(options: StreamOptions, providerN
|
|
|
58
58
|
pdfOptions: {
|
|
59
59
|
password?: string;
|
|
60
60
|
maxCanvasPixels?: number;
|
|
61
|
+
scale?: number;
|
|
62
|
+
maxPages?: number;
|
|
61
63
|
} | undefined;
|
|
62
64
|
systemPrompt: string | undefined;
|
|
63
65
|
conversationHistory: import("../types/conversation.js").ChatMessage[] | undefined;
|
|
@@ -18,6 +18,17 @@ export declare class PDFProcessor {
|
|
|
18
18
|
*/
|
|
19
19
|
static supportsNativePDF(provider: string): boolean;
|
|
20
20
|
static getProviderConfig(provider: string): PDFProviderConfig | null;
|
|
21
|
+
/**
|
|
22
|
+
* Best-effort page count for a PDF whose count the caller did not supply
|
|
23
|
+
* (#309). Mirrors what `process()` derives for the `input.pdfFiles` path:
|
|
24
|
+
* the accurate pdfjs count when the document parses, otherwise the header
|
|
25
|
+
* regex estimate. Returns null when neither can determine a count.
|
|
26
|
+
*
|
|
27
|
+
* Exists so the aggregate page-limit guard can enforce against PDFs handed
|
|
28
|
+
* in via `input.content`, where `metadata.pages` is optional and routinely
|
|
29
|
+
* omitted — without it, an absent count silently sums as zero.
|
|
30
|
+
*/
|
|
31
|
+
static resolvePageCount(buffer: Buffer): Promise<number | null>;
|
|
21
32
|
private static isValidPDF;
|
|
22
33
|
private static extractBasicMetadata;
|
|
23
34
|
/**
|
|
@@ -212,6 +212,23 @@ export class PDFProcessor {
|
|
|
212
212
|
static getProviderConfig(provider) {
|
|
213
213
|
return PDF_PROVIDER_CONFIGS[provider] || null;
|
|
214
214
|
}
|
|
215
|
+
/**
|
|
216
|
+
* Best-effort page count for a PDF whose count the caller did not supply
|
|
217
|
+
* (#309). Mirrors what `process()` derives for the `input.pdfFiles` path:
|
|
218
|
+
* the accurate pdfjs count when the document parses, otherwise the header
|
|
219
|
+
* regex estimate. Returns null when neither can determine a count.
|
|
220
|
+
*
|
|
221
|
+
* Exists so the aggregate page-limit guard can enforce against PDFs handed
|
|
222
|
+
* in via `input.content`, where `metadata.pages` is optional and routinely
|
|
223
|
+
* omitted — without it, an absent count silently sums as zero.
|
|
224
|
+
*/
|
|
225
|
+
static async resolvePageCount(buffer) {
|
|
226
|
+
const accurate = await PDFProcessor.getAccuratePageCount(buffer);
|
|
227
|
+
if (accurate !== null) {
|
|
228
|
+
return accurate;
|
|
229
|
+
}
|
|
230
|
+
return PDFProcessor.extractBasicMetadata(buffer).estimatedPages;
|
|
231
|
+
}
|
|
215
232
|
static isValidPDF(buffer) {
|
|
216
233
|
if (buffer.length < 5) {
|
|
217
234
|
return false;
|
|
@@ -360,6 +377,7 @@ export class PDFProcessor {
|
|
|
360
377
|
format,
|
|
361
378
|
scale,
|
|
362
379
|
maxCanvasPixels,
|
|
380
|
+
maxPages,
|
|
363
381
|
});
|
|
364
382
|
logger.debug("[PDF→Image] ✅ PDF validation passed", {
|
|
365
383
|
bufferSize: pdfBuffer.length,
|
|
@@ -518,6 +536,14 @@ export class PDFProcessor {
|
|
|
518
536
|
if (!Number.isFinite(opts.maxCanvasPixels) || opts.maxCanvasPixels <= 0) {
|
|
519
537
|
throw new Error(`Invalid maxCanvasPixels: ${opts.maxCanvasPixels}. Must be a finite number greater than 0.`);
|
|
520
538
|
}
|
|
539
|
+
// #297: maxPages became caller-controlled, and an unvalidated 0/-1/NaN
|
|
540
|
+
// silently converts nothing, surfacing later as a misleading
|
|
541
|
+
// "PDF has 0 pages" from deep inside the renderer. Reject it here where
|
|
542
|
+
// the message can still name the offending option.
|
|
543
|
+
if (opts.maxPages !== undefined &&
|
|
544
|
+
(!Number.isInteger(opts.maxPages) || opts.maxPages < 1)) {
|
|
545
|
+
throw new Error(`Invalid maxPages: ${opts.maxPages}. Must be a whole number of at least 1.`);
|
|
546
|
+
}
|
|
521
547
|
if (!pdfBuffer || pdfBuffer.length < 5) {
|
|
522
548
|
throw new Error("Invalid PDF: Buffer is too small or empty. " +
|
|
523
549
|
"A valid PDF must be at least 5 bytes (PDF header).");
|
|
@@ -553,6 +579,7 @@ export class PDFProcessor {
|
|
|
553
579
|
format,
|
|
554
580
|
scale,
|
|
555
581
|
maxCanvasPixels,
|
|
582
|
+
maxPages,
|
|
556
583
|
});
|
|
557
584
|
const pdfToImgModule = await import("pdf-to-img");
|
|
558
585
|
const pdf = pdfToImgModule.pdf;
|
|
@@ -44,7 +44,7 @@
|
|
|
44
44
|
* ```
|
|
45
45
|
*/
|
|
46
46
|
import { BaseFileProcessor } from "../base/BaseFileProcessor.js";
|
|
47
|
-
import type { FileInfo, ProcessedVideo, ProcessorFileProcessingResult, ProcessOptions } from "../../types/index.js";
|
|
47
|
+
import type { FileInfo, ProcessedVideo, ProcessorFileProcessingResult, ProcessOptions, VideoProcessorOptions } from "../../types/index.js";
|
|
48
48
|
/**
|
|
49
49
|
* Narrow a loaded `fluent-ffmpeg` export to the shape this file actually uses:
|
|
50
50
|
* a callable carrying the `ffprobe` and `setFfmpegPath` statics.
|
|
@@ -116,7 +116,7 @@ export declare class VideoProcessor extends BaseFileProcessor<ProcessedVideo> {
|
|
|
116
116
|
* @param options - Optional processing options
|
|
117
117
|
* @returns Processing result with extracted video data or error
|
|
118
118
|
*/
|
|
119
|
-
processFile(fileInfo: FileInfo, options?: ProcessOptions): Promise<ProcessorFileProcessingResult<ProcessedVideo>>;
|
|
119
|
+
processFile(fileInfo: FileInfo, options?: ProcessOptions & VideoProcessorOptions): Promise<ProcessorFileProcessingResult<ProcessedVideo>>;
|
|
120
120
|
/**
|
|
121
121
|
* Probe a video file to extract metadata using ffprobe.
|
|
122
122
|
*
|
|
@@ -137,6 +137,11 @@ export declare class VideoProcessor extends BaseFileProcessor<ProcessedVideo> {
|
|
|
137
137
|
* @returns Structured video metadata
|
|
138
138
|
*/
|
|
139
139
|
private buildMetadata;
|
|
140
|
+
/**
|
|
141
|
+
* Clamp a caller-supplied frame quality into sharp's valid 1-100 range,
|
|
142
|
+
* falling back to the default when absent or non-numeric (#478).
|
|
143
|
+
*/
|
|
144
|
+
private static resolveFrameQuality;
|
|
140
145
|
/**
|
|
141
146
|
* Extract keyframes from a video at calculated intervals.
|
|
142
147
|
*
|
|
@@ -153,10 +158,15 @@ export declare class VideoProcessor extends BaseFileProcessor<ProcessedVideo> {
|
|
|
153
158
|
* The interval is adaptive: if the tier interval would exceed MAX_FRAMES,
|
|
154
159
|
* the interval widens to duration/MAX_FRAMES for full-video coverage.
|
|
155
160
|
*
|
|
161
|
+
* A caller-supplied `options.frames` overrides the tier schedule entirely:
|
|
162
|
+
* that many frames are spread evenly across the clip, still capped at
|
|
163
|
+
* MAX_FRAMES. `options.quality` and `options.format` reach the encoder (#478).
|
|
164
|
+
*
|
|
156
165
|
* @param videoPath - Path to the video file
|
|
157
166
|
* @param tempDir - Temp directory for frame output
|
|
158
167
|
* @param durationSec - Video duration in seconds
|
|
159
|
-
* @
|
|
168
|
+
* @param options - Caller frame budget / encoder settings
|
|
169
|
+
* @returns Array of encoded frame buffers (JPEG unless png was requested)
|
|
160
170
|
*/
|
|
161
171
|
private extractKeyframes;
|
|
162
172
|
/**
|
|
@@ -313,7 +313,10 @@ export class VideoProcessor extends BaseFileProcessor {
|
|
|
313
313
|
* @param options - Optional processing options
|
|
314
314
|
* @returns Processing result with extracted video data or error
|
|
315
315
|
*/
|
|
316
|
-
async processFile(fileInfo,
|
|
316
|
+
async processFile(fileInfo,
|
|
317
|
+
// #478: widened with the keyframe knobs so `--video-frames`/`-quality`/
|
|
318
|
+
// `-format` can reach the encoder instead of being silently discarded.
|
|
319
|
+
options) {
|
|
317
320
|
const filename = this.getFilename(fileInfo);
|
|
318
321
|
const sizeBytes = fileInfo.size || fileInfo.buffer?.length || 0;
|
|
319
322
|
return withSpan({
|
|
@@ -438,7 +441,7 @@ export class VideoProcessor extends BaseFileProcessor {
|
|
|
438
441
|
// Step 5: Extract keyframes
|
|
439
442
|
let keyframes = [];
|
|
440
443
|
try {
|
|
441
|
-
keyframes = await this.extractKeyframes(tempVideoPath, tempDir, metadata.duration);
|
|
444
|
+
keyframes = await this.extractKeyframes(tempVideoPath, tempDir, metadata.duration, options);
|
|
442
445
|
}
|
|
443
446
|
catch {
|
|
444
447
|
// Non-fatal: continue without keyframes if extraction fails
|
|
@@ -642,6 +645,16 @@ export class VideoProcessor extends BaseFileProcessor {
|
|
|
642
645
|
// ===========================================================================
|
|
643
646
|
// KEYFRAME EXTRACTION
|
|
644
647
|
// ===========================================================================
|
|
648
|
+
/**
|
|
649
|
+
* Clamp a caller-supplied frame quality into sharp's valid 1-100 range,
|
|
650
|
+
* falling back to the default when absent or non-numeric (#478).
|
|
651
|
+
*/
|
|
652
|
+
static resolveFrameQuality(quality) {
|
|
653
|
+
if (typeof quality !== "number" || !Number.isFinite(quality)) {
|
|
654
|
+
return VIDEO_CONFIG.FRAME_JPEG_QUALITY;
|
|
655
|
+
}
|
|
656
|
+
return Math.min(100, Math.max(1, Math.round(quality)));
|
|
657
|
+
}
|
|
645
658
|
/**
|
|
646
659
|
* Extract keyframes from a video at calculated intervals.
|
|
647
660
|
*
|
|
@@ -658,20 +671,45 @@ export class VideoProcessor extends BaseFileProcessor {
|
|
|
658
671
|
* The interval is adaptive: if the tier interval would exceed MAX_FRAMES,
|
|
659
672
|
* the interval widens to duration/MAX_FRAMES for full-video coverage.
|
|
660
673
|
*
|
|
674
|
+
* A caller-supplied `options.frames` overrides the tier schedule entirely:
|
|
675
|
+
* that many frames are spread evenly across the clip, still capped at
|
|
676
|
+
* MAX_FRAMES. `options.quality` and `options.format` reach the encoder (#478).
|
|
677
|
+
*
|
|
661
678
|
* @param videoPath - Path to the video file
|
|
662
679
|
* @param tempDir - Temp directory for frame output
|
|
663
680
|
* @param durationSec - Video duration in seconds
|
|
664
|
-
* @
|
|
681
|
+
* @param options - Caller frame budget / encoder settings
|
|
682
|
+
* @returns Array of encoded frame buffers (JPEG unless png was requested)
|
|
665
683
|
*/
|
|
666
|
-
async extractKeyframes(videoPath, tempDir, durationSec) {
|
|
684
|
+
async extractKeyframes(videoPath, tempDir, durationSec, options) {
|
|
667
685
|
if (durationSec <= 0) {
|
|
668
686
|
return [];
|
|
669
687
|
}
|
|
670
|
-
//
|
|
671
|
-
|
|
688
|
+
// #478: honor the caller's frame budget, still bounded by MAX_FRAMES so a
|
|
689
|
+
// CLI flag can lower the cost but never raise it past the processor's own
|
|
690
|
+
// ceiling. A non-positive/non-finite request falls back to the default.
|
|
691
|
+
const requestedFrames = options?.frames;
|
|
692
|
+
const hasExplicitBudget = typeof requestedFrames === "number" &&
|
|
693
|
+
Number.isFinite(requestedFrames) &&
|
|
694
|
+
requestedFrames > 0;
|
|
695
|
+
const frameBudget = hasExplicitBudget
|
|
696
|
+
? Math.min(Math.floor(requestedFrames), VIDEO_CONFIG.MAX_FRAMES)
|
|
697
|
+
: VIDEO_CONFIG.MAX_FRAMES;
|
|
698
|
+
// Determine extraction interval based on duration. When the caller asked
|
|
699
|
+
// for a specific frame count, spread that many evenly across the whole
|
|
700
|
+
// video instead of using the duration tier — otherwise a short interval
|
|
701
|
+
// would hit the budget early and only cover the opening seconds.
|
|
702
|
+
//
|
|
703
|
+
// Keyed on whether a budget was REQUESTED, not on whether it happens to be
|
|
704
|
+
// below MAX_FRAMES: asking for exactly MAX_FRAMES is still an explicit
|
|
705
|
+
// request and must produce that many frames, not silently fall back to the
|
|
706
|
+
// tier schedule (which yields far fewer on a short clip).
|
|
707
|
+
const intervalSec = hasExplicitBudget
|
|
708
|
+
? Math.max(durationSec / frameBudget, Number.EPSILON)
|
|
709
|
+
: this.getFrameInterval(durationSec);
|
|
672
710
|
// Calculate timestamps to extract
|
|
673
711
|
const timestamps = [];
|
|
674
|
-
for (let t = 0; t < durationSec && timestamps.length <
|
|
712
|
+
for (let t = 0; t < durationSec && timestamps.length < frameBudget; t += intervalSec) {
|
|
675
713
|
timestamps.push(t);
|
|
676
714
|
}
|
|
677
715
|
if (timestamps.length === 0) {
|
|
@@ -691,13 +729,16 @@ export class VideoProcessor extends BaseFileProcessor {
|
|
|
691
729
|
const rawFrame = await fs.readFile(framePath);
|
|
692
730
|
// Resize to fit within max dimension while preserving aspect ratio
|
|
693
731
|
const sharp = (await import("sharp")).default;
|
|
694
|
-
const
|
|
695
|
-
.resize(VIDEO_CONFIG.FRAME_MAX_DIMENSION, VIDEO_CONFIG.FRAME_MAX_DIMENSION, {
|
|
732
|
+
const pipeline = sharp(rawFrame).resize(VIDEO_CONFIG.FRAME_MAX_DIMENSION, VIDEO_CONFIG.FRAME_MAX_DIMENSION, {
|
|
696
733
|
fit: "inside",
|
|
697
734
|
withoutEnlargement: true,
|
|
698
|
-
})
|
|
699
|
-
|
|
700
|
-
|
|
735
|
+
});
|
|
736
|
+
// #478: `--video-quality` / `--video-format` were accepted by the CLI
|
|
737
|
+
// and then dropped on the floor; both now reach the encoder.
|
|
738
|
+
const quality = VideoProcessor.resolveFrameQuality(options?.quality);
|
|
739
|
+
const resized = await (options?.format === "png"
|
|
740
|
+
? pipeline.png({ quality })
|
|
741
|
+
: pipeline.jpeg({ quality })).toBuffer();
|
|
701
742
|
keyframes.push(resized);
|
|
702
743
|
}
|
|
703
744
|
catch {
|
|
@@ -87,7 +87,14 @@ export class SkillsManager {
|
|
|
87
87
|
// name-find would resolve non-deterministically to the stale deprecated one
|
|
88
88
|
// across store backends. Fall back to any match only when no active skill
|
|
89
89
|
// carries the name.
|
|
90
|
-
|
|
90
|
+
//
|
|
91
|
+
// Matched case-insensitively to agree with assertNameAvailable, which
|
|
92
|
+
// enforces uniqueness that way: names differing only in case cannot
|
|
93
|
+
// coexist, so a case-sensitive lookup could only fail to find a skill that
|
|
94
|
+
// is definitively there (`get("DEPLOY")` returned null for "deploy").
|
|
95
|
+
const wanted = idOrName.toLowerCase();
|
|
96
|
+
const entry = index.find((item) => item.name.toLowerCase() === wanted &&
|
|
97
|
+
(item.status ?? "active") === "active") ?? index.find((item) => item.name.toLowerCase() === wanted);
|
|
91
98
|
return entry ? this.store.get(entry.id) : null;
|
|
92
99
|
}
|
|
93
100
|
/**
|
|
@@ -237,11 +244,23 @@ export class SkillsManager {
|
|
|
237
244
|
}
|
|
238
245
|
async assertNameAvailable(name, excludeId) {
|
|
239
246
|
const index = await this.getIndex(true);
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
247
|
+
// Deliberately NOT filtered to active (#1139). Soft-delete only flips
|
|
248
|
+
// status to "deprecated" — the entry stays in the index, so allowing a new
|
|
249
|
+
// skill to take the name left two entries sharing it. Every name-based
|
|
250
|
+
// lookup (CLI `skills show/delete <name>`, the skill_update/skill_delete
|
|
251
|
+
// tools, the `:id`-or-name REST routes) then had to guess which one the
|
|
252
|
+
// caller meant, and iteration order differs across store backends.
|
|
253
|
+
//
|
|
254
|
+
// A deprecated skill's name therefore stays reserved. Reusing it requires
|
|
255
|
+
// hard-deleting the old skill first, which is the explicit choice the
|
|
256
|
+
// ambiguity demands.
|
|
257
|
+
const clash = index.find((item) => item.id !== excludeId && item.name.toLowerCase() === name.toLowerCase());
|
|
243
258
|
if (clash) {
|
|
244
|
-
|
|
259
|
+
const suffix = (clash.status ?? "active") === "active"
|
|
260
|
+
? ""
|
|
261
|
+
: ` (that name belongs to a deprecated skill, id ${clash.id}; ` +
|
|
262
|
+
`remove it before reusing the name)`;
|
|
263
|
+
throw new Error(`A skill named "${name}" already exists${suffix}`);
|
|
245
264
|
}
|
|
246
265
|
}
|
|
247
266
|
}
|
package/dist/types/file.d.ts
CHANGED
|
@@ -302,6 +302,21 @@ export type AudioProcessorOptions = {
|
|
|
302
302
|
/** Maximum file size in megabytes */
|
|
303
303
|
maxSizeMB?: number;
|
|
304
304
|
};
|
|
305
|
+
/**
|
|
306
|
+
* Keyframe-extraction knobs for an attached video (#478).
|
|
307
|
+
*
|
|
308
|
+
* These back the `--video-frames` / `--video-quality` / `--video-format` CLI
|
|
309
|
+
* flags and `GenerateOptions.videoOptions`. Each is clamped to the processor's
|
|
310
|
+
* own ceiling — a caller cannot raise `frames` above VIDEO_CONFIG.MAX_FRAMES.
|
|
311
|
+
*/
|
|
312
|
+
export type VideoProcessorOptions = {
|
|
313
|
+
/** Max keyframes to extract. Clamped to the processor's MAX_FRAMES ceiling. */
|
|
314
|
+
frames?: number;
|
|
315
|
+
/** Encoder quality 1-100 for the extracted frames. */
|
|
316
|
+
quality?: number;
|
|
317
|
+
/** Frame encoding. Defaults to jpeg. */
|
|
318
|
+
format?: "jpeg" | "png";
|
|
319
|
+
};
|
|
305
320
|
/**
|
|
306
321
|
* Office processor options for Word, PowerPoint, and Excel documents
|
|
307
322
|
*
|
|
@@ -366,6 +381,7 @@ export type FileDetectorOptions = {
|
|
|
366
381
|
audioOptions?: AudioProcessorOptions;
|
|
367
382
|
csvOptions?: CSVProcessorOptions;
|
|
368
383
|
officeOptions?: OfficeProcessorOptions;
|
|
384
|
+
videoOptions?: VideoProcessorOptions;
|
|
369
385
|
confidenceThreshold?: number;
|
|
370
386
|
provider?: string;
|
|
371
387
|
/** Maximum number of retry attempts for network requests (default: 3) */
|
|
@@ -438,6 +454,31 @@ export type PDFImagePage = {
|
|
|
438
454
|
/** Populated when this page failed to render (#294). */
|
|
439
455
|
error?: string;
|
|
440
456
|
};
|
|
457
|
+
/**
|
|
458
|
+
* A single PDF queued for multimodal message building, normalised from either
|
|
459
|
+
* submission surface — `input.pdfFiles` or `input.content` with `type: "pdf"`
|
|
460
|
+
* — so both can share the aggregate page/size guard (#309).
|
|
461
|
+
*/
|
|
462
|
+
export type MultimodalPdfEntry = {
|
|
463
|
+
/** Raw PDF bytes. */
|
|
464
|
+
buffer: Buffer;
|
|
465
|
+
/** Display name; may be a full path, so log only its basename. */
|
|
466
|
+
filename: string;
|
|
467
|
+
/**
|
|
468
|
+
* Page count when known. Null/undefined on the `input.content` path whenever
|
|
469
|
+
* the caller omitted `metadata.pages`; the aggregate guard resolves those
|
|
470
|
+
* from `buffer` rather than treating them as zero.
|
|
471
|
+
*/
|
|
472
|
+
pageCount?: number | null;
|
|
473
|
+
/** Password for an encrypted PDF (#258). */
|
|
474
|
+
password?: string;
|
|
475
|
+
/** Per-page pixel ceiling for the image fallback (#260). */
|
|
476
|
+
maxCanvasPixels?: number;
|
|
477
|
+
/** Render scale for the image fallback (#297). */
|
|
478
|
+
scale?: number;
|
|
479
|
+
/** Max pages converted by the image fallback (#297). */
|
|
480
|
+
maxPages?: number;
|
|
481
|
+
};
|
|
441
482
|
/** Result of PDF to image conversion. */
|
|
442
483
|
export type PDFImageConversionResult = {
|
|
443
484
|
/** Array of base64-encoded PNG images (one per successfully converted page) */
|
package/dist/types/generate.d.ts
CHANGED
|
@@ -133,11 +133,26 @@ export type GenerateOptions = {
|
|
|
133
133
|
password?: string;
|
|
134
134
|
/** Max rendered-canvas pixels per page (#260 memory guard); oversized pages auto-downscale. */
|
|
135
135
|
maxCanvasPixels?: number;
|
|
136
|
+
/**
|
|
137
|
+
* Render scale for the image fallback used by providers without native PDF
|
|
138
|
+
* support (#297). Higher is sharper but costs roughly the square in memory
|
|
139
|
+
* and tokens. Range 0.1-10; defaults to PDF_LIMITS.DEFAULT_SCALE (1.5).
|
|
140
|
+
*/
|
|
141
|
+
scale?: number;
|
|
142
|
+
/**
|
|
143
|
+
* Max pages converted by the image fallback (#297). Pages beyond this are
|
|
144
|
+
* not sent to the model at all. Defaults to PDF_LIMITS.DEFAULT_MAX_PAGES (20).
|
|
145
|
+
*/
|
|
146
|
+
maxPages?: number;
|
|
136
147
|
};
|
|
137
148
|
videoOptions?: {
|
|
149
|
+
/** Frames to extract. Unset lets VideoProcessor pick from the clip's duration; clamped to 100. */
|
|
138
150
|
frames?: number;
|
|
151
|
+
/** Frame encoder quality, clamped to 1-100. Default 80. */
|
|
139
152
|
quality?: number;
|
|
153
|
+
/** Frame encoding. Default jpeg. */
|
|
140
154
|
format?: "jpeg" | "png";
|
|
155
|
+
/** Not implemented yet (#433) — warns rather than silently doing nothing. */
|
|
141
156
|
transcribeAudio?: boolean;
|
|
142
157
|
};
|
|
143
158
|
/**
|
|
@@ -1261,6 +1276,10 @@ export type TextGenerationOptions = {
|
|
|
1261
1276
|
password?: string;
|
|
1262
1277
|
/** Max rendered-canvas pixels per page (#260 memory guard); oversized pages auto-downscale. */
|
|
1263
1278
|
maxCanvasPixels?: number;
|
|
1279
|
+
/** Render scale for the image fallback (#297); defaults to PDF_LIMITS.DEFAULT_SCALE. */
|
|
1280
|
+
scale?: number;
|
|
1281
|
+
/** Max pages converted by the image fallback (#297); defaults to PDF_LIMITS.DEFAULT_MAX_PAGES. */
|
|
1282
|
+
maxPages?: number;
|
|
1264
1283
|
};
|
|
1265
1284
|
enableSummarization?: boolean;
|
|
1266
1285
|
/**
|