@juspay/neurolink 12.19.2 → 12.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -2
- package/dist/adapters/imageFormatSupport.d.ts +5 -2
- package/dist/adapters/imageFormatSupport.js +47 -24
- package/dist/browser/neurolink.min.js +359 -359
- package/dist/cli/loop/optionsSchema.d.ts +1 -1
- package/dist/core/modules/MessageBuilder.js +2 -0
- package/dist/neurolink.js +1 -0
- package/dist/processors/config/fileExtensions.d.ts +8 -1
- package/dist/processors/config/fileExtensions.js +8 -1
- package/dist/rag/pipeline/RAGPipeline.js +29 -0
- package/dist/rag/ragIntegration.js +21 -1
- package/dist/types/file.d.ts +10 -0
- package/dist/types/generate.d.ts +27 -1
- package/dist/types/stream.d.ts +13 -1
- package/dist/utils/imageProcessor.js +151 -6
- package/dist/utils/messageBuilder.d.ts +2 -2
- package/dist/utils/messageBuilder.js +23 -13
- package/dist/utils/multimodalOptionsBuilder.d.ts +3 -0
- package/dist/utils/multimodalOptionsBuilder.js +1 -0
- package/package.json +2 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,8 +1,12 @@
|
|
|
1
|
-
## [12.
|
|
1
|
+
## [12.20.0](https://github.com/juspay/neurolink/compare/v12.19.2...v12.20.0) (2026-09-22)
|
|
2
|
+
|
|
3
|
+
### Features
|
|
4
|
+
|
|
5
|
+
- **(images):** let a caller choose the transcode output format ([f536fd0](https://github.com/juspay/neurolink/commit/f536fd0916f9e688c723cf679e05f8d995d12e9f))
|
|
2
6
|
|
|
3
7
|
### Bug Fixes
|
|
4
8
|
|
|
5
|
-
- **(
|
|
9
|
+
- **(rag):** keep SVG out of the raster image path and scope the sanitized-markup claim ([0716f94](https://github.com/juspay/neurolink/commit/0716f944d097b2a8944e4036a72fccf3f7c636b6))
|
|
6
10
|
|
|
7
11
|
## [11.2.3](https://github.com/juspay/neurolink/compare/v11.2.2...v11.2.3) (2026-08-19)
|
|
8
12
|
|
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
*
|
|
20
20
|
* @module adapters/imageFormatSupport
|
|
21
21
|
*/
|
|
22
|
-
import type { ImageWithAltText, VisionImageConversion } from "../types/index.js";
|
|
22
|
+
import type { ImageWithAltText, VisionImageConversion, VisionImageOutputFormat } from "../types/index.js";
|
|
23
23
|
/**
|
|
24
24
|
* MIME types every vision-capable provider accepts as-is.
|
|
25
25
|
*
|
|
@@ -57,8 +57,11 @@ export declare function needsVisionTranscode(mimeType: string): boolean;
|
|
|
57
57
|
*
|
|
58
58
|
* @param buffer - Raw image bytes.
|
|
59
59
|
* @param mimeType - Detected MIME type of `buffer`.
|
|
60
|
+
* @param outputFormat - Transcode target. Defaults to `"png"` — see the
|
|
61
|
+
* module header for why that stays the default for every caller who
|
|
62
|
+
* doesn't opt into `"jpeg"` explicitly.
|
|
60
63
|
*/
|
|
61
|
-
export declare function toVisionCompatibleImage(buffer: Buffer, mimeType: string): Promise<VisionImageConversion>;
|
|
64
|
+
export declare function toVisionCompatibleImage(buffer: Buffer, mimeType: string, outputFormat?: VisionImageOutputFormat): Promise<VisionImageConversion>;
|
|
62
65
|
/**
|
|
63
66
|
* Unwrap an `input.images` entry to its payload.
|
|
64
67
|
*
|
|
@@ -23,7 +23,12 @@ import { extensionForMimeType } from "../processors/config/fileTypeRegistry.js";
|
|
|
23
23
|
import { withTimeout } from "../utils/errorHandling.js";
|
|
24
24
|
import { logger } from "../utils/logger.js";
|
|
25
25
|
import { tryImport } from "../utils/tryImport.js";
|
|
26
|
-
import { getFfmpegPath, runFfmpeg } from "./video/ffmpegAdapter.js";
|
|
26
|
+
import { getFfmpegPath, JPEG_QUALITY, runFfmpeg, } from "./video/ffmpegAdapter.js";
|
|
27
|
+
/**
|
|
28
|
+
* Default transcode target, and the one every existing caller keeps getting:
|
|
29
|
+
* see the module header for why PNG rather than JPEG is the right default.
|
|
30
|
+
*/
|
|
31
|
+
const DEFAULT_VISION_IMAGE_OUTPUT_FORMAT = "png";
|
|
27
32
|
/**
|
|
28
33
|
* Per-backend ceiling for one image conversion.
|
|
29
34
|
*
|
|
@@ -106,7 +111,7 @@ export function needsVisionTranscode(mimeType) {
|
|
|
106
111
|
* only — actual HEVC-coded HEIC fails inside libheif, and BMP, ICO and
|
|
107
112
|
* JPEG 2000 are not compiled in at all. Those fall through to ffmpeg below.
|
|
108
113
|
*/
|
|
109
|
-
async function transcodeWithSharp(buffer) {
|
|
114
|
+
async function transcodeWithSharp(buffer, outputFormat) {
|
|
110
115
|
const sharpModule = await tryImport("sharp", "Image format conversion for vision providers");
|
|
111
116
|
// A shape guard, not a crash guard: an absent or malformed sharp already
|
|
112
117
|
// fails safely, because `tryImport` throws a named install error and any
|
|
@@ -115,9 +120,9 @@ async function transcodeWithSharp(buffer) {
|
|
|
115
120
|
// failure list instead of "sharpModule.default is not a function".
|
|
116
121
|
//
|
|
117
122
|
// It deliberately throws rather than returning `buffer`: returning the input
|
|
118
|
-
// would report a successful conversion and relabel the original bytes
|
|
119
|
-
// and would also skip the ffmpeg backend — the one
|
|
120
|
-
// HEIC, BMP, ICO and JPEG 2000.
|
|
123
|
+
// would report a successful conversion and relabel the original bytes under
|
|
124
|
+
// the requested format, and would also skip the ffmpeg backend — the one
|
|
125
|
+
// that actually handles HEIC, BMP, ICO and JPEG 2000.
|
|
121
126
|
if (typeof sharpModule?.default !== "function") {
|
|
122
127
|
throw new Error("the installed sharp package does not expose a callable default export");
|
|
123
128
|
}
|
|
@@ -125,9 +130,15 @@ async function transcodeWithSharp(buffer) {
|
|
|
125
130
|
// a multi-page TIFF) from being flattened into one tall strip — the first
|
|
126
131
|
// frame is what a vision model should receive.
|
|
127
132
|
const pipeline = sharpModule.default(buffer, { pages: 1 });
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
133
|
+
if (outputFormat === "jpeg") {
|
|
134
|
+
// The instance shape is checked as well as the factory: a build that
|
|
135
|
+
// exports a callable but returns something without `.jpeg()` would
|
|
136
|
+
// otherwise fail as an opaque TypeError inside the backend loop.
|
|
137
|
+
if (typeof pipeline?.jpeg !== "function") {
|
|
138
|
+
throw new Error("the installed sharp package returned a pipeline without a jpeg() encoder");
|
|
139
|
+
}
|
|
140
|
+
return pipeline.jpeg().toBuffer();
|
|
141
|
+
}
|
|
131
142
|
if (typeof pipeline?.png !== "function") {
|
|
132
143
|
throw new Error("the installed sharp package returned a pipeline without a png() encoder");
|
|
133
144
|
}
|
|
@@ -151,7 +162,7 @@ async function transcodeWithSharp(buffer) {
|
|
|
151
162
|
* `mkdtemp`. This whole path is server-only — nothing in a browser is going to
|
|
152
163
|
* spawn ffmpeg — so the import belongs where it is used.
|
|
153
164
|
*/
|
|
154
|
-
async function transcodeWithFfmpeg(buffer, extension, binaryPath) {
|
|
165
|
+
async function transcodeWithFfmpeg(buffer, extension, outputFormat, binaryPath) {
|
|
155
166
|
const [{ randomUUID }, { mkdtemp, readFile, rm, writeFile }, { tmpdir }, { join },] = await Promise.all([
|
|
156
167
|
import("node:crypto"),
|
|
157
168
|
import("node:fs/promises"),
|
|
@@ -160,7 +171,13 @@ async function transcodeWithFfmpeg(buffer, extension, binaryPath) {
|
|
|
160
171
|
]);
|
|
161
172
|
const workDir = await mkdtemp(join(tmpdir(), "neurolink-img-"));
|
|
162
173
|
const inputPath = join(workDir, `${randomUUID()}${extension}`);
|
|
163
|
-
|
|
174
|
+
// "mjpeg" is ffmpeg's codec name for a JPEG image2 stream; "png" doubles as
|
|
175
|
+
// both the codec name and the output extension.
|
|
176
|
+
const outputExtension = outputFormat === "jpeg" ? ".jpg" : ".png";
|
|
177
|
+
const codecArgs = outputFormat === "jpeg"
|
|
178
|
+
? ["-c:v", "mjpeg", "-q:v", JPEG_QUALITY]
|
|
179
|
+
: ["-c:v", "png"];
|
|
180
|
+
const outputPath = join(workDir, `${randomUUID()}${outputExtension}`);
|
|
164
181
|
try {
|
|
165
182
|
await writeFile(inputPath, buffer);
|
|
166
183
|
await runFfmpeg([
|
|
@@ -169,14 +186,13 @@ async function transcodeWithFfmpeg(buffer, extension, binaryPath) {
|
|
|
169
186
|
"error",
|
|
170
187
|
"-i",
|
|
171
188
|
inputPath,
|
|
172
|
-
// Take a single frame so a multi-image container yields one
|
|
173
|
-
// than ffmpeg erroring on a missing
|
|
189
|
+
// Take a single frame so a multi-image container yields one output
|
|
190
|
+
// image rather than ffmpeg erroring on a missing sequence pattern.
|
|
174
191
|
"-frames:v",
|
|
175
192
|
"1",
|
|
176
193
|
"-f",
|
|
177
194
|
"image2",
|
|
178
|
-
|
|
179
|
-
"png",
|
|
195
|
+
...codecArgs,
|
|
180
196
|
outputPath,
|
|
181
197
|
], binaryPath ? { binaryPath } : {});
|
|
182
198
|
return await readFile(outputPath);
|
|
@@ -194,17 +210,20 @@ async function transcodeWithFfmpeg(buffer, extension, binaryPath) {
|
|
|
194
210
|
* so when the resolved binary is not already the system one it is retried
|
|
195
211
|
* explicitly rather than reporting a photo as unsupported.
|
|
196
212
|
*/
|
|
197
|
-
async function* transcodeBackends(buffer, extension) {
|
|
198
|
-
yield {
|
|
213
|
+
async function* transcodeBackends(buffer, extension, outputFormat) {
|
|
214
|
+
yield {
|
|
215
|
+
name: "sharp",
|
|
216
|
+
run: () => transcodeWithSharp(buffer, outputFormat),
|
|
217
|
+
};
|
|
199
218
|
const resolved = await getFfmpegPath().catch(() => "ffmpeg");
|
|
200
219
|
yield {
|
|
201
220
|
name: "ffmpeg",
|
|
202
|
-
run: () => transcodeWithFfmpeg(buffer, extension),
|
|
221
|
+
run: () => transcodeWithFfmpeg(buffer, extension, outputFormat),
|
|
203
222
|
};
|
|
204
223
|
if (resolved !== "ffmpeg") {
|
|
205
224
|
yield {
|
|
206
225
|
name: "system ffmpeg",
|
|
207
|
-
run: () => transcodeWithFfmpeg(buffer, extension, "ffmpeg"),
|
|
226
|
+
run: () => transcodeWithFfmpeg(buffer, extension, outputFormat, "ffmpeg"),
|
|
208
227
|
};
|
|
209
228
|
}
|
|
210
229
|
}
|
|
@@ -220,15 +239,19 @@ async function* transcodeBackends(buffer, extension) {
|
|
|
220
239
|
*
|
|
221
240
|
* @param buffer - Raw image bytes.
|
|
222
241
|
* @param mimeType - Detected MIME type of `buffer`.
|
|
242
|
+
* @param outputFormat - Transcode target. Defaults to `"png"` — see the
|
|
243
|
+
* module header for why that stays the default for every caller who
|
|
244
|
+
* doesn't opt into `"jpeg"` explicitly.
|
|
223
245
|
*/
|
|
224
|
-
export async function toVisionCompatibleImage(buffer, mimeType) {
|
|
246
|
+
export async function toVisionCompatibleImage(buffer, mimeType, outputFormat = DEFAULT_VISION_IMAGE_OUTPUT_FORMAT) {
|
|
225
247
|
if (!needsVisionTranscode(mimeType)) {
|
|
226
248
|
return { buffer, mimeType, converted: false };
|
|
227
249
|
}
|
|
228
250
|
const normalized = mimeType.split(";")[0].trim().toLowerCase();
|
|
229
251
|
const extension = extensionForMimeType(normalized) ?? ".bin";
|
|
252
|
+
const targetMimeType = outputFormat === "jpeg" ? "image/jpeg" : "image/png";
|
|
230
253
|
const failures = [];
|
|
231
|
-
for await (const backend of transcodeBackends(buffer, extension)) {
|
|
254
|
+
for await (const backend of transcodeBackends(buffer, extension, outputFormat)) {
|
|
232
255
|
try {
|
|
233
256
|
// Both backends can stall — sharp on a malformed stream, ffmpeg on a
|
|
234
257
|
// container it half-understands — and this runs inline on the request
|
|
@@ -238,16 +261,16 @@ export async function toVisionCompatibleImage(buffer, mimeType) {
|
|
|
238
261
|
if (converted.length === 0) {
|
|
239
262
|
throw new Error("produced an empty image");
|
|
240
263
|
}
|
|
241
|
-
logger.debug(`[imageFormatSupport] Transcoded ${normalized} →
|
|
264
|
+
logger.debug(`[imageFormatSupport] Transcoded ${normalized} → ${targetMimeType} via ${backend.name} ` +
|
|
242
265
|
`(${buffer.length} → ${converted.length} bytes) for vision compatibility`);
|
|
243
|
-
return { buffer: converted, mimeType:
|
|
266
|
+
return { buffer: converted, mimeType: targetMimeType, converted: true };
|
|
244
267
|
}
|
|
245
268
|
catch (error) {
|
|
246
269
|
failures.push(`${backend.name}: ${error instanceof Error ? error.message.split("\n")[0] : String(error)}`);
|
|
247
270
|
}
|
|
248
271
|
}
|
|
249
|
-
logger.warn(`[imageFormatSupport] Could not transcode ${normalized} to
|
|
250
|
-
`original bytes, which most vision providers will reject. Install a full ` +
|
|
272
|
+
logger.warn(`[imageFormatSupport] Could not transcode ${normalized} to ${targetMimeType} — sending ` +
|
|
273
|
+
`the original bytes, which most vision providers will reject. Install a full ` +
|
|
251
274
|
`ffmpeg build (or set FFMPEG_PATH to one) to enable this format. ` +
|
|
252
275
|
`Tried ${failures.join("; ")}`);
|
|
253
276
|
return { buffer, mimeType, converted: false };
|