@juspay/neurolink 10.8.6 → 10.8.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/browser/neurolink.min.js +336 -336
- package/dist/core/baseProvider.js +5 -1
- package/dist/lib/core/baseProvider.js +5 -1
- package/dist/lib/providers/googleVertex/client.d.ts +20 -0
- package/dist/lib/providers/googleVertex/client.js +39 -15
- package/dist/lib/proxy/rollingWorkerSupervisor.d.ts +1 -0
- package/dist/lib/proxy/rollingWorkerSupervisor.js +41 -15
- package/dist/lib/types/proxy.d.ts +2 -0
- package/dist/lib/utils/messageBuilder.d.ts +25 -0
- package/dist/lib/utils/messageBuilder.js +33 -12
- package/dist/lib/utils/multimodalOptionsBuilder.d.ts +5 -1
- package/dist/lib/utils/multimodalOptionsBuilder.js +8 -1
- package/dist/providers/googleVertex/client.d.ts +20 -0
- package/dist/providers/googleVertex/client.js +39 -15
- package/dist/proxy/rollingWorkerSupervisor.d.ts +1 -0
- package/dist/proxy/rollingWorkerSupervisor.js +41 -15
- package/dist/types/proxy.d.ts +2 -0
- package/dist/utils/messageBuilder.d.ts +25 -0
- package/dist/utils/messageBuilder.js +33 -12
- package/dist/utils/multimodalOptionsBuilder.d.ts +5 -1
- package/dist/utils/multimodalOptionsBuilder.js +8 -1
- package/package.json +1 -1
|
@@ -166,7 +166,11 @@ export class BaseProvider {
|
|
|
166
166
|
timestamp: Date.now(),
|
|
167
167
|
});
|
|
168
168
|
// ===== EARLY MULTIMODAL DETECTION =====
|
|
169
|
-
|
|
169
|
+
// #1259: audioFiles was missing here while videoFiles was present, so an
|
|
170
|
+
// audio-only stream skipped this branch entirely.
|
|
171
|
+
const hasFileInput = !!options.input?.files?.length ||
|
|
172
|
+
!!options.input?.videoFiles?.length ||
|
|
173
|
+
!!options.input?.audioFiles?.length;
|
|
170
174
|
if (hasFileInput) {
|
|
171
175
|
// ===== VIDEO ANALYSIS DETECTION =====
|
|
172
176
|
// Check if video frames are present and handle with fake streaming
|
|
@@ -166,7 +166,11 @@ export class BaseProvider {
|
|
|
166
166
|
timestamp: Date.now(),
|
|
167
167
|
});
|
|
168
168
|
// ===== EARLY MULTIMODAL DETECTION =====
|
|
169
|
-
|
|
169
|
+
// #1259: audioFiles was missing here while videoFiles was present, so an
|
|
170
|
+
// audio-only stream skipped this branch entirely.
|
|
171
|
+
const hasFileInput = !!options.input?.files?.length ||
|
|
172
|
+
!!options.input?.videoFiles?.length ||
|
|
173
|
+
!!options.input?.audioFiles?.length;
|
|
170
174
|
if (hasFileInput) {
|
|
171
175
|
// ===== VIDEO ANALYSIS DETECTION =====
|
|
172
176
|
// Check if video frames are present and handle with fake streaming
|
|
@@ -141,6 +141,26 @@ export declare class GoogleVertexProvider extends BaseProvider {
|
|
|
141
141
|
* Validate stream options
|
|
142
142
|
*/
|
|
143
143
|
private validateStreamOptionsOnly;
|
|
144
|
+
/**
|
|
145
|
+
* Preprocess file input before routing to the native SDKs.
|
|
146
|
+
*
|
|
147
|
+
* BaseProvider runs this via `buildMultimodalMessagesArray`, but Vertex
|
|
148
|
+
* overrides both `generate()` and `executeStream()` to reach the native
|
|
149
|
+
* @google/genai / @anthropic-ai/vertex-sdk clients directly, so neither
|
|
150
|
+
* inherits it. Without this the file content never reaches the model and
|
|
151
|
+
* the reply is an entirely plausible "no document is attached" — a silent
|
|
152
|
+
* wrong answer rather than an error.
|
|
153
|
+
*
|
|
154
|
+
* #1258: only `generate()` used to call this, so the same document that
|
|
155
|
+
* `generate()` read back correctly came back as "no documents attached"
|
|
156
|
+
* through `stream()`. Sharing one method is what keeps the two paths from
|
|
157
|
+
* drifting apart again.
|
|
158
|
+
*
|
|
159
|
+
* #1259: the alias fold has to happen *before* the `files` check, or
|
|
160
|
+
* requests carrying only `audioFiles`/`videoFiles` look empty here and skip
|
|
161
|
+
* preprocessing entirely.
|
|
162
|
+
*/
|
|
163
|
+
private preprocessNativeFileInput;
|
|
144
164
|
protected executeStream(options: StreamOptions, _analysisSchema?: ZodType<unknown> | Schema<unknown>): Promise<StreamResult>;
|
|
145
165
|
/**
|
|
146
166
|
* Emit `stream:end` so the Pipeline B observability listener creates a
|
|
@@ -14,7 +14,7 @@ import { AuthenticationError, InvalidModelError, NetworkError, ProviderError, Ra
|
|
|
14
14
|
import { ERROR_CODES, NeuroLinkError } from "../../utils/errorHandling.js";
|
|
15
15
|
import { applyVertexAnthropicCacheBreakpoints } from "../../utils/anthropicCacheBreakpoints.js";
|
|
16
16
|
import { FileDetector } from "../../utils/fileDetector.js";
|
|
17
|
-
import { processUnifiedFilesArray } from "../../utils/messageBuilder.js";
|
|
17
|
+
import { mergeMediaFileAliases, processUnifiedFilesArray, } from "../../utils/messageBuilder.js";
|
|
18
18
|
import { logger } from "../../utils/logger.js";
|
|
19
19
|
import { hasRestrictedOutputLimit, RESTRICTED_OUTPUT_TOKEN_LIMIT, toVertexAnthropicModelId, } from "../../utils/modelDetection.js";
|
|
20
20
|
import { detectImageMimeType } from "../../utils/imageDetection.js";
|
|
@@ -868,6 +868,40 @@ export class GoogleVertexProvider extends BaseProvider {
|
|
|
868
868
|
validateStreamOptionsOnly(options) {
|
|
869
869
|
this.validateStreamOptions(options);
|
|
870
870
|
}
|
|
871
|
+
/**
|
|
872
|
+
* Preprocess file input before routing to the native SDKs.
|
|
873
|
+
*
|
|
874
|
+
* BaseProvider runs this via `buildMultimodalMessagesArray`, but Vertex
|
|
875
|
+
* overrides both `generate()` and `executeStream()` to reach the native
|
|
876
|
+
* @google/genai / @anthropic-ai/vertex-sdk clients directly, so neither
|
|
877
|
+
* inherits it. Without this the file content never reaches the model and
|
|
878
|
+
* the reply is an entirely plausible "no document is attached" — a silent
|
|
879
|
+
* wrong answer rather than an error.
|
|
880
|
+
*
|
|
881
|
+
* #1258: only `generate()` used to call this, so the same document that
|
|
882
|
+
* `generate()` read back correctly came back as "no documents attached"
|
|
883
|
+
* through `stream()`. Sharing one method is what keeps the two paths from
|
|
884
|
+
* drifting apart again.
|
|
885
|
+
*
|
|
886
|
+
* #1259: the alias fold has to happen *before* the `files` check, or
|
|
887
|
+
* requests carrying only `audioFiles`/`videoFiles` look empty here and skip
|
|
888
|
+
* preprocessing entirely.
|
|
889
|
+
*/
|
|
890
|
+
async preprocessNativeFileInput(options) {
|
|
891
|
+
if (options.input) {
|
|
892
|
+
mergeMediaFileAliases(options.input);
|
|
893
|
+
}
|
|
894
|
+
if (!options.input?.files?.length) {
|
|
895
|
+
return;
|
|
896
|
+
}
|
|
897
|
+
try {
|
|
898
|
+
// Mutates options.input.text / .images / .pdfFiles in place.
|
|
899
|
+
await processUnifiedFilesArray(options, 100 * 1024 * 1024, this.providerName);
|
|
900
|
+
}
|
|
901
|
+
catch (fileError) {
|
|
902
|
+
logger.warn(`[GoogleVertex] processUnifiedFilesArray threw, continuing without file content: ${fileError instanceof Error ? fileError.message : String(fileError)}`);
|
|
903
|
+
}
|
|
904
|
+
}
|
|
871
905
|
async executeStream(options, _analysisSchema) {
|
|
872
906
|
// ALL models now use native SDKs - no more @ai-sdk/google-vertex dependency
|
|
873
907
|
const modelName = options.model || this.modelName || getDefaultVertexModel();
|
|
@@ -889,6 +923,9 @@ export class GoogleVertexProvider extends BaseProvider {
|
|
|
889
923
|
// Tool filter (a0269210): trust options.tools — caller (BaseProvider.stream)
|
|
890
924
|
// already merged MCP/built-in tools and applied any enabledToolNames filter.
|
|
891
925
|
const optionTools = options.tools || {};
|
|
926
|
+
// #1258: stream() must run the same file preprocessing generate()
|
|
927
|
+
// does, or attached files are dropped on this path alone.
|
|
928
|
+
await this.preprocessNativeFileInput(options);
|
|
892
929
|
// Emit a `neurolink.message.build` span for the native stream path
|
|
893
930
|
// so observability tooling sees the same hierarchy it sees on
|
|
894
931
|
// Pipeline A. Without this, test:tracing's "Message Build Span"
|
|
@@ -5742,20 +5779,7 @@ export class GoogleVertexProvider extends BaseProvider {
|
|
|
5742
5779
|
const baseTools = !options.disableTools
|
|
5743
5780
|
? await this.getToolsForStream(options)
|
|
5744
5781
|
: {};
|
|
5745
|
-
|
|
5746
|
-
// native SDK. BaseProvider.generate() runs this preprocessing via
|
|
5747
|
-
// buildMultimodalMessagesArray, but Vertex's override skips it,
|
|
5748
|
-
// which would otherwise drop text-file content (and the
|
|
5749
|
-
// mimetype-hint contract) on the floor. Mutates options.input.text /
|
|
5750
|
-
// options.input.images / options.input.pdfFiles in place.
|
|
5751
|
-
if (options.input?.files && options.input.files.length > 0) {
|
|
5752
|
-
try {
|
|
5753
|
-
await processUnifiedFilesArray(options, 100 * 1024 * 1024, this.providerName);
|
|
5754
|
-
}
|
|
5755
|
-
catch (fileError) {
|
|
5756
|
-
logger.warn(`[GoogleVertex] processUnifiedFilesArray threw, continuing without file content: ${fileError instanceof Error ? fileError.message : String(fileError)}`);
|
|
5757
|
-
}
|
|
5758
|
-
}
|
|
5782
|
+
await this.preprocessNativeFileInput(options);
|
|
5759
5783
|
// Emit a `neurolink.message.build` span so observability tooling
|
|
5760
5784
|
// sees the message-construction phase even on the native (Pipeline B)
|
|
5761
5785
|
// Vertex path. Pipeline A normally produces this via MessageBuilder;
|
|
@@ -287,21 +287,19 @@ export class RollingWorkerSupervisor {
|
|
|
287
287
|
generation,
|
|
288
288
|
version: expectedVersion,
|
|
289
289
|
dispose,
|
|
290
|
+
pendingTransfers: 0,
|
|
291
|
+
drainRequested: false,
|
|
290
292
|
};
|
|
291
293
|
this.active = activated;
|
|
292
294
|
this.candidate = null;
|
|
293
295
|
this.flushQueuedSockets();
|
|
294
296
|
if (previous) {
|
|
295
297
|
this.draining.set(previous.generation, previous);
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
}
|
|
302
|
-
catch {
|
|
303
|
-
previous.handle.terminate("SIGTERM");
|
|
304
|
-
}
|
|
298
|
+
// A worker can still own socket transfers accepted before this
|
|
299
|
+
// activation. Draining it before their IPC commits makes those
|
|
300
|
+
// clients fail even though the replacement is healthy.
|
|
301
|
+
previous.drainRequested = true;
|
|
302
|
+
this.maybeDrainWorker(previous);
|
|
305
303
|
}
|
|
306
304
|
this.options.log?.(`[proxy-supervisor] activated generation=${generation} pid=${handle.pid} version=${expectedVersion}`);
|
|
307
305
|
this.publishState();
|
|
@@ -340,6 +338,8 @@ export class RollingWorkerSupervisor {
|
|
|
340
338
|
handle,
|
|
341
339
|
generation,
|
|
342
340
|
version: expectedVersion,
|
|
341
|
+
pendingTransfers: 0,
|
|
342
|
+
drainRequested: false,
|
|
343
343
|
expectedVersion,
|
|
344
344
|
activationRequested: false,
|
|
345
345
|
dispose,
|
|
@@ -362,15 +362,41 @@ export class RollingWorkerSupervisor {
|
|
|
362
362
|
}
|
|
363
363
|
}
|
|
364
364
|
transferSocket(worker, socket) {
|
|
365
|
+
worker.pendingTransfers += 1;
|
|
366
|
+
let settled = false;
|
|
367
|
+
const complete = (error) => {
|
|
368
|
+
if (settled) {
|
|
369
|
+
return;
|
|
370
|
+
}
|
|
371
|
+
settled = true;
|
|
372
|
+
worker.pendingTransfers = Math.max(0, worker.pendingTransfers - 1);
|
|
373
|
+
if (error) {
|
|
374
|
+
this.handleTransferFailure(worker, socket, error);
|
|
375
|
+
}
|
|
376
|
+
this.maybeDrainWorker(worker);
|
|
377
|
+
};
|
|
365
378
|
try {
|
|
366
|
-
worker.handle.sendSocket(worker.generation, socket,
|
|
367
|
-
if (error) {
|
|
368
|
-
this.handleTransferFailure(worker, socket, error);
|
|
369
|
-
}
|
|
370
|
-
});
|
|
379
|
+
worker.handle.sendSocket(worker.generation, socket, complete);
|
|
371
380
|
}
|
|
372
381
|
catch (error) {
|
|
373
|
-
|
|
382
|
+
complete(error instanceof Error ? error : new Error(String(error)));
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
maybeDrainWorker(worker) {
|
|
386
|
+
if (!worker.drainRequested ||
|
|
387
|
+
worker.pendingTransfers > 0 ||
|
|
388
|
+
!this.draining.has(worker.generation)) {
|
|
389
|
+
return;
|
|
390
|
+
}
|
|
391
|
+
worker.drainRequested = false;
|
|
392
|
+
try {
|
|
393
|
+
worker.handle.sendControl({
|
|
394
|
+
type: "proxy-worker:drain",
|
|
395
|
+
generation: worker.generation,
|
|
396
|
+
});
|
|
397
|
+
}
|
|
398
|
+
catch {
|
|
399
|
+
worker.handle.terminate("SIGTERM");
|
|
374
400
|
}
|
|
375
401
|
}
|
|
376
402
|
handleTransferFailure(worker, socket, error) {
|
|
@@ -1949,6 +1949,8 @@ export type RollingManagedWorker = {
|
|
|
1949
1949
|
generation: number;
|
|
1950
1950
|
version: string;
|
|
1951
1951
|
dispose: () => void;
|
|
1952
|
+
pendingTransfers: number;
|
|
1953
|
+
drainRequested: boolean;
|
|
1952
1954
|
};
|
|
1953
1955
|
export type RollingCandidateWorker = RollingManagedWorker & {
|
|
1954
1956
|
expectedVersion: string;
|
|
@@ -12,6 +12,31 @@ export declare function convertToModelMessages(messages: MultimodalChatMessage[]
|
|
|
12
12
|
* Enhanced with CSV file processing support
|
|
13
13
|
*/
|
|
14
14
|
export declare function buildMessagesArray(options: TextGenerationOptions | StreamOptions): Promise<ModelMessage[]>;
|
|
15
|
+
/**
|
|
16
|
+
* Fold the `audioFiles` / `videoFiles` aliases into the unified `files` array.
|
|
17
|
+
*
|
|
18
|
+
* #284 gave audio and video their own input fields, but neither has a
|
|
19
|
+
* dedicated processor — both are meant to travel through the same
|
|
20
|
+
* auto-detecting `files` pipeline that already understands "audio"/"video"
|
|
21
|
+
* FileDetector results (see `appendDetectedFileResult`). The fold used to live
|
|
22
|
+
* inline in `buildMultimodalMessagesArray`, which meant any path that bypassed
|
|
23
|
+
* that builder never performed it and dropped the files silently: the model
|
|
24
|
+
* received the prompt alone and answered as though nothing were attached
|
|
25
|
+
* (#1259). GoogleVertex's native SDK path and `buildMultimodalOptions`
|
|
26
|
+
* (Bedrock) are both such paths, so the fold has to be callable from them.
|
|
27
|
+
*
|
|
28
|
+
* The aliases are cleared once merged, which makes the call idempotent: a
|
|
29
|
+
* provider override and the shared builder can both call this on the same
|
|
30
|
+
* options object without attaching every file twice.
|
|
31
|
+
*
|
|
32
|
+
* Mutates in place, matching `processUnifiedFilesArray` below — downstream
|
|
33
|
+
* stages all read `options.input.files`.
|
|
34
|
+
*/
|
|
35
|
+
export declare function mergeMediaFileAliases<TFile>(input: {
|
|
36
|
+
files?: Array<TFile | Buffer | string>;
|
|
37
|
+
audioFiles?: Array<Buffer | string>;
|
|
38
|
+
videoFiles?: Array<Buffer | string>;
|
|
39
|
+
}): void;
|
|
15
40
|
/**
|
|
16
41
|
* Process the unified files array with auto-detection.
|
|
17
42
|
* Handles lazy file registration, full processing, and preview injection.
|
|
@@ -777,6 +777,38 @@ function appendDetectedFileResult(result, file, options) {
|
|
|
777
777
|
logger.info(`[FileDetector] ⚠️ Unknown format (metadata extracted): ${filename}`);
|
|
778
778
|
}
|
|
779
779
|
}
|
|
780
|
+
/**
|
|
781
|
+
* Fold the `audioFiles` / `videoFiles` aliases into the unified `files` array.
|
|
782
|
+
*
|
|
783
|
+
* #284 gave audio and video their own input fields, but neither has a
|
|
784
|
+
* dedicated processor — both are meant to travel through the same
|
|
785
|
+
* auto-detecting `files` pipeline that already understands "audio"/"video"
|
|
786
|
+
* FileDetector results (see `appendDetectedFileResult`). The fold used to live
|
|
787
|
+
* inline in `buildMultimodalMessagesArray`, which meant any path that bypassed
|
|
788
|
+
* that builder never performed it and dropped the files silently: the model
|
|
789
|
+
* received the prompt alone and answered as though nothing were attached
|
|
790
|
+
* (#1259). GoogleVertex's native SDK path and `buildMultimodalOptions`
|
|
791
|
+
* (Bedrock) are both such paths, so the fold has to be callable from them.
|
|
792
|
+
*
|
|
793
|
+
* The aliases are cleared once merged, which makes the call idempotent: a
|
|
794
|
+
* provider override and the shared builder can both call this on the same
|
|
795
|
+
* options object without attaching every file twice.
|
|
796
|
+
*
|
|
797
|
+
* Mutates in place, matching `processUnifiedFilesArray` below — downstream
|
|
798
|
+
* stages all read `options.input.files`.
|
|
799
|
+
*/
|
|
800
|
+
export function mergeMediaFileAliases(input) {
|
|
801
|
+
if (!input.audioFiles?.length && !input.videoFiles?.length) {
|
|
802
|
+
return;
|
|
803
|
+
}
|
|
804
|
+
input.files = [
|
|
805
|
+
...(input.files ?? []),
|
|
806
|
+
...(input.audioFiles ?? []),
|
|
807
|
+
...(input.videoFiles ?? []),
|
|
808
|
+
];
|
|
809
|
+
input.audioFiles = undefined;
|
|
810
|
+
input.videoFiles = undefined;
|
|
811
|
+
}
|
|
780
812
|
/**
|
|
781
813
|
* Process the unified files array with auto-detection.
|
|
782
814
|
* Handles lazy file registration, full processing, and preview injection.
|
|
@@ -1115,18 +1147,7 @@ export async function buildMultimodalMessagesArray(options, provider, model) {
|
|
|
1115
1147
|
// local const so TypeScript sees the definite (non-optional) type in the
|
|
1116
1148
|
// rest of this function, avoiding 60+ "possibly undefined" errors.
|
|
1117
1149
|
const inp = options.input;
|
|
1118
|
-
|
|
1119
|
-
// through the same auto-detecting `files` pipeline that already
|
|
1120
|
-
// understands "audio"/"video" FileDetector results (see
|
|
1121
|
-
// appendDetectedFileResult), instead of silently dropping them once
|
|
1122
|
-
// detectMultimodal() routes an audio/video-only request here.
|
|
1123
|
-
if (inp.audioFiles?.length || inp.videoFiles?.length) {
|
|
1124
|
-
inp.files = [
|
|
1125
|
-
...(inp.files || []),
|
|
1126
|
-
...(inp.audioFiles || []),
|
|
1127
|
-
...(inp.videoFiles || []),
|
|
1128
|
-
];
|
|
1129
|
-
}
|
|
1150
|
+
mergeMediaFileAliases(inp);
|
|
1130
1151
|
// Compute provider-specific max PDF size once for consistent validation
|
|
1131
1152
|
const pdfConfig = PDFProcessor.getProviderConfig(provider);
|
|
1132
1153
|
const maxSize = pdfConfig
|
|
@@ -12,6 +12,8 @@ import type { StreamOptions } from "../types/index.js";
|
|
|
12
12
|
* - input.files: Auto-detected file types
|
|
13
13
|
* - input.csvFiles: CSV files for tabular data
|
|
14
14
|
* - input.pdfFiles: PDF documents (Buffer | string paths)
|
|
15
|
+
* - input.audioFiles: Audio files (Buffer | string paths)
|
|
16
|
+
* - input.videoFiles: Video files (Buffer | string paths)
|
|
15
17
|
* - csvOptions: CSV parsing options
|
|
16
18
|
* - systemPrompt: System-level instructions
|
|
17
19
|
* - conversationMessages: Chat history
|
|
@@ -23,7 +25,7 @@ import type { StreamOptions } from "../types/index.js";
|
|
|
23
25
|
* @param {string} providerName - Provider identifier (e.g., "vertex", "openai", "anthropic")
|
|
24
26
|
* @param {string} modelName - Model identifier (e.g., "gemini-2.5-flash", "gpt-4o")
|
|
25
27
|
* @returns {object} Normalized options object with:
|
|
26
|
-
* - input: { text, images, content, files, csvFiles, pdfFiles }
|
|
28
|
+
* - input: { text, images, content, files, csvFiles, pdfFiles, audioFiles, videoFiles }
|
|
27
29
|
* - csvOptions: CSV processing options
|
|
28
30
|
* - systemPrompt: System prompt string
|
|
29
31
|
* - conversationHistory: Message history array
|
|
@@ -49,6 +51,8 @@ export declare function buildMultimodalOptions(options: StreamOptions, providerN
|
|
|
49
51
|
files: (string | Buffer<ArrayBufferLike> | import("../types/file.js").FileWithMetadata)[] | undefined;
|
|
50
52
|
csvFiles: (string | Buffer<ArrayBufferLike>)[] | undefined;
|
|
51
53
|
pdfFiles: (string | Buffer<ArrayBufferLike>)[] | undefined;
|
|
54
|
+
audioFiles: (string | Buffer<ArrayBufferLike>)[] | undefined;
|
|
55
|
+
videoFiles: (string | Buffer<ArrayBufferLike>)[] | undefined;
|
|
52
56
|
};
|
|
53
57
|
csvOptions: import("../types/file.js").CSVProcessorOptions | undefined;
|
|
54
58
|
pdfOptions: {
|
|
@@ -11,6 +11,8 @@
|
|
|
11
11
|
* - input.files: Auto-detected file types
|
|
12
12
|
* - input.csvFiles: CSV files for tabular data
|
|
13
13
|
* - input.pdfFiles: PDF documents (Buffer | string paths)
|
|
14
|
+
* - input.audioFiles: Audio files (Buffer | string paths)
|
|
15
|
+
* - input.videoFiles: Video files (Buffer | string paths)
|
|
14
16
|
* - csvOptions: CSV parsing options
|
|
15
17
|
* - systemPrompt: System-level instructions
|
|
16
18
|
* - conversationMessages: Chat history
|
|
@@ -22,7 +24,7 @@
|
|
|
22
24
|
* @param {string} providerName - Provider identifier (e.g., "vertex", "openai", "anthropic")
|
|
23
25
|
* @param {string} modelName - Model identifier (e.g., "gemini-2.5-flash", "gpt-4o")
|
|
24
26
|
* @returns {object} Normalized options object with:
|
|
25
|
-
* - input: { text, images, content, files, csvFiles, pdfFiles }
|
|
27
|
+
* - input: { text, images, content, files, csvFiles, pdfFiles, audioFiles, videoFiles }
|
|
26
28
|
* - csvOptions: CSV processing options
|
|
27
29
|
* - systemPrompt: System prompt string
|
|
28
30
|
* - conversationHistory: Message history array
|
|
@@ -49,6 +51,11 @@ export function buildMultimodalOptions(options, providerName, modelName) {
|
|
|
49
51
|
files: options.input?.files,
|
|
50
52
|
csvFiles: options.input?.csvFiles,
|
|
51
53
|
pdfFiles: options.input?.pdfFiles,
|
|
54
|
+
// #1259: this is a whitelist — a field omitted here is dropped
|
|
55
|
+
// silently, and the model answers as though nothing were attached.
|
|
56
|
+
// audioFiles/videoFiles were missing, so Bedrock received neither.
|
|
57
|
+
audioFiles: options.input?.audioFiles,
|
|
58
|
+
videoFiles: options.input?.videoFiles,
|
|
52
59
|
},
|
|
53
60
|
csvOptions: options.csvOptions,
|
|
54
61
|
pdfOptions: options.pdfOptions,
|
|
@@ -141,6 +141,26 @@ export declare class GoogleVertexProvider extends BaseProvider {
|
|
|
141
141
|
* Validate stream options
|
|
142
142
|
*/
|
|
143
143
|
private validateStreamOptionsOnly;
|
|
144
|
+
/**
|
|
145
|
+
* Preprocess file input before routing to the native SDKs.
|
|
146
|
+
*
|
|
147
|
+
* BaseProvider runs this via `buildMultimodalMessagesArray`, but Vertex
|
|
148
|
+
* overrides both `generate()` and `executeStream()` to reach the native
|
|
149
|
+
* @google/genai / @anthropic-ai/vertex-sdk clients directly, so neither
|
|
150
|
+
* inherits it. Without this the file content never reaches the model and
|
|
151
|
+
* the reply is an entirely plausible "no document is attached" — a silent
|
|
152
|
+
* wrong answer rather than an error.
|
|
153
|
+
*
|
|
154
|
+
* #1258: only `generate()` used to call this, so the same document that
|
|
155
|
+
* `generate()` read back correctly came back as "no documents attached"
|
|
156
|
+
* through `stream()`. Sharing one method is what keeps the two paths from
|
|
157
|
+
* drifting apart again.
|
|
158
|
+
*
|
|
159
|
+
* #1259: the alias fold has to happen *before* the `files` check, or
|
|
160
|
+
* requests carrying only `audioFiles`/`videoFiles` look empty here and skip
|
|
161
|
+
* preprocessing entirely.
|
|
162
|
+
*/
|
|
163
|
+
private preprocessNativeFileInput;
|
|
144
164
|
protected executeStream(options: StreamOptions, _analysisSchema?: ZodType<unknown> | Schema<unknown>): Promise<StreamResult>;
|
|
145
165
|
/**
|
|
146
166
|
* Emit `stream:end` so the Pipeline B observability listener creates a
|
|
@@ -14,7 +14,7 @@ import { AuthenticationError, InvalidModelError, NetworkError, ProviderError, Ra
|
|
|
14
14
|
import { ERROR_CODES, NeuroLinkError } from "../../utils/errorHandling.js";
|
|
15
15
|
import { applyVertexAnthropicCacheBreakpoints } from "../../utils/anthropicCacheBreakpoints.js";
|
|
16
16
|
import { FileDetector } from "../../utils/fileDetector.js";
|
|
17
|
-
import { processUnifiedFilesArray } from "../../utils/messageBuilder.js";
|
|
17
|
+
import { mergeMediaFileAliases, processUnifiedFilesArray, } from "../../utils/messageBuilder.js";
|
|
18
18
|
import { logger } from "../../utils/logger.js";
|
|
19
19
|
import { hasRestrictedOutputLimit, RESTRICTED_OUTPUT_TOKEN_LIMIT, toVertexAnthropicModelId, } from "../../utils/modelDetection.js";
|
|
20
20
|
import { detectImageMimeType } from "../../utils/imageDetection.js";
|
|
@@ -868,6 +868,40 @@ export class GoogleVertexProvider extends BaseProvider {
|
|
|
868
868
|
validateStreamOptionsOnly(options) {
|
|
869
869
|
this.validateStreamOptions(options);
|
|
870
870
|
}
|
|
871
|
+
/**
|
|
872
|
+
* Preprocess file input before routing to the native SDKs.
|
|
873
|
+
*
|
|
874
|
+
* BaseProvider runs this via `buildMultimodalMessagesArray`, but Vertex
|
|
875
|
+
* overrides both `generate()` and `executeStream()` to reach the native
|
|
876
|
+
* @google/genai / @anthropic-ai/vertex-sdk clients directly, so neither
|
|
877
|
+
* inherits it. Without this the file content never reaches the model and
|
|
878
|
+
* the reply is an entirely plausible "no document is attached" — a silent
|
|
879
|
+
* wrong answer rather than an error.
|
|
880
|
+
*
|
|
881
|
+
* #1258: only `generate()` used to call this, so the same document that
|
|
882
|
+
* `generate()` read back correctly came back as "no documents attached"
|
|
883
|
+
* through `stream()`. Sharing one method is what keeps the two paths from
|
|
884
|
+
* drifting apart again.
|
|
885
|
+
*
|
|
886
|
+
* #1259: the alias fold has to happen *before* the `files` check, or
|
|
887
|
+
* requests carrying only `audioFiles`/`videoFiles` look empty here and skip
|
|
888
|
+
* preprocessing entirely.
|
|
889
|
+
*/
|
|
890
|
+
async preprocessNativeFileInput(options) {
|
|
891
|
+
if (options.input) {
|
|
892
|
+
mergeMediaFileAliases(options.input);
|
|
893
|
+
}
|
|
894
|
+
if (!options.input?.files?.length) {
|
|
895
|
+
return;
|
|
896
|
+
}
|
|
897
|
+
try {
|
|
898
|
+
// Mutates options.input.text / .images / .pdfFiles in place.
|
|
899
|
+
await processUnifiedFilesArray(options, 100 * 1024 * 1024, this.providerName);
|
|
900
|
+
}
|
|
901
|
+
catch (fileError) {
|
|
902
|
+
logger.warn(`[GoogleVertex] processUnifiedFilesArray threw, continuing without file content: ${fileError instanceof Error ? fileError.message : String(fileError)}`);
|
|
903
|
+
}
|
|
904
|
+
}
|
|
871
905
|
async executeStream(options, _analysisSchema) {
|
|
872
906
|
// ALL models now use native SDKs - no more @ai-sdk/google-vertex dependency
|
|
873
907
|
const modelName = options.model || this.modelName || getDefaultVertexModel();
|
|
@@ -889,6 +923,9 @@ export class GoogleVertexProvider extends BaseProvider {
|
|
|
889
923
|
// Tool filter (a0269210): trust options.tools — caller (BaseProvider.stream)
|
|
890
924
|
// already merged MCP/built-in tools and applied any enabledToolNames filter.
|
|
891
925
|
const optionTools = options.tools || {};
|
|
926
|
+
// #1258: stream() must run the same file preprocessing generate()
|
|
927
|
+
// does, or attached files are dropped on this path alone.
|
|
928
|
+
await this.preprocessNativeFileInput(options);
|
|
892
929
|
// Emit a `neurolink.message.build` span for the native stream path
|
|
893
930
|
// so observability tooling sees the same hierarchy it sees on
|
|
894
931
|
// Pipeline A. Without this, test:tracing's "Message Build Span"
|
|
@@ -5742,20 +5779,7 @@ export class GoogleVertexProvider extends BaseProvider {
|
|
|
5742
5779
|
const baseTools = !options.disableTools
|
|
5743
5780
|
? await this.getToolsForStream(options)
|
|
5744
5781
|
: {};
|
|
5745
|
-
|
|
5746
|
-
// native SDK. BaseProvider.generate() runs this preprocessing via
|
|
5747
|
-
// buildMultimodalMessagesArray, but Vertex's override skips it,
|
|
5748
|
-
// which would otherwise drop text-file content (and the
|
|
5749
|
-
// mimetype-hint contract) on the floor. Mutates options.input.text /
|
|
5750
|
-
// options.input.images / options.input.pdfFiles in place.
|
|
5751
|
-
if (options.input?.files && options.input.files.length > 0) {
|
|
5752
|
-
try {
|
|
5753
|
-
await processUnifiedFilesArray(options, 100 * 1024 * 1024, this.providerName);
|
|
5754
|
-
}
|
|
5755
|
-
catch (fileError) {
|
|
5756
|
-
logger.warn(`[GoogleVertex] processUnifiedFilesArray threw, continuing without file content: ${fileError instanceof Error ? fileError.message : String(fileError)}`);
|
|
5757
|
-
}
|
|
5758
|
-
}
|
|
5782
|
+
await this.preprocessNativeFileInput(options);
|
|
5759
5783
|
// Emit a `neurolink.message.build` span so observability tooling
|
|
5760
5784
|
// sees the message-construction phase even on the native (Pipeline B)
|
|
5761
5785
|
// Vertex path. Pipeline A normally produces this via MessageBuilder;
|
|
@@ -287,21 +287,19 @@ export class RollingWorkerSupervisor {
|
|
|
287
287
|
generation,
|
|
288
288
|
version: expectedVersion,
|
|
289
289
|
dispose,
|
|
290
|
+
pendingTransfers: 0,
|
|
291
|
+
drainRequested: false,
|
|
290
292
|
};
|
|
291
293
|
this.active = activated;
|
|
292
294
|
this.candidate = null;
|
|
293
295
|
this.flushQueuedSockets();
|
|
294
296
|
if (previous) {
|
|
295
297
|
this.draining.set(previous.generation, previous);
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
}
|
|
302
|
-
catch {
|
|
303
|
-
previous.handle.terminate("SIGTERM");
|
|
304
|
-
}
|
|
298
|
+
// A worker can still own socket transfers accepted before this
|
|
299
|
+
// activation. Draining it before their IPC commits makes those
|
|
300
|
+
// clients fail even though the replacement is healthy.
|
|
301
|
+
previous.drainRequested = true;
|
|
302
|
+
this.maybeDrainWorker(previous);
|
|
305
303
|
}
|
|
306
304
|
this.options.log?.(`[proxy-supervisor] activated generation=${generation} pid=${handle.pid} version=${expectedVersion}`);
|
|
307
305
|
this.publishState();
|
|
@@ -340,6 +338,8 @@ export class RollingWorkerSupervisor {
|
|
|
340
338
|
handle,
|
|
341
339
|
generation,
|
|
342
340
|
version: expectedVersion,
|
|
341
|
+
pendingTransfers: 0,
|
|
342
|
+
drainRequested: false,
|
|
343
343
|
expectedVersion,
|
|
344
344
|
activationRequested: false,
|
|
345
345
|
dispose,
|
|
@@ -362,15 +362,41 @@ export class RollingWorkerSupervisor {
|
|
|
362
362
|
}
|
|
363
363
|
}
|
|
364
364
|
transferSocket(worker, socket) {
|
|
365
|
+
worker.pendingTransfers += 1;
|
|
366
|
+
let settled = false;
|
|
367
|
+
const complete = (error) => {
|
|
368
|
+
if (settled) {
|
|
369
|
+
return;
|
|
370
|
+
}
|
|
371
|
+
settled = true;
|
|
372
|
+
worker.pendingTransfers = Math.max(0, worker.pendingTransfers - 1);
|
|
373
|
+
if (error) {
|
|
374
|
+
this.handleTransferFailure(worker, socket, error);
|
|
375
|
+
}
|
|
376
|
+
this.maybeDrainWorker(worker);
|
|
377
|
+
};
|
|
365
378
|
try {
|
|
366
|
-
worker.handle.sendSocket(worker.generation, socket,
|
|
367
|
-
if (error) {
|
|
368
|
-
this.handleTransferFailure(worker, socket, error);
|
|
369
|
-
}
|
|
370
|
-
});
|
|
379
|
+
worker.handle.sendSocket(worker.generation, socket, complete);
|
|
371
380
|
}
|
|
372
381
|
catch (error) {
|
|
373
|
-
|
|
382
|
+
complete(error instanceof Error ? error : new Error(String(error)));
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
maybeDrainWorker(worker) {
|
|
386
|
+
if (!worker.drainRequested ||
|
|
387
|
+
worker.pendingTransfers > 0 ||
|
|
388
|
+
!this.draining.has(worker.generation)) {
|
|
389
|
+
return;
|
|
390
|
+
}
|
|
391
|
+
worker.drainRequested = false;
|
|
392
|
+
try {
|
|
393
|
+
worker.handle.sendControl({
|
|
394
|
+
type: "proxy-worker:drain",
|
|
395
|
+
generation: worker.generation,
|
|
396
|
+
});
|
|
397
|
+
}
|
|
398
|
+
catch {
|
|
399
|
+
worker.handle.terminate("SIGTERM");
|
|
374
400
|
}
|
|
375
401
|
}
|
|
376
402
|
handleTransferFailure(worker, socket, error) {
|
package/dist/types/proxy.d.ts
CHANGED
|
@@ -1949,6 +1949,8 @@ export type RollingManagedWorker = {
|
|
|
1949
1949
|
generation: number;
|
|
1950
1950
|
version: string;
|
|
1951
1951
|
dispose: () => void;
|
|
1952
|
+
pendingTransfers: number;
|
|
1953
|
+
drainRequested: boolean;
|
|
1952
1954
|
};
|
|
1953
1955
|
export type RollingCandidateWorker = RollingManagedWorker & {
|
|
1954
1956
|
expectedVersion: string;
|
|
@@ -12,6 +12,31 @@ export declare function convertToModelMessages(messages: MultimodalChatMessage[]
|
|
|
12
12
|
* Enhanced with CSV file processing support
|
|
13
13
|
*/
|
|
14
14
|
export declare function buildMessagesArray(options: TextGenerationOptions | StreamOptions): Promise<ModelMessage[]>;
|
|
15
|
+
/**
|
|
16
|
+
* Fold the `audioFiles` / `videoFiles` aliases into the unified `files` array.
|
|
17
|
+
*
|
|
18
|
+
* #284 gave audio and video their own input fields, but neither has a
|
|
19
|
+
* dedicated processor — both are meant to travel through the same
|
|
20
|
+
* auto-detecting `files` pipeline that already understands "audio"/"video"
|
|
21
|
+
* FileDetector results (see `appendDetectedFileResult`). The fold used to live
|
|
22
|
+
* inline in `buildMultimodalMessagesArray`, which meant any path that bypassed
|
|
23
|
+
* that builder never performed it and dropped the files silently: the model
|
|
24
|
+
* received the prompt alone and answered as though nothing were attached
|
|
25
|
+
* (#1259). GoogleVertex's native SDK path and `buildMultimodalOptions`
|
|
26
|
+
* (Bedrock) are both such paths, so the fold has to be callable from them.
|
|
27
|
+
*
|
|
28
|
+
* The aliases are cleared once merged, which makes the call idempotent: a
|
|
29
|
+
* provider override and the shared builder can both call this on the same
|
|
30
|
+
* options object without attaching every file twice.
|
|
31
|
+
*
|
|
32
|
+
* Mutates in place, matching `processUnifiedFilesArray` below — downstream
|
|
33
|
+
* stages all read `options.input.files`.
|
|
34
|
+
*/
|
|
35
|
+
export declare function mergeMediaFileAliases<TFile>(input: {
|
|
36
|
+
files?: Array<TFile | Buffer | string>;
|
|
37
|
+
audioFiles?: Array<Buffer | string>;
|
|
38
|
+
videoFiles?: Array<Buffer | string>;
|
|
39
|
+
}): void;
|
|
15
40
|
/**
|
|
16
41
|
* Process the unified files array with auto-detection.
|
|
17
42
|
* Handles lazy file registration, full processing, and preview injection.
|