@juspay/neurolink 10.11.2 → 10.11.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/adapters/audioFormatSupport.d.ts +53 -0
- package/dist/adapters/audioFormatSupport.js +200 -0
- package/dist/browser/neurolink.min.js +398 -397
- package/dist/lib/adapters/audioFormatSupport.d.ts +53 -0
- package/dist/lib/adapters/audioFormatSupport.js +201 -0
- package/dist/lib/processors/archive/ArchiveProcessor.d.ts +37 -0
- package/dist/lib/processors/archive/ArchiveProcessor.js +347 -32
- package/dist/lib/providers/googleAiStudio/client.d.ts +17 -0
- package/dist/lib/providers/googleAiStudio/client.js +45 -19
- package/dist/lib/providers/googleNativeGemini3/utils.d.ts +22 -1
- package/dist/lib/providers/googleNativeGemini3/utils.js +54 -0
- package/dist/lib/providers/googleVertex/client.js +3 -0
- package/dist/lib/types/file.d.ts +41 -0
- package/dist/lib/types/generate.d.ts +12 -1
- package/dist/lib/types/processor.d.ts +20 -1
- package/dist/lib/types/providers.d.ts +7 -0
- package/dist/lib/utils/fileDetector.d.ts +27 -0
- package/dist/lib/utils/fileDetector.js +130 -7
- package/dist/lib/utils/imageProcessor.js +31 -0
- package/dist/lib/utils/messageBuilder.d.ts +0 -9
- package/dist/lib/utils/messageBuilder.js +380 -56
- package/dist/processors/archive/ArchiveProcessor.d.ts +37 -0
- package/dist/processors/archive/ArchiveProcessor.js +347 -32
- package/dist/providers/googleAiStudio/client.d.ts +17 -0
- package/dist/providers/googleAiStudio/client.js +45 -19
- package/dist/providers/googleNativeGemini3/utils.d.ts +22 -1
- package/dist/providers/googleNativeGemini3/utils.js +54 -0
- package/dist/providers/googleVertex/client.js +3 -0
- package/dist/types/file.d.ts +41 -0
- package/dist/types/generate.d.ts +12 -1
- package/dist/types/processor.d.ts +20 -1
- package/dist/types/providers.d.ts +7 -0
- package/dist/utils/fileDetector.d.ts +27 -0
- package/dist/utils/fileDetector.js +130 -7
- package/dist/utils/imageProcessor.js +31 -0
- package/dist/utils/messageBuilder.d.ts +0 -9
- package/dist/utils/messageBuilder.js +380 -56
- package/package.json +1 -1
|
@@ -12,6 +12,7 @@ import { randomUUID } from "node:crypto";
|
|
|
12
12
|
import { existsSync, readFileSync } from "node:fs";
|
|
13
13
|
import { extname } from "node:path";
|
|
14
14
|
import { DEFAULT_CONTEXT_GUARD_RATIO, DEFAULT_MAX_STEPS, DEFAULT_TOOL_MAX_RETRIES, DEFAULT_WRAPUP_TIME_LEAD_MS, } from "../../core/constants.js";
|
|
15
|
+
import { needsAudioTranscode, toProviderCompatibleAudio, } from "../../adapters/audioFormatSupport.js";
|
|
15
16
|
import { logger } from "../../utils/logger.js";
|
|
16
17
|
import { resolveSamplingParams } from "../../models/modelRegistry.js";
|
|
17
18
|
import { convertZodToJsonSchema, ensureNestedSchemaTypes, inlineJsonSchema, isZodSchema, normalizeJsonSchemaObject, } from "../../utils/schemaConversion.js";
|
|
@@ -1432,6 +1433,53 @@ conversationMessages) {
|
|
|
1432
1433
|
* is skipped rather than aborting the entire request, matching prior
|
|
1433
1434
|
* Vertex behaviour.
|
|
1434
1435
|
*/
|
|
1436
|
+
/**
|
|
1437
|
+
* Append audio to a Gemini request as `inlineData` parts.
|
|
1438
|
+
*
|
|
1439
|
+
* Shared by both Gemini front ends. Vertex assembles its request here and AI
|
|
1440
|
+
* Studio assembles it in `buildUserPartsWithMultimodal`; when this lived only in
|
|
1441
|
+
* the Vertex client, AI Studio advertised audio support through
|
|
1442
|
+
* `NATIVE_AUDIO_PROVIDERS` and then silently dropped the bytes.
|
|
1443
|
+
*
|
|
1444
|
+
* Gemini's native request shape is assembled directly rather than taken from the
|
|
1445
|
+
* AI SDK's `file` parts, so audio has to be added explicitly the same way PDFs
|
|
1446
|
+
* and images are — a `{ type: "file" }` part built upstream simply never
|
|
1447
|
+
* reaches this request body. That asymmetry is why attaching a recording
|
|
1448
|
+
* produced only the metadata summary even after the message builder learned to
|
|
1449
|
+
* carry the bytes.
|
|
1450
|
+
*
|
|
1451
|
+
* A container Gemini does not accept is converted first; one that cannot be
|
|
1452
|
+
* converted is skipped rather than sent, because an unsupported inlineData
|
|
1453
|
+
* mimeType fails the whole request, and the caller still has the metadata
|
|
1454
|
+
* summary in the text part.
|
|
1455
|
+
*/
|
|
1456
|
+
export async function appendNativeAudioParts(userParts, audioFiles, logPrefix = "[GeminiNative]") {
|
|
1457
|
+
if (!audioFiles || audioFiles.length === 0) {
|
|
1458
|
+
return;
|
|
1459
|
+
}
|
|
1460
|
+
for (const audio of audioFiles) {
|
|
1461
|
+
// Split on both separators: a Windows-style name reaching a POSIX host
|
|
1462
|
+
// would otherwise keep its whole path, and the extension lookup below
|
|
1463
|
+
// needs the bare filename.
|
|
1464
|
+
const base = audio.filename.split(/[\\/]/).pop() ?? audio.filename;
|
|
1465
|
+
const dot = base.lastIndexOf(".");
|
|
1466
|
+
const extension = dot > 0 ? base.slice(dot) : ".bin";
|
|
1467
|
+
const compatible = await toProviderCompatibleAudio(audio.buffer, audio.mimeType, extension);
|
|
1468
|
+
if (needsAudioTranscode(compatible.mimeType)) {
|
|
1469
|
+
logger.warn(`${logPrefix} Skipping native audio for ${base}: ${compatible.mimeType} ` +
|
|
1470
|
+
`is not accepted and could not be converted. The metadata summary was ` +
|
|
1471
|
+
`still included.`);
|
|
1472
|
+
continue;
|
|
1473
|
+
}
|
|
1474
|
+
userParts.push({
|
|
1475
|
+
inlineData: {
|
|
1476
|
+
mimeType: compatible.mimeType,
|
|
1477
|
+
data: compatible.buffer.toString("base64"),
|
|
1478
|
+
},
|
|
1479
|
+
});
|
|
1480
|
+
logger.debug(`${logPrefix} Added native audio part for ${base} (${compatible.mimeType})`);
|
|
1481
|
+
}
|
|
1482
|
+
}
|
|
1435
1483
|
export async function buildUserPartsWithMultimodal(input, textOverride, logPrefix = "[GeminiNative]") {
|
|
1436
1484
|
const text = typeof textOverride === "string" ? textOverride : (input?.text ?? "");
|
|
1437
1485
|
const parts = [{ text }];
|
|
@@ -1533,6 +1581,12 @@ export async function buildUserPartsWithMultimodal(input, textOverride, logPrefi
|
|
|
1533
1581
|
});
|
|
1534
1582
|
}
|
|
1535
1583
|
}
|
|
1584
|
+
// Audio last, and through the same helper the Vertex client uses. AI Studio
|
|
1585
|
+
// never touches `buildMultimodalMessagesArray` — it overrides generate() and
|
|
1586
|
+
// stream() and assembles its request here — so wiring audio only into the
|
|
1587
|
+
// Vertex client left this front end advertising native audio via
|
|
1588
|
+
// NATIVE_AUDIO_PROVIDERS and then dropping the bytes on the floor.
|
|
1589
|
+
await appendNativeAudioParts(parts, input?.nativeAudioFiles, logPrefix);
|
|
1536
1590
|
return parts;
|
|
1537
1591
|
}
|
|
1538
1592
|
//# sourceMappingURL=utils.js.map
|
|
@@ -6,6 +6,7 @@ import os from "os";
|
|
|
6
6
|
import { AIProviderName, ErrorCategory, ErrorSeverity, } from "../../constants/enums.js";
|
|
7
7
|
import { BaseProvider } from "../../core/baseProvider.js";
|
|
8
8
|
import { unwrapImagePayload } from "../../adapters/imageFormatSupport.js";
|
|
9
|
+
import { appendNativeAudioParts } from "../googleNativeGemini3/utils.js";
|
|
9
10
|
import { getMimeTypeForExtension } from "../../processors/config/mimeConstants.js";
|
|
10
11
|
import { DEFAULT_GEMINI_STREAM_TIMEOUT_MS, DEFAULT_MAX_STEPS, DEFAULT_TOOL_EXECUTION_TIMEOUT_MS, DEFAULT_TOOL_MAX_RETRIES, GLOBAL_LOCATION_MODELS, IMAGE_GENERATION_MODELS, TOOL_STORAGE_TIMEOUT_MS, } from "../../core/constants.js";
|
|
11
12
|
import { ModelConfigurationManager } from "../../core/modelConfiguration.js";
|
|
@@ -1403,6 +1404,7 @@ export class GoogleVertexProvider extends BaseProvider {
|
|
|
1403
1404
|
});
|
|
1404
1405
|
}
|
|
1405
1406
|
}
|
|
1407
|
+
await appendNativeAudioParts(userParts, multimodalInput?.nativeAudioFiles, "[GoogleVertex]");
|
|
1406
1408
|
// Add images as inlineData parts if present
|
|
1407
1409
|
if (multimodalInput?.images && multimodalInput.images.length > 0) {
|
|
1408
1410
|
logger.debug(`[GoogleVertex] Processing ${multimodalInput.images.length} image(s) for native stream`);
|
|
@@ -2408,6 +2410,7 @@ export class GoogleVertexProvider extends BaseProvider {
|
|
|
2408
2410
|
});
|
|
2409
2411
|
}
|
|
2410
2412
|
}
|
|
2413
|
+
await appendNativeAudioParts(userParts, multimodalInput?.nativeAudioFiles, "[GoogleVertex]");
|
|
2411
2414
|
// Add images as inlineData parts if present
|
|
2412
2415
|
if (multimodalInput?.images && multimodalInput.images.length > 0) {
|
|
2413
2416
|
logger.debug(`[GoogleVertex] Processing ${multimodalInput.images.length} image(s) for native generate`);
|
package/dist/lib/types/file.d.ts
CHANGED
|
@@ -22,6 +22,36 @@ export type VisionImageConversion = {
|
|
|
22
22
|
/** True when the bytes were re-encoded; false when they were left alone. */
|
|
23
23
|
readonly converted: boolean;
|
|
24
24
|
};
|
|
25
|
+
/**
|
|
26
|
+
* Outcome of an audio-compatibility pass over one file.
|
|
27
|
+
*
|
|
28
|
+
* See `adapters/audioFormatSupport.ts`. As with images, `converted` is false
|
|
29
|
+
* both when the container was already acceptable and when nothing could
|
|
30
|
+
* re-encode it, so it is not a success flag — the caller decides what to do
|
|
31
|
+
* from the resulting `mimeType`.
|
|
32
|
+
*/
|
|
33
|
+
export type AudioConversionResult = {
|
|
34
|
+
readonly buffer: Buffer;
|
|
35
|
+
readonly mimeType: string;
|
|
36
|
+
/** True when the bytes were re-encoded; false when they were left alone. */
|
|
37
|
+
readonly converted: boolean;
|
|
38
|
+
};
|
|
39
|
+
/**
|
|
40
|
+
* One audio file destined for native delivery to a provider.
|
|
41
|
+
*
|
|
42
|
+
* Carries the bytes rather than a path because the decision to send audio is
|
|
43
|
+
* made per provider, after detection has already read the file — re-reading it
|
|
44
|
+
* from disk at dispatch time would be a second read of something already in
|
|
45
|
+
* memory.
|
|
46
|
+
*/
|
|
47
|
+
export type MultimodalAudioEntry = {
|
|
48
|
+
/** Raw audio bytes, as detected. */
|
|
49
|
+
buffer: Buffer;
|
|
50
|
+
/** Display name; may be a full path, so log only its basename. */
|
|
51
|
+
filename: string;
|
|
52
|
+
/** Detected MIME type of `buffer`. */
|
|
53
|
+
mimeType: string;
|
|
54
|
+
};
|
|
25
55
|
/**
|
|
26
56
|
* Broad category a file format belongs to, as a human would name it.
|
|
27
57
|
*
|
|
@@ -445,6 +475,17 @@ export type FileDetectorOptions = {
|
|
|
445
475
|
* hint (the lazy FileReferenceRegistry path has its own hint-handling).
|
|
446
476
|
*/
|
|
447
477
|
mimetypeHint?: string;
|
|
478
|
+
/**
|
|
479
|
+
* Caller-provided filename hint, the companion to {@link mimetypeHint}.
|
|
480
|
+
*
|
|
481
|
+
* The unified file path unwraps a `FileWithMetadata` to its `buffer` before
|
|
482
|
+
* detection runs, so the object's `filename` is gone by the time extension
|
|
483
|
+
* resolution looks for one — and TAR in particular cannot be identified any
|
|
484
|
+
* other way, because its "ustar" marker sits at byte 257 rather than at
|
|
485
|
+
* offset 0. Passing the name alongside the bytes keeps `.odp`, `.rtf` and
|
|
486
|
+
* `.tar` routed to the processors that can actually read them.
|
|
487
|
+
*/
|
|
488
|
+
filenameHint?: string;
|
|
448
489
|
};
|
|
449
490
|
/**
|
|
450
491
|
* Google AI Studio Files API types
|
|
@@ -17,7 +17,7 @@ import type { AvatarOptions, AvatarResult } from "./avatar.js";
|
|
|
17
17
|
import type { MusicOptions, MusicResult } from "./music.js";
|
|
18
18
|
import type { StandardRecord, ValidationSchema, ZodUnknownSchema } from "./aliases.js";
|
|
19
19
|
import type { NeurolinkCredentials } from "./providers.js";
|
|
20
|
-
import type { CSVProcessorOptions, FileWithMetadata } from "./file.js";
|
|
20
|
+
import type { CSVProcessorOptions, FileWithMetadata, MultimodalAudioEntry } from "./file.js";
|
|
21
21
|
import type { WorkflowConfig } from "./workflow.js";
|
|
22
22
|
import type { Schema, Tool, ToolChoice } from "./tools.js";
|
|
23
23
|
import type { StepResult, LanguageModel } from "./providers.js";
|
|
@@ -56,6 +56,17 @@ export type GenerateOptions = {
|
|
|
56
56
|
csvFiles?: Array<Buffer | string>;
|
|
57
57
|
pdfFiles?: Array<Buffer | string>;
|
|
58
58
|
audioFiles?: Array<Buffer | string>;
|
|
59
|
+
/**
|
|
60
|
+
* Audio whose bytes should be delivered to the provider, populated during
|
|
61
|
+
* detection rather than by callers.
|
|
62
|
+
*
|
|
63
|
+
* Separate from `audioFiles` above, which is the caller-facing input that
|
|
64
|
+
* yields a metadata summary. This one carries the decoded bytes forward so
|
|
65
|
+
* a provider that can actually listen receives the audio instead of a
|
|
66
|
+
* description of it; providers that cannot fall back to the summary and
|
|
67
|
+
* this is ignored.
|
|
68
|
+
*/
|
|
69
|
+
nativeAudioFiles?: MultimodalAudioEntry[];
|
|
59
70
|
videoFiles?: Array<Buffer | string>;
|
|
60
71
|
files?: Array<Buffer | string | FileWithMetadata>;
|
|
61
72
|
content?: Content[];
|
|
@@ -741,7 +741,26 @@ export type ProcessedVideo = ProcessedFileBase & {
|
|
|
741
741
|
/**
|
|
742
742
|
* Supported archive format identifiers.
|
|
743
743
|
*/
|
|
744
|
-
export type ArchiveFormat = "zip" | "tar" | "tar.gz" | "tar.bz2" | "gz" | "rar" | "7z";
|
|
744
|
+
export type ArchiveFormat = "zip" | "tar" | "tar.gz" | "tar.bz2" | "gz" | "bz2" | "xz" | "zst" | "rar" | "7z";
|
|
745
|
+
/**
|
|
746
|
+
* Outcome of decompressing a single-stream archive (.bz2, .xz, .zst).
|
|
747
|
+
*
|
|
748
|
+
* A plain `Buffer | null` collapsed two very different failures into one: a
|
|
749
|
+
* machine that has no `xz` installed and a `.xz` file that is corrupt both
|
|
750
|
+
* returned null, and the caller reported both as "the command is unavailable on
|
|
751
|
+
* this machine" — actively misleading for the second. The reason is carried so
|
|
752
|
+
* the message can match the fact.
|
|
753
|
+
*/
|
|
754
|
+
export type ArchiveDecompressionResult = {
|
|
755
|
+
readonly status: "ok";
|
|
756
|
+
readonly buffer: Buffer;
|
|
757
|
+
} | {
|
|
758
|
+
readonly status: "tool-unavailable";
|
|
759
|
+
} | {
|
|
760
|
+
readonly status: "too-large";
|
|
761
|
+
} | {
|
|
762
|
+
readonly status: "failed";
|
|
763
|
+
};
|
|
745
764
|
/**
|
|
746
765
|
* Metadata about an individual entry within an archive.
|
|
747
766
|
*/
|
|
@@ -6,6 +6,7 @@ import type { NeuroLink } from "../neurolink.js";
|
|
|
6
6
|
import { AIProviderName, AnthropicModels, BedrockModels, DeepSeekModels, GoogleAIModels, LlamaCppModels, LMStudioModels, NvidiaNimModels, OpenAIModels, VertexModels } from "../constants/enums.js";
|
|
7
7
|
import type { ValidationSchema } from "./aliases.js";
|
|
8
8
|
import type { EnhancedGenerateResult, GenerateResult, TextGenerationOptions } from "./generate.js";
|
|
9
|
+
import type { MultimodalAudioEntry } from "./file.js";
|
|
9
10
|
import type { StreamOptions, StreamResult } from "./stream.js";
|
|
10
11
|
import type { ExternalMCPToolInfo } from "./externalMcp.js";
|
|
11
12
|
import type { ClaudeSubscriptionTier, AnthropicAuthMethod, AnthropicAuthConfig, SubscriptionInfo, OAuthToken } from "./subscription.js";
|
|
@@ -1821,6 +1822,12 @@ export type GeminiMultimodalInput = {
|
|
|
1821
1822
|
data: Buffer | string;
|
|
1822
1823
|
altText?: string;
|
|
1823
1824
|
}>;
|
|
1825
|
+
/**
|
|
1826
|
+
* Audio collected during file detection, carried through to the native
|
|
1827
|
+
* request as `inlineData`. Distinct from the user-facing `audioFiles`: these
|
|
1828
|
+
* are already-materialised bytes with a resolved mime type.
|
|
1829
|
+
*/
|
|
1830
|
+
nativeAudioFiles?: MultimodalAudioEntry[];
|
|
1824
1831
|
};
|
|
1825
1832
|
/**
|
|
1826
1833
|
* Internal helpers used by the conversation-history builder in
|
|
@@ -17,6 +17,13 @@ import type { FileDetectorOptions, FileInput, FileProcessingResult } from "../ty
|
|
|
17
17
|
export declare class FileDetector {
|
|
18
18
|
static readonly DEFAULT_NETWORK_TIMEOUT = 30000;
|
|
19
19
|
static readonly DEFAULT_HEAD_TIMEOUT = 5000;
|
|
20
|
+
/**
|
|
21
|
+
* Ceiling on an in-process document parse (unzip + XML walk). Generous
|
|
22
|
+
* relative to the work, because the cost of firing early on a large but
|
|
23
|
+
* legitimate file is a lost extraction, while the cost of never firing is a
|
|
24
|
+
* held request.
|
|
25
|
+
*/
|
|
26
|
+
static readonly DEFAULT_DOCUMENT_TIMEOUT = 30000;
|
|
20
27
|
/**
|
|
21
28
|
* Auto-detect file type and process in one call
|
|
22
29
|
*
|
|
@@ -80,6 +87,26 @@ export declare class FileDetector {
|
|
|
80
87
|
* Stops at first strategy with confidence >= threshold (default: 80%)
|
|
81
88
|
*/
|
|
82
89
|
private static detect;
|
|
90
|
+
/**
|
|
91
|
+
* Fill in `extension` from the input's name when detection did not set it.
|
|
92
|
+
*
|
|
93
|
+
* Content-based strategies identify a type from magic bytes and legitimately
|
|
94
|
+
* have no extension to report, so they return null. That is fine for the type
|
|
95
|
+
* itself but not for routing: several processors are chosen by extension
|
|
96
|
+
* *after* detection has settled the type, because one routing type covers
|
|
97
|
+
* several formats — `docx` covers .docx, .odt and .rtf.
|
|
98
|
+
*
|
|
99
|
+
* With a null extension those branches were unreachable. An .rtf scored high
|
|
100
|
+
* on its `{\\rtf1` signature, arrived as type "docx" with no extension, and
|
|
101
|
+
* fell through to the Word processor, which cannot read RTF — so a file whose
|
|
102
|
+
* dedicated processor extracts it perfectly reported "Could not extract
|
|
103
|
+
* content". The extension was known the whole time; it was simply dropped on
|
|
104
|
+
* the way through.
|
|
105
|
+
*
|
|
106
|
+
* Only fills a gap — a strategy that did determine an extension keeps it, so
|
|
107
|
+
* content still wins over a lying filename.
|
|
108
|
+
*/
|
|
109
|
+
private static withResolvedExtension;
|
|
83
110
|
/**
|
|
84
111
|
* Load file content from various sources
|
|
85
112
|
*/
|
|
@@ -290,6 +290,13 @@ export class FileDetector {
|
|
|
290
290
|
// These default ensure consistent timeout behavior across all file-detection logic.
|
|
291
291
|
static DEFAULT_NETWORK_TIMEOUT = 30000; // 30 seconds
|
|
292
292
|
static DEFAULT_HEAD_TIMEOUT = 5000; // 5 seconds
|
|
293
|
+
/**
|
|
294
|
+
* Ceiling on an in-process document parse (unzip + XML walk). Generous
|
|
295
|
+
* relative to the work, because the cost of firing early on a large but
|
|
296
|
+
* legitimate file is a lost extraction, while the cost of never firing is a
|
|
297
|
+
* held request.
|
|
298
|
+
*/
|
|
299
|
+
static DEFAULT_DOCUMENT_TIMEOUT = 30000; // 30 seconds
|
|
293
300
|
/**
|
|
294
301
|
* Auto-detect file type and process in one call
|
|
295
302
|
*
|
|
@@ -404,7 +411,14 @@ export class FileDetector {
|
|
|
404
411
|
if (Buffer.isBuffer(input)) {
|
|
405
412
|
return "buffer";
|
|
406
413
|
}
|
|
407
|
-
|
|
414
|
+
// Everything left is a `FileWithMetadata`, which states its own name, and
|
|
415
|
+
// `withResolvedExtension` reads this to recover an extension when a
|
|
416
|
+
// content-based strategy reported none. Falling straight through to
|
|
417
|
+
// "unknown-input" threw that name away, so an `.odp`, `.rtf` or `.tar`
|
|
418
|
+
// supplied as bytes-plus-name lost the extension its processor routes on.
|
|
419
|
+
// Still defensive about the value: the type says required, callers are
|
|
420
|
+
// untyped JavaScript often enough.
|
|
421
|
+
return input?.filename || "unknown-input";
|
|
408
422
|
}
|
|
409
423
|
/**
|
|
410
424
|
* Derive byte size from FileInput for tracing.
|
|
@@ -661,7 +675,7 @@ export class FileDetector {
|
|
|
661
675
|
source: FileDetector.deriveInputSource(input),
|
|
662
676
|
metadata: {
|
|
663
677
|
confidence: 95,
|
|
664
|
-
filename: FileDetector.deriveInputFilename(input),
|
|
678
|
+
filename: options?.filenameHint || FileDetector.deriveInputFilename(input),
|
|
665
679
|
size: FileDetector.deriveInputSize(input),
|
|
666
680
|
},
|
|
667
681
|
};
|
|
@@ -684,13 +698,68 @@ export class FileDetector {
|
|
|
684
698
|
}
|
|
685
699
|
if (result.metadata.confidence >= confidenceThreshold) {
|
|
686
700
|
logger.info(`[FileDetector] Type: ${result.type} (${result.metadata.confidence}%)`);
|
|
687
|
-
return result;
|
|
701
|
+
return FileDetector.withResolvedExtension(result, input, options);
|
|
688
702
|
}
|
|
689
703
|
}
|
|
690
704
|
// Below-threshold detection is the common case for any file under the
|
|
691
705
|
// ContentHeuristic ceiling — a debug detail, not a warning-worthy anomaly.
|
|
692
706
|
logger.debug(`[FileDetector] Best-effort type below threshold: ${best?.type ?? "unknown"} (${best?.metadata.confidence ?? 0}%, threshold ${confidenceThreshold}%)`);
|
|
693
|
-
return best;
|
|
707
|
+
return FileDetector.withResolvedExtension(best, input, options);
|
|
708
|
+
}
|
|
709
|
+
/**
|
|
710
|
+
* Fill in `extension` from the input's name when detection did not set it.
|
|
711
|
+
*
|
|
712
|
+
* Content-based strategies identify a type from magic bytes and legitimately
|
|
713
|
+
* have no extension to report, so they return null. That is fine for the type
|
|
714
|
+
* itself but not for routing: several processors are chosen by extension
|
|
715
|
+
* *after* detection has settled the type, because one routing type covers
|
|
716
|
+
* several formats — `docx` covers .docx, .odt and .rtf.
|
|
717
|
+
*
|
|
718
|
+
* With a null extension those branches were unreachable. An .rtf scored high
|
|
719
|
+
* on its `{\\rtf1` signature, arrived as type "docx" with no extension, and
|
|
720
|
+
* fell through to the Word processor, which cannot read RTF — so a file whose
|
|
721
|
+
* dedicated processor extracts it perfectly reported "Could not extract
|
|
722
|
+
* content". The extension was known the whole time; it was simply dropped on
|
|
723
|
+
* the way through.
|
|
724
|
+
*
|
|
725
|
+
* Only fills a gap — a strategy that did determine an extension keeps it, so
|
|
726
|
+
* content still wins over a lying filename.
|
|
727
|
+
*/
|
|
728
|
+
static withResolvedExtension(result, input, options) {
|
|
729
|
+
if (!result) {
|
|
730
|
+
return result;
|
|
731
|
+
}
|
|
732
|
+
// The caller's hint outranks a name derived from the input, because on the
|
|
733
|
+
// unified path the input has already been unwrapped to a bare Buffer and
|
|
734
|
+
// derives to the literal "buffer" — carrying no extension at all.
|
|
735
|
+
const filename = options?.filenameHint ||
|
|
736
|
+
result.metadata?.filename ||
|
|
737
|
+
FileDetector.deriveInputFilename(input);
|
|
738
|
+
if (!filename) {
|
|
739
|
+
return result;
|
|
740
|
+
}
|
|
741
|
+
// Split on both separators so a Windows-style path on a POSIX host still
|
|
742
|
+
// yields its basename, then take the final suffix.
|
|
743
|
+
const base = filename.split(/[\\/]/).pop() ?? filename;
|
|
744
|
+
const dot = base.lastIndexOf(".");
|
|
745
|
+
const extension = result.extension ??
|
|
746
|
+
(dot > 0 && dot < base.length - 1
|
|
747
|
+
? base.slice(dot + 1).toLowerCase()
|
|
748
|
+
: null);
|
|
749
|
+
// The name is carried alongside the extension for the same reason. A
|
|
750
|
+
// content strategy reports no filename, so every processor keyed on one
|
|
751
|
+
// received the literal fallback "archive" — and archive format detection
|
|
752
|
+
// reads the name, because TAR has no magic bytes at offset 0 (its "ustar"
|
|
753
|
+
// marker sits at byte 257). A .tar therefore arrived as an unidentifiable
|
|
754
|
+
// archive and reported "Could not extract content", while the same bytes
|
|
755
|
+
// handed to the processor WITH their name extract perfectly.
|
|
756
|
+
const metadata = result.metadata && !result.metadata.filename
|
|
757
|
+
? { ...result.metadata, filename: base }
|
|
758
|
+
: result.metadata;
|
|
759
|
+
if (extension === result.extension && metadata === result.metadata) {
|
|
760
|
+
return result;
|
|
761
|
+
}
|
|
762
|
+
return { ...result, extension, metadata };
|
|
694
763
|
}
|
|
695
764
|
/**
|
|
696
765
|
* Load file content from various sources
|
|
@@ -1343,6 +1412,40 @@ export class FileDetector {
|
|
|
1343
1412
|
static async processPptxFile(content, detection) {
|
|
1344
1413
|
const pptxFilename = detection.metadata.filename || "presentation";
|
|
1345
1414
|
try {
|
|
1415
|
+
// ODP is an OpenDocument package, not OOXML — the PPTX reader finds no
|
|
1416
|
+
// ppt/slides parts in it and returns nothing. It reaches this branch at
|
|
1417
|
+
// all because one routing type ("pptx") covers every presentation
|
|
1418
|
+
// format, the same way "docx" covers .odt.
|
|
1419
|
+
if (detection.extension?.toLowerCase() === "odp") {
|
|
1420
|
+
const { openDocumentProcessor } = await import("../processors/document/OpenDocumentProcessor.js");
|
|
1421
|
+
// Bounded per the project's async-timeout guideline: this unzips and
|
|
1422
|
+
// parses attacker-supplied bytes, and a stalled parse would otherwise
|
|
1423
|
+
// hold the request open with no ceiling. On timeout the throw lands in
|
|
1424
|
+
// this block's existing catch, which degrades to the placeholder.
|
|
1425
|
+
const odpResult = await withTimeout(openDocumentProcessor.processFile({
|
|
1426
|
+
id: pptxFilename,
|
|
1427
|
+
name: pptxFilename,
|
|
1428
|
+
mimetype: detection.mimeType ||
|
|
1429
|
+
"application/vnd.oasis.opendocument.presentation",
|
|
1430
|
+
size: content.length,
|
|
1431
|
+
buffer: content,
|
|
1432
|
+
}), FileDetector.DEFAULT_DOCUMENT_TIMEOUT);
|
|
1433
|
+
// Gated on success rather than on text, because a presentation of
|
|
1434
|
+
// nothing but images is a legitimate ODP that extracts to an empty
|
|
1435
|
+
// string. Requiring text sent that file on to the PPTX reader, which
|
|
1436
|
+
// cannot read OpenDocument at all — so a successful extraction was
|
|
1437
|
+
// discarded in favour of a guaranteed failure. Matches how the ODT and
|
|
1438
|
+
// ODS branches degrade.
|
|
1439
|
+
if (odpResult.success && odpResult.data) {
|
|
1440
|
+
return {
|
|
1441
|
+
type: "pptx",
|
|
1442
|
+
content: odpResult.data.textContent ||
|
|
1443
|
+
FileDetector.formatInformativePlaceholder("Presentation", pptxFilename, content, detection),
|
|
1444
|
+
mimeType: detection.mimeType,
|
|
1445
|
+
metadata: detection.metadata,
|
|
1446
|
+
};
|
|
1447
|
+
}
|
|
1448
|
+
}
|
|
1346
1449
|
const { PptxProcessor } = await import("../processors/document/PptxProcessor.js");
|
|
1347
1450
|
const pptxResult = await PptxProcessor.extractText(content);
|
|
1348
1451
|
if (pptxResult) {
|
|
@@ -2003,13 +2106,33 @@ class MagicBytesStrategy {
|
|
|
2003
2106
|
if (input.length >= 6 && input.toString("latin1", 0, 5) === "#!AMR") {
|
|
2004
2107
|
return this.result("audio", "audio/amr", 95);
|
|
2005
2108
|
}
|
|
2006
|
-
// JPEG 2000: 12-byte signature box
|
|
2007
|
-
|
|
2109
|
+
// JPEG 2000: the full 12-byte signature box, trailing 0D 0A 87 0A
|
|
2110
|
+
// included. Those four bytes are a deliberate line-ending probe — CR LF, a
|
|
2111
|
+
// high byte, LF — that any transfer which mangles newlines or strips the
|
|
2112
|
+
// eighth bit will visibly corrupt, so checking only the length and brand
|
|
2113
|
+
// accepts exactly the damaged files the signature exists to reject.
|
|
2114
|
+
if (input.length >= 12 &&
|
|
2008
2115
|
input[0] === 0x00 &&
|
|
2009
2116
|
input[1] === 0x00 &&
|
|
2010
2117
|
input[2] === 0x00 &&
|
|
2011
2118
|
input[3] === 0x0c &&
|
|
2012
|
-
input.toString("latin1", 4, 8) === "jP "
|
|
2119
|
+
input.toString("latin1", 4, 8) === "jP " &&
|
|
2120
|
+
input[8] === 0x0d &&
|
|
2121
|
+
input[9] === 0x0a &&
|
|
2122
|
+
input[10] === 0x87 &&
|
|
2123
|
+
input[11] === 0x0a) {
|
|
2124
|
+
return this.result("image", "image/jp2", 95);
|
|
2125
|
+
}
|
|
2126
|
+
// The other shape JPEG 2000 ships in: a bare codestream (.j2k/.j2c) opening
|
|
2127
|
+
// with the SOC + SIZ markers. `ImageProcessor.detectImageType` learned both
|
|
2128
|
+
// shapes; this strategy knew only the container, so a codestream uploaded
|
|
2129
|
+
// as bytes-plus-filename was typed "unknown" and delivered as a binary
|
|
2130
|
+
// blob — the codestream branch over there was unreachable from this path.
|
|
2131
|
+
if (input.length >= 4 &&
|
|
2132
|
+
input[0] === 0xff &&
|
|
2133
|
+
input[1] === 0x4f &&
|
|
2134
|
+
input[2] === 0xff &&
|
|
2135
|
+
input[3] === 0x51) {
|
|
2013
2136
|
return this.result("image", "image/jp2", 95);
|
|
2014
2137
|
}
|
|
2015
2138
|
// MP3: ID3 tag
|
|
@@ -468,6 +468,37 @@ export class ImageProcessor {
|
|
|
468
468
|
if (isoBmffMimeType) {
|
|
469
469
|
return isoBmffMimeType;
|
|
470
470
|
}
|
|
471
|
+
// JPEG 2000, in both shapes it ships in. The JP2 container opens with
|
|
472
|
+
// the 12-byte signature box `00 00 00 0C 6A 50 20 20 0D 0A 87 0A`; a
|
|
473
|
+
// bare codestream (.j2k/.j2c) opens with the SOC+SIZ markers FF 4F FF
|
|
474
|
+
// 51. Checked BEFORE ICO because the container's first three bytes are
|
|
475
|
+
// 00 00 00, which is one byte away from ICO's 00 00 01 00 and shares
|
|
476
|
+
// its leading zeros — an ordering mistake here would classify every
|
|
477
|
+
// JPEG 2000 as an icon rather than merely failing to recognise it.
|
|
478
|
+
// All twelve bytes are checked, not just the length and brand: the
|
|
479
|
+
// trailing 0D 0A 87 0A is the signature's whole point. It is a
|
|
480
|
+
// line-ending probe — CR LF, a high byte, LF — that a transfer which
|
|
481
|
+
// mangles newlines or strips the eighth bit will visibly corrupt, so
|
|
482
|
+
// skipping it accepts exactly the damaged files it exists to reject.
|
|
483
|
+
if (input.length >= 12 &&
|
|
484
|
+
input[0] === 0x00 &&
|
|
485
|
+
input[1] === 0x00 &&
|
|
486
|
+
input[2] === 0x00 &&
|
|
487
|
+
input[3] === 0x0c &&
|
|
488
|
+
input.subarray(4, 8).toString("latin1") === "jP " &&
|
|
489
|
+
input[8] === 0x0d &&
|
|
490
|
+
input[9] === 0x0a &&
|
|
491
|
+
input[10] === 0x87 &&
|
|
492
|
+
input[11] === 0x0a) {
|
|
493
|
+
return "image/jp2";
|
|
494
|
+
}
|
|
495
|
+
if (input.length >= 4 &&
|
|
496
|
+
input[0] === 0xff &&
|
|
497
|
+
input[1] === 0x4f &&
|
|
498
|
+
input[2] === 0xff &&
|
|
499
|
+
input[3] === 0x51) {
|
|
500
|
+
return "image/jp2";
|
|
501
|
+
}
|
|
471
502
|
// ICO: 00 00 01 00 (icon type=1)
|
|
472
503
|
if (input[0] === 0x00 &&
|
|
473
504
|
input[1] === 0x00 &&
|
|
@@ -37,15 +37,6 @@ export declare function mergeMediaFileAliases<TFile>(input: {
|
|
|
37
37
|
audioFiles?: Array<Buffer | string>;
|
|
38
38
|
videoFiles?: Array<Buffer | string>;
|
|
39
39
|
}): void;
|
|
40
|
-
/**
|
|
41
|
-
* Process the unified files array with auto-detection.
|
|
42
|
-
* Handles lazy file registration, full processing, and preview injection.
|
|
43
|
-
*
|
|
44
|
-
* Exported so providers that bypass BaseProvider.generate() (e.g.
|
|
45
|
-
* GoogleVertex's native @google/genai path) can still preprocess
|
|
46
|
-
* `input.files` — without this, mimetype-hint and text-file inputs
|
|
47
|
-
* would silently never reach the model on those paths.
|
|
48
|
-
*/
|
|
49
40
|
export declare function processUnifiedFilesArray(options: GenerateOptions, maxSize: number, provider: string): Promise<void>;
|
|
50
41
|
/**
|
|
51
42
|
* Build multimodal message array with image support
|