@juspay/neurolink 13.8.16 → 13.8.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -2
- package/README.md +1 -1
- package/dist/action/actionExecutor.d.ts +33 -3
- package/dist/action/actionExecutor.js +71 -15
- package/dist/action/actionInputs.js +4 -1
- package/dist/action/cliVersion.d.ts +25 -0
- package/dist/action/cliVersion.js +26 -0
- package/dist/action/index.d.ts +2 -1
- package/dist/action/index.js +2 -1
- package/dist/adapters/video/directorPipeline.js +46 -36
- package/dist/adapters/video/ffmpegAdapter.d.ts +17 -0
- package/dist/adapters/video/ffmpegAdapter.js +32 -5
- package/dist/adapters/video/frameExtractor.js +1 -1
- package/dist/adapters/video/videoMerger.js +5 -2
- package/dist/agent/arithmeticEvaluator.d.ts +32 -0
- package/dist/agent/arithmeticEvaluator.js +349 -0
- package/dist/agent/directTools.d.ts +6 -18
- package/dist/agent/directTools.js +17 -56
- package/dist/artifacts/artifactStore.d.ts +22 -1
- package/dist/artifacts/artifactStore.js +66 -6
- package/dist/auth/anthropicOAuth.d.ts +26 -1
- package/dist/auth/anthropicOAuth.js +81 -5
- package/dist/auth/secureFiles.d.ts +37 -0
- package/dist/auth/secureFiles.js +95 -0
- package/dist/auth/tokenStore.d.ts +43 -6
- package/dist/auth/tokenStore.js +260 -79
- package/dist/avatar/providers/DIDAvatar.js +49 -45
- package/dist/avatar/providers/ReplicateAvatar.js +33 -40
- package/dist/browser/neurolink.min.js +485 -558
- package/dist/cli/commands/auth.js +112 -70
- package/dist/cli/commands/proxy.d.ts +4 -0
- package/dist/cli/commands/proxy.js +128 -13
- package/dist/cli/commands/serve.d.ts +10 -1
- package/dist/cli/commands/serve.js +98 -17
- package/dist/cli/commands/server.js +17 -5
- package/dist/cli/index.js +146 -5
- package/dist/cli/loop/optionsSchema.d.ts +1 -1
- package/dist/cli/utils/envManager.js +17 -3
- package/dist/cli/utils/serverUtils.d.ts +39 -3
- package/dist/cli/utils/serverUtils.js +63 -6
- package/dist/core/baseProvider.js +7 -1
- package/dist/core/modules/MessageBuilder.js +5 -0
- package/dist/core/modules/ToolsManager.d.ts +46 -1
- package/dist/core/modules/ToolsManager.js +190 -43
- package/dist/files/fileReferenceRegistry.d.ts +2 -23
- package/dist/files/fileReferenceRegistry.js +55 -5
- package/dist/hitl/hitlManager.js +1 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +6 -1
- package/dist/mcp/auth/tokenStorage.d.ts +7 -0
- package/dist/mcp/auth/tokenStorage.js +23 -4
- package/dist/mcp/externalServerManager.d.ts +13 -3
- package/dist/mcp/externalServerManager.js +154 -136
- package/dist/mcp/index.d.ts +2 -0
- package/dist/mcp/index.js +3 -0
- package/dist/mcp/mcpClientFactory.d.ts +14 -0
- package/dist/mcp/mcpClientFactory.js +70 -7
- package/dist/mcp/projectConfigTrust.d.ts +72 -0
- package/dist/mcp/projectConfigTrust.js +237 -0
- package/dist/mcp/transportUrlPolicy.d.ts +39 -0
- package/dist/mcp/transportUrlPolicy.js +172 -0
- package/dist/music/providers/BeatovenMusic.js +35 -46
- package/dist/music/providers/ReplicateMusic.js +33 -40
- package/dist/neurolink.d.ts +16 -1
- package/dist/neurolink.js +64 -11
- package/dist/processors/archive/ArchiveProcessor.js +15 -1
- package/dist/processors/archive/zipEntryIndex.d.ts +22 -0
- package/dist/processors/archive/zipEntryIndex.js +33 -0
- package/dist/processors/base/BaseFileProcessor.d.ts +9 -21
- package/dist/processors/base/BaseFileProcessor.js +40 -111
- package/dist/processors/data/YamlProcessor.d.ts +13 -1
- package/dist/processors/data/YamlProcessor.js +93 -2
- package/dist/processors/document/OpenDocumentProcessor.js +7 -2
- package/dist/processors/document/PptxProcessor.js +12 -7
- package/dist/processors/media/VideoProcessor.js +93 -15
- package/dist/proxy/inboundAuth.d.ts +89 -0
- package/dist/proxy/inboundAuth.js +219 -0
- package/dist/proxy/tokenRefresh.d.ts +6 -0
- package/dist/proxy/tokenRefresh.js +13 -1
- package/dist/proxy/updateState.js +4 -4
- package/dist/rag/document/loaders.d.ts +0 -6
- package/dist/rag/document/loaders.js +16 -10
- package/dist/rag/ragIntegration.d.ts +2 -2
- package/dist/rag/ragIntegration.js +35 -10
- package/dist/rag/retrieval/vectorQueryTool.d.ts +7 -0
- package/dist/rag/retrieval/vectorQueryTool.js +54 -17
- package/dist/server/abstract/baseServerAdapter.d.ts +43 -0
- package/dist/server/abstract/baseServerAdapter.js +119 -6
- package/dist/server/adapters/expressAdapter.d.ts +3 -0
- package/dist/server/adapters/expressAdapter.js +83 -8
- package/dist/server/adapters/fastifyAdapter.d.ts +3 -0
- package/dist/server/adapters/fastifyAdapter.js +165 -70
- package/dist/server/adapters/honoAdapter.d.ts +9 -1
- package/dist/server/adapters/honoAdapter.js +99 -21
- package/dist/server/adapters/koaAdapter.d.ts +1 -0
- package/dist/server/adapters/koaAdapter.js +153 -74
- package/dist/server/index.d.ts +2 -0
- package/dist/server/index.js +4 -0
- package/dist/server/middleware/auth.js +6 -2
- package/dist/server/middleware/cache.js +6 -1
- package/dist/server/middleware/rateLimit.js +29 -4
- package/dist/server/routes/agentRoutes.js +24 -11
- package/dist/server/routes/claudeProxyRoutes.d.ts +1 -0
- package/dist/server/routes/claudeProxyRoutes.js +273 -84
- package/dist/server/routes/codexProxyRoutes.d.ts +2 -2
- package/dist/server/routes/codexProxyRoutes.js +125 -40
- package/dist/server/routes/index.d.ts +4 -0
- package/dist/server/routes/index.js +35 -4
- package/dist/server/routes/mcpRoutes.d.ts +6 -2
- package/dist/server/routes/mcpRoutes.js +21 -7
- package/dist/server/routes/memoryRoutes.js +7 -6
- package/dist/server/routes/toolRoutes.d.ts +12 -2
- package/dist/server/routes/toolRoutes.js +33 -3
- package/dist/server/routes/transcribeRoutes.js +60 -13
- package/dist/server/utils/clientAddress.d.ts +23 -0
- package/dist/server/utils/clientAddress.js +65 -0
- package/dist/server/utils/clientError.d.ts +35 -0
- package/dist/server/utils/clientError.js +129 -0
- package/dist/server/utils/expressBody.d.ts +33 -0
- package/dist/server/utils/expressBody.js +353 -0
- package/dist/server/utils/pathMatch.d.ts +21 -0
- package/dist/server/utils/pathMatch.js +41 -0
- package/dist/server/utils/validation.d.ts +9 -0
- package/dist/server/utils/validation.js +9 -0
- package/dist/server/websocket/WebSocketHandler.js +22 -3
- package/dist/server/websocket/frameQueue.d.ts +19 -0
- package/dist/server/websocket/frameQueue.js +41 -3
- package/dist/server/websocket/transcribeWebSocket.d.ts +8 -1
- package/dist/server/websocket/transcribeWebSocket.js +223 -34
- package/dist/skills/skillStores.js +5 -0
- package/dist/tasks/store/fileTaskStore.d.ts +2 -0
- package/dist/tasks/store/fileTaskStore.js +11 -4
- package/dist/types/attachments.d.ts +117 -0
- package/dist/types/attachments.js +15 -0
- package/dist/types/auth.d.ts +6 -0
- package/dist/types/cli.d.ts +4 -0
- package/dist/types/config.d.ts +56 -0
- package/dist/types/file.d.ts +8 -0
- package/dist/types/hitl.d.ts +8 -0
- package/dist/types/index.d.ts +1 -0
- package/dist/types/index.js +1 -0
- package/dist/types/mcp.d.ts +98 -0
- package/dist/types/multimodal.js +0 -41
- package/dist/types/processor.d.ts +14 -1
- package/dist/types/proxy.d.ts +62 -3
- package/dist/types/rag.d.ts +9 -0
- package/dist/types/safeFetch.d.ts +40 -0
- package/dist/types/server.d.ts +255 -10
- package/dist/types/tools.d.ts +57 -0
- package/dist/types/transcribe.d.ts +6 -0
- package/dist/utils/attachmentPolicy.d.ts +89 -0
- package/dist/utils/attachmentPolicy.js +327 -0
- package/dist/utils/dotenvBootstrap.d.ts +48 -19
- package/dist/utils/dotenvBootstrap.js +172 -31
- package/dist/utils/fileDetector.js +115 -65
- package/dist/utils/json/coerce.js +11 -1
- package/dist/utils/logSanitize.d.ts +5 -0
- package/dist/utils/logSanitize.js +8 -0
- package/dist/utils/logger.js +73 -1
- package/dist/utils/messageBuilder.d.ts +2 -2
- package/dist/utils/messageBuilder.js +118 -105
- package/dist/utils/multimodalOptionsBuilder.d.ts +1 -0
- package/dist/utils/multimodalOptionsBuilder.js +2 -0
- package/dist/utils/redis.d.ts +12 -0
- package/dist/utils/redis.js +72 -0
- package/dist/utils/safeFetch.d.ts +22 -1
- package/dist/utils/safeFetch.js +132 -38
- package/dist/utils/safeStoreId.d.ts +19 -0
- package/dist/utils/safeStoreId.js +29 -0
- package/dist/utils/schemaConversion.d.ts +30 -3
- package/dist/utils/schemaConversion.js +457 -84
- package/dist/utils/sizeGuard.js +2 -2
- package/dist/utils/ssrfGuard.d.ts +3 -3
- package/dist/utils/ssrfGuard.js +71 -39
- package/dist/voice/eval/sttEvalStore.d.ts +26 -3
- package/dist/voice/eval/sttEvalStore.js +77 -12
- package/dist/voice/livekit/agentDefinition.d.ts +18 -0
- package/dist/voice/livekit/agentDefinition.js +22 -0
- package/dist/voice/livekit/realtimeMcpTools.d.ts +1 -1
- package/dist/voice/livekit/realtimeMcpTools.js +8 -7
- package/dist/voice/livekit/realtimeVoiceAgent.d.ts +2 -4
- package/dist/voice/livekit/realtimeVoiceAgent.js +18 -7
- package/dist/voice/livekit/voiceAgent.d.ts +2 -2
- package/dist/voice/livekit/voiceAgent.js +2 -1
- package/dist/voice/livekit/xaiSessionWatch.d.ts +0 -5
- package/dist/voice/livekit/xaiSessionWatch.js +30 -0
- package/dist/voice/providers/GeminiLive.js +21 -6
- package/dist/voice/providers/GoogleSTT.js +8 -5
- package/dist/voice/transcribe.d.ts +2 -2
- package/dist/voice/transcribe.js +34 -18
- package/dist/voice/whistle/audio.js +11 -5
- package/docs-site/static/search-index.json +75 -52
- package/package.json +171 -156
- package/scripts/observability/import-openobserve-dashboard.mjs +11 -3
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { open as openAsync,
|
|
1
|
+
import { constants as fsConstants, existsSync, statSync } from "fs";
|
|
2
|
+
import { open as openAsync, stat as statAsync } from "fs/promises";
|
|
3
3
|
import pLimit from "p-limit";
|
|
4
|
-
import {
|
|
4
|
+
import { downloadAttachment, getAttachmentPolicy, readAttachmentFile, resolveAttachmentPath, resolveAttachmentPolicy, } from "./attachmentPolicy.js";
|
|
5
5
|
import { MultimodalLogger, ProviderImageAdapter, } from "../adapters/providerImageAdapter.js";
|
|
6
6
|
import { CONVERSATION_INSTRUCTIONS, STRUCTURED_OUTPUT_INSTRUCTIONS, } from "../config/conversationMemory.js";
|
|
7
7
|
import { getAvailableInputTokens } from "../constants/contextWindows.js";
|
|
@@ -18,6 +18,7 @@ import { needsAudioTranscode, supportsNativeAudio, toProviderCompatibleAudio, }
|
|
|
18
18
|
import { supportsNativeVideo } from "../adapters/videoFormatSupport.js";
|
|
19
19
|
import { getImageCache } from "./imageCache.js";
|
|
20
20
|
import { ImageProcessor, imageUtils } from "./imageProcessor.js";
|
|
21
|
+
import { SIZE_LIMITS_BYTES } from "../processors/config/sizeLimits.js";
|
|
21
22
|
import { logger } from "./logger.js";
|
|
22
23
|
import { looksLikeSvgMarkup } from "./markupSniff.js";
|
|
23
24
|
import { redactUrlForError } from "./logSanitize.js";
|
|
@@ -914,6 +915,13 @@ const PREPROCESSED_FILES = new WeakMap();
|
|
|
914
915
|
* page cache the read usually hits — detection has just read the same bytes.
|
|
915
916
|
*/
|
|
916
917
|
const FILE_READ_TIMEOUT_MS = 30_000;
|
|
918
|
+
/**
|
|
919
|
+
* Ceiling for a local attachment read by path on the unified `files` path and
|
|
920
|
+
* the lazy registry. Matches FileDetector's own 200 MB default, so a file the
|
|
921
|
+
* detector would have accepted is not refused here; `attachments.maxBytes`
|
|
922
|
+
* lowers it.
|
|
923
|
+
*/
|
|
924
|
+
const ATTACHED_FILE_READ_CAP_BYTES = 200 * 1024 * 1024;
|
|
917
925
|
/**
|
|
918
926
|
* How many files may be detected and processed at once.
|
|
919
927
|
*
|
|
@@ -941,7 +949,7 @@ const FILE_PROCESSING_CONCURRENCY = 4;
|
|
|
941
949
|
* materialised by the time detection runs, and re-fetching a remote URL here
|
|
942
950
|
* would issue a second network request behind the caller's back.
|
|
943
951
|
*/
|
|
944
|
-
async function readFileInputBytes(file) {
|
|
952
|
+
async function readFileInputBytes(file, policy) {
|
|
945
953
|
try {
|
|
946
954
|
if (isFileWithMetadata(file)) {
|
|
947
955
|
return file.buffer;
|
|
@@ -950,35 +958,18 @@ async function readFileInputBytes(file) {
|
|
|
950
958
|
return file;
|
|
951
959
|
}
|
|
952
960
|
if (typeof file === "string") {
|
|
953
|
-
|
|
954
|
-
//
|
|
955
|
-
//
|
|
956
|
-
//
|
|
957
|
-
//
|
|
958
|
-
//
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
// in flight) — and it matters, because racing the promise alone leaves
|
|
966
|
-
// the read filling a buffer nobody will collect. Aborting in `finally`
|
|
967
|
-
// covers both exits; after a resolved read it is a no-op.
|
|
968
|
-
//
|
|
969
|
-
// stat-then-read is a TOCTOU window, but a narrow one, and it is strictly
|
|
970
|
-
// better than the unbounded read it replaces.
|
|
971
|
-
const stats = await withTimeout(stat(file), FILE_READ_TIMEOUT_MS);
|
|
972
|
-
if (!stats.isFile()) {
|
|
973
|
-
return null;
|
|
974
|
-
}
|
|
975
|
-
const controller = new AbortController();
|
|
976
|
-
try {
|
|
977
|
-
return await withTimeout(readFile(file, { signal: controller.signal }), FILE_READ_TIMEOUT_MS);
|
|
978
|
-
}
|
|
979
|
-
finally {
|
|
980
|
-
controller.abort();
|
|
981
|
-
}
|
|
961
|
+
// `readAttachmentFile` applies the instance's roots, opens the real path
|
|
962
|
+
// O_NOFOLLOW|O_NONBLOCK (so a FIFO cannot park a worker inside the read
|
|
963
|
+
// syscall — the hang a timeout alone can never reach), refuses anything
|
|
964
|
+
// that is not a regular file, and drains under the size cap through the
|
|
965
|
+
// same descriptor it stat'd. The timeout still bounds a slow or hung
|
|
966
|
+
// mount, where control does return between chunks.
|
|
967
|
+
const read = await withTimeout(readAttachmentFile(file, {
|
|
968
|
+
policy,
|
|
969
|
+
maxBytes: ATTACHED_FILE_READ_CAP_BYTES,
|
|
970
|
+
label: "attached file",
|
|
971
|
+
}), FILE_READ_TIMEOUT_MS);
|
|
972
|
+
return read.buffer;
|
|
982
973
|
}
|
|
983
974
|
}
|
|
984
975
|
catch {
|
|
@@ -1046,7 +1037,7 @@ async function appendDetectedFileResult(result, file, options) {
|
|
|
1046
1037
|
// and is no worse off than before. This matters most where the frames are
|
|
1047
1038
|
// absent — ffmpeg is optional, and without it the summary was everything
|
|
1048
1039
|
// the model saw.
|
|
1049
|
-
const videoBytes = await readFileInputBytes(file);
|
|
1040
|
+
const videoBytes = await readFileInputBytes(file, getAttachmentPolicy(options));
|
|
1050
1041
|
if (videoBytes) {
|
|
1051
1042
|
options.input.nativeVideoFiles = [
|
|
1052
1043
|
...(options.input.nativeVideoFiles || []),
|
|
@@ -1069,7 +1060,7 @@ async function appendDetectedFileResult(result, file, options) {
|
|
|
1069
1060
|
// Carry the bytes forward as well as the summary. Whether they are used is
|
|
1070
1061
|
// decided later, per provider: one that can listen receives the audio, one
|
|
1071
1062
|
// that cannot still gets the summary above and is no worse off than before.
|
|
1072
|
-
const audioBytes = await readFileInputBytes(file);
|
|
1063
|
+
const audioBytes = await readFileInputBytes(file, getAttachmentPolicy(options));
|
|
1073
1064
|
if (audioBytes) {
|
|
1074
1065
|
options.input.nativeAudioFiles = [
|
|
1075
1066
|
...(options.input.nativeAudioFiles || []),
|
|
@@ -1267,6 +1258,9 @@ async function detectFileForUnifiedArray(file, options, provider, genericFileMax
|
|
|
1267
1258
|
provider: provider,
|
|
1268
1259
|
mimetypeHint: fileMimetypeHint,
|
|
1269
1260
|
filenameHint: fileFilenameHint,
|
|
1261
|
+
// The instance's roots / URL rules / size cap, so a path or URL named in
|
|
1262
|
+
// `files` is read under the same policy as every other attachment.
|
|
1263
|
+
attachmentPolicy: getAttachmentPolicy(options),
|
|
1270
1264
|
});
|
|
1271
1265
|
}
|
|
1272
1266
|
/**
|
|
@@ -1379,7 +1373,7 @@ export async function processUnifiedFilesArray(options, maxSize, provider) {
|
|
|
1379
1373
|
// ─── Lazy file registration path ──────────────────────────────
|
|
1380
1374
|
const fileSize = fileRegistry ? getFileSize(file) : 0;
|
|
1381
1375
|
if (fileRegistry && isLazyCandidate[fileIdx]) {
|
|
1382
|
-
const registered = await tryRegisterFileReference(file, fileSize, fileRegistry, fileIdx);
|
|
1376
|
+
const registered = await tryRegisterFileReference(file, fileSize, fileRegistry, fileIdx, getAttachmentPolicy(options));
|
|
1383
1377
|
if (registered) {
|
|
1384
1378
|
logger.info(`[NEUROLINK] File lazily registered: ${filename} (${fileSize} bytes) — deferred processing`);
|
|
1385
1379
|
includedCount++;
|
|
@@ -1486,6 +1480,7 @@ async function processExplicitCsvFiles(options) {
|
|
|
1486
1480
|
const settled = await Promise.allSettled(csvFiles.map((csvFile) => limit(() => FileDetector.detectAndProcess(csvFile, {
|
|
1487
1481
|
allowedTypes: ["csv"],
|
|
1488
1482
|
csvOptions: options.csvOptions,
|
|
1483
|
+
attachmentPolicy: getAttachmentPolicy(options),
|
|
1489
1484
|
}))));
|
|
1490
1485
|
for (let i = 0; i < csvFiles.length; i++) {
|
|
1491
1486
|
const csvFile = csvFiles[i];
|
|
@@ -1652,6 +1647,7 @@ async function processExplicitPdfFiles(options, maxSize, provider) {
|
|
|
1652
1647
|
maxSize,
|
|
1653
1648
|
allowedTypes: ["pdf"],
|
|
1654
1649
|
provider: provider,
|
|
1650
|
+
attachmentPolicy: getAttachmentPolicy(options),
|
|
1655
1651
|
});
|
|
1656
1652
|
if (Buffer.isBuffer(result.content)) {
|
|
1657
1653
|
pdfFiles.push({
|
|
@@ -1782,7 +1778,7 @@ export async function buildMultimodalModelMessages(options, provider, model, rep
|
|
|
1782
1778
|
await processUnifiedFilesArray(options, maxSize, provider);
|
|
1783
1779
|
// Detection can append images (PDF page renders, video keyframes, a HEIC
|
|
1784
1780
|
// photo), so compatibility conversion has to run after it, not before.
|
|
1785
|
-
await normalizeVisionImageFormats(inp, options.imageOptions?.outputFormat);
|
|
1781
|
+
await normalizeVisionImageFormats(inp, options.imageOptions?.outputFormat, getAttachmentPolicy(options));
|
|
1786
1782
|
// Process explicit CSV files array
|
|
1787
1783
|
await processExplicitCsvFiles(options);
|
|
1788
1784
|
// Post-processing budget enforcement and token usage logging
|
|
@@ -1957,7 +1953,11 @@ export async function buildMultimodalModelMessages(options, provider, model, rep
|
|
|
1957
1953
|
// the multimodal converter below, so the bytes were dropped and only the
|
|
1958
1954
|
// metadata summary folded into `text` survived. Exactly the failure this
|
|
1959
1955
|
// change exists to remove, reintroduced through the other door.
|
|
1960
|
-
userContent = await convertContentToProviderFormat(inp.content, provider, model, options.pdfOptions,
|
|
1956
|
+
userContent = await convertContentToProviderFormat(inp.content, provider, model, options.pdfOptions, {
|
|
1957
|
+
audioFiles: inp.nativeAudioFiles ?? [],
|
|
1958
|
+
outputFormat: options.imageOptions?.outputFormat,
|
|
1959
|
+
attachmentPolicy: getAttachmentPolicy(options),
|
|
1960
|
+
});
|
|
1961
1961
|
}
|
|
1962
1962
|
else if ((inp.images && inp.images.length > 0) ||
|
|
1963
1963
|
pdfFiles.length > 0 ||
|
|
@@ -1973,6 +1973,7 @@ export async function buildMultimodalModelMessages(options, provider, model, rep
|
|
|
1973
1973
|
userContent = await convertMultimodalToProviderFormat(inp.text ?? "", inp.images || [], pdfFiles, provider, model, {
|
|
1974
1974
|
audioFiles: inp.nativeAudioFiles ?? [],
|
|
1975
1975
|
outputFormat: options.imageOptions?.outputFormat,
|
|
1976
|
+
attachmentPolicy: getAttachmentPolicy(options),
|
|
1976
1977
|
});
|
|
1977
1978
|
}
|
|
1978
1979
|
else {
|
|
@@ -2066,7 +2067,8 @@ async function appendCsvContentToText(csvItems, baseText) {
|
|
|
2066
2067
|
/**
|
|
2067
2068
|
* Convert advanced content format to provider-specific format
|
|
2068
2069
|
*/
|
|
2069
|
-
async function convertContentToProviderFormat(content, provider, model, pdfOptions,
|
|
2070
|
+
async function convertContentToProviderFormat(content, provider, model, pdfOptions, mediaOptions = {}) {
|
|
2071
|
+
const { audioFiles = [], outputFormat, attachmentPolicy } = mediaOptions;
|
|
2070
2072
|
const textContent = content.find((c) => c.type === "text");
|
|
2071
2073
|
const imageContent = content.filter((c) => c.type === "image");
|
|
2072
2074
|
const pdfContent = content.filter((c) => c.type === "pdf");
|
|
@@ -2129,6 +2131,7 @@ async function convertContentToProviderFormat(content, provider, model, pdfOptio
|
|
|
2129
2131
|
return await convertMultimodalToProviderFormat(text, images, pdfFiles, provider, model, {
|
|
2130
2132
|
audioFiles,
|
|
2131
2133
|
outputFormat,
|
|
2134
|
+
attachmentPolicy,
|
|
2132
2135
|
});
|
|
2133
2136
|
}
|
|
2134
2137
|
/**
|
|
@@ -2144,8 +2147,12 @@ function isInternetUrl(input) {
|
|
|
2144
2147
|
* Download image from URL and convert to base64 data URI
|
|
2145
2148
|
* Rate-limited to 10 downloads per second to prevent DoS
|
|
2146
2149
|
* Uses LRU cache to avoid redundant downloads of the same URL
|
|
2150
|
+
*
|
|
2151
|
+
* The fetch goes through `downloadAttachment`: HTTPS to a public host unless
|
|
2152
|
+
* the instance's policy says otherwise, the connection pinned to the vetted
|
|
2153
|
+
* addresses, redirects re-validated per hop, body bounded while streaming.
|
|
2147
2154
|
*/
|
|
2148
|
-
async function downloadImageFromUrl(url) {
|
|
2155
|
+
async function downloadImageFromUrl(url, policy) {
|
|
2149
2156
|
// Check cache first (before rate limiting)
|
|
2150
2157
|
const cache = getImageCache();
|
|
2151
2158
|
const cached = cache.get(url);
|
|
@@ -2156,40 +2163,19 @@ async function downloadImageFromUrl(url) {
|
|
|
2156
2163
|
// Apply rate limiting only if cache missed
|
|
2157
2164
|
await urlDownloadRateLimiter.acquire();
|
|
2158
2165
|
try {
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
dispatcher: await redirectFollowingDispatcher(5),
|
|
2165
|
-
method: "GET",
|
|
2166
|
-
headersTimeout: 10000, // 10 second timeout for headers
|
|
2167
|
-
bodyTimeout: 30000, // 30 second timeout for body,
|
|
2166
|
+
const downloaded = await downloadAttachment(url, {
|
|
2167
|
+
policy,
|
|
2168
|
+
maxBytes: IMAGE_URL_MAX_BYTES,
|
|
2169
|
+
label: "image URL",
|
|
2170
|
+
timeoutMs: IMAGE_URL_TIMEOUT_MS,
|
|
2168
2171
|
});
|
|
2169
|
-
if (response.statusCode !== 200) {
|
|
2170
|
-
throw new Error(`HTTP ${response.statusCode}: Failed to download image from ${url}`);
|
|
2171
|
-
}
|
|
2172
2172
|
// Get content type from headers
|
|
2173
|
-
const contentType =
|
|
2173
|
+
const contentType = downloaded.contentType || "image/jpeg";
|
|
2174
2174
|
// Validate it's an image
|
|
2175
2175
|
if (!contentType.startsWith("image/")) {
|
|
2176
2176
|
throw new Error(`URL does not point to an image. Content-Type: ${contentType}`);
|
|
2177
2177
|
}
|
|
2178
|
-
|
|
2179
|
-
// misbehaving/malicious server on a user-supplied URL must not be able to
|
|
2180
|
-
// force unbounded memory growth by streaming gigabytes before we ever
|
|
2181
|
-
// check the total (the previous code concat'd everything first).
|
|
2182
|
-
const maxSize = 10 * 1024 * 1024; // 10MB
|
|
2183
|
-
const chunks = [];
|
|
2184
|
-
let totalSize = 0;
|
|
2185
|
-
for await (const chunk of response.body) {
|
|
2186
|
-
totalSize += chunk.length;
|
|
2187
|
-
if (totalSize > maxSize) {
|
|
2188
|
-
throw new Error(`Image too large: exceeds ${maxSize} bytes while downloading from ${url}`);
|
|
2189
|
-
}
|
|
2190
|
-
chunks.push(chunk);
|
|
2191
|
-
}
|
|
2192
|
-
const buffer = Buffer.concat(chunks);
|
|
2178
|
+
const buffer = downloaded.buffer;
|
|
2193
2179
|
// Convert to base64 data URI
|
|
2194
2180
|
const base64 = buffer.toString("base64");
|
|
2195
2181
|
const dataUri = `data:${contentType};base64,${base64}`;
|
|
@@ -2199,9 +2185,13 @@ async function downloadImageFromUrl(url) {
|
|
|
2199
2185
|
}
|
|
2200
2186
|
catch (error) {
|
|
2201
2187
|
MultimodalLogger.logError("URL_DOWNLOAD_FAILED", error, { url });
|
|
2202
|
-
throw new Error(`Failed to download image from ${url}: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
2188
|
+
throw new Error(`Failed to download image from ${redactUrlForError(url)}: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
2203
2189
|
}
|
|
2204
2190
|
}
|
|
2191
|
+
/** 10 MB — the cap `downloadImageFromUrl` has always applied. */
|
|
2192
|
+
const IMAGE_URL_MAX_BYTES = 10 * 1024 * 1024;
|
|
2193
|
+
/** Headers plus body for an image URL; the previous 10 s + 30 s budgets, combined. */
|
|
2194
|
+
const IMAGE_URL_TIMEOUT_MS = 40_000;
|
|
2205
2195
|
/**
|
|
2206
2196
|
* Get MIME type from an image file extension.
|
|
2207
2197
|
*
|
|
@@ -2338,34 +2328,41 @@ function detectMimeTypeFromBuffer(buffer) {
|
|
|
2338
2328
|
* Uses async fs so a large/slow-filesystem image never blocks the event loop
|
|
2339
2329
|
* while the size guard (below) or the read itself is in flight.
|
|
2340
2330
|
*/
|
|
2341
|
-
async function convertFilePathToBase64(filePath) {
|
|
2342
|
-
|
|
2331
|
+
async function convertFilePathToBase64(filePath, policy) {
|
|
2332
|
+
const context = `image file "${filePath}"`;
|
|
2333
|
+
let buffer;
|
|
2343
2334
|
try {
|
|
2344
|
-
|
|
2335
|
+
// Roots containment, O_NOFOLLOW open of the real path, regular files
|
|
2336
|
+
// only, and a read bounded at the image cap through the same descriptor
|
|
2337
|
+
// that was stat'd — so neither a symlink swap nor a file growing under
|
|
2338
|
+
// the read gets past the size guard (#257).
|
|
2339
|
+
({ buffer } = await readAttachmentFile(filePath, {
|
|
2340
|
+
policy,
|
|
2341
|
+
maxBytes: SIZE_LIMITS_BYTES.IMAGE_MAX,
|
|
2342
|
+
label: "image file",
|
|
2343
|
+
}));
|
|
2345
2344
|
}
|
|
2346
2345
|
catch (error) {
|
|
2346
|
+
if (error instanceof NeuroLinkError) {
|
|
2347
|
+
throw error;
|
|
2348
|
+
}
|
|
2349
|
+
const code = error instanceof Error ? error.code : undefined;
|
|
2347
2350
|
// Only ENOENT means "not found" — EACCES/ELOOP/etc. are real access or
|
|
2348
2351
|
// filesystem problems that must not be misreported as a missing file.
|
|
2349
|
-
const code = error instanceof Error
|
|
2350
|
-
? error.code
|
|
2351
|
-
: undefined;
|
|
2352
2352
|
if (code === "ENOENT") {
|
|
2353
2353
|
throw new Error(`Image file not found: ${filePath}`, { cause: error });
|
|
2354
2354
|
}
|
|
2355
|
+
if (code === "ATTACHMENT_NOT_A_FILE") {
|
|
2356
|
+
throw new Error(`Image path is not a file: ${filePath}`, {
|
|
2357
|
+
cause: error,
|
|
2358
|
+
});
|
|
2359
|
+
}
|
|
2360
|
+
if (code === "ATTACHMENT_TOO_LARGE") {
|
|
2361
|
+
// Preserve the typed IMAGE_TOO_LARGE error callers already match on.
|
|
2362
|
+
ImageProcessor.validateSize(Number.MAX_SAFE_INTEGER, context);
|
|
2363
|
+
}
|
|
2355
2364
|
throw new Error(`Cannot access image file ${filePath}: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
2356
2365
|
}
|
|
2357
|
-
if (!stats.isFile()) {
|
|
2358
|
-
throw new Error(`Image path is not a file: ${filePath}`);
|
|
2359
|
-
}
|
|
2360
|
-
// #257: check the file size via stat BEFORE reading it into memory, so a
|
|
2361
|
-
// huge image path never triggers an unbounded read + base64 allocation.
|
|
2362
|
-
// Delegates to ImageProcessor's shared guard (no local reimplementation).
|
|
2363
|
-
const context = `image file "${filePath}"`;
|
|
2364
|
-
ImageProcessor.validateSize(stats.size, context);
|
|
2365
|
-
const buffer = await readFileAsync(filePath);
|
|
2366
|
-
// TOCTOU: the file can grow, or a symlink can be swapped, between the stat
|
|
2367
|
-
// above and this read completing. Re-validate the bytes actually read
|
|
2368
|
-
// (not just the pre-read stat) before the base64 allocation.
|
|
2369
2366
|
return ImageProcessor.safeBase64Convert(buffer, context);
|
|
2370
2367
|
}
|
|
2371
2368
|
/**
|
|
@@ -2377,7 +2374,7 @@ async function convertFilePathToBase64(filePath) {
|
|
|
2377
2374
|
* Passing raw base64 avoids this because `new URL(base64string)` throws and
|
|
2378
2375
|
* the SDK treats the string as inline base64 data instead.
|
|
2379
2376
|
*/
|
|
2380
|
-
async function processImageToBase64(image, index, outputFormat) {
|
|
2377
|
+
async function processImageToBase64(image, index, outputFormat, policy = resolveAttachmentPolicy()) {
|
|
2381
2378
|
let imageData;
|
|
2382
2379
|
let mimeType = "image/jpeg"; // Default mime type
|
|
2383
2380
|
if (typeof image === "string") {
|
|
@@ -2412,7 +2409,7 @@ async function processImageToBase64(image, index, outputFormat) {
|
|
|
2412
2409
|
else {
|
|
2413
2410
|
// File path string - convert to raw base64
|
|
2414
2411
|
try {
|
|
2415
|
-
imageData = await convertFilePathToBase64(image);
|
|
2412
|
+
imageData = await convertFilePathToBase64(image, policy);
|
|
2416
2413
|
mimeType = getMimeTypeFromExtension(image);
|
|
2417
2414
|
}
|
|
2418
2415
|
catch (error) {
|
|
@@ -2508,7 +2505,7 @@ async function processImageToBase64(image, index, outputFormat) {
|
|
|
2508
2505
|
* Entries that are `http(s)` URLs are left alone — they have no bytes yet, and
|
|
2509
2506
|
* `processImageToBase64` converts them once they are downloaded.
|
|
2510
2507
|
*/
|
|
2511
|
-
export async function normalizeVisionImageFormats(input, outputFormat) {
|
|
2508
|
+
export async function normalizeVisionImageFormats(input, outputFormat, attachmentPolicy = resolveAttachmentPolicy()) {
|
|
2512
2509
|
const images = input?.images;
|
|
2513
2510
|
if (!images || images.length === 0) {
|
|
2514
2511
|
return;
|
|
@@ -2521,7 +2518,7 @@ export async function normalizeVisionImageFormats(input, outputFormat) {
|
|
|
2521
2518
|
const payload = isWrapped
|
|
2522
2519
|
? entry.data
|
|
2523
2520
|
: entry;
|
|
2524
|
-
const source = await readImageSourceForConversion(payload);
|
|
2521
|
+
const source = await readImageSourceForConversion(payload, attachmentPolicy);
|
|
2525
2522
|
if (!source ||
|
|
2526
2523
|
(!needsVisionTranscode(source.mimeType) &&
|
|
2527
2524
|
!mayCarryExifOrientation(source.mimeType))) {
|
|
@@ -2679,7 +2676,9 @@ async function probeFileExifOrientation(path, mimeType, size) {
|
|
|
2679
2676
|
}
|
|
2680
2677
|
let handle;
|
|
2681
2678
|
try {
|
|
2682
|
-
|
|
2679
|
+
// `path` is the symlink-resolved real path from `resolveAttachmentPath`;
|
|
2680
|
+
// O_NOFOLLOW refuses a link swapped in since.
|
|
2681
|
+
handle = await openAsync(path, fsConstants.O_RDONLY | (fsConstants.O_NOFOLLOW ?? 0));
|
|
2683
2682
|
const prefix = Buffer.alloc(window);
|
|
2684
2683
|
const { bytesRead } = await handle.read(prefix, 0, prefix.length, 0);
|
|
2685
2684
|
return probeExifOrientation(prefix.subarray(0, bytesRead), mimeType);
|
|
@@ -2691,7 +2690,7 @@ async function probeFileExifOrientation(path, mimeType, size) {
|
|
|
2691
2690
|
await handle?.close().catch(() => undefined);
|
|
2692
2691
|
}
|
|
2693
2692
|
}
|
|
2694
|
-
async function readImageSourceForConversion(payload) {
|
|
2693
|
+
async function readImageSourceForConversion(payload, policy) {
|
|
2695
2694
|
// Both in-memory shapes are size-checked before they can reach a decoder,
|
|
2696
2695
|
// the same way the file-path branch below is. `withinConversionLimit`
|
|
2697
2696
|
// reports rather than throws, because this whole pass is best-effort: an
|
|
@@ -2732,7 +2731,11 @@ async function readImageSourceForConversion(payload) {
|
|
|
2732
2731
|
// without this a large local HEIC/TIFF/BMP was read fully into memory and
|
|
2733
2732
|
// then re-encoded, with no limit applied at either step. The same applies
|
|
2734
2733
|
// to a local JPEG read only for its EXIF tag: still bounded before the read.
|
|
2735
|
-
|
|
2734
|
+
// Containment first: a path outside the instance's roots is refused
|
|
2735
|
+
// before anything about it is read, and everything below opens the real
|
|
2736
|
+
// path this returned.
|
|
2737
|
+
const real = await resolveAttachmentPath(payload, policy, "image file");
|
|
2738
|
+
const { size } = await statAsync(real);
|
|
2736
2739
|
ImageProcessor.validateSize(size, `image at ${safeBasename(payload)}`);
|
|
2737
2740
|
// A file whose only reason to be here is the orientation question gets
|
|
2738
2741
|
// that question answered from its header, and is not read at all when the
|
|
@@ -2740,10 +2743,14 @@ async function readImageSourceForConversion(payload) {
|
|
|
2740
2743
|
// whatever the orientation turns out to be, so a probe would be pure
|
|
2741
2744
|
// overhead.
|
|
2742
2745
|
if (!transcodable &&
|
|
2743
|
-
(await probeFileExifOrientation(
|
|
2746
|
+
(await probeFileExifOrientation(real, mimeType, size)) === "absent") {
|
|
2744
2747
|
return undefined;
|
|
2745
2748
|
}
|
|
2746
|
-
const buffer = await
|
|
2749
|
+
const { buffer } = await readAttachmentFile(real, {
|
|
2750
|
+
policy,
|
|
2751
|
+
maxBytes: SIZE_LIMITS_BYTES.IMAGE_MAX,
|
|
2752
|
+
label: "image file",
|
|
2753
|
+
});
|
|
2747
2754
|
ImageProcessor.validateBufferSize(buffer, `image at ${safeBasename(payload)}`);
|
|
2748
2755
|
return { buffer, mimeType };
|
|
2749
2756
|
}
|
|
@@ -2762,7 +2769,7 @@ async function readImageSourceForConversion(payload) {
|
|
|
2762
2769
|
* - Buffers/Data URIs: Processed normally
|
|
2763
2770
|
* - Supports alt text for accessibility (included as context in text parts)
|
|
2764
2771
|
*/
|
|
2765
|
-
async function convertSimpleImagesToProviderFormat(text, images, provider, _model, outputFormat) {
|
|
2772
|
+
async function convertSimpleImagesToProviderFormat(text, images, provider, _model, outputFormat, attachmentPolicy = resolveAttachmentPolicy()) {
|
|
2766
2773
|
// Validate image count against provider-specific limits before processing
|
|
2767
2774
|
ProviderImageAdapter.validateImageCount(images.length, provider, _model);
|
|
2768
2775
|
// For Vercel AI SDK, we need to return the content in the standard format
|
|
@@ -2801,7 +2808,7 @@ async function convertSimpleImagesToProviderFormat(text, images, provider, _mode
|
|
|
2801
2808
|
// Download URL images and add to actual images
|
|
2802
2809
|
for (const { url, altText } of urlImages) {
|
|
2803
2810
|
try {
|
|
2804
|
-
const downloadedDataUri = await downloadImageFromUrl(url);
|
|
2811
|
+
const downloadedDataUri = await downloadImageFromUrl(url, attachmentPolicy);
|
|
2805
2812
|
actualImages.push({ data: downloadedDataUri, altText });
|
|
2806
2813
|
}
|
|
2807
2814
|
catch (error) {
|
|
@@ -2819,7 +2826,7 @@ async function convertSimpleImagesToProviderFormat(text, images, provider, _mode
|
|
|
2819
2826
|
for (const [index, { data: image }] of actualImages.entries()) {
|
|
2820
2827
|
try {
|
|
2821
2828
|
// Use helper function to process image and reduce nesting depth
|
|
2822
|
-
const { imageData, mimeType } = await processImageToBase64(image, index, outputFormat);
|
|
2829
|
+
const { imageData, mimeType } = await processImageToBase64(image, index, outputFormat, attachmentPolicy);
|
|
2823
2830
|
content.push({
|
|
2824
2831
|
type: "image",
|
|
2825
2832
|
image: imageData,
|
|
@@ -2847,12 +2854,13 @@ pdfFiles, provider, model,
|
|
|
2847
2854
|
// repo's `max-params` ceiling of 6.
|
|
2848
2855
|
mediaOptions = {}) {
|
|
2849
2856
|
const { audioFiles = [], outputFormat } = mediaOptions;
|
|
2857
|
+
const attachmentPolicy = mediaOptions.attachmentPolicy ?? resolveAttachmentPolicy();
|
|
2850
2858
|
const content = [
|
|
2851
2859
|
{ type: "text", text },
|
|
2852
2860
|
];
|
|
2853
2861
|
// Add images if present
|
|
2854
2862
|
if (images.length > 0) {
|
|
2855
|
-
const imageContent = await convertSimpleImagesToProviderFormat("", images, provider, model, outputFormat);
|
|
2863
|
+
const imageContent = await convertSimpleImagesToProviderFormat("", images, provider, model, outputFormat, attachmentPolicy);
|
|
2856
2864
|
if (Array.isArray(imageContent)) {
|
|
2857
2865
|
imageContent.forEach((item) => {
|
|
2858
2866
|
if (item.type !== "text") {
|
|
@@ -3165,16 +3173,21 @@ function getFileSize(file) {
|
|
|
3165
3173
|
* For file paths: reads the file.
|
|
3166
3174
|
* For URLs/data URIs: returns null (not supported for lazy registration).
|
|
3167
3175
|
*/
|
|
3168
|
-
async function getFileBuffer(file) {
|
|
3176
|
+
async function getFileBuffer(file, policy) {
|
|
3169
3177
|
if (isFileWithMetadata(file)) {
|
|
3170
3178
|
return file.buffer;
|
|
3171
3179
|
}
|
|
3172
3180
|
if (Buffer.isBuffer(file)) {
|
|
3173
3181
|
return file;
|
|
3174
3182
|
}
|
|
3175
|
-
if (typeof file === "string" &&
|
|
3183
|
+
if (typeof file === "string" && getFileSource(file) === "path") {
|
|
3176
3184
|
try {
|
|
3177
|
-
|
|
3185
|
+
const read = await readAttachmentFile(file, {
|
|
3186
|
+
policy,
|
|
3187
|
+
maxBytes: ATTACHED_FILE_READ_CAP_BYTES,
|
|
3188
|
+
label: "attached file",
|
|
3189
|
+
});
|
|
3190
|
+
return read.buffer;
|
|
3178
3191
|
}
|
|
3179
3192
|
catch {
|
|
3180
3193
|
return null;
|
|
@@ -3349,9 +3362,9 @@ function inferFileTypeFromMimetype(mimetype) {
|
|
|
3349
3362
|
* Returns true if registration succeeded, false if it failed (caller should
|
|
3350
3363
|
* fall through to full processing).
|
|
3351
3364
|
*/
|
|
3352
|
-
async function tryRegisterFileReference(file, fileSize, registry, index = 0) {
|
|
3365
|
+
async function tryRegisterFileReference(file, fileSize, registry, index = 0, policy = resolveAttachmentPolicy()) {
|
|
3353
3366
|
try {
|
|
3354
|
-
const buffer = await getFileBuffer(file);
|
|
3367
|
+
const buffer = await getFileBuffer(file, policy);
|
|
3355
3368
|
if (!buffer) {
|
|
3356
3369
|
return false;
|
|
3357
3370
|
}
|
|
@@ -80,4 +80,5 @@ export declare function buildMultimodalOptions(options: StreamOptions, providerN
|
|
|
80
80
|
enableEvaluation: boolean | undefined;
|
|
81
81
|
context: import("../index.js").UnknownRecord | undefined;
|
|
82
82
|
fileRegistry: unknown;
|
|
83
|
+
attachmentPolicy: import("../index.js").AttachmentPolicy;
|
|
83
84
|
};
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { getAttachmentPolicy } from "./attachmentPolicy.js";
|
|
1
2
|
/**
|
|
2
3
|
* Builds a normalized multimodal options payload for streaming providers.
|
|
3
4
|
*
|
|
@@ -84,5 +85,6 @@ export function buildMultimodalOptions(options, providerName, modelName) {
|
|
|
84
85
|
enableEvaluation: options.enableEvaluation,
|
|
85
86
|
context: options.context,
|
|
86
87
|
fileRegistry: options.fileRegistry,
|
|
88
|
+
attachmentPolicy: getAttachmentPolicy(options),
|
|
87
89
|
};
|
|
88
90
|
}
|
package/dist/utils/redis.d.ts
CHANGED
|
@@ -3,6 +3,18 @@
|
|
|
3
3
|
* Helper functions for Redis storage operations
|
|
4
4
|
*/
|
|
5
5
|
import type { ChatMessage, RedisClient, RedisConversationObject, RedisStorageConfig } from "../types/index.js";
|
|
6
|
+
/**
|
|
7
|
+
* Why an id is not usable as a key segment, or `null` when it is.
|
|
8
|
+
*/
|
|
9
|
+
export declare function describeRedisIdProblem(value: string, kind?: "sessionId" | "userId"): string | null;
|
|
10
|
+
/**
|
|
11
|
+
* Refuse an id that would alias another key or break a SCAN pattern.
|
|
12
|
+
*
|
|
13
|
+
* @throws {Error} with `code: "INVALID_REDIS_ID"` unless the legacy switch is
|
|
14
|
+
* set, in which case the problem is logged once per id shape and the id
|
|
15
|
+
* is used as given.
|
|
16
|
+
*/
|
|
17
|
+
export declare function assertSafeRedisId(value: string, label: "sessionId" | "userId"): void;
|
|
6
18
|
/**
|
|
7
19
|
* Redact the userinfo of every `scheme://user:pass@host` in `text`.
|
|
8
20
|
*
|
package/dist/utils/redis.js
CHANGED
|
@@ -5,6 +5,73 @@
|
|
|
5
5
|
import { randomUUID } from "crypto";
|
|
6
6
|
import { logger } from "./logger.js";
|
|
7
7
|
const SESSION_ONLY_PREFIX = "session-only:";
|
|
8
|
+
/**
|
|
9
|
+
* Longest id that may become part of a key. Redis itself allows far more, but
|
|
10
|
+
* a session or user id past this length is not an id.
|
|
11
|
+
*/
|
|
12
|
+
const MAX_REDIS_ID_LENGTH = 512;
|
|
13
|
+
/**
|
|
14
|
+
* Characters an id may not contain: `:` is the key-segment separator (so
|
|
15
|
+
* `user "a:b"` + session `"c"` would alias `user "a"` + session `"b:c"`), and
|
|
16
|
+
* `*`, `?`, `[` are glob metacharacters for SCAN/KEYS patterns; line breaks
|
|
17
|
+
* never belong in a key.
|
|
18
|
+
*/
|
|
19
|
+
const FORBIDDEN_REDIS_ID_CHARS = /[:*?[\r\n]/;
|
|
20
|
+
// userId may carry ':' (multi-tenant ids such as `cust:merchant:shop` are established
|
|
21
|
+
// usage); the key `${prefix}${userId}:${sessionId}` still splits unambiguously at the
|
|
22
|
+
// last colon because sessionId may not contain one. Glob characters and line breaks
|
|
23
|
+
// stay forbidden so an id can never widen a SCAN pattern or break a log line.
|
|
24
|
+
const FORBIDDEN_REDIS_USER_ID_CHARS = /[*?[\r\n]/;
|
|
25
|
+
/**
|
|
26
|
+
* Set `NEUROLINK_REDIS_ALLOW_LEGACY_IDS=1` to accept ids that fail the check
|
|
27
|
+
* with a warning instead of an error, for a deployment whose stored sessions
|
|
28
|
+
* already carry `:` in their ids and needs time to migrate.
|
|
29
|
+
*/
|
|
30
|
+
const ENV_ALLOW_LEGACY_IDS = "NEUROLINK_REDIS_ALLOW_LEGACY_IDS";
|
|
31
|
+
/**
|
|
32
|
+
* Why an id is not usable as a key segment, or `null` when it is.
|
|
33
|
+
*/
|
|
34
|
+
export function describeRedisIdProblem(value, kind = "sessionId") {
|
|
35
|
+
if (value.length === 0) {
|
|
36
|
+
return "is empty";
|
|
37
|
+
}
|
|
38
|
+
if (value.length > MAX_REDIS_ID_LENGTH) {
|
|
39
|
+
return `is longer than ${MAX_REDIS_ID_LENGTH} characters`;
|
|
40
|
+
}
|
|
41
|
+
if (kind === "userId") {
|
|
42
|
+
if (FORBIDDEN_REDIS_USER_ID_CHARS.test(value)) {
|
|
43
|
+
return "contains a reserved character ('*', '?', '[' or a line break)";
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
else if (FORBIDDEN_REDIS_ID_CHARS.test(value)) {
|
|
47
|
+
return "contains a reserved character (':', '*', '?', '[' or a line break)";
|
|
48
|
+
}
|
|
49
|
+
if (value === SESSION_ONLY_PREFIX.slice(0, -1)) {
|
|
50
|
+
return "is the reserved no-user namespace";
|
|
51
|
+
}
|
|
52
|
+
return null;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Refuse an id that would alias another key or break a SCAN pattern.
|
|
56
|
+
*
|
|
57
|
+
* @throws {Error} with `code: "INVALID_REDIS_ID"` unless the legacy switch is
|
|
58
|
+
* set, in which case the problem is logged once per id shape and the id
|
|
59
|
+
* is used as given.
|
|
60
|
+
*/
|
|
61
|
+
export function assertSafeRedisId(value, label) {
|
|
62
|
+
const problem = describeRedisIdProblem(value, label);
|
|
63
|
+
if (problem === null) {
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
const message = `[REDIS] ${label} ${problem}; it cannot be used in a Redis key`;
|
|
67
|
+
if (process.env[ENV_ALLOW_LEGACY_IDS] === "1") {
|
|
68
|
+
logger.warn(`${message} (allowed by ${ENV_ALLOW_LEGACY_IDS})`);
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
const error = new Error(`${message}. Set ${ENV_ALLOW_LEGACY_IDS}=1 to accept legacy ids while migrating.`);
|
|
72
|
+
error.code = "INVALID_REDIS_ID";
|
|
73
|
+
throw error;
|
|
74
|
+
}
|
|
8
75
|
/** Characters that end the authority (`user:pass@host:port`) of a URL in text. */
|
|
9
76
|
const AUTHORITY_TERMINATORS = new Set(["/", "?", "#", " ", "\t", "\n", "\r"]);
|
|
10
77
|
/**
|
|
@@ -222,6 +289,10 @@ export async function createRedisClient(config) {
|
|
|
222
289
|
* Generates a Redis key for session messages
|
|
223
290
|
*/
|
|
224
291
|
export function getSessionKey(config, sessionId, userId) {
|
|
292
|
+
assertSafeRedisId(sessionId, "sessionId");
|
|
293
|
+
if (userId !== undefined && userId !== "") {
|
|
294
|
+
assertSafeRedisId(userId, "userId");
|
|
295
|
+
}
|
|
225
296
|
if (!userId) {
|
|
226
297
|
logger.warn("[REDIS] getSessionKey called without userId", { sessionId });
|
|
227
298
|
return `${config.keyPrefix}${SESSION_ONLY_PREFIX}${sessionId}`;
|
|
@@ -239,6 +310,7 @@ export function getSessionKey(config, sessionId, userId) {
|
|
|
239
310
|
* Generates a Redis key for user sessions mapping
|
|
240
311
|
*/
|
|
241
312
|
export function getUserSessionsKey(config, userId) {
|
|
313
|
+
assertSafeRedisId(userId, "userId");
|
|
242
314
|
return `${config.userSessionsKeyPrefix}${userId}`;
|
|
243
315
|
}
|
|
244
316
|
/**
|
|
@@ -14,6 +14,13 @@
|
|
|
14
14
|
* Use this for **every** download of an external (caller-supplied or
|
|
15
15
|
* third-party-returned) URL. Direct `fetch(url)` of such URLs is unsafe.
|
|
16
16
|
*
|
|
17
|
+
* Redirects are refused by default. A caller that opts in (`maxRedirects`)
|
|
18
|
+
* gets each hop validated and pinned exactly like the first request, and the
|
|
19
|
+
* caller's `headers` are sent to the first host only — a credential meant for
|
|
20
|
+
* one origin is never replayed to the one it redirects to. `allowInsecure`
|
|
21
|
+
* and `allowPrivateNetwork` relax the URL policy for SDK callers that have
|
|
22
|
+
* said so explicitly; the server path never passes them.
|
|
23
|
+
*
|
|
17
24
|
* When the environment configures a proxy for the URL (HTTPS_PROXY / ALL_PROXY,
|
|
18
25
|
* and the host is not in NO_PROXY) the download goes through that proxy and
|
|
19
26
|
* nowhere else: there is no fallback to a direct connection. The proxy resolves
|
|
@@ -22,7 +29,19 @@
|
|
|
22
29
|
*
|
|
23
30
|
* @module utils/safeFetch
|
|
24
31
|
*/
|
|
25
|
-
import type { SafeDownloadOptions } from "../types/index.js";
|
|
32
|
+
import type { SafeDownloadOptions, SafeDownloadResult } from "../types/index.js";
|
|
33
|
+
/**
|
|
34
|
+
* Safely download an external URL and report what came back.
|
|
35
|
+
*
|
|
36
|
+
* Same guarantees as {@link safeDownload}; additionally returns the final
|
|
37
|
+
* response's `Content-Type`/`Content-Encoding`, the URL that produced the body
|
|
38
|
+
* and how many redirect hops were taken.
|
|
39
|
+
*
|
|
40
|
+
* @throws {Error} if any URL on the way is unsafe, the response is too large,
|
|
41
|
+
* a redirect is encountered beyond what `maxRedirects` allows, or the HTTP
|
|
42
|
+
* status indicates failure.
|
|
43
|
+
*/
|
|
44
|
+
export declare function safeDownloadWithMeta(url: string, options: SafeDownloadOptions): Promise<SafeDownloadResult>;
|
|
26
45
|
/**
|
|
27
46
|
* Safely download a binary asset from an external URL.
|
|
28
47
|
*
|
|
@@ -30,3 +49,5 @@ import type { SafeDownloadOptions } from "../types/index.js";
|
|
|
30
49
|
* is encountered, or the HTTP status indicates failure.
|
|
31
50
|
*/
|
|
32
51
|
export declare function safeDownload(url: string, options: SafeDownloadOptions): Promise<Buffer>;
|
|
52
|
+
/** Hop count applied when a caller opts into redirects without naming one. */
|
|
53
|
+
export declare function defaultRedirectHops(follow: boolean): number;
|