@juspay/neurolink 13.8.17 → 13.8.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5 -2
- package/README.md +1 -1
- package/dist/action/actionExecutor.d.ts +33 -3
- package/dist/action/actionExecutor.js +71 -15
- package/dist/action/actionInputs.js +4 -1
- package/dist/action/cliVersion.d.ts +25 -0
- package/dist/action/cliVersion.js +26 -0
- package/dist/action/index.d.ts +2 -1
- package/dist/action/index.js +2 -1
- package/dist/adapters/video/directorPipeline.js +46 -36
- package/dist/adapters/video/ffmpegAdapter.d.ts +17 -0
- package/dist/adapters/video/ffmpegAdapter.js +32 -5
- package/dist/adapters/video/frameExtractor.js +1 -1
- package/dist/adapters/video/videoMerger.js +5 -2
- package/dist/agent/arithmeticEvaluator.d.ts +32 -0
- package/dist/agent/arithmeticEvaluator.js +349 -0
- package/dist/agent/directTools.d.ts +6 -18
- package/dist/agent/directTools.js +17 -56
- package/dist/artifacts/artifactStore.d.ts +22 -1
- package/dist/artifacts/artifactStore.js +66 -6
- package/dist/auth/anthropicOAuth.d.ts +26 -1
- package/dist/auth/anthropicOAuth.js +81 -5
- package/dist/auth/secureFiles.d.ts +37 -0
- package/dist/auth/secureFiles.js +95 -0
- package/dist/auth/tokenStore.d.ts +43 -6
- package/dist/auth/tokenStore.js +260 -79
- package/dist/avatar/providers/DIDAvatar.js +49 -45
- package/dist/avatar/providers/ReplicateAvatar.js +33 -40
- package/dist/browser/neurolink.min.js +485 -558
- package/dist/cli/commands/auth.js +112 -70
- package/dist/cli/commands/proxy.d.ts +4 -0
- package/dist/cli/commands/proxy.js +128 -13
- package/dist/cli/commands/serve.d.ts +10 -1
- package/dist/cli/commands/serve.js +98 -17
- package/dist/cli/commands/server.js +17 -5
- package/dist/cli/index.js +146 -5
- package/dist/cli/loop/optionsSchema.d.ts +1 -1
- package/dist/cli/utils/envManager.js +17 -3
- package/dist/cli/utils/serverUtils.d.ts +39 -3
- package/dist/cli/utils/serverUtils.js +63 -6
- package/dist/core/baseProvider.js +7 -1
- package/dist/core/modules/MessageBuilder.js +5 -0
- package/dist/core/modules/ToolsManager.d.ts +46 -1
- package/dist/core/modules/ToolsManager.js +190 -43
- package/dist/files/fileReferenceRegistry.d.ts +2 -23
- package/dist/files/fileReferenceRegistry.js +55 -5
- package/dist/hitl/hitlManager.js +1 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +6 -1
- package/dist/mcp/auth/tokenStorage.d.ts +7 -0
- package/dist/mcp/auth/tokenStorage.js +23 -4
- package/dist/mcp/externalServerManager.d.ts +13 -3
- package/dist/mcp/externalServerManager.js +154 -136
- package/dist/mcp/index.d.ts +2 -0
- package/dist/mcp/index.js +3 -0
- package/dist/mcp/mcpClientFactory.d.ts +14 -0
- package/dist/mcp/mcpClientFactory.js +70 -7
- package/dist/mcp/projectConfigTrust.d.ts +72 -0
- package/dist/mcp/projectConfigTrust.js +237 -0
- package/dist/mcp/transportUrlPolicy.d.ts +39 -0
- package/dist/mcp/transportUrlPolicy.js +172 -0
- package/dist/music/providers/BeatovenMusic.js +35 -46
- package/dist/music/providers/ReplicateMusic.js +33 -40
- package/dist/neurolink.d.ts +16 -1
- package/dist/neurolink.js +64 -11
- package/dist/processors/archive/ArchiveProcessor.js +15 -1
- package/dist/processors/archive/zipEntryIndex.d.ts +22 -0
- package/dist/processors/archive/zipEntryIndex.js +33 -0
- package/dist/processors/base/BaseFileProcessor.d.ts +9 -21
- package/dist/processors/base/BaseFileProcessor.js +40 -111
- package/dist/processors/data/YamlProcessor.d.ts +13 -1
- package/dist/processors/data/YamlProcessor.js +93 -2
- package/dist/processors/document/OpenDocumentProcessor.js +7 -2
- package/dist/processors/document/PptxProcessor.js +12 -7
- package/dist/processors/media/VideoProcessor.js +93 -15
- package/dist/providers/openaiChatCompletionsClient.js +12 -3
- package/dist/proxy/accountLedger.js +6 -8
- package/dist/proxy/geminiFormat.d.ts +3 -11
- package/dist/proxy/geminiFormat.js +11 -2
- package/dist/proxy/inboundAuth.d.ts +89 -0
- package/dist/proxy/inboundAuth.js +219 -0
- package/dist/proxy/openaiFormat.d.ts +3 -9
- package/dist/proxy/openaiFormat.js +21 -15
- package/dist/proxy/proxyTranslationEngine.d.ts +2 -6
- package/dist/proxy/proxyTranslationEngine.js +25 -14
- package/dist/proxy/routingPolicy.d.ts +3 -1
- package/dist/proxy/routingPolicy.js +7 -8
- package/dist/proxy/tokenRefresh.d.ts +6 -0
- package/dist/proxy/tokenRefresh.js +13 -1
- package/dist/proxy/updateState.js +4 -4
- package/dist/rag/document/loaders.d.ts +0 -6
- package/dist/rag/document/loaders.js +16 -10
- package/dist/rag/ragIntegration.d.ts +2 -2
- package/dist/rag/ragIntegration.js +35 -10
- package/dist/rag/retrieval/vectorQueryTool.d.ts +7 -0
- package/dist/rag/retrieval/vectorQueryTool.js +54 -17
- package/dist/server/abstract/baseServerAdapter.d.ts +43 -0
- package/dist/server/abstract/baseServerAdapter.js +119 -6
- package/dist/server/adapters/expressAdapter.d.ts +3 -0
- package/dist/server/adapters/expressAdapter.js +78 -6
- package/dist/server/adapters/fastifyAdapter.d.ts +3 -0
- package/dist/server/adapters/fastifyAdapter.js +165 -70
- package/dist/server/adapters/honoAdapter.d.ts +9 -1
- package/dist/server/adapters/honoAdapter.js +99 -21
- package/dist/server/adapters/koaAdapter.d.ts +1 -0
- package/dist/server/adapters/koaAdapter.js +153 -74
- package/dist/server/index.d.ts +2 -0
- package/dist/server/index.js +4 -0
- package/dist/server/middleware/auth.js +6 -2
- package/dist/server/middleware/cache.js +6 -1
- package/dist/server/middleware/rateLimit.js +29 -4
- package/dist/server/routes/agentRoutes.js +24 -11
- package/dist/server/routes/claudeProxyRoutes.d.ts +1 -0
- package/dist/server/routes/claudeProxyRoutes.js +273 -84
- package/dist/server/routes/codexProxyRoutes.d.ts +2 -2
- package/dist/server/routes/codexProxyRoutes.js +125 -40
- package/dist/server/routes/index.d.ts +4 -0
- package/dist/server/routes/index.js +35 -4
- package/dist/server/routes/mcpRoutes.d.ts +6 -2
- package/dist/server/routes/mcpRoutes.js +21 -7
- package/dist/server/routes/memoryRoutes.js +7 -6
- package/dist/server/routes/toolRoutes.d.ts +12 -2
- package/dist/server/routes/toolRoutes.js +33 -3
- package/dist/server/routes/transcribeRoutes.js +60 -13
- package/dist/server/utils/clientAddress.d.ts +23 -0
- package/dist/server/utils/clientAddress.js +65 -0
- package/dist/server/utils/clientError.d.ts +35 -0
- package/dist/server/utils/clientError.js +129 -0
- package/dist/server/utils/pathMatch.d.ts +21 -0
- package/dist/server/utils/pathMatch.js +41 -0
- package/dist/server/utils/validation.d.ts +9 -0
- package/dist/server/utils/validation.js +9 -0
- package/dist/server/websocket/WebSocketHandler.js +22 -3
- package/dist/server/websocket/frameQueue.d.ts +19 -0
- package/dist/server/websocket/frameQueue.js +41 -3
- package/dist/server/websocket/transcribeWebSocket.d.ts +8 -1
- package/dist/server/websocket/transcribeWebSocket.js +223 -34
- package/dist/skills/skillStores.js +5 -0
- package/dist/tasks/store/fileTaskStore.d.ts +2 -0
- package/dist/tasks/store/fileTaskStore.js +11 -4
- package/dist/types/attachments.d.ts +117 -0
- package/dist/types/attachments.js +15 -0
- package/dist/types/auth.d.ts +6 -0
- package/dist/types/cli.d.ts +4 -0
- package/dist/types/config.d.ts +56 -0
- package/dist/types/file.d.ts +8 -0
- package/dist/types/hitl.d.ts +8 -0
- package/dist/types/index.d.ts +1 -0
- package/dist/types/index.js +1 -0
- package/dist/types/mcp.d.ts +98 -0
- package/dist/types/multimodal.js +0 -41
- package/dist/types/openaiCompatible.d.ts +1 -1
- package/dist/types/processor.d.ts +14 -1
- package/dist/types/proxy.d.ts +71 -16
- package/dist/types/proxyClient.d.ts +2 -0
- package/dist/types/rag.d.ts +9 -0
- package/dist/types/safeFetch.d.ts +40 -0
- package/dist/types/server.d.ts +206 -10
- package/dist/types/tools.d.ts +57 -0
- package/dist/types/transcribe.d.ts +6 -0
- package/dist/utils/attachmentPolicy.d.ts +89 -0
- package/dist/utils/attachmentPolicy.js +327 -0
- package/dist/utils/dotenvBootstrap.d.ts +48 -19
- package/dist/utils/dotenvBootstrap.js +172 -31
- package/dist/utils/fileDetector.js +115 -65
- package/dist/utils/json/coerce.js +11 -1
- package/dist/utils/logSanitize.d.ts +5 -0
- package/dist/utils/logSanitize.js +8 -0
- package/dist/utils/logger.js +73 -1
- package/dist/utils/messageBuilder.d.ts +2 -2
- package/dist/utils/messageBuilder.js +118 -105
- package/dist/utils/multimodalOptionsBuilder.d.ts +1 -0
- package/dist/utils/multimodalOptionsBuilder.js +2 -0
- package/dist/utils/redis.d.ts +12 -0
- package/dist/utils/redis.js +72 -0
- package/dist/utils/retryAfter.d.ts +32 -0
- package/dist/utils/retryAfter.js +135 -11
- package/dist/utils/safeFetch.d.ts +22 -1
- package/dist/utils/safeFetch.js +132 -38
- package/dist/utils/safeStoreId.d.ts +19 -0
- package/dist/utils/safeStoreId.js +29 -0
- package/dist/utils/schemaConversion.d.ts +30 -3
- package/dist/utils/schemaConversion.js +457 -84
- package/dist/utils/sizeGuard.js +2 -2
- package/dist/utils/ssrfGuard.d.ts +3 -3
- package/dist/utils/ssrfGuard.js +71 -39
- package/dist/voice/eval/sttEvalStore.d.ts +26 -3
- package/dist/voice/eval/sttEvalStore.js +77 -12
- package/dist/voice/livekit/agentDefinition.d.ts +18 -0
- package/dist/voice/livekit/agentDefinition.js +22 -0
- package/dist/voice/livekit/realtimeMcpTools.d.ts +1 -1
- package/dist/voice/livekit/realtimeMcpTools.js +8 -7
- package/dist/voice/livekit/realtimeVoiceAgent.d.ts +2 -4
- package/dist/voice/livekit/realtimeVoiceAgent.js +18 -7
- package/dist/voice/livekit/voiceAgent.d.ts +2 -2
- package/dist/voice/livekit/voiceAgent.js +2 -1
- package/dist/voice/livekit/xaiSessionWatch.d.ts +0 -5
- package/dist/voice/livekit/xaiSessionWatch.js +30 -0
- package/dist/voice/providers/GeminiLive.js +21 -6
- package/dist/voice/providers/GoogleSTT.js +8 -5
- package/dist/voice/transcribe.d.ts +2 -2
- package/dist/voice/transcribe.js +34 -18
- package/dist/voice/whistle/audio.js +11 -5
- package/dist/workflow/workflows/adaptiveWorkflow.d.ts +2 -2
- package/dist/workflow/workflows/adaptiveWorkflow.js +10 -10
- package/dist/workflow/workflows/consensusWorkflow.d.ts +4 -4
- package/dist/workflow/workflows/consensusWorkflow.js +12 -12
- package/dist/workflow/workflows/fallbackWorkflow.d.ts +3 -3
- package/dist/workflow/workflows/fallbackWorkflow.js +7 -7
- package/dist/workflow/workflows/multiJudgeWorkflow.d.ts +7 -7
- package/dist/workflow/workflows/multiJudgeWorkflow.js +31 -31
- package/docs-site/static/search-index.json +74 -52
- package/package.json +173 -156
- package/scripts/observability/import-openobserve-dashboard.mjs +11 -3
package/dist/utils/retryAfter.js
CHANGED
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Retrieves a header value case-insensitively from either a Headers object or a plain record.
|
|
3
|
+
*
|
|
4
|
+
* @param headers - The HTTP headers object or record
|
|
5
|
+
* @param headerName - The case-insensitive name of the header to retrieve
|
|
6
|
+
* @returns The header value, or null if not found
|
|
7
|
+
*/
|
|
1
8
|
function getHeader(headers, headerName) {
|
|
2
9
|
if ("get" in headers && typeof headers.get === "function") {
|
|
3
10
|
return headers.get(headerName);
|
|
@@ -10,6 +17,128 @@ function getHeader(headers, headerName) {
|
|
|
10
17
|
}
|
|
11
18
|
return null;
|
|
12
19
|
}
|
|
20
|
+
/**
|
|
21
|
+
* Parses a duration string (e.g., "6m0s", "30s", "1h30m", "500ms") into milliseconds.
|
|
22
|
+
*
|
|
23
|
+
* @param raw - The raw duration string
|
|
24
|
+
* @returns Duration in milliseconds, or undefined if invalid
|
|
25
|
+
*/
|
|
26
|
+
export function parseDurationMs(raw) {
|
|
27
|
+
const trimmed = raw.trim();
|
|
28
|
+
if (!trimmed) {
|
|
29
|
+
return undefined;
|
|
30
|
+
}
|
|
31
|
+
// Must consist of one or more [number][unit] components
|
|
32
|
+
const durationPattern = /^(\d+(?:\.\d+)?(?:ms|s|m|h))+$/i;
|
|
33
|
+
if (!durationPattern.test(trimmed)) {
|
|
34
|
+
return undefined;
|
|
35
|
+
}
|
|
36
|
+
const tokenRegex = /(\d+(?:\.\d+)?)(ms|s|m|h)/gi;
|
|
37
|
+
let totalMs = 0;
|
|
38
|
+
let match;
|
|
39
|
+
let matchedLength = 0;
|
|
40
|
+
while ((match = tokenRegex.exec(trimmed)) !== null) {
|
|
41
|
+
matchedLength += match[0].length;
|
|
42
|
+
const num = parseFloat(match[1]);
|
|
43
|
+
const unit = match[2].toLowerCase();
|
|
44
|
+
switch (unit) {
|
|
45
|
+
case "h":
|
|
46
|
+
totalMs += num * 3600 * 1000;
|
|
47
|
+
break;
|
|
48
|
+
case "m":
|
|
49
|
+
totalMs += num * 60 * 1000;
|
|
50
|
+
break;
|
|
51
|
+
case "s":
|
|
52
|
+
totalMs += num * 1000;
|
|
53
|
+
break;
|
|
54
|
+
case "ms":
|
|
55
|
+
totalMs += num;
|
|
56
|
+
break;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
return matchedLength === trimmed.length ? Math.round(totalMs) : undefined;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Parses a single Retry-After header value into milliseconds.
|
|
63
|
+
*
|
|
64
|
+
* Checks in strict order:
|
|
65
|
+
* 1. Strict numeric seconds (e.g. "5", "120", "1.5") without truncation of dates or durations
|
|
66
|
+
* 2. HTTP-date or RFC 3339 / ISO 8601 date (e.g. "2026-10-18T12:00:00Z" or "Sun, 18 Oct 2026 12:00:00 GMT")
|
|
67
|
+
* 3. Duration string (e.g. "6m0s", "30s", "1h30m", "500ms")
|
|
68
|
+
*
|
|
69
|
+
* @param value - The header value string
|
|
70
|
+
* @returns Milliseconds to wait, or undefined if not parseable
|
|
71
|
+
*/
|
|
72
|
+
export function parseRetryAfterHeaderValue(value) {
|
|
73
|
+
if (!value) {
|
|
74
|
+
return undefined;
|
|
75
|
+
}
|
|
76
|
+
const trimmed = value.trim();
|
|
77
|
+
if (!trimmed) {
|
|
78
|
+
return undefined;
|
|
79
|
+
}
|
|
80
|
+
// 1. Strict numeric representation (delta seconds)
|
|
81
|
+
// Note: Avoid parseInt, which reads leading digits from dates ("2026-10-13..." -> 2026) or durations ("6m0s" -> 6)
|
|
82
|
+
if (/^\d+(?:\.\d+)?$/.test(trimmed)) {
|
|
83
|
+
const seconds = Number(trimmed);
|
|
84
|
+
if (Number.isFinite(seconds) && seconds >= 0) {
|
|
85
|
+
return Math.round(seconds * 1000);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
// 2. HTTP-date or RFC 3339 / ISO 8601 date
|
|
89
|
+
const parsedDate = Date.parse(trimmed);
|
|
90
|
+
if (Number.isFinite(parsedDate)) {
|
|
91
|
+
return Math.max(0, parsedDate - Date.now());
|
|
92
|
+
}
|
|
93
|
+
// 3. Duration string (e.g. "6m0s", "30s", "500ms")
|
|
94
|
+
const duration = parseDurationMs(trimmed);
|
|
95
|
+
if (duration !== undefined && duration >= 0) {
|
|
96
|
+
return duration;
|
|
97
|
+
}
|
|
98
|
+
return undefined;
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Parses X-RateLimit-Reset header value into milliseconds until reset.
|
|
102
|
+
*
|
|
103
|
+
* Handles:
|
|
104
|
+
* - Delta seconds when value < 1e9 (e.g. "30", "60")
|
|
105
|
+
* - Epoch seconds when 1e9 <= value <= 1e12
|
|
106
|
+
* - Epoch milliseconds when value > 1e12
|
|
107
|
+
* - Fallback to date or duration strings
|
|
108
|
+
*
|
|
109
|
+
* @param value - The X-RateLimit-Reset header value
|
|
110
|
+
* @returns Milliseconds until reset, or undefined if not parseable
|
|
111
|
+
*/
|
|
112
|
+
export function parseRateLimitResetValue(value) {
|
|
113
|
+
if (!value) {
|
|
114
|
+
return undefined;
|
|
115
|
+
}
|
|
116
|
+
const trimmed = value.trim();
|
|
117
|
+
if (!trimmed) {
|
|
118
|
+
return undefined;
|
|
119
|
+
}
|
|
120
|
+
if (/^\d+(?:\.\d+)?$/.test(trimmed)) {
|
|
121
|
+
const num = Number(trimmed);
|
|
122
|
+
if (Number.isFinite(num) && num >= 0) {
|
|
123
|
+
if (num < 1e9) {
|
|
124
|
+
// Delta seconds (e.g. 30, 60, 300)
|
|
125
|
+
return Math.round(num * 1000);
|
|
126
|
+
}
|
|
127
|
+
// Unix epoch timestamp
|
|
128
|
+
const resetTime = num > 1e12 ? num : num * 1000;
|
|
129
|
+
return Math.max(0, Math.round(resetTime - Date.now()));
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
const parsedDate = Date.parse(trimmed);
|
|
133
|
+
if (Number.isFinite(parsedDate)) {
|
|
134
|
+
return Math.max(0, parsedDate - Date.now());
|
|
135
|
+
}
|
|
136
|
+
const duration = parseDurationMs(trimmed);
|
|
137
|
+
if (duration !== undefined && duration >= 0) {
|
|
138
|
+
return duration;
|
|
139
|
+
}
|
|
140
|
+
return undefined;
|
|
141
|
+
}
|
|
13
142
|
/**
|
|
14
143
|
* Parse standard and commonly used rate-limit response headers.
|
|
15
144
|
*
|
|
@@ -19,21 +148,16 @@ function getHeader(headers, headerName) {
|
|
|
19
148
|
export function parseRetryAfterMs(headers) {
|
|
20
149
|
const retryAfter = getHeader(headers, "Retry-After");
|
|
21
150
|
if (retryAfter) {
|
|
22
|
-
const
|
|
23
|
-
if (
|
|
24
|
-
return
|
|
25
|
-
}
|
|
26
|
-
const retryDate = new Date(retryAfter);
|
|
27
|
-
if (!Number.isNaN(retryDate.getTime())) {
|
|
28
|
-
return Math.max(0, retryDate.getTime() - Date.now());
|
|
151
|
+
const delay = parseRetryAfterHeaderValue(retryAfter);
|
|
152
|
+
if (delay !== undefined) {
|
|
153
|
+
return delay;
|
|
29
154
|
}
|
|
30
155
|
}
|
|
31
156
|
const rateLimitReset = getHeader(headers, "X-RateLimit-Reset");
|
|
32
157
|
if (rateLimitReset) {
|
|
33
|
-
const
|
|
34
|
-
if (
|
|
35
|
-
|
|
36
|
-
return Math.max(0, resetTime - Date.now());
|
|
158
|
+
const delay = parseRateLimitResetValue(rateLimitReset);
|
|
159
|
+
if (delay !== undefined) {
|
|
160
|
+
return delay;
|
|
37
161
|
}
|
|
38
162
|
}
|
|
39
163
|
if (getHeader(headers, "X-RateLimit-Remaining") === "0") {
|
|
@@ -14,6 +14,13 @@
|
|
|
14
14
|
* Use this for **every** download of an external (caller-supplied or
|
|
15
15
|
* third-party-returned) URL. Direct `fetch(url)` of such URLs is unsafe.
|
|
16
16
|
*
|
|
17
|
+
* Redirects are refused by default. A caller that opts in (`maxRedirects`)
|
|
18
|
+
* gets each hop validated and pinned exactly like the first request, and the
|
|
19
|
+
* caller's `headers` are sent to the first host only — a credential meant for
|
|
20
|
+
* one origin is never replayed to the one it redirects to. `allowInsecure`
|
|
21
|
+
* and `allowPrivateNetwork` relax the URL policy for SDK callers that have
|
|
22
|
+
* said so explicitly; the server path never passes them.
|
|
23
|
+
*
|
|
17
24
|
* When the environment configures a proxy for the URL (HTTPS_PROXY / ALL_PROXY,
|
|
18
25
|
* and the host is not in NO_PROXY) the download goes through that proxy and
|
|
19
26
|
* nowhere else: there is no fallback to a direct connection. The proxy resolves
|
|
@@ -22,7 +29,19 @@
|
|
|
22
29
|
*
|
|
23
30
|
* @module utils/safeFetch
|
|
24
31
|
*/
|
|
25
|
-
import type { SafeDownloadOptions } from "../types/index.js";
|
|
32
|
+
import type { SafeDownloadOptions, SafeDownloadResult } from "../types/index.js";
|
|
33
|
+
/**
|
|
34
|
+
* Safely download an external URL and report what came back.
|
|
35
|
+
*
|
|
36
|
+
* Same guarantees as {@link safeDownload}; additionally returns the final
|
|
37
|
+
* response's `Content-Type`/`Content-Encoding`, the URL that produced the body
|
|
38
|
+
* and how many redirect hops were taken.
|
|
39
|
+
*
|
|
40
|
+
* @throws {Error} if any URL on the way is unsafe, the response is too large,
|
|
41
|
+
* a redirect is encountered beyond what `maxRedirects` allows, or the HTTP
|
|
42
|
+
* status indicates failure.
|
|
43
|
+
*/
|
|
44
|
+
export declare function safeDownloadWithMeta(url: string, options: SafeDownloadOptions): Promise<SafeDownloadResult>;
|
|
26
45
|
/**
|
|
27
46
|
* Safely download a binary asset from an external URL.
|
|
28
47
|
*
|
|
@@ -30,3 +49,5 @@ import type { SafeDownloadOptions } from "../types/index.js";
|
|
|
30
49
|
* is encountered, or the HTTP status indicates failure.
|
|
31
50
|
*/
|
|
32
51
|
export declare function safeDownload(url: string, options: SafeDownloadOptions): Promise<Buffer>;
|
|
52
|
+
/** Hop count applied when a caller opts into redirects without naming one. */
|
|
53
|
+
export declare function defaultRedirectHops(follow: boolean): number;
|
package/dist/utils/safeFetch.js
CHANGED
|
@@ -14,6 +14,13 @@
|
|
|
14
14
|
* Use this for **every** download of an external (caller-supplied or
|
|
15
15
|
* third-party-returned) URL. Direct `fetch(url)` of such URLs is unsafe.
|
|
16
16
|
*
|
|
17
|
+
* Redirects are refused by default. A caller that opts in (`maxRedirects`)
|
|
18
|
+
* gets each hop validated and pinned exactly like the first request, and the
|
|
19
|
+
* caller's `headers` are sent to the first host only — a credential meant for
|
|
20
|
+
* one origin is never replayed to the one it redirects to. `allowInsecure`
|
|
21
|
+
* and `allowPrivateNetwork` relax the URL policy for SDK callers that have
|
|
22
|
+
* said so explicitly; the server path never passes them.
|
|
23
|
+
*
|
|
17
24
|
* When the environment configures a proxy for the URL (HTTPS_PROXY / ALL_PROXY,
|
|
18
25
|
* and the host is not in NO_PROXY) the download goes through that proxy and
|
|
19
26
|
* nowhere else: there is no fallback to a direct connection. The proxy resolves
|
|
@@ -86,6 +93,12 @@ function isBoundedBufferResponse(value) {
|
|
|
86
93
|
candidate.headers !== null &&
|
|
87
94
|
typeof candidate.headers.get === "function");
|
|
88
95
|
}
|
|
96
|
+
/**
|
|
97
|
+
* Hop limit when a caller opts into redirects but names no number.
|
|
98
|
+
*/
|
|
99
|
+
const DEFAULT_MAX_REDIRECTS = 5;
|
|
100
|
+
/** Ceiling on the hop count, whatever the caller asks for. */
|
|
101
|
+
const MAX_REDIRECTS_CEILING = 10;
|
|
89
102
|
/**
|
|
90
103
|
* Check the response the same way for both transports, then read it under the
|
|
91
104
|
* size cap.
|
|
@@ -97,6 +110,12 @@ async function readDownloadResponse(response, url, options) {
|
|
|
97
110
|
if (!response.ok) {
|
|
98
111
|
throw new Error(`safeDownload(${options.label}) failed: HTTP ${response.status} for ${url}`);
|
|
99
112
|
}
|
|
113
|
+
if (options.method === "HEAD") {
|
|
114
|
+
// A HEAD has no body to read; drain whatever the transport queued so the
|
|
115
|
+
// connection is released, and hand back nothing.
|
|
116
|
+
await response.body?.cancel().catch(() => undefined);
|
|
117
|
+
return Buffer.alloc(0);
|
|
118
|
+
}
|
|
100
119
|
// readBoundedBuffer expects a Response that exposes Content-Length and
|
|
101
120
|
// arrayBuffer(). undici Response satisfies both.
|
|
102
121
|
if (!isBoundedBufferResponse(response)) {
|
|
@@ -105,51 +124,118 @@ async function readDownloadResponse(response, url, options) {
|
|
|
105
124
|
return readBoundedBuffer(response, options.maxBytes, options.label);
|
|
106
125
|
}
|
|
107
126
|
/**
|
|
108
|
-
*
|
|
109
|
-
*
|
|
110
|
-
*
|
|
111
|
-
*
|
|
127
|
+
* Vet `url` for the chosen transport and return the dispatcher to send it
|
|
128
|
+
* through. Through a proxy the proxy resolves the name, so only the URL text
|
|
129
|
+
* can be vetted; without one the connection is pinned to the addresses the
|
|
130
|
+
* guard cleared. Runs once per hop so a redirect gets the same treatment as
|
|
131
|
+
* the request that produced it.
|
|
112
132
|
*/
|
|
113
|
-
|
|
114
|
-
// Dynamic imports keep undici and the proxy module off the package's static
|
|
115
|
-
// import graph — they are only needed once a download actually happens. See
|
|
116
|
-
// test/continuous-test-suite-import-cost.ts.
|
|
133
|
+
async function prepareHop(url, options) {
|
|
117
134
|
const { getProxyDispatcherForUrl } = await import("../proxy/proxyFetch.js");
|
|
118
135
|
const proxyDispatcher = await getProxyDispatcherForUrl(url);
|
|
119
|
-
const
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
let validatedUrl;
|
|
124
|
-
let dispatcher;
|
|
125
|
-
let closeDispatcher = false;
|
|
136
|
+
const guard = {
|
|
137
|
+
allowInsecure: options.allowInsecure === true,
|
|
138
|
+
allowPrivateNetwork: options.allowPrivateNetwork === true,
|
|
139
|
+
};
|
|
126
140
|
if (proxyDispatcher) {
|
|
127
|
-
assertSafeUrlForProxy(url);
|
|
128
|
-
|
|
129
|
-
dispatcher = proxyDispatcher;
|
|
141
|
+
assertSafeUrlForProxy(url, guard);
|
|
142
|
+
return { dispatcher: proxyDispatcher, owned: false };
|
|
130
143
|
}
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
144
|
+
const resolved = await validateAndResolveUrl(url, guard);
|
|
145
|
+
const hostname = new URL(resolved.url).hostname.replace(/^\[|\]$/g, "");
|
|
146
|
+
return {
|
|
147
|
+
dispatcher: await buildPinnedAgent(hostname, resolved.addresses),
|
|
148
|
+
owned: true,
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* Resolve a `Location` header against the URL that produced it, refusing
|
|
153
|
+
* anything that is not an absolute HTTP(S) URL after resolution.
|
|
154
|
+
*/
|
|
155
|
+
function resolveRedirectTarget(location, from, label, status) {
|
|
156
|
+
if (!location) {
|
|
157
|
+
throw new Error(`safeDownload(${label}): HTTP ${status} redirect without a Location header (for ${from})`);
|
|
137
158
|
}
|
|
159
|
+
let target;
|
|
160
|
+
try {
|
|
161
|
+
target = new URL(location, from);
|
|
162
|
+
}
|
|
163
|
+
catch {
|
|
164
|
+
throw new Error(`safeDownload(${label}): HTTP ${status} redirect to an unparseable Location (for ${from})`);
|
|
165
|
+
}
|
|
166
|
+
// Strip any userinfo a redirect tries to smuggle in; credentials never
|
|
167
|
+
// travel on the URL here.
|
|
168
|
+
target.username = "";
|
|
169
|
+
target.password = "";
|
|
170
|
+
return target.toString();
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* Safely download an external URL and report what came back.
|
|
174
|
+
*
|
|
175
|
+
* Same guarantees as {@link safeDownload}; additionally returns the final
|
|
176
|
+
* response's `Content-Type`/`Content-Encoding`, the URL that produced the body
|
|
177
|
+
* and how many redirect hops were taken.
|
|
178
|
+
*
|
|
179
|
+
* @throws {Error} if any URL on the way is unsafe, the response is too large,
|
|
180
|
+
* a redirect is encountered beyond what `maxRedirects` allows, or the HTTP
|
|
181
|
+
* status indicates failure.
|
|
182
|
+
*/
|
|
183
|
+
export async function safeDownloadWithMeta(url, options) {
|
|
184
|
+
// Dynamic imports keep undici off the package's static import graph — it is
|
|
185
|
+
// only needed once a download actually happens. See
|
|
186
|
+
// test/continuous-test-suite-import-cost.ts.
|
|
187
|
+
const { fetch: undiciFetch } = await import("undici");
|
|
188
|
+
const maxRedirects = Math.min(Math.max(0, Math.floor(options.maxRedirects ?? 0)), MAX_REDIRECTS_CEILING);
|
|
138
189
|
const timeoutCtrl = new AbortController();
|
|
139
190
|
const timeoutId = setTimeout(() => timeoutCtrl.abort(), options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
|
|
140
191
|
const composedSignal = options.signal
|
|
141
192
|
? AbortSignal.any([options.signal, timeoutCtrl.signal])
|
|
142
193
|
: timeoutCtrl.signal;
|
|
194
|
+
let currentUrl = url;
|
|
195
|
+
let hops = 0;
|
|
143
196
|
try {
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
197
|
+
for (;;) {
|
|
198
|
+
const { dispatcher, owned } = await prepareHop(currentUrl, options);
|
|
199
|
+
try {
|
|
200
|
+
const response = await undiciFetch(currentUrl, {
|
|
201
|
+
method: options.method ?? "GET",
|
|
202
|
+
// The caller's headers ride on the FIRST request only. A redirect
|
|
203
|
+
// target is a different authority as far as a credential is
|
|
204
|
+
// concerned, even when the host name happens to match.
|
|
205
|
+
headers: hops === 0 && options.headers ? options.headers : undefined,
|
|
206
|
+
signal: composedSignal,
|
|
207
|
+
redirect: "manual", // a 3xx → private-IP redirect would bypass the guard
|
|
208
|
+
dispatcher,
|
|
209
|
+
});
|
|
210
|
+
if (response.status >= 300 &&
|
|
211
|
+
response.status < 400 &&
|
|
212
|
+
hops < maxRedirects) {
|
|
213
|
+
const next = resolveRedirectTarget(response.headers.get("location"), currentUrl, options.label, response.status);
|
|
214
|
+
// Release the redirect response before the next hop.
|
|
215
|
+
await response.body?.cancel().catch(() => undefined);
|
|
216
|
+
hops += 1;
|
|
217
|
+
currentUrl = next;
|
|
218
|
+
continue;
|
|
219
|
+
}
|
|
220
|
+
// fetch resolves at the headers. Keep the same deadline and abort
|
|
221
|
+
// signal active while the bounded reader drains the body as well.
|
|
222
|
+
const buffer = await readDownloadResponse(response, currentUrl, options);
|
|
223
|
+
return {
|
|
224
|
+
buffer,
|
|
225
|
+
contentType: response.headers.get("content-type") ?? "",
|
|
226
|
+
contentEncoding: response.headers.get("content-encoding") ?? "",
|
|
227
|
+
finalUrl: currentUrl,
|
|
228
|
+
redirects: hops,
|
|
229
|
+
};
|
|
230
|
+
}
|
|
231
|
+
finally {
|
|
232
|
+
if (owned) {
|
|
233
|
+
// Close the per-hop agent so the pinned connection isn't pooled.
|
|
234
|
+
// The proxy agent is shared process-wide and stays open.
|
|
235
|
+
dispatcher.close().catch(() => undefined);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
}
|
|
153
239
|
}
|
|
154
240
|
catch (error) {
|
|
155
241
|
// A status/size rejection may happen before the body is consumed. Stop
|
|
@@ -160,10 +246,18 @@ export async function safeDownload(url, options) {
|
|
|
160
246
|
}
|
|
161
247
|
finally {
|
|
162
248
|
clearTimeout(timeoutId);
|
|
163
|
-
if (closeDispatcher) {
|
|
164
|
-
// Close the per-request agent so the pinned connection isn't pooled.
|
|
165
|
-
// The proxy agent is shared process-wide and stays open.
|
|
166
|
-
dispatcher.close().catch(() => undefined);
|
|
167
|
-
}
|
|
168
249
|
}
|
|
169
250
|
}
|
|
251
|
+
/**
|
|
252
|
+
* Safely download a binary asset from an external URL.
|
|
253
|
+
*
|
|
254
|
+
* @throws {Error} if the URL is unsafe, the response is too large, a redirect
|
|
255
|
+
* is encountered, or the HTTP status indicates failure.
|
|
256
|
+
*/
|
|
257
|
+
export async function safeDownload(url, options) {
|
|
258
|
+
return (await safeDownloadWithMeta(url, options)).buffer;
|
|
259
|
+
}
|
|
260
|
+
/** Hop count applied when a caller opts into redirects without naming one. */
|
|
261
|
+
export function defaultRedirectHops(follow) {
|
|
262
|
+
return follow ? DEFAULT_MAX_REDIRECTS : 0;
|
|
263
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shape check for an id that becomes a file name or a key segment.
|
|
3
|
+
*
|
|
4
|
+
* The same rule `isSafeArtifactId` applies to model-supplied artifact ids,
|
|
5
|
+
* widened by `.` for host-registered skill ids: one leading alphanumeric, then
|
|
6
|
+
* up to 127 of `[A-Za-z0-9._-]`. Nothing that could be a path separator, a
|
|
7
|
+
* traversal segment (a leading `.` is refused, so `..` cannot start one), a
|
|
8
|
+
* glob metacharacter or a line break gets through.
|
|
9
|
+
*
|
|
10
|
+
* @module utils/safeStoreId
|
|
11
|
+
*/
|
|
12
|
+
/** Whether `id` may be joined into a path or a key. */
|
|
13
|
+
export declare function isSafeStoreId(id: string): boolean;
|
|
14
|
+
/**
|
|
15
|
+
* Refuse an unsafe id.
|
|
16
|
+
*
|
|
17
|
+
* @throws {Error} with `code: "INVALID_STORE_ID"`
|
|
18
|
+
*/
|
|
19
|
+
export declare function assertSafeStoreId(id: string, label: string): void;
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shape check for an id that becomes a file name or a key segment.
|
|
3
|
+
*
|
|
4
|
+
* The same rule `isSafeArtifactId` applies to model-supplied artifact ids,
|
|
5
|
+
* widened by `.` for host-registered skill ids: one leading alphanumeric, then
|
|
6
|
+
* up to 127 of `[A-Za-z0-9._-]`. Nothing that could be a path separator, a
|
|
7
|
+
* traversal segment (a leading `.` is refused, so `..` cannot start one), a
|
|
8
|
+
* glob metacharacter or a line break gets through.
|
|
9
|
+
*
|
|
10
|
+
* @module utils/safeStoreId
|
|
11
|
+
*/
|
|
12
|
+
const SAFE_STORE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/;
|
|
13
|
+
/** Whether `id` may be joined into a path or a key. */
|
|
14
|
+
export function isSafeStoreId(id) {
|
|
15
|
+
return SAFE_STORE_ID.test(id);
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Refuse an unsafe id.
|
|
19
|
+
*
|
|
20
|
+
* @throws {Error} with `code: "INVALID_STORE_ID"`
|
|
21
|
+
*/
|
|
22
|
+
export function assertSafeStoreId(id, label) {
|
|
23
|
+
if (isSafeStoreId(id)) {
|
|
24
|
+
return;
|
|
25
|
+
}
|
|
26
|
+
const error = new Error(`${label} "${String(id).slice(0, 64)}" is not a valid id: use 1-128 characters of letters, digits, '.', '_' or '-', starting with a letter or digit`);
|
|
27
|
+
error.code = "INVALID_STORE_ID";
|
|
28
|
+
throw error;
|
|
29
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ZodUnknownSchema } from "../types/index.js";
|
|
1
|
+
import type { SchemaRegexDrop, ZodUnknownSchema } from "../types/index.js";
|
|
2
2
|
/**
|
|
3
3
|
* Inline a JSON Schema by recursively resolving all $ref references.
|
|
4
4
|
* zodToJsonSchema with 'name' option produces schemas with $ref pointing to definitions.
|
|
@@ -59,7 +59,34 @@ export declare function isZodSchema(value: unknown): boolean;
|
|
|
59
59
|
*/
|
|
60
60
|
export declare function isZod4Schema(value: unknown): boolean;
|
|
61
61
|
/**
|
|
62
|
-
* Convert JSON Schema to Zod schema
|
|
63
|
-
*
|
|
62
|
+
* Convert a JSON Schema object to a Zod schema.
|
|
63
|
+
*
|
|
64
|
+
* Built directly from the schema object — no generated source, no
|
|
65
|
+
* `new Function` — so a hostile `description`, `pattern`, `default`, `const`,
|
|
66
|
+
* `enum` or property name is data, never code. Covers the subset tool
|
|
67
|
+
* schemas use: object/array/string/number/integer/boolean/null, `enum`,
|
|
68
|
+
* `const`, `anyOf`/`oneOf`/`allOf`, type unions, local `$ref`s, `nullable`,
|
|
69
|
+
* `required`, `additionalProperties`, `description`, `default` and the common
|
|
70
|
+
* length/range bounds. A `pattern` is compiled only after `vetRegexPattern`
|
|
71
|
+
* accepts it; a pattern that could stall the process is dropped from the
|
|
72
|
+
* schema and reported once at warn level. Unknown constructs validate as
|
|
73
|
+
* `unknown` rather than failing the conversion.
|
|
74
|
+
*
|
|
75
|
+
* Returns `z.object({}).passthrough()` for input that is not a schema object,
|
|
76
|
+
* exactly as the previous implementation did on any failure.
|
|
64
77
|
*/
|
|
65
78
|
export declare function convertJsonSchemaToZod(jsonSchema: Record<string, unknown>): ZodUnknownSchema;
|
|
79
|
+
/**
|
|
80
|
+
* Remove from a JSON Schema every `pattern` and every `patternProperties` key
|
|
81
|
+
* that `vetRegexPattern` refuses, so a validator built from the result (the
|
|
82
|
+
* `@cfworker/json-schema` validator compiles each with `new RegExp` on every
|
|
83
|
+
* check) cannot be stalled by a schema a remote server chose.
|
|
84
|
+
*
|
|
85
|
+
* Returns the input object itself when nothing had to be removed, so the
|
|
86
|
+
* common case costs one walk and no allocation. Otherwise a deep copy with the
|
|
87
|
+
* offending keywords left out, and `drops` names each one. Never throws.
|
|
88
|
+
*/
|
|
89
|
+
export declare function stripUnsafeRegexPatterns(schema: Record<string, unknown>): {
|
|
90
|
+
schema: Record<string, unknown>;
|
|
91
|
+
drops: SchemaRegexDrop[];
|
|
92
|
+
};
|