@inkeep/agents-api 0.0.0-dev-20260320143859 → 0.0.0-dev-20260320150253
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.well-known/workflow/v1/manifest.json +19 -19
- package/dist/createApp.d.ts +2 -2
- package/dist/data/db/runDbClient.d.ts +2 -2
- package/dist/data-reconciliation/handlers/agent.d.ts +2 -2
- package/dist/data-reconciliation/handlers/context-configs.d.ts +2 -2
- package/dist/data-reconciliation/handlers/sub-agents.d.ts +2 -2
- package/dist/data-reconciliation/handlers/tools.d.ts +2 -2
- package/dist/domains/evals/routes/datasetTriggers.d.ts +2 -2
- package/dist/domains/evals/routes/index.d.ts +2 -2
- package/dist/domains/evals/workflow/routes.d.ts +2 -2
- package/dist/domains/manage/routes/authLookup.d.ts +2 -2
- package/dist/domains/manage/routes/availableAgents.d.ts +2 -2
- package/dist/domains/manage/routes/conversations.d.ts +2 -2
- package/dist/domains/manage/routes/index.d.ts +2 -2
- package/dist/domains/manage/routes/invitations.d.ts +2 -2
- package/dist/domains/manage/routes/passwordResetLinks.d.ts +2 -2
- package/dist/domains/manage/routes/signoz.d.ts +2 -2
- package/dist/domains/manage/routes/users.d.ts +2 -2
- package/dist/domains/mcp/routes/mcp.d.ts +2 -2
- package/dist/domains/run/agents/agent-types.d.ts +8 -2
- package/dist/domains/run/agents/generation/conversation-history.d.ts +2 -2
- package/dist/domains/run/agents/generation/conversation-history.js +34 -11
- package/dist/domains/run/agents/generation/generate.js +7 -2
- package/dist/domains/run/agents/generation/tool-result-for-conversation-history.js +2 -2
- package/dist/domains/run/agents/relationTools.d.ts +2 -2
- package/dist/domains/run/routes/auth.d.ts +2 -2
- package/dist/domains/run/routes/chat.js +10 -6
- package/dist/domains/run/routes/chatDataStream.js +3 -43
- package/dist/domains/run/services/blob-storage/external-file-downloader.d.ts +7 -0
- package/dist/domains/run/services/blob-storage/{external-image-downloader.js → external-file-downloader.js} +19 -19
- package/dist/domains/run/services/blob-storage/file-content-security.d.ts +18 -0
- package/dist/domains/run/services/blob-storage/file-content-security.js +55 -0
- package/dist/domains/run/services/blob-storage/{image-security-constants.d.ts → file-security-constants.d.ts} +3 -3
- package/dist/domains/run/services/blob-storage/{image-security-constants.js → file-security-constants.js} +4 -4
- package/dist/domains/run/services/blob-storage/file-security-errors.d.ts +69 -0
- package/dist/domains/run/services/blob-storage/file-security-errors.js +115 -0
- package/dist/domains/run/services/blob-storage/{image-upload-helpers.d.ts → file-upload-helpers.d.ts} +2 -2
- package/dist/domains/run/services/blob-storage/{image-upload-helpers.js → file-upload-helpers.js} +6 -6
- package/dist/domains/run/services/blob-storage/{image-upload.d.ts → file-upload.d.ts} +3 -3
- package/dist/domains/run/services/blob-storage/{image-upload.js → file-upload.js} +24 -12
- package/dist/domains/run/services/blob-storage/file-url-security.d.ts +6 -0
- package/dist/domains/run/services/blob-storage/{image-url-security.js → file-url-security.js} +10 -10
- package/dist/domains/run/services/blob-storage/local-provider.js +1 -1
- package/dist/domains/run/types/chat.d.ts +134 -2
- package/dist/domains/run/types/chat.js +71 -4
- package/dist/domains/run/utils/message-parts.js +31 -25
- package/dist/domains/run/utils/token-estimator.d.ts +2 -2
- package/dist/domains/run/workflow/steps/scheduledTriggerSteps.d.ts +6 -6
- package/dist/index.d.ts +2 -2
- package/dist/middleware/evalsAuth.d.ts +2 -2
- package/dist/middleware/manageAuth.d.ts +3 -3
- package/dist/middleware/projectConfig.d.ts +3 -3
- package/dist/middleware/requirePermission.d.ts +2 -2
- package/dist/middleware/runAuth.d.ts +4 -4
- package/dist/middleware/sessionAuth.d.ts +3 -3
- package/dist/middleware/tenantAccess.d.ts +2 -2
- package/dist/middleware/tracing.d.ts +3 -3
- package/package.json +5 -5
- package/dist/domains/run/services/blob-storage/external-image-downloader.d.ts +0 -7
- package/dist/domains/run/services/blob-storage/image-content-security.d.ts +0 -11
- package/dist/domains/run/services/blob-storage/image-content-security.js +0 -38
- package/dist/domains/run/services/blob-storage/image-security-errors.d.ts +0 -66
- package/dist/domains/run/services/blob-storage/image-security-errors.js +0 -110
- package/dist/domains/run/services/blob-storage/image-url-security.d.ts +0 -6
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
import { EXTERNAL_FETCH_TIMEOUT_MS,
|
|
2
|
-
import { BlockedConnectionToPrivateIpError,
|
|
3
|
-
import { resolveDownloadedImageMimeType } from "./
|
|
4
|
-
import { isBlockedIpAddress,
|
|
1
|
+
import { EXTERNAL_FETCH_TIMEOUT_MS, MAX_EXTERNAL_REDIRECTS, MAX_FILE_BYTES } from "./file-security-constants.js";
|
|
2
|
+
import { BlockedConnectionToPrivateIpError, BlockedExternalFileExceedingError, BlockedExternalFileLargerThanError, ExternalFileResponseBodyEmptyError, FailedToDownloadError, FileSecurityError, RedirectMissingLocationError, TimedOutDownloadingError, TooManyRedirectsError, UnableToResolveHostError, UnexpectedRedirectStateError } from "./file-security-errors.js";
|
|
3
|
+
import { resolveDownloadedImageMimeType } from "./file-content-security.js";
|
|
4
|
+
import { isBlockedIpAddress, validateExternalFileUrl, validateUrlResolvesToPublicIp } from "./file-url-security.js";
|
|
5
5
|
import { retryWithBackoff } from "@inkeep/agents-core";
|
|
6
6
|
import { lookup } from "node:dns";
|
|
7
7
|
import { Agent } from "undici";
|
|
8
8
|
|
|
9
|
-
//#region src/domains/run/services/blob-storage/external-
|
|
9
|
+
//#region src/domains/run/services/blob-storage/external-file-downloader.ts
|
|
10
10
|
const externalImageDispatcher = new Agent({ connect: { lookup(hostname, options, callback) {
|
|
11
11
|
lookup(hostname, options, (error, address, family) => {
|
|
12
12
|
if (error) {
|
|
@@ -34,8 +34,8 @@ const externalImageDispatcher = new Agent({ connect: { lookup(hostname, options,
|
|
|
34
34
|
});
|
|
35
35
|
} } });
|
|
36
36
|
const MAX_EXTERNAL_FETCH_ATTEMPTS = 3;
|
|
37
|
-
async function
|
|
38
|
-
let currentUrl =
|
|
37
|
+
async function downloadExternalFile(url) {
|
|
38
|
+
let currentUrl = validateExternalFileUrl(url);
|
|
39
39
|
await validateUrlResolvesToPublicIp(currentUrl);
|
|
40
40
|
for (let redirectCount = 0; redirectCount <= MAX_EXTERNAL_REDIRECTS; redirectCount++) {
|
|
41
41
|
const response = await fetchWithRetry(currentUrl);
|
|
@@ -43,15 +43,15 @@ async function downloadExternalImage(url) {
|
|
|
43
43
|
const location = response.headers.get("location");
|
|
44
44
|
if (!location) throw new RedirectMissingLocationError(toSanitizedUrl(currentUrl));
|
|
45
45
|
if (redirectCount === MAX_EXTERNAL_REDIRECTS) throw new TooManyRedirectsError(toSanitizedUrl(url));
|
|
46
|
-
currentUrl =
|
|
46
|
+
currentUrl = validateExternalFileUrl(new URL(location, currentUrl).toString());
|
|
47
47
|
await validateUrlResolvesToPublicIp(currentUrl);
|
|
48
48
|
continue;
|
|
49
49
|
}
|
|
50
50
|
if (!response.ok) throw new FailedToDownloadError(toSanitizedUrl(currentUrl), `${response.status} ${response.statusText}`);
|
|
51
51
|
const headerContentType = (response.headers.get("content-type") || "").split(";")[0].trim().toLowerCase();
|
|
52
52
|
const contentLength = response.headers.get("content-length");
|
|
53
|
-
if (contentLength && Number.isFinite(Number(contentLength)) && Number(contentLength) >
|
|
54
|
-
const data = await readResponseBytesWithLimit(response,
|
|
53
|
+
if (contentLength && Number.isFinite(Number(contentLength)) && Number(contentLength) > MAX_FILE_BYTES) throw new BlockedExternalFileLargerThanError(MAX_FILE_BYTES, contentLength);
|
|
54
|
+
const data = await readResponseBytesWithLimit(response, MAX_FILE_BYTES);
|
|
55
55
|
return {
|
|
56
56
|
data,
|
|
57
57
|
mimeType: await resolveDownloadedImageMimeType(data, headerContentType)
|
|
@@ -77,7 +77,7 @@ async function fetchWithRetry(url) {
|
|
|
77
77
|
}, {
|
|
78
78
|
maxAttempts: MAX_EXTERNAL_FETCH_ATTEMPTS,
|
|
79
79
|
maxDelayMs: 2e3,
|
|
80
|
-
label: `
|
|
80
|
+
label: `file-download ${toSanitizedUrl(url)}`
|
|
81
81
|
});
|
|
82
82
|
}
|
|
83
83
|
function isRedirectStatus(status) {
|
|
@@ -87,7 +87,7 @@ function isRetryableStatus(status) {
|
|
|
87
87
|
return status === 429 || status >= 500 && status <= 599;
|
|
88
88
|
}
|
|
89
89
|
async function readResponseBytesWithLimit(response, maxBytes) {
|
|
90
|
-
if (!response.body) throw new
|
|
90
|
+
if (!response.body) throw new ExternalFileResponseBodyEmptyError();
|
|
91
91
|
const reader = response.body.getReader();
|
|
92
92
|
const chunks = [];
|
|
93
93
|
let totalBytes = 0;
|
|
@@ -97,7 +97,7 @@ async function readResponseBytesWithLimit(response, maxBytes) {
|
|
|
97
97
|
if (done) break;
|
|
98
98
|
if (!value) continue;
|
|
99
99
|
totalBytes += value.byteLength;
|
|
100
|
-
if (totalBytes > maxBytes) throw new
|
|
100
|
+
if (totalBytes > maxBytes) throw new BlockedExternalFileExceedingError(maxBytes);
|
|
101
101
|
chunks.push(value);
|
|
102
102
|
}
|
|
103
103
|
} finally {
|
|
@@ -119,15 +119,15 @@ async function fetchWithConnectionIpValidation(url) {
|
|
|
119
119
|
dispatcher: externalImageDispatcher
|
|
120
120
|
});
|
|
121
121
|
} catch (error) {
|
|
122
|
-
const
|
|
123
|
-
if (
|
|
122
|
+
const fileSecurityError = extractFileSecurityError(error);
|
|
123
|
+
if (fileSecurityError instanceof BlockedConnectionToPrivateIpError || fileSecurityError instanceof UnableToResolveHostError) throw fileSecurityError;
|
|
124
124
|
if (error instanceof Error && (error.name === "AbortError" || error.name === "TimeoutError")) throw new TimedOutDownloadingError(toSanitizedUrl(url));
|
|
125
125
|
throw new FailedToDownloadError(toSanitizedUrl(url), void 0, { cause: error });
|
|
126
126
|
}
|
|
127
127
|
}
|
|
128
|
-
function
|
|
129
|
-
if (error instanceof
|
|
130
|
-
if (error instanceof Error && error.cause) return
|
|
128
|
+
function extractFileSecurityError(error) {
|
|
129
|
+
if (error instanceof FileSecurityError) return error;
|
|
130
|
+
if (error instanceof Error && error.cause) return extractFileSecurityError(error.cause);
|
|
131
131
|
return null;
|
|
132
132
|
}
|
|
133
133
|
function toSanitizedUrl(url) {
|
|
@@ -138,4 +138,4 @@ function toSanitizedUrl(url) {
|
|
|
138
138
|
}
|
|
139
139
|
|
|
140
140
|
//#endregion
|
|
141
|
-
export {
|
|
141
|
+
export { downloadExternalFile };
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
//#region src/domains/run/services/blob-storage/file-content-security.d.ts
|
|
2
|
+
declare function normalizeInlineImageBytes(file: {
|
|
3
|
+
bytes: string;
|
|
4
|
+
mimeType?: string;
|
|
5
|
+
}): Promise<{
|
|
6
|
+
data: Uint8Array;
|
|
7
|
+
mimeType: string;
|
|
8
|
+
}>;
|
|
9
|
+
declare function normalizeInlineFileBytes(file: {
|
|
10
|
+
bytes: string;
|
|
11
|
+
mimeType?: string;
|
|
12
|
+
}): Promise<{
|
|
13
|
+
data: Uint8Array;
|
|
14
|
+
mimeType: string;
|
|
15
|
+
}>;
|
|
16
|
+
declare function resolveDownloadedImageMimeType(data: Uint8Array, headerContentType: string): Promise<string>;
|
|
17
|
+
//#endregion
|
|
18
|
+
export { normalizeInlineFileBytes, normalizeInlineImageBytes, resolveDownloadedImageMimeType };
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { ALLOWED_EXTERNAL_IMAGE_MIME_TYPES, MAX_FILE_BYTES } from "./file-security-constants.js";
|
|
2
|
+
import { BlockedExternalUnsupportedBytesError, BlockedInlineFileExceedingError, BlockedInlineUnsupportedFileBytesError, InvalidInlineFileMalformedBase64Error } from "./file-security-errors.js";
|
|
3
|
+
import { fileTypeFromBuffer } from "file-type";
|
|
4
|
+
|
|
5
|
+
//#region src/domains/run/services/blob-storage/file-content-security.ts
|
|
6
|
+
async function normalizeInlineImageBytes(file) {
|
|
7
|
+
const normalized = await normalizeInlineFileBytes(file);
|
|
8
|
+
if (normalized.mimeType.startsWith("image/")) return normalized;
|
|
9
|
+
throw new BlockedInlineUnsupportedFileBytesError(file.mimeType || "unknown");
|
|
10
|
+
}
|
|
11
|
+
async function normalizeInlineFileBytes(file) {
|
|
12
|
+
const data = decodeBase64Bytes(file.bytes);
|
|
13
|
+
validateInlineFileSize(data);
|
|
14
|
+
const requestedMimeType = file.mimeType?.split(";")[0]?.trim().toLowerCase();
|
|
15
|
+
const sniffedMime = await sniffAllowedInlineFileMimeType(data, requestedMimeType);
|
|
16
|
+
if (sniffedMime) return {
|
|
17
|
+
data,
|
|
18
|
+
mimeType: sniffedMime
|
|
19
|
+
};
|
|
20
|
+
throw new BlockedInlineUnsupportedFileBytesError(file.mimeType || "unknown");
|
|
21
|
+
}
|
|
22
|
+
async function resolveDownloadedImageMimeType(data, headerContentType) {
|
|
23
|
+
const sniffedMime = await sniffAllowedImageMimeType(data);
|
|
24
|
+
if (sniffedMime) return sniffedMime;
|
|
25
|
+
throw new BlockedExternalUnsupportedBytesError(headerContentType || "unknown");
|
|
26
|
+
}
|
|
27
|
+
function validateInlineFileSize(data) {
|
|
28
|
+
if (data.length > MAX_FILE_BYTES) throw new BlockedInlineFileExceedingError(MAX_FILE_BYTES);
|
|
29
|
+
}
|
|
30
|
+
async function sniffAllowedImageMimeType(data) {
|
|
31
|
+
const sniffedMime = (await fileTypeFromBuffer(data))?.mime?.toLowerCase();
|
|
32
|
+
if (sniffedMime && ALLOWED_EXTERNAL_IMAGE_MIME_TYPES.has(sniffedMime)) return sniffedMime;
|
|
33
|
+
return null;
|
|
34
|
+
}
|
|
35
|
+
async function sniffAllowedInlineFileMimeType(data, requestedMimeType) {
|
|
36
|
+
if (requestedMimeType === "application/pdf") {
|
|
37
|
+
if (!looksLikePdf(data)) throw new BlockedInlineUnsupportedFileBytesError(requestedMimeType);
|
|
38
|
+
return "application/pdf";
|
|
39
|
+
}
|
|
40
|
+
return await sniffAllowedImageMimeType(data);
|
|
41
|
+
}
|
|
42
|
+
function decodeBase64Bytes(base64Bytes) {
|
|
43
|
+
const normalized = base64Bytes.replace(/\s+/g, "");
|
|
44
|
+
if (normalized.length === 0 || normalized.length % 4 !== 0 || !/^[A-Za-z0-9+/]*={0,2}$/.test(normalized)) throw new InvalidInlineFileMalformedBase64Error();
|
|
45
|
+
const decoded = Buffer.from(normalized, "base64");
|
|
46
|
+
if (decoded.length === 0 || decoded.toString("base64") !== normalized) throw new InvalidInlineFileMalformedBase64Error();
|
|
47
|
+
return Uint8Array.from(decoded);
|
|
48
|
+
}
|
|
49
|
+
function looksLikePdf(data) {
|
|
50
|
+
if (data.length < 5) return false;
|
|
51
|
+
return data[0] === 37 && data[1] === 80 && data[2] === 68 && data[3] === 70 && data[4] === 45;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
//#endregion
|
|
55
|
+
export { normalizeInlineFileBytes, normalizeInlineImageBytes, resolveDownloadedImageMimeType };
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
//#region src/domains/run/services/blob-storage/
|
|
2
|
-
declare const
|
|
1
|
+
//#region src/domains/run/services/blob-storage/file-security-constants.d.ts
|
|
2
|
+
declare const MAX_FILE_BYTES: number;
|
|
3
3
|
declare const EXTERNAL_FETCH_TIMEOUT_MS = 10000;
|
|
4
4
|
declare const MAX_EXTERNAL_REDIRECTS = 3;
|
|
5
5
|
declare const ALLOWED_EXTERNAL_IMAGE_MIME_TYPES: Set<string>;
|
|
6
6
|
declare const ALLOWED_HTTP_PORTS: Set<string>;
|
|
7
7
|
//#endregion
|
|
8
|
-
export { ALLOWED_EXTERNAL_IMAGE_MIME_TYPES, ALLOWED_HTTP_PORTS, EXTERNAL_FETCH_TIMEOUT_MS,
|
|
8
|
+
export { ALLOWED_EXTERNAL_IMAGE_MIME_TYPES, ALLOWED_HTTP_PORTS, EXTERNAL_FETCH_TIMEOUT_MS, MAX_EXTERNAL_REDIRECTS, MAX_FILE_BYTES };
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { ALLOWED_IMAGE_MIME_TYPES } from "@inkeep/agents-core/constants/allowed-
|
|
1
|
+
import { ALLOWED_IMAGE_MIME_TYPES } from "@inkeep/agents-core/constants/allowed-file-formats";
|
|
2
2
|
|
|
3
|
-
//#region src/domains/run/services/blob-storage/
|
|
4
|
-
const
|
|
3
|
+
//#region src/domains/run/services/blob-storage/file-security-constants.ts
|
|
4
|
+
const MAX_FILE_BYTES = 10 * 1024 * 1024;
|
|
5
5
|
const EXTERNAL_FETCH_TIMEOUT_MS = 1e4;
|
|
6
6
|
const MAX_EXTERNAL_REDIRECTS = 3;
|
|
7
7
|
const ALLOWED_EXTERNAL_IMAGE_MIME_TYPES = ALLOWED_IMAGE_MIME_TYPES;
|
|
@@ -12,4 +12,4 @@ const ALLOWED_HTTP_PORTS = new Set([
|
|
|
12
12
|
]);
|
|
13
13
|
|
|
14
14
|
//#endregion
|
|
15
|
-
export { ALLOWED_EXTERNAL_IMAGE_MIME_TYPES, ALLOWED_HTTP_PORTS, EXTERNAL_FETCH_TIMEOUT_MS,
|
|
15
|
+
export { ALLOWED_EXTERNAL_IMAGE_MIME_TYPES, ALLOWED_HTTP_PORTS, EXTERNAL_FETCH_TIMEOUT_MS, MAX_EXTERNAL_REDIRECTS, MAX_FILE_BYTES };
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
//#region src/domains/run/services/blob-storage/file-security-errors.d.ts
|
|
2
|
+
declare class FileSecurityError extends Error {
|
|
3
|
+
constructor(message: string, options?: ErrorOptions);
|
|
4
|
+
}
|
|
5
|
+
declare class InvalidExternalFileUrlError extends FileSecurityError {
|
|
6
|
+
constructor(rawUrl: string);
|
|
7
|
+
}
|
|
8
|
+
declare class BlockedUnsupportedSchemeError extends FileSecurityError {
|
|
9
|
+
constructor(protocol: string);
|
|
10
|
+
}
|
|
11
|
+
declare class BlockedDisallowedPortError extends FileSecurityError {
|
|
12
|
+
constructor(port: string);
|
|
13
|
+
}
|
|
14
|
+
declare class BlockedEmbeddedCredentialsError extends FileSecurityError {
|
|
15
|
+
constructor();
|
|
16
|
+
}
|
|
17
|
+
declare class NoIpResolvedError extends FileSecurityError {
|
|
18
|
+
constructor(hostname: string);
|
|
19
|
+
}
|
|
20
|
+
declare class BlockedUrlResolvingToPrivateIpError extends FileSecurityError {
|
|
21
|
+
constructor(ip: string);
|
|
22
|
+
}
|
|
23
|
+
declare class UnableToResolveHostError extends FileSecurityError {
|
|
24
|
+
constructor(hostname: string, options?: ErrorOptions);
|
|
25
|
+
}
|
|
26
|
+
declare class BlockedConnectionToPrivateIpError extends FileSecurityError {
|
|
27
|
+
constructor(address: string);
|
|
28
|
+
}
|
|
29
|
+
declare class RedirectMissingLocationError extends FileSecurityError {
|
|
30
|
+
constructor(url: string);
|
|
31
|
+
}
|
|
32
|
+
declare class TooManyRedirectsError extends FileSecurityError {
|
|
33
|
+
constructor(url: string);
|
|
34
|
+
}
|
|
35
|
+
declare class FailedToDownloadError extends FileSecurityError {
|
|
36
|
+
constructor(url: string, statusText?: string, options?: ErrorOptions);
|
|
37
|
+
}
|
|
38
|
+
declare class BlockedExternalFileLargerThanError extends FileSecurityError {
|
|
39
|
+
constructor(maxBytes: number, contentLength: string);
|
|
40
|
+
}
|
|
41
|
+
declare class BlockedExternalFileExceedingError extends FileSecurityError {
|
|
42
|
+
constructor(maxBytes: number);
|
|
43
|
+
}
|
|
44
|
+
declare class ExternalFileResponseBodyEmptyError extends FileSecurityError {
|
|
45
|
+
constructor();
|
|
46
|
+
}
|
|
47
|
+
declare class UnexpectedRedirectStateError extends FileSecurityError {
|
|
48
|
+
constructor(url: string);
|
|
49
|
+
}
|
|
50
|
+
declare class TimedOutDownloadingError extends FileSecurityError {
|
|
51
|
+
constructor(url: string);
|
|
52
|
+
}
|
|
53
|
+
declare class BlockedInlineFileExceedingError extends FileSecurityError {
|
|
54
|
+
constructor(maxBytes: number);
|
|
55
|
+
}
|
|
56
|
+
declare class BlockedInlineUnsupportedFileBytesError extends FileSecurityError {
|
|
57
|
+
constructor(claimedContentType: string);
|
|
58
|
+
}
|
|
59
|
+
declare class BlockedExternalUnsupportedBytesError extends FileSecurityError {
|
|
60
|
+
constructor(contentType: string);
|
|
61
|
+
}
|
|
62
|
+
declare class BlockedExternalPdfUrlNotSupportedError extends FileSecurityError {
|
|
63
|
+
constructor();
|
|
64
|
+
}
|
|
65
|
+
declare class InvalidInlineFileMalformedBase64Error extends FileSecurityError {
|
|
66
|
+
constructor();
|
|
67
|
+
}
|
|
68
|
+
//#endregion
|
|
69
|
+
export { BlockedConnectionToPrivateIpError, BlockedDisallowedPortError, BlockedEmbeddedCredentialsError, BlockedExternalFileExceedingError, BlockedExternalFileLargerThanError, BlockedExternalPdfUrlNotSupportedError, BlockedExternalUnsupportedBytesError, BlockedInlineFileExceedingError, BlockedInlineUnsupportedFileBytesError, BlockedUnsupportedSchemeError, BlockedUrlResolvingToPrivateIpError, ExternalFileResponseBodyEmptyError, FailedToDownloadError, FileSecurityError, InvalidExternalFileUrlError, InvalidInlineFileMalformedBase64Error, NoIpResolvedError, RedirectMissingLocationError, TimedOutDownloadingError, TooManyRedirectsError, UnableToResolveHostError, UnexpectedRedirectStateError };
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
//#region src/domains/run/services/blob-storage/file-security-errors.ts
|
|
2
|
+
var FileSecurityError = class extends Error {
|
|
3
|
+
constructor(message, options) {
|
|
4
|
+
super(message, options);
|
|
5
|
+
this.name = new.target.name;
|
|
6
|
+
}
|
|
7
|
+
};
|
|
8
|
+
var InvalidExternalFileUrlError = class extends FileSecurityError {
|
|
9
|
+
constructor(rawUrl) {
|
|
10
|
+
super(`Invalid external file URL: ${rawUrl}`);
|
|
11
|
+
}
|
|
12
|
+
};
|
|
13
|
+
var BlockedUnsupportedSchemeError = class extends FileSecurityError {
|
|
14
|
+
constructor(protocol) {
|
|
15
|
+
super(`Blocked external file URL with unsupported scheme: ${protocol}`);
|
|
16
|
+
}
|
|
17
|
+
};
|
|
18
|
+
var BlockedDisallowedPortError = class extends FileSecurityError {
|
|
19
|
+
constructor(port) {
|
|
20
|
+
super(`Blocked external file URL with disallowed port: ${port}`);
|
|
21
|
+
}
|
|
22
|
+
};
|
|
23
|
+
var BlockedEmbeddedCredentialsError = class extends FileSecurityError {
|
|
24
|
+
constructor() {
|
|
25
|
+
super("Blocked external file URL with embedded credentials");
|
|
26
|
+
}
|
|
27
|
+
};
|
|
28
|
+
var NoIpResolvedError = class extends FileSecurityError {
|
|
29
|
+
constructor(hostname) {
|
|
30
|
+
super(`No IP addresses resolved for host: ${hostname}`);
|
|
31
|
+
}
|
|
32
|
+
};
|
|
33
|
+
var BlockedUrlResolvingToPrivateIpError = class extends FileSecurityError {
|
|
34
|
+
constructor(ip) {
|
|
35
|
+
super(`Blocked external file URL resolving to private or reserved IP: ${ip}`);
|
|
36
|
+
}
|
|
37
|
+
};
|
|
38
|
+
var UnableToResolveHostError = class extends FileSecurityError {
|
|
39
|
+
constructor(hostname, options) {
|
|
40
|
+
super(`Unable to resolve external file host: ${hostname}`, options);
|
|
41
|
+
}
|
|
42
|
+
};
|
|
43
|
+
var BlockedConnectionToPrivateIpError = class extends FileSecurityError {
|
|
44
|
+
constructor(address) {
|
|
45
|
+
super(`Blocked external file connection to private or reserved IP: ${address}`);
|
|
46
|
+
}
|
|
47
|
+
};
|
|
48
|
+
var RedirectMissingLocationError = class extends FileSecurityError {
|
|
49
|
+
constructor(url) {
|
|
50
|
+
super(`Redirect response missing location header: ${url}`);
|
|
51
|
+
}
|
|
52
|
+
};
|
|
53
|
+
var TooManyRedirectsError = class extends FileSecurityError {
|
|
54
|
+
constructor(url) {
|
|
55
|
+
super(`Too many redirects while downloading file: ${url}`);
|
|
56
|
+
}
|
|
57
|
+
};
|
|
58
|
+
var FailedToDownloadError = class extends FileSecurityError {
|
|
59
|
+
constructor(url, statusText, options) {
|
|
60
|
+
super(statusText ? `Failed to download file from ${url}: ${statusText}` : `Failed to download file from ${url}`, options);
|
|
61
|
+
}
|
|
62
|
+
};
|
|
63
|
+
var BlockedExternalFileLargerThanError = class extends FileSecurityError {
|
|
64
|
+
constructor(maxBytes, contentLength) {
|
|
65
|
+
super(`Blocked external file larger than ${maxBytes} bytes: ${contentLength}`);
|
|
66
|
+
}
|
|
67
|
+
};
|
|
68
|
+
var BlockedExternalFileExceedingError = class extends FileSecurityError {
|
|
69
|
+
constructor(maxBytes) {
|
|
70
|
+
super(`Blocked external file exceeding ${maxBytes} bytes`);
|
|
71
|
+
}
|
|
72
|
+
};
|
|
73
|
+
var ExternalFileResponseBodyEmptyError = class extends FileSecurityError {
|
|
74
|
+
constructor() {
|
|
75
|
+
super("External file response body is empty");
|
|
76
|
+
}
|
|
77
|
+
};
|
|
78
|
+
var UnexpectedRedirectStateError = class extends FileSecurityError {
|
|
79
|
+
constructor(url) {
|
|
80
|
+
super(`Unexpected redirect handling state for URL: ${url}`);
|
|
81
|
+
}
|
|
82
|
+
};
|
|
83
|
+
var TimedOutDownloadingError = class extends FileSecurityError {
|
|
84
|
+
constructor(url) {
|
|
85
|
+
super(`Timed out downloading file from ${url}`);
|
|
86
|
+
}
|
|
87
|
+
};
|
|
88
|
+
var BlockedInlineFileExceedingError = class extends FileSecurityError {
|
|
89
|
+
constructor(maxBytes) {
|
|
90
|
+
super(`Blocked inline file exceeding ${maxBytes} bytes`);
|
|
91
|
+
}
|
|
92
|
+
};
|
|
93
|
+
var BlockedInlineUnsupportedFileBytesError = class extends FileSecurityError {
|
|
94
|
+
constructor(claimedContentType) {
|
|
95
|
+
super(`Blocked inline file with unsupported bytes signature (claimed content-type: ${claimedContentType})`);
|
|
96
|
+
}
|
|
97
|
+
};
|
|
98
|
+
var BlockedExternalUnsupportedBytesError = class extends FileSecurityError {
|
|
99
|
+
constructor(contentType) {
|
|
100
|
+
super(`Blocked external file with unsupported bytes signature (content-type: ${contentType})`);
|
|
101
|
+
}
|
|
102
|
+
};
|
|
103
|
+
var BlockedExternalPdfUrlNotSupportedError = class extends FileSecurityError {
|
|
104
|
+
constructor() {
|
|
105
|
+
super("External PDF URLs are not supported; provide PDFs as data URIs");
|
|
106
|
+
}
|
|
107
|
+
};
|
|
108
|
+
var InvalidInlineFileMalformedBase64Error = class extends FileSecurityError {
|
|
109
|
+
constructor() {
|
|
110
|
+
super("Invalid inline file: malformed base64 payload");
|
|
111
|
+
}
|
|
112
|
+
};
|
|
113
|
+
|
|
114
|
+
//#endregion
|
|
115
|
+
export { BlockedConnectionToPrivateIpError, BlockedDisallowedPortError, BlockedEmbeddedCredentialsError, BlockedExternalFileExceedingError, BlockedExternalFileLargerThanError, BlockedExternalPdfUrlNotSupportedError, BlockedExternalUnsupportedBytesError, BlockedInlineFileExceedingError, BlockedInlineUnsupportedFileBytesError, BlockedUnsupportedSchemeError, BlockedUrlResolvingToPrivateIpError, ExternalFileResponseBodyEmptyError, FailedToDownloadError, FileSecurityError, InvalidExternalFileUrlError, InvalidInlineFileMalformedBase64Error, NoIpResolvedError, RedirectMissingLocationError, TimedOutDownloadingError, TooManyRedirectsError, UnableToResolveHostError, UnexpectedRedirectStateError };
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { UploadContext } from "./
|
|
1
|
+
import { UploadContext } from "./file-upload.js";
|
|
2
2
|
import { MessageContent, Part } from "@inkeep/agents-core";
|
|
3
3
|
|
|
4
|
-
//#region src/domains/run/services/blob-storage/
|
|
4
|
+
//#region src/domains/run/services/blob-storage/file-upload-helpers.d.ts
|
|
5
5
|
declare function buildPersistedMessageContent(text: string, parts: Part[], ctx: UploadContext): Promise<MessageContent>;
|
|
6
6
|
//#endregion
|
|
7
7
|
export { buildPersistedMessageContent };
|
package/dist/domains/run/services/blob-storage/{image-upload-helpers.js → file-upload-helpers.js}
RENAMED
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
import { getLogger } from "../../../../logger.js";
|
|
2
|
-
import { hasFileParts, makeMessageContentParts,
|
|
2
|
+
import { hasFileParts, makeMessageContentParts, uploadPartsFiles } from "./file-upload.js";
|
|
3
3
|
|
|
4
|
-
//#region src/domains/run/services/blob-storage/
|
|
5
|
-
const logger = getLogger("
|
|
4
|
+
//#region src/domains/run/services/blob-storage/file-upload-helpers.ts
|
|
5
|
+
const logger = getLogger("file-upload-helpers");
|
|
6
6
|
async function buildPersistedMessageContent(text, parts, ctx) {
|
|
7
7
|
if (!hasFileParts(parts)) return { text };
|
|
8
8
|
try {
|
|
9
|
-
const contentParts = makeMessageContentParts(await
|
|
9
|
+
const contentParts = makeMessageContentParts(await uploadPartsFiles(parts, ctx));
|
|
10
10
|
logger.debug({
|
|
11
11
|
messageId: ctx.messageId,
|
|
12
12
|
originalParts: parts.length,
|
|
13
13
|
uploadedParts: contentParts.length,
|
|
14
14
|
fileParts: contentParts.filter((p) => p.kind === "file").length
|
|
15
|
-
}, "Built persisted message content with uploaded
|
|
15
|
+
}, "Built persisted message content with uploaded files");
|
|
16
16
|
return {
|
|
17
17
|
text,
|
|
18
18
|
parts: contentParts
|
|
@@ -21,7 +21,7 @@ async function buildPersistedMessageContent(text, parts, ctx) {
|
|
|
21
21
|
logger.error({
|
|
22
22
|
error: error instanceof Error ? error.message : String(error),
|
|
23
23
|
messageId: ctx.messageId
|
|
24
|
-
}, "Failed to upload
|
|
24
|
+
}, "Failed to upload files, persisting text only");
|
|
25
25
|
return { text };
|
|
26
26
|
}
|
|
27
27
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { MessageContent, Part } from "@inkeep/agents-core";
|
|
2
2
|
|
|
3
|
-
//#region src/domains/run/services/blob-storage/
|
|
3
|
+
//#region src/domains/run/services/blob-storage/file-upload.d.ts
|
|
4
4
|
type MessageContentPart = NonNullable<MessageContent['parts']>[number];
|
|
5
5
|
interface UploadContext {
|
|
6
6
|
tenantId: string;
|
|
@@ -8,8 +8,8 @@ interface UploadContext {
|
|
|
8
8
|
conversationId: string;
|
|
9
9
|
messageId: string;
|
|
10
10
|
}
|
|
11
|
-
declare function
|
|
11
|
+
declare function uploadPartsFiles(parts: Part[], ctx: UploadContext): Promise<Part[]>;
|
|
12
12
|
declare function makeMessageContentParts(parts: Part[]): MessageContentPart[];
|
|
13
13
|
declare function hasFileParts(parts: Part[]): boolean;
|
|
14
14
|
//#endregion
|
|
15
|
-
export { UploadContext, hasFileParts, makeMessageContentParts,
|
|
15
|
+
export { UploadContext, hasFileParts, makeMessageContentParts, uploadPartsFiles };
|
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
import { getLogger } from "../../../../logger.js";
|
|
2
2
|
import { getBlobStorageProvider, toBlobUri } from "./index.js";
|
|
3
3
|
import { buildStorageKey } from "./storage-keys.js";
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
4
|
+
import { BlockedExternalPdfUrlNotSupportedError } from "./file-security-errors.js";
|
|
5
|
+
import { normalizeInlineFileBytes } from "./file-content-security.js";
|
|
6
|
+
import { downloadExternalFile } from "./external-file-downloader.js";
|
|
6
7
|
import { createHash } from "node:crypto";
|
|
7
|
-
import { getExtensionFromMimeType } from "@inkeep/agents-core/constants/allowed-
|
|
8
|
+
import { getExtensionFromMimeType } from "@inkeep/agents-core/constants/allowed-file-formats";
|
|
8
9
|
|
|
9
|
-
//#region src/domains/run/services/blob-storage/
|
|
10
|
-
const logger = getLogger("
|
|
10
|
+
//#region src/domains/run/services/blob-storage/file-upload.ts
|
|
11
|
+
const logger = getLogger("file-upload");
|
|
11
12
|
const FILE_UPLOAD_CONCURRENCY = 3;
|
|
12
13
|
async function uploadFilePart(part, ctx, index) {
|
|
13
14
|
const storage = getBlobStorageProvider();
|
|
@@ -15,11 +16,12 @@ async function uploadFilePart(part, ctx, index) {
|
|
|
15
16
|
let data;
|
|
16
17
|
let mimeType;
|
|
17
18
|
if ("bytes" in file && file.bytes) {
|
|
18
|
-
const normalized = await
|
|
19
|
+
const normalized = await normalizeInlineFileBytes(file);
|
|
19
20
|
data = normalized.data;
|
|
20
21
|
mimeType = normalized.mimeType;
|
|
21
22
|
} else if ("uri" in file && file.uri) {
|
|
22
|
-
|
|
23
|
+
if (file.mimeType?.toLowerCase().startsWith("application/pdf")) throw new BlockedExternalPdfUrlNotSupportedError();
|
|
24
|
+
const downloaded = await downloadExternalFile(file.uri);
|
|
23
25
|
data = downloaded.data;
|
|
24
26
|
mimeType = downloaded.mimeType;
|
|
25
27
|
} else {
|
|
@@ -46,7 +48,7 @@ async function uploadFilePart(part, ctx, index) {
|
|
|
46
48
|
key,
|
|
47
49
|
mimeType,
|
|
48
50
|
size: data.length
|
|
49
|
-
}, "Uploaded
|
|
51
|
+
}, "Uploaded file to blob storage");
|
|
50
52
|
return {
|
|
51
53
|
kind: "file",
|
|
52
54
|
file: {
|
|
@@ -56,7 +58,7 @@ async function uploadFilePart(part, ctx, index) {
|
|
|
56
58
|
...part.metadata && { metadata: part.metadata }
|
|
57
59
|
};
|
|
58
60
|
}
|
|
59
|
-
async function
|
|
61
|
+
async function uploadPartsFiles(parts, ctx) {
|
|
60
62
|
const results = parts.map((part) => part.kind === "file" ? null : part);
|
|
61
63
|
const fileIndices = parts.flatMap((part, index) => part.kind === "file" ? [index] : []);
|
|
62
64
|
let nextFileCursor = 0;
|
|
@@ -71,10 +73,20 @@ async function uploadPartsImages(parts, ctx) {
|
|
|
71
73
|
try {
|
|
72
74
|
results[index] = await uploadFilePart(part, ctx, index);
|
|
73
75
|
} catch (error) {
|
|
76
|
+
const file = part.kind === "file" ? {
|
|
77
|
+
...part.file.mimeType ? { mimeType: part.file.mimeType } : {},
|
|
78
|
+
..."uri" in part.file && part.file.uri ? { uri: part.file.uri } : {},
|
|
79
|
+
..."bytes" in part.file && part.file.bytes ? { bytesLength: part.file.bytes.length } : {}
|
|
80
|
+
} : void 0;
|
|
74
81
|
logger.error({
|
|
75
82
|
error: error instanceof Error ? error.message : String(error),
|
|
76
|
-
index
|
|
77
|
-
|
|
83
|
+
index,
|
|
84
|
+
tenantId: ctx.tenantId,
|
|
85
|
+
projectId: ctx.projectId,
|
|
86
|
+
conversationId: ctx.conversationId,
|
|
87
|
+
messageId: ctx.messageId,
|
|
88
|
+
file
|
|
89
|
+
}, "Failed to upload file part, dropping from persisted message to avoid storing base64 in DB");
|
|
78
90
|
}
|
|
79
91
|
}
|
|
80
92
|
};
|
|
@@ -111,4 +123,4 @@ function hasFileParts(parts) {
|
|
|
111
123
|
}
|
|
112
124
|
|
|
113
125
|
//#endregion
|
|
114
|
-
export { hasFileParts, makeMessageContentParts,
|
|
126
|
+
export { hasFileParts, makeMessageContentParts, uploadPartsFiles };
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
//#region src/domains/run/services/blob-storage/file-url-security.d.ts
|
|
2
|
+
declare function validateExternalFileUrl(rawUrl: string): URL;
|
|
3
|
+
declare function validateUrlResolvesToPublicIp(url: URL): Promise<void>;
|
|
4
|
+
declare function isBlockedIpAddress(ipAddress: string): boolean;
|
|
5
|
+
//#endregion
|
|
6
|
+
export { isBlockedIpAddress, validateExternalFileUrl, validateUrlResolvesToPublicIp };
|
package/dist/domains/run/services/blob-storage/{image-url-security.js → file-url-security.js}
RENAMED
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
import { getLogger } from "../../../../logger.js";
|
|
2
|
-
import { ALLOWED_HTTP_PORTS } from "./
|
|
3
|
-
import { BlockedDisallowedPortError, BlockedEmbeddedCredentialsError, BlockedUnsupportedSchemeError, BlockedUrlResolvingToPrivateIpError,
|
|
2
|
+
import { ALLOWED_HTTP_PORTS } from "./file-security-constants.js";
|
|
3
|
+
import { BlockedDisallowedPortError, BlockedEmbeddedCredentialsError, BlockedUnsupportedSchemeError, BlockedUrlResolvingToPrivateIpError, FileSecurityError, InvalidExternalFileUrlError, NoIpResolvedError, UnableToResolveHostError } from "./file-security-errors.js";
|
|
4
4
|
import { lookup } from "node:dns/promises";
|
|
5
5
|
import { isIP } from "node:net";
|
|
6
6
|
import * as ipaddr from "ipaddr.js";
|
|
7
7
|
|
|
8
|
-
//#region src/domains/run/services/blob-storage/
|
|
9
|
-
const logger = getLogger("
|
|
10
|
-
function
|
|
8
|
+
//#region src/domains/run/services/blob-storage/file-url-security.ts
|
|
9
|
+
const logger = getLogger("file-security");
|
|
10
|
+
function validateExternalFileUrl(rawUrl) {
|
|
11
11
|
let parsed;
|
|
12
12
|
try {
|
|
13
13
|
parsed = new URL(rawUrl);
|
|
14
14
|
} catch {
|
|
15
|
-
throw new
|
|
15
|
+
throw new InvalidExternalFileUrlError(rawUrl);
|
|
16
16
|
}
|
|
17
17
|
const protocol = parsed.protocol.toLowerCase();
|
|
18
18
|
if (protocol !== "http:" && protocol !== "https:") throw new BlockedUnsupportedSchemeError(protocol);
|
|
@@ -26,7 +26,7 @@ async function validateUrlResolvesToPublicIp(url) {
|
|
|
26
26
|
try {
|
|
27
27
|
candidateIps = await resolveCandidateIps(hostname);
|
|
28
28
|
} catch (error) {
|
|
29
|
-
if (error instanceof
|
|
29
|
+
if (error instanceof FileSecurityError) throw error;
|
|
30
30
|
throw new UnableToResolveHostError(hostname, { cause: error });
|
|
31
31
|
}
|
|
32
32
|
if (candidateIps.length === 0) throw new NoIpResolvedError(hostname);
|
|
@@ -34,7 +34,7 @@ async function validateUrlResolvesToPublicIp(url) {
|
|
|
34
34
|
logger.warn({
|
|
35
35
|
host: hostname,
|
|
36
36
|
ip
|
|
37
|
-
}, "Blocked external
|
|
37
|
+
}, "Blocked external file URL resolving to private IP");
|
|
38
38
|
throw new BlockedUrlResolvingToPrivateIpError(ip);
|
|
39
39
|
}
|
|
40
40
|
}
|
|
@@ -49,7 +49,7 @@ async function resolveCandidateIps(hostname) {
|
|
|
49
49
|
logger.warn({
|
|
50
50
|
host: hostname,
|
|
51
51
|
error
|
|
52
|
-
}, "DNS resolution failed for external
|
|
52
|
+
}, "DNS resolution failed for external file URL");
|
|
53
53
|
throw new UnableToResolveHostError(hostname, { cause: error });
|
|
54
54
|
}
|
|
55
55
|
}
|
|
@@ -66,4 +66,4 @@ function isBlockedIpAddress(ipAddress) {
|
|
|
66
66
|
}
|
|
67
67
|
|
|
68
68
|
//#endregion
|
|
69
|
-
export { isBlockedIpAddress,
|
|
69
|
+
export { isBlockedIpAddress, validateExternalFileUrl, validateUrlResolvesToPublicIp };
|
|
@@ -2,7 +2,7 @@ import { getLogger } from "../../../../logger.js";
|
|
|
2
2
|
import { env } from "../../../../env.js";
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { access, lstat, mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
|
5
|
-
import { getMimeTypeFromExtension } from "@inkeep/agents-core/constants/allowed-
|
|
5
|
+
import { getMimeTypeFromExtension } from "@inkeep/agents-core/constants/allowed-file-formats";
|
|
6
6
|
|
|
7
7
|
//#region src/domains/run/services/blob-storage/local-provider.ts
|
|
8
8
|
const logger = getLogger("LocalBlobStorageProvider");
|