@inkeep/agents-core 0.75.4 → 0.77.0

package/dist/external-fetch/external-file-downloader.d.ts

@@ -0,0 +1,14 @@
+import { LookupAddress } from "node:dns";
+
+//#region src/external-fetch/external-file-downloader.d.ts
+type LookupCallback = (error: Error | null, address: string | LookupAddress[], family?: number) => void;
+declare function forwardLookupResult(hostname: string, address: string | LookupAddress[], family: number | undefined, callback: LookupCallback): void;
+declare function downloadExternalFile(url: string, options?: {
+  expectedMimeType?: string;
+  signal?: AbortSignal;
+}): Promise<{
+  data: Uint8Array;
+  mimeType: string;
+}>;
+//#endregion
+export { downloadExternalFile, forwardLookupResult };

package/dist/external-fetch/external-file-downloader.js

@@ -0,0 +1,146 @@
+import { retryWithBackoff } from "../utils/retry.js";
+import { BlockedConnectionToPrivateIpError, BlockedExternalFileExceedingError, BlockedExternalFileLargerThanError, ExternalFileResponseBodyEmptyError, FailedToDownloadError, FileSecurityError, RedirectMissingLocationError, TimedOutDownloadingError, TooManyRedirectsError, UnableToResolveHostError, UnexpectedRedirectStateError } from "./file-security-errors.js";
+import { EXTERNAL_FETCH_TIMEOUT_MS, MAX_EXTERNAL_REDIRECTS, MAX_FILE_BYTES } from "./file-security-constants.js";
+import { resolveDownloadedFileMimeType } from "./file-content-security.js";
+import { isBlockedIpAddress, validateExternalFileUrl, validateUrlResolvesToPublicIp } from "./file-url-security.js";
+import { lookup } from "node:dns";
+import { Agent } from "undici";
+
+//#region src/external-fetch/external-file-downloader.ts
+const externalImageDispatcher = new Agent({ connect: { lookup(hostname, options, callback) {
+	lookup(hostname, options, (error, address, family) => {
+		if (error) {
+			callback(new UnableToResolveHostError(hostname, { cause: error }), "", 0);
+			return;
+		}
+		forwardLookupResult(hostname, address, family, callback);
+	});
+} } });
+const MAX_EXTERNAL_FETCH_ATTEMPTS = 3;
+function forwardLookupResult(hostname, address, family, callback) {
+	if (Array.isArray(address)) {
+		if (!address[0]) {
+			callback(new UnableToResolveHostError(hostname), "", 0);
+			return;
+		}
+		const blocked = address.find((candidate) => isBlockedIpAddress(candidate.address));
+		if (blocked) {
+			callback(new BlockedConnectionToPrivateIpError(blocked.address), "", 0);
+			return;
+		}
+		callback(null, address);
+		return;
+	}
+	if (isBlockedIpAddress(address)) {
+		callback(new BlockedConnectionToPrivateIpError(address), "", 0);
+		return;
+	}
+	callback(null, address, family);
+}
+async function downloadExternalFile(url, options) {
+	let currentUrl = validateExternalFileUrl(url);
+	await validateUrlResolvesToPublicIp(currentUrl);
+	for (let redirectCount = 0; redirectCount <= MAX_EXTERNAL_REDIRECTS; redirectCount++) {
+		const response = await fetchWithRetry(currentUrl, options?.signal);
+		if (isRedirectStatus(response.status)) {
+			const location = response.headers.get("location");
+			if (!location) throw new RedirectMissingLocationError(toSanitizedUrl(currentUrl));
+			if (redirectCount === MAX_EXTERNAL_REDIRECTS) throw new TooManyRedirectsError(toSanitizedUrl(url));
+			currentUrl = validateExternalFileUrl(new URL(location, currentUrl).toString());
+			await validateUrlResolvesToPublicIp(currentUrl);
+			continue;
+		}
+		if (!response.ok) throw new FailedToDownloadError(toSanitizedUrl(currentUrl), `${response.status} ${response.statusText}`);
+		const headerContentType = (response.headers.get("content-type") || "").split(";")[0].trim().toLowerCase();
+		const contentLength = response.headers.get("content-length");
+		if (contentLength && Number.isFinite(Number(contentLength)) && Number(contentLength) > MAX_FILE_BYTES) throw new BlockedExternalFileLargerThanError(MAX_FILE_BYTES, contentLength);
+		const data = await readResponseBytesWithLimit(response, MAX_FILE_BYTES);
+		return {
+			data,
+			mimeType: await resolveDownloadedFileMimeType(data, headerContentType, options?.expectedMimeType)
+		};
+	}
+	throw new UnexpectedRedirectStateError(toSanitizedUrl(url));
+}
+async function fetchWithRetry(url, signal) {
+	return retryWithBackoff(async () => {
+		let response;
+		try {
+			response = await fetchWithConnectionIpValidation(url, signal);
+		} catch (error) {
+			if (error instanceof TimedOutDownloadingError || error instanceof FailedToDownloadError) error.status = 502;
+			throw error;
+		}
+		if (isRetryableStatus(response.status)) {
+			const err = new FailedToDownloadError(toSanitizedUrl(url), `${response.status} ${response.statusText}`);
+			err.status = response.status;
+			throw err;
+		}
+		return response;
+	}, {
+		maxAttempts: MAX_EXTERNAL_FETCH_ATTEMPTS,
+		maxDelayMs: 2e3,
+		label: `file-download ${toSanitizedUrl(url)}`
+	});
+}
+function isRedirectStatus(status) {
+	return status === 301 || status === 302 || status === 303 || status === 307 || status === 308;
+}
+function isRetryableStatus(status) {
+	return status === 429 || status >= 500 && status <= 599;
+}
+async function readResponseBytesWithLimit(response, maxBytes) {
+	if (!response.body) throw new ExternalFileResponseBodyEmptyError();
+	const reader = response.body.getReader();
+	const chunks = [];
+	let totalBytes = 0;
+	try {
+		while (true) {
+			const { done, value } = await reader.read();
+			if (done) break;
+			if (!value) continue;
+			totalBytes += value.byteLength;
+			if (totalBytes > maxBytes) throw new BlockedExternalFileExceedingError(maxBytes);
+			chunks.push(value);
+		}
+	} finally {
+		reader.releaseLock();
+	}
+	const merged = new Uint8Array(totalBytes);
+	let offset = 0;
+	for (const chunk of chunks) {
+		merged.set(chunk, offset);
+		offset += chunk.byteLength;
+	}
+	return merged;
+}
+async function fetchWithConnectionIpValidation(url, callerSignal) {
+	const timeoutSignal = AbortSignal.timeout(EXTERNAL_FETCH_TIMEOUT_MS);
+	const signal = callerSignal ? AbortSignal.any([timeoutSignal, callerSignal]) : timeoutSignal;
+	try {
+		return await fetch(url.toString(), {
+			redirect: "manual",
+			signal,
+			dispatcher: externalImageDispatcher
+		});
+	} catch (error) {
+		const fileSecurityError = extractFileSecurityError(error);
+		if (fileSecurityError instanceof BlockedConnectionToPrivateIpError || fileSecurityError instanceof UnableToResolveHostError) throw fileSecurityError;
+		if (error instanceof Error && (error.name === "AbortError" || error.name === "TimeoutError")) throw new TimedOutDownloadingError(toSanitizedUrl(url));
+		throw new FailedToDownloadError(toSanitizedUrl(url), void 0, { cause: error });
+	}
+}
+function extractFileSecurityError(error) {
+	if (error instanceof FileSecurityError) return error;
+	if (error instanceof Error && error.cause) return extractFileSecurityError(error.cause);
+	return null;
+}
+function toSanitizedUrl(url) {
+	const parsed = typeof url === "string" ? new URL(url) : new URL(url.toString());
+	parsed.search = "";
+	parsed.hash = "";
+	return parsed.toString();
+}
+
+//#endregion
+export { downloadExternalFile, forwardLookupResult };

package/dist/external-fetch/file-content-security.d.ts

@@ -0,0 +1,18 @@
+//#region src/external-fetch/file-content-security.d.ts
+declare function normalizeInlineImageBytes(file: {
+  bytes: string;
+  mimeType?: string;
+}): Promise<{
+  data: Uint8Array;
+  mimeType: string;
+}>;
+declare function normalizeInlineFileBytes(file: {
+  bytes: string;
+  mimeType?: string;
+}): Promise<{
+  data: Uint8Array;
+  mimeType: string;
+}>;
+declare function resolveDownloadedFileMimeType(data: Uint8Array, headerContentType: string, expectedMimeType?: string): Promise<string>;
+//#endregion
+export { normalizeInlineFileBytes, normalizeInlineImageBytes, resolveDownloadedFileMimeType };

package/dist/external-fetch/file-content-security.js

@@ -0,0 +1,81 @@
+import { isOfficeDocumentMimeType } from "../constants/allowed-file-formats.js";
+import { BlockedExternalUnsupportedBytesError, BlockedInlineFileExceedingError, BlockedInlineUnsupportedFileBytesError, InvalidInlineFileMalformedBase64Error } from "./file-security-errors.js";
+import { decodeTextDocumentBytes, isTextDocumentMimeType } from "../text-attachments/text-document-attachments.js";
+import { ALLOWED_EXTERNAL_IMAGE_MIME_TYPES, MAX_FILE_BYTES, TEXT_DOCUMENT_MAX_BYTES } from "./file-security-constants.js";
+import { fileTypeFromBuffer } from "file-type";
+
+//#region src/external-fetch/file-content-security.ts
+async function normalizeInlineImageBytes(file) {
+	const normalized = await normalizeInlineFileBytes(file);
+	if (normalized.mimeType.startsWith("image/")) return normalized;
+	throw new BlockedInlineUnsupportedFileBytesError(file.mimeType || "unknown");
+}
+async function normalizeInlineFileBytes(file) {
+	const data = decodeBase64Bytes(file.bytes);
+	const requestedMimeType = file.mimeType?.split(";")[0]?.trim().toLowerCase();
+	validateInlineFileSize(data, requestedMimeType);
+	const sniffedMime = await sniffAllowedInlineFileMimeType(data, requestedMimeType);
+	if (sniffedMime) return {
+		data,
+		mimeType: sniffedMime
+	};
+	throw new BlockedInlineUnsupportedFileBytesError(file.mimeType || "unknown");
+}
+async function resolveDownloadedFileMimeType(data, headerContentType, expectedMimeType) {
+	const expected = expectedMimeType?.split(";")[0]?.trim().toLowerCase();
+	if (expected === "application/pdf") {
+		if (looksLikePdf(data)) return "application/pdf";
+		throw new BlockedExternalUnsupportedBytesError(headerContentType || expected || "unknown");
+	}
+	if (isOfficeDocumentMimeType(expected)) {
+		if (looksLikeZip(data)) return expected;
+		throw new BlockedExternalUnsupportedBytesError(headerContentType || expected || "unknown");
+	}
+	const sniffedMime = await sniffAllowedImageMimeType(data);
+	if (sniffedMime) return sniffedMime;
+	throw new BlockedExternalUnsupportedBytesError(headerContentType || expected || "unknown");
+}
+function validateInlineFileSize(data, requestedMimeType) {
+	const maxBytes = isTextDocumentMimeType(requestedMimeType) ? TEXT_DOCUMENT_MAX_BYTES : MAX_FILE_BYTES;
+	if (data.length > maxBytes) throw new BlockedInlineFileExceedingError(maxBytes);
+}
+async function sniffAllowedImageMimeType(data) {
+	const sniffedMime = (await fileTypeFromBuffer(data))?.mime?.toLowerCase();
+	if (sniffedMime && ALLOWED_EXTERNAL_IMAGE_MIME_TYPES.has(sniffedMime)) return sniffedMime;
+	return null;
+}
+async function sniffAllowedInlineFileMimeType(data, requestedMimeType) {
+	if (requestedMimeType === "application/pdf") {
+		if (!looksLikePdf(data)) throw new BlockedInlineUnsupportedFileBytesError(requestedMimeType);
+		return "application/pdf";
+	}
+	if (isOfficeDocumentMimeType(requestedMimeType)) {
+		if (!looksLikeZip(data)) throw new BlockedInlineUnsupportedFileBytesError(requestedMimeType ?? "unknown");
+		return requestedMimeType;
+	}
+	if (isTextDocumentMimeType(requestedMimeType)) try {
+		decodeTextDocumentBytes(data);
+		return requestedMimeType;
+	} catch {
+		throw new BlockedInlineUnsupportedFileBytesError(requestedMimeType);
+	}
+	return await sniffAllowedImageMimeType(data);
+}
+function decodeBase64Bytes(base64Bytes) {
+	const normalized = base64Bytes.replace(/\s+/g, "");
+	if (normalized.length === 0 || normalized.length % 4 !== 0 || !/^[A-Za-z0-9+/]*={0,2}$/.test(normalized)) throw new InvalidInlineFileMalformedBase64Error();
+	const decoded = Buffer.from(normalized, "base64");
+	if (decoded.length === 0 || decoded.toString("base64") !== normalized) throw new InvalidInlineFileMalformedBase64Error();
+	return Uint8Array.from(decoded);
+}
+function looksLikePdf(data) {
+	if (data.length < 5) return false;
+	return data[0] === 37 && data[1] === 80 && data[2] === 68 && data[3] === 70 && data[4] === 45;
+}
+function looksLikeZip(data) {
+	if (data.length < 4) return false;
+	return data[0] === 80 && data[1] === 75 && data[2] === 3 && data[3] === 4;
+}
+
+//#endregion
+export { normalizeInlineFileBytes, normalizeInlineImageBytes, resolveDownloadedFileMimeType };

package/dist/external-fetch/file-security-constants.d.ts

@@ -0,0 +1,12 @@
+import { AllowedImageMimeType } from "../constants/allowed-file-formats.js";
+
+//#region src/external-fetch/file-security-constants.d.ts
+type AllowedExternalImageMimeType = AllowedImageMimeType;
+declare const MAX_FILE_BYTES: number;
+declare const TEXT_DOCUMENT_MAX_BYTES: number;
+declare const EXTERNAL_FETCH_TIMEOUT_MS = 10000;
+declare const MAX_EXTERNAL_REDIRECTS = 3;
+declare const ALLOWED_EXTERNAL_IMAGE_MIME_TYPES: Set<string>;
+declare const ALLOWED_HTTP_PORTS: Set<string>;
+//#endregion
+export { ALLOWED_EXTERNAL_IMAGE_MIME_TYPES, ALLOWED_HTTP_PORTS, AllowedExternalImageMimeType, EXTERNAL_FETCH_TIMEOUT_MS, MAX_EXTERNAL_REDIRECTS, MAX_FILE_BYTES, TEXT_DOCUMENT_MAX_BYTES };

package/dist/external-fetch/file-security-constants.js

@@ -0,0 +1,16 @@
+import { ALLOWED_IMAGE_MIME_TYPES } from "../constants/allowed-file-formats.js";
+
+//#region src/external-fetch/file-security-constants.ts
+const MAX_FILE_BYTES = 10 * 1024 * 1024;
+const TEXT_DOCUMENT_MAX_BYTES = 256 * 1024;
+const EXTERNAL_FETCH_TIMEOUT_MS = 1e4;
+const MAX_EXTERNAL_REDIRECTS = 3;
+const ALLOWED_EXTERNAL_IMAGE_MIME_TYPES = ALLOWED_IMAGE_MIME_TYPES;
+const ALLOWED_HTTP_PORTS = new Set([
+	"",
+	"80",
+	"443"
+]);
+
+//#endregion
+export { ALLOWED_EXTERNAL_IMAGE_MIME_TYPES, ALLOWED_HTTP_PORTS, EXTERNAL_FETCH_TIMEOUT_MS, MAX_EXTERNAL_REDIRECTS, MAX_FILE_BYTES, TEXT_DOCUMENT_MAX_BYTES };

package/dist/external-fetch/file-security-errors.d.ts

@@ -0,0 +1,96 @@
+//#region src/external-fetch/file-security-errors.d.ts
+declare class FileSecurityError extends Error {
+  constructor(message: string, options?: ErrorOptions);
+}
+declare class InvalidExternalFileUrlError extends FileSecurityError {
+  constructor(rawUrl: string);
+}
+declare class BlockedUnsupportedSchemeError extends FileSecurityError {
+  constructor(protocol: string);
+}
+declare class BlockedDisallowedPortError extends FileSecurityError {
+  constructor(port: string);
+}
+declare class BlockedEmbeddedCredentialsError extends FileSecurityError {
+  constructor();
+}
+declare class NoIpResolvedError extends FileSecurityError {
+  constructor(hostname: string);
+}
+declare class BlockedUrlResolvingToPrivateIpError extends FileSecurityError {
+  constructor(ip: string);
+}
+declare class UnableToResolveHostError extends FileSecurityError {
+  constructor(hostname: string, options?: ErrorOptions);
+}
+declare class BlockedConnectionToPrivateIpError extends FileSecurityError {
+  constructor(address: string);
+}
+declare class RedirectMissingLocationError extends FileSecurityError {
+  constructor(url: string);
+}
+declare class TooManyRedirectsError extends FileSecurityError {
+  constructor(url: string);
+}
+declare class FailedToDownloadError extends FileSecurityError {
+  constructor(url: string, statusText?: string, options?: ErrorOptions);
+}
+declare class BlockedExternalFileLargerThanError extends FileSecurityError {
+  constructor(maxBytes: number, contentLength: string);
+}
+declare class BlockedExternalFileExceedingError extends FileSecurityError {
+  constructor(maxBytes: number);
+}
+declare class ExternalFileResponseBodyEmptyError extends FileSecurityError {
+  constructor();
+}
+declare class UnexpectedRedirectStateError extends FileSecurityError {
+  constructor(url: string);
+}
+declare class TimedOutDownloadingError extends FileSecurityError {
+  constructor(url: string);
+}
+declare class BlockedInlineFileExceedingError extends FileSecurityError {
+  constructor(maxBytes: number);
+}
+declare class BlockedInlineUnsupportedFileBytesError extends FileSecurityError {
+  constructor(claimedContentType: string);
+}
+declare class BlockedExternalUnsupportedBytesError extends FileSecurityError {
+  constructor(contentType: string);
+}
+declare class InvalidInlineFileMalformedBase64Error extends FileSecurityError {
+  constructor();
+}
+declare class TextDocumentAttachmentError extends FileSecurityError {
+  constructor(message: string, options?: ErrorOptions);
+}
+declare class InvalidUtf8TextDocumentError extends TextDocumentAttachmentError {
+  constructor(options?: ErrorOptions);
+}
+declare class TextDocumentControlCharacterError extends TextDocumentAttachmentError {
+  constructor(options?: ErrorOptions);
+}
+declare class UnsupportedTextAttachmentSourceError extends TextDocumentAttachmentError {
+  constructor(mimeType: string, options?: ErrorOptions);
+}
+declare class PdfUrlIngestionError extends FileSecurityError {
+  readonly sourceUrl: string;
+  constructor(sourceUrl: string, options?: ErrorOptions);
+}
+/**
+ * Transient / content-format errors that occur on the *external-fetch* path:
+ * the server told us "no" (404/5xx/timeout/DNS), we couldn't ingest the
+ * response (empty body), or the auto-fetched file's MIME type isn't one the
+ * platform handles (gif, svg, bmp, etc.). Callers may drop the affected file
+ * and continue rather than failing the whole request — the user didn't
+ * directly upload it, the producer auto-extracted a URL on their behalf.
+ *
+ * Inline (user-uploaded) byte rejections like `BlockedInlineUnsupportedFileBytesError`
+ * remain FATAL: when a user explicitly pastes an unsupported file we should
+ * surface that to them, not silently drop. Real security guards (private-IP/SSRF,
+ * embedded credentials, disallowed schemes and ports) likewise remain fatal.
+ */
+declare function isTransientDownloadError(error: unknown): boolean;
+//#endregion
+export { BlockedConnectionToPrivateIpError, BlockedDisallowedPortError, BlockedEmbeddedCredentialsError, BlockedExternalFileExceedingError, BlockedExternalFileLargerThanError, BlockedExternalUnsupportedBytesError, BlockedInlineFileExceedingError, BlockedInlineUnsupportedFileBytesError, BlockedUnsupportedSchemeError, BlockedUrlResolvingToPrivateIpError, ExternalFileResponseBodyEmptyError, FailedToDownloadError, FileSecurityError, InvalidExternalFileUrlError, InvalidInlineFileMalformedBase64Error, InvalidUtf8TextDocumentError, NoIpResolvedError, PdfUrlIngestionError, RedirectMissingLocationError, TextDocumentAttachmentError, TextDocumentControlCharacterError, TimedOutDownloadingError, TooManyRedirectsError, UnableToResolveHostError, UnexpectedRedirectStateError, UnsupportedTextAttachmentSourceError, isTransientDownloadError };

package/dist/external-fetch/file-security-errors.js

@@ -0,0 +1,154 @@
+//#region src/external-fetch/file-security-errors.ts
+var FileSecurityError = class extends Error {
+	constructor(message, options) {
+		super(message, options);
+		this.name = new.target.name;
+	}
+};
+var InvalidExternalFileUrlError = class extends FileSecurityError {
+	constructor(rawUrl) {
+		super(`Invalid external file URL: ${rawUrl}`);
+	}
+};
+var BlockedUnsupportedSchemeError = class extends FileSecurityError {
+	constructor(protocol) {
+		super(`Blocked external file URL with unsupported scheme: ${protocol}`);
+	}
+};
+var BlockedDisallowedPortError = class extends FileSecurityError {
+	constructor(port) {
+		super(`Blocked external file URL with disallowed port: ${port}`);
+	}
+};
+var BlockedEmbeddedCredentialsError = class extends FileSecurityError {
+	constructor() {
+		super("Blocked external file URL with embedded credentials");
+	}
+};
+var NoIpResolvedError = class extends FileSecurityError {
+	constructor(hostname) {
+		super(`No IP addresses resolved for host: ${hostname}`);
+	}
+};
+var BlockedUrlResolvingToPrivateIpError = class extends FileSecurityError {
+	constructor(ip) {
+		super(`Blocked external file URL resolving to private or reserved IP: ${ip}`);
+	}
+};
+var UnableToResolveHostError = class extends FileSecurityError {
+	constructor(hostname, options) {
+		super(`Unable to resolve external file host: ${hostname}`, options);
+	}
+};
+var BlockedConnectionToPrivateIpError = class extends FileSecurityError {
+	constructor(address) {
+		super(`Blocked external file connection to private or reserved IP: ${address}`);
+	}
+};
+var RedirectMissingLocationError = class extends FileSecurityError {
+	constructor(url) {
+		super(`Redirect response missing location header: ${url}`);
+	}
+};
+var TooManyRedirectsError = class extends FileSecurityError {
+	constructor(url) {
+		super(`Too many redirects while downloading file: ${url}`);
+	}
+};
+var FailedToDownloadError = class extends FileSecurityError {
+	constructor(url, statusText, options) {
+		super(statusText ? `Failed to download file from ${url}: ${statusText}` : `Failed to download file from ${url}`, options);
+	}
+};
+var BlockedExternalFileLargerThanError = class extends FileSecurityError {
+	constructor(maxBytes, contentLength) {
+		super(`Blocked external file larger than ${maxBytes} bytes: ${contentLength}`);
+	}
+};
+var BlockedExternalFileExceedingError = class extends FileSecurityError {
+	constructor(maxBytes) {
+		super(`Blocked external file exceeding ${maxBytes} bytes`);
+	}
+};
+var ExternalFileResponseBodyEmptyError = class extends FileSecurityError {
+	constructor() {
+		super("External file response body is empty");
+	}
+};
+var UnexpectedRedirectStateError = class extends FileSecurityError {
+	constructor(url) {
+		super(`Unexpected redirect handling state for URL: ${url}`);
+	}
+};
+var TimedOutDownloadingError = class extends FileSecurityError {
+	constructor(url) {
+		super(`Timed out downloading file from ${url}`);
+	}
+};
+var BlockedInlineFileExceedingError = class extends FileSecurityError {
+	constructor(maxBytes) {
+		super(`Blocked inline file exceeding ${maxBytes} bytes`);
+	}
+};
+var BlockedInlineUnsupportedFileBytesError = class extends FileSecurityError {
+	constructor(claimedContentType) {
+		super(`Blocked inline file with unsupported bytes signature (claimed content-type: ${claimedContentType})`);
+	}
+};
+var BlockedExternalUnsupportedBytesError = class extends FileSecurityError {
+	constructor(contentType) {
+		super(`Blocked external file with unsupported bytes signature (content-type: ${contentType})`);
+	}
+};
+var InvalidInlineFileMalformedBase64Error = class extends FileSecurityError {
+	constructor() {
+		super("Invalid inline file: malformed base64 payload");
+	}
+};
+var TextDocumentAttachmentError = class extends FileSecurityError {
+	constructor(message, options) {
+		super(message, options);
+		this.name = new.target.name;
+	}
+};
+var InvalidUtf8TextDocumentError = class extends TextDocumentAttachmentError {
+	constructor(options) {
+		super("Invalid UTF-8 text document", options);
+	}
+};
+var TextDocumentControlCharacterError = class extends TextDocumentAttachmentError {
+	constructor(options) {
+		super("Text document contains disallowed control characters", options);
+	}
+};
+var UnsupportedTextAttachmentSourceError = class extends TextDocumentAttachmentError {
+	constructor(mimeType, options) {
+		super(`Unsupported text attachment source for mime type: ${mimeType}`, options);
+	}
+};
+var PdfUrlIngestionError = class extends FileSecurityError {
+	sourceUrl;
+	constructor(sourceUrl, options) {
+		super(`Failed to ingest PDF URL: ${sourceUrl}`, options);
+		this.sourceUrl = sourceUrl;
+	}
+};
+/**
+* Transient / content-format errors that occur on the *external-fetch* path:
+* the server told us "no" (404/5xx/timeout/DNS), we couldn't ingest the
+* response (empty body), or the auto-fetched file's MIME type isn't one the
+* platform handles (gif, svg, bmp, etc.). Callers may drop the affected file
+* and continue rather than failing the whole request — the user didn't
+* directly upload it, the producer auto-extracted a URL on their behalf.
+*
+* Inline (user-uploaded) byte rejections like `BlockedInlineUnsupportedFileBytesError`
+* remain FATAL: when a user explicitly pastes an unsupported file we should
+* surface that to them, not silently drop. Real security guards (private-IP/SSRF,
+* embedded credentials, disallowed schemes and ports) likewise remain fatal.
+*/
+function isTransientDownloadError(error) {
+	return error instanceof FailedToDownloadError || error instanceof TimedOutDownloadingError || error instanceof UnableToResolveHostError || error instanceof NoIpResolvedError || error instanceof ExternalFileResponseBodyEmptyError || error instanceof BlockedExternalUnsupportedBytesError;
+}
+
+//#endregion
+export { BlockedConnectionToPrivateIpError, BlockedDisallowedPortError, BlockedEmbeddedCredentialsError, BlockedExternalFileExceedingError, BlockedExternalFileLargerThanError, BlockedExternalUnsupportedBytesError, BlockedInlineFileExceedingError, BlockedInlineUnsupportedFileBytesError, BlockedUnsupportedSchemeError, BlockedUrlResolvingToPrivateIpError, ExternalFileResponseBodyEmptyError, FailedToDownloadError, FileSecurityError, InvalidExternalFileUrlError, InvalidInlineFileMalformedBase64Error, InvalidUtf8TextDocumentError, NoIpResolvedError, PdfUrlIngestionError, RedirectMissingLocationError, TextDocumentAttachmentError, TextDocumentControlCharacterError, TimedOutDownloadingError, TooManyRedirectsError, UnableToResolveHostError, UnexpectedRedirectStateError, UnsupportedTextAttachmentSourceError, isTransientDownloadError };

package/dist/external-fetch/file-url-security.d.ts

@@ -0,0 +1,7 @@
+//#region src/external-fetch/file-url-security.d.ts
+declare function makeSanitizedSourceUrl(rawUrl: string): string;
+declare function validateExternalFileUrl(rawUrl: string): URL;
+declare function validateUrlResolvesToPublicIp(url: URL): Promise<void>;
+declare function isBlockedIpAddress(ipAddress: string): boolean;
+//#endregion
+export { isBlockedIpAddress, makeSanitizedSourceUrl, validateExternalFileUrl, validateUrlResolvesToPublicIp };

package/dist/external-fetch/file-url-security.js

@@ -0,0 +1,75 @@
+import { getLogger } from "../utils/logger.js";
+import { BlockedDisallowedPortError, BlockedEmbeddedCredentialsError, BlockedUnsupportedSchemeError, BlockedUrlResolvingToPrivateIpError, FileSecurityError, InvalidExternalFileUrlError, NoIpResolvedError, UnableToResolveHostError } from "./file-security-errors.js";
+import { ALLOWED_HTTP_PORTS } from "./file-security-constants.js";
+import { lookup } from "node:dns/promises";
+import { isIP } from "node:net";
+import ipaddr from "ipaddr.js";
+
+//#region src/external-fetch/file-url-security.ts
+const logger = getLogger("file-security");
+function makeSanitizedSourceUrl(rawUrl) {
+	const parsed = new URL(rawUrl);
+	parsed.search = "";
+	parsed.hash = "";
+	return parsed.toString();
+}
+function validateExternalFileUrl(rawUrl) {
+	let parsed;
+	try {
+		parsed = new URL(rawUrl);
+	} catch {
+		throw new InvalidExternalFileUrlError(rawUrl);
+	}
+	const protocol = parsed.protocol.toLowerCase();
+	if (protocol !== "http:" && protocol !== "https:") throw new BlockedUnsupportedSchemeError(protocol);
+	if (!ALLOWED_HTTP_PORTS.has(parsed.port)) throw new BlockedDisallowedPortError(parsed.port);
+	if (parsed.username || parsed.password) throw new BlockedEmbeddedCredentialsError();
+	return parsed;
+}
+async function validateUrlResolvesToPublicIp(url) {
+	const hostname = url.hostname;
+	let candidateIps;
+	try {
+		candidateIps = await resolveCandidateIps(hostname);
+	} catch (error) {
+		if (error instanceof FileSecurityError) throw error;
+		throw new UnableToResolveHostError(hostname, { cause: error });
+	}
+	if (candidateIps.length === 0) throw new NoIpResolvedError(hostname);
+	for (const ip of candidateIps) if (isBlockedIpAddress(ip)) {
+		logger.warn({
+			host: hostname,
+			ip
+		}, "Blocked external file URL resolving to private IP");
+		throw new BlockedUrlResolvingToPrivateIpError(ip);
+	}
+}
+async function resolveCandidateIps(hostname) {
+	if (isIP(hostname) !== 0) return [hostname];
+	try {
+		return (await lookup(hostname, {
+			all: true,
+			verbatim: true
+		})).map((result) => result.address);
+	} catch (error) {
+		logger.warn({
+			host: hostname,
+			error
+		}, "DNS resolution failed for external file URL");
+		throw new UnableToResolveHostError(hostname, { cause: error });
+	}
+}
+function isBlockedIpAddress(ipAddress) {
+	if (ipaddr.IPv4.isValid(ipAddress)) {
+		const range = ipaddr.IPv4.parse(ipAddress).range();
+		return range === "private" || range === "loopback" || range === "linkLocal" || range === "multicast" || range === "carrierGradeNat" || range === "reserved" || range === "unspecified" || range === "broadcast";
+	}
+	if (ipaddr.IPv6.isValid(ipAddress)) {
+		const range = ipaddr.IPv6.parse(ipAddress).range();
+		return range === "uniqueLocal" || range === "loopback" || range === "linkLocal" || range === "multicast" || range === "ipv4Mapped" || range === "rfc6145" || range === "rfc6052" || range === "6to4" || range === "teredo" || range === "reserved" || range === "unspecified";
+	}
+	return true;
+}
+
+//#endregion
+export { isBlockedIpAddress, makeSanitizedSourceUrl, validateExternalFileUrl, validateUrlResolvesToPublicIp };

package/dist/external-fetch/index.d.ts

@@ -0,0 +1,6 @@
+import { downloadExternalFile } from "./external-file-downloader.js";
+import { normalizeInlineFileBytes, normalizeInlineImageBytes, resolveDownloadedFileMimeType } from "./file-content-security.js";
+import { ALLOWED_EXTERNAL_IMAGE_MIME_TYPES, ALLOWED_HTTP_PORTS, AllowedExternalImageMimeType, EXTERNAL_FETCH_TIMEOUT_MS, MAX_EXTERNAL_REDIRECTS, MAX_FILE_BYTES, TEXT_DOCUMENT_MAX_BYTES } from "./file-security-constants.js";
+import { BlockedConnectionToPrivateIpError, BlockedDisallowedPortError, BlockedEmbeddedCredentialsError, BlockedExternalFileExceedingError, BlockedExternalFileLargerThanError, BlockedExternalUnsupportedBytesError, BlockedInlineFileExceedingError, BlockedInlineUnsupportedFileBytesError, BlockedUnsupportedSchemeError, BlockedUrlResolvingToPrivateIpError, ExternalFileResponseBodyEmptyError, FailedToDownloadError, FileSecurityError, InvalidExternalFileUrlError, InvalidInlineFileMalformedBase64Error, InvalidUtf8TextDocumentError, NoIpResolvedError, PdfUrlIngestionError, RedirectMissingLocationError, TextDocumentAttachmentError, TextDocumentControlCharacterError, TimedOutDownloadingError, TooManyRedirectsError, UnableToResolveHostError, UnexpectedRedirectStateError, UnsupportedTextAttachmentSourceError, isTransientDownloadError } from "./file-security-errors.js";
+import { isBlockedIpAddress, makeSanitizedSourceUrl, validateExternalFileUrl, validateUrlResolvesToPublicIp } from "./file-url-security.js";
+export { ALLOWED_EXTERNAL_IMAGE_MIME_TYPES, ALLOWED_HTTP_PORTS, type AllowedExternalImageMimeType, BlockedConnectionToPrivateIpError, BlockedDisallowedPortError, BlockedEmbeddedCredentialsError, BlockedExternalFileExceedingError, BlockedExternalFileLargerThanError, BlockedExternalUnsupportedBytesError, BlockedInlineFileExceedingError, BlockedInlineUnsupportedFileBytesError, BlockedUnsupportedSchemeError, BlockedUrlResolvingToPrivateIpError, EXTERNAL_FETCH_TIMEOUT_MS, ExternalFileResponseBodyEmptyError, FailedToDownloadError, FileSecurityError, InvalidExternalFileUrlError, InvalidInlineFileMalformedBase64Error, InvalidUtf8TextDocumentError, MAX_EXTERNAL_REDIRECTS, MAX_FILE_BYTES, NoIpResolvedError, PdfUrlIngestionError, RedirectMissingLocationError, TEXT_DOCUMENT_MAX_BYTES, TextDocumentAttachmentError, TextDocumentControlCharacterError, TimedOutDownloadingError, TooManyRedirectsError, UnableToResolveHostError, UnexpectedRedirectStateError, UnsupportedTextAttachmentSourceError, downloadExternalFile, isBlockedIpAddress, isTransientDownloadError, makeSanitizedSourceUrl, normalizeInlineFileBytes, normalizeInlineImageBytes, resolveDownloadedFileMimeType, validateExternalFileUrl, validateUrlResolvesToPublicIp };

package/dist/external-fetch/index.js

@@ -0,0 +1,7 @@
+import { BlockedConnectionToPrivateIpError, BlockedDisallowedPortError, BlockedEmbeddedCredentialsError, BlockedExternalFileExceedingError, BlockedExternalFileLargerThanError, BlockedExternalUnsupportedBytesError, BlockedInlineFileExceedingError, BlockedInlineUnsupportedFileBytesError, BlockedUnsupportedSchemeError, BlockedUrlResolvingToPrivateIpError, ExternalFileResponseBodyEmptyError, FailedToDownloadError, FileSecurityError, InvalidExternalFileUrlError, InvalidInlineFileMalformedBase64Error, InvalidUtf8TextDocumentError, NoIpResolvedError, PdfUrlIngestionError, RedirectMissingLocationError, TextDocumentAttachmentError, TextDocumentControlCharacterError, TimedOutDownloadingError, TooManyRedirectsError, UnableToResolveHostError, UnexpectedRedirectStateError, UnsupportedTextAttachmentSourceError, isTransientDownloadError } from "./file-security-errors.js";
+import { ALLOWED_EXTERNAL_IMAGE_MIME_TYPES, ALLOWED_HTTP_PORTS, EXTERNAL_FETCH_TIMEOUT_MS, MAX_EXTERNAL_REDIRECTS, MAX_FILE_BYTES, TEXT_DOCUMENT_MAX_BYTES } from "./file-security-constants.js";
+import { normalizeInlineFileBytes, normalizeInlineImageBytes, resolveDownloadedFileMimeType } from "./file-content-security.js";
+import { isBlockedIpAddress, makeSanitizedSourceUrl, validateExternalFileUrl, validateUrlResolvesToPublicIp } from "./file-url-security.js";
+import { downloadExternalFile } from "./external-file-downloader.js";
+
+export { ALLOWED_EXTERNAL_IMAGE_MIME_TYPES, ALLOWED_HTTP_PORTS, BlockedConnectionToPrivateIpError, BlockedDisallowedPortError, BlockedEmbeddedCredentialsError, BlockedExternalFileExceedingError, BlockedExternalFileLargerThanError, BlockedExternalUnsupportedBytesError, BlockedInlineFileExceedingError, BlockedInlineUnsupportedFileBytesError, BlockedUnsupportedSchemeError, BlockedUrlResolvingToPrivateIpError, EXTERNAL_FETCH_TIMEOUT_MS, ExternalFileResponseBodyEmptyError, FailedToDownloadError, FileSecurityError, InvalidExternalFileUrlError, InvalidInlineFileMalformedBase64Error, InvalidUtf8TextDocumentError, MAX_EXTERNAL_REDIRECTS, MAX_FILE_BYTES, NoIpResolvedError, PdfUrlIngestionError, RedirectMissingLocationError, TEXT_DOCUMENT_MAX_BYTES, TextDocumentAttachmentError, TextDocumentControlCharacterError, TimedOutDownloadingError, TooManyRedirectsError, UnableToResolveHostError, UnexpectedRedirectStateError, UnsupportedTextAttachmentSourceError, downloadExternalFile, isBlockedIpAddress, isTransientDownloadError, makeSanitizedSourceUrl, normalizeInlineFileBytes, normalizeInlineImageBytes, resolveDownloadedFileMimeType, validateExternalFileUrl, validateUrlResolvesToPublicIp };