@x12i/content-fetcher 0.1.0

package/dist/create-content-fetcher.js

@@ -0,0 +1,836 @@
+import { ArtifactStoreError, ConfigurationError, InvalidFetchRequestError, SourcePolicyViolationError, UnknownSourcePolicyError, UnsafeUrlError, UnsupportedChannelError, } from "./errors.js";
+import { validateConfiguration } from "./config.js";
+import { contentTypeMatches, extractFeedEntryId, FALLTHROUGH_STATUSES, isTokenHeaderName, parseArchiveReference, sanitizedMessageFor, selectChannelPlan, } from "./channels.js";
+import { classifyHttpResponse, shouldSniffSuccessBody } from "./classify.js";
+import { AddressValidator, SystemDnsResolver } from "./security/dns.js";
+import { fileNameFromUrl, hostMatchesAllowedDomains, parseOutboundUrl, sanitizeFileName, sanitizeUrl, } from "./security/url.js";
+import { getHeader, parseRetryAfter, retainedResponseHeaders } from "./security/headers.js";
+import { readBody } from "./util/bytes.js";
+import { sha256ContentHash } from "./util/hash.js";
+import { newAcquisitionId } from "./util/ids.js";
+import { RateLimiterRegistry } from "./ratelimit.js";
+import { RobotsService } from "./robots/robots-service.js";
+import { resourceKindOf, validateFetchRequest } from "./validate-request.js";
+import { ArtifactPersistence } from "./storage.js";
+const TYPED_ERRORS = [
+    InvalidFetchRequestError,
+    UnknownSourcePolicyError,
+    SourcePolicyViolationError,
+    UnsafeUrlError,
+    UnsupportedChannelError,
+    ArtifactStoreError,
+    ConfigurationError,
+];
+function isTypedError(error) {
+    return TYPED_ERRORS.some((type) => error instanceof type);
+}
+function isAbortLike(error) {
+    if (!(error instanceof Error))
+        return false;
+    return error.name === "AbortError" || /abort|timed?\s*out/i.test(error.message);
+}
+class FetcherImpl {
+    policies;
+    httpClient;
+    browserAdapter;
+    authProvider;
+    robots;
+    limiter;
+    addresses;
+    persistence;
+    userAgent;
+    requestTimeoutMs;
+    nowFn;
+    constructor(options) {
+        validateConfiguration(options);
+        this.policies = new Map(options.sourcePolicies.map((p) => [p.sourceId, p]));
+        this.httpClient = options.httpClient;
+        this.browserAdapter = options.browserAdapter;
+        this.authProvider = options.authProvider;
+        const clockMs = () => this.now().getTime();
+        this.addresses = new AddressValidator(options.dnsResolver ?? new SystemDnsResolver(), clockMs);
+        this.userAgent = options.userAgent ?? "x12i-content-fetcher/1.0";
+        this.requestTimeoutMs = options.requestTimeoutMs ?? 30_000;
+        this.robots = new RobotsService(this.httpClient, this.addresses, this.userAgent, options.robotsCacheTtlMs ?? 10 * 60_000, this.requestTimeoutMs, clockMs);
+        this.limiter = new RateLimiterRegistry(clockMs);
+        this.persistence = new ArtifactPersistence(options.artifactStore);
+        this.nowFn = options.now ?? (() => new Date());
+    }
+    now() {
+        return this.nowFn();
+    }
+    async fetch(rawRequest) {
+        const request = validateFetchRequest(rawRequest);
+        const policy = this.policies.get(request.sourceId);
+        if (policy === undefined) {
+            throw new UnknownSourcePolicyError(`No source policy is registered for sourceId '${request.sourceId}'.`, request.sourceId);
+        }
+        const acquisitionId = newAcquisitionId();
+        const requestedAt = this.now();
+        const kind = resourceKindOf(request.resource);
+        const plan = selectChannelPlan(kind, policy);
+        const targetUrl = readResourceUrl(request, kind);
+        const attempts = [];
+        if (plan.length === 0) {
+            return this.finalize(request, policy, undefined, acquisitionId, requestedAt, {
+                status: "source_not_authorised",
+                reasonCode: "channel_not_permitted",
+                ...(targetUrl !== undefined ? { requestedUrl: sanitizeUrl(targetUrl) } : {}),
+            }, attempts);
+        }
+        let lastOutcome;
+        for (const channel of plan) {
+            const outcome = await this.attempt(request, policy, kind, channel, targetUrl);
+            attempts.push({ channel, status: outcome.status, reasonCode: outcome.reasonCode });
+            const hasFurtherChannels = plan.indexOf(channel) < plan.length - 1;
+            if (FALLTHROUGH_STATUSES.includes(outcome.status) && hasFurtherChannels) {
+                lastOutcome = outcome;
+                continue;
+            }
+            return this.finalize(request, policy, channel, acquisitionId, requestedAt, outcome, attempts);
+        }
+        // Every attempt was a fallthrough status; report the last one.
+        return this.finalize(request, policy, plan[plan.length - 1], acquisitionId, requestedAt, lastOutcome, attempts);
+    }
+    attempt(request, policy, kind, channel, targetUrl) {
+        if (channel === "manual_file") {
+            return this.attemptManualFile(request, policy);
+        }
+        if (targetUrl === undefined) {
+            return Promise.resolve({
+                status: "source_not_authorised",
+                reasonCode: "channel_not_permitted",
+            });
+        }
+        if (channel === "permitted_browser") {
+            return this.attemptBrowser(request, policy, targetUrl);
+        }
+        return this.attemptHttp(request, policy, channel, targetUrl);
+    }
+    /* ----------------------------------------------------------------------
+     * manual_file
+     * -------------------------------------------------------------------- */
+    attemptManualFile(request, policy) {
+        const file = request.resource.manualFile;
+        const contentHash = sha256ContentHash(file.bytes);
+        if (request.previousVersion?.contentHash !== undefined &&
+            request.previousVersion.contentHash !== contentHash) {
+            return Promise.resolve({
+                status: "integrity_error",
+                reasonCode: "previous_hash_mismatch",
+                metadataExtras: { providedFileName: sanitizeFileName(file.fileName) },
+            });
+        }
+        if (request.expectedContentTypes !== undefined &&
+            !contentTypeMatches(file.contentType, request.expectedContentTypes)) {
+            return Promise.resolve({
+                status: "unsupported_content_type",
+                reasonCode: "content_type_mismatch",
+                metadataExtras: { providedFileName: sanitizeFileName(file.fileName) },
+            });
+        }
+        if (file.bytes.length > policy.limits.maxResponseBytes) {
+            return Promise.resolve({
+                status: "content_too_large",
+                reasonCode: "max_bytes_exceeded",
+                metadataExtras: { providedFileName: sanitizeFileName(file.fileName) },
+            });
+        }
+        return Promise.resolve({
+            status: "collected",
+            reasonCode: "manual_file_supplied",
+            bytes: file.bytes,
+            contentType: file.contentType,
+            fileName: sanitizeFileName(file.fileName),
+            metadataExtras: {
+                providedFileName: file.fileName,
+                providedContentType: file.contentType,
+            },
+        });
+    }
+    /* ----------------------------------------------------------------------
+     * permitted_browser
+     * -------------------------------------------------------------------- */
+    async attemptBrowser(request, policy, targetUrl) {
+        const requestedSanitized = sanitizeUrl(targetUrl);
+        const gate = await this.gateTarget(request, policy, "permitted_browser", targetUrl, requestedSanitized);
+        if (gate !== undefined)
+            return gate;
+        if (this.browserAdapter === undefined) {
+            throw new UnsupportedChannelError("Channel 'permitted_browser' was selected but no browser adapter is configured.", "permitted_browser");
+        }
+        const slot = this.limiter.tryAcquire(policy.sourceId, policy);
+        if (!slot.ok) {
+            return {
+                status: "rate_limited",
+                reasonCode: "local_rate_limit",
+                retryAfterSeconds: slot.retryAfterSeconds,
+                requestedUrl: requestedSanitized,
+            };
+        }
+        try {
+            const output = await this.browserAdapter.fetchPage({
+                url: targetUrl,
+                userAgent: this.userAgent,
+            });
+            return this.mapBrowserOutput(request, policy, output, targetUrl, requestedSanitized);
+        }
+        finally {
+            slot.release();
+        }
+    }
+    mapBrowserOutput(request, policy, output, targetUrl, requestedSanitized) {
+        if (output.blockedReason !== undefined) {
+            const statusByReason = {
+                captcha: "waf_or_bot_challenge",
+                waf: "waf_or_bot_challenge",
+                login: "login_required",
+                paywall: "paywalled",
+                geo: "geo_restricted",
+            };
+            return {
+                status: statusByReason[output.blockedReason],
+                reasonCode: "browser_adapter_reported_block",
+                requestedUrl: requestedSanitized,
+                finalUrl: output.finalUrl !== undefined ? sanitizeUrl(output.finalUrl) : undefined,
+                metadataExtras: { browserFinalUrl: sanitizeUrl(output.finalUrl ?? targetUrl) },
+            };
+        }
+        if (output.status < 200 || output.status >= 300) {
+            const classified = classifyHttpResponse(output.status, null);
+            return {
+                status: classified.status,
+                reasonCode: classified.reasonCode,
+                httpStatus: output.status,
+                requestedUrl: requestedSanitized,
+                finalUrl: sanitizeUrl(output.finalUrl ?? targetUrl),
+            };
+        }
+        if (output.bytes === undefined || output.bytes.length === 0) {
+            return {
+                status: "temporarily_unavailable",
+                reasonCode: "empty_browser_response",
+                httpStatus: output.status,
+                requestedUrl: requestedSanitized,
+            };
+        }
+        const contentType = output.contentType ?? "text/html";
+        if (request.expectedContentTypes !== undefined &&
+            !contentTypeMatches(contentType, request.expectedContentTypes)) {
+            return {
+                status: "unsupported_content_type",
+                reasonCode: "content_type_mismatch",
+                httpStatus: output.status,
+                requestedUrl: requestedSanitized,
+            };
+        }
+        if (output.bytes.length > policy.limits.maxResponseBytes) {
+            return {
+                status: "content_too_large",
+                reasonCode: "max_bytes_exceeded",
+                httpStatus: output.status,
+                requestedUrl: requestedSanitized,
+            };
+        }
+        return {
+            status: "collected",
+            reasonCode: "http_success",
+            bytes: output.bytes,
+            contentType,
+            fileName: fileNameFromUrl(output.finalUrl ?? targetUrl),
+            etag: output.headers !== undefined ? getHeader(output.headers, "etag") : undefined,
+            lastModified: output.headers !== undefined ? getHeader(output.headers, "last-modified") : undefined,
+            requestedUrl: requestedSanitized,
+            finalUrl: sanitizeUrl(output.finalUrl ?? targetUrl),
+            metadataExtras: { browserFinalUrl: sanitizeUrl(output.finalUrl ?? targetUrl) },
+        };
+    }
+    /* ----------------------------------------------------------------------
+     * HTTP channels: direct_http, official_api, rss_or_atom, approved_archive,
+     * licensed_provider
+     * -------------------------------------------------------------------- */
+    async attemptHttp(request, policy, channel, targetUrl) {
+        const requestedSanitized = sanitizeUrl(targetUrl);
+        const provenanceExtras = {};
+        const metadataExtras = {};
+        if (channel === "rss_or_atom") {
+            const entryId = extractFeedEntryId(targetUrl);
+            if (entryId !== undefined)
+                provenanceExtras.feedEntryId = entryId;
+            metadataExtras.feedUrl = requestedSanitized;
+        }
+        else if (channel === "official_api") {
+            provenanceExtras.apiResource = requestedSanitized;
+        }
+        else if (channel === "approved_archive") {
+            provenanceExtras.archiveReference = requestedSanitized;
+        }
+        else if (channel === "licensed_provider") {
+            metadataExtras.licensedResourceId = requestedSanitized;
+        }
+        const gate = await this.gateTarget(request, policy, channel, targetUrl, requestedSanitized, provenanceExtras, metadataExtras);
+        if (gate !== undefined)
+            return gate;
+        const auth = await this.buildAuthHeaders(policy, channel, targetUrl);
+        if (auth.outcome !== undefined) {
+            return {
+                ...auth.outcome,
+                requestedUrl: requestedSanitized,
+                provenanceExtras,
+                metadataExtras,
+            };
+        }
+        const conditionalHeaders = {};
+        if (request.previousVersion?.etag !== undefined) {
+            conditionalHeaders["if-none-match"] = request.previousVersion.etag;
+        }
+        if (request.previousVersion?.lastModified !== undefined) {
+            conditionalHeaders["if-modified-since"] = request.previousVersion.lastModified;
+        }
+        const slot = this.limiter.tryAcquire(policy.sourceId, policy);
+        if (!slot.ok) {
+            return {
+                status: "rate_limited",
+                reasonCode: "local_rate_limit",
+                retryAfterSeconds: slot.retryAfterSeconds,
+                requestedUrl: requestedSanitized,
+                provenanceExtras,
+                metadataExtras,
+            };
+        }
+        try {
+            let current = parseOutboundUrl(targetUrl);
+            let redirects = 0;
+            while (true) {
+                let response;
+                try {
+                    response = await this.timedRequest(current.url.toString(), {
+                        method: "GET",
+                        headers: { ...auth.headers, ...conditionalHeaders },
+                    });
+                }
+                catch (error) {
+                    if (isTypedError(error))
+                        throw error;
+                    return {
+                        status: "temporarily_unavailable",
+                        reasonCode: isAbortLike(error) ? "request_timed_out" : "network_error",
+                        requestedUrl: requestedSanitized,
+                        provenanceExtras,
+                        metadataExtras,
+                    };
+                }
+                if (REDIRECT_STATUSES.has(response.status)) {
+                    const verdict = await this.evaluateRedirect(response, current, policy, channel, redirects);
+                    if (verdict.next !== undefined) {
+                        current = verdict.next;
+                        redirects += 1;
+                        continue;
+                    }
+                    return {
+                        ...verdict.outcome,
+                        requestedUrl: requestedSanitized,
+                        provenanceExtras,
+                        metadataExtras,
+                    };
+                }
+                return await this.handleTerminalResponse(request, policy, channel, response, current, requestedSanitized, provenanceExtras, metadataExtras);
+            }
+        }
+        finally {
+            slot.release();
+        }
+    }
+    /**
+     * Shared pre-network gate for browser and HTTP channels. Address safety
+     * (hostname blocklist, DNS resolution, resolved-IP ranges) runs FIRST: it is
+     * a security invariant that holds regardless of the source policy. Policy
+     * protocol/domain checks and robots evaluation follow.
+     */
+    async gateTarget(request, policy, channel, targetUrl, requestedSanitized, provenanceExtras, metadataExtras) {
+        void request;
+        const parsed = parseOutboundUrl(targetUrl);
+        // Address safety. UnsafeUrlError propagates: security violations on the
+        // originally requested URL are usage errors, not retrieval outcomes.
+        const validation = await this.addresses.validate(parsed.hostname);
+        if (!validation.ok) {
+            return {
+                status: "temporarily_unavailable",
+                reasonCode: "dns_resolution_failed",
+                requestedUrl: requestedSanitized,
+                ...(provenanceExtras !== undefined ? { provenanceExtras } : {}),
+                ...(metadataExtras !== undefined ? { metadataExtras } : {}),
+            };
+        }
+        if (!policy.allowedProtocols.includes(parsed.protocol)) {
+            return this.policyRefusal("protocol_not_allowed", requestedSanitized, provenanceExtras, metadataExtras);
+        }
+        if (!hostMatchesAllowedDomains(parsed.hostname, policy.allowedDomains)) {
+            return this.policyRefusal("domain_not_allowed", requestedSanitized, provenanceExtras, metadataExtras);
+        }
+        if (policy.robotsPolicy === "required" && isRobotsGoverned(channel)) {
+            const decision = await this.robots.decide(parsed.url);
+            if (decision.outcome === "disallowed") {
+                return {
+                    status: "robots_disallowed",
+                    reasonCode: "robots_disallowed_path",
+                    requestedUrl: requestedSanitized,
+                    ...(provenanceExtras !== undefined ? { provenanceExtras } : {}),
+                    ...(metadataExtras !== undefined ? { metadataExtras } : {}),
+                };
+            }
+            if (decision.outcome === "unavailable") {
+                return {
+                    status: "manual_review_required",
+                    reasonCode: "robots_txt_unavailable",
+                    requestedUrl: requestedSanitized,
+                    ...(provenanceExtras !== undefined ? { provenanceExtras } : {}),
+                    ...(metadataExtras !== undefined ? { metadataExtras } : {}),
+                };
+            }
+        }
+        return undefined;
+    }
+    async evaluateRedirect(response, current, policy, channel, redirectsSoFar) {
+        const location = getHeader(response.headers, "location");
+        if (location === undefined) {
+            return {
+                outcome: {
+                    status: "temporarily_unavailable",
+                    reasonCode: "redirect_missing_location",
+                    httpStatus: response.status,
+                },
+            };
+        }
+        if (redirectsSoFar >= policy.limits.maxRedirects) {
+            return {
+                outcome: {
+                    status: "redirect_disallowed",
+                    reasonCode: "max_redirects_exceeded",
+                    httpStatus: response.status,
+                },
+            };
+        }
+        let nextUrl;
+        try {
+            nextUrl = new URL(location, current.url);
+        }
+        catch {
+            return {
+                outcome: {
+                    status: "redirect_disallowed",
+                    reasonCode: "redirect_location_invalid",
+                    httpStatus: response.status,
+                },
+            };
+        }
+        const hostname = nextUrl.hostname.replace(/^\[|\]$/g, "").toLowerCase();
+        const protocol = nextUrl.protocol.replace(":", "");
+        if (protocol !== "https" && protocol !== "http") {
+            return {
+                outcome: {
+                    status: "redirect_disallowed",
+                    reasonCode: "unsafe_redirect_scheme",
+                    httpStatus: response.status,
+                },
+            };
+        }
+        if (nextUrl.username !== "" || nextUrl.password !== "") {
+            return {
+                outcome: {
+                    status: "redirect_disallowed",
+                    reasonCode: "redirect_credentials_forbidden",
+                    httpStatus: response.status,
+                },
+            };
+        }
+        if (!policy.allowedProtocols.includes(protocol)) {
+            return {
+                outcome: {
+                    status: "redirect_disallowed",
+                    reasonCode: "redirect_protocol_not_allowed",
+                    httpStatus: response.status,
+                },
+            };
+        }
+        if (!hostMatchesAllowedDomains(hostname, policy.allowedDomains)) {
+            return {
+                outcome: {
+                    status: "redirect_disallowed",
+                    reasonCode: "redirect_domain_not_allowed",
+                    httpStatus: response.status,
+                },
+            };
+        }
+        try {
+            const validation = await this.addresses.validate(hostname);
+            if (!validation.ok) {
+                return {
+                    outcome: {
+                        status: "temporarily_unavailable",
+                        reasonCode: "dns_resolution_failed",
+                        httpStatus: response.status,
+                    },
+                };
+            }
+        }
+        catch (error) {
+            if (error instanceof UnsafeUrlError) {
+                return {
+                    outcome: {
+                        status: "redirect_disallowed",
+                        reasonCode: "unsafe_redirect_address",
+                        httpStatus: response.status,
+                    },
+                };
+            }
+            throw error;
+        }
+        if (policy.robotsPolicy === "required" && isRobotsGoverned(channel)) {
+            const decision = await this.robots.decide(nextUrl);
+            if (decision.outcome === "disallowed") {
+                return {
+                    outcome: {
+                        status: "robots_disallowed",
+                        reasonCode: "redirect_target_robots_disallowed",
+                        httpStatus: response.status,
+                    },
+                };
+            }
+            if (decision.outcome === "unavailable") {
+                return {
+                    outcome: {
+                        status: "manual_review_required",
+                        reasonCode: "robots_txt_unavailable",
+                        httpStatus: response.status,
+                    },
+                };
+            }
+        }
+        return { next: { url: nextUrl, hostname, protocol } };
+    }
+    async handleTerminalResponse(request, policy, channel, response, current, requestedSanitized, provenanceExtras, metadataExtras) {
+        const finalSanitized = sanitizeUrl(current.url.toString());
+        metadataExtras.finalRetainedHeaders = retainedResponseHeaders(response.headers);
+        if (channel === "approved_archive") {
+            const details = parseArchiveReference(current.url);
+            const headerTimestamp = getHeader(response.headers, "x-archive-capture-timestamp");
+            const headerOriginal = getHeader(response.headers, "x-archive-original-url");
+            metadataExtras.archiveProvider = current.url.host;
+            metadataExtras.captureTimestamp = headerTimestamp ?? details.captureTimestamp;
+            if (headerOriginal !== undefined && /^https?:\/\//i.test(headerOriginal)) {
+                metadataExtras.originalResourceUrl = sanitizeUrl(headerOriginal);
+            }
+            else if (details.originalResourceUrl !== undefined) {
+                metadataExtras.originalResourceUrl = details.originalResourceUrl;
+            }
+        }
+        if (channel === "official_api" || channel === "licensed_provider") {
+            metadataExtras.provider = current.url.host;
+        }
+        const etag = getHeader(response.headers, "etag");
+        const lastModified = getHeader(response.headers, "last-modified");
+        if (response.status === 304) {
+            return {
+                status: "not_modified",
+                reasonCode: "http_304",
+                httpStatus: 304,
+                etag: etag ?? request.previousVersion?.etag,
+                lastModified: lastModified ?? request.previousVersion?.lastModified,
+                requestedUrl: requestedSanitized,
+                finalUrl: finalSanitized,
+                provenanceExtras,
+                metadataExtras,
+            };
+        }
+        if (response.status < 200 || response.status >= 300) {
+            const prefix = await readBody(response.body, {
+                maxBytes: 4096,
+                prefixOnly: true,
+                prefixBytes: 4096,
+            });
+            const prefixText = Buffer.from(prefix.bytes.subarray(0, 4096)).toString("utf8");
+            const classified = classifyHttpResponse(response.status, prefixText);
+            return {
+                status: classified.status,
+                reasonCode: classified.reasonCode,
+                httpStatus: response.status,
+                retryAfterSeconds: parseRetryAfter(response.headers, this.now()),
+                requestedUrl: requestedSanitized,
+                finalUrl: finalSanitized,
+                provenanceExtras,
+                metadataExtras,
+            };
+        }
+        // 2xx path: only identity encoding is accepted so stored bytes are exact.
+        const encoding = getHeader(response.headers, "content-encoding");
+        if (encoding !== undefined && !/^identity$/i.test(encoding.trim())) {
+            return {
+                status: "unsupported_content_type",
+                reasonCode: "unsupported_content_encoding",
+                httpStatus: response.status,
+                requestedUrl: requestedSanitized,
+                finalUrl: finalSanitized,
+                provenanceExtras,
+                metadataExtras,
+            };
+        }
+        const contentType = normalizeContentType(getHeader(response.headers, "content-type"));
+        if (request.expectedContentTypes !== undefined &&
+            !contentTypeMatches(contentType, request.expectedContentTypes)) {
+            return {
+                status: "unsupported_content_type",
+                reasonCode: "content_type_mismatch",
+                httpStatus: response.status,
+                requestedUrl: requestedSanitized,
+                finalUrl: finalSanitized,
+                provenanceExtras,
+                metadataExtras,
+            };
+        }
+        const declaredLength = parseContentLength(getHeader(response.headers, "content-length"));
+        if (declaredLength !== undefined && declaredLength > policy.limits.maxResponseBytes) {
+            return {
+                status: "content_too_large",
+                reasonCode: "declared_length_exceeded",
+                httpStatus: response.status,
+                requestedUrl: requestedSanitized,
+                finalUrl: finalSanitized,
+                provenanceExtras,
+                metadataExtras,
+            };
+        }
+        const read = await readBody(response.body, { maxBytes: policy.limits.maxResponseBytes });
+        if (read.exceeded) {
+            return {
+                status: "content_too_large",
+                reasonCode: "max_bytes_exceeded",
+                httpStatus: response.status,
+                requestedUrl: requestedSanitized,
+                finalUrl: finalSanitized,
+                provenanceExtras,
+                metadataExtras,
+            };
+        }
+        if (declaredLength !== undefined && read.bytes.length !== declaredLength) {
+            return {
+                status: "integrity_error",
+                reasonCode: "content_length_mismatch",
+                httpStatus: response.status,
+                requestedUrl: requestedSanitized,
+                finalUrl: finalSanitized,
+                provenanceExtras,
+                metadataExtras,
+            };
+        }
+        if (shouldSniffSuccessBody(contentType, read.bytes.length)) {
+            const prefixText = Buffer.from(read.bytes).toString("utf8");
+            const classified = classifyHttpResponse(response.status, prefixText);
+            if (classified.status !== "collected") {
+                return {
+                    status: classified.status,
+                    reasonCode: classified.reasonCode,
+                    httpStatus: response.status,
+                    requestedUrl: requestedSanitized,
+                    finalUrl: finalSanitized,
+                    provenanceExtras,
+                    metadataExtras,
+                };
+            }
+        }
+        return {
+            status: "collected",
+            reasonCode: "http_success",
+            httpStatus: response.status,
+            bytes: read.bytes,
+            contentType,
+            fileName: fileNameFromUrl(current.url.toString()),
+            etag,
+            lastModified,
+            requestedUrl: requestedSanitized,
+            finalUrl: finalSanitized,
+            provenanceExtras,
+            metadataExtras,
+        };
+    }
+    policyRefusal(reasonCode, requestedSanitized, provenanceExtras, metadataExtras) {
+        return {
+            status: "source_not_authorised",
+            reasonCode,
+            requestedUrl: requestedSanitized,
+            ...(provenanceExtras !== undefined ? { provenanceExtras } : {}),
+            ...(metadataExtras !== undefined ? { metadataExtras } : {}),
+        };
+    }
+    async buildAuthHeaders(policy, channel, url) {
+        if (policy.authProfileId === undefined)
+            return { headers: {} };
+        if (this.authProvider === undefined) {
+            throw new UnsupportedChannelError(`Channel '${channel}' requires authentication profile '${policy.authProfileId}' but no auth provider is configured.`, channel);
+        }
+        const credentials = await this.authProvider.getCredentials({
+            authProfileId: policy.authProfileId,
+            url,
+            channel,
+        });
+        if (credentials === null) {
+            return {
+                outcome: { status: "source_not_authorised", reasonCode: "auth_unavailable" },
+            };
+        }
+        if (credentials.type === "bearer") {
+            return { headers: { authorization: `Bearer ${credentials.value}` } };
+        }
+        const name = (credentials.headerName ?? "x-api-key").toLowerCase();
+        if (!isTokenHeaderName(name)) {
+            return {
+                outcome: { status: "source_not_authorised", reasonCode: "auth_invalid" },
+            };
+        }
+        return { headers: { [name]: credentials.value } };
+    }
+    async timedRequest(url, init) {
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(new Error("request timed out")), this.requestTimeoutMs);
+        try {
+            return await this.httpClient.request(url, {
+                method: init.method,
+                headers: init.headers,
+                signal: controller.signal,
+            });
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    }
+    /* ----------------------------------------------------------------------
+     * Result assembly, persistence, audit
+     * -------------------------------------------------------------------- */
+    async finalize(request, policy, channel, acquisitionId, requestedAt, outcome, attempts) {
+        const completedAt = this.now();
+        const provenance = {
+            ...(outcome.requestedUrl !== undefined ? { requestedUrl: outcome.requestedUrl } : {}),
+            ...(outcome.finalUrl !== undefined ? { finalUrl: outcome.finalUrl } : {}),
+            retrievedAt: completedAt.toISOString(),
+            publisher: policy.publisher,
+            ...(outcome.provenanceExtras ?? {}),
+        };
+        let status = outcome.status;
+        let reasonCode = outcome.reasonCode;
+        let artifact;
+        let version;
+        if (outcome.status === "collected" && outcome.bytes !== undefined) {
+            const persisted = await this.persistence.persistArtifact({
+                acquisitionId,
+                sourceId: policy.sourceId,
+                fileName: outcome.fileName ?? "artifact",
+                contentType: outcome.contentType ?? "application/octet-stream",
+                bytes: outcome.bytes,
+                rights: policy.rights,
+            });
+            artifact = persisted.artifact;
+            if (persisted.artifact === undefined) {
+                status = "metadata_only";
+                reasonCode = "raw_retention_not_allowed";
+            }
+            version = compactVersion({
+                etag: outcome.etag,
+                lastModified: outcome.lastModified,
+                previousVersionId: await this.persistence.resolvePreviousVersionId(request, policy),
+                duplicateOfArtifactId: persisted.duplicateOfArtifactId,
+            });
+        }
+        else if (outcome.status === "not_modified") {
+            version = compactVersion({
+                etag: outcome.etag,
+                lastModified: outcome.lastModified,
+                previousVersionId: await this.persistence.resolvePreviousVersionId(request, policy),
+            });
+        }
+        const result = {
+            requestId: request.requestId,
+            acquisitionId,
+            status,
+            sourceId: policy.sourceId,
+            ...(channel !== undefined ? { channel } : {}),
+            ...(artifact !== undefined ? { artifact } : {}),
+            ...(version !== undefined ? { version } : {}),
+            provenance,
+            rights: policy.rights,
+            diagnostics: {
+                ...(outcome.httpStatus !== undefined ? { httpStatus: outcome.httpStatus } : {}),
+                ...(outcome.retryAfterSeconds !== undefined
+                    ? { retryAfterSeconds: outcome.retryAfterSeconds }
+                    : {}),
+                reasonCode,
+                sanitizedMessage: sanitizedMessageFor(reasonCode),
+            },
+        };
+        await this.persistence.writeAcquisitionMetadata(result, {
+            requestedAt: requestedAt.toISOString(),
+            completedAt: completedAt.toISOString(),
+            attempts,
+            ...(outcome.metadataExtras ?? {}),
+        });
+        await this.persistence.writeAuditRecord(result, requestedAt, completedAt);
+        return result;
+    }
+}
+/* --------------------------------------------------------------------------
+ * Local helpers
+ * ------------------------------------------------------------------------ */
+function readResourceUrl(request, kind) {
+    switch (kind) {
+        case "url":
+            return request.resource.url;
+        case "api_resource":
+            return request.resource.apiResource;
+        case "feed_entry":
+            return request.resource.feedEntryUrl;
+        case "archive_reference":
+            return request.resource.archiveReference;
+        case "licensed_resource":
+            return request.resource.licensedResourceId;
+        default:
+            return undefined;
+    }
+}
+function isRobotsGoverned(channel) {
+    return (channel === "direct_http" || channel === "rss_or_atom" || channel === "permitted_browser");
+}
+/** Statuses whose Location header must be followed (validated per hop). 304 is conditional retrieval, not a redirect. */
+const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
+function compactVersion(parts) {
+    const version = {};
+    if (parts.etag !== undefined)
+        version.etag = parts.etag;
+    if (parts.lastModified !== undefined)
+        version.lastModified = parts.lastModified;
+    if (parts.previousVersionId !== undefined)
+        version.previousVersionId = parts.previousVersionId;
+    if (parts.duplicateOfArtifactId !== undefined) {
+        version.duplicateOfArtifactId = parts.duplicateOfArtifactId;
+    }
+    return Object.keys(version).length > 0 ? version : undefined;
+}
+function normalizeContentType(raw) {
+    if (raw === undefined)
+        return "application/octet-stream";
+    const trimmed = raw.split(";")[0].trim().toLowerCase();
+    return trimmed === "" ? "application/octet-stream" : trimmed;
+}
+function parseContentLength(raw) {
+    if (raw === undefined || !/^\d+$/.test(raw.trim()))
+        return undefined;
+    return Number(raw.trim());
+}
+/**
+ * Create a policy-aware content fetcher.
+ *
+ * @throws ConfigurationError / SourcePolicyViolationError when options or any
+ * source policy is invalid.
+ */
+export function createContentFetcher(options) {
+    return new FetcherImpl(options);
+}
+//# sourceMappingURL=create-content-fetcher.js.map