@x12i/content-fetcher 0.1.0

package/README.md

@@ -0,0 +1,133 @@
+# @x12i/content-fetcher
+
+Policy-aware content acquisition for known, approved sources.
+
+The fetcher receives an **exact** resource (URL, feed entry, API resource, archive reference, licensed-provider reference, or manually supplied file), validates it against a source-policy registry, retrieves the permitted content through an approved channel, and preserves the original bytes together with provenance, rights, and acquisition outcomes. Downstream packages (extraction, OCR, transcription, Markdown normalization) consume the retained artifacts.
+
+It **never** discovers URLs, crawls, schedules, extracts text, uses an LLM, or bypasses access restrictions (CAPTCHAs, WAFs, bot challenges, logins, paywalls, geo-blocks, rate limits). Restrictions are detected and reported, not circumvented.
+
+- **Runtime:** Node.js ≥ 20, ESM only, zero runtime dependencies.
+- **Not included:** URL discovery, crawling, scheduling, text extraction, Markdown generation, classification, summarization.
+
+## Installation
+
+```bash
+npm install @x12i/content-fetcher
+```
+
+## Usage
+
+```ts
+import {
+  createContentFetcher,
+  toFetchedArtifactReference,
+  type SourcePolicy,
+} from "@x12i/content-fetcher";
+
+const policies: SourcePolicy[] = [
+  {
+    sourceId: "metr",
+    publisher: "Metropolitan Review",
+    allowedDomains: ["example.com"],
+    allowedProtocols: ["https"],
+    allowedChannels: ["direct_http", "rss_or_atom"],
+    channelOrder: ["direct_http"],
+    robotsPolicy: "required",
+    browserAccess: "disabled",
+    archiveAccess: "disabled",
+    licensedProviderAccess: "disabled",
+    rateLimit: { maxRequestsPerMinute: 30, maxConcurrentRequests: 2 },
+    limits: { maxRedirects: 3, maxResponseBytes: 8 * 1024 * 1024 },
+    rights: {
+      profileId: "metr-public-v1",
+      allowRawRetention: true,
+      allowExtraction: true,
+      allowDisplay: true,
+      allowTraining: false,
+      retentionClass: "public-web",
+    },
+  },
+];
+
+const fetcher = createContentFetcher({
+  sourcePolicies: policies,
+  artifactStore: myArtifactStore, // your implementation
+  // httpClient, browserAdapter, authProvider, dnsResolver are optional
+  // adapters; see "Adapters" below. Default HTTP and DNS adapters are used
+  // when omitted (httpClient is required if any policy enables a network
+  // channel).
+});
+
+const result = await fetcher.fetch({
+  requestId: "req_0001",
+  sourceId: "metr",
+  resource: { url: "https://example.com/articles/article.html" },
+  previousVersion: { etag: '"v1"' },
+});
+
+if (result.status === "collected") {
+  const reference = toFetchedArtifactReference(result);
+  // hand `reference` to the extraction / normalization pipeline
+}
+```
+
+## How a request flows
+
+1. **Request validation** — malformed requests throw `InvalidFetchRequestError`; an unknown `sourceId` throws `UnknownSourcePolicyError`. Expected retrieval outcomes are returned as statuses, never thrown.
+2. **Channel selection** — the resource kind maps to candidate channels (`url` → `direct_http`/`permitted_browser`, `feedEntryUrl` → `rss_or_atom`, `apiResource` → `official_api`, `archiveReference` → `approved_archive`, `licensedResourceId` → `licensed_provider`, `manualFile` → `manual_file`). The policy's `channelOrder` (restricted to `allowedChannels`) decides the plan. Fallback to the next channel happens only for `temporarily_unavailable`; access-restriction outcomes are terminal — retrying through another channel would amount to a bypass attempt.
+3. **Security gates (before any network request)** — URL parsing rejects non-HTTP(S) schemes and embedded credentials (`UnsafeUrlError`); blocked hostnames (`localhost`, cloud-metadata names, `*.internal`) and blocked IP ranges (loopback, private, shared/CGNAT, link-local, multicast, reserved, documentation — including IPv4-mapped IPv6) throw `UnsafeUrlError`; DNS is resolved and **every** resolved address is re-validated. A hostname that cannot be resolved yields `temporarily_unavailable` (`dns_resolution_failed`) — nothing is fetched unverifiable.
+4. **Policy gates** — protocol and domain allowlists; violations return `source_not_authorised` before any request is made.
+5. **Robots** — when `robotsPolicy: "required"` (applies to `direct_http`, `rss_or_atom`, `permitted_browser`), robots.txt is fetched (per-origin TTL cache), parsed per RFC 9309 semantics, and evaluated for the configured user agent. Disallowed → `robots_disallowed` without fetching the resource. robots.txt unavailable (401/403/5xx/network) → `manual_review_required`. 404 → permissive.
+6. **Rate limiting** — per-source sliding one-minute window plus concurrency cap. Local refusals return `rate_limited` with `retryAfterSeconds`.
+7. **Retrieval** — `If-None-Match`/`If-Modified-Since` are sent from `previousVersion`; `304` → `not_modified`. Redirects are followed one hop at a time; **each destination is re-validated** (scheme, credentials, protocol/domain policy, address safety, robots) before the next request. Violations → `redirect_disallowed`.
+8. **Classification** — status codes plus high-precision body markers map to explicit statuses (`login_required`, `paywalled`, `waf_or_bot_challenge`, `rate_limited`, `geo_restricted`, …). Detection never triggers a bypass.
+9. **Content gates** — non-identity content encodings are rejected (`unsupported_content_type` / `unsupported_content_encoding`) so retained bytes are exactly what the server sent; `expectedContentTypes` mismatches are rejected; responses are streamed under `maxResponseBytes`; declared `Content-Length` mismatches are flagged `integrity_error`.
+10. **Persistence** — SHA-256 is computed; identical content per source is deduplicated (`findByContentHash`, never overwriting an existing artifact, `version.duplicateOfArtifactId` set) while every acquisition keeps its own metadata record and audit record. When `rights.allowRawRetention` is false the bytes are hashed but **not** stored and the result is `metadata_only`.
+
+## Acquisition statuses
+
+`collected`, `not_modified`, `metadata_only`, `robots_disallowed`, `rate_limited`, `waf_or_bot_challenge`, `login_required`, `paywalled`, `geo_restricted`, `source_not_authorised`, `not_found`, `temporarily_unavailable`, `redirect_disallowed`, `content_too_large`, `unsupported_content_type`, `integrity_error`, `manual_review_required`
+
+There is deliberately no generic `failed` status.
+
+## Adapters
+
+The package has no I/O of its own; everything injectable:
+
+| Adapter | Purpose | Default |
+| --- | --- | --- |
+| `ArtifactStore` | Immutable raw artifact + metadata + audit storage. **Required.** | none |
+| `HttpClient` | HTTP transport. Must not follow redirects itself; the fetcher validates each hop. | `DefaultHttpClient` (platform `fetch`, manual redirects, `accept-encoding: identity`, per-request timeout) |
+| `DnsResolver` | Address resolution so SSRF validation can run after DNS. | `SystemDnsResolver` (`node:dns`) |
+| `BrowserAdapter` | `permitted_browser` channel. Must report `blockedReason` instead of attempting to bypass challenges. Required at fetch time when a browser acquisition is attempted. | none |
+| `AuthProvider` | Supplies credentials for a policy's `authProfileId`. Credentials are attached to requests only — never stored, logged, or echoed in results. Required at fetch time when an authenticated channel is used. | none |
+
+Notes on resource fields: `apiResource`, `feedEntryUrl`, `archiveReference`, and `licensedResourceId` must each be an absolute URL to the exact resource; they are validated against the same policy as `url`. Wayback-style archive references additionally record capture timestamp and original resource URL in the acquisition metadata.
+
+## Security model
+
+- Validation occurs before the first request, after DNS resolution, and before/after every redirect hop (the fetcher drives redirects manually, so every destination is validated before it is contacted).
+- Stored/returned URLs are sanitized: credentials, query strings, and fragments are removed before anything is persisted, returned, or audited.
+- Only an allowlist of benign response headers (`content-type`, `etag`, `last-modified`, `date`, `cache-control`, `expires`, `content-length`, `content-encoding`, `x-archive-*`) is retained; cookies and authentication material never reach storage.
+- Audit records contain identifiers, statuses, timestamps, and hashes only.
+- Diagnostic messages come from a fixed vocabulary (`sanitizedMessage`); raw error text is never surfaced.
+- Known limitation: validate-then-connect leaves a small DNS-rebinding window. Deployments that need connection-level pinning should wrap `undici`'s `Agent` with a custom `connect` that pins the validated address, or front the process with an egress proxy enforcing the same policy.
+
+## Downstream handoff
+
+`toFetchedArtifactReference(result)` returns the `FetchedArtifactReference` (acquisition id, artifact id, source, file name, content type, hash, provenance, rights) for any result with a retained artifact, or `null` otherwise (e.g. `metadata_only`). Extraction, OCR, transcription, canonical construction, and Markdown normalization are the next package's job.
+
+## Typed errors
+
+`InvalidFetchRequestError`, `UnknownSourcePolicyError`, `SourcePolicyViolationError`, `UnsafeUrlError`, `UnsupportedChannelError`, `ArtifactStoreError`, `ConfigurationError` — thrown for invalid usage, configuration, or deployment wiring. Everything expected during retrieval is an `AcquisitionResult` status.
+
+## Development
+
+```bash
+npm install
+npm run build     # tsc -> dist/
+npm test          # vitest
+npm run typecheck
+```
+
+The test suite covers all acceptance criteria from the package requirement: byte-exact retrieval, domain/redirect policy enforcement, robots compliance, conditional retrieval, deduplication, rights gating, browser gating, explicit restriction statuses, SSRF blocking, output sanitization, and downstream handoff.

package/dist/adapters/default-http-client.d.ts

@@ -0,0 +1,17 @@
+import type { HttpClient, HttpClientRequestInit, HttpClientResponse } from "../types.js";
+/**
+ * Default HTTP transport built on the platform `fetch`.
+ *
+ * - Redirects are NEVER followed automatically; the fetcher validates each
+ *   hop against the source policy before following it.
+ * - `accept-encoding: identity` keeps the stored bytes identical to what the
+ *   server sent, honouring the preserve-original-bytes requirement.
+ * - The injected AbortSignal (timeouts, size aborts) is chained through.
+ */
+export declare class DefaultHttpClient implements HttpClient {
+    private readonly userAgent;
+    private readonly requestTimeoutMs;
+    constructor(userAgent: string, requestTimeoutMs: number);
+    request(url: string, init?: HttpClientRequestInit): Promise<HttpClientResponse>;
+}
+//# sourceMappingURL=default-http-client.d.ts.map

package/dist/adapters/default-http-client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"default-http-client.d.ts","sourceRoot":"","sources":["../../src/adapters/default-http-client.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,UAAU,EACV,qBAAqB,EACrB,kBAAkB,EACnB,MAAM,aAAa,CAAC;AAErB;;;;;;;;GAQG;AACH,qBAAa,iBAAkB,YAAW,UAAU;IAEhD,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,gBAAgB;gBADhB,SAAS,EAAE,MAAM,EACjB,gBAAgB,EAAE,MAAM;IAGrC,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,qBAAqB,GAAG,OAAO,CAAC,kBAAkB,CAAC;CA0CtF"}

package/dist/adapters/default-http-client.js

@@ -0,0 +1,74 @@
+/**
+ * Default HTTP transport built on the platform `fetch`.
+ *
+ * - Redirects are NEVER followed automatically; the fetcher validates each
+ *   hop against the source policy before following it.
+ * - `accept-encoding: identity` keeps the stored bytes identical to what the
+ *   server sent, honouring the preserve-original-bytes requirement.
+ * - The injected AbortSignal (timeouts, size aborts) is chained through.
+ */
+export class DefaultHttpClient {
+    userAgent;
+    requestTimeoutMs;
+    constructor(userAgent, requestTimeoutMs) {
+        this.userAgent = userAgent;
+        this.requestTimeoutMs = requestTimeoutMs;
+    }
+    async request(url, init) {
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(new Error("request timed out")), this.requestTimeoutMs);
+        const onCallerAbort = () => controller.abort(new Error("request aborted"));
+        init?.signal?.addEventListener("abort", onCallerAbort, { once: true });
+        try {
+            const response = await fetch(url, {
+                method: init?.method ?? "GET",
+                headers: {
+                    "user-agent": this.userAgent,
+                    accept: "*/*",
+                    "accept-encoding": "identity",
+                    ...(init?.headers ?? {}),
+                },
+                redirect: "manual",
+                signal: controller.signal,
+            });
+            const headers = {};
+            const seen = new Map();
+            response.headers.forEach((value, name) => {
+                const key = name.toLowerCase();
+                const list = seen.get(key) ?? [];
+                list.push(value);
+                seen.set(key, list);
+            });
+            for (const [key, values] of seen)
+                headers[key] = values.join(", ");
+            const body = response.body
+                ? streamToAsyncIterable(response.body)
+                : undefined;
+            return { status: response.status, headers, body };
+        }
+        finally {
+            clearTimeout(timer);
+            init?.signal?.removeEventListener("abort", onCallerAbort);
+        }
+    }
+}
+function streamToAsyncIterable(stream) {
+    return {
+        [Symbol.asyncIterator]() {
+            const reader = stream.getReader();
+            return {
+                async next() {
+                    const { done, value } = await reader.read();
+                    if (done)
+                        return { done: true, value: undefined };
+                    return { done: false, value: value };
+                },
+                async return() {
+                    await reader.cancel().catch(() => undefined);
+                    return { done: true, value: undefined };
+                },
+            };
+        },
+    };
+}
+//# sourceMappingURL=default-http-client.js.map

package/dist/adapters/default-http-client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"default-http-client.js","sourceRoot":"","sources":["../../src/adapters/default-http-client.ts"],"names":[],"mappings":"AAMA;;;;;;;;GAQG;AACH,MAAM,OAAO,iBAAiB;IAET;IACA;IAFnB,YACmB,SAAiB,EACjB,gBAAwB;QADxB,cAAS,GAAT,SAAS,CAAQ;QACjB,qBAAgB,GAAhB,gBAAgB,CAAQ;IACxC,CAAC;IAEJ,KAAK,CAAC,OAAO,CAAC,GAAW,EAAE,IAA4B;QACrD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,UAAU,CACtB,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC,EACtD,IAAI,CAAC,gBAAgB,CACtB,CAAC;QACF,MAAM,aAAa,GAAG,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC;QAC3E,IAAI,EAAE,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAEvE,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAChC,MAAM,EAAE,IAAI,EAAE,MAAM,IAAI,KAAK;gBAC7B,OAAO,EAAE;oBACP,YAAY,EAAE,IAAI,CAAC,SAAS;oBAC5B,MAAM,EAAE,KAAK;oBACb,iBAAiB,EAAE,UAAU;oBAC7B,GAAG,CAAC,IAAI,EAAE,OAAO,IAAI,EAAE,CAAC;iBACzB;gBACD,QAAQ,EAAE,QAAQ;gBAClB,MAAM,EAAE,UAAU,CAAC,MAAM;aAC1B,CAAC,CAAC;YAEH,MAAM,OAAO,GAA2B,EAAE,CAAC;YAC3C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAoB,CAAC;YACzC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;gBACvC,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;gBAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;gBACjC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACjB,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;YACtB,CAAC,CAAC,CAAC;YACH,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,IAAI;gBAAE,OAAO,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAEnE,MAAM,IAAI,GAA0C,QAAQ,CAAC,IAAI;gBAC/D,CAAC,CAAC,qBAAqB,CAAC,QAAQ,CAAC,IAAI,CAAC;gBACtC,CAAC,CAAC,SAAS,CAAC;YAEd,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QACpD,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,IAAI,EAAE,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC;CACF;AAED,SAAS,qBAAqB,CAAC,MAAkC;IAC/D,OAAO;QACL,CAAC,MAAM,CAAC,aAAa,CAAC;YACpB,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;YAClC,OAAO;gBACL,KAAK,CAAC,IAAI;oBACR,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;oBAC5C,IAAI,IAAI;wBAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;oBAClD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,KAAmB,EAAE,CAAC;gBACrD,CAAC;gBACD,KAAK,CAAC,MAAM;oBACV,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;oBAC7C,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;gBAC1C,CAAC;aACF,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/channels.d.ts

@@ -0,0 +1,34 @@
+import type { ResourceKind } from "./validate-request.js";
+import type { AcquisitionStatus, AcquisitionChannel, SourcePolicy } from "./types.js";
+/**
+ * Maps a request's resource kind to the channels that can acquire it.
+ * Channel fallback then follows the policy's channelOrder restricted to this
+ * candidate set.
+ */
+export declare const KIND_CANDIDATE_CHANNELS: Record<ResourceKind, readonly AcquisitionChannel[]>;
+/** Channels governed by robots.txt when the policy requires it. */
+export declare const ROBOTS_GOVERNED_CHANNELS: readonly AcquisitionChannel[];
+/**
+ * Select the ordered channel plan for a request: policy channelOrder entries
+ * that are both allowed by the policy and compatible with the resource kind.
+ */
+export declare function selectChannelPlan(kind: ResourceKind, policy: SourcePolicy): AcquisitionChannel[];
+/** Additional policy gates beyond channel allowlisting. */
+export declare function channelPermittedByPolicy(channel: AcquisitionChannel, policy: SourcePolicy): boolean;
+export declare const FALLTHROUGH_STATUSES: readonly AcquisitionStatus[];
+export declare function sanitizedMessageFor(reasonCode: string): string;
+export declare function requireBrowserAdapter(browserAdapter: unknown): asserts browserAdapter is NonNullable<unknown>;
+export declare function isTokenHeaderName(name: string): boolean;
+export declare function contentTypeMatches(actual: string, expected: readonly string[]): boolean;
+/** Extract a feed entry identifier from a fragment or common query parameters, if present. */
+export declare function extractFeedEntryId(raw: string): string | undefined;
+export interface ArchiveReferenceDetails {
+    captureTimestamp?: string;
+    originalResourceUrl?: string;
+}
+/**
+ * Parse wayback-style archive references (`/web/<timestamp>/<original-url>`)
+ * for provenance. Returns empty details for other archive URL shapes.
+ */
+export declare function parseArchiveReference(url: URL): ArchiveReferenceDetails;
+//# sourceMappingURL=channels.d.ts.map

package/dist/channels.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"channels.d.ts","sourceRoot":"","sources":["../src/channels.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAC1D,OAAO,KAAK,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAEtF;;;;GAIG;AACH,eAAO,MAAM,uBAAuB,EAAE,MAAM,CAAC,YAAY,EAAE,SAAS,kBAAkB,EAAE,CAOvF,CAAC;AAEF,mEAAmE;AACnE,eAAO,MAAM,wBAAwB,EAAE,SAAS,kBAAkB,EAIjE,CAAC;AAEF;;;GAGG;AACH,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,YAAY,EAClB,MAAM,EAAE,YAAY,GACnB,kBAAkB,EAAE,CAKtB;AAED,2DAA2D;AAC3D,wBAAgB,wBAAwB,CACtC,OAAO,EAAE,kBAAkB,EAC3B,MAAM,EAAE,YAAY,GACnB,OAAO,CAWT;AAED,eAAO,MAAM,oBAAoB,EAAE,SAAS,iBAAiB,EAE5D,CAAC;AA+DF,wBAAgB,mBAAmB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAE9D;AAED,wBAAgB,qBAAqB,CACnC,cAAc,EAAE,OAAO,GACtB,OAAO,CAAC,cAAc,IAAI,WAAW,CAAC,OAAO,CAAC,CAOhD;AAID,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAEvD;AAID,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAWvF;AAED,8FAA8F;AAC9F,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAalE;AAED,MAAM,WAAW,uBAAuB;IACtC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,GAAG,GAAG,uBAAuB,CAYvE"}

package/dist/channels.js

@@ -0,0 +1,164 @@
+import { UnsupportedChannelError } from "./errors.js";
+/**
+ * Maps a request's resource kind to the channels that can acquire it.
+ * Channel fallback then follows the policy's channelOrder restricted to this
+ * candidate set.
+ */
+export const KIND_CANDIDATE_CHANNELS = {
+    url: ["direct_http", "permitted_browser"],
+    api_resource: ["official_api"],
+    feed_entry: ["rss_or_atom"],
+    archive_reference: ["approved_archive"],
+    licensed_resource: ["licensed_provider"],
+    manual_file: ["manual_file"],
+};
+/** Channels governed by robots.txt when the policy requires it. */
+export const ROBOTS_GOVERNED_CHANNELS = [
+    "direct_http",
+    "rss_or_atom",
+    "permitted_browser",
+];
+/**
+ * Select the ordered channel plan for a request: policy channelOrder entries
+ * that are both allowed by the policy and compatible with the resource kind.
+ */
+export function selectChannelPlan(kind, policy) {
+    const candidates = KIND_CANDIDATE_CHANNELS[kind];
+    return policy.channelOrder.filter((channel) => candidates.includes(channel) && policy.allowedChannels.includes(channel));
+}
+/** Additional policy gates beyond channel allowlisting. */
+export function channelPermittedByPolicy(channel, policy) {
+    if (!policy.allowedChannels.includes(channel))
+        return false;
+    if (channel === "permitted_browser" && policy.browserAccess !== "permitted")
+        return false;
+    if (channel === "approved_archive" && policy.archiveAccess !== "permitted")
+        return false;
+    if (channel === "licensed_provider" &&
+        policy.licensedProviderAccess !== "permitted") {
+        return false;
+    }
+    return true;
+}
+export const FALLTHROUGH_STATUSES = [
+    "temporarily_unavailable",
+];
+/** Fixed, safe diagnostic messages. Raw error text never reaches results. */
+const SANITIZED_MESSAGES = {
+    domain_not_allowed: "The resource host is not allowed by the source policy.",
+    protocol_not_allowed: "The resource protocol is not allowed by the source policy.",
+    channel_not_permitted: "No permitted acquisition channel supports this resource.",
+    local_rate_limit: "The source rate limit defined by policy has been reached.",
+    window_exhausted: "The source rate limit defined by policy has been reached.",
+    concurrency_exhausted: "The source concurrency limit defined by policy has been reached.",
+    robots_disallowed_path: "robots.txt disallows this resource for the configured user agent.",
+    redirect_target_robots_disallowed: "robots.txt disallows the redirect destination for the configured user agent.",
+    robots_txt_unavailable: "robots.txt could not be retrieved; permission to fetch cannot be verified.",
+    http_429: "The source responded with a rate-limit status.",
+    challenge_detected: "A bot challenge or WAF interstitial was detected.",
+    login_marker_detected: "The resource requires authentication.",
+    paywall_marker_detected: "The resource is paywalled.",
+    geo_marker_detected: "The resource is geo-restricted.",
+    http_401: "The resource requires authentication.",
+    http_402: "The resource requires payment.",
+    http_451: "The resource is unavailable for legal or geographic reasons.",
+    max_redirects_exceeded: "The redirect chain exceeded the policy limit.",
+    redirect_domain_not_allowed: "A redirect destination is not allowed by the source policy.",
+    unsafe_redirect_address: "A redirect destination targets an address the package must never contact.",
+    unsafe_redirect_scheme: "A redirect destination uses a protocol other than http or https.",
+    redirect_credentials_forbidden: "A redirect destination contained credentials.",
+    redirect_location_invalid: "A redirect destination could not be parsed.",
+    redirect_missing_location: "A redirect response carried no Location header.",
+    redirect_protocol_not_allowed: "A redirect destination protocol is not allowed by the source policy.",
+    dns_resolution_failed: "The resource host could not be resolved.",
+    network_error: "The request failed at the network level.",
+    request_timed_out: "The request timed out.",
+    unsupported_content_encoding: "The response used a content encoding the package does not accept.",
+    content_type_mismatch: "The response content type does not match the expected content types.",
+    max_bytes_exceeded: "The response exceeded the maximum response size.",
+    declared_length_exceeded: "The declared response length exceeded the maximum response size.",
+    content_length_mismatch: "The response length did not match its declared Content-Length.",
+    auth_unavailable: "Authentication required by the source policy could not be supplied.",
+    auth_invalid: "The configured authentication produced an invalid header.",
+    browser_adapter_reported_block: "The browser adapter reported an access restriction; no bypass was attempted.",
+    empty_browser_response: "The browser adapter returned no document.",
+    previous_hash_mismatch: "The supplied bytes do not match the declared previous content hash.",
+    http_success: "The resource was retrieved successfully.",
+    http_304: "The resource was not modified since the previous acquisition.",
+    duplicate_content: "The content matches an already retained artifact.",
+    manual_file_supplied: "A manually supplied file was accepted for acquisition.",
+    raw_retention_not_allowed: "The rights profile for this source does not permit raw retention; only metadata was retained.",
+    empty_response: "The response contained no content.",
+};
+export function sanitizedMessageFor(reasonCode) {
+    return SANITIZED_MESSAGES[reasonCode] ?? "The acquisition ended with an explicit outcome.";
+}
+export function requireBrowserAdapter(browserAdapter) {
+    if (browserAdapter === undefined || browserAdapter === null) {
+        throw new UnsupportedChannelError("Channel 'permitted_browser' was selected but no browser adapter is configured.", "permitted_browser");
+    }
+}
+const HEADER_NAME_RE = /^[\w-]+$/;
+export function isTokenHeaderName(name) {
+    return HEADER_NAME_RE.test(name);
+}
+// Match a response content type against expected patterns such as
+// "text/html", "application/" + "*", or the any-type wildcard pattern.
+export function contentTypeMatches(actual, expected) {
+    const [type, subtype] = actual.split(";")[0].trim().toLowerCase().split("/");
+    if (type === undefined || subtype === undefined)
+        return false;
+    return expected.some((pattern) => {
+        const [expectedType, expectedSubtype] = pattern.trim().toLowerCase().split("/");
+        if (expectedType === undefined || expectedSubtype === undefined)
+            return false;
+        return ((expectedType === "*" || expectedType === type) &&
+            (expectedSubtype === "*" || expectedSubtype === subtype));
+    });
+}
+/** Extract a feed entry identifier from a fragment or common query parameters, if present. */
+export function extractFeedEntryId(raw) {
+    try {
+        const url = new URL(raw);
+        const fragment = url.hash.replace(/^#/, "");
+        if (fragment !== "")
+            return fragment;
+        for (const key of ["entry", "id", "guid"]) {
+            const value = url.searchParams.get(key);
+            if (value !== null && value !== "")
+                return value;
+        }
+    }
+    catch {
+        return undefined;
+    }
+    return undefined;
+}
+/**
+ * Parse wayback-style archive references (`/web/<timestamp>/<original-url>`)
+ * for provenance. Returns empty details for other archive URL shapes.
+ */
+export function parseArchiveReference(url) {
+    let pathname;
+    try {
+        pathname = decodeURIComponent(url.pathname);
+    }
+    catch {
+        pathname = url.pathname;
+    }
+    const match = pathname.match(/^\/web\/[^/]*?(\d{8,14})[^/]*?\/(https?:\/\/.+)$/i);
+    if (match?.[1] && match[2]) {
+        return { captureTimestamp: match[1], originalResourceUrl: sanitizeForProvenance(match[2]) };
+    }
+    return {};
+}
+import { sanitizeUrl } from "./security/url.js";
+function sanitizeForProvenance(raw) {
+    try {
+        return sanitizeUrl(raw);
+    }
+    catch {
+        return raw;
+    }
+}
+//# sourceMappingURL=channels.js.map

package/dist/channels.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"channels.js","sourceRoot":"","sources":["../src/channels.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AAItD;;;;GAIG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAwD;IAC1F,GAAG,EAAE,CAAC,aAAa,EAAE,mBAAmB,CAAC;IACzC,YAAY,EAAE,CAAC,cAAc,CAAC;IAC9B,UAAU,EAAE,CAAC,aAAa,CAAC;IAC3B,iBAAiB,EAAE,CAAC,kBAAkB,CAAC;IACvC,iBAAiB,EAAE,CAAC,mBAAmB,CAAC;IACxC,WAAW,EAAE,CAAC,aAAa,CAAC;CAC7B,CAAC;AAEF,mEAAmE;AACnE,MAAM,CAAC,MAAM,wBAAwB,GAAkC;IACrE,aAAa;IACb,aAAa;IACb,mBAAmB;CACpB,CAAC;AAEF;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAC/B,IAAkB,EAClB,MAAoB;IAEpB,MAAM,UAAU,GAAG,uBAAuB,CAAC,IAAI,CAAC,CAAC;IACjD,OAAO,MAAM,CAAC,YAAY,CAAC,MAAM,CAC/B,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,eAAe,CAAC,QAAQ,CAAC,OAAO,CAAC,CACtF,CAAC;AACJ,CAAC;AAED,2DAA2D;AAC3D,MAAM,UAAU,wBAAwB,CACtC,OAA2B,EAC3B,MAAoB;IAEpB,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,QAAQ,CAAC,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAC5D,IAAI,OAAO,KAAK,mBAAmB,IAAI,MAAM,CAAC,aAAa,KAAK,WAAW;QAAE,OAAO,KAAK,CAAC;IAC1F,IAAI,OAAO,KAAK,kBAAkB,IAAI,MAAM,CAAC,aAAa,KAAK,WAAW;QAAE,OAAO,KAAK,CAAC;IACzF,IACE,OAAO,KAAK,mBAAmB;QAC/B,MAAM,CAAC,sBAAsB,KAAK,WAAW,EAC7C,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,MAAM,oBAAoB,GAAiC;IAChE,yBAAyB;CAC1B,CAAC;AAEF,6EAA6E;AAC7E,MAAM,kBAAkB,GAA2B;IACjD,kBAAkB,EAAE,wDAAwD;IAC5E,oBAAoB,EAAE,4DAA4D;IAClF,qBAAqB,EAAE,0DAA0D;IACjF,gBAAgB,EAAE,2DAA2D;IAC7E,gBAAgB,EAAE,2DAA2D;IAC7E,qBAAqB,EAAE,kEAAkE;IACzF,sBAAsB,EAAE,mEAAmE;IAC3F,iCAAiC,EAC/B,8EAA8E;IAChF,sBAAsB,EACpB,4EAA4E;IAC9E,QAAQ,EAAE,gDAAgD;IAC1D,kBAAkB,EAAE,mDAAmD;IACvE,qBAAqB,EAAE,uCAAuC;IAC9D,uBAAuB,EAAE,4BAA4B;IACrD,mBAAmB,EAAE,iCAAiC;IACtD,QAAQ,EAAE,uCAAuC;IACjD,QAAQ,EAAE,gCAAgC;IAC1C,QAAQ,EAAE,8DAA8D;IACxE,sBAAsB,EAAE,+CAA+C;IACvE,2BAA2B,EACzB,6DAA6D;IAC/D,uBAAuB,EACrB,2EAA2E;IAC7E,sBAAsB,EACpB,kEAAkE;IACpE,8BAA8B,EAAE,+CAA+C;IAC/E,yBAAyB,EAAE,6CAA6C;IACxE,yBAAyB,EAAE,iDAAiD;IAC5E,6BAA6B,EAC3B,sEAAsE;IACxE,qBAAqB,EAAE,0CAA0C;IACjE,aAAa,EAAE,0CAA0C;IACzD,iBAAiB,EAAE,wBAAwB;IAC3C,4BAA4B,EAC1B,mEAAmE;IACrE,qBAAqB,EACnB,sEAAsE;IACxE,kBAAkB,EAAE,kDAAkD;IACtE,wBAAwB,EAAE,kEAAkE;IAC5F,uBAAuB,EACrB,gEAAgE;IAClE,gBAAgB,EACd,qEAAqE;IACvE,YAAY,EAAE,2DAA2D;IACzE,8BAA8B,EAC5B,8EAA8E;IAChF,sBAAsB,EAAE,2CAA2C;IACnE,sBAAsB,EACpB,qEAAqE;IACvE,YAAY,EAAE,0CAA0C;IACxD,QAAQ,EAAE,+DAA+D;IACzE,iBAAiB,EAAE,mDAAmD;IACtE,oBAAoB,EAAE,wDAAwD;IAC9E,yBAAyB,EACvB,+FAA+F;IACjG,cAAc,EAAE,oCAAoC;CACrD,CAAC;AAEF,MAAM,UAAU,mBAAmB,CAAC,UAAkB;IACpD,OAAO,kBAAkB,CAAC,UAAU,CAAC,IAAI,iDAAiD,CAAC;AAC7F,CAAC;AAED,MAAM,UAAU,qBAAqB,CACnC,cAAuB;IAEvB,IAAI,cAAc,KAAK,SAAS,IAAI,cAAc,KAAK,IAAI,EAAE,CAAC;QAC5D,MAAM,IAAI,uBAAuB,CAC/B,gFAAgF,EAChF,mBAAmB,CACpB,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,cAAc,GAAG,UAAU,CAAC;AAElC,MAAM,UAAU,iBAAiB,CAAC,IAAY;IAC5C,OAAO,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACnC,CAAC;AAED,kEAAkE;AAClE,uEAAuE;AACvE,MAAM,UAAU,kBAAkB,CAAC,MAAc,EAAE,QAA2B;IAC5E,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC9E,IAAI,IAAI,KAAK,SAAS,IAAI,OAAO,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC;IAC9D,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE;QAC/B,MAAM,CAAC,YAAY,EAAE,eAAe,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAChF,IAAI,YAAY,KAAK,SAAS,IAAI,eAAe,KAAK,SAAS;YAAE,OAAO,KAAK,CAAC;QAC9E,OAAO,CACL,CAAC,YAAY,KAAK,GAAG,IAAI,YAAY,KAAK,IAAI,CAAC;YAC/C,CAAC,eAAe,KAAK,GAAG,IAAI,eAAe,KAAK,OAAO,CAAC,CACzD,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,8FAA8F;AAC9F,MAAM,UAAU,kBAAkB,CAAC,GAAW;IAC5C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACzB,MAAM,QAAQ,GAAG,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC5C,IAAI,QAAQ,KAAK,EAAE;YAAE,OAAO,QAAQ,CAAC;QACrC,KAAK,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,EAAE,CAAC;YAC1C,MAAM,KAAK,GAAG,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACxC,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,EAAE;gBAAE,OAAO,KAAK,CAAC;QACnD,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAOD;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CAAC,GAAQ;IAC5C,IAAI,QAAgB,CAAC;IACrB,IAAI,CAAC;QACH,QAAQ,GAAG,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACP,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;IAC1B,CAAC;IACD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,mDAAmD,CAAC,CAAC;IAClF,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3B,OAAO,EAAE,gBAAgB,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,qBAAqB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC9F,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,SAAS,qBAAqB,CAAC,GAAW;IACxC,IAAI,CAAC;QACH,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,CAAC;IACb,CAAC;AACH,CAAC"}

package/dist/classify.d.ts

@@ -0,0 +1,13 @@
+import type { AcquisitionStatus } from "./types.js";
+export interface ClassifiedResponse {
+    status: AcquisitionStatus;
+    reasonCode: string;
+}
+/**
+ * Classify a response from its status code. `sniffPrefix` (first bytes of the
+ * body, text-decoded) refines 403-family and 200-family outcomes.
+ */
+export declare function classifyHttpResponse(httpStatus: number, sniffPrefix: string | null): ClassifiedResponse;
+/** True when a 2xx body should be sniffed for challenge markers. */
+export declare function shouldSniffSuccessBody(contentType: string, byteLength: number): boolean;
+//# sourceMappingURL=classify.d.ts.map

package/dist/classify.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"classify.d.ts","sourceRoot":"","sources":["../src/classify.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AA4DpD,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,iBAAiB,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAClC,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,GAAG,IAAI,GACzB,kBAAkB,CAqDpB;AAED,oEAAoE;AACpE,wBAAgB,sBAAsB,CAAC,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAEvF"}

package/dist/classify.js

@@ -0,0 +1,122 @@
+/**
+ * Maps an HTTP response to an acquisition status. Uses high-precision markers
+ * only, to avoid misclassifying legitimate content that merely mentions
+ * CAPTCHAs or subscriptions. Detection never triggers bypass behaviour; the
+ * package simply reports what it found.
+ */
+const WAF_CHALLENGE_MARKERS = [
+    "just a moment",
+    "checking your browser",
+    "attention required",
+    "cf-chl",
+    "cf_chl_",
+    "cdn-cgi/challenge-platform",
+    "verify you are human",
+    "verify you are not a robot",
+    "enable javascript and cookies",
+    "distil_r_captcha",
+    "geo-captcha",
+    "px-captcha",
+    "_incapsula_resource",
+    "sucuri website cloud",
+    "bot verification",
+    "blocked by ibm security",
+];
+const LOGIN_MARKERS = [
+    "sign in to continue",
+    "log in to continue",
+    "login required",
+    "please log in",
+    "sign in to view",
+];
+const PAYWALL_MARKERS = [
+    "subscribe to continue",
+    "subscription required",
+    "subscribe to read",
+    "premium content",
+    "purchase to access",
+    "this article is for subscribers",
+];
+const GEO_MARKERS = [
+    "not available in your country",
+    "not available in your region",
+    "access denied due to geographic",
+    "geo-restricted",
+];
+function containsMarker(prefix, markers) {
+    const lower = prefix.toLowerCase();
+    for (const marker of markers) {
+        if (lower.includes(marker))
+            return marker;
+    }
+    return null;
+}
+/**
+ * Classify a response from its status code. `sniffPrefix` (first bytes of the
+ * body, text-decoded) refines 403-family and 200-family outcomes.
+ */
+export function classifyHttpResponse(httpStatus, sniffPrefix) {
+    if (httpStatus === 304)
+        return { status: "not_modified", reasonCode: "http_304" };
+    if (httpStatus === 401)
+        return { status: "login_required", reasonCode: "http_401" };
+    if (httpStatus === 402)
+        return { status: "paywalled", reasonCode: "http_402" };
+    if (httpStatus === 429)
+        return { status: "rate_limited", reasonCode: "http_429" };
+    if (httpStatus === 451)
+        return { status: "geo_restricted", reasonCode: "http_451" };
+    if (httpStatus === 404 || httpStatus === 410) {
+        return { status: "not_found", reasonCode: `http_${httpStatus}` };
+    }
+    if (httpStatus === 408) {
+        return { status: "temporarily_unavailable", reasonCode: "http_408" };
+    }
+    if (httpStatus === 413)
+        return { status: "content_too_large", reasonCode: "http_413" };
+    if (httpStatus === 415) {
+        return { status: "unsupported_content_type", reasonCode: "http_415" };
+    }
+    if (httpStatus === 418 || httpStatus === 403) {
+        // Teapot and forbidden are challenge-shaped unless a more specific marker says otherwise.
+        if (sniffPrefix !== null) {
+            if (containsMarker(sniffPrefix, WAF_CHALLENGE_MARKERS)) {
+                return { status: "waf_or_bot_challenge", reasonCode: "challenge_detected" };
+            }
+            if (containsMarker(sniffPrefix, LOGIN_MARKERS)) {
+                return { status: "login_required", reasonCode: "login_marker_detected" };
+            }
+            if (containsMarker(sniffPrefix, PAYWALL_MARKERS)) {
+                return { status: "paywalled", reasonCode: "paywall_marker_detected" };
+            }
+            if (containsMarker(sniffPrefix, GEO_MARKERS)) {
+                return { status: "geo_restricted", reasonCode: "geo_marker_detected" };
+            }
+        }
+        return { status: "waf_or_bot_challenge", reasonCode: `http_${httpStatus}` };
+    }
+    if (httpStatus >= 400 && httpStatus < 500) {
+        // Remaining 4xx: the request as issued is not permitted by the server.
+        return { status: "source_not_authorised", reasonCode: `http_${httpStatus}` };
+    }
+    if (httpStatus >= 500) {
+        return { status: "temporarily_unavailable", reasonCode: `http_${httpStatus}` };
+    }
+    if (httpStatus >= 200 && httpStatus < 300) {
+        // A successful status with an interstitial challenge shell (small HTML).
+        if (sniffPrefix !== null && containsMarker(sniffPrefix, WAF_CHALLENGE_MARKERS)) {
+            return { status: "waf_or_bot_challenge", reasonCode: "challenge_detected" };
+        }
+        return { status: "collected", reasonCode: "http_success" };
+    }
+    if (httpStatus >= 300 && httpStatus < 400) {
+        // Should have been handled by the redirect loop.
+        return { status: "temporarily_unavailable", reasonCode: "unresolved_redirect" };
+    }
+    return { status: "temporarily_unavailable", reasonCode: `http_${httpStatus}` };
+}
+/** True when a 2xx body should be sniffed for challenge markers. */
+export function shouldSniffSuccessBody(contentType, byteLength) {
+    return contentType.includes("text/html") && byteLength <= 65_536;
+}
+//# sourceMappingURL=classify.js.map

package/dist/classify.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"classify.js","sourceRoot":"","sources":["../src/classify.ts"],"names":[],"mappings":"AAEA;;;;;GAKG;AAEH,MAAM,qBAAqB,GAAG;IAC5B,eAAe;IACf,uBAAuB;IACvB,oBAAoB;IACpB,QAAQ;IACR,SAAS;IACT,4BAA4B;IAC5B,sBAAsB;IACtB,4BAA4B;IAC5B,+BAA+B;IAC/B,kBAAkB;IAClB,aAAa;IACb,YAAY;IACZ,qBAAqB;IACrB,sBAAsB;IACtB,kBAAkB;IAClB,yBAAyB;CAC1B,CAAC;AAEF,MAAM,aAAa,GAAG;IACpB,qBAAqB;IACrB,oBAAoB;IACpB,gBAAgB;IAChB,eAAe;IACf,iBAAiB;CAClB,CAAC;AAEF,MAAM,eAAe,GAAG;IACtB,uBAAuB;IACvB,uBAAuB;IACvB,mBAAmB;IACnB,iBAAiB;IACjB,oBAAoB;IACpB,iCAAiC;CAClC,CAAC;AAEF,MAAM,WAAW,GAAG;IAClB,+BAA+B;IAC/B,8BAA8B;IAC9B,iCAAiC;IACjC,gBAAgB;CACjB,CAAC;AAEF,SAAS,cAAc,CAAC,MAAc,EAAE,OAAiB;IACvD,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;IACnC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,MAAM,CAAC;IAC5C,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAOD;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAClC,UAAkB,EAClB,WAA0B;IAE1B,IAAI,UAAU,KAAK,GAAG;QAAE,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;IAClF,IAAI,UAAU,KAAK,GAAG;QAAE,OAAO,EAAE,MAAM,EAAE,gBAAgB,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;IACpF,IAAI,UAAU,KAAK,GAAG;QAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;IAC/E,IAAI,UAAU,KAAK,GAAG;QAAE,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;IAClF,IAAI,UAAU,KAAK,GAAG;QAAE,OAAO,EAAE,MAAM,EAAE,gBAAgB,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;IACpF,IAAI,UAAU,KAAK,GAAG,IAAI,UAAU,KAAK,GAAG,EAAE,CAAC;QAC7C,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,UAAU,EAAE,EAAE,CAAC;IACnE,CAAC;IACD,IAAI,UAAU,KAAK,GAAG,EAAE,CAAC;QACvB,OAAO,EAAE,MAAM,EAAE,yBAAyB,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;IACvE,CAAC;IACD,IAAI,UAAU,KAAK,GAAG;QAAE,OAAO,EAAE,MAAM,EAAE,mBAAmB,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;IACvF,IAAI,UAAU,KAAK,GAAG,EAAE,CAAC;QACvB,OAAO,EAAE,MAAM,EAAE,0BAA0B,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;IACxE,CAAC;IACD,IAAI,UAAU,KAAK,GAAG,IAAI,UAAU,KAAK,GAAG,EAAE,CAAC;QAC7C,0FAA0F;QAC1F,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;YACzB,IAAI,cAAc,CAAC,WAAW,EAAE,qBAAqB,CAAC,EAAE,CAAC;gBACvD,OAAO,EAAE,MAAM,EAAE,sBAAsB,EAAE,UAAU,EAAE,oBAAoB,EAAE,CAAC;YAC9E,CAAC;YACD,IAAI,cAAc,CAAC,WAAW,EAAE,aAAa,CAAC,EAAE,CAAC;gBAC/C,OAAO,EAAE,MAAM,EAAE,gBAAgB,EAAE,UAAU,EAAE,uBAAuB,EAAE,CAAC;YAC3E,CAAC;YACD,IAAI,cAAc,CAAC,WAAW,EAAE,eAAe,CAAC,EAAE,CAAC;gBACjD,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,yBAAyB,EAAE,CAAC;YACxE,CAAC;YACD,IAAI,cAAc,CAAC,WAAW,EAAE,WAAW,CAAC,EAAE,CAAC;gBAC7C,OAAO,EAAE,MAAM,EAAE,gBAAgB,EAAE,UAAU,EAAE,qBAAqB,EAAE,CAAC;YACzE,CAAC;QACH,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,sBAAsB,EAAE,UAAU,EAAE,QAAQ,UAAU,EAAE,EAAE,CAAC;IAC9E,CAAC;IACD,IAAI,UAAU,IAAI,GAAG,IAAI,UAAU,GAAG,GAAG,EAAE,CAAC;QAC1C,uEAAuE;QACvE,OAAO,EAAE,MAAM,EAAE,uBAAuB,EAAE,UAAU,EAAE,QAAQ,UAAU,EAAE,EAAE,CAAC;IAC/E,CAAC;IACD,IAAI,UAAU,IAAI,GAAG,EAAE,CAAC;QACtB,OAAO,EAAE,MAAM,EAAE,yBAAyB,EAAE,UAAU,EAAE,QAAQ,UAAU,EAAE,EAAE,CAAC;IACjF,CAAC;IACD,IAAI,UAAU,IAAI,GAAG,IAAI,UAAU,GAAG,GAAG,EAAE,CAAC;QAC1C,yEAAyE;QACzE,IAAI,WAAW,KAAK,IAAI,IAAI,cAAc,CAAC,WAAW,EAAE,qBAAqB,CAAC,EAAE,CAAC;YAC/E,OAAO,EAAE,MAAM,EAAE,sBAAsB,EAAE,UAAU,EAAE,oBAAoB,EAAE,CAAC;QAC9E,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,cAAc,EAAE,CAAC;IAC7D,CAAC;IACD,IAAI,UAAU,IAAI,GAAG,IAAI,UAAU,GAAG,GAAG,EAAE,CAAC;QAC1C,iDAAiD;QACjD,OAAO,EAAE,MAAM,EAAE,yBAAyB,EAAE,UAAU,EAAE,qBAAqB,EAAE,CAAC;IAClF,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,yBAAyB,EAAE,UAAU,EAAE,QAAQ,UAAU,EAAE,EAAE,CAAC;AACjF,CAAC;AAED,oEAAoE;AACpE,MAAM,UAAU,sBAAsB,CAAC,WAAmB,EAAE,UAAkB;IAC5E,OAAO,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,UAAU,IAAI,MAAM,CAAC;AACnE,CAAC"}

package/dist/config.d.ts

@@ -0,0 +1,7 @@
+import type { AcquisitionChannel, ContentFetcherOptions, SourcePolicy } from "./types.js";
+export declare function isAcquisitionChannel(value: unknown): value is AcquisitionChannel;
+/** Deep runtime validation of one source policy. Throws SourcePolicyViolationError. */
+export declare function validateSourcePolicy(policy: unknown): SourcePolicy;
+/** Validate the whole fetcher configuration. Throws ConfigurationError / SourcePolicyViolationError. */
+export declare function validateConfiguration(options: unknown): asserts options is ContentFetcherOptions;
+//# sourceMappingURL=config.d.ts.map

package/dist/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,kBAAkB,EAAE,qBAAqB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAqB1F,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,kBAAkB,CAEhF;AA8BD,uFAAuF;AACvF,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,OAAO,GAAG,YAAY,CA4KlE;AAED,wGAAwG;AACxG,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,qBAAqB,CAkEhG"}