@bluefields/primitives 0.2.0

package/dist/logger.js

@@ -0,0 +1,118 @@
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Structured logging (pino, JSON output).
+ *
+ * One root logger; child loggers per layer to attach consistent bindings
+ * without callers having to remember them.
+ *
+ * ## Conventions
+ *
+ * Every log call passes a structured object as the first argument with these
+ * fields when relevant:
+ *   - layer:           'poll' | 'fetcher' | 'extractor' | 'differ' | 'events' | 'api' | 'control'
+ *   - subscription_id: UUID
+ *   - event_id:        UUID
+ *   - customer_id:     UUID
+ *   - duration_ms:     int
+ *   - outcome:         'ok' | 'cached' | 'blocked' | 'skipped' | 'error' | 'partial'
+ *
+ * The message string is the second argument and stays short — searchable
+ * tokens, not prose.
+ *
+ * ## What NOT to log
+ *
+ * Configured via pino's `redact` so accidental inclusion in a bindings
+ * object or a log argument silently becomes `[Redacted]`:
+ *   - webhook secrets (current + previous)
+ *   - API keys (full + hash)
+ *   - Stripe webhook signing secret
+ *   - Authorization headers + bearer tokens
+ *   - Anthropic API key
+ *   - Resend API key
+ *   - Raw HTML / extraction body — caller's responsibility (we just don't log it)
+ *
+ * Raw HTML and raw extraction output aren't redactable by path (the field
+ * names vary by call site); the convention is: log size + hash, never bytes.
+ * Reviewers should grep `logger.*(.*html|.*markdown|.*structuredData|.*body:)`
+ * before merging changes that touch logging.
+ */
+import { pino } from 'pino';
+/**
+ * Redact paths — pino walks these and replaces matches with [Redacted].
+ * Glob `*` matches a single level; `**` matches any depth.
+ *
+ * Be generous; false positives (redacting something harmless) are cheap,
+ * leaking a secret is not.
+ */
+const REDACT_PATHS = [
+    // Webhook secrets
+    '*.webhook_secret',
+    '*.webhookSecret',
+    '*.webhookSecretCurrent',
+    '*.webhookSecretPrevious',
+    '*.webhook_secret_current',
+    '*.webhook_secret_previous',
+    // API keys (both formats)
+    '*.api_key',
+    '*.apiKey',
+    '*.key_hash',
+    '*.keyHash',
+    // Generic secrets
+    '*.secret',
+    '*.password',
+    '*.token',
+    // HTTP headers
+    '*.authorization',
+    '*.Authorization',
+    'headers.authorization',
+    'headers.Authorization',
+    'headers["x-stripe-signature"]',
+    'headers["stripe-signature"]',
+    'headers["bluefields-signature"]',
+    // Provider keys
+    '*.STRIPE_SECRET_KEY',
+    '*.STRIPE_WEBHOOK_SECRET',
+    '*.ANTHROPIC_API_KEY',
+    '*.RESEND_API_KEY',
+    '*.IPROYAL_PASSWORD',
+    '*.R2_SECRET_KEY',
+    // Customer plaintext on creation
+    '*.apiKeyPlaintext',
+    '*.api_key_plaintext',
+];
+let rootLogger = null;
+/**
+ * Build the root logger. Idempotent — subsequent calls return the same
+ * instance. Tests can pass a custom `level` override.
+ */
+export function getRootLogger(env = process.env) {
+    if (rootLogger)
+        return rootLogger;
+    const built = pino({
+        level: env.LOG_LEVEL ?? (env.NODE_ENV === 'production' ? 'info' : 'debug'),
+        redact: {
+            paths: REDACT_PATHS,
+            censor: '[Redacted]',
+        },
+        // ISO timestamps — easier to grep than epoch ms.
+        timestamp: () => `,"time":"${new Date().toISOString()}"`,
+        base: {
+            service: 'bluefields',
+        },
+    });
+    rootLogger = built;
+    return built;
+}
+/**
+ * Build a child logger pre-bound to a layer (and optional subscription /
+ * customer / event ids). Pass through callers so log calls don't have to
+ * repeat the same bindings.
+ */
+export function createLayerLogger(layer, bindings = {}, env) {
+    return getRootLogger(env).child({ layer, ...bindings });
+}
+/** Test-only — reset the cached root logger. */
+export function _resetLoggerForTests() {
+    rootLogger = null;
+}
+//# sourceMappingURL=logger.js.map

package/dist/logger.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"logger.js","sourceRoot":"","sources":["../src/logger.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAEH,OAAO,EAA6B,IAAI,EAAE,MAAM,MAAM,CAAC;AA0BvD;;;;;;GAMG;AACH,MAAM,YAAY,GAAG;IACnB,kBAAkB;IAClB,kBAAkB;IAClB,iBAAiB;IACjB,wBAAwB;IACxB,yBAAyB;IACzB,0BAA0B;IAC1B,2BAA2B;IAC3B,0BAA0B;IAC1B,WAAW;IACX,UAAU;IACV,YAAY;IACZ,WAAW;IACX,kBAAkB;IAClB,UAAU;IACV,YAAY;IACZ,SAAS;IACT,eAAe;IACf,iBAAiB;IACjB,iBAAiB;IACjB,uBAAuB;IACvB,uBAAuB;IACvB,+BAA+B;IAC/B,6BAA6B;IAC7B,iCAAiC;IACjC,gBAAgB;IAChB,qBAAqB;IACrB,yBAAyB;IACzB,qBAAqB;IACrB,kBAAkB;IAClB,oBAAoB;IACpB,iBAAiB;IACjB,iCAAiC;IACjC,mBAAmB;IACnB,qBAAqB;CACtB,CAAC;AAEF,IAAI,UAAU,GAAkB,IAAI,CAAC;AAErC;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,MAAyB,OAAO,CAAC,GAAG;IAChE,IAAI,UAAU;QAAE,OAAO,UAAU,CAAC;IAClC,MAAM,KAAK,GAAG,IAAI,CAAC;QACjB,KAAK,EAAE,GAAG,CAAC,SAAS,IAAI,CAAC,GAAG,CAAC,QAAQ,KAAK,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;QAC1E,MAAM,EAAE;YACN,KAAK,EAAE,YAAY;YACnB,MAAM,EAAE,YAAY;SACrB;QACD,iDAAiD;QACjD,SAAS,EAAE,GAAG,EAAE,CAAC,YAAY,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,GAAG;QACxD,IAAI,EAAE;YACJ,OAAO,EAAE,YAAY;SACtB;KACF,CAAC,CAAC;IACH,UAAU,GAAG,KAAK,CAAC;IACnB,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAC/B,KAAe,EACf,WAA0B,EAAE,EAC5B,GAAuB;IAEvB,OAAO,aAAa,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,GAAG,QAAQ,EAAE,CAAC,CAAC;AAC1D,CAAC;AAUD,gDAAgD;AAChD,MAAM,UAAU,oBAAoB;IAClC,UAAU,GAAG,IAAI,CAAC;AACpB,CAAC"}

package/dist/robots-txt.d.ts

@@ -0,0 +1,65 @@
+/**
+ * robots.txt parser + cache (chunk 9).
+ *
+ * Per the preface's "legal stuff is real" guidance: respect robots.txt
+ * by default. Fetched lazily per host with a 1-hour in-memory TTL.
+ *
+ * Defaults to ALLOWED on:
+ *   - robots.txt fetch failure (404, 5xx, network) — robots being down
+ *     shouldn't block customer subscriptions
+ *   - robots.txt missing — same
+ *   - SSRF rejection of the robots URL itself — log + allow
+ *
+ * Crawl-delay is parsed (see `parseCrawlDelays` / `evaluateRobots`) for the
+ * politeness floor. The authorized "ignore robots.txt" opt-out is enforced one
+ * layer up (the robots-gated fetcher + an API-level authorization attestation),
+ * not here — see `ROBOTS_POLICY.md`. Staging-environment robots stays out of scope.
+ */
+/**
+ * Thrown when a URL is disallowed by robots.txt and the request carries no
+ * authorized opt-out. Carries a stable `reason` so callers (the API routes) map
+ * it to a 403 `robots_disallowed` without string-matching the message.
+ */
+export declare class RobotsDisallowedError extends Error {
+    readonly url: string;
+    readonly reason: "robots_disallowed";
+    constructor(url: string);
+}
+export declare function resetRobotsCache(): void;
+export interface IsAllowedOptions {
+    /** Test seam: override the fetch. */
+    fetchImpl?: typeof fetch;
+    /** Test seam: override Date.now() for TTL checks. */
+    nowMsFn?: () => number;
+    /** Test seam: skip the cache. */
+    bypassCache?: boolean;
+}
+export interface RobotsEvaluation {
+    /** False iff a Disallow rule matches the path for this user-agent. */
+    allowed: boolean;
+    /** Crawl-delay (seconds) for the matched agent group, or null if none. */
+    crawlDelaySeconds: number | null;
+}
+/**
+ * Evaluate robots.txt for `url` + `userAgent` from a single cached fetch+parse:
+ * whether the path is allowed AND the applicable Crawl-delay. Fail-open
+ * (`{ allowed: true, crawlDelaySeconds: null }`) when robots.txt is
+ * missing/failed/SSRF-rejected — see the file header.
+ *
+ * Cache key is `${url.origin}` so http vs https are separate entries.
+ */
+export declare function evaluateRobots(url: string, userAgent: string, options?: IsAllowedOptions): Promise<RobotsEvaluation>;
+/**
+ * Returns true iff `url` is allowed by the host's robots.txt for the given
+ * user-agent. Thin wrapper over `evaluateRobots`, preserved for existing
+ * callers. Fail-open if robots.txt is missing or fetch fails.
+ */
+export declare function isAllowedByRobotsTxt(url: string, userAgent: string, options?: IsAllowedOptions): Promise<boolean>;
+/**
+ * Disallow prefixes per lowercased user-agent ('*' fallback). Exported for testing.
+ */
+export declare function parseRobotsTxt(text: string): Record<string, string[]>;
+/**
+ * Crawl-delay (seconds) per lowercased user-agent ('*' fallback). Exported for testing.
+ */
+export declare function parseCrawlDelays(text: string): Record<string, number>;

package/dist/robots-txt.js

@@ -0,0 +1,250 @@
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * robots.txt parser + cache (chunk 9).
+ *
+ * Per the preface's "legal stuff is real" guidance: respect robots.txt
+ * by default. Fetched lazily per host with a 1-hour in-memory TTL.
+ *
+ * Defaults to ALLOWED on:
+ *   - robots.txt fetch failure (404, 5xx, network) — robots being down
+ *     shouldn't block customer subscriptions
+ *   - robots.txt missing — same
+ *   - SSRF rejection of the robots URL itself — log + allow
+ *
+ * Crawl-delay is parsed (see `parseCrawlDelays` / `evaluateRobots`) for the
+ * politeness floor. The authorized "ignore robots.txt" opt-out is enforced one
+ * layer up (the robots-gated fetcher + an API-level authorization attestation),
+ * not here — see `ROBOTS_POLICY.md`. Staging-environment robots stays out of scope.
+ */
+import { createLayerLogger } from './logger.js';
+import { SsrfValidationError, validateUrlForExternalRequest } from './ssrf.js';
+const log = createLayerLogger('fetcher');
+const CACHE_TTL_MS = 60 * 60 * 1000; // 1 hour
+const ROBOTS_FETCH_TIMEOUT_MS = 5_000;
+/**
+ * Thrown when a URL is disallowed by robots.txt and the request carries no
+ * authorized opt-out. Carries a stable `reason` so callers (the API routes) map
+ * it to a 403 `robots_disallowed` without string-matching the message.
+ */
+export class RobotsDisallowedError extends Error {
+    url;
+    reason = 'robots_disallowed';
+    constructor(url) {
+        super(`robots.txt disallows fetching ${url}`);
+        this.url = url;
+        this.name = 'RobotsDisallowedError';
+    }
+}
+/**
+ * Mutable module-level cache. Tests reset via `resetRobotsCache`.
+ * Production process restarts naturally invalidate; for v1 that's
+ * acceptable (no shared cache across instances).
+ */
+const cache = new Map();
+export function resetRobotsCache() {
+    cache.clear();
+}
+/**
+ * Evaluate robots.txt for `url` + `userAgent` from a single cached fetch+parse:
+ * whether the path is allowed AND the applicable Crawl-delay. Fail-open
+ * (`{ allowed: true, crawlDelaySeconds: null }`) when robots.txt is
+ * missing/failed/SSRF-rejected — see the file header.
+ *
+ * Cache key is `${url.origin}` so http vs https are separate entries.
+ */
+export async function evaluateRobots(url, userAgent, options = {}) {
+    const parsed = safeParseUrl(url);
+    // Can't parse → don't block; the fetcher will reject this URL anyway.
+    if (!parsed)
+        return { allowed: true, crawlDelaySeconds: null };
+    const origin = parsed.origin;
+    const path = parsed.pathname + parsed.search;
+    const now = options.nowMsFn ? options.nowMsFn() : Date.now();
+    let rules;
+    if (!options.bypassCache) {
+        const hit = cache.get(origin);
+        if (hit && now - hit.cachedAtMs < CACHE_TTL_MS) {
+            rules = hit;
+        }
+    }
+    if (!rules) {
+        rules = await fetchAndParse(origin, now, options.fetchImpl);
+        if (!options.bypassCache)
+            cache.set(origin, rules);
+    }
+    const ua = userAgent.toLowerCase();
+    return {
+        allowed: !isDisallowed(rules, ua, path),
+        crawlDelaySeconds: crawlDelayFor(rules, ua),
+    };
+}
+/**
+ * Returns true iff `url` is allowed by the host's robots.txt for the given
+ * user-agent. Thin wrapper over `evaluateRobots`, preserved for existing
+ * callers. Fail-open if robots.txt is missing or fetch fails.
+ */
+export async function isAllowedByRobotsTxt(url, userAgent, options = {}) {
+    return (await evaluateRobots(url, userAgent, options)).allowed;
+}
+// ── Internals ───────────────────────────────────────────────────────────
+function safeParseUrl(s) {
+    try {
+        return new URL(s);
+    }
+    catch {
+        return null;
+    }
+}
+async function fetchAndParse(origin, now, fetchImpl) {
+    const robotsUrl = `${origin}/robots.txt`;
+    // SSRF-validate. Allow on SSRF rejection (robots.txt URL is the same
+    // host as a URL we'll likely fetch anyway; rejecting here would just
+    // double-error.)
+    try {
+        await validateUrlForExternalRequest(robotsUrl);
+    }
+    catch (err) {
+        if (err instanceof SsrfValidationError) {
+            return { disallows: {}, crawlDelays: {}, cachedAtMs: now };
+        }
+        throw err;
+    }
+    const f = fetchImpl ?? fetch;
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), ROBOTS_FETCH_TIMEOUT_MS);
+    try {
+        const res = await f(robotsUrl, { method: 'GET', signal: controller.signal });
+        if (!res.ok) {
+            // 404 is the common case (no robots.txt) — debug level. 5xx is
+            // worth a warn so an operator can see if a target host's robots
+            // is degraded.
+            log[res.status >= 500 ? 'warn' : 'debug']({ outcome: 'skipped', origin, status: res.status }, 'robots.txt fetch non-2xx; failing open');
+            return { disallows: {}, crawlDelays: {}, cachedAtMs: now };
+        }
+        const text = await res.text();
+        const groups = parseGroups(text);
+        return { disallows: groups.disallows, crawlDelays: groups.crawlDelays, cachedAtMs: now };
+    }
+    catch (err) {
+        // Network failure, timeout, etc. → fail open per spec. Log so it's
+        // visible — fail-open is a deliberate trade-off but invisible
+        // failures hide actual outages.
+        log.warn({ outcome: 'error', origin, error: err.message }, 'robots.txt fetch threw; failing open');
+        return { disallows: {}, crawlDelays: {}, cachedAtMs: now };
+    }
+    finally {
+        clearTimeout(timeout);
+    }
+}
+/**
+ * Single-pass robots.txt group parser → Disallow prefixes + Crawl-delay seconds
+ * per lowercased user-agent ('*' fallback). Handles consecutive User-agent lines
+ * forming one group, comments, and blank-line group separators. Defensive:
+ * anything unexpected is ignored, never throws. Allow-vs-Disallow precedence and
+ * `*`/`$` wildcard path matching remain out of scope for v1 (see ROBOTS_POLICY.md).
+ */
+function parseGroups(text) {
+    const disallows = {};
+    const crawlDelays = {};
+    let currentAgents = [];
+    // collectingAgents: inside a stretch of consecutive User-agent lines that all
+    // belong to the same group. The next directive ends the agent-collecting phase.
+    let collectingAgents = false;
+    for (const rawLine of text.split(/\r?\n/)) {
+        // Check the RAW line for blankness before stripping comments. A comment-only
+        // line is not a group separator per RFC 9309.
+        if (rawLine.trim().length === 0) {
+            currentAgents = [];
+            collectingAgents = false;
+            continue;
+        }
+        const line = rawLine.replace(/#.*$/, '').trim();
+        if (line.length === 0)
+            continue; // pure comment → skip without ending the group
+        const colon = line.indexOf(':');
+        if (colon === -1)
+            continue;
+        const key = line.slice(0, colon).trim().toLowerCase();
+        const value = line.slice(colon + 1).trim();
+        if (key === 'user-agent') {
+            if (!collectingAgents) {
+                currentAgents = [value.toLowerCase()];
+                collectingAgents = true;
+            }
+            else {
+                currentAgents.push(value.toLowerCase());
+            }
+        }
+        else if (key === 'disallow') {
+            collectingAgents = false;
+            if (currentAgents.length === 0)
+                continue;
+            if (value.length === 0)
+                continue; // empty disallow = no rule per RFC
+            for (const agent of currentAgents) {
+                if (!disallows[agent])
+                    disallows[agent] = [];
+                disallows[agent].push(value);
+            }
+        }
+        else if (key === 'crawl-delay') {
+            collectingAgents = false;
+            if (currentAgents.length === 0)
+                continue;
+            const seconds = Number(value);
+            // Ignore non-numeric / negative crawl-delays (defensive).
+            if (Number.isFinite(seconds) && seconds >= 0) {
+                for (const agent of currentAgents)
+                    crawlDelays[agent] = seconds;
+            }
+        }
+        else if (key === 'allow') {
+            collectingAgents = false;
+            // Allow lines don't add disallows; full precedence (RFC 9309) is v1-out-of-scope.
+        }
+    }
+    return { disallows, crawlDelays };
+}
+/**
+ * Disallow prefixes per lowercased user-agent ('*' fallback). Exported for testing.
+ */
+export function parseRobotsTxt(text) {
+    return parseGroups(text).disallows;
+}
+/**
+ * Crawl-delay (seconds) per lowercased user-agent ('*' fallback). Exported for testing.
+ */
+export function parseCrawlDelays(text) {
+    return parseGroups(text).crawlDelays;
+}
+function isDisallowed(rules, userAgent, path) {
+    const ua = userAgent.toLowerCase();
+    // Match either by full UA or by token prefix (most robots.txt match a
+    // bare product token like 'bluefields-fetcher').
+    const token = ua.split('/')[0]?.trim() ?? ua;
+    // RFC 9309: when a per-agent group matches, the wildcard group is
+    // NOT consulted. Only fall through to '*' when no per-agent group
+    // exists for this UA.
+    const perAgent = rules.disallows[ua] ?? rules.disallows[token];
+    if (perAgent !== undefined) {
+        return perAgent.some((p) => pathStartsWith(path, p));
+    }
+    const wildcard = rules.disallows['*'];
+    return wildcard?.some((p) => pathStartsWith(path, p)) ?? false;
+}
+/** Crawl-delay (seconds) for `userAgent`, per-agent overriding '*'. Null if none. */
+function crawlDelayFor(rules, userAgent) {
+    const ua = userAgent.toLowerCase();
+    const token = ua.split('/')[0]?.trim() ?? ua;
+    const perAgent = rules.crawlDelays[ua] ?? rules.crawlDelays[token];
+    if (perAgent !== undefined)
+        return perAgent;
+    return rules.crawlDelays['*'] ?? null;
+}
+function pathStartsWith(path, prefix) {
+    // robots.txt prefixes match by URL path prefix. We don't implement
+    // the full RFC 9309 wildcard syntax (* and $) — too much surface for
+    // v1; defaults to "literal prefix" semantics which most robots use.
+    return path.startsWith(prefix);
+}
+//# sourceMappingURL=robots-txt.js.map

package/dist/robots-txt.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"robots-txt.js","sourceRoot":"","sources":["../src/robots-txt.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,EAAE,mBAAmB,EAAE,6BAA6B,EAAE,MAAM,WAAW,CAAC;AAE/E,MAAM,GAAG,GAAG,iBAAiB,CAAC,SAAS,CAAC,CAAC;AAEzC,MAAM,YAAY,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,SAAS;AAC9C,MAAM,uBAAuB,GAAG,KAAK,CAAC;AAWtC;;;;GAIG;AACH,MAAM,OAAO,qBAAsB,SAAQ,KAAK;IAEzB;IADZ,MAAM,GAAG,mBAA4B,CAAC;IAC/C,YAAqB,GAAW;QAC9B,KAAK,CAAC,iCAAiC,GAAG,EAAE,CAAC,CAAC;QAD3B,QAAG,GAAH,GAAG,CAAQ;QAE9B,IAAI,CAAC,IAAI,GAAG,uBAAuB,CAAC;IACtC,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,KAAK,GAAG,IAAI,GAAG,EAAuB,CAAC;AAE7C,MAAM,UAAU,gBAAgB;IAC9B,KAAK,CAAC,KAAK,EAAE,CAAC;AAChB,CAAC;AAkBD;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,GAAW,EACX,SAAiB,EACjB,UAA4B,EAAE;IAE9B,MAAM,MAAM,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;IACjC,sEAAsE;IACtE,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,iBAAiB,EAAE,IAAI,EAAE,CAAC;IAE/D,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAC7B,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;IAC7C,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7D,IAAI,KAA8B,CAAC;IACnC,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC9B,IAAI,GAAG,IAAI,GAAG,GAAG,GAAG,CAAC,UAAU,GAAG,YAAY,EAAE,CAAC;YAC/C,KAAK,GAAG,GAAG,CAAC;QACd,CAAC;IACH,CAAC;IAED,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,GAAG,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;QAC5D,IAAI,CAAC,OAAO,CAAC,WAAW;YAAE,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IACrD,CAAC;IAED,MAAM,EAAE,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC;IACnC,OAAO;QACL,OAAO,EAAE,CAAC,YAAY,CAAC,KAAK,EAAE,EAAE,EAAE,IAAI,CAAC;QACvC,iBAAiB,EAAE,aAAa,CAAC,KAAK,EAAE,EAAE,CAAC;KAC5C,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,GAAW,EACX,SAAiB,EACjB,UAA4B,EAAE;IAE9B,OAAO,CAAC,MAAM,cAAc,CAAC,GAAG,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;AACjE,CAAC;AAED,2EAA2E;AAE3E,SAAS,YAAY,CAAC,CAAS;IAC7B,IAAI,CAAC;QACH,OAAO,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,MAAc,EACd,GAAW,EACX,SAAmC;IAEnC,MAAM,SAAS,GAAG,GAAG,MAAM,aAAa,CAAC;IAEzC,qEAAqE;IACrE,qEAAqE;IACrE,iBAAiB;IACjB,IAAI,CAAC;QACH,MAAM,6BAA6B,CAAC,SAAS,CAAC,CAAC;IACjD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,GAAG,YAAY,mBAAmB,EAAE,CAAC;YACvC,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC;QAC7D,CAAC;QACD,MAAM,GAAG,CAAC;IACZ,CAAC;IAED,MAAM,CAAC,GAAG,SAAS,IAAI,KAAK,CAAC;IAC7B,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,uBAAuB,CAAC,CAAC;IAE9E,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;QAC7E,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,+DAA+D;YAC/D,gEAAgE;YAChE,eAAe;YACf,GAAG,CAAC,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CACvC,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,EAClD,wCAAwC,CACzC,CAAC;YACF,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC;QAC7D,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACjC,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC;IAC3F,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,mEAAmE;QACnE,8DAA8D;QAC9D,gCAAgC;QAChC,GAAG,CAAC,IAAI,CACN,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAG,GAAa,CAAC,OAAO,EAAE,EAC3D,sCAAsC,CACvC,CAAC;QACF,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC;IAC7D,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,OAAO,CAAC,CAAC;IACxB,CAAC;AACH,CAAC;AAOD;;;;;;GAMG;AACH,SAAS,WAAW,CAAC,IAAY;IAC/B,MAAM,SAAS,GAA6B,EAAE,CAAC;IAC/C,MAAM,WAAW,GAA2B,EAAE,CAAC;IAC/C,IAAI,aAAa,GAAa,EAAE,CAAC;IACjC,8EAA8E;IAC9E,gFAAgF;IAChF,IAAI,gBAAgB,GAAG,KAAK,CAAC;IAE7B,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1C,6EAA6E;QAC7E,8CAA8C;QAC9C,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChC,aAAa,GAAG,EAAE,CAAC;YACnB,gBAAgB,GAAG,KAAK,CAAC;YACzB,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAChD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS,CAAC,+CAA+C;QAEhF,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,KAAK,KAAK,CAAC,CAAC;YAAE,SAAS;QAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACtD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAE3C,IAAI,GAAG,KAAK,YAAY,EAAE,CAAC;YACzB,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACtB,aAAa,GAAG,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;gBACtC,gBAAgB,GAAG,IAAI,CAAC;YAC1B,CAAC;iBAAM,CAAC;gBACN,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC;aAAM,IAAI,GAAG,KAAK,UAAU,EAAE,CAAC;YAC9B,gBAAgB,GAAG,KAAK,CAAC;YACzB,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YACzC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS,CAAC,mCAAmC;YACrE,KAAK,MAAM,KAAK,IAAI,aAAa,EAAE,CAAC;gBAClC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;oBAAE,SAAS,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;gBAC7C,SAAS,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;aAAM,IAAI,GAAG,KAAK,aAAa,EAAE,CAAC;YACjC,gBAAgB,GAAG,KAAK,CAAC;YACzB,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YACzC,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;YAC9B,0DAA0D;YAC1D,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,IAAI,CAAC,EAAE,CAAC;gBAC7C,KAAK,MAAM,KAAK,IAAI,aAAa;oBAAE,WAAW,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC;YAClE,CAAC;QACH,CAAC;aAAM,IAAI,GAAG,KAAK,OAAO,EAAE,CAAC;YAC3B,gBAAgB,GAAG,KAAK,CAAC;YACzB,kFAAkF;QACpF,CAAC;IACH,CAAC;IACD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC;AACpC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC;AACrC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC,WAAW,CAAC;AACvC,CAAC;AAED,SAAS,YAAY,CAAC,KAAkB,EAAE,SAAiB,EAAE,IAAY;IACvE,MAAM,EAAE,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC;IACnC,sEAAsE;IACtE,iDAAiD;IACjD,MAAM,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAE7C,kEAAkE;IAClE,kEAAkE;IAClE,sBAAsB;IACtB,MAAM,QAAQ,GAAG,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC/D,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC3B,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACvD,CAAC;IACD,MAAM,QAAQ,GAAG,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACtC,OAAO,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC;AACjE,CAAC;AAED,qFAAqF;AACrF,SAAS,aAAa,CAAC,KAAkB,EAAE,SAAiB;IAC1D,MAAM,EAAE,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC;IACnC,MAAM,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAC7C,MAAM,QAAQ,GAAG,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;IACnE,IAAI,QAAQ,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IAC5C,OAAO,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC;AACxC,CAAC;AAED,SAAS,cAAc,CAAC,IAAY,EAAE,MAAc;IAClD,mEAAmE;IACnE,qEAAqE;IACrE,oEAAoE;IACpE,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;AACjC,CAAC"}

package/dist/ssrf.d.ts

@@ -0,0 +1,77 @@
+/**
+ * SSRF-safe URL validation and HTTP client (Reviews #6, #7, #10).
+ *
+ * Every URL that crosses a trust boundary into our platform — customer
+ * webhook URLs, customer watch URLs, MCP scrape_url arguments — must pass
+ * through `validateUrlForExternalRequest` before we make any outbound
+ * request. The validator is the first line of defense; `createSsrfSafeHttpClient`
+ * is the second, catching DNS rebinding by re-validating at connect time.
+ *
+ * This module is reused widely. Two rules to keep in mind when extending it:
+ *   1. Default-deny. Add new categories to the deny side; don't allowlist exceptions.
+ *   2. Validate, don't sanitize. We reject bad URLs; we never massage them into
+ *      "fixed" URLs and proceed.
+ */
+import type { LookupOptions } from 'node:dns';
+import { Agent } from 'undici';
+export type SsrfRejectionReason = 'invalid_url' | 'unsupported_protocol' | 'metadata_endpoint' | 'internal_domain' | 'private_ip' | 'dns_failure';
+export declare class SsrfValidationError extends Error {
+    readonly reason: SsrfRejectionReason;
+    constructor(message: string, reason: SsrfRejectionReason);
+}
+/**
+ * Validate that `url` is safe to use for an outbound HTTPS request.
+ *
+ * Throws `SsrfValidationError` on any rejection. Callers should treat the
+ * error as a 4xx validation failure (not a 5xx server error).
+ */
+export declare function validateUrlForExternalRequest(url: string): Promise<void>;
+/**
+ * Returns true if `ip` is in any private, reserved, loopback, link-local,
+ * multicast, broadcast, documentation, or AWS-metadata range — anything we
+ * don't want our platform's egress to reach. Exported so tests can verify
+ * each category and other callers (e.g. attestation manifest integrity
+ * checks on `proxy_egress_ip`) can reuse the same predicate.
+ */
+export declare function isPrivateOrReservedIp(ip: string): boolean;
+/**
+ * Undici lookup callback that re-validates each candidate address at
+ * connect time. Exported for testability; created via
+ * `createSsrfSafeHttpClient` for production use.
+ *
+ * The signature matches Node's `dns.lookup` callback so undici can call it
+ * as its `connect.lookup`. We always do an `all: true` lookup internally so
+ * a multi-record DNS response can't slip a private IP past us by chance of
+ * record ordering — we only return a public one.
+ */
+export declare function ssrfSafeLookup(hostname: string, options: LookupOptions, callback: (err: NodeJS.ErrnoException | null, address: string, family: number) => void): void;
+/**
+ * Diagnostic helper — fetch a URL through an arbitrary HTTP proxy.
+ * Used by the /debug/diagnostics/proxy-test endpoint to verify
+ * provider reachability from inside the api container without going
+ * through Patchright/Chromium. Not for production traffic; the
+ * standard fetcher path uses ssrfSafeFetch.
+ *
+ * The target `url` is SSRF-validated up front (it can't go through the
+ * connect-time `ssrfSafeLookup`, since the dispatcher connects to the proxy,
+ * not the target). The proxy itself is operator-supplied via the admin-gated
+ * diagnostic, so it isn't validated here.
+ */
+export declare function proxyFetch(url: string, proxy: {
+    server: string;
+    username: string;
+    password: string;
+}, timeoutMs?: number): Promise<Response>;
+export declare function ssrfSafeFetch(url: string | URL, init?: RequestInit): Promise<Response>;
+/**
+ * Build an undici Agent that pins each connection to a public IP and
+ * times out at 10s. Use this dispatcher wherever the platform makes an
+ * outbound request on behalf of a customer (webhook delivery, fetcher
+ * Tier 0, MCP scrape_url backend).
+ *
+ * Redirects: undici re-runs the dispatcher (and thus our lookup) for each
+ * new host encountered, so a redirect to a private IP is rejected at
+ * connect time the same way the original URL would be. Callers should
+ * still pass `maxRedirections` explicitly per-request — the default is 0.
+ */
+export declare function createSsrfSafeHttpClient(): Agent;