@expandai/sdk 0.19.0 → 0.20.0

package/build/esm/Crawls.js

@@ -0,0 +1,786 @@
+import { BackoffDefaults, stream as durableStream, FetchError } from '@durable-streams/client';
+import { Duration } from 'effect';
+import { ExpandClientApiError, ExpandClientConnectionError, ExpandClientError, ExpandClientTimeoutError, } from './Error.js';
+import { isRetryable } from './Internal.js';
+import { MAX_RETRY_AFTER } from './RetryAfter.js';
+import { readStreamPage } from './StreamPage.js';
+export class CrawlClientError extends ExpandClientError {
+    code;
+    status;
+    retryable;
+    name = 'CrawlClientError';
+    constructor(message, code, status, retryable = false) {
+        super(message);
+        this.code = code;
+        this.status = status;
+        this.retryable = retryable;
+    }
+}
+/** API refusal with the Crawl error code and original response envelope. */
+export class CrawlClientApiError extends ExpandClientApiError {
+    code;
+    retryable;
+    constructor(message, code, status, retryable = false, body) {
+        super(message, { status, body });
+        this.code = code;
+        this.retryable = retryable;
+        this.name = 'CrawlClientApiError';
+    }
+}
+export class CrawlOffsetExpiredError extends CrawlClientApiError {
+    name = 'CrawlOffsetExpiredError';
+}
+function transportSignal(transport, signal) {
+    return transport.signal === undefined
+        ? signal
+        : AbortSignal.any([transport.signal, ...(signal === undefined ? [] : [signal])]);
+}
+/** Await one page, iterate individual items, or use pages() for bounded batches. */
+export class CrawlList {
+    fetchPage;
+    cursor;
+    signal;
+    firstPage;
+    constructor(fetchPage, cursor, signal) {
+        this.fetchPage = fetchPage;
+        this.cursor = cursor;
+        this.signal = signal;
+    }
+    first() {
+        if (this.signal?.aborted === true)
+            return Promise.reject(abortError(this.signal));
+        this.firstPage ??= this.fetchPage(this.cursor);
+        return this.firstPage;
+    }
+    then(onfulfilled, onrejected) {
+        return this.first().then(onfulfilled, onrejected);
+    }
+    async *pages() {
+        let page = await this.first();
+        const visited = new Set(this.cursor === undefined ? [] : [this.cursor]);
+        while (true) {
+            if (signalIsAborted(this.signal))
+                throw abortError(this.signal);
+            yield page;
+            if (page.nextCursor === undefined)
+                return;
+            if (visited.has(page.nextCursor))
+                throw new CrawlClientError('The server repeated a pagination cursor.');
+            visited.add(page.nextCursor);
+            page = await this.fetchPage(page.nextCursor);
+        }
+    }
+    async *[Symbol.asyncIterator]() {
+        for await (const page of this.pages())
+            for (const item of page.items) {
+                if (signalIsAborted(this.signal))
+                    throw abortError(this.signal);
+                yield item;
+            }
+    }
+    async collect(options) {
+        if (!Number.isSafeInteger(options.limit) || options.limit < 1)
+            throw new CrawlClientError('collect limit must be positive.');
+        const items = [];
+        for await (const item of this) {
+            items.push(item);
+            if (items.length >= options.limit)
+                break;
+        }
+        return items;
+    }
+}
+function projectionQuery(path, values) {
+    const query = new URLSearchParams();
+    for (const [key, value] of Object.entries(values))
+        if (value !== undefined)
+            query.set(key, String(value));
+    return query.size === 0 ? path : `${path}?${query}`;
+}
+async function projectionResponse(transport, path, options) {
+    const target = new URL(absoluteUrl(transport.baseUrl, path));
+    const base = new URL(transport.baseUrl);
+    if (target.origin !== base.origin ||
+        target.username !== '' ||
+        target.password !== '' ||
+        !target.pathname.startsWith('/v1/')) {
+        throw new CrawlClientError('Projection URLs must belong to the configured Expand API.');
+    }
+    const timeoutMs = options.timeoutMs ?? transport.timeoutMs ?? 60_000;
+    if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0)
+        throw new CrawlClientError('Projection timeoutMs must be a positive integer.');
+    const deadline = AbortSignal.timeout(timeoutMs);
+    const callerSignal = transportSignal(transport, options.signal);
+    const signal = callerSignal === undefined ? deadline : AbortSignal.any([callerSignal, deadline]);
+    if (signal.aborted)
+        throw abortError(signal);
+    const response = await (transport.fetch ?? globalThis.fetch)(target, {
+        headers: { 'x-expand-api-key': transport.apiKey },
+        signal,
+        redirect: 'error',
+    });
+    if (!response.ok)
+        throw await apiError(response);
+    return response;
+}
+async function projectionJson(transport, path, options) {
+    // Mirrors the typed API contract, like requestJson; arbitrary body reads use unknown.
+    return (await (await projectionResponse(transport, path, options)).json());
+}
+export class CrawlCaptureResource {
+    id;
+    transport;
+    constructor(id, transport) {
+        this.id = id;
+        this.transport = transport;
+    }
+    get(options = {}) {
+        return projectionJson(this.transport, `/v1/captures/${encodeURIComponent(this.id)}`, options);
+    }
+    stateJson(options = {}) {
+        return projectionJson(this.transport, `/v1/captures/${encodeURIComponent(this.id)}/state-json`, options);
+    }
+    evidence(key, options = {}) {
+        return projectionJson(this.transport, `/v1/captures/${encodeURIComponent(this.id)}/state-json/${encodeURIComponent(key)}`, options);
+    }
+    async markdown(options = {}) {
+        return (await projectionResponse(this.transport, `/v1/captures/${encodeURIComponent(this.id)}/markdown`, options)).text();
+    }
+    async html(options = {}) {
+        return (await projectionResponse(this.transport, `/v1/captures/${encodeURIComponent(this.id)}/html`, options)).text();
+    }
+    playgroundUrl() {
+        return `https://expand.land/s/${encodeURIComponent(this.id)}`;
+    }
+}
+export class CrawlPageResource {
+    id;
+    transport;
+    constructor(id, transport) {
+        this.id = id;
+        this.transport = transport;
+    }
+    versions(options = {}) {
+        return new CrawlList(async (before) => {
+            const result = await projectionJson(this.transport, projectionQuery(`/v1/pages/${encodeURIComponent(this.id)}/versions`, {
+                before,
+                at: options.at,
+                limit: options.limit,
+            }), options);
+            return { items: result.versions, nextCursor: options.at === undefined ? result.nextCursor : undefined };
+        }, options.before, transportSignal(this.transport, options.signal));
+    }
+    diff(options) {
+        return projectionJson(this.transport, projectionQuery(`/v1/pages/${encodeURIComponent(this.id)}/diff`, { from: options.from, to: options.to }), options);
+    }
+}
+/** Shares the Crawl transport without exposing its API key to arbitrary body URLs. */
+export function makeCrawlProjectionsClient(transport) {
+    return {
+        captures(id) {
+            if (id.length === 0)
+                throw new CrawlClientError('Capture id must not be empty.');
+            return new CrawlCaptureResource(id, transport);
+        },
+        pages(id) {
+            if (id.length === 0)
+                throw new CrawlClientError('Page id must not be empty.');
+            return new CrawlPageResource(id, transport);
+        },
+        fetchJson(url, options = {}) {
+            return projectionJson(transport, url, options);
+        },
+    };
+}
+function requireIdempotencyKey(value, operation) {
+    if (value === undefined || value.length < 1 || value.length > 255) {
+        throw new CrawlClientError(`${operation} requires an idempotencyKey between 1 and 255 characters.`);
+    }
+    return value;
+}
+function isRecord(value) {
+    return typeof value === 'object' && value !== null;
+}
+async function apiError(response) {
+    let body;
+    try {
+        body = await response.json();
+    }
+    catch {
+        // A status and status text are still useful when a proxy returned non-JSON.
+    }
+    const error = isRecord(body) && isRecord(body['error']) ? body['error'] : undefined;
+    const code = typeof error?.['code'] === 'string' ? error['code'] : undefined;
+    const message = typeof error?.['message'] === 'string' ? error['message'] : response.statusText;
+    const retryable = error?.['retryable'] === true;
+    return response.status === 410
+        ? new CrawlOffsetExpiredError(message, code, response.status, retryable, body)
+        : new CrawlClientApiError(message, code, response.status, retryable, body);
+}
+function absoluteUrl(baseUrl, path) {
+    return new URL(path, `${baseUrl.replace(/\/+$/u, '')}/`).toString();
+}
+const CRAWL_STREAM_REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
+// durable-streams 0.2.6 cannot interrupt its retry timer. Retries belong to
+// the transport below, which keeps both setup and backoff tied to the caller.
+const CRAWL_STREAM_BACKOFF = { ...BackoffDefaults, maxRetries: 0 };
+// The same ceiling the Effect client applies to `Retry-After`: a live iterator must not sleep
+// for an hour because one response asked for it.
+const MAX_RETRY_AFTER_MS = Duration.toMillis(MAX_RETRY_AFTER);
+function retryAfterMilliseconds(error) {
+    if (!(error instanceof FetchError))
+        return 0;
+    const value = error.headers['retry-after'];
+    if (value === undefined)
+        return 0;
+    const seconds = Number(value);
+    if (Number.isFinite(seconds) && seconds > 0)
+        return Math.min(seconds * 1000, MAX_RETRY_AFTER_MS);
+    const date = Date.parse(value);
+    return Number.isFinite(date) ? Math.max(0, Math.min(date - Date.now(), MAX_RETRY_AFTER_MS)) : 0;
+}
+function isDeadlineReason(reason) {
+    return reason instanceof Error && reason.name === 'TimeoutError';
+}
+function crawlTimeoutMs(transport) {
+    const timeoutMs = transport.timeoutMs ?? 60_000;
+    if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0 || timeoutMs > 2_147_483_647)
+        throw new CrawlClientError('Crawl timeoutMs must be a positive integer no greater than 2147483647.');
+    return timeoutMs;
+}
+function streamError(error, timeoutMs) {
+    if (error instanceof ExpandClientError)
+        return error;
+    if (error instanceof FetchError) {
+        const body = error.json ?? error.text;
+        const envelope = isRecord(body) && isRecord(body['error']) ? body['error'] : undefined;
+        const code = typeof envelope?.['code'] === 'string' ? envelope['code'] : undefined;
+        const message = typeof envelope?.['message'] === 'string' ? envelope['message'] : error.message;
+        const retryable = envelope?.['retryable'] === true ||
+            (envelope?.['retryable'] === undefined && (error.status === 408 || error.status === 429 || error.status >= 500));
+        return error.status === 410
+            ? new CrawlOffsetExpiredError(message, code ?? 'offset_expired', error.status, retryable, body)
+            : new CrawlClientApiError(message, code, error.status, retryable, body);
+    }
+    if (error instanceof Error && error.name === 'TimeoutError')
+        return new ExpandClientTimeoutError(`Request timed out after ${timeoutMs}ms`, { timeoutMs, cause: error });
+    return new ExpandClientConnectionError(error instanceof Error ? error.message : 'Crawl request failed.', {
+        cause: error,
+    });
+}
+function createCrawlStreamRequest(transport, callerSignal, live = false) {
+    const redirect = new AbortController();
+    const execute = transport.fetch ?? globalThis.fetch;
+    const timeoutMs = crawlTimeoutMs(transport);
+    const maxRetries = transport.maxRetries ?? 2;
+    if (!Number.isSafeInteger(maxRetries) || maxRetries < 0)
+        throw new CrawlClientError('Crawl maxRetries must be a non-negative integer.');
+    const signal = AbortSignal.any([
+        redirect.signal,
+        ...(callerSignal === undefined ? [] : [callerSignal]),
+        ...(live ? [] : [AbortSignal.timeout(timeoutMs)]),
+    ]);
+    return {
+        signal,
+        redirectSignal: redirect.signal,
+        fetch: async (input, init) => {
+            const requestSignal = init?.signal == null ? signal : AbortSignal.any([signal, init.signal]);
+            let delay = BackoffDefaults.initialDelay;
+            let retries = 0;
+            while (true) {
+                if (requestSignal.aborted)
+                    throw abortError(requestSignal);
+                try {
+                    const response = await execute(input, { ...init, signal: requestSignal, redirect: 'manual' });
+                    if (CRAWL_STREAM_REDIRECT_STATUSES.has(response.status) || response.type === 'opaqueredirect') {
+                        const error = new CrawlClientError('Authenticated Crawl requests cannot redirect.', 'redirect_not_allowed', response.status || undefined, false);
+                        // Keep policy failures out of transport and iterator reconnect loops.
+                        redirect.abort(error);
+                        try {
+                            await response.body?.cancel();
+                        }
+                        catch {
+                            // Aborting the request may already have canceled its body.
+                        }
+                        throw error;
+                    }
+                    if (!response.ok)
+                        throw await FetchError.fromResponse(response, String(input));
+                    return response;
+                }
+                catch (error) {
+                    if (requestSignal.aborted)
+                        throw abortError(requestSignal);
+                    if (!isRetryable(streamError(error, timeoutMs), timeoutMs))
+                        throw error;
+                    if (!live && retries >= maxRetries)
+                        throw error;
+                    retries += 1;
+                    try {
+                        await abortableDelay(Math.max(retryAfterMilliseconds(error), Math.random() * delay), requestSignal);
+                    }
+                    catch (abort) {
+                        // The request deadline expired while waiting to retry. The refusal being retried
+                        // explains the failure; a timeout is reserved for an attempt that was in flight.
+                        if (!live && isDeadlineReason(requestSignal.reason))
+                            throw streamError(error, timeoutMs);
+                        throw abort;
+                    }
+                    delay = Math.min(delay * BackoffDefaults.multiplier, BackoffDefaults.maxDelay);
+                }
+            }
+        },
+    };
+}
+async function requestJson(transport, path, init = {}) {
+    const timeoutMs = crawlTimeoutMs(transport);
+    const callerSignal = transportSignal(transport, init.signal ?? undefined);
+    const request = createCrawlStreamRequest(transport, callerSignal);
+    const signal = request.signal;
+    if (signal.aborted)
+        throw abortError(signal);
+    try {
+        const response = await request.fetch(absoluteUrl(transport.baseUrl, path), {
+            ...init,
+            redirect: 'error',
+            signal,
+            headers: {
+                'x-expand-api-key': transport.apiKey,
+                ...(init.body === undefined ? {} : { 'content-type': 'application/json' }),
+                ...init.headers,
+            },
+        });
+        if (!response.ok)
+            throw await apiError(response);
+        // The SDK's public response interfaces mirror the versioned HTTP contract.
+        return (await response.json());
+    }
+    catch (error) {
+        // A typed transport failure already names the cause, even when the deadline has since fired.
+        if (error instanceof ExpandClientError)
+            throw error;
+        throw streamError(signal.aborted ? abortError(signal) : error, timeoutMs);
+    }
+}
+function abortError(signal) {
+    return signal?.reason instanceof Error ? signal.reason : new CrawlClientError('The Crawl operation was aborted.');
+}
+function abortableDelay(milliseconds, signal) {
+    return new Promise((resolve, reject) => {
+        if (signal?.aborted === true) {
+            reject(abortError(signal));
+            return;
+        }
+        const onAbort = () => {
+            clearTimeout(timer);
+            signal?.removeEventListener('abort', onAbort);
+            reject(abortError(signal));
+        };
+        const timer = setTimeout(() => {
+            signal?.removeEventListener('abort', onAbort);
+            resolve();
+        }, milliseconds);
+        signal?.addEventListener('abort', onAbort, { once: true });
+    });
+}
+function signalIsAborted(signal) {
+    return signal?.aborted === true;
+}
+const CRAWL_ITERATOR_RETURN = Symbol('crawl-iterator-return');
+const CRAWL_ITERATOR_ABORT = Symbol('crawl-iterator-abort');
+function iteratorWasCanceled(signal) {
+    return signal?.reason === CRAWL_ITERATOR_RETURN || signal?.reason === CRAWL_ITERATOR_ABORT;
+}
+class CrawlIterable {
+    factory;
+    externalSignal;
+    constructor(factory, externalSignal) {
+        this.factory = factory;
+        this.externalSignal = externalSignal;
+    }
+    [Symbol.asyncIterator]() {
+        const controller = new AbortController();
+        const abort = () => controller.abort(CRAWL_ITERATOR_ABORT);
+        if (this.externalSignal?.aborted === true)
+            abort();
+        else
+            this.externalSignal?.addEventListener('abort', abort, { once: true });
+        const iterator = this.factory(controller.signal);
+        const cleanup = () => this.externalSignal?.removeEventListener('abort', abort);
+        return {
+            next: async () => {
+                try {
+                    const result = await iterator.next();
+                    if (result.done)
+                        cleanup();
+                    return result;
+                }
+                catch (error) {
+                    cleanup();
+                    if (iteratorWasCanceled(controller.signal))
+                        return { done: true, value: undefined };
+                    throw error;
+                }
+            },
+            return: async (value) => {
+                controller.abort(CRAWL_ITERATOR_RETURN);
+                cleanup();
+                return (await iterator.return?.(value)) ?? { done: true, value: undefined };
+            },
+            throw: async (error) => {
+                controller.abort(error);
+                cleanup();
+                if (iterator.throw === undefined)
+                    throw error;
+                return iterator.throw(error);
+            },
+        };
+    }
+    async collect(options) {
+        if (!Number.isSafeInteger(options.limit) || options.limit < 1)
+            throw new CrawlClientError('collect limit must be positive.');
+        const events = [];
+        for await (const event of this) {
+            events.push(event);
+            if (events.length >= options.limit)
+                break;
+        }
+        return events;
+    }
+}
+export class CrawlStreamPageIterable extends CrawlIterable {
+    then(onfulfilled, onrejected) {
+        const first = async () => {
+            const iterator = this[Symbol.asyncIterator]();
+            try {
+                const result = await iterator.next();
+                return result.done ? undefined : result.value;
+            }
+            finally {
+                await iterator.return?.();
+            }
+        };
+        return first().then(onfulfilled, onrejected);
+    }
+}
+export class CrawlStreamIterable extends CrawlIterable {
+    pageFactory;
+    constructor(factory, pageFactory, signal) {
+        super(factory, signal);
+        this.pageFactory = pageFactory;
+    }
+    pages() {
+        return new CrawlStreamPageIterable(this.pageFactory, this.externalSignal);
+    }
+    then(onfulfilled, onrejected) {
+        return this.pages().then(onfulfilled, onrejected);
+    }
+}
+export class CrawlEventPageIterable extends CrawlStreamPageIterable {
+}
+export class CrawlEventIterable extends CrawlStreamIterable {
+}
+export class CrawlActivityPageIterable extends CrawlStreamPageIterable {
+}
+export class CrawlActivityIterable extends CrawlStreamIterable {
+}
+const PAGE_STREAM = { path: 'events' };
+const ACTIVITY_STREAM = { path: 'activity' };
+export class CrawlResource {
+    id;
+    transport;
+    constructor(id, transport) {
+        this.id = id;
+        this.transport = transport;
+    }
+    events(options = {}) {
+        return new CrawlEventIterable((signal) => this.readStream(PAGE_STREAM, { ...options, signal }), (signal) => this.readStreamPages(PAGE_STREAM, { ...options, signal }), transportSignal(this.transport, options.signal));
+    }
+    /**
+     * The activity stream: batches of URL transitions without content, for progress and
+     * operator views. Same shape as `events`; a one-off's iteration ends on stream closure.
+     */
+    activity(options = {}) {
+        return new CrawlActivityIterable((signal) => this.readStream(ACTIVITY_STREAM, { ...options, signal }), (signal) => this.readStreamPages(ACTIVITY_STREAM, { ...options, signal }), transportSignal(this.transport, options.signal));
+    }
+    /** A pinned live-catalog snapshot. Resume /events from the final page's nextOffset. */
+    snapshot(options = {}) {
+        return new CrawlEventIterable((signal) => this.readSnapshotItems({ ...options, signal }), (signal) => this.readSnapshotPages({ ...options, signal }), transportSignal(this.transport, options.signal));
+    }
+    async *readSnapshotItems(options) {
+        for await (const page of this.readSnapshotPages(options)) {
+            for (const event of page.items)
+                yield event;
+        }
+    }
+    async *readSnapshotPages(options) {
+        let offset = options.from;
+        while (!signalIsAborted(options.signal)) {
+            const page = await projectionJson(this.transport, projectionQuery(`/v1/crawls/${encodeURIComponent(this.id)}/snapshot`, { offset }), options);
+            yield {
+                crawlId: this.id,
+                items: page.events,
+                nextOffset: page.nextOffset,
+                upToDate: page.upToDate,
+                streamClosed: false,
+            };
+            if (page.upToDate)
+                return;
+            if (page.nextOffset === offset)
+                throw new CrawlClientError('Snapshot cursor did not advance.');
+            offset = page.nextOffset;
+        }
+    }
+    async *readStreamPages(spec, options) {
+        let offset = options.from ?? '-1';
+        let caughtUp = false;
+        let cursor;
+        let recurring;
+        while (!signalIsAborted(options.signal)) {
+            const url = new URL(absoluteUrl(this.transport.baseUrl, `/v1/crawls/${encodeURIComponent(this.id)}/${spec.path}`));
+            if (caughtUp)
+                url.searchParams.set('live', 'long-poll');
+            if (cursor !== undefined)
+                url.searchParams.set('cursor', cursor);
+            const request = createCrawlStreamRequest(this.transport, options.signal, options.live === true);
+            let page;
+            try {
+                const response = await durableStream({
+                    url: url.toString(),
+                    headers: { 'x-expand-api-key': this.transport.apiKey },
+                    offset,
+                    live: false,
+                    json: true,
+                    signal: request.signal,
+                    fetch: request.fetch,
+                    backoffOptions: CRAWL_STREAM_BACKOFF,
+                });
+                page = await readStreamPage(response);
+                if (request.redirectSignal.aborted)
+                    throw abortError(request.redirectSignal);
+            }
+            catch (error) {
+                if (iteratorWasCanceled(options.signal))
+                    return;
+                if (request.redirectSignal.aborted)
+                    throw abortError(request.redirectSignal);
+                const retryable = isRetryable(streamError(error, crawlTimeoutMs(this.transport)), crawlTimeoutMs(this.transport));
+                if (options.live === true && retryable) {
+                    await abortableDelay(500, options.signal);
+                    continue;
+                }
+                throw streamError(request.signal.aborted ? abortError(request.signal) : error, crawlTimeoutMs(this.transport));
+            }
+            if (signalIsAborted(options.signal))
+                return;
+            offset = page.offset;
+            cursor = page.cursor;
+            caughtUp ||= page.upToDate;
+            yield {
+                crawlId: this.id,
+                items: page.items,
+                nextOffset: offset,
+                upToDate: page.upToDate,
+                streamClosed: page.streamClosed,
+            };
+            if (page.streamClosed)
+                return;
+            if (!page.upToDate || options.live === true)
+                continue;
+            // Recurring reads default to catch-up. One-off completion is signaled only by EOF;
+            // offsets are opaque and are never compared to an out-of-band watermark.
+            if (recurring === undefined)
+                recurring = (await this.status({ signal: options.signal })).sync !== undefined;
+            if (recurring)
+                return;
+        }
+    }
+    pages(options = {}) {
+        return new CrawlList(async (cursor) => {
+            const result = await projectionJson(this.transport, projectionQuery(`/v1/crawls/${encodeURIComponent(this.id)}/pages`, {
+                cursor,
+                limit: options.limit,
+                prefix: options.prefix,
+                status: options.status,
+                changedSince: options.changedSince,
+            }), options);
+            return { items: result.pages, nextCursor: result.nextCursor };
+        }, options.cursor, transportSignal(this.transport, options.signal));
+    }
+    /**
+     * Every discovered URL in any admission state, cursor-paginated and filterable by state,
+     * source and a URL substring (`search`, matched literally and case-insensitively across the
+     * whole crawl). The first page's `activityOffset` is where to start an activity tail that
+     * keeps the snapshot current: apply an event to a URL only when `event.offset > url.activitySeq`.
+     */
+    urls(options = {}) {
+        return new CrawlList(async (cursor) => {
+            const result = await projectionJson(this.transport, projectionQuery(`/v1/crawls/${encodeURIComponent(this.id)}/urls`, {
+                cursor,
+                limit: options.limit,
+                state: options.state,
+                source: options.source,
+                search: options.search,
+            }), options);
+            return { items: result.urls, nextCursor: result.nextCursor };
+        }, options.cursor, transportSignal(this.transport, options.signal));
+    }
+    /** One page of the discovered-URLs view with its counts and activity offset, for callers that need the envelope. */
+    urlsPage(options = {}) {
+        return projectionJson(this.transport, projectionQuery(`/v1/crawls/${encodeURIComponent(this.id)}/urls`, {
+            cursor: options.cursor,
+            limit: options.limit,
+            state: options.state,
+            source: options.source,
+            search: options.search,
+        }), options);
+    }
+    page(id, options = {}) {
+        return projectionJson(this.transport, `/v1/crawls/${encodeURIComponent(this.id)}/pages/${encodeURIComponent(id)}`, options);
+    }
+    pageByUrl(url, options = {}) {
+        return projectionJson(this.transport, projectionQuery(`/v1/crawls/${encodeURIComponent(this.id)}/pages/by-url`, { url }), options);
+    }
+    async *readStream(spec, options) {
+        for await (const page of this.readStreamPages(spec, options)) {
+            for (const event of page.items) {
+                if (signalIsAborted(options.signal))
+                    return;
+                yield event;
+            }
+        }
+    }
+    status(options = {}) {
+        return requestJson(this.transport, `/v1/crawls/${encodeURIComponent(this.id)}`, {
+            signal: options.signal,
+        });
+    }
+    async done(options = {}) {
+        if (options.pollIntervalMs !== undefined &&
+            (!Number.isFinite(options.pollIntervalMs) || options.pollIntervalMs > 2_147_483_647))
+            throw new CrawlClientError('Crawl pollIntervalMs must be finite and no greater than 2147483647.');
+        const interval = Math.max(50, options.pollIntervalMs ?? 500);
+        const signal = transportSignal(this.transport, options.signal);
+        while (!signalIsAborted(signal)) {
+            const status = await this.status({ signal });
+            if (signalIsAborted(signal))
+                break;
+            if (status.state === 'done' ||
+                status.state === 'failed' ||
+                status.state === 'canceled' ||
+                (status.sync !== undefined && status.progress.queued === 0 && status.progress.inFlight === 0)) {
+                return { lastOffset: status.offset, status };
+            }
+            await abortableDelay(interval, signal);
+        }
+        const reason = signal?.reason;
+        throw reason instanceof Error ? reason : new CrawlClientError('Crawl.done was aborted.');
+    }
+    update(patch) {
+        return requestJson(this.transport, `/v1/crawls/${encodeURIComponent(this.id)}`, {
+            method: 'PATCH',
+            body: JSON.stringify(patch),
+        });
+    }
+    cancel() {
+        return requestJson(this.transport, `/v1/crawls/${encodeURIComponent(this.id)}/cancel`, {
+            method: 'POST',
+        });
+    }
+    /** Cancels the crawl if it is live and hides it from `list()`. It is deleted 90 days later unless unarchived. */
+    archive() {
+        return requestJson(this.transport, `/v1/crawls/${encodeURIComponent(this.id)}/archive`, {
+            method: 'POST',
+        });
+    }
+    /** Returns an archived crawl to `list()`, still canceled, and keeps it for at least 90 more days. */
+    unarchive() {
+        return requestJson(this.transport, `/v1/crawls/${encodeURIComponent(this.id)}/unarchive`, {
+            method: 'POST',
+        });
+    }
+    run(options) {
+        const idempotencyKey = requireIdempotencyKey(options?.idempotencyKey, 'Crawl.run');
+        return requestJson(this.transport, `/v1/crawls/${encodeURIComponent(this.id)}/runs${options.replayOnly ? '/replay' : ''}`, {
+            method: 'POST',
+            headers: {
+                'x-idempotency-key': idempotencyKey,
+            },
+        });
+    }
+}
+export class MergedCrawlResource {
+    crawls;
+    constructor(crawls) {
+        this.crawls = crawls;
+    }
+    events(options = {}) {
+        if (typeof options.from === 'string')
+            throw new CrawlClientError('Merged Crawl reads require an offset map keyed by Crawl id.');
+        return new CrawlEventIterable((signal) => this.readItems({ ...options, signal }, (events) => events), (signal) => this.readItems({ ...options, signal }, (events) => events.pages()), options.signal);
+    }
+    async *readItems(options, select) {
+        if (signalIsAborted(options.signal))
+            return;
+        const controllers = this.crawls.map(() => new AbortController());
+        const abortChildren = () => {
+            for (const controller of controllers)
+                controller.abort(options.signal?.reason);
+        };
+        options.signal?.addEventListener('abort', abortChildren, { once: true });
+        const iterators = this.crawls.map((crawl, index) => select(crawl.events({ from: options.from?.[crawl.id], live: options.live, signal: controllers[index]?.signal }))[Symbol.asyncIterator]());
+        const pending = new Map();
+        for (const [index, iterator] of iterators.entries()) {
+            pending.set(index, iterator.next().then((result) => ({ index, result })));
+        }
+        try {
+            while (pending.size > 0) {
+                const next = await Promise.race(pending.values());
+                if (next.result.done) {
+                    pending.delete(next.index);
+                    continue;
+                }
+                yield next.result.value;
+                const iterator = iterators[next.index];
+                if (iterator !== undefined)
+                    pending.set(next.index, iterator.next().then((result) => ({ index: next.index, result })));
+            }
+        }
+        finally {
+            abortChildren();
+            options.signal?.removeEventListener('abort', abortChildren);
+            await Promise.allSettled(iterators.map((iterator) => iterator.return?.(undefined)));
+        }
+    }
+}
+export function makeCrawlsClient(options) {
+    function byId(id) {
+        if (id.length === 0)
+            throw new CrawlClientError('Crawl id must not be empty.');
+        return new CrawlResource(id, options);
+    }
+    return Object.assign(byId, {
+        async create(request, createOptions) {
+            const idempotencyKey = requireIdempotencyKey(createOptions?.idempotencyKey, 'Crawls.create');
+            const created = await requestJson(options, '/v1/crawls', {
+                method: 'POST',
+                headers: { 'x-idempotency-key': idempotencyKey },
+                body: JSON.stringify(request),
+            });
+            return created.map(({ id }) => new CrawlResource(id, options));
+        },
+        list(listOptions = {}) {
+            const params = new URLSearchParams();
+            if (listOptions.limit !== undefined)
+                params.set('limit', String(listOptions.limit));
+            if (listOptions.archived !== undefined)
+                params.set('archived', String(listOptions.archived));
+            const query = params.size === 0 ? '' : `?${params.toString()}`;
+            return requestJson(options, `/v1/crawls${query}`, { signal: listOptions.signal });
+        },
+        merge(crawls) {
+            return new MergedCrawlResource(crawls);
+        },
+    });
+}
+//# sourceMappingURL=Crawls.js.map