@expandai/sdk 0.19.0 → 0.20.0

package/src/Crawls.ts

@@ -0,0 +1,1387 @@
+import { BackoffDefaults, stream as durableStream, FetchError, type JsonBatch } from '@durable-streams/client'
+import { Duration } from 'effect'
+import {
+	ExpandClientApiError,
+	ExpandClientConnectionError,
+	ExpandClientError,
+	ExpandClientTimeoutError,
+} from './Error.js'
+import { isRetryable } from './Internal.js'
+import { MAX_RETRY_AFTER } from './RetryAfter.js'
+import { readStreamPage } from './StreamPage.js'
+
+export type CrawlOffset = string
+export type CrawlId = string
+export type CrawlPageId = string
+
+export type CrawlJsonValue =
+	| string
+	| number
+	| boolean
+	| null
+	| readonly CrawlJsonValue[]
+	| { readonly [key: string]: CrawlJsonValue }
+
+export interface CrawlSource {
+	readonly url: string
+	readonly kind?: 'seed' | 'sitemap'
+	/** Which pages end up in the results. Non-matching pages may still be opened to follow their links. */
+	readonly include?: readonly string[]
+	/** Pages the crawler never opens. */
+	readonly exclude?: readonly string[]
+	readonly maxPages?: number
+	/**
+	 * Clicks from `url`: 0 is the URL alone, 1 adds the pages it links to, the site's sitemap and
+	 * search results. A sitemap source's own listings count as depth 0. Defaults to 50.
+	 */
+	readonly maxDepth?: number
+}
+
+export interface CrawlChangePolicy {
+	readonly ignore?: readonly string[]
+	readonly volatile?: readonly string[]
+}
+
+export interface CrawlSyncPolicy {
+	/** How often every page is checked for changes, such as `24h` or `7d`. */
+	readonly freshness: string
+	readonly budget?: { readonly rendersPerDay?: number }
+}
+
+export interface CrawlRequest {
+	readonly sources: readonly (string | CrawlSource)[]
+	readonly formats?: readonly ('markdown' | 'stateJson' | 'html')[]
+	readonly maxPages?: number
+	readonly changePolicy?: CrawlChangePolicy
+	readonly sync?: CrawlSyncPolicy
+	/** A label for the crawl, 1–120 characters. Every crawl the request creates gets it. */
+	readonly name?: string
+}
+
+export interface CrawlPageMeta {
+	readonly title?: string
+	readonly description?: string
+	readonly canonicalUrl?: string
+	readonly language?: string
+	readonly pageType: string
+	readonly author?: string
+	readonly publishedAt?: string
+	readonly updatedAt?: string
+	readonly image?: string
+	readonly openGraph?: Readonly<Record<string, string>>
+	readonly twitter?: Readonly<Record<string, string>>
+	/**
+	 * Present when the capture behind this page ran at reduced fidelity, naming what was
+	 * reduced (`bestEffort`, `skipped-modelInputTooLarge`, `hfc-local-fallback`, …). The
+	 * page and its content are real; the field says how much of the extraction ran.
+	 */
+	readonly reducedCapture?: string
+}
+
+export interface CrawlEvidence {
+	readonly key: string
+	readonly kind: 'inline' | 'network'
+	readonly role: string
+	/** Evidence relevance weight. Higher means more relevant; not a probability, so it can exceed 1. */
+	readonly score: number
+	readonly url?: string
+	readonly method?: string
+	readonly body?: CrawlJsonValue
+	readonly bodyUrl?: string
+	readonly approxTokens?: number
+	readonly keys?: readonly string[]
+}
+
+export interface CrawlStateJson {
+	readonly captureId: string
+	readonly evidence: readonly CrawlEvidence[]
+}
+
+export interface CrawlPageContent {
+	readonly markdown?: string
+	readonly stateJson?: CrawlStateJson
+	readonly html?: string
+	readonly truncated: readonly ('markdown' | 'stateJson' | 'html')[]
+}
+
+interface CrawlEventBase {
+	readonly offset: CrawlOffset
+	readonly id: string
+	readonly observedAt: string
+	readonly crawlId: CrawlId
+}
+
+export interface CrawlPageAdded extends CrawlEventBase {
+	readonly type: 'page.added'
+	readonly pageId: CrawlPageId
+	readonly url: string
+	readonly captureId: string
+	readonly discoveredBy: 'seed' | 'sitemap' | 'link' | 'search'
+	readonly depth: number
+	readonly meta: CrawlPageMeta
+	readonly page: CrawlPageContent
+}
+
+export interface CrawlPageChanged extends CrawlEventBase {
+	readonly type: 'page.changed'
+	readonly pageId: CrawlPageId
+	readonly url: string
+	readonly captureId: string
+	readonly previousCaptureId: string
+	readonly changed: { readonly content: boolean; readonly stateJson: boolean; readonly extraction: boolean }
+	readonly diff: {
+		readonly stateJson: Readonly<Record<string, { readonly from?: CrawlJsonValue; readonly to?: CrawlJsonValue }>>
+		readonly markdown: string
+		readonly truncated: boolean
+	}
+	readonly meta: CrawlPageMeta
+	readonly page: CrawlPageContent
+}
+
+export interface CrawlPageRemoved extends CrawlEventBase {
+	readonly type: 'page.removed'
+	readonly pageId: CrawlPageId
+	readonly url: string
+	readonly previousCaptureId: string
+	readonly evidence: {
+		readonly kind: 'http-404' | 'http-410' | 'content-gone' | 'redirected-away'
+		readonly checks: readonly {
+			readonly captureId: string
+			readonly observedAt: string
+			readonly status: number
+			readonly finalUrl: string
+		}[]
+		readonly sitemapDisappeared: boolean
+	}
+}
+
+export interface CrawlPageFailed extends CrawlEventBase {
+	readonly type: 'page.failed'
+	readonly url: string
+	readonly reason:
+		| 'blocked'
+		| 'not-found'
+		| 'timeout'
+		| 'error'
+		| 'robots-disallowed'
+		| 'robots-unreachable'
+		| 'out-of-scope'
+	/** The precise cause behind `reason`, when the check knew one (`quality-degraded`, `http-503`, …). */
+	readonly detail?: string
+	readonly retriesExhausted: boolean
+}
+
+export type CrawlEvent = CrawlPageAdded | CrawlPageChanged | CrawlPageRemoved | CrawlPageFailed
+
+/**
+ * One batch of URL transitions on the activity stream (`GET /v1/crawls/:id/activity`): what was
+ * discovered, admitted, claimed, rendered or deferred, which sitemaps and robots files resolved,
+ * and the run lifecycle. No content; the page stream (`events`) carries that. `eventsOffset` is a
+ * lower bound on the page stream: every page event at or below it committed before this one.
+ */
+export interface CrawlActivityEvent {
+	readonly offset: CrawlOffset
+	readonly id: string
+	readonly crawlId: CrawlId
+	readonly runId: string | null
+	/** ISO date-time. */
+	readonly observedAt: string
+	/** `discovery.batch`, `admission.batch`, `claim.batch`, `render.settled`, `sitemap.fetched`, `sitemap.failed`, `robots.resolved`, `search.resolved`, `alias.batch`, `run.started`, `run.discovery-closed`, `run.finished`, `run.canceled`, … */
+	readonly type: string
+	readonly eventsOffset: CrawlOffset
+	readonly payload: Record<string, unknown>
+}
+
+export type CrawlAdmissionReason = 'no-fleet-snapshot' | 'no-fleet-capacity' | 'billing-blocked' | 'disabled' | 'none'
+
+export interface CrawlAdmission {
+	readonly paused: boolean
+	readonly reason: CrawlAdmissionReason
+}
+
+/** The canonical source configuration a crawl was created with. */
+export interface CrawlConfigSummary {
+	readonly sources: readonly {
+		readonly url: string
+		readonly kind: 'seed' | 'sitemap'
+		readonly include: readonly string[]
+		readonly exclude: readonly string[]
+		readonly maxPages?: number
+		readonly maxDepth: number
+	}[]
+	/** How the include and exclude entries are read: RE2 searched in the path, or path globs. */
+	readonly patternSyntax: 're2' | 'glob'
+	readonly formats: readonly ('markdown' | 'stateJson' | 'html')[]
+	readonly maxPages: number
+	readonly changePolicy: { readonly ignore: readonly string[]; readonly volatile: readonly string[] }
+}
+
+/** The most recent run of a crawl, whatever its outcome. */
+export interface CrawlRunSummary {
+	readonly kind: 'INITIAL' | 'MANUAL' | 'DISCOVERY'
+	readonly status: 'RUNNING' | 'DONE' | 'FAILED' | 'CANCELED'
+	/** ISO date-time. */
+	readonly startedAt: string
+	/** ISO date-time; absent while the run is still going. */
+	readonly completedAt?: string
+	/** `stalled` when the run was stopped because nothing settled for the stall timeout (an hour). */
+	readonly endReason?: 'stalled'
+}
+
+export interface CrawlStatus {
+	readonly id: CrawlId
+	readonly state: 'crawling' | 'done' | 'syncing' | 'failed' | 'paused' | 'canceled'
+	/** ISO date-time the crawl was created. */
+	readonly createdAt?: string
+	/** Registrable domain the crawl is scoped to. */
+	readonly domain?: string
+	/** The crawl's label; absent when it is shown by its domain. */
+	readonly name?: string
+	/** ISO date-time the crawl was archived. Archived crawls are canceled and left out of `list()`. */
+	readonly archivedAt?: string
+	/** ISO date-time retention deletes an archived crawl, unless it is unarchived first. */
+	readonly deletesAt?: string
+	readonly pages: number
+	readonly failed: number
+	/** How many of `failed` were skips: URLs given up after one attempt so their slot could move on. */
+	readonly skipped?: number
+	readonly config?: CrawlConfigSummary
+	readonly run?: CrawlRunSummary
+	readonly sync?: CrawlSyncPolicy
+	readonly freshness?: {
+		readonly pagesPastDue: number
+		readonly maxOverdue: string
+		readonly probeMisses: number
+		readonly causes: readonly { readonly kind: string; readonly pages: number }[]
+	}
+	readonly coverage: {
+		readonly complete: boolean
+		readonly pendingAtCap: number
+		readonly sourcePartial: number
+		readonly causes: readonly string[]
+	}
+	readonly progress: {
+		readonly queued: number
+		readonly inFlight: number
+		readonly done: number
+		/** Page work alone; `queued` and `inFlight` also count robots and sitemap operations. */
+		readonly pages?: { readonly queued: number; readonly inFlight: number; readonly skipped?: number }
+	}
+	/** Only meaningful while work is queued: why nothing is being rendered right now. */
+	readonly admission?: CrawlAdmission
+	/** Trailing 24 hours; `budgetPerDay` is null when the sync policy sets no render budget. */
+	readonly renders?: { readonly usedToday: number; readonly budgetPerDay: number | null }
+	/** Renders settled in each of the last 60 minutes, oldest first, and the latest settle time. Single-crawl status only. */
+	readonly throughput?: { readonly perMinute: ReadonlyArray<number>; readonly lastSettledAt?: string }
+	readonly offset: CrawlOffset
+	/** The activity stream's tail; absent for a crawl that has no activity yet. */
+	readonly activityOffset?: CrawlOffset
+}
+
+export interface CrawlReadOptions {
+	readonly from?: CrawlOffset
+	readonly live?: boolean
+	readonly signal?: AbortSignal
+}
+
+export interface MergedCrawlReadOptions {
+	readonly from?: Readonly<Record<CrawlId, CrawlOffset>>
+	readonly live?: boolean
+	readonly signal?: AbortSignal
+}
+
+export interface CrawlListOptions {
+	/** 1..200, defaults to 50. */
+	readonly limit?: number
+	/** Lists archived crawls instead of the rest. */
+	readonly archived?: boolean
+	readonly signal?: AbortSignal
+}
+
+export interface CrawlListResponse {
+	readonly crawls: CrawlStatus[]
+}
+
+export interface CrawlCreateOptions {
+	readonly idempotencyKey: string
+}
+
+export interface CrawlRunOptions {
+	readonly idempotencyKey: string
+	/** Recover an existing command, or fail without creating a replacement run. */
+	readonly replayOnly?: boolean
+}
+
+export interface CrawlDoneResult {
+	readonly lastOffset: CrawlOffset
+	readonly status: CrawlStatus
+}
+
+export class CrawlClientError extends ExpandClientError {
+	readonly name: string = 'CrawlClientError'
+	constructor(
+		message: string,
+		readonly code?: string,
+		readonly status?: number,
+		readonly retryable = false,
+	) {
+		super(message)
+	}
+}
+
+/** API refusal with the Crawl error code and original response envelope. */
+export class CrawlClientApiError extends ExpandClientApiError {
+	constructor(
+		message: string,
+		readonly code: string | undefined,
+		status: number,
+		readonly retryable = false,
+		body?: unknown,
+	) {
+		super(message, { status, body })
+		this.name = 'CrawlClientApiError'
+	}
+}
+
+export class CrawlOffsetExpiredError extends CrawlClientApiError {
+	readonly name: string = 'CrawlOffsetExpiredError'
+}
+
+interface CrawlTransportOptions {
+	readonly apiKey: string
+	readonly baseUrl: string
+	readonly fetch?: typeof globalThis.fetch
+	readonly timeoutMs?: number
+	readonly maxRetries?: number
+	readonly signal?: AbortSignal
+}
+
+function transportSignal(transport: CrawlTransportOptions, signal?: AbortSignal): AbortSignal | undefined {
+	return transport.signal === undefined
+		? signal
+		: AbortSignal.any([transport.signal, ...(signal === undefined ? [] : [signal])])
+}
+
+export interface CrawlProjectionOptions {
+	readonly signal?: AbortSignal
+	readonly timeoutMs?: number
+}
+
+export interface CrawlListPage<T> {
+	readonly items: readonly T[]
+	readonly nextCursor?: string
+}
+
+/** Await one page, iterate individual items, or use pages() for bounded batches. */
+export class CrawlList<T> implements AsyncIterable<T>, PromiseLike<CrawlListPage<T>> {
+	private firstPage?: Promise<CrawlListPage<T>>
+	constructor(
+		private readonly fetchPage: (cursor?: string) => Promise<CrawlListPage<T>>,
+		private readonly cursor?: string,
+		private readonly signal?: AbortSignal,
+	) {}
+
+	private first(): Promise<CrawlListPage<T>> {
+		if (this.signal?.aborted === true) return Promise.reject(abortError(this.signal))
+		this.firstPage ??= this.fetchPage(this.cursor)
+		return this.firstPage
+	}
+
+	then<TResult1 = CrawlListPage<T>, TResult2 = never>(
+		onfulfilled?: ((value: CrawlListPage<T>) => TResult1 | PromiseLike<TResult1>) | null,
+		onrejected?: ((reason: unknown) => TResult2 | PromiseLike<TResult2>) | null,
+	): PromiseLike<TResult1 | TResult2> {
+		return this.first().then(onfulfilled, onrejected)
+	}
+
+	async *pages(): AsyncGenerator<CrawlListPage<T>> {
+		let page = await this.first()
+		const visited = new Set<string>(this.cursor === undefined ? [] : [this.cursor])
+		while (true) {
+			if (signalIsAborted(this.signal)) throw abortError(this.signal)
+			yield page
+			if (page.nextCursor === undefined) return
+			if (visited.has(page.nextCursor)) throw new CrawlClientError('The server repeated a pagination cursor.')
+			visited.add(page.nextCursor)
+			page = await this.fetchPage(page.nextCursor)
+		}
+	}
+
+	async *[Symbol.asyncIterator](): AsyncGenerator<T> {
+		for await (const page of this.pages())
+			for (const item of page.items) {
+				if (signalIsAborted(this.signal)) throw abortError(this.signal)
+				yield item
+			}
+	}
+
+	async collect(options: { readonly limit: number }): Promise<T[]> {
+		if (!Number.isSafeInteger(options.limit) || options.limit < 1)
+			throw new CrawlClientError('collect limit must be positive.')
+		const items: T[] = []
+		for await (const item of this) {
+			items.push(item)
+			if (items.length >= options.limit) break
+		}
+		return items
+	}
+}
+
+/** A discovered URL in any admission state (`GET /v1/crawls/:id/urls`), the snapshot half of the URL view. */
+export interface CrawlUrl {
+	readonly id: string
+	readonly url: string
+	readonly state: 'pending' | 'candidate' | 'reserved' | 'admitted' | 'traversal'
+	readonly sourceId: string
+	readonly depth: number
+	readonly discoveredBy: 'seed' | 'sitemap' | 'link' | 'search'
+	/** ISO date-time. */
+	readonly discoveredAt: string
+	/** Apply an activity event to this row only when `event.offset > activitySeq`. */
+	readonly activitySeq: CrawlOffset
+	readonly pageId?: string
+	/** Why the URL is where it is (`canceled`, `out-of-scope`, `skipped`, …). */
+	readonly reason?: string
+	readonly excludedReason?: string
+	/** What the render told us when `reason` is `skipped` (`http-500`, `timeout`). */
+	readonly skipDetail?: string
+	/** ISO date-time: when a skipped URL may be rearmed by a later run. */
+	readonly availableAt?: string
+}
+
+export interface CrawlUrlListPage {
+	readonly urls: readonly CrawlUrl[]
+	readonly nextCursor?: string
+	readonly truncated: boolean
+	/** The activity stream's tail, read before the rows: tail `crawl.activity({ from })` from here. */
+	readonly activityOffset: CrawlOffset
+	/** Rows per state for the whole crawl under the same `source` and `search` filters, not just this page. */
+	readonly counts: Readonly<Record<string, number>>
+}
+
+export interface CrawlCatalogPage {
+	readonly id: string
+	readonly url: string
+	readonly status: 'live' | 'removed'
+	readonly latestCaptureId?: string
+	readonly changedAt?: string
+}
+
+export interface CrawlCurrentPage {
+	readonly id: string
+	readonly url: string
+	readonly status: 'live' | 'removed'
+	readonly captureId?: string
+	readonly meta?: CrawlPageMeta
+	readonly page?: CrawlPageContent
+}
+
+export interface CrawlArtifactLinks {
+	readonly markdown: string
+	readonly html: string
+	readonly stateJson: string
+	readonly screenshot?: { readonly url: string; readonly fullPage: boolean }
+	readonly replay: string
+	readonly playground: string
+	readonly assets: string
+}
+
+export interface CrawlPageVersion {
+	readonly captureId: string
+	readonly observedAt: string
+	readonly offset: CrawlOffset
+	readonly contentSha256: string
+	readonly http: { readonly status: number; readonly finalUrl: string }
+	readonly quality: 'complete' | 'best-effort' | 'degraded'
+	readonly meta: CrawlPageMeta
+	readonly artifacts: CrawlArtifactLinks
+}
+
+export interface CrawlCaptureContent {
+	readonly captureId: string
+	readonly meta: CrawlPageMeta
+	readonly page: CrawlPageContent
+	readonly artifacts: CrawlArtifactLinks
+}
+
+function projectionQuery(path: string, values: Readonly<Record<string, string | number | undefined>>): string {
+	const query = new URLSearchParams()
+	for (const [key, value] of Object.entries(values)) if (value !== undefined) query.set(key, String(value))
+	return query.size === 0 ? path : `${path}?${query}`
+}
+
+async function projectionResponse(
+	transport: CrawlTransportOptions,
+	path: string,
+	options: CrawlProjectionOptions,
+): Promise<Response> {
+	const target = new URL(absoluteUrl(transport.baseUrl, path))
+	const base = new URL(transport.baseUrl)
+	if (
+		target.origin !== base.origin ||
+		target.username !== '' ||
+		target.password !== '' ||
+		!target.pathname.startsWith('/v1/')
+	) {
+		throw new CrawlClientError('Projection URLs must belong to the configured Expand API.')
+	}
+	const timeoutMs = options.timeoutMs ?? transport.timeoutMs ?? 60_000
+	if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0)
+		throw new CrawlClientError('Projection timeoutMs must be a positive integer.')
+	const deadline = AbortSignal.timeout(timeoutMs)
+	const callerSignal = transportSignal(transport, options.signal)
+	const signal = callerSignal === undefined ? deadline : AbortSignal.any([callerSignal, deadline])
+	if (signal.aborted) throw abortError(signal)
+	const response = await (transport.fetch ?? globalThis.fetch)(target, {
+		headers: { 'x-expand-api-key': transport.apiKey },
+		signal,
+		redirect: 'error',
+	})
+	if (!response.ok) throw await apiError(response)
+	return response
+}
+
+async function projectionJson<T>(
+	transport: CrawlTransportOptions,
+	path: string,
+	options: CrawlProjectionOptions,
+): Promise<T> {
+	// Mirrors the typed API contract, like requestJson; arbitrary body reads use unknown.
+	return (await (await projectionResponse(transport, path, options)).json()) as unknown as T
+}
+
+export class CrawlCaptureResource {
+	constructor(
+		readonly id: string,
+		private readonly transport: CrawlTransportOptions,
+	) {}
+	get(options: CrawlProjectionOptions = {}): Promise<CrawlCaptureContent> {
+		return projectionJson(this.transport, `/v1/captures/${encodeURIComponent(this.id)}`, options)
+	}
+	stateJson(options: CrawlProjectionOptions = {}): Promise<CrawlStateJson> {
+		return projectionJson(this.transport, `/v1/captures/${encodeURIComponent(this.id)}/state-json`, options)
+	}
+	evidence(key: string, options: CrawlProjectionOptions = {}): Promise<CrawlJsonValue> {
+		return projectionJson(
+			this.transport,
+			`/v1/captures/${encodeURIComponent(this.id)}/state-json/${encodeURIComponent(key)}`,
+			options,
+		)
+	}
+	async markdown(options: CrawlProjectionOptions = {}): Promise<string> {
+		return (
+			await projectionResponse(this.transport, `/v1/captures/${encodeURIComponent(this.id)}/markdown`, options)
+		).text()
+	}
+	async html(options: CrawlProjectionOptions = {}): Promise<string> {
+		return (
+			await projectionResponse(this.transport, `/v1/captures/${encodeURIComponent(this.id)}/html`, options)
+		).text()
+	}
+	playgroundUrl(): string {
+		return `https://expand.land/s/${encodeURIComponent(this.id)}`
+	}
+}
+
+export class CrawlPageResource {
+	constructor(
+		readonly id: string,
+		private readonly transport: CrawlTransportOptions,
+	) {}
+	versions(
+		options: CrawlProjectionOptions & { readonly limit?: number; readonly before?: string; readonly at?: string } = {},
+	): CrawlList<CrawlPageVersion> {
+		return new CrawlList(
+			async (before) => {
+				const result = await projectionJson<{ readonly versions: CrawlPageVersion[]; readonly nextCursor?: string }>(
+					this.transport,
+					projectionQuery(`/v1/pages/${encodeURIComponent(this.id)}/versions`, {
+						before,
+						at: options.at,
+						limit: options.limit,
+					}),
+					options,
+				)
+				return { items: result.versions, nextCursor: options.at === undefined ? result.nextCursor : undefined }
+			},
+			options.before,
+			transportSignal(this.transport, options.signal),
+		)
+	}
+	diff(
+		options: CrawlProjectionOptions & { readonly from: string; readonly to: string },
+	): Promise<CrawlPageChanged['diff']> {
+		return projectionJson(
+			this.transport,
+			projectionQuery(`/v1/pages/${encodeURIComponent(this.id)}/diff`, { from: options.from, to: options.to }),
+			options,
+		)
+	}
+}
+
+/** Shares the Crawl transport without exposing its API key to arbitrary body URLs. */
+export function makeCrawlProjectionsClient(transport: CrawlTransportOptions) {
+	return {
+		captures(id: string): CrawlCaptureResource {
+			if (id.length === 0) throw new CrawlClientError('Capture id must not be empty.')
+			return new CrawlCaptureResource(id, transport)
+		},
+		pages(id: string): CrawlPageResource {
+			if (id.length === 0) throw new CrawlClientError('Page id must not be empty.')
+			return new CrawlPageResource(id, transport)
+		},
+		fetchJson(url: string, options: CrawlProjectionOptions = {}): Promise<unknown> {
+			return projectionJson(transport, url, options)
+		},
+	}
+}
+
+interface CreatedCrawlResponse {
+	readonly id: CrawlId
+}
+
+function requireIdempotencyKey(value: string | undefined, operation: string): string {
+	if (value === undefined || value.length < 1 || value.length > 255) {
+		throw new CrawlClientError(`${operation} requires an idempotencyKey between 1 and 255 characters.`)
+	}
+	return value
+}
+
+function isRecord(value: unknown): value is Readonly<Record<string, unknown>> {
+	return typeof value === 'object' && value !== null
+}
+
+async function apiError(response: Response): Promise<CrawlClientApiError> {
+	let body: unknown
+	try {
+		body = await response.json()
+	} catch {
+		// A status and status text are still useful when a proxy returned non-JSON.
+	}
+	const error = isRecord(body) && isRecord(body['error']) ? body['error'] : undefined
+	const code = typeof error?.['code'] === 'string' ? error['code'] : undefined
+	const message = typeof error?.['message'] === 'string' ? error['message'] : response.statusText
+	const retryable = error?.['retryable'] === true
+	return response.status === 410
+		? new CrawlOffsetExpiredError(message, code, response.status, retryable, body)
+		: new CrawlClientApiError(message, code, response.status, retryable, body)
+}
+
+function absoluteUrl(baseUrl: string, path: string): string {
+	return new URL(path, `${baseUrl.replace(/\/+$/u, '')}/`).toString()
+}
+
+interface CrawlStreamRequest {
+	readonly signal: AbortSignal
+	readonly redirectSignal: AbortSignal
+	readonly fetch: typeof globalThis.fetch
+}
+
+const CRAWL_STREAM_REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308])
+// durable-streams 0.2.6 cannot interrupt its retry timer. Retries belong to
+// the transport below, which keeps both setup and backoff tied to the caller.
+const CRAWL_STREAM_BACKOFF = { ...BackoffDefaults, maxRetries: 0 }
+
+// The same ceiling the Effect client applies to `Retry-After`: a live iterator must not sleep
+// for an hour because one response asked for it.
+const MAX_RETRY_AFTER_MS = Duration.toMillis(MAX_RETRY_AFTER)
+
+function retryAfterMilliseconds(error: unknown): number {
+	if (!(error instanceof FetchError)) return 0
+	const value = error.headers['retry-after']
+	if (value === undefined) return 0
+	const seconds = Number(value)
+	if (Number.isFinite(seconds) && seconds > 0) return Math.min(seconds * 1000, MAX_RETRY_AFTER_MS)
+	const date = Date.parse(value)
+	return Number.isFinite(date) ? Math.max(0, Math.min(date - Date.now(), MAX_RETRY_AFTER_MS)) : 0
+}
+
+function isDeadlineReason(reason: unknown): boolean {
+	return reason instanceof Error && reason.name === 'TimeoutError'
+}
+
+function crawlTimeoutMs(transport: CrawlTransportOptions): number {
+	const timeoutMs = transport.timeoutMs ?? 60_000
+	if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0 || timeoutMs > 2_147_483_647)
+		throw new CrawlClientError('Crawl timeoutMs must be a positive integer no greater than 2147483647.')
+	return timeoutMs
+}
+
+function streamError(error: unknown, timeoutMs: number): ExpandClientError {
+	if (error instanceof ExpandClientError) return error
+	if (error instanceof FetchError) {
+		const body: unknown = error.json ?? error.text
+		const envelope = isRecord(body) && isRecord(body['error']) ? body['error'] : undefined
+		const code = typeof envelope?.['code'] === 'string' ? envelope['code'] : undefined
+		const message = typeof envelope?.['message'] === 'string' ? envelope['message'] : error.message
+		const retryable =
+			envelope?.['retryable'] === true ||
+			(envelope?.['retryable'] === undefined && (error.status === 408 || error.status === 429 || error.status >= 500))
+		return error.status === 410
+			? new CrawlOffsetExpiredError(message, code ?? 'offset_expired', error.status, retryable, body)
+			: new CrawlClientApiError(message, code, error.status, retryable, body)
+	}
+	if (error instanceof Error && error.name === 'TimeoutError')
+		return new ExpandClientTimeoutError(`Request timed out after ${timeoutMs}ms`, { timeoutMs, cause: error })
+	return new ExpandClientConnectionError(error instanceof Error ? error.message : 'Crawl request failed.', {
+		cause: error,
+	})
+}
+
+function createCrawlStreamRequest(
+	transport: CrawlTransportOptions,
+	callerSignal?: AbortSignal,
+	live = false,
+): CrawlStreamRequest {
+	const redirect = new AbortController()
+	const execute = transport.fetch ?? globalThis.fetch
+	const timeoutMs = crawlTimeoutMs(transport)
+	const maxRetries = transport.maxRetries ?? 2
+	if (!Number.isSafeInteger(maxRetries) || maxRetries < 0)
+		throw new CrawlClientError('Crawl maxRetries must be a non-negative integer.')
+	const signal = AbortSignal.any([
+		redirect.signal,
+		...(callerSignal === undefined ? [] : [callerSignal]),
+		...(live ? [] : [AbortSignal.timeout(timeoutMs)]),
+	])
+	return {
+		signal,
+		redirectSignal: redirect.signal,
+		fetch: async (input, init) => {
+			const requestSignal = init?.signal == null ? signal : AbortSignal.any([signal, init.signal])
+			let delay = BackoffDefaults.initialDelay
+			let retries = 0
+			while (true) {
+				if (requestSignal.aborted) throw abortError(requestSignal)
+				try {
+					const response = await execute(input, { ...init, signal: requestSignal, redirect: 'manual' })
+					if (CRAWL_STREAM_REDIRECT_STATUSES.has(response.status) || response.type === 'opaqueredirect') {
+						const error = new CrawlClientError(
+							'Authenticated Crawl requests cannot redirect.',
+							'redirect_not_allowed',
+							response.status || undefined,
+							false,
+						)
+						// Keep policy failures out of transport and iterator reconnect loops.
+						redirect.abort(error)
+						try {
+							await response.body?.cancel()
+						} catch {
+							// Aborting the request may already have canceled its body.
+						}
+						throw error
+					}
+					if (!response.ok) throw await FetchError.fromResponse(response, String(input))
+					return response
+				} catch (error: unknown) {
+					if (requestSignal.aborted) throw abortError(requestSignal)
+					if (!isRetryable(streamError(error, timeoutMs), timeoutMs)) throw error
+					if (!live && retries >= maxRetries) throw error
+					retries += 1
+					try {
+						await abortableDelay(Math.max(retryAfterMilliseconds(error), Math.random() * delay), requestSignal)
+					} catch (abort: unknown) {
+						// The request deadline expired while waiting to retry. The refusal being retried
+						// explains the failure; a timeout is reserved for an attempt that was in flight.
+						if (!live && isDeadlineReason(requestSignal.reason)) throw streamError(error, timeoutMs)
+						throw abort
+					}
+					delay = Math.min(delay * BackoffDefaults.multiplier, BackoffDefaults.maxDelay)
+				}
+			}
+		},
+	}
+}
+
+async function requestJson<T>(transport: CrawlTransportOptions, path: string, init: RequestInit = {}): Promise<T> {
+	const timeoutMs = crawlTimeoutMs(transport)
+	const callerSignal = transportSignal(transport, init.signal ?? undefined)
+	const request = createCrawlStreamRequest(transport, callerSignal)
+	const signal = request.signal
+	if (signal.aborted) throw abortError(signal)
+	try {
+		const response = await request.fetch(absoluteUrl(transport.baseUrl, path), {
+			...init,
+			redirect: 'error',
+			signal,
+			headers: {
+				'x-expand-api-key': transport.apiKey,
+				...(init.body === undefined ? {} : { 'content-type': 'application/json' }),
+				...init.headers,
+			},
+		})
+		if (!response.ok) throw await apiError(response)
+		// The SDK's public response interfaces mirror the versioned HTTP contract.
+		return (await response.json()) as unknown as T
+	} catch (error: unknown) {
+		// A typed transport failure already names the cause, even when the deadline has since fired.
+		if (error instanceof ExpandClientError) throw error
+		throw streamError(signal.aborted ? abortError(signal) : error, timeoutMs)
+	}
+}
+
+function abortError(signal?: AbortSignal): Error {
+	return signal?.reason instanceof Error ? signal.reason : new CrawlClientError('The Crawl operation was aborted.')
+}
+
+function abortableDelay(milliseconds: number, signal?: AbortSignal): Promise<void> {
+	return new Promise((resolve, reject) => {
+		if (signal?.aborted === true) {
+			reject(abortError(signal))
+			return
+		}
+		const onAbort = () => {
+			clearTimeout(timer)
+			signal?.removeEventListener('abort', onAbort)
+			reject(abortError(signal))
+		}
+		const timer = setTimeout(() => {
+			signal?.removeEventListener('abort', onAbort)
+			resolve()
+		}, milliseconds)
+		signal?.addEventListener('abort', onAbort, { once: true })
+	})
+}
+
+function signalIsAborted(signal?: AbortSignal): boolean {
+	return signal?.aborted === true
+}
+
+const CRAWL_ITERATOR_RETURN = Symbol('crawl-iterator-return')
+const CRAWL_ITERATOR_ABORT = Symbol('crawl-iterator-abort')
+
+function iteratorWasCanceled(signal?: AbortSignal): boolean {
+	return signal?.reason === CRAWL_ITERATOR_RETURN || signal?.reason === CRAWL_ITERATOR_ABORT
+}
+
+export interface CrawlStreamPage<T> {
+	readonly crawlId: CrawlId
+	readonly items: readonly T[]
+	readonly nextOffset: CrawlOffset
+	readonly upToDate: boolean
+	readonly streamClosed: boolean
+}
+
+export type CrawlEventPage = CrawlStreamPage<CrawlEvent>
+export type CrawlActivityPage = CrawlStreamPage<CrawlActivityEvent>
+
+class CrawlIterable<T> implements AsyncIterable<T> {
+	constructor(
+		private readonly factory: (signal: AbortSignal) => AsyncGenerator<T>,
+		protected readonly externalSignal?: AbortSignal,
+	) {}
+
+	[Symbol.asyncIterator](): AsyncIterator<T> {
+		const controller = new AbortController()
+		const abort = () => controller.abort(CRAWL_ITERATOR_ABORT)
+		if (this.externalSignal?.aborted === true) abort()
+		else this.externalSignal?.addEventListener('abort', abort, { once: true })
+		const iterator = this.factory(controller.signal)
+		const cleanup = () => this.externalSignal?.removeEventListener('abort', abort)
+		return {
+			next: async () => {
+				try {
+					const result = await iterator.next()
+					if (result.done) cleanup()
+					return result
+				} catch (error: unknown) {
+					cleanup()
+					if (iteratorWasCanceled(controller.signal)) return { done: true, value: undefined }
+					throw error
+				}
+			},
+			return: async (value?: unknown) => {
+				controller.abort(CRAWL_ITERATOR_RETURN)
+				cleanup()
+				return (await iterator.return?.(value)) ?? { done: true, value: undefined }
+			},
+			throw: async (error?: unknown) => {
+				controller.abort(error)
+				cleanup()
+				if (iterator.throw === undefined) throw error
+				return iterator.throw(error)
+			},
+		}
+	}
+
+	async collect(options: { readonly limit: number }): Promise<T[]> {
+		if (!Number.isSafeInteger(options.limit) || options.limit < 1)
+			throw new CrawlClientError('collect limit must be positive.')
+		const events: T[] = []
+		for await (const event of this) {
+			events.push(event)
+			if (events.length >= options.limit) break
+		}
+		return events
+	}
+}
+
+export class CrawlStreamPageIterable<T>
+	extends CrawlIterable<CrawlStreamPage<T>>
+	implements PromiseLike<CrawlStreamPage<T> | undefined>
+{
+	then<TResult1 = CrawlStreamPage<T> | undefined, TResult2 = never>(
+		onfulfilled?: ((page: CrawlStreamPage<T> | undefined) => TResult1 | PromiseLike<TResult1>) | null,
+		onrejected?: ((reason: unknown) => TResult2 | PromiseLike<TResult2>) | null,
+	): PromiseLike<TResult1 | TResult2> {
+		const first = async () => {
+			const iterator = this[Symbol.asyncIterator]()
+			try {
+				const result = await iterator.next()
+				return result.done ? undefined : result.value
+			} finally {
+				await iterator.return?.()
+			}
+		}
+		return first().then(onfulfilled, onrejected)
+	}
+}
+
+export class CrawlStreamIterable<T> extends CrawlIterable<T> implements PromiseLike<CrawlStreamPage<T> | undefined> {
+	constructor(
+		factory: (signal: AbortSignal) => AsyncGenerator<T>,
+		private readonly pageFactory: (signal: AbortSignal) => AsyncGenerator<CrawlStreamPage<T>>,
+		signal?: AbortSignal,
+	) {
+		super(factory, signal)
+	}
+	pages(): CrawlStreamPageIterable<T> {
+		return new CrawlStreamPageIterable(this.pageFactory, this.externalSignal)
+	}
+	then<TResult1 = CrawlStreamPage<T> | undefined, TResult2 = never>(
+		onfulfilled?: ((page: CrawlStreamPage<T> | undefined) => TResult1 | PromiseLike<TResult1>) | null,
+		onrejected?: ((reason: unknown) => TResult2 | PromiseLike<TResult2>) | null,
+	): PromiseLike<TResult1 | TResult2> {
+		return this.pages().then(onfulfilled, onrejected)
+	}
+}
+
+export class CrawlEventPageIterable extends CrawlStreamPageIterable<CrawlEvent> {}
+export class CrawlEventIterable extends CrawlStreamIterable<CrawlEvent> {}
+export class CrawlActivityPageIterable extends CrawlStreamPageIterable<CrawlActivityEvent> {}
+export class CrawlActivityIterable extends CrawlStreamIterable<CrawlActivityEvent> {}
+
+/** Which per-crawl stream a read tails. */
+interface CrawlStreamSpec {
+	readonly path: 'events' | 'activity'
+}
+
+const PAGE_STREAM: CrawlStreamSpec = { path: 'events' }
+const ACTIVITY_STREAM: CrawlStreamSpec = { path: 'activity' }
+
+export class CrawlResource {
+	constructor(
+		readonly id: CrawlId,
+		private readonly transport: CrawlTransportOptions,
+	) {}
+
+	events(options: CrawlReadOptions = {}): CrawlEventIterable {
+		return new CrawlEventIterable(
+			(signal) => this.readStream<CrawlEvent>(PAGE_STREAM, { ...options, signal }),
+			(signal) => this.readStreamPages<CrawlEvent>(PAGE_STREAM, { ...options, signal }),
+			transportSignal(this.transport, options.signal),
+		)
+	}
+
+	/**
+	 * The activity stream: batches of URL transitions without content, for progress and
+	 * operator views. Same shape as `events`; a one-off's iteration ends on stream closure.
+	 */
+	activity(options: CrawlReadOptions = {}): CrawlActivityIterable {
+		return new CrawlActivityIterable(
+			(signal) => this.readStream<CrawlActivityEvent>(ACTIVITY_STREAM, { ...options, signal }),
+			(signal) => this.readStreamPages<CrawlActivityEvent>(ACTIVITY_STREAM, { ...options, signal }),
+			transportSignal(this.transport, options.signal),
+		)
+	}
+
+	/** A pinned live-catalog snapshot. Resume /events from the final page's nextOffset. */
+	snapshot(options: CrawlProjectionOptions & { readonly from?: string } = {}): CrawlEventIterable {
+		return new CrawlEventIterable(
+			(signal) => this.readSnapshotItems({ ...options, signal }),
+			(signal) => this.readSnapshotPages({ ...options, signal }),
+			transportSignal(this.transport, options.signal),
+		)
+	}
+
+	private async *readSnapshotItems(
+		options: CrawlProjectionOptions & { readonly from?: string },
+	): AsyncGenerator<CrawlEvent> {
+		for await (const page of this.readSnapshotPages(options)) {
+			for (const event of page.items) yield event
+		}
+	}
+
+	private async *readSnapshotPages(
+		options: CrawlProjectionOptions & { readonly from?: string },
+	): AsyncGenerator<CrawlEventPage> {
+		let offset = options.from
+		while (!signalIsAborted(options.signal)) {
+			const page = await projectionJson<{
+				readonly events: readonly CrawlEvent[]
+				readonly nextOffset: CrawlOffset
+				readonly upToDate: boolean
+			}>(this.transport, projectionQuery(`/v1/crawls/${encodeURIComponent(this.id)}/snapshot`, { offset }), options)
+			yield {
+				crawlId: this.id,
+				items: page.events,
+				nextOffset: page.nextOffset,
+				upToDate: page.upToDate,
+				streamClosed: false,
+			}
+			if (page.upToDate) return
+			if (page.nextOffset === offset) throw new CrawlClientError('Snapshot cursor did not advance.')
+			offset = page.nextOffset
+		}
+	}
+
+	private async *readStreamPages<T extends { readonly offset: CrawlOffset }>(
+		spec: CrawlStreamSpec,
+		options: CrawlReadOptions,
+	): AsyncGenerator<CrawlStreamPage<T>> {
+		let offset = options.from ?? '-1'
+		let caughtUp = false
+		let cursor: string | undefined
+		let recurring: boolean | undefined
+		while (!signalIsAborted(options.signal)) {
+			const url = new URL(absoluteUrl(this.transport.baseUrl, `/v1/crawls/${encodeURIComponent(this.id)}/${spec.path}`))
+			if (caughtUp) url.searchParams.set('live', 'long-poll')
+			if (cursor !== undefined) url.searchParams.set('cursor', cursor)
+			const request = createCrawlStreamRequest(this.transport, options.signal, options.live === true)
+			let page: JsonBatch<T>
+			try {
+				const response = await durableStream<T>({
+					url: url.toString(),
+					headers: { 'x-expand-api-key': this.transport.apiKey },
+					offset,
+					live: false,
+					json: true,
+					signal: request.signal,
+					fetch: request.fetch,
+					backoffOptions: CRAWL_STREAM_BACKOFF,
+				})
+				page = await readStreamPage(response)
+				if (request.redirectSignal.aborted) throw abortError(request.redirectSignal)
+			} catch (error: unknown) {
+				if (iteratorWasCanceled(options.signal)) return
+				if (request.redirectSignal.aborted) throw abortError(request.redirectSignal)
+				const retryable = isRetryable(
+					streamError(error, crawlTimeoutMs(this.transport)),
+					crawlTimeoutMs(this.transport),
+				)
+				if (options.live === true && retryable) {
+					await abortableDelay(500, options.signal)
+					continue
+				}
+				throw streamError(request.signal.aborted ? abortError(request.signal) : error, crawlTimeoutMs(this.transport))
+			}
+			if (signalIsAborted(options.signal)) return
+			offset = page.offset
+			cursor = page.cursor
+			caughtUp ||= page.upToDate
+			yield {
+				crawlId: this.id,
+				items: page.items,
+				nextOffset: offset,
+				upToDate: page.upToDate,
+				streamClosed: page.streamClosed,
+			}
+			if (page.streamClosed) return
+			if (!page.upToDate || options.live === true) continue
+			// Recurring reads default to catch-up. One-off completion is signaled only by EOF;
+			// offsets are opaque and are never compared to an out-of-band watermark.
+			if (recurring === undefined) recurring = (await this.status({ signal: options.signal })).sync !== undefined
+			if (recurring) return
+		}
+	}
+
+	pages(
+		options: CrawlProjectionOptions & {
+			readonly limit?: number
+			readonly cursor?: string
+			readonly prefix?: string
+			readonly status?: 'live' | 'removed'
+			readonly changedSince?: string
+		} = {},
+	): CrawlList<CrawlCatalogPage> {
+		return new CrawlList(
+			async (cursor) => {
+				const result = await projectionJson<{ readonly pages: CrawlCatalogPage[]; readonly nextCursor?: string }>(
+					this.transport,
+					projectionQuery(`/v1/crawls/${encodeURIComponent(this.id)}/pages`, {
+						cursor,
+						limit: options.limit,
+						prefix: options.prefix,
+						status: options.status,
+						changedSince: options.changedSince,
+					}),
+					options,
+				)
+				return { items: result.pages, nextCursor: result.nextCursor }
+			},
+			options.cursor,
+			transportSignal(this.transport, options.signal),
+		)
+	}
+
+	/**
+	 * Every discovered URL in any admission state, cursor-paginated and filterable by state,
+	 * source and a URL substring (`search`, matched literally and case-insensitively across the
+	 * whole crawl). The first page's `activityOffset` is where to start an activity tail that
+	 * keeps the snapshot current: apply an event to a URL only when `event.offset > url.activitySeq`.
+	 */
+	urls(
+		options: CrawlProjectionOptions & {
+			readonly limit?: number
+			readonly cursor?: string
+			readonly state?: CrawlUrl['state']
+			readonly source?: string
+			readonly search?: string
+		} = {},
+	): CrawlList<CrawlUrl> {
+		return new CrawlList(
+			async (cursor) => {
+				const result = await projectionJson<CrawlUrlListPage>(
+					this.transport,
+					projectionQuery(`/v1/crawls/${encodeURIComponent(this.id)}/urls`, {
+						cursor,
+						limit: options.limit,
+						state: options.state,
+						source: options.source,
+						search: options.search,
+					}),
+					options,
+				)
+				return { items: result.urls, nextCursor: result.nextCursor }
+			},
+			options.cursor,
+			transportSignal(this.transport, options.signal),
+		)
+	}
+
+	/** One page of the discovered-URLs view with its counts and activity offset, for callers that need the envelope. */
+	urlsPage(
+		options: CrawlProjectionOptions & {
+			readonly limit?: number
+			readonly cursor?: string
+			readonly state?: CrawlUrl['state']
+			readonly source?: string
+			readonly search?: string
+		} = {},
+	): Promise<CrawlUrlListPage> {
+		return projectionJson<CrawlUrlListPage>(
+			this.transport,
+			projectionQuery(`/v1/crawls/${encodeURIComponent(this.id)}/urls`, {
+				cursor: options.cursor,
+				limit: options.limit,
+				state: options.state,
+				source: options.source,
+				search: options.search,
+			}),
+			options,
+		)
+	}
+
+	page(id: string, options: CrawlProjectionOptions = {}): Promise<CrawlCurrentPage> {
+		return projectionJson(
+			this.transport,
+			`/v1/crawls/${encodeURIComponent(this.id)}/pages/${encodeURIComponent(id)}`,
+			options,
+		)
+	}
+
+	pageByUrl(url: string, options: CrawlProjectionOptions = {}): Promise<CrawlCurrentPage> {
+		return projectionJson(
+			this.transport,
+			projectionQuery(`/v1/crawls/${encodeURIComponent(this.id)}/pages/by-url`, { url }),
+			options,
+		)
+	}
+
+	private async *readStream<T extends { readonly offset: CrawlOffset }>(
+		spec: CrawlStreamSpec,
+		options: CrawlReadOptions,
+	): AsyncGenerator<T> {
+		for await (const page of this.readStreamPages<T>(spec, options)) {
+			for (const event of page.items) {
+				if (signalIsAborted(options.signal)) return
+				yield event
+			}
+		}
+	}
+
+	status(options: { readonly signal?: AbortSignal } = {}): Promise<CrawlStatus> {
+		return requestJson<CrawlStatus>(this.transport, `/v1/crawls/${encodeURIComponent(this.id)}`, {
+			signal: options.signal,
+		})
+	}
+
+	async done(
+		options: { readonly signal?: AbortSignal; readonly pollIntervalMs?: number } = {},
+	): Promise<CrawlDoneResult> {
+		if (
+			options.pollIntervalMs !== undefined &&
+			(!Number.isFinite(options.pollIntervalMs) || options.pollIntervalMs > 2_147_483_647)
+		)
+			throw new CrawlClientError('Crawl pollIntervalMs must be finite and no greater than 2147483647.')
+		const interval = Math.max(50, options.pollIntervalMs ?? 500)
+		const signal = transportSignal(this.transport, options.signal)
+		while (!signalIsAborted(signal)) {
+			const status = await this.status({ signal })
+			if (signalIsAborted(signal)) break
+			if (
+				status.state === 'done' ||
+				status.state === 'failed' ||
+				status.state === 'canceled' ||
+				(status.sync !== undefined && status.progress.queued === 0 && status.progress.inFlight === 0)
+			) {
+				return { lastOffset: status.offset, status }
+			}
+			await abortableDelay(interval, signal)
+		}
+		const reason: unknown = signal?.reason
+		throw reason instanceof Error ? reason : new CrawlClientError('Crawl.done was aborted.')
+	}
+
+	update(patch: {
+		readonly sync?: CrawlSyncPolicy | null
+		readonly changePolicy?: CrawlChangePolicy
+		readonly maxPages?: number
+		/** `null` clears the name. A name can be changed even on a closed or archived crawl. */
+		readonly name?: string | null
+	}): Promise<CrawlStatus> {
+		return requestJson<CrawlStatus>(this.transport, `/v1/crawls/${encodeURIComponent(this.id)}`, {
+			method: 'PATCH',
+			body: JSON.stringify(patch),
+		})
+	}
+
+	cancel(): Promise<CrawlStatus> {
+		return requestJson<CrawlStatus>(this.transport, `/v1/crawls/${encodeURIComponent(this.id)}/cancel`, {
+			method: 'POST',
+		})
+	}
+
+	/** Cancels the crawl if it is live and hides it from `list()`. It is deleted 90 days later unless unarchived. */
+	archive(): Promise<CrawlStatus> {
+		return requestJson<CrawlStatus>(this.transport, `/v1/crawls/${encodeURIComponent(this.id)}/archive`, {
+			method: 'POST',
+		})
+	}
+
+	/** Returns an archived crawl to `list()`, still canceled, and keeps it for at least 90 more days. */
+	unarchive(): Promise<CrawlStatus> {
+		return requestJson<CrawlStatus>(this.transport, `/v1/crawls/${encodeURIComponent(this.id)}/unarchive`, {
+			method: 'POST',
+		})
+	}
+
+	run(options: CrawlRunOptions): Promise<CrawlStatus> {
+		const idempotencyKey = requireIdempotencyKey(options?.idempotencyKey, 'Crawl.run')
+		return requestJson<CrawlStatus>(
+			this.transport,
+			`/v1/crawls/${encodeURIComponent(this.id)}/runs${options.replayOnly ? '/replay' : ''}`,
+			{
+				method: 'POST',
+				headers: {
+					'x-idempotency-key': idempotencyKey,
+				},
+			},
+		)
+	}
+}
+
+export class MergedCrawlResource {
+	constructor(private readonly crawls: readonly CrawlResource[]) {}
+
+	events(options: MergedCrawlReadOptions = {}): CrawlEventIterable {
+		if (typeof options.from === 'string')
+			throw new CrawlClientError('Merged Crawl reads require an offset map keyed by Crawl id.')
+		return new CrawlEventIterable(
+			(signal) => this.readItems({ ...options, signal }, (events) => events),
+			(signal) => this.readItems({ ...options, signal }, (events) => events.pages()),
+			options.signal,
+		)
+	}
+
+	private async *readItems<T>(
+		options: MergedCrawlReadOptions,
+		select: (events: CrawlEventIterable) => AsyncIterable<T>,
+	): AsyncGenerator<T> {
+		if (signalIsAborted(options.signal)) return
+		const controllers = this.crawls.map(() => new AbortController())
+		const abortChildren = () => {
+			for (const controller of controllers) controller.abort(options.signal?.reason)
+		}
+		options.signal?.addEventListener('abort', abortChildren, { once: true })
+		const iterators = this.crawls.map((crawl, index) =>
+			select(crawl.events({ from: options.from?.[crawl.id], live: options.live, signal: controllers[index]?.signal }))[
+				Symbol.asyncIterator
+			](),
+		)
+		const pending = new Map<number, Promise<{ readonly index: number; readonly result: IteratorResult<T> }>>()
+		for (const [index, iterator] of iterators.entries()) {
+			pending.set(
+				index,
+				iterator.next().then((result) => ({ index, result })),
+			)
+		}
+		try {
+			while (pending.size > 0) {
+				const next = await Promise.race(pending.values())
+				if (next.result.done) {
+					pending.delete(next.index)
+					continue
+				}
+				yield next.result.value
+				const iterator = iterators[next.index]
+				if (iterator !== undefined)
+					pending.set(
+						next.index,
+						iterator.next().then((result) => ({ index: next.index, result })),
+					)
+			}
+		} finally {
+			abortChildren()
+			options.signal?.removeEventListener('abort', abortChildren)
+			await Promise.allSettled(iterators.map((iterator) => iterator.return?.(undefined)))
+		}
+	}
+}
+
+/** Create, list and control crawls. */
+export interface CrawlsClient {
+	(id: string): CrawlResource
+	create(request: CrawlRequest, options: CrawlCreateOptions): Promise<CrawlResource[]>
+	/** The organization's crawls, newest first, each with its current status. */
+	list(options?: CrawlListOptions): Promise<CrawlListResponse>
+	merge(crawls: readonly CrawlResource[]): MergedCrawlResource
+}
+
+export function makeCrawlsClient(options: CrawlTransportOptions): CrawlsClient {
+	function byId(id: string): CrawlResource {
+		if (id.length === 0) throw new CrawlClientError('Crawl id must not be empty.')
+		return new CrawlResource(id, options)
+	}
+	return Object.assign(byId, {
+		async create(request: CrawlRequest, createOptions: CrawlCreateOptions): Promise<CrawlResource[]> {
+			const idempotencyKey = requireIdempotencyKey(createOptions?.idempotencyKey, 'Crawls.create')
+			const created = await requestJson<CreatedCrawlResponse[]>(options, '/v1/crawls', {
+				method: 'POST',
+				headers: { 'x-idempotency-key': idempotencyKey },
+				body: JSON.stringify(request),
+			})
+			return created.map(({ id }) => new CrawlResource(id, options))
+		},
+		list(listOptions: CrawlListOptions = {}): Promise<CrawlListResponse> {
+			const params = new URLSearchParams()
+			if (listOptions.limit !== undefined) params.set('limit', String(listOptions.limit))
+			if (listOptions.archived !== undefined) params.set('archived', String(listOptions.archived))
+			const query = params.size === 0 ? '' : `?${params.toString()}`
+			return requestJson<CrawlListResponse>(options, `/v1/crawls${query}`, { signal: listOptions.signal })
+		},
+		merge(crawls: readonly CrawlResource[]): MergedCrawlResource {
+			return new MergedCrawlResource(crawls)
+		},
+	})
+}