@expandai/sdk 0.19.0 → 0.20.0

package/src/Service.ts

@@ -1,16 +1,26 @@
 import { HttpClient, HttpClientRequest, HttpClientResponse } from '@effect/platform'
-import { NodeHttpClient } from '@effect/platform-node'
-import { Context, Duration, Effect, flow, Layer, Option, Random, Schedule, Schema } from 'effect'
-import { ExpandSdkError } from './Error.js'
+import { Cause, Context, Duration, Effect, flow, Layer, MutableRef, Option, Random, Schedule, Schema } from 'effect'
+import type {
+	CrawlChangePolicy,
+	CrawlCreateOptions,
+	CrawlListResponse,
+	CrawlRequest,
+	CrawlRunOptions,
+	CrawlStatus,
+	CrawlSyncPolicy,
+} from './Crawls.js'
+import * as EnvProxyHttpClient from './EnvProxyHttpClient.js'
+import { ExpandApiError, type ExpandError, ExpandSdkError } from './Error.js'
 import {
-	type FetchObjectModeResultWithJson,
-	type FetchSnapshotSearchResultWithJson,
+	type FetchObjectModeJsonResponse,
+	type FetchSnapshotHighlightsResultWithJson,
 	fetchObjectModeJson,
-	fetchSnapshotSearch,
+	fetchSnapshotHighlights,
 } from './FetchJson.js'
 import * as Generated from './Generated.js'
 import { decodeError, unexpectedStatus } from './HttpClientHelpers.js'
 import { isRetryable, toExpandError } from './Internal.js'
+import { nextRetryDelay, recordRetryAfter } from './RetryAfter.js'
 
 const DEFAULT_BASE_URL = 'https://api.expand.ai'
 const DEFAULT_TIMEOUT_MS = 60_000
@@ -74,11 +84,24 @@ export type FetchResponse = string
 export type FetchRequestOptions = RequestOptions & typeof Generated.FetchFetchParams.Encoded
 export type FetchJsonParams = typeof Generated.FetchFetchJsonRequest.Encoded
 // `*WithJson` rather than the generated result: the codegen drops the statejson snippet `json`, so
-// the service decodes the body with the recovering schemas (see `FetchJson.ts`).
-export type FetchJsonResponse = FetchObjectModeResultWithJson
+// the service decodes the body with the recovering schemas (see `FetchJson.ts`). A union because an
+// origin that never answered has no content to describe — narrow on `_tag`.
+export type FetchJsonResponse = FetchObjectModeJsonResponse
 export type FetchJsonRequestOptions = RequestOptions & typeof Generated.FetchFetchJsonParams.Encoded
-export type FetchSearchParams = typeof Generated.FetchFetchSearchRequest.Encoded
-export type FetchSearchResponse = FetchSnapshotSearchResultWithJson
+export interface SnapshotHighlightsParams extends Schema.Schema.Encoded<typeof Generated.SnapshotHighlightsInput> {
+	readonly snapshotId: string
+}
+export interface SnapshotCatalogParams {
+	readonly snapshotId: string
+	readonly highlights?: undefined
+}
+export interface SnapshotGetParams {
+	readonly snapshotId: string
+	readonly highlights?: SnapshotHighlightsParams['highlights']
+	readonly include?: SnapshotHighlightsParams['include']
+}
+export type SnapshotCatalog = typeof Generated.SnapshotCatalog.Type
+export type SnapshotHighlightsResponse = FetchSnapshotHighlightsResultWithJson
 export type BatchedParams = typeof Generated.FetchBatchedRequest.Encoded
 export type BatchedResponse = typeof Generated.BatchedRun.Type
 export interface BatchedRequestOptions extends RequestOptions {
@@ -126,6 +149,12 @@ export const buildServiceConfig = Effect.fn('buildServiceConfig')(function* (opt
 	if (!apiKey) {
 		return yield* new ExpandSdkError({ message: 'Missing Expand API key. Pass apiKey or set EXPAND_API_KEY.' })
 	}
+	// Checked here, not only in the HttpClient layer, so `new ExpandClient()` keeps its
+	// documented sync-throw contract for bad configuration instead of rejecting later.
+	const invalidProxy = EnvProxyHttpClient.invalidProxyEnvMessage()
+	if (Option.isSome(invalidProxy)) {
+		return yield* new ExpandSdkError({ message: invalidProxy.value })
+	}
 	return yield* Schema.decode(ServiceConfig)({
 		apiKey,
 		baseUrl: options.baseUrl ?? DEFAULT_BASE_URL,
@@ -148,7 +177,7 @@ export const buildServiceConfig = Effect.fn('buildServiceConfig')(function* (opt
  * ```
  */
 export class ExpandService extends Effect.Service<ExpandService>()('@expandai/sdk/ExpandService', {
-	dependencies: [NodeHttpClient.layer],
+	dependencies: [EnvProxyHttpClient.layer],
 	effect: Effect.gen(function* () {
 		const config = yield* ServiceConfigTag
 		const httpClient = yield* HttpClient.HttpClient
@@ -159,6 +188,9 @@ export class ExpandService extends Effect.Service<ExpandService>()('@expandai/sd
 					HttpClientRequest.setHeader('x-expand-api-key', config.apiKey),
 				),
 			),
+			// Wraps every path, generated and hand-written alike, so `withRetries` can honour a
+			// `Retry-After` no matter which call produced the failure.
+			recordRetryAfter,
 		)
 		const generated = Generated.make(authedClient)
 		const fetchMarkdown = (params: FetchParams, queryParams: typeof Generated.FetchFetchParams.Encoded = {}) =>
@@ -175,11 +207,15 @@ export class ExpandService extends Effect.Service<ExpandService>()('@expandai/sd
 							'400': (response) => decodeError(response, Generated.FetchFetch400),
 							'401': (response) => decodeError(response, Generated.AuthFailed),
 							'403': (response) => decodeError(response, Generated.FetchFetch403),
-							'413': (response) => decodeError(response, Generated.PayloadTooLarge),
-							'429': (response) => decodeError(response, Generated.TooManyRequests),
+							'404': (response) => decodeError(response, Generated.FetchOriginNotFound),
+							'410': (response) => decodeError(response, Generated.FetchOriginGone),
+							'413': (response) => decodeError(response, Generated.FetchFetch413),
+							'415': (response) => decodeError(response, Generated.FetchUnsupportedContentType),
+							'422': (response) => decodeError(response, Generated.FetchVideoUnavailable),
+							'429': (response) => decodeError(response, Generated.FetchFetch429),
 							'500': (response) => decodeError(response, Generated.InternalError),
-							'502': (response) => decodeError(response, Generated.FetchNavigationFailed),
-							'503': (response) => decodeError(response, Generated.ServiceUnavailable),
+							'502': (response) => decodeError(response, Generated.FetchFetch502),
+							'503': (response) => decodeError(response, Generated.FetchFetch503),
 							'504': (response) => decodeError(response, Generated.FetchCaptureTimeout),
 							'529': (response) => decodeError(response, Generated.FetchCapacityTimeout),
 							orElse: (response) => unexpectedStatus(request, response),
@@ -196,16 +232,38 @@ export class ExpandService extends Effect.Service<ExpandService>()('@expandai/sd
 		) {
 			const totalTimeoutMs = requestOptions.timeoutMs ?? config.timeoutMs
 			const maxRetries = requestOptions.maxRetries ?? config.maxRetries
+			// The failure the schedule last chose to wait on, while the fiber is inside that wait.
+			// If the wall-clock budget expires mid-wait, the caller is handed this refusal rather
+			// than a timeout: the server said "not before X", the caller said "no later than Y",
+			// and a timeout would hide the 429/503 the caller branches on. Cleared before every
+			// attempt, so a budget that expires during a live attempt still reports as a timeout.
+			const pendingRefusal = MutableRef.make<Option.Option<unknown>>(Option.none())
 			// `timeoutMs` is one wall-clock budget for the operation, including retries and backoff. A slow
 			// valid capture may use most of that budget; only failures that arrive before it expires can retry.
 			const retrySchedule = Schedule.exponential(Duration.millis(250)).pipe(
 				Schedule.jittered,
 				Schedule.compose(Schedule.recurs(maxRetries)),
 				Schedule.whileInput((error: unknown) => isRetryable(error, totalTimeoutMs)),
+				// `passthrough` republishes the failure as the schedule's output so the delay function
+				// can see which error it is about to wait on.
+				Schedule.passthrough,
+				// Floors the delay at the server's requested wait (re-jittered) and caps it at
+				// `MAX_RETRY_AFTER`; the whole policy lives with its inputs in `RetryAfter.ts`.
+				Schedule.modifyDelayEffect((error: unknown, backoff) =>
+					Effect.zipLeft(
+						nextRetryDelay(error, backoff),
+						Effect.sync(() => MutableRef.set(pendingRefusal, Option.some(error))),
+					),
+				),
 			)
-			return yield* operation.pipe(
+			return yield* Effect.sync(() => MutableRef.set(pendingRefusal, Option.none())).pipe(
+				Effect.zipRight(operation),
 				Effect.retry(retrySchedule),
-				Effect.timeout(Duration.millis(totalTimeoutMs)),
+				Effect.timeoutFail({
+					duration: Duration.millis(totalTimeoutMs),
+					onTimeout: (): unknown =>
+						Option.getOrElse(MutableRef.get(pendingRefusal), () => new Cause.TimeoutException()),
+				}),
 				Effect.mapError((error) => toExpandError(error, totalTimeoutMs)),
 			)
 		})
@@ -231,13 +289,39 @@ export class ExpandService extends Effect.Service<ExpandService>()('@expandai/sd
 			)
 		})
 
-		const fetchSearch = Effect.fn('ExpandService.fetchSearch')(function* (
-			params: FetchSearchParams,
-			requestOptions: RequestOptions = {},
-		) {
-			// `fetchSnapshotSearch` mirrors the generated call but preserves the statejson snippet `json`.
-			return yield* withRetries(fetchSnapshotSearch(authedClient, params), requestOptions)
-		})
+		const publicClient = httpClient.pipe(
+			HttpClient.mapRequest(HttpClientRequest.prependUrl(config.baseUrl)),
+			recordRetryAfter,
+		)
+		function getSnapshot(
+			params: SnapshotHighlightsParams,
+			options?: RequestOptions,
+		): Effect.Effect<SnapshotHighlightsResponse, ExpandError>
+		function getSnapshot(
+			params: SnapshotCatalogParams,
+			options?: RequestOptions,
+		): Effect.Effect<SnapshotCatalog, ExpandError>
+		function getSnapshot(
+			params: SnapshotGetParams,
+			options?: RequestOptions,
+		): Effect.Effect<SnapshotCatalog | SnapshotHighlightsResponse, ExpandError>
+		function getSnapshot(
+			params: SnapshotGetParams,
+			options: RequestOptions = {},
+		): Effect.Effect<SnapshotCatalog | SnapshotHighlightsResponse, ExpandError> {
+			return params.highlights === undefined
+				? withRetries(
+						Generated.make(publicClient).publicSnapshotsGetCatalog(encodeURIComponent(params.snapshotId), {}),
+						options,
+					)
+				: withRetries(
+						fetchSnapshotHighlights(authedClient, params.snapshotId, {
+							highlights: params.highlights,
+							include: params.include,
+						}),
+						options,
+					)
+		}
 
 		const batched = Effect.fn('ExpandService.batched')(function* (
 			params: BatchedParams,
@@ -301,7 +385,106 @@ export class ExpandService extends Effect.Service<ExpandService>()('@expandai/sd
 			return yield* withRetries(generated.fetchCancelBatched(id), requestOptions)
 		})
 
-		return { fetch, fetchJson, fetchSearch, batched, getBatched, cancelBatched } as const
+		// Crawl control routes intentionally do not use the fetch-only generator. They still
+		// share this service's authenticated client, wall-clock budget, and retry policy.
+		const crawlRequest = Effect.fn('ExpandService.crawlRequest')(function* <A>(
+			path: string,
+			method: 'GET' | 'POST' | 'PATCH',
+			options: RequestOptions = {},
+			body?: unknown,
+			idempotencyKey?: string,
+		) {
+			const request = HttpClientRequest.make(method)(path)
+			const keyed =
+				idempotencyKey === undefined
+					? request
+					: HttpClientRequest.setHeader(request, 'x-idempotency-key', idempotencyKey)
+			const encoded =
+				body === undefined
+					? keyed
+					: yield* HttpClientRequest.bodyJson(keyed, body).pipe(
+							Effect.mapError((cause) => new ExpandSdkError({ message: 'Invalid Crawl request body.', cause })),
+						)
+			return yield* withRetries(
+				authedClient.execute(encoded).pipe(
+					Effect.flatMap((response) =>
+						(response.status >= 200 && response.status < 300
+							? response.json
+							: response.json.pipe(Effect.catchAll(() => Effect.void))
+						).pipe(
+							Effect.flatMap((value) => {
+								if (response.status >= 200 && response.status < 300) {
+									// The hand-written interfaces mirror the versioned Crawl HTTP contract.
+									return Effect.succeed(value as unknown as A)
+								}
+								const error = typeof value === 'object' && value !== null && 'error' in value ? value.error : undefined
+								const message =
+									typeof error === 'object' && error !== null && 'message' in error && typeof error.message === 'string'
+										? error.message
+										: `HTTP ${response.status}`
+								return new ExpandApiError({ message, status: response.status, body: value })
+							}),
+						),
+					),
+				),
+				options,
+			)
+		})
+		// Mirrors the Promise client: a durable caller key is required before any request is issued,
+		// or an unretryable failure could not be replayed by key.
+		// Takes `string | undefined`: TypeScript does not stop a JavaScript caller, and
+		// reading `.length` off undefined threw where the contract promises a typed failure.
+		const requireCrawlIdempotencyKey = (
+			value: string | undefined,
+			operation: string,
+		): Effect.Effect<string, ExpandSdkError> =>
+			value !== undefined && value.length >= 1 && value.length <= 255
+				? Effect.succeed(value)
+				: new ExpandSdkError({ message: `${operation} requires an idempotencyKey between 1 and 255 characters.` })
+		function crawlById(id: string) {
+			const path = `/v1/crawls/${encodeURIComponent(id)}`
+			return {
+				id,
+				status: (options: RequestOptions = {}) => crawlRequest<CrawlStatus>(path, 'GET', options),
+				cancel: (options: RequestOptions = {}) => crawlRequest<CrawlStatus>(`${path}/cancel`, 'POST', options),
+				update: (
+					patch: {
+						readonly sync?: CrawlSyncPolicy | null
+						readonly changePolicy?: CrawlChangePolicy
+						readonly maxPages?: number
+					},
+					options: RequestOptions = {},
+				) => crawlRequest<CrawlStatus>(path, 'PATCH', options, patch),
+				run: (options: CrawlRunOptions & RequestOptions) =>
+					requireCrawlIdempotencyKey(options.idempotencyKey, 'Crawl.run').pipe(
+						Effect.flatMap((idempotencyKey) =>
+							crawlRequest<CrawlStatus>(
+								`${path}/runs${options.replayOnly ? '/replay' : ''}`,
+								'POST',
+								options,
+								undefined,
+								idempotencyKey,
+							),
+						),
+					),
+			}
+		}
+		const crawls = Object.assign(crawlById, {
+			list: (options: { readonly limit?: number } & RequestOptions = {}) =>
+				crawlRequest<CrawlListResponse>(
+					options.limit === undefined ? '/v1/crawls' : `/v1/crawls?limit=${encodeURIComponent(options.limit)}`,
+					'GET',
+					options,
+				),
+			create: (request: CrawlRequest, options: CrawlCreateOptions & RequestOptions) =>
+				requireCrawlIdempotencyKey(options.idempotencyKey, 'Crawls.create').pipe(
+					Effect.flatMap((idempotencyKey) =>
+						crawlRequest<readonly { readonly id: string }[]>('/v1/crawls', 'POST', options, request, idempotencyKey),
+					),
+					Effect.map((created) => created.map(({ id }) => crawlById(id))),
+				),
+		})
+		return { fetch, fetchJson, snapshots: { get: getSnapshot }, batched, getBatched, cancelBatched, crawls } as const
 	}),
 }) {
 	/**

package/src/StreamPage.ts

@@ -0,0 +1,47 @@
+import { BackoffDefaults, type JsonBatch, type StreamResponse, stream } from '@durable-streams/client'
+
+/** live:false reads one bounded server response, never the complete retained history. */
+export async function readStreamPage<T>(response: StreamResponse<T>): Promise<JsonBatch<T>> {
+	if (response.live !== false) throw new Error('A bounded stream page requires live:false.')
+	try {
+		const items = await response.json<T>()
+		return {
+			items,
+			offset: response.offset,
+			upToDate: response.upToDate,
+			streamClosed: response.streamClosed,
+			cursor: response.cursor,
+		}
+	} finally {
+		response.cancel()
+	}
+}
+
+/** Decode bounded responses with the official protocol client while preserving the caller's auth transport. */
+export class DurableStreamReader {
+	private readonly cursors = new Map<string, string>()
+
+	cursor(key: string): string | undefined {
+		return this.cursors.get(key)
+	}
+
+	async read<T>(key: string, response: Response, offset: string): Promise<JsonBatch<T>> {
+		const page = await readStreamPage(
+			await stream<T>({
+				url: response.url || 'https://stream.invalid/',
+				offset,
+				live: false,
+				json: true,
+				fetch: async () => response,
+				backoffOptions: { ...BackoffDefaults, maxRetries: 0 },
+			}),
+		)
+		this.cursors.delete(key)
+		if (!page.streamClosed && page.cursor !== undefined) this.cursors.set(key, page.cursor)
+		if (this.cursors.size > 128) {
+			const oldest = this.cursors.keys().next().value
+			if (oldest !== undefined) this.cursors.delete(oldest)
+		}
+		return page
+	}
+}

package/src/index.ts

@@ -1,3 +1,13 @@
+export type { AppendixMarker } from './AppendixMarkers.js'
+export {
+	APPENDIX_MARKER_PREFIX,
+	AppendixMarkerLevel,
+	isAppendixMarkerLine,
+	parseAppendixMarkerLine,
+	parseAppendixMarkers,
+	stripAppendixMarkers,
+	withoutFencedCode,
+} from './AppendixMarkers.js'
 export type {
 	BatchedParams,
 	BatchedRequestOptions,
@@ -10,12 +20,16 @@ export type {
 	FetchParams,
 	FetchRequestOptions,
 	FetchResponse,
-	FetchSearchParams,
-	FetchSearchResponse,
 	GetBatchedParams,
 	GetBatchedResponse,
+	SnapshotCatalog,
+	SnapshotCatalogParams,
+	SnapshotGetParams,
+	SnapshotHighlightsParams,
+	SnapshotHighlightsResponse,
 } from './Client.js'
-export { ExpandClient } from './Client.js'
+export { Expand, ExpandClient } from './Client.js'
+export * from './Crawls.js'
 export type { ExpandError } from './Error.js'
 export {
 	ExpandApiError,
@@ -27,4 +41,16 @@ export {
 	ExpandSdkError,
 	ExpandTimeoutError,
 } from './Error.js'
+export type { FetchOriginErrorResponse } from './FetchJson.js'
+export { isFetchOriginError } from './FetchJson.js'
+export type { SnapshotHandle } from './Playground.js'
+export {
+	citationUrl,
+	DEFAULT_PLAYGROUND_HOST,
+	parseSnapshotHandle,
+	playgroundAppendixUrl,
+	playgroundBase,
+	playgroundContentUrl,
+	resolvePlaygroundHost,
+} from './Playground.js'
 export { ExpandService, ExpandServiceOptions, RequestOptions } from './Service.js'