@expandai/sdk 0.20.4 → 0.21.0

package/src/Generated.ts

@@ -169,8 +169,8 @@ export class FetchBlocked extends S.Struct({
 
 export class FetchFetch403 extends S.Union(
 	S.Struct({
-		organizationId: S.String.pipe(S.minLength(1)),
 		blockedUntil: S.optionalWith(S.Union(DateTimeUtc, S.Null), { nullable: true }),
+		retryable: S.optionalWith(S.Literal(false), { default: () => false }),
 		_tag: S.Literal('AccessBlocked'),
 	}),
 	FetchBlocked,
@@ -326,12 +326,6 @@ export class FetchOriginHttpError extends S.Struct({
 
 export class FetchFetch502 extends S.Union(FetchCaptureFailed, FetchNavigationFailed, FetchOriginHttpError) {}
 
-export class ServiceUnavailableDispatchOutcome extends S.Literal(
-	'cancelled_before_start',
-	'cancelled_after_start',
-	'unknown',
-) {}
-
 export class ServiceUnavailableRetryable extends S.Literal(true) {}
 
 export class ServiceUnavailableTag extends S.Literal('ServiceUnavailable') {}
@@ -339,7 +333,6 @@ export class ServiceUnavailableTag extends S.Literal('ServiceUnavailable') {}
 export class ServiceUnavailable extends S.Struct({
 	message: S.optionalWith(S.String, { nullable: true }),
 	requestId: S.optionalWith(S.String, { nullable: true }),
-	dispatchOutcome: S.optionalWith(ServiceUnavailableDispatchOutcome, { nullable: true }),
 	retryable: S.optionalWith(ServiceUnavailableRetryable, { default: () => true }),
 	_tag: ServiceUnavailableTag,
 }) {}
@@ -579,7 +572,8 @@ export class FetchAssetReference extends S.Struct({
 	id: S.String,
 	kind: FetchAssetReferenceKind,
 	placeholder: S.String,
-	archiveKey: S.String,
+	/** @deprecated An opaque identifier equal to `sha256`. It will become optional and then be removed; use `id` or `sha256` instead. */
+	archiveKey: S.optionalWith(S.String, { nullable: true }),
 	mimeType: S.String,
 	sha256: S.String,
 	bytes: S.Number,
@@ -813,8 +807,8 @@ export class FetchFetchJson400 extends S.Union(HttpApiDecodeError, BadRequest) {
 
 export class FetchFetchJson403 extends S.Union(
 	S.Struct({
-		organizationId: S.String.pipe(S.minLength(1)),
 		blockedUntil: S.optionalWith(S.Union(DateTimeUtc, S.Null), { nullable: true }),
+		retryable: S.optionalWith(S.Literal(false), { default: () => false }),
 		_tag: S.Literal('AccessBlocked'),
 	}),
 	FetchBlocked,
@@ -903,11 +897,13 @@ export class BatchedRun extends S.Class<BatchedRun>('BatchedRun')({
 
 export class FetchBatched400 extends S.Union(HttpApiDecodeError, BadRequest) {}
 
+export class FetchBatched403Retryable extends S.Literal(false) {}
+
 export class FetchBatched403Tag extends S.Literal('AccessBlocked') {}
 
 export class FetchBatched403 extends S.Struct({
-	organizationId: S.String.pipe(S.minLength(1)),
 	blockedUntil: S.optionalWith(S.Union(DateTimeUtc, S.Null), { nullable: true }),
+	retryable: S.optionalWith(FetchBatched403Retryable, { default: () => false }),
 	_tag: FetchBatched403Tag,
 }) {}
 
@@ -1183,6 +1179,32 @@ export class BatchedCancellationConflict extends S.Class<BatchedCancellationConf
 	_tag: BatchedCancellationConflictTag,
 }) {}
 
+export class SearchSearchRequest extends S.Class<SearchSearchRequest>('SearchSearchRequest')({
+	query: S.String,
+	maxResults: S.optionalWith(Int, { nullable: true }),
+}) {}
+
+export class SearchResult extends S.Struct({
+	url: S.String.pipe(S.minLength(1)),
+	title: S.String,
+	excerpt: S.String,
+	score: S.Number,
+}) {}
+
+export class SearchResponse extends S.Class<SearchResponse>('SearchResponse')({
+	results: S.Array(SearchResult),
+}) {}
+
+export class SearchSearch403Retryable extends S.Literal(false) {}
+
+export class SearchSearch403Tag extends S.Literal('AccessBlocked') {}
+
+export class SearchSearch403 extends S.Struct({
+	blockedUntil: S.optionalWith(S.Union(DateTimeUtc, S.Null), { nullable: true }),
+	retryable: S.optionalWith(SearchSearch403Retryable, { default: () => false }),
+	_tag: SearchSearch403Tag,
+}) {}
+
 export class SnapshotHighlightsInputIncludeAppendixMarkers extends S.Literal('off', 'compact', 'detailed') {}
 
 export class SnapshotHighlightsInput extends S.Class<SnapshotHighlightsInput>('SnapshotHighlightsInput')({
@@ -1251,11 +1273,13 @@ export class FetchSnapshotHighlightsResult extends S.Class<FetchSnapshotHighligh
 
 export class SnapshotsHighlights400 extends S.Union(HttpApiDecodeError, BadRequest) {}
 
+export class SnapshotsHighlights403Retryable extends S.Literal(false) {}
+
 export class SnapshotsHighlights403Tag extends S.Literal('AccessBlocked') {}
 
 export class SnapshotsHighlights403 extends S.Struct({
-	organizationId: S.String.pipe(S.minLength(1)),
 	blockedUntil: S.optionalWith(S.Union(DateTimeUtc, S.Null), { nullable: true }),
+	retryable: S.optionalWith(SnapshotsHighlights403Retryable, { default: () => false }),
 	_tag: SnapshotsHighlights403Tag,
 }) {}
 
@@ -1308,6 +1332,140 @@ export class SnapshotNotFound extends S.Class<SnapshotNotFound>('SnapshotNotFoun
 	_tag: SnapshotNotFoundTag,
 }) {}
 
+export class PublicSnapshotsGetNetworkParamsBodies extends S.Literal('none', 'inline') {}
+
+export class PublicSnapshotsGetNetworkParamsDetail extends S.Literal('summary', 'full') {}
+
+export class PublicSnapshotsGetNetworkParams extends S.Struct({
+	q: S.optionalWith(S.String.pipe(S.minLength(1), S.maxLength(2048)), { nullable: true }),
+	urlContains: S.optionalWith(S.String.pipe(S.minLength(1), S.maxLength(2048)), { nullable: true }),
+	mime: S.optionalWith(S.String, { nullable: true }),
+	type: S.optionalWith(S.String, { nullable: true }),
+	status: S.optionalWith(S.String, { nullable: true }),
+	bodies: S.optionalWith(PublicSnapshotsGetNetworkParamsBodies, { nullable: true }),
+	detail: S.optionalWith(PublicSnapshotsGetNetworkParamsDetail, { nullable: true }),
+	limit: S.optionalWith(S.String, { nullable: true }),
+	cursor: S.optionalWith(S.String, { nullable: true }),
+}) {}
+
+export class SnapshotNetworkEntryResourceType extends S.Literal(
+	'document',
+	'fetch',
+	'script',
+	'stylesheet',
+	'image',
+	'font',
+	'media',
+	'other',
+) {}
+
+export class SnapshotNetworkHeader extends S.Struct({
+	name: S.String,
+	value: S.String,
+}) {}
+
+export class SnapshotNetworkQueryParam extends S.Struct({
+	name: S.String,
+	value: S.String,
+}) {}
+
+export class SnapshotNetworkCookie extends S.Struct({
+	name: S.String,
+	value: S.String,
+}) {}
+
+export class SnapshotNetworkRequest extends S.Struct({
+	method: S.String,
+	url: S.String,
+	httpVersion: S.String,
+	headers: S.Array(SnapshotNetworkHeader),
+	queryString: S.Array(SnapshotNetworkQueryParam),
+	cookies: S.Array(SnapshotNetworkCookie),
+	headersSize: S.Number,
+	bodySize: S.Number,
+	postData: S.optionalWith(
+		S.Struct({
+			mimeType: S.String,
+			text: S.String,
+		}),
+		{ nullable: true },
+	),
+}) {}
+
+export class SnapshotNetworkContent extends S.Struct({
+	size: S.Number,
+	mimeType: S.String,
+	text: S.optionalWith(S.String, { nullable: true }),
+	encoding: S.optionalWith(S.String, { nullable: true }),
+}) {}
+
+export class SnapshotNetworkResponse extends S.Struct({
+	status: S.Number,
+	statusText: S.String,
+	httpVersion: S.String,
+	headers: S.Array(SnapshotNetworkHeader),
+	cookies: S.Array(SnapshotNetworkCookie),
+	content: SnapshotNetworkContent,
+	redirectURL: S.String,
+	headersSize: S.Number,
+	bodySize: S.Number,
+}) {}
+
+export class SnapshotNetworkTimings extends S.Struct({
+	blocked: S.Number,
+	dns: S.Number,
+	connect: S.Number,
+	send: S.Number,
+	wait: S.Number,
+	receive: S.Number,
+	ssl: S.Number,
+}) {}
+
+export class SnapshotNetworkEntry extends S.Struct({
+	id: S.String,
+	method: S.String,
+	url: S.String,
+	status: S.Number,
+	mimeType: S.String,
+	resourceType: SnapshotNetworkEntryResourceType,
+	size: S.Number,
+	startedDateTime: S.String,
+	time: S.Number,
+	content: S.String,
+	body: S.optionalWith(
+		S.Struct({
+			text: S.String,
+			encoding: S.optionalWith(S.String, { nullable: true }),
+		}),
+		{ nullable: true },
+	),
+	request: S.optionalWith(SnapshotNetworkRequest, { nullable: true }),
+	response: S.optionalWith(SnapshotNetworkResponse, { nullable: true }),
+	requestHeaders: S.optionalWith(S.Array(SnapshotNetworkHeader), { nullable: true }),
+	requestCookies: S.optionalWith(S.Array(SnapshotNetworkCookie), { nullable: true }),
+	responseHeaders: S.optionalWith(S.Array(SnapshotNetworkHeader), { nullable: true }),
+	timings: S.optionalWith(SnapshotNetworkTimings, { nullable: true }),
+	artifact: S.optionalWith(S.String, { nullable: true }),
+}) {}
+
+export class SnapshotNetworkPage extends S.Class<SnapshotNetworkPage>('SnapshotNetworkPage')({
+	items: S.Array(SnapshotNetworkEntry),
+	next: S.optionalWith(S.String, { nullable: true }),
+}) {}
+
+export class PublicSnapshotsGetNetwork400 extends S.Union(HttpApiDecodeError, BadRequest) {}
+
+export class ArtifactUnavailableReason extends S.Literal('archive_missing', 'document_not_archived', 'not_html') {}
+
+export class ArtifactUnavailableTag extends S.Literal('ArtifactUnavailable') {}
+
+export class ArtifactUnavailable extends S.Struct({
+	reason: ArtifactUnavailableReason,
+	_tag: ArtifactUnavailableTag,
+}) {}
+
+export class PublicSnapshotsGetNetwork404 extends S.Union(SnapshotNotFound, ArtifactUnavailable) {}
+
 export const make = (
 	httpClient: HttpClient.HttpClient,
 	options: {
@@ -1472,6 +1630,28 @@ export const make = (
 					),
 				),
 			),
+		searchSearch: (options) =>
+			HttpClientRequest.make('POST')(`/v1/search`).pipe(
+				(req) => Effect.orDie(HttpClientRequest.bodyJson(req, options)),
+				Effect.flatMap((request) =>
+					Effect.flatMap(applyClientTransform(httpClient), (httpClient) =>
+						Effect.flatMap(
+							httpClient.execute(request),
+							HttpClientResponse.matchStatus({
+								'200': (r) => HttpClientResponse.schemaBodyJson(SearchResponse)(r),
+								'400': (r) => decodeError(r, HttpApiDecodeError),
+								'401': (r) => decodeError(r, AuthFailed),
+								'403': (r) => decodeError(r, SearchSearch403),
+								'413': (r) => decodeError(r, PayloadTooLarge),
+								'429': (r) => decodeError(r, TooManyRequests),
+								'500': (r) => decodeError(r, InternalError),
+								'503': (r) => decodeError(r, ServiceUnavailable),
+								orElse: (response) => unexpectedStatus(request, response),
+							}),
+						),
+					),
+				),
+			),
 		snapshotsHighlights: (snapshotId, options) =>
 			HttpClientRequest.make('POST')(`/v1/snapshots/${snapshotId}/highlights`).pipe(
 				(req) => Effect.orDie(HttpClientRequest.bodyJson(req, options)),
@@ -1517,6 +1697,38 @@ export const make = (
 					),
 				),
 			),
+		publicSnapshotsGetNetwork: (snapshotId, options) =>
+			HttpClientRequest.make('GET')(`/v1/public/snapshots/${snapshotId}/network`).pipe(
+				HttpClientRequest.setUrlParams({
+					q: options['q'] as UrlParams.Coercible,
+					urlContains: options['urlContains'] as UrlParams.Coercible,
+					mime: options['mime'] as UrlParams.Coercible,
+					type: options['type'] as UrlParams.Coercible,
+					status: options['status'] as UrlParams.Coercible,
+					bodies: options['bodies'] as UrlParams.Coercible,
+					detail: options['detail'] as UrlParams.Coercible,
+					limit: options['limit'] as UrlParams.Coercible,
+					cursor: options['cursor'] as UrlParams.Coercible,
+				}),
+				Effect.succeed,
+				Effect.flatMap((request) =>
+					Effect.flatMap(applyClientTransform(httpClient), (httpClient) =>
+						Effect.flatMap(
+							httpClient.execute(request),
+							HttpClientResponse.matchStatus({
+								'200': (r) => HttpClientResponse.schemaBodyJson(SnapshotNetworkPage)(r),
+								'400': (r) => decodeError(r, PublicSnapshotsGetNetwork400),
+								'404': (r) => decodeError(r, PublicSnapshotsGetNetwork404),
+								'413': (r) => decodeError(r, PayloadTooLarge),
+								'429': (r) => decodeError(r, TooManyRequests),
+								'500': (r) => decodeError(r, InternalError),
+								'503': (r) => decodeError(r, ServiceUnavailable),
+								orElse: (response) => unexpectedStatus(request, response),
+							}),
+						),
+					),
+				),
+			),
 	}
 }
 
@@ -1609,6 +1821,20 @@ export interface Client {
 		| typeof InternalError.Type
 		| typeof ServiceUnavailable.Type
 	>
+	readonly searchSearch: (
+		options: typeof SearchSearchRequest.Encoded,
+	) => Effect.Effect<
+		typeof SearchResponse.Type,
+		| HttpClientError.HttpClientError
+		| ParseError
+		| typeof HttpApiDecodeError.Type
+		| typeof AuthFailed.Type
+		| typeof SearchSearch403.Type
+		| typeof PayloadTooLarge.Type
+		| typeof TooManyRequests.Type
+		| typeof InternalError.Type
+		| typeof ServiceUnavailable.Type
+	>
 	readonly snapshotsHighlights: (
 		snapshotId: string,
 		options: typeof SnapshotHighlightsInput.Encoded,
@@ -1639,4 +1865,18 @@ export interface Client {
 		| typeof InternalError.Type
 		| typeof ServiceUnavailable.Type
 	>
+	readonly publicSnapshotsGetNetwork: (
+		snapshotId: string,
+		options: typeof PublicSnapshotsGetNetworkParams.Encoded,
+	) => Effect.Effect<
+		typeof SnapshotNetworkPage.Type,
+		| HttpClientError.HttpClientError
+		| ParseError
+		| typeof PublicSnapshotsGetNetwork400.Type
+		| typeof PublicSnapshotsGetNetwork404.Type
+		| typeof PayloadTooLarge.Type
+		| typeof TooManyRequests.Type
+		| typeof InternalError.Type
+		| typeof ServiceUnavailable.Type
+	>
 }

package/src/Internal.ts

@@ -29,9 +29,13 @@ import {
 	type GetBatchedParams,
 	type GetBatchedResponse,
 	type RequestOptions,
+	type SearchParams,
+	type SearchResponse,
 	type SnapshotCatalog,
 	type SnapshotGetParams,
 	type SnapshotHighlightsResponse,
+	type SnapshotNetworkPage,
+	type SnapshotNetworkParams,
 } from './Service.js'
 
 const RETRYABLE_STATUS_CODES = HashSet.make(408, 409, 429, 500, 502, 503, 504, 529)
@@ -302,12 +306,24 @@ export class InternalClient extends Effect.Service<InternalClient>()('@expandai/
 		): Effect.Effect<FetchJsonResponse, ExpandClientError> =>
 			service.fetchJson(params, options).pipe(Effect.mapError((error) => error.toPublicError()))
 
+		const search = (
+			params: SearchParams,
+			options: RequestOptions = {},
+		): Effect.Effect<SearchResponse, ExpandClientError> =>
+			service.search(params, options).pipe(Effect.mapError((error) => error.toPublicError()))
+
 		const getSnapshot = (
 			params: SnapshotGetParams,
 			options: RequestOptions = {},
 		): Effect.Effect<SnapshotCatalog | SnapshotHighlightsResponse, ExpandClientError> =>
 			service.snapshots.get(params, options).pipe(Effect.mapError((error) => error.toPublicError()))
 
+		const getSnapshotNetwork = (
+			params: SnapshotNetworkParams,
+			options: RequestOptions = {},
+		): Effect.Effect<SnapshotNetworkPage, ExpandClientError> =>
+			service.snapshots.network(params, options).pipe(Effect.mapError((error) => error.toPublicError()))
+
 		const batched = (
 			params: BatchedParams,
 			options: BatchedRequestOptions,
@@ -326,7 +342,7 @@ export class InternalClient extends Effect.Service<InternalClient>()('@expandai/
 		): Effect.Effect<CancelBatchedResponse, ExpandClientError> =>
 			service.cancelBatched(id, options).pipe(Effect.mapError((error) => error.toPublicError()))
 
-		return { fetch, fetchJson, getSnapshot, batched, getBatched, cancelBatched } as const
+		return { fetch, fetchJson, search, getSnapshot, getSnapshotNetwork, batched, getBatched, cancelBatched } as const
 	}),
 }) {
 	/**

package/src/Service.ts

@@ -88,6 +88,9 @@ interface LocalOptions {
 	readonly local?: boolean
 }
 
+export type SearchParams = typeof Generated.SearchSearchRequest.Encoded
+export type SearchResponse = typeof Generated.SearchResponse.Type
+
 export type FetchParams = typeof Generated.FetchFetchRequest.Encoded
 export type FetchResponse = string
 export type FetchRequestOptions = RequestOptions & typeof Generated.FetchFetchParams.Encoded & LocalOptions
@@ -110,6 +113,47 @@ export interface SnapshotGetParams {
 	readonly include?: SnapshotHighlightsParams['include']
 }
 export type SnapshotCatalog = typeof Generated.SnapshotCatalog.Type
+/**
+ * `json`, `csv`, `html` and `file` match the response content; the rest match how the browser
+ * requested the resource. `xhr` is the same as `fetch`.
+ */
+export type SnapshotNetworkType =
+	| 'json'
+	| 'csv'
+	| 'html'
+	| 'file'
+	| 'document'
+	| 'fetch'
+	| 'xhr'
+	| 'script'
+	| 'stylesheet'
+	| 'image'
+	| 'font'
+	| 'media'
+	| 'other'
+/** Filters and paging for a Snapshot's captured network requests. Lists combine with OR, fields with AND. */
+export interface SnapshotNetworkParams {
+	readonly snapshotId: string
+	/** Case-insensitive substring of the request URL. */
+	readonly q?: string
+	/** Case-insensitive substring of the request URL; with `q`, both must match. */
+	readonly urlContains?: string
+	/** Case-insensitive substrings of the response MIME type, such as `json` or `text/csv`. */
+	readonly mime?: string | readonly string[]
+	readonly type?: SnapshotNetworkType | readonly SnapshotNetworkType[]
+	/** Status codes or classes, such as `200` or `'4xx'`. `0` is a request that got no response. */
+	readonly status?: number | string | ReadonlyArray<number | string>
+	/** `inline` (default) returns the response bodies the capture kept; `none` omits them. Request payloads stay in `detail: 'full'`. */
+	readonly bodies?: 'none' | 'inline'
+	/** `full` (default) adds HAR request and response detail; `summary` keeps each entry small. */
+	readonly detail?: 'summary' | 'full'
+	/** Entries per page, 1 to 1000. Without it the server returns every matching entry and no `next`. */
+	readonly limit?: number
+	/** The previous page's `next`, with the same filters. Without `limit`, returns the rest of the listing. */
+	readonly cursor?: string
+}
+export type SnapshotNetworkPage = typeof Generated.SnapshotNetworkPage.Type
+export type SnapshotNetworkEntry = typeof Generated.SnapshotNetworkEntry.Type
 export type SnapshotHighlightsResponse = FetchSnapshotHighlightsResultWithJson
 export type BatchedParams = typeof Generated.FetchBatchedRequest.Encoded
 export type BatchedResponse = typeof Generated.BatchedRun.Type
@@ -329,6 +373,13 @@ export class ExpandService extends Effect.Service<ExpandService>()('@expandai/sd
 			)
 		})
 
+		const search = Effect.fn('ExpandService.search')(function* (
+			params: SearchParams,
+			requestOptions: RequestOptions = {},
+		) {
+			return yield* withRetries(generated.searchSearch(params), requestOptions)
+		})
+
 		const publicClient = httpClient.pipe(
 			HttpClient.mapRequest(HttpClientRequest.prependUrl(config.baseUrl)),
 			recordRetryAfter,
@@ -363,6 +414,25 @@ export class ExpandService extends Effect.Service<ExpandService>()('@expandai/sd
 					)
 		}
 
+		const commaList = (value: string | number | ReadonlyArray<string | number> | undefined) =>
+			value === undefined ? undefined : typeof value === 'object' ? value.join(',') : String(value)
+		const listSnapshotNetwork = (
+			params: SnapshotNetworkParams,
+			options: RequestOptions = {},
+		): Effect.Effect<SnapshotNetworkPage, ExpandError> => {
+			const { snapshotId, mime, type, status, limit, ...filters } = params
+			return withRetries(
+				Generated.make(publicClient).publicSnapshotsGetNetwork(encodeURIComponent(snapshotId), {
+					...filters,
+					mime: commaList(mime),
+					type: commaList(type),
+					status: commaList(status),
+					limit: commaList(limit),
+				}),
+				options,
+			)
+		}
+
 		const batched = Effect.fn('ExpandService.batched')(function* (
 			params: BatchedParams,
 			requestOptions: BatchedRequestOptions = {},
@@ -524,7 +594,16 @@ export class ExpandService extends Effect.Service<ExpandService>()('@expandai/sd
 					Effect.map((created) => created.map(({ id }) => crawlById(id))),
 				),
 		})
-		return { fetch, fetchJson, snapshots: { get: getSnapshot }, batched, getBatched, cancelBatched, crawls } as const
+		return {
+			fetch,
+			fetchJson,
+			search,
+			snapshots: { get: getSnapshot, network: listSnapshotNetwork },
+			batched,
+			getBatched,
+			cancelBatched,
+			crawls,
+		} as const
 	}),
 }) {
 	/**

package/src/index.ts

@@ -22,11 +22,17 @@ export type {
 	FetchResponse,
 	GetBatchedParams,
 	GetBatchedResponse,
+	SearchParams,
+	SearchResponse,
 	SnapshotCatalog,
 	SnapshotCatalogParams,
 	SnapshotGetParams,
 	SnapshotHighlightsParams,
 	SnapshotHighlightsResponse,
+	SnapshotNetworkEntry,
+	SnapshotNetworkPage,
+	SnapshotNetworkParams,
+	SnapshotNetworkType,
 } from './Client.js'
 export { Expand, ExpandClient } from './Client.js'
 export * from './Crawls.js'