@expandai/sdk 0.20.1 → 0.20.2

package/src/Tunnel/Tunnel.ts

@@ -0,0 +1,420 @@
+import { createServer } from 'node:http'
+import {
+	Cookies,
+	FetchHttpClient,
+	HttpBody,
+	HttpClient,
+	HttpClientRequest,
+	type HttpMethod,
+	Path,
+	Socket,
+} from '@effect/platform'
+import { NodeFileSystem, NodeHttpServer, NodePath } from '@effect/platform-node'
+import { Config, Deferred, Duration, Effect, FiberMap, Iterable, Layer, Match, Option, Schema, Stream } from 'effect'
+import { WebSocket } from 'ws'
+import { ExpandLocalTargetError } from '../Error.js'
+import { endToEndHeaders } from './Headers.js'
+import { LocalTarget, resolveLocalTarget, resolveSafeDirectory } from './LocalTarget.js'
+import {
+	type ClientToRelay,
+	decodeBodyFrame,
+	encodeBodyFrame,
+	encodeClientToRelay,
+	type Hello,
+	MAX_BODY_BYTES,
+	MAX_INFLIGHT_REQUESTS,
+	PROTOCOL_VERSION,
+	RelayToClient,
+	type RequestId,
+	type RequestStart,
+	splitBodyChunk,
+} from './Protocol.js'
+import { serveDirectory, serveFile } from './StaticServer.js'
+
+export class TunnelError extends Schema.TaggedError<TunnelError>()('TunnelError', {
+	reason: Schema.Literal('Unauthorized', 'Configuration', 'RelayUnreachable'),
+	message: Schema.String,
+	cause: Schema.optional(Schema.Defect),
+}) {}
+
+/** What the relay needs to accept a tunnel: the API base URL it derives from, and the caller's credentials. */
+interface TunnelCredentials {
+	readonly baseUrl: string
+	/** `x-expand-api-key`, or the CLI's OAuth `authorization`. */
+	readonly headers: Readonly<Record<string, string>>
+}
+
+const HELLO_TIMEOUT = Duration.seconds(10)
+const MAX_QUEUED_RELAY_BYTES = 16 * 1024 * 1024
+// A masked WebSocket frame has at most 14 bytes of framing in addition to its payload.
+const MAX_WEBSOCKET_FRAME_OVERHEAD = 14
+const RELAY_URLS = new Map([
+	['https://api.expand.ai', 'https://expandusercontent.com'],
+	['https://api.staging.expand.ai', 'https://staging.expandusercontent.com'],
+])
+const decodeControlFrame = Schema.decodeUnknownOption(Schema.parseJson(RelayToClient))
+const decodeBody = Option.liftThrowable(decodeBodyFrame)
+// Hop-by-hop or recomputed by `fetch`; forwarding them breaks the local request.
+const DROPPED_REQUEST_HEADERS = new Set(['host', 'content-length', 'accept-encoding'])
+// `fetch` decoded the body; cookies are forwarded separately to preserve repeated fields.
+const DROPPED_RESPONSE_HEADERS = new Set(['content-encoding', 'content-length', 'set-cookie'])
+
+function rewriteLocalLocation(value: string, localUrl: string, publicUrl: string): string {
+	if (!URL.canParse(value)) return value
+	const location = new URL(value)
+	return location.origin === new URL(localUrl).origin
+		? `${publicUrl}${location.pathname}${location.search}${location.hash}`
+		: value
+}
+
+// `fetch` reports "fetch failed" and hides the reason in `cause`; a refused connection is an
+// `AggregateError` (one attempt per address) with an empty message and the reason in `code`.
+function describeFetchError(error: unknown): string {
+	const cause = error instanceof Error && error.cause instanceof Error ? error.cause : error
+	if (!(cause instanceof Error)) return String(cause)
+	return cause.message || ('code' in cause && typeof cause.code === 'string' ? cause.code : cause.name)
+}
+
+/** Uses the relay for the selected API environment, unless `EXPAND_TUNNEL_URL` says otherwise. */
+const relayUrlFor = Effect.fn('Tunnel.relayUrlFor')(function* (baseUrl: string) {
+	const override = yield* Config.option(Config.string('EXPAND_TUNNEL_URL')).pipe(Effect.orDie)
+	const configured = Option.orElse(override, () => Option.fromNullable(RELAY_URLS.get(baseUrl.replace(/\/+$/u, ''))))
+	if (Option.isNone(configured)) {
+		return yield* new TunnelError({
+			reason: 'Configuration',
+			message: `No tunnel relay is known for ${baseUrl}; set EXPAND_TUNNEL_URL.`,
+		})
+	}
+	const relayUrl = configured.value
+	if (!URL.canParse(relayUrl) || !['http:', 'https:'].includes(new URL(relayUrl).protocol)) {
+		return yield* new TunnelError({
+			reason: 'Configuration',
+			message: 'EXPAND_TUNNEL_URL must be an http:// or https:// relay address.',
+		})
+	}
+	return relayUrl.replace(/\/+$/u, '')
+})
+
+function tunnelOpenError(relayUrl: string, status: number): TunnelError {
+	if (status === 401 || status === 403) {
+		return new TunnelError({ reason: 'Unauthorized', message: 'The tunnel relay rejected the Expand credentials.' })
+	}
+	const retryable = status === 0 || status === 408 || status === 429 || status >= 500
+	return new TunnelError({
+		reason: retryable ? 'RelayUnreachable' : 'Configuration',
+		message: `Could not open a tunnel at ${relayUrl}${status === 0 ? '' : ` (HTTP ${status})`}.`,
+	})
+}
+
+// Effect owns receiving and socket lifetime; native send callbacks supply the backpressure its writer lacks.
+const makeRelaySocket = Effect.fn('Tunnel.makeRelaySocket')(function* (
+	relayUrl: string,
+	headers: Readonly<Record<string, string>>,
+) {
+	const opened = yield* Deferred.make<WebSocket>()
+	let rejectedStatus = 0
+	const acquire = Effect.acquireRelease(
+		Effect.sync(() => {
+			// Disable compression so bufferedAmount plus frame overhead bounds the actual native send queue.
+			const ws = new WebSocket(`${relayUrl.replace(/^http/u, 'ws')}/v1/connect`, { headers, perMessageDeflate: false })
+			ws.once('open', () => Deferred.unsafeDone(opened, Effect.succeed(ws)))
+			ws.once('unexpected-response', (_request, response) => {
+				rejectedStatus = response.statusCode ?? 0
+				ws.terminate()
+			})
+			return ws
+		}),
+		// Graceful close queues behind body frames and can retain them for ws's 30-second close timeout.
+		(ws) => Effect.sync(() => ws.terminate()),
+	)
+	const socket = yield* Socket.fromWebSocket(
+		// `ws` implements the WHATWG event interface `fromWebSocket` uses.
+		acquire.pipe(Effect.map((ws) => ws as unknown as globalThis.WebSocket)),
+	)
+	const writer = Effect.map(socket.writer, (closeWrite) =>
+		Effect.fnUntraced(function* (chunk: Uint8Array | string | Socket.CloseEvent) {
+			if (Socket.isCloseEvent(chunk)) return yield* closeWrite(chunk)
+			const ws = yield* Deferred.await(opened)
+			return yield* Effect.async<void, Socket.SocketError>((resume) => {
+				const fail = (cause: unknown) => Effect.fail(new Socket.SocketGenericError({ reason: 'Write', cause }))
+				if (ws.readyState !== WebSocket.OPEN) {
+					resume(fail(new Error('Tunnel connection closed.')))
+					return
+				}
+				const bytes = typeof chunk === 'string' ? Buffer.byteLength(chunk) : chunk.byteLength
+				if (ws.bufferedAmount + bytes + MAX_WEBSOCKET_FRAME_OVERHEAD > MAX_QUEUED_RELAY_BYTES) {
+					// Cancelled requests leave native writes queued, so overflow must end the whole transport.
+					ws.terminate()
+					resume(fail(new Error('Tunnel relay stopped reading.')))
+					return
+				}
+				let complete: typeof resume | undefined = resume
+				try {
+					ws.send(chunk, (error) => {
+						complete?.(error ? fail(error) : Effect.void)
+						complete = undefined
+					})
+				} catch (error: unknown) {
+					complete = undefined
+					resume(fail(error))
+				}
+				// ws retains the send callback until flushing or close; don't retain an interrupted request fiber.
+				return Effect.sync(() => {
+					complete = undefined
+				})
+			})
+		}),
+	)
+	return { socket: Socket.Socket.of({ ...socket, writer }), rejectedStatus: () => rejectedStatus }
+})
+
+// Where the relay's requests go, and the path on the public URL that shows the target.
+const localOrigin = Effect.fn('Tunnel.localOrigin')(
+	function* (target: LocalTarget) {
+		if (LocalTarget.$is('Port')(target)) return { host: target.host, port: target.port, path: target.path }
+		const path = yield* Path.Path
+		const isFile = LocalTarget.$is('File')(target)
+		const directory = isFile ? target.path : yield* resolveSafeDirectory(target.path)
+		// Build into the tunnel's scope so the listener stays open after this function returns.
+		const port = yield* Layer.build(NodeHttpServer.layer(() => createServer(), { host: '127.0.0.1', port: 0 })).pipe(
+			Effect.flatMap((context) =>
+				(isFile ? serveFile(target.path) : serveDirectory(directory)).pipe(Effect.provide(context)),
+			),
+			Effect.mapError(
+				(cause) =>
+					new ExpandLocalTargetError({
+						reason: 'Unavailable',
+						input: target.path,
+						cause,
+						message: `Could not start a local file server for "${target.path}". ${cause.message}`,
+					}),
+			),
+		)
+		return { host: '127.0.0.1', port, path: isFile ? `/${encodeURIComponent(path.basename(target.path))}` : '/' }
+	},
+	Effect.provide(Layer.merge(NodePath.layer, NodeFileSystem.layer)),
+)
+
+const makeResponder = Effect.fn('Tunnel.makeResponder')(function* (
+	socket: Socket.Socket,
+	localUrl: string,
+	hello: Deferred.Deferred<typeof Hello.Type, TunnelError>,
+) {
+	// Loopback requests must not inherit API proxies or trace user paths and forwarded credentials.
+	const http = HttpClient.withScope(yield* HttpClient.HttpClient).pipe(
+		HttpClient.withTracerDisabledWhen(() => true),
+		HttpClient.withTracerPropagation(false),
+	)
+	const write = yield* socket.writer
+	const send = (frame: ClientToRelay) => write(encodeClientToRelay(frame))
+	return Effect.fn('Tunnel.respond')(
+		function* (start: typeof RequestStart.Type, body: Option.Option<Uint8Array<ArrayBuffer>>) {
+			const { url: publicUrl } = yield* Deferred.await(hello)
+			const response = yield* http
+				.execute(
+					// Effect's method union is narrower than fetch; preserve custom methods from the relay.
+					HttpClientRequest.make(start.method as HttpMethod.HttpMethod)(`${localUrl}${start.path}`, {
+						headers: endToEndHeaders(start.headers).filter(
+							([name]) => !DROPPED_REQUEST_HEADERS.has(name.toLowerCase()),
+						),
+						body: Option.isSome(body) ? HttpBody.raw(body.value) : undefined,
+					}),
+				)
+				.pipe(Effect.mapError((error) => describeFetchError(error.cause ?? error)))
+			yield* send({
+				_tag: 'ResponseStart',
+				v: PROTOCOL_VERSION,
+				id: start.id,
+				status: response.status,
+				headers: endToEndHeaders(Object.entries(response.headers))
+					.filter(([name]) => !DROPPED_RESPONSE_HEADERS.has(name))
+					// A dev server redirecting to its own absolute URL must send the fetcher back through the tunnel.
+					.map(([name, value]): readonly [string, string] => [
+						name,
+						name === 'location' ? rewriteLocalLocation(value, localUrl, publicUrl) : value,
+					])
+					// Effect's flat Headers record loses repeated Set-Cookie fields.
+					.concat(Cookies.toSetCookieHeaders(response.cookies).map((cookie) => ['set-cookie', cookie])),
+			})
+			let responseBytes = 0
+			yield* response.stream.pipe(
+				Stream.catchTag('ResponseError', (error) =>
+					error.reason === 'EmptyBody' ? Stream.empty : Stream.fail(describeFetchError(error.cause ?? error)),
+				),
+				Stream.runForEach(
+					Effect.fnUntraced(function* (chunk: Uint8Array) {
+						responseBytes += chunk.byteLength
+						// Enforce the response bound independently of transport backpressure and relay cancellation.
+						if (responseBytes > MAX_BODY_BYTES) return yield* Effect.fail('response body too large')
+						yield* Effect.forEach(splitBodyChunk(chunk), (slice) => write(encodeBodyFrame(start.id, slice)))
+					}),
+				),
+			)
+			yield* send({ _tag: 'ResponseEnd', v: PROTOCOL_VERSION, id: start.id })
+		},
+		(effect, start) =>
+			effect.pipe(
+				Effect.scoped,
+				Effect.catchAll((error) =>
+					send({
+						_tag: 'ResponseError',
+						v: PROTOCOL_VERSION,
+						id: start.id,
+						message: `${localUrl} did not answer: ${String(error)}`,
+					}),
+				),
+				Effect.ignore,
+			),
+	)
+})
+
+const runRequests = Effect.fn('Tunnel.runRequests')(function* (
+	socket: Socket.Socket,
+	localUrl: string,
+	hello: Deferred.Deferred<typeof Hello.Type, TunnelError>,
+) {
+	const write = yield* socket.writer
+	const send = (frame: ClientToRelay) => write(encodeClientToRelay(frame))
+	const requestFibers = yield* FiberMap.make<RequestId, void, never>()
+	const run = yield* FiberMap.runtime(requestFibers)()
+
+	const respond = yield* makeResponder(socket, localUrl, hello)
+	function runRequest(start: typeof RequestStart.Type, body: Option.Option<Uint8Array<ArrayBuffer>>) {
+		run(start.id, respond(start, body))
+	}
+	// Bookkeeping runs synchronously in frame order so body chunks always land after their
+	// `RequestStart` and before their `RequestEnd`; only the returned effects run concurrently.
+	const bodies = new Map<
+		RequestId,
+		{ readonly start: typeof RequestStart.Type; readonly chunks: Uint8Array[]; bytes: number }
+	>()
+	function handleBodyFrame(data: Uint8Array) {
+		const body = decodeBody(data)
+		if (Option.isNone(body)) return
+		const pending = bodies.get(body.value.id)
+		if (pending === undefined) return
+		pending.chunks.push(body.value.chunk)
+		pending.bytes += body.value.chunk.byteLength
+		// The public URL is open to anyone who learns it; do not let a request body exhaust memory.
+		if (pending.bytes <= MAX_BODY_BYTES) return
+		bodies.delete(body.value.id)
+		return send({
+			_tag: 'ResponseError',
+			v: PROTOCOL_VERSION,
+			id: body.value.id,
+			message: 'request body too large',
+		})
+	}
+	function handleControlFrame(data: string) {
+		return Option.match(decodeControlFrame(data), {
+			onNone: () => Effect.logDebug('Ignoring unrecognised tunnel frame'),
+			onSome: Match.valueTags({
+				Hello: (frame) => Deferred.succeed(hello, frame),
+				RequestStart: (start) => {
+					if (bodies.has(start.id) || FiberMap.unsafeHas(requestFibers, start.id)) return
+					if (bodies.size + Iterable.size(requestFibers) >= MAX_INFLIGHT_REQUESTS) {
+						return send({
+							_tag: 'ResponseError',
+							v: PROTOCOL_VERSION,
+							id: start.id,
+							message: 'too many local requests',
+						})
+					}
+					if (!start.hasBody) {
+						runRequest(start, Option.none())
+						return
+					}
+					bodies.set(start.id, { start, chunks: [], bytes: 0 })
+					return Effect.void
+				},
+				RequestCancel: ({ id }) => {
+					bodies.delete(id)
+					return FiberMap.remove(requestFibers, id)
+				},
+				RequestEnd: ({ id }) => {
+					const pending = Option.fromNullable(bodies.get(id))
+					bodies.delete(id)
+					return Option.match(pending, {
+						onNone: () => Effect.void,
+						onSome: ({ start, chunks }) => {
+							runRequest(start, Option.some(Buffer.concat(chunks)))
+							return Effect.void
+						},
+					})
+				},
+			}),
+		})
+	}
+	return yield* socket
+		.runRaw((data) => (typeof data === 'string' ? handleControlFrame(data) : handleBodyFrame(data)))
+		.pipe(Effect.ensuring(Effect.sync(() => bodies.clear())))
+})
+
+/**
+ * Opens a tunnel from the relay to `target` for the lifetime of the scope and returns its public URL.
+ * Closing the scope closes the socket and, for file targets, the static server.
+ */
+export const openTunnel = Effect.fn('Tunnel.openTunnel')(
+	function* (target: LocalTarget, relayUrl: string, headers: Readonly<Record<string, string>>) {
+		const origin = yield* localOrigin(target)
+		const { socket, rejectedStatus } = yield* makeRelaySocket(relayUrl, headers)
+		const hello = yield* Deferred.make<typeof Hello.Type, TunnelError>()
+
+		// If the socket ends before Hello the open fails; after Hello, the API simply sees the origin go away.
+		yield* runRequests(socket, `http://${origin.host}:${origin.port}`, hello).pipe(
+			Effect.scoped,
+			Effect.exit,
+			Effect.flatMap(() => Deferred.fail(hello, tunnelOpenError(relayUrl, rejectedStatus()))),
+			Effect.forkScoped,
+		)
+
+		const { url } = yield* Deferred.await(hello).pipe(
+			Effect.timeoutFail({
+				duration: HELLO_TIMEOUT,
+				onTimeout: () =>
+					new TunnelError({
+						reason: 'RelayUnreachable',
+						message: `No answer from the tunnel relay at ${relayUrl}.`,
+					}),
+			}),
+		)
+		return `${url}${origin.path}`
+	},
+	Effect.provide(FetchHttpClient.layer),
+	Effect.provideService(FetchHttpClient.RequestInit, { redirect: 'manual' }),
+)
+
+/**
+ * Runs `use` with a public URL for `input`, which must be a local target (see `resolveLocalTarget`),
+ * and closes the tunnel afterwards.
+ */
+export const withLocalTunnel: <A, E, R, CredentialsError>(
+	input: string,
+	credentials: Effect.Effect<TunnelCredentials, CredentialsError>,
+	use: (url: string) => Effect.Effect<A, E, R>,
+) => Effect.Effect<A, E | CredentialsError | TunnelError | ExpandLocalTargetError, R> = Effect.fn(
+	'Tunnel.withLocalTunnel',
+)(
+	function* <A, E, R, CredentialsError>(
+		input: string,
+		credentials: Effect.Effect<TunnelCredentials, CredentialsError>,
+		use: (url: string) => Effect.Effect<A, E, R>,
+	) {
+		const target = yield* resolveLocalTarget(input).pipe(
+			Effect.provide(Layer.merge(NodeFileSystem.layer, NodePath.layer)),
+		)
+		const { baseUrl, headers } = yield* credentials
+		const relayUrl = yield* relayUrlFor(baseUrl)
+		const url = yield* openTunnel(target, relayUrl, headers)
+		return yield* use(url)
+	},
+	(effect, input) =>
+		effect.pipe(
+			Effect.mapError((error) =>
+				error instanceof ExpandLocalTargetError && error.input !== input
+					? new ExpandLocalTargetError({ reason: error.reason, input, message: error.message, cause: error.cause })
+					: error,
+			),
+			Effect.scoped,
+		),
+)

package/src/index.ts

@@ -30,14 +30,16 @@ export type {
 } from './Client.js'
 export { Expand, ExpandClient } from './Client.js'
 export * from './Crawls.js'
-export type { ExpandError } from './Error.js'
+export type { ExpandError, LocalTargetErrorReason } from './Error.js'
 export {
 	ExpandApiError,
 	ExpandClientApiError,
 	ExpandClientConnectionError,
 	ExpandClientError,
+	ExpandClientLocalTargetError,
 	ExpandClientTimeoutError,
 	ExpandConnectionError,
+	ExpandLocalTargetError,
 	ExpandSdkError,
 	ExpandTimeoutError,
 } from './Error.js'