@dxos/edge-client 0.8.4-main.84f28bd → 0.8.4-main.937b3ca

package/src/edge-http-client.ts

@@ -2,47 +2,55 @@
 // Copyright 2024 DXOS.org
 //
 
-import { FetchHttpClient, HttpClient } from '@effect/platform';
-import { Effect, pipe } from 'effect';
+import * as FetchHttpClient from '@effect/platform/FetchHttpClient';
+import * as HttpClient from '@effect/platform/HttpClient';
+import * as Effect from 'effect/Effect';
+import * as Function from 'effect/Function';
 
 import { sleep } from '@dxos/async';
 import { Context } from '@dxos/context';
+import { runAndForwardErrors } from '@dxos/effect';
+import { invariant } from '@dxos/invariant';
 import { type PublicKey, type SpaceId } from '@dxos/keys';
 import { log } from '@dxos/log';
 import {
-  type CreateAgentResponseBody,
   type CreateAgentRequestBody,
+  type CreateAgentResponseBody,
   type CreateSpaceRequest,
   type CreateSpaceResponseBody,
   EdgeAuthChallengeError,
   EdgeCallFailedError,
-  type EdgeHttpResponse,
+  type EdgeFailure,
+  type EdgeStatus,
   type ExecuteWorkflowResponseBody,
+  type ExportBundleRequest,
+  type ExportBundleResponse,
   type GetAgentStatusResponseBody,
   type GetNotarizationResponseBody,
+  type ImportBundleRequest,
   type InitiateOAuthFlowRequest,
   type InitiateOAuthFlowResponse,
   type JoinSpaceRequest,
   type JoinSpaceResponseBody,
-  type RecoverIdentityRequest,
-  type RecoverIdentityResponseBody,
   type ObjectId,
   type PostNotarizationRequestBody,
-  type QueueQuery,
   type QueryResult,
+  type QueueQuery,
+  type RecoverIdentityRequest,
+  type RecoverIdentityResponseBody,
   type UploadFunctionRequest,
   type UploadFunctionResponseBody,
-  type EdgeStatus,
 } from '@dxos/protocols';
 import { createUrl } from '@dxos/util';
 
 import { type EdgeIdentity, handleAuthChallenge } from './edge-identity';
-import { encodeAuthHeader, HttpConfig, withLogging, withRetryConfig } from './http-client';
+import { HttpConfig, encodeAuthHeader, withLogging, withRetryConfig } from './http-client';
 import { getEdgeUrlWithProtocol } from './utils';
 
 const DEFAULT_RETRY_TIMEOUT = 1500;
 const DEFAULT_RETRY_JITTER = 500;
 const DEFAULT_MAX_RETRIES_COUNT = 3;
+const WARNING_BODY_SIZE = 10 * 1024 * 1024; // 10MB
 
 export type RetryConfig = {
   /**
@@ -64,10 +72,25 @@ type EdgeHttpRequestArgs = {
   context?: Context;
   retry?: RetryConfig;
   body?: any;
+  /**
+   * @default true
+   */
+  json?: boolean;
+
+  /**
+   * Force authentication.
+   * This should be used for requests with large bodies to avoid sending the body twice.
+   * The client will call /auth endpoint to generate the auth header.
+   */
+  auth?: boolean;
 };
 
-export type EdgeHttpGetArgs = Pick<EdgeHttpRequestArgs, 'context' | 'retry'>;
-export type EdgeHttpPostArgs = Pick<EdgeHttpRequestArgs, 'context' | 'retry' | 'body'>;
+export type EdgeHttpGetArgs = Pick<EdgeHttpRequestArgs, 'context' | 'retry' | 'auth'>;
+export type EdgeHttpPostArgs = Pick<EdgeHttpRequestArgs, 'context' | 'retry' | 'body' | 'auth'>;
+
+export type GetCronTriggersResponse = {
+  cronIds: string[];
+};
 
 export class EdgeHttpClient {
   private readonly _baseUrl: string;
@@ -100,7 +123,7 @@ export class EdgeHttpClient {
   //
 
   public async getStatus(args?: EdgeHttpGetArgs): Promise<EdgeStatus> {
-    return this._call(new URL('/status', this.baseUrl), { ...args, method: 'GET' });
+    return this._call(new URL('/status', this.baseUrl), { ...args, method: 'GET', auth: true });
   }
 
   //
@@ -189,7 +212,8 @@ export class EdgeHttpClient {
     query: QueueQuery,
     args?: EdgeHttpGetArgs,
   ): Promise<QueryResult> {
-    const { queueId } = query;
+    const queueId = query.queueIds?.[0];
+    invariant(queueId, 'queueId required');
     return this._call(
       createUrl(new URL(`/spaces/${subspaceTag}/${spaceId}/queue/${queueId}/query`, this.baseUrl), {
         after: query.after,
@@ -246,8 +270,63 @@ export class EdgeHttpClient {
     body: UploadFunctionRequest,
     args?: EdgeHttpGetArgs,
   ): Promise<UploadFunctionResponseBody> {
+    const formData = new FormData();
+    formData.append('name', body.name ?? '');
+    formData.append('version', body.version);
+    formData.append('ownerPublicKey', body.ownerPublicKey);
+    formData.append('entryPoint', body.entryPoint);
+    body.runtime && formData.append('runtime', body.runtime);
+    for (const [filename, content] of Object.entries(body.assets)) {
+      formData.append(
+        'assets',
+        new Blob([content as Uint8Array<ArrayBuffer>], { type: getFileMimeType(filename) }),
+        filename,
+      );
+    }
+
     const path = ['functions', ...(pathParts.functionId ? [pathParts.functionId] : [])].join('/');
-    return this._call(new URL(path, this.baseUrl), { ...args, body, method: 'PUT' });
+    return this._call(new URL(path, this.baseUrl), {
+      ...args,
+      body: formData,
+      method: 'PUT',
+      json: false,
+    });
+  }
+
+  public async listFunctions(args?: EdgeHttpGetArgs): Promise<any> {
+    return this._call(new URL('/functions', this.baseUrl), { ...args, method: 'GET' });
+  }
+
+  public async invokeFunction(
+    params: {
+      functionId: string;
+      version?: string;
+      spaceId?: SpaceId;
+      cpuTimeLimit?: number;
+      subrequestsLimit?: number;
+    },
+    input: unknown,
+    args?: EdgeHttpGetArgs,
+  ): Promise<any> {
+    const url = new URL(`/functions/${params.functionId}`, this.baseUrl);
+    if (params.version) {
+      url.searchParams.set('version', params.version);
+    }
+    if (params.spaceId) {
+      url.searchParams.set('spaceId', params.spaceId.toString());
+    }
+    if (params.cpuTimeLimit) {
+      url.searchParams.set('cpuTimeLimit', params.cpuTimeLimit.toString());
+    }
+    if (params.subrequestsLimit) {
+      url.searchParams.set('subrequestsLimit', params.subrequestsLimit.toString());
+    }
+
+    return this._call(url, {
+      ...args,
+      body: input,
+      method: 'POST',
+    });
   }
 
   //
@@ -267,63 +346,120 @@ export class EdgeHttpClient {
     });
   }
 
+  //
+  // Triggers
+  //
+
+  public async getCronTriggers(spaceId: SpaceId): Promise<GetCronTriggersResponse> {
+    return this._call<GetCronTriggersResponse>(new URL(`/test/functions/${spaceId}/triggers/crons`, this.baseUrl), {
+      method: 'GET',
+    });
+  }
+
+  public async forceRunCronTrigger(spaceId: SpaceId, triggerId: ObjectId) {
+    return this._call(new URL(`/test/functions/${spaceId}/triggers/crons/${triggerId}/run`, this.baseUrl), {
+      method: 'POST',
+    });
+  }
+
+  //
+  // Import/Export space.
+  //
+
+  public async importBundle(
+    spaceId: SpaceId, //
+    body: ImportBundleRequest,
+    args?: EdgeHttpGetArgs,
+  ): Promise<void> {
+    return this._call(new URL(`/spaces/${spaceId}/import`, this.baseUrl), { ...args, body, method: 'PUT' });
+  }
+
+  public async exportBundle(
+    spaceId: SpaceId,
+    body: ExportBundleRequest,
+    args?: EdgeHttpGetArgs,
+  ): Promise<ExportBundleResponse> {
+    return this._call(new URL(`/spaces/${spaceId}/export`, this.baseUrl), {
+      ...args,
+      body,
+      method: 'POST',
+    });
+  }
+
   //
   // Internal
   //
 
-  private async _fetch<T>(url: URL, args: EdgeHttpRequestArgs): Promise<T> {
-    return pipe(
+  private async _fetch<T>(url: URL, _args: EdgeHttpRequestArgs): Promise<T> {
+    return Function.pipe(
       HttpClient.get(url),
       withLogging,
       withRetryConfig,
       Effect.provide(FetchHttpClient.layer),
       Effect.provide(HttpConfig.default),
       Effect.withSpan('EdgeHttpClient'),
-      Effect.runPromise,
+      runAndForwardErrors,
     ) as T;
   }
 
   // TODO(burdon): Refactor with effect (see edge-http-client.test.ts).
   private async _call<T>(url: URL, args: EdgeHttpRequestArgs): Promise<T> {
     const shouldRetry = createRetryHandler(args);
-    const requestContext = args.context ?? new Context();
+    const requestContext = args.context ?? Context.default();
     log('fetch', { url, request: args.body });
 
     let handledAuth = false;
+    const tryCount = 1;
     while (true) {
-      let processingError: EdgeCallFailedError;
-      let retryAfterHeaderValue: number = Number.NaN;
+      let processingError: EdgeCallFailedError | undefined = undefined;
       try {
+        if (!this._authHeader && args.auth) {
+          const response = await fetch(new URL(`/auth`, this.baseUrl));
+          if (response.status === 401) {
+            this._authHeader = await this._handleUnauthorized(response);
+          }
+        }
+
         const request = createRequest(args, this._authHeader);
+        log('call edge', { url, tryCount, authHeader: !!this._authHeader });
         const response = await fetch(url, request);
-        retryAfterHeaderValue = Number(response.headers.get('Retry-After'));
+
         if (response.ok) {
-          const body = (await response.json()) as EdgeHttpResponse<T>;
+          const body = await response.clone().json();
+          invariant(body, 'Expected body to be present');
+          if (!('success' in body)) {
+            return body;
+          }
           if (body.success) {
             return body.data;
           }
-
-          log.warn('unsuccessful edge response', { url, body });
-          if (body.errorData?.type === 'auth_challenge' && typeof body.errorData?.challenge === 'string') {
-            processingError = new EdgeAuthChallengeError(body.errorData.challenge, body.errorData);
-          } else {
-            processingError = EdgeCallFailedError.fromUnsuccessfulResponse(response, body);
-          }
         } else if (response.status === 401 && !handledAuth) {
           this._authHeader = await this._handleUnauthorized(response);
           handledAuth = true;
           continue;
+        }
+
+        const body: EdgeFailure =
+          response.headers.get('Content-Type') === 'application/json' ? await response.clone().json() : undefined;
+
+        invariant(!body?.success, 'Expected body to not be a failure response or undefined.');
+
+        if (body?.data?.type === 'auth_challenge' && typeof body?.data?.challenge === 'string') {
+          processingError = new EdgeAuthChallengeError(body.data.challenge, body.data);
+        } else if (body?.success === false) {
+          processingError = EdgeCallFailedError.fromUnsuccessfulResponse(response, body);
         } else {
-          processingError = EdgeCallFailedError.fromHttpFailure(response);
+          invariant(!response.ok, 'Expected response to not be ok.');
+          processingError = await EdgeCallFailedError.fromHttpFailure(response);
         }
       } catch (error: any) {
         processingError = EdgeCallFailedError.fromProcessingFailureCause(error);
       }
 
-      if (processingError.isRetryable && (await shouldRetry(requestContext, retryAfterHeaderValue))) {
-        log('retrying edge request', { url, processingError });
+      if (processingError?.isRetryable && (await shouldRetry(requestContext, processingError.retryAfterMs))) {
+        log.verbose('retrying edge request', { url, processingError });
       } else {
-        throw processingError;
+        throw processingError!;
       }
     }
   }
@@ -331,7 +467,7 @@ export class EdgeHttpClient {
   private async _handleUnauthorized(response: Response): Promise<string> {
     if (!this._edgeIdentity) {
       log.warn('unauthorized response received before identity was set');
-      throw EdgeCallFailedError.fromHttpFailure(response);
+      throw await EdgeCallFailedError.fromHttpFailure(response);
     }
 
     const challenge = await handleAuthChallenge(response, this._edgeIdentity);
@@ -339,11 +475,32 @@ export class EdgeHttpClient {
   }
 }
 
-const createRequest = ({ method, body }: EdgeHttpRequestArgs, authHeader: string | undefined): RequestInit => {
+const createRequest = (
+  { method, body, json = true }: EdgeHttpRequestArgs,
+  authHeader: string | undefined,
+): RequestInit => {
+  let requestBody: BodyInit | undefined;
+  const headers: HeadersInit = {};
+
+  if (json) {
+    requestBody = body && JSON.stringify(body);
+    headers['Content-Type'] = 'application/json';
+  } else {
+    requestBody = body;
+  }
+
+  if (typeof requestBody === 'string' && requestBody.length > WARNING_BODY_SIZE) {
+    log.warn('Request with large body', { bodySize: requestBody.length });
+  }
+
+  if (authHeader) {
+    headers['Authorization'] = authHeader;
+  }
+
   return {
     method,
-    body: body && JSON.stringify(body),
-    headers: authHeader ? { Authorization: authHeader } : undefined,
+    body: requestBody,
+    headers,
   };
 };
 
@@ -359,7 +516,7 @@ const createRetryHandler = ({ retry }: EdgeHttpRequestArgs) => {
   const maxRetries = retry.count ?? DEFAULT_MAX_RETRIES_COUNT;
   const baseTimeout = retry.timeout ?? DEFAULT_RETRY_TIMEOUT;
   const jitter = retry.jitter ?? DEFAULT_RETRY_JITTER;
-  return async (ctx: Context, retryAfter: number) => {
+  return async (ctx: Context, retryAfter?: number) => {
     if (++retries > maxRetries || ctx.disposed) {
       return false;
     }
@@ -374,3 +531,10 @@ const createRetryHandler = ({ retry }: EdgeHttpRequestArgs) => {
     return true;
   };
 };
+
+const getFileMimeType = (filename: string) =>
+  ['.js', '.mjs'].some((codeExtension) => filename.endsWith(codeExtension))
+    ? 'application/javascript+module'
+    : filename.endsWith('.wasm')
+      ? 'application/wasm'
+      : 'application/octet-stream';

package/src/edge-ws-connection.ts

@@ -10,7 +10,7 @@ import { invariant } from '@dxos/invariant';
 import { log, logInfo } from '@dxos/log';
 import { EdgeWebsocketProtocol } from '@dxos/protocols';
 import { buf } from '@dxos/protocols/buf';
-import { MessageSchema, type Message } from '@dxos/protocols/buf/dxos/edge/messenger_pb';
+import { type Message, MessageSchema } from '@dxos/protocols/buf/dxos/edge/messenger_pb';
 
 import { protocol } from './defs';
 import { type EdgeIdentity } from './edge-identity';
@@ -30,6 +30,23 @@ export class EdgeWsConnection extends Resource {
   private _inactivityTimeoutCtx: Context | undefined;
   private _ws: WebSocket | undefined;
   private _wsMuxer: WebSocketMuxer | undefined;
+  private _lastReceivedMessageTimestamp = Date.now();
+
+  private _openTimestamp: number | undefined;
+
+  // Latency tracking.
+  private _pingTimestamp: number | undefined;
+  private _rtt = 0;
+
+  // Rate tracking with sliding window.
+  private _uploadRate = 0;
+  private _downloadRate = 0;
+  private readonly _rateWindow = 10000; // 10 second sliding window.
+  private readonly _rateUpdateInterval = 1000; // Update rates every second.
+  private _bytesSamples: Array<{ timestamp: number; sent: number; received: number }> = [];
+
+  private _messagesSent = 0;
+  private _messagesReceived = 0;
 
   constructor(
     private readonly _identity: EdgeIdentity,
@@ -48,10 +65,35 @@ export class EdgeWsConnection extends Resource {
     };
   }
 
+  public get rtt(): number {
+    return this._rtt;
+  }
+
+  public get uptime(): number {
+    return this._openTimestamp ? (Date.now() - this._openTimestamp) / 1000 : 0;
+  }
+
+  public get uploadRate(): number {
+    return this._uploadRate;
+  }
+
+  public get downloadRate(): number {
+    return this._downloadRate;
+  }
+
+  public get messagesSent(): number {
+    return this._messagesSent;
+  }
+
+  public get messagesReceived(): number {
+    return this._messagesReceived;
+  }
+
   public send(message: Message): void {
     invariant(this._ws);
     invariant(this._wsMuxer);
     log('sending...', { peerKey: this._identity.peerKey, payload: protocol.getPayloadType(message) });
+    this._messagesSent++;
     if (this._ws?.protocol.includes(EdgeWebsocketProtocol.V0)) {
       const binary = buf.toBinary(MessageSchema, message);
       if (binary.length > CLOUDFLARE_MESSAGE_MAX_BYTES) {
@@ -62,8 +104,12 @@ export class EdgeWsConnection extends Resource {
         });
         return;
       }
+      this._recordBytes(binary.byteLength, 0);
       this._ws.send(binary);
     } else {
+      // For muxer, we need to track the size of the message being sent.
+      const binary = buf.toBinary(MessageSchema, message);
+      this._recordBytes(binary.byteLength, 0);
       this._wsMuxer.send(message).catch((e) => log.catch(e));
     }
   }
@@ -82,20 +128,22 @@ export class EdgeWsConnection extends Resource {
     this._ws.onopen = () => {
       if (this.isOpen) {
         log('connected');
+        this._openTimestamp = Date.now();
         this._callbacks.onConnected();
         this._scheduleHeartbeats();
+        this._scheduleRateCalculation();
       } else {
         log.verbose('connected after becoming inactive', { currentIdentity: this._identity });
       }
     };
-    this._ws.onclose = (event) => {
+    this._ws.onclose = (event: WebSocket.CloseEvent) => {
       if (this.isOpen) {
-        log.warn('disconnected while being open', { code: event.code, reason: event.reason });
+        log.warn('server disconnected', { code: event.code, reason: event.reason });
         this._callbacks.onRestartRequired();
         muxer.destroy();
       }
     };
-    this._ws.onerror = (event) => {
+    this._ws.onerror = (event: WebSocket.ErrorEvent) => {
       if (this.isOpen) {
         log.warn('edge connection socket error', { error: event.error, info: event.message });
         this._callbacks.onRestartRequired();
@@ -106,20 +154,29 @@ export class EdgeWsConnection extends Resource {
     /**
      * https://developer.mozilla.org/en-US/docs/Web/API/MessageEvent/data
      */
-    this._ws.onmessage = async (event) => {
+    this._ws.onmessage = async (event: WebSocket.MessageEvent) => {
       if (!this.isOpen) {
         log.verbose('message ignored on closed connection', { event: event.type });
         return;
       }
+      this._lastReceivedMessageTimestamp = Date.now();
       if (event.data === '__pong__') {
+        // Calculate latency.
+        if (this._pingTimestamp) {
+          this._rtt = Date.now() - this._pingTimestamp;
+          this._pingTimestamp = undefined;
+        }
         this._rescheduleHeartbeatTimeout();
         return;
       }
       const bytes = await toUint8Array(event.data);
+      this._recordBytes(0, bytes.byteLength);
       if (!this.isOpen) {
         return;
       }
 
+      this._messagesReceived++;
+
       const message = this._ws?.protocol?.includes(EdgeWebsocketProtocol.V0)
         ? buf.fromBinary(MessageSchema, bytes)
         : muxer.receiveData(bytes);
@@ -143,7 +200,7 @@ export class EdgeWsConnection extends Resource {
       if (err instanceof Error && err.message.includes('WebSocket is closed before the connection is established.')) {
         return;
       }
-      log.warn('Error closing websocket', { err });
+      log.warn('error closing websocket', { err });
     }
   }
 
@@ -154,10 +211,12 @@ export class EdgeWsConnection extends Resource {
       async () => {
         // TODO(mykola): use RFC6455 ping/pong once implemented in the browser?
         // Cloudflare's worker responds to this `without interrupting hibernation`. https://developers.cloudflare.com/durable-objects/api/websockets/#setwebsocketautoresponse
+        this._pingTimestamp = Date.now();
         this._ws?.send('__ping__');
       },
       SIGNAL_KEEPALIVE_INTERVAL,
     );
+    this._pingTimestamp = Date.now();
     this._ws.send('__ping__');
     this._rescheduleHeartbeatTimeout();
   }
@@ -172,11 +231,73 @@ export class EdgeWsConnection extends Resource {
       this._inactivityTimeoutCtx,
       () => {
         if (this.isOpen) {
-          log.warn('restart due to inactivity timeout');
-          this._callbacks.onRestartRequired();
+          if (Date.now() - this._lastReceivedMessageTimestamp > SIGNAL_KEEPALIVE_TIMEOUT) {
+            log.warn('restart due to inactivity timeout', {
+              lastReceivedMessageTimestamp: this._lastReceivedMessageTimestamp,
+            });
+            this._callbacks.onRestartRequired();
+          } else {
+            this._rescheduleHeartbeatTimeout();
+          }
         }
       },
       SIGNAL_KEEPALIVE_TIMEOUT,
     );
   }
+
+  private _recordBytes(sent: number, received: number): void {
+    const now = Date.now();
+
+    // Find if we have a sample for the current second.
+    const currentSecond = Math.floor(now / 1000) * 1000;
+    const existingSample = this._bytesSamples.find((s) => Math.floor(s.timestamp / 1000) * 1000 === currentSecond);
+
+    if (existingSample) {
+      existingSample.sent += sent;
+      existingSample.received += received;
+    } else {
+      this._bytesSamples.push({ timestamp: now, sent, received });
+    }
+  }
+
+  private _scheduleRateCalculation(): void {
+    scheduleTaskInterval(
+      this._ctx,
+      async () => {
+        this._calculateRates();
+      },
+      this._rateUpdateInterval,
+    );
+    // Calculate initial rates.
+    this._calculateRates();
+  }
+
+  private _calculateRates(): void {
+    const now = Date.now();
+    const cutoff = now - this._rateWindow;
+
+    // Remove old samples.
+    this._bytesSamples = this._bytesSamples.filter((s) => s.timestamp > cutoff);
+
+    if (this._bytesSamples.length === 0) {
+      this._uploadRate = 0;
+      this._downloadRate = 0;
+      return;
+    }
+
+    // Calculate total bytes and time span.
+    let totalSent = 0;
+    let totalReceived = 0;
+    const oldestTimestamp = Math.min(...this._bytesSamples.map((s) => s.timestamp));
+    const timeSpan = (now - oldestTimestamp) / 1000; // Convert to seconds.
+
+    for (const sample of this._bytesSamples) {
+      totalSent += sample.sent;
+      totalReceived += sample.received;
+    }
+
+    // Calculate rates (bytes per second).
+    this._uploadRate = timeSpan > 0 ? Math.round(totalSent / timeSpan) : 0;
+    this._downloadRate = timeSpan > 0 ? Math.round(totalReceived / timeSpan) : 0;
+  }
 }

package/src/edge-ws-muxer.ts

@@ -5,7 +5,7 @@
 import { Trigger } from '@dxos/async';
 import { log } from '@dxos/log';
 import { buf } from '@dxos/protocols/buf';
-import { MessageSchema, type Message } from '@dxos/protocols/buf/dxos/edge/messenger_pb';
+import { type Message, MessageSchema } from '@dxos/protocols/buf/dxos/edge/messenger_pb';
 
 import { protocol } from './defs';
 

package/src/http-client.test.ts

@@ -2,10 +2,13 @@
 // Copyright 2025 DXOS.org
 //
 
-import { FetchHttpClient, HttpClient } from '@effect/platform';
-import { Effect, pipe } from 'effect';
+import * as FetchHttpClient from '@effect/platform/FetchHttpClient';
+import * as HttpClient from '@effect/platform/HttpClient';
+import * as Effect from 'effect/Effect';
+import * as Function from 'effect/Function';
 import { afterEach, beforeEach, describe, it } from 'vitest';
 
+import { runAndForwardErrors } from '@dxos/effect';
 import { invariant } from '@dxos/invariant';
 
 import { HttpConfig, withLogging, withRetry, withRetryConfig } from './http-client';
@@ -18,36 +21,36 @@ describe('HttpClient', () => {
     server = await createTestServer(responseHandler((attempt) => (attempt > 2 ? { value: 100 } : false)));
   });
 
-  // eslint-disable-next-line mocha/no-top-level-hooks
   afterEach(() => {
     server?.close();
     server = undefined;
   });
 
-  // TODO(burdon): Auth headers.
+  // TODO(burdon): Auth headers/API key for admin.
   // TODO(burdon): Add request/response schema type checking.
+  // TODO(burdon): Test swarm.
   it.skipIf(process.env.CI)('should retry', async ({ expect }) => {
     invariant(server);
 
     {
-      const result = await pipe(
+      const result = await Function.pipe(
         withRetry(HttpClient.get(server.url)),
         Effect.provide(FetchHttpClient.layer),
         Effect.withSpan('EdgeHttpClient'),
-        Effect.runPromise,
+        runAndForwardErrors,
       );
       expect(result).toMatchObject({ success: true, data: { value: 100 } });
     }
 
     {
-      const result = await pipe(
+      const result = await Function.pipe(
         HttpClient.get(server.url),
         withLogging,
         withRetryConfig,
         Effect.provide(FetchHttpClient.layer),
         Effect.provide(HttpConfig.default), // TODO(burdon): Swap out to mock.
         Effect.withSpan('EdgeHttpClient'), // TODO(burdon): OTEL.
-        Effect.runPromise,
+        runAndForwardErrors,
       );
       expect(result).toMatchObject({ success: true, data: { value: 100 } });
     }