@dxos/edge-client 0.8.4-main.fd6878d → 0.8.4-main.fffef41

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dxos/edge-client",
-  "version": "0.8.4-main.fd6878d",
+  "version": "0.8.4-main.fffef41",
   "description": "EDGE Client",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
@@ -42,24 +42,24 @@
     "README.md"
   ],
   "dependencies": {
-    "@effect/platform": "^0.90.2",
+    "@effect/platform": "^0.92.1",
     "isomorphic-ws": "^5.0.0",
     "ws": "^8.14.2",
-    "@dxos/async": "0.8.4-main.fd6878d",
-    "@dxos/context": "0.8.4-main.fd6878d",
-    "@dxos/crypto": "0.8.4-main.fd6878d",
-    "@dxos/credentials": "0.8.4-main.fd6878d",
-    "@dxos/debug": "0.8.4-main.fd6878d",
-    "@dxos/invariant": "0.8.4-main.fd6878d",
-    "@dxos/keyring": "0.8.4-main.fd6878d",
-    "@dxos/keys": "0.8.4-main.fd6878d",
-    "@dxos/log": "0.8.4-main.fd6878d",
-    "@dxos/node-std": "0.8.4-main.fd6878d",
-    "@dxos/protocols": "0.8.4-main.fd6878d",
-    "@dxos/util": "0.8.4-main.fd6878d"
+    "@dxos/async": "0.8.4-main.fffef41",
+    "@dxos/context": "0.8.4-main.fffef41",
+    "@dxos/credentials": "0.8.4-main.fffef41",
+    "@dxos/crypto": "0.8.4-main.fffef41",
+    "@dxos/debug": "0.8.4-main.fffef41",
+    "@dxos/invariant": "0.8.4-main.fffef41",
+    "@dxos/node-std": "0.8.4-main.fffef41",
+    "@dxos/log": "0.8.4-main.fffef41",
+    "@dxos/keys": "0.8.4-main.fffef41",
+    "@dxos/keyring": "0.8.4-main.fffef41",
+    "@dxos/protocols": "0.8.4-main.fffef41",
+    "@dxos/util": "0.8.4-main.fffef41"
   },
   "devDependencies": {
-    "@dxos/test-utils": "0.8.4-main.fd6878d"
+    "@dxos/test-utils": "0.8.4-main.fffef41"
   },
   "peerDependencies": {
     "effect": "^3.13.3"

package/src/edge-client.test.ts

@@ -41,17 +41,17 @@ describe('EdgeClient', () => {
 
     const { client } = await openNewClient(endpoint);
 
-    expect(client.status).toBe(EdgeStatus.NOT_CONNECTED);
+    expect(client.status.state).toBe(EdgeStatus.ConnectionState.NOT_CONNECTED);
     admitConnection.wake();
-    await expect.poll(() => client.status).toBe(EdgeStatus.CONNECTED);
+    await expect.poll(() => client.status.state).toBe(EdgeStatus.ConnectionState.CONNECTED);
 
     admitConnection.reset();
     await closeConnection();
     expect(client.isOpen).is.true;
-    await expect.poll(() => client.status).toBe(EdgeStatus.NOT_CONNECTED);
+    await expect.poll(() => client.status.state).toBe(EdgeStatus.ConnectionState.NOT_CONNECTED);
 
     admitConnection.wake();
-    await expect.poll(() => client.status).toBe(EdgeStatus.CONNECTED);
+    await expect.poll(() => client.status.state).toBe(EdgeStatus.ConnectionState.CONNECTED);
   });
 
   test('set identity reconnects', async () => {

package/src/edge-client.ts

@@ -2,7 +2,14 @@
 // Copyright 2024 DXOS.org
 //
 
-import { Event, PersistentLifecycle, Trigger, TriggerState, scheduleMicroTask } from '@dxos/async';
+import {
+  Event,
+  PersistentLifecycle,
+  Trigger,
+  TriggerState,
+  scheduleMicroTask,
+  scheduleTaskInterval,
+} from '@dxos/async';
 import { type Lifecycle, Resource } from '@dxos/context';
 import { log, logInfo } from '@dxos/log';
 import { type Message } from '@dxos/protocols/buf/dxos/edge/messenger_pb';
@@ -17,9 +24,19 @@ import { getEdgeUrlWithProtocol } from './utils';
 
 const DEFAULT_TIMEOUT = 10_000;
 
+// Refresh status every second: rtt, rate counters.
+const STATUS_REFRESH_INTERVAL = 1000;
+
 export type MessageListener = (message: Message) => void;
 export type ReconnectListener = () => void;
 
+export type MessengerConfig = {
+  socketEndpoint: string;
+  timeout?: number;
+  protocol?: Protocol;
+  disableAuth?: boolean;
+};
+
 export interface EdgeConnection extends Required<Lifecycle> {
   statusChanged: Event<EdgeStatus>;
   get info(): any;
@@ -28,18 +45,11 @@ export interface EdgeConnection extends Required<Lifecycle> {
   get isOpen(): boolean;
   get status(): EdgeStatus;
   setIdentity(identity: EdgeIdentity): void;
+  send(message: Message): Promise<void>;
   onMessage(listener: MessageListener): () => void;
   onReconnected(listener: ReconnectListener): () => void;
-  send(message: Message): Promise<void>;
 }
 
-export type MessengerConfig = {
-  socketEndpoint: string;
-  timeout?: number;
-  protocol?: Protocol;
-  disableAuth?: boolean;
-};
-
 /**
  * Messenger client for EDGE:
  *  - While open, uses PersistentLifecycle to keep an open EdgeWsConnection, reconnecting on failures.
@@ -81,9 +91,18 @@ export class EdgeClient extends Resource implements EdgeConnection {
   }
 
   get status(): EdgeStatus {
-    return Boolean(this._currentConnection) && this._ready.state === TriggerState.RESOLVED
-      ? EdgeStatus.CONNECTED
-      : EdgeStatus.NOT_CONNECTED;
+    return {
+      state:
+        Boolean(this._currentConnection) && this._ready.state === TriggerState.RESOLVED
+          ? EdgeStatus.ConnectionState.CONNECTED
+          : EdgeStatus.ConnectionState.NOT_CONNECTED,
+      uptime: this._currentConnection?.uptime ?? 0,
+      rtt: this._currentConnection?.rtt ?? 0,
+      rateBytesUp: this._currentConnection?.uploadRate ?? 0,
+      rateBytesDown: this._currentConnection?.downloadRate ?? 0,
+      messagesSent: this._currentConnection?.messagesSent ?? 0,
+      messagesReceived: this._currentConnection?.messagesReceived ?? 0,
+    };
   }
 
   get identityKey() {
@@ -94,7 +113,7 @@ export class EdgeClient extends Resource implements EdgeConnection {
     return this._identity.peerKey;
   }
 
-  setIdentity(identity: EdgeIdentity): void {
+  setIdentity(identity: EdgeIdentity) {
     if (identity.identityKey !== this._identity.identityKey || identity.peerKey !== this._identity.peerKey) {
       log('Edge identity changed', { identity, oldIdentity: this._identity });
       this._identity = identity;
@@ -103,12 +122,36 @@ export class EdgeClient extends Resource implements EdgeConnection {
     }
   }
 
-  public onMessage(listener: MessageListener): () => void {
+  /**
+   * Send message.
+   * NOTE: The message is guaranteed to be delivered but the service must respond with a message to confirm processing.
+   */
+  public async send(message: Message) {
+    if (this._ready.state !== TriggerState.RESOLVED) {
+      log('waiting for websocket');
+      await this._ready.wait({ timeout: this._config.timeout ?? DEFAULT_TIMEOUT });
+    }
+
+    if (!this._currentConnection) {
+      throw new EdgeConnectionClosedError();
+    }
+
+    if (
+      message.source &&
+      (message.source.peerKey !== this._identity.peerKey || message.source.identityKey !== this.identityKey)
+    ) {
+      throw new EdgeIdentityChangedError();
+    }
+
+    this._currentConnection.send(message);
+  }
+
+  public onMessage(listener: MessageListener) {
     this._messageListeners.add(listener);
     return () => this._messageListeners.delete(listener);
   }
 
-  public onReconnected(listener: () => void): () => void {
+  public onReconnected(listener: () => void) {
     this._reconnectListeners.add(listener);
     if (this._ready.state === TriggerState.RESOLVED) {
       // Microtask so that listener is always called asynchronously, no matter the state of the ready trigger
@@ -123,6 +166,7 @@ export class EdgeClient extends Resource implements EdgeConnection {
         }
       });
     }
+
     return () => this._reconnectListeners.delete(listener);
   }
 
@@ -134,6 +178,18 @@ export class EdgeClient extends Resource implements EdgeConnection {
     this._persistentLifecycle.open().catch((err) => {
       log.warn('Error while opening connection', { err });
     });
+
+    // Notify about status changes (rtt, rate counters).
+    scheduleTaskInterval(
+      this._ctx,
+      async () => {
+        if (!this._currentConnection) {
+          return;
+        }
+        this.statusChanged.emit(this.status);
+      },
+      STATUS_REFRESH_INTERVAL,
+    );
   }
 
   /**
@@ -200,7 +256,6 @@ export class EdgeClient extends Resource implements EdgeConnection {
     // Race with restartRequired so that restart is not blocked by _connect execution.
     // Wait on ready to attempt a reconnect if it times out.
     await Promise.race([this._ready.wait({ timeout: this._config.timeout ?? DEFAULT_TIMEOUT }), restartRequired]);
-
     return connection;
   }
 
@@ -237,30 +292,6 @@ export class EdgeClient extends Resource implements EdgeConnection {
     }
   }
 
-  /**
-   * Send message.
-   * NOTE: The message is guaranteed to be delivered but the service must respond with a message to confirm processing.
-   */
-  public async send(message: Message): Promise<void> {
-    if (this._ready.state !== TriggerState.RESOLVED) {
-      log('waiting for websocket to become ready');
-      await this._ready.wait({ timeout: this._config.timeout ?? DEFAULT_TIMEOUT });
-    }
-
-    if (!this._currentConnection) {
-      throw new EdgeConnectionClosedError();
-    }
-
-    if (
-      message.source &&
-      (message.source.peerKey !== this._identity.peerKey || message.source.identityKey !== this.identityKey)
-    ) {
-      throw new EdgeIdentityChangedError();
-    }
-
-    this._currentConnection.send(message);
-  }
-
   private async _createAuthHeader(path: string): Promise<string | undefined> {
     const httpUrl = new URL(path, this._baseHttpUrl);
     httpUrl.protocol = getEdgeUrlWithProtocol(this._baseWsUrl.toString(), 'http');
@@ -277,7 +308,7 @@ export class EdgeClient extends Resource implements EdgeConnection {
 }
 
 const encodePresentationWsAuthHeader = (encodedPresentation: Uint8Array): string => {
-  // = and / characters are not allowed in the WebSocket subprotocol header.
+  // '=' and '/' characters are not allowed in the WebSocket subprotocol header.
   const encodedToken = Buffer.from(encodedPresentation).toString('base64').replace(/=*$/, '').replaceAll('/', '|');
   return `base64url.bearer.authorization.dxos.org.${encodedToken}`;
 };

package/src/edge-http-client.test.ts

@@ -11,7 +11,7 @@ import { EdgeHttpClient } from './edge-http-client';
 const DEV_SERVER = 'https://edge.dxos.workers.dev';
 
 describe.skipIf(process.env.CI)('EdgeHttpClient', () => {
-  it.only('should get status', async ({ expect }) => {
+  it.skip('should get status', async ({ expect }) => {
     const client = new EdgeHttpClient(DEV_SERVER);
     const identity = await createEphemeralEdgeIdentity();
     client.setIdentity(identity);

package/src/edge-http-client.ts

@@ -2,11 +2,14 @@
 // Copyright 2024 DXOS.org
 //
 
-import { FetchHttpClient, HttpClient } from '@effect/platform';
-import { Effect, pipe } from 'effect';
+import * as FetchHttpClient from '@effect/platform/FetchHttpClient';
+import * as HttpClient from '@effect/platform/HttpClient';
+import * as Effect from 'effect/Effect';
+import * as Function from 'effect/Function';
 
 import { sleep } from '@dxos/async';
 import { Context } from '@dxos/context';
+import { invariant } from '@dxos/invariant';
 import { type PublicKey, type SpaceId } from '@dxos/keys';
 import { log } from '@dxos/log';
 import {
@@ -16,11 +19,13 @@ import {
   type CreateSpaceResponseBody,
   EdgeAuthChallengeError,
   EdgeCallFailedError,
-  type EdgeHttpResponse,
   type EdgeStatus,
   type ExecuteWorkflowResponseBody,
+  type ExportBundleRequest,
+  type ExportBundleResponse,
   type GetAgentStatusResponseBody,
   type GetNotarizationResponseBody,
+  type ImportBundleRequest,
   type InitiateOAuthFlowRequest,
   type InitiateOAuthFlowResponse,
   type JoinSpaceRequest,
@@ -43,6 +48,7 @@ import { getEdgeUrlWithProtocol } from './utils';
 const DEFAULT_RETRY_TIMEOUT = 1500;
 const DEFAULT_RETRY_JITTER = 500;
 const DEFAULT_MAX_RETRIES_COUNT = 3;
+const WARNING_BODY_SIZE = 10 * 1024 * 1024; // 10MB
 
 export type RetryConfig = {
   /**
@@ -64,10 +70,27 @@ type EdgeHttpRequestArgs = {
   context?: Context;
   retry?: RetryConfig;
   body?: any;
+  /**
+   * @default true
+   */
+  json?: boolean;
+
+  /**
+   * Do not expect a standard EDGE JSON response with a `success` field.
+   * @deprecated Use only for debugging.
+   */
+  rawResponse?: boolean;
+
+  /**
+   * Force authentication.
+   * This should be used for requests with large bodies to avoid sending the body twice.
+   * The client will call /auth endpoint to generate the auth header.
+   */
+  auth?: boolean;
 };
 
-export type EdgeHttpGetArgs = Pick<EdgeHttpRequestArgs, 'context' | 'retry'>;
-export type EdgeHttpPostArgs = Pick<EdgeHttpRequestArgs, 'context' | 'retry' | 'body'>;
+export type EdgeHttpGetArgs = Pick<EdgeHttpRequestArgs, 'context' | 'retry' | 'auth'>;
+export type EdgeHttpPostArgs = Pick<EdgeHttpRequestArgs, 'context' | 'retry' | 'body' | 'auth'>;
 
 export class EdgeHttpClient {
   private readonly _baseUrl: string;
@@ -164,10 +187,6 @@ export class EdgeHttpClient {
   // OAuth and credentials
   //
 
-  public async listFunctions(args?: EdgeHttpGetArgs): Promise<any> {
-    return this._call(new URL('/functions', this.baseUrl), { ...args, method: 'GET' });
-  }
-
   public async initiateOAuthFlow(
     body: InitiateOAuthFlowRequest,
     args?: EdgeHttpGetArgs,
@@ -250,8 +269,64 @@ export class EdgeHttpClient {
     body: UploadFunctionRequest,
     args?: EdgeHttpGetArgs,
   ): Promise<UploadFunctionResponseBody> {
+    const formData = new FormData();
+    formData.append('name', body.name ?? '');
+    formData.append('version', body.version);
+    formData.append('ownerPublicKey', body.ownerPublicKey);
+    formData.append('entryPoint', body.entryPoint);
+    body.runtime && formData.append('runtime', body.runtime);
+    for (const [filename, content] of Object.entries(body.assets)) {
+      formData.append(
+        'assets',
+        new Blob([content as Uint8Array<ArrayBuffer>], { type: getFileMimeType(filename) }),
+        filename,
+      );
+    }
+
     const path = ['functions', ...(pathParts.functionId ? [pathParts.functionId] : [])].join('/');
-    return this._call(new URL(path, this.baseUrl), { ...args, body, method: 'PUT' });
+    return this._call(new URL(path, this.baseUrl), {
+      ...args,
+      body: formData,
+      method: 'PUT',
+      json: false,
+    });
+  }
+
+  public async listFunctions(args?: EdgeHttpGetArgs): Promise<any> {
+    return this._call(new URL('/functions', this.baseUrl), { ...args, method: 'GET' });
+  }
+
+  public async invokeFunction(
+    params: {
+      functionId: string;
+      version?: string;
+      spaceId?: SpaceId;
+      cpuTimeLimit?: number;
+      subrequestsLimit?: number;
+    },
+    input: unknown,
+    args?: EdgeHttpGetArgs,
+  ): Promise<any> {
+    const url = new URL(`/functions/${params.functionId}`, this.baseUrl);
+    if (params.version) {
+      url.searchParams.set('version', params.version);
+    }
+    if (params.spaceId) {
+      url.searchParams.set('spaceId', params.spaceId.toString());
+    }
+    if (params.cpuTimeLimit) {
+      url.searchParams.set('cpuTimeLimit', params.cpuTimeLimit.toString());
+    }
+    if (params.subrequestsLimit) {
+      url.searchParams.set('subrequestsLimit', params.subrequestsLimit.toString());
+    }
+
+    return this._call(url, {
+      ...args,
+      body: input,
+      method: 'POST',
+      rawResponse: true,
+    });
   }
 
   //
@@ -271,12 +346,44 @@ export class EdgeHttpClient {
     });
   }
 
+  //
+  // Triggers
+  //
+
+  public async getCronTriggers(spaceId: SpaceId) {
+    return this._call(new URL(`/test/functions/${spaceId}/triggers/crons`, this.baseUrl), { method: 'GET' });
+  }
+
+  //
+  // Import/Export space.
+  //
+
+  public async importBundle(
+    spaceId: SpaceId, //
+    body: ImportBundleRequest,
+    args?: EdgeHttpGetArgs,
+  ): Promise<void> {
+    return this._call(new URL(`/spaces/${spaceId}/import`, this.baseUrl), { ...args, body, method: 'PUT' });
+  }
+
+  public async exportBundle(
+    spaceId: SpaceId,
+    body: ExportBundleRequest,
+    args?: EdgeHttpGetArgs,
+  ): Promise<ExportBundleResponse> {
+    return this._call(new URL(`/spaces/${spaceId}/export`, this.baseUrl), {
+      ...args,
+      body,
+      method: 'POST',
+    });
+  }
+
   //
   // Internal
   //
 
-  private async _fetch<T>(url: URL, args: EdgeHttpRequestArgs): Promise<T> {
-    return pipe(
+  private async _fetch<T>(url: URL, _args: EdgeHttpRequestArgs): Promise<T> {
+    return Function.pipe(
       HttpClient.get(url),
       withLogging,
       withRetryConfig,
@@ -290,44 +397,64 @@ export class EdgeHttpClient {
   // TODO(burdon): Refactor with effect (see edge-http-client.test.ts).
   private async _call<T>(url: URL, args: EdgeHttpRequestArgs): Promise<T> {
     const shouldRetry = createRetryHandler(args);
-    const requestContext = args.context ?? new Context();
+    const requestContext = args.context ?? Context.default();
     log('fetch', { url, request: args.body });
 
     let handledAuth = false;
+    const tryCount = 1;
     while (true) {
-      let processingError: EdgeCallFailedError;
-      let retryAfterHeaderValue: number = Number.NaN;
+      let processingError: EdgeCallFailedError | undefined = undefined;
       try {
+        if (!this._authHeader && args.auth) {
+          const response = await fetch(new URL(`/auth`, this.baseUrl));
+          if (response.status === 401) {
+            this._authHeader = await this._handleUnauthorized(response);
+          }
+        }
+
         const request = createRequest(args, this._authHeader);
+        log('call edge', { url, tryCount, authHeader: !!this._authHeader });
         const response = await fetch(url, request);
-        retryAfterHeaderValue = Number(response.headers.get('Retry-After'));
+
         if (response.ok) {
-          const body = (await response.json()) as EdgeHttpResponse<T>;
+          const body = await response.clone().json();
+          if (args.rawResponse) {
+            return body as any;
+          }
+          invariant(body, 'Expected body to be present');
+          if (!('success' in body)) {
+            return body;
+          }
           if (body.success) {
             return body.data;
           }
-
-          log.warn('unsuccessful edge response', { url, body });
-          if (body.errorData?.type === 'auth_challenge' && typeof body.errorData?.challenge === 'string') {
-            processingError = new EdgeAuthChallengeError(body.errorData.challenge, body.errorData);
-          } else {
-            processingError = EdgeCallFailedError.fromUnsuccessfulResponse(response, body);
-          }
         } else if (response.status === 401 && !handledAuth) {
           this._authHeader = await this._handleUnauthorized(response);
           handledAuth = true;
           continue;
+        }
+
+        const body =
+          response.headers.get('Content-Type') === 'application/json' ? await response.clone().json() : undefined;
+
+        invariant(!body?.success, 'Expected body to not be a failure response or undefined.');
+
+        if (body?.errorData?.type === 'auth_challenge' && typeof body?.errorData?.challenge === 'string') {
+          processingError = new EdgeAuthChallengeError(body.errorData.challenge, body.errorData);
+        } else if (body?.success === false) {
+          processingError = EdgeCallFailedError.fromUnsuccessfulResponse(response, body);
         } else {
-          processingError = EdgeCallFailedError.fromHttpFailure(response);
+          invariant(!response.ok, 'Expected response to not be ok.');
+          processingError = await EdgeCallFailedError.fromHttpFailure(response);
         }
       } catch (error: any) {
         processingError = EdgeCallFailedError.fromProcessingFailureCause(error);
       }
 
-      if (processingError.isRetryable && (await shouldRetry(requestContext, retryAfterHeaderValue))) {
-        log('retrying edge request', { url, processingError });
+      if (processingError?.isRetryable && (await shouldRetry(requestContext, processingError.retryAfterMs))) {
+        log.verbose('retrying edge request', { url, processingError });
       } else {
-        throw processingError;
+        throw processingError!;
       }
     }
   }
@@ -335,7 +462,7 @@ export class EdgeHttpClient {
   private async _handleUnauthorized(response: Response): Promise<string> {
     if (!this._edgeIdentity) {
       log.warn('unauthorized response received before identity was set');
-      throw EdgeCallFailedError.fromHttpFailure(response);
+      throw await EdgeCallFailedError.fromHttpFailure(response);
     }
 
     const challenge = await handleAuthChallenge(response, this._edgeIdentity);
@@ -343,11 +470,32 @@ export class EdgeHttpClient {
   }
 }
 
-const createRequest = ({ method, body }: EdgeHttpRequestArgs, authHeader: string | undefined): RequestInit => {
+const createRequest = (
+  { method, body, json = true }: EdgeHttpRequestArgs,
+  authHeader: string | undefined,
+): RequestInit => {
+  let requestBody: BodyInit | undefined;
+  const headers: HeadersInit = {};
+
+  if (json) {
+    requestBody = body && JSON.stringify(body);
+    headers['Content-Type'] = 'application/json';
+  } else {
+    requestBody = body;
+  }
+
+  if (typeof requestBody === 'string' && requestBody.length > WARNING_BODY_SIZE) {
+    log.warn('Request with large body', { bodySize: requestBody.length });
+  }
+
+  if (authHeader) {
+    headers['Authorization'] = authHeader;
+  }
+
   return {
     method,
-    body: body && JSON.stringify(body),
-    headers: authHeader ? { Authorization: authHeader } : undefined,
+    body: requestBody,
+    headers,
   };
 };
 
@@ -363,7 +511,7 @@ const createRetryHandler = ({ retry }: EdgeHttpRequestArgs) => {
   const maxRetries = retry.count ?? DEFAULT_MAX_RETRIES_COUNT;
   const baseTimeout = retry.timeout ?? DEFAULT_RETRY_TIMEOUT;
   const jitter = retry.jitter ?? DEFAULT_RETRY_JITTER;
-  return async (ctx: Context, retryAfter: number) => {
+  return async (ctx: Context, retryAfter?: number) => {
     if (++retries > maxRetries || ctx.disposed) {
       return false;
     }
@@ -378,3 +526,10 @@ const createRetryHandler = ({ retry }: EdgeHttpRequestArgs) => {
     return true;
   };
 };
+
+const getFileMimeType = (filename: string) =>
+  ['.js', '.mjs'].some((codeExtension) => filename.endsWith(codeExtension))
+    ? 'application/javascript+module'
+    : filename.endsWith('.wasm')
+      ? 'application/wasm'
+      : 'application/octet-stream';