@dxos/functions-runtime-cloudflare 0.8.4-main.03d5cd7b56

package/package.json

@@ -0,0 +1,52 @@
+{
+  "name": "@dxos/functions-runtime-cloudflare",
+  "version": "0.8.4-main.03d5cd7b56",
+  "description": "Functions runtime for Cloudflare.",
+  "homepage": "https://dxos.org",
+  "bugs": "https://github.com/dxos/dxos/issues",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dxos/dxos"
+  },
+  "license": "MIT",
+  "author": "info@dxos.org",
+  "sideEffects": false,
+  "type": "module",
+  "exports": {
+    ".": {
+      "source": "./src/index.ts",
+      "types": "./dist/types/src/index.d.ts",
+      "browser": "./dist/lib/browser/index.mjs",
+      "node": "./dist/lib/node-esm/index.mjs"
+    }
+  },
+  "types": "dist/types/src/index.d.ts",
+  "files": [
+    "dist",
+    "schema",
+    "src"
+  ],
+  "dependencies": {
+    "effect": "3.20.0",
+    "@dxos/codec-protobuf": "0.8.4-main.03d5cd7b56",
+    "@dxos/context": "0.8.4-main.03d5cd7b56",
+    "@dxos/echo": "0.8.4-main.03d5cd7b56",
+    "@dxos/debug": "0.8.4-main.03d5cd7b56",
+    "@dxos/echo-db": "0.8.4-main.03d5cd7b56",
+    "@dxos/invariant": "0.8.4-main.03d5cd7b56",
+    "@dxos/echo-protocol": "0.8.4-main.03d5cd7b56",
+    "@dxos/log": "0.8.4-main.03d5cd7b56",
+    "@dxos/keys": "0.8.4-main.03d5cd7b56",
+    "@dxos/protocols": "0.8.4-main.03d5cd7b56",
+    "@dxos/errors": "0.8.4-main.03d5cd7b56",
+    "@dxos/util": "0.8.4-main.03d5cd7b56"
+  },
+  "devDependencies": {
+    "@cloudflare/workers-types": "^4.20260302.0",
+    "@dxos/types": "0.8.4-main.03d5cd7b56"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {}
+}

package/src/functions-client.ts

@@ -0,0 +1,88 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { Resource } from '@dxos/context';
+import { EchoClient } from '@dxos/echo-db';
+import { invariant } from '@dxos/invariant';
+import { type SpaceId } from '@dxos/keys';
+import { type EdgeFunctionEnv } from '@dxos/protocols';
+
+import { ServiceContainer } from './internal';
+import { SpaceProxy } from './space-proxy';
+
+type Services = {
+  dataService: EdgeFunctionEnv.DataService;
+  queueService: EdgeFunctionEnv.QueueService;
+  functionsAiService: EdgeFunctionEnv.FunctionsAiService;
+};
+
+/**
+ * API for functions to integrate with ECHO and HALO.
+ * @deprecated
+ */
+export class FunctionsClient extends Resource {
+  private readonly _serviceContainer;
+  private readonly _echoClient;
+  private readonly _executionContext: EdgeFunctionEnv.TraceContext = {};
+
+  private readonly _spaces = new Map<SpaceId, SpaceProxy>();
+
+  constructor(services: Services) {
+    super();
+    invariant(typeof services.dataService !== 'undefined', 'DataService is required');
+    invariant(typeof services.queueService !== 'undefined', 'QueueService is required');
+    this._serviceContainer = new ServiceContainer(
+      this._executionContext,
+      services.dataService,
+      services.queueService,
+      services.functionsAiService,
+    );
+    this._echoClient = new EchoClient({});
+  }
+
+  get echo(): EchoClient {
+    return this._echoClient;
+  }
+
+  protected override async _open() {
+    const { dataService, queryService } = await this._serviceContainer.createServices();
+    this._echoClient.connectToService({ dataService, queryService });
+    await this._echoClient.open();
+  }
+
+  protected override async _close() {
+    for (const space of this._spaces.values()) {
+      await space.close();
+    }
+    this._spaces.clear();
+
+    await this._echoClient.close();
+  }
+
+  async getSpace(spaceId: SpaceId): Promise<SpaceProxy> {
+    if (!this._spaces.has(spaceId)) {
+      const space = new SpaceProxy(this._serviceContainer, this._echoClient, spaceId);
+      this._spaces.set(spaceId, space);
+    }
+    const space = this._spaces.get(spaceId)!;
+    await space.open(); // No-op if already open.
+    return space;
+  }
+}
+
+export const createClientFromEnv = async (env: any): Promise<FunctionsClient> => {
+  const client = new FunctionsClient({
+    dataService: env.DATA_SERVICE,
+    queueService: env.QUEUE_SERVICE,
+    functionsAiService: env.FUNCTIONS_AI_SERVICE,
+  });
+  await client.open();
+  return client;
+};
+
+/**
+ - Provides data access capabilities for user functions.
+ - No real-time replication or reactive queries -- function receives a snapshot.
+ - Function event contains the metadata but doesn't need to include the data.
+ */

package/src/index.ts

@@ -0,0 +1,9 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export * from './functions-client';
+export * from './internal';
+export * from './types';
+export * from './wrap-handler-for-cloudflare';
+export * from './logger';

package/src/internal/data-service-impl.ts

@@ -0,0 +1,143 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import type { RequestOptions } from '@dxos/codec-protobuf';
+import { Stream } from '@dxos/codec-protobuf/stream';
+import { raise } from '@dxos/debug';
+import { NotImplementedError, RuntimeServiceError } from '@dxos/errors';
+import { invariant } from '@dxos/invariant';
+import { SpaceId } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { type EdgeFunctionEnv } from '@dxos/protocols';
+import type {
+  BatchedDocumentUpdates,
+  CreateDocumentRequest,
+  CreateDocumentResponse,
+  DataService as DataServiceProto,
+  GetDocumentHeadsRequest,
+  GetDocumentHeadsResponse,
+  GetSpaceSyncStateRequest,
+  ReIndexHeadsRequest,
+  SpaceSyncState,
+  UpdateRequest,
+  UpdateSubscriptionRequest,
+} from '@dxos/protocols/proto/dxos/echo/service';
+
+import { copyUint8Array } from './utils';
+
+export class DataServiceImpl implements DataServiceProto {
+  private dataSubscriptions = new Map<string, { spaceId: SpaceId; next: (msg: BatchedDocumentUpdates) => void }>();
+
+  constructor(
+    private _executionContext: EdgeFunctionEnv.TraceContext,
+    private _dataService: EdgeFunctionEnv.DataService,
+  ) {}
+
+  subscribe({ subscriptionId, spaceId }: { subscriptionId: string; spaceId: string }): Stream<BatchedDocumentUpdates> {
+    return new Stream(({ next }) => {
+      invariant(SpaceId.isValid(spaceId));
+      this.dataSubscriptions.set(subscriptionId, { spaceId, next });
+
+      return () => {
+        this.dataSubscriptions.delete(subscriptionId);
+      };
+    });
+  }
+
+  async updateSubscription({ subscriptionId, addIds }: UpdateSubscriptionRequest): Promise<void> {
+    const sub =
+      this.dataSubscriptions.get(subscriptionId) ??
+      raise(
+        new RuntimeServiceError({
+          message: 'Subscription not found.',
+          context: { subscriptionId },
+        }),
+      );
+
+    if (addIds) {
+      log.info('request documents', { count: addIds.length });
+      // TODO(dmaretskyi): Batch.
+      for (const documentId of addIds) {
+        using document = await this._dataService.getDocument(this._executionContext, sub.spaceId, documentId);
+        log.info('document loaded', { documentId, spaceId: sub.spaceId, found: !!document });
+        if (!document) {
+          log.warn('not found', { documentId });
+          continue;
+        }
+        sub.next({
+          updates: [
+            {
+              documentId,
+              // Copy returned object to avoid hanging RPC stub
+              // See https://developers.cloudflare.com/workers/runtime-apis/rpc/lifecycle/
+              mutation: copyUint8Array(document.data),
+            },
+          ],
+        });
+      }
+    }
+  }
+
+  async createDocument({ spaceId, initialValue }: CreateDocumentRequest): Promise<CreateDocumentResponse> {
+    invariant(SpaceId.isValid(spaceId));
+    using response = await this._dataService.createDocument(this._executionContext, spaceId, initialValue);
+    return { documentId: response.documentId };
+  }
+
+  async update({ updates, subscriptionId }: UpdateRequest): Promise<void> {
+    const sub =
+      this.dataSubscriptions.get(subscriptionId) ??
+      raise(
+        new RuntimeServiceError({
+          message: 'Subscription not found.',
+          context: { subscriptionId },
+        }),
+      );
+    // TODO(dmaretskyi): Batch.
+    try {
+      for (const update of updates ?? []) {
+        await this._dataService.changeDocument(this._executionContext, sub.spaceId, update.documentId, update.mutation);
+      }
+    } catch (error) {
+      throw RuntimeServiceError.wrap({
+        message: 'Failed to apply document updates.',
+        context: { subscriptionId },
+        ifTypeDiffers: true,
+      })(error);
+    }
+  }
+
+  async flush(): Promise<void> {
+    // No-op.
+  }
+
+  subscribeSpaceSyncState(_request: GetSpaceSyncStateRequest, _options?: RequestOptions): Stream<SpaceSyncState> {
+    throw new NotImplementedError({
+      message: 'subscribeSpaceSyncState is not implemented.',
+    });
+  }
+
+  async getDocumentHeads({ documentIds: _documentIds }: GetDocumentHeadsRequest): Promise<GetDocumentHeadsResponse> {
+    throw new NotImplementedError({
+      message: 'getDocumentHeads is not implemented.',
+    });
+  }
+
+  async reIndexHeads({ documentIds: _documentIds }: ReIndexHeadsRequest): Promise<void> {
+    throw new NotImplementedError({
+      message: 'reIndexHeads is not implemented.',
+    });
+  }
+
+  async updateIndexes(): Promise<void> {
+    log.error('updateIndexes is not available in EDGE env.');
+    // No-op.
+  }
+
+  async waitUntilHeadsReplicated({ heads: _heads }: { heads: any }): Promise<void> {
+    throw new NotImplementedError({
+      message: 'waitUntilHeadsReplicated is not implemented.',
+    });
+  }
+}

package/src/internal/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './service-container';

package/src/internal/query-service-impl.ts

@@ -0,0 +1,53 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { Stream } from '@dxos/codec-protobuf/stream';
+import { NotImplementedError, RuntimeServiceError } from '@dxos/errors';
+import { log } from '@dxos/log';
+import { type EdgeFunctionEnv } from '@dxos/protocols';
+import { type QueryRequest, type QueryResponse, type QueryService } from '@dxos/protocols/proto/dxos/echo/query';
+
+export class QueryServiceImpl implements QueryService {
+  private _queryCount = 0;
+
+  constructor(
+    private readonly _executionContext: EdgeFunctionEnv.TraceContext,
+    private readonly _dataService: EdgeFunctionEnv.DataService,
+  ) {}
+
+  execQuery(request: QueryRequest): Stream<QueryResponse> {
+    log.info('execQuery', { request });
+
+    return Stream.fromPromise<QueryResponse>(
+      (async () => {
+        try {
+          this._queryCount++;
+          log.info('begin query', { request });
+          using queryResponse = await this._dataService.execQuery(this._executionContext, request);
+          log.info('query response', { resultCount: queryResponse.results?.length });
+          return structuredClone(queryResponse);
+        } catch (error) {
+          log.error('query failed', { err: error });
+          throw new RuntimeServiceError({
+            message: `Query execution failed (queryCount=${this._queryCount})`,
+            context: { queryCount: this._queryCount },
+            cause: error,
+          });
+        }
+      })(),
+    );
+  }
+
+  async reindex() {
+    throw new NotImplementedError({
+      message: 'Reindex is not implemented.',
+    });
+  }
+
+  async setConfig() {
+    throw new NotImplementedError({
+      message: 'SetConfig is not implemented.',
+    });
+  }
+}

package/src/internal/queue-service-impl.ts

@@ -0,0 +1,66 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { RuntimeServiceError } from '@dxos/errors';
+import { type EdgeFunctionEnv, type FeedProtocol } from '@dxos/protocols';
+
+export class QueueServiceImpl implements FeedProtocol.QueueService {
+  constructor(
+    protected _ctx: EdgeFunctionEnv.TraceContext,
+    private readonly _queueService: EdgeFunctionEnv.QueueService,
+  ) {}
+
+  async queryQueue(request: FeedProtocol.QueryQueueRequest): Promise<FeedProtocol.QueryResult> {
+    try {
+      using result = await this._queueService.queryQueue(this._ctx, request);
+      // Copy to avoid hanging RPC stub (Workers RPC lifecycle).
+      return {
+        objects: structuredClone(result.objects),
+        nextCursor: result.nextCursor,
+        prevCursor: result.prevCursor,
+      };
+    } catch (error) {
+      const { query } = request;
+      throw RuntimeServiceError.wrap({
+        message: 'Queue query failed.',
+        context: {
+          subspaceTag: query?.queuesNamespace,
+          spaceId: query?.spaceId,
+          queueId: query?.queueIds?.[0],
+        },
+        ifTypeDiffers: true,
+      })(error);
+    }
+  }
+
+  async insertIntoQueue(request: FeedProtocol.InsertIntoQueueRequest): Promise<void> {
+    try {
+      using _ = await this._queueService.insertIntoQueue(this._ctx, request);
+    } catch (error) {
+      const { subspaceTag, spaceId, queueId } = request;
+      throw RuntimeServiceError.wrap({
+        message: 'Queue append failed.',
+        context: { subspaceTag, spaceId, queueId },
+        ifTypeDiffers: true,
+      })(error);
+    }
+  }
+
+  async deleteFromQueue(request: FeedProtocol.DeleteFromQueueRequest): Promise<void> {
+    try {
+      using _ = await this._queueService.deleteFromQueue(this._ctx, request);
+    } catch (error) {
+      const { subspaceTag, spaceId, queueId } = request;
+      throw RuntimeServiceError.wrap({
+        message: 'Queue delete failed.',
+        context: { subspaceTag, spaceId, queueId },
+        ifTypeDiffers: true,
+      })(error);
+    }
+  }
+
+  async syncQueue(_: FeedProtocol.SyncQueueRequest): Promise<void> {
+    // No-op in Cloudflare runtime.
+  }
+}

package/src/internal/service-container.ts

@@ -0,0 +1,89 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { type AnyEntity } from '@dxos/echo/internal';
+import { type DXN, type SpaceId } from '@dxos/keys';
+import { type EdgeFunctionEnv, type FeedProtocol } from '@dxos/protocols';
+import { type QueryService as QueryServiceProto } from '@dxos/protocols/proto/dxos/echo/query';
+import type { DataService as DataServiceProto } from '@dxos/protocols/proto/dxos/echo/service';
+
+import { DataServiceImpl } from './data-service-impl';
+import { QueryServiceImpl } from './query-service-impl';
+import { QueueServiceImpl } from './queue-service-impl';
+
+/**
+ * TODO: make this implement DataService and QueryService to unify API over edge and web backend
+ */
+export class ServiceContainer {
+  constructor(
+    private readonly _executionContext: EdgeFunctionEnv.TraceContext,
+    private readonly _dataService: EdgeFunctionEnv.DataService,
+    private readonly _queueService: EdgeFunctionEnv.QueueService,
+    private readonly _functionsService: EdgeFunctionEnv.FunctionsAiService,
+  ) {}
+
+  async getSpaceMeta(spaceId: SpaceId): Promise<EdgeFunctionEnv.SpaceMeta | undefined> {
+    using result = await this._dataService.getSpaceMeta(this._executionContext, spaceId);
+    // Copy returned object to avoid hanging RPC stub
+    // See https://developers.cloudflare.com/workers/runtime-apis/rpc/lifecycle/
+    return result
+      ? {
+          spaceKey: result.spaceKey,
+          rootDocumentId: result.rootDocumentId,
+        }
+      : undefined;
+  }
+
+  async createServices(): Promise<{
+    dataService: DataServiceProto;
+    queryService: QueryServiceProto;
+    queueService: FeedProtocol.QueueService;
+    functionsAiService: EdgeFunctionEnv.FunctionsAiService;
+  }> {
+    const dataService = new DataServiceImpl(this._executionContext, this._dataService);
+    const queryService = new QueryServiceImpl(this._executionContext, this._dataService);
+    const queueService = new QueueServiceImpl(this._executionContext, this._queueService);
+
+    return {
+      dataService,
+      queryService,
+      queueService,
+      functionsAiService: this._functionsService,
+    };
+  }
+
+  async queryQueue(queue: DXN): Promise<FeedProtocol.QueryResult> {
+    const parts = queue.asQueueDXN();
+    if (!parts) {
+      throw new Error('Invalid queue DXN');
+    }
+    const { subspaceTag, spaceId, queueId } = parts;
+    const result = await this._queueService.queryQueue(this._executionContext, {
+      query: {
+        spaceId,
+        queuesNamespace: subspaceTag,
+        queueIds: [queueId],
+      },
+    });
+    return {
+      objects: structuredClone(result.objects),
+      nextCursor: result.nextCursor ?? null,
+      prevCursor: result.prevCursor ?? null,
+    };
+  }
+
+  async insertIntoQueue(queue: DXN, objects: AnyEntity[]): Promise<void> {
+    const parts = queue.asQueueDXN();
+    if (!parts) {
+      throw new Error('Invalid queue DXN');
+    }
+    const { subspaceTag, spaceId, queueId } = parts;
+    await this._queueService.insertIntoQueue(this._executionContext, {
+      subspaceTag,
+      spaceId,
+      queueId,
+      objects: objects.map((obj) => JSON.stringify(obj)),
+    });
+  }
+}

package/src/internal/utils.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export const copyUint8Array = (value: Uint8Array): Uint8Array => new Uint8Array(value);

package/src/logger.ts

@@ -0,0 +1,44 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+/* eslint-disable no-console */
+
+import { LogLevel, type LogProcessor, log, shouldLog } from '@dxos/log';
+
+export const setupFunctionsLogger = () => {
+  log.runtimeConfig.processors.length = 0;
+  log.runtimeConfig.processors.push(functionLogProcessor);
+};
+
+const functionLogProcessor: LogProcessor = (config, entry) => {
+  if (!shouldLog(entry, config.filters)) {
+    return;
+  }
+
+  const context = entry.computedContext;
+  const error = entry.computedError;
+  const extras = [Object.keys(context).length > 0 ? context : undefined, error].filter((value) => value !== undefined);
+
+  switch (entry.level) {
+    case LogLevel.DEBUG:
+    case LogLevel.TRACE:
+      console.debug(entry.message, ...extras);
+      break;
+    case LogLevel.VERBOSE:
+      console.log(entry.message, ...extras);
+      break;
+    case LogLevel.INFO:
+      console.info(entry.message, ...extras);
+      break;
+    case LogLevel.WARN:
+      console.warn(entry.message, ...extras);
+      break;
+    case LogLevel.ERROR:
+      console.error(entry.message, ...extras);
+      break;
+    default:
+      console.log(entry.message, ...extras);
+      break;
+  }
+};

package/src/queues-api.ts

@@ -0,0 +1,57 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { type AnyEntity } from '@dxos/echo/internal';
+import type { DXN, SpaceId } from '@dxos/keys';
+import { log } from '@dxos/log';
+
+import type { ServiceContainer } from './internal';
+
+export interface QueuesQueryResult {
+  objects: AnyEntity[];
+  nextCursor: string | null;
+  prevCursor: string | null;
+}
+
+// TODO(dmaretskyi): Temporary API to get the queues working.
+// TODO(dmaretskyi): To be replaced with integrating queues into echo.
+/**
+ * @deprecated
+ */
+export interface QueuesAPI {
+  queryQueue(queue: DXN, options?: {}): Promise<QueuesQueryResult>;
+  insertIntoQueue(queue: DXN, objects: AnyEntity[]): Promise<void>;
+}
+
+/**
+ * @deprecated
+ */
+export class QueuesAPIImpl implements QueuesAPI {
+  constructor(
+    private readonly _serviceContainer: ServiceContainer,
+    private readonly _spaceId: SpaceId,
+  ) {}
+
+  async queryQueue(queue: DXN, options?: {}): Promise<QueuesQueryResult> {
+    const result = await this._serviceContainer.queryQueue(queue);
+    const objects = (result.objects ?? []).flatMap((encoded): AnyEntity[] => {
+      try {
+        return [JSON.parse(encoded) as AnyEntity];
+      } catch (err) {
+        log.verbose('queue object JSON parse failed; object ignored', { encoded, error: err });
+        return [];
+      }
+    });
+    return {
+      objects,
+      nextCursor: result.nextCursor ?? null,
+      prevCursor: result.prevCursor ?? null,
+    };
+  }
+
+  insertIntoQueue(queue: DXN, objects: AnyEntity[]): Promise<void> {
+    // TODO(dmaretskyi): Ugly.
+    return this._serviceContainer.insertIntoQueue(queue, JSON.parse(JSON.stringify(objects)));
+  }
+}

package/src/space-proxy.ts

@@ -0,0 +1,66 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { Resource } from '@dxos/context';
+import { type Database } from '@dxos/echo';
+import { type CoreDatabase, type EchoClient, type EchoDatabaseImpl } from '@dxos/echo-db';
+import { invariant } from '@dxos/invariant';
+import { PublicKey, type SpaceId } from '@dxos/keys';
+
+import type { ServiceContainer } from './internal';
+import { type QueuesAPI, QueuesAPIImpl } from './queues-api';
+
+/**
+ * @deprecated
+ */
+export class SpaceProxy extends Resource {
+  private _db?: EchoDatabaseImpl = undefined;
+  private _queuesApi: QueuesAPIImpl;
+
+  constructor(
+    private readonly _serviceContainer: ServiceContainer,
+    private readonly _echoClient: EchoClient,
+    private readonly _id: SpaceId,
+  ) {
+    super();
+    this._queuesApi = new QueuesAPIImpl(this._serviceContainer, this._id);
+  }
+
+  get id(): SpaceId {
+    return this._id;
+  }
+
+  get db(): Database.Database {
+    invariant(this._db);
+    return this._db;
+  }
+
+  /**
+   * @deprecated Use db API.
+   */
+  get crud(): CoreDatabase {
+    invariant(this._db);
+    return this._db.coreDatabase;
+  }
+
+  get queues(): QueuesAPI {
+    return this._queuesApi;
+  }
+
+  protected override async _open() {
+    const meta = await this._serviceContainer.getSpaceMeta(this._id);
+    if (!meta) {
+      throw new Error(`Space not found: ${this._id}`);
+    }
+
+    this._db = this._echoClient.constructDatabase({
+      spaceId: this._id,
+      spaceKey: PublicKey.from(meta.spaceKey),
+      reactiveSchemaQuery: false,
+      owningObject: this,
+    });
+
+    await this._db.coreDatabase.open(this._ctx, { rootUrl: meta.rootDocumentId });
+  }
+}