@dxos/functions-runtime-cloudflare 0.8.4-main.21d9917

package/package.json

@@ -0,0 +1,52 @@
+{
+  "name": "@dxos/functions-runtime-cloudflare",
+  "version": "0.8.4-main.21d9917",
+  "description": "Functions runtime for Cloudflare.",
+  "homepage": "https://dxos.org",
+  "bugs": "https://github.com/dxos/dxos/issues",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dxos/dxos"
+  },
+  "license": "MIT",
+  "author": "info@dxos.org",
+  "sideEffects": false,
+  "type": "module",
+  "exports": {
+    ".": {
+      "source": "./src/index.ts",
+      "types": "./dist/types/src/index.d.ts",
+      "browser": "./dist/lib/browser/index.mjs",
+      "node": "./dist/lib/node-esm/index.mjs"
+    }
+  },
+  "types": "dist/types/src/index.d.ts",
+  "files": [
+    "dist",
+    "schema",
+    "src"
+  ],
+  "dependencies": {
+    "effect": "3.19.11",
+    "@dxos/context": "0.8.4-main.21d9917",
+    "@dxos/debug": "0.8.4-main.21d9917",
+    "@dxos/codec-protobuf": "0.8.4-main.21d9917",
+    "@dxos/echo-db": "0.8.4-main.21d9917",
+    "@dxos/echo": "0.8.4-main.21d9917",
+    "@dxos/echo-protocol": "0.8.4-main.21d9917",
+    "@dxos/errors": "0.8.4-main.21d9917",
+    "@dxos/invariant": "0.8.4-main.21d9917",
+    "@dxos/keys": "0.8.4-main.21d9917",
+    "@dxos/protocols": "0.8.4-main.21d9917",
+    "@dxos/log": "0.8.4-main.21d9917",
+    "@dxos/util": "0.8.4-main.21d9917"
+  },
+  "devDependencies": {
+    "@cloudflare/workers-types": "^4.20251210.0",
+    "@dxos/types": "0.8.4-main.21d9917"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {}
+}

package/src/functions-client.ts

@@ -0,0 +1,88 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { Resource } from '@dxos/context';
+import { EchoClient } from '@dxos/echo-db';
+import { invariant } from '@dxos/invariant';
+import { type SpaceId } from '@dxos/keys';
+import { type EdgeFunctionEnv } from '@dxos/protocols';
+
+import { ServiceContainer } from './internal';
+import { SpaceProxy } from './space-proxy';
+
+type Services = {
+  dataService: EdgeFunctionEnv.DataService;
+  queueService: EdgeFunctionEnv.QueueService;
+  functionsAiService: EdgeFunctionEnv.FunctionsAiService;
+};
+
+/**
+ * API for functions to integrate with ECHO and HALO.
+ * @deprecated
+ */
+export class FunctionsClient extends Resource {
+  private readonly _serviceContainer;
+  private readonly _echoClient;
+  private readonly _executionContext: EdgeFunctionEnv.ExecutionContext = {};
+
+  private readonly _spaces = new Map<SpaceId, SpaceProxy>();
+
+  constructor(services: Services) {
+    super();
+    invariant(typeof services.dataService !== 'undefined', 'DataService is required');
+    invariant(typeof services.queueService !== 'undefined', 'QueueService is required');
+    this._serviceContainer = new ServiceContainer(
+      this._executionContext,
+      services.dataService,
+      services.queueService,
+      services.functionsAiService,
+    );
+    this._echoClient = new EchoClient({});
+  }
+
+  get echo(): EchoClient {
+    return this._echoClient;
+  }
+
+  protected override async _open() {
+    const { dataService, queryService } = await this._serviceContainer.createServices();
+    this._echoClient.connectToService({ dataService, queryService });
+    await this._echoClient.open();
+  }
+
+  protected override async _close() {
+    for (const space of this._spaces.values()) {
+      await space.close();
+    }
+    this._spaces.clear();
+
+    await this._echoClient.close();
+  }
+
+  async getSpace(spaceId: SpaceId): Promise<SpaceProxy> {
+    if (!this._spaces.has(spaceId)) {
+      const space = new SpaceProxy(this._serviceContainer, this._echoClient, spaceId);
+      this._spaces.set(spaceId, space);
+    }
+    const space = this._spaces.get(spaceId)!;
+    await space.open(); // No-op if already open.
+    return space;
+  }
+}
+
+export const createClientFromEnv = async (env: any): Promise<FunctionsClient> => {
+  const client = new FunctionsClient({
+    dataService: env.DATA_SERVICE,
+    queueService: env.QUEUE_SERVICE,
+    functionsAiService: env.FUNCTIONS_AI_SERVICE,
+  });
+  await client.open();
+  return client;
+};
+
+/**
+ - Provides data access capabilities for user functions.
+ - No real-time replication or reactive queries -- function receives a snapshot.
+ - Function event contains the metadata but doesn't need to include the data.
+ */

package/src/index.ts

@@ -0,0 +1,9 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export * from './functions-client';
+export * from './internal';
+export * from './types';
+export * from './wrap-handler-for-cloudflare';
+export * from './logger';

package/src/internal/adapter.ts

@@ -0,0 +1,48 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { failUndefined } from '@dxos/debug';
+import { type QueryAST } from '@dxos/echo-protocol';
+import { invariant } from '@dxos/invariant';
+import { SpaceId } from '@dxos/keys';
+import { type EdgeFunctionEnv } from '@dxos/protocols';
+
+export const queryToDataServiceRequest = (query: QueryAST.Query): EdgeFunctionEnv.QueryRequest => {
+  const { filter, options } = isSimpleSelectionQuery(query) ?? failUndefined();
+  invariant(options?.spaceIds?.length === 1, 'Only one space is supported');
+  invariant(filter.type === 'object', 'Only object filters are supported');
+
+  const spaceId = options.spaceIds[0];
+  invariant(SpaceId.isValid(spaceId));
+
+  return {
+    spaceId,
+    type: filter.typename ?? undefined,
+    objectIds: [...(filter.id ?? [])],
+  };
+};
+
+/**
+ * Extracts the filter and options from a query.
+ * Supports Select(...) and Options(Select(...)) queries.
+ */
+export const isSimpleSelectionQuery = (
+  query: QueryAST.Query,
+): { filter: QueryAST.Filter; options?: QueryAST.QueryOptions } | null => {
+  switch (query.type) {
+    case 'options': {
+      const maybeFilter = isSimpleSelectionQuery(query.query);
+      if (!maybeFilter) {
+        return null;
+      }
+      return { filter: maybeFilter.filter, options: query.options };
+    }
+    case 'select': {
+      return { filter: query.filter, options: undefined };
+    }
+    default: {
+      return null;
+    }
+  }
+};

package/src/internal/data-service-impl.ts

@@ -0,0 +1,142 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import type { RequestOptions } from '@dxos/codec-protobuf';
+import { Stream } from '@dxos/codec-protobuf/stream';
+import { raise } from '@dxos/debug';
+import { NotImplementedError, RuntimeServiceError } from '@dxos/errors';
+import { invariant } from '@dxos/invariant';
+import { SpaceId } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { type EdgeFunctionEnv } from '@dxos/protocols';
+import type {
+  BatchedDocumentUpdates,
+  CreateDocumentRequest,
+  CreateDocumentResponse,
+  DataService as DataServiceProto,
+  GetDocumentHeadsRequest,
+  GetDocumentHeadsResponse,
+  GetSpaceSyncStateRequest,
+  ReIndexHeadsRequest,
+  SpaceSyncState,
+  UpdateRequest,
+  UpdateSubscriptionRequest,
+} from '@dxos/protocols/proto/dxos/echo/service';
+
+import { copyUint8Array } from './utils';
+
+export class DataServiceImpl implements DataServiceProto {
+  private dataSubscriptions = new Map<string, { spaceId: SpaceId; next: (msg: BatchedDocumentUpdates) => void }>();
+
+  constructor(
+    private _executionContext: EdgeFunctionEnv.ExecutionContext,
+    private _dataService: EdgeFunctionEnv.DataService,
+  ) {}
+
+  subscribe({ subscriptionId, spaceId }: { subscriptionId: string; spaceId: string }): Stream<BatchedDocumentUpdates> {
+    return new Stream(({ next }) => {
+      invariant(SpaceId.isValid(spaceId));
+      this.dataSubscriptions.set(subscriptionId, { spaceId, next });
+
+      return () => {
+        this.dataSubscriptions.delete(subscriptionId);
+      };
+    });
+  }
+
+  async updateSubscription({ subscriptionId, addIds }: UpdateSubscriptionRequest): Promise<void> {
+    const sub =
+      this.dataSubscriptions.get(subscriptionId) ??
+      raise(
+        new RuntimeServiceError({
+          message: 'Subscription not found.',
+          context: { subscriptionId },
+        }),
+      );
+
+    if (addIds) {
+      log.info('request documents', { count: addIds.length });
+      // TODO(dmaretskyi): Batch.
+      for (const documentId of addIds) {
+        using document = await this._dataService.getDocument(this._executionContext, sub.spaceId, documentId);
+        log.info('document loaded', { documentId, spaceId: sub.spaceId, found: !!document });
+        if (!document) {
+          log.warn('not found', { documentId });
+          continue;
+        }
+        sub.next({
+          updates: [
+            {
+              documentId,
+              // Copy returned object to avoid hanging RPC stub
+              // See https://developers.cloudflare.com/workers/runtime-apis/rpc/lifecycle/
+              mutation: copyUint8Array(document.data),
+            },
+          ],
+        });
+      }
+    }
+  }
+
+  async createDocument({ spaceId, initialValue }: CreateDocumentRequest): Promise<CreateDocumentResponse> {
+    using response = await this._dataService.createDocument(this._executionContext, { spaceId, initialValue });
+    return { documentId: response.documentId };
+  }
+
+  async update({ updates, subscriptionId }: UpdateRequest): Promise<void> {
+    const sub =
+      this.dataSubscriptions.get(subscriptionId) ??
+      raise(
+        new RuntimeServiceError({
+          message: 'Subscription not found.',
+          context: { subscriptionId },
+        }),
+      );
+    // TODO(dmaretskyi): Batch.
+    try {
+      for (const update of updates ?? []) {
+        await this._dataService.changeDocument(this._executionContext, sub.spaceId, update.documentId, update.mutation);
+      }
+    } catch (error) {
+      throw RuntimeServiceError.wrap({
+        message: 'Failed to apply document updates.',
+        context: { subscriptionId },
+        ifTypeDiffers: true,
+      })(error);
+    }
+  }
+
+  async flush(): Promise<void> {
+    // No-op.
+  }
+
+  subscribeSpaceSyncState(_request: GetSpaceSyncStateRequest, _options?: RequestOptions): Stream<SpaceSyncState> {
+    throw new NotImplementedError({
+      message: 'subscribeSpaceSyncState is not implemented.',
+    });
+  }
+
+  async getDocumentHeads({ documentIds: _documentIds }: GetDocumentHeadsRequest): Promise<GetDocumentHeadsResponse> {
+    throw new NotImplementedError({
+      message: 'getDocumentHeads is not implemented.',
+    });
+  }
+
+  async reIndexHeads({ documentIds: _documentIds }: ReIndexHeadsRequest): Promise<void> {
+    throw new NotImplementedError({
+      message: 'reIndexHeads is not implemented.',
+    });
+  }
+
+  async updateIndexes(): Promise<void> {
+    log.error('updateIndexes is not available in EDGE env.');
+    // No-op.
+  }
+
+  async waitUntilHeadsReplicated({ heads: _heads }: { heads: any }): Promise<void> {
+    throw new NotImplementedError({
+      message: 'waitUntilHeadsReplicated is not implemented.',
+    });
+  }
+}

package/src/internal/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './service-container';

package/src/internal/query-service-impl.ts

@@ -0,0 +1,107 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import * as Schema from 'effect/Schema';
+
+import { Stream } from '@dxos/codec-protobuf/stream';
+import { QueryAST } from '@dxos/echo-protocol';
+import { NotImplementedError, RuntimeServiceError } from '@dxos/errors';
+import { invariant } from '@dxos/invariant';
+import { PublicKey } from '@dxos/keys';
+import { SpaceId } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { type EdgeFunctionEnv } from '@dxos/protocols';
+import {
+  type QueryRequest,
+  type QueryResponse,
+  type QueryResult as QueryResultProto,
+  type QueryService as QueryServiceProto,
+} from '@dxos/protocols/proto/dxos/echo/query';
+
+import { queryToDataServiceRequest } from './adapter';
+import { copyUint8Array } from './utils';
+
+export class QueryServiceImpl implements QueryServiceProto {
+  private _queryCount = 0;
+
+  constructor(
+    private readonly _executionContext: EdgeFunctionEnv.ExecutionContext,
+    private readonly _dataService: EdgeFunctionEnv.DataService,
+  ) {}
+
+  execQuery(request: QueryRequest): Stream<QueryResponse> {
+    log.info('execQuery', { request });
+    const query = QueryAST.Query.pipe(Schema.decodeUnknownSync)(JSON.parse(request.query));
+    const requestedSpaceIds = getTargetSpacesForQuery(query);
+    invariant(requestedSpaceIds.length === 1, 'Only one space is supported');
+    const spaceId = requestedSpaceIds[0];
+
+    return Stream.fromPromise<QueryResponse>(
+      (async () => {
+        try {
+          this._queryCount++;
+          log.info('begin query', { spaceId });
+          using queryResponse = await this._dataService.queryDocuments(
+            this._executionContext,
+            queryToDataServiceRequest(query),
+          );
+          log.info('query response', { spaceId, filter: request.filter, resultCount: queryResponse.results.length });
+          return {
+            results: queryResponse.results.map(
+              (object): QueryResultProto => ({
+                id: object.objectId,
+                spaceId,
+                spaceKey: PublicKey.ZERO,
+                documentId: object.document.documentId,
+                // Rank 1 for predicate matches where ranking is not determined.
+                rank: 1,
+                // Copy returned object to avoid hanging RPC stub.
+                // See https://developers.cloudflare.com/workers/runtime-apis/rpc/lifecycle/
+                documentAutomerge: copyUint8Array(object.document.data),
+              }),
+            ),
+          } satisfies QueryResponse;
+        } catch (error) {
+          log.error('query failed', { err: error });
+          throw new RuntimeServiceError({
+            message: `Query execution failed (queryCount=${this._queryCount})`,
+            context: { spaceId, filter: request.filter, queryCount: this._queryCount },
+            cause: error,
+          });
+        }
+      })(),
+    );
+  }
+
+  async reindex() {
+    throw new NotImplementedError({
+      message: 'Reindex is not implemented.',
+    });
+  }
+
+  async setConfig() {
+    throw new NotImplementedError({
+      message: 'SetConfig is not implemented.',
+    });
+  }
+}
+
+/**
+ * Lists spaces this query will select from.
+ */
+export const getTargetSpacesForQuery = (query: QueryAST.Query): SpaceId[] => {
+  const spaces = new Set<SpaceId>();
+
+  const visitor = (node: QueryAST.Query) => {
+    if (node.type === 'options') {
+      if (node.options.spaceIds) {
+        for (const spaceId of node.options.spaceIds) {
+          spaces.add(SpaceId.make(spaceId));
+        }
+      }
+    }
+  };
+  QueryAST.visit(query, visitor);
+  return [...spaces];
+};

package/src/internal/queue-service-impl.ts

@@ -0,0 +1,69 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { NotImplementedError, RuntimeServiceError } from '@dxos/errors';
+import { invariant } from '@dxos/invariant';
+import { type QueueService as QueueServiceProto } from '@dxos/protocols';
+import type {
+  DeleteFromQueueRequest,
+  EdgeFunctionEnv,
+  InsertIntoQueueRequest,
+  QueryQueueRequest,
+  QueryResult,
+} from '@dxos/protocols';
+
+export class QueueServiceImpl implements QueueServiceProto {
+  constructor(
+    protected _ctx: EdgeFunctionEnv.ExecutionContext,
+    private readonly _queueService: EdgeFunctionEnv.QueueService,
+  ) {}
+  async queryQueue(request: QueryQueueRequest): Promise<QueryResult> {
+    const { query } = request;
+    const { queueIds, ...filter } = query!;
+    const spaceId = query!.spaceId;
+    const queueId = queueIds?.[0];
+    invariant(request.query.queuesNamespace);
+    try {
+      using result = await this._queueService.query(
+        this._ctx,
+        `dxn:queue:${request.query.queuesNamespace}:${spaceId}:${queueId}`,
+        filter,
+      );
+      return {
+        // Copy returned object to avoid hanging RPC stub
+        // See https://developers.cloudflare.com/workers/runtime-apis/rpc/lifecycle/
+        objects: structuredClone(result.objects),
+        nextCursor: result.nextCursor,
+        prevCursor: result.prevCursor,
+      };
+    } catch (error) {
+      throw RuntimeServiceError.wrap({
+        message: 'Queue query failed.',
+        context: { subspaceTag: request.query.queuesNamespace, spaceId, queueId },
+        ifTypeDiffers: true,
+      })(error);
+    }
+  }
+
+  async insertIntoQueue(request: InsertIntoQueueRequest): Promise<void> {
+    const { subspaceTag, spaceId, queueId, objects } = request;
+    try {
+      await this._queueService.append(this._ctx, `dxn:queue:${subspaceTag}:${spaceId}:${queueId}`, objects ?? []);
+    } catch (error) {
+      throw RuntimeServiceError.wrap({
+        message: 'Queue append failed.',
+        context: { subspaceTag, spaceId, queueId },
+        ifTypeDiffers: true,
+      })(error);
+    }
+  }
+
+  deleteFromQueue(request: DeleteFromQueueRequest): Promise<void> {
+    const { subspaceTag, spaceId, queueId } = request;
+    throw new NotImplementedError({
+      message: 'Deleting from queue is not supported.',
+      context: { subspaceTag, spaceId, queueId },
+    });
+  }
+}

package/src/internal/service-container.ts

@@ -0,0 +1,73 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { type AnyEntity } from '@dxos/echo/internal';
+import { type DXN, type SpaceId } from '@dxos/keys';
+import type { QueryResult } from '@dxos/protocols';
+import { type EdgeFunctionEnv } from '@dxos/protocols';
+import { type QueueService as QueueServiceProto } from '@dxos/protocols';
+import { type QueryService as QueryServiceProto } from '@dxos/protocols/proto/dxos/echo/query';
+import type { DataService as DataServiceProto } from '@dxos/protocols/proto/dxos/echo/service';
+
+import { DataServiceImpl } from './data-service-impl';
+import { QueryServiceImpl } from './query-service-impl';
+import { QueueServiceImpl } from './queue-service-impl';
+
+/**
+ * TODO: make this implement DataService and QueryService to unify API over edge and web backend
+ */
+export class ServiceContainer {
+  constructor(
+    private readonly _executionContext: EdgeFunctionEnv.ExecutionContext,
+    private readonly _dataService: EdgeFunctionEnv.DataService,
+    private readonly _queueService: EdgeFunctionEnv.QueueService,
+    private readonly _functionsService: EdgeFunctionEnv.FunctionsAiService,
+  ) {}
+
+  async getSpaceMeta(spaceId: SpaceId): Promise<EdgeFunctionEnv.SpaceMeta | undefined> {
+    using result = await this._dataService.getSpaceMeta(this._executionContext, spaceId);
+    // Copy returned object to avoid hanging RPC stub
+    // See https://developers.cloudflare.com/workers/runtime-apis/rpc/lifecycle/
+    return result
+      ? {
+          spaceKey: result.spaceKey,
+          rootDocumentId: result.rootDocumentId,
+        }
+      : undefined;
+  }
+
+  async createServices(): Promise<{
+    dataService: DataServiceProto;
+    queryService: QueryServiceProto;
+    queueService: QueueServiceProto;
+    functionsAiService: EdgeFunctionEnv.FunctionsAiService;
+  }> {
+    const dataService = new DataServiceImpl(this._executionContext, this._dataService);
+    const queryService = new QueryServiceImpl(this._executionContext, this._dataService);
+    const queueService = new QueueServiceImpl(this._executionContext, this._queueService);
+
+    return {
+      dataService,
+      queryService,
+      queueService,
+      functionsAiService: this._functionsService,
+    };
+  }
+
+  async queryQueue(queue: DXN): Promise<QueryResult> {
+    const { spaceId } = queue.asQueueDXN() ?? {};
+    using result = (await this._queueService.query({}, queue.toString(), { spaceId: spaceId! })) as any;
+    // Copy returned object to avoid hanging RPC stub
+    // See https://developers.cloudflare.com/workers/runtime-apis/rpc/lifecycle/
+    return {
+      objects: structuredClone(result.objects),
+      nextCursor: result.nextCursor ?? null,
+      prevCursor: result.prevCursor ?? null,
+    };
+  }
+
+  async insertIntoQueue(queue: DXN, objects: AnyEntity[]): Promise<void> {
+    await this._queueService.append({}, queue.toString(), objects);
+  }
+}

package/src/internal/utils.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export const copyUint8Array = (value: Uint8Array): Uint8Array => new Uint8Array(value);

package/src/logger.ts

@@ -0,0 +1,42 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+/* eslint-disable no-console */
+
+import { LogLevel, type LogProcessor, log, shouldLog } from '@dxos/log';
+
+export const setupFunctionsLogger = () => {
+  log.runtimeConfig.processors.length = 0;
+  log.runtimeConfig.processors.push(functionLogProcessor);
+};
+
+const functionLogProcessor: LogProcessor = (config, entry) => {
+  if (!shouldLog(entry, config.filters)) {
+    return;
+  }
+
+  switch (entry.level) {
+    case LogLevel.DEBUG:
+      console.debug(entry.message, entry.context);
+      break;
+    case LogLevel.TRACE:
+      console.debug(entry.message, entry.context);
+      break;
+    case LogLevel.VERBOSE:
+      console.log(entry.message, entry.context);
+      break;
+    case LogLevel.INFO:
+      console.info(entry.message, entry.context);
+      break;
+    case LogLevel.WARN:
+      console.warn(entry.message, entry.context);
+      break;
+    case LogLevel.ERROR:
+      console.error(entry.message, entry.context);
+      break;
+    default:
+      console.log(entry.message, entry.context);
+      break;
+  }
+};

package/src/queues-api.ts

@@ -0,0 +1,38 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { type AnyEntity } from '@dxos/echo/internal';
+import type { DXN, SpaceId } from '@dxos/keys';
+import type { QueryResult } from '@dxos/protocols';
+
+import type { ServiceContainer } from './internal';
+
+// TODO(dmaretskyi): Temporary API to get the queues working.
+// TODO(dmaretskyi): To be replaced with integrating queues into echo.
+/**
+ * @deprecated
+ */
+export interface QueuesAPI {
+  queryQueue(queue: DXN, options?: {}): Promise<QueryResult>;
+  insertIntoQueue(queue: DXN, objects: AnyEntity[]): Promise<void>;
+}
+
+/**
+ * @deprecated
+ */
+export class QueuesAPIImpl implements QueuesAPI {
+  constructor(
+    private readonly _serviceContainer: ServiceContainer,
+    private readonly _spaceId: SpaceId,
+  ) {}
+
+  queryQueue(queue: DXN, options?: {}): Promise<QueryResult> {
+    return this._serviceContainer.queryQueue(queue);
+  }
+
+  insertIntoQueue(queue: DXN, objects: AnyEntity[]): Promise<void> {
+    // TODO(dmaretskyi): Ugly.
+    return this._serviceContainer.insertIntoQueue(queue, JSON.parse(JSON.stringify(objects)));
+  }
+}