@dxos/functions 0.7.5-labs.8a82073 → 0.7.5-labs.c0e040f

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dxos/functions",
-  "version": "0.7.5-labs.8a82073",
+  "version": "0.7.5-labs.c0e040f",
   "description": "Functions API and runtime.",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
@@ -13,6 +13,11 @@
       "browser": "./dist/lib/browser/index.mjs",
       "node": "./dist/lib/node-esm/index.mjs"
     },
+    "./bundler": {
+      "types": "./dist/types/src/bundler/index.d.ts",
+      "browser": "./dist/lib/browser/bundler/index.mjs",
+      "node": "./dist/lib/node-esm/bundler/index.mjs"
+    },
     "./edge": {
       "types": "./dist/types/src/edge/index.d.ts",
       "browser": "./dist/lib/browser/edge/index.mjs",
@@ -32,6 +37,12 @@
   "types": "dist/types/src/index.d.ts",
   "typesVersions": {
     "*": {
+      "bundler": [
+        "dist/types/src/bundler/index.d.ts"
+      ],
+      "edge": [
+        "dist/types/src/edge/index.d.ts"
+      ],
       "testing": [
         "dist/types/src/testing/index.d.ts"
       ],
@@ -50,30 +61,32 @@
     "@preact/signals-core": "^1.6.0",
     "cron": "^3.1.6",
     "effect": "^3.12.3",
+    "esbuild-wasm": "^0.16.14",
     "express": "^4.19.2",
     "get-port-please": "^3.1.1",
     "iso-did": "^1.6.0",
     "ws": "^8.14.2",
-    "@dxos/async": "0.7.5-labs.8a82073",
-    "@dxos/context": "0.7.5-labs.8a82073",
-    "@dxos/client": "0.7.5-labs.8a82073",
-    "@dxos/echo-db": "0.7.5-labs.8a82073",
-    "@dxos/echo-protocol": "0.7.5-labs.8a82073",
-    "@dxos/echo-schema": "0.7.5-labs.8a82073",
-    "@dxos/invariant": "0.7.5-labs.8a82073",
-    "@dxos/keys": "0.7.5-labs.8a82073",
-    "@dxos/edge-client": "0.7.5-labs.8a82073",
-    "@dxos/live-object": "0.7.5-labs.8a82073",
-    "@dxos/log": "0.7.5-labs.8a82073",
-    "@dxos/node-std": "0.7.5-labs.8a82073",
-    "@dxos/protocols": "0.7.5-labs.8a82073",
-    "@dxos/util": "0.7.5-labs.8a82073",
-    "@dxos/schema": "0.7.5-labs.8a82073"
+    "@dxos/async": "0.7.5-labs.c0e040f",
+    "@dxos/client": "0.7.5-labs.c0e040f",
+    "@dxos/context": "0.7.5-labs.c0e040f",
+    "@dxos/crypto": "0.7.5-labs.c0e040f",
+    "@dxos/echo-db": "0.7.5-labs.c0e040f",
+    "@dxos/echo-schema": "0.7.5-labs.c0e040f",
+    "@dxos/echo-protocol": "0.7.5-labs.c0e040f",
+    "@dxos/edge-client": "0.7.5-labs.c0e040f",
+    "@dxos/invariant": "0.7.5-labs.c0e040f",
+    "@dxos/keys": "0.7.5-labs.c0e040f",
+    "@dxos/live-object": "0.7.5-labs.c0e040f",
+    "@dxos/node-std": "0.7.5-labs.c0e040f",
+    "@dxos/log": "0.7.5-labs.c0e040f",
+    "@dxos/schema": "0.7.5-labs.c0e040f",
+    "@dxos/util": "0.7.5-labs.c0e040f",
+    "@dxos/protocols": "0.7.5-labs.c0e040f"
   },
   "devDependencies": {
     "@types/express": "^4.17.17",
     "@types/ws": "^7.4.0",
-    "@dxos/agent": "0.7.5-labs.8a82073"
+    "@dxos/agent": "0.7.5-labs.c0e040f"
   },
   "publishConfig": {
     "access": "public"

package/src/bundler/bundler.test.ts

@@ -0,0 +1,59 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+// @ts-ignore
+import wasmUrl from 'esbuild-wasm/esbuild.wasm?url';
+import { beforeAll, describe, expect, test } from 'vitest';
+
+import { isNode } from '@dxos/util';
+
+import { Bundler, initializeBundler } from './bundler';
+
+describe('Bundler', () => {
+  beforeAll(async () => {
+    if (!isNode()) {
+      await initializeBundler({ wasmUrl });
+    }
+  });
+
+  test('Basic', async () => {
+    const bundler = new Bundler({ platform: 'node', sandboxedModules: [], remoteModules: {} });
+    const result = await bundler.bundle({ source: 'const x = 100' }); // TODO(burdon): Test import.
+    expect(result.bundle).to.exist;
+    expect(result.error).to.not.exist;
+  });
+
+  test('Import', async () => {
+    const bundler = new Bundler({ platform: 'node', sandboxedModules: [], remoteModules: {} });
+    const result = await bundler.bundle({
+      source: `
+      import { Filter } from './runtime.js';
+
+      const query = Filter.typename('dxos.org/type/Example');
+    `,
+    });
+    expect(result.bundle).to.exist;
+    expect(result.error).to.not.exist;
+  });
+
+  // TODO(dmaretskyi): Flaky on CI: https://cloud.nx.app/runs/Hjcifa8Ccq/task/plugin-script%3Atest
+  test.skip('HTTPS Import', async () => {
+    const bundler = new Bundler({ platform: 'node', sandboxedModules: [], remoteModules: {} });
+    const result = await bundler.bundle({
+      source: `
+      import { invariant } from 'https://esm.sh/@dxos/invariant';
+      invariant(true);
+    `,
+    });
+    expect(result.bundle).to.exist;
+    expect(result.error).to.not.exist;
+  });
+
+  test('Error', async () => {
+    const bundler = new Bundler({ platform: 'node', sandboxedModules: [], remoteModules: {} });
+    const result = await bundler.bundle({ source: "import missing from './module'; missing();" });
+    expect(result.bundle).to.not.exist;
+    expect(result.error).to.exist;
+  });
+});

package/src/bundler/bundler.ts

@@ -0,0 +1,264 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import { type BuildOptions } from 'esbuild';
+import { build, initialize, type BuildResult, type Plugin } from 'esbuild-wasm';
+
+import { subtleCrypto } from '@dxos/crypto';
+import { invariant } from '@dxos/invariant';
+import { log } from '@dxos/log';
+
+export type Import = {
+  moduleUrl: string;
+  defaultImport: boolean;
+  namedImports: string[];
+};
+
+export type BundleOptions = {
+  /**
+   * Path to the source file on the local file system.
+   * If provided, the path will be used instead of the `source` code.
+   */
+  path?: string;
+
+  /**
+   * Source code to bundle.
+   * Required if `path` is not provided.
+   */
+  source?: string;
+};
+
+export type BundleResult = {
+  timestamp: number;
+  sourceHash?: Buffer;
+  imports?: Import[];
+  bundle?: string;
+  error?: any;
+};
+
+export type BundlerOptions = {
+  platform: BuildOptions['platform'];
+  sandboxedModules: string[];
+  remoteModules: Record<string, string>;
+};
+
+let initialized: Promise<void>;
+export const initializeBundler = async (options: { wasmUrl: string }) => {
+  await (initialized ??= initialize({
+    wasmURL: options.wasmUrl,
+  }));
+};
+
+/**
+ * ESBuild bundler.
+ */
+export class Bundler {
+  constructor(private readonly _options: BundlerOptions) {}
+
+  async bundle({ path, source }: BundleOptions): Promise<BundleResult> {
+    const { sandboxedModules: providedModules, ...options } = this._options;
+
+    const createResult = async (result?: Partial<BundleResult>) => {
+      return {
+        timestamp: Date.now(),
+        sourceHash: source ? Buffer.from(await subtleCrypto.digest('SHA-256', Buffer.from(source))) : undefined,
+        ...result,
+      };
+    };
+
+    if (this._options.platform === 'browser') {
+      invariant(initialized, 'Compiler not initialized.');
+      await initialized;
+    }
+
+    const imports = source ? analyzeSourceFileImports(source) : [];
+
+    // https://esbuild.github.io/api/#build
+    try {
+      const result = await build({
+        platform: options.platform,
+        conditions: ['workerd', 'browser'],
+        metafile: true,
+        write: false,
+        entryPoints: [path ?? 'memory:main.tsx'],
+        bundle: true,
+        format: 'esm',
+        plugins: [
+          {
+            name: 'memory',
+            setup: (build) => {
+              build.onResolve({ filter: /^\.\/runtime\.js$/ }, ({ path }) => {
+                return { path, external: true };
+              });
+
+              build.onResolve({ filter: /^dxos:functions$/ }, ({ path }) => {
+                return { path: './runtime.js', external: true };
+              });
+
+              build.onResolve({ filter: /^memory:/ }, ({ path }) => {
+                return { path: path.split(':')[1], namespace: 'memory' };
+              });
+
+              build.onLoad({ filter: /.*/, namespace: 'memory' }, ({ path }) => {
+                if (path === 'main.tsx') {
+                  return {
+                    contents: source,
+                    loader: 'tsx',
+                  };
+                }
+              });
+
+              for (const module of providedModules) {
+                build.onResolve({ filter: new RegExp(`^${module}$`) }, ({ path }) => {
+                  return { path, namespace: 'injected-module' };
+                });
+              }
+
+              build.onLoad({ filter: /.*/, namespace: 'injected-module' }, ({ path }) => {
+                const namedImports = imports.find((entry) => entry.moduleIdentifier === path)?.namedImports ?? [];
+                return {
+                  contents: `
+                  const { ${namedImports.join(',')} } = window.__DXOS_SANDBOX_MODULES__[${JSON.stringify(path)}];
+                  export { ${namedImports.join(',')} };
+                  export default window.__DXOS_SANDBOX_MODULES__[${JSON.stringify(path)}].default;
+                `,
+                  loader: 'tsx',
+                };
+              });
+            },
+          },
+          httpPlugin,
+        ],
+      });
+
+      log('compile complete', result.metafile);
+
+      return await createResult({
+        imports: this.analyzeImports(result),
+        bundle: result.outputFiles![0].text,
+      });
+    } catch (err) {
+      return await createResult({ error: err });
+    }
+  }
+
+  // TODO(dmaretskyi): In the future we can replace the compiler with SWC with plugins running in WASM.
+  analyzeImports(result: BuildResult): Import[] {
+    invariant(result.outputFiles);
+
+    // TODO(dmaretskyi): Support import aliases and wildcard imports.
+    const parsedImports = allMatches(IMPORT_REGEX, result.outputFiles[0].text);
+    return Object.values(result.metafile!.outputs)[0].imports.map((entry): Import => {
+      const namedImports: string[] = [];
+
+      const parsedImport = parsedImports.find((capture) => capture?.[4] === entry.path);
+      if (parsedImport?.[2]) {
+        NAMED_IMPORTS_REGEX.lastIndex = 0;
+        const namedImportsMatch = NAMED_IMPORTS_REGEX.exec(parsedImport[2]);
+        if (namedImportsMatch) {
+          namedImportsMatch[1].split(',').forEach((importName) => {
+            namedImports.push(importName.trim());
+          });
+        }
+      }
+
+      return {
+        moduleUrl: entry.path,
+        defaultImport: !!parsedImport?.[1],
+        namedImports,
+      };
+    });
+  }
+
+  analyzeSourceFileImports(code: string) {
+    // TODO(dmaretskyi): Support import aliases and wildcard imports.
+    const parsedImports = allMatches(IMPORT_REGEX, code);
+    return parsedImports.map((capture) => {
+      return {
+        defaultImportName: capture[1],
+        namedImports: capture[2]?.split(',').map((importName) => importName.trim()),
+        wildcardImportName: capture[3],
+        moduleIdentifier: capture[4],
+        quotes: capture[5],
+      };
+    });
+  }
+}
+
+// https://regex101.com/r/FEN5ks/1
+// https://stackoverflow.com/a/73265022
+// $1 = default import name (can be non-existent)
+// $2 = destructured exports (can be non-existent)
+// $3 = wildcard import name (can be non-existent)
+// $4 = module identifier
+// $5 = quotes used (either ' or ")
+const IMPORT_REGEX =
+  /import(?:(?:(?:[ \n\t]+([^ *\n\t{},]+)[ \n\t]*(?:,|[ \n\t]+))?([ \n\t]*{(?:[ \n\t]*[^ \n\t"'{}]+[ \n\t]*,?)+})?[ \n\t]*)|[ \n\t]*\*[ \n\t]*as[ \n\t]+([^ \n\t{}]+)[ \n\t]+)from[ \n\t]*(?:['"])([^'"\n]+)(['"])/gm;
+
+const NAMED_IMPORTS_REGEX = /[ \n\t]*{((?:[ \n\t]*[^ \n\t"'{}]+[ \n\t]*,?)+)}[ \n\t]*/gm;
+
+const allMatches = (regex: RegExp, str: string) => {
+  regex.lastIndex = 0;
+
+  let match;
+  const matches = [];
+  while ((match = regex.exec(str))) {
+    matches.push(match);
+  }
+
+  return matches;
+};
+
+type ParsedImport = {
+  defaultImportName?: string;
+  namedImports: string[];
+  wildcardImportName?: string;
+  moduleIdentifier: string;
+  quotes: string;
+};
+
+const analyzeSourceFileImports = (code: string): ParsedImport[] => {
+  // TODO(dmaretskyi): Support import aliases and wildcard imports.
+  const parsedImports = allMatches(IMPORT_REGEX, code);
+  return parsedImports.map((capture) => {
+    return {
+      defaultImportName: capture[1],
+      namedImports: capture[2]
+        ?.trim()
+        .slice(1, -1)
+        .split(',')
+        .map((importName) => importName.trim()),
+      wildcardImportName: capture[3],
+      moduleIdentifier: capture[4],
+      quotes: capture[5],
+    };
+  });
+};
+
+const httpPlugin: Plugin = {
+  name: 'http',
+  setup: (build) => {
+    // Intercept import paths starting with "http:" and "https:" so esbuild doesn't attempt to map them to a file system location.
+    // Tag them with the "http-url" namespace to associate them with this plugin.
+    build.onResolve({ filter: /^https?:\/\// }, (args) => ({
+      path: args.path,
+      namespace: 'http-url',
+    }));
+
+    // We also want to intercept all import paths inside downloaded files and resolve them against the original URL.
+    // All of these files will be in the "http-url" namespace.
+    // Make sure to keep the newly resolved URL in the "http-url" namespace so imports inside it will also be resolved as URLs recursively.
+    build.onResolve({ filter: /.*/, namespace: 'http-url' }, (args) => ({
+      path: new URL(args.path, args.importer).toString(),
+      namespace: 'http-url',
+    }));
+
+    // When a URL is loaded, we want to actually download the content from the internet.
+    // This has just enough logic to be able to handle the example import from unpkg.com but in reality this would probably need to be more complex.
+    build.onLoad({ filter: /.*/, namespace: 'http-url' }, async (args) => {
+      const response = await fetch(args.path);
+      return { contents: await response.text(), loader: 'jsx' };
+    });
+  },
+};

package/src/bundler/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+export * from './bundler';

package/src/handler.ts

@@ -8,7 +8,7 @@ import { type Client, PublicKey } from '@dxos/client';
 import { type Space, type SpaceId } from '@dxos/client/echo';
 import type { CoreDatabase, ReactiveEchoObject } from '@dxos/echo-db';
 import { log } from '@dxos/log';
-import { nonNullable } from '@dxos/util';
+import { isNonNullable } from '@dxos/util';
 
 // TODO(burdon): Model after http request. Ref Lambda/OpenFaaS.
 // https://docs.aws.amazon.com/lambda/latest/dg/typescript-handler.html
@@ -136,7 +136,9 @@ export const subscriptionHandler = <TMeta>(
 
     registerTypes(space, types);
     const objects = space
-      ? data.objects?.map<ReactiveEchoObject<any> | undefined>((id) => space!.db.getObjectById(id)).filter(nonNullable)
+      ? data.objects
+          ?.map<ReactiveEchoObject<any> | undefined>((id) => space!.db.getObjectById(id))
+          .filter(isNonNullable)
       : [];
 
     if (!!data.spaceKey && !space) {