@silverbulletmd/silverbullet 2.4.1

package/client/plugos/manifest_cache.ts

@@ -0,0 +1,67 @@
+import type { KvPrimitives } from "../data/kv_primitives.ts";
+import type { Plug } from "./plug.ts";
+import type { Manifest } from "./types.ts";
+
+export interface ManifestCache<T> {
+  getManifest(
+    plug: Plug<T>,
+    cacheKey: string,
+    cacheHash: number,
+  ): Promise<Manifest<T>>;
+}
+
+export class KVPrimitivesManifestCache<T> implements ManifestCache<T> {
+  constructor(private kv: KvPrimitives, private manifestPrefix: string) {
+  }
+
+  async getManifest(
+    plug: Plug<T>,
+    cacheKey: string,
+    cacheHash: number,
+  ): Promise<Manifest<T>> {
+    const [cached] = await this.kv.batchGet([[
+      this.manifestPrefix,
+      cacheKey,
+    ]]);
+    if (cached && cached.hash === cacheHash) {
+      // console.log("Using KV cached manifest for", plug.name);
+      return cached.manifest;
+    }
+    await plug.sandbox.init();
+    const manifest = plug.sandbox.manifest!;
+    await this.kv.batchSet([{
+      key: [this.manifestPrefix, cacheKey],
+      // Deliverately removing the assets from the manifest to preserve space, will be re-added upon load of actual worker
+      value: { manifest: { ...manifest, assets: undefined }, hash: cacheHash },
+    }]);
+    return manifest;
+  }
+}
+
+export class InMemoryManifestCache<T> implements ManifestCache<T> {
+  private cache = new Map<string, {
+    manifest: Manifest<T>;
+    hash: number;
+  }>();
+
+  async getManifest(
+    plug: Plug<T>,
+    cacheKey: string,
+    cacheHash: number,
+  ): Promise<Manifest<T>> {
+    const cached = this.cache.get(cacheKey);
+    if (cached && cached.hash === cacheHash) {
+      // console.log("Using memory cached manifest for", plug.name);
+      return cached.manifest;
+    }
+    await plug.sandbox.init();
+    const manifest = plug.sandbox.manifest!;
+
+    // Deliverately removing the assets from the manifest to preserve space, will be re-added upon load of actual worker
+    this.cache.set(cacheKey, {
+      manifest: { ...manifest, assets: undefined },
+      hash: cacheHash,
+    });
+    return manifest;
+  }
+}

package/client/plugos/plug.ts

@@ -0,0 +1,99 @@
+import type { Manifest } from "./types.ts";
+import type { System } from "./system.ts";
+import type { AssetBundle } from "../asset_bundle/bundle.ts";
+import type { Sandbox, SandboxFactory } from "./sandboxes/sandbox.ts";
+
+export class Plug<HookT> {
+  readonly runtimeEnv?: string;
+
+  public grantedPermissions: string[] = [];
+  public sandbox: Sandbox<HookT>;
+
+  public manifest!: Manifest<HookT>;
+  public assets?: AssetBundle;
+
+  constructor(
+    readonly system: System<HookT>,
+    sandboxFactory: SandboxFactory<HookT>,
+  ) {
+    this.runtimeEnv = system.env;
+    this.sandbox = sandboxFactory(this);
+  }
+
+  static async createLazily<HookT>(
+    system: System<HookT>,
+    cacheKey: string,
+    cacheHash: number,
+    sandboxFactory: SandboxFactory<HookT>,
+  ): Promise<Plug<HookT>> {
+    const plug = new Plug(
+      system,
+      sandboxFactory,
+    );
+
+    // Retrieve the manifest, which may either come from a cache or be loaded from the worker
+    plug.manifest = await system.options.manifestCache!.getManifest(
+      plug,
+      cacheKey,
+      cacheHash,
+    );
+
+    // TODO: These need to be explicitly granted, not just taken
+    plug.grantedPermissions = plug.manifest.requiredPermissions || [];
+
+    return plug;
+  }
+
+  // Invoke a syscall
+  syscall(name: string, args: any[]): Promise<any> {
+    return this.system.syscall({ plug: this.manifest.name }, name, args);
+  }
+
+  /**
+   * Checks if a function can be invoked (it may be restricted on its execution environment)
+   */
+  canInvoke(name: string) {
+    const funDef = this.manifest!.functions[name];
+    if (!funDef) {
+      throw new Error(`Function ${name} not found in manifest`);
+    }
+    return !funDef.env || !this.runtimeEnv || funDef.env === this.runtimeEnv;
+  }
+
+  // Invoke a function
+  async invoke(name: string, args: any[]): Promise<any> {
+    // Before we access the manifest
+    const funDef = this.manifest!.functions[name];
+    if (!funDef) {
+      throw new Error(`Function ${name} not found in manifest`);
+    }
+    const sandbox = this.sandbox!;
+    if (funDef.redirect) {
+      // Function redirect, look up
+      // deno-lint-ignore no-this-alias
+      let plug: Plug<HookT> | undefined = this;
+      if (funDef.redirect.indexOf(".") !== -1) {
+        const [plugName, functionName] = funDef.redirect.split(".");
+        plug = this.system.loadedPlugs.get(plugName);
+        if (!plug) {
+          throw Error(`Plug ${plugName} redirected to not found`);
+        }
+        name = functionName;
+      } else {
+        name = funDef.redirect;
+      }
+      return plug.invoke(name, args);
+    }
+    if (!await this.canInvoke(name)) {
+      throw new Error(
+        `Function ${name} is not available in ${this.runtimeEnv}`,
+      );
+    }
+    return sandbox.invoke(name, args);
+  }
+
+  stop() {
+    console.log("Stopping sandbox for", this.manifest.name);
+    this.sandbox.stop();
+  }
+}

package/client/plugos/plug_compile.ts

@@ -0,0 +1,202 @@
+import * as path from "node:path";
+import { readFile, writeFile, mkdtemp, rm, mkdir } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import * as YAML from "js-yaml";
+
+import { denoPlugin, esbuild } from "../../build_deps.ts";
+import { bundleAssets } from "../asset_bundle/builder.ts";
+import type { Manifest } from "./types.ts";
+import { version } from "../../version.ts";
+
+// const workerRuntimeUrl = new URL(
+//   "../lib/plugos/worker_runtime.ts",
+//   import.meta.url,
+// );
+const workerRuntimeUrl =
+  `https://deno.land/x/silverbullet@${version}/client/plugos/worker_runtime.ts`;
+
+export type CompileOptions = {
+  debug?: boolean;
+  runtimeUrl?: string;
+  // path to config file
+  configPath?: string;
+  // Print info on bundle size
+  info?: boolean;
+};
+
+export async function compileManifest(
+  manifestPath: string,
+  destPath: string,
+  options: CompileOptions = {},
+): Promise<string> {
+  const rootPath = path.dirname(manifestPath);
+  const manifestContent = await readFile(manifestPath, "utf-8");
+  const manifest = YAML.load(manifestContent) as Manifest<any>;
+
+  if (!manifest.name) {
+    throw new Error(`Missing 'name' in ${manifestPath}`);
+  }
+
+  // Assets
+  const assetsBundle = await bundleAssets(
+    path.resolve(rootPath),
+    manifest.assets as string[] || [],
+  );
+  manifest.assets = assetsBundle.toJSON();
+
+  // Normalize the edge case of a plug with no functions
+  if (!manifest.functions) {
+    manifest.functions = {};
+  }
+
+  const jsFile = `
+import { setupMessageListener } from "${
+    options.runtimeUrl || workerRuntimeUrl
+  }";
+
+// Imports
+${
+    Object.entries(manifest.functions).map(([funcName, def]) => {
+      if (!def.path) {
+        return "";
+      }
+      let [filePath, jsFunctionName] = def.path.split(":");
+      // Resolve path
+      filePath = path.join(rootPath, filePath);
+
+      return `import {${jsFunctionName} as ${funcName}} from "file://${
+        // Replacing \ with / for Windows
+        path.resolve(filePath).replaceAll(
+          "\\",
+          "\\\\",
+        )}";\n`;
+    }).join("")
+  }
+
+// Function mapping
+const functionMapping = {
+${
+    Object.entries(manifest.functions).map(([funcName, def]) => {
+      if (!def.path) {
+        return "";
+      }
+      return `  ${funcName}: ${funcName},\n`;
+    }).join("")
+  }
+};
+
+// Manifest
+const manifest = ${JSON.stringify(manifest, null, 2)};
+
+export const plug = {manifest, functionMapping};
+
+setupMessageListener(functionMapping, manifest, self.postMessage);
+`;
+
+  // console.log("Code:", jsFile);
+
+  const tempDir = await mkdtemp(path.join(tmpdir(), "plug-compile-"));
+  const inFile = path.join(tempDir, "input.js");
+  const outFile = `${destPath}/${manifest.name}.plug.js`;
+  await writeFile(inFile, jsFile, "utf-8");
+
+  const result = await esbuild.build({
+    entryPoints: [inFile],
+    bundle: true,
+    format: "esm",
+    globalName: "mod",
+    platform: "browser",
+    sourcemap: "linked",
+    minify: !options.debug,
+    outfile: outFile,
+    metafile: options.info,
+    treeShaking: true,
+    plugins: [
+      denoPlugin({
+        configPath: options.configPath &&
+          path.resolve(process.cwd(), options.configPath),
+      }),
+    ],
+  });
+
+  if (options.info) {
+    const text = await esbuild.analyzeMetafile(result.metafile!);
+    console.log("Bundle info for", manifestPath, text);
+  }
+
+  let jsCode = await readFile(outFile, "utf-8");
+  jsCode = patchDenoLibJS(jsCode);
+  await writeFile(outFile, jsCode, "utf-8");
+  
+  // Clean up temp directory
+  await rm(tempDir, { recursive: true, force: true });
+  
+  console.log(`Plug ${manifest.name} written to ${outFile}.`);
+  return outFile;
+}
+
+export async function compileManifests(
+  manifestFiles: string[],
+  dist: string,
+  options: CompileOptions = {},
+) {
+  let building = false;
+  dist = path.resolve(dist);
+
+  async function buildAll() {
+    if (building) {
+      return;
+    }
+    console.log("Building", manifestFiles);
+    building = true;
+    await mkdir(dist, { recursive: true });
+    const startTime = Date.now();
+    // Build all plugs in parallel
+    await Promise.all(manifestFiles.map(async (plugManifestPath) => {
+      const manifestPath = plugManifestPath as string;
+      try {
+        await compileManifest(
+          manifestPath,
+          dist,
+          options,
+        );
+      } catch (e: any) {
+        console.error(`Error building ${manifestPath}:`, e.message);
+        throw e;
+      }
+    }));
+    console.log(`Done building plugs in ${Date.now() - startTime}ms`);
+    building = false;
+  }
+
+  await buildAll();
+}
+
+export function patchDenoLibJS(code: string): string {
+  // The Deno std lib has one occurence of a regex that Webkit JS doesn't (yet parse), we'll strip it because it's likely never invoked anyway, YOLO
+  return code.replaceAll("/(?<=\\n)/", "/()/");
+}
+
+export async function plugCompileCommand(
+  { dist, debug, info, config, runtimeUrl }: {
+    dist: string;
+    debug: boolean;
+    info: boolean;
+    config?: string;
+    runtimeUrl?: string;
+  },
+  ...manifestPaths: string[]
+) {
+  await compileManifests(
+    manifestPaths,
+    dist,
+    {
+      debug: debug,
+      info: info,
+      runtimeUrl,
+      configPath: config,
+    },
+  );
+  esbuild.stop();
+  process.exit(0);
+}

package/client/plugos/protocol.ts

@@ -0,0 +1,40 @@
+import type { Manifest } from "@silverbulletmd/silverbullet/type/manifest";
+
+// Messages received from the worker
+export type ControllerMessage =
+  | {
+    // Parsed manifest when worker is initialized
+    type: "manifest";
+    manifest: Manifest;
+  }
+  | {
+    // Function invocation result
+    type: "invr";
+    id: number;
+    error?: string;
+    result?: any;
+  }
+  | {
+    // Syscall
+    type: "sys";
+    id: number;
+    name: string;
+    args: any[];
+  };
+
+// Messages received inside the worker
+export type WorkerMessage =
+  | {
+    // Function invocation
+    type: "inv";
+    id: number;
+    name: string;
+    args: any[];
+  }
+  | {
+    // Syscall result
+    type: "sysr";
+    id: number;
+    result?: any;
+    error?: any;
+  };

package/client/plugos/proxy_fetch.ts

@@ -0,0 +1,53 @@
+import { base64Decode, base64Encode } from "../../plug-api/lib/crypto.ts";
+
+export type ProxyFetchRequest64 = {
+  method?: string;
+  headers?: Record<string, string>;
+  base64Body?: string;
+};
+
+export type ProxyFetchResponse64 = {
+  ok: boolean;
+  status: number;
+  headers: Record<string, string>;
+  // We base64 encode the body because the body can be binary data that we have to push through the worker boundary
+  base64Body: string;
+};
+
+export type ProxyFetchRequest = {
+  method?: string;
+  headers?: Record<string, string>;
+  responseEncoding?: string;
+  body?: Uint8Array | string | any;
+};
+
+export type ProxyFetchResponse = {
+  ok: boolean;
+  status: number;
+  headers: Record<string, string>;
+  // We base64 encode the body because the body can be binary data that we have to push through the worker boundary
+  body: Uint8Array | string | any;
+};
+
+export async function performLocalFetch(
+  url: string,
+  req: ProxyFetchRequest64,
+): Promise<ProxyFetchResponse64> {
+  const result = await fetch(
+    url,
+    req && {
+      method: req.method,
+      headers: req.headers,
+      body: req.base64Body && base64Decode(req.base64Body),
+      // Casting  to "any" for now, since of weird Deno typing
+    } as any,
+  );
+  return {
+    ok: result.ok,
+    status: result.status,
+    headers: Object.fromEntries(result.headers.entries()),
+    base64Body: base64Encode(
+      new Uint8Array(await (await result.blob()).arrayBuffer()),
+    ),
+  };
+}

package/client/plugos/sandboxes/deno_worker_sandbox.ts

@@ -0,0 +1,6 @@
+import { WorkerSandbox } from "./worker_sandbox.ts";
+import type { SandboxFactory } from "./sandbox.ts";
+
+export function createSandbox<HookT>(workerUrl: URL): SandboxFactory<HookT> {
+  return (plug) => new WorkerSandbox(plug, workerUrl);
+}

package/client/plugos/sandboxes/sandbox.ts

@@ -0,0 +1,14 @@
+import type { Plug } from "../plug.ts";
+import type { Manifest } from "../types.ts";
+
+export type SandboxFactory<HookT> = (plug: Plug<HookT>) => Sandbox<HookT>;
+
+export interface Sandbox<HookT> {
+  manifest?: Manifest<HookT>;
+
+  init(): Promise<void>;
+
+  invoke(name: string, args: any[]): Promise<any>;
+
+  stop(): void;
+}

package/client/plugos/sandboxes/web_worker_sandbox.ts

@@ -0,0 +1,17 @@
+import { WorkerSandbox } from "./worker_sandbox.ts";
+import type { Plug } from "../plug.ts";
+import type { SandboxFactory } from "./sandbox.ts";
+import { fsEndpoint } from "../../spaces/constants.ts";
+
+export function createWorkerSandboxFromLocalPath<HookT>(
+  name: string,
+): SandboxFactory<HookT> {
+  return (plug: Plug<HookT>) =>
+    new WorkerSandbox(
+      plug,
+      new URL(
+        name,
+        document.baseURI.slice(0, -1) + fsEndpoint + "/", // We're NOT striping trailing '/', this used to be `location.origin`
+      ),
+    );
+}

package/client/plugos/sandboxes/worker_sandbox.ts

@@ -0,0 +1,132 @@
+import type { Manifest } from "../types.ts";
+import type { ControllerMessage, WorkerMessage } from "../protocol.ts";
+import type { Plug } from "../plug.ts";
+import { AssetBundle, type AssetJson } from "../../asset_bundle/bundle.ts";
+import type { Sandbox } from "./sandbox.ts";
+import { race, timeout } from "@silverbulletmd/silverbullet/lib/async";
+
+/**
+ * Represents a "safe" execution environment for plug code
+ * Effectively this wraps a web worker, the reason to have this split from Plugs is to allow plugs to manage multiple sandboxes, e.g. for performance in the future
+ */
+export class WorkerSandbox<HookT> implements Sandbox<HookT> {
+  public manifest?: Manifest<HookT>;
+  private worker?: Worker;
+  private reqId = 0;
+  private outstandingInvocations = new Map<
+    number,
+    { resolve: (result: any) => void; reject: (e: any) => void }
+  >();
+
+  constructor(
+    readonly plug: Plug<HookT>,
+    public workerUrl: URL,
+    private workerOptions = {},
+  ) {
+  }
+
+  /**
+   * Should only invoked lazily (either by invoke, or by a ManifestCache to load the manifest)
+   */
+  init(): Promise<void> {
+    console.log("Booting up worker from", this.workerUrl.pathname);
+    if (this.worker) {
+      // Race condition
+      console.warn("Double init of sandbox, ignoring");
+      return Promise.resolve();
+    }
+    this.worker = new Worker(this.workerUrl, {
+      ...this.workerOptions,
+      type: "module",
+    });
+
+    return race([
+      // We're adding a timeout of 5s here to handle the case where a plug blows up during initialization
+      timeout(5000).catch((_) =>
+        Promise.reject(new Error("Plug timed out during creation"))
+      ),
+      new Promise((resolve) => {
+        this.worker!.onmessage = (ev) => {
+          if (ev.data.type === "manifest") {
+            this.manifest = ev.data.manifest;
+            // Set manifest in the plug
+            this.plug.manifest = this.manifest!;
+
+            // Set assets in the plug
+            this.plug.assets = new AssetBundle(
+              this.manifest?.assets ? this.manifest.assets as AssetJson : {},
+            );
+
+            return resolve();
+          }
+
+          this.onMessage(ev.data);
+        };
+      }),
+    ]);
+  }
+
+  async onMessage(data: ControllerMessage) {
+    if (!this.worker) {
+      console.warn("Received message for terminated worker, ignoring");
+      return;
+    }
+    switch (data.type) {
+      case "sys":
+        try {
+          const result = await this.plug.syscall(data.name!, data.args!);
+
+          this.worker && this.worker!.postMessage({
+            type: "sysr",
+            id: data.id,
+            result: result,
+          } as WorkerMessage);
+        } catch (e: any) {
+          // console.error("Syscall fail", e);
+          this.worker && this.worker!.postMessage({
+            type: "sysr",
+            id: data.id,
+            error: e.message,
+          } as WorkerMessage);
+        }
+        break;
+      case "invr": {
+        const resultCbs = this.outstandingInvocations.get(data.id!);
+        this.outstandingInvocations.delete(data.id!);
+        if (data.error) {
+          resultCbs &&
+            resultCbs.reject(new Error(data.error));
+        } else {
+          resultCbs && resultCbs.resolve(data.result);
+        }
+        break;
+      }
+      default:
+        console.error("Unknown message type", data);
+    }
+  }
+
+  async invoke(name: string, args: any[]): Promise<any> {
+    if (!this.worker) {
+      // Lazy initialization
+      await this.init();
+    }
+    this.reqId++;
+    this.worker!.postMessage({
+      type: "inv",
+      id: this.reqId,
+      name,
+      args,
+    } as WorkerMessage);
+    return new Promise((resolve, reject) => {
+      this.outstandingInvocations.set(this.reqId, { resolve, reject });
+    });
+  }
+
+  stop() {
+    if (this.worker) {
+      this.worker.terminate();
+      this.worker = undefined;
+    }
+  }
+}

package/client/plugos/syscalls/asset.ts

@@ -0,0 +1,35 @@
+import type { SysCallMapping, System } from "../system.ts";
+import type { FileMeta } from "../../../plug-api/types/index.ts";
+
+export default function assetSyscalls(system: System<any>): SysCallMapping {
+  return {
+    "asset.readAsset": (_ctx, plugName: string, name: string): string => {
+      return system.loadedPlugs.get(plugName)!.assets!.readFileAsDataUrl(
+        name,
+      );
+    },
+    "asset.listFiles": (_ctx, plugName: string): FileMeta[] => {
+      const assets = system.loadedPlugs.get(plugName)!.assets!;
+      const fileNames = assets.listFiles();
+      return fileNames.map((name) => ({
+        name,
+        contentType: assets.getMimeType(name),
+        created: assets.getMtime(name),
+        lastModified: assets.getMtime(name),
+        size: -1,
+        perm: "ro",
+      }));
+    },
+    "asset.getFileMeta": (_ctx, plugName: string, name: string): FileMeta => {
+      const assets = system.loadedPlugs.get(plugName)!.assets!;
+      return {
+        name,
+        contentType: assets.getMimeType(name),
+        created: assets.getMtime(name),
+        lastModified: assets.getMtime(name),
+        size: -1,
+        perm: "ro",
+      };
+    },
+  };
+}

package/client/plugos/syscalls/clientStore.ts

@@ -0,0 +1,21 @@
+import type { SysCallMapping } from "../../plugos/system.ts";
+import type { DataStore } from "../../data/datastore.ts";
+
+import type { KvKey } from "@silverbulletmd/silverbullet/type/datastore";
+
+export function clientStoreSyscalls(
+  ds: DataStore,
+  prefix: KvKey = ["client"],
+): SysCallMapping {
+  return {
+    "clientStore.get": (_ctx, key: string): Promise<any> => {
+      return ds.get([...prefix, key]);
+    },
+    "clientStore.set": (_ctx, key: string, val: any): Promise<void> => {
+      return ds.set([...prefix, key], val);
+    },
+    "clientStore.delete": (_ctx, key: string): Promise<void> => {
+      return ds.delete([...prefix, key]);
+    },
+  };
+}