@silverbulletmd/silverbullet 2.5.3 → 2.7.0

package/client/plugos/plug.ts

@@ -11,6 +11,8 @@ export class Plug<HookT> {
 
   public manifest!: Manifest<HookT>;
   public assets?: AssetBundle;
+  // File path the plug was loaded from; also used as the manifest cache key.
+  public path!: string;
 
   constructor(
     readonly system: System<HookT>,
@@ -22,19 +24,17 @@ export class Plug<HookT> {
 
   static async createLazily<HookT>(
     system: System<HookT>,
-    cacheKey: string,
+    path: string,
     cacheHash: number,
     sandboxFactory: SandboxFactory<HookT>,
   ): Promise<Plug<HookT>> {
-    const plug = new Plug(
-      system,
-      sandboxFactory,
-    );
+    const plug = new Plug(system, sandboxFactory);
+    plug.path = path;
 
     // Retrieve the manifest, which may either come from a cache or be loaded from the worker
     plug.manifest = await system.options.manifestCache!.getManifest(
       plug,
-      cacheKey,
+      path,
       cacheHash,
     );
 
@@ -83,7 +83,7 @@ export class Plug<HookT> {
       }
       return plug.invoke(name, args);
     }
-    if (!await this.canInvoke(name)) {
+    if (!(await this.canInvoke(name))) {
       throw new Error(
         `Function ${name} is not available in ${this.runtimeEnv}`,
       );

package/client/plugos/plug_compile.ts

@@ -1,30 +1,44 @@
 import * as path from "node:path";
-import { readFile, writeFile, mkdtemp, rm, mkdir } from "node:fs/promises";
+import {
+  copyFile,
+  mkdir,
+  mkdtemp,
+  readFile,
+  rm,
+  writeFile,
+} from "node:fs/promises";
 import { tmpdir } from "node:os";
 import * as YAML from "js-yaml";
 
 import * as esbuild from "esbuild";
+import * as sass from "sass";
 import { bundleAssets } from "../asset_bundle/builder.ts";
-import type { Manifest } from "./types.ts";
+import type { BuildStep, Manifest } from "./types.ts";
 
-import { existsSync } from "node:fs";
-
-// Resolve the pre-built worker_runtime bundle path
-// When running from source: ../../dist/worker_runtime_bundle.js (from client/plugos/)
-// When bundled: ./worker_runtime_bundle.js (from dist/)
-const currentDir = import.meta.dirname;
-const bundledPath = path.join(currentDir, "worker_runtime_bundle.js");
-const sourcePath = path.join(currentDir, "../../dist/worker_runtime_bundle.js");
-const workerRuntimeBundlePath = existsSync(bundledPath)
-  ? bundledPath
-  : sourcePath;
+// When bundled by build_plug_compile.ts, this is replaced with the pre-bundled
+// worker runtime JS. When running from source, it's undefined and esbuild
+// resolves the .ts file directly (it handles TypeScript natively).
+declare const __EMBEDDED_WORKER_RUNTIME_JS__: string | undefined;
 
 const workerRuntimePlugin: esbuild.Plugin = {
   name: "worker-runtime",
   setup(build) {
-    build.onResolve({ filter: /^worker-runtime$/ }, () => ({
-      path: workerRuntimeBundlePath,
-    }));
+    if (typeof __EMBEDDED_WORKER_RUNTIME_JS__ !== "undefined") {
+      // Bundled CLI: serve pre-bundled JS from the embedded constant
+      build.onResolve({ filter: /^worker-runtime$/ }, () => ({
+        path: "worker-runtime",
+        namespace: "worker-runtime",
+      }));
+      build.onLoad({ filter: /.*/, namespace: "worker-runtime" }, () => ({
+        contents: __EMBEDDED_WORKER_RUNTIME_JS__,
+        loader: "js",
+      }));
+    } else {
+      // From source: point directly at the .ts file, esbuild handles it
+      build.onResolve({ filter: /^worker-runtime$/ }, () => ({
+        path: path.join(import.meta.dirname, "worker_runtime.ts"),
+      }));
+    }
   },
 };
 
@@ -47,10 +61,19 @@ export async function compileManifest(
     throw new Error(`Missing 'name' in ${manifestPath}`);
   }
 
+  // Build steps (run before asset bundling so produced files can be picked
+  // up by the `assets` glob).
+  if (manifest.build) {
+    await Promise.all(
+      manifest.build.map((step) => runBuildStep(step, rootPath, options)),
+    );
+    delete manifest.build;
+  }
+
   // Assets
   const assetsBundle = await bundleAssets(
     path.resolve(rootPath),
-    manifest.assets as string[] || [],
+    (manifest.assets as string[]) || [],
   );
   manifest.assets = assetsBundle.toJSON();
 
@@ -63,34 +86,32 @@ export async function compileManifest(
 import { setupMessageListener } from "worker-runtime";
 
 // Imports
-${
-    Object.entries(manifest.functions).map(([funcName, def]) => {
-      if (!def.path) {
-        return "";
-      }
-      let [filePath, jsFunctionName] = def.path.split(":");
-      // Resolve path
-      filePath = path.join(rootPath, filePath);
-
-      return `import {${jsFunctionName} as ${funcName}} from "${
-        // Replacing \ with / for Windows
-        path.resolve(filePath).replaceAll(
-          "\\",
-          "\\\\",
-        )}";\n`;
-    }).join("")
-  }
+${Object.entries(manifest.functions)
+  .map(([funcName, def]) => {
+    if (!def.path) {
+      return "";
+    }
+    let [filePath, jsFunctionName] = def.path.split(":");
+    // Resolve path
+    filePath = path.join(rootPath, filePath);
+
+    return `import {${jsFunctionName} as ${funcName}} from "${
+      // Replacing \ with / for Windows
+      path.resolve(filePath).replaceAll("\\", "\\\\")
+    }";\n`;
+  })
+  .join("")}
 
 // Function mapping
 const functionMapping = {
-${
-    Object.entries(manifest.functions).map(([funcName, def]) => {
-      if (!def.path) {
-        return "";
-      }
-      return `  ${funcName}: ${funcName},\n`;
-    }).join("")
-  }
+${Object.entries(manifest.functions)
+  .map(([funcName, def]) => {
+    if (!def.path) {
+      return "";
+    }
+    return `  ${funcName}: ${funcName},\n`;
+  })
+  .join("")}
 };
 
 // Manifest
@@ -130,10 +151,10 @@ setupMessageListener(functionMapping, manifest, self.postMessage);
   let jsCode = await readFile(outFile, "utf-8");
   jsCode = patchBundledJS(jsCode);
   await writeFile(outFile, jsCode, "utf-8");
-  
+
   // Clean up temp directory
   await rm(tempDir, { recursive: true, force: true });
-  
+
   console.log(`Plug ${manifest.name} written to ${outFile}.`);
   return outFile;
 }
@@ -155,19 +176,17 @@ export async function compileManifests(
     await mkdir(dist, { recursive: true });
     const startTime = Date.now();
     // Build all plugs in parallel
-    await Promise.all(manifestFiles.map(async (plugManifestPath) => {
-      const manifestPath = plugManifestPath as string;
-      try {
-        await compileManifest(
-          manifestPath,
-          dist,
-          options,
-        );
-      } catch (e: any) {
-        console.error(`Error building ${manifestPath}:`, e.message);
-        throw e;
-      }
-    }));
+    await Promise.all(
+      manifestFiles.map(async (plugManifestPath) => {
+        const manifestPath = plugManifestPath as string;
+        try {
+          await compileManifest(manifestPath, dist, options);
+        } catch (e: any) {
+          console.error(`Error building ${manifestPath}:`, e.message);
+          throw e;
+        }
+      }),
+    );
     console.log(`Done building plugs in ${Date.now() - startTime}ms`);
     building = false;
   }
@@ -175,27 +194,60 @@ export async function compileManifests(
   await buildAll();
 }
 
+async function runBuildStep(
+  step: BuildStep,
+  rootPath: string,
+  options: CompileOptions,
+) {
+  const type = step.type ?? "esbuild";
+  const inFile = path.resolve(rootPath, step.in);
+  const outFile = path.resolve(rootPath, step.out);
+  await mkdir(path.dirname(outFile), { recursive: true });
+  if (type === "esbuild") {
+    await esbuild.build({
+      entryPoints: [inFile],
+      bundle: true,
+      format: "iife",
+      platform: "browser",
+      minify: !options.debug,
+      outfile: outFile,
+      treeShaking: true,
+    });
+  } else if (type === "sass") {
+    const scss = await readFile(inFile, "utf-8");
+    const result = sass.compileString(scss, {
+      loadPaths: [path.dirname(inFile)],
+      style: options.debug ? "expanded" : "compressed",
+    });
+    await writeFile(outFile, result.css, "utf-8");
+  } else if (type === "copy") {
+    await copyFile(inFile, outFile);
+  } else {
+    throw new Error(`Unsupported build step type: ${type}`);
+  }
+}
+
 export function patchBundledJS(code: string): string {
   // One bundled dependency has a lookbehind regex that WebKit can't parse; replace it with a no-op
   return code.replaceAll("/(?<=\\n)/", "/()/");
 }
 
 export async function plugCompileCommand(
-  { dist, debug, info }: {
+  {
+    dist,
+    debug,
+    info,
+  }: {
     dist: string;
     debug: boolean;
     info: boolean;
   },
   ...manifestPaths: string[]
 ) {
-  await compileManifests(
-    manifestPaths,
-    dist,
-    {
-      debug: debug,
-      info: info,
-    },
-  );
-  esbuild.stop();
+  await compileManifests(manifestPaths, dist, {
+    debug: debug,
+    info: info,
+  });
+  await esbuild.stop();
   process.exit(0);
 }

package/client/plugos/protocol.ts

@@ -3,38 +3,38 @@ import type { Manifest } from "@silverbulletmd/silverbullet/type/manifest";
 // Messages received from the worker
 export type ControllerMessage =
   | {
-    // Parsed manifest when worker is initialized
-    type: "manifest";
-    manifest: Manifest;
-  }
+      // Parsed manifest when worker is initialized
+      type: "manifest";
+      manifest: Manifest;
+    }
   | {
-    // Function invocation result
-    type: "invr";
-    id: number;
-    error?: string;
-    result?: any;
-  }
+      // Function invocation result
+      type: "invr";
+      id: number;
+      error?: string;
+      result?: any;
+    }
   | {
-    // Syscall
-    type: "sys";
-    id: number;
-    name: string;
-    args: any[];
-  };
+      // Syscall
+      type: "sys";
+      id: number;
+      name: string;
+      args: any[];
+    };
 
 // Messages received inside the worker
 export type WorkerMessage =
   | {
-    // Function invocation
-    type: "inv";
-    id: number;
-    name: string;
-    args: any[];
-  }
+      // Function invocation
+      type: "inv";
+      id: number;
+      name: string;
+      args: any[];
+    }
   | {
-    // Syscall result
-    type: "sysr";
-    id: number;
-    result?: any;
-    error?: any;
-  };
+      // Syscall result
+      type: "sysr";
+      id: number;
+      result?: any;
+      error?: any;
+    };

package/client/plugos/proxy_fetch.ts

@@ -35,12 +35,13 @@ export async function performLocalFetch(
 ): Promise<ProxyFetchResponse64> {
   const result = await fetch(
     url,
-    req && {
-      method: req.method,
-      headers: req.headers,
-      body: req.base64Body && base64Decode(req.base64Body),
-      // Casting to any due to TypeScript fetch type limitations
-    } as any,
+    req &&
+      ({
+        method: req.method,
+        headers: req.headers,
+        body: req.base64Body && base64Decode(req.base64Body),
+        // Casting to any due to TypeScript fetch type limitations
+      } as any),
   );
   return {
     ok: result.ok,

package/client/plugos/sandboxes/worker_sandbox.ts

@@ -2,8 +2,9 @@ import type { Manifest } from "../types.ts";
 import type { ControllerMessage, WorkerMessage } from "../protocol.ts";
 import type { Plug } from "../plug.ts";
 import { AssetBundle, type AssetJson } from "../../asset_bundle/bundle.ts";
-import type { Sandbox } from "./sandbox.ts";
+import type { Sandbox, SandboxFactory } from "./sandbox.ts";
 import { race, timeout } from "@silverbulletmd/silverbullet/lib/async";
+import { fsEndpoint } from "../../spaces/constants.ts";
 
 /**
  * Represents a "safe" execution environment for plug code
@@ -22,7 +23,15 @@ export class WorkerSandbox<HookT> implements Sandbox<HookT> {
     readonly plug: Plug<HookT>,
     public workerUrl: URL,
     private workerOptions = {},
-  ) {
+  ) {}
+
+  /** Factory producing a WorkerSandbox for a plug file at a local space path. */
+  static forPath<HookT>(path: string): SandboxFactory<HookT> {
+    return (plug) =>
+      new WorkerSandbox(
+        plug,
+        new URL(path, `${document.baseURI.slice(0, -1) + fsEndpoint}/`),
+      );
   }
 
   /**
@@ -43,7 +52,7 @@ export class WorkerSandbox<HookT> implements Sandbox<HookT> {
     return race([
       // We're adding a timeout of 5s here to handle the case where a plug blows up during initialization
       timeout(5000).catch((_) =>
-        Promise.reject(new Error("Plug timed out during creation"))
+        Promise.reject(new Error("Plug timed out during creation")),
       ),
       new Promise((resolve) => {
         this.worker!.onmessage = (ev) => {
@@ -54,13 +63,13 @@ export class WorkerSandbox<HookT> implements Sandbox<HookT> {
 
             // Set assets in the plug
             this.plug.assets = new AssetBundle(
-              this.manifest?.assets ? this.manifest.assets as AssetJson : {},
+              this.manifest?.assets ? (this.manifest.assets as AssetJson) : {},
             );
 
             return resolve();
           }
 
-          this.onMessage(ev.data);
+          void this.onMessage(ev.data);
         };
       }),
     ]);
@@ -76,18 +85,20 @@ export class WorkerSandbox<HookT> implements Sandbox<HookT> {
         try {
           const result = await this.plug.syscall(data.name!, data.args!);
 
-          this.worker && this.worker!.postMessage({
-            type: "sysr",
-            id: data.id,
-            result: result,
-          } as WorkerMessage);
+          this.worker &&
+            this.worker!.postMessage({
+              type: "sysr",
+              id: data.id,
+              result: result,
+            } as WorkerMessage);
         } catch (e: any) {
           // console.error("Syscall fail", e);
-          this.worker && this.worker!.postMessage({
-            type: "sysr",
-            id: data.id,
-            error: e.message,
-          } as WorkerMessage);
+          this.worker &&
+            this.worker!.postMessage({
+              type: "sysr",
+              id: data.id,
+              error: e.message,
+            } as WorkerMessage);
         }
         break;
       case "invr": {

package/client/plugos/syscalls/asset.ts

@@ -4,9 +4,7 @@ import type { FileMeta } from "../../../plug-api/types/index.ts";
 export default function assetSyscalls(system: System<any>): SysCallMapping {
   return {
     "asset.readAsset": (_ctx, plugName: string, name: string): string => {
-      return system.loadedPlugs.get(plugName)!.assets!.readFileAsDataUrl(
-        name,
-      );
+      return system.loadedPlugs.get(plugName)!.assets!.readFileAsDataUrl(name);
     },
     "asset.listFiles": (_ctx, plugName: string): FileMeta[] => {
       const assets = system.loadedPlugs.get(plugName)!.assets!;

package/client/plugos/syscalls/code_widget.ts

@@ -12,9 +12,7 @@ export function codeWidgetSyscalls(
       body: string,
       pageName: string,
     ): Promise<CodeWidgetContent | null> => {
-      const langCallback = codeWidgetHook.codeWidgetCallbacks.get(
-        lang,
-      );
+      const langCallback = codeWidgetHook.codeWidgetCallbacks.get(lang);
       if (!langCallback) {
         throw new Error(`Code widget ${lang} not found`);
       }

package/client/plugos/syscalls/config.ts

@@ -1,6 +1,7 @@
 import type { SysCallMapping } from "../system.ts";
 import type { Config } from "../../config.ts";
 import { LuaStackFrame, luaValueToJS } from "../../space_lua/runtime.ts";
+import { stripFunctions } from "../util.ts";
 
 export function configSyscalls(config: Config): SysCallMapping {
   return {
@@ -23,11 +24,7 @@ export function configSyscalls(config: Config): SysCallMapping {
       keyOrValues = await luaValueToJS(keyOrValues, LuaStackFrame.lostFrame);
       config.set(keyOrValues as any, value);
     },
-    "config.insert": (
-      _ctx,
-      key: string | string[],
-      value: any,
-    ) => {
+    "config.insert": (_ctx, key: string | string[], value: any) => {
       config.insert(key, value);
     },
     "config.has": (_ctx, path: string) => {
@@ -36,11 +33,17 @@ export function configSyscalls(config: Config): SysCallMapping {
     "config.define": (_ctx, key: string, schema: any) => {
       config.define(key, schema);
     },
+    "config.defineCategory": (_ctx, definition: any) => {
+      config.defineCategory(definition);
+    },
     "config.getValues": () => {
-      return config.values;
+      return stripFunctions(config.values);
     },
     "config.getSchemas": () => {
-      return config.schemas;
+      return stripFunctions(config.schemas);
+    },
+    "config.getCategories": () => {
+      return { ...config.categories };
     },
   };
 }

package/client/plugos/syscalls/datastore.ts

@@ -61,7 +61,7 @@ export function dataStoreReadSyscalls(
         }
       }
       return (await queryLua<any>(ds.kv, prefix, query, env, sf)).map((item) =>
-        luaValueToJS(item, sf)
+        luaValueToJS(item, sf),
       );
     },
   };