@silverbulletmd/silverbullet 2.9.0 → 2.11.0

package/client/plugos/hooks/syscall.ts

@@ -1,5 +1,5 @@
 import type { Hook, Manifest } from "../types.ts";
-import type { SysCallMapping, System } from "../system.ts";
+import type { SysCallMapping, SyscallContext, System } from "../system.ts";
 import type { SyscallHookT } from "@silverbulletmd/silverbullet/type/manifest";
 
 export class SyscallHook implements Hook<SyscallHookT> {
@@ -13,7 +13,6 @@ export class SyscallHook implements Hook<SyscallHookT> {
   }
 
   registerSyscalls(system: System<SyscallHookT>) {
-    // Register syscalls from all loaded plugs
     for (const plug of system.loadedPlugs.values()) {
       const syscalls: SysCallMapping = {};
 
@@ -24,18 +23,27 @@ export class SyscallHook implements Hook<SyscallHookT> {
           continue;
         }
 
-        const syscallName = functionDef.syscall;
+        const syscallDefinition = functionDef.syscall;
+        const syscallName =
+          typeof syscallDefinition === "string"
+            ? syscallDefinition
+            : syscallDefinition.name;
 
-        // Add the syscall to our mapping
-        syscalls[syscallName] = (ctx, ...args) => {
-          // Delegate to the system to invoke the function
-          return system.syscall(ctx, "system.invokeFunction", [
+        const callback = (ctx: SyscallContext, ...args: any[]) =>
+          system.syscall(ctx, "system.invokeFunction", [
             `${plug.manifest!.name}.${name}`,
             ...args,
           ]);
-        };
+        if (typeof syscallDefinition === "string") {
+          syscalls[syscallName] = callback;
+        } else {
+          const { name: _name, ...metadata } = syscallDefinition;
+          syscalls[syscallName] = {
+            callback,
+            ...metadata,
+          };
+        }
 
-        // Register the syscalls with no required permissions
         system.registerSyscalls([], syscalls);
       }
     }
@@ -48,16 +56,18 @@ export class SyscallHook implements Hook<SyscallHookT> {
         continue;
       }
 
-      // Validate syscall name is provided
-      if (!functionDef.syscall) {
+      const syscallName =
+        typeof functionDef.syscall === "string"
+          ? functionDef.syscall
+          : functionDef.syscall.name;
+      if (!syscallName) {
         errors.push(`Function ${name} has a syscall but no name`);
         continue;
       }
 
-      // Validate syscall name format (should be namespaced)
-      if (!functionDef.syscall.includes(".")) {
+      if (!syscallName.includes(".")) {
         errors.push(
-          `Function ${name} has invalid syscall name "${functionDef.syscall}" - must be in format "namespace.name"`,
+          `Function ${name} has invalid syscall name "${syscallName}" - must be in format "namespace.name"`,
         );
       }
     }

package/client/plugos/manifest_cache.ts

@@ -23,7 +23,6 @@ export class KVPrimitivesManifestCache<T> implements ManifestCache<T> {
   ): Promise<Manifest<T>> {
     const [cached] = await this.kv.batchGet([[this.manifestPrefix, cacheKey]]);
     if (cached && cached.hash === cacheHash) {
-      // console.log("Using KV cached manifest for", plug.name);
       return cached.manifest;
     }
     await plug.sandbox.init();
@@ -41,34 +40,3 @@ export class KVPrimitivesManifestCache<T> implements ManifestCache<T> {
     return manifest;
   }
 }
-
-export class InMemoryManifestCache<T> implements ManifestCache<T> {
-  private cache = new Map<
-    string,
-    {
-      manifest: Manifest<T>;
-      hash: number;
-    }
-  >();
-
-  async getManifest(
-    plug: Plug<T>,
-    cacheKey: string,
-    cacheHash: number,
-  ): Promise<Manifest<T>> {
-    const cached = this.cache.get(cacheKey);
-    if (cached && cached.hash === cacheHash) {
-      // console.log("Using memory cached manifest for", plug.name);
-      return cached.manifest;
-    }
-    await plug.sandbox.init();
-    const manifest = plug.sandbox.manifest!;
-
-    // Deliverately removing the assets from the manifest to preserve space, will be re-added upon load of actual worker
-    this.cache.set(cacheKey, {
-      manifest: { ...manifest, assets: undefined },
-      hash: cacheHash,
-    });
-    return manifest;
-  }
-}

package/client/plugos/plug.ts

@@ -31,7 +31,6 @@ export class Plug<HookT> {
     const plug = new Plug(system, sandboxFactory);
     plug.path = path;
 
-    // Retrieve the manifest, which may either come from a cache or be loaded from the worker
     plug.manifest = await system.options.manifestCache!.getManifest(
       plug,
       path,
@@ -62,14 +61,12 @@ export class Plug<HookT> {
 
   // Invoke a function
   async invoke(name: string, args: any[]): Promise<any> {
-    // Before we access the manifest
     const funDef = this.manifest!.functions[name];
     if (!funDef) {
       throw new Error(`Function ${name} not found in manifest`);
     }
     const sandbox = this.sandbox!;
     if (funDef.redirect) {
-      // Function redirect, look up
       let plug: Plug<HookT> | undefined = this;
       if (funDef.redirect.indexOf(".") !== -1) {
         const [plugName, functionName] = funDef.redirect.split(".");

package/client/plugos/plug_compile.ts

@@ -24,7 +24,6 @@ const workerRuntimePlugin: esbuild.Plugin = {
   name: "worker-runtime",
   setup(build) {
     if (typeof __EMBEDDED_WORKER_RUNTIME_JS__ !== "undefined") {
-      // Bundled CLI: serve pre-bundled JS from the embedded constant
       build.onResolve({ filter: /^worker-runtime$/ }, () => ({
         path: "worker-runtime",
         namespace: "worker-runtime",
@@ -34,7 +33,6 @@ const workerRuntimePlugin: esbuild.Plugin = {
         loader: "js",
       }));
     } else {
-      // From source: point directly at the .ts file, esbuild handles it
       build.onResolve({ filter: /^worker-runtime$/ }, () => ({
         path: path.join(import.meta.dirname, "worker_runtime.ts"),
       }));
@@ -70,14 +68,12 @@ export async function compileManifest(
     delete manifest.build;
   }
 
-  // Assets
   const assetsBundle = await bundleAssets(
     path.resolve(rootPath),
     (manifest.assets as string[]) || [],
   );
   manifest.assets = assetsBundle.toJSON();
 
-  // Normalize the edge case of a plug with no functions
   if (!manifest.functions) {
     manifest.functions = {};
   }
@@ -92,13 +88,11 @@ ${Object.entries(manifest.functions)
       return "";
     }
     let [filePath, jsFunctionName] = def.path.split(":");
-    // Resolve path
     filePath = path.join(rootPath, filePath);
 
-    return `import {${jsFunctionName} as ${funcName}} from "${
-      // Replacing \ with / for Windows
-      path.resolve(filePath).replaceAll("\\", "\\\\")
-    }";\n`;
+    return `import {${jsFunctionName} as ${funcName}} from "${path
+      .resolve(filePath)
+      .replaceAll("\\", "\\\\")}";\n`;
   })
   .join("")}
 
@@ -122,8 +116,6 @@ export const plug = {manifest, functionMapping};
 setupMessageListener(functionMapping, manifest, self.postMessage);
 `;
 
-  // console.log("Code:", jsFile);
-
   const tempDir = await mkdtemp(path.join(tmpdir(), "plug-compile-"));
   const inFile = path.join(tempDir, "input.js");
   const outFile = `${destPath}/${manifest.name}.plug.js`;
@@ -152,7 +144,6 @@ setupMessageListener(functionMapping, manifest, self.postMessage);
   jsCode = patchBundledJS(jsCode);
   await writeFile(outFile, jsCode, "utf-8");
 
-  // Clean up temp directory
   await rm(tempDir, { recursive: true, force: true });
 
   console.log(`Plug ${manifest.name} written to ${outFile}.`);
@@ -175,7 +166,6 @@ export async function compileManifests(
     building = true;
     await mkdir(dist, { recursive: true });
     const startTime = Date.now();
-    // Build all plugs in parallel
     await Promise.all(
       manifestFiles.map(async (plugManifestPath) => {
         const manifestPath = plugManifestPath as string;

package/client/plugos/sandboxes/worker_sandbox.ts

@@ -7,7 +7,6 @@ import { race, timeout } from "@silverbulletmd/silverbullet/lib/async";
 import { fsEndpoint } from "../../spaces/constants.ts";
 
 /**
- * Represents a "safe" execution environment for plug code
  * Effectively this wraps a web worker, the reason to have this split from Plugs is to allow plugs to manage multiple sandboxes, e.g. for performance in the future
  */
 export class WorkerSandbox<HookT> implements Sandbox<HookT> {
@@ -25,7 +24,6 @@ export class WorkerSandbox<HookT> implements Sandbox<HookT> {
     private workerOptions = {},
   ) {}
 
-  /** Factory producing a WorkerSandbox for a plug file at a local space path. */
   static forPath<HookT>(path: string): SandboxFactory<HookT> {
     return (plug) =>
       new WorkerSandbox(
@@ -40,7 +38,6 @@ export class WorkerSandbox<HookT> implements Sandbox<HookT> {
   init(): Promise<void> {
     console.log("Booting up worker from", this.workerUrl.pathname);
     if (this.worker) {
-      // Race condition
       console.warn("Double init of sandbox, ignoring");
       return Promise.resolve();
     }
@@ -54,14 +51,12 @@ export class WorkerSandbox<HookT> implements Sandbox<HookT> {
       timeout(5000).catch((_) =>
         Promise.reject(new Error("Plug timed out during creation")),
       ),
-      new Promise((resolve) => {
+      new Promise<void>((resolve) => {
         this.worker!.onmessage = (ev) => {
           if (ev.data.type === "manifest") {
             this.manifest = ev.data.manifest;
-            // Set manifest in the plug
             this.plug.manifest = this.manifest!;
 
-            // Set assets in the plug
             this.plug.assets = new AssetBundle(
               this.manifest?.assets ? (this.manifest.assets as AssetJson) : {},
             );
@@ -72,7 +67,15 @@ export class WorkerSandbox<HookT> implements Sandbox<HookT> {
           void this.onMessage(ev.data);
         };
       }),
-    ]);
+    ]).catch((e) => {
+      // A worker that failed to come up must not linger: its script fetch may
+      // still be in flight (e.g. on a slow link), and a late boot would
+      // resolve into a sandbox the system already gave up on. Terminating it
+      // also lets a retry start from a clean slate.
+      this.worker?.terminate();
+      this.worker = undefined;
+      throw e;
+    });
   }
 
   async onMessage(data: ControllerMessage) {
@@ -92,7 +95,6 @@ export class WorkerSandbox<HookT> implements Sandbox<HookT> {
               result: result,
             } as WorkerMessage);
         } catch (e: any) {
-          // console.error("Syscall fail", e);
           this.worker &&
             this.worker!.postMessage({
               type: "sysr",
@@ -118,7 +120,6 @@ export class WorkerSandbox<HookT> implements Sandbox<HookT> {
 
   async invoke(name: string, args: any[]): Promise<any> {
     if (!this.worker) {
-      // Lazy initialization
       await this.init();
     }
     this.reqId++;
@@ -138,5 +139,14 @@ export class WorkerSandbox<HookT> implements Sandbox<HookT> {
       this.worker.terminate();
       this.worker = undefined;
     }
+    // A terminated worker will never post the `invr` that would otherwise
+    // settle these.
+    if (this.outstandingInvocations.size > 0) {
+      const error = new Error("Plug sandbox stopped");
+      for (const { reject } of this.outstandingInvocations.values()) {
+        reject(error);
+      }
+      this.outstandingInvocations.clear();
+    }
   }
 }

package/client/plugos/syscalls/asset.ts

@@ -3,31 +3,69 @@ import type { FileMeta } from "../../../plug-api/types/index.ts";
 
 export default function assetSyscalls(system: System<any>): SysCallMapping {
   return {
-    "asset.readAsset": (_ctx, plugName: string, name: string): string => {
-      return system.loadedPlugs.get(plugName)!.assets!.readFileAsDataUrl(name);
+    "asset.readAsset": {
+      callback: (_ctx, plugName: string, name: string): string => {
+        return system.loadedPlugs
+          .get(plugName)!
+          .assets!.readFileAsDataUrl(name);
+      },
+      description: "Reads an asset embedded in a plug as a data URL.",
+      parameters: [
+        { name: "plugName", type: "string", description: "Plug name." },
+        { name: "name", type: "string", description: "Asset path." },
+      ],
+      returns: [{ type: "string", description: "Asset data URL." }],
+      examples: [
+        { code: 'local image = asset.readAsset("myplug", "image.png")' },
+      ],
     },
-    "asset.listFiles": (_ctx, plugName: string): FileMeta[] => {
-      const assets = system.loadedPlugs.get(plugName)!.assets!;
-      const fileNames = assets.listFiles();
-      return fileNames.map((name) => ({
-        name,
-        contentType: assets.getMimeType(name),
-        created: assets.getMtime(name),
-        lastModified: assets.getMtime(name),
-        size: -1,
-        perm: "ro",
-      }));
+    "asset.listFiles": {
+      callback: (_ctx, plugName: string): FileMeta[] => {
+        const assets = system.loadedPlugs.get(plugName)!.assets!;
+        const fileNames = assets.listFiles();
+        return fileNames.map((name) => ({
+          name,
+          contentType: assets.getMimeType(name),
+          created: assets.getMtime(name),
+          lastModified: assets.getMtime(name),
+          size: -1,
+          perm: "ro",
+        }));
+      },
+      description: "Lists the assets embedded in a plug.",
+      parameters: [
+        { name: "plugName", type: "string", description: "Plug name." },
+      ],
+      returns: [{ type: "table", description: "List of file metadata." }],
+      examples: [
+        {
+          code: 'for _, file in ipairs(asset.listFiles("myplug")) do\n  print(file.name)\nend',
+        },
+      ],
     },
-    "asset.getFileMeta": (_ctx, plugName: string, name: string): FileMeta => {
-      const assets = system.loadedPlugs.get(plugName)!.assets!;
-      return {
-        name,
-        contentType: assets.getMimeType(name),
-        created: assets.getMtime(name),
-        lastModified: assets.getMtime(name),
-        size: -1,
-        perm: "ro",
-      };
+    "asset.getFileMeta": {
+      callback: (_ctx, plugName: string, name: string): FileMeta => {
+        const assets = system.loadedPlugs.get(plugName)!.assets!;
+        return {
+          name,
+          contentType: assets.getMimeType(name),
+          created: assets.getMtime(name),
+          lastModified: assets.getMtime(name),
+          size: -1,
+          perm: "ro",
+        };
+      },
+      description: "Gets metadata for an asset embedded in a plug.",
+      parameters: [
+        { name: "plugName", type: "string", description: "Plug name." },
+        { name: "name", type: "string", description: "Asset path." },
+      ],
+      returns: [{ type: "table", description: "File metadata." }],
+      examples: [
+        {
+          code: 'local meta = asset.getFileMeta("myplug", "data.txt")\nprint(meta.lastModified)',
+        },
+      ],
     },
   };
 }

package/client/plugos/syscalls/clientStore.ts

@@ -8,14 +8,40 @@ export function clientStoreSyscalls(
   prefix: KvKey = ["client"],
 ): SysCallMapping {
   return {
-    "clientStore.get": (_ctx, key: string): Promise<any> => {
-      return ds.get([...prefix, key]);
+    "clientStore.get": {
+      callback: (_ctx, key: string): Promise<any> => {
+        return ds.get([...prefix, key]);
+      },
+      description:
+        "Gets a client-specific value from the local key-value store.",
+      parameters: [
+        { name: "key", type: "string", description: "Key to read." },
+      ],
+      returns: [{ description: "Stored value, or nil when absent." }],
+      examples: [{ code: 'local theme = clientStore.get("theme")' }],
     },
-    "clientStore.set": (_ctx, key: string, val: any): Promise<void> => {
-      return ds.set([...prefix, key], val);
+    "clientStore.set": {
+      callback: (_ctx, key: string, val: any): Promise<void> => {
+        return ds.set([...prefix, key], val);
+      },
+      description:
+        "Stores a client-specific value in the local key-value store.",
+      parameters: [
+        { name: "key", type: "string", description: "Key to set." },
+        { name: "value", description: "Value to store." },
+      ],
+      examples: [{ code: 'clientStore.set("theme", "dark")' }],
     },
-    "clientStore.delete": (_ctx, key: string): Promise<void> => {
-      return ds.delete([...prefix, key]);
+    "clientStore.delete": {
+      callback: (_ctx, key: string): Promise<void> => {
+        return ds.delete([...prefix, key]);
+      },
+      description:
+        "Deletes a client-specific value from the local key-value store.",
+      parameters: [
+        { name: "key", type: "string", description: "Key to delete." },
+      ],
+      examples: [{ code: 'clientStore.delete("theme")' }],
     },
   };
 }

package/client/plugos/syscalls/client_code_widget.ts

@@ -5,10 +5,15 @@ import { broadcastReload } from "../../components/widget_sandbox_iframe.ts";
 
 export function clientCodeWidgetSyscalls(client: Client): SysCallMapping {
   return {
-    "codeWidget.refreshAll": () => {
-      client.widgetCache.clearPrewarm();
-      broadcastReload();
-      return reloadAllWidgets();
+    "codeWidget.refreshAll": {
+      callback: () => {
+        client.widgetCache.clearPrewarm();
+        broadcastReload();
+        return reloadAllWidgets();
+      },
+      description:
+        "Refreshes all code widgets on the current page that support refreshing.",
+      examples: [{ code: "codeWidget.refreshAll()" }],
     },
   };
 }

package/client/plugos/syscalls/code_widget.ts

@@ -6,17 +6,32 @@ export function codeWidgetSyscalls(
   codeWidgetHook: CodeWidgetHook,
 ): SysCallMapping {
   return {
-    "codeWidget.render": (
-      _ctx,
-      lang: string,
-      body: string,
-      pageName: string,
-    ): Promise<CodeWidgetContent | null> => {
-      const langCallback = codeWidgetHook.codeWidgetCallbacks.get(lang);
-      if (!langCallback) {
-        throw new Error(`Code widget ${lang} not found`);
-      }
-      return langCallback(body, pageName);
+    "codeWidget.render": {
+      callback: (
+        _ctx,
+        lang: string,
+        body: string,
+        pageName: string,
+      ): Promise<CodeWidgetContent | null> => {
+        const langCallback = codeWidgetHook.codeWidgetCallbacks.get(lang);
+        if (!langCallback) {
+          throw new Error(`Code widget ${lang} not found`);
+        }
+        return langCallback(body, pageName);
+      },
+      description: "Renders code through the widget registered for a language.",
+      parameters: [
+        { name: "language", type: "string", description: "Widget language." },
+        { name: "body", type: "string", description: "Code block body." },
+        {
+          name: "pageName",
+          type: "string",
+          description: "Containing page name.",
+        },
+      ],
+      returns: [
+        { type: "table", description: "Rendered widget content, or nil." },
+      ],
     },
   };
 }