@silverbulletmd/silverbullet 2.9.0 → 2.10.0

package/README.md

@@ -27,27 +27,30 @@ You were told there’s no such thing as a silver bullet. You were told wrong.
 Check out the [instructions](https://silverbullet.md/Install).
 
 ## Developing SilverBullet
-SilverBullet's frontend is written in [TypeScript](https://www.typescriptlang.org/) and built on top of the excellent [CodeMirror 6](https://codemirror.net/) editor component. Additional UI is built using [Preact](https://preactjs.com). [ESBuild](https://esbuild.github.io) is used to build the frontend.
+SilverBullet's frontend ("client") is written in [TypeScript](https://www.typescriptlang.org/) and built on top of the excellent [CodeMirror 6](https://codemirror.net/) editor component. Additional UI is built using [Preact](https://preactjs.com). [ESBuild](https://esbuild.github.io) is used to build the frontend.
 
-The server backend is written in Go.
+The server backend is written in [Rust](https://www.rust-lang.org/) (a Cargo workspace). It serves the pre-built client bundle, the file/space HTTP API, authentication, and a headless-Chrome runtime for server-side Lua.
 
 ## Code structure
-* `client/`: The SilverBullet client, implemented with TypeScript
-* `server/`: The SilverBullet server, written in Go
-* `plugs`: Set of built-in plugs that are distributed with SilverBullet
-* `libraries`: A set of libraries (space scripts, page templates, slash templates) distributed with SilverBullet
-* `plug-api/`: Useful APIs for use in plugs
+* `bin/silverbullet/`: The standalone `silverbullet` server binary (Rust) (this is the one you generally run)
+* `bin/sb/`: The `sb` CLI client (Rust)
+* `bin/plug-compile.ts`: the plug compiler
+* `client/`: The SilverBullet client (browser, TypeScript)
+* `server/`: The SilverBullet server library (Rust): HTTP router, handlers, auth, runtime seam
+* `server-common/`: Shared Rust crate (space primitives, shared types)
+* `server-runtime-chrome/`: Headless-Chrome runtime backend (Rust)
+* `plugs`: Set of built-in plugs (TypeScript) that are distributed with SilverBullet
+* `libraries`: A set of libraries (space scripts, page templates, slash templates) distributed with SilverBullet (Space Lua)
+* `plug-api/`: Useful APIs for use in plugs and published to NPM
   * `lib/`: Useful libraries to be used in plugs
   * `syscalls/`: TypeScript wrappers around syscalls
   * `types/`: Various (client) types that can be references from plugs
-* `bin`
-  * `plug_compile.ts` the plug compiler
 * `scripts/`: Useful scripts
-* `website/`: silverbullet.md website content
+* `docs/`: documentation (also serves a silverbullet.md website content)
 
 ### Requirements
-* [Node.js](https://nodejs.org/) 24+ and npm 10+: Used to build the frontend and plugs
-* [Go](https://go.dev/): Used to build the backend
+* [Node.js](https://nodejs.org/) 24+ and npm 10+: Used to build the frontend (client) and plugs
+* [Rust](https://www.rust-lang.org/tools/install) (stable, via `rustup`): Used to build the server
 
 The project includes `.nvmrc` and `.node-version` files. If you use [nvm](https://github.com/nvm-sh/nvm) or another Node version manager, it will automatically use the correct Node.js version:
 
@@ -55,35 +58,45 @@ The project includes `.nvmrc` and `.node-version` files. If you use [nvm](https:
 nvm use  # If using nvm
 ```
 
-It's convenient to also install [air](https://github.com/air-verse/air) for development, this will automatically rebuild both the frontend and backend when changes are made:
+First, install dependencies:
 
 ```shell
-go install github.com/air-verse/air@latest
+make setup
 ```
-Make sure your `$GOPATH/bin` is in your $PATH.
 
-First, install dependencies:
+To create a (release) build:
 
 ```shell
-make setup
+make
 ```
 
-To build everything and run the server (which automatically restarts upon file changing):
+This will produce `silverbullet` and `cli` binaries in `./target/release/`, run `./target/release/silverbullet` to run the server.
+
+### Development workflow
+
+SilverBullet has two halves you rebuild **independently**:
+
+* The **client** (TypeScript: `client/`) is built by ESBuild into `client_bundle/`, which the server then serves.
+* The **server** (Rust: `server/`, `server-common/`, `server-runtime-chrome/`, `bin/silverbullet/`) is a compiled binary.
+
+Run the server in development with `cargo run`. A **debug** build serves the client bundle **live from `client_bundle/` on disk** (a release build embeds it instead):
 
 ```shell
-air <PATH-TO-YOUR-SPACE>
+npm run build
+cargo run <PATH-TO-YOUR-SPACE>
 ```
 
-Alternatively, to build:
+**When you change only the client** (TypeScript in `client/`): you do **not** need to restart the server. Rebuild just the client bundle and reload the page in your browser — the debug server serves the freshly-built bundle straight from disk:
 
 ```shell
-make
+npm run build
 ```
 
-To run the resulting server:
+To build a self-contained **release** binary (with the client bundle embedded), and run it:
 
 ```shell
-./silverbullet <PATH-TO-YOUR-SPACE>
+make
+./target/release/silverbullet <PATH-TO-YOUR-SPACE>
 ```
 
 ### Useful development tasks

package/client/plugos/hooks/syscall.ts

@@ -1,5 +1,5 @@
 import type { Hook, Manifest } from "../types.ts";
-import type { SysCallMapping, System } from "../system.ts";
+import type { SysCallMapping, SyscallContext, System } from "../system.ts";
 import type { SyscallHookT } from "@silverbulletmd/silverbullet/type/manifest";
 
 export class SyscallHook implements Hook<SyscallHookT> {
@@ -24,16 +24,27 @@ export class SyscallHook implements Hook<SyscallHookT> {
           continue;
         }
 
-        const syscallName = functionDef.syscall;
+        const syscallDefinition = functionDef.syscall;
+        const syscallName =
+          typeof syscallDefinition === "string"
+            ? syscallDefinition
+            : syscallDefinition.name;
 
         // Add the syscall to our mapping
-        syscalls[syscallName] = (ctx, ...args) => {
-          // Delegate to the system to invoke the function
-          return system.syscall(ctx, "system.invokeFunction", [
+        const callback = (ctx: SyscallContext, ...args: any[]) =>
+          system.syscall(ctx, "system.invokeFunction", [
             `${plug.manifest!.name}.${name}`,
             ...args,
           ]);
-        };
+        if (typeof syscallDefinition === "string") {
+          syscalls[syscallName] = callback;
+        } else {
+          const { name: _name, ...metadata } = syscallDefinition;
+          syscalls[syscallName] = {
+            callback,
+            ...metadata,
+          };
+        }
 
         // Register the syscalls with no required permissions
         system.registerSyscalls([], syscalls);
@@ -48,16 +59,19 @@ export class SyscallHook implements Hook<SyscallHookT> {
         continue;
       }
 
-      // Validate syscall name is provided
-      if (!functionDef.syscall) {
+      const syscallName =
+        typeof functionDef.syscall === "string"
+          ? functionDef.syscall
+          : functionDef.syscall.name;
+      if (!syscallName) {
         errors.push(`Function ${name} has a syscall but no name`);
         continue;
       }
 
       // Validate syscall name format (should be namespaced)
-      if (!functionDef.syscall.includes(".")) {
+      if (!syscallName.includes(".")) {
         errors.push(
-          `Function ${name} has invalid syscall name "${functionDef.syscall}" - must be in format "namespace.name"`,
+          `Function ${name} has invalid syscall name "${syscallName}" - must be in format "namespace.name"`,
         );
       }
     }

package/client/plugos/syscalls/asset.ts

@@ -3,31 +3,69 @@ import type { FileMeta } from "../../../plug-api/types/index.ts";
 
 export default function assetSyscalls(system: System<any>): SysCallMapping {
   return {
-    "asset.readAsset": (_ctx, plugName: string, name: string): string => {
-      return system.loadedPlugs.get(plugName)!.assets!.readFileAsDataUrl(name);
+    "asset.readAsset": {
+      callback: (_ctx, plugName: string, name: string): string => {
+        return system.loadedPlugs
+          .get(plugName)!
+          .assets!.readFileAsDataUrl(name);
+      },
+      description: "Reads an asset embedded in a plug as a data URL.",
+      parameters: [
+        { name: "plugName", type: "string", description: "Plug name." },
+        { name: "name", type: "string", description: "Asset path." },
+      ],
+      returns: [{ type: "string", description: "Asset data URL." }],
+      examples: [
+        { code: 'local image = asset.readAsset("myplug", "image.png")' },
+      ],
     },
-    "asset.listFiles": (_ctx, plugName: string): FileMeta[] => {
-      const assets = system.loadedPlugs.get(plugName)!.assets!;
-      const fileNames = assets.listFiles();
-      return fileNames.map((name) => ({
-        name,
-        contentType: assets.getMimeType(name),
-        created: assets.getMtime(name),
-        lastModified: assets.getMtime(name),
-        size: -1,
-        perm: "ro",
-      }));
+    "asset.listFiles": {
+      callback: (_ctx, plugName: string): FileMeta[] => {
+        const assets = system.loadedPlugs.get(plugName)!.assets!;
+        const fileNames = assets.listFiles();
+        return fileNames.map((name) => ({
+          name,
+          contentType: assets.getMimeType(name),
+          created: assets.getMtime(name),
+          lastModified: assets.getMtime(name),
+          size: -1,
+          perm: "ro",
+        }));
+      },
+      description: "Lists the assets embedded in a plug.",
+      parameters: [
+        { name: "plugName", type: "string", description: "Plug name." },
+      ],
+      returns: [{ type: "table", description: "List of file metadata." }],
+      examples: [
+        {
+          code: 'for _, file in ipairs(asset.listFiles("myplug")) do\n  print(file.name)\nend',
+        },
+      ],
     },
-    "asset.getFileMeta": (_ctx, plugName: string, name: string): FileMeta => {
-      const assets = system.loadedPlugs.get(plugName)!.assets!;
-      return {
-        name,
-        contentType: assets.getMimeType(name),
-        created: assets.getMtime(name),
-        lastModified: assets.getMtime(name),
-        size: -1,
-        perm: "ro",
-      };
+    "asset.getFileMeta": {
+      callback: (_ctx, plugName: string, name: string): FileMeta => {
+        const assets = system.loadedPlugs.get(plugName)!.assets!;
+        return {
+          name,
+          contentType: assets.getMimeType(name),
+          created: assets.getMtime(name),
+          lastModified: assets.getMtime(name),
+          size: -1,
+          perm: "ro",
+        };
+      },
+      description: "Gets metadata for an asset embedded in a plug.",
+      parameters: [
+        { name: "plugName", type: "string", description: "Plug name." },
+        { name: "name", type: "string", description: "Asset path." },
+      ],
+      returns: [{ type: "table", description: "File metadata." }],
+      examples: [
+        {
+          code: 'local meta = asset.getFileMeta("myplug", "data.txt")\nprint(meta.lastModified)',
+        },
+      ],
     },
   };
 }

package/client/plugos/syscalls/clientStore.ts

@@ -8,14 +8,40 @@ export function clientStoreSyscalls(
   prefix: KvKey = ["client"],
 ): SysCallMapping {
   return {
-    "clientStore.get": (_ctx, key: string): Promise<any> => {
-      return ds.get([...prefix, key]);
+    "clientStore.get": {
+      callback: (_ctx, key: string): Promise<any> => {
+        return ds.get([...prefix, key]);
+      },
+      description:
+        "Gets a client-specific value from the local key-value store.",
+      parameters: [
+        { name: "key", type: "string", description: "Key to read." },
+      ],
+      returns: [{ description: "Stored value, or nil when absent." }],
+      examples: [{ code: 'local theme = clientStore.get("theme")' }],
     },
-    "clientStore.set": (_ctx, key: string, val: any): Promise<void> => {
-      return ds.set([...prefix, key], val);
+    "clientStore.set": {
+      callback: (_ctx, key: string, val: any): Promise<void> => {
+        return ds.set([...prefix, key], val);
+      },
+      description:
+        "Stores a client-specific value in the local key-value store.",
+      parameters: [
+        { name: "key", type: "string", description: "Key to set." },
+        { name: "value", description: "Value to store." },
+      ],
+      examples: [{ code: 'clientStore.set("theme", "dark")' }],
     },
-    "clientStore.delete": (_ctx, key: string): Promise<void> => {
-      return ds.delete([...prefix, key]);
+    "clientStore.delete": {
+      callback: (_ctx, key: string): Promise<void> => {
+        return ds.delete([...prefix, key]);
+      },
+      description:
+        "Deletes a client-specific value from the local key-value store.",
+      parameters: [
+        { name: "key", type: "string", description: "Key to delete." },
+      ],
+      examples: [{ code: 'clientStore.delete("theme")' }],
     },
   };
 }

package/client/plugos/syscalls/client_code_widget.ts

@@ -5,10 +5,15 @@ import { broadcastReload } from "../../components/widget_sandbox_iframe.ts";
 
 export function clientCodeWidgetSyscalls(client: Client): SysCallMapping {
   return {
-    "codeWidget.refreshAll": () => {
-      client.widgetCache.clearPrewarm();
-      broadcastReload();
-      return reloadAllWidgets();
+    "codeWidget.refreshAll": {
+      callback: () => {
+        client.widgetCache.clearPrewarm();
+        broadcastReload();
+        return reloadAllWidgets();
+      },
+      description:
+        "Refreshes all code widgets on the current page that support refreshing.",
+      examples: [{ code: "codeWidget.refreshAll()" }],
     },
   };
 }

package/client/plugos/syscalls/code_widget.ts

@@ -6,17 +6,32 @@ export function codeWidgetSyscalls(
   codeWidgetHook: CodeWidgetHook,
 ): SysCallMapping {
   return {
-    "codeWidget.render": (
-      _ctx,
-      lang: string,
-      body: string,
-      pageName: string,
-    ): Promise<CodeWidgetContent | null> => {
-      const langCallback = codeWidgetHook.codeWidgetCallbacks.get(lang);
-      if (!langCallback) {
-        throw new Error(`Code widget ${lang} not found`);
-      }
-      return langCallback(body, pageName);
+    "codeWidget.render": {
+      callback: (
+        _ctx,
+        lang: string,
+        body: string,
+        pageName: string,
+      ): Promise<CodeWidgetContent | null> => {
+        const langCallback = codeWidgetHook.codeWidgetCallbacks.get(lang);
+        if (!langCallback) {
+          throw new Error(`Code widget ${lang} not found`);
+        }
+        return langCallback(body, pageName);
+      },
+      description: "Renders code through the widget registered for a language.",
+      parameters: [
+        { name: "language", type: "string", description: "Widget language." },
+        { name: "body", type: "string", description: "Code block body." },
+        {
+          name: "pageName",
+          type: "string",
+          description: "Containing page name.",
+        },
+      ],
+      returns: [
+        { type: "table", description: "Rendered widget content, or nil." },
+      ],
     },
   };
 }

package/client/plugos/syscalls/config.ts

@@ -5,45 +5,161 @@ import { stripFunctions } from "../util.ts";
 
 export function configSyscalls(config: Config): SysCallMapping {
   return {
-    "config.get": (_ctx, path: string, defaultValue: any) => {
-      return config.get(path, defaultValue);
+    "config.get": {
+      callback: (_ctx, path: string, defaultValue: any) => {
+        return config.get(path, defaultValue);
+      },
+      description:
+        "Gets a configuration value by path, with dot notation support.",
+      parameters: [
+        { name: "path", type: "string", description: "Configuration path." },
+        {
+          name: "defaultValue",
+          description: "Value returned when the path is absent.",
+        },
+      ],
+      returns: [{ description: "Configured value or the supplied default." }],
+      examples: [{ code: 'local theme = config.get("theme", "light")' }],
     },
-    "config.set": (
-      _ctx,
-      keyOrValues: string | string[] | Record<string, any>,
-      value?: any,
-    ) => {
-      config.set(keyOrValues as any, value);
+    "config.set": {
+      callback: (
+        _ctx,
+        keyOrValues: string | string[] | Record<string, any>,
+        value?: any,
+      ) => {
+        config.set(keyOrValues as any, value);
+      },
+      description: "Sets one configuration value or multiple values at once.",
+      signatures: ["config.set(path, value)", "config.set(values)"],
+      parameters: [
+        {
+          name: "pathOrValues",
+          description: "Configuration path or table of values.",
+        },
+        {
+          name: "value",
+          description: "Value to set when a path is supplied.",
+          optional: true,
+        },
+      ],
+      examples: [
+        { code: 'config.set("theme", "dark")' },
+        { code: 'config.set({theme = "dark", fontSize = 14})' },
+      ],
     },
-    "lua:config.setLuaValue": async (
-      _ctx,
-      keyOrValues: string | string[] | Record<string, any>,
-      value?: any,
-    ) => {
-      // This is for special cases where we explicitly want to NOT convert a value to a JS value, but maintain its Lua version (mostly for LuaTables) — main use case: metatable
-      keyOrValues = await luaValueToJS(keyOrValues, LuaStackFrame.lostFrame);
-      config.set(keyOrValues as any, value);
+    "lua:config.setLuaValue": {
+      callback: async (
+        _ctx,
+        keyOrValues: string | string[] | Record<string, any>,
+        value?: any,
+      ) => {
+        // This is for special cases where we explicitly want to NOT convert a value to a JS value, but maintain its Lua version (mostly for LuaTables) — main use case: metatable
+        keyOrValues = await luaValueToJS(keyOrValues, LuaStackFrame.lostFrame);
+        config.set(keyOrValues as any, value);
+      },
+      description:
+        "Sets configuration while preserving the supplied Lua value representation.",
+      signatures: [
+        "config.setLuaValue(path, value)",
+        "config.setLuaValue(values)",
+      ],
+      parameters: [
+        {
+          name: "pathOrValues",
+          description: "Configuration path or table of values.",
+        },
+        {
+          name: "value",
+          description: "Lua value to preserve.",
+          optional: true,
+        },
+      ],
     },
-    "config.insert": (_ctx, key: string | string[], value: any) => {
-      config.insert(key, value);
+    "config.insert": {
+      callback: (_ctx, key: string | string[], value: any) => {
+        config.insert(key, value);
+      },
+      description: "Appends a value to the configuration array at a path.",
+      parameters: [
+        { name: "path", description: "Configuration path." },
+        { name: "value", description: "Value to append." },
+      ],
     },
-    "config.has": (_ctx, path: string) => {
-      return config.has(path);
+    "config.has": {
+      callback: (_ctx, path: string) => {
+        return config.has(path);
+      },
+      description: "Checks whether a configuration path exists.",
+      parameters: [
+        { name: "path", type: "string", description: "Configuration path." },
+      ],
+      returns: [{ type: "boolean", description: "Whether the path exists." }],
     },
-    "config.define": (_ctx, key: string, schema: any) => {
-      config.define(key, schema);
+    "config.define": {
+      callback: (_ctx, key: string, schema: any) => {
+        config.define(key, schema);
+      },
+      description: "Defines a JSON schema for a configuration key.",
+      parameters: [
+        { name: "key", type: "string", description: "Configuration key." },
+        {
+          name: "schema",
+          type: "table",
+          description:
+            "JSON Schema definition; default applies a missing value and ui annotations expose it in the Configuration Manager.",
+        },
+      ],
+      examples: [
+        {
+          code: 'config.define("shortWikiLinks", {\n  description = "Render short wiki link labels",\n  type = "boolean",\n  default = true,\n  ui = {category = "Editor", label = "Short wiki links", priority = 1},\n})',
+        },
+      ],
     },
-    "config.defineCategory": (_ctx, definition: any) => {
-      config.defineCategory(definition);
+    "config.defineCategory": {
+      callback: (_ctx, definition: any) => {
+        config.defineCategory(definition);
+      },
+      description: "Defines or updates a Configuration Manager UI category.",
+      parameters: [
+        {
+          name: "definition",
+          type: "table",
+          description: "Category name, description, and priority.",
+        },
+      ],
+      examples: [
+        {
+          code: 'config.defineCategory {\n  name = "Editor",\n  description = "Page editor behavior.",\n  priority = 50,\n}',
+        },
+      ],
     },
-    "config.getValues": () => {
-      return stripFunctions(config.values);
+    "config.getValues": {
+      callback: () => {
+        return stripFunctions(config.values);
+      },
+      description: "Gets all configuration values as a single table.",
+      returns: [{ type: "table", description: "All configuration values." }],
     },
-    "config.getSchemas": () => {
-      return stripFunctions(config.schemas);
+    "config.getSchemas": {
+      callback: () => {
+        return stripFunctions(config.schemas);
+      },
+      description: "Gets all defined configuration schemas.",
+      returns: [
+        {
+          type: "table",
+          description: "Schemas keyed by configuration path.",
+        },
+      ],
     },
-    "config.getCategories": () => {
-      return { ...config.categories };
+    "config.getCategories": {
+      callback: () => {
+        return { ...config.categories };
+      },
+      description: "Gets all Configuration Manager UI categories.",
+      returns: [
+        { type: "table", description: "Category definitions keyed by name." },
+      ],
     },
   };
 }