@silverbulletmd/silverbullet 2.9.0 → 2.11.0

package/README.md

@@ -27,27 +27,30 @@ You were told there’s no such thing as a silver bullet. You were told wrong.
 Check out the [instructions](https://silverbullet.md/Install).
 
 ## Developing SilverBullet
-SilverBullet's frontend is written in [TypeScript](https://www.typescriptlang.org/) and built on top of the excellent [CodeMirror 6](https://codemirror.net/) editor component. Additional UI is built using [Preact](https://preactjs.com). [ESBuild](https://esbuild.github.io) is used to build the frontend.
+SilverBullet's frontend ("client") is written in [TypeScript](https://www.typescriptlang.org/) and built on top of the excellent [CodeMirror 6](https://codemirror.net/) editor component. Additional UI is built using [Preact](https://preactjs.com). [ESBuild](https://esbuild.github.io) is used to build the frontend.
 
-The server backend is written in Go.
+The server backend is written in [Rust](https://www.rust-lang.org/) (a Cargo workspace). It serves the pre-built client bundle, the file/space HTTP API, authentication, and a headless-Chrome runtime for server-side Lua.
 
 ## Code structure
-* `client/`: The SilverBullet client, implemented with TypeScript
-* `server/`: The SilverBullet server, written in Go
-* `plugs`: Set of built-in plugs that are distributed with SilverBullet
-* `libraries`: A set of libraries (space scripts, page templates, slash templates) distributed with SilverBullet
-* `plug-api/`: Useful APIs for use in plugs
+* `bin/silverbullet/`: The standalone `silverbullet` server binary (Rust) (this is the one you generally run)
+* `bin/sb/`: The `sb` CLI client (Rust)
+* `bin/plug-compile.ts`: the plug compiler
+* `client/`: The SilverBullet client (browser, TypeScript)
+* `server/`: The SilverBullet server library (Rust): HTTP router, handlers, auth, runtime seam
+* `server-common/`: Shared Rust crate (space primitives, shared types)
+* `server-runtime-chrome/`: Headless-Chrome runtime backend (Rust)
+* `plugs`: Set of built-in plugs (TypeScript) that are distributed with SilverBullet
+* `libraries`: A set of libraries (space scripts, page templates, slash templates) distributed with SilverBullet (Space Lua)
+* `plug-api/`: Useful APIs for use in plugs and published to NPM
   * `lib/`: Useful libraries to be used in plugs
   * `syscalls/`: TypeScript wrappers around syscalls
   * `types/`: Various (client) types that can be references from plugs
-* `bin`
-  * `plug_compile.ts` the plug compiler
 * `scripts/`: Useful scripts
-* `website/`: silverbullet.md website content
+* `docs/`: documentation (also serves a silverbullet.md website content)
 
 ### Requirements
-* [Node.js](https://nodejs.org/) 24+ and npm 10+: Used to build the frontend and plugs
-* [Go](https://go.dev/): Used to build the backend
+* [Node.js](https://nodejs.org/) 24+ and npm 10+: Used to build the frontend (client) and plugs
+* [Rust](https://www.rust-lang.org/tools/install) (stable, via `rustup`): Used to build the server
 
 The project includes `.nvmrc` and `.node-version` files. If you use [nvm](https://github.com/nvm-sh/nvm) or another Node version manager, it will automatically use the correct Node.js version:
 
@@ -55,35 +58,45 @@ The project includes `.nvmrc` and `.node-version` files. If you use [nvm](https:
 nvm use  # If using nvm
 ```
 
-It's convenient to also install [air](https://github.com/air-verse/air) for development, this will automatically rebuild both the frontend and backend when changes are made:
+First, install dependencies:
 
 ```shell
-go install github.com/air-verse/air@latest
+make setup
 ```
-Make sure your `$GOPATH/bin` is in your $PATH.
 
-First, install dependencies:
+To create a (release) build:
 
 ```shell
-make setup
+make
 ```
 
-To build everything and run the server (which automatically restarts upon file changing):
+This will produce `silverbullet` and `cli` binaries in `./target/release/`, run `./target/release/silverbullet` to run the server.
+
+### Development workflow
+
+SilverBullet has two halves you rebuild **independently**:
+
+* The **client** (TypeScript: `client/`) is built by ESBuild into `client_bundle/`, which the server then serves.
+* The **server** (Rust: `server/`, `server-common/`, `server-runtime-chrome/`, `bin/silverbullet/`) is a compiled binary.
+
+Run the server in development with `cargo run`. A **debug** build serves the client bundle **live from `client_bundle/` on disk** (a release build embeds it instead):
 
 ```shell
-air <PATH-TO-YOUR-SPACE>
+npm run build
+cargo run <PATH-TO-YOUR-DATA-FOLDER>
 ```
 
-Alternatively, to build:
+**When you change only the client** (TypeScript in `client/`): you do **not** need to restart the server. Rebuild just the client bundle and reload the page in your browser — the debug server serves the freshly-built bundle straight from disk:
 
 ```shell
-make
+npm run build
 ```
 
-To run the resulting server:
+To build a self-contained **release** binary (with the client bundle embedded), and run it:
 
 ```shell
-./silverbullet <PATH-TO-YOUR-SPACE>
+make
+./target/release/silverbullet <PATH-TO-YOUR-DATA-FOLDER>
 ```
 
 ### Useful development tasks
@@ -102,7 +115,7 @@ make bench
 ```
 
 ### Build a docker container
-Note, you do not need Node.js nor Go locally installed for this to work:
+Note, you do not need Node.js nor Rust locally installed for this to work:
 
 ```shell
 docker build -t silverbullet .
@@ -110,5 +123,5 @@ docker build -t silverbullet .
 
 To run:
 ```shell
-docker run -p 3000:3000 -v <PATH-TO-YOUR-SPACE>:/space silverbullet
+docker run -p 3000:3000 -v <PATH-TO-YOUR-DATA-FOLDER>:/data silverbullet
 ```

package/client/asset_bundle/bundle.ts

@@ -66,7 +66,6 @@ export class AssetBundle {
     data: Uint8Array,
     mtime: number = Date.now(),
   ) {
-    // Replace \ with / for windows
     path = path.replaceAll("\\", "/");
     this.bundle[path] = {
       data: base64EncodedDataUrl(mimeType, data),

package/client/config.ts

@@ -0,0 +1,343 @@
+import { type OutputUnit, Validator, format } from "@cfworker/json-schema";
+import { stripFunctions } from "./plugos/util.ts";
+
+format.email = (data: string) => data.includes("@");
+format["page-ref"] = (data: string) =>
+  data.startsWith("[[") && data.endsWith("]]");
+
+function cfwFormatErrors(errors: OutputUnit[]): string {
+  // Filter out "properties" wrapper errors, keep only the specific leaf errors
+  const leafErrors = errors.filter((e) => e.keyword !== "properties");
+  const errorsToUse = leafErrors.length > 0 ? leafErrors : errors;
+
+  return errorsToUse
+    .map((e) => {
+      // Convert instanceLocation from "#/foo/bar" to "foo.bar"
+      const path =
+        e.instanceLocation === "#"
+          ? ""
+          : e.instanceLocation.slice(2).replaceAll("/", ".");
+      return path ? `${path}: ${e.error}` : e.error;
+    })
+    .join(", ");
+}
+
+/**
+ * Validates that a value looks like a valid JSON schema.
+ * Uses a lightweight structural check rather than full meta-schema validation.
+ */
+function isValidJsonSchema(schema: any): { valid: boolean; error?: string } {
+  if (schema === null || schema === undefined) {
+    return { valid: false, error: "schema must not be null or undefined" };
+  }
+  if (typeof schema === "boolean") {
+    return { valid: true };
+  }
+  if (typeof schema !== "object" || Array.isArray(schema)) {
+    return { valid: false, error: "schema must be an object or boolean" };
+  }
+  if (schema.type !== undefined) {
+    const validTypes = [
+      "string",
+      "number",
+      "integer",
+      "boolean",
+      "object",
+      "array",
+      "null",
+    ];
+    const types = Array.isArray(schema.type) ? schema.type : [schema.type];
+    for (const t of types) {
+      if (!validTypes.includes(t)) {
+        return {
+          valid: false,
+          error: `schema.type must be one of ${validTypes.join(", ")}`,
+        };
+      }
+    }
+  }
+  return { valid: true };
+}
+
+/**
+ * Configuration management (config.* APIs) for the client
+ */
+export type CategoryDefinition = {
+  name: string;
+  description?: string;
+  priority?: number;
+};
+
+export class Config {
+  public schemas: Record<string, any> = {
+    type: "object",
+    properties: {},
+  };
+
+  /**
+   * Registered UI categories for the configuration manager. Categories appear
+   * in the UI in descending `priority` (higher first); categories referenced
+   * by a schema's `ui.category` but never registered fall to the bottom in
+   * alphabetical order.
+   */
+  public categories: Record<string, CategoryDefinition> = {};
+
+  constructor(public values: Record<string, any> = {}) {}
+
+  public clear() {
+    this.schemas = {
+      type: "object",
+      properties: {},
+    };
+    this.values = {};
+    this.categories = {};
+  }
+
+  /**
+   * Defines (or updates) a UI category for the configuration manager. The
+   * category's `name` is the same string used in a schema field's
+   * `ui.category`.
+   */
+  defineCategory(definition: CategoryDefinition): void {
+    if (typeof definition?.name !== "string" || !definition.name) {
+      throw new Error("defineCategory requires a non-empty name");
+    }
+    this.categories[definition.name] = { ...definition };
+  }
+
+  /**
+   * Defines a JSON schema for a configuration key
+   * @param key The configuration key to define a schema for
+   * @param schema The JSON schema to validate against
+   */
+  define(key: string | string[], schema: any): void {
+    const result = isValidJsonSchema(schema);
+    if (!result.valid) {
+      throw new Error(`Invalid schema for key ${key}: ${result.error}`);
+    }
+
+    if (typeof key === "string") {
+      key = key.split(".");
+    }
+
+    let current = this.schemas;
+    for (let i = 0; i < key.length - 1; i++) {
+      const part = key[i];
+      if (!current.properties[part]) {
+        current.properties[part] = {
+          type: "object",
+          properties: {},
+        };
+      }
+      current = current.properties[part];
+    }
+
+    const finalKey = key[key.length - 1];
+    current.properties[finalKey] = schema;
+
+    this.applySchemaDefaults(key, schema);
+  }
+
+  /**
+   * Recursively applies default values from a schema definition.
+   * Only sets a value if none exists at that path.
+   */
+  private applySchemaDefaults(path: string[], schema: any): void {
+    if ("default" in schema) {
+      if (!this.has(path)) {
+        this.set(path, structuredClone(schema.default));
+      }
+    }
+    if (schema.properties) {
+      for (const [propKey, propSchema] of Object.entries(schema.properties)) {
+        this.applySchemaDefaults([...path, propKey], propSchema as any);
+      }
+    }
+  }
+
+  /**
+   * Gets a value from the config
+   * @param path The path to get, supports dot notation (e.g. "foo.bar.baz")
+   * @param defaultValue The default value to return if the path doesn't exist
+   * @returns The value at the path, or the default value
+   */
+  get<T>(path: string | string[], defaultValue: T): T {
+    if (typeof path === "string") {
+      path = path.split(".");
+    }
+    const resolved = resolvePath(this.values, path);
+    if (!resolved) {
+      return defaultValue;
+    }
+
+    return (resolved.obj[resolved.key] ?? defaultValue) as T;
+  }
+
+  /**
+   * Sets a value in the config
+   * @param path The path to set, supports dot notation (e.g. "foo.bar.baz")
+   * @param value The value to set
+   */
+  set<T>(path: string, value: T): void;
+  /**
+   * Sets a value in the config
+   * @param path The path to set (one path element per array element)
+   * @param value The value to set
+   */
+  set<T>(path: string[], value: T): void;
+
+  /**
+   * Sets multiple values in the config
+   * @param values An object containing key-value pairs to set
+   */
+  set(values: Record<string, any>): void;
+
+  set<T>(
+    keyOrValues: string | string[] | Record<string, any>,
+    value?: T,
+  ): void {
+    if (typeof keyOrValues === "string") {
+      keyOrValues = keyOrValues.split(".");
+    }
+    if (Array.isArray(keyOrValues)) {
+      const key = keyOrValues as string[];
+
+      const resolved = resolvePath(this.values, key, true);
+      if (resolved) {
+        resolved.obj[resolved.key] = value;
+      } else {
+        throw new Error(`Invalid key ${key}`);
+      }
+
+      // Re-apply nested schema defaults so partial object writes don't wipe
+      // out defaults for nested properties the user didn't specify.
+      const schema = this.getSchemaAtPath(key);
+      if (schema) {
+        this.applySchemaDefaults(key, schema);
+      }
+
+      this.validatePath(key);
+    } else {
+      for (const [key, val] of Object.entries(keyOrValues)) {
+        this.set(key, val);
+      }
+    }
+  }
+
+  insert<T>(key: string | string[], value?: T): void {
+    if (typeof key === "string") {
+      key = key.split(".");
+    }
+    const resolved = resolvePath(this.values, key, true);
+    if (resolved) {
+      if (!Array.isArray(resolved.obj[resolved.key])) {
+        resolved.obj[resolved.key] = [];
+      }
+      resolved.obj[resolved.key].push(value);
+    } else {
+      throw new Error(`Invalid key ${key}`);
+    }
+
+    this.validatePath(key);
+  }
+
+  /**
+   * Checks if a path exists in the config
+   * @param path The path to check, supports dot notation (e.g. "foo.bar.baz")
+   * @returns True if the path exists, false otherwise
+   */
+  has(path: string | string[]): boolean {
+    if (typeof path === "string") {
+      path = path.split(".");
+    }
+    const resolved = resolvePath(this.values, path);
+    if (!resolved) {
+      return false;
+    }
+
+    return resolved.key in resolved.obj;
+  }
+
+  /**
+   * Lists all configuration keys in the config
+   * @returns An array of keys
+   */
+  keys(): string[] {
+    return Object.keys(this.values);
+  }
+
+  /**
+   * Validates a specific path against its schema
+   */
+  private validatePath(path: string[]): void {
+    for (let i = path.length; i > 0; i--) {
+      const schemaPath = path.slice(0, i);
+      const schema = this.getSchemaAtPath(schemaPath);
+      if (schema) {
+        const valueAtPath = this.get(schemaPath, undefined);
+        if (valueAtPath !== undefined) {
+          const validator = new Validator(schema, "7");
+          const result = validator.validate(stripFunctions(valueAtPath));
+          if (!result.valid) {
+            const errorText = cfwFormatErrors(result.errors);
+            throw new Error(
+              `Validation error for ${schemaPath.join(".")}:> ${errorText}`,
+            );
+          }
+        }
+        break; // Only validate the most specific schema
+      }
+    }
+  }
+
+  /**
+   * Gets the schema at a specific path
+   */
+  private getSchemaAtPath(path: string[]): any | null {
+    let current = this.schemas;
+    for (const part of path) {
+      if (!current.properties || !current.properties[part]) {
+        return null;
+      }
+      current = current.properties[part];
+    }
+    return current.type ? current : null;
+  }
+}
+
+/**
+ * Resolves a configuration path to the containing object and final key
+ * @param path The path to resolve (e.g. ["foo", "bar", "baz"])
+ * @param create Whether to create objects along the path if they don't exist
+ * @returns The containing object and the final key, or null if the path cannot be resolved
+ */
+function resolvePath(
+  inObject: Record<string, any>,
+  path: string[],
+  create = false,
+): { obj: any; key: string } | null {
+  path = [...path];
+
+  const lastKey = path.pop()!;
+  let current = inObject;
+
+  for (const part of path) {
+    if (current[part] === undefined) {
+      if (create) {
+        current[part] = {};
+      } else {
+        return null;
+      }
+    } else if (typeof current[part] !== "object" || current[part] === null) {
+      if (create) {
+        current[part] = {};
+      } else {
+        return null;
+      }
+    }
+
+    current = current[part];
+  }
+
+  return { obj: current, key: lastKey };
+}

package/client/markdown_parser/constants.ts

@@ -3,7 +3,9 @@ export const wikiLinkRegex =
 export const mdLinkRegex =
   /!?\[(?<title>[^\]\\]*(?:\\.[^\]\\]*)*)\]\((?<url>.+)\)/g;
 export const tagRegex =
-  /#(?:(?:\d*[^\d\s!@#$%^&*(),.?":{}|<>\\][^\s!@#$%^&*(),.?":{}|<>\\]*)|(?:<[^>\n]+>))/;
+  /#(?:(?:\d*[^\d\s!@#$%^&*(),.?":{}|<>\\';\/–—][^\s!@#$%^&*(),.?":{}|<>\\]*)|(?:<[^>\n]+>))(?<!-)/;
+export const atMentionRegex =
+  /@(?:\d*[^\d\s!@#$%^&*(),.?":{}|<>\\\/][^\s!@#$%^&*(),.?":{}|<>\\\/]*(?:\.[^\s!@#$%^&*(),.?":{}|<>\\\/]+)*)/;
 export const nakedUrlRegex =
   /(^https?:\/\/([-a-zA-Z0-9@:%_+~#=]|(?:[.](?!(\s|$)))){1,256})(([-a-zA-Z0-9(@:%_+~#?&=/]|(?:[.,:;)](?!(\s|$))))*)/;
 export const frontmatterQuotesRegex = /["'].*["']/g;

package/client/plugos/hooks/command.ts

@@ -39,7 +39,6 @@ export class CommandHook
     cmd: Command,
   ) {
     if (commands.has(name)) {
-      // Existing command, let's do some inline patching
       const existingCommand = commands.get(name)!;
       const command: Command = {
         ...existingCommand,
@@ -68,9 +67,7 @@ export class CommandHook
       }
       commands.set(name, cmd);
     }
-    // Add commands from plugs
     if (!this.system) {
-      // Not initialized yet
       return commands;
     }
     for (const plug of this.system.loadedPlugs.values()) {
@@ -82,7 +79,6 @@ export class CommandHook
         }
         const cmd = functionDef.command;
         if (cmd.requireMode === "rw" && this.readOnly) {
-          // Bit hacky, but don't expose commands that require write mode in read-only mode
           continue;
         }
         this.mergeCommand(commands, cmd.name, {
@@ -112,7 +108,6 @@ export class CommandHook
         this.throttledBuildAllCommandsAndEmit();
       },
     });
-    // On next tick
     setTimeout(() => {
       this.throttledBuildAllCommandsAndEmit();
     });

package/client/plugos/hooks/event.ts

@@ -79,9 +79,7 @@ export class EventHook implements EventHookI {
               event === eventName ||
               eventNameToRegex(event).test(eventName)
             ) {
-              // Only dispatch functions that can run in this environment
               if (plug.canInvoke(name)) {
-                // Queue the promise
                 promises.push(
                   (async () => {
                     try {
@@ -101,11 +99,9 @@ export class EventHook implements EventHookI {
       }
     }
 
-    // Local listeners
     for (const [name, localListeners] of this.localListeners) {
       if (eventNameToRegex(name).test(eventName)) {
         for (const localListener of localListeners) {
-          // Queue the promise
           promises.push(
             (async () => {
               return await Promise.resolve(localListener(...args));
@@ -115,7 +111,6 @@ export class EventHook implements EventHookI {
       }
     }
 
-    // Space Lua listeners
     if (this.config) {
       const configListeners: Record<string, Function[]> = this.config.get(
         "eventListeners",
@@ -140,7 +135,6 @@ export class EventHook implements EventHookI {
       }
     }
 
-    // Wait for all promises to resolve
     return (await Promise.allSettled(promises))
       .filter((result) => {
         if (result.status === "rejected") {

package/client/plugos/hooks/mq.ts

@@ -48,7 +48,6 @@ export class MQHook implements Hook<MQHookT> {
 
   reloadQueues() {
     this.stop();
-    // Plug based subscriptions
     for (const plug of this.system.loadedPlugs.values()) {
       if (!plug.manifest) {
         continue;
@@ -96,14 +95,12 @@ export class MQHook implements Hook<MQHookT> {
         }
       }
     }
-    // Space Lua based subscriptions
     const configListeners: Record<string, MQListenerSpec[]> = this.config.get(
       "mqSubscriptions",
       {},
     );
     for (const [queue, listeners] of Object.entries(configListeners)) {
       for (const listener of listeners) {
-        // console.log("Subscribing to", queue, listener);
         this.subscriptions.push(
           this.mq.subscribe(
             queue,

package/client/plugos/hooks/slash_command.ts

@@ -46,7 +46,6 @@ export class SlashCommandHook implements Hook<SlashCommandHookT> {
         });
       }
     }
-    // Iterate over script defined slash commands
     for (const command of Object.values(
       this.client.config.get<Record<string, SlashCommand>>("slashCommands", {}),
     )) {
@@ -70,13 +69,12 @@ export class SlashCommandHook implements Hook<SlashCommandHookT> {
     // the user is referring to preceding text, not what comes after.
     const currentNode = syntaxTree(ctx.state).resolveInner(ctx.pos, -1);
     if (
-      currentNode.type.name === "CommentBlock" ||
+      currentNode.type.name === "CommentMarkerBlock" ||
       currentNode.type.name === "Link"
     ) {
       return null;
     }
 
-    // Check if the slash command is available in the current context
     const parentNodes = this.client.extractParentNodes(ctx.state, currentNode);
     for (const def of this.slashCommands) {
       if (
@@ -99,7 +97,6 @@ export class SlashCommandHook implements Hook<SlashCommandHookT> {
         detail: def.description,
         boost: def.priority,
         apply: () => {
-          // Delete slash command part
           this.client.editorView.dispatch({
             changes: {
               from: prefix!.from + prefixText.indexOf("/"),
@@ -107,7 +104,6 @@ export class SlashCommandHook implements Hook<SlashCommandHookT> {
               insert: "",
             },
           });
-          // Replace with whatever the completion is
           safeRun(async () => {
             await def.run!();
             this.client.focus();
@@ -126,7 +122,6 @@ export class SlashCommandHook implements Hook<SlashCommandHookT> {
           detail: slashCompletion.detail,
           boost: slashCompletion.order && -slashCompletion.order,
           apply: () => {
-            // Delete slash command part
             this.client.editorView.dispatch({
               changes: {
                 from: prefix!.from + prefixText.indexOf("/"),
@@ -134,7 +129,6 @@ export class SlashCommandHook implements Hook<SlashCommandHookT> {
                 insert: "",
               },
             });
-            // Replace with whatever the completion is
             safeRun(async () => {
               await this.client.clientSystem.system.invokeFunction(
                 slashCompletion.invoke,