@silverbulletmd/silverbullet 2.6.1 → 2.7.0

package/README.md

@@ -107,7 +107,6 @@ docker build -t silverbullet .
 ```
 
 To run:
-
 ```shell
 docker run -p 3000:3000 -v <PATH-TO-YOUR-SPACE>:/space silverbullet
 ```

package/client/markdown_parser/constants.ts

@@ -12,3 +12,4 @@ export const frontmatterWikiLinkRegex =
   /(?<leadingTrivia>!?\[\[)(?<stringRef>.*?)(?:\|(?<alias>.*?))?(?<trailingTrivia>\]\])/g;
 export const frontmatterMailtoRegex = /(mailto:[^@\s]+@[^@\s"']+)/gi;
 export const pWikiLinkRegex = new RegExp(`^${wikiLinkRegex.source}`); // Modified regex used only in parser
+export const anchorRegex = /\$[A-Za-z_][A-Za-z0-9_/:-]*/;

package/client/plugos/hooks/command.ts

@@ -14,6 +14,8 @@ export class CommandHook
     this.buildAllCommandsAndEmit();
   }, 200);
 
+  private registeredCommands = new Map<string, Command>();
+
   constructor(
     private readOnly: boolean,
     private additionalCommands: Map<string, Command>,
@@ -21,11 +23,51 @@ export class CommandHook
     super();
   }
 
+  /**
+   * Register a command directly with the hook. Intended for client-side code
+   * that wants to define commands without going through a plug manifest.
+   * Overwrites any previously-registered command with the same name.
+   */
+  registerCommand(command: Command): void {
+    this.registeredCommands.set(command.name, command);
+    this.throttledBuildAllCommandsAndEmit();
+  }
+
+  private mergeCommand(
+    commands: Map<string, Command>,
+    name: string,
+    cmd: Command,
+  ) {
+    if (commands.has(name)) {
+      // Existing command, let's do some inline patching
+      const existingCommand = commands.get(name)!;
+      const command: Command = {
+        ...existingCommand,
+        ...cmd,
+      };
+      if (cmd.run) {
+        command.run = cmd.run;
+      }
+      commands.set(name, command);
+    } else {
+      commands.set(name, cmd);
+    }
+  }
+
   /**
    * Build the command map
    */
   buildAllCommands(): Map<string, Command> {
     const commands = new Map<string, Command>();
+    // Start with directly-registered commands (built-in client commands).
+    // These form the base layer and can be overridden by plug commands or
+    // script commands further down.
+    for (const [name, cmd] of this.registeredCommands) {
+      if (cmd.requireMode === "rw" && this.readOnly) {
+        continue;
+      }
+      commands.set(name, cmd);
+    }
     // Add commands from plugs
     if (!this.system) {
       // Not initialized yet
@@ -43,7 +85,7 @@ export class CommandHook
           // Bit hacky, but don't expose commands that require write mode in read-only mode
           continue;
         }
-        commands.set(cmd.name, {
+        this.mergeCommand(commands, cmd.name, {
           ...cmd,
           run: (args?: string[]) => {
             return plug.invoke(name, [cmd, ...(args ?? [])]);
@@ -51,22 +93,10 @@ export class CommandHook
         });
       }
     }
+    // Script commands come last — they always win over built-ins and plugs,
+    // so users can rebind keys / replace run bodies from Lua.
     for (const [name, cmd] of this.additionalCommands) {
-      if (commands.has(name)) {
-        // Existing command, let's do some inline patching
-        const existingCommand = commands.get(name)!;
-        const command: Command = {
-          ...existingCommand,
-          ...cmd,
-        };
-        if (cmd.run) {
-          command.run = cmd.run;
-        }
-        commands.set(name, command);
-      } else {
-        // New command, let's just set
-        commands.set(name, cmd);
-      }
+      this.mergeCommand(commands, name, cmd);
     }
     return commands;
   }

package/client/plugos/hooks/slash_command.ts

@@ -66,7 +66,9 @@ export class SlashCommandHook implements Hook<SlashCommandHookT> {
     const options: Completion[] = [];
 
     // No slash commands in comment blocks (queries and such) or links
-    const currentNode = syntaxTree(ctx.state).resolveInner(ctx.pos);
+    // Pass -1 so we resolve into the node ending at the cursor: when typing,
+    // the user is referring to preceding text, not what comes after.
+    const currentNode = syntaxTree(ctx.state).resolveInner(ctx.pos, -1);
     if (
       currentNode.type.name === "CommentBlock" ||
       currentNode.type.name === "Link"

package/client/plugos/plug.ts

@@ -11,6 +11,8 @@ export class Plug<HookT> {
 
   public manifest!: Manifest<HookT>;
   public assets?: AssetBundle;
+  // File path the plug was loaded from; also used as the manifest cache key.
+  public path!: string;
 
   constructor(
     readonly system: System<HookT>,
@@ -22,16 +24,17 @@ export class Plug<HookT> {
 
   static async createLazily<HookT>(
     system: System<HookT>,
-    cacheKey: string,
+    path: string,
     cacheHash: number,
     sandboxFactory: SandboxFactory<HookT>,
   ): Promise<Plug<HookT>> {
     const plug = new Plug(system, sandboxFactory);
+    plug.path = path;
 
     // Retrieve the manifest, which may either come from a cache or be loaded from the worker
     plug.manifest = await system.options.manifestCache!.getManifest(
       plug,
-      cacheKey,
+      path,
       cacheHash,
     );
 

package/client/plugos/plug_compile.ts

@@ -1,11 +1,19 @@
 import * as path from "node:path";
-import { readFile, writeFile, mkdtemp, rm, mkdir } from "node:fs/promises";
+import {
+  copyFile,
+  mkdir,
+  mkdtemp,
+  readFile,
+  rm,
+  writeFile,
+} from "node:fs/promises";
 import { tmpdir } from "node:os";
 import * as YAML from "js-yaml";
 
 import * as esbuild from "esbuild";
+import * as sass from "sass";
 import { bundleAssets } from "../asset_bundle/builder.ts";
-import type { Manifest } from "./types.ts";
+import type { BuildStep, Manifest } from "./types.ts";
 
 // When bundled by build_plug_compile.ts, this is replaced with the pre-bundled
 // worker runtime JS. When running from source, it's undefined and esbuild
@@ -53,6 +61,15 @@ export async function compileManifest(
     throw new Error(`Missing 'name' in ${manifestPath}`);
   }
 
+  // Build steps (run before asset bundling so produced files can be picked
+  // up by the `assets` glob).
+  if (manifest.build) {
+    await Promise.all(
+      manifest.build.map((step) => runBuildStep(step, rootPath, options)),
+    );
+    delete manifest.build;
+  }
+
   // Assets
   const assetsBundle = await bundleAssets(
     path.resolve(rootPath),
@@ -177,6 +194,39 @@ export async function compileManifests(
   await buildAll();
 }
 
+async function runBuildStep(
+  step: BuildStep,
+  rootPath: string,
+  options: CompileOptions,
+) {
+  const type = step.type ?? "esbuild";
+  const inFile = path.resolve(rootPath, step.in);
+  const outFile = path.resolve(rootPath, step.out);
+  await mkdir(path.dirname(outFile), { recursive: true });
+  if (type === "esbuild") {
+    await esbuild.build({
+      entryPoints: [inFile],
+      bundle: true,
+      format: "iife",
+      platform: "browser",
+      minify: !options.debug,
+      outfile: outFile,
+      treeShaking: true,
+    });
+  } else if (type === "sass") {
+    const scss = await readFile(inFile, "utf-8");
+    const result = sass.compileString(scss, {
+      loadPaths: [path.dirname(inFile)],
+      style: options.debug ? "expanded" : "compressed",
+    });
+    await writeFile(outFile, result.css, "utf-8");
+  } else if (type === "copy") {
+    await copyFile(inFile, outFile);
+  } else {
+    throw new Error(`Unsupported build step type: ${type}`);
+  }
+}
+
 export function patchBundledJS(code: string): string {
   // One bundled dependency has a lookbehind regex that WebKit can't parse; replace it with a no-op
   return code.replaceAll("/(?<=\\n)/", "/()/");

package/client/plugos/sandboxes/worker_sandbox.ts

@@ -2,8 +2,9 @@ import type { Manifest } from "../types.ts";
 import type { ControllerMessage, WorkerMessage } from "../protocol.ts";
 import type { Plug } from "../plug.ts";
 import { AssetBundle, type AssetJson } from "../../asset_bundle/bundle.ts";
-import type { Sandbox } from "./sandbox.ts";
+import type { Sandbox, SandboxFactory } from "./sandbox.ts";
 import { race, timeout } from "@silverbulletmd/silverbullet/lib/async";
+import { fsEndpoint } from "../../spaces/constants.ts";
 
 /**
  * Represents a "safe" execution environment for plug code
@@ -24,6 +25,15 @@ export class WorkerSandbox<HookT> implements Sandbox<HookT> {
     private workerOptions = {},
   ) {}
 
+  /** Factory producing a WorkerSandbox for a plug file at a local space path. */
+  static forPath<HookT>(path: string): SandboxFactory<HookT> {
+    return (plug) =>
+      new WorkerSandbox(
+        plug,
+        new URL(path, `${document.baseURI.slice(0, -1) + fsEndpoint}/`),
+      );
+  }
+
   /**
    * Should only invoked lazily (either by invoke, or by a ManifestCache to load the manifest)
    */

package/client/plugos/syscalls/config.ts

@@ -1,6 +1,7 @@
 import type { SysCallMapping } from "../system.ts";
 import type { Config } from "../../config.ts";
 import { LuaStackFrame, luaValueToJS } from "../../space_lua/runtime.ts";
+import { stripFunctions } from "../util.ts";
 
 export function configSyscalls(config: Config): SysCallMapping {
   return {
@@ -32,11 +33,17 @@ export function configSyscalls(config: Config): SysCallMapping {
     "config.define": (_ctx, key: string, schema: any) => {
       config.define(key, schema);
     },
+    "config.defineCategory": (_ctx, definition: any) => {
+      config.defineCategory(definition);
+    },
     "config.getValues": () => {
-      return config.values;
+      return stripFunctions(config.values);
     },
     "config.getSchemas": () => {
-      return config.schemas;
+      return stripFunctions(config.schemas);
+    },
+    "config.getCategories": () => {
+      return { ...config.categories };
     },
   };
 }

package/client/plugos/syscalls/editor.ts

@@ -7,15 +7,59 @@ import {
   unfoldCode,
 } from "@codemirror/language";
 import {
+  cursorCharLeft,
+  cursorCharRight,
+  cursorDocEnd,
+  cursorDocStart,
+  cursorGroupLeft,
+  cursorGroupRight,
+  cursorLineBoundaryLeft,
+  cursorLineBoundaryRight,
+  cursorLineDown,
+  cursorLineEnd,
+  cursorLineStart,
+  cursorLineUp,
+  cursorPageDown,
+  cursorPageUp,
+  deleteCharBackward,
+  deleteCharForward,
+  deleteGroupBackward,
+  deleteGroupForward,
   deleteLine,
+  deleteLineBoundaryBackward,
+  deleteLineBoundaryForward,
+  indentLess,
+  indentMore,
   insertNewline,
   insertNewlineAndIndent,
   moveLineDown,
   moveLineUp,
   redo,
+  selectAll,
+  selectCharLeft,
+  selectCharRight,
+  selectDocEnd,
+  selectDocStart,
+  selectGroupLeft,
+  selectGroupRight,
+  selectLineBoundaryLeft,
+  selectLineBoundaryRight,
+  selectLineDown,
+  selectLineEnd,
+  selectLineStart,
+  selectLineUp,
+  selectPageDown,
+  selectPageUp,
   toggleComment,
+  transposeChars,
   undo,
 } from "@codemirror/commands";
+import {
+  acceptCompletion,
+  closeCompletion,
+  moveCompletionSelection,
+  startCompletion,
+} from "@codemirror/autocomplete";
 import type { Transaction } from "@codemirror/state";
 import { EditorView } from "@codemirror/view";
 import { getVimModule } from "../../vim_loader.ts";
@@ -324,6 +368,9 @@ export function editorSyscalls(client: Client): SysCallMapping {
         client.editorView.dispatch({});
       });
     },
+    "editor.focus": () => {
+      client.focus();
+    },
     "editor.hidePanel": (_ctx, id: string) => {
       client.ui.viewDispatch({
         type: "hide-panel",
@@ -600,6 +647,24 @@ export function editorSyscalls(client: Client): SysCallMapping {
     "editor.deleteLine": () => {
       deleteLine(client.editorView);
     },
+    "editor.selectAll": () => {
+      return selectAll({
+        state: client.editorView.state,
+        dispatch: client.editorView.dispatch,
+      });
+    },
+    "editor.indentMore": () => {
+      return indentMore({
+        state: client.editorView.state,
+        dispatch: client.editorView.dispatch,
+      });
+    },
+    "editor.indentLess": () => {
+      return indentLess({
+        state: client.editorView.state,
+        dispatch: client.editorView.dispatch,
+      });
+    },
     "editor.toggleComment": () => {
       return toggleComment({
         state: client.editorView.state,
@@ -640,6 +705,77 @@ export function editorSyscalls(client: Client): SysCallMapping {
     "editor.redo": () => {
       return redo(client.editorView);
     },
+    // Cursor motion syscalls
+    "editor.cursorCharLeft": () => cursorCharLeft(client.editorView),
+    "editor.cursorCharRight": () => cursorCharRight(client.editorView),
+    "editor.cursorGroupLeft": () => cursorGroupLeft(client.editorView),
+    "editor.cursorGroupRight": () => cursorGroupRight(client.editorView),
+    "editor.cursorLineBoundaryLeft": () =>
+      cursorLineBoundaryLeft(client.editorView),
+    "editor.cursorLineBoundaryRight": () =>
+      cursorLineBoundaryRight(client.editorView),
+    "editor.cursorLineStart": () => cursorLineStart(client.editorView),
+    "editor.cursorLineEnd": () => cursorLineEnd(client.editorView),
+    "editor.cursorDocStart": () => cursorDocStart(client.editorView),
+    "editor.cursorDocEnd": () => cursorDocEnd(client.editorView),
+    // Cursor motions that also navigate the completion popup if it's open
+    "editor.cursorLineUp": () => {
+      const view = client.editorView;
+      if (moveCompletionSelection(false)(view)) return true;
+      return cursorLineUp(view);
+    },
+    "editor.cursorLineDown": () => {
+      const view = client.editorView;
+      if (moveCompletionSelection(true)(view)) return true;
+      return cursorLineDown(view);
+    },
+    "editor.cursorPageUp": () => {
+      const view = client.editorView;
+      if (moveCompletionSelection(false, "page")(view)) return true;
+      return cursorPageUp(view);
+    },
+    "editor.cursorPageDown": () => {
+      const view = client.editorView;
+      if (moveCompletionSelection(true, "page")(view)) return true;
+      return cursorPageDown(view);
+    },
+    // Selection-extending motions
+    "editor.selectCharLeft": () => selectCharLeft(client.editorView),
+    "editor.selectCharRight": () => selectCharRight(client.editorView),
+    "editor.selectGroupLeft": () => selectGroupLeft(client.editorView),
+    "editor.selectGroupRight": () => selectGroupRight(client.editorView),
+    "editor.selectLineBoundaryLeft": () =>
+      selectLineBoundaryLeft(client.editorView),
+    "editor.selectLineBoundaryRight": () =>
+      selectLineBoundaryRight(client.editorView),
+    "editor.selectLineStart": () => selectLineStart(client.editorView),
+    "editor.selectLineEnd": () => selectLineEnd(client.editorView),
+    "editor.selectDocStart": () => selectDocStart(client.editorView),
+    "editor.selectDocEnd": () => selectDocEnd(client.editorView),
+    "editor.selectLineUp": () => selectLineUp(client.editorView),
+    "editor.selectLineDown": () => selectLineDown(client.editorView),
+    "editor.selectPageUp": () => selectPageUp(client.editorView),
+    "editor.selectPageDown": () => selectPageDown(client.editorView),
+    // Delete
+    "editor.deleteCharBackward": () => deleteCharBackward(client.editorView),
+    "editor.deleteCharForward": () => deleteCharForward(client.editorView),
+    "editor.deleteGroupBackward": () => deleteGroupBackward(client.editorView),
+    "editor.deleteGroupForward": () => deleteGroupForward(client.editorView),
+    "editor.deleteLineBoundaryBackward": () =>
+      deleteLineBoundaryBackward(client.editorView),
+    "editor.deleteLineBoundaryForward": () =>
+      deleteLineBoundaryForward(client.editorView),
+    "editor.transposeChars": () => transposeChars(client.editorView),
+    // Enter: accept completion if popup is open, else newline-and-indent
+    "editor.insertNewline": () => {
+      const view = client.editorView;
+      if (acceptCompletion(view)) return true;
+      return insertNewlineAndIndent(view);
+    },
+    // Completion popup control
+    "editor.acceptCompletion": () => acceptCompletion(client.editorView),
+    "editor.startCompletion": () => startCompletion(client.editorView),
+    "editor.closeCompletion": () => closeCompletion(client.editorView),
     "editor.openSearchPanel": () => {
       openSearchPanel(client.editorView);
     },

package/client/plugos/syscalls/index.ts

@@ -67,7 +67,7 @@ export function indexSyscalls(
       tag: string,
       ref: string,
     ): Promise<ObjectValue | undefined> => {
-      return objectIndex.getObjectByRef(ref, page, tag);
+      return objectIndex.getObjectByRef(page, tag, ref);
     },
     "index.queryLuaObjects": (
       _ctx,

package/client/plugos/syscalls/jsonschema.ts

@@ -1,5 +1,6 @@
 import type { SysCallMapping } from "../system.ts";
 import { type OutputUnit, Validator, format } from "@cfworker/json-schema";
+import { stripFunctions } from "../util.ts";
 
 // Register custom formats
 format.email = (data: string) => data.includes("@");
@@ -8,25 +9,6 @@ format["page-ref"] = (data: string) =>
 
 const schemaCache = new Map<string, Validator>();
 
-/**
- * Deep-clone a value, replacing any functions with null.
- * JSON schema can't validate functions, so we strip them before validation.
- */
-function stripFunctions(value: any): any {
-  if (typeof value === "function") return null;
-  if (value === null || value === undefined || typeof value !== "object") {
-    return value;
-  }
-  if (Array.isArray(value)) {
-    return value.map(stripFunctions);
-  }
-  const result: Record<string, any> = {};
-  for (const key of Object.keys(value)) {
-    result[key] = stripFunctions(value[key]);
-  }
-  return result;
-}
-
 function formatErrors(errors: OutputUnit[]): string {
   // Filter out "properties" wrapper errors, keep only the specific leaf errors
   const leafErrors = errors.filter((e) => e.keyword !== "properties");

package/client/plugos/syscalls/system.ts

@@ -60,6 +60,11 @@ export function systemSyscalls(
           mac: def.mac,
           hide: def.hide,
           requireMode: def.requireMode,
+          requireEditor: def.requireEditor,
+          menu: def.menu,
+          menuMac: def.menuMac,
+          menuWindows: def.menuWindows,
+          menuLinux: def.menuLinux,
         };
       }
       return allCommands;
@@ -82,6 +87,14 @@ export function systemSyscalls(
       }
       return client.loadPlugs();
     },
+    "system.loadPlug": async (_ctx, path: string) => {
+      const meta = await client.space.spacePrimitives.getFileMeta(path);
+      await client.clientSystem.loadPlugFromPath(path, meta.lastModified);
+      await client.dispatchAppEvent("plugs:loaded");
+    },
+    "system.unloadPlug": (_ctx, path: string) => {
+      return client.clientSystem.system.unloadByPath(path);
+    },
     "system.reloadConfig": (): Record<string, any> => {
       console.warn("system.reloadConfig is deprecated, it's now a no-op");
       return client.config.values;

package/client/plugos/system.ts

@@ -133,26 +133,20 @@ export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
   }
 
   /**
-   * @param cacheKey Used to cache the manifest of the worker. Should be equal to the filepath
+   * @param path The plug file's space path; also used as the manifest cache key.
    * @param cacheHash Used to determine if the manifest is up to date. Should be `lastModified`
    */
   async loadPlug(
     sandboxFactory: SandboxFactory<HookT>,
-    cacheKey: string,
+    path: string,
     cacheHash: number = -1,
   ): Promise<Plug<HookT>> {
-    const plug = await Plug.createLazily(
-      this,
-      cacheKey,
-      cacheHash,
-      sandboxFactory,
-    );
+    const plug = await Plug.createLazily(this, path, cacheHash, sandboxFactory);
 
     const manifest = plug.manifest;
 
-    // This depends on cacheKey being the file path
     if (
-      !builtinPlugPaths.includes(cacheKey) &&
+      !builtinPlugPaths.includes(path) &&
       builtinPlugNames.includes(manifest.name)
     ) {
       plug.stop();
@@ -177,6 +171,16 @@ export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
     return plug;
   }
 
+  unloadByPath(path: string): boolean {
+    for (const [name, plug] of this.plugs) {
+      if (plug.path === path) {
+        this.unload(name);
+        return true;
+      }
+    }
+    return false;
+  }
+
   unload(name: string) {
     const plug = this.plugs.get(name);
     if (!plug) {

package/client/plugos/types.ts

@@ -34,6 +34,31 @@ export interface Manifest<HookT> {
    * A map of configuration options for the plug (to be merged with the system configuration).
    */
   config?: any;
+
+  /** Build steps run before asset bundling. Each step transforms an `in` file
+   * into an `out` file, which is then picked up by the `assets` glob.
+   *
+   * Currently only `type: "esbuild"` is supported (the default when omitted),
+   * which runs esbuild in bundle mode on `in` and writes the result to `out`.
+   */
+  build?: BuildStep[];
+}
+
+/** A pre-asset-bundling transformation, e.g. bundling a TypeScript entry point
+ * into a single JS file that is then loaded from plug assets.
+ *
+ * Supported types:
+ *   - `esbuild` (default): bundle `in` into `out` as a minified browser IIFE.
+ *   - `sass`: compile an SCSS/SASS file at `in` to CSS at `out`.
+ *   - `copy`: copy `in` verbatim to `out`.
+ */
+export interface BuildStep {
+  /** Transformation type. Defaults to "esbuild" when omitted. */
+  type?: "esbuild" | "sass" | "copy";
+  /** Input path relative to the manifest. */
+  in: string;
+  /** Output path relative to the manifest. */
+  out: string;
 }
 
 /** Associates hooks with a function. This is the generic base structure, that identifies the function. Hooks are defined by the type parameter. */

package/client/plugos/util.ts

@@ -1,3 +1,23 @@
+/**
+ * Deep-clone a value, replacing any functions with null.
+ * Needed because values containing functions can't be serialized
+ * via postMessage or validated with JSON Schema.
+ */
+export function stripFunctions(value: any): any {
+  if (typeof value === "function") return null;
+  if (value === null || value === undefined || typeof value !== "object") {
+    return value;
+  }
+  if (Array.isArray(value)) {
+    return value.map(stripFunctions);
+  }
+  const result: Record<string, any> = {};
+  for (const key of Object.keys(value)) {
+    result[key] = stripFunctions(value[key]);
+  }
+  return result;
+}
+
 /**
  * Checks if an object is sendable across the plugos worker boundary.
  *

package/client/space_lua/aggregates.ts

@@ -1,5 +1,5 @@
 /**
- * Aggregate function definitions and execution for LIQ.
+ * Aggregate function definitions and execution for SLIQ.
  *
  * Builtins implement ILuaFunction via plain objects rather than
  * LuaBuiltinFunction instances.  This avoids ES module TDZ issues:
@@ -16,7 +16,7 @@ import {
   luaValueToJS,
   type LuaValue,
 } from "./runtime.ts";
-import { isSqlNull } from "./liq_null.ts";
+import { isSqlNull } from "./sliq_null.ts";
 import type { LuaExpression, LuaOrderBy } from "./ast.ts";
 import { buildItemEnv } from "./query_env.ts";
 import { asyncMergeSort } from "./util.ts";