@silverbulletmd/silverbullet 2.5.3 → 2.7.0

package/client/plugos/syscalls/jsonschema.ts

@@ -1,40 +1,44 @@
 import type { SysCallMapping } from "../system.ts";
-import { Ajv, type ValidateFunction } from "ajv";
+import { type OutputUnit, Validator, format } from "@cfworker/json-schema";
+import { stripFunctions } from "../util.ts";
 
-const ajv = new Ajv();
+// Register custom formats
+format.email = (data: string) => data.includes("@");
+format["page-ref"] = (data: string) =>
+  data.startsWith("[[") && data.endsWith("]]");
 
-ajv.addFormat("email", {
-  validate: (data: string) => {
-    // TODO: Implement email validation
-    return data.includes("@");
-  },
-  async: false,
-});
+const schemaCache = new Map<string, Validator>();
 
-ajv.addFormat("page-ref", {
-  validate: (data: string) => {
-    return data.startsWith("[[") && data.endsWith("]]");
-  },
-  async: false,
-});
+function formatErrors(errors: OutputUnit[]): string {
+  // Filter out "properties" wrapper errors, keep only the specific leaf errors
+  const leafErrors = errors.filter((e) => e.keyword !== "properties");
+  const errorsToUse = leafErrors.length > 0 ? leafErrors : errors;
 
-const schemaCache = new Map<string, ValidateFunction>();
+  return errorsToUse
+    .map((e) => {
+      // Convert instanceLocation from "#/foo/bar" to "foo.bar"
+      const path =
+        e.instanceLocation === "#"
+          ? ""
+          : e.instanceLocation.slice(2).replaceAll("/", ".");
+      return path ? `${path}: ${e.error}` : e.error;
+    })
+    .join(", ");
+}
 
 export function validateObject(schema: any, object: any): undefined | string {
   try {
     const schemaKey = JSON.stringify(schema);
     if (!schemaCache.has(schemaKey)) {
-      const validate = ajv.compile(schema);
-      schemaCache.set(schemaKey, validate);
+      const validator = new Validator(schema, "7");
+      schemaCache.set(schemaKey, validator);
     }
-    const validate = schemaCache.get(schemaKey)!;
-    if (validate(object)) {
+    const validator = schemaCache.get(schemaKey)!;
+    const result = validator.validate(stripFunctions(object));
+    if (result.valid) {
       return;
     } else {
-      let text = ajv.errorsText(validate.errors);
-      text = text.replaceAll("/", ".");
-      text = text.replace(/^data[.\s]/, "");
-      return text;
+      return formatErrors(result.errors);
     }
   } catch (e: any) {
     return e.message;
@@ -42,12 +46,33 @@ export function validateObject(schema: any, object: any): undefined | string {
 }
 
 export function validateSchema(schema: any): undefined | string {
-  const valid = ajv.validateSchema(schema);
-  if (valid) {
+  if (schema === null || schema === undefined) {
+    return "schema must not be null or undefined";
+  }
+  if (typeof schema === "boolean") {
     return;
-  } else {
-    return ajv.errorsText(ajv.errors);
   }
+  if (typeof schema !== "object" || Array.isArray(schema)) {
+    return "schema must be an object or boolean";
+  }
+  if (schema.type !== undefined) {
+    const validTypes = [
+      "string",
+      "number",
+      "integer",
+      "boolean",
+      "object",
+      "array",
+      "null",
+    ];
+    const types = Array.isArray(schema.type) ? schema.type : [schema.type];
+    for (const t of types) {
+      if (!validTypes.includes(t)) {
+        return `schema.type must be one of ${validTypes.join(", ")}`;
+      }
+    }
+  }
+  return;
 }
 
 export function jsonschemaSyscalls(): SysCallMapping {
@@ -59,10 +84,7 @@ export function jsonschemaSyscalls(): SysCallMapping {
     ): undefined | string => {
       return validateObject(schema, object);
     },
-    "jsonschema.validateSchema": (
-      _ctx,
-      schema: any,
-    ): undefined | string => {
+    "jsonschema.validateSchema": (_ctx, schema: any): undefined | string => {
       return validateSchema(schema);
     },
   };

package/client/plugos/syscalls/language.ts

@@ -1,23 +1,27 @@
 import type { SysCallMapping } from "../system.ts";
 import { parse } from "../../markdown_parser/parse_tree.ts";
 import type { ParseTree } from "@silverbulletmd/silverbullet/lib/tree";
-import { builtinLanguages, languageFor } from "../../languages.ts";
+import {
+  allLanguageNames,
+  languageFor,
+  loadLanguageFor,
+} from "../../languages.ts";
 
 export function languageSyscalls(): SysCallMapping {
   return {
-    "language.parseLanguage": (
+    "language.parseLanguage": async (
       _ctx,
       language: string,
       code: string,
-    ): ParseTree => {
-      const lang = languageFor(language);
+    ): Promise<ParseTree> => {
+      const lang = languageFor(language) ?? await loadLanguageFor(language);
       if (!lang) {
         throw new Error(`Unknown language ${language}`);
       }
       return parse(lang, code);
     },
     "language.listLanguages": (): string[] => {
-      return Object.keys(builtinLanguages);
+      return allLanguageNames;
     },
   };
 }

package/client/plugos/syscalls/markdown.ts

@@ -4,7 +4,7 @@ import {
   type ParseTree,
   renderToText,
 } from "@silverbulletmd/silverbullet/lib/tree";
-import { extendedMarkdownLanguage } from "../../markdown_parser/parser.ts";
+import { buildExtendedMarkdownLanguage } from "../../markdown_parser/parser.ts";
 import {
   expandMarkdown,
   type MarkdownExpandOptions,
@@ -18,11 +18,12 @@ import {
   jsonToMDTable,
   refCellTransformer,
 } from "../../markdown_renderer/result_render.ts";
+import * as TagConstants from "../../../plugs/index/constants.ts";
 
 export function markdownSyscalls(client: Client): SysCallMapping {
   return {
     "markdown.parseMarkdown": (_ctx, text: string): ParseTree => {
-      return parse(extendedMarkdownLanguage, text);
+      return parse(markdownLanguageWithUserExtensions(client), text);
     },
     "markdown.renderParseTree": (_ctx, tree: ParseTree): string => {
       return renderToText(tree);
@@ -34,7 +35,10 @@ export function markdownSyscalls(client: Client): SysCallMapping {
     ): Promise<ParseTree | string> => {
       const outputString = typeof treeOrText === "string";
       if (typeof treeOrText === "string") {
-        treeOrText = parse(extendedMarkdownLanguage, treeOrText);
+        treeOrText = parse(
+          markdownLanguageWithUserExtensions(client),
+          treeOrText,
+        );
       }
       const result = await expandMarkdownWithClient(
         client,
@@ -52,23 +56,38 @@ export function markdownSyscalls(client: Client): SysCallMapping {
       text: string,
       options: MarkdownRenderOptions = {},
     ) => {
-      let mdTree = parse(extendedMarkdownLanguage, text);
+      let mdTree = parse(markdownLanguageWithUserExtensions(client), text);
       if (options.expand) {
         mdTree = await expandMarkdownWithClient(client, mdTree);
       }
+      if (!options.resolveTagHref) {
+        options.resolveTagHref = (tagName: string) => {
+          return client.config.get<string | null>(
+            ["tags", tagName, "tagPage"],
+            null,
+          ) ?? TagConstants.tagPrefix + tagName;
+        };
+      }
       return renderMarkdownToHtml(mdTree, options);
     },
     "markdown.objectsToTable": (
       _ctx,
       data: any[],
-      options: { renderCell?: (val: any, key: string) => Promise<any> | any } =
-        {},
+      options: {
+        renderCell?: (val: any, key: string) => Promise<any> | any;
+      } = {},
     ) => {
       return jsonToMDTable(data, options.renderCell || refCellTransformer);
     },
   };
 }
 
+function markdownLanguageWithUserExtensions(client: Client) {
+  return buildExtendedMarkdownLanguage(
+    client.config.get("syntaxExtensions", {}),
+  );
+}
+
 function expandMarkdownWithClient(
   client: Client,
   tree: ParseTree,
@@ -79,6 +98,9 @@ function expandMarkdownWithClient(
     client.currentName(),
     tree,
     client.clientSystem.spaceLuaEnv,
-    options,
+    {
+      ...options,
+      syntaxExtensions: client.config.get("syntaxExtensions", {}),
+    },
   );
 }

package/client/plugos/syscalls/mq.ts

@@ -6,23 +6,15 @@ export type EventSubscription = MQListenerSpec & {
   run: (...args: any[]) => Promise<any>;
 };
 
-export function mqSyscalls(
-  mq: DataStoreMQ,
-): SysCallMapping {
+export function mqSyscalls(mq: DataStoreMQ): SysCallMapping {
   return {
     /**
      * Define a Lua event listener
      */
-    "mq.subscribe": (
-      _ctx,
-      def: MQListenerSpec,
-    ) => {
+    "mq.subscribe": (_ctx, def: MQListenerSpec) => {
       def.autoAck = def.autoAck !== false;
       // console.log("Registering Lua event listener: ", def.name);
-      client.config.insert([
-        "mqSubscriptions",
-        def.queue,
-      ], def);
+      client.config.insert(["mqSubscriptions", def.queue], def);
     },
     "mq.send": (_ctx, queue: string, body: any) => {
       return mq.send(queue, body);

package/client/plugos/syscalls/service_registry.ts

@@ -12,10 +12,7 @@ export function serviceRegistrySyscalls(
     /**
      * Define a Lua event listener
      */
-    "service.define": (
-      _ctx,
-      def: ServiceSpec,
-    ) => {
+    "service.define": (_ctx, def: ServiceSpec) => {
       return serviceRegistry.define(def);
     },
     "service.discover": (

package/client/plugos/syscalls/shell.ts

@@ -2,9 +2,7 @@ import type { SysCallMapping } from "../system.ts";
 import type { Client } from "../../client.ts";
 import { fsEndpoint } from "../../spaces/constants.ts";
 
-export function shellSyscalls(
-  client: Client,
-): SysCallMapping {
+export function shellSyscalls(client: Client): SysCallMapping {
   return {
     "shell.run": async (
       _ctx,
@@ -34,6 +32,5 @@ export function shellSyscalls(
 
 function buildShellUrl(client: Client) {
   // Strip off the /.fs and replace with /.shell
-  return client.httpSpacePrimitives.url.slice(0, -fsEndpoint.length) +
-    "/.shell";
+  return `${client.httpSpacePrimitives.url.slice(0, -fsEndpoint.length)}/.shell`;
 }

package/client/plugos/syscalls/sync.ts

@@ -1,77 +1,86 @@
 import type { SysCallMapping } from "../system.ts";
 import type { Client } from "../../client.ts";
 
-// TODO: Reimplement this
 export function syncSyscalls(client: Client): SysCallMapping {
-  return {
-    "sync.hasInitialSyncCompleted": (): boolean => {
-      return client.fullSyncCompleted;
-    },
-    "sync.performFileSync": (_ctx, path: string): Promise<void> => {
-      client.postServiceWorkerMessage({ type: "perform-file-sync", path });
-      return waitForServiceWorkerActivation(path);
-    },
-    "sync.performSpaceSync": (): Promise<number> => {
-      client.postServiceWorkerMessage({ type: "perform-space-sync" });
-      return waitForServiceWorkerActivation();
-    },
-  };
-}
+  const syncTimeoutMs = 30000;
 
-function waitForServiceWorkerActivation(path?: string): Promise<any> {
-  return new Promise<any>((resolve, reject) => {
-    client.eventHook.addLocalListener(
-      "service-worker:file-sync-complete",
-      eventHandler,
-    );
-    client.eventHook.addLocalListener(
-      "service-worker:space-sync-complete",
-      eventHandler,
-    );
-    client.eventHook.addLocalListener(
-      "service-worker:sync-error",
-      errorHandler,
-    );
-    function eventHandler(data: any) {
-      // If data.path is set, we are notified about a specific file sync -> all good, even for an individual file sync
-      // If data.path is not set, we are notified about a full space sync
-      // If we were waiting for a specific path, ignore other paths
-      if (data.path && path && data.path !== path) {
-        // Event for other file sync
-        return;
+  function waitForServiceWorkerActivation(path?: string): Promise<any> {
+    return new Promise<any>((resolve, reject) => {
+      const timeout = setTimeout(() => {
+        cleanup();
+        reject(new Error(`Sync timeout after ${syncTimeoutMs / 1000}s`));
+      }, syncTimeoutMs);
+
+      function cleanup() {
+        clearTimeout(timeout);
+        client.eventHook.removeLocalListener(
+          "service-worker:file-sync-complete",
+          eventHandler,
+        );
+        client.eventHook.removeLocalListener(
+          "service-worker:space-sync-complete",
+          eventHandler,
+        );
+        client.eventHook.removeLocalListener(
+          "service-worker:sync-error",
+          errorHandler,
+        );
       }
-      // If we were waiting for a specific path, ignore other paths
-      resolve(data);
 
-      // Unsubscribe from all these events
-      client.eventHook.removeLocalListener(
-        "service-worker:file-sync-complete",
-        eventHandler,
-      );
-      client.eventHook.removeLocalListener(
-        "service-worker:space-sync-complete",
-        eventHandler,
-      );
-      client.eventHook.removeLocalListener(
-        "service-worker:sync-error",
-        errorHandler,
-      );
-    }
-    function errorHandler(e: any) {
-      reject(e);
-      // Unsubscribe from all these events
-      client.eventHook.removeLocalListener(
+      client.eventHook.addLocalListener(
         "service-worker:file-sync-complete",
         eventHandler,
       );
-      client.eventHook.removeLocalListener(
+      client.eventHook.addLocalListener(
         "service-worker:space-sync-complete",
         eventHandler,
       );
-      client.eventHook.removeLocalListener(
+      client.eventHook.addLocalListener(
         "service-worker:sync-error",
         errorHandler,
       );
-    }
-  });
+
+      function eventHandler(data: any) {
+        if (data.path && path && data.path !== path) {
+          return;
+        }
+        cleanup();
+        resolve(data);
+      }
+
+      function errorHandler(e: any) {
+        // Only reject if the error is for our specific path, or if no path was specified
+        if (e.path && path && e.path !== path) {
+          return;
+        }
+        cleanup();
+        reject(e);
+      }
+    });
+  }
+
+  return {
+    "sync.hasInitialSyncCompleted": (): boolean => {
+      return client.fullSyncCompleted;
+    },
+    "sync.performFileSync": async (_ctx, path: string): Promise<void> => {
+      await client.postServiceWorkerMessage({
+        type: "perform-file-sync",
+        path,
+      });
+      // postServiceWorkerMessage returns silently if no SW, so only wait if SW is active
+      const registration = await navigator.serviceWorker.getRegistration();
+      if (registration?.active) {
+        return waitForServiceWorkerActivation(path);
+      }
+    },
+    "sync.performSpaceSync": async (): Promise<number> => {
+      await client.postServiceWorkerMessage({ type: "perform-space-sync" });
+      const registration = await navigator.serviceWorker.getRegistration();
+      if (registration?.active) {
+        return waitForServiceWorkerActivation();
+      }
+      return 0;
+    },
+  };
 }

package/client/plugos/syscalls/system.ts

@@ -60,15 +60,19 @@ export function systemSyscalls(
           mac: def.mac,
           hide: def.hide,
           requireMode: def.requireMode,
+          requireEditor: def.requireEditor,
+          menu: def.menu,
+          menuMac: def.menuMac,
+          menuWindows: def.menuWindows,
+          menuLinux: def.menuLinux,
         };
       }
       return allCommands;
     },
     "system.listSyscalls": (): SyscallMeta[] => {
       const syscalls: SyscallMeta[] = [];
-      for (
-        const [name, info] of client.clientSystem.system.registeredSyscalls
-      ) {
+      for (const [name, info] of client.clientSystem.system
+        .registeredSyscalls) {
         syscalls.push({
           name,
           requiredPermissions: info.requiredPermissions,
@@ -83,6 +87,14 @@ export function systemSyscalls(
       }
       return client.loadPlugs();
     },
+    "system.loadPlug": async (_ctx, path: string) => {
+      const meta = await client.space.spacePrimitives.getFileMeta(path);
+      await client.clientSystem.loadPlugFromPath(path, meta.lastModified);
+      await client.dispatchAppEvent("plugs:loaded");
+    },
+    "system.unloadPlug": (_ctx, path: string) => {
+      return client.clientSystem.system.unloadByPath(path);
+    },
     "system.reloadConfig": (): Record<string, any> => {
       console.warn("system.reloadConfig is deprecated, it's now a no-op");
       return client.config.values;

package/client/plugos/system.ts

@@ -24,9 +24,7 @@ export type SyscallContext = {
   plug?: string;
 };
 
-type SyscallSignature = (
-  ...args: any[]
-) => Promise<any> | any;
+type SyscallSignature = (...args: any[]) => Promise<any> | any;
 
 type Syscall = {
   requiredPermissions: string[];
@@ -112,11 +110,7 @@ export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
     return this.syscall({}, name, args);
   }
 
-  syscall(
-    ctx: SyscallContext,
-    name: string,
-    args: any[],
-  ): Promise<any> {
+  syscall(ctx: SyscallContext, name: string, args: any[]): Promise<any> {
     const syscall = this.registeredSyscalls.get(name);
     if (!syscall) {
       throw Error(`Unregistered syscall ${name}`);
@@ -139,26 +133,20 @@ export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
   }
 
   /**
-   * @param cacheKey Used to cache the manifest of the worker. Should be equal to the filepath
+   * @param path The plug file's space path; also used as the manifest cache key.
    * @param cacheHash Used to determine if the manifest is up to date. Should be `lastModified`
    */
   async loadPlug(
     sandboxFactory: SandboxFactory<HookT>,
-    cacheKey: string,
+    path: string,
     cacheHash: number = -1,
   ): Promise<Plug<HookT>> {
-    const plug = await Plug.createLazily(
-      this,
-      cacheKey,
-      cacheHash,
-      sandboxFactory,
-    );
+    const plug = await Plug.createLazily(this, path, cacheHash, sandboxFactory);
 
     const manifest = plug.manifest;
 
-    // This depends on cacheKey being the file path
     if (
-      !builtinPlugPaths.includes(cacheKey) &&
+      !builtinPlugPaths.includes(path) &&
       builtinPlugNames.includes(manifest.name)
     ) {
       plug.stop();
@@ -183,13 +171,23 @@ export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
     return plug;
   }
 
+  unloadByPath(path: string): boolean {
+    for (const [name, plug] of this.plugs) {
+      if (plug.path === path) {
+        this.unload(name);
+        return true;
+      }
+    }
+    return false;
+  }
+
   unload(name: string) {
     const plug = this.plugs.get(name);
     if (!plug) {
       return;
     }
-    plug.stop();
-    this.emit("plugUnloaded", name);
+    void plug.stop();
+    void this.emit("plugUnloaded", name);
     this.plugs.delete(name);
   }
 

package/client/plugos/types.ts

@@ -34,6 +34,31 @@ export interface Manifest<HookT> {
    * A map of configuration options for the plug (to be merged with the system configuration).
    */
   config?: any;
+
+  /** Build steps run before asset bundling. Each step transforms an `in` file
+   * into an `out` file, which is then picked up by the `assets` glob.
+   *
+   * Currently only `type: "esbuild"` is supported (the default when omitted),
+   * which runs esbuild in bundle mode on `in` and writes the result to `out`.
+   */
+  build?: BuildStep[];
+}
+
+/** A pre-asset-bundling transformation, e.g. bundling a TypeScript entry point
+ * into a single JS file that is then loaded from plug assets.
+ *
+ * Supported types:
+ *   - `esbuild` (default): bundle `in` into `out` as a minified browser IIFE.
+ *   - `sass`: compile an SCSS/SASS file at `in` to CSS at `out`.
+ *   - `copy`: copy `in` verbatim to `out`.
+ */
+export interface BuildStep {
+  /** Transformation type. Defaults to "esbuild" when omitted. */
+  type?: "esbuild" | "sass" | "copy";
+  /** Input path relative to the manifest. */
+  in: string;
+  /** Output path relative to the manifest. */
+  out: string;
 }
 
 /** Associates hooks with a function. This is the generic base structure, that identifies the function. Hooks are defined by the type parameter. */

package/client/plugos/util.ts

@@ -1,3 +1,23 @@
+/**
+ * Deep-clone a value, replacing any functions with null.
+ * Needed because values containing functions can't be serialized
+ * via postMessage or validated with JSON Schema.
+ */
+export function stripFunctions(value: any): any {
+  if (typeof value === "function") return null;
+  if (value === null || value === undefined || typeof value !== "object") {
+    return value;
+  }
+  if (Array.isArray(value)) {
+    return value.map(stripFunctions);
+  }
+  const result: Record<string, any> = {};
+  for (const key of Object.keys(value)) {
+    result[key] = stripFunctions(value[key]);
+  }
+  return result;
+}
+
 /**
  * Checks if an object is sendable across the plugos worker boundary.
  *

package/client/plugos/worker_runtime.ts

@@ -19,7 +19,8 @@ let workerPostMessage = (_msg: ControllerMessage): void => {
 // - in a browser's main thread, typeof window is "object"
 // - in a browser's worker threads, typeof window === "undefined"
 // - in Cloudflare workers typeof window === "undefined", but typeof globalThis.WebSocketPair is defined
-const runningAsWebWorker = typeof window === "undefined" &&
+const runningAsWebWorker =
+  typeof window === "undefined" &&
   // @ts-expect-error: globalThis
   typeof globalThis.WebSocketPair === "undefined";
 
@@ -147,8 +148,8 @@ export function monkeyPatchFetch() {
   ): Promise<Response> => {
     const encodedBody = init?.body
       ? base64Encode(
-        new Uint8Array(await (new Response(init.body)).arrayBuffer()),
-      )
+          new Uint8Array(await new Response(init.body).arrayBuffer()),
+        )
       : undefined;
     const r = await sandboxFetch(
       reqInfo,