@silverbulletmd/silverbullet 2.4.1

package/LICENSE.md

@@ -0,0 +1,18 @@
+Copyright 2022, Zef Hemel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/README.md

@@ -0,0 +1,98 @@
+![GitHub Repo stars](https://img.shields.io/github/stars/silverbulletmd/silverbullet)
+![Docker Pulls](https://img.shields.io/docker/pulls/zefhemel/silverbullet)
+![GitHub Downloads (all assets, all releases)](https://img.shields.io/github/downloads/silverbulletmd/silverbullet/total)
+![GitHub contributors](https://img.shields.io/github/contributors/silverbulletmd/silverbullet)
+
+# SilverBullet
+SilverBullet is a Programmable, Private, Browser-based, Open Source, Self Hosted, Personal Knowledge Management Platform.
+
+_Yowza!_ That surely is a lot of adjectives to describe a browser-based Markdown editor programmable with Lua.
+
+Let’s get more specific.
+
+In SilverBullet you keep your content as a collection of Markdown Pages (called a Space). You navigate your space using the Page Picker like a traditional notes app, or through Links like a wiki (except they are bi-directional).
+
+If you are the **writer** type, you’ll appreciate SilverBullet as a clean Markdown editor with Live Preview. If you have more of an **outliner** personality, SilverBullet has Outlining tools for you. Productivity freak? Have a look at Tasks. More of a **database** person? You will appreciate Objects and Queries. 
+
+And if you are comfortable **programming** a little bit — now we’re really talking. You will love _dynamically generating content_ with Space Lua (SilverBullet’s Lua dialect), or to use it to create custom Commands, Page Templates or Widgets.
+
+[Much more detail can be found on silverbullet.md](https://silverbullet.md)
+
+## Installing SilverBullet
+Check out the [instructions](https://silverbullet.md/Install).
+
+## Developing SilverBullet
+SilverBullet's frontend is written in [TypeScript](https://www.typescriptlang.org/) and built on top of the excellent [CodeMirror 6](https://codemirror.net/) editor component. Additional UI is built using [Preact](https://preactjs.com). [ESBuild](https://esbuild.github.io)) running through Deno is used to build both the front-end.
+
+The server backend is written in Go.
+
+If you're considering contributing changes, be aware of the [LLM use policy](https://silverbullet.md/LLM%20Use).
+
+## Code structure
+* `client/`: The SilverBullet client, implemented with TypeScript
+* `server/`: The SilverBullet server, written in Go
+* `plugs`: Set of built-in plugs that are distributed with SilverBullet
+* `libraries`: A set of libraries (space scripts, page templates, slash templates) distributed with SilverBullet
+* `plug-api/`: Useful APIs for use in plugs
+  * `lib/`: Useful libraries to be used in plugs
+  * `syscalls/`: TypeScript wrappers around syscalls
+  * `types/`: Various (client) types that can be references from plugs
+* `bin`
+  * `plug_compile.ts` the plug compiler
+* `scripts/`: Useful scripts
+* `website/`: silverbullet.md website content
+
+### Requirements
+* [Deno](https://deno.com/): Used to build the frontend and plugs
+* [Go](https://go.dev/): Used to build the backend
+
+It's convenient to also install [air](https://github.com/air-verse/air) for development, this will automatically rebuild both the frontend and backend when changes are made:
+
+```shell
+go install github.com/air-verse/air@latest
+```
+Make sure your `$GOPATH/bin` is in your $PATH.
+
+To build everything and run the server:
+
+```shell
+air <PATH-TO-YOUR-SPACE>
+```
+
+Alternatively, to build:
+
+```shell
+make
+```
+
+To run the resulting server:
+
+```shell
+./silverbullet <PATH-TO-YOUR-SPACE>
+```
+
+### Useful development tasks
+
+```shell
+# Clean all generated files
+make clean
+# Typecheck and lint all code
+make check
+# Format all code
+make fmt
+# Run all tests
+make test
+```
+
+### Build a docker container
+Note, you do not need Deno nor Go locally installed for this to work:
+
+```shell
+docker build -t silverbullet .
+```
+
+To run:
+
+```shell
+docker run -p 3000:3000 -v <PATH-TO-YOUR-SPACE>:/space silverbullet
+```

package/client/asset_bundle/bundle.ts

@@ -0,0 +1,95 @@
+import {
+  base64Decode,
+  base64EncodedDataUrl,
+} from "../../plug-api/lib/crypto.ts";
+
+type DataUrl = string;
+
+// Mapping from path -> `data:mimetype;base64,base64-encoded-data` strings
+export type AssetJson = Record<string, { data: DataUrl; mtime: number }>;
+
+export class AssetBundle {
+  readonly bundle: AssetJson;
+
+  constructor(bundle: AssetJson = {}) {
+    this.bundle = bundle;
+  }
+
+  has(path: string): boolean {
+    return path in this.bundle;
+  }
+
+  listFiles(): string[] {
+    return Object.keys(this.bundle);
+  }
+
+  readFileSync(
+    path: string,
+  ): Uint8Array {
+    const content = this.bundle[path];
+    if (!content) {
+      throw new Error(`No such file ${path}`);
+    }
+    const data = content.data.split(",", 2)[1];
+    return base64Decode(data);
+  }
+
+  readFileAsDataUrl(path: string): string {
+    const content = this.bundle[path];
+    if (!content) {
+      throw new Error(`No such file ${path}`);
+    }
+    return content.data;
+  }
+
+  readTextFileSync(
+    path: string,
+  ): string {
+    return new TextDecoder().decode(this.readFileSync(path));
+  }
+
+  getMimeType(
+    path: string,
+  ): string {
+    const entry = this.bundle[path];
+    if (!entry) {
+      throw new Error(`No such file ${path}`);
+    }
+    return entry.data.split(";")[0].split(":")[1];
+  }
+
+  getMtime(path: string): number {
+    const entry = this.bundle[path];
+    if (!entry) {
+      throw new Error(`No such file ${path}`);
+    }
+    return entry.mtime;
+  }
+
+  writeFileSync(
+    path: string,
+    mimeType: string,
+    data: Uint8Array,
+    mtime: number = Date.now(),
+  ) {
+    // Replace \ with / for windows
+    path = path.replaceAll("\\", "/");
+    this.bundle[path] = {
+      data: base64EncodedDataUrl(mimeType, data),
+      mtime,
+    };
+  }
+
+  writeTextFileSync(
+    path: string,
+    mimeType: string,
+    s: string,
+    mtime: number = Date.now(),
+  ) {
+    this.writeFileSync(path, mimeType, new TextEncoder().encode(s), mtime);
+  }
+
+  toJSON(): AssetJson {
+    return this.bundle;
+  }
+}

package/client/data/datastore.ts

@@ -0,0 +1,85 @@
+import {
+  type LuaCollectionQuery,
+  queryLua,
+} from "../space_lua/query_collection.ts";
+import { LuaEnv, LuaStackFrame } from "../space_lua/runtime.ts";
+import type { KvPrimitives, KvQueryOptions } from "./kv_primitives.ts";
+
+import type { KV, KvKey } from "../../plug-api/types/datastore.ts";
+
+/**
+ * This is the data store class you'll actually want to use, wrapping the primitives
+ * in a more user-friendly way
+ */
+export class DataStore {
+  constructor(
+    readonly kv: KvPrimitives,
+  ) {
+  }
+
+  async get<T = any>(key: KvKey): Promise<T | null> {
+    return (await this.batchGet([key]))[0];
+  }
+
+  batchGet<T = any>(keys: KvKey[]): Promise<(T | null)[]> {
+    if (keys.length === 0) {
+      return Promise.resolve([]);
+    }
+    return this.kv.batchGet(keys);
+  }
+
+  set(key: KvKey, value: any): Promise<void> {
+    return this.batchSet([{ key, value }]);
+  }
+
+  batchSet<T = any>(entries: KV<T>[]): Promise<void> {
+    if (entries.length === 0) {
+      return Promise.resolve();
+    }
+    const allKeyStrings = new Set<string>();
+    const uniqueEntries: KV[] = [];
+    for (const { key, value } of entries) {
+      const keyString = JSON.stringify(key);
+      if (allKeyStrings.has(keyString)) {
+        console.warn(`Duplicate key ${keyString} in batchSet, skipping`);
+      } else {
+        allKeyStrings.add(keyString);
+        uniqueEntries.push({ key, value });
+      }
+    }
+    return this.kv.batchSet(uniqueEntries);
+  }
+
+  delete(key: KvKey): Promise<void> {
+    return this.batchDelete([key]);
+  }
+
+  batchDelete(keys: KvKey[]): Promise<void> {
+    if (keys.length === 0) {
+      return Promise.resolve();
+    }
+    return this.kv.batchDelete(keys);
+  }
+
+  async batchDeletePrefix(prefix: KvKey): Promise<void> {
+    const keys: KvKey[] = [];
+    for await (const { key } of this.kv.query({ prefix })) {
+      keys.push(key);
+    }
+    return this.batchDelete(keys);
+  }
+
+  query<T = any>(options: KvQueryOptions): AsyncIterableIterator<KV<T>> {
+    return this.kv.query<T>(options);
+  }
+
+  luaQuery<T = any>(
+    prefix: KvKey,
+    query: LuaCollectionQuery,
+    env: LuaEnv = new LuaEnv(),
+    sf: LuaStackFrame = LuaStackFrame.lostFrame,
+    enricher?: (key: KvKey, item: any) => any,
+  ): Promise<T[]> {
+    return queryLua(this.kv, prefix, query, env, sf, enricher);
+  }
+}

package/client/data/kv_primitives.ts

@@ -0,0 +1,25 @@
+import type { KV, KvKey } from "../../plug-api/types/datastore.ts";
+
+export type KvQueryOptions = {
+  prefix?: KvKey;
+};
+
+export interface KvPrimitives {
+  batchGet(keys: KvKey[]): Promise<(any | undefined)[]>;
+
+  batchSet(entries: KV[]): Promise<void>;
+
+  batchDelete(keys: KvKey[]): Promise<void>;
+
+  query<T = any>(options: KvQueryOptions): AsyncIterableIterator<KV<T>>;
+
+  /**
+   * A more efficient way to do counts
+   */
+  countQuery(options: KvQueryOptions): Promise<number>;
+
+  // Completely clear all data from this datastore
+  clear(): Promise<void>;
+
+  close(): void;
+}

package/client/markdown_parser/constants.ts

@@ -0,0 +1,13 @@
+export const wikiLinkRegex =
+  /(?<leadingTrivia>!?\[\[)(?<stringRef>.*?)(?:\|(?<alias>.*?))?(?<trailingTrivia>\]\])/g;
+export const mdLinkRegex = /!?\[(?<title>[^\]]*)\]\((?<url>.+)\)/g;
+export const tagRegex =
+  /#(?:(?:\d*[^\d\s!@#$%^&*(),.?":{}|<>\\][^\s!@#$%^&*(),.?":{}|<>\\]*)|(?:<[^>\n]+>))/;
+export const nakedUrlRegex =
+  /(^https?:\/\/([-a-zA-Z0-9@:%_\+~#=]|(?:[.](?!(\s|$)))){1,256})(([-a-zA-Z0-9(@:%_\+~#?&=\/]|(?:[.,:;)](?!(\s|$))))*)/;
+export const frontmatterQuotesRegex = /["'].*["']/g;
+export const frontmatterUrlRegex = /([a-zA-Z][a-zA-Z0-9+.-]*:\/\/[^\s"']+)/g;
+export const frontmatterWikiLinkRegex =
+  /(?<leadingTrivia>!?\[\[)(?<stringRef>.*?)(?:\|(?<alias>.*?))?(?<trailingTrivia>\]\])/g;
+export const frontmatterMailtoRegex = /(mailto:[^@\s]+@[^@\s"']+)/ig;
+export const pWikiLinkRegex = new RegExp("^" + wikiLinkRegex.source); // Modified regex used only in parser

package/client/plugos/event.ts

@@ -0,0 +1,36 @@
+/**
+ * EventEmitter implementation, similar to the one used in CodeMirror
+ */
+export abstract class EventEmitter<HandlerT> {
+  private handlers: Partial<HandlerT>[] = [];
+
+  /**
+   * Subscribe to events
+   * @param handlers
+   */
+  on(handlers: Partial<HandlerT>) {
+    this.handlers.push(handlers);
+  }
+
+  /**
+   * Unsubscribe from events
+   * @param handlers
+   */
+  off(handlers: Partial<HandlerT>) {
+    this.handlers = this.handlers.filter((h) => h !== handlers);
+  }
+
+  /**
+   * Broadcast an event to all subscribers
+   * @param eventName
+   * @param args
+   */
+  async emit(eventName: keyof HandlerT, ...args: any[]): Promise<void> {
+    for (const handler of this.handlers) {
+      const fn: any = handler[eventName];
+      if (fn) {
+        await Promise.resolve(fn(...args));
+      }
+    }
+  }
+}

package/client/plugos/eventhook.ts

@@ -0,0 +1,8 @@
+import type { EventHookT } from "@silverbulletmd/silverbullet/type/manifest";
+import type { Hook } from "./types.ts";
+
+export interface EventHookI extends Hook<EventHookT> {
+  dispatchEvent(eventName: string, ...args: unknown[]): Promise<unknown[]>;
+
+  listEvents(): string[];
+}

package/client/plugos/hooks/code_widget.ts

@@ -0,0 +1,59 @@
+import type { Hook, Manifest } from "../types.ts";
+import type { System } from "../system.ts";
+import type { CodeWidgetT } from "@silverbulletmd/silverbullet/type/manifest";
+import type { CodeWidgetCallback } from "@silverbulletmd/silverbullet/type/client";
+
+export class CodeWidgetHook implements Hook<CodeWidgetT> {
+  codeWidgetCallbacks = new Map<string, CodeWidgetCallback>();
+  codeWidgetModes = new Map<string, "markdown" | "iframe">();
+
+  constructor() {
+  }
+
+  collectAllCodeWidgets(system: System<CodeWidgetT>) {
+    this.codeWidgetCallbacks.clear();
+    for (const plug of system.loadedPlugs.values()) {
+      for (
+        const [name, functionDef] of Object.entries(
+          plug.manifest!.functions,
+        )
+      ) {
+        if (!functionDef.codeWidget) {
+          continue;
+        }
+        this.codeWidgetModes.set(
+          functionDef.codeWidget,
+          functionDef.renderMode || "iframe",
+        );
+        this.codeWidgetCallbacks.set(
+          functionDef.codeWidget,
+          (bodyText, pageName) => {
+            return plug.invoke(name, [bodyText, pageName]);
+          },
+        );
+      }
+    }
+  }
+
+  apply(system: System<CodeWidgetT>): void {
+    this.collectAllCodeWidgets(system);
+    system.on({
+      plugLoaded: () => {
+        this.collectAllCodeWidgets(system);
+      },
+    });
+  }
+
+  validateManifest(manifest: Manifest<CodeWidgetT>): string[] {
+    const errors = [];
+    for (const functionDef of Object.values(manifest.functions)) {
+      if (!functionDef.codeWidget) {
+        continue;
+      }
+      if (typeof functionDef.codeWidget !== "string") {
+        errors.push(`Codewidgets require a string name.`);
+      }
+    }
+    return errors;
+  }
+}

package/client/plugos/hooks/command.ts

@@ -0,0 +1,104 @@
+import type { Hook, Manifest } from "../types.ts";
+import type { System } from "../system.ts";
+import { EventEmitter } from "../event.ts";
+import { throttle } from "@silverbulletmd/silverbullet/lib/async";
+import type { Command, CommandHookEvents } from "../../types/command.ts";
+import type { CommandHookT } from "@silverbulletmd/silverbullet/type/manifest";
+
+export class CommandHook extends EventEmitter<CommandHookEvents>
+  implements Hook<CommandHookT> {
+  system?: System<CommandHookT>;
+  public throttledBuildAllCommandsAndEmit = throttle(() => {
+    this.buildAllCommandsAndEmit();
+  }, 200);
+
+  constructor(
+    private readOnly: boolean,
+    private additionalCommands: Map<string, Command>,
+  ) {
+    super();
+  }
+
+  /**
+   * Build the command map
+   */
+  buildAllCommands(): Map<string, Command> {
+    const commands = new Map<string, Command>();
+    // Add commands from plugs
+    if (!this.system) {
+      // Not initialized yet
+      return commands;
+    }
+    for (const plug of this.system.loadedPlugs.values()) {
+      for (
+        const [name, functionDef] of Object.entries(
+          plug.manifest!.functions,
+        )
+      ) {
+        if (!functionDef.command) {
+          continue;
+        }
+        const cmd = functionDef.command;
+        if (cmd.requireMode === "rw" && this.readOnly) {
+          // Bit hacky, but don't expose commands that require write mode in read-only mode
+          continue;
+        }
+        commands.set(cmd.name, {
+          ...cmd,
+          run: (args?: string[]) => {
+            return plug.invoke(name, [cmd, ...args ?? []]);
+          },
+        });
+      }
+    }
+    for (const [name, cmd] of this.additionalCommands) {
+      if (commands.has(name)) {
+        // Existing command, let's do some inline patching
+        const existingCommand = commands.get(name)!;
+        const command: Command = {
+          ...existingCommand,
+          ...cmd,
+        };
+        if (cmd.run) {
+          command.run = cmd.run;
+        }
+        commands.set(name, command);
+      } else {
+        // New command, let's just set
+        commands.set(name, cmd);
+      }
+    }
+    return commands;
+  }
+
+  public buildAllCommandsAndEmit() {
+    this.emit("commandsUpdated", this.buildAllCommands());
+  }
+
+  apply(system: System<CommandHookT>): void {
+    this.system = system;
+    system.on({
+      plugLoaded: () => {
+        this.throttledBuildAllCommandsAndEmit();
+      },
+    });
+    // On next tick
+    setTimeout(() => {
+      this.throttledBuildAllCommandsAndEmit();
+    });
+  }
+
+  validateManifest(manifest: Manifest<CommandHookT>): string[] {
+    const errors = [];
+    for (const [name, functionDef] of Object.entries(manifest.functions)) {
+      if (!functionDef.command) {
+        continue;
+      }
+      const cmd = functionDef.command;
+      if (!cmd.name) {
+        errors.push(`Function ${name} has a command but no name`);
+      }
+    }
+    return [];
+  }
+}

package/client/plugos/hooks/document_editor.ts

@@ -0,0 +1,77 @@
+import type { Hook, Manifest } from "../types.ts";
+import type { DocumentEditorT } from "@silverbulletmd/silverbullet/type/manifest";
+import type { System } from "../system.ts";
+import type { DocumentEditorCallback } from "@silverbulletmd/silverbullet/type/client";
+
+export class DocumentEditorHook implements Hook<DocumentEditorT> {
+  documentEditors = new Map<
+    string,
+    { extensions: string[]; callback: DocumentEditorCallback }
+  >();
+
+  constructor() {
+  }
+
+  collectAllDocumentEditors(system: System<DocumentEditorT>) {
+    this.documentEditors.clear();
+    for (const plug of system.loadedPlugs.values()) {
+      for (
+        const [name, functionDef] of Object.entries(
+          plug.manifest!.functions,
+        )
+      ) {
+        if (!functionDef.editor) {
+          continue;
+        }
+
+        const keys = Array.isArray(functionDef.editor)
+          ? functionDef.editor
+          : [functionDef.editor];
+
+        const conflict = Array.from(this.documentEditors.entries()).find((
+          [_, { extensions }],
+        ) => keys.some((key) => extensions.includes(key)));
+
+        if (conflict) {
+          console.log(
+            `Extension definition of document editor ${name}: [${keys}] conflicts with the one from ${
+              conflict[0]
+            }: [${conflict[1].extensions}]! Using the latter.`,
+          );
+        }
+
+        this.documentEditors.set(
+          name,
+          { extensions: keys, callback: () => plug.invoke(name, []) },
+        );
+      }
+    }
+  }
+
+  apply(system: System<DocumentEditorT>): void {
+    this.collectAllDocumentEditors(system);
+    system.on({
+      plugLoaded: () => {
+        this.collectAllDocumentEditors(system);
+      },
+    });
+  }
+
+  validateManifest(manifest: Manifest<DocumentEditorT>): string[] {
+    const errors = [];
+    for (const functionDef of Object.values(manifest.functions)) {
+      if (!functionDef.editor) {
+        continue;
+      }
+      if (
+        typeof functionDef.editor !== "string" &&
+        !Array.isArray(functionDef.editor)
+      ) {
+        errors.push(
+          `Document editors require a string name or an array of string names.`,
+        );
+      }
+    }
+    return errors;
+  }
+}