@silverbulletmd/silverbullet 2.4.1

package/package.json

@@ -0,0 +1,120 @@
+{
+  "name": "@silverbulletmd/silverbullet",
+  "version": "2.4.1",
+  "type": "module",
+  "description": "A self-hosted, web-based note taking app",
+  "publishConfig": {
+    "access": "public"
+  },
+  "exports": {
+    "./syscall": "./plug-api/syscall.ts",
+    "./syscalls": "./plug-api/syscalls.ts",
+    "./constants": "./plug-api/constants.ts",
+    "./lib/json": "./plug-api/lib/json.ts",
+    "./lib/tree": "./plug-api/lib/tree.ts",
+    "./lib/ref": "./plug-api/lib/ref.ts",
+    "./lib/resolve": "./plug-api/lib/resolve.ts",
+    "./lib/dates": "./plug-api/lib/dates.ts",
+    "./lib/async": "./plug-api/lib/async.ts",
+    "./lib/crypto": "./plug-api/lib/crypto.ts",
+    "./lib/limited_map": "./plug-api/lib/limited_map.ts",
+    "./lib/tags": "./plug-api/lib/tags.ts",
+    "./lib/transclusion": "./plug-api/lib/transclusion.ts",
+    "./lib/native_fetch": "./plug-api/lib/native_fetch.ts",
+    "./type/client": "./plug-api/types/client.ts",
+    "./type/config": "./plug-api/types/config.ts",
+    "./type/manifest": "./plug-api/types/manifest.ts",
+    "./type/namespace": "./plug-api/types/namespace.ts",
+    "./type/datastore": "./plug-api/types/datastore.ts",
+    "./type/event": "./plug-api/types/event.ts",
+    "./type/index": "./plug-api/types/index.ts"
+  },
+  "bin": {
+    "plug-compile": "dist/plug-compile.js"
+  },
+  "files": [
+    "plug-api/**/*.ts",
+    "!plug-api/**/*.test.ts",
+    "!plug-api/**/*.bench.ts",
+    "client/plugos/**/*.ts",
+    "!client/plugos/**/*.test.ts",
+    "client/asset_bundle/bundle.ts",
+    "client/data/kv_primitives.ts",
+    "client/data/datastore.ts",
+    "client/space_lua/**/*.ts",
+    "!client/space_lua/**/*.test.ts",
+    "!client/space_lua/**/*.bench.ts",
+    "client/markdown_parser/constants.ts",
+    "plugs/builtin_plugs.ts",
+    "dist/plug-compile.js"
+  ],
+  "scripts": {
+    "build": "npm run build:plugs && npm run build:client",
+    "build:plugs": "tsx build_plugs_libraries.ts",
+    "build:client": "tsx build_client.ts",
+    "build:plug-compile": "tsx build_plug_compile.ts",
+    "prepublishOnly": "npm run build:plug-compile",
+    "test": "vitest run",
+    "check": "tsc --noEmit",
+    "bench": "vitest bench"
+  },
+  "dependencies": {
+    "@codemirror/autocomplete": "6.20.0",
+    "@codemirror/commands": "6.10.1",
+    "@codemirror/lang-css": "6.3.1",
+    "@codemirror/lang-html": "6.4.11",
+    "@codemirror/lang-javascript": "6.2.4",
+    "@codemirror/lang-markdown": "6.5.0",
+    "@codemirror/language": "6.12.1",
+    "@codemirror/legacy-modes": "6.5.2",
+    "@codemirror/lint": "6.9.2",
+    "@codemirror/search": "6.6.0",
+    "@codemirror/state": "6.5.4",
+    "@codemirror/view": "6.39.11",
+    "@joplin/turndown-plugin-gfm": "1.0.64",
+    "@lezer/common": "1.5.0",
+    "@lezer/css": "1.3.0",
+    "@lezer/highlight": "1.2.3",
+    "@lezer/html": "1.3.13",
+    "@lezer/javascript": "1.5.4",
+    "@lezer/lr": "1.4.7",
+    "@lezer/markdown": "1.6.3",
+    "@msgpack/msgpack": "3.1.3",
+    "@replit/codemirror-lang-nix": "6.0.1",
+    "@replit/codemirror-vim": "6.3.0",
+    "ajv": "8.17.1",
+    "crelt": "1.0.6",
+    "fast-diff": "1.3.0",
+    "fuse.js": "7.1.0",
+    "gitignore-parser": "0.0.2",
+    "idb": "8.0.3",
+    "js-yaml": "4.1.0",
+    "mime": "4.1.0",
+    "preact": "10.28.2",
+    "preact-feather": "4.2.1",
+    "react-icons": "5.5.0",
+    "style-mod": "4.1.2",
+    "turndown": "7.2.2"
+  },
+  "devDependencies": {
+    "@preact/preset-vite": "^2.10.3",
+    "@types/gitignore-parser": "^0.0.3",
+    "@types/js-yaml": "^4.0.9",
+    "@types/node": "^22.0.0",
+    "@types/picomatch": "^4.0.2",
+    "@types/turndown": "^5.0.6",
+    "commander": "^14.0.3",
+    "esbuild": "^0.27.3",
+    "fake-indexeddb": "6.0.1",
+    "fast-glob": "^3.3.0",
+    "picomatch": "^4.0.0",
+    "sass": "^1.97.3",
+    "tsx": "^4.19.0",
+    "typescript": "^5.9.3",
+    "vite": "^7.3.1",
+    "vitest": "^4.0.18"
+  },
+  "engines": {
+    "node": ">=20.0.0"
+  }
+}

package/plug-api/constants.ts

@@ -0,0 +1,42 @@
+export const maximumDocumentSize: number = 10; // MiB
+export const defaultLinkStyle: string = "wikilink";
+export const offlineError: Error = new Error("Offline");
+export const notFoundError: Error = new Error("Not found");
+export const notAuthenticatedError: Error = new Error("Unauthenticated");
+export const wrongSpacePathError: Error = new Error(
+  "Space folder path different on server, reloading the page",
+);
+export const pingTimeout: number = 2000;
+export const pingInterval: number = 5000;
+
+/**
+ * HTTP status codes that should be treated as "offline" conditions.
+ *
+ * This is particularly useful for cases where a proxy (such as Cloudflare or other reverse proxies)
+ * indicates that the backend server is down, but there is still network connectivity between
+ * the user and the proxy. In these scenarios, we want to allow the user to continue working
+ * with their cached data rather than showing an error, even though technically there is network
+ * connectivity to the proxy.
+ *
+ * This enables SilverBullet to work in a true "offline-first" manner, falling back to cached
+ * content when the backend is unavailable through no fault of the user's network connection.
+ *
+ * All 5xx server errors are included to prevent the client from caching error HTML pages
+ * (e.g., Nginx 500 error pages) which would prevent the client from booting in offline mode.
+ */
+export const offlineStatusCodes = {
+  500: "Internal Server Error", // Server encountered an unexpected condition
+  501: "Not Implemented", // Server does not support the functionality required
+  502: "Bad Gateway", // Proxy server received invalid response from upstream server
+  503: "Service Unavailable", // Server is temporarily unable to handle the request
+  504: "Gateway Timeout", // Proxy server did not receive a timely response from upstream server
+  505: "HTTP Version Not Supported", // Server does not support the HTTP version
+  506: "Variant Also Negotiates", // Server has an internal configuration error
+  507: "Insufficient Storage", // Server is unable to store the representation
+  508: "Loop Detected", // Server detected an infinite loop while processing
+  509: "Bandwidth Limit Exceeded", // Server bandwidth limit has been exceeded
+  510: "Not Extended", // Further extensions to the request are required
+  511: "Network Authentication Required", // Client needs to authenticate to gain network access
+
+  530: "Unable to resolve origin hostname", // Served when cloudflared is down on the host
+} as const;

package/plug-api/lib/async.ts

@@ -0,0 +1,162 @@
+export function throttle(func: () => void, limit: number): () => void {
+  let timer: any = null;
+  return function () {
+    if (!timer) {
+      timer = setTimeout(() => {
+        func();
+        timer = null;
+      }, limit);
+    }
+  };
+}
+
+export function throttleImmediately(
+  func: () => void,
+  limit: number,
+): () => void {
+  let timer: any = null;
+  return function () {
+    if (!timer) {
+      func();
+      timer = setTimeout(() => {
+        timer = null;
+      }, limit);
+    }
+  };
+}
+
+// race for promises returns first promise that resolves
+export function race<T>(promises: Promise<T>[]): Promise<T> {
+  return new Promise((resolve, reject) => {
+    for (const p of promises) {
+      p.then(resolve, reject);
+    }
+  });
+}
+
+export function timeout(ms: number): Promise<never> {
+  return new Promise((_resolve, reject) =>
+    setTimeout(() => {
+      reject(new Error("timeout"));
+    }, ms)
+  );
+}
+
+export function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+export class PromiseQueue {
+  private queue: {
+    fn: () => Promise<any>;
+    resolve: (value: any) => void;
+    reject: (error: any) => void;
+  }[] = [];
+  private processing = false;
+
+  runInQueue(fn: () => Promise<any>): Promise<any> {
+    return new Promise((resolve, reject) => {
+      this.queue.push({ fn, resolve, reject });
+      if (!this.processing) {
+        this.process();
+      }
+    });
+  }
+
+  private async process(): Promise<void> {
+    if (this.queue.length === 0) {
+      this.processing = false;
+      return;
+    }
+
+    this.processing = true;
+    const { fn, resolve, reject } = this.queue.shift()!;
+
+    try {
+      const result = await fn();
+      resolve(result);
+    } catch (error) {
+      reject(error);
+    }
+
+    this.process(); // Continue processing the next promise in the queue
+  }
+}
+
+/**
+ * Batches up values, and processes in batches of batchSize in parallel
+ * then merges the results in the appropriate order.
+ * @param values - The values to batch.
+ * @param fn - The function to run on each batch.
+ * @param batchSize - The size of each batch.
+ */
+export async function batchRequests<I, O>(
+  values: I[],
+  fn: (batch: I[]) => Promise<O[]>,
+  batchSize: number,
+): Promise<O[]> {
+  const results: O[] = [];
+  // Split values into batches of batchSize
+  const batches: I[][] = [];
+  for (let i = 0; i < values.length; i += batchSize) {
+    batches.push(values.slice(i, i + batchSize));
+  }
+  // Run fn on them in parallel
+  const batchResults = await Promise.all(batches.map(fn));
+  // Flatten the results
+  for (const batchResult of batchResults) {
+    if (Array.isArray(batchResult)) { // If fn returns an array, collect them
+      results.push(...batchResult);
+    }
+  }
+  return results;
+}
+
+/**
+ * Processes items in parallel with a specified concurrency limit.
+ * @param items - The items to process.
+ * @param handler - The function to run on each item.
+ * @param concurrency - The maximum number of concurrent operations.
+ */
+export async function processWithConcurrency<I, O>(
+  items: I[],
+  handler: (item: I) => Promise<O>,
+  concurrency: number,
+): Promise<O[]> {
+  const results: O[] = [];
+  let idx = 0;
+
+  async function worker() {
+    while (idx < items.length) {
+      const currentIdx = idx++;
+      const item = items[currentIdx];
+      const result = await handler(item);
+      results[currentIdx] = result;
+    }
+  }
+
+  const workers = [];
+  for (let i = 0; i < Math.min(concurrency, items.length); i++) {
+    workers.push(worker());
+  }
+
+  await Promise.all(workers);
+  return results;
+}
+
+/**
+ * Runs a function safely by catching any errors and logging them to the console.
+ * @param fn - The function to run.
+ */
+export function safeRun(fn: () => Promise<void>): void {
+  fn().catch((e) => {
+    console.error(e);
+  });
+}
+
+/**
+ * Generates a random delay between 0 and 1000 milliseconds.
+ */
+export function jitter(maxLength = 1000): number {
+  return Math.floor(Math.random() * maxLength);
+}

package/plug-api/lib/crypto.ts

@@ -0,0 +1,202 @@
+export function base64Decode(s: string): Uint8Array {
+  const binString = atob(s);
+  const len = binString.length;
+  const bytes = new Uint8Array(len);
+  for (let i = 0; i < len; i++) {
+    bytes[i] = binString.charCodeAt(i);
+  }
+  return bytes;
+}
+
+export function base64Encode(buffer: Uint8Array | string): string {
+  if (typeof buffer === "string") {
+    buffer = new TextEncoder().encode(buffer);
+  }
+  let binary = "";
+  const len = buffer.byteLength;
+  for (let i = 0; i < len; i++) {
+    binary += String.fromCharCode(buffer[i]);
+  }
+  return btoa(binary);
+}
+
+export function base64EncodedDataUrl(
+  mimeType: string,
+  buffer: Uint8Array,
+): string {
+  return `data:${mimeType};base64,${base64Encode(buffer)}`;
+}
+
+export function base64DecodeDataUrl(dataUrl: string): Uint8Array {
+  const b64Encoded = dataUrl.split(",", 2)[1];
+  return base64Decode(b64Encoded);
+}
+
+/**
+ * Perform sha256 hash using the browser's crypto APIs
+ * Note: this will only work over HTTPS
+ * @param message
+ */
+export async function hashSHA256(
+  message: string | Uint8Array,
+): Promise<string> {
+  // Transform the string into an ArrayBuffer
+  const encoder = new TextEncoder();
+  const data: Uint8Array = typeof message === "string"
+    ? encoder.encode(message)
+    : message;
+
+  // Generate the hash
+  const hashBuffer = await globalThis.crypto.subtle.digest(
+    "SHA-256",
+    data as BufferSource,
+  );
+
+  // Transform the hash into a hex string
+  return Array.from(new Uint8Array(hashBuffer)).map((b) =>
+    b.toString(16).padStart(2, "0")
+  ).join("");
+}
+
+/**
+ * To avoid database clashes based on space folder path name, base URLs and encryption keys we derive
+ * a database name from a hash of all these combined together
+ */
+export async function deriveDbName(
+  type: "data" | "files",
+  spaceFolderPath: string,
+  baseURI: string,
+  encryptionKey?: CryptoKey,
+): Promise<string> {
+  let keyPart = "";
+  if (encryptionKey) {
+    keyPart = await exportKey(encryptionKey);
+  }
+  const spaceHash = await hashSHA256(
+    `${spaceFolderPath}:${baseURI}:${keyPart}`,
+  );
+  return `sb_${type}_${spaceHash}`;
+}
+
+// Fixed counter for AES-CTR all zeroes, for determinism
+const fixedCounter = new Uint8Array(16);
+
+export async function encryptStringDeterministic(
+  key: CryptoKey,
+  clearText: string,
+): Promise<string> {
+  const encrypted = await crypto.subtle.encrypt(
+    { name: "AES-CTR", counter: fixedCounter, length: fixedCounter.length * 8 },
+    key,
+    new TextEncoder().encode(clearText),
+  );
+  return base64Encode(new Uint8Array(encrypted));
+}
+
+export async function decryptStringDeterministic(
+  key: CryptoKey,
+  cipherText: string,
+): Promise<string> {
+  const decrypted = await crypto.subtle.decrypt(
+    { name: "AES-CTR", counter: fixedCounter, length: fixedCounter.length * 8 },
+    key,
+    base64Decode(cipherText) as BufferSource,
+  );
+  return new TextDecoder().decode(decrypted);
+}
+
+// Encrypt using AES-GCM with random IV; output = IV + ciphertext
+export async function encryptAesGcm(
+  key: CryptoKey,
+  data: Uint8Array,
+): Promise<Uint8Array> {
+  const iv = crypto.getRandomValues(new Uint8Array(12)); // 96-bit IV recommended for GCM
+  const encryptedBuffer = await crypto.subtle.encrypt(
+    { name: "AES-GCM", iv },
+    key,
+    data as BufferSource,
+  );
+  const encrypted = new Uint8Array(encryptedBuffer);
+
+  // Prepend IV to ciphertext
+  const result = new Uint8Array(iv.length + encrypted.length);
+  result.set(iv, 0);
+  result.set(encrypted, iv.length);
+  return result;
+}
+
+// Decrypt using AES-GCM assuming input format IV + ciphertext
+export async function decryptAesGcm(
+  key: CryptoKey,
+  encryptedData: Uint8Array,
+): Promise<Uint8Array> {
+  const iv = encryptedData.slice(0, 12); // extract IV (first 12 bytes)
+  const ciphertext = encryptedData.slice(12);
+  const decryptedBuffer = await crypto.subtle.decrypt(
+    { name: "AES-GCM", iv },
+    key,
+    ciphertext,
+  );
+  return new Uint8Array(decryptedBuffer);
+}
+
+export async function deriveCTRKeyFromPassword(
+  password: string,
+  salt: Uint8Array,
+): Promise<CryptoKey> {
+  // Encode password to ArrayBuffer
+  const passwordBytes = new TextEncoder().encode(password);
+
+  // Import password as a CryptoKey
+  const baseKey = await crypto.subtle.importKey(
+    "raw",
+    passwordBytes,
+    { name: "PBKDF2" },
+    false,
+    ["deriveBits", "deriveKey"],
+  );
+
+  return crypto.subtle.deriveKey(
+    {
+      name: "PBKDF2",
+      salt: salt as BufferSource,
+      iterations: 100000,
+      hash: "SHA-256",
+    },
+    baseKey,
+    {
+      name: "AES-CTR",
+      length: 256,
+    },
+    true, // extractable
+    ["encrypt", "decrypt"],
+  );
+}
+
+export function importKey(b64EncodedKey: string): Promise<CryptoKey> {
+  return crypto.subtle.importKey(
+    "raw",
+    base64Decode(b64EncodedKey) as BufferSource,
+    { name: "AES-CTR" },
+    true,
+    ["encrypt", "decrypt"],
+  );
+}
+
+export async function exportKey(ctrKey: CryptoKey): Promise<string> {
+  const key = await crypto.subtle.exportKey("raw", ctrKey);
+  return base64Encode(new Uint8Array(key));
+}
+
+export async function deriveGCMKeyFromCTR(
+  ctrKey: CryptoKey,
+): Promise<CryptoKey> {
+  const rawKey = await crypto.subtle.exportKey("raw", ctrKey);
+  return crypto.subtle.importKey(
+    "raw",
+    rawKey,
+    { name: "AES-GCM" },
+    true,
+    ["encrypt", "decrypt"],
+  );
+}

package/plug-api/lib/dates.ts

@@ -0,0 +1,13 @@
+export function niceDate(d: Date): string {
+  return localDateString(d).split("T")[0];
+}
+
+export function localDateString(d: Date): string {
+  return d.getFullYear() +
+    "-" + String(d.getMonth() + 1).padStart(2, "0") +
+    "-" + String(d.getDate()).padStart(2, "0") +
+    "T" + String(d.getHours()).padStart(2, "0") +
+    ":" + String(d.getMinutes()).padStart(2, "0") +
+    ":" + String(d.getSeconds()).padStart(2, "0") +
+    "." + String(d.getMilliseconds()).padStart(3, "0");
+}

package/plug-api/lib/json.ts

@@ -0,0 +1,136 @@
+/**
+ * Performs a deep comparison of two objects, returning true if they are equal
+ * @param a first object
+ * @param b second object
+ * @returns
+ */
+export function deepEqual(a: any, b: any): boolean {
+  if (a === b) {
+    return true;
+  }
+  if (typeof a !== typeof b) {
+    return false;
+  }
+  if (a === null || b === null) {
+    return false;
+  }
+  if (a === undefined || b === undefined) {
+    return false;
+  }
+  if (typeof a === "object") {
+    if (Array.isArray(a) && Array.isArray(b)) {
+      if (a.length !== b.length) {
+        return false;
+      }
+      for (let i = 0; i < a.length; i++) {
+        if (!deepEqual(a[i], b[i])) {
+          return false;
+        }
+      }
+      return true;
+    } else {
+      const aKeys = Object.keys(a);
+      const bKeys = Object.keys(b);
+      if (aKeys.length !== bKeys.length) {
+        return false;
+      }
+      for (const key of aKeys) {
+        if (!deepEqual(a[key], b[key])) {
+          return false;
+        }
+      }
+      return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * Converts a Date object to a date string in the format YYYY-MM-DD if it just contains a date (and no significant time), or a full ISO string otherwise
+ * @param d the date to convert
+ */
+export function cleanStringDate(d: Date): string {
+  // If no significant time, return a date string only
+  if (
+    d.getUTCHours() === 0 && d.getUTCMinutes() === 0 && d.getUTCSeconds() === 0
+  ) {
+    return d.getUTCFullYear() + "-" +
+      String(d.getUTCMonth() + 1).padStart(2, "0") + "-" +
+      String(d.getUTCDate()).padStart(2, "0");
+  } else {
+    return d.toISOString();
+  }
+}
+
+/**
+ * Processes a JSON (typically coming from parse YAML frontmatter) in two ways:
+ * 1. Expands property names in an object containing a .-separated path
+ * 2. Converts dates to strings in sensible ways
+ * @param a
+ * @returns
+ */
+export function cleanupJSON(a: any): any {
+  if (!a) {
+    return a;
+  }
+  if (typeof a !== "object") {
+    return a;
+  }
+  if (Array.isArray(a)) {
+    return a.map(cleanupJSON);
+  }
+  // If a is a date, convert to a string
+  if (a instanceof Date) {
+    return cleanStringDate(a);
+  }
+  const expanded: any = {};
+  for (const key of Object.keys(a)) {
+    const parts = key.split(".");
+    let target = expanded;
+    for (let i = 0; i < parts.length - 1; i++) {
+      const part = parts[i];
+      if (!target[part]) {
+        target[part] = {};
+      }
+      target = target[part];
+    }
+    target[parts[parts.length - 1]] = cleanupJSON(a[key]);
+  }
+  return expanded;
+}
+
+export function deepClone<T>(obj: T, ignoreKeys: string[] = []): T {
+  // Handle null, undefined, or primitive types (string, number, boolean, symbol, bigint)
+  if (obj === null || typeof obj !== "object") {
+    return obj;
+  }
+
+  // Handle Date
+  if (obj instanceof Date) {
+    return new Date(obj.getTime()) as any;
+  }
+
+  // Handle Array
+  if (Array.isArray(obj)) {
+    const arrClone: any[] = [];
+    for (let i = 0; i < obj.length; i++) {
+      arrClone[i] = deepClone(obj[i], ignoreKeys);
+    }
+    return arrClone as any;
+  }
+
+  // Handle Object
+  if (obj instanceof Object) {
+    const objClone: { [key: string]: any } = {};
+    for (const key in obj) {
+      if (ignoreKeys.includes(key)) {
+        objClone[key] = obj[key];
+      } else if (Object.prototype.hasOwnProperty.call(obj, key)) {
+        objClone[key] = deepClone(obj[key], ignoreKeys);
+      }
+    }
+    return objClone as T;
+  }
+
+  throw new Error("Unsupported data type.");
+}