@silverbulletmd/silverbullet 2.9.0 → 2.11.0

package/client/plugos/system.ts

@@ -2,14 +2,15 @@ import type { Hook } from "./types.ts";
 import { EventEmitter } from "./event.ts";
 import type { SandboxFactory } from "./sandboxes/sandbox.ts";
 import { Plug } from "./plug.ts";
-import { InMemoryManifestCache, type ManifestCache } from "./manifest_cache.ts";
+import type { ManifestCache } from "./manifest_cache.ts";
 import {
   builtinPlugNames,
   builtinPlugPaths,
 } from "../../plugs/builtin_plugs.ts";
+import type { LuaFunctionDocumentation } from "../../plug-api/types/index.ts";
 
 export interface SysCallMapping {
-  [key: string]: (ctx: SyscallContext, ...args: any) => Promise<any> | any;
+  [key: string]: SyscallSignature | SyscallDefinition;
 }
 
 export type SystemEvents<HookT> = {
@@ -26,10 +27,14 @@ export type SyscallContext = {
 
 type SyscallSignature = (...args: any[]) => Promise<any> | any;
 
-type Syscall = {
+export type SyscallDefinition = {
+  callback: SyscallSignature;
+} & LuaFunctionDocumentation;
+
+type RegisteredSyscall = {
   requiredPermissions: string[];
   callback: SyscallSignature;
-};
+} & LuaFunctionDocumentation;
 
 export type SystemOptions = {
   manifestCache?: ManifestCache<any>;
@@ -37,7 +42,7 @@ export type SystemOptions = {
 };
 
 export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
-  registeredSyscalls = new Map<string, Syscall>();
+  registeredSyscalls = new Map<string, RegisteredSyscall>();
   protected plugs = new Map<string, Plug<HookT>>();
   protected enabledHooks = new Set<Hook<HookT>>();
 
@@ -50,7 +55,12 @@ export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
   ) {
     super();
     if (!options.manifestCache) {
-      options.manifestCache = new InMemoryManifestCache();
+      options.manifestCache = {
+        async getManifest(plug) {
+          await plug.sandbox.init();
+          return plug.sandbox.manifest!;
+        },
+      };
     }
   }
 
@@ -68,10 +78,21 @@ export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
     ...registrationObjects: SysCallMapping[]
   ) {
     for (const registrationObject of registrationObjects) {
-      for (const [name, callback] of Object.entries(registrationObject)) {
+      for (const [name, registration] of Object.entries(registrationObject)) {
+        const definition =
+          typeof registration === "function"
+            ? { callback: registration }
+            : registration;
+        const cleanName = name.startsWith("lua:")
+          ? name.slice("lua:".length)
+          : name;
+        const namespace = cleanName.split(".")[0];
+        const { callback, ...metadata } = definition;
         this.registeredSyscalls.set(name, {
           requiredPermissions: requiredCapabilities,
           callback,
+          see: `API/${namespace}`,
+          ...metadata,
         });
       }
     }
@@ -83,7 +104,6 @@ export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
    * @param args an array of arguments to pass to the function
    */
   invokeFunction(name: string, args: any[]): Promise<any> {
-    // Some sanity type checks
     if (typeof name !== "string") {
       throw new Error(
         `invokeFunction: function name should be a string, got ${typeof name}`,
@@ -96,7 +116,6 @@ export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
     }
     const [plugName, functionName] = name.split(".");
     if (!functionName) {
-      // Sanity check
       throw new Error(`Missing function name: ${name}`);
     }
     const plug = this.loadedPlugs.get(plugName);
@@ -153,7 +172,6 @@ export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
       throw new Error("Plug tried to overwrite internal plug");
     }
 
-    // Validate the manifest
     let errors: string[] = [];
     for (const feature of this.enabledHooks) {
       errors = [...errors, ...feature.validateManifest(manifest)];

package/client/plugos/worker_runtime.ts

@@ -112,7 +112,6 @@ export function setupMessageListener(
       }
     })().catch(console.error);
   });
-  // Signal initialization with manifest
   workerPostMessage({
     type: "manifest",
     manifest,
@@ -139,7 +138,6 @@ export async function sandboxFetch(
 
 globalThis.nativeFetch = globalThis.fetch;
 
-// Monkey patch fetch()
 export function monkeyPatchFetch() {
   // @ts-expect-error: monkey patching fetch
   globalThis.fetch = async (

package/client/space_lua/aggregates.ts

@@ -112,7 +112,6 @@ function covarIterate(state: CovarState, x: any, y: any): CovarState {
   return state;
 }
 
-// Quantile interpolation methods
 type QuantileMethod =
   | "linear" // percentile_cont
   | "lower" // percentile_disc
@@ -126,7 +125,6 @@ interface QuantileState {
   method: QuantileMethod;
 }
 
-// Default method based on aggregate invocation name
 const quantileNameDefaults: Record<string, QuantileMethod> = {
   percentile_cont: "linear",
   percentile_disc: "lower",
@@ -159,7 +157,6 @@ function quantileFinish(state: QuantileState): number | null {
   }
 }
 
-// Shared spec — branching on `ctx.name` for the default method
 function makeQuantileSpec(name: string, description: string): AggregateSpec {
   return {
     name,
@@ -187,9 +184,7 @@ function makeQuantileSpec(name: string, description: string): AggregateSpec {
   };
 }
 
-// Built-in aggregate specs
 const builtinAggregates: Record<string, AggregateSpec> = {
-  // General purpose
   count: {
     name: "count",
     description:
@@ -299,7 +294,6 @@ const builtinAggregates: Record<string, AggregateSpec> = {
       return value;
     }),
   },
-  // Collection and format
   array_agg: {
     name: "array_agg",
     description: "Input values concatenated into an array",
@@ -365,7 +359,6 @@ const builtinAggregates: Record<string, AggregateSpec> = {
       return JSON.stringify(state);
     }),
   },
-  // Bitwise and boolean
   bit_and: {
     name: "bit_and",
     description: "Bitwise AND of all non-null input values",
@@ -447,7 +440,6 @@ const builtinAggregates: Record<string, AggregateSpec> = {
       return state.hasValue ? state.result : null;
     }),
   },
-  // Statistical
   stddev_pop: {
     name: "stddev_pop",
     description: "Population standard deviation of non-null inputs",
@@ -558,7 +550,6 @@ const builtinAggregates: Record<string, AggregateSpec> = {
       return state.bestCount > 0 ? state.best : null;
     }),
   },
-  // Quantile and percentile
   quantile: makeQuantileSpec(
     "quantile",
     "Quantile of ordered set of non-null inputs; arguments: value, quantile (0-1), interpolation ('lower', 'higher', 'nearest', 'midpoint' and default: 'linear')",
@@ -617,14 +608,12 @@ export function getAggregateSpec(
     const spec: any = config.get(`aggregates.${current}`, null);
     if (!spec) break;
 
-    // Check for alias redirect
     const alias = spec instanceof LuaTable ? spec.rawGet("alias") : spec.alias;
     if (typeof alias === "string") {
       current = alias;
       continue;
     }
 
-    // Full definition in config
     let candidate: AggregateSpec | null = null;
     if (spec instanceof LuaTable) {
       const init = spec.rawGet("initialize");
@@ -700,16 +689,13 @@ export async function executeAggregate(
     }
   }
 
-  // Initialize
   let state = await luaCall(spec.initialize, [ctx, ...extraArgs], noCtx, sf);
 
-  // Collect filtered items
   const filteredItems: LuaValue[] = [];
   const len = items.length;
   for (let i = 1; i <= len; i++) {
     const item = items.rawGet(i);
 
-    // Filter
     if (filterExpr) {
       const filterEnv = buildItemEnv(objectVariable, item, env, sf);
       const filterResult = await evalExprFn(filterExpr, filterEnv, sf);
@@ -747,7 +733,6 @@ export async function executeAggregate(
     });
   }
 
-  // Iterate
   for (const item of filteredItems) {
     const itemEnv = buildItemEnv(objectVariable, item, env, sf);
     let value: LuaValue;
@@ -773,7 +758,6 @@ export async function executeAggregate(
     );
   }
 
-  // Finish
   if (spec.finish) {
     state = await luaCall(spec.finish, [state, ctx, ...extraArgs], noCtx, sf);
   }

package/client/space_lua/api_documentation.ts

@@ -0,0 +1,107 @@
+import type {
+  LuaFunctionExampleDocumentation,
+  LuaFunctionInfo,
+  LuaFunctionParameterDocumentation,
+  LuaFunctionReturnDocumentation,
+} from "../../plug-api/types/index.ts";
+
+function compareNames(a: LuaFunctionInfo, b: LuaFunctionInfo): number {
+  const aName = a.name ?? "";
+  const bName = b.name ?? "";
+  return aName < bName ? -1 : aName > bName ? 1 : 0;
+}
+
+function signature(info: LuaFunctionInfo): string {
+  const parameters = (info.parameters ?? [])
+    .map((parameter) => `${parameter.name}${parameter.optional ? "?" : ""}`)
+    .join(", ");
+  return `${info.name ?? "<anonymous>"}(${parameters})`;
+}
+
+function renderParameter(parameter: LuaFunctionParameterDocumentation): string {
+  const type = parameter.type ? ` (\`${parameter.type}\`)` : "";
+  const description = parameter.description
+    ? ` — ${parameter.description}`
+    : "";
+  return `- \`${parameter.name}${parameter.optional ? "?" : ""}\`${type}${description}`;
+}
+
+function renderReturn(value: LuaFunctionReturnDocumentation): string {
+  const type = value.type ? `\`${value.type}\`` : "Value";
+  return `- ${type}${value.description ? ` — ${value.description}` : ""}`;
+}
+
+function codeFence(example: LuaFunctionExampleDocumentation): string[] {
+  const longestFence = Math.max(
+    2,
+    ...[...example.code.matchAll(/`+/g)].map((match) => match[0].length),
+  );
+  const fence = "`".repeat(longestFence + 1);
+  return [`${fence}${example.language ?? "lua"}`, example.code, fence];
+}
+
+function renderFunction(
+  info: LuaFunctionInfo,
+  apiNamespace?: string,
+): string[] {
+  const lines = [`## ${info.name ?? "<anonymous>"}`, ""];
+  const signatures = info.signatures?.length
+    ? info.signatures
+    : [signature(info)];
+  for (const value of signatures) lines.push(`\`${value}\``);
+
+  if (info.deprecated) {
+    lines.push(
+      "",
+      `> **Deprecated:** ${
+        typeof info.deprecated === "string"
+          ? info.deprecated
+          : "This function is deprecated."
+      }`,
+    );
+  }
+  if (info.description) lines.push("", info.description);
+  if (info.parameters?.length) {
+    lines.push("", "**Parameters:**", "");
+    lines.push(...info.parameters.map(renderParameter));
+  }
+  if (info.returns?.length) {
+    lines.push("", "**Returns:**", "");
+    lines.push(...info.returns.map(renderReturn));
+  }
+  if (info.examples?.length) {
+    lines.push(
+      "",
+      info.examples.length === 1 ? "**Example:**" : "**Examples:**",
+    );
+    for (const example of info.examples) {
+      if (example.description) lines.push("", example.description);
+      lines.push("", ...codeFence(example));
+    }
+  }
+  const namespacePage = apiNamespace ? `API/${apiNamespace}` : undefined;
+  if (info.see && info.see !== namespacePage) {
+    lines.push("", `**See:** [[${info.see}]]`);
+  }
+  return lines;
+}
+
+/** Render function metadata as deterministic Markdown for live `${...}` directives. */
+export function renderApiDocumentationMarkdown(
+  functions: LuaFunctionInfo[],
+  context?: string,
+): string {
+  const documented = functions.filter((info) => info.name).sort(compareNames);
+  if (documented.length === 0) {
+    return context
+      ? `_No documented API functions found for \`${context}\`._`
+      : "_No functions found._";
+  }
+  return documented
+    .flatMap((info, index) => [
+      ...(index > 0 ? [""] : []),
+      ...renderFunction(info, context),
+    ])
+    .join("\n")
+    .trim();
+}

package/client/space_lua/ast.ts

@@ -1,3 +1,5 @@
+import type { LuaFunctionDocumentation } from "../../plug-api/types/index.ts";
+
 type ASTContext = {
   ctx: ASTCtx;
 };
@@ -12,6 +14,8 @@ export type ASTCtx = {
 export type LuaBlock = {
   type: "Block";
   statements: LuaStatement[];
+  // Comments lexically owned by this block, in source order.
+  comments?: LuaComment[];
   hasLabel?: boolean;
   hasGoto?: boolean;
   // the first duplicated label is recorded here for later eval
@@ -26,7 +30,14 @@ export type LuaBlock = {
   hasFunctionDef?: boolean;
 } & ASTContext;
 
-// STATEMENTS
+export type LuaComment = {
+  type: "Comment";
+  /** Exact source text, including delimiters. */
+  text: string;
+  kind: "line" | "long";
+  ctx: ASTCtx;
+};
+
 export type LuaReturnStatement = {
   type: "Return";
   expressions: LuaExpression[];
@@ -128,6 +139,7 @@ export type LuaFunctionBody = {
   type: "FunctionBody";
   parameters: string[];
   block: LuaBlock;
+  documentation?: LuaFunctionDocumentation;
 } & ASTContext;
 
 export type LuaAssignmentStatement = {
@@ -164,7 +176,6 @@ export type LuaFunctionCallStatement = {
   call: LuaFunctionCallExpression;
 } & ASTContext;
 
-// EXPRESSIONS
 export type LuaExpression =
   | LuaNilLiteral
   | LuaBooleanLiteral
@@ -281,21 +292,18 @@ export type LuaFunctionDefinition = {
   body: LuaFunctionBody;
 } & ASTContext;
 
-// Aggregate with per-row filter
 export type LuaFilteredCallExpression = {
   type: "FilteredCall";
   call: LuaFunctionCallExpression;
   filter: LuaExpression;
 } & ASTContext;
 
-// Aggregate call with intra-aggregate order by
 export type LuaAggregateCallExpression = {
   type: "AggregateCall";
   call: LuaFunctionCallExpression;
   orderBy: LuaOrderBy[];
 } & ASTContext;
 
-// Query stuff
 export type LuaQueryExpression = {
   type: "Query";
   clauses: LuaQueryClause[];

package/client/space_lua/ast_narrow.ts

@@ -2,13 +2,11 @@
 
 import type { ASTCtx, LuaExpression, LuaLValue, LuaStatement } from "./ast.ts";
 
-// Extract by `type` discriminant
 type NarrowByType<
   U,
   K extends U extends { type: infer T } ? T : never,
 > = Extract<U, { type: K }>;
 
-// Expressions
 export const asStringExpr = (e: LuaExpression) =>
   e as NarrowByType<LuaExpression, "String">;
 export const asNumberExpr = (e: LuaExpression) =>
@@ -38,7 +36,6 @@ export const asFunctionDef = (e: LuaExpression) =>
 export const asQueryExpr = (e: LuaExpression) =>
   e as NarrowByType<LuaExpression, "Query">;
 
-// L-values
 export const asLValueVariable = (l: LuaLValue) =>
   l as Extract<LuaLValue, { type: "Variable" }>;
 export const asLValueTableAccess = (l: LuaLValue) =>
@@ -46,7 +43,6 @@ export const asLValueTableAccess = (l: LuaLValue) =>
 export const asLValuePropertyAccess = (l: LuaLValue) =>
   l as Extract<LuaLValue, { type: "PropertyAccess" }>;
 
-// Statements
 export const asAssignment = (s: LuaStatement) =>
   s as NarrowByType<LuaStatement, "Assignment">;
 export const asLocal = (s: LuaStatement) =>
@@ -79,5 +75,4 @@ export const asGoto = (s: LuaStatement) =>
 export const asSemicolon = (s: LuaStatement) =>
   s as NarrowByType<LuaStatement, "Semicolon">;
 
-// Pull ctx with a single shape
 export const ctxOf = (node: { ctx: ASTCtx }): ASTCtx => node.ctx;

package/client/space_lua/budget.ts

@@ -0,0 +1,113 @@
+export const BUDGET_TICKS = 5000;
+export const YIELD_AFTER_MS = 500;
+export const YIELD_EVERY_MS = 50;
+export const YIELD_COUNT_LIMIT = 200;
+export const BUSY_LIMIT_DEFAULT_MS = 2000;
+export const BUSY_LIMIT_COMMAND_MS = 10000;
+export const KEEP_GOING_MS = 10000;
+
+export class LuaBudgetStopped extends Error {
+  constructor() {
+    super("Script stopped by the user");
+    this.name = "LuaBudgetStopped";
+  }
+}
+
+export type LuaBudget = {
+  ticks: number;
+  lastCheck: number;
+  lastYield: number;
+  busyMs: number;
+  yieldCount: number;
+  awaited: boolean;
+  stopped: boolean;
+  /** Set once the invocation this budget was created for has completed. */
+  finished: boolean;
+  limitReported: boolean;
+  busyLimitMs: number;
+  yieldAfterMs: number;
+  onLimit: ((b: LuaBudget) => void) | undefined;
+  now: () => number;
+  doYield: () => Promise<void>;
+};
+
+export function macrotaskYield(): Promise<void> {
+  // Do NOT use scheduler.yield() here: it resumes at a *higher* priority than
+  // ordinary tasks, so it never lets timers/input/rendering run and defeats
+  // the whole point of yielding. MessageChannel posts a genuine macrotask.
+  if (typeof MessageChannel === "function") {
+    return new Promise<void>((resolve) => {
+      const ch = new MessageChannel();
+      ch.port1.onmessage = () => {
+        ch.port1.close();
+        resolve();
+      };
+      ch.port2.postMessage(undefined);
+    });
+  }
+  return new Promise<void>((resolve) => setTimeout(resolve, 0));
+}
+
+export function makeLuaBudget(
+  opts: Partial<
+    Pick<
+      LuaBudget,
+      "busyLimitMs" | "yieldAfterMs" | "onLimit" | "now" | "doYield"
+    >
+  > = {},
+): LuaBudget {
+  const now = opts.now ?? (() => performance.now());
+  return {
+    ticks: BUDGET_TICKS,
+    lastCheck: now(),
+    lastYield: now(),
+    busyMs: 0,
+    yieldCount: 0,
+    awaited: false,
+    stopped: false,
+    finished: false,
+    limitReported: false,
+    busyLimitMs: opts.busyLimitMs ?? BUSY_LIMIT_DEFAULT_MS,
+    yieldAfterMs: opts.yieldAfterMs ?? YIELD_AFTER_MS,
+    onLimit: opts.onLimit,
+    now,
+    doYield: opts.doYield ?? macrotaskYield,
+  };
+}
+
+export function budgetTick(b: LuaBudget): Promise<void> | undefined {
+  if (b.stopped) {
+    throw new LuaBudgetStopped();
+  }
+  b.ticks = BUDGET_TICKS;
+
+  const now = b.now();
+  const sinceCheck = now - b.lastCheck;
+  b.lastCheck = now;
+  if (b.awaited) {
+    b.awaited = false;
+  } else {
+    b.busyMs += sinceCheck;
+  }
+
+  if (
+    !b.limitReported &&
+    (b.busyMs >= b.busyLimitMs || b.yieldCount >= YIELD_COUNT_LIMIT)
+  ) {
+    b.limitReported = true;
+    b.onLimit?.(b);
+  }
+
+  const threshold = b.yieldCount === 0 ? b.yieldAfterMs : YIELD_EVERY_MS;
+  if (now - b.lastYield < threshold) {
+    return undefined;
+  }
+  b.lastYield = now;
+  b.yieldCount++;
+  // The event-loop turn spent yielding (rendering, sync, other scripts) must
+  // not be billed to this script's busyMs at the next tick, so re-read the
+  // clock on resume rather than trusting the pre-yield lastCheck.
+  return b.doYield().then(() => {
+    b.lastCheck = b.now();
+  });
+}

package/client/space_lua/budget_ui.ts

@@ -0,0 +1,49 @@
+import { KEEP_GOING_MS, type LuaBudget } from "./budget.ts";
+
+export type BudgetNotifier = {
+  flashNotification(
+    message: string,
+    type?: string,
+    options?: {
+      timeout?: number;
+      actions?: { name: string; run: () => void }[];
+    },
+  ): void;
+};
+
+export function offerStopNotification(
+  ui: BudgetNotifier,
+  message: string,
+  budget: LuaBudget,
+  onStop?: () => void,
+): void {
+  ui.flashNotification(message, "warning", {
+    timeout: 0,
+    actions: [
+      {
+        name: "Stop",
+        run: () => {
+          // The script may have finished running (this toast is persistent)
+          // between the limit tripping and the user clicking Stop; don't
+          // quarantine a script that isn't running anymore.
+          if (budget.finished) {
+            return;
+          }
+          budget.stopped = true;
+          onStop?.();
+        },
+      },
+      {
+        name: "Keep going",
+        run: () => {
+          budget.busyLimitMs = budget.busyMs + KEEP_GOING_MS;
+          // Resetting to 0 would restore the 500ms first-yield threshold,
+          // making the tab unresponsive again right after the user asked
+          // to continue.
+          budget.yieldCount = 1;
+          budget.limitReported = false;
+        },
+      },
+    ],
+  });
+}