@silverbulletmd/silverbullet 2.4.1

package/client/plugos/syscalls/fetch.ts

@@ -0,0 +1,128 @@
+import type { SysCallMapping } from "../system.ts";
+import type {
+  ProxyFetchRequest,
+  ProxyFetchRequest64,
+  ProxyFetchResponse,
+  ProxyFetchResponse64,
+} from "../proxy_fetch.ts";
+import type { Client } from "../../client.ts";
+import {
+  base64Decode,
+  base64Encode,
+} from "@silverbulletmd/silverbullet/lib/crypto";
+import { fsEndpoint } from "../../spaces/constants.ts";
+
+export function sandboxFetchSyscalls(
+  client: Client,
+): SysCallMapping {
+  return {
+    // For use in Lua
+    "http.request": async (
+      _ctx,
+      url: string,
+      options: ProxyFetchRequest = {},
+    ): Promise<ProxyFetchResponse> => {
+      console.warn("Deprecated: use net.proxyFetch() instead");
+      // JSONify any non-serializable body
+      if (
+        options?.body && typeof options.body !== "string" &&
+        !(options.body instanceof Uint8Array)
+      ) {
+        options.body = JSON.stringify(options.body);
+      }
+      const fetchOptions = options
+        ? {
+          method: options.method,
+          headers: {} as Record<string, string>,
+          body: options.body,
+        }
+        : {};
+
+      fetchOptions.headers = buildProxyHeaders(options?.headers);
+
+      const resp = await client.httpSpacePrimitives.authenticatedFetch(
+        buildProxyUrl(client, url),
+        fetchOptions,
+      );
+      // Do sensible things with the body based on the content type
+      let body: any;
+      const contentTypeHeader = options.responseEncoding ||
+        resp.headers.get("x-proxy-header-content-type");
+      const statusCode = +(resp.headers.get("x-proxy-status-code") || "200");
+      if (contentTypeHeader?.startsWith("application/json")) {
+        body = await resp.json();
+      } else if (
+        contentTypeHeader?.startsWith("application/xml") ||
+        contentTypeHeader?.startsWith("text/")
+      ) {
+        body = await resp.text();
+      } else {
+        body = new Uint8Array(await resp.arrayBuffer());
+      }
+      return {
+        ok: resp.ok,
+        status: statusCode,
+        headers: extractProxyHeaders(resp.headers),
+        body: body,
+      };
+    },
+    "sandboxFetch.fetch": async (
+      _ctx,
+      url: string,
+      options?: ProxyFetchRequest64,
+    ): Promise<ProxyFetchResponse64> => {
+      // console.log("Got sandbox fetch ", url, op);
+      const fetchOptions = options
+        ? {
+          method: options.method,
+          headers: options.headers,
+          body: options.base64Body && base64Decode(options.base64Body),
+        }
+        : {};
+      fetchOptions.headers = buildProxyHeaders(options?.headers);
+      const resp = await client.httpSpacePrimitives.authenticatedFetch(
+        buildProxyUrl(client, url),
+        // Casting this to any because of weird Deno typing
+        fetchOptions as any,
+      );
+      const statusCode = +(resp.headers.get("x-proxy-status-code") || "200");
+      const body = await resp.arrayBuffer();
+      return {
+        ok: resp.ok,
+        status: statusCode,
+        headers: extractProxyHeaders(resp.headers),
+        base64Body: base64Encode(new Uint8Array(body)),
+      };
+    },
+  };
+}
+
+function buildProxyUrl(client: Client, url: string) {
+  url = url.replace(/^https?:\/\//, "");
+  // Strip off the /.fs and replace with /.proxy
+  return client.httpSpacePrimitives.url.slice(0, -fsEndpoint.length) +
+    "/.proxy/" + url;
+}
+
+function buildProxyHeaders(headers?: Record<string, any>): Record<string, any> {
+  const newHeaders: Record<string, any> = { "X-Proxy-Request": "true" };
+  if (!headers) {
+    return newHeaders;
+  }
+  for (const [key, value] of Object.entries(headers)) {
+    newHeaders[`X-Proxy-Header-${key}`] = value;
+  }
+  return newHeaders;
+}
+
+function extractProxyHeaders(
+  headers: Headers,
+): Record<string, any> {
+  const newHeaders: Record<string, any> = {};
+  for (const [key, value] of headers.entries()) {
+    if (key.toLowerCase().startsWith("x-proxy-header-")) {
+      newHeaders[key.slice("x-proxy-header-".length)] = value;
+    }
+  }
+  return newHeaders;
+}

package/client/plugos/syscalls/index.ts

@@ -0,0 +1,102 @@
+import type { SysCallMapping } from "../system.ts";
+import type {
+  LuaCollectionQuery,
+  LuaQueryCollection,
+} from "../../space_lua/query_collection.ts";
+
+import {
+  type ObjectIndex,
+  ObjectValidationError,
+} from "../../data/object_index.ts";
+import type { ObjectValue } from "@silverbulletmd/silverbullet/type/index";
+import type { Client } from "../../client.ts";
+import type { LuaTable } from "../../space_lua/runtime.ts";
+
+export function indexSyscalls(
+  objectIndex: ObjectIndex,
+  client: Client,
+): SysCallMapping {
+  return {
+    "index.tag": (_ctx, tagName: string): LuaQueryCollection => {
+      return objectIndex.tag(tagName);
+    },
+    "index.ensureFullIndex": (_ctx) => {
+      return objectIndex.ensureFullIndex(client.space);
+    },
+    "index.reindexSpace": () => {
+      return objectIndex.reindexSpace(client.space);
+    },
+    "index.indexObjects": (
+      _ctx,
+      page: string,
+      objects: ObjectValue[],
+    ): Promise<void> => {
+      return objectIndex.indexObjects(page, objects);
+    },
+    "index.validateObjects": async (
+      _ctx,
+      page: string,
+      objects: ObjectValue[],
+    ): Promise<{ error: string; object: ObjectValue } | null> => {
+      try {
+        await objectIndex.validateObjects(page, objects);
+        return null;
+      } catch (e: any) {
+        if (e instanceof ObjectValidationError) {
+          return {
+            error: e.message,
+            object: e.object,
+          };
+        } else {
+          throw e;
+        }
+      }
+    },
+    "index.getObjectByRef": (
+      _ctx,
+      page: string,
+      tag: string,
+      ref: string,
+    ): Promise<ObjectValue | undefined> => {
+      return objectIndex.getObjectByRef(ref, page, tag);
+    },
+    "index.queryLuaObjects": (
+      _ctx,
+      tag: string,
+      query: LuaCollectionQuery,
+      scopedVariables?: Record<string, any>,
+    ): Promise<ObjectValue[]> => {
+      return objectIndex.queryLuaObjects(
+        client.clientSystem.spaceLuaEnv.env,
+        tag,
+        query,
+        scopedVariables,
+      );
+    },
+
+    "index.deleteObject": (
+      _ctx,
+      page: string,
+      tag: string,
+      ref: string,
+    ): Promise<void> => {
+      return objectIndex.deleteObject(page, tag, ref);
+    },
+    "lua:index.defineTag": (_ctx, tagDef: LuaTable) => {
+      // Using 'lua:' prefix to _not_ convert tagDef to a JS version (but keep original LuaTable)
+      if (!tagDef.has("name")) {
+        throw new Error("A tag name is required");
+      }
+      const currentTag = client.config.get(["tags", tagDef.get("name")], null);
+      if (!currentTag) {
+        client.config.set(["tags", tagDef.get("name")], {
+          name: tagDef.get("name"),
+        });
+      }
+      client.config.set(
+        ["tags", tagDef.get("name"), "metatable"],
+        tagDef.get("metatable"),
+      );
+    },
+  };
+}

package/client/plugos/syscalls/jsonschema.ts

@@ -0,0 +1,69 @@
+import type { SysCallMapping } from "../system.ts";
+import { Ajv, type ValidateFunction } from "ajv";
+
+const ajv = new Ajv();
+
+ajv.addFormat("email", {
+  validate: (data: string) => {
+    // TODO: Implement email validation
+    return data.includes("@");
+  },
+  async: false,
+});
+
+ajv.addFormat("page-ref", {
+  validate: (data: string) => {
+    return data.startsWith("[[") && data.endsWith("]]");
+  },
+  async: false,
+});
+
+const schemaCache = new Map<string, ValidateFunction>();
+
+export function validateObject(schema: any, object: any): undefined | string {
+  try {
+    const schemaKey = JSON.stringify(schema);
+    if (!schemaCache.has(schemaKey)) {
+      const validate = ajv.compile(schema);
+      schemaCache.set(schemaKey, validate);
+    }
+    const validate = schemaCache.get(schemaKey)!;
+    if (validate(object)) {
+      return;
+    } else {
+      let text = ajv.errorsText(validate.errors);
+      text = text.replaceAll("/", ".");
+      text = text.replace(/^data[\.\s]/, "");
+      return text;
+    }
+  } catch (e: any) {
+    return e.message;
+  }
+}
+
+export function validateSchema(schema: any): undefined | string {
+  const valid = ajv.validateSchema(schema);
+  if (valid) {
+    return;
+  } else {
+    return ajv.errorsText(ajv.errors);
+  }
+}
+
+export function jsonschemaSyscalls(): SysCallMapping {
+  return {
+    "jsonschema.validateObject": (
+      _ctx,
+      schema: any,
+      object: any,
+    ): undefined | string => {
+      return validateObject(schema, object);
+    },
+    "jsonschema.validateSchema": (
+      _ctx,
+      schema: any,
+    ): undefined | string => {
+      return validateSchema(schema);
+    },
+  };
+}

package/client/plugos/syscalls/language.ts

@@ -0,0 +1,23 @@
+import type { SysCallMapping } from "../system.ts";
+import { parse } from "../../markdown_parser/parse_tree.ts";
+import type { ParseTree } from "@silverbulletmd/silverbullet/lib/tree";
+import { builtinLanguages, languageFor } from "../../languages.ts";
+
+export function languageSyscalls(): SysCallMapping {
+  return {
+    "language.parseLanguage": (
+      _ctx,
+      language: string,
+      code: string,
+    ): ParseTree => {
+      const lang = languageFor(language);
+      if (!lang) {
+        throw new Error(`Unknown language ${language}`);
+      }
+      return parse(lang, code);
+    },
+    "language.listLanguages": (): string[] => {
+      return Object.keys(builtinLanguages);
+    },
+  };
+}

package/client/plugos/syscalls/lua.ts

@@ -0,0 +1,58 @@
+import type { SysCallMapping } from "../system.ts";
+import { evalExpression } from "../../space_lua/eval.ts";
+import { parse, parseExpressionString } from "../../space_lua/parse.ts";
+import {
+  LuaStackFrame,
+  luaToString,
+  luaValueToJS,
+} from "../../space_lua/runtime.ts";
+import { buildThreadLocalEnv } from "../../space_lua_api.ts";
+import { isSendable } from "../util.ts";
+import type { LuaBlock, LuaExpression } from "../../space_lua/ast.ts";
+import type { ClientSystem } from "../../client_system.ts";
+
+export function luaSyscalls(clientSystem: ClientSystem): SysCallMapping {
+  return {
+    "lua.parse": (_ctx, code: string): LuaBlock => {
+      return parse(code);
+    },
+    "lua.parseExpression": (_ctx, expression: string): LuaExpression => {
+      return parseExpressionString(expression);
+    },
+    /**
+     * Evaluates a Lua expression and returns the result as a JavaScript value
+     * @param _ctx
+     * @param expression
+     * @returns
+     */
+    "lua.evalExpression": async (_ctx, expression: string) => {
+      try {
+        const ast = parseExpressionString(expression);
+        const env = await buildThreadLocalEnv(
+          clientSystem.system,
+          clientSystem.spaceLuaEnv.env,
+        );
+        const sf = new LuaStackFrame(env, null);
+        const luaResult = await evalExpression(
+          ast,
+          clientSystem.spaceLuaEnv.env,
+          sf,
+        );
+        const jsResult = luaValueToJS(luaResult, sf);
+        if (isSendable(jsResult)) {
+          return jsResult;
+        } else {
+          // This may evaluate to e.g. a function, which is not sendable, in this case we'll console.warn and return a stringified version of the result
+          console.warn(
+            "Lua eval result is not sendable, returning stringified version",
+            jsResult,
+          );
+          return luaToString(luaResult);
+        }
+      } catch (e: any) {
+        console.error("Lua eval error: ", e.message, e.sf?.astCtx);
+        throw e;
+      }
+    },
+  };
+}

package/client/plugos/syscalls/markdown.ts

@@ -0,0 +1,84 @@
+import type { SysCallMapping } from "../system.ts";
+import { parse } from "../../markdown_parser/parse_tree.ts";
+import {
+  type ParseTree,
+  renderToText,
+} from "@silverbulletmd/silverbullet/lib/tree";
+import { extendedMarkdownLanguage } from "../../markdown_parser/parser.ts";
+import {
+  expandMarkdown,
+  type MarkdownExpandOptions,
+} from "../../markdown_renderer/inline.ts";
+import type { Client } from "../../client.ts";
+import {
+  type MarkdownRenderOptions,
+  renderMarkdownToHtml,
+} from "../../markdown_renderer/markdown_render.ts";
+import {
+  jsonToMDTable,
+  refCellTransformer,
+} from "../../markdown_renderer/result_render.ts";
+
+export function markdownSyscalls(client: Client): SysCallMapping {
+  return {
+    "markdown.parseMarkdown": (_ctx, text: string): ParseTree => {
+      return parse(extendedMarkdownLanguage, text);
+    },
+    "markdown.renderParseTree": (_ctx, tree: ParseTree): string => {
+      return renderToText(tree);
+    },
+    "markdown.expandMarkdown": async (
+      _ctx,
+      treeOrText: ParseTree | string,
+      options?: MarkdownExpandOptions,
+    ): Promise<ParseTree | string> => {
+      const outputString = typeof treeOrText === "string";
+      if (typeof treeOrText === "string") {
+        treeOrText = parse(extendedMarkdownLanguage, treeOrText);
+      }
+      const result = await expandMarkdownWithClient(
+        client,
+        treeOrText,
+        options,
+      );
+      if (outputString) {
+        return renderToText(result);
+      } else {
+        return result;
+      }
+    },
+    "markdown.markdownToHtml": async (
+      _ctx,
+      text: string,
+      options: MarkdownRenderOptions = {},
+    ) => {
+      let mdTree = parse(extendedMarkdownLanguage, text);
+      if (options.expand) {
+        mdTree = await expandMarkdownWithClient(client, mdTree);
+      }
+      return renderMarkdownToHtml(mdTree, options);
+    },
+    "markdown.objectsToTable": (
+      _ctx,
+      data: any[],
+      options: { renderCell?: (val: any, key: string) => Promise<any> | any } =
+        {},
+    ) => {
+      return jsonToMDTable(data, options.renderCell || refCellTransformer);
+    },
+  };
+}
+
+function expandMarkdownWithClient(
+  client: Client,
+  tree: ParseTree,
+  options?: MarkdownExpandOptions,
+) {
+  return expandMarkdown(
+    client.space,
+    client.currentName(),
+    tree,
+    client.clientSystem.spaceLuaEnv,
+    options,
+  );
+}

package/client/plugos/syscalls/mq.ts

@@ -0,0 +1,52 @@
+import type { DataStoreMQ } from "../../data/mq.datastore.ts";
+import type { MQListenerSpec } from "../hooks/mq.ts";
+import type { SysCallMapping } from "../system.ts";
+
+export type EventSubscription = MQListenerSpec & {
+  run: (...args: any[]) => Promise<any>;
+};
+
+export function mqSyscalls(
+  mq: DataStoreMQ,
+): SysCallMapping {
+  return {
+    /**
+     * Define a Lua event listener
+     */
+    "mq.subscribe": (
+      _ctx,
+      def: MQListenerSpec,
+    ) => {
+      def.autoAck = def.autoAck != false;
+      // console.log("Registering Lua event listener: ", def.name);
+      client.config.insert([
+        "mqSubscriptions",
+        def.queue,
+      ], def);
+    },
+    "mq.send": (_ctx, queue: string, body: any) => {
+      return mq.send(queue, body);
+    },
+    "mq.batchSend": (_ctx, queue: string, bodies: any[]) => {
+      return mq.batchSend(queue, bodies);
+    },
+    "mq.ack": (_ctx, queue: string, id: string) => {
+      return mq.ack(queue, id);
+    },
+    "mq.batchAck": (_ctx, queue: string, ids: string[]) => {
+      return mq.batchAck(queue, ids);
+    },
+    "mq.flushQueue": (_ctx, queue: string) => {
+      return mq.flushQueue(queue);
+    },
+    "mq.flushAllQueues": () => {
+      return mq.flushAllQueues();
+    },
+    "mq.getQueueStats": (_ctx, queue: string) => {
+      return mq.getQueueStats(queue);
+    },
+    "mq.awaitEmptyQueue": (_ctx, queue: string) => {
+      return mq.awaitEmptyQueue(queue);
+    },
+  };
+}

package/client/plugos/syscalls/service_registry.ts

@@ -0,0 +1,43 @@
+import type {
+  ServiceMatch,
+  ServiceRegistry,
+  ServiceSpec,
+} from "../../service_registry.ts";
+import type { SysCallMapping } from "../system.ts";
+
+export function serviceRegistrySyscalls(
+  serviceRegistry: ServiceRegistry,
+): SysCallMapping {
+  return {
+    /**
+     * Define a Lua event listener
+     */
+    "service.define": (
+      _ctx,
+      def: ServiceSpec,
+    ) => {
+      return serviceRegistry.define(def);
+    },
+    "service.discover": (
+      _ctx,
+      selector: string,
+      data: any,
+    ): Promise<ServiceMatch[]> => {
+      return serviceRegistry.discover(selector, data);
+    },
+    "service.invoke": (
+      _ctx,
+      service: ServiceMatch,
+      data: any,
+    ): Promise<any> => {
+      return serviceRegistry.invoke(service, data);
+    },
+    "service.invokeBestMatch": (
+      _ctx,
+      selector: string,
+      data: any,
+    ): Promise<any> => {
+      return serviceRegistry.invokeBestMatch(selector, data);
+    },
+  };
+}

package/client/plugos/syscalls/shell.ts

@@ -0,0 +1,39 @@
+import type { SysCallMapping } from "../system.ts";
+import type { Client } from "../../client.ts";
+import { fsEndpoint } from "../../spaces/constants.ts";
+
+export function shellSyscalls(
+  client: Client,
+): SysCallMapping {
+  return {
+    "shell.run": async (
+      _ctx,
+      cmd: string,
+      args: string[],
+      stdin?: string,
+    ): Promise<{ stdout: string; stderr: string; code: number }> => {
+      if (!client.httpSpacePrimitives) {
+        throw new Error("Not supported in fully local mode");
+      }
+      const resp = client.httpSpacePrimitives.authenticatedFetch(
+        buildShellUrl(client),
+        {
+          method: "POST",
+          body: JSON.stringify({
+            cmd,
+            args,
+            stdin,
+          }),
+        },
+      );
+      const { code, stderr, stdout } = await (await resp).json();
+      return { code, stderr, stdout };
+    },
+  };
+}
+
+function buildShellUrl(client: Client) {
+  // Strip off the /.fs and replace with /.shell
+  return client.httpSpacePrimitives.url.slice(0, -fsEndpoint.length) +
+    "/.shell";
+}