@marimo-team/frontend 0.14.18-dev5 → 0.14.18-dev51

package/src/core/islands/parse.ts

@@ -0,0 +1,161 @@
+/* Copyright 2024 Marimo. All rights reserved. */
+
+import { MarimoIslandElement } from "@/core/islands/components/web-components";
+import { Logger } from "@/utils/Logger";
+
+/**
+ * DOM elements look like this:
+ * <marimo-island data-app-id="{self._app_id}">
+ *   <marimo-cell-output>
+ *     <div> Hello, world! </div>
+ *   </marimo-cell-output>
+ *   <marimo-cell-code>
+ *     encoded(print("Hello, world!"))
+ *   </marimo-cell-code>
+ * </marimo-island>
+ */
+
+export interface MarimoIslandApp {
+  /**
+   * ID since we allow multiple apps on the same page.
+   */
+  id: string;
+  /**
+   * Cells in the app.
+   */
+  cells: MarimoIslandCell[];
+}
+
+interface MarimoIslandCell {
+  /**
+   * Output of the cell.
+   */
+  output: string;
+  /**
+   * Code of the cell.
+   */
+  code: string;
+  /**
+   * Index of the cell.
+   */
+  idx: number;
+}
+
+export function parseMarimoIslandApps(): MarimoIslandApp[] {
+  const apps = new Map<string, MarimoIslandApp>();
+
+  const embeds = document.querySelectorAll<HTMLElement>(
+    MarimoIslandElement.tagName,
+  );
+  if (embeds.length === 0) {
+    Logger.warn("No embedded marimo apps found.");
+    return [];
+  }
+
+  for (const embed of embeds) {
+    const id = embed.dataset.appId;
+    if (!id) {
+      Logger.warn("Embedded marimo cell missing data-app-id attribute.");
+      continue;
+    }
+
+    const cellOutput = embed.querySelector<HTMLElement>(
+      MarimoIslandElement.outputTagName,
+    );
+    const code = extractIslandCodeFromEmbed(embed);
+
+    if (!cellOutput || !code) {
+      Logger.warn(`Embedded marimo app ${id} missing cell output or code.`);
+      continue;
+    }
+
+    if (!apps.has(id)) {
+      apps.set(id, { id, cells: [] });
+    }
+    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+    const app = apps.get(id)!;
+    const idx = app.cells.length;
+    app.cells.push({
+      output: cellOutput.innerHTML,
+      code: code,
+      idx: idx,
+    });
+
+    // Add data-cell-idx attribute to the island element
+    embed.dataset.cellIdx = idx.toString();
+  }
+
+  return [...apps.values()];
+}
+
+export function createMarimoFile(app: {
+  cells: Array<{ code: string }>;
+}): string {
+  const lines = [
+    "import marimo",
+    "app = marimo.App()",
+    app.cells
+      .map((cell) => {
+        // Add 4 spaces to each line
+        const code = cell.code
+          .split("\n")
+          .map((line) => `    ${line}`)
+          .join("\n");
+
+        // TODO: Handle async cells better
+        // This is probably not the best way to check if the code is async
+        // Ideally this is pushed into the Python code
+        const isAsync = code.includes("await ");
+        const prefix = isAsync ? "async def" : "def";
+
+        // Wrap in a function
+        return `@app.cell\n${prefix} __():\n${code}\n    return`;
+      })
+      .join("\n"),
+  ];
+
+  return lines.join("\n");
+}
+
+export function parseIslandEditor(code: string | undefined | null): string {
+  if (!code) {
+    return "";
+  }
+  try {
+    return `${JSON.parse(code)}`;
+  } catch {
+    return code;
+  }
+}
+
+export function parseIslandCode(code: string | undefined | null): string {
+  if (!code) {
+    return "";
+  }
+  return decodeURIComponent(code).trim();
+}
+
+export function extractIslandCodeFromEmbed(embed: HTMLElement): string {
+  const reactive = embed.dataset.reactive === "true";
+  // Non-reactive cells are not guaranteed to have code, and should be treated as
+  // such.
+  if (!reactive) {
+    return "";
+  }
+
+  const cellCodeElement = embed.querySelector<HTMLElement>(
+    MarimoIslandElement.codeTagName,
+  );
+  if (cellCodeElement) {
+    return parseIslandCode(cellCodeElement.textContent);
+  }
+
+  const editorCodeElement = embed.querySelector<HTMLElement>(
+    MarimoIslandElement.editorTagName,
+  );
+  if (editorCodeElement) {
+    return parseIslandEditor(editorCodeElement.dataset.initialValue);
+  }
+
+  return "";
+}

package/src/core/islands/state.ts

@@ -0,0 +1,30 @@
+/* Copyright 2024 Marimo. All rights reserved. */
+import { atom } from "jotai";
+
+/**
+ * Whether the islands have been initialized.
+ *
+ * false: Islands have not been initialized.
+ * true: Islands have been initialized.
+ * string: If there was an error initializing the islands, the string will
+ *         contain the error message.
+ */
+export const islandsInitializedAtom = atom<boolean | string>(false);
+
+/**
+ * Whether the user has tried to interact with the islands.
+ */
+export const userTriedToInteractWithIslandsAtom = atom(false);
+
+/**
+ * Whether to show the islands warning indicator.
+ *
+ * This only happens if the user has tried to interact with the islands, but
+ * they haven't been initialized yet.
+ */
+export const shouldShowIslandsWarningIndicatorAtom = atom((get) => {
+  return (
+    get(userTriedToInteractWithIslandsAtom) &&
+    get(islandsInitializedAtom) === false
+  );
+});

package/src/core/islands/toast.ts

@@ -0,0 +1,41 @@
+/* Copyright 2024 Marimo. All rights reserved. */
+let didToast = false;
+let toastElement: HTMLDivElement | undefined;
+
+/**
+ * Show a toast message in the center of the screen.
+ *
+ * Intentionally outside React and not using our styles since we place
+ * this in the global scope.
+ *
+ * We only show this once.
+ */
+export function toastIslandsLoading() {
+  if (didToast) {
+    return;
+  }
+  didToast = true;
+  toastElement = document.createElement("div");
+  toastElement.classList.add("marimo-islands-loading-toast");
+  toastElement.textContent = "⚡ reactivity initializing...";
+  toastElement.style.position = "fixed";
+  toastElement.style.top = "0.5rem";
+  toastElement.style.left = "50%";
+  toastElement.style.transform = "translateX(-50%)";
+  toastElement.style.zIndex = "1000";
+  toastElement.style.backgroundColor = "#0f172a";
+  toastElement.style.color = "#f8fafc";
+  toastElement.style.padding = "0.25rem 0.5rem";
+  toastElement.style.borderRadius = "0.5rem";
+  toastElement.style.fontSize = "0.875rem";
+  document.body.append(toastElement);
+}
+
+/**
+ * Remove the toast message from the screen.
+ */
+export function dismissIslandsLoadingToast() {
+  if (toastElement) {
+    toastElement.remove();
+  }
+}

package/src/core/islands/utils.ts

@@ -0,0 +1,5 @@
+/* Copyright 2024 Marimo. All rights reserved. */
+
+export function isIslands() {
+  return import.meta.env.VITE_MARIMO_ISLANDS === "true";
+}

package/src/core/islands/worker/controller.ts

@@ -0,0 +1,52 @@
+/* Copyright 2024 Marimo. All rights reserved. */
+import type { PyodideInterface } from "pyodide";
+import { defaultUserConfig } from "@/core/config/config-schema";
+import type { OperationMessage } from "@/core/kernel/messages";
+import { DefaultWasmController } from "@/core/wasm/worker/bootstrap";
+import { WasmFileSystem } from "@/core/wasm/worker/fs";
+import type { SerializedBridge } from "@/core/wasm/worker/types";
+import type { JsonString } from "@/utils/json/base64";
+import { Logger } from "../../../utils/Logger";
+
+export class ReadonlyWasmController extends DefaultWasmController {
+  override async bootstrap(opts: {
+    version: string;
+    pyodideVersion: string;
+  }): Promise<PyodideInterface> {
+    const pyodide = await super.bootstrap(opts);
+    return pyodide;
+  }
+
+  override async mountFilesystem(opts: { code: string; filename: string }) {
+    const { code, filename } = opts;
+    // Write file
+    try {
+      WasmFileSystem.createHomeDir(this.requirePyodide);
+      return WasmFileSystem.initNotebookCode({
+        pyodide: this.requirePyodide,
+        code,
+        filename,
+      });
+    } catch (error) {
+      Logger.error(error);
+    }
+
+    return { code, filename };
+  }
+
+  override async startSession(opts: {
+    code: string;
+    filename: string | null;
+    onMessage: (message: JsonString<OperationMessage>) => void;
+  }): Promise<SerializedBridge> {
+    const bridge = super.startSession({
+      queryParameters: {},
+      code: opts.code,
+      filename: opts.filename,
+      onMessage: opts.onMessage,
+      userConfig: defaultUserConfig(),
+    });
+
+    return bridge;
+  }
+}

package/src/core/islands/worker/worker.tsx

@@ -0,0 +1,178 @@
+/* Copyright 2024 Marimo. All rights reserved. */
+
+import type { PyodideInterface } from "pyodide";
+import {
+  createRPC,
+  createRPCRequestHandler,
+  createWorkerParentTransport,
+  type RPCSchema,
+} from "rpc-anywhere";
+import type { OperationMessage } from "@/core/kernel/messages";
+import type { ParentSchema } from "@/core/wasm/rpc";
+import { TRANSPORT_ID } from "@/core/wasm/worker/constants";
+import { getPyodideVersion } from "@/core/wasm/worker/getPyodideVersion";
+import { MessageBuffer } from "@/core/wasm/worker/message-buffer";
+import type { RawBridge, SerializedBridge } from "@/core/wasm/worker/types";
+import type { JsonString } from "@/utils/json/base64";
+import { Logger } from "@/utils/Logger";
+import { Deferred } from "../../../utils/Deferred";
+import { prettyError } from "../../../utils/errors";
+import { invariant } from "../../../utils/invariant";
+import { ReadonlyWasmController } from "./controller";
+
+declare const self: Window & {
+  pyodide: PyodideInterface;
+  controller: ReadonlyWasmController;
+};
+
+// Initialize pyodide
+async function loadPyodideAndPackages() {
+  const marimoVersion = getMarimoVersion();
+  const pyodideVersion = getPyodideVersion(marimoVersion);
+  try {
+    self.controller = new ReadonlyWasmController();
+    self.pyodide = await self.controller.bootstrap({
+      version: marimoVersion,
+      pyodideVersion: pyodideVersion,
+    });
+  } catch (error) {
+    Logger.error("Error bootstrapping", error);
+    rpc.send.initializedError({
+      error: prettyError(error),
+    });
+  }
+}
+
+const pyodideReadyPromise = loadPyodideAndPackages();
+const messageBuffer = new MessageBuffer(
+  (message: JsonString<OperationMessage>) => {
+    rpc.send.kernelMessage({ message });
+  },
+);
+const bridgeReady = new Deferred<SerializedBridge>();
+
+// Handle RPC requests
+const requestHandler = createRPCRequestHandler({
+  /**
+   * Start the session
+   */
+  startSession: async (opts: { code: string; appId: string }) => {
+    await pyodideReadyPromise; // Make sure loading is done
+
+    try {
+      invariant(self.controller, "Controller not loaded");
+      const notebook = await self.controller.mountFilesystem({
+        code: opts.code,
+        filename: `app-${opts.appId}.py`,
+      });
+      const bridge = await self.controller.startSession({
+        ...notebook,
+        onMessage: messageBuffer.push,
+      });
+      bridgeReady.resolve(bridge);
+      rpc.send.initialized({});
+    } catch (error) {
+      rpc.send.initializedError({
+        error: prettyError(error),
+      });
+    }
+    return;
+  },
+
+  /**
+   * Load packages
+   */
+  loadPackages: async (code: string) => {
+    await pyodideReadyPromise; // Make sure loading is done
+
+    if (code.includes("mo.sql")) {
+      // Add pandas and duckdb to the code
+      code = `import pandas\n${code}`;
+      code = `import duckdb\n${code}`;
+      code = `import sqlglot\n${code}`;
+
+      // Polars + SQL requires pyarrow, and installing
+      // after notebook load does not work. As a heuristic,
+      // if it appears that the notebook uses polars, add pyarrow.
+      if (code.includes("polars")) {
+        code = `import pyarrow\n${code}`;
+      }
+    }
+
+    await self.pyodide.loadPackagesFromImports(code, {
+      messageCallback: Logger.log,
+      errorCallback: Logger.error,
+    });
+  },
+
+  /**
+   * Call a function on the bridge
+   */
+  bridge: async (opts: {
+    functionName: keyof RawBridge;
+    payload: {} | undefined | null;
+  }) => {
+    await pyodideReadyPromise; // Make sure loading is done
+
+    const { functionName, payload } = opts;
+
+    // Perform the function call to the Python bridge
+    const bridge = await bridgeReady.promise;
+
+    // Serialize the payload
+    const payloadString =
+      payload == null
+        ? null
+        : typeof payload === "string"
+          ? payload
+          : JSON.stringify(payload);
+
+    // Make the request
+    const response =
+      payloadString == null
+        ? // @ts-expect-error ehh TypeScript
+          await bridge[functionName]()
+        : // @ts-expect-error ehh TypeScript
+          await bridge[functionName](payloadString);
+
+    // Post the response back to the main thread
+    return typeof response === "string" ? JSON.parse(response) : response;
+  },
+});
+
+// create the iframe's schema
+export type WorkerSchema = RPCSchema<
+  {
+    messages: {
+      // Emitted when the worker is ready
+      ready: {};
+      // Emitted when the kernel sends a message
+      kernelMessage: { message: JsonString<OperationMessage> };
+      // Emitted when the Pyodide is initialized
+      initialized: {};
+      // Emitted when the Pyodide fails to initialize
+      initializedError: { error: string };
+    };
+  },
+  typeof requestHandler
+>;
+
+const rpc = createRPC<WorkerSchema, ParentSchema>({
+  transport: createWorkerParentTransport({
+    transportId: TRANSPORT_ID,
+  }),
+  requestHandler,
+});
+
+rpc.send("ready", {});
+
+/// Listeners
+// When the consumer is ready, start the message buffer
+rpc.addMessageListener("consumerReady", async () => {
+  await pyodideReadyPromise; // Make sure loading is done
+  messageBuffer.start();
+});
+
+function getMarimoVersion() {
+  return self.name; // We store the version in the worker name
+}

package/src/core/kernel/RuntimeState.ts

@@ -0,0 +1,94 @@
+/* Copyright 2024 Marimo. All rights reserved. */
+
+import { Logger } from "@/utils/Logger";
+import {
+  MarimoValueReadyEvent,
+  type MarimoValueReadyEventType,
+} from "../dom/events";
+import { UI_ELEMENT_REGISTRY, type UIElementRegistry } from "../dom/uiregistry";
+import type { RunRequests } from "../network/types";
+
+/**
+ * Manager to track running cells.
+ */
+export class RuntimeState {
+  private hasStarted = false;
+
+  /**
+   * Shared instance of RuntimeState since this must be a singleton.
+   */
+  static get INSTANCE(): RuntimeState {
+    const KEY = "_marimo_private_RuntimeState";
+    if (!window[KEY]) {
+      window[KEY] = new RuntimeState(UI_ELEMENT_REGISTRY);
+    }
+    return window[KEY] as RuntimeState;
+  }
+
+  /**
+   * ObjectIds of UIElements whose values need to be updated in the kernel
+   */
+  private _sendComponentValues: RunRequests["sendComponentValues"] | undefined;
+
+  constructor(private uiElementRegistry: UIElementRegistry) {}
+
+  private get sendComponentValues(): RunRequests["sendComponentValues"] {
+    if (!this._sendComponentValues) {
+      throw new Error("sendComponentValues is not set");
+    }
+    return this._sendComponentValues;
+  }
+
+  /**
+   * Start listening for events from UIElements
+   */
+  start(sendComponentValues: RunRequests["sendComponentValues"]) {
+    if (this.hasStarted) {
+      Logger.warn("RuntimeState already started");
+      return;
+    }
+    this._sendComponentValues = sendComponentValues;
+    document.addEventListener(
+      MarimoValueReadyEvent.TYPE,
+      this.handleReadyEvent,
+    );
+    this.hasStarted = true;
+  }
+
+  /**
+   * Stop listening for events from UIElements
+   */
+  stop() {
+    if (!this.hasStarted) {
+      Logger.warn("RuntimeState already stopped");
+      return;
+    }
+    document.removeEventListener(
+      MarimoValueReadyEvent.TYPE,
+      this.handleReadyEvent,
+    );
+    this.hasStarted = false;
+  }
+
+  private handleReadyEvent = (e: MarimoValueReadyEventType) => {
+    const objectId = e.detail.objectId;
+    if (!this.uiElementRegistry.has(objectId)) {
+      return;
+    }
+
+    const value = this.uiElementRegistry.lookupValue(objectId);
+    if (value !== undefined) {
+      this.sendComponentValues({
+        objectIds: [objectId],
+        values: [value],
+      }).catch(
+        // This happens if the run failed to register (401, 403, network
+        // error, etc.) A run may fail if the kernel is restarted or the
+        // notebook is closed.
+        (error) => {
+          Logger.warn(error);
+        },
+      );
+    }
+  };
+}

package/src/core/kernel/__tests__/RuntimeState.test.ts

@@ -0,0 +1,57 @@
+/* Copyright 2024 Marimo. All rights reserved. */
+
+import {
+  beforeEach,
+  describe,
+  expect,
+  type MockedFunction,
+  test,
+  vi,
+} from "vitest";
+import { MarimoValueReadyEvent } from "@/core/dom/events";
+import { UIElementRegistry } from "@/core/dom/uiregistry";
+import type { RunRequests } from "@/core/network/types";
+import { RuntimeState } from "../RuntimeState";
+
+const addEventListenerSpy = vi.spyOn(document, "addEventListener");
+const removeEventListenerSpy = vi.spyOn(document, "removeEventListener");
+
+describe("RuntimeState", () => {
+  let runtimeState: RuntimeState;
+  let mockSendComponentValues: MockedFunction<
+    RunRequests["sendComponentValues"]
+  >;
+  let uiElementRegistry: UIElementRegistry;
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    mockSendComponentValues = vi.fn((args) => Promise.resolve(null));
+    uiElementRegistry = UIElementRegistry.INSTANCE;
+    uiElementRegistry.entries.clear();
+
+    runtimeState = new RuntimeState(uiElementRegistry);
+  });
+
+  test("start should register event listener", () => {
+    runtimeState.start(mockSendComponentValues);
+    expect(addEventListenerSpy).toHaveBeenCalledWith(
+      MarimoValueReadyEvent.TYPE,
+      expect.any(Function),
+    );
+  });
+
+  test("it can only start once", () => {
+    runtimeState.start(mockSendComponentValues);
+    runtimeState.start(mockSendComponentValues);
+    expect(addEventListenerSpy).toHaveBeenCalledTimes(1);
+  });
+
+  test("stop should remove event listener", () => {
+    runtimeState.start(mockSendComponentValues);
+    runtimeState.stop();
+    expect(removeEventListenerSpy).toHaveBeenCalledWith(
+      MarimoValueReadyEvent.TYPE,
+      expect.any(Function),
+    );
+  });
+});

package/src/core/kernel/__tests__/handlers.test.ts

@@ -0,0 +1,61 @@
+/* Copyright 2024 Marimo. All rights reserved. */
+import { describe, expect, it, vi } from "vitest";
+import { queryParamHandlers } from "../queryParamHandlers";
+
+// Helper to set up URL and searchParams
+function setupURL(search = "") {
+  const url = new URL("http://localhost:3000");
+  url.search = search;
+  window.history.pushState({}, "", `${url.pathname}${url.search}`);
+  return url;
+}
+
+vi.spyOn(window.history, "pushState");
+
+describe("queryParamHandlers", () => {
+  it("should append a query parameter", () => {
+    setupURL();
+    queryParamHandlers.append({ key: "test", value: "123" });
+    expect(window.location.href).toContain("test=123");
+    expect(window.history.pushState).toHaveBeenCalled();
+  });
+
+  it("should set a query parameter", () => {
+    setupURL();
+    queryParamHandlers.set({ key: "test", value: "123" });
+    expect(window.location.href).toContain("test=123");
+    expect(window.history.pushState).toHaveBeenCalled();
+  });
+
+  it("should delete a specific query parameter", () => {
+    setupURL("?test=123&sample=456");
+    queryParamHandlers.delete({ key: "test", value: "123" });
+    expect(window.location.href).not.toContain("test=123");
+    expect(window.location.href).toContain("sample=456");
+    expect(window.history.pushState).toHaveBeenCalled();
+  });
+
+  it("shouldn't delete a specific query parameter if the value doesn't match", () => {
+    setupURL("?test=abc&sample=456");
+    queryParamHandlers.delete({ key: "test", value: "123" });
+    expect(window.location.href).toContain("test=abc");
+    expect(window.location.href).toContain("sample=456");
+    expect(window.history.pushState).toHaveBeenCalled();
+  });
+
+  it("should delete all instances of a query parameter", () => {
+    setupURL("?test=123&test=456");
+    queryParamHandlers.delete({ key: "test", value: null });
+    expect(window.location.href).not.toContain("test=123");
+    expect(window.location.href).not.toContain("test=456");
+    expect(window.history.pushState).toHaveBeenCalled();
+  });
+
+  it("should clear all query parameters", () => {
+    setupURL("?test=123&sample=456");
+    queryParamHandlers.clear();
+    expect(window.location.href).not.toContain("test=123");
+    expect(window.location.href).not.toContain("sample=456");
+    expect(window.history.pushState).toHaveBeenCalled();
+  });
+});

package/src/core/kernel/__tests__/session.test.ts

@@ -0,0 +1,10 @@
+/* Copyright 2024 Marimo. All rights reserved. */
+import { describe, expect, it } from "vitest";
+import { generateSessionId, isSessionId } from "../session";
+
+describe("Session", () => {
+  it("should create a session", () => {
+    const id = generateSessionId();
+    expect(isSessionId(id)).toBe(true);
+  });
+});