@marimo-team/islands 0.17.8 → 0.18.1

package/src/core/cells/logs.ts

@@ -2,7 +2,8 @@
 
 import { fromUnixTime } from "date-fns";
 import { toast } from "@/components/ui/use-toast";
-import { invariant } from "@/utils/invariant";
+import { parseHtmlContent } from "@/utils/dom";
+import { Strings } from "@/utils/strings";
 import type { CellMessage, OutputMessage } from "../kernel/messages";
 import { isErrorMime } from "../mime";
 import type { CellId } from "./ids";
@@ -21,21 +22,31 @@ export function getCellLogsForMessage(cell: CellMessage): CellLog[] {
   const consoleOutputs: OutputMessage[] = [cell.console].filter(Boolean).flat();
 
   for (const output of consoleOutputs) {
-    if (output.mimetype === "text/plain") {
-      invariant(typeof output.data === "string", "expected string");
+    // Handle text/plain, text/html, and traceback MIME types
+    const isTextPlain = output.mimetype === "text/plain";
+    const isHtml =
+      output.mimetype === "text/html" ||
+      output.mimetype === "application/vnd.marimo+traceback";
+
+    if (isTextPlain || isHtml) {
       const isError =
         output.channel === "stderr" || output.channel === "marimo-error";
       switch (output.channel) {
         case "stdout":
         case "stderr":
-        case "marimo-error":
+        case "marimo-error": {
+          // Convert data to string and process based on MIME type
+          const rawData = Strings.asString(output.data);
+          const message = isHtml ? parseHtmlContent(rawData) : rawData;
+
           logs.push({
             timestamp: output.timestamp || Date.now(),
             level: isError ? "stderr" : "stdout",
-            message: output.data,
+            message: message,
             cellId: cell.cell_id as CellId,
           });
           break;
+        }
         default:
           break;
       }

package/src/core/cells/session.ts

@@ -246,7 +246,6 @@ export function notebookStateFromSession(
     mergeSessionAndNotebookCells(session, notebook);
 
   if (cellIds.length === 0) {
-    Logger.warn("Session and notebook must have at least one cell");
     return null;
   }
 

package/src/core/codemirror/__tests__/format.test.ts

@@ -11,6 +11,7 @@ import { getNotebook } from "@/core/cells/cells";
 import type { CellId } from "@/core/cells/ids";
 import { notebookCellEditorViews } from "@/core/cells/utils";
 import { getResolvedMarimoConfig } from "@/core/config/config";
+import type { ConnectionName } from "@/core/datasets/engines";
 import { OverridingHotkeyProvider } from "@/core/hotkeys/hotkeys";
 import { requestClientAtom } from "@/core/network/requests";
 import type { MarimoConfig } from "@/core/network/types";
@@ -198,7 +199,7 @@ describe("format", () => {
       const editor = createEditor("SELECT * FROM table WHERE id = 1", cellId);
       switchLanguage(editor, { language: "sql" });
 
-      await formatSQL(editor);
+      await formatSQL(editor, "duckdb" as ConnectionName);
 
       // Check that the SQL was formatted
       expect(editor.state.doc.toString()).toMatchInlineSnapshot(`
@@ -221,7 +222,7 @@ describe("format", () => {
       const editor = createEditor("SELECT * FROM table WHERE id = 1", cellId);
       switchLanguage(editor, { language: "python" });
 
-      await formatSQL(editor);
+      await formatSQL(editor, "duckdb" as ConnectionName);
 
       // Check that the SQL was not formatted
       expect(editor.state.doc.toString()).toBe(
@@ -230,4 +231,29 @@ describe("format", () => {
       expect(updateCellCode).not.toHaveBeenCalled();
     });
   });
+
+  it("should format SQL code with different dialect", async () => {
+    const cellId = "1" as CellId;
+    const editor = createEditor(
+      "SELECT * FROM `table.dot` WHERE id = 1",
+      cellId,
+    );
+    switchLanguage(editor, { language: "sql" });
+
+    await formatSQL(editor, "mysql" as ConnectionName); // mysql uses backticks for identifiers
+
+    expect(editor.state.doc.toString()).toMatchInlineSnapshot(`
+      "SELECT
+        *
+      FROM
+        \`table.dot\`
+      WHERE
+        id = 1"
+    `);
+    expect(updateCellCode).toHaveBeenCalledWith({
+      cellId: cellId,
+      code: editor.state.doc.toString(),
+      formattingChange: true,
+    });
+  });
 });

package/src/core/codemirror/copilot/client.ts

@@ -89,9 +89,29 @@ class LazyWebsocketTransport extends Transport {
     data: JSONRPCRequestData,
     timeout: number | null | undefined,
   ) {
+    // If delegate is undefined, try to reconnect
+    if (!this.delegate) {
+      Logger.log("Copilot#sendData: Delegate not initialized, reconnecting...");
+      try {
+        await this.tryConnect();
+      } catch (error) {
+        Logger.error("Copilot#sendData: Failed to reconnect transport", error);
+        throw new Error(
+          "Unable to connect to GitHub Copilot. Please check your settings and try again.",
+        );
+      }
+    }
+
+    // After reconnection, delegate should be initialized
+    if (!this.delegate) {
+      throw new Error(
+        "Failed to initialize GitHub Copilot connection. Please try again.",
+      );
+    }
+
     // Clamp timeout to 5 seconds
     timeout = Math.min(timeout ?? 5000, 5000);
-    return this.delegate?.sendData(data, timeout);
+    return this.delegate.sendData(data, timeout);
   }
 }
 

package/src/core/codemirror/copilot/copilot-config.tsx

@@ -24,13 +24,41 @@ export const CopilotConfig = memo(() => {
     evt.preventDefault();
     setLoading(true);
     try {
-      const { verificationUri, status, userCode } = await initiateSignIn();
+      const result = await initiateSignIn();
+
+      // Validate the response has required fields
+      if (!result?.verificationUri || !result.userCode) {
+        Logger.error("Copilot#trySignIn: Invalid response from sign-in", {
+          result,
+        });
+        setStep("connectionError");
+        toast({
+          title: "GitHub Copilot Connection Error",
+          description:
+            "Unable to connect to GitHub Copilot. Please check your settings and try again.",
+          variant: "danger",
+        });
+        return;
+      }
+
+      const { verificationUri, status, userCode } = result;
       if (isSignedIn(status)) {
         copilotChangeSignIn(true);
       } else {
         setStep("signingIn");
         setLocalData({ url: verificationUri, code: userCode });
       }
+    } catch (error) {
+      Logger.error("Copilot#trySignIn: Error during sign-in", error);
+      setStep("connectionError");
+      toast({
+        title: "GitHub Copilot Connection Error",
+        description:
+          error instanceof Error
+            ? error.message
+            : "Unable to connect to GitHub Copilot. Please check your settings and try again.",
+        variant: "danger",
+      });
     } finally {
       setLoading(false);
     }

package/src/core/codemirror/editing/commands.ts

@@ -1,16 +1,17 @@
 /* Copyright 2024 Marimo. All rights reserved. */
 import { foldAll, unfoldAll } from "@codemirror/language";
 import type { Command, EditorView } from "@codemirror/view";
-import type { Nullable } from "vitest";
 
-export type BulkCommand = (targets: Nullable<EditorView>[]) => boolean;
+type MaybeEditorView = EditorView | undefined | null;
+
+export type BulkCommand = (targets: MaybeEditorView[]) => boolean;
 
 /**
  * Make a bulk command from a single {@type Command} that applies
  * the given command to all targets.
  */
 export function makeBulkCommand(command: Command) {
-  return (targets: Nullable<EditorView>[]) => {
+  return (targets: MaybeEditorView[]) => {
     let changed = false;
     for (const target of targets) {
       if (target) {

package/src/core/codemirror/format.ts

@@ -3,16 +3,21 @@
 import { getIndentUnit } from "@codemirror/language";
 import { StateEffect } from "@codemirror/state";
 import type { EditorView } from "@codemirror/view";
+import type { DialectOptions } from "sql-formatter";
+import { logNever } from "@/utils/assertNever";
 import { Logger } from "@/utils/Logger";
 import { Objects } from "../../utils/objects";
 import { getNotebook } from "../cells/cells";
 import type { CellId } from "../cells/ids";
 import { notebookCellEditorViews } from "../cells/utils";
 import { getResolvedMarimoConfig } from "../config/config";
+import type { ConnectionName } from "../datasets/engines";
 import { getRequestClient } from "../network/requests";
 import { cellActionsState } from "./cells/state";
 import { cellIdState } from "./config/extension";
 import { languageAdapterState } from "./language/extension";
+import { SCHEMA_CACHE } from "./language/languages/sql/completion-store";
+import { isKnownDialect } from "./language/languages/sql/utils";
 import {
   getEditorCodeAsPython,
   updateEditorCodeFromPython,
@@ -77,9 +82,12 @@ export function formatAll() {
  * SQL formatting is much more opinionated than Python formatting, and we
  * don't want to tie the two together (just yet).
  */
-export async function formatSQL(editor: EditorView) {
+export async function formatSQL(editor: EditorView, engine: ConnectionName) {
   // Lazy import sql-formatter
-  const { formatDialect, duckdb } = await import("sql-formatter");
+  const { formatDialect } = await import("sql-formatter");
+
+  const sqlDialect = SCHEMA_CACHE.getInternalDialect(engine);
+  const formatterDialect = await getSqlFormatterDialect(sqlDialect);
 
   // Get language adapter
   const languageAdapter = editor.state.field(languageAdapterState);
@@ -90,11 +98,17 @@ export async function formatSQL(editor: EditorView) {
   }
 
   const codeAsSQL = editor.state.doc.toString();
-  const formattedSQL = formatDialect(codeAsSQL, {
-    dialect: duckdb,
-    tabWidth: tabWidth,
-    useTabs: false,
-  });
+  let formattedSQL: string;
+  try {
+    formattedSQL = formatDialect(codeAsSQL, {
+      dialect: formatterDialect,
+      tabWidth: tabWidth,
+      useTabs: false,
+    });
+  } catch (error) {
+    Logger.error("Error formatting SQL", { error });
+    return;
+  }
 
   // Update Python in the notebook state
   const codeAsPython = languageAdapter.transformIn(formattedSQL)[0];
@@ -111,3 +125,87 @@ export async function formatSQL(editor: EditorView) {
     effects: [formattingChangeEffect.of(true)],
   });
 }
+
+async function getSqlFormatterDialect(
+  sqlDialect: string | null,
+): Promise<DialectOptions> {
+  const {
+    bigquery,
+    db2,
+    db2i,
+    duckdb,
+    hive,
+    mariadb,
+    mysql,
+    tidb,
+    n1ql,
+    plsql,
+    postgresql,
+    redshift,
+    spark,
+    sqlite,
+    sql,
+    trino,
+    transactsql,
+    singlestoredb,
+    snowflake,
+  } = await import("sql-formatter");
+
+  const defaultDialect = sql;
+
+  if (!sqlDialect || !isKnownDialect(sqlDialect)) {
+    return defaultDialect;
+  }
+  switch (sqlDialect) {
+    case "mysql":
+      return mysql;
+    case "mariadb":
+      return mariadb;
+    case "postgres":
+    case "postgresql":
+      return postgresql;
+    case "sqlite":
+      return sqlite;
+    case "db2":
+      return db2;
+    case "db2i":
+      return db2i;
+    case "hive":
+      return hive;
+    case "redshift":
+      return redshift;
+    case "snowflake":
+      return snowflake;
+    case "trino":
+      return trino;
+    case "tidb":
+      return tidb;
+    case "oracle":
+    case "oracledb":
+      return plsql;
+    case "spark":
+      return spark;
+    case "singlestoredb":
+      return singlestoredb;
+    case "couchbase":
+      return n1ql;
+    case "bigquery":
+      return bigquery;
+    case "duckdb":
+      return duckdb;
+    case "mssql":
+    case "sqlserver":
+      return transactsql;
+    case "athena":
+    case "awsathena":
+    case "cassandra":
+    case "noql":
+    case "flink":
+    case "mongodb":
+    case "timescaledb":
+      return sql;
+    default:
+      logNever(sqlDialect);
+      return defaultDialect;
+  }
+}

package/src/core/codemirror/language/languages/sql/completion-store.ts

@@ -122,7 +122,7 @@ class SQLCompletionStore {
     if (!connection) {
       return null;
     }
-    return connection.dialect;
+    return connection.dialect.toLowerCase();
   }
 
   /**

package/src/core/codemirror/language/languages/sql/sql.ts

@@ -137,7 +137,7 @@ export class SQLLanguageAdapter
     _completionConfig: CompletionConfig,
     _hotkeys: HotkeyProvider,
     _placeholderType: PlaceholderType,
-    lspConfig: LSPConfig & { diagnostics: DiagnosticsConfig },
+    _lspConfig: LSPConfig & { diagnostics: DiagnosticsConfig },
   ): Extension[] {
     const extensions = [
       // This can be updated with a dispatch effect
@@ -366,6 +366,7 @@ function connectionNameToParserDialect(
     case "postgres":
       return "PostgreSQL";
     case "db2":
+    case "db2i":
       return "DB2";
     case "mysql":
       return "MySQL";
@@ -381,6 +382,7 @@ function connectionNameToParserDialect(
     case "cassandra":
       return "Noql";
     case "athena":
+    case "awsathena":
       return "Athena";
     case "bigquery":
       return "BigQuery";
@@ -398,6 +400,11 @@ function connectionNameToParserDialect(
     case "oracle":
     case "oracledb":
     case "timescaledb":
+    case "couchbase":
+    case "trino":
+    case "tidb":
+    case "singlestoredb":
+    case "spark":
       Logger.debug("Unsupported dialect", { dialect });
       return null;
     default:

package/src/core/codemirror/language/languages/sql/utils.ts

@@ -23,7 +23,10 @@ import { Logger } from "@/utils/Logger";
 const KNOWN_DIALECTS_ARRAY = [
   "postgresql",
   "postgres",
+  "couchbase",
   "db2",
+  "db2i",
+  "tidb",
   "mysql",
   "sqlite",
   "mssql",
@@ -32,6 +35,8 @@ const KNOWN_DIALECTS_ARRAY = [
   "mariadb",
   "cassandra",
   "noql",
+  "spark",
+  "awsathena",
   "athena",
   "bigquery",
   "hive",
@@ -39,8 +44,10 @@ const KNOWN_DIALECTS_ARRAY = [
   "snowflake",
   "flink",
   "mongodb",
+  "trino",
   "oracle",
   "oracledb",
+  "singlestoredb",
   "timescaledb",
 ] as const;
 const KNOWN_DIALECTS: ReadonlySet<string> = new Set(KNOWN_DIALECTS_ARRAY);
@@ -59,6 +66,7 @@ export function guessDialect(
 ): SQLDialect {
   const dialect = connection.dialect;
   if (!isKnownDialect(dialect)) {
+    Logger.debug("Unknown dialect", { dialect });
     return ModifiedStandardSQL;
   }
 
@@ -86,7 +94,9 @@ export function guessDialect(
       return BigQueryDialect;
     case "timescaledb":
       return PostgreSQL; // TimescaleDB is a PostgreSQL dialect
+    case "awsathena":
     case "athena":
+    case "db2i":
     case "db2":
     case "hive":
     case "redshift":
@@ -94,6 +104,11 @@ export function guessDialect(
     case "flink":
     case "mongodb":
     case "noql":
+    case "couchbase":
+    case "trino":
+    case "tidb":
+    case "singlestoredb":
+    case "spark":
       Logger.debug("Unsupported dialect", { dialect });
       return ModifiedStandardSQL;
     default:

package/src/core/codemirror/language/panel/panel.tsx

@@ -101,7 +101,7 @@ export const LanguagePanelComponent: React.FC<{
               variant="text"
               size="icon"
               onClick={async () => {
-                await formatSQL(view);
+                await formatSQL(view, metadata.engine);
               }}
             >
               <PaintRollerIcon className="h-3 w-3" />

package/src/core/config/__tests__/config-schema.test.ts

@@ -87,6 +87,7 @@ test("default UserConfig - empty", () => {
         "default_auto_download": [],
         "default_sql_output": "auto",
         "on_cell_change": "autorun",
+        "reactive_tests": true,
         "watcher_on_save": "lazy",
       },
       "save": {
@@ -154,6 +155,7 @@ test("default UserConfig - one level", () => {
         "default_auto_download": [],
         "default_sql_output": "auto",
         "on_cell_change": "autorun",
+        "reactive_tests": true,
         "watcher_on_save": "lazy",
       },
       "save": {

package/src/core/config/config-schema.ts

@@ -2,7 +2,7 @@
 import { z } from "zod";
 import { invariant } from "@/utils/invariant";
 import { Logger } from "@/utils/Logger";
-import type { MarimoConfig } from "../network/types";
+import type { MarimoConfig, schemas } from "../network/types";
 
 // This has to be defined in the same file as the zod schema to satisfy zod
 export const PackageManagerNames = [
@@ -44,8 +44,8 @@ export const DEFAULT_AI_MODEL = "openai/gpt-4o";
  */
 const AUTO_DOWNLOAD_FORMATS = ["html", "markdown", "ipynb"] as const;
 
-const COPILOT_MODES = ["manual", "ask", "agent"] as const;
-export type CopilotMode = (typeof COPILOT_MODES)[number];
+export type CopilotMode = NonNullable<schemas["AiConfig"]["mode"]>;
+export const COPILOT_MODES: CopilotMode[] = ["manual", "ask", "agent"];
 
 const AiConfigSchema = z
   .object({
@@ -119,6 +119,7 @@ export const UserConfigSchema = z
         auto_instantiate: z.boolean().prefault(true),
         on_cell_change: z.enum(["lazy", "autorun"]).prefault("autorun"),
         auto_reload: z.enum(["off", "lazy", "autorun"]).prefault("off"),
+        reactive_tests: z.boolean().prefault(true),
         watcher_on_save: z.enum(["lazy", "autorun"]).prefault("lazy"),
         default_sql_output: z.enum(VALID_SQL_OUTPUT_FORMATS).prefault("auto"),
         default_auto_download: z

package/src/core/packages/__tests__/package-input-utils.test.ts

@@ -0,0 +1,93 @@
+/* Copyright 2024 Marimo. All rights reserved. */
+import { describe, expect, it } from "vitest";
+import { stripPackageManagerPrefix } from "../package-input-utils";
+
+describe("stripPackageManagerPrefix", () => {
+  it("should remove 'pip install' prefix", () => {
+    expect(stripPackageManagerPrefix("pip install httpx")).toBe("httpx");
+    expect(stripPackageManagerPrefix("pip install httpx requests")).toBe(
+      "httpx requests",
+    );
+  });
+
+  it("should remove 'pip3 install' prefix", () => {
+    expect(stripPackageManagerPrefix("pip3 install pandas")).toBe("pandas");
+  });
+
+  it("should remove 'uv add' prefix", () => {
+    expect(stripPackageManagerPrefix("uv add numpy")).toBe("numpy");
+    expect(stripPackageManagerPrefix("uv add pandas numpy")).toBe(
+      "pandas numpy",
+    );
+  });
+
+  it("should remove 'uv pip install' prefix", () => {
+    expect(stripPackageManagerPrefix("uv pip install scipy")).toBe("scipy");
+  });
+
+  it("should remove 'poetry add' prefix", () => {
+    expect(stripPackageManagerPrefix("poetry add flask")).toBe("flask");
+  });
+
+  it("should remove 'conda install' prefix", () => {
+    expect(stripPackageManagerPrefix("conda install matplotlib")).toBe(
+      "matplotlib",
+    );
+  });
+
+  it("should remove 'pipenv install' prefix", () => {
+    expect(stripPackageManagerPrefix("pipenv install django")).toBe("django");
+  });
+
+  it("should be case insensitive", () => {
+    expect(stripPackageManagerPrefix("PIP INSTALL httpx")).toBe("httpx");
+    expect(stripPackageManagerPrefix("Pip Install requests")).toBe("requests");
+    expect(stripPackageManagerPrefix("UV ADD numpy")).toBe("numpy");
+  });
+
+  it("should handle extra whitespace", () => {
+    expect(stripPackageManagerPrefix("  pip install   httpx  ")).toBe("httpx");
+    expect(stripPackageManagerPrefix("uv add    pandas   ")).toBe("pandas");
+  });
+
+  it("should return input unchanged if no prefix matches", () => {
+    expect(stripPackageManagerPrefix("httpx")).toBe("httpx");
+    expect(stripPackageManagerPrefix("pandas numpy")).toBe("pandas numpy");
+    expect(stripPackageManagerPrefix("httpx==0.27.0")).toBe("httpx==0.27.0");
+  });
+
+  it("should handle package specifications with versions", () => {
+    expect(stripPackageManagerPrefix("pip install httpx==0.27.0")).toBe(
+      "httpx==0.27.0",
+    );
+    expect(stripPackageManagerPrefix("uv add pandas>=2.0.0")).toBe(
+      "pandas>=2.0.0",
+    );
+  });
+
+  it("should handle git URLs", () => {
+    expect(
+      stripPackageManagerPrefix(
+        "pip install git+https://github.com/encode/httpx",
+      ),
+    ).toBe("git+https://github.com/encode/httpx");
+  });
+
+  it("should handle multiple packages", () => {
+    expect(stripPackageManagerPrefix("pip install httpx requests pandas")).toBe(
+      "httpx requests pandas",
+    );
+  });
+
+  it("should only remove the first matching prefix", () => {
+    // Edge case: input contains prefix-like text multiple times
+    expect(stripPackageManagerPrefix("pip install pip install httpx")).toBe(
+      "pip install httpx",
+    );
+  });
+
+  it("should handle empty string", () => {
+    expect(stripPackageManagerPrefix("")).toBe("");
+    expect(stripPackageManagerPrefix("   ")).toBe("");
+  });
+});

package/src/core/packages/package-input-utils.ts

@@ -0,0 +1,36 @@
+/* Copyright 2024 Marimo. All rights reserved. */
+
+/**
+ * Removes common package manager command prefixes from an input string.
+ * This allows users to paste commands like "pip install httpx" and have
+ * the "pip install" prefix automatically removed.
+ *
+ * @param input - The raw input string that may contain a package manager prefix
+ * @returns The input with any recognized prefix removed and trimmed
+ *
+ * @example
+ * stripPackageManagerPrefix("pip install httpx") // returns "httpx"
+ * stripPackageManagerPrefix("uv add pandas numpy") // returns "pandas numpy"
+ * stripPackageManagerPrefix("httpx") // returns "httpx"
+ */
+export function stripPackageManagerPrefix(input: string): string {
+  const trimmedInput = input.trim();
+
+  const prefixes = [
+    "pip install",
+    "pip3 install",
+    "uv add",
+    "uv pip install",
+    "poetry add",
+    "conda install",
+    "pipenv install",
+  ];
+
+  for (const prefix of prefixes) {
+    if (trimmedInput.toLowerCase().startsWith(prefix.toLowerCase())) {
+      return trimmedInput.slice(prefix.length).trim();
+    }
+  }
+
+  return trimmedInput;
+}

package/src/core/websocket/useMarimoWebSocket.tsx

@@ -331,7 +331,7 @@ export function useMarimoWebSocket(opts: {
       try {
         handleMessage(e);
       } catch (error) {
-        Logger.error("Failed to handle message", error);
+        Logger.error("Failed to handle message", e.data, error);
         toast({
           title: "Failed to handle message",
           description: prettyError(error),

package/src/css/app/Cell.css

@@ -359,10 +359,15 @@
   position: relative;
   z-index: 1;
 
-  &:first-child .cm-editor {
+  /* Round the top of the code editor unless there is output above or the AI prompt input is open */
+  &[data-has-output-above="false"] .cm-editor {
     border-top-left-radius: 9px;
     border-top-right-radius: 9px;
   }
+  div[data-ai-input-open="true"] .cm-editor {
+    border-top-left-radius: 0;
+    border-top-right-radius: 0;
+  }
 
   &:last-child .cm-editor {
     border-bottom-left-radius: 9px;