@marimo-team/islands 0.18.0 → 0.18.1

package/src/core/cells/__tests__/logs.test.ts

@@ -0,0 +1,371 @@
+/* Copyright 2024 Marimo. All rights reserved. */
+
+import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
+import type { CellMessage } from "../../kernel/messages";
+import { formatLogTimestamp, getCellLogsForMessage } from "../logs";
+
+describe("getCellLogsForMessage", () => {
+  beforeEach(() => {
+    // Mock console.log to avoid cluttering test output
+    vi.spyOn(console, "log").mockImplementation(() => {
+      // no-op
+    });
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  test("handles text/plain MIME type on stdout", () => {
+    const cellMessage: CellMessage = {
+      cell_id: "cell-1",
+      console: [
+        {
+          mimetype: "text/plain",
+          channel: "stdout",
+          data: "Hello, World!",
+          timestamp: 1_234_567_890,
+        },
+      ],
+      output: null,
+      status: "idle",
+      stale_inputs: null,
+      timestamp: 0,
+    };
+
+    const logs = getCellLogsForMessage(cellMessage);
+
+    expect(logs).toHaveLength(1);
+    expect(logs[0]).toEqual({
+      timestamp: 1_234_567_890,
+      level: "stdout",
+      message: "Hello, World!",
+      cellId: "cell-1",
+    });
+  });
+
+  test("handles text/plain MIME type on stderr", () => {
+    const cellMessage: CellMessage = {
+      cell_id: "cell-2",
+      console: [
+        {
+          mimetype: "text/plain",
+          channel: "stderr",
+          data: "Error occurred",
+          timestamp: 1_234_567_890,
+        },
+      ],
+      output: null,
+      status: "idle",
+      stale_inputs: null,
+      timestamp: 0,
+    };
+
+    const logs = getCellLogsForMessage(cellMessage);
+
+    expect(logs).toHaveLength(1);
+    expect(logs[0]).toEqual({
+      timestamp: 1_234_567_890,
+      level: "stderr",
+      message: "Error occurred",
+      cellId: "cell-2",
+    });
+  });
+
+  test("handles text/html MIME type and strips HTML tags", () => {
+    const cellMessage: CellMessage = {
+      cell_id: "cell-3",
+      console: [
+        {
+          mimetype: "text/html",
+          channel: "stdout",
+          data: '<span style="color: red;">Error: Something went wrong</span>',
+          timestamp: 1_234_567_890,
+        },
+      ],
+      output: null,
+      status: "idle",
+      stale_inputs: null,
+      timestamp: 0,
+    };
+
+    const logs = getCellLogsForMessage(cellMessage);
+
+    expect(logs).toHaveLength(1);
+    expect(logs[0]).toEqual({
+      timestamp: 1_234_567_890,
+      level: "stdout",
+      message: "Error: Something went wrong",
+      cellId: "cell-3",
+    });
+  });
+
+  test("handles text/html MIME type on stderr", () => {
+    const cellMessage: CellMessage = {
+      cell_id: "cell-4",
+      console: [
+        {
+          mimetype: "text/html",
+          channel: "stderr",
+          data: "<div><strong>Critical Error:</strong> System failure</div>",
+          timestamp: 1_234_567_890,
+        },
+      ],
+      output: null,
+      status: "idle",
+      stale_inputs: null,
+      timestamp: 0,
+    };
+
+    const logs = getCellLogsForMessage(cellMessage);
+
+    expect(logs).toHaveLength(1);
+    expect(logs[0]).toEqual({
+      timestamp: 1_234_567_890,
+      level: "stderr",
+      message: "Critical Error: System failure",
+      cellId: "cell-4",
+    });
+  });
+
+  test("handles application/vnd.marimo+traceback MIME type and strips HTML", () => {
+    const cellMessage: CellMessage = {
+      cell_id: "cell-5",
+      console: [
+        {
+          mimetype: "application/vnd.marimo+traceback",
+          channel: "marimo-error",
+          data: '<div class="traceback"><span style="color: red;">Traceback (most recent call last):</span><pre>  File "test.py", line 1</pre></div>',
+          timestamp: 1_234_567_890,
+        },
+      ],
+      output: null,
+      status: "idle",
+      stale_inputs: null,
+      timestamp: 0,
+    };
+
+    const logs = getCellLogsForMessage(cellMessage);
+
+    expect(logs).toHaveLength(1);
+    expect(logs[0].level).toBe("stderr"); // marimo-error should be treated as stderr
+    expect(logs[0].message).toContain("Traceback (most recent call last):");
+    expect(logs[0].message).toContain('File "test.py", line 1');
+    expect(logs[0].cellId).toBe("cell-5");
+  });
+
+  test("handles multiple console outputs with different MIME types", () => {
+    const cellMessage: CellMessage = {
+      cell_id: "cell-7",
+      console: [
+        {
+          mimetype: "text/plain",
+          channel: "stdout",
+          data: "Plain text output",
+          timestamp: 1_234_567_890,
+        },
+        {
+          mimetype: "text/html",
+          channel: "stdout",
+          data: "<span>HTML output</span>",
+          timestamp: 1_234_567_891,
+        },
+        {
+          mimetype: "application/vnd.marimo+traceback",
+          channel: "stderr",
+          data: "<div>Traceback error</div>",
+          timestamp: 1_234_567_892,
+        },
+      ],
+      output: null,
+      status: "idle",
+      stale_inputs: null,
+      timestamp: 0,
+    };
+
+    const logs = getCellLogsForMessage(cellMessage);
+
+    expect(logs).toHaveLength(3);
+    expect(logs[0].message).toBe("Plain text output");
+    expect(logs[1].message).toBe("HTML output");
+    expect(logs[2].message).toBe("Traceback error");
+  });
+
+  test("uses Date.now() when timestamp is missing", () => {
+    const now = Date.now();
+    vi.spyOn(Date, "now").mockReturnValue(now);
+
+    const cellMessage: CellMessage = {
+      cell_id: "cell-8",
+      console: [
+        {
+          mimetype: "text/plain",
+          channel: "stdout",
+          data: "No timestamp",
+          // timestamp is undefined
+        },
+      ],
+      output: null,
+      status: "idle",
+      stale_inputs: null,
+      timestamp: 0,
+    };
+
+    const logs = getCellLogsForMessage(cellMessage);
+
+    expect(logs).toHaveLength(1);
+    expect(logs[0].timestamp).toBe(now);
+  });
+
+  test("ignores unsupported MIME types", () => {
+    const cellMessage: CellMessage = {
+      cell_id: "cell-9",
+      console: [
+        {
+          mimetype: "application/json",
+          channel: "stdout",
+          data: '{"key": "value"}',
+          timestamp: 1_234_567_890,
+        },
+      ],
+      output: null,
+      status: "idle",
+      stale_inputs: null,
+      timestamp: 0,
+    };
+
+    const logs = getCellLogsForMessage(cellMessage);
+
+    expect(logs).toHaveLength(0);
+  });
+
+  test("ignores non-logging channels", () => {
+    const cellMessage: CellMessage = {
+      cell_id: "cell-10",
+      console: [
+        {
+          mimetype: "text/plain",
+          channel: "pdb" as unknown as "stdout", // Non-logging channel
+          data: "Should be ignored",
+          timestamp: 1_234_567_890,
+        },
+      ],
+      output: null,
+      status: "idle",
+      stale_inputs: null,
+      timestamp: 0,
+    };
+
+    const logs = getCellLogsForMessage(cellMessage);
+
+    expect(logs).toHaveLength(0);
+  });
+
+  test("returns empty array when console is null", () => {
+    const cellMessage: CellMessage = {
+      cell_id: "cell-11",
+      console: null as unknown as CellMessage["console"],
+      output: null,
+      status: "idle",
+      stale_inputs: null,
+      timestamp: 0,
+    };
+
+    const logs = getCellLogsForMessage(cellMessage);
+
+    expect(logs).toHaveLength(0);
+  });
+
+  test("handles complex HTML with nested elements in text/html", () => {
+    const cellMessage: CellMessage = {
+      cell_id: "cell-12",
+      console: [
+        {
+          mimetype: "text/html",
+          channel: "stdout",
+          data: "<div><span>Nested</span> <strong>HTML</strong> <em>content</em></div>",
+          timestamp: 1_234_567_890,
+        },
+      ],
+      output: null,
+      status: "idle",
+      stale_inputs: null,
+      timestamp: 0,
+    };
+
+    const logs = getCellLogsForMessage(cellMessage);
+
+    expect(logs).toHaveLength(1);
+    expect(logs[0].message).toBe("Nested HTML content");
+  });
+
+  test("handles marimo-error channel as stderr level", () => {
+    const cellMessage: CellMessage = {
+      cell_id: "cell-13",
+      console: [
+        {
+          mimetype: "text/plain",
+          channel: "marimo-error",
+          data: "Internal error",
+          timestamp: 1_234_567_890,
+        },
+      ],
+      output: null,
+      status: "idle",
+      stale_inputs: null,
+      timestamp: 0,
+    };
+
+    const logs = getCellLogsForMessage(cellMessage);
+
+    expect(logs).toHaveLength(1);
+    expect(logs[0].level).toBe("stderr");
+  });
+});
+
+describe("formatLogTimestamp", () => {
+  test("formats unix timestamp correctly", () => {
+    // January 1, 2024, 12:00:00 PM UTC
+    const timestamp = 1_704_110_400;
+    const result = formatLogTimestamp(timestamp);
+
+    // The result depends on the timezone, so we just check it's not empty
+    // and contains expected time format elements
+    expect(result).toBeTruthy();
+    expect(result).toMatch(/\d+:\d+:\d+/); // Should contain time format
+  });
+
+  test("formats timestamp with AM/PM notation", () => {
+    const timestamp = 1_704_110_400; // Noon
+    const result = formatLogTimestamp(timestamp);
+
+    // Should contain AM or PM
+    expect(result).toMatch(/AM|PM/);
+  });
+
+  test("returns 'Invalid Date' for invalid timestamp", () => {
+    const invalidTimestamp = Number.NaN;
+    const result = formatLogTimestamp(invalidTimestamp);
+
+    expect(result).toBe("Invalid Date");
+  });
+
+  test("handles edge case: zero timestamp", () => {
+    const timestamp = 0; // Unix epoch
+    const result = formatLogTimestamp(timestamp);
+
+    // Should format successfully (Jan 1, 1970)
+    expect(result).toBeTruthy();
+    expect(result).toMatch(/\d+:\d+:\d+/);
+  });
+
+  test("handles recent timestamp", () => {
+    // Use a recent timestamp (seconds since epoch)
+    const timestamp = Math.floor(Date.now() / 1000);
+    const result = formatLogTimestamp(timestamp);
+
+    expect(result).toBeTruthy();
+    expect(result).toMatch(/\d+:\d+:\d+/);
+    expect(result).toMatch(/AM|PM/);
+  });
+});

package/src/core/cells/logs.ts

@@ -2,7 +2,8 @@
 
 import { fromUnixTime } from "date-fns";
 import { toast } from "@/components/ui/use-toast";
-import { invariant } from "@/utils/invariant";
+import { parseHtmlContent } from "@/utils/dom";
+import { Strings } from "@/utils/strings";
 import type { CellMessage, OutputMessage } from "../kernel/messages";
 import { isErrorMime } from "../mime";
 import type { CellId } from "./ids";
@@ -21,21 +22,31 @@ export function getCellLogsForMessage(cell: CellMessage): CellLog[] {
   const consoleOutputs: OutputMessage[] = [cell.console].filter(Boolean).flat();
 
   for (const output of consoleOutputs) {
-    if (output.mimetype === "text/plain") {
-      invariant(typeof output.data === "string", "expected string");
+    // Handle text/plain, text/html, and traceback MIME types
+    const isTextPlain = output.mimetype === "text/plain";
+    const isHtml =
+      output.mimetype === "text/html" ||
+      output.mimetype === "application/vnd.marimo+traceback";
+
+    if (isTextPlain || isHtml) {
       const isError =
         output.channel === "stderr" || output.channel === "marimo-error";
       switch (output.channel) {
         case "stdout":
         case "stderr":
-        case "marimo-error":
+        case "marimo-error": {
+          // Convert data to string and process based on MIME type
+          const rawData = Strings.asString(output.data);
+          const message = isHtml ? parseHtmlContent(rawData) : rawData;
+
           logs.push({
             timestamp: output.timestamp || Date.now(),
             level: isError ? "stderr" : "stdout",
-            message: output.data,
+            message: message,
             cellId: cell.cell_id as CellId,
           });
           break;
+        }
         default:
           break;
       }

package/src/core/codemirror/__tests__/format.test.ts

@@ -11,6 +11,7 @@ import { getNotebook } from "@/core/cells/cells";
 import type { CellId } from "@/core/cells/ids";
 import { notebookCellEditorViews } from "@/core/cells/utils";
 import { getResolvedMarimoConfig } from "@/core/config/config";
+import type { ConnectionName } from "@/core/datasets/engines";
 import { OverridingHotkeyProvider } from "@/core/hotkeys/hotkeys";
 import { requestClientAtom } from "@/core/network/requests";
 import type { MarimoConfig } from "@/core/network/types";
@@ -198,7 +199,7 @@ describe("format", () => {
       const editor = createEditor("SELECT * FROM table WHERE id = 1", cellId);
       switchLanguage(editor, { language: "sql" });
 
-      await formatSQL(editor);
+      await formatSQL(editor, "duckdb" as ConnectionName);
 
       // Check that the SQL was formatted
       expect(editor.state.doc.toString()).toMatchInlineSnapshot(`
@@ -221,7 +222,7 @@ describe("format", () => {
       const editor = createEditor("SELECT * FROM table WHERE id = 1", cellId);
       switchLanguage(editor, { language: "python" });
 
-      await formatSQL(editor);
+      await formatSQL(editor, "duckdb" as ConnectionName);
 
       // Check that the SQL was not formatted
       expect(editor.state.doc.toString()).toBe(
@@ -230,4 +231,29 @@ describe("format", () => {
       expect(updateCellCode).not.toHaveBeenCalled();
     });
   });
+
+  it("should format SQL code with different dialect", async () => {
+    const cellId = "1" as CellId;
+    const editor = createEditor(
+      "SELECT * FROM `table.dot` WHERE id = 1",
+      cellId,
+    );
+    switchLanguage(editor, { language: "sql" });
+
+    await formatSQL(editor, "mysql" as ConnectionName); // mysql uses backticks for identifiers
+
+    expect(editor.state.doc.toString()).toMatchInlineSnapshot(`
+      "SELECT
+        *
+      FROM
+        \`table.dot\`
+      WHERE
+        id = 1"
+    `);
+    expect(updateCellCode).toHaveBeenCalledWith({
+      cellId: cellId,
+      code: editor.state.doc.toString(),
+      formattingChange: true,
+    });
+  });
 });

package/src/core/codemirror/copilot/copilot-config.tsx

@@ -27,7 +27,7 @@ export const CopilotConfig = memo(() => {
       const result = await initiateSignIn();
 
       // Validate the response has required fields
-      if (!result || !result.verificationUri || !result.userCode) {
+      if (!result?.verificationUri || !result.userCode) {
         Logger.error("Copilot#trySignIn: Invalid response from sign-in", {
           result,
         });

package/src/core/codemirror/editing/commands.ts

@@ -1,16 +1,17 @@
 /* Copyright 2024 Marimo. All rights reserved. */
 import { foldAll, unfoldAll } from "@codemirror/language";
 import type { Command, EditorView } from "@codemirror/view";
-import type { Nullable } from "vitest";
 
-export type BulkCommand = (targets: Nullable<EditorView>[]) => boolean;
+type MaybeEditorView = EditorView | undefined | null;
+
+export type BulkCommand = (targets: MaybeEditorView[]) => boolean;
 
 /**
  * Make a bulk command from a single {@type Command} that applies
  * the given command to all targets.
  */
 export function makeBulkCommand(command: Command) {
-  return (targets: Nullable<EditorView>[]) => {
+  return (targets: MaybeEditorView[]) => {
     let changed = false;
     for (const target of targets) {
       if (target) {

package/src/core/codemirror/format.ts

@@ -3,16 +3,21 @@
 import { getIndentUnit } from "@codemirror/language";
 import { StateEffect } from "@codemirror/state";
 import type { EditorView } from "@codemirror/view";
+import type { DialectOptions } from "sql-formatter";
+import { logNever } from "@/utils/assertNever";
 import { Logger } from "@/utils/Logger";
 import { Objects } from "../../utils/objects";
 import { getNotebook } from "../cells/cells";
 import type { CellId } from "../cells/ids";
 import { notebookCellEditorViews } from "../cells/utils";
 import { getResolvedMarimoConfig } from "../config/config";
+import type { ConnectionName } from "../datasets/engines";
 import { getRequestClient } from "../network/requests";
 import { cellActionsState } from "./cells/state";
 import { cellIdState } from "./config/extension";
 import { languageAdapterState } from "./language/extension";
+import { SCHEMA_CACHE } from "./language/languages/sql/completion-store";
+import { isKnownDialect } from "./language/languages/sql/utils";
 import {
   getEditorCodeAsPython,
   updateEditorCodeFromPython,
@@ -77,9 +82,12 @@ export function formatAll() {
  * SQL formatting is much more opinionated than Python formatting, and we
  * don't want to tie the two together (just yet).
  */
-export async function formatSQL(editor: EditorView) {
+export async function formatSQL(editor: EditorView, engine: ConnectionName) {
   // Lazy import sql-formatter
-  const { formatDialect, duckdb } = await import("sql-formatter");
+  const { formatDialect } = await import("sql-formatter");
+
+  const sqlDialect = SCHEMA_CACHE.getInternalDialect(engine);
+  const formatterDialect = await getSqlFormatterDialect(sqlDialect);
 
   // Get language adapter
   const languageAdapter = editor.state.field(languageAdapterState);
@@ -90,11 +98,17 @@ export async function formatSQL(editor: EditorView) {
   }
 
   const codeAsSQL = editor.state.doc.toString();
-  const formattedSQL = formatDialect(codeAsSQL, {
-    dialect: duckdb,
-    tabWidth: tabWidth,
-    useTabs: false,
-  });
+  let formattedSQL: string;
+  try {
+    formattedSQL = formatDialect(codeAsSQL, {
+      dialect: formatterDialect,
+      tabWidth: tabWidth,
+      useTabs: false,
+    });
+  } catch (error) {
+    Logger.error("Error formatting SQL", { error });
+    return;
+  }
 
   // Update Python in the notebook state
   const codeAsPython = languageAdapter.transformIn(formattedSQL)[0];
@@ -111,3 +125,87 @@ export async function formatSQL(editor: EditorView) {
     effects: [formattingChangeEffect.of(true)],
   });
 }
+
+async function getSqlFormatterDialect(
+  sqlDialect: string | null,
+): Promise<DialectOptions> {
+  const {
+    bigquery,
+    db2,
+    db2i,
+    duckdb,
+    hive,
+    mariadb,
+    mysql,
+    tidb,
+    n1ql,
+    plsql,
+    postgresql,
+    redshift,
+    spark,
+    sqlite,
+    sql,
+    trino,
+    transactsql,
+    singlestoredb,
+    snowflake,
+  } = await import("sql-formatter");
+
+  const defaultDialect = sql;
+
+  if (!sqlDialect || !isKnownDialect(sqlDialect)) {
+    return defaultDialect;
+  }
+  switch (sqlDialect) {
+    case "mysql":
+      return mysql;
+    case "mariadb":
+      return mariadb;
+    case "postgres":
+    case "postgresql":
+      return postgresql;
+    case "sqlite":
+      return sqlite;
+    case "db2":
+      return db2;
+    case "db2i":
+      return db2i;
+    case "hive":
+      return hive;
+    case "redshift":
+      return redshift;
+    case "snowflake":
+      return snowflake;
+    case "trino":
+      return trino;
+    case "tidb":
+      return tidb;
+    case "oracle":
+    case "oracledb":
+      return plsql;
+    case "spark":
+      return spark;
+    case "singlestoredb":
+      return singlestoredb;
+    case "couchbase":
+      return n1ql;
+    case "bigquery":
+      return bigquery;
+    case "duckdb":
+      return duckdb;
+    case "mssql":
+    case "sqlserver":
+      return transactsql;
+    case "athena":
+    case "awsathena":
+    case "cassandra":
+    case "noql":
+    case "flink":
+    case "mongodb":
+    case "timescaledb":
+      return sql;
+    default:
+      logNever(sqlDialect);
+      return defaultDialect;
+  }
+}

package/src/core/codemirror/language/languages/sql/completion-store.ts

@@ -122,7 +122,7 @@ class SQLCompletionStore {
     if (!connection) {
       return null;
     }
-    return connection.dialect;
+    return connection.dialect.toLowerCase();
   }
 
   /**

package/src/core/codemirror/language/languages/sql/sql.ts

@@ -137,7 +137,7 @@ export class SQLLanguageAdapter
     _completionConfig: CompletionConfig,
     _hotkeys: HotkeyProvider,
     _placeholderType: PlaceholderType,
-    lspConfig: LSPConfig & { diagnostics: DiagnosticsConfig },
+    _lspConfig: LSPConfig & { diagnostics: DiagnosticsConfig },
   ): Extension[] {
     const extensions = [
       // This can be updated with a dispatch effect
@@ -366,6 +366,7 @@ function connectionNameToParserDialect(
     case "postgres":
       return "PostgreSQL";
     case "db2":
+    case "db2i":
       return "DB2";
     case "mysql":
       return "MySQL";
@@ -381,6 +382,7 @@ function connectionNameToParserDialect(
     case "cassandra":
       return "Noql";
     case "athena":
+    case "awsathena":
       return "Athena";
     case "bigquery":
       return "BigQuery";
@@ -398,6 +400,11 @@ function connectionNameToParserDialect(
     case "oracle":
     case "oracledb":
     case "timescaledb":
+    case "couchbase":
+    case "trino":
+    case "tidb":
+    case "singlestoredb":
+    case "spark":
       Logger.debug("Unsupported dialect", { dialect });
       return null;
     default: