@marimo-team/frontend 0.16.1 → 0.16.2

package/src/components/editor/output/MarimoErrorOutput.tsx

@@ -1,6 +1,10 @@
 /* Copyright 2024 Marimo. All rights reserved. */
 
-import { NotebookPenIcon, SquareArrowOutUpRightIcon } from "lucide-react";
+import {
+  InfoIcon,
+  NotebookPenIcon,
+  SquareArrowOutUpRightIcon,
+} from "lucide-react";
 import { Fragment, type JSX } from "react";
 import {
   Accordion,
@@ -72,6 +76,8 @@ export const MarimoErrorOutput = ({
     titleContents = "Ancestor stopped";
     alertVariant = "default";
     titleColor = "text-secondary-foreground";
+  } else if (errors.some((e) => e.type === "sql-error")) {
+    titleContents = "SQL Error in statement";
   } else {
     // Check for exception type
     const exceptionError = errors.find((e) => e.type === "exception");
@@ -126,6 +132,10 @@ export const MarimoErrorOutput = ({
   const unknownErrors = errors.filter(
     (e): e is Extract<MarimoError, { type: "unknown" }> => e.type === "unknown",
   );
+  const sqlErrors = errors.filter(
+    (e): e is Extract<MarimoError, { type: "sql-error" }> =>
+      e.type === "sql-error",
+  );
 
   const openScratchpad = () => {
     chromeActions.openApplication("scratchpad");
@@ -485,6 +495,55 @@ export const MarimoErrorOutput = ({
       );
     }
 
+    if (sqlErrors.length > 0) {
+      messages.push(
+        <div key="sql-errors">
+          {sqlErrors.map((error, idx) => {
+            const line =
+              error.sql_line != null ? (error?.sql_line | 0) + 1 : null;
+            const col = error.sql_col != null ? (error?.sql_col | 0) + 1 : null;
+            return (
+              <div key={`sql-error-${idx}`} className="space-y-2">
+                <p className="text-muted-foreground">{error.msg}</p>
+                {error.hint && (
+                  <div className="flex items-start gap-2">
+                    <InfoIcon className="h-4 w-4 text-muted-foreground mt-0.5 flex-shrink-0" />
+                    <pre className="whitespace-pre-wrap text-sm text-muted-foreground">
+                      {error.hint}
+                    </pre>
+                  </div>
+                )}
+                {error.sql_statement && (
+                  <div className="bg-muted/50 p-2 rounded text-xs font-mono">
+                    <pre className="whitespace-pre-wrap">
+                      {error.sql_statement}
+                    </pre>
+                  </div>
+                )}
+                {line !== null && col !== null && (
+                  <p className="text-xs text-muted-foreground">
+                    Error at line {line}, column {col}
+                  </p>
+                )}
+              </div>
+            );
+          })}
+          {cellId && <AutoFixButton errors={sqlErrors} cellId={cellId} />}
+          <Tip title="How to fix SQL errors">
+            <p className="pb-2">
+              SQL parsing errors often occur due to invalid syntax, missing
+              keywords, or unsupported SQL features.
+            </p>
+            <p className="py-2">
+              Check your SQL syntax and ensure you're using supported SQL
+              dialect features. The error location can help you identify the
+              problematic part of your query.
+            </p>
+          </Tip>
+        </div>,
+      );
+    }
+
     return messages;
   };
 

package/src/core/ai/context/providers/cell-output.ts

@@ -9,6 +9,7 @@ import { displayCellName } from "@/core/cells/names";
 import { isOutputEmpty } from "@/core/cells/outputs";
 import type { OutputMessage } from "@/core/kernel/messages";
 import type { JotaiStore } from "@/core/state/jotai";
+import { parseHtmlContent } from "@/utils/dom";
 import { Logger } from "@/utils/Logger";
 import { type AIContextItem, AIContextProvider } from "../registry";
 import { contextToXml } from "../utils";
@@ -64,24 +65,6 @@ function isMediaMimetype(
   return false;
 }
 
-function parseHtmlContent(htmlString: string): string {
-  try {
-    // Create a temporary DOM element to parse HTML
-    const tempDiv = document.createElement("div");
-    tempDiv.innerHTML = htmlString;
-
-    // Extract text content, removing HTML tags
-    const textContent = tempDiv.textContent || tempDiv.innerText || "";
-
-    // Clean up extra whitespace
-    return textContent.replaceAll(/\s+/g, " ").trim();
-  } catch (error) {
-    Logger.error("Error parsing HTML content:", error);
-    // If parsing fails, return the original string
-    return htmlString;
-  }
-}
-
 export class CellOutputContextProvider extends AIContextProvider<CellOutputContextItem> {
   readonly title = "Cell Outputs";
   readonly mentionPrefix = "@";

package/src/core/codemirror/language/__tests__/extension.test.ts

@@ -14,6 +14,7 @@ import {
   languageAdapterState,
   switchLanguage,
 } from "../extension";
+import { exportedForTesting as sqlValidationErrorsForTesting } from "../languages/sql/validation-errors";
 import { languageMetadataField } from "../metadata";
 
 let view: EditorView | null = null;
@@ -258,3 +259,26 @@ describe("switchLanguage", () => {
     });
   });
 });
+
+describe("sqlValidationErrors", () => {
+  const { splitErrorMessage } = sqlValidationErrorsForTesting;
+
+  describe("split error message", () => {
+    it("should split the error message into error type and error message", () => {
+      const error = "SyntaxError: SELECT * FROM df";
+      const { errorType, errorMessage } = splitErrorMessage(error);
+      expect(errorType).toBe("SyntaxError");
+      expect(errorMessage).toBe("SELECT * FROM df");
+    });
+
+    it("should handle multiple colons", () => {
+      const error =
+        "SyntaxError: SELECT * FROM df:SyntaxError: SELECT * FROM df";
+      const { errorType, errorMessage } = splitErrorMessage(error);
+      expect(errorType).toBe("SyntaxError");
+      expect(errorMessage).toBe(
+        "SELECT * FROM df:SyntaxError: SELECT * FROM df",
+      );
+    });
+  });
+});

package/src/core/codemirror/language/__tests__/sql-validation.test.ts

@@ -0,0 +1,133 @@
+/* Copyright 2024 Marimo. All rights reserved. */
+
+import { describe, expect, it } from "vitest";
+import { exportedForTesting } from "../languages/sql/validation-errors";
+
+describe("Error Message Splitting", () => {
+  it("should handle error message splitting correctly", () => {
+    const { splitErrorMessage } = exportedForTesting;
+
+    const result1 = splitErrorMessage("Syntax error: unexpected token");
+    expect(result1.errorType).toBe("Syntax error");
+    expect(result1.errorMessage).toBe("unexpected token");
+
+    const result2 = splitErrorMessage("Multiple: colons: in error");
+    expect(result2.errorType).toBe("Multiple");
+    expect(result2.errorMessage).toBe("colons: in error");
+
+    const result3 = splitErrorMessage("No colon error");
+    expect(result3.errorType).toBe("No colon error");
+    expect(result3.errorMessage).toBe("");
+  });
+});
+
+describe("DuckDB Error Handling", () => {
+  it("should extract codeblock from error with LINE information", () => {
+    const { handleDuckdbError } = exportedForTesting;
+
+    const error =
+      'Binder Error: Referenced column "attacks" not found in FROM clause! Candidate bindings: "Attack", "Total" LINE 1:... from pokemon WHERE \'type_2\' = 32 and attack = 32 and not attacks = \'hi\' ^';
+
+    const result = handleDuckdbError(error);
+
+    expect(result.errorType).toBe("Binder Error");
+    expect(result.errorMessage).toBe(
+      'Referenced column "attacks" not found in FROM clause! Candidate bindings: "Attack", "Total"',
+    );
+    expect(result.codeblock).toBe(
+      "LINE 1:... from pokemon WHERE 'type_2' = 32 and attack = 32 and not attacks = 'hi' ^",
+    );
+  });
+
+  it("should handle error without LINE information", () => {
+    const { handleDuckdbError } = exportedForTesting;
+
+    const error = "Syntax Error: Invalid syntax near WHERE";
+
+    const result = handleDuckdbError(error);
+
+    expect(result.errorType).toBe("Syntax Error");
+    expect(result.errorMessage).toBe("Invalid syntax near WHERE");
+    expect(result.codeblock).toBeUndefined();
+  });
+
+  it("should handle error with LINE at the beginning", () => {
+    const { handleDuckdbError } = exportedForTesting;
+
+    const error = "LINE 1: SELECT * FROM table WHERE invalid_column = 1 ^";
+
+    const result = handleDuckdbError(error);
+
+    expect(result.errorType).toBe("LINE 1");
+    expect(result.errorMessage).toBe(
+      "SELECT * FROM table WHERE invalid_column = 1 ^",
+    );
+    expect(result.codeblock).toBeUndefined();
+  });
+
+  it("should handle error with multiple LINE occurrences", () => {
+    const { handleDuckdbError } = exportedForTesting;
+
+    const error =
+      "Error: Something went wrong LINE 1: SELECT * FROM table WHERE invalid_column = 1 ^";
+
+    const result = handleDuckdbError(error);
+
+    expect(result.errorType).toBe("Error");
+    expect(result.errorMessage).toBe("Something went wrong");
+    expect(result.codeblock).toBe(
+      "LINE 1: SELECT * FROM table WHERE invalid_column = 1 ^",
+    );
+  });
+
+  it("should handle complex error with nested quotes", () => {
+    const { handleDuckdbError } = exportedForTesting;
+
+    const error =
+      "Binder Error: Column \"name\" not found! LINE 1: SELECT * FROM users WHERE name = 'John' AND age > 25 ^";
+
+    const result = handleDuckdbError(error);
+
+    expect(result.errorType).toBe("Binder Error");
+    expect(result.errorMessage).toBe('Column "name" not found!');
+    expect(result.codeblock).toBe(
+      "LINE 1: SELECT * FROM users WHERE name = 'John' AND age > 25 ^",
+    );
+  });
+
+  it("should handle error with LINE but no caret", () => {
+    const { handleDuckdbError } = exportedForTesting;
+
+    const error = "Error: Invalid query LINE 1: SELECT * FROM table";
+
+    const result = handleDuckdbError(error);
+
+    expect(result.errorType).toBe("Error");
+    expect(result.errorMessage).toBe("Invalid query");
+    expect(result.codeblock).toBe("LINE 1: SELECT * FROM table");
+  });
+
+  it("should trim whitespace from codeblock", () => {
+    const { handleDuckdbError } = exportedForTesting;
+
+    const error = "Error: Something wrong   LINE 1: SELECT * FROM table   ^   ";
+
+    const result = handleDuckdbError(error);
+
+    expect(result.errorType).toBe("Error");
+    expect(result.errorMessage).toBe("Something wrong");
+    expect(result.codeblock).toBe("LINE 1: SELECT * FROM table   ^");
+  });
+
+  it("should handle empty error message", () => {
+    const { handleDuckdbError } = exportedForTesting;
+
+    const error = "";
+
+    const result = handleDuckdbError(error);
+
+    expect(result.errorType).toBe("");
+    expect(result.errorMessage).toBe("");
+    expect(result.codeblock).toBeUndefined();
+  });
+});

package/src/core/codemirror/language/languages/sql/sql-mode.ts

@@ -0,0 +1,20 @@
+/* Copyright 2024 Marimo. All rights reserved. */
+
+import { useAtom } from "jotai";
+import { atomWithStorage } from "jotai/utils";
+import { store } from "@/core/state/jotai";
+
+const BASE_KEY = "marimo:notebook-sql-mode";
+
+export type SQLMode = "validate" | "default";
+
+const sqlModeAtom = atomWithStorage<SQLMode>(BASE_KEY, "default");
+
+export function useSQLMode() {
+  const [sqlMode, setSQLMode] = useAtom(sqlModeAtom);
+  return { sqlMode, setSQLMode };
+}
+
+export function getSQLMode() {
+  return store.get(sqlModeAtom);
+}

package/src/core/codemirror/language/languages/sql/sql.ts

@@ -5,7 +5,7 @@ import { insertTab } from "@codemirror/commands";
 import { type SQLDialect, type SQLNamespace, sql } from "@codemirror/lang-sql";
 import type { EditorState, Extension } from "@codemirror/state";
 import { Compartment } from "@codemirror/state";
-import { type EditorView, keymap } from "@codemirror/view";
+import { EditorView, keymap } from "@codemirror/view";
 import type { SyntaxNode, TreeCursor } from "@lezer/common";
 import { parser } from "@lezer/python";
 import {
@@ -16,12 +16,19 @@ import {
 } from "@marimo-team/codemirror-sql";
 import { DuckDBDialect } from "@marimo-team/codemirror-sql/dialects";
 import dedent from "string-dedent";
+import { cellIdState } from "@/core/codemirror/cells/state";
 import { getFeatureFlag } from "@/core/config/feature-flag";
 import {
   dataSourceConnectionsAtom,
   setLatestEngineSelected,
 } from "@/core/datasets/data-source-connections";
-import { type ConnectionName, DUCKDB_ENGINE } from "@/core/datasets/engines";
+import {
+  type ConnectionName,
+  DUCKDB_ENGINE,
+  INTERNAL_SQL_ENGINES,
+} from "@/core/datasets/engines";
+import { ValidateSQL } from "@/core/datasets/request-registry";
+import type { ValidateSQLResult } from "@/core/kernel/messages";
 import { store } from "@/core/state/jotai";
 import { resolvedThemeAtom } from "@/theme/useTheme";
 import { Logger } from "@/utils/Logger";
@@ -37,6 +44,11 @@ import {
   tablesCompletionSource,
 } from "./completion-sources";
 import { SCHEMA_CACHE } from "./completion-store";
+import { getSQLMode } from "./sql-mode";
+import {
+  clearSqlValidationError,
+  setSqlValidationError,
+} from "./validation-errors";
 
 const DEFAULT_DIALECT = DuckDBDialect;
 const DEFAULT_PARSER_DIALECT = "DuckDB";
@@ -64,12 +76,15 @@ export class SQLLanguageAdapter
 {
   readonly type = "sql";
   sqlLinterEnabled: boolean;
+  sqlModeEnabled: boolean;
 
   constructor() {
     try {
       this.sqlLinterEnabled = getFeatureFlag("sql_linter");
+      this.sqlModeEnabled = getFeatureFlag("sql_mode");
     } catch {
       this.sqlLinterEnabled = false;
+      this.sqlModeEnabled = false;
     }
   }
 
@@ -265,6 +280,10 @@ export class SQLLanguageAdapter
       );
     }
 
+    if (this.sqlModeEnabled) {
+      extensions.push(sqlValidationExtension());
+    }
+
     return extensions;
   }
 }
@@ -315,9 +334,14 @@ function getSchema(view: EditorView): SQLNamespace {
 function guessParserDialect(state: EditorState): ParserDialects | null {
   const metadata = getSQLMetadata(state);
   const connectionName = metadata.engine;
+  return connectionNameToParserDialect(connectionName);
+}
+
+function connectionNameToParserDialect(
+  connectionName: ConnectionName,
+): ParserDialects | null {
   const dialect =
     SCHEMA_CACHE.getInternalDialect(connectionName)?.toLowerCase();
-
   switch (dialect) {
     case "postgresql":
     case "postgres":
@@ -543,3 +567,66 @@ function safeDedent(code: string): string {
     return code;
   }
 }
+
+function sqlValidationExtension(): Extension {
+  let debounceTimeout: NodeJS.Timeout | null = null;
+  let lastValidationRequest: string | null = null;
+
+  return EditorView.updateListener.of((update) => {
+    const sqlMode = getSQLMode();
+    if (sqlMode !== "validate") {
+      return;
+    }
+
+    const metadata = getSQLMetadata(update.state);
+    const connectionName = metadata.engine;
+    if (!INTERNAL_SQL_ENGINES.has(connectionName)) {
+      // Currently only internal engines are supported
+      return;
+    }
+
+    if (!update.docChanged) {
+      return;
+    }
+
+    const doc = update.state.doc;
+    const sqlContent = doc.toString();
+
+    // Clear existing timeout
+    if (debounceTimeout) {
+      clearTimeout(debounceTimeout);
+    }
+
+    // Debounce the validation call
+    debounceTimeout = setTimeout(async () => {
+      // Skip if content hasn't changed since last validation
+      if (lastValidationRequest === sqlContent) {
+        return;
+      }
+
+      lastValidationRequest = sqlContent;
+      const cellId = update.view.state.facet(cellIdState);
+
+      if (sqlContent === "") {
+        clearSqlValidationError(cellId);
+        return;
+      }
+
+      try {
+        const result: ValidateSQLResult = await ValidateSQL.request({
+          engine: connectionName,
+          query: sqlContent,
+        });
+
+        if (result.error) {
+          const dialect = connectionNameToParserDialect(connectionName);
+          setSqlValidationError({ cellId, error: result.error, dialect });
+        } else {
+          clearSqlValidationError(cellId);
+        }
+      } catch (error) {
+        Logger.warn("Failed to validate SQL", { error });
+      }
+    }, 300);
+  });
+}

package/src/core/codemirror/language/languages/sql/validation-errors.ts

@@ -0,0 +1,79 @@
+/* Copyright 2024 Marimo. All rights reserved. */
+
+import type { SupportedDialects } from "@marimo-team/codemirror-sql";
+import { atom, useAtomValue } from "jotai";
+import type { CellId } from "@/core/cells/ids";
+import { store } from "@/core/state/jotai";
+
+export interface SQLValidationError {
+  errorType: string;
+  errorMessage: string;
+  codeblock?: string; // Code block that caused the error
+}
+
+type CellToSQLErrors = Map<CellId, SQLValidationError>;
+
+export const sqlValidationErrorsAtom = atom<CellToSQLErrors>(
+  new Map<CellId, SQLValidationError>(),
+);
+
+export const useSqlValidationErrorsForCell = (cellId: CellId) => {
+  const sqlValidationErrors = useAtomValue(sqlValidationErrorsAtom);
+  return sqlValidationErrors.get(cellId);
+};
+
+export function clearSqlValidationError(cellId: CellId) {
+  const sqlValidationErrors = store.get(sqlValidationErrorsAtom);
+  const newErrors = new Map(sqlValidationErrors);
+  newErrors.delete(cellId);
+  store.set(sqlValidationErrorsAtom, newErrors);
+}
+
+export function setSqlValidationError({
+  cellId,
+  error,
+  dialect,
+}: {
+  cellId: CellId;
+  error: string;
+  dialect: SupportedDialects | null;
+}) {
+  const sqlValidationErrors = store.get(sqlValidationErrorsAtom);
+  const newErrors = new Map(sqlValidationErrors);
+
+  const errorResult: SQLValidationError =
+    dialect === "DuckDB" ? handleDuckdbError(error) : splitErrorMessage(error);
+
+  newErrors.set(cellId, errorResult);
+  store.set(sqlValidationErrorsAtom, newErrors);
+}
+
+function handleDuckdbError(error: string): SQLValidationError {
+  const { errorType, errorMessage } = splitErrorMessage(error);
+  let newErrorMessage = errorMessage;
+
+  // Extract the LINE and the rest of the message as codeblock, keep errorMessage as whatever is before
+  let codeblock: string | undefined;
+  const lineIndex = errorMessage.indexOf("LINE ");
+  if (lineIndex !== -1) {
+    codeblock = errorMessage.slice(Math.max(0, lineIndex)).trim();
+    newErrorMessage = errorMessage.slice(0, Math.max(0, lineIndex)).trim();
+  }
+
+  return {
+    errorType,
+    errorMessage: newErrorMessage,
+    codeblock,
+  };
+}
+
+function splitErrorMessage(error: string) {
+  const errorType = error.split(":")[0].trim();
+  const errorMessage = error.split(":").slice(1).join(":").trim();
+  return { errorType, errorMessage };
+}
+
+export const exportedForTesting = {
+  splitErrorMessage,
+  handleDuckdbError,
+};

package/src/core/codemirror/language/panel/panel.tsx

@@ -5,7 +5,8 @@ import { Button } from "@/components/ui/button";
 import { Checkbox } from "@/components/ui/checkbox";
 import { Tooltip, TooltipProvider } from "@/components/ui/tooltip";
 import { normalizeName } from "@/core/cells/names";
-import type { ConnectionName } from "@/core/datasets/engines";
+import { getFeatureFlag } from "@/core/config/feature-flag";
+import { type ConnectionName, DUCKDB_ENGINE } from "@/core/datasets/engines";
 import { useAutoGrowInputProps } from "@/hooks/useAutoGrowInputProps";
 import { formatSQL } from "../../format";
 import { languageAdapterState } from "../extension";
@@ -22,7 +23,7 @@ import {
 import type { LanguageMetadataOf } from "../types";
 import type { QuotePrefixKind } from "../utils/quotes";
 import { getQuotePrefix, MarkdownQuotePrefixTooltip } from "./markdown";
-import { SQLEngineSelect } from "./sql";
+import { SQLEngineSelect, SQLModeSelect } from "./sql";
 
 const Divider = () => <div className="h-4 border-r border-border" />;
 
@@ -70,6 +71,8 @@ export const LanguagePanelComponent: React.FC<{
       updateSQLDialectFromConnection(view, engine);
     };
 
+    const sqlModeEnabled = getFeatureFlag("sql_mode");
+
     actions = (
       <div className="flex flex-1 gap-2 items-center">
         <label className="flex gap-2 items-center">
@@ -95,6 +98,9 @@ export const LanguagePanelComponent: React.FC<{
           onChange={switchEngine}
         />
         <div className="flex items-center gap-2 ml-auto">
+          {sqlModeEnabled && metadata.engine === DUCKDB_ENGINE && (
+            <SQLModeSelect />
+          )}
           <Tooltip content="Format SQL">
             <Button
               variant="text"