@marimo-team/frontend 0.23.16-dev51 → 0.23.16-dev53

package/src/core/codemirror/language/languages/sql/sql.ts

@@ -66,7 +66,7 @@ import { getSQLMode, type SQLMode } from "./sql-mode";
 import { isKnownDialect } from "./utils";
 
 const DEFAULT_DIALECT = DuckDBDialect;
-const DEFAULT_PARSER_DIALECT = "DuckDB";
+const DEFAULT_PARSER_DIALECT: ParserDialects = "DuckDB";
 
 // A compartment for the SQL config, so we can update the config of codemirror
 const sqlConfigCompartment = new Compartment();
@@ -372,6 +372,8 @@ class DialectAwareSqlStructureAnalyzer extends SqlStructureAnalyzer {
 
 class CustomSqlParser extends NodeSqlParser {
   private validationTimeout: number | null = null;
+  private pendingValidationResolve: ((errors: SqlParseError[]) => void) | null =
+    null;
   private readonly VALIDATION_DELAY_MS = 300; // Wait 300ms after user stops typing
   private isFocused = false; // Only validate if the editor is focused
 
@@ -379,37 +381,51 @@ class CustomSqlParser extends NodeSqlParser {
     this.isFocused = focused;
   }
 
+  private resolvePendingValidation(errors: SqlParseError[]): void {
+    this.pendingValidationResolve?.(errors);
+    this.pendingValidationResolve = null;
+  }
+
   private async validateWithDelay(
     sql: string,
-    engine: string,
+    engine: ConnectionName,
     dialect: ParserDialects | null,
   ): Promise<SqlParseError[]> {
-    // Clear any existing delay call
+    // Clear any existing delay call, resolving its promise so a superseded
+    // request doesn't hang forever.
     if (this.validationTimeout) {
       window.clearTimeout(this.validationTimeout);
+      this.resolvePendingValidation([]);
     }
 
     // Set up a new request to be called after the delay
     return new Promise((resolve) => {
+      this.pendingValidationResolve = resolve;
       this.validationTimeout = window.setTimeout(async () => {
+        this.validationTimeout = null;
+
         // Only validate if the editor is still focused
         if (!this.isFocused) {
-          resolve([]);
+          this.resolvePendingValidation([]);
           return;
         }
 
         try {
-          const sqlMode = getSQLMode();
+          // For validate mode, we run EXPLAIN queries on the engine, which can be
+          // expensive for remote databases. So, we only run for internal engines.
+          const sqlMode = INTERNAL_SQL_ENGINES.has(engine)
+            ? getSQLMode()
+            : "default";
           const result = await validateSQL(sql, engine, dialect, sqlMode);
           if (result.error) {
             Logger.error("Failed to validate SQL", { error: result.error });
-            resolve([]);
+            this.resolvePendingValidation([]);
             return;
           }
-          resolve(result.parse_result?.errors ?? []);
+          this.resolvePendingValidation(result.parse_result?.errors ?? []);
         } catch (error) {
           Logger.error("Failed to validate SQL", { error });
-          resolve([]);
+          this.resolvePendingValidation([]);
         }
       }, this.VALIDATION_DELAY_MS);
     });
@@ -420,19 +436,23 @@ class CustomSqlParser extends NodeSqlParser {
     opts: { state: EditorState },
   ): Promise<SqlParseError[]> {
     const metadata = getSQLMetadata(opts.state);
+    const dialect = connectionNameToParserDialect(metadata.engine);
 
     // Only validate if the editor is focused
     if (!this.isFocused) {
       return [];
     }
 
-    // Only perform custom validation for DuckDB
-    if (!INTERNAL_SQL_ENGINES.has(metadata.engine)) {
+    // Only perform custom validation for DuckDB as we have a custom validation endpoint for it.
+    if (!isDuckDBConnection(metadata.engine, dialect)) {
       return super.validateSql(sql, opts);
     }
 
-    const dialect = guessParserDialect(opts.state);
-    return this.validateWithDelay(sql, metadata.engine, dialect);
+    return this.validateWithDelay(
+      sql,
+      metadata.engine,
+      dialect ?? DEFAULT_PARSER_DIALECT,
+    );
   }
 
   override async parse(
@@ -441,9 +461,10 @@ class CustomSqlParser extends NodeSqlParser {
   ): Promise<NodeSqlParserResult> {
     const metadata = getSQLMetadata(opts.state);
     const engine = metadata.engine;
+    const dialect = connectionNameToParserDialect(engine);
 
     // For now, always return success for DuckDB
-    if (engine === DUCKDB_ENGINE) {
+    if (isDuckDBConnection(engine, dialect)) {
       return { success: true, errors: [] };
     }
 
@@ -451,6 +472,13 @@ class CustomSqlParser extends NodeSqlParser {
   }
 }
 
+function isDuckDBConnection(
+  engine: ConnectionName,
+  dialect: ParserDialects | null,
+): boolean {
+  return engine === DUCKDB_ENGINE || dialect === "DuckDB";
+}
+
 /**
  * Update the SQL dialect in the editor view.
  */

package/src/core/datasets/data-source-discovery.ts

@@ -0,0 +1,5 @@
+/* Copyright 2026 Marimo. All rights reserved. */
+
+import type { components } from "@marimo-team/marimo-api";
+
+export type DetectedDataSource = components["schemas"]["DetectedDataSource"];

package/src/core/datasets/request-registry.ts

@@ -1,5 +1,6 @@
 /* Copyright 2026 Marimo. All rights reserved. */
 import type {
+  DataSourceDiscoveryResult,
   SQLSchemaListPreview,
   SQLTableListPreview,
   SQLTablePreview,
@@ -19,6 +20,16 @@ import type {
 // The backend returns data tables, which could also exist in other engines, dbs, schemas
 // Thus, we use the request ID pattern to match the response to the request
 
+export const DiscoverDataSources = new DeferredRequestRegistry<
+  {},
+  DataSourceDiscoveryResult
+>("data-source-discovery-result", async (requestId) => {
+  const client = getRequestClient();
+  await client.discoverDataSources({
+    requestId,
+  });
+});
+
 export const PreviewSQLTable = new DeferredRequestRegistry<
   Omit<PreviewSQLTableRequest, "requestId">,
   SQLTablePreview

package/src/core/islands/bootstrap.ts

@@ -194,6 +194,7 @@ function handleMessage(
       case "sql-schema-list-preview":
       case "datasets":
       case "data-source-connections":
+      case "data-source-discovery-result":
       case "validate-sql-result":
       case "storage-namespaces":
       case "storage-entries":

package/src/core/islands/bridge.ts

@@ -341,6 +341,7 @@ export class IslandsPyodideBridge implements RunRequests, EditRequests {
   previewSQLTableList = throwNotImplemented;
   previewSQLSchemaList = throwNotImplemented;
   previewDataSourceConnection = throwNotImplemented;
+  discoverDataSources = throwNotImplemented;
   validateSQL = throwNotImplemented;
   openFile = throwNotImplemented;
   sendListFiles = throwNotImplemented;

package/src/core/kernel/messages.ts

@@ -44,6 +44,8 @@ export type SQLTableListPreview =
 export type SQLSchemaListPreview =
   NotificationMessageData<"sql-schema-list-preview">;
 export type ValidateSQLResult = NotificationMessageData<"validate-sql-result">;
+export type DataSourceDiscoveryResult =
+  NotificationMessageData<"data-source-discovery-result">;
 export type SecretKeysResult = NotificationMessageData<"secret-keys-result">;
 export type StartupLogs = NotificationMessageData<"startup-logs">;
 export type CellMessage = NotificationMessageData<"cell-op">;

package/src/core/network/__tests__/requests-lazy.test.ts

@@ -114,6 +114,21 @@ describe("createLazyRequests", () => {
     expect(mockInit).toHaveBeenCalledTimes(1);
   });
 
+  it("starts the kernel for datasource discovery", async () => {
+    mockDelegate.discoverDataSources = vi.fn().mockResolvedValue(null);
+    const lazyRequests = createLazyRequests(
+      mockDelegate,
+      mockGetRuntimeManager,
+    );
+
+    await lazyRequests.discoverDataSources({
+      requestId: requestId("discovery"),
+    });
+
+    expect(mockInit).toHaveBeenCalledOnce();
+    expect(mockDelegate.discoverDataSources).toHaveBeenCalledOnce();
+  });
+
   it("should only call init once across multiple requests", async () => {
     const lazyRequests = createLazyRequests(
       mockDelegate,

package/src/core/network/__tests__/requests-network.test.ts

@@ -130,6 +130,20 @@ describe("createNetworkRequests", () => {
       expect(mockClient.GET).toHaveBeenCalledWith("/api/export/availability");
     });
 
+    it("discoverDataSources should POST to the discovery endpoint", async () => {
+      const requests = createNetworkRequests();
+      const request = { requestId: "discovery-request" } as any;
+      await requests.discoverDataSources(request);
+
+      expect(mockClient.POST).toHaveBeenCalledWith(
+        "/api/datasources/discover",
+        expect.objectContaining({
+          body: request,
+          params: expect.anything(),
+        }),
+      );
+    });
+
     it("getPackageList should not require a kernel connection", async () => {
       const { waitForConnectionOpen, waitForConnectionOpenIfNotebook } =
         await import("../connection");

package/src/core/network/requests-lazy.ts

@@ -51,6 +51,7 @@ const ACTIONS: Record<keyof AllRequests, Action> = {
   sendRunScratchpad: "startConnection",
   saveAppConfig: "startConnection",
   saveCellConfig: "startConnection",
+  discoverDataSources: "startConnection",
 
   // Export operations start a connection
   exportAsHTML: "startConnection",

package/src/core/network/requests-network.ts

@@ -267,6 +267,14 @@ export function createNetworkRequests(): EditRequests & RunRequests {
         })
         .then(handleResponseReturnNull);
     },
+    discoverDataSources: (request) => {
+      return getClient()
+        .POST("/api/datasources/discover", {
+          body: request,
+          params: getParams(),
+        })
+        .then(handleResponseReturnNull);
+    },
     validateSQL: (request) => {
       return getClient()
         .POST("/api/sql/validate", {

package/src/core/network/requests-static.ts

@@ -62,6 +62,7 @@ export function createStaticRequests(): EditRequests & RunRequests {
     previewSQLTableList: throwNotInEditMode,
     previewSQLSchemaList: throwNotInEditMode,
     previewDataSourceConnection: throwNotInEditMode,
+    discoverDataSources: throwNotInEditMode,
     validateSQL: throwNotInEditMode,
     openFile: throwNotInEditMode,
     getUsageStats: throwNotInEditMode,

package/src/core/network/requests-toasting.tsx

@@ -43,6 +43,7 @@ export function createErrorToastingRequests(
     previewSQLTableList: "Failed to fetch SQL table list",
     previewSQLSchemaList: "Failed to fetch SQL schema list",
     previewDataSourceConnection: "Failed to preview data source connection",
+    discoverDataSources: "Failed to discover data sources",
     validateSQL: "Failed to validate SQL",
     openFile: "Failed to open file",
     getUsageStats: "", // No toast

package/src/core/network/types.ts

@@ -75,6 +75,7 @@ export type ListSQLTablesRequest = schemas["ListSQLTablesRequest"];
 export type ListSQLSchemasRequest = schemas["ListSQLSchemasRequest"];
 export type ListDataSourceConnectionRequest =
   schemas["ListDataSourceConnectionRequest"];
+export type DiscoverDataSourcesRequest = schemas["DiscoverDataSourcesRequest"];
 export type ValidateSQLRequest = schemas["ValidateSQLRequest"];
 export type DebugCellRequest = schemas["DebugCellRequest"];
 export type SetBreakpointsRequest = schemas["SetBreakpointsRequest"];
@@ -181,6 +182,7 @@ export interface EditRequests {
   previewDataSourceConnection: (
     request: ListDataSourceConnectionRequest,
   ) => Promise<null>;
+  discoverDataSources: (request: DiscoverDataSourcesRequest) => Promise<null>;
   validateSQL: (request: ValidateSQLRequest) => Promise<null>;
   openFile: (request: { path: string; lineNumber?: number }) => Promise<null>;
   getUsageStats: () => Promise<UsageResponse>;

package/src/core/wasm/bridge.ts

@@ -630,6 +630,16 @@ export class PyodideBridge implements RunRequests, EditRequests {
     return null;
   };
 
+  discoverDataSources: EditRequests["discoverDataSources"] = async (
+    request,
+  ) => {
+    await this.putControlRequest({
+      type: "discover-data-sources",
+      ...request,
+    });
+    return null;
+  };
+
   getUsageStats = throwNotImplemented;
   getEnvironmentInfo: EditRequests["getEnvironmentInfo"] = async () => {
     const response = await this.rpc.proxy.request.bridge({

package/src/core/websocket/useMarimoKernelConnection.tsx

@@ -42,6 +42,7 @@ import { connectionTransportTypeAtom, useSetAppConfig } from "../config/config";
 import { useDataSourceActions } from "../datasets/data-source-connections";
 import type { ConnectionName } from "../datasets/engines";
 import {
+  DiscoverDataSources,
   PreviewSQLSchemaList,
   PreviewSQLTable,
   PreviewSQLTableList,
@@ -394,6 +395,9 @@ export function useMarimoKernelConnection(opts: {
       case "validate-sql-result":
         ValidateSQL.resolve(msg.data.request_id as RequestId, msg.data);
         return;
+      case "data-source-discovery-result":
+        DiscoverDataSources.resolve(msg.data.request_id as RequestId, msg.data);
+        return;
       case "secret-keys-result":
         SECRETS_REGISTRY.resolve(msg.data.request_id, msg.data);
         return;

package/src/hooks/__tests__/useDataSourceDiscovery.test.ts

@@ -0,0 +1,80 @@
+/* Copyright 2026 Marimo. All rights reserved. */
+
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { DiscoverDataSources } from "@/core/datasets/request-registry";
+import { loadDataSourceDiscovery } from "../useDataSourceDiscovery";
+
+describe("loadDataSourceDiscovery", () => {
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it("returns the kernel's secret-free discovery model", async () => {
+    const sources = [
+      {
+        id: "pyiceberg-prod",
+        integration: "pyiceberg",
+        category: "catalog" as const,
+        displayName: "PyIceberg (prod)",
+        confidence: "high" as const,
+        origins: [
+          {
+            type: "configuration" as const,
+            label: "Resolved PyIceberg configuration",
+          },
+        ],
+        configuration: [
+          {
+            field: "Catalog",
+            value: {
+              kind: "safe-literal" as const,
+              value: "prod",
+            },
+          },
+        ],
+        code: 'catalog = load_catalog("prod")',
+      },
+    ];
+    const request = vi.spyOn(DiscoverDataSources, "request").mockResolvedValue({
+      request_id: "request-id",
+      sources,
+    });
+
+    const detected = await loadDataSourceDiscovery();
+
+    expect({ detected, requests: request.mock.calls }).toMatchInlineSnapshot(`
+      {
+        "detected": [
+          {
+            "category": "catalog",
+            "code": "catalog = load_catalog("prod")",
+            "confidence": "high",
+            "configuration": [
+              {
+                "field": "Catalog",
+                "value": {
+                  "kind": "safe-literal",
+                  "value": "prod",
+                },
+              },
+            ],
+            "displayName": "PyIceberg (prod)",
+            "id": "pyiceberg-prod",
+            "integration": "pyiceberg",
+            "origins": [
+              {
+                "label": "Resolved PyIceberg configuration",
+                "type": "configuration",
+              },
+            ],
+          },
+        ],
+        "requests": [
+          [
+            {},
+          ],
+        ],
+      }
+    `);
+  });
+});

package/src/hooks/useDataSourceDiscovery.ts

@@ -0,0 +1,18 @@
+/* Copyright 2026 Marimo. All rights reserved. */
+
+import type { DetectedDataSource } from "@/core/datasets/data-source-discovery";
+import { DiscoverDataSources } from "@/core/datasets/request-registry";
+import { useAsyncData } from "./useAsyncData";
+
+export async function loadDataSourceDiscovery(): Promise<DetectedDataSource[]> {
+  const result = await DiscoverDataSources.request({});
+  return result.sources;
+}
+
+/**
+ * Reusable UI-facing hook for kernel-managed datasource discovery.
+ * Consumers decide how to render, filter, or act on the detected model.
+ */
+export function useDataSourceDiscovery() {
+  return useAsyncData(loadDataSourceDiscovery, []);
+}