@marimo-team/frontend 0.24.1-dev3 → 0.24.1-dev30

package/src/components/datasets/icons.tsx

@@ -7,6 +7,7 @@ import {
   CurlyBracesIcon,
   HashIcon,
   type LucideIcon,
+  MapPinIcon,
   ToggleLeftIcon,
   TypeIcon,
 } from "lucide-react";
@@ -27,6 +28,7 @@ export const DATA_TYPE_ICON: Record<DataType | SelectableDataType, LucideIcon> =
     number: HashIcon,
     string: TypeIcon,
     integer: HashIcon,
+    geometry: MapPinIcon,
     unknown: CurlyBracesIcon,
   };
 
@@ -56,6 +58,8 @@ export function getDataTypeColor(
       return "bg-(--purple-4)";
     case "string":
       return "bg-(--blue-4)";
+    case "geometry":
+      return "bg-(--cyan-4) dark:bg-(--cyan-5)";
     case "unknown":
       return "bg-(--slate-4) dark:bg-(--slate-6)";
     default:

package/src/components/datasources/__tests__/components.test.tsx

@@ -12,7 +12,7 @@ describe("ColumnName", () => {
 
     try {
       const unrecognized = render(
-        <ColumnName columnName="geom" dataType={"geometry" as DataType} />,
+        <ColumnName columnName="geom" dataType={"bogus_type" as DataType} />,
       );
       const unknown = render(
         <ColumnName columnName="geom" dataType="unknown" />,
@@ -26,4 +26,17 @@ describe("ColumnName", () => {
       warn.mockRestore();
     }
   });
+
+  it("renders geometry with a distinct icon", () => {
+    const geometry = render(
+      <ColumnName columnName="geom" dataType="geometry" />,
+    );
+    const unknown = render(<ColumnName columnName="geom" dataType="unknown" />);
+
+    expect(geometry.container.querySelector(".lucide-map-pin")).not.toBeNull();
+    expect(unknown.container.querySelector(".lucide-braces")).not.toBeNull();
+    expect(geometry.container.innerHTML).not.toEqual(
+      unknown.container.innerHTML,
+    );
+  });
 });

package/src/components/datasources/datasources.tsx

@@ -21,15 +21,11 @@ import { cellIdsAtom, useCellActions } from "@/core/cells/cells";
 import { useLastFocusedCellId } from "@/core/cells/focus";
 import { autoInstantiateAtom } from "@/core/config/config";
 import {
-  dataConnectionsMapAtom,
+  connectionsAtom,
   type SQLTableContext,
   useDataSourceActions,
 } from "@/core/datasets/data-source-connections";
-import {
-  DEFAULT_DUCKDB_DATABASE,
-  DUCKDB_ENGINE,
-  INTERNAL_SQL_ENGINES,
-} from "@/core/datasets/engines";
+import { DUCKDB_ENGINE } from "@/core/datasets/engines";
 import {
   PreviewSQLSchemaList,
   PreviewSQLTable,
@@ -224,40 +220,6 @@ export function filterEmptyDatabases(databases: Database[]): Database[] {
   return changed ? result : databases;
 }
 
-/**
- * This atom is used to get the data connections that are available to the user.
- * It filters out the internal engines if it has no databases or if it has only the in-memory database and no schemas.
- */
-export const connectionsAtom = atom((get) => {
-  const dataConnections = new Map(get(dataConnectionsMapAtom));
-
-  // Filter out the internal engines if it has no databases
-  // Or if it has only the in-memory database and no schemas
-  for (const engine of INTERNAL_SQL_ENGINES) {
-    const connection = dataConnections.get(engine);
-    if (!connection) {
-      continue;
-    }
-
-    if (connection.databases.length === 0) {
-      dataConnections.delete(engine);
-    }
-
-    if (
-      connection.databases.length === 1 &&
-      connection.databases[0].name === DEFAULT_DUCKDB_DATABASE &&
-      connection.databases[0].schemas.length === 0
-    ) {
-      dataConnections.delete(engine);
-    }
-  }
-
-  // Put internal engines last to prioritize user-defined connections
-  return sortBy([...dataConnections.values()], (connection) =>
-    INTERNAL_SQL_ENGINES.has(connection.name) ? 1 : 0,
-  );
-});
-
 export const DataSources: React.FC = () => {
   const [searchValue, setSearchValue] = React.useState<string>("");
   const [hideEmpty, setHideEmpty] = useAtom(hideEmptyDatasourcesAtom);

package/src/components/editor/chrome/panels/components.tsx

@@ -8,6 +8,9 @@ import {
 } from "@/components/ui/accordion";
 import { Badge } from "@/components/ui/badge";
 import { cn } from "@/utils/cn";
+import { SparklesIcon } from "lucide-react";
+import { Tooltip } from "@/components/ui/tooltip";
+import type { DataSourceDiscoveryGroup } from "@/core/datasets/data-source-discovery";
 
 /**
  * Styled AccordionItem for sidebar panels.
@@ -80,9 +83,41 @@ const PanelBadge = ({
 );
 PanelBadge.displayName = "PanelBadge";
 
+const DiscoveredSourcesBadge = ({
+  count,
+  type,
+}: {
+  count: number;
+  type: DataSourceDiscoveryGroup;
+}) => {
+  if (count === 0) {
+    return null;
+  }
+
+  let noun: string;
+  if (type === "database") {
+    noun = count === 1 ? "database" : "databases";
+  } else {
+    noun = count === 1 ? "remote storage" : "remote storages";
+  }
+  const content = `${count} ${noun} detected in your environment, ready to quick-add`;
+
+  return (
+    <Tooltip content={content}>
+      <span>
+        <PanelBadge className="gap-0.5 border-(--blue-6) bg-(--blue-3) text-(--blue-11)">
+          <SparklesIcon className="h-2.5 w-2.5" />
+          {count}
+        </PanelBadge>
+      </span>
+    </Tooltip>
+  );
+};
+
 export {
   PanelAccordionItem,
   PanelAccordionTrigger,
   PanelAccordionContent,
   PanelBadge,
+  DiscoveredSourcesBadge,
 };

package/src/components/editor/chrome/panels/file-explorer-panel.tsx

@@ -7,11 +7,13 @@ import useResizeObserver from "use-resize-observer";
 import { StorageInspector } from "@/components/storage/storage-inspector";
 import { Accordion } from "@/components/ui/accordion";
 import { storageNamespacesAtom } from "@/core/storage/state";
+import { useDetectedDataSources } from "@/hooks/useDataSourceDiscovery";
 import { cn } from "@/utils/cn";
 import { TreeDndProvider } from "../../file-tree/dnd-wrapper";
 import { FileExplorer } from "../../file-tree/file-explorer";
 import { useFileExplorerUpload } from "../../file-tree/upload";
 import {
+  DiscoveredSourcesBadge,
   PanelAccordionContent,
   PanelAccordionItem,
   PanelAccordionTrigger,
@@ -58,6 +60,7 @@ const FileExplorerPanel: React.FC = () => {
 
   const storageNamespaces = useAtomValue(storageNamespacesAtom);
   const remoteStorageConnections = storageNamespaces.length;
+  const pendingDataSources = useDetectedDataSources("storage");
 
   const openSections = useMemo<FileExplorerPanelSection[]>(() => {
     if (!state.hasUserInteracted && remoteStorageConnections > 0) {
@@ -81,6 +84,7 @@ const FileExplorerPanel: React.FC = () => {
 
   const availableContent = panelHeight - TRIGGER_HEIGHT * 2;
   const storageIsOpen = openSections.includes("remote-storage");
+  const showDiscoveredStorageBadge = pendingDataSources.length > 0;
   const bothOpen = storageIsOpen && openSections.includes("files");
 
   const storageMaxHeight = bothOpen
@@ -104,6 +108,12 @@ const FileExplorerPanel: React.FC = () => {
             {remoteStorageConnections > 0 && (
               <PanelBadge>{remoteStorageConnections}</PanelBadge>
             )}
+            {showDiscoveredStorageBadge && (
+              <DiscoveredSourcesBadge
+                count={pendingDataSources.length}
+                type="storage"
+              />
+            )}
           </PanelAccordionTrigger>
           <PanelAccordionContent
             className="overflow-auto"

package/src/components/editor/chrome/panels/session-panel.tsx

@@ -3,16 +3,16 @@
 import { useAtom, useAtomValue } from "jotai";
 import { DatabaseIcon, VariableIcon } from "lucide-react";
 import React, { useCallback } from "react";
-import {
-  connectionsAtom,
-  DataSources,
-} from "@/components/datasources/datasources";
+import { DataSources } from "@/components/datasources/datasources";
 import { Accordion } from "@/components/ui/accordion";
 import { VariableTable } from "@/components/variables/variables-table";
 import { useCellIds } from "@/core/cells/cells";
+import { connectionsAtom } from "@/core/datasets/data-source-connections";
 import { datasetTablesAtom } from "@/core/datasets/state";
 import { useVariables } from "@/core/variables/state";
+import { useDetectedDataSources } from "@/hooks/useDataSourceDiscovery";
 import {
+  DiscoveredSourcesBadge,
   PanelAccordionContent,
   PanelAccordionItem,
   PanelAccordionTrigger,
@@ -28,6 +28,7 @@ const SessionPanel: React.FC = () => {
   const cellIds = useCellIds();
   const tables = useAtomValue(datasetTablesAtom);
   const dataConnections = useAtomValue(connectionsAtom);
+  const pendingDataSources = useDetectedDataSources("database");
   const [state, setState] = useAtom(sessionPanelAtom);
 
   const datasourcesCount = tables.length + dataConnections.length;
@@ -50,6 +51,7 @@ const SessionPanel: React.FC = () => {
 
   const isDatasourcesOpen = openSections.includes("datasources");
   const showDatasourcesBadge = !isDatasourcesOpen && datasourcesCount > 0;
+  const showDiscoveredDataSourcesBadge = pendingDataSources.length > 0;
 
   return (
     <Accordion
@@ -63,6 +65,12 @@ const SessionPanel: React.FC = () => {
           <DatabaseIcon className="w-4 h-4" />
           Data sources
           {showDatasourcesBadge && <PanelBadge>{datasourcesCount}</PanelBadge>}
+          {showDiscoveredDataSourcesBadge && (
+            <DiscoveredSourcesBadge
+              count={pendingDataSources.length}
+              type="database"
+            />
+          )}
         </PanelAccordionTrigger>
         <PanelAccordionContent>
           <DataSources />

package/src/components/editor/connections/__tests__/quick-add-data-sources.test.tsx

@@ -28,6 +28,7 @@ const sources: DetectedDataSource[] = [
       },
     ],
     code: "engine = create_engine()",
+    hidesWhen: { kind: "dialect", substrings: ["postgres"] },
   },
   {
     id: "pyiceberg-prod",
@@ -52,6 +53,7 @@ const sources: DetectedDataSource[] = [
       },
     ],
     code: 'catalog = load_catalog("prod")',
+    hidesWhen: { kind: "dialect", substrings: ["iceberg"] },
   },
 ];
 

package/src/components/editor/connections/add-connection-dialog.tsx

@@ -87,7 +87,11 @@ export const AddConnectionDialogContent: React.FC<{
           <span className="block">{codeSnippetHint}</span>
         </DialogDescription>
       </DialogHeader>
-      <AutoDiscoveredDataSources onSubmit={onClose} className="-mt-2" />
+      <AutoDiscoveredDataSources
+        onSubmit={onClose}
+        className="-mt-2"
+        group={activeTab === "storage" ? "storage" : "database"}
+      />
       <Tabs
         value={activeTab}
         onValueChange={(v) => setActiveTab(v as ConnectionTab)}

package/src/components/editor/connections/quick-add-data-sources.tsx

@@ -2,8 +2,11 @@
 
 import { PlusIcon, SparklesIcon } from "lucide-react";
 import { Tooltip, TooltipProvider } from "@/components/ui/tooltip";
-import type { DetectedDataSource } from "@/core/datasets/data-source-discovery";
-import { useDataSourceDiscovery } from "@/hooks/useDataSourceDiscovery";
+import type {
+  DataSourceDiscoveryGroup,
+  DetectedDataSource,
+} from "@/core/datasets/data-source-discovery";
+import { useDetectedDataSources } from "@/hooks/useDataSourceDiscovery";
 import { useInsertCode } from "./components";
 import { cn } from "@/utils/cn";
 
@@ -94,19 +97,20 @@ const DetectedDataSourceDetails: React.FC<{
 );
 
 export const AutoDiscoveredDataSources: React.FC<{
-  onSubmit: () => void;
+  onSubmit?: () => void;
   className?: string;
-}> = ({ onSubmit, className }) => {
+  group?: DataSourceDiscoveryGroup;
+}> = ({ onSubmit, className, group }) => {
   const insertCode = useInsertCode();
-  const { data } = useDataSourceDiscovery();
+  const sources = useDetectedDataSources(group);
 
   return (
     <QuickAddDataSources
       className={className}
-      sources={data ?? []}
+      sources={sources}
       onAdd={(source) => {
         insertCode(source.code);
-        onSubmit();
+        onSubmit?.();
       }}
     />
   );

package/src/components/editor/connections/storage/__tests__/__snapshots__/as-code.test.ts.snap

@@ -117,6 +117,49 @@ exports[`generateStorageCode > GCS > without service account key (default creden
 store = GCSStore("my-bucket")"
 `;
 
+exports[`generateStorageCode > GitHub > public repo 1`] = `
+"from fsspec.implementations.github import GithubFileSystem
+
+fs = GithubFileSystem(
+    org="marimo-team",
+    repo="marimo",
+)"
+`;
+
+exports[`generateStorageCode > GitHub > with sha 1`] = `
+"from fsspec.implementations.github import GithubFileSystem
+
+fs = GithubFileSystem(
+    org="marimo-team",
+    repo="marimo",
+    sha="main",
+)"
+`;
+
+exports[`generateStorageCode > GitHub > with username and token 1`] = `
+"from fsspec.implementations.github import GithubFileSystem
+
+fs = GithubFileSystem(
+    org="marimo-team",
+    repo="marimo",
+    username="octocat",
+    token="ghp_example",
+)"
+`;
+
+exports[`generateStorageCode > GitHub > with username and token from secrets 1`] = `
+"from fsspec.implementations.github import GithubFileSystem
+import os
+
+_token = os.environ.get("GITHUB_TOKEN")
+fs = GithubFileSystem(
+    org="marimo-team",
+    repo="marimo",
+    username="octocat",
+    token=_token,
+)"
+`;
+
 exports[`generateStorageCode > Google Drive > with default auth (no credentials) 1`] = `
 "from gdrive_fsspec import GoogleDriveFileSystem
 

package/src/components/editor/connections/storage/__tests__/as-code.test.ts

@@ -329,6 +329,83 @@ describe("generateStorageCode", () => {
     });
   });
 
+  describe("GitHub", () => {
+    it("public repo", () => {
+      expect(
+        generateStorageCode(
+          { type: "github", org: "marimo-team", repo: "marimo" },
+          { library: "fsspec" },
+        ),
+      ).toMatchSnapshot();
+    });
+
+    it("with sha", () => {
+      expect(
+        generateStorageCode(
+          {
+            type: "github",
+            org: "marimo-team",
+            repo: "marimo",
+            sha: "main",
+          },
+          { library: "fsspec" },
+        ),
+      ).toMatchSnapshot();
+    });
+
+    it("escapes quotes and backslashes in sha", () => {
+      expect(
+        generateStorageCode(
+          {
+            type: "github",
+            org: "marimo-team",
+            repo: "marimo",
+            sha: 'feature/\\"new"',
+          },
+          { library: "fsspec" },
+        ),
+      ).toMatchInlineSnapshot(`
+        "from fsspec.implementations.github import GithubFileSystem
+
+        fs = GithubFileSystem(
+            org="marimo-team",
+            repo="marimo",
+            sha="feature/\\\\\\"new\\"",
+        )"
+      `);
+    });
+
+    it("with username and token", () => {
+      expect(
+        generateStorageCode(
+          {
+            type: "github",
+            org: "marimo-team",
+            repo: "marimo",
+            username: "octocat",
+            token: "ghp_example",
+          },
+          { library: "fsspec" },
+        ),
+      ).toMatchSnapshot();
+    });
+
+    it("with username and token from secrets", () => {
+      expect(
+        generateStorageCode(
+          {
+            type: "github",
+            org: "marimo-team",
+            repo: "marimo",
+            username: "octocat",
+            token: prefixSecret("GITHUB_TOKEN"),
+          },
+          { library: "fsspec" },
+        ),
+      ).toMatchSnapshot();
+    });
+  });
+
   describe("invalid cases", () => {
     it("throws for empty S3 bucket", () => {
       expect(() =>
@@ -397,5 +474,55 @@ describe("generateStorageCode", () => {
         ),
       ).toThrow();
     });
+
+    it("throws for empty GitHub org", () => {
+      expect(() =>
+        generateStorageCode(
+          { type: "github", org: "", repo: "marimo" } as StorageConnection,
+          { library: "fsspec" },
+        ),
+      ).toThrow();
+    });
+
+    it("throws for empty GitHub repo", () => {
+      expect(() =>
+        generateStorageCode(
+          {
+            type: "github",
+            org: "marimo-team",
+            repo: "",
+          } as StorageConnection,
+          { library: "fsspec" },
+        ),
+      ).toThrow();
+    });
+
+    it("throws when GitHub username is set without token", () => {
+      expect(() =>
+        generateStorageCode(
+          {
+            type: "github",
+            org: "marimo-team",
+            repo: "marimo",
+            username: "octocat",
+          },
+          { library: "fsspec" },
+        ),
+      ).toThrow();
+    });
+
+    it("throws when GitHub token is set without username", () => {
+      expect(() =>
+        generateStorageCode(
+          {
+            type: "github",
+            org: "marimo-team",
+            repo: "marimo",
+            token: "ghp_example",
+          },
+          { library: "fsspec" },
+        ),
+      ).toThrow();
+    });
   });
 });

package/src/components/editor/connections/storage/add-storage-form.tsx

@@ -16,6 +16,7 @@ import {
   AzureStorageSchema,
   CoreWeaveStorageSchema,
   GCSStorageSchema,
+  GithubStorageSchema,
   GoogleDriveStorageSchema,
   HuggingfaceStorageSchema,
   S3StorageSchema,
@@ -89,6 +90,15 @@ const STORAGE_PROVIDERS = [
       preferred: "huggingface_hub",
     },
   },
+  {
+    name: "GitHub",
+    schema: GithubStorageSchema,
+    protocol: "github",
+    storageLibraries: {
+      libraries: ["fsspec"],
+      preferred: "fsspec",
+    },
+  },
 ] satisfies StorageProviderSchema[];
 
 const StorageProviderSelector: React.FC<{
@@ -105,7 +115,7 @@ const StorageProviderSelector: React.FC<{
             <ProtocolIcon
               protocol={protocol as KnownStorageProtocol}
               forceDark={true}
-              className="w-7.5 h-7.5"
+              className="w-7.5 h-7.5 text-white"
             />
           }
           onSelect={() => onSelect(schema)}

package/src/components/editor/connections/storage/as-code.ts

@@ -43,7 +43,7 @@ class SecretContainer {
       this.secrets[privateVar] = `os.environ.get("${envVar}")`;
       return privateVar;
     }
-    return `"${value}"`;
+    return `"${escapePythonString(value)}"`;
   }
 
   formatSecrets(): string {
@@ -260,6 +260,32 @@ function generateHuggingfaceCode(
   return { imports, code: `hf = HfApi(token=${token})` };
 }
 
+function generateGithubCode(
+  connection: Extract<StorageConnection, { type: "github" }>,
+  secrets: SecretContainer,
+): { imports: Set<string>; code: string } {
+  const imports = new Set([
+    "from fsspec.implementations.github import GithubFileSystem",
+  ]);
+  const params: string[] = [
+    `org=${secrets.print("org", connection.org)},`,
+    `repo=${secrets.print("repo", connection.repo)},`,
+  ];
+
+  if (connection.sha) {
+    params.push(`sha=${secrets.print("sha", connection.sha)},`);
+  }
+  if (connection.username && connection.token) {
+    params.push(
+      `username=${secrets.print("username", connection.username)},`,
+      `token=${secrets.print("token", connection.token)},`,
+    );
+  }
+
+  const paramsStr = params.map((param) => `    ${param}`).join("\n");
+  return { imports, code: `fs = GithubFileSystem(\n${paramsStr}\n)` };
+}
+
 export function generateStorageCode(
   connection: StorageConnection,
   options: StorageCodeOptions,
@@ -291,6 +317,9 @@ export function generateStorageCode(
     case "huggingface":
       result = generateHuggingfaceCode(connection, secrets);
       break;
+    case "github":
+      result = generateGithubCode(connection, secrets);
+      break;
     default:
       assertNever(connection);
   }

package/src/components/editor/connections/storage/schemas.ts

@@ -238,6 +238,68 @@ export const HuggingfaceStorageSchema = z
   })
   .describe(FieldOptions.of({ direction: "two-columns" }));
 
+function isFilled(value: string | undefined): boolean {
+  return value != null && value !== "";
+}
+
+export const GithubStorageSchema = z
+  .object({
+    type: z.literal("github"),
+    org: z
+      .string()
+      .nonempty()
+      .describe(
+        FieldOptions.of({
+          label: "Organization or user",
+        }),
+      ),
+    repo: z
+      .string()
+      .nonempty()
+      .describe(
+        FieldOptions.of({
+          label: "Repository",
+        }),
+      ),
+    sha: z
+      .string()
+      .optional()
+      .describe(
+        FieldOptions.of({
+          label: "Branch, tag, or commit",
+          placeholder: "main",
+          description: "Leave empty to use the default branch",
+        }),
+      ),
+    username: z
+      .string()
+      .optional()
+      .describe(
+        FieldOptions.of({
+          label: "Username",
+          description: "Set this together with access token",
+          optionRegex: "(github.?username|github.?user)",
+        }),
+      ),
+    token: z
+      .string()
+      .optional()
+      .describe(
+        FieldOptions.of({
+          label: "Access Token",
+          description:
+            "Set this together with username. Required for private repos and to avoid GitHub's rate limits.",
+          inputType: "password",
+          optionRegex: "(github.?token|gh.?token|github.?pat)",
+        }),
+      ),
+  })
+  .refine((value) => isFilled(value.username) === isFilled(value.token), {
+    message: "Username and access token are required together",
+    path: ["token"],
+  })
+  .describe(FieldOptions.of({ direction: "two-columns" }));
+
 export const StorageConnectionSchema = z.discriminatedUnion("type", [
   S3StorageSchema,
   GCSStorageSchema,
@@ -245,6 +307,7 @@ export const StorageConnectionSchema = z.discriminatedUnion("type", [
   CoreWeaveStorageSchema,
   GoogleDriveStorageSchema,
   HuggingfaceStorageSchema,
+  GithubStorageSchema,
 ]);
 
 export type StorageConnection = z.infer<typeof StorageConnectionSchema>;