@marimo-team/islands 0.24.1-dev3 → 0.24.1-dev31

package/src/components/editor/connections/quick-add-data-sources.tsx

@@ -2,8 +2,11 @@
 
 import { PlusIcon, SparklesIcon } from "lucide-react";
 import { Tooltip, TooltipProvider } from "@/components/ui/tooltip";
-import type { DetectedDataSource } from "@/core/datasets/data-source-discovery";
-import { useDataSourceDiscovery } from "@/hooks/useDataSourceDiscovery";
+import type {
+  DataSourceDiscoveryGroup,
+  DetectedDataSource,
+} from "@/core/datasets/data-source-discovery";
+import { useDetectedDataSources } from "@/hooks/useDataSourceDiscovery";
 import { useInsertCode } from "./components";
 import { cn } from "@/utils/cn";
 
@@ -94,19 +97,20 @@ const DetectedDataSourceDetails: React.FC<{
 );
 
 export const AutoDiscoveredDataSources: React.FC<{
-  onSubmit: () => void;
+  onSubmit?: () => void;
   className?: string;
-}> = ({ onSubmit, className }) => {
+  group?: DataSourceDiscoveryGroup;
+}> = ({ onSubmit, className, group }) => {
   const insertCode = useInsertCode();
-  const { data } = useDataSourceDiscovery();
+  const sources = useDetectedDataSources(group);
 
   return (
     <QuickAddDataSources
       className={className}
-      sources={data ?? []}
+      sources={sources}
       onAdd={(source) => {
         insertCode(source.code);
-        onSubmit();
+        onSubmit?.();
       }}
     />
   );

package/src/components/editor/connections/storage/__tests__/__snapshots__/as-code.test.ts.snap

@@ -117,6 +117,49 @@ exports[`generateStorageCode > GCS > without service account key (default creden
 store = GCSStore("my-bucket")"
 `;
 
+exports[`generateStorageCode > GitHub > public repo 1`] = `
+"from fsspec.implementations.github import GithubFileSystem
+
+fs = GithubFileSystem(
+    org="marimo-team",
+    repo="marimo",
+)"
+`;
+
+exports[`generateStorageCode > GitHub > with sha 1`] = `
+"from fsspec.implementations.github import GithubFileSystem
+
+fs = GithubFileSystem(
+    org="marimo-team",
+    repo="marimo",
+    sha="main",
+)"
+`;
+
+exports[`generateStorageCode > GitHub > with username and token 1`] = `
+"from fsspec.implementations.github import GithubFileSystem
+
+fs = GithubFileSystem(
+    org="marimo-team",
+    repo="marimo",
+    username="octocat",
+    token="ghp_example",
+)"
+`;
+
+exports[`generateStorageCode > GitHub > with username and token from secrets 1`] = `
+"from fsspec.implementations.github import GithubFileSystem
+import os
+
+_token = os.environ.get("GITHUB_TOKEN")
+fs = GithubFileSystem(
+    org="marimo-team",
+    repo="marimo",
+    username="octocat",
+    token=_token,
+)"
+`;
+
 exports[`generateStorageCode > Google Drive > with default auth (no credentials) 1`] = `
 "from gdrive_fsspec import GoogleDriveFileSystem
 

package/src/components/editor/connections/storage/__tests__/as-code.test.ts

@@ -329,6 +329,83 @@ describe("generateStorageCode", () => {
     });
   });
 
+  describe("GitHub", () => {
+    it("public repo", () => {
+      expect(
+        generateStorageCode(
+          { type: "github", org: "marimo-team", repo: "marimo" },
+          { library: "fsspec" },
+        ),
+      ).toMatchSnapshot();
+    });
+
+    it("with sha", () => {
+      expect(
+        generateStorageCode(
+          {
+            type: "github",
+            org: "marimo-team",
+            repo: "marimo",
+            sha: "main",
+          },
+          { library: "fsspec" },
+        ),
+      ).toMatchSnapshot();
+    });
+
+    it("escapes quotes and backslashes in sha", () => {
+      expect(
+        generateStorageCode(
+          {
+            type: "github",
+            org: "marimo-team",
+            repo: "marimo",
+            sha: 'feature/\\"new"',
+          },
+          { library: "fsspec" },
+        ),
+      ).toMatchInlineSnapshot(`
+        "from fsspec.implementations.github import GithubFileSystem
+
+        fs = GithubFileSystem(
+            org="marimo-team",
+            repo="marimo",
+            sha="feature/\\\\\\"new\\"",
+        )"
+      `);
+    });
+
+    it("with username and token", () => {
+      expect(
+        generateStorageCode(
+          {
+            type: "github",
+            org: "marimo-team",
+            repo: "marimo",
+            username: "octocat",
+            token: "ghp_example",
+          },
+          { library: "fsspec" },
+        ),
+      ).toMatchSnapshot();
+    });
+
+    it("with username and token from secrets", () => {
+      expect(
+        generateStorageCode(
+          {
+            type: "github",
+            org: "marimo-team",
+            repo: "marimo",
+            username: "octocat",
+            token: prefixSecret("GITHUB_TOKEN"),
+          },
+          { library: "fsspec" },
+        ),
+      ).toMatchSnapshot();
+    });
+  });
+
   describe("invalid cases", () => {
     it("throws for empty S3 bucket", () => {
       expect(() =>
@@ -397,5 +474,55 @@ describe("generateStorageCode", () => {
         ),
       ).toThrow();
     });
+
+    it("throws for empty GitHub org", () => {
+      expect(() =>
+        generateStorageCode(
+          { type: "github", org: "", repo: "marimo" } as StorageConnection,
+          { library: "fsspec" },
+        ),
+      ).toThrow();
+    });
+
+    it("throws for empty GitHub repo", () => {
+      expect(() =>
+        generateStorageCode(
+          {
+            type: "github",
+            org: "marimo-team",
+            repo: "",
+          } as StorageConnection,
+          { library: "fsspec" },
+        ),
+      ).toThrow();
+    });
+
+    it("throws when GitHub username is set without token", () => {
+      expect(() =>
+        generateStorageCode(
+          {
+            type: "github",
+            org: "marimo-team",
+            repo: "marimo",
+            username: "octocat",
+          },
+          { library: "fsspec" },
+        ),
+      ).toThrow();
+    });
+
+    it("throws when GitHub token is set without username", () => {
+      expect(() =>
+        generateStorageCode(
+          {
+            type: "github",
+            org: "marimo-team",
+            repo: "marimo",
+            token: "ghp_example",
+          },
+          { library: "fsspec" },
+        ),
+      ).toThrow();
+    });
   });
 });

package/src/components/editor/connections/storage/add-storage-form.tsx

@@ -16,6 +16,7 @@ import {
   AzureStorageSchema,
   CoreWeaveStorageSchema,
   GCSStorageSchema,
+  GithubStorageSchema,
   GoogleDriveStorageSchema,
   HuggingfaceStorageSchema,
   S3StorageSchema,
@@ -89,6 +90,15 @@ const STORAGE_PROVIDERS = [
       preferred: "huggingface_hub",
     },
   },
+  {
+    name: "GitHub",
+    schema: GithubStorageSchema,
+    protocol: "github",
+    storageLibraries: {
+      libraries: ["fsspec"],
+      preferred: "fsspec",
+    },
+  },
 ] satisfies StorageProviderSchema[];
 
 const StorageProviderSelector: React.FC<{
@@ -105,7 +115,7 @@ const StorageProviderSelector: React.FC<{
             <ProtocolIcon
               protocol={protocol as KnownStorageProtocol}
               forceDark={true}
-              className="w-7.5 h-7.5"
+              className="w-7.5 h-7.5 text-white"
             />
           }
           onSelect={() => onSelect(schema)}

package/src/components/editor/connections/storage/as-code.ts

@@ -43,7 +43,7 @@ class SecretContainer {
       this.secrets[privateVar] = `os.environ.get("${envVar}")`;
       return privateVar;
     }
-    return `"${value}"`;
+    return `"${escapePythonString(value)}"`;
   }
 
   formatSecrets(): string {
@@ -260,6 +260,32 @@ function generateHuggingfaceCode(
   return { imports, code: `hf = HfApi(token=${token})` };
 }
 
+function generateGithubCode(
+  connection: Extract<StorageConnection, { type: "github" }>,
+  secrets: SecretContainer,
+): { imports: Set<string>; code: string } {
+  const imports = new Set([
+    "from fsspec.implementations.github import GithubFileSystem",
+  ]);
+  const params: string[] = [
+    `org=${secrets.print("org", connection.org)},`,
+    `repo=${secrets.print("repo", connection.repo)},`,
+  ];
+
+  if (connection.sha) {
+    params.push(`sha=${secrets.print("sha", connection.sha)},`);
+  }
+  if (connection.username && connection.token) {
+    params.push(
+      `username=${secrets.print("username", connection.username)},`,
+      `token=${secrets.print("token", connection.token)},`,
+    );
+  }
+
+  const paramsStr = params.map((param) => `    ${param}`).join("\n");
+  return { imports, code: `fs = GithubFileSystem(\n${paramsStr}\n)` };
+}
+
 export function generateStorageCode(
   connection: StorageConnection,
   options: StorageCodeOptions,
@@ -291,6 +317,9 @@ export function generateStorageCode(
     case "huggingface":
       result = generateHuggingfaceCode(connection, secrets);
       break;
+    case "github":
+      result = generateGithubCode(connection, secrets);
+      break;
     default:
       assertNever(connection);
   }

package/src/components/editor/connections/storage/schemas.ts

@@ -238,6 +238,68 @@ export const HuggingfaceStorageSchema = z
   })
   .describe(FieldOptions.of({ direction: "two-columns" }));
 
+function isFilled(value: string | undefined): boolean {
+  return value != null && value !== "";
+}
+
+export const GithubStorageSchema = z
+  .object({
+    type: z.literal("github"),
+    org: z
+      .string()
+      .nonempty()
+      .describe(
+        FieldOptions.of({
+          label: "Organization or user",
+        }),
+      ),
+    repo: z
+      .string()
+      .nonempty()
+      .describe(
+        FieldOptions.of({
+          label: "Repository",
+        }),
+      ),
+    sha: z
+      .string()
+      .optional()
+      .describe(
+        FieldOptions.of({
+          label: "Branch, tag, or commit",
+          placeholder: "main",
+          description: "Leave empty to use the default branch",
+        }),
+      ),
+    username: z
+      .string()
+      .optional()
+      .describe(
+        FieldOptions.of({
+          label: "Username",
+          description: "Set this together with access token",
+          optionRegex: "(github.?username|github.?user)",
+        }),
+      ),
+    token: z
+      .string()
+      .optional()
+      .describe(
+        FieldOptions.of({
+          label: "Access Token",
+          description:
+            "Set this together with username. Required for private repos and to avoid GitHub's rate limits.",
+          inputType: "password",
+          optionRegex: "(github.?token|gh.?token|github.?pat)",
+        }),
+      ),
+  })
+  .refine((value) => isFilled(value.username) === isFilled(value.token), {
+    message: "Username and access token are required together",
+    path: ["token"],
+  })
+  .describe(FieldOptions.of({ direction: "two-columns" }));
+
 export const StorageConnectionSchema = z.discriminatedUnion("type", [
   S3StorageSchema,
   GCSStorageSchema,
@@ -245,6 +307,7 @@ export const StorageConnectionSchema = z.discriminatedUnion("type", [
   CoreWeaveStorageSchema,
   GoogleDriveStorageSchema,
   HuggingfaceStorageSchema,
+  GithubStorageSchema,
 ]);
 
 export type StorageConnection = z.infer<typeof StorageConnectionSchema>;

package/src/core/codemirror/code-lens/__tests__/popover.test.ts

@@ -56,6 +56,19 @@ describe("CachePopover getCacheInfo throttling", () => {
     return () => act(() => dispose());
   }
 
+  it("renders content synchronously for CodeMirror's initial measurement", async () => {
+    const dom = document.createElement("div");
+
+    // Keep the mount outside `act()`. It would flush an asynchronous render
+    // and make this assertion pass without `flushSync`.
+    const dispose = mountLensPopover(dom, CACHE_SPEC);
+
+    expect(dom.textContent).toContain("my_cache");
+    await act(async () => {
+      dispose();
+    });
+  });
+
   it("does not refetch on every hover, only after the throttle interval elapses", () => {
     const disposeFirst = mount();
     expect(getCacheInfoMock).toHaveBeenCalledTimes(1);

package/src/core/codemirror/code-lens/popover.tsx

@@ -4,6 +4,7 @@ import { Provider, useAtomValue } from "jotai";
 import { DatabaseZapIcon } from "lucide-react";
 import type React from "react";
 import { useEffect } from "react";
+import { flushSync } from "react-dom";
 import { createRoot } from "react-dom/client";
 import { useLocale } from "react-aria";
 import { ProtocolIcon } from "@/components/storage/components";
@@ -39,11 +40,16 @@ export function mountLensPopover(
   spec: CodeLensSpec,
 ): () => void {
   const root = createRoot(dom);
-  root.render(
-    <Provider store={store}>
-      <LensPopover spec={spec} />
-    </Provider>,
-  );
+  // CodeMirror measures and positions the tooltip as soon as this function
+  // returns. Render now so it measures the popover content, not an empty
+  // container that later grows over the hovered icon.
+  flushSync(() => {
+    root.render(
+      <Provider store={store}>
+        <LensPopover spec={spec} />
+      </Provider>,
+    );
+  });
   // Defer so tearing the tooltip down from inside a React event (e.g. the
   // icon's click handler) doesn't unmount mid-render
   return () => queueMicrotask(() => root.unmount());

package/src/core/codemirror/language/languages/sql/__tests__/renderers.test.tsx

@@ -9,13 +9,18 @@ import type {
 } from "@/core/kernel/messages";
 import { renderColumnInfo, renderTableInfo } from "../renderers";
 
-const geometryColumn = {
+const bogusColumn = {
   name: "geom",
-  type: "geometry" as DataType,
+  type: "bogus_type" as DataType,
   external_type: "geometry",
   sample_values: [],
 } as DataTableColumn;
 
+const geometryColumn = {
+  ...bogusColumn,
+  type: "geometry" as DataType,
+} as DataTableColumn;
+
 describe("SQL renderers", () => {
   it("renders table information with an unrecognized column type", () => {
     const table: DataTable = {
@@ -26,7 +31,7 @@ describe("SQL renderers", () => {
       num_rows: null,
       num_columns: 1,
       variable_name: null,
-      columns: [geometryColumn],
+      columns: [bogusColumn],
       primary_keys: [],
       indexes: [],
     };
@@ -37,8 +42,14 @@ describe("SQL renderers", () => {
   });
 
   it("renders column information with an unrecognized column type", () => {
-    const result = render(renderColumnInfo(geometryColumn));
+    const result = render(renderColumnInfo(bogusColumn));
 
     expect(result.container.querySelector("svg")).not.toBeNull();
   });
+
+  it("renders geometry column information with the map pin icon", () => {
+    const result = render(renderColumnInfo(geometryColumn));
+
+    expect(result.container.querySelector(".lucide-map-pin")).not.toBeNull();
+  });
 });

package/src/core/codemirror/language/languages/sql/renderers.tsx

@@ -41,6 +41,7 @@ const DATA_TYPE_COLORS: Record<DataType, string> = {
   number: "bg-(--purple-4) text-(--purple-11)",
   integer: "bg-(--purple-4) text-(--purple-11)",
   string: "bg-(--blue-4) text-(--blue-11)",
+  geometry: "bg-(--cyan-4) text-(--cyan-11)",
   unknown: "bg-(--slate-4) text-(--slate-11)",
 };