@malloy-publisher/sdk 0.0.137 → 0.0.139

package/src/components/Notebook/Notebook.tsx

@@ -1,24 +1,45 @@
 import "@malloydata/malloy-explorer/styles.css";
-import { Stack, Typography } from "@mui/material";
-import { CompiledNotebook } from "../../client";
+import * as Malloy from "@malloydata/malloy-interfaces";
+import { Box, Paper, Stack, Typography } from "@mui/material";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
+import { RawNotebook } from "../../client";
+import { useDimensionalFilterRangeData } from "../../hooks/useDimensionalFilterRangeData";
+import {
+   FilterSelection,
+   useDimensionFilters,
+} from "../../hooks/useDimensionFilters";
 import { useQueryWithApiError } from "../../hooks/useQueryWithApiError";
 import { ApiErrorDisplay } from "../ApiErrorDisplay";
+import { DimensionFilter, RetrievalFunction } from "../filter/DimensionFilter";
+import {
+   extractDimensionSpecs,
+   extractSourceFromQuery,
+   generateFilterClause,
+   getJoinedSources,
+   injectWhereClause,
+   parseAllSourceInfos,
+   parseNotebookFilterAnnotation,
+} from "../filter/utils";
 
 import { parseResourceUri } from "../../utils/formatting";
 import { Loading } from "../Loading";
 import { useServer } from "../ServerProvider";
 import { CleanNotebookContainer, CleanNotebookSection } from "../styles";
 import { NotebookCell } from "./NotebookCell";
+import { EnhancedNotebookCell } from "./types";
 
 interface NotebookProps {
    resourceUri: string;
    maxResultSize?: number;
+   /** Optional retrieval function for semantic search filters */
+   retrievalFn?: RetrievalFunction;
 }
 
 // Requires PackageProvider
 export default function Notebook({
    resourceUri,
    maxResultSize = 0,
+   retrievalFn,
 }: NotebookProps) {
    const { apiClients } = useServer();
    const {
@@ -27,12 +48,14 @@ export default function Notebook({
       versionId,
       modelPath: notebookPath,
    } = parseResourceUri(resourceUri);
+
+   // Fetch the raw notebook cells
    const {
       data: notebook,
       isSuccess,
       isError,
       error,
-   } = useQueryWithApiError<CompiledNotebook>({
+   } = useQueryWithApiError<RawNotebook>({
       queryKey: [resourceUri],
       queryFn: async () => {
          const response = await apiClients.notebooks.getNotebook(
@@ -45,15 +68,364 @@ export default function Notebook({
       },
    });
 
+   // State to store executed cells with results
+   const [enhancedCells, setEnhancedCells] = useState<EnhancedNotebookCell[]>(
+      [],
+   );
+   const [isExecuting, setIsExecuting] = useState(false);
+   const [executionError, setExecutionError] = useState<Error | null>(null);
+
+   // Parse filter configuration from notebook annotations
+   const filterConfig = useMemo(() => {
+      if (!notebook) return null;
+      return parseNotebookFilterAnnotation(notebook.annotations);
+   }, [notebook]);
+
+   // Parse all SourceInfos from notebook cells and create a map
+   const sourceData = useMemo(() => {
+      if (!notebook?.notebookCells) return null;
+      return parseAllSourceInfos(notebook.notebookCells);
+   }, [notebook]);
+
+   const sourceInfoMap = useMemo(
+      () => sourceData?.sourceInfoMap ?? new Map<string, Malloy.SourceInfo>(),
+      [sourceData],
+   );
+   const modelPath = sourceData?.modelPath ?? null;
+
+   // Build dimension specs from filter config and source info map
+   // Each spec includes source and model for proper query routing
+   const dimensionSpecs = useMemo(() => {
+      if (!filterConfig || sourceInfoMap.size === 0 || !modelPath) return [];
+      return extractDimensionSpecs(
+         sourceInfoMap,
+         filterConfig.filters,
+         modelPath,
+      );
+   }, [filterConfig, sourceInfoMap, modelPath]);
+
+   // Initialize dimension filters hook
+   const { filterStates, updateFilter, getActiveFilters } = useDimensionFilters(
+      {
+         dimensionSpecs,
+      },
+   );
+
+   // Get active filters - include filterStates in deps to ensure updates when individual items change
+   const activeFilters = useMemo(
+      () => getActiveFilters(),
+      // eslint-disable-next-line react-hooks/exhaustive-deps
+      [filterStates, getActiveFilters],
+   );
+
+   // Create a map of dimension name -> source name for quick lookup
+   const dimensionToSourceMap = useMemo(() => {
+      const map = new Map<string, string>();
+      for (const spec of dimensionSpecs) {
+         map.set(spec.dimensionName, spec.source);
+      }
+      return map;
+   }, [dimensionSpecs]);
+
+   // Create a map of source name -> set of joined source names
+   const sourceJoinsMap = useMemo(() => {
+      const map = new Map<string, Set<string>>();
+      for (const [sourceName, sourceInfo] of sourceInfoMap) {
+         map.set(sourceName, getJoinedSources(sourceInfo));
+      }
+      return map;
+   }, [sourceInfoMap]);
+
+   // Fetch filter range data when we have dimension specs
+   // The hook now handles multiple source/model combos internally
+   const { data: filterValuesData } = useDimensionalFilterRangeData({
+      project: projectName,
+      package: packageName,
+      dimensionSpecs,
+      versionId,
+      enabled: dimensionSpecs.length > 0,
+      activeFilters,
+   });
+
+   // Unified cell execution function
+   // Executes all notebook cells, optionally applying filters to query cells
+   const executeCells = useCallback(
+      async (filtersToApply: FilterSelection[] = []) => {
+         if (!isSuccess || !notebook?.notebookCells) return;
+
+         setIsExecuting(true);
+         setExecutionError(null);
+         const cells: EnhancedNotebookCell[] = [];
+
+         try {
+            // Execute cells sequentially
+            for (let i = 0; i < notebook.notebookCells.length; i++) {
+               const rawCell = notebook.notebookCells[i];
+
+               // Markdown cells don't need execution - use raw content directly
+               if (rawCell.type === "markdown") {
+                  cells.push({
+                     type: rawCell.type,
+                     text: rawCell.text,
+                  });
+                  continue;
+               }
+
+               // Execute code cells
+               const cellText = rawCell.text || "";
+               const hasQuery =
+                  cellText.includes("run:") ||
+                  cellText.includes("->") ||
+                  /^\s*(run|query)\s*:/m.test(cellText);
+
+               try {
+                  if (hasQuery && modelPath) {
+                     // Query cell - use models API with optional filters
+                     let queryToExecute = cellText;
+
+                     // Apply filters if any match this query's source
+                     if (filtersToApply.length > 0) {
+                        const querySourceName =
+                           extractSourceFromQuery(cellText);
+
+                        // Get the set of joined sources for this query's source
+                        const joinedSources =
+                           (querySourceName &&
+                              sourceJoinsMap.get(querySourceName)) ||
+                           new Set<string>();
+
+                        // Filter to only include those matching this query's source or joined sources
+                        const filtersForSource = querySourceName
+                           ? filtersToApply.filter((filter) => {
+                                const filterSourceName =
+                                   dimensionToSourceMap.get(
+                                      filter.dimensionName,
+                                   );
+                                if (!filterSourceName) return false;
+                                return (
+                                   filterSourceName === querySourceName ||
+                                   joinedSources.has(filterSourceName)
+                                );
+                             })
+                           : [];
+
+                        if (filtersForSource.length > 0) {
+                           const filterClause = generateFilterClause(
+                              filtersForSource,
+                              dimensionToSourceMap,
+                              querySourceName,
+                           );
+                           if (filterClause) {
+                              queryToExecute = injectWhereClause(
+                                 cellText,
+                                 filterClause,
+                              );
+                           }
+                        }
+                     }
+
+                     // Execute using models API
+                     const response = await apiClients.models.executeQueryModel(
+                        projectName,
+                        packageName,
+                        modelPath,
+                        {
+                           query: queryToExecute,
+                           versionId,
+                        },
+                     );
+
+                     cells.push({
+                        type: rawCell.type,
+                        text: rawCell.text,
+                        result: response.data.result,
+                     });
+                  } else {
+                     // Non-query code cell - use notebook cell execution API
+                     const response =
+                        await apiClients.notebooks.executeNotebookCell(
+                           projectName,
+                           packageName,
+                           notebookPath,
+                           i,
+                           versionId,
+                        );
+
+                     const executedCell = response.data;
+                     cells.push({
+                        type: rawCell.type,
+                        text: rawCell.text,
+                        result: executedCell.result,
+                        newSources: executedCell.newSources,
+                     });
+                  }
+               } catch (cellError) {
+                  // If a cell fails, add it without execution results
+                  console.error(`Error executing cell ${i}:`, cellError);
+                  cells.push({
+                     type: rawCell.type,
+                     text: rawCell.text,
+                  });
+               }
+            }
+
+            setEnhancedCells(cells);
+         } catch (error) {
+            console.error("Error executing notebook cells:", error);
+            setExecutionError(error as Error);
+         } finally {
+            setIsExecuting(false);
+         }
+      },
+      [
+         isSuccess,
+         notebook,
+         modelPath,
+         dimensionToSourceMap,
+         sourceJoinsMap,
+         projectName,
+         packageName,
+         notebookPath,
+         versionId,
+         apiClients.models,
+         apiClients.notebooks,
+      ],
+   );
+
+   // Execute cells when notebook is loaded (no filters initially)
+   useEffect(() => {
+      if (!isSuccess || !notebook?.notebookCells) return;
+      executeCells([]);
+   }, [isSuccess, notebook, executeCells]);
+
+   // Re-execute when filters change
+   // Track previous activeFilters to detect actual changes (not just reference changes)
+   const prevActiveFiltersRef = useRef<string>("");
+
+   useEffect(() => {
+      // Serialize activeFilters to detect actual value changes
+      const serialized = JSON.stringify(
+         activeFilters.map((f) => ({
+            dim: f.dimensionName,
+            type: f.matchType,
+            val: f.value,
+            val2: f.value2,
+         })),
+      );
+
+      // Skip if no actual change or if this is the initial empty state
+      if (serialized === prevActiveFiltersRef.current) {
+         return;
+      }
+
+      // Skip the initial render (when prevActiveFiltersRef is empty and filters are also empty)
+      if (prevActiveFiltersRef.current === "" && activeFilters.length === 0) {
+         prevActiveFiltersRef.current = serialized;
+         return;
+      }
+
+      prevActiveFiltersRef.current = serialized;
+
+      // Re-execute with current filters (or no filters if cleared)
+      if (!isExecuting) {
+         executeCells(activeFilters);
+      }
+   }, [activeFilters, isExecuting, executeCells]);
+
+   // Handle filter change
+   const handleFilterChange = useCallback(
+      (dimensionName: string) => (selection: FilterSelection | null) => {
+         updateFilter(dimensionName, selection);
+      },
+      [updateFilter],
+   );
+
+   // Check if retrieval is supported
+   const hasRetrievalFilters = dimensionSpecs.some(
+      (spec) => spec.filterType === "Retrieval",
+   );
+   const _retrievalSupported = !hasRetrievalFilters || !!retrievalFn;
+
    return (
       <CleanNotebookContainer>
          <CleanNotebookSection>
             <Stack spacing={3} component="section">
-               {!isSuccess && !isError && (
-                  <Loading text="Fetching Notebook..." />
+               {/* Filter Panel */}
+               {dimensionSpecs.length > 0 && filterValuesData && (
+                  <Paper
+                     elevation={0}
+                     sx={{
+                        p: 3,
+                        backgroundColor: "#f8f9fa",
+                        border: "1px solid #e0e0e0",
+                        borderRadius: 2,
+                     }}
+                  >
+                     <Typography
+                        variant="subtitle2"
+                        sx={{
+                           fontWeight: 600,
+                           mb: 2,
+                           color: "#333",
+                        }}
+                     >
+                        Filters
+                     </Typography>
+                     <Box
+                        sx={{
+                           display: "grid",
+                           gridTemplateColumns:
+                              "repeat(auto-fill, minmax(250px, 1fr))",
+                           gap: 3,
+                        }}
+                     >
+                        {dimensionSpecs.map((spec) => {
+                           const values =
+                              filterValuesData.get(spec.dimensionName) || [];
+                           const filterState = filterStates.get(
+                              spec.dimensionName,
+                           );
+                           // Skip Retrieval filters if no retrievalFn provided
+                           if (
+                              spec.filterType === "Retrieval" &&
+                              !retrievalFn
+                           ) {
+                              return null;
+                           }
+
+                           return (
+                              <Box key={spec.dimensionName}>
+                                 <DimensionFilter
+                                    spec={spec}
+                                    values={values}
+                                    selection={filterState?.selection}
+                                    onChange={handleFilterChange(
+                                       spec.dimensionName,
+                                    )}
+                                    retrievalFn={retrievalFn}
+                                 />
+                              </Box>
+                           );
+                        })}
+                     </Box>
+                  </Paper>
+               )}
+
+               {/* Loading State */}
+               {(!isSuccess || isExecuting) && !isError && (
+                  <Loading
+                     text={
+                        isExecuting
+                           ? "Executing Notebook..."
+                           : "Fetching Notebook..."
+                     }
+                  />
                )}
+
+               {/* Notebook Cells */}
                {isSuccess &&
-                  notebook.notebookCells?.map((cell, index) => (
+                  !isExecuting &&
+                  enhancedCells.map((cell, index) => (
                      <NotebookCell
                         cell={cell}
                         key={index}
@@ -62,6 +434,8 @@ export default function Notebook({
                         maxResultSize={maxResultSize}
                      />
                   ))}
+
+               {/* Error States */}
                {isError && error.status === 404 && (
                   <Typography variant="body2" sx={{ color: "#666666" }}>
                      <code>{`${projectName} > ${packageName} > ${notebookPath}`}</code>{" "}
@@ -75,6 +449,17 @@ export default function Notebook({
                      context={`${projectName} > ${packageName} > ${notebookPath}`}
                   />
                )}
+
+               {executionError && (
+                  <ApiErrorDisplay
+                     error={{
+                        message: executionError.message,
+                        status: 500,
+                        name: "ExecutionError",
+                     }}
+                     context="Notebook Execution"
+                  />
+               )}
             </Stack>
          </CleanNotebookSection>
       </CleanNotebookContainer>

package/src/components/Notebook/NotebookCell.tsx

@@ -16,16 +16,16 @@ import {
 } from "@mui/material";
 import Markdown from "markdown-to-jsx";
 import React, { useEffect, useState } from "react";
-import { NotebookCell as ClientNotebookCell } from "../../client";
 import { highlight } from "../highlighter";
 import { ModelExplorerDialog } from "../Model/ModelExplorerDialog";
 import { createEmbeddedQueryResult } from "../QueryResult/QueryResult";
 import ResultContainer from "../RenderedResult/ResultContainer";
 import ResultsDialog from "../ResultsDialog";
 import { CleanMetricCard, CleanNotebookCell } from "../styles";
+import { EnhancedNotebookCell } from "./types";
 
 interface NotebookCellProps {
-   cell: ClientNotebookCell;
+   cell: EnhancedNotebookCell;
    expandCodeCell?: boolean;
    hideCodeCellIcon?: boolean;
    expandEmbedding?: boolean;

package/src/components/Notebook/types.ts

@@ -0,0 +1,24 @@
+import { NotebookCell as ClientNotebookCell } from "../../client";
+
+/**
+ * Enhanced notebook cell that extends the base API cell with execution results.
+ * The base ClientNotebookCell only contains type and text (raw content).
+ * This interface adds the runtime execution data.
+ */
+export interface EnhancedNotebookCell extends ClientNotebookCell {
+   /**
+    * Name of the query that was executed (for code cells that run queries)
+    */
+   queryName?: string;
+
+   /**
+    * JSON string containing the query execution results
+    */
+   result?: string;
+
+   /**
+    * Array of JSON strings containing SourceInfo objects for data sources
+    * that become available in this cell (e.g., from imports or source definitions)
+    */
+   newSources?: string[];
+}

package/src/components/Package/Connections.tsx

@@ -121,6 +121,7 @@ export default function Connections({ resourceUri }: ConnectionsProps) {
       projectName: projectName,
       connectionName: selectedConnection,
    });
+
    const { data, isSuccess, isError, error } = useQueryWithApiError({
       queryKey: ["connections", projectName],
       queryFn: () => apiClients.connections.listConnections(projectName),
@@ -275,15 +276,11 @@ export default function Connections({ resourceUri }: ConnectionsProps) {
                                     updateConnection.mutateAsync(payload)
                                  }
                                  onDelete={(payload) => {
-                                    if (
-                                       !selectedConnectionResourceUri.startsWith(
-                                          "publisher:",
-                                       )
-                                    ) {
+                                    if (!conn.resource) {
                                        deleteConnection.mutateAsync(payload);
                                     } else {
                                        setNotificationMessage(
-                                          "Cannot delete this connection (publisher: resource)",
+                                          "Cannot delete this connection",
                                        );
                                     }
                                  }}

package/src/components/RenderedResult/RenderedResult.tsx

@@ -1,9 +1,9 @@
 import React, {
+   Suspense,
    useEffect,
+   useLayoutEffect,
    useRef,
    useState,
-   useLayoutEffect,
-   Suspense,
 } from "react";
 
 type MalloyRenderElement = HTMLElement & Record<string, unknown>;
@@ -45,6 +45,21 @@ const createRenderer = async (onDrill?: (element: unknown) => void) => {
    return renderer.createViz();
 };
 
+function RenderResultSimple({ result, onDrill }: RenderedResultProps) {
+   const ref = useRef<HTMLDivElement>(null);
+
+   useLayoutEffect(() => {
+      if (!ref.current || !result) return;
+      const element = ref.current;
+
+      createRenderer(onDrill).then((viz) => {
+         viz.setResult(JSON.parse(result));
+         viz.render(element);
+      });
+   }, [result, onDrill]);
+
+   return <div ref={ref} style={{ width: "100%", height: "100%" }} />;
+}
 // Inner component that actually renders the visualization
 function RenderedResultInner({
    result,
@@ -219,7 +234,11 @@ export default function RenderedResult(props: RenderedResultProps) {
             </div>
          }
       >
-         <RenderedResultInner {...props} />
+         {props.onSizeChange ? (
+            <RenderedResultInner {...props} />
+         ) : (
+            <RenderResultSimple {...props} />
+         )}
       </Suspense>
    );
 }