@manta-eu/react 0.8.0 → 0.9.0

package/src/routes/data-explorer/uploads.ts

@@ -0,0 +1,205 @@
+import { userIriPrefix } from "@manta-eu/client/shared/stakeholder-iri";
+import { isIngestableExtension } from "@manta-eu/client/shared/upload";
+import { ingestManual } from "@manta-eu/ldo";
+import { ingest, nfo, nmo, prov } from "@manta-eu/ldo/namespaces";
+import { getExtension } from "@manta-eu/lib/file";
+import { useQueryClient } from "@tanstack/react-query";
+import type { RDF } from "ldkit/rdf";
+import { useCallback, useMemo } from "react";
+import { bindingCount, bindingValue, useSparqlSelect } from "#ldo-ui";
+
+export const uploadsLimit = 500;
+
+export const ingestPollIntervalMs = 15_000;
+
+const uploadsQueryKey = ["uploaded-files"] as const;
+
+export type UploadedFile = {
+  iri: string;
+  fileName: string;
+  mediaType?: string;
+  byteSize?: number;
+  extension?: string;
+  uploadedAt?: string;
+  uploader?: string;
+  purpose?: string;
+};
+
+export type IngestStatus = "ingested" | "unparsable" | "gone" | "too_large";
+
+export type IngestRun = {
+  iri: string;
+  file: string;
+  status: IngestStatus;
+  endedAt?: string;
+  recordsRead?: number;
+  recordsDropped?: number;
+  dataset?: string;
+};
+
+export type FileIngestState = { kind: "waiting" } | { kind: "stored" } | { kind: "run"; run: IngestRun };
+
+const ingestStatuses = new Set<string>(["ingested", "unparsable", "gone", "too_large"]);
+
+function isIngestStatus(value: string | undefined): value is IngestStatus {
+  return value !== undefined && ingestStatuses.has(value);
+}
+
+// Operators see every file node; members see the files a user uploaded.
+export function uploadedFilesQuery(includeAllFiles: boolean): string {
+  const uploaderPattern = includeAllFiles
+    ? `OPTIONAL { ?file <${prov.wasAttributedTo}> ?uploader }`
+    : `?file <${prov.wasAttributedTo}> ?uploader .
+  FILTER(STRSTARTS(STR(?uploader), "${userIriPrefix}"))`;
+
+  return `SELECT ?file ?fileName ?mediaType ?byteSize ?extension ?uploadedAt ?uploader ?purpose WHERE {
+  ?file a <${nfo.FileDataObject}> .
+  FILTER(isIRI(?file))
+  ${uploaderPattern}
+  OPTIONAL { ?file <${nfo.fileName}> ?fileName }
+  OPTIONAL { ?file <${nmo.contentMimeType}> ?mediaType }
+  OPTIONAL { ?file <${nfo.fileSize}> ?byteSize }
+  OPTIONAL { ?file <${nfo.fileExtension}> ?extension }
+  OPTIONAL { ?file <${prov.generatedAtTime}> ?uploadedAt }
+  OPTIONAL { ?file <${ingestManual.uploadPurpose}> ?purpose }
+}
+ORDER BY DESC(STR(?uploadedAt)) STR(?file)
+LIMIT ${uploadsLimit}`;
+}
+
+export const ingestRunsQuery = `SELECT ?run ?file ?status ?endedAt ?recordsRead ?recordsDropped ?dataset WHERE {
+  ?run a <${ingest.IngestRun}> ;
+    <${prov.used}> ?file ;
+    <${ingest.status}> ?status .
+  FILTER(isIRI(?run))
+  OPTIONAL { ?run <${prov.endedAtTime}> ?endedAt }
+  OPTIONAL { ?run <${ingest.recordsRead}> ?recordsRead }
+  OPTIONAL { ?run <${ingest.recordsDropped}> ?recordsDropped }
+  OPTIONAL { ?run <${prov.generated}> ?dataset }
+}
+ORDER BY DESC(STR(?endedAt)) STR(?run)
+LIMIT ${uploadsLimit}`;
+
+export function uploadedFilesFromBindings(bindings: RDF.Bindings[]): UploadedFile[] {
+  const files = new Map<string, UploadedFile>();
+
+  for (const binding of bindings) {
+    const iri = bindingValue(binding, "file");
+
+    if (!iri || files.has(iri)) continue;
+
+    const fileName = bindingValue(binding, "fileName") ?? iri;
+
+    files.set(iri, {
+      iri,
+      fileName,
+      mediaType: bindingValue(binding, "mediaType"),
+      byteSize: bindingCount(binding, "byteSize"),
+      extension: bindingValue(binding, "extension") ?? getExtension(fileName),
+      uploadedAt: bindingValue(binding, "uploadedAt"),
+      uploader: bindingValue(binding, "uploader"),
+      purpose: bindingValue(binding, "purpose"),
+    });
+  }
+
+  return [...files.values()];
+}
+
+export function ingestRunsFromBindings(bindings: RDF.Bindings[]): IngestRun[] {
+  const runs = new Map<string, IngestRun>();
+
+  for (const binding of bindings) {
+    const iri = bindingValue(binding, "run");
+    const file = bindingValue(binding, "file");
+    const status = bindingValue(binding, "status");
+
+    if (!iri || !file || !isIngestStatus(status) || runs.has(iri)) continue;
+
+    runs.set(iri, {
+      iri,
+      file,
+      status,
+      endedAt: bindingValue(binding, "endedAt"),
+      recordsRead: bindingCount(binding, "recordsRead"),
+      recordsDropped: bindingCount(binding, "recordsDropped"),
+      dataset: bindingValue(binding, "dataset"),
+    });
+  }
+
+  return [...runs.values()];
+}
+
+export function latestRunByFile(runs: readonly IngestRun[]): Map<string, IngestRun> {
+  const latest = new Map<string, IngestRun>();
+
+  for (const run of runs) {
+    const current = latest.get(run.file);
+
+    if (!current || (run.endedAt ?? "") > (current.endedAt ?? "")) latest.set(run.file, run);
+  }
+
+  return latest;
+}
+
+// Only a file recorded with the "file" purpose is picked up by ingest; a bundle or an older upload is only stored.
+export function fileIngestState(file: UploadedFile, run: IngestRun | undefined): FileIngestState {
+  if (run) return { kind: "run", run };
+
+  return file.purpose === "file" && isIngestableExtension(file.extension) ? { kind: "waiting" } : { kind: "stored" };
+}
+
+export function filesWithIngestState(
+  files: readonly UploadedFile[],
+  runs: readonly IngestRun[],
+): UploadedFileWithState[] {
+  const latest = latestRunByFile(runs);
+
+  return files.map((file) => ({ ...file, state: fileIngestState(file, latest.get(file.iri)) }));
+}
+
+export function hasWaitingFile(files: readonly UploadedFileWithState[]): boolean {
+  return files.some((file) => file.state.kind === "waiting");
+}
+
+export type UploadedFileWithState = UploadedFile & { state: FileIngestState };
+
+type UseUploadsOptions = { includeAllFiles: boolean };
+
+export function useUploads(options: UseUploadsOptions) {
+  const { includeAllFiles } = options;
+
+  const filesQuery = useSparqlSelect({
+    query: uploadedFilesQuery(includeAllFiles),
+    queryKey: [...uploadsQueryKey, "files", includeAllFiles],
+  });
+
+  const files = useMemo(() => uploadedFilesFromBindings(filesQuery.data ?? []), [filesQuery.data]);
+
+  const runsQuery = useSparqlSelect({
+    query: ingestRunsQuery,
+    queryKey: [...uploadsQueryKey, "runs"],
+    refetchInterval: (query) =>
+      hasWaitingFile(filesWithIngestState(files, ingestRunsFromBindings(query.state.data ?? [])))
+        ? ingestPollIntervalMs
+        : false,
+  });
+
+  const runs = useMemo(() => ingestRunsFromBindings(runsQuery.data ?? []), [runsQuery.data]);
+  const filesWithState = useMemo(() => filesWithIngestState(files, runs), [files, runs]);
+  const fileByIri = useMemo(() => new Map(files.map((file) => [file.iri, file])), [files]);
+
+  return {
+    files: filesWithState,
+    runs,
+    fileByIri,
+    hasWaiting: hasWaitingFile(filesWithState),
+    isLoading: filesQuery.isLoading || runsQuery.isLoading,
+    error: filesQuery.error ?? runsQuery.error,
+  };
+}
+
+export function useInvalidateUploads() {
+  const queryClient = useQueryClient();
+
+  return useCallback(() => queryClient.invalidateQueries({ queryKey: uploadsQueryKey }), [queryClient]);
+}

package/src/routes/datastores/components/LinkUnlinkedDatastoreDialog.tsx

@@ -1,5 +1,6 @@
 import { useLinkUnlinkedDatastoreMutation, useDatastoreTreeQuery, useUnlinkedDatastoresQuery } from "@manta-eu/client";
 import { parseError } from "@manta-eu/client/shared/error-utils";
+import { normalizeGroupPath, pathSegments } from "@manta-eu/client/shared/team";
 import { isPresent } from "@manta-eu/lib/present";
 import { useQueryClient } from "@tanstack/react-query";
 import { useState } from "react";
@@ -30,6 +31,7 @@ export type SuggestedLinkTarget = {
 export type LinkUnlinkedDatastoreDialogProps = {
   datastoreId: string;
   organizationNames: string[];
+  groupPathsByOrganization: Map<string, string[]>;
   existingOrganizationNames: Set<string>;
   suggestedMatch?: SuggestedLinkTarget;
   onClose: () => void;
@@ -37,8 +39,17 @@ export type LinkUnlinkedDatastoreDialogProps = {
 
 type Mode = "new" | "existing";
 
+const organizationRootPath = "/";
+
 export function LinkUnlinkedDatastoreDialog(props: LinkUnlinkedDatastoreDialogProps) {
-  const { datastoreId, organizationNames, existingOrganizationNames, suggestedMatch, onClose } = props;
+  const {
+    datastoreId,
+    organizationNames,
+    groupPathsByOrganization,
+    existingOrganizationNames,
+    suggestedMatch,
+    onClose,
+  } = props;
   const { mantaClient } = useMantaClient();
   const queryClient = useQueryClient();
 
@@ -46,6 +57,11 @@ export function LinkUnlinkedDatastoreDialog(props: LinkUnlinkedDatastoreDialogPr
   const [organizationName, setOrganizationName] = useState("");
   const [existingOrganizationName, setExistingOrganizationName] = useState(suggestedMatch?.organizationName ?? "");
   const [name, setName] = useState(suggestedMatch?.name ?? "");
+  const [parentGroupPath, setParentGroupPath] = useState(
+    suggestedMatch ? normalizeGroupPath(suggestedMatch.parentPath.join("/")) : organizationRootPath,
+  );
+  const parentGroupPaths = groupPathsByOrganization.get(existingOrganizationName) ?? [];
+  const offeredParentGroupPath = parentGroupPaths.includes(parentGroupPath) ? parentGroupPath : organizationRootPath;
 
   const linkMutation = useLinkUnlinkedDatastoreMutation(mantaClient, {
     onSuccess: (result) => {
@@ -53,7 +69,11 @@ export function LinkUnlinkedDatastoreDialog(props: LinkUnlinkedDatastoreDialogPr
         void queryClient.invalidateQueries({ queryKey: useDatastoreTreeQuery.getKey() });
         void queryClient.invalidateQueries({ queryKey: useUnlinkedDatastoresQuery.getKey() });
         void queryClient.invalidateQueries({ queryKey: ["DatastoreProvisioningStatuses"] });
-        toast.success(`Linked to the existing datastore '${datastoreId}'`);
+        toast.success(
+          result.linkUnlinkedDatastore.pending
+            ? `Linked to '${datastoreId}', which is still being built`
+            : `Linked to the existing datastore '${datastoreId}'`,
+        );
         toastProviderWarning(result.linkUnlinkedDatastore.providerWarning);
         onClose();
       } else {
@@ -72,12 +92,12 @@ export function LinkUnlinkedDatastoreDialog(props: LinkUnlinkedDatastoreDialogPr
 
   const canSubmit = mode === "new" ? !!organizationName.trim() && !tenantCheck.blocked : !!existingOrganizationName;
   const atExistingOrganizationRoot = mode === "existing" && !name.trim();
-  const suggestedParentPath =
-    suggestedMatch &&
-    existingOrganizationName === suggestedMatch.organizationName &&
-    name.trim() === suggestedMatch.name
-      ? suggestedMatch.parentPath
-      : [];
+  const parentPath = pathSegments(offeredParentGroupPath);
+
+  function selectOrganization(organization: string) {
+    setExistingOrganizationName(organization);
+    setParentGroupPath(organizationRootPath);
+  }
 
   function submit() {
     if (!canSubmit) return;
@@ -107,7 +127,7 @@ export function LinkUnlinkedDatastoreDialog(props: LinkUnlinkedDatastoreDialogPr
           : {
               datastoreId,
               organizationName: existingOrganizationName,
-              ...(isPresent(suggestedParentPath) ? { parentPath: suggestedParentPath } : {}),
+              ...(isPresent(parentPath) ? { parentPath } : {}),
               name: name.trim(),
               atOrganizationRoot: false,
             };
@@ -143,7 +163,7 @@ export function LinkUnlinkedDatastoreDialog(props: LinkUnlinkedDatastoreDialogPr
           </>
         ) : (
           <>
-            <Select value={existingOrganizationName} onValueChange={setExistingOrganizationName}>
+            <Select value={existingOrganizationName} onValueChange={selectOrganization}>
               <SelectTrigger label="Organization" id="link-organization">
                 <SelectValue placeholder="Select an organization…" />
               </SelectTrigger>
@@ -155,6 +175,25 @@ export function LinkUnlinkedDatastoreDialog(props: LinkUnlinkedDatastoreDialogPr
                 ))}
               </SelectContent>
             </Select>
+            {isPresent(parentGroupPaths) && (
+              <Select
+                value={offeredParentGroupPath}
+                onValueChange={setParentGroupPath}
+                disabled={atExistingOrganizationRoot}
+              >
+                <SelectTrigger label="Under" id="link-parent-group">
+                  <SelectValue />
+                </SelectTrigger>
+                <SelectContent>
+                  <SelectItem value={organizationRootPath}>Organization root</SelectItem>
+                  {parentGroupPaths.map((path) => (
+                    <SelectItem key={path} value={path}>
+                      {path}
+                    </SelectItem>
+                  ))}
+                </SelectContent>
+              </Select>
+            )}
             <Input
               label="Datastore name"
               placeholder="Leave empty to link to the organization root"

package/src/routes/datastores/components/StatusBadge.tsx

@@ -1,3 +1,4 @@
+import { enableDatastoreRemoval } from "@manta-eu/client/shared/constants";
 import { isPresent } from "@manta-eu/lib/present";
 import { LoaderCircle } from "lucide-react";
 import { Badge, type BadgeVariant } from "#components/badge/index";
@@ -7,13 +8,13 @@ import {
   type DatastoreStatusCategory,
 } from "#components/datastore-management/tree";
 
-export type DatastoreStatusFilterValue = "noDatastore" | "checking" | DatastoreStatusCategory;
+export type DatastoreStatusFilterValue = "noDatastore" | "checking" | "notBuilt" | DatastoreStatusCategory;
 
-export function datastoreStatusFilterValue(
-  node: { datastoreId?: string; removed: boolean; isUnlinked?: boolean },
-  status?: DatastoreStatus,
-): DatastoreStatusFilterValue {
+type StatusNode = { datastoreId?: string; removed: boolean; isUnlinked?: boolean; unbuilt?: boolean };
+
+export function datastoreStatusFilterValue(node: StatusNode, status?: DatastoreStatus): DatastoreStatusFilterValue {
   if (!isPresent(node.datastoreId)) return "noDatastore";
+  if (node.unbuilt === true && status?.exists !== true) return "notBuilt";
   return datastoreStatusCategory(node, status) ?? "checking";
 }
 
@@ -33,10 +34,23 @@ const badgeConfigByFilterValue: Record<DatastoreStatusFilterValue, BadgeConfig>
     variant: "destructive",
     title: "This id never reached the datastore registry, so nothing will build it. Provision it again.",
   },
+  notBuilt: {
+    label: "Not built",
+    variant: "warning",
+    title: "Registered in Infisical, but no datastore exists for it. Link it to build it, or delete it.",
+  },
   offline: { label: "Offline", variant: "destructive" },
   unavailable: { label: "Status unavailable", variant: "copy" },
   checking: { label: "Checking…", variant: "copy", spinning: true },
-  removed: { label: "Removed", variant: "copy" },
+  removed: enableDatastoreRemoval
+    ? { label: "Removed", variant: "copy" }
+    : {
+        label: "Removed",
+        variant: "warning",
+        title:
+          "Out of use: a permanent delete stopped partway, or it was removed before removal was turned off. " +
+          "Delete it permanently only if it should go for good.",
+      },
   noDatastore: { label: "No datastore", variant: "copy" },
 };
 
@@ -44,6 +58,7 @@ const filterOptionOrder: DatastoreStatusFilterValue[] = [
   "ready",
   "materializing",
   "notRegistered",
+  "notBuilt",
   "offline",
   "unavailable",
   "removed",
@@ -55,7 +70,7 @@ export const datastoreStatusFilterOptions: { id: "ALL" | DatastoreStatusFilterVa
 ];
 
 export type DatastoreStatusBadgeProps = {
-  node: { datastoreId?: string; removed: boolean; isUnlinked?: boolean };
+  node: StatusNode;
   status?: DatastoreStatus;
 };
 

package/src/routes/datastores/flatten.ts

@@ -25,6 +25,7 @@ export type DatastoreRow = {
   // Real in the datastore, but no Keycloak node behind it, so it can only be selected.
   isRealmDefault?: boolean;
   isUnlinked?: boolean;
+  unbuilt?: boolean;
 };
 
 export function realmDefaultRow(datastoreId: Realm): DatastoreRow {
@@ -105,6 +106,15 @@ export function hiddenRowCount(
   return visibleDatastoreRows(rows, { ...visibility, [key]: true }).length - shownCount;
 }
 
+export function hiddenRemovedRowCount(rows: DatastoreRow[], visibility: DatastoreRowVisibility): number {
+  if (visibility.showArchived) return 0;
+
+  const shown = new Set(visibleDatastoreRows(rows, visibility));
+  return rows.filter(
+    (row) => row.removed && !shown.has(row) && matchesVisibility(row, { ...visibility, showArchived: true }),
+  ).length;
+}
+
 // Matches Keycloak's own org lookup (ensureOrganization), which is case-insensitive and trims too.
 export function organizationNameKey(name: string): string {
   return name.trim().toLowerCase();
@@ -118,6 +128,33 @@ export function organizationNames(rows: DatastoreRow[]): string[] {
   return [...new Set(rows.filter((row) => row.isOrganization).map((row) => row.organizationName))].sort();
 }
 
+export function removedSubtreeRowIds(rows: DatastoreRow[]): Set<string> {
+  const ids = new Set<string>();
+  let removedDepth: number | undefined;
+  for (const row of rows) {
+    if (removedDepth === undefined || row.depth <= removedDepth) removedDepth = row.removed ? row.depth : undefined;
+    if (removedDepth !== undefined) ids.add(row.id);
+  }
+  return ids;
+}
+
+export function outsideRemovedSubtrees(rows: DatastoreRow[]): DatastoreRow[] {
+  const removed = removedSubtreeRowIds(rows);
+  return rows.filter((row) => !removed.has(row.id));
+}
+
+export function organizationGroupPaths(rows: DatastoreRow[]): Map<string, string[]> {
+  const paths = new Map<string, string[]>();
+  for (const row of outsideRemovedSubtrees(rows)) {
+    if (row.isOrganization || row.isRealmDefault === true || row.isUnlinked === true) continue;
+
+    const organizationPaths = paths.get(row.organizationName);
+    if (organizationPaths) organizationPaths.push(row.path);
+    else paths.set(row.organizationName, [row.path]);
+  }
+  return paths;
+}
+
 export function visibilityForStatusFilter(
   visibility: DatastoreRowVisibility,
   status: DatastoreStatusFilterValue | "ALL",
@@ -125,11 +162,21 @@ export function visibilityForStatusFilter(
   return match(status)
     .with("removed", () => ({ ...visibility, showArchived: true }))
     .with("noDatastore", () => ({ ...visibility, showUnprovisioned: true }))
-    .with("ALL", "ready", "materializing", "notRegistered", "offline", "unavailable", "checking", () => visibility)
+    .with(
+      "ALL",
+      "ready",
+      "materializing",
+      "notRegistered",
+      "notBuilt",
+      "offline",
+      "unavailable",
+      "checking",
+      () => visibility,
+    )
     .exhaustive();
 }
 
-export function unlinkedDatastoreRow(datastoreId: string): DatastoreRow {
+export function unlinkedDatastoreRow(datastoreId: string, built = true): DatastoreRow {
   return {
     id: `unlinked:${datastoreId}`,
     name: datastoreId,
@@ -142,6 +189,7 @@ export function unlinkedDatastoreRow(datastoreId: string): DatastoreRow {
     removed: false,
     isOrganization: false,
     isUnlinked: true,
+    unbuilt: !built,
   };
 }
 

package/src/routes/datastores/hooks/useDatastoreColumns.tsx

@@ -6,6 +6,7 @@ import {
   Blocks,
   Check,
   DatabaseZap,
+  Hammer,
   Link2,
   MoreHorizontal,
   Pencil,
@@ -113,6 +114,8 @@ type UseDatastoreColumnsArgs = {
   onDelete: (row: DatastoreRow) => void;
   onRename: (row: DatastoreRow) => void;
   onRegister: (row: DatastoreRow) => void;
+  registeringDatastoreIds: ReadonlySet<string>;
+  removedSubtreeRowIds: ReadonlySet<string>;
   onManageUsers: (row: DatastoreRow) => void;
   onManageCustomApps: (row: DatastoreRow) => void;
   onBringOnline: (row: DatastoreRow) => void;
@@ -141,6 +144,8 @@ export function useDatastoreColumns(args: UseDatastoreColumnsArgs): ColumnDef<Da
     onDelete,
     onRename,
     onRegister,
+    registeringDatastoreIds,
+    removedSubtreeRowIds,
     onManageUsers,
     onManageCustomApps,
     onBringOnline,
@@ -311,6 +316,8 @@ export function useDatastoreColumns(args: UseDatastoreColumnsArgs): ColumnDef<Da
           const status = hasDatastore ? statusByDatastoreId[datastoreId] : undefined;
           const category = datastoreStatusCategory(datastoreRow, status);
           const notRegistered = category === "notRegistered";
+          const buildable = category === "materializing";
+          const registering = hasDatastore && registeringDatastoreIds.has(datastoreId);
           const reason = match({ notRegistered, materializing: isMaterializing(status) && !isRealmDefault })
             .with({ notRegistered: true }, () => "Not registered - use Register to fix, then try selecting again.")
             .with({ materializing: true }, () => "Still materializing - this can take a few minutes.")
@@ -359,12 +366,13 @@ export function useDatastoreColumns(args: UseDatastoreColumnsArgs): ColumnDef<Da
                   Bring online
                 </Button>
               )}
-              {!isUnlinked && hasDatastore && !removed && notRegistered && (
+              {!isUnlinked && hasDatastore && !removedSubtreeRowIds.has(datastoreRow.id) && notRegistered && (
                 <Button
                   data-testid={`datastore-register-${datastoreRow.id}`}
                   size="sm"
                   variant="secondary"
                   icon={RefreshCw}
+                  disabled={registering}
                   onClick={() => onRegister(datastoreRow)}
                 >
                   Register
@@ -439,6 +447,16 @@ export function useDatastoreColumns(args: UseDatastoreColumnsArgs): ColumnDef<Da
                               Provision datastore
                             </DropdownItem>
                           ))}
+                        {hasDatastore && !removedSubtreeRowIds.has(datastoreRow.id) && buildable && (
+                          <DropdownItem
+                            icon={<Hammer className="size-4" />}
+                            disabled={registering}
+                            title="Starts a build, or joins the one already running"
+                            onSelect={() => onRegister(datastoreRow)}
+                          >
+                            Build datastore now
+                          </DropdownItem>
+                        )}
                         <DropdownItem icon={<Plus className="size-4" />} onSelect={() => onAddChild(datastoreRow)}>
                           Add datastore
                         </DropdownItem>
@@ -505,6 +523,8 @@ export function useDatastoreColumns(args: UseDatastoreColumnsArgs): ColumnDef<Da
       onRestore,
       onDelete,
       onRegister,
+      registeringDatastoreIds,
+      removedSubtreeRowIds,
       onRename,
       onManageUsers,
       onManageCustomApps,