@geode/opengeodeweb-front 10.40.1-rc.1 → 10.41.0-rc.1

package/.build/shared/utils/file.js

@@ -5,7 +5,7 @@ const BYTES_PER_KIBIBYTE = 1024;
 const CHUNK_SIZE_BYTES = 8192 * BYTES_PER_KIBIBYTE;
 function assertFile(file) {
 	consola.debug("[ASSERT_FILE] Asserting file", { file });
-	if (!(file instanceof File)) throw new Error("file must be an instance of File");
+	if (!(file instanceof File)) throw new TypeError("file must be an instance of File");
 }
 function hasBody(params) {
 	if (params instanceof FormData || params instanceof Blob) return true;

package/.build/shared/utils/response_handlers/load.js

@@ -46,5 +46,23 @@ function resolveAllowedObjects(filenames, allowedObjectsList) {
 		selectedGeodeObject: selectGeodeObject(mergedAllowedObjects)
 	};
 }
+function sortCandidates(objectMap) {
+	return Object.keys(objectMap).filter((key) => getObject(objectMap, key).is_loadable > 0).toSorted((keyA, keyB) => {
+		const objectA = getObject(objectMap, keyA);
+		const objectB = getObject(objectMap, keyB);
+		if (objectA.is_loadable !== objectB.is_loadable) return objectB.is_loadable - objectA.is_loadable;
+		return (objectB.object_priority ?? -Infinity) - (objectA.object_priority ?? -Infinity);
+	});
+}
+function resolveAllowedObjectsPerFile(filenames, allowedObjectsList) {
+	return filenames.map((filename, index) => {
+		const objectMap = allowedObjectsList[index] ?? {};
+		return {
+			filename,
+			candidates: sortCandidates(objectMap),
+			selectedGeodeObject: selectGeodeObject(objectMap)
+		};
+	});
+}
 //#endregion
-export { getFileExtension, resolveAllowedObjects };
+export { getFileExtension, resolveAllowedObjects, resolveAllowedObjectsPerFile };

package/app/components/MissingFilesSelector.vue

@@ -15,12 +15,17 @@ interface FilePlan {
   mandatory_files: string[];
   additional_files: string[];
 }
+interface FileMissingFiles {
+  filename: string;
+  mandatory_files: string[];
+  additional_files: string[];
+}
 
 const schema = schemas.opengeodeweb_back.data_missing_files;
 const time_series_schema = schemas.opengeodeweb_back.time_series_missing_files;
 
 interface Emits {
-  update_values: [value: { additional_files: UploadFile[] }];
+  update_values: [value: { additional_files?: UploadFile[]; skipped_files?: string[] }];
   increment_step: [];
   decrement_step: [];
 }
@@ -29,7 +34,8 @@ const emit = defineEmits<Emits>();
 
 interface Props {
   multiple: boolean;
-  geodeObjectType: string;
+  // Filename -> geode object type, each file is checked with its own type (unused for time series)
+  geodeObjectTypes?: Record<string, string>;
   filenames: string[];
   files?: UploadFile[];
   timeSeries?: boolean;
@@ -37,7 +43,7 @@ interface Props {
 
 const {
   multiple,
-  geodeObjectType,
+  geodeObjectTypes = {},
   filenames,
   files = [],
   timeSeries = false,
@@ -48,11 +54,36 @@ const loading = ref<boolean>(false);
 const has_missing_files = ref<boolean>(false);
 const mandatory_files = ref<string[]>([]);
 const additional_files = ref<string[]>([]);
+const missing_by_file = ref<FileMissingFiles[]>([]);
+// Files left out at the data type step are not checked
+const checked_filenames = computed(() =>
+  timeSeries ? filenames : filenames.filter((filename) => filename in geodeObjectTypes),
+);
+// Files whose mandatory files are missing are skipped when the step is skipped
+const files_without_mandatory = computed(() =>
+  checked_filenames.value.filter(
+    (filename) =>
+      !missing_by_file.value.some(
+        (missing) => missing.filename === filename && missing.mandatory_files.length > 0,
+      ),
+  ),
+);
+const can_skip = computed(
+  () => missing_by_file.value.length > 0 && files_without_mandatory.value.length > 0,
+);
 const toggle_loading = useToggle(loading);
 const uploading = ref<boolean>(false);
 
 function files_uploaded_event(value: UploadFile[]): void {
-  emit("update_values", { additional_files: value });
+  emit("update_values", { additional_files: value, skipped_files: [] });
+  emit("increment_step");
+}
+
+function skip_step(): void {
+  const skipped_files = checked_filenames.value.filter(
+    (filename) => !files_without_mandatory.value.includes(filename),
+  );
+  emit("update_values", { skipped_files });
   emit("increment_step");
 }
 
@@ -65,9 +96,10 @@ async function missing_files(): Promise<void> {
   has_missing_files.value = false;
   mandatory_files.value = [];
   additional_files.value = [];
+  missing_by_file.value = [];
   const backStore = useBackStore();
 
-  const promise_array: Promise<FilePlan>[] = filenames.map((filename): Promise<FilePlan> => {
+  const promise_array = checked_filenames.value.map((filename): Promise<FilePlan> => {
     if (isCsvFile(filename)) {
       return Promise.resolve({
         has_missing_files: false,
@@ -78,13 +110,18 @@ async function missing_files(): Promise<void> {
     if (timeSeries) {
       return backStore.request({ schema: time_series_schema, params: { filename } });
     }
-    const params = { geode_object_type: geodeObjectType, filename };
+    const params = { geode_object_type: geodeObjectTypes[filename] ?? "", filename };
     return backStore.request({ schema, params });
   });
   const values = await Promise.all(promise_array);
-  for (const value of values) {
+  for (const [index, value] of values.entries()) {
     if (value.has_missing_files) {
       has_missing_files.value = true;
+      missing_by_file.value.push({
+        filename: checked_filenames.value[index] ?? "",
+        mandatory_files: value.mandatory_files,
+        additional_files: value.additional_files,
+      });
     }
     mandatory_files.value = [...mandatory_files.value, ...value.mandatory_files];
     additional_files.value = [...additional_files.value, ...value.additional_files];
@@ -98,6 +135,7 @@ async function missing_files(): Promise<void> {
   }
 
   if (!has_missing_files.value) {
+    emit("update_values", { skipped_files: [] });
     emit("increment_step");
   }
   toggle_loading();
@@ -159,7 +197,40 @@ await missing_files();
 <template>
   <FetchingData v-if="loading || uploading" />
   <v-container v-else-if="has_missing_files">
-    <v-row v-if="mandatory_files.length" align="center">
+    <template v-if="checked_filenames.length > 1">
+      <div v-for="missing in missing_by_file" :key="missing.filename" class="mb-2">
+        <v-row align="center">
+          <p class="pa-1 font-weight-bold">{{ missing.filename }}</p>
+          <v-chip
+            v-if="missing.mandatory_files.length && files_without_mandatory.length"
+            class="ma-1"
+            color="warning"
+            size="small"
+          >
+            Skipped if mandatory files are not added
+          </v-chip>
+        </v-row>
+        <v-row v-if="missing.mandatory_files.length" align="center">
+          <v-col cols="auto" class="pa-0">
+            <v-icon color="warning" icon="mdi-file-document-alert-outline" />
+          </v-col>
+          <p class="pa-1">Mandatory files:</p>
+          <v-col v-for="mandatory_file in missing.mandatory_files" cols="auto" class="pa-0">
+            <v-chip>{{ mandatory_file }}</v-chip>
+          </v-col>
+        </v-row>
+        <v-row v-if="missing.additional_files.length" align="center">
+          <v-col cols="auto" class="pa-0">
+            <v-icon color="accent" icon="mdi-file-document-plus-outline" />
+          </v-col>
+          <p class="pa-1">Additional files:</p>
+          <v-col v-for="additional_file in missing.additional_files" cols="auto" class="pa-0">
+            <v-chip>{{ additional_file }}</v-chip>
+          </v-col>
+        </v-row>
+      </div>
+    </template>
+    <v-row v-else-if="mandatory_files.length" align="center">
       <v-col cols="auto" class="pa-0">
         <v-icon color="warning" icon="mdi-file-document-alert-outline" />
       </v-col>
@@ -168,7 +239,7 @@ await missing_files();
         <v-chip>{{ mandatory_file }}</v-chip>
       </v-col>
     </v-row>
-    <v-row v-if="additional_files.length" align="center">
+    <v-row v-if="!(checked_filenames.length > 1) && additional_files.length" align="center">
       <v-col cols="auto" class="pa-0">
         <v-icon color="accent" icon="mdi-file-document-plus-outline" />
       </v-col>
@@ -194,8 +265,10 @@ await missing_files();
       </v-col>
     </v-row>
     <v-row>
-      <v-col v-if="mandatory_files.length === 0 && additional_files.length > 0" cols="auto">
-        <v-btn @click="emit('increment_step')" color="warning">Skip step</v-btn>
+      <v-col v-if="can_skip" cols="auto">
+        <v-btn @click="skip_step" color="warning" data-testid="skipMissingFilesButton">
+          Skip step
+        </v-btn>
       </v-col>
     </v-row>
   </v-container>

package/app/components/ObjectSelector.vue

@@ -1,7 +1,11 @@
 <script setup lang="ts">
+import {
+  type FileAllowedObjects,
+  resolveAllowedObjects,
+  resolveAllowedObjectsPerFile,
+} from "@ogw_shared/utils/response_handlers/load.js";
 import FetchingData from "@ogw_front/components/FetchingData.vue";
 import { geode_objects } from "@ogw_front/assets/geode_objects";
-import { resolveAllowedObjects } from "@ogw_shared/utils/response_handlers/load.js";
 import schemas from "@geode/opengeodeweb-back/opengeodeweb_back_typed_schemas.js";
 import { useBackStore } from "@ogw_front/stores/back";
 
@@ -14,8 +18,15 @@ type AllowedObjectMap = Record<string, AllowedObject>;
 
 const schema = schemas.opengeodeweb_back.allowed_objects;
 
+interface PendingGroup {
+  filenames: string[];
+  candidates: string[];
+  selected: string;
+}
+
 const emit = defineEmits<{
-  update_values: [value: { geode_object_type: string }];
+  // Filename -> geode object type, files that cannot be loaded are left out
+  update_values: [value: { geode_object_types: Record<string, string> }];
   increment_step: [];
 }>();
 
@@ -31,6 +42,14 @@ const loading = ref<boolean>(false);
 const allowedGeodeObjects = ref<AllowedObjectMap>({});
 const toggleLoading = useToggle(loading);
 const multipleFilesNoCommon = ref<boolean>(false);
+// Types already known, files left to choose for, and files that cannot be loaded
+const resolvedTypes = ref<Record<string, string>>({});
+const pendingFilenames = ref<string[]>([]);
+const pendingGroups = ref<PendingGroup[]>([]);
+const skippedFiles = ref<string[]>([]);
+const showPerFile = ref<boolean>(false);
+// Every file got its type without the user, nothing to display
+const autoResolved = ref<boolean>(false);
 
 async function fetchAllowedObjectsList(): Promise<AllowedObjectMap[]> {
   const promiseArray = filenames.map((filename): Promise<{ allowed_objects: AllowedObjectMap }> => {
@@ -41,11 +60,43 @@ async function fetchAllowedObjectsList(): Promise<AllowedObjectMap[]> {
   return responses.map((response) => response.allowed_objects);
 }
 
+function emitGeodeObjectTypes(geode_object_types: Record<string, string>): void {
+  emit("update_values", { geode_object_types });
+  emit("increment_step");
+}
+
 function setGeodeObject(geode_object_type: string | undefined): void {
-  if (geode_object_type) {
-    emit("update_values", { geode_object_type });
-    emit("increment_step");
+  if (!geode_object_type) {
+    return;
+  }
+  const chosen = pendingFilenames.value.map((filename) => [filename, geode_object_type]);
+  emitGeodeObjectTypes({ ...resolvedTypes.value, ...Object.fromEntries(chosen) });
+}
+
+function confirmPerFile(): void {
+  const chosen = pendingGroups.value.flatMap((group) =>
+    group.filenames.map((filename) => [filename, group.selected]),
+  );
+  emitGeodeObjectTypes({ ...resolvedTypes.value, ...Object.fromEntries(chosen) });
+}
+
+// Files with the same candidate types share one choice
+function groupPendingFiles(pending: readonly FileAllowedObjects[]): PendingGroup[] {
+  const groups = new Map<string, PendingGroup>();
+  for (const file of pending) {
+    const key = file.candidates.join(",");
+    const group = groups.get(key);
+    if (group) {
+      group.filenames.push(file.filename);
+    } else {
+      groups.set(key, {
+        filenames: [file.filename],
+        candidates: file.candidates,
+        selected: file.candidates[0] ?? "",
+      });
+    }
   }
+  return [...groups.values()];
 }
 
 // The geode_objects lookup is fixed and keyed by known object type names; a missing entry only happens if the backend reports a type this table doesn't know about.
@@ -59,19 +110,63 @@ function geodeObjectImage(key: string): string | undefined {
   return geode_objects[key]?.image;
 }
 
+function showCommonGeodeObjects(
+  pendingNames: readonly string[],
+  allowedObjectsList: readonly AllowedObjectMap[],
+): string | undefined {
+  const resolved = resolveAllowedObjects(pendingNames, allowedObjectsList);
+  allowedGeodeObjects.value = resolved.mergedAllowedObjects;
+  multipleFilesNoCommon.value = resolved.multipleFilesNoCommon;
+  return resolved.selectedGeodeObject;
+}
+
+// Each file gets its own type. Returns true when the per file panel is needed, i.e. the files cannot be handled by a single choice
+function resolvePerFile(allowedObjectsList: readonly AllowedObjectMap[]): boolean {
+  const files = resolveAllowedObjectsPerFile(filenames, allowedObjectsList);
+  const loadable = files.filter((file) => file.candidates.length > 0);
+  if (loadable.length === 0) {
+    // Nothing can be loaded: show why with the common types
+    showCommonGeodeObjects(filenames, allowedObjectsList);
+    return false;
+  }
+  const pending = loadable.filter((file) => file.selectedGeodeObject === undefined);
+  resolvedTypes.value = Object.fromEntries(
+    loadable.flatMap((file) =>
+      file.selectedGeodeObject === undefined ? [] : [[file.filename, file.selectedGeodeObject]],
+    ),
+  );
+  pendingFilenames.value = pending.map((file) => file.filename);
+  skippedFiles.value = files
+    .filter((file) => file.candidates.length === 0)
+    .map((file) => file.filename);
+  if (skippedFiles.value.length === 0) {
+    if (pending.length === 0) {
+      autoResolved.value = true;
+      emitGeodeObjectTypes(resolvedTypes.value);
+      return false;
+    }
+    const pendingAllowedObjects = pending.map(
+      (file) => allowedObjectsList[filenames.indexOf(file.filename)] ?? {},
+    );
+    // Pending files sharing a loadable type: one choice for all of them
+    const selected = showCommonGeodeObjects(pendingFilenames.value, pendingAllowedObjects);
+    if (Object.values(allowedGeodeObjects.value).some((object) => object.is_loadable > 0)) {
+      setGeodeObject(selected);
+      return false;
+    }
+  }
+  pendingGroups.value = groupPendingFiles(pending);
+  return true;
+}
+
 async function getAllowedGeodeObjects(): Promise<void> {
   toggleLoading();
   allowedGeodeObjects.value = {};
   multipleFilesNoCommon.value = false;
+  showPerFile.value = false;
 
   const allowedGeodeObjectsList = await fetchAllowedObjectsList();
-  const resolved = resolveAllowedObjects(filenames, allowedGeodeObjectsList);
-
-  allowedGeodeObjects.value = resolved.mergedAllowedObjects;
-  multipleFilesNoCommon.value = resolved.multipleFilesNoCommon;
-  if (resolved.selectedGeodeObject) {
-    setGeodeObject(resolved.selectedGeodeObject);
-  }
+  showPerFile.value = resolvePerFile(allowedGeodeObjectsList);
   toggleLoading();
 }
 // oxlint-disable-next-line no-top-level-await
@@ -80,7 +175,46 @@ await getAllowedGeodeObjects();
 
 <template>
   <FetchingData v-if="loading" />
+  <v-container v-else-if="showPerFile" data-testid="perFileObjectSelector">
+    <v-row v-for="group in pendingGroups" :key="group.filenames.join()" align="center">
+      <v-col cols="auto" class="pa-1">
+        <v-chip v-for="filename in group.filenames" :key="filename" class="ma-1" size="small">
+          {{ filename }}
+        </v-chip>
+      </v-col>
+      <v-col class="pa-1">
+        <v-select
+          v-model="group.selected"
+          :items="group.candidates"
+          density="compact"
+          hide-details
+          variant="outlined"
+        />
+      </v-col>
+    </v-row>
+    <v-row v-for="(type, filename) in resolvedTypes" :key="filename" align="center">
+      <v-chip class="ma-1" size="small">{{ filename }}</v-chip>
+      <v-chip class="ma-1" size="small" variant="tonal">{{ type }}</v-chip>
+    </v-row>
+    <v-row v-if="skippedFiles.length" align="center">
+      <v-icon color="warning" icon="mdi-file-alert-outline" class="ma-1" />
+      <p class="pa-1">Not supported, will be skipped:</p>
+      <v-chip v-for="filename in skippedFiles" :key="filename" class="ma-1" size="small">
+        {{ filename }}
+      </v-chip>
+    </v-row>
+    <v-row>
+      <v-col cols="auto">
+        <v-btn color="primary" data-testid="confirmDataTypesButton" @click="confirmPerFile">
+          Continue
+        </v-btn>
+      </v-col>
+    </v-row>
+  </v-container>
   <v-row v-else-if="Object.keys(allowedGeodeObjects).length" class="justify-left">
+    <v-col v-if="pendingFilenames.length && pendingFilenames.length < filenames.length" cols="12">
+      Choose the data type for: {{ pendingFilenames.join(", ") }}
+    </v-col>
     <v-col v-for="(value, key) in allowedGeodeObjects" :key="key" cols="3" md="4">
       <v-tooltip :text="geodeObjectTooltip(key, Boolean(value['is_loadable']))" location="bottom">
         <template v-slot:activator="{ props }">
@@ -112,7 +246,7 @@ await getAllowedGeodeObjects();
       </v-card-text>
     </v-card>
   </v-row>
-  <v-row v-else class="pa-5">
+  <v-row v-else-if="!autoResolved" class="pa-5">
     <v-card class="card" variant="tonal" rounded>
       <v-card-text>
         This file format isn't supported! Please check the

package/app/components/csv-preview/CsvSettings.vue

@@ -26,7 +26,7 @@ const emit = defineEmits<Emits>();
 const separators = [
   { title: "Comma (,)", value: "," },
   { title: "Semicolon (;)", value: ";" },
-  { title: "Tab (\\t)", value: "\t" },
+  { title: String.raw`Tab (\t)`, value: "\t" },
   { title: "Space ( )", value: " " },
   { title: "Pipe (|)", value: "|" },
   { title: "Custom", value: "custom" },

package/app/utils/import_workflow.ts

@@ -96,18 +96,26 @@ async function importWorkflow(files: readonly FileToImport[]): Promise<string[]>
   }
 
   const results: string[] = [];
+  const errors: unknown[] = [];
   async function processChunk(chunkIndex: number): Promise<void> {
     if (chunkIndex >= chunks.length) {
       return;
     }
     const chunk = chunks[chunkIndex] ?? [];
-    const chunk_results = await Promise.all(
+    // A failing file does not prevent the other files from being imported
+    const chunk_results = await Promise.allSettled(
       chunk.map(async ({ filename, geode_object_type }) => {
         const id = await importFile(filename, geode_object_type);
         return id;
       }),
     );
-    results.push(...chunk_results);
+    for (const result of chunk_results) {
+      if (result.status === "fulfilled") {
+        results.push(result.value);
+      } else {
+        errors.push(result.reason);
+      }
+    }
     await processChunk(chunkIndex + 1);
   }
   await processChunk(0);
@@ -116,6 +124,9 @@ async function importWorkflow(files: readonly FileToImport[]): Promise<string[]>
     hybridViewerStore.resetCamera();
   }
   await hybridViewerStore.remoteRender();
+  if (errors.length > 0) {
+    throw new AggregateError(errors, `${errors.length} file(s) could not be imported`);
+  }
   return results;
 }
 

package/internal/utils/upload_file.ts

@@ -44,7 +44,7 @@ async function upload_file(
   const feedbackStore = useFeedbackStore();
 
   if (!(file instanceof File)) {
-    throw new Error("file must be an instance of File");
+    throw new TypeError("file must be an instance of File");
   }
 
   const route = schema.$id;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@geode/opengeodeweb-front",
-  "version": "10.40.1-rc.1",
+  "version": "10.41.0-rc.1",
   "description": "OpenSource Vue/Nuxt/Pinia/Vuetify framework for web applications",
   "homepage": "https://github.com/Geode-solutions/OpenGeodeWeb-Front",
   "bugs": {

package/shared/utils/file.ts

@@ -10,7 +10,7 @@ const CHUNK_SIZE_BYTES = CHUNK_SIZE_MEBIBYTES * KIBIBYTES_PER_MEBIBYTE * BYTES_P
 function assertFile(file: unknown): asserts file is File {
   consola.debug("[ASSERT_FILE] Asserting file", { file });
   if (!(file instanceof File)) {
-    throw new Error("file must be an instance of File");
+    throw new TypeError("file must be an instance of File");
   }
 }
 

package/shared/utils/response_handlers/load.ts

@@ -106,4 +106,40 @@ function resolveAllowedObjects(
   };
 }
 
-export { resolveAllowedObjects, getFileExtension };
+interface FileAllowedObjects {
+  readonly filename: string;
+  // Loadable types only, sorted by best is_loadable then best object_priority
+  readonly candidates: string[];
+  readonly selectedGeodeObject: string | undefined;
+}
+
+function sortCandidates(objectMap: AllowedObjectMap): string[] {
+  return Object.keys(objectMap)
+    .filter((key) => getObject(objectMap, key).is_loadable > 0)
+    .toSorted((keyA, keyB) => {
+      const objectA = getObject(objectMap, keyA);
+      const objectB = getObject(objectMap, keyB);
+      if (objectA.is_loadable !== objectB.is_loadable) {
+        return objectB.is_loadable - objectA.is_loadable;
+      }
+      return (objectB.object_priority ?? -Infinity) - (objectA.object_priority ?? -Infinity);
+    });
+}
+
+// Each file gets its own geode object type; files are no longer required to share one
+function resolveAllowedObjectsPerFile(
+  filenames: readonly string[],
+  allowedObjectsList: readonly AllowedObjectMap[],
+): FileAllowedObjects[] {
+  return filenames.map((filename, index) => {
+    const objectMap = allowedObjectsList[index] ?? {};
+    return {
+      filename,
+      candidates: sortCandidates(objectMap),
+      selectedGeodeObject: selectGeodeObject(objectMap),
+    };
+  });
+}
+
+export { resolveAllowedObjects, resolveAllowedObjectsPerFile, getFileExtension };
+export type { FileAllowedObjects };