geonetwork-ui 2.10.3-dev.4154db07d → 2.11.0-dev.0ddf1324b

package/src/libs/ui/inputs/src/lib/dropdown-multiselect/dropdown-multiselect.component.ts

@@ -24,7 +24,7 @@ import {
   propagateToDocumentOnly,
 } from '../../../../../../libs/util/shared/src'
 import { ButtonComponent } from '../button/button.component'
-import { NgIcon, provideIcons } from '@ng-icons/core'
+import { NgIcon, provideIcons, provideNgIconsConfig } from '@ng-icons/core'
 import { FormsModule } from '@angular/forms'
 import { TranslatePipe } from '@ngx-translate/core'
 
@@ -46,19 +46,22 @@ import {
       matExpandMore,
       matExpandLess,
     }),
+    provideNgIconsConfig({
+      size: '1.5rem',
+    }),
   ],
   standalone: true,
 })
-export class DropdownMultiselectComponent {
+export class DropdownMultiselectComponent<T = unknown> {
   private scrollStrategies = inject(ScrollStrategyOptions)
 
   @Input() title: string
-  @Input() choices: Choice[]
-  @Input() selected: unknown[] = []
+  @Input() choices: Choice<T>[]
+  @Input() selected: T[] = []
   @Input() allowSearch = true
   @Input() maxRows: number
   @Input() searchInputValue = ''
-  @Output() selectValues = new EventEmitter<unknown[]>()
+  @Output() selectValues = new EventEmitter<T[]>()
   @ViewChild('overlayOrigin') overlayOrigin: CdkOverlayOrigin
   @ViewChild(CdkConnectedOverlay) overlay: CdkConnectedOverlay
   @ViewChild('overlayContainer', { read: ElementRef })
@@ -115,7 +118,7 @@ export class DropdownMultiselectComponent {
 
   private setFocus() {
     setTimeout(() => {
-      this.searchFieldInput.nativeElement.focus()
+      this.searchFieldInput?.nativeElement.focus()
     }, 0)
   }
 
@@ -194,18 +197,18 @@ export class DropdownMultiselectComponent {
     this.checkboxes.get(newIndex).nativeElement.focus()
   }
 
-  isSelected(choice: Choice) {
+  isSelected(choice: Choice<T>) {
     return this.selected.indexOf(choice.value) > -1
   }
 
-  select(choice: Choice, selected: boolean) {
+  select(choice: Choice<T>, selected: boolean) {
     this.selected = selected
       ? [...this.selected.filter((v) => v !== choice.value), choice.value]
       : this.selected.filter((v) => v !== choice.value)
     this.selectValues.emit(this.selected)
   }
 
-  toggle(choice: Choice) {
+  toggle(choice: Choice<T>) {
     this.select(choice, !this.isSelected(choice))
   }
 

package/src/libs/ui/inputs/src/lib/dropdown-multiselect/dropdown-multiselect.model.ts

@@ -1,4 +1,4 @@
-export interface Choice {
-  value: unknown
+export interface Choice<T = unknown> {
+  value: T
   label: string
 }

package/src/libs/ui/inputs/src/lib/spatial-extent-dropdown/spatial-extent-dropdown.component.html

@@ -0,0 +1,111 @@
+<gn-ui-button
+  type="outline"
+  extraClass="bg-background w-full !p-[8px] !pl-[16px]"
+  [title]="title"
+  (buttonClick)="toggleOverlay()"
+  cdkOverlayOrigin
+  #overlayOrigin="cdkOverlayOrigin"
+>
+  <div class="grow flex items-center mr-2 gap-2 overflow-hidden">
+    <div class="text-left font-medium truncate py-1">
+      {{ title }}
+    </div>
+    @if (hasSelection) {
+      <div
+        class="bg-primary-lighter text-white shrink-0 rounded-full font-bold text-[12px] w-[22px] h-[22px] flex items-center justify-center"
+      >
+        1
+      </div>
+    }
+  </div>
+  <button class="h-6 w-6 flex items-center" data-cy="clearSelection">
+    @if (hasSelection && !overlayOpen) {
+      <ng-icon
+        class="shrink-0 opacity-40 mr-1.5 hover:opacity-80 transition-colors clear-btn"
+        (click)="removeSelection($event)"
+        name="matClose"
+      ></ng-icon>
+    }
+  </button>
+  <ng-icon
+    [name]="overlayOpen ? 'matExpandLess' : 'matExpandMore'"
+    class="shrink-0 opacity-40"
+  >
+  </ng-icon>
+</gn-ui-button>
+
+<ng-template
+  cdkConnectedOverlay
+  cdkConnectedOverlayHasBackdrop
+  cdkConnectedOverlayBackdropClass="cdk-overlay-transparent-backdrop"
+  [cdkConnectedOverlayOrigin]="overlayOrigin"
+  [cdkConnectedOverlayOpen]="overlayOpen"
+  [cdkConnectedOverlayPositions]="overlayPositions"
+  [cdkConnectedOverlayScrollStrategy]="scrollStrategy"
+  [cdkConnectedOverlayFlexibleDimensions]="true"
+  (overlayOutsideClick)="closeOverlay()"
+  (detach)="closeOverlay()"
+>
+  <div
+    class="bg-white border border-gray-400 rounded shadow-lg pt-[12px] px-[16px] overlay-container w-[280px] flex flex-col gap-2"
+    [style.min-width]="overlayMinWidth"
+  >
+    <div class="text-xs text-black">
+      {{ 'search.filters.spatialExtent.helpText' | translate }}
+    </div>
+
+    <div
+      [class]="
+        (hasSelection
+          ? 'border-b-[1px] border-gray-400 pb-[20px]'
+          : 'pb-[8px]') + ' text-black'
+      "
+    >
+      <gn-ui-drag-and-drop-file-input
+        accept=".json,.geojson"
+        [placeholder]="'search.filters.spatialExtent.import' | translate"
+        [maxFileSizeMb]="maxFileSizeMb"
+        [showFileName]="false"
+        textClass="text-black truncate"
+        dropzoneBackgroundColor="transparent"
+        icon="iconoirImport"
+        extraClass="gn-ui-gray-outline-input w-full !h-[32px] px-[8px] py-[4px] justify-between"
+        (fileChange)="handleFileSelected($event)"
+        (errorChange)="handleFileError($event)"
+        data-cy="importGeojson"
+      ></gn-ui-drag-and-drop-file-input>
+    </div>
+
+    @if (hasSelection) {
+      <gn-ui-button
+        [type]="'primary-light'"
+        [title]="selectionDeleteLabelKey | translate: selectionLabelParams"
+        (buttonClick)="removeSelection($event)"
+        extraClass="group gap-x-2 w-full h-[28px] px-[8px] py-[4px] mb-[12px] bg-primary-lighter text-white"
+        data-test="spatial-extent-selected-item"
+      >
+        <ng-icon
+          name="iconoirSquareDashed"
+          class="shrink-0"
+          size="15px"
+        ></ng-icon>
+        <span class="group-hover:hidden truncate text-sm m-auto">
+          {{ selectionLabelKey | translate: selectionLabelParams }}
+        </span>
+        <span class="hidden group-hover:block truncate text-sm m-auto">
+          {{ selectionDeleteLabelKey | translate: selectionLabelParams }}
+        </span>
+        <ng-icon
+          name="iconoirCheckCircle"
+          class="group-hover:hidden shrink-0"
+          size="14px"
+        ></ng-icon>
+        <ng-icon
+          name="iconoirTrash"
+          class="!hidden group-hover:!block shrink-0"
+          size="15px"
+        ></ng-icon>
+      </gn-ui-button>
+    }
+  </div>
+</ng-template>

package/src/libs/ui/inputs/src/lib/spatial-extent-dropdown/spatial-extent-dropdown.component.ts

@@ -0,0 +1,215 @@
+import {
+  ChangeDetectionStrategy,
+  ChangeDetectorRef,
+  Component,
+  EventEmitter,
+  Input,
+  Output,
+  ViewChild,
+  inject,
+} from '@angular/core'
+import {
+  CdkConnectedOverlay,
+  CdkOverlayOrigin,
+  ConnectedPosition,
+  OverlayModule,
+  ScrollStrategyOptions,
+} from '@angular/cdk/overlay'
+
+import { ButtonComponent } from '../button/button.component'
+import { NgIcon, provideIcons } from '@ng-icons/core'
+import {
+  iconoirCheckCircle,
+  iconoirImport,
+  iconoirSquareDashed,
+  iconoirTrash,
+} from '@ng-icons/iconoir'
+import {
+  matClose,
+  matExpandLess,
+  matExpandMore,
+} from '@ng-icons/material-icons/baseline'
+import { TranslatePipe } from '@ngx-translate/core'
+import { marker } from '@biesbjerg/ngx-translate-extract-marker'
+import {
+  BoundingBox,
+  getGeometryBoundingBox,
+  getGeometryFromGeoJSON,
+  propagateToDocumentOnly,
+  readFileAsText,
+} from '../../../../../../libs/util/shared/src'
+import {
+  DragAndDropFileInputComponent,
+  DragAndDropFileInputError,
+} from '../drag-and-drop-file-input/drag-and-drop-file-input.component'
+
+marker('search.filters.spatialExtent.import')
+marker('search.filters.spatialExtent.helpText')
+marker('search.filters.spatialExtent.error.title')
+
+const LABEL_FROM_FILE = marker('search.filters.spatialExtent.bboxFromFile')
+const LABEL_FROM_FILE_DELETE = marker(
+  'search.filters.spatialExtent.bboxFromFileDelete'
+)
+const LABEL_INITIAL = marker('search.filters.spatialExtent.bboxInitial')
+const LABEL_INITIAL_DELETE = marker(
+  'search.filters.spatialExtent.bboxInitialDelete'
+)
+
+export interface SpatialExtentDropdownError {
+  key: string
+  params?: Record<string, string | number>
+}
+
+@Component({
+  selector: 'gn-ui-spatial-extent-dropdown',
+  standalone: true,
+  imports: [
+    ButtonComponent,
+    NgIcon,
+    OverlayModule,
+    TranslatePipe,
+    DragAndDropFileInputComponent,
+  ],
+  providers: [
+    provideIcons({
+      iconoirCheckCircle,
+      iconoirImport,
+      iconoirSquareDashed,
+      iconoirTrash,
+      matClose,
+      matExpandLess,
+      matExpandMore,
+    }),
+  ],
+  templateUrl: './spatial-extent-dropdown.component.html',
+  styleUrls: ['./spatial-extent-dropdown.component.css'],
+  changeDetection: ChangeDetectionStrategy.OnPush,
+})
+export class SpatialExtentDropdownComponent {
+  private cd = inject(ChangeDetectorRef)
+  private scrollStrategies = inject(ScrollStrategyOptions)
+
+  @Input() title: string
+  @Input() maxFileSizeMb: number | null = null
+  @Input() set initialBbox(value: BoundingBox | null) {
+    if (!this.bbox && value) {
+      this.bbox = value
+    }
+  }
+
+  @Output() bboxChange = new EventEmitter<BoundingBox | null>()
+  @Output() errorChange = new EventEmitter<SpatialExtentDropdownError>()
+
+  bbox: BoundingBox | null = null
+  fileName = ''
+
+  @ViewChild('overlayOrigin') overlayOrigin: CdkOverlayOrigin
+  @ViewChild(CdkConnectedOverlay) overlay: CdkConnectedOverlay
+  @ViewChild(DragAndDropFileInputComponent)
+  fileInput: DragAndDropFileInputComponent
+
+  overlayPositions: ConnectedPosition[] = [
+    {
+      originX: 'start',
+      originY: 'bottom',
+      overlayX: 'start',
+      overlayY: 'top',
+      offsetY: 8,
+    },
+    {
+      originX: 'start',
+      originY: 'top',
+      overlayX: 'start',
+      overlayY: 'bottom',
+      offsetY: -8,
+    },
+  ]
+  scrollStrategy = this.scrollStrategies.reposition()
+  overlayOpen = false
+  overlayMinWidth = 'none'
+
+  errorKey: string | null = null
+
+  get hasSelection() {
+    return !!this.bbox
+  }
+
+  get selectionLabelKey() {
+    return this.fileName ? LABEL_FROM_FILE : LABEL_INITIAL
+  }
+
+  get selectionDeleteLabelKey() {
+    return this.fileName ? LABEL_FROM_FILE_DELETE : LABEL_INITIAL_DELETE
+  }
+
+  get selectionLabelParams() {
+    return { fileName: this.fileName }
+  }
+
+  openOverlay() {
+    this.overlayMinWidth =
+      this.overlayOrigin.elementRef.nativeElement.getBoundingClientRect()
+        .width + 'px'
+    this.overlayOpen = true
+  }
+
+  closeOverlay() {
+    this.overlayOpen = false
+  }
+
+  toggleOverlay() {
+    if (this.overlayOpen) {
+      this.closeOverlay()
+    } else {
+      this.openOverlay()
+    }
+  }
+
+  async handleFileSelected(file: File) {
+    this.errorKey = null
+    let content: string
+    try {
+      content = await readFileAsText(file)
+      const parsed = JSON.parse(content)
+      const geometry = getGeometryFromGeoJSON(parsed)
+      if (!geometry) {
+        this.setError(marker('search.filters.spatialExtent.error.noGeometry'))
+        return
+      }
+      const bbox = getGeometryBoundingBox(geometry)
+      this.bbox = bbox
+      this.fileName = file.name
+      this.bboxChange.emit(bbox)
+      this.cd.markForCheck()
+    } catch {
+      this.setError(marker('search.filters.spatialExtent.error.invalidFormat'))
+      return
+    }
+  }
+
+  handleFileError(error: DragAndDropFileInputError) {
+    if (error === 'file-too-large') {
+      this.setError(marker('search.filters.spatialExtent.error.fileTooLarge'), {
+        maxSize: this.maxFileSizeMb,
+      })
+    } else {
+      this.setError(marker('search.filters.spatialExtent.error.invalidFormat'))
+    }
+  }
+
+  private setError(errorKey: string, params?: Record<string, string | number>) {
+    this.errorKey = errorKey
+    this.errorChange.emit({ key: errorKey, params })
+    this.cd.markForCheck()
+  }
+
+  removeSelection(event: Event) {
+    this.bbox = null
+    this.fileName = ''
+    this.errorKey = null
+    this.fileInput?.clear()
+    this.bboxChange.emit(null)
+    propagateToDocumentOnly(event)
+  }
+}

package/src/libs/util/app-config/src/lib/app-config.ts

@@ -252,6 +252,7 @@ export function loadAppConfig(configUrl = 'assets/configuration/default.toml') {
           'search_preset',
           'advanced_filters',
           'limit',
+          'spatial_extent_max_file_size',
         ],
         warnings,
         errors
@@ -281,6 +282,8 @@ export function loadAppConfig(configUrl = 'assets/configuration/default.toml') {
               })),
               ADVANCED_FILTERS: parsedSearchSection.advanced_filters,
               LIMIT: parsedSearchSection.limit,
+              SPATIAL_EXTENT_MAX_FILE_SIZE:
+                parsedSearchSection.spatial_extent_max_file_size,
             } as SearchConfig)
 
       const parsedMetadataQualitySection = parseConfigSection(

package/src/libs/util/app-config/src/lib/model.ts

@@ -63,6 +63,7 @@ export interface SearchConfig {
   SEARCH_PRESET?: SearchPreset[]
   ADVANCED_FILTERS?: []
   LIMIT?: number
+  SPATIAL_EXTENT_MAX_FILE_SIZE?: number
 }
 
 export interface MetadataQualityConfig {

package/src/libs/util/data-fetcher/src/lib/data-fetcher.ts

@@ -7,20 +7,19 @@ import { DataItem, DatasetHeaders, FetchError, SupportedType } from './model'
 import { inferDatasetType } from './utils'
 import { BaseReader } from './readers/base'
 import { GmlReader } from './readers/gml'
-import { WfsVersion } from '@camptocamp/ogc-client'
 import { WfsReader } from './readers/wfs'
 
+interface OpenDatasetOptions {
+  typeHint?: SupportedType
+  wfsFeatureType?: string
+  enableCache?: boolean // by default cache will be used if this is not set to false
+}
+
 export async function openDataset(
   url: string,
-  typeHint?: SupportedType,
-  options?: {
-    namespace?: string
-    wfsVersion?: WfsVersion
-    wfsFeatureType?: string
-  },
-  cacheActive?: boolean
+  options?: OpenDatasetOptions
 ): Promise<BaseReader> {
-  const fileType = await inferDatasetType(url, typeHint)
+  const fileType = await inferDatasetType(url, options?.typeHint)
   let reader:
     | CsvReader
     | JsonReader
@@ -43,19 +42,19 @@ export async function openDataset(
         reader = new ExcelReader(url)
         break
       case 'gml':
-        reader = new GmlReader(url, options.namespace, options.wfsVersion)
+        reader = new GmlReader(url)
         break
       case 'wfs':
-        reader = await WfsReader.createReader(url, options.wfsFeatureType)
+        reader = new WfsReader(url, options?.wfsFeatureType)
         break
     }
-    reader.setCacheActive(cacheActive)
+    reader.enableCache(options?.enableCache ?? true)
     reader.load()
     return reader
-  } catch (e: any) {
-    //WfsReader may already raise a FetchError
+  } catch (e: unknown) {
+    // WfsReader may already raise a FetchError
     if (e instanceof FetchError) throw e
-    else throw FetchError.parsingFailed(e.message)
+    else throw FetchError.parsingFailed((e as Error).message)
   }
 }
 
@@ -63,22 +62,20 @@ export async function openDataset(
  * This fetches the full dataset at the given URL and parses it according to its mime type.
  * All items in the dataset are converted to GeoJSON features, even if they do not bear any spatial geometry.
  * File type can be either inferred (from the HTTP headers or the URL), or hinted using the 2nd argument
- * File type is determined liked so:
+ * File type is determined like so:
  *  1. if a type hint is given, use it
  *  2. otherwise, look for a Content-Type header in the response with a supported mime type
  *  3. if no valid mime type was found, look for an explicit file extension in the url (.csv, .geojson etc.)
  */
 export async function readDataset(
   url: string,
-  typeHint?: SupportedType,
-  options?: any,
-  cacheActive = true
+  options?: OpenDatasetOptions
 ): Promise<DataItem[]> {
-  const reader = await openDataset(url, typeHint, options, cacheActive)
+  const reader = await openDataset(url, options)
   try {
     return await reader.read()
-  } catch (e: any) {
-    throw FetchError.parsingFailed(e.message)
+  } catch (e: unknown) {
+    throw FetchError.parsingFailed((e as Error).message)
   }
 }
 

package/src/libs/util/data-fetcher/src/lib/engine/duckdb.ts

@@ -0,0 +1,185 @@
+import * as duckdb from '@duckdb/duckdb-wasm'
+import { DataItem, FetchError, PropertyInfo } from '../model'
+import { arrowTableToDataItems } from './results'
+import { Table } from 'apache-arrow'
+
+// init code taken from https://github.com/duckdb/duckdb-wasm/blob/main/packages/duckdb-wasm/README.md
+const JSDELIVR_BUNDLES = duckdb.getJsDelivrBundles()
+
+// from https://duckdb.org/docs/current/sql/data_types/overview
+const typesMapping: Record<string, PropertyInfo['type']> = {
+  INTEGER: 'number',
+  SMALLINT: 'number',
+  TINYINT: 'number',
+  BIGINT: 'number',
+  HUGEINT: 'number',
+  UINTEGER: 'number',
+  USMALLINT: 'number',
+  UTINYINT: 'number',
+  UBIGINT: 'number',
+  UHUGEINT: 'number',
+  DOUBLE: 'number',
+  BIGNUM: 'number',
+  DECIMAL: 'number',
+  NUMERIC: 'number',
+  FLOAT: 'number',
+  REAL: 'number',
+  'TIMESTAMP WITH TIME ZONE': 'date',
+  TIMESTAMP: 'date',
+  DATE: 'date',
+  CHAR: 'string',
+  VARCHAR: 'string',
+  TEXT: 'string',
+  UUID: 'string',
+  BLOB: 'string',
+  BIT: 'string',
+  INTERVAL: 'string',
+  LIST: 'string',
+  BOOLEAN: 'boolean',
+}
+
+export class Engine {
+  private db: duckdb.AsyncDuckDB
+  private init_: Promise<Engine>
+
+  private async makeInit(): Promise<Engine> {
+    const bundle = await duckdb.selectBundle(JSDELIVR_BUNDLES)
+    let worker_url = bundle.mainWorker
+
+    // this is necessary to let browsers execute WASM code coming from a cross-origin host
+    if (worker_url.startsWith('https://')) {
+      worker_url = URL.createObjectURL(
+        new Blob([`importScripts("${worker_url}");`], {
+          type: 'text/javascript',
+        })
+      )
+    }
+    const worker = new Worker(worker_url)
+    const logger = new duckdb.ConsoleLogger(duckdb.LogLevel.WARNING)
+    this.db = new duckdb.AsyncDuckDB(logger, worker)
+    await this.db.instantiate(bundle.mainModule, bundle.pthreadWorker)
+
+    // setup duckdb options
+    const conn = await this.db.connect()
+    const timezone = Intl.DateTimeFormat().resolvedOptions().timeZone
+    await conn.query(`INSTALL spatial; LOAD spatial;`)
+    await conn.query(`SET TimeZone = '${timezone}';`)
+    await conn.query(`SET threads = 1;`)
+    // await conn.query(`SET memory_limit = '2GB';`) // used for experimenting when we encounter memory issues
+    await conn.query(`SET preserve_insertion_order = false;`) // this reduces memory usage when loading a dataset
+    conn.close()
+    return this
+  }
+
+  isReady(): Promise<Engine> {
+    if (!this.init_) {
+      this.init_ = this.makeInit()
+    }
+    return this.init_
+  }
+
+  /**
+   * Returns information about a dataset once it's loaded:
+   * - a list of properties description
+   * - the name of the dataset geometry column (null if no geometry present)
+   * @param datasetId name of the table under which the dataset will be stored
+   * @param loadQuery duckdb-specific query for creating a table out of the data
+   * @param forceReload if true, any existing data will be dropped and redownloaded
+   */
+  async loadFile(
+    datasetId: string,
+    loadQuery: string,
+    forceReload = false
+  ): Promise<{
+    properties: PropertyInfo[]
+    geometryColumn: string | null
+    rowsCount: number
+  }> {
+    const conn = await this.db.connect()
+    let results: Table
+
+    // either we want to recreate the table, or we keep it if it already exists
+    const safeLoadQuery = forceReload
+      ? `DROP TABLE IF EXISTS ${datasetId};
+${loadQuery}`
+      : loadQuery.replace(
+          /CREATE TABLE(?! IF NOT EXISTS)/gi,
+          'CREATE TABLE IF NOT EXISTS'
+        )
+
+    // create the table
+    try {
+      results = await conn.query(safeLoadQuery)
+    } catch (e: unknown) {
+      throw new FetchError(
+        'parse',
+        `DuckDB encountered an error when loading the data: ${(e as Error).message}`
+      )
+    }
+
+    // read rows count
+    results = await conn.query(`SELECT count(*) FROM ${datasetId}`)
+    const { 'count_star()': recordsCount } = results.toArray()[0].toJSON()
+
+    // read columns
+    results = await conn.query(
+      `SELECT * FROM information_schema.columns WHERE table_name = '${datasetId}';`
+    )
+    let geometryColumn: string | null = null
+
+    const properties = results
+      .toArray()
+      .map((row) => {
+        const rowObj = row.toJSON()
+        if (rowObj['data_type'] === 'GEOMETRY') {
+          // the geometry is not part of the properties
+          // note: right now we only keep one geometry column name, but if there are multiple they will
+          // all get discarded
+          geometryColumn = rowObj['column_name']
+          return null
+        }
+        return {
+          name: rowObj['column_name'],
+          label: rowObj['column_name'],
+          type: typesMapping[rowObj['data_type']] ?? 'other',
+        } as PropertyInfo
+      })
+      .filter((prop) => prop !== null)
+
+    conn.close()
+
+    return {
+      properties,
+      geometryColumn,
+      rowsCount: Number(recordsCount),
+    }
+  }
+
+  // register a Uint8 buffer using a handle in the duckdb instance
+  async registerData(name: string, buffer: Uint8Array) {
+    return this.db.registerFileBuffer(name, buffer)
+  }
+
+  /**
+   * @param query duckdb-specific query for fetching items
+   */
+  async queryItems(query: string): Promise<DataItem[]> {
+    const conn = await this.db.connect()
+    const results = await conn.query(query)
+    conn.close()
+    return arrowTableToDataItems(results)
+  }
+
+  close() {
+    this.db?.terminate()
+  }
+}
+
+let engine: Engine | null = null
+
+export async function getEngine(): Promise<Engine> {
+  if (!engine) {
+    engine = new Engine()
+  }
+  return engine.isReady()
+}