geonetwork-ui 2.10.3-dev.4154db07d → 2.11.0-dev.4976ded4f

package/src/libs/ui/inputs/src/lib/spatial-extent-dropdown/spatial-extent-dropdown.component.ts

@@ -0,0 +1,191 @@
+import {
+  ChangeDetectionStrategy,
+  ChangeDetectorRef,
+  Component,
+  EventEmitter,
+  Input,
+  Output,
+  ViewChild,
+  inject,
+} from '@angular/core'
+import {
+  CdkConnectedOverlay,
+  CdkOverlayOrigin,
+  ConnectedPosition,
+  OverlayModule,
+  ScrollStrategyOptions,
+} from '@angular/cdk/overlay'
+
+import { ButtonComponent } from '../button/button.component'
+import { NgIcon, provideIcons } from '@ng-icons/core'
+import {
+  iconoirCheckCircle,
+  iconoirImport,
+  iconoirSquareDashed,
+  iconoirTrash,
+} from '@ng-icons/iconoir'
+import {
+  matClose,
+  matExpandLess,
+  matExpandMore,
+} from '@ng-icons/material-icons/baseline'
+import { TranslatePipe } from '@ngx-translate/core'
+import { marker } from '@biesbjerg/ngx-translate-extract-marker'
+import {
+  BoundingBox,
+  getGeometryBoundingBox,
+  getGeometryFromGeoJSON,
+  propagateToDocumentOnly,
+  readFileAsText,
+} from '../../../../../../libs/util/shared/src'
+import {
+  DragAndDropFileInputComponent,
+  DragAndDropFileInputError,
+} from '../drag-and-drop-file-input/drag-and-drop-file-input.component'
+
+marker('search.filters.spatialExtent.import')
+marker('search.filters.spatialExtent.helpText')
+marker('search.filters.spatialExtent.error.title')
+marker('search.filters.spatialExtent.bboxPrefix')
+
+export interface SpatialExtentDropdownError {
+  key: string
+  params?: Record<string, string | number>
+}
+marker('search.filters.spatialExtent.bboxDelete')
+
+@Component({
+  selector: 'gn-ui-spatial-extent-dropdown',
+  standalone: true,
+  imports: [
+    ButtonComponent,
+    NgIcon,
+    OverlayModule,
+    TranslatePipe,
+    DragAndDropFileInputComponent,
+  ],
+  providers: [
+    provideIcons({
+      iconoirCheckCircle,
+      iconoirImport,
+      iconoirSquareDashed,
+      iconoirTrash,
+      matClose,
+      matExpandLess,
+      matExpandMore,
+    }),
+  ],
+  templateUrl: './spatial-extent-dropdown.component.html',
+  styleUrls: ['./spatial-extent-dropdown.component.css'],
+  changeDetection: ChangeDetectionStrategy.OnPush,
+})
+export class SpatialExtentDropdownComponent {
+  private cd = inject(ChangeDetectorRef)
+  private scrollStrategies = inject(ScrollStrategyOptions)
+
+  @Input() title: string
+  @Input() maxFileSizeMb: number | null = null
+
+  @Output() bboxChange = new EventEmitter<BoundingBox | null>()
+  @Output() errorChange = new EventEmitter<SpatialExtentDropdownError>()
+
+  bbox: BoundingBox | null = null
+  fileName = ''
+
+  @ViewChild('overlayOrigin') overlayOrigin: CdkOverlayOrigin
+  @ViewChild(CdkConnectedOverlay) overlay: CdkConnectedOverlay
+  @ViewChild(DragAndDropFileInputComponent)
+  fileInput: DragAndDropFileInputComponent
+
+  overlayPositions: ConnectedPosition[] = [
+    {
+      originX: 'start',
+      originY: 'bottom',
+      overlayX: 'start',
+      overlayY: 'top',
+      offsetY: 8,
+    },
+    {
+      originX: 'start',
+      originY: 'top',
+      overlayX: 'start',
+      overlayY: 'bottom',
+      offsetY: -8,
+    },
+  ]
+  scrollStrategy = this.scrollStrategies.reposition()
+  overlayOpen = false
+  overlayMinWidth = 'none'
+
+  errorKey: string | null = null
+
+  get hasSelection() {
+    return !!this.bbox
+  }
+
+  openOverlay() {
+    this.overlayMinWidth =
+      this.overlayOrigin.elementRef.nativeElement.getBoundingClientRect()
+        .width + 'px'
+    this.overlayOpen = true
+  }
+
+  closeOverlay() {
+    this.overlayOpen = false
+  }
+
+  toggleOverlay() {
+    if (this.overlayOpen) {
+      this.closeOverlay()
+    } else {
+      this.openOverlay()
+    }
+  }
+
+  async handleFileSelected(file: File) {
+    this.errorKey = null
+    let content: string
+    try {
+      content = await readFileAsText(file)
+      const parsed = JSON.parse(content)
+      const geometry = getGeometryFromGeoJSON(parsed)
+      if (!geometry) {
+        this.setError(marker('search.filters.spatialExtent.error.noGeometry'))
+        return
+      }
+      const bbox = getGeometryBoundingBox(geometry)
+      this.bbox = bbox
+      this.fileName = file.name
+      this.bboxChange.emit(bbox)
+      this.cd.markForCheck()
+    } catch {
+      this.setError(marker('search.filters.spatialExtent.error.invalidFormat'))
+      return
+    }
+  }
+
+  handleFileError(error: DragAndDropFileInputError) {
+    if (error === 'file-too-large') {
+      this.setError(marker('search.filters.spatialExtent.error.fileTooLarge'), {
+        maxSize: this.maxFileSizeMb,
+      })
+    } else {
+      this.setError(marker('search.filters.spatialExtent.error.invalidFormat'))
+    }
+  }
+
+  private setError(errorKey: string, params?: Record<string, string | number>) {
+    this.errorKey = errorKey
+    this.errorChange.emit({ key: errorKey, params })
+    this.cd.markForCheck()
+  }
+
+  removeSelection(event: Event) {
+    this.bbox = null
+    this.fileName = ''
+    this.errorKey = null
+    this.fileInput?.clear()
+    this.bboxChange.emit(null)
+    propagateToDocumentOnly(event)
+  }
+}

package/src/libs/util/app-config/src/lib/app-config.ts

@@ -252,6 +252,7 @@ export function loadAppConfig(configUrl = 'assets/configuration/default.toml') {
           'search_preset',
           'advanced_filters',
           'limit',
+          'spatial_extent_max_file_size',
         ],
         warnings,
         errors
@@ -281,6 +282,8 @@ export function loadAppConfig(configUrl = 'assets/configuration/default.toml') {
               })),
               ADVANCED_FILTERS: parsedSearchSection.advanced_filters,
               LIMIT: parsedSearchSection.limit,
+              SPATIAL_EXTENT_MAX_FILE_SIZE:
+                parsedSearchSection.spatial_extent_max_file_size,
             } as SearchConfig)
 
       const parsedMetadataQualitySection = parseConfigSection(

package/src/libs/util/app-config/src/lib/model.ts

@@ -63,6 +63,7 @@ export interface SearchConfig {
   SEARCH_PRESET?: SearchPreset[]
   ADVANCED_FILTERS?: []
   LIMIT?: number
+  SPATIAL_EXTENT_MAX_FILE_SIZE?: number
 }
 
 export interface MetadataQualityConfig {

package/src/libs/util/data-fetcher/src/lib/data-fetcher.ts

@@ -7,20 +7,19 @@ import { DataItem, DatasetHeaders, FetchError, SupportedType } from './model'
 import { inferDatasetType } from './utils'
 import { BaseReader } from './readers/base'
 import { GmlReader } from './readers/gml'
-import { WfsVersion } from '@camptocamp/ogc-client'
 import { WfsReader } from './readers/wfs'
 
+interface OpenDatasetOptions {
+  typeHint?: SupportedType
+  wfsFeatureType?: string
+  enableCache?: boolean // by default cache will be used if this is not set to false
+}
+
 export async function openDataset(
   url: string,
-  typeHint?: SupportedType,
-  options?: {
-    namespace?: string
-    wfsVersion?: WfsVersion
-    wfsFeatureType?: string
-  },
-  cacheActive?: boolean
+  options?: OpenDatasetOptions
 ): Promise<BaseReader> {
-  const fileType = await inferDatasetType(url, typeHint)
+  const fileType = await inferDatasetType(url, options?.typeHint)
   let reader:
     | CsvReader
     | JsonReader
@@ -43,19 +42,19 @@ export async function openDataset(
         reader = new ExcelReader(url)
         break
       case 'gml':
-        reader = new GmlReader(url, options.namespace, options.wfsVersion)
+        reader = new GmlReader(url)
         break
       case 'wfs':
-        reader = await WfsReader.createReader(url, options.wfsFeatureType)
+        reader = new WfsReader(url, options?.wfsFeatureType)
         break
     }
-    reader.setCacheActive(cacheActive)
+    reader.enableCache(options?.enableCache ?? true)
     reader.load()
     return reader
-  } catch (e: any) {
-    //WfsReader may already raise a FetchError
+  } catch (e: unknown) {
+    // WfsReader may already raise a FetchError
     if (e instanceof FetchError) throw e
-    else throw FetchError.parsingFailed(e.message)
+    else throw FetchError.parsingFailed((e as Error).message)
   }
 }
 
@@ -63,22 +62,20 @@ export async function openDataset(
  * This fetches the full dataset at the given URL and parses it according to its mime type.
  * All items in the dataset are converted to GeoJSON features, even if they do not bear any spatial geometry.
  * File type can be either inferred (from the HTTP headers or the URL), or hinted using the 2nd argument
- * File type is determined liked so:
+ * File type is determined like so:
  *  1. if a type hint is given, use it
  *  2. otherwise, look for a Content-Type header in the response with a supported mime type
  *  3. if no valid mime type was found, look for an explicit file extension in the url (.csv, .geojson etc.)
  */
 export async function readDataset(
   url: string,
-  typeHint?: SupportedType,
-  options?: any,
-  cacheActive = true
+  options?: OpenDatasetOptions
 ): Promise<DataItem[]> {
-  const reader = await openDataset(url, typeHint, options, cacheActive)
+  const reader = await openDataset(url, options)
   try {
     return await reader.read()
-  } catch (e: any) {
-    throw FetchError.parsingFailed(e.message)
+  } catch (e: unknown) {
+    throw FetchError.parsingFailed((e as Error).message)
   }
 }
 

package/src/libs/util/data-fetcher/src/lib/engine/duckdb.ts

@@ -0,0 +1,185 @@
+import * as duckdb from '@duckdb/duckdb-wasm'
+import { DataItem, FetchError, PropertyInfo } from '../model'
+import { arrowTableToDataItems } from './results'
+import { Table } from 'apache-arrow'
+
+// init code taken from https://github.com/duckdb/duckdb-wasm/blob/main/packages/duckdb-wasm/README.md
+const JSDELIVR_BUNDLES = duckdb.getJsDelivrBundles()
+
+// from https://duckdb.org/docs/current/sql/data_types/overview
+const typesMapping: Record<string, PropertyInfo['type']> = {
+  INTEGER: 'number',
+  SMALLINT: 'number',
+  TINYINT: 'number',
+  BIGINT: 'number',
+  HUGEINT: 'number',
+  UINTEGER: 'number',
+  USMALLINT: 'number',
+  UTINYINT: 'number',
+  UBIGINT: 'number',
+  UHUGEINT: 'number',
+  DOUBLE: 'number',
+  BIGNUM: 'number',
+  DECIMAL: 'number',
+  NUMERIC: 'number',
+  FLOAT: 'number',
+  REAL: 'number',
+  'TIMESTAMP WITH TIME ZONE': 'date',
+  TIMESTAMP: 'date',
+  DATE: 'date',
+  CHAR: 'string',
+  VARCHAR: 'string',
+  TEXT: 'string',
+  UUID: 'string',
+  BLOB: 'string',
+  BIT: 'string',
+  INTERVAL: 'string',
+  LIST: 'string',
+  BOOLEAN: 'boolean',
+}
+
+export class Engine {
+  private db: duckdb.AsyncDuckDB
+  private init_: Promise<Engine>
+
+  private async makeInit(): Promise<Engine> {
+    const bundle = await duckdb.selectBundle(JSDELIVR_BUNDLES)
+    let worker_url = bundle.mainWorker
+
+    // this is necessary to let browsers execute WASM code coming from a cross-origin host
+    if (worker_url.startsWith('https://')) {
+      worker_url = URL.createObjectURL(
+        new Blob([`importScripts("${worker_url}");`], {
+          type: 'text/javascript',
+        })
+      )
+    }
+    const worker = new Worker(worker_url)
+    const logger = new duckdb.ConsoleLogger(duckdb.LogLevel.WARNING)
+    this.db = new duckdb.AsyncDuckDB(logger, worker)
+    await this.db.instantiate(bundle.mainModule, bundle.pthreadWorker)
+
+    // setup duckdb options
+    const conn = await this.db.connect()
+    const timezone = Intl.DateTimeFormat().resolvedOptions().timeZone
+    await conn.query(`INSTALL spatial; LOAD spatial;`)
+    await conn.query(`SET TimeZone = '${timezone}';`)
+    await conn.query(`SET threads = 1;`)
+    // await conn.query(`SET memory_limit = '2GB';`) // used for experimenting when we encounter memory issues
+    await conn.query(`SET preserve_insertion_order = false;`) // this reduces memory usage when loading a dataset
+    conn.close()
+    return this
+  }
+
+  isReady(): Promise<Engine> {
+    if (!this.init_) {
+      this.init_ = this.makeInit()
+    }
+    return this.init_
+  }
+
+  /**
+   * Returns information about a dataset once it's loaded:
+   * - a list of properties description
+   * - the name of the dataset geometry column (null if no geometry present)
+   * @param datasetId name of the table under which the dataset will be stored
+   * @param loadQuery duckdb-specific query for creating a table out of the data
+   * @param forceReload if true, any existing data will be dropped and redownloaded
+   */
+  async loadFile(
+    datasetId: string,
+    loadQuery: string,
+    forceReload = false
+  ): Promise<{
+    properties: PropertyInfo[]
+    geometryColumn: string | null
+    rowsCount: number
+  }> {
+    const conn = await this.db.connect()
+    let results: Table
+
+    // either we want to recreate the table, or we keep it if it already exists
+    const safeLoadQuery = forceReload
+      ? `DROP TABLE IF EXISTS ${datasetId};
+${loadQuery}`
+      : loadQuery.replace(
+          /CREATE TABLE(?! IF NOT EXISTS)/gi,
+          'CREATE TABLE IF NOT EXISTS'
+        )
+
+    // create the table
+    try {
+      results = await conn.query(safeLoadQuery)
+    } catch (e: unknown) {
+      throw new FetchError(
+        'parse',
+        `DuckDB encountered an error when loading the data: ${(e as Error).message}`
+      )
+    }
+
+    // read rows count
+    results = await conn.query(`SELECT count(*) FROM ${datasetId}`)
+    const { 'count_star()': recordsCount } = results.toArray()[0].toJSON()
+
+    // read columns
+    results = await conn.query(
+      `SELECT * FROM information_schema.columns WHERE table_name = '${datasetId}';`
+    )
+    let geometryColumn: string | null = null
+
+    const properties = results
+      .toArray()
+      .map((row) => {
+        const rowObj = row.toJSON()
+        if (rowObj['data_type'] === 'GEOMETRY') {
+          // the geometry is not part of the properties
+          // note: right now we only keep one geometry column name, but if there are multiple they will
+          // all get discarded
+          geometryColumn = rowObj['column_name']
+          return null
+        }
+        return {
+          name: rowObj['column_name'],
+          label: rowObj['column_name'],
+          type: typesMapping[rowObj['data_type']] ?? 'other',
+        } as PropertyInfo
+      })
+      .filter((prop) => prop !== null)
+
+    conn.close()
+
+    return {
+      properties,
+      geometryColumn,
+      rowsCount: Number(recordsCount),
+    }
+  }
+
+  // register a Uint8 buffer using a handle in the duckdb instance
+  async registerData(name: string, buffer: Uint8Array) {
+    return this.db.registerFileBuffer(name, buffer)
+  }
+
+  /**
+   * @param query duckdb-specific query for fetching items
+   */
+  async queryItems(query: string): Promise<DataItem[]> {
+    const conn = await this.db.connect()
+    const results = await conn.query(query)
+    conn.close()
+    return arrowTableToDataItems(results)
+  }
+
+  close() {
+    this.db?.terminate()
+  }
+}
+
+let engine: Engine | null = null
+
+export async function getEngine(): Promise<Engine> {
+  if (!engine) {
+    engine = new Engine()
+  }
+  return engine.isReady()
+}

package/src/libs/util/data-fetcher/src/lib/engine/results.ts

@@ -0,0 +1,63 @@
+import { DataItem } from '../model'
+import { Feature } from 'geojson'
+import { DataType, StructRow, Table, Vector } from 'apache-arrow'
+import { GEOMETRY_COLUMN_ALIAS } from './sql-utils'
+
+export function arrowTableToDataItems(
+  table: Table<{ [key: string]: DataType }>
+): DataItem[] {
+  const fields = table.schema.fields
+
+  return table.toArray().map((row: StructRow) => {
+    const rowJson = row.toJSON()
+    const feature: Feature = {
+      type: 'Feature',
+      geometry: null,
+      properties: {},
+    }
+    const keys = Object.keys(rowJson)
+    for (let i = 0; i < keys.length; i++) {
+      const key = keys[i]
+      const dataType = fields[i].type
+      let value = rowJson[key]
+      // this might happen if we get an array inside a field
+      if (value instanceof Vector) {
+        value = Array.from(value)
+      }
+      // cast bigints to ints
+      if (typeof value === 'bigint') {
+        value = Number(value)
+      }
+      // rename columns with empty name
+      if (!key) {
+        feature.properties['unknown'] = value
+        continue
+      }
+      // assign properties that look like an id to the geojson `id` field
+      if (
+        /^(object|feature)?_?id$/.test(key.toLowerCase()) &&
+        (typeof value == 'string' || typeof value === 'number')
+      ) {
+        feature.id = value
+      }
+      // if a date is in timestamp (number) format, convert it to native
+      if (
+        typeof value === 'number' &&
+        (DataType.isTimestamp(dataType) || DataType.isDate(dataType))
+      ) {
+        value = new Date(value)
+      }
+      // if a binary field (most likely a geometry column): skip
+      if (DataType.isBinary(dataType)) {
+        continue
+      }
+      // geometry column
+      if (key === GEOMETRY_COLUMN_ALIAS && DataType.isUtf8(dataType)) {
+        feature.geometry = JSON.parse(value)
+        continue
+      }
+      feature.properties[key] = value
+    }
+    return feature
+  })
+}

package/src/libs/util/data-fetcher/src/lib/{sql-utils.ts → engine/sql-utils.ts}

@@ -4,7 +4,13 @@ import {
   FieldGroupBy,
   FieldName,
   FieldSort,
-} from './model'
+} from '../model'
+
+export const GEOMETRY_COLUMN_ALIAS = '__geometry__'
+
+function fieldToSql(name: string): string {
+  return `"${name.replace(/"/g, '""')}"` // escape double quotes in field names
+}
 
 function filterToSql(filter: FieldFilter): string {
   const operator = filter[0]
@@ -20,12 +26,12 @@ function filterToSql(filter: FieldFilter): string {
     case '=':
     case '!=':
     case 'like':
-      return `[${args[0]}] ${operator.toUpperCase()} ${valueToSql(
+      return `${fieldToSql(args[0] as string)} ${operator.toUpperCase()} ${valueToSql(
         args[1] as string | number
       )}`
     case 'in': {
       const values = args.slice(1) as string[] | number[]
-      return `[${args[0]}] IN (${values.map(valueToSql).join(', ')})`
+      return `${fieldToSql(args[0] as string)} IN (${values.map(valueToSql).join(', ')})`
     }
     case 'and':
     case 'or': {
@@ -48,18 +54,19 @@ function aggregationToSql(aggregation: FieldAggregation): string {
 
   switch (operation) {
     case 'average':
-      return `AVG([${field}]) as [average(${field})]`
+      return `CAST(AVG(${fieldToSql(field)}) AS DOUBLE) as ${fieldToSql(`average(${field})`)}`
     case 'sum':
     case 'max':
     case 'min':
-      return `${operation.toUpperCase()}([${field}]) as [${operation}(${field})]`
+      return `CAST(${operation.toUpperCase()}(${fieldToSql(field)}) AS DOUBLE) as ${fieldToSql(`${operation}(${field})`)}`
     case 'count':
-      return 'COUNT(*) as [count()]'
+      return 'CAST(COUNT(*) AS INTEGER) as "count()"' // we don't need Bigint precision here
   }
 }
 
 /**
  * Leave arguments at null if not used
+ * @param tableName
  * @param selected
  * @param filter
  * @param sort
@@ -69,29 +76,34 @@ function aggregationToSql(aggregation: FieldAggregation): string {
  * @param aggregations
  */
 export function generateSqlQuery(
+  tableName: string,
   selected: FieldName[] = null,
   filter: FieldFilter = null,
   sort: FieldSort[] = null,
   startIndex: number = null,
   count: number = null,
   groupBy: FieldGroupBy[] = null,
-  aggregations: FieldAggregation[] = null
+  aggregations: FieldAggregation[] = null,
+  geometryColumn: string = null
 ): string {
   let sqlSelect = 'SELECT *'
-  const sqlFrom = ' FROM ?'
+  const sqlFrom = ` FROM ${tableName}`
   let sqlOrderBy = ''
   let sqlWhere = ''
   let sqlLimit = ''
   let sqlGroupBy = ''
   if (selected !== null) {
-    sqlSelect = `SELECT ${selected.map((name) => `[${name}]`).join(', ')}`
+    sqlSelect = `SELECT ${selected.map(fieldToSql).join(', ')}`
+  }
+  if (geometryColumn !== null) {
+    sqlSelect += `, ST_AsGeoJSON(${fieldToSql(geometryColumn)}) as ${GEOMETRY_COLUMN_ALIAS}`
   }
   if (filter !== null) {
     sqlWhere = ` WHERE ${filterToSql(filter)}`
   }
   if (sort?.length) {
     sqlOrderBy = ` ORDER BY ${sort
-      .map((sort) => `[${sort[1]}] ${sort[0].toUpperCase()}`)
+      .map((sort) => `${fieldToSql(sort[1])} ${sort[0].toUpperCase()}`)
       .join(', ')}`
   }
   if (startIndex !== null && count !== null) {
@@ -101,15 +113,18 @@ export function generateSqlQuery(
     sqlSelect = `SELECT ${aggregations.map(aggregationToSql).join(', ')}`
     const groupedByDistinct = groupBy.filter((group) => group[0] === 'distinct')
     const sqlGroupByFields = groupedByDistinct
-      .map((group) => `[${group[1]}]`)
+      .map((group) => fieldToSql(group[1]))
       .join(', ')
     const sqlGroupBySelect = groupedByDistinct
-      .map((group) => `[${group[1]}] as [distinct(${group[1]})]`)
+      .map(
+        (group) =>
+          `${fieldToSql(group[1])} as ${fieldToSql(`distinct(${group[1]})`)}`
+      )
       .join(', ')
     if (sqlGroupByFields && sqlGroupBySelect) {
       sqlGroupBy = ` GROUP BY ${sqlGroupByFields}`
       sqlSelect += `, ${sqlGroupBySelect}`
     }
   }
-  return sqlSelect + sqlFrom + sqlGroupBy + sqlOrderBy + sqlWhere + sqlLimit
+  return sqlSelect + sqlFrom + sqlGroupBy + sqlWhere + sqlOrderBy + sqlLimit
 }

package/src/libs/util/data-fetcher/src/lib/model.ts

@@ -97,11 +97,12 @@ export interface DatasetHeaders {
 export interface PropertyInfo {
   name: string
   label: string
-  type: 'number' | 'date' | 'url' | 'string'
+  type: 'number' | 'date' | 'boolean' | 'string' | 'other'
 }
 
 export interface DatasetInfo {
   itemsCount: number
+  hasGeometry: boolean
 }
 
 export type FieldName = string