geonetwork-ui 2.11.0-dev.e7b8ec331 → 2.11.0-dev.f5ee66b24

package/src/libs/util/data-fetcher/src/lib/engine/duckdb.ts

@@ -0,0 +1,185 @@
+import * as duckdb from '@duckdb/duckdb-wasm'
+import { DataItem, FetchError, PropertyInfo } from '../model'
+import { arrowTableToDataItems } from './results'
+import { Table } from 'apache-arrow'
+
+// init code taken from https://github.com/duckdb/duckdb-wasm/blob/main/packages/duckdb-wasm/README.md
+const JSDELIVR_BUNDLES = duckdb.getJsDelivrBundles()
+
+// from https://duckdb.org/docs/current/sql/data_types/overview
+const typesMapping: Record<string, PropertyInfo['type']> = {
+  INTEGER: 'number',
+  SMALLINT: 'number',
+  TINYINT: 'number',
+  BIGINT: 'number',
+  HUGEINT: 'number',
+  UINTEGER: 'number',
+  USMALLINT: 'number',
+  UTINYINT: 'number',
+  UBIGINT: 'number',
+  UHUGEINT: 'number',
+  DOUBLE: 'number',
+  BIGNUM: 'number',
+  DECIMAL: 'number',
+  NUMERIC: 'number',
+  FLOAT: 'number',
+  REAL: 'number',
+  'TIMESTAMP WITH TIME ZONE': 'date',
+  TIMESTAMP: 'date',
+  DATE: 'date',
+  CHAR: 'string',
+  VARCHAR: 'string',
+  TEXT: 'string',
+  UUID: 'string',
+  BLOB: 'string',
+  BIT: 'string',
+  INTERVAL: 'string',
+  LIST: 'string',
+  BOOLEAN: 'boolean',
+}
+
+export class Engine {
+  private db: duckdb.AsyncDuckDB
+  private init_: Promise<Engine>
+
+  private async makeInit(): Promise<Engine> {
+    const bundle = await duckdb.selectBundle(JSDELIVR_BUNDLES)
+    let worker_url = bundle.mainWorker
+
+    // this is necessary to let browsers execute WASM code coming from a cross-origin host
+    if (worker_url.startsWith('https://')) {
+      worker_url = URL.createObjectURL(
+        new Blob([`importScripts("${worker_url}");`], {
+          type: 'text/javascript',
+        })
+      )
+    }
+    const worker = new Worker(worker_url)
+    const logger = new duckdb.ConsoleLogger(duckdb.LogLevel.WARNING)
+    this.db = new duckdb.AsyncDuckDB(logger, worker)
+    await this.db.instantiate(bundle.mainModule, bundle.pthreadWorker)
+
+    // setup duckdb options
+    const conn = await this.db.connect()
+    const timezone = Intl.DateTimeFormat().resolvedOptions().timeZone
+    await conn.query(`INSTALL spatial; LOAD spatial;`)
+    await conn.query(`SET TimeZone = '${timezone}';`)
+    await conn.query(`SET threads = 1;`)
+    // await conn.query(`SET memory_limit = '2GB';`) // used for experimenting when we encounter memory issues
+    await conn.query(`SET preserve_insertion_order = false;`) // this reduces memory usage when loading a dataset
+    conn.close()
+    return this
+  }
+
+  isReady(): Promise<Engine> {
+    if (!this.init_) {
+      this.init_ = this.makeInit()
+    }
+    return this.init_
+  }
+
+  /**
+   * Returns information about a dataset once it's loaded:
+   * - a list of properties description
+   * - the name of the dataset geometry column (null if no geometry present)
+   * @param datasetId name of the table under which the dataset will be stored
+   * @param loadQuery duckdb-specific query for creating a table out of the data
+   * @param forceReload if true, any existing data will be dropped and redownloaded
+   */
+  async loadFile(
+    datasetId: string,
+    loadQuery: string,
+    forceReload = false
+  ): Promise<{
+    properties: PropertyInfo[]
+    geometryColumn: string | null
+    rowsCount: number
+  }> {
+    const conn = await this.db.connect()
+    let results: Table
+
+    // either we want to recreate the table, or we keep it if it already exists
+    const safeLoadQuery = forceReload
+      ? `DROP TABLE IF EXISTS ${datasetId};
+${loadQuery}`
+      : loadQuery.replace(
+          /CREATE TABLE(?! IF NOT EXISTS)/gi,
+          'CREATE TABLE IF NOT EXISTS'
+        )
+
+    // create the table
+    try {
+      results = await conn.query(safeLoadQuery)
+    } catch (e: unknown) {
+      throw new FetchError(
+        'parse',
+        `DuckDB encountered an error when loading the data: ${(e as Error).message}`
+      )
+    }
+
+    // read rows count
+    results = await conn.query(`SELECT count(*) FROM ${datasetId}`)
+    const { 'count_star()': recordsCount } = results.toArray()[0].toJSON()
+
+    // read columns
+    results = await conn.query(
+      `SELECT * FROM information_schema.columns WHERE table_name = '${datasetId}';`
+    )
+    let geometryColumn: string | null = null
+
+    const properties = results
+      .toArray()
+      .map((row) => {
+        const rowObj = row.toJSON()
+        if (rowObj['data_type'] === 'GEOMETRY') {
+          // the geometry is not part of the properties
+          // note: right now we only keep one geometry column name, but if there are multiple they will
+          // all get discarded
+          geometryColumn = rowObj['column_name']
+          return null
+        }
+        return {
+          name: rowObj['column_name'],
+          label: rowObj['column_name'],
+          type: typesMapping[rowObj['data_type']] ?? 'other',
+        } as PropertyInfo
+      })
+      .filter((prop) => prop !== null)
+
+    conn.close()
+
+    return {
+      properties,
+      geometryColumn,
+      rowsCount: Number(recordsCount),
+    }
+  }
+
+  // register a Uint8 buffer using a handle in the duckdb instance
+  async registerData(name: string, buffer: Uint8Array) {
+    return this.db.registerFileBuffer(name, buffer)
+  }
+
+  /**
+   * @param query duckdb-specific query for fetching items
+   */
+  async queryItems(query: string): Promise<DataItem[]> {
+    const conn = await this.db.connect()
+    const results = await conn.query(query)
+    conn.close()
+    return arrowTableToDataItems(results)
+  }
+
+  close() {
+    this.db?.terminate()
+  }
+}
+
+let engine: Engine | null = null
+
+export async function getEngine(): Promise<Engine> {
+  if (!engine) {
+    engine = new Engine()
+  }
+  return engine.isReady()
+}

package/src/libs/util/data-fetcher/src/lib/engine/results.ts

@@ -0,0 +1,63 @@
+import { DataItem } from '../model'
+import { Feature } from 'geojson'
+import { DataType, StructRow, Table, Vector } from 'apache-arrow'
+import { GEOMETRY_COLUMN_ALIAS } from './sql-utils'
+
+export function arrowTableToDataItems(
+  table: Table<{ [key: string]: DataType }>
+): DataItem[] {
+  const fields = table.schema.fields
+
+  return table.toArray().map((row: StructRow) => {
+    const rowJson = row.toJSON()
+    const feature: Feature = {
+      type: 'Feature',
+      geometry: null,
+      properties: {},
+    }
+    const keys = Object.keys(rowJson)
+    for (let i = 0; i < keys.length; i++) {
+      const key = keys[i]
+      const dataType = fields[i].type
+      let value = rowJson[key]
+      // this might happen if we get an array inside a field
+      if (value instanceof Vector) {
+        value = Array.from(value)
+      }
+      // cast bigints to ints
+      if (typeof value === 'bigint') {
+        value = Number(value)
+      }
+      // rename columns with empty name
+      if (!key) {
+        feature.properties['unknown'] = value
+        continue
+      }
+      // assign properties that look like an id to the geojson `id` field
+      if (
+        /^(object|feature)?_?id$/.test(key.toLowerCase()) &&
+        (typeof value == 'string' || typeof value === 'number')
+      ) {
+        feature.id = value
+      }
+      // if a date is in timestamp (number) format, convert it to native
+      if (
+        typeof value === 'number' &&
+        (DataType.isTimestamp(dataType) || DataType.isDate(dataType))
+      ) {
+        value = new Date(value)
+      }
+      // if a binary field (most likely a geometry column): skip
+      if (DataType.isBinary(dataType)) {
+        continue
+      }
+      // geometry column
+      if (key === GEOMETRY_COLUMN_ALIAS && DataType.isUtf8(dataType)) {
+        feature.geometry = JSON.parse(value)
+        continue
+      }
+      feature.properties[key] = value
+    }
+    return feature
+  })
+}

package/src/libs/util/data-fetcher/src/lib/{sql-utils.ts → engine/sql-utils.ts}

@@ -4,7 +4,13 @@ import {
   FieldGroupBy,
   FieldName,
   FieldSort,
-} from './model'
+} from '../model'
+
+export const GEOMETRY_COLUMN_ALIAS = '__geometry__'
+
+function fieldToSql(name: string): string {
+  return `"${name.replace(/"/g, '""')}"` // escape double quotes in field names
+}
 
 function filterToSql(filter: FieldFilter): string {
   const operator = filter[0]
@@ -20,12 +26,12 @@ function filterToSql(filter: FieldFilter): string {
     case '=':
     case '!=':
     case 'like':
-      return `[${args[0]}] ${operator.toUpperCase()} ${valueToSql(
+      return `${fieldToSql(args[0] as string)} ${operator.toUpperCase()} ${valueToSql(
         args[1] as string | number
       )}`
     case 'in': {
       const values = args.slice(1) as string[] | number[]
-      return `[${args[0]}] IN (${values.map(valueToSql).join(', ')})`
+      return `${fieldToSql(args[0] as string)} IN (${values.map(valueToSql).join(', ')})`
     }
     case 'and':
     case 'or': {
@@ -48,18 +54,19 @@ function aggregationToSql(aggregation: FieldAggregation): string {
 
   switch (operation) {
     case 'average':
-      return `AVG([${field}]) as [average(${field})]`
+      return `CAST(AVG(${fieldToSql(field)}) AS DOUBLE) as ${fieldToSql(`average(${field})`)}`
     case 'sum':
     case 'max':
     case 'min':
-      return `${operation.toUpperCase()}([${field}]) as [${operation}(${field})]`
+      return `CAST(${operation.toUpperCase()}(${fieldToSql(field)}) AS DOUBLE) as ${fieldToSql(`${operation}(${field})`)}`
     case 'count':
-      return 'COUNT(*) as [count()]'
+      return 'CAST(COUNT(*) AS INTEGER) as "count()"' // we don't need Bigint precision here
   }
 }
 
 /**
  * Leave arguments at null if not used
+ * @param tableName
  * @param selected
  * @param filter
  * @param sort
@@ -69,29 +76,34 @@ function aggregationToSql(aggregation: FieldAggregation): string {
  * @param aggregations
  */
 export function generateSqlQuery(
+  tableName: string,
   selected: FieldName[] = null,
   filter: FieldFilter = null,
   sort: FieldSort[] = null,
   startIndex: number = null,
   count: number = null,
   groupBy: FieldGroupBy[] = null,
-  aggregations: FieldAggregation[] = null
+  aggregations: FieldAggregation[] = null,
+  geometryColumn: string = null
 ): string {
   let sqlSelect = 'SELECT *'
-  const sqlFrom = ' FROM ?'
+  const sqlFrom = ` FROM ${tableName}`
   let sqlOrderBy = ''
   let sqlWhere = ''
   let sqlLimit = ''
   let sqlGroupBy = ''
   if (selected !== null) {
-    sqlSelect = `SELECT ${selected.map((name) => `[${name}]`).join(', ')}`
+    sqlSelect = `SELECT ${selected.map(fieldToSql).join(', ')}`
+  }
+  if (geometryColumn !== null) {
+    sqlSelect += `, ST_AsGeoJSON(${fieldToSql(geometryColumn)}) as ${GEOMETRY_COLUMN_ALIAS}`
   }
   if (filter !== null) {
     sqlWhere = ` WHERE ${filterToSql(filter)}`
   }
   if (sort?.length) {
     sqlOrderBy = ` ORDER BY ${sort
-      .map((sort) => `[${sort[1]}] ${sort[0].toUpperCase()}`)
+      .map((sort) => `${fieldToSql(sort[1])} ${sort[0].toUpperCase()}`)
       .join(', ')}`
   }
   if (startIndex !== null && count !== null) {
@@ -101,15 +113,18 @@ export function generateSqlQuery(
     sqlSelect = `SELECT ${aggregations.map(aggregationToSql).join(', ')}`
     const groupedByDistinct = groupBy.filter((group) => group[0] === 'distinct')
     const sqlGroupByFields = groupedByDistinct
-      .map((group) => `[${group[1]}]`)
+      .map((group) => fieldToSql(group[1]))
       .join(', ')
     const sqlGroupBySelect = groupedByDistinct
-      .map((group) => `[${group[1]}] as [distinct(${group[1]})]`)
+      .map(
+        (group) =>
+          `${fieldToSql(group[1])} as ${fieldToSql(`distinct(${group[1]})`)}`
+      )
       .join(', ')
     if (sqlGroupByFields && sqlGroupBySelect) {
       sqlGroupBy = ` GROUP BY ${sqlGroupByFields}`
       sqlSelect += `, ${sqlGroupBySelect}`
     }
   }
-  return sqlSelect + sqlFrom + sqlGroupBy + sqlOrderBy + sqlWhere + sqlLimit
+  return sqlSelect + sqlFrom + sqlGroupBy + sqlWhere + sqlOrderBy + sqlLimit
 }

package/src/libs/util/data-fetcher/src/lib/model.ts

@@ -97,11 +97,12 @@ export interface DatasetHeaders {
 export interface PropertyInfo {
   name: string
   label: string
-  type: 'number' | 'date' | 'url' | 'string'
+  type: 'number' | 'date' | 'boolean' | 'string' | 'other'
 }
 
 export interface DatasetInfo {
   itemsCount: number
+  hasGeometry: boolean
 }
 
 export type FieldName = string

package/src/libs/util/data-fetcher/src/lib/readers/base-file.ts

@@ -1,65 +1,80 @@
 import { DataItem, DatasetInfo, PropertyInfo } from '../model'
-import { getJsonDataItemsProxy, jsonToGeojsonFeature } from '../utils'
-import { generateSqlQuery } from '../sql-utils'
-import { BaseCacheReader } from './base-cache'
+import { generateSqlQuery } from '../engine/sql-utils'
+import { Engine, getEngine } from '../engine/duckdb'
+import { BaseReader } from './base'
 
-type ParseResult = {
-  items: DataItem[]
-  properties: PropertyInfo[]
-}
+/**
+ * This reader handles file formats supported natively by DuckDB
+ */
+export class BaseFileReader extends BaseReader {
+  protected engine: Engine
+  protected datasetId: string
+  protected properties_: PropertyInfo[]
+  protected geometryColumn: string
+  protected rowsCount: number
 
-export class BaseFileReader extends BaseCacheReader {
-  private parseResult_: Promise<ParseResult>
+  // a table id should not exceed 63 chars
+  protected generateDatasetId() {
+    // generate a hash out of the url
+    let hash = 0
+    for (const char of this.url) {
+      hash = (hash << 5) - hash + char.charCodeAt(0)
+      hash = hash >>> 0 // make it unsigned
+    }
+    return `datafetcher_${hash.toString(16)}`
+  }
 
-  protected getData(): Promise<ParseResult> {
+  protected async getLoadQuery(): Promise<string> {
     throw new Error('not implemented')
   }
 
-  load() {
-    this.parseResult_ = this.getData()
+  async load() {
+    this.datasetId = this.generateDatasetId()
+    this.loadPromise_ = getEngine()
+      .then((engine) => {
+        this.engine = engine
+        return this.getLoadQuery()
+      })
+      .then((loadQuery) =>
+        this.engine.loadFile(this.datasetId, loadQuery, !this.cacheEnabled)
+      )
+      .then((datasetInfo) => {
+        this.properties_ = datasetInfo.properties
+        this.geometryColumn = datasetInfo.geometryColumn
+        this.rowsCount = datasetInfo.rowsCount
+        // returns void for the loaded promise
+      })
   }
 
   get properties(): Promise<PropertyInfo[]> {
-    return this.parseResult_.then((result) => result.properties)
+    return this.isLoaded.then(() => this.properties_)
   }
 
   get info(): Promise<DatasetInfo> {
-    return this.parseResult_.then(
-      (result) =>
-        ({
-          itemsCount: result.items.length,
-        }) as DatasetInfo
-    )
+    return this.isLoaded.then(() => ({
+      itemsCount: this.rowsCount,
+      hasGeometry: !!this.geometryColumn,
+    }))
   }
 
   async read(): Promise<DataItem[]> {
-    const items = (await this.parseResult_).items
-    // no query defined: return the full results as is
-    if (
-      this.groupedBy == null &&
-      this.aggregations == null &&
-      this.selected == null &&
-      this.sort == null &&
-      this.filter == null &&
-      this.startIndex == null &&
-      this.count == null
-    ) {
-      return items
-    }
+    await this.isLoaded
+
+    // if only certain fields are selected, omit the geometry
+    const geometryColumn = this.selected === null ? this.geometryColumn : null
 
-    const jsonItems = getJsonDataItemsProxy(items)
     const query = generateSqlQuery(
+      this.datasetId,
       this.selected,
       this.filter,
       this.sort,
       this.startIndex,
       this.count,
       this.groupedBy,
-      this.aggregations
+      this.aggregations,
+      geometryColumn
     )
-    const result: any[] = await import('alasql').then((module) =>
-      module.default(query, [jsonItems])
-    )
-    return result.map(jsonToGeojsonFeature)
+
+    return this.engine.queryItems(query)
   }
 }

package/src/libs/util/data-fetcher/src/lib/readers/base.ts

@@ -17,13 +17,24 @@ export class BaseReader {
   protected sort: FieldSort[] = null
   protected startIndex: number = null
   protected count: number = null
+  protected loadPromise_: Promise<void> = Promise.resolve()
+
+  protected cacheEnabled = false
 
   constructor(protected url: string) {}
 
+  enableCache(enabled: boolean) {
+    this.cacheEnabled = enabled
+  }
+
   load() {
     throw new Error('not implemented')
   }
 
+  get isLoaded() {
+    return this.loadPromise_
+  }
+
   get properties(): Promise<PropertyInfo[]> {
     throw new Error('not implemented')
   }

package/src/libs/util/data-fetcher/src/lib/readers/csv.ts

@@ -1,52 +1,14 @@
-import * as Papa from 'papaparse'
-import { DataItem, PropertyInfo } from '../model'
-import {
-  fetchDataAsText,
-  jsonToGeojsonFeature,
-  processItemProperties,
-} from '../utils'
 import { BaseFileReader } from './base-file'
 
-export function parseCsv(text: string): {
-  items: DataItem[]
-  properties: PropertyInfo[]
-} {
-  // first parse the header to guess the delimiter
-  // note that we do that to not rely on Papaparse logic for guessing delimiter
-  let delimiter
-  try {
-    const header = text.split('\n')[0]
-    const result = Papa.parse(header, {
-      header: false,
-    })
-    delimiter = result.meta.delimiter
-  } catch (e) {
-    throw new Error('CSV parsing failed: the delimiter could not be guessed')
-  }
-
-  const parsed = Papa.parse(text, {
-    header: true,
-    skipEmptyLines: true,
-    delimiter,
-  })
-  if (parsed.errors.length) {
-    throw new Error(
-      'CSV parsing failed for the following reasons:\n' +
-        parsed.errors
-          .map(
-            (error) =>
-              `* ${error.message} at row ${error.row}, column ${error.index}`
-          )
-          .join('\n')
-    )
-  }
-
-  const items = (parsed.data as any[]).map(jsonToGeojsonFeature)
-  return processItemProperties(items, true)
-}
-
 export class CsvReader extends BaseFileReader {
-  getData() {
-    return fetchDataAsText(this.url, this.cacheActive).then(parseCsv)
+  async getLoadQuery() {
+    // first we get a list of columns hich have a detected type of VARCHAR
+    // then we make sure that for those columns, we don't cast an empty string to null; instead we want to keep the empty string
+    return `
+SET VARIABLE strColumns = (SELECT list(name) FROM (SELECT unnest(Columns, recursive := true) FROM sniff_csv("${this.url}")) WHERE type = 'VARCHAR');
+CREATE TABLE ${this.datasetId} AS SELECT * FROM read_csv("${this.url}",
+  force_not_null = getvariable('strColumns'),
+  auto_type_candidates = ['NULL', 'BOOLEAN', 'INTEGER', 'DOUBLE', 'DATE', 'VARCHAR']
+);`
   }
 }

package/src/libs/util/data-fetcher/src/lib/readers/excel.ts

@@ -1,33 +1,33 @@
-import { DataItem, PropertyInfo } from '../model'
-import {
-  fetchDataAsArrayBuffer,
-  jsonToGeojsonFeature,
-  processItemProperties,
-} from '../utils'
 import { BaseFileReader } from './base-file'
+import { fetchDataAsArrayBuffer } from '../utils'
 
-/**
- * This will read the first sheet of the excel workbook and expect the first
- * line to contain the properties names
- * @param buffer
- */
-export function parseExcel(buffer: ArrayBuffer): Promise<{
-  items: DataItem[]
-  properties: PropertyInfo[]
-}> {
-  return import('xlsx').then(({ read, utils }) => {
-    const workbook = read(buffer)
-    const sheet = workbook.Sheets[workbook.SheetNames[0]]
-    let json = utils.sheet_to_json(sheet)
-    if (!json.length) {
-      json = []
+export class ExcelReader extends BaseFileReader {
+  protected async getLoadQuery(): Promise<string> {
+    // we download the file as an array buffer first, in order to be able to check if it's an XLS file
+    let buffer = await fetchDataAsArrayBuffer(this.url, this.cacheEnabled)
+    const bufferHandle = `B${this.datasetId}`
+
+    // checking against the magic number at the beginning of XLS files, see https://en.wikipedia.org/wiki/List_of_file_signatures
+    const magicNumber = new Uint8Array(buffer, 0, 8) // first 8 bytes
+    const isXls =
+      Array.from(magicNumber)
+        .map((n) => n.toString(16).toUpperCase())
+        .join(' ') === 'D0 CF 11 E0 A1 B1 1A E1'
+
+    // uh oh, this is an XLS file (not supported by duckdb); convert it to CSV using the xlsx package
+    if (isXls) {
+      buffer = await import('xlsx').then(({ read, utils }) => {
+        const workbook = read(buffer)
+        const json = utils.sheet_to_json(
+          workbook.Sheets[workbook.SheetNames[0]]
+        )
+        return new TextEncoder().encode(JSON.stringify(json)).buffer
+      })
     }
-    return processItemProperties(json.map(jsonToGeojsonFeature), true)
-  })
-}
+    const duckDbFn = isXls ? 'read_json' : 'read_xlsx'
 
-export class ExcelReader extends BaseFileReader {
-  getData() {
-    return fetchDataAsArrayBuffer(this.url, this.cacheActive).then(parseExcel)
+    await this.engine.registerData(bufferHandle, new Uint8Array(buffer))
+    return `
+CREATE TABLE ${this.datasetId} AS SELECT * FROM ${duckDbFn}("${bufferHandle}", ignore_errors = true);`
   }
 }

package/src/libs/util/data-fetcher/src/lib/readers/geojson.ts

@@ -1,29 +1,10 @@
-import { DataItem, PropertyInfo } from '../model'
-import { fetchDataAsText, processItemProperties } from '../utils'
 import { BaseFileReader } from './base-file'
 
-/**
- * This parser supports both Geojson Feature collections or arrays
- * of Features
- * @param text
- */
-export function parseGeojson(text: string): {
-  items: DataItem[]
-  properties: PropertyInfo[]
-} {
-  const parsed = JSON.parse(text)
-  const features =
-    parsed.type === 'FeatureCollection' ? parsed.features : parsed
-  if (!Array.isArray(features)) {
-    throw new Error(
-      'Could not parse GeoJSON, expected a features collection or an array of features at root level'
-    )
-  }
-  return processItemProperties(features)
-}
-
 export class GeojsonReader extends BaseFileReader {
-  getData() {
-    return fetchDataAsText(this.url, this.cacheActive).then(parseGeojson)
+  async getLoadQuery() {
+    return `
+CREATE TABLE ${this.datasetId} AS SELECT * FROM st_read("${this.url}",
+    allowed_drivers = ['GeoJSON']
+);`
   }
 }