geonetwork-ui 2.10.3-dev.4154db07d → 2.11.0-dev.1cf4ab0c6

package/src/libs/util/data-fetcher/src/lib/readers/base-file.ts

@@ -1,65 +1,80 @@
 import { DataItem, DatasetInfo, PropertyInfo } from '../model'
-import { getJsonDataItemsProxy, jsonToGeojsonFeature } from '../utils'
-import { generateSqlQuery } from '../sql-utils'
-import { BaseCacheReader } from './base-cache'
+import { generateSqlQuery } from '../engine/sql-utils'
+import { Engine, getEngine } from '../engine/duckdb'
+import { BaseReader } from './base'
 
-type ParseResult = {
-  items: DataItem[]
-  properties: PropertyInfo[]
-}
+/**
+ * This reader handles file formats supported natively by DuckDB
+ */
+export class BaseFileReader extends BaseReader {
+  protected engine: Engine
+  protected datasetId: string
+  protected properties_: PropertyInfo[]
+  protected geometryColumn: string
+  protected rowsCount: number
 
-export class BaseFileReader extends BaseCacheReader {
-  private parseResult_: Promise<ParseResult>
+  // a table id should not exceed 63 chars
+  protected generateDatasetId() {
+    // generate a hash out of the url
+    let hash = 0
+    for (const char of this.url) {
+      hash = (hash << 5) - hash + char.charCodeAt(0)
+      hash = hash >>> 0 // make it unsigned
+    }
+    return `datafetcher_${hash.toString(16)}`
+  }
 
-  protected getData(): Promise<ParseResult> {
+  protected async getLoadQuery(): Promise<string> {
     throw new Error('not implemented')
   }
 
-  load() {
-    this.parseResult_ = this.getData()
+  async load() {
+    this.datasetId = this.generateDatasetId()
+    this.loadPromise_ = getEngine()
+      .then((engine) => {
+        this.engine = engine
+        return this.getLoadQuery()
+      })
+      .then((loadQuery) =>
+        this.engine.loadFile(this.datasetId, loadQuery, !this.cacheEnabled)
+      )
+      .then((datasetInfo) => {
+        this.properties_ = datasetInfo.properties
+        this.geometryColumn = datasetInfo.geometryColumn
+        this.rowsCount = datasetInfo.rowsCount
+        // returns void for the loaded promise
+      })
   }
 
   get properties(): Promise<PropertyInfo[]> {
-    return this.parseResult_.then((result) => result.properties)
+    return this.isLoaded.then(() => this.properties_)
   }
 
   get info(): Promise<DatasetInfo> {
-    return this.parseResult_.then(
-      (result) =>
-        ({
-          itemsCount: result.items.length,
-        }) as DatasetInfo
-    )
+    return this.isLoaded.then(() => ({
+      itemsCount: this.rowsCount,
+      hasGeometry: !!this.geometryColumn,
+    }))
   }
 
   async read(): Promise<DataItem[]> {
-    const items = (await this.parseResult_).items
-    // no query defined: return the full results as is
-    if (
-      this.groupedBy == null &&
-      this.aggregations == null &&
-      this.selected == null &&
-      this.sort == null &&
-      this.filter == null &&
-      this.startIndex == null &&
-      this.count == null
-    ) {
-      return items
-    }
+    await this.isLoaded
+
+    // if only certain fields are selected, omit the geometry
+    const geometryColumn = this.selected === null ? this.geometryColumn : null
 
-    const jsonItems = getJsonDataItemsProxy(items)
     const query = generateSqlQuery(
+      this.datasetId,
       this.selected,
       this.filter,
       this.sort,
       this.startIndex,
       this.count,
       this.groupedBy,
-      this.aggregations
+      this.aggregations,
+      geometryColumn
     )
-    const result: any[] = await import('alasql').then((module) =>
-      module.default(query, [jsonItems])
-    )
-    return result.map(jsonToGeojsonFeature)
+
+    return this.engine.queryItems(query)
   }
 }

package/src/libs/util/data-fetcher/src/lib/readers/base.ts

@@ -17,13 +17,24 @@ export class BaseReader {
   protected sort: FieldSort[] = null
   protected startIndex: number = null
   protected count: number = null
+  protected loadPromise_: Promise<void> = Promise.resolve()
+
+  protected cacheEnabled = false
 
   constructor(protected url: string) {}
 
+  enableCache(enabled: boolean) {
+    this.cacheEnabled = enabled
+  }
+
   load() {
     throw new Error('not implemented')
   }
 
+  get isLoaded() {
+    return this.loadPromise_
+  }
+
   get properties(): Promise<PropertyInfo[]> {
     throw new Error('not implemented')
   }

package/src/libs/util/data-fetcher/src/lib/readers/csv.ts

@@ -1,52 +1,14 @@
-import * as Papa from 'papaparse'
-import { DataItem, PropertyInfo } from '../model'
-import {
-  fetchDataAsText,
-  jsonToGeojsonFeature,
-  processItemProperties,
-} from '../utils'
 import { BaseFileReader } from './base-file'
 
-export function parseCsv(text: string): {
-  items: DataItem[]
-  properties: PropertyInfo[]
-} {
-  // first parse the header to guess the delimiter
-  // note that we do that to not rely on Papaparse logic for guessing delimiter
-  let delimiter
-  try {
-    const header = text.split('\n')[0]
-    const result = Papa.parse(header, {
-      header: false,
-    })
-    delimiter = result.meta.delimiter
-  } catch (e) {
-    throw new Error('CSV parsing failed: the delimiter could not be guessed')
-  }
-
-  const parsed = Papa.parse(text, {
-    header: true,
-    skipEmptyLines: true,
-    delimiter,
-  })
-  if (parsed.errors.length) {
-    throw new Error(
-      'CSV parsing failed for the following reasons:\n' +
-        parsed.errors
-          .map(
-            (error) =>
-              `* ${error.message} at row ${error.row}, column ${error.index}`
-          )
-          .join('\n')
-    )
-  }
-
-  const items = (parsed.data as any[]).map(jsonToGeojsonFeature)
-  return processItemProperties(items, true)
-}
-
 export class CsvReader extends BaseFileReader {
-  getData() {
-    return fetchDataAsText(this.url, this.cacheActive).then(parseCsv)
+  async getLoadQuery() {
+    // first we get a list of columns hich have a detected type of VARCHAR
+    // then we make sure that for those columns, we don't cast an empty string to null; instead we want to keep the empty string
+    return `
+SET VARIABLE strColumns = (SELECT list(name) FROM (SELECT unnest(Columns, recursive := true) FROM sniff_csv("${this.url}")) WHERE type = 'VARCHAR');
+CREATE TABLE ${this.datasetId} AS SELECT * FROM read_csv("${this.url}",
+  force_not_null = getvariable('strColumns'),
+  auto_type_candidates = ['NULL', 'BOOLEAN', 'INTEGER', 'DOUBLE', 'DATE', 'VARCHAR']
+);`
   }
 }

package/src/libs/util/data-fetcher/src/lib/readers/excel.ts

@@ -1,33 +1,33 @@
-import { DataItem, PropertyInfo } from '../model'
-import {
-  fetchDataAsArrayBuffer,
-  jsonToGeojsonFeature,
-  processItemProperties,
-} from '../utils'
 import { BaseFileReader } from './base-file'
+import { fetchDataAsArrayBuffer } from '../utils'
 
-/**
- * This will read the first sheet of the excel workbook and expect the first
- * line to contain the properties names
- * @param buffer
- */
-export function parseExcel(buffer: ArrayBuffer): Promise<{
-  items: DataItem[]
-  properties: PropertyInfo[]
-}> {
-  return import('xlsx').then(({ read, utils }) => {
-    const workbook = read(buffer)
-    const sheet = workbook.Sheets[workbook.SheetNames[0]]
-    let json = utils.sheet_to_json(sheet)
-    if (!json.length) {
-      json = []
+export class ExcelReader extends BaseFileReader {
+  protected async getLoadQuery(): Promise<string> {
+    // we download the file as an array buffer first, in order to be able to check if it's an XLS file
+    let buffer = await fetchDataAsArrayBuffer(this.url, this.cacheEnabled)
+    const bufferHandle = `B${this.datasetId}`
+
+    // checking against the magic number at the beginning of XLS files, see https://en.wikipedia.org/wiki/List_of_file_signatures
+    const magicNumber = new Uint8Array(buffer, 0, 8) // first 8 bytes
+    const isXls =
+      Array.from(magicNumber)
+        .map((n) => n.toString(16).toUpperCase())
+        .join(' ') === 'D0 CF 11 E0 A1 B1 1A E1'
+
+    // uh oh, this is an XLS file (not supported by duckdb); convert it to CSV using the xlsx package
+    if (isXls) {
+      buffer = await import('xlsx').then(({ read, utils }) => {
+        const workbook = read(buffer)
+        const json = utils.sheet_to_json(
+          workbook.Sheets[workbook.SheetNames[0]]
+        )
+        return new TextEncoder().encode(JSON.stringify(json)).buffer
+      })
     }
-    return processItemProperties(json.map(jsonToGeojsonFeature), true)
-  })
-}
+    const duckDbFn = isXls ? 'read_json' : 'read_xlsx'
 
-export class ExcelReader extends BaseFileReader {
-  getData() {
-    return fetchDataAsArrayBuffer(this.url, this.cacheActive).then(parseExcel)
+    await this.engine.registerData(bufferHandle, new Uint8Array(buffer))
+    return `
+CREATE TABLE ${this.datasetId} AS SELECT * FROM ${duckDbFn}("${bufferHandle}", ignore_errors = true);`
   }
 }

package/src/libs/util/data-fetcher/src/lib/readers/geojson.ts

@@ -1,29 +1,10 @@
-import { DataItem, PropertyInfo } from '../model'
-import { fetchDataAsText, processItemProperties } from '../utils'
 import { BaseFileReader } from './base-file'
 
-/**
- * This parser supports both Geojson Feature collections or arrays
- * of Features
- * @param text
- */
-export function parseGeojson(text: string): {
-  items: DataItem[]
-  properties: PropertyInfo[]
-} {
-  const parsed = JSON.parse(text)
-  const features =
-    parsed.type === 'FeatureCollection' ? parsed.features : parsed
-  if (!Array.isArray(features)) {
-    throw new Error(
-      'Could not parse GeoJSON, expected a features collection or an array of features at root level'
-    )
-  }
-  return processItemProperties(features)
-}
-
 export class GeojsonReader extends BaseFileReader {
-  getData() {
-    return fetchDataAsText(this.url, this.cacheActive).then(parseGeojson)
+  async getLoadQuery() {
+    return `
+CREATE TABLE ${this.datasetId} AS SELECT * FROM st_read("${this.url}",
+    allowed_drivers = ['GeoJSON']
+);`
   }
 }

package/src/libs/util/data-fetcher/src/lib/readers/gml.ts

@@ -1,54 +1,10 @@
 import { BaseFileReader } from './base-file'
-import { DataItem, PropertyInfo } from '../model'
-import { fetchDataAsText, processItemProperties } from '../utils'
-import { GeoJSON, WFS } from 'ol/format.js'
-import { WfsVersion } from '@camptocamp/ogc-client'
-
-export function parseGml(
-  text: string,
-  namespace: string,
-  version: string
-): {
-  items: DataItem[]
-  properties: PropertyInfo[]
-} {
-  const splittedNamespace = namespace.split(':')
-  const regex = new RegExp(`xmlns:${splittedNamespace[0]}=["']([^'"]*)["']`)
-  const match = regex.exec(text)
-
-  if (match && match.length >= 2) {
-    const wf = new WFS({
-      featureNS: match[1],
-      featureType: splittedNamespace[1],
-      version: version,
-    })
-
-    let features
-    try {
-      features = wf.readFeatures(text)
-    } catch (e) {
-      throw Error("Couldn't parse WFS with GML features")
-    }
-
-    const geojsonItem = new GeoJSON().writeFeaturesObject(features)
-    return processItemProperties(geojsonItem.features, true)
-  }
-  throw Error("Couldn't retrieve namespace url")
-}
 
 export class GmlReader extends BaseFileReader {
-  constructor(
-    protected url: string,
-    protected namespace: string,
-    protected version: WfsVersion,
-    protected cacheActive = true
-  ) {
-    super(url)
-  }
-
-  protected getData() {
-    return fetchDataAsText(this.url, this.cacheActive).then((text) =>
-      parseGml(text, this.namespace, this.version)
-    )
+  async getLoadQuery() {
+    return `
+CREATE TABLE ${this.datasetId} AS SELECT * FROM st_read("${this.url}",
+    allowed_drivers = ['GML']
+);`
   }
 }

package/src/libs/util/data-fetcher/src/lib/readers/json.ts

@@ -1,28 +1,10 @@
-import { DataItem, PropertyInfo } from '../model'
-import {
-  fetchDataAsText,
-  jsonToGeojsonFeature,
-  processItemProperties,
-} from '../utils'
 import { BaseFileReader } from './base-file'
 
-/**
- * This parser only supports arrays of simple flat objects with properties
- * @param text
- */
-export function parseJson(text: string): {
-  items: DataItem[]
-  properties: PropertyInfo[]
-} {
-  const parsed = JSON.parse(text) as any[]
-  if (!Array.isArray(parsed)) {
-    throw new Error('Could not parse JSON, expected an array at root level')
-  }
-  return processItemProperties(parsed.map(jsonToGeojsonFeature))
-}
-
 export class JsonReader extends BaseFileReader {
-  getData() {
-    return fetchDataAsText(this.url, this.cacheActive).then(parseJson)
+  async getLoadQuery() {
+    return `
+CREATE TABLE ${this.datasetId} AS SELECT * FROM read_json("${this.url}",
+    maximum_object_size = 536870912 -- 500MB
+);`
   }
 }