geonetwork-ui 2.11.0-dev.96d0c6ce2 → 2.11.0-dev.a5c259d9e

package/src/libs/util/data-fetcher/src/lib/readers/excel.ts

@@ -1,33 +1,33 @@
-import { DataItem, PropertyInfo } from '../model'
-import {
-  fetchDataAsArrayBuffer,
-  jsonToGeojsonFeature,
-  processItemProperties,
-} from '../utils'
 import { BaseFileReader } from './base-file'
+import { fetchDataAsArrayBuffer } from '../utils'
 
-/**
- * This will read the first sheet of the excel workbook and expect the first
- * line to contain the properties names
- * @param buffer
- */
-export function parseExcel(buffer: ArrayBuffer): Promise<{
-  items: DataItem[]
-  properties: PropertyInfo[]
-}> {
-  return import('xlsx').then(({ read, utils }) => {
-    const workbook = read(buffer)
-    const sheet = workbook.Sheets[workbook.SheetNames[0]]
-    let json = utils.sheet_to_json(sheet)
-    if (!json.length) {
-      json = []
+export class ExcelReader extends BaseFileReader {
+  protected async getLoadQuery(): Promise<string> {
+    // we download the file as an array buffer first, in order to be able to check if it's an XLS file
+    let buffer = await fetchDataAsArrayBuffer(this.url, this.cacheEnabled)
+    const bufferHandle = `B${this.datasetId}`
+
+    // checking against the magic number at the beginning of XLS files, see https://en.wikipedia.org/wiki/List_of_file_signatures
+    const magicNumber = new Uint8Array(buffer, 0, 8) // first 8 bytes
+    const isXls =
+      Array.from(magicNumber)
+        .map((n) => n.toString(16).toUpperCase())
+        .join(' ') === 'D0 CF 11 E0 A1 B1 1A E1'
+
+    // uh oh, this is an XLS file (not supported by duckdb); convert it to CSV using the xlsx package
+    if (isXls) {
+      buffer = await import('xlsx').then(({ read, utils }) => {
+        const workbook = read(buffer)
+        const json = utils.sheet_to_json(
+          workbook.Sheets[workbook.SheetNames[0]]
+        )
+        return new TextEncoder().encode(JSON.stringify(json)).buffer
+      })
     }
-    return processItemProperties(json.map(jsonToGeojsonFeature), true)
-  })
-}
+    const duckDbFn = isXls ? 'read_json' : 'read_xlsx'
 
-export class ExcelReader extends BaseFileReader {
-  getData() {
-    return fetchDataAsArrayBuffer(this.url, this.cacheActive).then(parseExcel)
+    await this.engine.registerData(bufferHandle, new Uint8Array(buffer))
+    return `
+CREATE TABLE ${this.datasetId} AS SELECT * FROM ${duckDbFn}("${bufferHandle}", ignore_errors = true);`
   }
 }

package/src/libs/util/data-fetcher/src/lib/readers/geojson.ts

@@ -1,29 +1,10 @@
-import { DataItem, PropertyInfo } from '../model'
-import { fetchDataAsText, processItemProperties } from '../utils'
 import { BaseFileReader } from './base-file'
 
-/**
- * This parser supports both Geojson Feature collections or arrays
- * of Features
- * @param text
- */
-export function parseGeojson(text: string): {
-  items: DataItem[]
-  properties: PropertyInfo[]
-} {
-  const parsed = JSON.parse(text)
-  const features =
-    parsed.type === 'FeatureCollection' ? parsed.features : parsed
-  if (!Array.isArray(features)) {
-    throw new Error(
-      'Could not parse GeoJSON, expected a features collection or an array of features at root level'
-    )
-  }
-  return processItemProperties(features)
-}
-
 export class GeojsonReader extends BaseFileReader {
-  getData() {
-    return fetchDataAsText(this.url, this.cacheActive).then(parseGeojson)
+  async getLoadQuery() {
+    return `
+CREATE TABLE ${this.datasetId} AS SELECT * FROM st_read("${this.url}",
+    allowed_drivers = ['GeoJSON']
+);`
   }
 }

package/src/libs/util/data-fetcher/src/lib/readers/gml.ts

@@ -1,54 +1,10 @@
 import { BaseFileReader } from './base-file'
-import { DataItem, PropertyInfo } from '../model'
-import { fetchDataAsText, processItemProperties } from '../utils'
-import { GeoJSON, WFS } from 'ol/format.js'
-import { WfsVersion } from '@camptocamp/ogc-client'
-
-export function parseGml(
-  text: string,
-  namespace: string,
-  version: string
-): {
-  items: DataItem[]
-  properties: PropertyInfo[]
-} {
-  const splittedNamespace = namespace.split(':')
-  const regex = new RegExp(`xmlns:${splittedNamespace[0]}=["']([^'"]*)["']`)
-  const match = regex.exec(text)
-
-  if (match && match.length >= 2) {
-    const wf = new WFS({
-      featureNS: match[1],
-      featureType: splittedNamespace[1],
-      version: version,
-    })
-
-    let features
-    try {
-      features = wf.readFeatures(text)
-    } catch (e) {
-      throw Error("Couldn't parse WFS with GML features")
-    }
-
-    const geojsonItem = new GeoJSON().writeFeaturesObject(features)
-    return processItemProperties(geojsonItem.features, true)
-  }
-  throw Error("Couldn't retrieve namespace url")
-}
 
 export class GmlReader extends BaseFileReader {
-  constructor(
-    protected url: string,
-    protected namespace: string,
-    protected version: WfsVersion,
-    protected cacheActive = true
-  ) {
-    super(url)
-  }
-
-  protected getData() {
-    return fetchDataAsText(this.url, this.cacheActive).then((text) =>
-      parseGml(text, this.namespace, this.version)
-    )
+  async getLoadQuery() {
+    return `
+CREATE TABLE ${this.datasetId} AS SELECT * FROM st_read("${this.url}",
+    allowed_drivers = ['GML']
+);`
   }
 }

package/src/libs/util/data-fetcher/src/lib/readers/json.ts

@@ -1,28 +1,10 @@
-import { DataItem, PropertyInfo } from '../model'
-import {
-  fetchDataAsText,
-  jsonToGeojsonFeature,
-  processItemProperties,
-} from '../utils'
 import { BaseFileReader } from './base-file'
 
-/**
- * This parser only supports arrays of simple flat objects with properties
- * @param text
- */
-export function parseJson(text: string): {
-  items: DataItem[]
-  properties: PropertyInfo[]
-} {
-  const parsed = JSON.parse(text) as any[]
-  if (!Array.isArray(parsed)) {
-    throw new Error('Could not parse JSON, expected an array at root level')
-  }
-  return processItemProperties(parsed.map(jsonToGeojsonFeature))
-}
-
 export class JsonReader extends BaseFileReader {
-  getData() {
-    return fetchDataAsText(this.url, this.cacheActive).then(parseJson)
+  async getLoadQuery() {
+    return `
+CREATE TABLE ${this.datasetId} AS SELECT * FROM read_json("${this.url}",
+    maximum_object_size = 536870912 -- 500MB
+);`
   }
 }

package/src/libs/util/data-fetcher/src/lib/readers/wfs.ts

@@ -1,14 +1,58 @@
-import { EndpointError, WfsEndpoint, WfsVersion } from '@camptocamp/ogc-client'
-import { DataItem, DatasetInfo, FetchError, PropertyInfo } from '../model'
 import {
-  fetchDataAsText,
-  getJsonDataItemsProxy,
-  jsonToGeojsonFeature,
-} from '../utils'
-import { GmlReader, parseGml } from './gml'
-import { GeojsonReader, parseGeojson } from './geojson'
-import { BaseCacheReader } from './base-cache'
-import { generateSqlQuery } from '../sql-utils'
+  EndpointError,
+  WfsEndpoint,
+  WfsFeatureTypeFull,
+} from '@camptocamp/ogc-client'
+import { DataItem, DatasetInfo, FetchError, PropertyInfo } from '../model'
+import { fetchDataAsText } from '../utils'
+import WFS from 'ol/format/WFS'
+import GeoJSON from 'ol/format/GeoJSON'
+import Feature from 'ol/Feature'
+import { GeojsonReader } from './geojson'
+import { GmlReader } from './gml'
+import { BaseReader } from './base'
+
+const formatGeojson = new GeoJSON()
+
+export function parseGeojson(text: string): DataItem[] {
+  const parsed = JSON.parse(text)
+  const features =
+    parsed.type === 'FeatureCollection' ? parsed.features : parsed
+  if (!Array.isArray(features)) {
+    throw new Error(
+      'Could not parse GeoJSON, expected a features collection or an array of features at root level'
+    )
+  }
+  return features
+}
+
+export function parseGml(
+  text: string,
+  featureType: string,
+  version: string
+): DataItem[] {
+  const parts = featureType.split(':')
+  const regex = new RegExp(`xmlns:${parts[0]}=["']([^'"]*)["']`)
+  const match = regex.exec(text)
+  if (match && match.length >= 2) {
+    const wfs = new WFS({
+      featureNS: match[1],
+      featureType: parts[1],
+      version: version,
+    })
+    let features: Feature[]
+    try {
+      features = wfs.readFeatures(text)
+    } catch (e: unknown) {
+      throw Error(
+        `Couldn't parse WFS with GML features: ${(e as Error).message}`
+      )
+    }
+    const geojsonItem = formatGeojson.writeFeaturesObject(features)
+    return geojsonItem.features
+  }
+  throw Error("Couldn't retrieve namespace url")
+}
 
 export async function getWfsEndpoint(wfsUrl: string): Promise<WfsEndpoint> {
   try {
@@ -36,151 +80,163 @@ export async function getWfsEndpoint(wfsUrl: string): Promise<WfsEndpoint> {
   }
 }
 
-export class WfsReader extends BaseCacheReader {
-  endpoint: WfsEndpoint
-  featureTypeName: string
-  version: WfsVersion
-
-  constructor(
-    url: string,
-    wfsEndpoint: WfsEndpoint,
-    featureTypeName: string,
-    cacheActive?: boolean
-  ) {
-    super(url, cacheActive)
-    this.endpoint = wfsEndpoint
-    this.featureTypeName = featureTypeName
-    this.version = this.endpoint.getVersion()
+export class WfsReader extends BaseReader {
+  endpoint: Promise<WfsEndpoint>
+  featureType: Promise<WfsFeatureTypeFull>
+
+  /**
+   * This dataset reader will read the whole data as geojson or gml instead of
+   * using the WFS protocol; it is used in the following situations:
+   * - the endpoint does not support pagination
+   * - aggregations or grouping are set on the dataset (i.e. for chart view)
+   */
+  backupReader_: Promise<BaseReader>
+
+  constructor(url: string, featureTypeName: string) {
+    super(url)
+    this.endpoint = getWfsEndpoint(url)
+
+    this.featureType = this.endpoint
+      .then((endpoint) => {
+        const featureTypes = endpoint.getFeatureTypes()
+        return endpoint.getFeatureTypeFull(
+          featureTypes.length === 1 && !featureTypeName
+            ? featureTypes[0].name
+            : featureTypeName
+        )
+      })
+      .then((featureType) => {
+        if (!featureType) {
+          throw new Error('wfs.featuretype.notfound')
+        }
+        return featureType
+      })
   }
 
-  get properties(): Promise<PropertyInfo[]> {
-    return this.endpoint
-      .getFeatureTypeFull(this.featureTypeName)
-      .then((featureType) =>
-        Object.keys(featureType.properties).map((prop) => {
-          const originalType = featureType.properties[prop]
-          const type =
-            originalType === 'float' || originalType === 'integer'
-              ? 'number'
-              : (originalType as PropertyInfo['type']) // FIXME: ogc-client typing is incorrect, should be a string union
-          return {
-            name: prop,
-            label: prop,
-            type,
+  get backupReader(): Promise<BaseReader> {
+    if (this.backupReader_) {
+      return this.backupReader_
+    }
+
+    this.backupReader_ = Promise.all([this.endpoint, this.featureType]).then(
+      ([endpoint, featureType]) => {
+        let reader: BaseReader
+        if (endpoint.supportsJson(featureType.name)) {
+          reader = new GeojsonReader(
+            endpoint.getFeatureUrl(featureType.name, {
+              asJson: true,
+              outputCrs: 'EPSG:4326',
+            })
+          )
+        } else {
+          if (
+            featureType.outputFormats.find((f) =>
+              f.toLowerCase().includes('gml')
+            ) &&
+            (featureType.defaultCrs === 'EPSG:4326' ||
+              featureType.otherCrs?.includes('EPSG:4326'))
+          ) {
+            reader = new GmlReader(
+              endpoint.getFeatureUrl(featureType.name, {
+                outputFormat: featureType.outputFormats.find((f) =>
+                  f.toLowerCase().includes('gml')
+                ),
+                outputCrs: 'EPSG:4326',
+              })
+            )
           }
-        })
-      )
+        }
+        reader.enableCache(this.cacheEnabled)
+        reader.load()
+        return reader
+      }
+    )
+    return this.backupReader_
+  }
+
+  get properties(): Promise<PropertyInfo[]> {
+    return this.featureType.then((featureType) =>
+      Object.keys(featureType.properties).map((prop) => {
+        const originalType = featureType.properties[prop]
+        let type: PropertyInfo['type']
+        if (originalType === 'float' || originalType === 'integer') {
+          type = 'number'
+        } else if (originalType === 'boolean') {
+          type = 'string' // we don't handle booleans yet in the data fetcher
+        } else {
+          type = originalType
+        }
+        return {
+          name: prop,
+          label: prop,
+          type,
+        }
+      })
+    )
   }
 
   get info(): Promise<DatasetInfo> {
-    return this.endpoint.getFeatureTypeFull(this.featureTypeName).then(
+    return this.featureType.then(
       (result) =>
         ({
           itemsCount: result.objectCount,
+          hasGeometry: !!result.geometryName,
         }) as DatasetInfo
     )
   }
 
-  static async createReader(wfsUrlEndpoint: string, featureTypeName?: string) {
-    const wfsEndpoint = await getWfsEndpoint(wfsUrlEndpoint)
-    const featureTypes = wfsEndpoint.getFeatureTypes()
-    const featureType = wfsEndpoint.getFeatureTypeSummary(
-      featureTypes.length === 1 && !featureTypeName
-        ? featureTypes[0].name
-        : featureTypeName
-    )
-    if (!featureType) {
-      throw new Error('wfs.featuretype.notfound')
-    }
+  load() {
+    // Nothing to load for Wfs
+  }
 
-    if (wfsEndpoint.supportsStartIndex()) {
-      return new WfsReader(wfsUrlEndpoint, wfsEndpoint, featureType.name)
-    } else if (wfsEndpoint.supportsJson(featureType.name)) {
-      return new GeojsonReader(
-        wfsEndpoint.getFeatureUrl(featureType.name, {
-          asJson: true,
-          outputCrs: 'EPSG:4326',
-        })
-      )
-    } else {
-      if (
-        featureType.outputFormats.find((f) =>
-          f.toLowerCase().includes('gml')
-        ) &&
-        (featureType.defaultCrs === 'EPSG:4326' ||
-          featureType.otherCrs?.includes('EPSG:4326'))
-      ) {
-        return new GmlReader(
-          wfsEndpoint.getFeatureUrl(featureType.name, {
-            outputFormat: featureType.outputFormats.find((f) =>
-              f.toLowerCase().includes('gml')
-            ),
-            outputCrs: 'EPSG:4326',
-          }),
-          featureType.name,
-          wfsEndpoint.getVersion()
-        )
+  async read(): Promise<DataItem[]> {
+    const endpoint = await this.endpoint
+    const featureType = await this.featureType
+
+    // if we can't use the WFS protocol we fall back to the backup reader
+    if (this.aggregations || this.groupedBy || !endpoint.supportsStartIndex()) {
+      const backupReader = await this.backupReader
+      backupReader.selectAll()
+      if (this.selected) {
+        backupReader.select(...this.selected)
+      }
+      if (this.aggregations) {
+        backupReader.aggregate(...this.aggregations)
+      }
+      if (this.groupedBy) {
+        backupReader.groupBy(...this.groupedBy)
+      }
+      if (this.sort) {
+        backupReader.orderBy(...this.sort)
       }
-      throw new Error('wfs.geojsongml.notsupported')
+      if (this.startIndex !== null && this.count !== null) {
+        backupReader.limit(this.startIndex, this.count)
+      }
+      return backupReader.read()
     }
-  }
 
-  public async getData(aggregation?, groupedBy?) {
-    if (aggregation || groupedBy) {
-      return { items: await this.getQueryData() }
-    }
-    const asJson = this.endpoint.supportsJson(this.featureTypeName)
+    const asJson = endpoint.supportsJson(featureType.name)
     const attributes = this.selected ?? undefined
-    let url = this.endpoint.getFeatureUrl(this.featureTypeName, {
+    let sortBy = null
+    if (this.sort) {
+      const mapSort = (s) => [s[0] === 'desc' ? 'D' : 'A', s[1]]
+      sortBy = Array.isArray(this.sort[0])
+        ? this.sort.map(mapSort)
+        : mapSort(this.sort)
+    }
+    const url = endpoint.getFeatureUrl(featureType.name, {
       ...(this.startIndex !== null && { startIndex: this.startIndex }),
       ...(this.count !== null && { maxFeatures: this.count }),
       asJson,
       outputCrs: 'EPSG:4326',
       attributes,
-      // sortBy: this.sort // TODO: no sort in ogc-client?
+      sortBy,
     })
 
-    if (Array.isArray(this.sort) && this.sort.length > 0) {
-      const finalUrl = new URL(url)
-      const sorts = this.sort
-        .map(
-          (fieldSort) => `${fieldSort[1]}+${fieldSort[0] === 'asc' ? 'A' : 'D'}`
-        )
-        .join(',')
-      // Direct update on string url to prevent encoding of +A and +D
-      url = `${url}${finalUrl.search ? '&' : ''}SORTBY=${sorts}`
-    }
-
-    return fetchDataAsText(url, this.cacheActive).then((text) =>
+    return fetchDataAsText(url, this.cacheEnabled).then((text) =>
       asJson
         ? parseGeojson(text)
-        : parseGml(text, this.featureTypeName, this.version)
-    )
-  }
-
-  public async getQueryData() {
-    const items = (await this.getData()).items
-    const jsonItems = getJsonDataItemsProxy(items)
-    const query = generateSqlQuery(
-      this.selected,
-      this.filter,
-      this.sort,
-      this.startIndex,
-      this.count,
-      this.groupedBy,
-      this.aggregations
-    )
-    const result: any[] = await import('alasql').then((module) =>
-      module.default(query, [jsonItems])
+        : parseGml(text, featureType.name, endpoint.getVersion())
     )
-    return result.map(jsonToGeojsonFeature)
-  }
-
-  load() {
-    // Nothing to load for Wfs
-  }
-
-  async read(): Promise<DataItem[]> {
-    return (await this.getData(this.aggregations, this.groupedBy)).items
   }
 }