geonetwork-ui 2.10.0-dev.fc6515c0b → 2.11.0-dev.2478042b2

package/src/libs/util/data-fetcher/src/lib/readers/gml.ts

@@ -1,54 +1,10 @@
 import { BaseFileReader } from './base-file'
-import { DataItem, PropertyInfo } from '../model'
-import { fetchDataAsText, processItemProperties } from '../utils'
-import { GeoJSON, WFS } from 'ol/format.js'
-import { WfsVersion } from '@camptocamp/ogc-client'
-
-export function parseGml(
-  text: string,
-  namespace: string,
-  version: string
-): {
-  items: DataItem[]
-  properties: PropertyInfo[]
-} {
-  const splittedNamespace = namespace.split(':')
-  const regex = new RegExp(`xmlns:${splittedNamespace[0]}=["']([^'"]*)["']`)
-  const match = regex.exec(text)
-
-  if (match && match.length >= 2) {
-    const wf = new WFS({
-      featureNS: match[1],
-      featureType: splittedNamespace[1],
-      version: version,
-    })
-
-    let features
-    try {
-      features = wf.readFeatures(text)
-    } catch (e) {
-      throw Error("Couldn't parse WFS with GML features")
-    }
-
-    const geojsonItem = new GeoJSON().writeFeaturesObject(features)
-    return processItemProperties(geojsonItem.features, true)
-  }
-  throw Error("Couldn't retrieve namespace url")
-}
 
 export class GmlReader extends BaseFileReader {
-  constructor(
-    protected url: string,
-    protected namespace: string,
-    protected version: WfsVersion,
-    protected cacheActive = true
-  ) {
-    super(url)
-  }
-
-  protected getData() {
-    return fetchDataAsText(this.url, this.cacheActive).then((text) =>
-      parseGml(text, this.namespace, this.version)
-    )
+  async getLoadQuery() {
+    return `
+CREATE TABLE ${this.datasetId} AS SELECT * FROM st_read("${this.url}",
+    allowed_drivers = ['GML']
+);`
   }
 }

package/src/libs/util/data-fetcher/src/lib/readers/json.ts

@@ -1,28 +1,10 @@
-import { DataItem, PropertyInfo } from '../model'
-import {
-  fetchDataAsText,
-  jsonToGeojsonFeature,
-  processItemProperties,
-} from '../utils'
 import { BaseFileReader } from './base-file'
 
-/**
- * This parser only supports arrays of simple flat objects with properties
- * @param text
- */
-export function parseJson(text: string): {
-  items: DataItem[]
-  properties: PropertyInfo[]
-} {
-  const parsed = JSON.parse(text) as any[]
-  if (!Array.isArray(parsed)) {
-    throw new Error('Could not parse JSON, expected an array at root level')
-  }
-  return processItemProperties(parsed.map(jsonToGeojsonFeature))
-}
-
 export class JsonReader extends BaseFileReader {
-  getData() {
-    return fetchDataAsText(this.url, this.cacheActive).then(parseJson)
+  async getLoadQuery() {
+    return `
+CREATE TABLE ${this.datasetId} AS SELECT * FROM read_json("${this.url}",
+    maximum_object_size = 536870912 -- 500MB
+);`
   }
 }

package/src/libs/util/data-fetcher/src/lib/readers/wfs.ts

@@ -1,11 +1,58 @@
-import { EndpointError, WfsEndpoint, WfsVersion } from '@camptocamp/ogc-client'
+import {
+  EndpointError,
+  WfsEndpoint,
+  WfsFeatureTypeFull,
+} from '@camptocamp/ogc-client'
 import { DataItem, DatasetInfo, FetchError, PropertyInfo } from '../model'
 import { fetchDataAsText } from '../utils'
-import { GmlReader, parseGml } from './gml'
-import { GeojsonReader, parseGeojson } from './geojson'
-import { BaseCacheReader } from './base-cache'
-import { getJsonDataItemsProxy, jsonToGeojsonFeature } from '../utils'
-import { generateSqlQuery } from '../sql-utils'
+import WFS from 'ol/format/WFS'
+import GeoJSON from 'ol/format/GeoJSON'
+import Feature from 'ol/Feature'
+import { GeojsonReader } from './geojson'
+import { GmlReader } from './gml'
+import { BaseReader } from './base'
+
+const formatGeojson = new GeoJSON()
+
+export function parseGeojson(text: string): DataItem[] {
+  const parsed = JSON.parse(text)
+  const features =
+    parsed.type === 'FeatureCollection' ? parsed.features : parsed
+  if (!Array.isArray(features)) {
+    throw new Error(
+      'Could not parse GeoJSON, expected a features collection or an array of features at root level'
+    )
+  }
+  return features
+}
+
+export function parseGml(
+  text: string,
+  featureType: string,
+  version: string
+): DataItem[] {
+  const parts = featureType.split(':')
+  const regex = new RegExp(`xmlns:${parts[0]}=["']([^'"]*)["']`)
+  const match = regex.exec(text)
+  if (match && match.length >= 2) {
+    const wfs = new WFS({
+      featureNS: match[1],
+      featureType: parts[1],
+      version: version,
+    })
+    let features: Feature[]
+    try {
+      features = wfs.readFeatures(text)
+    } catch (e: unknown) {
+      throw Error(
+        `Couldn't parse WFS with GML features: ${(e as Error).message}`
+      )
+    }
+    const geojsonItem = formatGeojson.writeFeaturesObject(features)
+    return geojsonItem.features
+  }
+  throw Error("Couldn't retrieve namespace url")
+}
 
 export async function getWfsEndpoint(wfsUrl: string): Promise<WfsEndpoint> {
   try {
@@ -33,151 +80,163 @@ export async function getWfsEndpoint(wfsUrl: string): Promise<WfsEndpoint> {
   }
 }
 
-export class WfsReader extends BaseCacheReader {
-  endpoint: WfsEndpoint
-  featureTypeName: string
-  version: WfsVersion
-
-  constructor(
-    url: string,
-    wfsEndpoint: WfsEndpoint,
-    featureTypeName: string,
-    cacheActive?: boolean
-  ) {
-    super(url, cacheActive)
-    this.endpoint = wfsEndpoint
-    this.featureTypeName = featureTypeName
-    this.version = this.endpoint.getVersion()
+export class WfsReader extends BaseReader {
+  endpoint: Promise<WfsEndpoint>
+  featureType: Promise<WfsFeatureTypeFull>
+
+  /**
+   * This dataset reader will read the whole data as geojson or gml instead of
+   * using the WFS protocol; it is used in the following situations:
+   * - the endpoint does not support pagination
+   * - aggregations or grouping are set on the dataset (i.e. for chart view)
+   */
+  backupReader_: Promise<BaseReader>
+
+  constructor(url: string, featureTypeName: string) {
+    super(url)
+    this.endpoint = getWfsEndpoint(url)
+
+    this.featureType = this.endpoint
+      .then((endpoint) => {
+        const featureTypes = endpoint.getFeatureTypes()
+        return endpoint.getFeatureTypeFull(
+          featureTypes.length === 1 && !featureTypeName
+            ? featureTypes[0].name
+            : featureTypeName
+        )
+      })
+      .then((featureType) => {
+        if (!featureType) {
+          throw new Error('wfs.featuretype.notfound')
+        }
+        return featureType
+      })
   }
 
-  get properties(): Promise<PropertyInfo[]> {
-    return this.endpoint
-      .getFeatureTypeFull(this.featureTypeName)
-      .then((featureType) =>
-        Object.keys(featureType.properties).map((prop) => {
-          const originalType = featureType.properties[prop]
-          const type =
-            originalType === 'float' || originalType === 'integer'
-              ? 'number'
-              : (originalType as PropertyInfo['type']) // FIXME: ogc-client typing is incorrect, should be a string union
-          return {
-            name: prop,
-            label: prop,
-            type,
+  get backupReader(): Promise<BaseReader> {
+    if (this.backupReader_) {
+      return this.backupReader_
+    }
+
+    this.backupReader_ = Promise.all([this.endpoint, this.featureType]).then(
+      ([endpoint, featureType]) => {
+        let reader: BaseReader
+        if (endpoint.supportsJson(featureType.name)) {
+          reader = new GeojsonReader(
+            endpoint.getFeatureUrl(featureType.name, {
+              asJson: true,
+              outputCrs: 'EPSG:4326',
+            })
+          )
+        } else {
+          if (
+            featureType.outputFormats.find((f) =>
+              f.toLowerCase().includes('gml')
+            ) &&
+            (featureType.defaultCrs === 'EPSG:4326' ||
+              featureType.otherCrs?.includes('EPSG:4326'))
+          ) {
+            reader = new GmlReader(
+              endpoint.getFeatureUrl(featureType.name, {
+                outputFormat: featureType.outputFormats.find((f) =>
+                  f.toLowerCase().includes('gml')
+                ),
+                outputCrs: 'EPSG:4326',
+              })
+            )
           }
-        })
-      )
+        }
+        reader.enableCache(this.cacheEnabled)
+        reader.load()
+        return reader
+      }
+    )
+    return this.backupReader_
+  }
+
+  get properties(): Promise<PropertyInfo[]> {
+    return this.featureType.then((featureType) =>
+      Object.keys(featureType.properties).map((prop) => {
+        const originalType = featureType.properties[prop]
+        let type: PropertyInfo['type']
+        if (originalType === 'float' || originalType === 'integer') {
+          type = 'number'
+        } else if (originalType === 'boolean') {
+          type = 'string' // we don't handle booleans yet in the data fetcher
+        } else {
+          type = originalType
+        }
+        return {
+          name: prop,
+          label: prop,
+          type,
+        }
+      })
+    )
   }
 
   get info(): Promise<DatasetInfo> {
-    return this.endpoint.getFeatureTypeFull(this.featureTypeName).then(
+    return this.featureType.then(
       (result) =>
         ({
           itemsCount: result.objectCount,
+          hasGeometry: !!result.geometryName,
         }) as DatasetInfo
     )
   }
 
-  static async createReader(wfsUrlEndpoint: string, featureTypeName?: string) {
-    const wfsEndpoint = await getWfsEndpoint(wfsUrlEndpoint)
-    const featureTypes = wfsEndpoint.getFeatureTypes()
-    const featureType = wfsEndpoint.getFeatureTypeSummary(
-      featureTypes.length === 1 && !featureTypeName
-        ? featureTypes[0].name
-        : featureTypeName
-    )
-    if (!featureType) {
-      throw new Error('wfs.featuretype.notfound')
-    }
+  load() {
+    // Nothing to load for Wfs
+  }
 
-    if (wfsEndpoint.supportsStartIndex()) {
-      return new WfsReader(wfsUrlEndpoint, wfsEndpoint, featureType.name)
-    } else if (wfsEndpoint.supportsJson(featureType.name)) {
-      return new GeojsonReader(
-        wfsEndpoint.getFeatureUrl(featureType.name, {
-          asJson: true,
-          outputCrs: 'EPSG:4326',
-        })
-      )
-    } else {
-      if (
-        featureType.outputFormats.find((f) =>
-          f.toLowerCase().includes('gml')
-        ) &&
-        (featureType.defaultCrs === 'EPSG:4326' ||
-          featureType.otherCrs?.includes('EPSG:4326'))
-      ) {
-        return new GmlReader(
-          wfsEndpoint.getFeatureUrl(featureType.name, {
-            outputFormat: featureType.outputFormats.find((f) =>
-              f.toLowerCase().includes('gml')
-            ),
-            outputCrs: 'EPSG:4326',
-          }),
-          featureType.name,
-          wfsEndpoint.getVersion()
-        )
+  async read(): Promise<DataItem[]> {
+    const endpoint = await this.endpoint
+    const featureType = await this.featureType
+
+    // if we can't use the WFS protocol we fall back to the backup reader
+    if (this.aggregations || this.groupedBy || !endpoint.supportsStartIndex()) {
+      const backupReader = await this.backupReader
+      backupReader.selectAll()
+      if (this.selected) {
+        backupReader.select(...this.selected)
       }
-      throw new Error('wfs.geojsongml.notsupported')
+      if (this.aggregations) {
+        backupReader.aggregate(...this.aggregations)
+      }
+      if (this.groupedBy) {
+        backupReader.groupBy(...this.groupedBy)
+      }
+      if (this.sort) {
+        backupReader.orderBy(...this.sort)
+      }
+      if (this.startIndex !== null && this.count !== null) {
+        backupReader.limit(this.startIndex, this.count)
+      }
+      return backupReader.read()
     }
-  }
 
-  public async getData(aggregation?, groupedBy?) {
-    if (aggregation || groupedBy) {
-      return { items: await this.getQueryData() }
-    }
-    const asJson = this.endpoint.supportsJson(this.featureTypeName)
+    const asJson = endpoint.supportsJson(featureType.name)
     const attributes = this.selected ?? undefined
-    let url = this.endpoint.getFeatureUrl(this.featureTypeName, {
+    let sortBy = null
+    if (this.sort) {
+      const mapSort = (s) => [s[0] === 'desc' ? 'D' : 'A', s[1]]
+      sortBy = Array.isArray(this.sort[0])
+        ? this.sort.map(mapSort)
+        : mapSort(this.sort)
+    }
+    const url = endpoint.getFeatureUrl(featureType.name, {
       ...(this.startIndex !== null && { startIndex: this.startIndex }),
       ...(this.count !== null && { maxFeatures: this.count }),
       asJson,
       outputCrs: 'EPSG:4326',
       attributes,
-      // sortBy: this.sort // TODO: no sort in ogc-client?
+      sortBy,
     })
 
-    if (Array.isArray(this.sort) && this.sort.length > 0) {
-      const finalUrl = new URL(url)
-      const sorts = this.sort
-        .map(
-          (fieldSort) => `${fieldSort[1]}+${fieldSort[0] === 'asc' ? 'A' : 'D'}`
-        )
-        .join(',')
-      // Direct update on string url to prevent encoding of +A and +D
-      url = `${url}${finalUrl.search ? '&' : ''}SORTBY=${sorts}`
-    }
-
-    return fetchDataAsText(url, this.cacheActive).then((text) =>
+    return fetchDataAsText(url, this.cacheEnabled).then((text) =>
       asJson
         ? parseGeojson(text)
-        : parseGml(text, this.featureTypeName, this.version)
-    )
-  }
-
-  public async getQueryData() {
-    const items = (await this.getData()).items
-    const jsonItems = getJsonDataItemsProxy(items)
-    const query = generateSqlQuery(
-      this.selected,
-      this.filter,
-      this.sort,
-      this.startIndex,
-      this.count,
-      this.groupedBy,
-      this.aggregations
-    )
-    const result = await import('alasql').then((module) =>
-      module.default(query, [jsonItems])
+        : parseGml(text, featureType.name, endpoint.getVersion())
     )
-    return result.map(jsonToGeojsonFeature)
-  }
-
-  load() {
-    // Nothing to load for Wfs
-  }
-
-  async read(): Promise<DataItem[]> {
-    return (await this.getData(this.aggregations, this.groupedBy)).items
   }
 }

package/src/libs/util/data-fetcher/src/lib/utils.ts

@@ -2,14 +2,11 @@ import {
   DataItem,
   DatasetHeaders,
   FetchError,
-  PropertyInfo,
   SupportedType,
   SupportedTypes,
 } from './model'
 import { sharedFetch, useCache } from '@camptocamp/ogc-client'
 import { parseHeaders } from './headers'
-import { parse as parseDate } from 'date-fns/parse'
-import { parseISO as parseIsoDate } from 'date-fns/parseISO'
 
 export async function inferDatasetType(
   url: string,
@@ -95,146 +92,6 @@ export function fetchDataAsArrayBuffer(
   })
 }
 
-export function tryParseDate(input: unknown): Date | null {
-  if (typeof input !== 'string') return null
-
-  function tryIso(value: string) {
-    const parsed = parseIsoDate(value)
-    return isNaN(parsed.getDate()) ? null : parsed
-  }
-  function tryFormat(value: string, format: string) {
-    const parsed = parseDate(value, format, new Date())
-    return isNaN(parsed.getDate()) ? null : parsed
-  }
-  return (
-    tryIso(input) ||
-    tryFormat(input, 'dd/MM/yyyy') ||
-    tryFormat(input, 'dd.MM.yyyy') ||
-    tryFormat(input, 'MM/dd/yyyy') ||
-    null
-  )
-}
-
-export function tryParseNumber(input: string): number | null {
-  if (isNaN(input as any)) return null
-  const parsed = parseFloat(input)
-  return isNaN(parsed) ? null : parsed
-}
-
-export function jsonToGeojsonFeature(object: { [key: string]: any }): DataItem {
-  const { id, properties } = Object.keys(object)
-    .map((property) => (property ? property : 'unknown')) //prevent empty strings
-    .reduce(
-      (prev, curr) =>
-        curr.toLowerCase().endsWith('id')
-          ? {
-              ...prev,
-              id: object[curr],
-            }
-          : {
-              ...prev,
-              properties: { ...prev.properties, [curr]: object[curr] },
-            },
-      { id: undefined, properties: {} }
-    )
-  return {
-    type: 'Feature',
-    geometry: null,
-    properties,
-    ...(id !== undefined && { id }),
-  }
-}
-
-function mutateProperties(
-  items: DataItem[],
-  mutators: { [fieldName: string]: (value: unknown) => unknown }
-): DataItem[] {
-  const mutatorKeys = Object.keys(mutators)
-  for (let i = 0, ii = items.length; i < ii; i++) {
-    const item = items[i]
-    for (const mutatorField of mutatorKeys) {
-      if (!(mutatorField in item.properties)) continue
-      item.properties[mutatorField] = mutators[mutatorField](
-        item.properties[mutatorField]
-      )
-    }
-  }
-  return items
-}
-
-const SAMPLE_SIZE = 20
-
-/**
- * This will infer field types from a list of data items and cast the values accordingly
- * @param items
- * @param inferTypes
- */
-export function processItemProperties(
-  items: DataItem[],
-  inferTypes = false
-): {
-  items: DataItem[]
-  properties: PropertyInfo[]
-} {
-  const foundFields: { [name: string]: PropertyInfo } = {}
-  for (let i = 0, ii = Math.min(SAMPLE_SIZE, items.length); i < ii; i++) {
-    const item = items[i]
-    const fields = Object.keys(item.properties)
-    for (const field of fields) {
-      if (!(field in foundFields)) {
-        foundFields[field] = {
-          label: field,
-          name: field,
-          type: null,
-        }
-      }
-      const value = item.properties[field]
-      const info = foundFields[field]
-      if (value === undefined || value === '' || value === null) continue
-
-      if (!inferTypes) {
-        if (info.type === null && typeof value === 'number') {
-          info.type = 'number'
-        } else if (info.type === 'number' && typeof value !== 'number') {
-          info.type = 'string'
-        }
-        continue
-      }
-
-      const parsedNumber = tryParseNumber(value)
-      if (info.type === null && parsedNumber !== null) {
-        info.type = 'number'
-        continue
-      } else if (info.type === 'number' && parsedNumber === null) {
-        info.type = 'string'
-        continue
-      }
-      const parsedDate = tryParseDate(value)
-      if (info.type === null && parsedDate !== null) {
-        info.type = 'date'
-      } else if (info.type === 'date' && parsedDate === null) {
-        info.type = 'string'
-      }
-    }
-  }
-
-  const properties: PropertyInfo[] = []
-  const mutators = {}
-  for (const field in foundFields) {
-    const info = foundFields[field]
-    if (info.type === 'number') {
-      mutators[field] = tryParseNumber
-    } else if (info.type === 'date') {
-      mutators[field] = tryParseDate
-    }
-    properties.push({ ...info, type: info.type || 'string' })
-  }
-  if (inferTypes) {
-    mutateProperties(items, mutators)
-  }
-  return { items, properties }
-}
-
 /**
  * This creates a Proxy that allows reading and writing to the data item properties
  * as if it was a simple array of JSON objects

package/src/libs/util/i18n/src/index.ts

@@ -2,6 +2,5 @@ export * from './lib/i18n.providers'
 export * from './lib/i18n.constants'
 export * from './lib/i18n.interceptor'
 export * from './lib/file.translate.loader'
-export * from './lib/embedded.translate.loader'
 export * from './lib/translate.loader.utils'
 export * from './lib/language-codes'

package/src/libs/util/i18n/src/lib/i18n.constants.ts

@@ -1,6 +1,4 @@
-import { HttpClient } from '@angular/common/http'
 import { TranslateMessageFormatCompiler } from 'ngx-translate-messageformat-compiler'
-import { FileTranslateLoader } from './file.translate.loader'
 import {
   TranslateCompiler,
   TranslateDefaultParser,
@@ -9,26 +7,34 @@ import {
   TranslateParser,
 } from '@ngx-translate/core'
 import { Injectable } from '@angular/core'
+import { TRANSLATE_HTTP_LOADER_CONFIG } from '@ngx-translate/http-loader'
+import { HttpClient } from '@angular/common/http'
+import { FileTranslateLoader } from './file.translate.loader'
 
 export const DEFAULT_LANG = 'en'
 
 // Caution: changing this can break language selection from third parties!
 export const LANGUAGE_STORAGE_KEY = 'geonetwork-ui-language'
 
-export function HttpLoaderFactory(http: HttpClient) {
-  return new FileTranslateLoader(http, './assets/i18n/')
-}
-
 export const TRANSLATE_DEFAULT_CONFIG: TranslateModuleConfig = {
   compiler: {
     provide: TranslateCompiler,
     useClass: TranslateMessageFormatCompiler,
   },
-  loader: {
-    provide: TranslateLoader,
-    useFactory: HttpLoaderFactory,
-    deps: [HttpClient],
-  },
+  loader: [
+    {
+      provide: TRANSLATE_HTTP_LOADER_CONFIG,
+      useValue: {
+        prefix: './assets/i18n/',
+        suffix: '.json',
+      },
+    },
+    {
+      provide: TranslateLoader,
+      useClass: FileTranslateLoader,
+      deps: [HttpClient, TRANSLATE_HTTP_LOADER_CONFIG],
+    },
+  ],
 }
 
 @Injectable()

package/src/libs/util/i18n/src/lib/i18n.interceptor.ts

@@ -6,7 +6,7 @@ import {
 } from '@angular/common/http'
 import { DEFAULT_LANG } from './i18n.constants'
 import { Observable } from 'rxjs'
-import { Injectable, Injector, inject } from '@angular/core'
+import { inject, Injectable, Injector } from '@angular/core'
 import { TranslateService } from '@ngx-translate/core'
 
 @Injectable()
@@ -21,7 +21,7 @@ export class I18nInterceptor implements HttpInterceptor {
     const translate = this.injector.get(TranslateService)
     request = request.clone({
       setHeaders: {
-        'Accept-Language': translate.currentLang || DEFAULT_LANG,
+        'Accept-Language': translate.getCurrentLang() || DEFAULT_LANG,
       },
     })
     return next.handle(request)