geonetwork-ui 2.10.3-dev.4154db07d → 2.11.0-dev.0559cfd9d

package/src/libs/api/repository/src/lib/gn4/elasticsearch/elasticsearch.service.ts

@@ -11,6 +11,7 @@ import {
   AggregationsParams,
   FieldFilter,
   FieldFilters,
+  FieldSort,
   FilterQuery,
   FiltersAggregationParams,
   SortByField,
@@ -33,7 +34,12 @@ import {
   LanguageCode,
 } from '../../../../../../../libs/common/domain/src/lib/model/record'
 import { TranslateService } from '@ngx-translate/core'
-import { getGeometryBoundingBox } from '../../../../../../../libs/util/shared/src'
+import {
+  bboxToPolygon,
+  BoundingBox,
+  getGeometryBoundingBox,
+  isBoundingBox,
+} from '../../../../../../../libs/util/shared/src'
 import { getLength as getGeodesicLength } from 'ol/sphere.js'
 import { LineString } from 'ol/geom.js'
 
@@ -48,7 +54,12 @@ export class ElasticsearchService {
 
   // runtime fields are computed using a Painless script
   // see: https://www.elastic.co/guide/en/elasticsearch/reference/current/runtime-mapping-fields.html
-  private runtimeFields: Record<string, string> = {}
+  private runtimeFields: Record<
+    string,
+    { script: string; type: 'keyword' | 'date' }
+  > = {}
+
+  private fieldAliases: Record<string, string> = {}
 
   // we're using getters in case the defined languages change over time
   private get metadataLang(): LanguageCode {
@@ -90,8 +101,8 @@ export class ElasticsearchService {
     const addMapping = (fieldName: string) => {
       if (!payload.runtime_mappings) payload.runtime_mappings = {}
       payload.runtime_mappings[fieldName] = {
-        type: 'keyword',
-        script: this.runtimeFields[fieldName],
+        type: this.runtimeFields[fieldName].type,
+        script: this.runtimeFields[fieldName].script,
       }
     }
     const lookForField = (node: unknown) => {
@@ -112,6 +123,14 @@ export class ElasticsearchService {
         ) {
           addMapping(runtimeField)
         }
+        if (
+          runtimeField in node &&
+          typeof node[runtimeField] === 'object' &&
+          node[runtimeField] !== null &&
+          ('gte' in node[runtimeField] || 'lte' in node[runtimeField])
+        ) {
+          addMapping(runtimeField)
+        }
         if (
           'query' in node &&
           typeof node.query === 'string' &&
@@ -130,8 +149,16 @@ export class ElasticsearchService {
     return payload
   }
 
-  registerRuntimeField(fieldName: string, expression: string) {
-    this.runtimeFields[fieldName] = expression
+  registerRuntimeField(
+    fieldName: string,
+    expression: string,
+    type: 'keyword' | 'date' = 'keyword'
+  ) {
+    this.runtimeFields[fieldName] = { script: expression, type }
+  }
+
+  registerFieldAlias(fieldIdentifier: string, esFieldName: string) {
+    this.fieldAliases[fieldIdentifier] = esFieldName
   }
 
   getMetadataByIdsPayload(uuids: string[]): EsSearchParams {
@@ -141,6 +168,7 @@ export class ElasticsearchService {
           values: uuids,
         },
       },
+      size: uuids.length,
     }
   }
 
@@ -199,9 +227,27 @@ export class ElasticsearchService {
   }
 
   private buildPayloadSort(sortBy: SortByField): SortParams {
-    if (sortBy === null) return undefined
-    const fields = Array.isArray(sortBy[0]) ? sortBy : [sortBy]
-    return fields.map((field) => ({ [field[1]]: field[0] }))
+    if (!sortBy || sortBy.length === 0) return undefined
+    const fields = Array.isArray(sortBy[0])
+      ? (sortBy as FieldSort[])
+      : [sortBy as FieldSort]
+    return fields.map((field) => {
+      // Sort by nested array of dates only works with the explicit syntax
+      if (field[1].endsWith('.date')) {
+        const nestedPath = field[1].slice(0, field[1].lastIndexOf('.date'))
+        return {
+          [field[1]]: {
+            order: field[0] as 'desc' | 'asc',
+            mode: field[0] === 'desc' ? 'max' : 'min',
+            missing: '_last',
+            nested: {
+              path: nestedPath,
+            },
+          },
+        }
+      }
+      return { [field[1]]: field[0] }
+    })
   }
 
   private injectLangInQueryStringFields(
@@ -244,8 +290,68 @@ export class ElasticsearchService {
     return this.metadataLang === 'current'
   }
 
-  private filtersToQuery(
+  private findSpatialFilterExtent(
     filters: FieldFilters | FiltersAggregationParams | string
+  ): BoundingBox | undefined {
+    if (typeof filters === 'string') {
+      return undefined
+    }
+    return Object.values(filters).find(isBoundingBox)
+  }
+
+  // builds a query matching records whose dates intersect the filter range
+  private buildDateRangeQuery(searchField: string, dateRange: DateRange) {
+    // fields ending with 'DateRange' maps as ES range type
+    if (searchField.endsWith('DateRange')) {
+      return {
+        range: {
+          [searchField]: {
+            ...(dateRange.start && { gte: formatDate(dateRange.start) }),
+            ...(dateRange.end && { lte: formatDate(dateRange.end) }),
+            format: 'yyyy-MM-dd',
+            relation: 'intersects',
+          },
+        },
+      }
+    }
+    const minField = `${searchField}Min`
+    const maxField = `${searchField}Max`
+    if (minField in this.runtimeFields && maxField in this.runtimeFields) {
+      // min/max runtime fields form an interval per record; match if it intersects the filter range
+      const filter = [
+        dateRange.start && {
+          range: {
+            [maxField]: {
+              gte: formatDate(dateRange.start),
+              format: 'yyyy-MM-dd',
+            },
+          },
+        },
+        dateRange.end && {
+          range: {
+            [minField]: {
+              lte: formatDate(dateRange.end),
+              format: 'yyyy-MM-dd',
+            },
+          },
+        },
+      ].filter(Boolean)
+      return { bool: { filter } }
+    }
+    return {
+      range: {
+        [searchField]: {
+          ...(dateRange.start && { gte: formatDate(dateRange.start) }),
+          ...(dateRange.end && { lte: formatDate(dateRange.end) }),
+          format: 'yyyy-MM-dd',
+        },
+      },
+    }
+  }
+
+  private filtersToQuery(
+    filters: FieldFilters | FiltersAggregationParams | string,
+    spatialFilterExtent = this.findSpatialFilterExtent(filters)
   ): FilterQuery {
     const addQuote = (key: string) => (/^\/.+\/$/.test(key) ? key : `"${key}"`)
     const makeQuery = (filter: FieldFilter): string => {
@@ -266,50 +372,49 @@ export class ElasticsearchService {
         ? filters
         : Object.keys(filters)
             .filter((fieldname) => fieldname !== 'gn-ui-crossFieldFilter')
+            .filter((fieldname) => !isBoundingBox(filters[fieldname]))
             .filter((fieldname) => !isDateRange(filters[fieldname]))
+            .filter((fieldname) => !Array.isArray(filters[fieldname]))
             .filter(
               (fieldname) =>
                 filters[fieldname] &&
                 JSON.stringify(filters[fieldname]) !== '{}'
             )
             .map(
-              (fieldname) => `${fieldname}:(${makeQuery(filters[fieldname])})`
+              (fieldname) =>
+                `${this.fieldAliases[fieldname] ?? fieldname}:(${makeQuery(filters[fieldname])})`
             )
             .join(' AND ')
     if (filters['gn-ui-crossFieldFilter']) {
       queryString = `${queryString} AND (${filters['gn-ui-crossFieldFilter']})`
     }
-    const queryRange = Object.entries(filters)
-      .filter(([, value]) => isDateRange(value))
-      .map(([searchField, dateRange]) => {
-        return {
-          searchField,
-          dateRange,
-        } as {
-          searchField: string
-          dateRange: DateRange
-        }
-      })[0]
+    const queryRanges = Object.entries(filters).filter(([, value]) =>
+      isDateRange(value)
+    ) as [string, DateRange][]
     const queryParts = [
       queryString && {
         query_string: {
           query: queryString,
         },
       },
-      queryRange &&
-        queryRange.dateRange && {
-          range: {
-            [queryRange.searchField]: {
-              ...(queryRange.dateRange.start && {
-                gte: formatDate(queryRange.dateRange.start),
-              }),
-              ...(queryRange.dateRange.end && {
-                lte: formatDate(queryRange.dateRange.end),
-              }),
-              format: 'yyyy-MM-dd',
+      ...queryRanges.map(([searchField, dateRange]) =>
+        this.buildDateRangeQuery(searchField, dateRange)
+      ),
+      spatialFilterExtent && {
+        geo_shape: {
+          geom: {
+            shape: {
+              type: 'envelope',
+              // spatialFilterExtent is [minX, minY, maxX, maxY]; envelope coordinates are [top-left, bottom-right]
+              coordinates: [
+                [spatialFilterExtent[0], spatialFilterExtent[3]],
+                [spatialFilterExtent[2], spatialFilterExtent[1]],
+              ],
             },
+            relation: 'intersects',
           },
         },
+      },
     ].filter(Boolean)
     return queryParts.length > 0 ? (queryParts as FilterQuery) : undefined
   }
@@ -348,7 +453,12 @@ export class ElasticsearchService {
         },
       })
     }
-    const queryFilters = this.filtersToQuery(fieldSearchFilters)
+    // a spatial extent filter takes precedence over the preference geometry for boosting
+    const spatialFilterExtent = this.findSpatialFilterExtent(fieldSearchFilters)
+    const queryFilters = this.filtersToQuery(
+      fieldSearchFilters,
+      spatialFilterExtent
+    )
     if (queryFilters) {
       filter.push(...queryFilters)
     }
@@ -359,7 +469,10 @@ export class ElasticsearchService {
         },
       })
     }
-    if (geometry) {
+    const boostGeometry = spatialFilterExtent
+      ? bboxToPolygon(spatialFilterExtent)
+      : geometry
+    if (boostGeometry) {
       // boosts applied using the filter geometry:
       // * records completely within the geometry receive a boost of 5
       // * records intersecting the geometry receive a boost of 2
@@ -369,7 +482,7 @@ export class ElasticsearchService {
         {
           geo_shape: {
             geom: {
-              shape: geometry,
+              shape: boostGeometry,
               relation: 'within',
             },
             boost: 5.0,
@@ -378,7 +491,7 @@ export class ElasticsearchService {
         {
           geo_shape: {
             geom: {
-              shape: geometry,
+              shape: boostGeometry,
               relation: 'intersects',
             },
             boost: 2.0,
@@ -389,7 +502,7 @@ export class ElasticsearchService {
       // this will boost the results variably depending on their distance from the given geometry
       // note: this takes into account the `location` field of a record; this is generally the center of all spatial extents
       // combined, and thus the actual size/coverage of the record spatial extent isn't relevant here
-      const bbox = getGeometryBoundingBox(geometry)
+      const bbox = getGeometryBoundingBox(boostGeometry)
       const center = [(bbox[0] + bbox[2]) / 2, (bbox[1] + bbox[3]) / 2]
       const northToCenter = new LineString([
         [center[0], bbox[3]],
@@ -615,6 +728,12 @@ export class ElasticsearchService {
               order: {
                 [`_${aggregation.sort[1]}`]: aggregation.sort[0],
               },
+              ...(aggregation.includeValues && {
+                include: aggregation.includeValues,
+              }),
+              ...(aggregation.excludeValues && {
+                exclude: aggregation.excludeValues,
+              }),
             },
           }
         case 'histogram':

package/src/libs/api/repository/src/lib/gn4/gn4-repository.ts

@@ -14,10 +14,12 @@ import {
 } from '../../../../../../libs/api/metadata-converter/src'
 import { PublicationVersionError } from '../../../../../../libs/common/domain/src/lib/model/error'
 import {
+  AssociatedRecord,
   CatalogRecord,
   DatasetFeatureCatalog,
-  DatasetFeatureType,
   LanguageCode,
+  LinkedRecord,
+  RecordRelation,
 } from '../../../../../../libs/common/domain/src/lib/model/record'
 import {
   Aggregations,
@@ -145,7 +147,7 @@ export class Gn4Repository implements RecordsRepositoryInterface {
     return this.gn4SearchApi
       .search(
         'bucket',
-        ['fcats', 'hassources'],
+        ['fcats', 'hassources', 'siblings', 'associated'],
         JSON.stringify(
           this.gn4SearchHelper.getMetadataByIdsPayload([uniqueIdentifier])
         )
@@ -179,43 +181,15 @@ export class Gn4Repository implements RecordsRepositoryInterface {
       )
   }
 
-  private mapEmbeddedFeatureCatalog(
-    featureTypes: Array<DatasetFeatureType>
-  ): DatasetFeatureCatalog {
-    return {
-      featureTypes: featureTypes.map((featureType) => ({
-        name: featureType.typeName || '',
-        definition: featureType.definition || '',
-        attributes: Array.isArray(featureType.attributeTable)
-          ? featureType.attributeTable.map((attr) => {
-              const values = attr.values
-                ?.filter((v) => v.code || v.label)
-                .map((v) => ({
-                  code: v.code,
-                  label: v.label,
-                }))
-              return {
-                name: attr.name,
-                code: attr.code,
-                definition: attr.definition,
-                type: attr.type,
-                ...(values?.length > 0 ? { values } : {}),
-              }
-            })
-          : [],
-      })),
-    }
-  }
-  getFeatureCatalog(
+  queryLinkedFeatureCatalog(
     record: CatalogRecord,
     visited: Set<string> = new Set() // prevent looping
   ): Observable<DatasetFeatureCatalog | null> {
     if (
-      record.extras?.['featureTypes'] &&
-      Array.isArray(record.extras['featureTypes']) &&
-      record.extras['featureTypes'].length > 0
+      'featureTypeDescriptions' in record &&
+      record.featureTypeDescriptions?.length > 0
     ) {
-      return of(this.mapEmbeddedFeatureCatalog(record.extras['featureTypes']))
+      return of({ featureTypes: record.featureTypeDescriptions })
     }
 
     const featureCatalogIdentifier = record.extras[
@@ -225,7 +199,7 @@ export class Gn4Repository implements RecordsRepositoryInterface {
       visited.add(featureCatalogIdentifier)
       return this.getRecord(featureCatalogIdentifier).pipe(
         switchMap((record) =>
-          record ? this.getFeatureCatalog(record, visited) : of(null)
+          record ? this.queryLinkedFeatureCatalog(record, visited) : of(null)
         )
       )
     }
@@ -249,22 +223,43 @@ export class Gn4Repository implements RecordsRepositoryInterface {
       )
   }
 
-  getSources(record: CatalogRecord): Observable<CatalogRecord[]> {
-    const sourcesIdentifiers = record.extras?.['sourcesIdentifiers'] as string[]
-    if (sourcesIdentifiers && sourcesIdentifiers.length > 0) {
-      return this.getMultipleRecords(sourcesIdentifiers)
-    }
-    return of(null)
-  }
-
-  getSourceOf(record: CatalogRecord): Observable<CatalogRecord[]> {
-    const sourceOfIdentifiers = record.extras?.[
-      'sourceOfIdentifiers'
-    ] as string[]
-    if (sourceOfIdentifiers && sourceOfIdentifiers.length > 0) {
-      return this.getMultipleRecords(sourceOfIdentifiers)
+  getLinkedRecords(record: CatalogRecord): Observable<LinkedRecord[]> {
+    const siblings = (record.extras?.['siblings'] ?? []) as AssociatedRecord[]
+    const relations: Array<[RecordRelation, string[]]> = [
+      ['source', (record.extras?.['sourcesIdentifiers'] ?? []) as string[]],
+      ['sourceOf', (record.extras?.['sourceOfIdentifiers'] ?? []) as string[]],
+      ['sibling', siblings.map(({ uniqueIdentifier }) => uniqueIdentifier)],
+      [
+        'associated',
+        (record.extras?.['associatedIdentifiers'] ?? []) as string[],
+      ],
+    ]
+    const requested = relations.filter(
+      ([, identifiers]) => identifiers.length > 0
+    )
+    if (requested.length === 0) {
+      return of([])
     }
-    return of(null)
+    return forkJoin(
+      requested.map(([relation, identifiers]) =>
+        this.getMultipleRecords(identifiers).pipe(
+          map((records) =>
+            (records ?? []).map((record) => ({
+              record,
+              relation,
+              associationType:
+                relation === 'sibling'
+                  ? siblings.find(
+                      ({ uniqueIdentifier }) =>
+                        uniqueIdentifier === record.uniqueIdentifier
+                    )?.associationType
+                  : undefined,
+            }))
+          ),
+          catchError(() => of([]))
+        )
+      )
+    ).pipe(map((groups) => groups.flat()))
   }
 
   aggregate(params: AggregationsParams): Observable<Aggregations> {

package/src/libs/common/domain/src/lib/model/record/metadata.model.ts

@@ -260,12 +260,50 @@ export interface SourceRecord {
   href?: string
 }
 
+marker('domain.record.associationType.crossReference')
+marker('domain.record.associationType.largerWorkCitation')
+marker('domain.record.associationType.partOfSeamlessDatabase')
+marker('domain.record.associationType.stereoMate')
+marker('domain.record.associationType.isComposedOf')
+marker('domain.record.associationType.collectiveTitle')
+marker('domain.record.associationType.series')
+marker('domain.record.associationType.dependency')
+marker('domain.record.associationType.revisionOf')
+
+// DS_AssociationTypeCode values of the ISO19115-3 codelist, any unrecognized value
+// being mapped to 'crossReference'
+export const associationTypeValues = [
+  'crossReference',
+  'largerWorkCitation',
+  'partOfSeamlessDatabase',
+  'stereoMate',
+  'isComposedOf',
+  'collectiveTitle',
+  'series',
+  'dependency',
+  'revisionOf',
+] as const
+
+export type AssociationType = (typeof associationTypeValues)[number]
+
 /**
  * Represents another record associated with this one (e.g. a parent/sibling dataset).
  */
 export interface AssociatedRecord {
-  uuid: string
-  associationType: string
+  uniqueIdentifier: string
+  associationType: AssociationType
+}
+
+/**
+ * Nature of the link between a record and another one; the association type is
+ * only known for 'sibling' relations, as it is held by the declaring record.
+ */
+export type RecordRelation = 'source' | 'sourceOf' | 'sibling' | 'associated'
+
+export interface LinkedRecord {
+  record: CatalogRecord
+  relation: RecordRelation
+  associationType?: AssociationType
 }
 
 export interface DatasetRecord extends BaseRecord {
@@ -278,6 +316,7 @@ export interface DatasetRecord extends BaseRecord {
   spatialExtents: Array<DatasetSpatialExtent>
   temporalExtents: Array<DatasetTemporalExtent>
   spatialRepresentation?: SpatialRepresentationType
+  featureTypeDescriptions?: Array<DatasetFeatureType>
 }
 
 export interface ServiceEndpoint {
@@ -309,33 +348,28 @@ export interface ReuseRecord extends BaseRecord {
   temporalExtents: Array<DatasetTemporalExtent>
 }
 
+export interface DatasetFeatureType {
+  name: string
+  label?: string
+  description?: string
+  attributes: DatasetFeatureAttribute[]
+}
 export type DatasetFeatureCatalog = {
-  featureTypes: Array<{
-    name: string
-    definition: string
-    attributes: DatasetFeatureAttribute[]
-  }>
+  featureTypes: Array<DatasetFeatureType>
 }
 export interface DatasetFeatureAttributeValue {
-  code?: string
-  definition?: string
+  value: string // empty string when the value has no code
   label?: string
+  description?: string
 }
 export interface DatasetFeatureAttribute {
   name: string
-  definition: string
-  type: string
-  code: string
+  label?: string
+  description?: string
+  type?: string
+  code?: string
   values?: Array<DatasetFeatureAttributeValue>
 }
-export interface DatasetFeatureType {
-  aliases: string
-  code: string
-  isAbstract: string
-  typeName: string
-  definition: string
-  attributeTable: Array<DatasetFeatureAttribute>
-}
 
 export type ReuseType = 'application' | 'map' | 'other'
 

package/src/libs/common/domain/src/lib/model/search/aggregation.model.ts

@@ -7,6 +7,8 @@ export interface TermsAggregationParams {
   limit: number
   sort: AggregationSort
   filter?: string
+  includeValues?: string[]
+  excludeValues?: string[]
 }
 export interface HistogramAggregationParams {
   type: 'histogram'

package/src/libs/common/domain/src/lib/model/search/filter.model.ts

@@ -6,17 +6,39 @@ export type FieldFilterByRange = {
   start?: Date
   end?: Date
 }
+// matches util/shared's BoundingBox; duplicated here to avoid a circular
+// dependency (util/shared already depends on common/domain)
+export type FieldFilterByBoundingBox = [number, number, number, number]
 
 export type FieldFilter =
   | FieldFilterByExpression
   | FieldFilterByValues
   | FieldFilterByRange
+  | FieldFilterByBoundingBox
 export type FieldFilters = Record<FieldName, FieldFilter>
 
 export type QueryString = {
   query_string: string
 }
 export type QueryRange = {
-  range: Record<string, { format: string; gte: string; lte: string }>
+  range: Record<
+    string,
+    { format: string; gte?: string; lte?: string; relation?: string }
+  >
 }
-export type FilterQuery = Array<QueryString | QueryRange>
+export type QueryGeoShape = {
+  geo_shape: {
+    geom: {
+      shape: { type: string; coordinates: number[][] }
+      relation: string
+    }
+  }
+}
+export type QueryBool = {
+  bool: {
+    filter: QueryRange[]
+  }
+}
+export type FilterQuery = Array<
+  QueryString | QueryRange | QueryGeoShape | QueryBool
+>

package/src/libs/common/domain/src/lib/model/search/sort-by.model.ts

@@ -6,9 +6,5 @@ export const SortByEnum: Record<string, SortByField> = {
   RELEVANCY: ['desc', '_score'],
   QUALITY_SCORE: ['desc', 'qualityScore'],
   CHANGE_DATE: ['desc', 'changeDate'],
-  RESOURCE_DATES: [
-    ['desc', 'revisionDateForResource'],
-    ['desc', 'publicationDateForResource'],
-    ['desc', 'creationDateForResource'],
-  ],
+  RESOURCE_DATE: ['desc', 'resourceDate.date'],
 }

package/src/libs/common/domain/src/lib/repository/records-repository.interface.ts

@@ -10,21 +10,21 @@ import {
   CatalogRecord,
   DatasetFeatureCatalog,
   LanguageCode,
+  LinkedRecord,
 } from '../model/record'
 
 export abstract class RecordsRepositoryInterface {
   abstract search(params: SearchParams): Observable<SearchResults>
   abstract getMatchesCount(filters: FieldFilters): Observable<number>
   abstract getRecord(uniqueIdentifier: string): Observable<CatalogRecord | null>
-  abstract getFeatureCatalog(
+  abstract queryLinkedFeatureCatalog(
     record: CatalogRecord
   ): Observable<DatasetFeatureCatalog | null>
   abstract aggregate(params: AggregationsParams): Observable<Aggregations>
   abstract getSimilarRecords(
     similarTo: CatalogRecord
   ): Observable<CatalogRecord[]>
-  abstract getSources(record: CatalogRecord): Observable<CatalogRecord[]>
-  abstract getSourceOf(record: CatalogRecord): Observable<CatalogRecord[]>
+  abstract getLinkedRecords(record: CatalogRecord): Observable<LinkedRecord[]>
   abstract fuzzySearch(query: string): Observable<SearchResults>
   abstract canDuplicate(record: CatalogRecord): boolean
   abstract canDelete(record: CatalogRecord): Observable<boolean>