@kalisio/kdk 2.4.0 → 2.5.1

package/map/api/hooks/hooks.catalog.js

@@ -2,51 +2,54 @@ import _ from 'lodash'
 import siftModule from 'sift'
 import common from 'feathers-hooks-common'
 import makeDebug from 'debug'
+import { toString, toJson } from '../../../core/api/hooks/index.js'
 
 const { getItems, replaceItems } = common
 const sift = siftModule.default
 const debug = makeDebug('kdk:map:catalog:hooks')
 
+function isQueryForType(query, type) {
+  // Use sift to support MongoDB operators like $in, $nin, etc.
+  const filter = [{ type }].filter(sift(_.pick(query, ['type'])))
+  return !_.isEmpty(filter)
+}
+
+function addDefaultItems(query, type, items, defaultItems) {
+  // Add implicit type as not provided in default items config
+  defaultItems = _.map(defaultItems, item => Object.assign(item, { type }))
+  // Then filter according to query
+  defaultItems = defaultItems.filter(sift(_.omit(query, ['$sort', '$limit', '$skip'])))
+  return items.concat(defaultItems.map(item => Object.assign(item, { type })))
+}
+
 // By default we only return default categories
 export function getDefaultCategories (hook) {
   const query = _.get(hook, 'params.query', {})
-  if ((query.type === 'Category') && (query.$limit !== 0)) {
+  const type = 'Category'
+  if ((query.$limit !== 0) && isQueryForType(query, type)) {
     const service = hook.service
-    // Read default category config
-    const catalogConfig = hook.app.get('catalog') || { categories: [] }
-    // Check for specific service override (e.g. contextual catalog different from global catalog)
-    let defaultCategories = _.get(service, 'options.categories', catalogConfig.categories)
-    // Add implicit type
-    defaultCategories = _.map(defaultCategories, category => Object.assign(category, { type: 'Category' }))
-    // Then filter according to query
-    defaultCategories = defaultCategories.filter(sift(_.omit(query, ['$sort', '$limit', '$skip'])))
-    const item = getItems(hook)
-    replaceItems(hook, item.concat(defaultCategories.map(category => Object.assign(category, { type: 'Category' }))))
+    // Check for default categories config
+    const defaultCategories = _.get(service, 'options.categories', [])
+    replaceItems(hook, addDefaultItems(query, type, getItems(hook), defaultCategories))
   }
 }
 
 // By default we only return default sublegends
 export function getDefaultSublegends (hook) {
   const query = _.get(hook, 'params.query', {})
-  if ((query.type === 'Sublegend') && (query.$limit !== 0)) {
+  const type = 'Sublegend'
+  if ((query.$limit !== 0) && isQueryForType(query, type)) {
     const service = hook.service
-    // Read default sublegends config
-    const catalogConfig = hook.app.get('catalog') || { sublegends: [] }
-    // Check for specific service override (e.g. contextual catalog different from global catalog)
-    let defaultSublegends = _.get(service, 'options.sublegends', catalogConfig.sublegends)
-    // Add implicit type
-    defaultSublegends = _.map(defaultSublegends, sublegend => Object.assign(sublegend, { type: 'Sublegend' }))
-    // Then filter according to query
-    defaultSublegends = defaultSublegends.filter(sift(_.omit(query, ['$sort', '$limit', '$skip'])))
-    const item = getItems(hook)
-    replaceItems(hook, item.concat(defaultSublegends.map(sublegend => Object.assign(sublegend, { type: 'Sublegend' }))))
+    // Check for default sublegends config
+    const defaultSublegends = _.get(service, 'options.sublegends', [])
+    replaceItems(hook, addDefaultItems(query, type, getItems(hook), defaultSublegends))
   }
 }
 
 // By default we only return layers and not other objects in catalog
 export function filterLayers (hook) {
   const query = _.get(hook, 'params.query', {})
-  if (!query.type) query.type = { $nin: ['Context', 'Service', 'Category'] }
+  if (!query.type) query.type = { $nin: ['Context', 'Service', 'Category', 'Sublegend'] }
   _.set(hook, 'params.query', query)
 }
 
@@ -122,3 +125,33 @@ export async function updateProjects (hook) {
   }
   return hook
 }
+
+export function convertFilterQueriesToString (hook) {
+  let items = getItems(hook)
+  const isArray = Array.isArray(items)
+  items = (isArray ? items : [items])
+  items.forEach(item => {
+    const filters = _.get(item, 'filters', [])
+    _.forEach(filters, filter => {
+      toString(filter, ['active', 'inactive'])
+    })
+  })
+  replaceItems(hook, isArray ? items : items[0])
+
+  return hook
+}
+
+export function convertFilterQueriesToObject (hook) {
+  let items = getItems(hook)
+  const isArray = Array.isArray(items)
+  items = (isArray ? items : [items])
+  items.forEach(item => {
+    const filters = _.get(item, 'filters', [])
+    _.forEach(filters, filter => {
+      toJson(filter, ['active', 'inactive'])
+    })
+  })
+  replaceItems(hook, isArray ? items : items[0])
+
+  return hook
+}

package/map/api/hooks/hooks.query.js

@@ -137,7 +137,7 @@ export function asGeoJson (options = {}) {
     const params = hook.params
     const query = params.query
     if (!options.force && !params.asGeoJson) return
-    if (query.$distinct || query.$aggregation) return // Not applicable in this case
+    if (_.has(query, '$distinct') || _.has(query, '$aggregation')) return // Not applicable in this case
     const longitudeProperty = (options.longitudeProperty || 'longitude')
     const latitudeProperty = (options.latitudeProperty || 'latitude')
     const altitudeProperty = (options.altitudeProperty || 'altitude')
@@ -303,6 +303,8 @@ export async function aggregateFeaturesQuery (hook) {
     // Ensure we do not mix results with/without relevant element values
     // by separately querying each element then merging
     let aggregatedResults
+    // Associative map used to optimize merging between aggregated elements
+    const aggregatedResultsMap = new Map()
     const aggregateOptions = {
       allowDiskUse: true
     }
@@ -338,29 +340,10 @@ export async function aggregateFeaturesQuery (hook) {
       pipeline.forEach(stage => {
         _.forOwn(stage, (value, key) => debug('Stage', key, value))
       })
-      // Provide a hint to the aggregation targeting feature ID and aggregation elements.
-      // The problem with the aggregation hint option is that it should correspond
-      // exactly to an existing index otherwise it raises an error.
-      // We use a convention to get the order right: geometry => feature ID => aggregated element => time.
-      // We check anyway if the index does exist to avoid any error
-      // FIXME: Instead of assuming the appropriate index is defined in the right order,
-      // we might select the "best" available index (ie having the most similarities with the required one).
-      const hint = {}
-      if (isGeometry || query.$geoNear || match.geometry) Object.assign(hint, { geometry: '2dsphere' })
-      featureId.forEach(id => {
-        hint['properties.' + id] = 1
-      })
-      Object.assign(hint, { ['properties.' + element]: 1 })
-      // Use provided sort time option if any
-      hint.time = _.get(query, '$sort.time', 1)
-      const hintIndexName = _.reduce(hint, (name, value, key) => name ? `${name}_${key}_${value}` : `${key}_${value}`, '')
-      debug('Best aggregation hint index found', hintIndexName)
-      const hintIndex = _.find(indexes, { name: hintIndexName })
       const aggregateElementOptions = Object.assign({}, aggregateOptions)
-      if (hintIndex) aggregateElementOptions.hint = hintIndexName
       debug('Aggregation options', aggregateElementOptions)
       const elementResults = await collection.aggregate(pipeline, aggregateElementOptions).toArray()
-      debug(`Generated ${elementResults.length} feature(s) for ${element} element`, elementResults)
+      debug(`Generated ${elementResults.length} feature(s) for ${element} element, picked first two`, elementResults.slice(0,2))
       // Rearrange data so that we get ordered arrays indexed by element
       elementResults.forEach(result => {
         result.time = { [element]: result.time }
@@ -372,17 +355,26 @@ export async function aggregateFeaturesQuery (hook) {
           // Delete accumulator
           _.unset(result, element)
         }
+        // Keep track of result in map to improve search later
+        const resultKeys = keys.map(key => _.get(singleTime ? result : result._id, key))
+        const resultKey = resultKeys.join('-')
+        if (!aggregatedResultsMap.has(resultKey)) aggregatedResultsMap.set(resultKey, result)
       })
       // Now merge with previous element results
       if (!aggregatedResults) {
         aggregatedResults = elementResults
       } else {
         elementResults.forEach(result => {
-          const resultKeys = _.pick(result, keys)
+          // When single time no aggregation is performed at all so we only have raw features
+          const resultKeys = keys.map(key => _.get(singleTime ? result : result._id, key))
+          /* Optimize previous result search with map, kept the naive code here for reference/debug purpose
           const previousResult = aggregatedResults.find(aggregatedResult => {
-            const aggregatedResultKeys = _.pick(aggregatedResult, keys)
+            const aggregatedResultKeys = keys.map(key => _.get(singleTime ? result : result._id, key))
             return _.isEqual(aggregatedResultKeys, resultKeys)
           })
+          */
+          const resultKey = resultKeys.join('-')
+          const previousResult = aggregatedResultsMap.get(resultKey)
           // Merge with previous matching feature if any
           if (previousResult) {
             Object.assign(previousResult.time, result.time)
@@ -402,7 +394,7 @@ export async function aggregateFeaturesQuery (hook) {
     delete query.$aggregate
     delete query.$geoNear
     // Set result to avoid service DB call
-    hook.result = aggregatedResults
+    hook.result = aggregatedResults || []
   }
   return hook
 }

package/map/api/models/catalog.model.mongodb.js

@@ -1,3 +1,5 @@
+import _ from 'lodash'
+
 export default function (app, options) {
   const db = options.db || app.db
   options.Model = db.collection('catalog')
@@ -9,6 +11,18 @@ export default function (app, options) {
   options.Model.indexExists('name_1_type_1').then(() => options.Model.dropIndex('name_1_type_1')).catch(() => {})
   // Collation provided in query ensure sorting to be case insensitive w.r.t. user's language
   // We built indices with collation to cover the most used languages, it requires different naming...
-  options.Model.createIndex({ name: 1, type: 1 }, { unique: true, name: 'name-type-en', collation: { locale: 'en', strength: 1 } })
-  options.Model.createIndex({ name: 1, type: 1 }, { unique: true, name: 'name-type-fr', collation: { locale: 'fr', strength: 1 } })
+  const languages = ['en', 'fr']
+  options.Model.indexInformation({ full: true }).then(indexes => {
+    for (let i = 0; i < languages.length; i++) {
+      const language = languages[i]
+      const name = `name-type-${language}`
+      const createIndex = () => {
+        return options.Model.createIndex({ name: 1, type: 1 }, { name, collation: { locale: language, strength: 1 } })
+      }
+      // We also previously had a unique constraint that we have relaxed
+      let uniqueIndex = _.find(indexes, { name, unique: true })
+      if (uniqueIndex) options.Model.dropIndex(name).then(createIndex).catch(() => {})
+      else createIndex()
+    }
+  })
 }

package/{core/api/models/groups.model.mongodb.js → map/api/models/styles.model.mongodb.js}

@@ -1,8 +1,9 @@
 export default function (app, options) {
   const db = options.db || app.db
-  options.Model = db.collection('groups')
+  options.Model = db.collection('styles')
   // Collation provided in query ensure sorting to be case insensitive w.r.t. user's language
   // We built indices with collation to cover the most used languages, it requires different naming...
   options.Model.createIndex({ name: 1 }, { name: 'name-en', collation: { locale: 'en', strength: 1 } })
   options.Model.createIndex({ name: 1 }, { name: 'name-fr', collation: { locale: 'fr', strength: 1 } })
+  options.Model.createIndex({ name: 'text' })
 }

package/map/api/services/catalog/catalog.hooks.js

@@ -2,7 +2,9 @@ import _ from 'lodash'
 import common from 'feathers-hooks-common'
 import fuzzySearch from 'feathers-mongodb-fuzzy-search'
 import { hooks as coreHooks } from '../../../../core/api/index.js'
-import { filterLayers, updateLayerReferences, updateProjects, getDefaultCategories, getDefaultSublegends } from '../../hooks/index.js'
+import { filterLayers, updateLayerReferences, updateProjects,
+         getDefaultCategories, getDefaultSublegends, 
+         convertFilterQueriesToString, convertFilterQueriesToObject } from '../../hooks/index.js'
 
 const { setNow, discard, when } = common
 
@@ -15,7 +17,8 @@ export default {
     get: [],
     create: [
       coreHooks.checkUnique({ field: 'name', query: (query, hook) => { query.type = _.get(hook, 'data.type') } }),
-      coreHooks.convertToString(['baseQuery', 'schema.content', 'filters']),
+      coreHooks.convertToString(['baseQuery', 'schema.content']),
+      convertFilterQueriesToString,
       setNow('createdAt', 'updatedAt'),
       // This allow to use keys for base queries like 'properties.xxx': 'yyy'
       (hook) => { _.set(hook, 'params.mongodb.checkKeys', false) }
@@ -23,14 +26,16 @@ export default {
     update: [
       coreHooks.populatePreviousObject,
       coreHooks.checkUnique({ field: 'name', query: (query, hook) => { query.type = _.get(hook, 'params.previousItem.type') } }),
-      coreHooks.convertToString(['baseQuery', 'schema.content', 'filters']),
+      coreHooks.convertToString(['baseQuery', 'schema.content']),
+      convertFilterQueriesToString,
       discard('createdAt', 'updatedAt'),
       setNow('updatedAt')
     ],
     patch: [
       coreHooks.populatePreviousObject,
       coreHooks.checkUnique({ field: 'name', query: (query, hook) => { query.type = _.get(hook, 'params.previousItem.type') } }),
-      coreHooks.convertToString(['baseQuery', 'schema.content', 'filters']),
+      coreHooks.convertToString(['baseQuery', 'schema.content']),
+      convertFilterQueriesToString,
       discard('createdAt', 'updatedAt'),
       setNow('updatedAt')
     ],
@@ -41,12 +46,13 @@ export default {
 
   after: {
     all: [
-      coreHooks.convertToJson(['baseQuery', 'schema.content', 'filters']),
+      coreHooks.convertToJson(['baseQuery', 'schema.content']),
+      convertFilterQueriesToObject,
       coreHooks.convertObjectIDs(['baseQuery.layer'])
     ],
     find: [
-      // Merge built-in categoriessublegends with user-defined ones for global catalog only
-      when(hook => !hook.service.getContextId(), getDefaultCategories, getDefaultSublegends)
+      // Merge built-in categories/sublegends (if provided) with user-defined ones
+      getDefaultCategories, getDefaultSublegends
     ],
     get: [],
     create: [],

package/map/api/services/features/features.hooks.js

@@ -1,10 +1,13 @@
 import _ from 'lodash'
+import common from 'feathers-hooks-common'
 import { hooks as coreHooks } from '../../../../core/api/index.js'
 import {
   marshallSpatialQuery, aggregateFeaturesQuery, asGeoJson,
   simplifyResult, simplifyEvents, skipEvents, fuzzySearch
 } from '../../hooks/index.js'
 
+const { getItems, replaceItems } = common
+
 // Allow to control real-time events emission if required
 const emitEvents = (hook) => {
   _.set(hook, 'params.emitEvents', _.get(hook, 'params.query.emitEvents'))
@@ -15,6 +18,25 @@ const fullResult = (hook) => {
   _.set(hook, 'params.fullResult', _.get(hook, 'params.query.fullResult'))
   _.unset(hook, 'params.query.fullResult')
 }
+// Allow upsert if required
+const upsert = (hook) => {
+  _.set(hook, 'params.mongodb', { upsert: _.get(hook, 'params.query.upsert', false) })
+  _.unset(hook, 'params.query.upsert')
+}
+// Feathers does not return anything in this case
+const upsertResult = async (hook) => {
+  if (_.get(hook, 'params.mongodb.upsert', false)) {
+    let items = getItems(hook)
+    const isArray = Array.isArray(items)
+    items = (isArray ? items : [items])
+    const collection = await hook.service.find({
+      query: _.get(hook, 'params.query', {}),
+      paginate: false
+    })
+    replaceItems(hook, isArray ? collection.features : collection.features[0])
+  }
+  return hook
+}
 
 export default {
   before: {
@@ -23,9 +45,10 @@ export default {
       coreHooks.distinct, aggregateFeaturesQuery, coreHooks.aggregationQuery, fuzzySearch, coreHooks.diacriticSearch()],
     get: [],
     create: [coreHooks.processTimes(['time']), fullResult, emitEvents],
-    update: [coreHooks.processTimes(['time']), fullResult, emitEvents],
-    patch: [coreHooks.processTimes(['time']), fullResult, emitEvents],
-    remove: [coreHooks.marshallComparisonQuery, marshallSpatialQuery, fullResult, emitEvents]
+    update: [upsert, coreHooks.processTimes(['time']), fullResult, emitEvents],
+    patch: [upsert, coreHooks.processTimes(['time']), fullResult, emitEvents],
+    remove: [coreHooks.marshallComparisonQuery, marshallSpatialQuery, fullResult, emitEvents],
+    formatGeoJSON: []
   },
 
   after: {
@@ -35,8 +58,8 @@ export default {
     // By default avoid emitting events and full result on feature edition because we usually proceed with big batches
     // On a per-case basis each application can activate all events/results if required or define a more evolved strategy
     create: [coreHooks.unprocessTimes(['time']), simplifyEvents, simplifyResult, skipEvents],
-    update: [coreHooks.unprocessTimes(['time']), simplifyEvents, simplifyResult, skipEvents],
-    patch: [coreHooks.unprocessTimes(['time']), simplifyEvents, simplifyResult, skipEvents],
+    update: [coreHooks.unprocessTimes(['time']), upsertResult, simplifyEvents, simplifyResult, skipEvents],
+    patch: [coreHooks.unprocessTimes(['time']), upsertResult, simplifyEvents, simplifyResult, skipEvents],
     remove: [simplifyEvents, simplifyResult, skipEvents]
   },
 

package/map/api/services/features/features.service.js

@@ -47,6 +47,107 @@ export default {
     // Result is like [{ _id: { hour: 16, dayOfWeek: 1 }, count: 2 }, { _id: { hour: 18, dayOfWeek: 1 }, count: 4 }]
     // but we make it easier to read like this [{ hour: 16, dayOfWeek: 1, count: 2 }, { hour: 18, dayOfWeek: 1, count: 4 }]
     return results.map(result => Object.assign(result._id, { count: result.count }))
+  },
+
+  async formatGeoJSON(data, params) {
+    let { query, sort, type } = data
+    marshallComparisonFields(query)
+    const pipeline = []
+    const collection = this.Model
+
+    if (query) pipeline.push({ $match: query })
+    if (sort) pipeline.push({ $sort: sort })
+    switch (type) {
+      // Simple line string from a collection of points
+      case 'LineString':
+        pipeline.push(
+          // Match stage: Filter for documents that have a "Point" geometry type, which is required for a LineString.
+          {
+            $match: {
+              "geometry.type": "Point"
+            }
+          },
+
+          // Group stage: Group all documents into a single document, aggregating coordinates and properties.
+          {
+            $group: {
+              _id: null,
+              allCoordinates: {
+                $push: "$geometry.coordinates"
+              }
+            }
+          },
+
+          // Project stage: Construct the GeoJSON LineString and add properties.
+          {
+            $project: {
+              _id: 0,
+              type: { $literal: "Feature" },
+              geometry: {
+                type: { $literal: "LineString" },
+                coordinates: "$allCoordinates"
+              },
+              properties: {
+                name: "Generated LineString"
+              }
+            }
+          }
+        );
+        break;
+
+      // a feature collection of points and a linestring connecting them
+      case 'PointedLineString':
+        pipeline.push({
+          $match: {
+            "geometry.type": "Point"
+          }
+        },
+          {
+            // Group stage: Collect all documents of type "Point" into an array and all coordinates for the LineString.
+            $group: {
+              _id: null,
+              features: {
+                $push: "$$ROOT"  // Push entire documents of "Point" features.
+              },
+              allCoordinates: {
+                $push: "$geometry.coordinates"  // Collect coordinates for the LineString.
+              }
+            }
+          },
+          {
+            // Project stage: Create a FeatureCollection including both points and a LineString.
+            $project: {
+              _id: 0,
+              type: { "$literal": "FeatureCollection" },
+              features: {
+                $concatArrays: [
+                  "$features",  // Include original Point features.
+                  [
+                    {
+                      type: "Feature",
+                      geometry: {
+                        type: "LineString",
+                        coordinates: "$allCoordinates"  // Use all coordinates to create a LineString.
+                      },
+                      properties: {
+                        name: "Generated LineString"
+                      }
+                    }
+                  ]
+                ]
+              }
+            }
+          });
+
+
+
+        break
+      default:
+        throw new Error('Unknown type : ' + type)
+    }
+    console.log('Executing formatGeoJSON pipeline', JSON.stringify(pipeline))
+    const results = await collection.aggregate(pipeline).toArray()
+    return results
   }
 
 }