@data-fair/catalog-onegeo-suite 0.1.1 → 0.1.3

package/lib/imports.ts

@@ -154,7 +154,7 @@ export const getResource = async ({ catalogConfig, importConfig, resourceId, tmp
     },
     keywords: catalog._source['metadata-fr'].keyword,
     updatedAt: catalog._source['metadata-fr'].lastUpdateDate ?? undefined,
-    image: catalog._source['metadata-fr'].image.find((x: { type: string, url: string | null }) => { return x.type === 'thumbnail' && !!x.url }).url ?? null,
+    image: catalog._source['metadata-fr'].image.find((x: { type: string, url: string | null }) => { return x.type === 'thumbnail' && !!x.url })?.url ?? null,
     origin
   }
 }

package/lib/list.ts

@@ -10,13 +10,28 @@ const formatsList = [
   'CSV', 'ODS', 'Excel non structuré', 'Microsoft Excel',
   'ZIP', 'Shapefile (zip)', 'SHAPE-ZIP', 'GeoJSON', 'JSON', 'XML', 'GML', 'KML']
 
+const extensionTable: Record<string, string> = {
+  CSV: '.csv',
+  GeoJSON: '.geojson',
+  JSON: '.json',
+  'Shapefile (zip)': '.zip',
+  'SHAPE-ZIP': '.zip',
+  ZIP: '.zip',
+  GML: '.gml',
+  KML: '.kml',
+  XML: '.xml',
+  ODS: '.ods',
+  'Excel non structuré': '.xlsx',
+  'Microsoft Excel': '.xls',
+}
+
 const getBestFormat = (formats: string[]) => {
   return [...formats].sort((a, b) =>
     (formatsList.indexOf(a) === -1 ? formatsList.length : formatsList.indexOf(a)) -
     (formatsList.indexOf(b) === -1 ? formatsList.length : formatsList.indexOf(b))
   )
 }
-const baseReqDataset = (input: string = '*', size: number = 50, from: number = 1) => {
+const baseReqDataset = (input: string = '*', size: number = 100, from: number = 1) => {
   return {
     from: (from - 1) * size,
     size: Math.min(size, 10000),
@@ -55,12 +70,64 @@ const baseReqDataset = (input: string = '*', size: number = 50, from: number = 1
   }
 }
 
+const countReq = (input: string = '*') => {
+  return {
+    size: 0,
+    track_total_hits:
+    false,
+    query:
+    {
+      bool: {
+        must: [{
+          bool: {
+            should: [{
+              query_string: {
+                query: input || '*',
+                fields: ['data_and_metadata', 'metadata-fr.title^5', 'metadata-fr.abstract^3', 'content-fr.title^5', 'content-fr.excerpt^3', 'content-fr.plaintext'],
+                analyzer: 'my_search_analyzer',
+                fuzziness: 'AUTO',
+                minimum_should_match: '90%',
+                default_operator: 'AND',
+                boost: 5
+              }
+            }]
+          }
+        }, { term: { is_metadata: true } }, {
+          bool: {
+            should: [
+              ...apiList.filter((x: any) => x).map((x: any) => ({ term: { 'metadata-fr.link.service.keyword': x } })),
+              ...formatsList.filter((x: any) => x).map((x: any) => ({ term: { 'metadata-fr.link.formats.keyword': x } }))
+            ],
+          }
+        }],
+        must_not:
+        [{ term: { 'content-fr.status.keyword': 'draft' } }],
+        filter:
+        {
+          terms: {
+            'type.keyword':
+            ['dataset', 'nonGeographicDataset']
+          }
+        }
+      }
+    },
+    aggs: {
+      unique_datasets: {
+        cardinality: {
+          field: 'uuid.keyword',
+          precision_threshold:
+          40000
+        }
+      }
+    }
+  }
+}
+
 export const list = async ({ catalogConfig, params }: ListContext<OneGeoSuiteConfig, OneGeoCapabilities>): ReturnType<CatalogPlugin['list']> => {
   const url = catalogConfig.url
   const listResources = async (params: Record<any, any>) => {
-    let catalogs = (await axios.post(new URL('fr/indexer/elastic/_search/', url).href, baseReqDataset(params.q, params.size, params.page))).data
-    const count = catalogs.hits.total.value
-    catalogs = catalogs.hits.hits
+    const catalogs = (await axios.post(new URL('fr/indexer/elastic/_search/', url).href, baseReqDataset(params.q || '*', params.size, params.page))).data.hits.hits
+    const count = (await axios.post(new URL('fr/indexer/elastic/_search/', 'https://www.datasud.fr').href, countReq(params.q))).data.aggregations.unique_datasets.value
     const res = []
 
     for (const catalog of catalogs) {
@@ -78,11 +145,21 @@ export const list = async ({ catalogConfig, params }: ListContext<OneGeoSuiteCon
         return apiList.indexOf(x.service) - apiList.indexOf(y.service)
       })
 
+      const formatsSet: Set<string> = new Set()
+
+      for (const source of sources) {
+        for (const format of source.formats) {
+          if (formatsList.includes(format)) formatsSet.add(format)
+        }
+      }
+      let formats = (new Array(...formatsSet)).sort((a: string, b: string) => { return formatsList.indexOf(a) - formatsList.indexOf(b) })
+      formats = formats.map(x => extensionTable[x]?.slice(1) ?? x)
+
       res.push({
         id: `${catalog._id}`,
         title: catalog._source['metadata-fr'].title,
         description: sources[0].description,
-        format: '',
+        format: formats.slice(0, 3).join(', ') + (formats.length ? '..' : ''),
         type: 'resource'
       } as ResourceList[number])
     }
@@ -90,7 +167,6 @@ export const list = async ({ catalogConfig, params }: ListContext<OneGeoSuiteCon
   }
   // List datasets
   const [resources, count] = await listResources(params)
-
   return {
     count,
     results: resources,

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@data-fair/catalog-onegeo-suite",
   "description": "OneGeoSuite plugin for the Data Fair catalogs service.",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "main": "index.ts",
   "type": "module",
   "scripts": {

package/types/importConfig/.type/index.js

@@ -32,7 +32,7 @@ export const schema = {
       "layout": {
         "cols": 6,
         "getItems": {
-          "url": "${context.catalogConfig}/fr/indexer/elastic/_search/?q=_id:${context.resourceId}",
+          "url": "${context.catalogConfig.url}/fr/indexer/elastic/_search/?q=_id:${context.resourceId}",
           "itemsResults": "(function() { const links = data.hits.hits[0]._source['metadata-fr'].link; return links.filter(l => l.formats.find(f => ['CSV', 'ODS', 'Excel non structuré', 'Microsoft Excel','ZIP', 'Shapefile (zip)', 'GeoJSON', 'JSON', 'XML', 'GML', 'KML'].includes(f)) || ['WS', 'AFS', 'WFS'].includes(l.service)).map((l, i) =>  [l, i] );  })()",
           "itemTitle": "item[0].service?? 'Local N°' + (item[1] + 1)",
           "itemValue": "item[0].service??  item[0].url"
@@ -46,7 +46,7 @@ export const schema = {
         "if": "parent.data.service != null",
         "cols": 6,
         "getItems": {
-          "url": "${context.catalogConfig}/fr/indexer/elastic/_search/?q=${parent.data.service}&q=_id:${context.resourceId}",
+          "url": "${context.catalogConfig.url}/fr/indexer/elastic/_search/?q=${parent.data.service}&q=_id:${context.resourceId}",
           "itemsResults": "(function(){const service = data.hits.hits[0]._source['metadata-fr'].link.find(x => x.service === parent.data.service || x.url === parent.data.service); let formats = service.formats; if (['WS', 'WFS', 'AFS'].includes(service.service) && !formats.includes('CSV')) {formats.push('CSV');} return formats; })()"
         }
       }

package/types/importConfig/schema.json

@@ -24,7 +24,7 @@
       "layout": {
         "cols": 6,
         "getItems": {
-          "url": "${context.catalogConfig}/fr/indexer/elastic/_search/?q=_id:${context.resourceId}",
+          "url": "${context.catalogConfig.url}/fr/indexer/elastic/_search/?q=_id:${context.resourceId}",
           "itemsResults": "(function() { const links = data.hits.hits[0]._source['metadata-fr'].link; return links.filter(l => l.formats.find(f => ['CSV', 'ODS', 'Excel non structuré', 'Microsoft Excel','ZIP', 'Shapefile (zip)', 'GeoJSON', 'JSON', 'XML', 'GML', 'KML'].includes(f)) || ['WS', 'AFS', 'WFS'].includes(l.service)).map((l, i) =>  [l, i] );  })()",
           "itemTitle": "item[0].service?? 'Local N°' + (item[1] + 1)",
           "itemValue": "item[0].service??  item[0].url"
@@ -38,7 +38,7 @@
         "if": "parent.data.service != null",
         "cols": 6,
         "getItems": {
-          "url": "${context.catalogConfig}/fr/indexer/elastic/_search/?q=${parent.data.service}&q=_id:${context.resourceId}",
+          "url": "${context.catalogConfig.url}/fr/indexer/elastic/_search/?q=${parent.data.service}&q=_id:${context.resourceId}",
           "itemsResults": "(function(){const service = data.hits.hits[0]._source['metadata-fr'].link.find(x => x.service === parent.data.service || x.url === parent.data.service); let formats = service.formats; if (['WS', 'WFS', 'AFS'].includes(service.service) && !formats.includes('CSV')) {formats.push('CSV');} return formats; })()"
         }
       }