@data-fair/catalog-onegeo-suite 0.1.3 → 0.1.6

package/lib/imports.ts

@@ -1,36 +1,44 @@
 import type { CatalogPlugin, GetResourceContext } from '@data-fair/types-catalogs'
 import type { OneGeoSuiteConfig, Link } from '#types'
+import { apiList, formatsList, sortList } from './list.ts'
 
 import axios from '@data-fair/lib-node/axios.js'
 
 export const getResource = async ({ catalogConfig, importConfig, resourceId, tmpDir, log }: GetResourceContext<OneGeoSuiteConfig>): ReturnType<CatalogPlugin['getResource']> => {
-  const format: string = importConfig.format
-  const service: string = importConfig.service
-  const catalog = (await axios.get(new URL(`fr/indexer/elastic/_search/?q=_id:${resourceId}`, catalogConfig.url).href)).data.hits.hits[0]
-
-  if (!catalog) {
-    throw Error(`resource ${service} not found for ${resourceId} in ${catalogConfig.url}`)
+  let service: string = importConfig.service
+  let format: string = service ? importConfig.format : (importConfig.format2 || undefined)
+  const catalog = (await axios.get(new URL(`fr/indexer/elastic/_search/?q=uuid.keyword:${resourceId}%20AND%20is_metadata:true`, catalogConfig.url).href)).data.hits.hits[0]
+  if (!format) {
+    if (!service) {
+      const links = catalog._source['metadata-fr'].link.filter((x: any) => { return apiList.includes(x.service) && x.formats.find((y: string) => { return formatsList.includes(y) }) })
+      service = sortList(links.map((x: Link) => x.service), apiList)[0]
+    }
+    format = sortList(catalog._source['metadata-fr'].link.find((x: Link) => { return (x.service === service || x.url === service) && x.formats.find((y: string) => { return formatsList.includes(y) }) }).formats, formatsList)[0]
+    if (!format) throw Error(`resource not found for service ${service}`)
+  } else {
+    if (!service) {
+      const links = catalog._source['metadata-fr'].link.filter((x: Link) => { return x.formats.includes(format) })
+      service = sortList(links.map((x: Link) => x.service), apiList)[0]
+      if (!service) throw Error(`resource not found for format ${format}`)
+    }
   }
 
+  if (!catalog) throw Error(`resource not found for ${resourceId} in ${catalogConfig.url}`)
+  if (!service) throw Error('resource not found')
+  if (!format) throw Error(`resource not found for service ${service}`)
+
   // filter links by format and service
   const source: Link = catalog._source['metadata-fr'].link.find((x: Link) => {
     return x.service === service || x.url === service
   })
-
-  // table of format for make AFS url
-  const afsTable: Record<string, string> = {
-    CSV: 'text/csv',
-    JSON: 'application/json',
-    GeoJSON: 'application/Geo%2Bjson',
-    GML: 'application/gml%2Bxml',
-    KML: 'application/vnd.google-earth.kml%2Bxml',
-  }
+  if (!source) throw Error('resource not found')
   // table of format for make WFS url
   const wfsTable: Record<string, string> = {
     CSV: 'csv',
+    JSON: 'application/json',
     GeoJSON: 'application/json',
     'Shapefile (zip)': 'SHAPE-ZIP',
-    GML: 'GML3',
+    'SHAPE-ZIP': 'SHAPE-ZIP',
     KML: 'kml',
   }
   // table of format
@@ -39,23 +47,20 @@ export const getResource = async ({ catalogConfig, importConfig, resourceId, tmp
     GeoJSON: '.geojson',
     JSON: '.json',
     'Shapefile (zip)': '.zip',
-    ZIP: '.zip',
-    GML: '.gml',
+    'SHAPE-ZIP': '.zip',
     KML: '.kml',
-    XML: '.xml',
-    ODS: '.ods',
     'Excel non structuré': '.xlsx',
     'Microsoft Excel': '.xls',
   }
 
   let downloadUrl: string
   if (source.service === 'WS') {
+    if (extensionTable[format] === undefined) throw Error(`Format ${format} not valid for ${service}`)
     downloadUrl = `${source.url}/${source.name}/all${extensionTable[format]}`
   } else if (source.service === undefined) {
     downloadUrl = `${source.url}`
-  } else if (source.service === 'AFS') {
-    downloadUrl = `${source.url}${source.name}/items?&crs=${source.projections![0]}&f=${afsTable[format]}&sortby=gid`
   } else if (source.service === 'WFS') {
+    if (wfsTable[format] === undefined) throw Error(`Format ${format} not valid for ${service}`)
     downloadUrl = `${source.url}?SERVICE=WFS&VERSION=2.0.0&request=GetFeature&typename=${source.name}&outputFormat=${wfsTable[format]}&startIndex=0&sortby=gid`
   } else {
     downloadUrl = `${source.url}`

package/lib/list.ts

@@ -5,10 +5,9 @@ import axios from '@data-fair/lib-node/axios.js'
 
 type ResourceList = Awaited<ReturnType<CatalogPlugin['list']>>['results']
 
-const apiList: Array<string | undefined> = ['WS', 'AFS', 'WFS', undefined]
-const formatsList = [
-  'CSV', 'ODS', 'Excel non structuré', 'Microsoft Excel',
-  'ZIP', 'Shapefile (zip)', 'SHAPE-ZIP', 'GeoJSON', 'JSON', 'XML', 'GML', 'KML']
+export const apiList: Array<string | undefined> = ['WS', 'WFS', undefined]
+export const formatsList = [
+  'CSV', 'Excel non structuré', 'Microsoft Excel', 'Shapefile (zip)', 'SHAPE-ZIP', 'GeoJSON', 'JSON', 'KML']
 
 const extensionTable: Record<string, string> = {
   CSV: '.csv',
@@ -16,22 +15,18 @@ const extensionTable: Record<string, string> = {
   JSON: '.json',
   'Shapefile (zip)': '.zip',
   'SHAPE-ZIP': '.zip',
-  ZIP: '.zip',
-  GML: '.gml',
   KML: '.kml',
-  XML: '.xml',
-  ODS: '.ods',
   'Excel non structuré': '.xlsx',
   'Microsoft Excel': '.xls',
 }
 
-const getBestFormat = (formats: string[]) => {
+export const sortList = (formats: any[], reference: any[]) => {
   return [...formats].sort((a, b) =>
-    (formatsList.indexOf(a) === -1 ? formatsList.length : formatsList.indexOf(a)) -
-    (formatsList.indexOf(b) === -1 ? formatsList.length : formatsList.indexOf(b))
+    (reference.indexOf(a) === -1 ? reference.length : reference.indexOf(a)) -
+    (reference.indexOf(b) === -1 ? reference.length : reference.indexOf(b))
   )
 }
-const baseReqDataset = (input: string = '*', size: number = 100, from: number = 1) => {
+const baseReqDataset = (input: string = '*', size: number = 500, from: number = 1) => {
   return {
     from: (from - 1) * size,
     size: Math.min(size, 10000),
@@ -52,7 +47,7 @@ const baseReqDataset = (input: string = '*', size: number = 100, from: number =
               }
             }]
           }
-        }, { term: { is_metadata: true } }, {
+        }, { term: { is_metadata: true } }, { term: { 'editorial-metadata.isOpenAccess': true } }, {
           bool: {
             should: [
               ...apiList.filter((x: any) => { return x }).map((x: any) => { return { term: { 'metadata-fr.link.service.keyword': x } } }),
@@ -92,7 +87,7 @@ const countReq = (input: string = '*') => {
               }
             }]
           }
-        }, { term: { is_metadata: true } }, {
+        }, { term: { is_metadata: true } }, { term: { 'editorial-metadata.isOpenAccess': true } }, {
           bool: {
             should: [
               ...apiList.filter((x: any) => x).map((x: any) => ({ term: { 'metadata-fr.link.service.keyword': x } })),
@@ -126,8 +121,14 @@ const countReq = (input: string = '*') => {
 export const list = async ({ catalogConfig, params }: ListContext<OneGeoSuiteConfig, OneGeoCapabilities>): ReturnType<CatalogPlugin['list']> => {
   const url = catalogConfig.url
   const listResources = async (params: Record<any, any>) => {
-    const catalogs = (await axios.post(new URL('fr/indexer/elastic/_search/', url).href, baseReqDataset(params.q || '*', params.size, params.page))).data.hits.hits
-    const count = (await axios.post(new URL('fr/indexer/elastic/_search/', 'https://www.datasud.fr').href, countReq(params.q))).data.aggregations.unique_datasets.value
+    let catalogs
+    try {
+      catalogs = (await axios.post(new URL('fr/indexer/elastic/_search/', url).href, baseReqDataset(params.q || '*', params.size, params.page))).data.hits.hits
+    } catch (e) {
+      // @ts-ignore
+      throw Error(`Axios error: ${e?.status ?? ''} ${e?.message}`)
+    }
+    const count = (await axios.post(new URL('fr/indexer/elastic/_search/', url).href, countReq(params.q))).data.aggregations.unique_datasets.value
     const res = []
 
     for (const catalog of catalogs) {
@@ -135,8 +136,8 @@ export const list = async ({ catalogConfig, params }: ListContext<OneGeoSuiteCon
 
       // sort source by priority (services / format)
       sources.sort((x: Link, y: Link) => {
-        const bestFormatX = formatsList.indexOf(getBestFormat(x.formats)[0]) === -1 ? formatsList.length : formatsList.indexOf(getBestFormat(x.formats)[0])
-        const bestFormatY = formatsList.indexOf(getBestFormat(y.formats)[0]) === -1 ? formatsList.length : formatsList.indexOf(getBestFormat(y.formats)[0])
+        const bestFormatX = formatsList.indexOf(sortList(x.formats, formatsList)[0]) === -1 ? formatsList.length : formatsList.indexOf(sortList(x.formats, formatsList)[0])
+        const bestFormatY = formatsList.indexOf(sortList(y.formats, formatsList)[0]) === -1 ? formatsList.length : formatsList.indexOf(sortList(y.formats, formatsList)[0])
 
         if (bestFormatX !== bestFormatY) {
           return bestFormatX - bestFormatY
@@ -156,7 +157,7 @@ export const list = async ({ catalogConfig, params }: ListContext<OneGeoSuiteCon
       formats = formats.map(x => extensionTable[x]?.slice(1) ?? x)
 
       res.push({
-        id: `${catalog._id}`,
+        id: `${catalog._source.uuid}`,
         title: catalog._source['metadata-fr'].title,
         description: sources[0].description,
         format: formats.slice(0, 3).join(', ') + (formats.length ? '..' : ''),

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@data-fair/catalog-onegeo-suite",
   "description": "OneGeoSuite plugin for the Data Fair catalogs service.",
-  "version": "0.1.3",
+  "version": "0.1.6",
   "main": "index.ts",
   "type": "module",
   "scripts": {

package/types/importConfig/.type/index.js

@@ -14,10 +14,7 @@ export const schema = {
   "title": "ImportConfig",
   "type": "object",
   "additionalProperties": false,
-  "required": [
-    "service",
-    "format"
-  ],
+  "required": [],
   "properties": {
     "service": {
       "type": "string",
@@ -32,8 +29,8 @@ export const schema = {
       "layout": {
         "cols": 6,
         "getItems": {
-          "url": "${context.catalogConfig.url}/fr/indexer/elastic/_search/?q=_id:${context.resourceId}",
-          "itemsResults": "(function() { const links = data.hits.hits[0]._source['metadata-fr'].link; return links.filter(l => l.formats.find(f => ['CSV', 'ODS', 'Excel non structuré', 'Microsoft Excel','ZIP', 'Shapefile (zip)', 'GeoJSON', 'JSON', 'XML', 'GML', 'KML'].includes(f)) || ['WS', 'AFS', 'WFS'].includes(l.service)).map((l, i) =>  [l, i] );  })()",
+          "url": "${context.catalogConfig.url}/fr/indexer/elastic/_search/?q=uuid.keyword:${context.resourceId}%20AND%20is_metadata:true",
+          "itemsResults": "(function() { const links = data.hits.hits[0]._source['metadata-fr'].link; return links.filter(l => l.formats.find(f => ['CSV', 'Excel non structuré', 'Microsoft Excel','ZIP', 'Shapefile (zip)', 'SHAPE-ZIP', 'GeoJSON', 'JSON', 'KML'].includes(f)) && ['WS', 'WFS', undefined].includes(l.service)).map((l, i) =>  [l, i] );  })()",
           "itemTitle": "item[0].service?? 'Local N°' + (item[1] + 1)",
           "itemValue": "item[0].service??  item[0].url"
         }
@@ -42,12 +39,32 @@ export const schema = {
     "format": {
       "type": "string",
       "title": "Format",
+      "description": "Import format, if no service is set, the possible formats are unknown.",
+      "x-i18n-description": {
+        "fr": "Format d’importation, si aucun service n’est défini, les formats possibles sont inconnus."
+      },
       "layout": {
         "if": "parent.data.service != null",
         "cols": 6,
         "getItems": {
-          "url": "${context.catalogConfig.url}/fr/indexer/elastic/_search/?q=${parent.data.service}&q=_id:${context.resourceId}",
-          "itemsResults": "(function(){const service = data.hits.hits[0]._source['metadata-fr'].link.find(x => x.service === parent.data.service || x.url === parent.data.service); let formats = service.formats; if (['WS', 'WFS', 'AFS'].includes(service.service) && !formats.includes('CSV')) {formats.push('CSV');} return formats; })()"
+          "url": "${context.catalogConfig.url}/fr/indexer/elastic/_search/?q=${parent.data.service}&q=uuid.keyword:${context.resourceId}%20AND%20is_metadata:true",
+          "itemsResults": "(function(){ const service = data.hits.hits[0]._source['metadata-fr'].link.find(x => x.service === parent.data.service || x.url === parent.data.service); let formats = service.formats; if (['WS', 'WFS'].includes(service.service) && !formats.includes('CSV')) {formats.push('CSV');} return formats.filter(f => ['CSV', 'Excel non structuré', 'Microsoft Excel','ZIP', 'Shapefile (zip)', 'SHAPE-ZIP', 'GeoJSON', 'JSON', 'KML'].includes(f));})()"
+        }
+      }
+    },
+    "format2": {
+      "type": "string",
+      "title": "Format",
+      "description": "Import format, if no service is set, the possible formats are unknown.",
+      "x-i18n-description": {
+        "fr": "Format d’importation, si aucun service n’est défini, les formats possibles sont inconnus."
+      },
+      "layout": {
+        "if": "parent.data.service == null",
+        "cols": 6,
+        "getItems": {
+          "url": "https://httpbin.org/get",
+          "itemsResults": "(function(){return ['CSV', 'Excel non structuré', 'Microsoft Excel','ZIP', 'Shapefile (zip)', 'SHAPE-ZIP', 'GeoJSON', 'JSON', 'KML']})()"
         }
       }
     }

package/types/importConfig/schema.json

@@ -6,10 +6,7 @@
   "title": "ImportConfig",
   "type": "object",
   "additionalProperties": false,
-  "required": [
-    "service",
-    "format"
-  ],
+  "required": [],
   "properties": {
     "service": {
       "type": "string",
@@ -24,8 +21,8 @@
       "layout": {
         "cols": 6,
         "getItems": {
-          "url": "${context.catalogConfig.url}/fr/indexer/elastic/_search/?q=_id:${context.resourceId}",
-          "itemsResults": "(function() { const links = data.hits.hits[0]._source['metadata-fr'].link; return links.filter(l => l.formats.find(f => ['CSV', 'ODS', 'Excel non structuré', 'Microsoft Excel','ZIP', 'Shapefile (zip)', 'GeoJSON', 'JSON', 'XML', 'GML', 'KML'].includes(f)) || ['WS', 'AFS', 'WFS'].includes(l.service)).map((l, i) =>  [l, i] );  })()",
+          "url": "${context.catalogConfig.url}/fr/indexer/elastic/_search/?q=uuid.keyword:${context.resourceId}%20AND%20is_metadata:true",
+          "itemsResults": "(function() { const links = data.hits.hits[0]._source['metadata-fr'].link; return links.filter(l => l.formats.find(f => ['CSV', 'Excel non structuré', 'Microsoft Excel','ZIP', 'Shapefile (zip)', 'SHAPE-ZIP', 'GeoJSON', 'JSON', 'KML'].includes(f)) && ['WS', 'WFS', undefined].includes(l.service)).map((l, i) =>  [l, i] );  })()",
           "itemTitle": "item[0].service?? 'Local N°' + (item[1] + 1)",
           "itemValue": "item[0].service??  item[0].url"
         }
@@ -34,12 +31,32 @@
     "format": {
       "type": "string",
       "title": "Format",
+      "description": "Import format, if no service is set, the possible formats are unknown.",
+      "x-i18n-description": {
+        "fr": "Format d’importation, si aucun service n’est défini, les formats possibles sont inconnus."
+      },
       "layout": {
         "if": "parent.data.service != null",
         "cols": 6,
         "getItems": {
-          "url": "${context.catalogConfig.url}/fr/indexer/elastic/_search/?q=${parent.data.service}&q=_id:${context.resourceId}",
-          "itemsResults": "(function(){const service = data.hits.hits[0]._source['metadata-fr'].link.find(x => x.service === parent.data.service || x.url === parent.data.service); let formats = service.formats; if (['WS', 'WFS', 'AFS'].includes(service.service) && !formats.includes('CSV')) {formats.push('CSV');} return formats; })()"
+          "url": "${context.catalogConfig.url}/fr/indexer/elastic/_search/?q=${parent.data.service}&q=uuid.keyword:${context.resourceId}%20AND%20is_metadata:true",
+          "itemsResults": "(function(){ const service = data.hits.hits[0]._source['metadata-fr'].link.find(x => x.service === parent.data.service || x.url === parent.data.service); let formats = service.formats; if (['WS', 'WFS'].includes(service.service) && !formats.includes('CSV')) {formats.push('CSV');} return formats.filter(f => ['CSV', 'Excel non structuré', 'Microsoft Excel','ZIP', 'Shapefile (zip)', 'SHAPE-ZIP', 'GeoJSON', 'JSON', 'KML'].includes(f));})()"
+        }
+      }
+    },
+    "format2": {
+      "type": "string",
+      "title": "Format",
+      "description": "Import format, if no service is set, the possible formats are unknown.",
+      "x-i18n-description": {
+        "fr": "Format d’importation, si aucun service n’est défini, les formats possibles sont inconnus."
+      },
+      "layout": {
+        "if": "parent.data.service == null",
+        "cols": 6,
+        "getItems": {
+          "url": "https://httpbin.org/get",
+          "itemsResults": "(function(){return ['CSV', 'Excel non structuré', 'Microsoft Excel','ZIP', 'Shapefile (zip)', 'SHAPE-ZIP', 'GeoJSON', 'JSON', 'KML']})()"
         }
       }
     }