@data-fair/catalog-onegeo-suite 0.1.4 → 0.1.8

package/index.ts

@@ -1,5 +1,5 @@
 import type CatalogPlugin from '@data-fair/types-catalogs'
-import { importConfigSchema, configSchema, assertConfigValid, type OneGeoSuiteConfig } from '#types'
+import { configSchema, assertConfigValid, type OneGeoSuiteConfig } from '#types'
 import { type OneGeoCapabilities, capabilities } from './lib/capabilities.ts'
 import i18n from './lib/i18n.ts'
 
@@ -30,7 +30,6 @@ const plugin: CatalogPlugin<OneGeoSuiteConfig, OneGeoCapabilities> = {
     capabilities
   },
 
-  importConfigSchema,
   configSchema,
   assertConfigValid
 }

package/lib/capabilities.ts

@@ -11,7 +11,6 @@ export const capabilities = [
   'pagination',
 
   'import',
-  'importConfig',
 
 ] satisfies Capability[]
 

package/lib/imports.ts

@@ -4,65 +4,38 @@ import { apiList, formatsList, sortList } from './list.ts'
 
 import axios from '@data-fair/lib-node/axios.js'
 
-export const getResource = async ({ catalogConfig, importConfig, resourceId, tmpDir, log }: GetResourceContext<OneGeoSuiteConfig>): ReturnType<CatalogPlugin['getResource']> => {
-  let format: string = importConfig.format
-  let service: string = importConfig.service
-  const catalog = (await axios.get(new URL(`fr/indexer/elastic/_search/?q=_id:${resourceId}`, catalogConfig.url).href)).data.hits.hits[0]
-  if (!format) {
-    if (!service) {
-      const links = catalog._source['metadata-fr'].link.filter((x: any) => { return apiList.includes(x.service) && x.formats.find((y: string) => { return formatsList.includes(y) }) })
-      service = sortList(links.map((x: Link) => x.service), apiList)[0]
-    }
-    format = sortList(catalog._source['metadata-fr'].link.find((x: Link) => { return x.service === service && x.formats.find((y: string) => { return formatsList.includes(y) }) }).formats, formatsList)[0]
-  }
-
-  if (!catalog) {
-    throw Error(`resource ${service} not found for ${resourceId} in ${catalogConfig.url}`)
-  }
-
-  // filter links by format and service
-  const source: Link = catalog._source['metadata-fr'].link.find((x: Link) => {
-    return x.service === service || x.url === service
-  })
-
-  // table of format for make WFS url
-  const wfsTable: Record<string, string> = {
-    CSV: 'csv',
-    JSON: 'application/json',
-    GeoJSON: 'application/json',
-    'Shapefile (zip)': 'SHAPE-ZIP',
-    'SHAPE-ZIP': 'SHAPE-ZIP',
-    KML: 'kml',
-  }
-  // table of format
-  const extensionTable: Record<string, string> = {
-    CSV: '.csv',
-    GeoJSON: '.geojson',
-    JSON: '.json',
-    'Shapefile (zip)': '.zip',
-    'SHAPE-ZIP': '.zip',
-    KML: '.kml',
-    'Excel non structuré': '.xlsx',
-    'Microsoft Excel': '.xls',
-  }
+// table of format -> extension
+const wfsTable: Record<string, string> = {
+  CSV: 'csv',
+  JSON: 'application/json',
+  GeoJSON: 'application/json',
+  'Shapefile (zip)': 'SHAPE-ZIP',
+  'SHAPE-ZIP': 'SHAPE-ZIP',
+  KML: 'kml',
+}
 
-  let downloadUrl: string
-  if (source.service === 'WS') {
-    downloadUrl = `${source.url}/${source.name}/all${extensionTable[format]}`
-  } else if (source.service === undefined) {
-    downloadUrl = `${source.url}`
-  } else if (source.service === 'WFS') {
-    downloadUrl = `${source.url}?SERVICE=WFS&VERSION=2.0.0&request=GetFeature&typename=${source.name}&outputFormat=${wfsTable[format]}&startIndex=0&sortby=gid`
-  } else {
-    downloadUrl = `${source.url}`
-  }
+// table of format -> extension
+const extensionTable: Record<string, string> = {
+  CSV: '.csv',
+  GeoJSON: '.geojson',
+  JSON: '.json',
+  'Shapefile (zip)': '.zip',
+  'SHAPE-ZIP': '.zip',
+  KML: '.kml',
+  'Excel non structuré': '.xlsx',
+  'Microsoft Excel': '.xls',
+}
 
-  await log.step(`Downloading the file ${downloadUrl}`)
-
-  // Download the resource
-  const fs = await import('node:fs')
-  const path = await import('path')
+export const getResource = async ({
+  catalogConfig,
+  resourceId,
+  tmpDir,
+  log
+}: GetResourceContext<OneGeoSuiteConfig>): ReturnType<CatalogPlugin['getResource']> => {
+  const catalog = (await axios.get(new URL(`fr/indexer/elastic/_search/?q=uuid.keyword:${resourceId}%20AND%20is_metadata:true`, catalogConfig.url).href)).data.hits.hits[0]
+  if (!catalog) throw Error(`resource not found for ${resourceId} in ${catalogConfig.url}`)
 
+  // get origine url
   const axiosPortail = axios.create({
     validateStatus: function (status) {
       return status >= 200 && status < 500
@@ -77,19 +50,57 @@ export const getResource = async ({ catalogConfig, importConfig, resourceId, tmp
   }
   const origin = portail ? `${catalogConfig.url}/${portail}/jeux-de-donnees/${catalog._source.slug}/info` : ''
 
-  let response
-  try {
-    response = await axios.get(downloadUrl, {
-      responseType: 'stream',
+  const links: Link[] = catalog._source['metadata-fr'].link.filter((x: Link) => {
+    return apiList.includes(x.service) && x.formats.find((y: string) => {
+      return formatsList.includes(y)
+    })
+  })
+
+  // list all url possible
+  let downloadUrls: { url: string, format: string, service: string | undefined, description: string | undefined }[] = []
+
+  for (const link of links) {
+    const formats = link.formats.filter((f: string) => {
+      return formatsList.includes(f)
     })
-    if (response.headers['content-type'] === 'text/html') {
-      response = undefined
-      throw Error('return HTML page')
+    for (const format of formats) {
+      if (link.service === 'WS' && extensionTable[format]) {
+        downloadUrls.push({ url: `${link.url}/${link.name}/all${extensionTable[format]}`, format, service: link.service, description: link.description })
+      } else if (link.service === undefined) {
+        downloadUrls.push({ url: `${link.url}`, format, service: link.service, description: link.description })
+      } else if (link.service === 'WFS' && wfsTable[format]) {
+        downloadUrls.push({ url: `${link.url}?SERVICE=WFS&VERSION=2.0.0&request=GetFeature&typename=${link.name}&outputFormat=${wfsTable[format]}`, format, service: link.service, description: link.description })
+      }
     }
+  }
+  downloadUrls = sortList(downloadUrls, apiList, (x: any) => { return x.service })
+  downloadUrls = sortList(downloadUrls, formatsList, (x: any) => { return x.format })
 
-    await log.info(`Get file with ${downloadUrl} successfully! ${response.status}`)
-  } catch (e) {
-    await log.warning(`Get file fail with this url: ${downloadUrl}; ${e}`)
+  // Download the resource
+  const fs = await import('node:fs')
+  const path = await import('path')
+  let response
+  let format: string
+  let description: string | undefined
+
+  for (const downloadUrl of downloadUrls) {
+    await log.step(`Downloading the file ${downloadUrl.url}; format: ${downloadUrl.format}; service: ${downloadUrl.service}`)
+
+    try {
+      response = await axios.get(downloadUrl.url, {
+        responseType: 'stream',
+      })
+      if (response.headers['content-type'] === 'text/html') {
+        response = undefined
+        throw Error('return HTML page')
+      }
+      format = downloadUrl.format
+      description = downloadUrl.description
+      await log.info(`Get file with ${downloadUrl.url} successfully! ${response.status}`)
+    } catch (e) {
+      await log.warning(`Downloading fail with this url: ${downloadUrl}; ${e}`)
+    }
+    if (response) break
   }
 
   if (!response) {
@@ -97,7 +108,7 @@ export const getResource = async ({ catalogConfig, importConfig, resourceId, tmp
   }
 
   // Create a filename
-  const fileName = catalog._source.slug + extensionTable[format]
+  const fileName = catalog._source.slug + extensionTable[format!]
   const filePath = path.join(tmpDir, fileName)
   await log.info(`Downloading resource to ${fileName}`)
 
@@ -140,9 +151,9 @@ export const getResource = async ({ catalogConfig, importConfig, resourceId, tmp
     id: resourceId,
     slug: catalog._source.slug,
     title: catalog._source['metadata-fr'].title,
-    description: source.description ?? catalog._source['metadata-fr'].abstratc,
+    description: description ?? catalog._source['metadata-fr'].abstratc,
     filePath,
-    format,
+    format: format!,
     frequency,
     license: {
       href: '',
@@ -150,7 +161,9 @@ export const getResource = async ({ catalogConfig, importConfig, resourceId, tmp
     },
     keywords: catalog._source['metadata-fr'].keyword,
     updatedAt: catalog._source['metadata-fr'].lastUpdateDate ?? undefined,
-    image: catalog._source['metadata-fr'].image.find((x: { type: string, url: string | null }) => { return x.type === 'thumbnail' && !!x.url })?.url ?? null,
+    image: catalog._source['metadata-fr'].image.find((x: { type: string, url: string | null }) => {
+      return x.type === 'thumbnail' && !!x.url
+    })?.url ?? null,
     origin
   }
 }

package/lib/list.ts

@@ -7,7 +7,7 @@ type ResourceList = Awaited<ReturnType<CatalogPlugin['list']>>['results']
 
 export const apiList: Array<string | undefined> = ['WS', 'WFS', undefined]
 export const formatsList = [
-  'CSV', 'Excel non structuré', 'Microsoft Excel', 'Shapefile (zip)', 'SHAPE-ZIP', 'GeoJSON', 'JSON', 'KML']
+  'GeoJSON', 'SHAPE-ZIP', 'Shapefile (zip)', 'CSV', 'JSON', 'Excel non structuré', 'Microsoft Excel', 'KML']
 
 const extensionTable: Record<string, string> = {
   CSV: '.csv',
@@ -20,13 +20,13 @@ const extensionTable: Record<string, string> = {
   'Microsoft Excel': '.xls',
 }
 
-export const sortList = (formats: any[], reference: any[]) => {
+export const sortList = (formats: any[], reference: any[], func = (x: any) => { return x }) => {
   return [...formats].sort((a, b) =>
-    (reference.indexOf(a) === -1 ? reference.length : reference.indexOf(a)) -
-    (reference.indexOf(b) === -1 ? reference.length : reference.indexOf(b))
+    (reference.indexOf(func(a)) === -1 ? reference.length : reference.indexOf(func(a))) -
+    (reference.indexOf(func(b)) === -1 ? reference.length : reference.indexOf(func(b)))
   )
 }
-const baseReqDataset = (input: string = '*', size: number = 100, from: number = 1) => {
+const baseReqDataset = (input: string = '*', size: number = 500, from: number = 1) => {
   return {
     from: (from - 1) * size,
     size: Math.min(size, 10000),
@@ -47,7 +47,7 @@ const baseReqDataset = (input: string = '*', size: number = 100, from: number =
               }
             }]
           }
-        }, { term: { is_metadata: true } }, {
+        }, { term: { is_metadata: true } }, { term: { 'editorial-metadata.defaultPermissionLevel': 3 } }, {
           bool: {
             should: [
               ...apiList.filter((x: any) => { return x }).map((x: any) => { return { term: { 'metadata-fr.link.service.keyword': x } } }),
@@ -87,7 +87,7 @@ const countReq = (input: string = '*') => {
               }
             }]
           }
-        }, { term: { is_metadata: true } }, {
+        }, { term: { is_metadata: true } }, { term: { 'editorial-metadata.defaultPermissionLevel': 3 } }, {
           bool: {
             should: [
               ...apiList.filter((x: any) => x).map((x: any) => ({ term: { 'metadata-fr.link.service.keyword': x } })),
@@ -121,7 +121,13 @@ const countReq = (input: string = '*') => {
 export const list = async ({ catalogConfig, params }: ListContext<OneGeoSuiteConfig, OneGeoCapabilities>): ReturnType<CatalogPlugin['list']> => {
   const url = catalogConfig.url
   const listResources = async (params: Record<any, any>) => {
-    const catalogs = (await axios.post(new URL('fr/indexer/elastic/_search/', url).href, baseReqDataset(params.q || '*', params.size, params.page))).data.hits.hits
+    let catalogs
+    try {
+      catalogs = (await axios.post(new URL('fr/indexer/elastic/_search/', url).href, baseReqDataset(params.q || '*', params.size, params.page))).data.hits.hits
+    } catch (e) {
+      // @ts-ignore
+      throw Error(`Axios error: ${e?.status ?? ''} ${e?.message}`)
+    }
     const count = (await axios.post(new URL('fr/indexer/elastic/_search/', url).href, countReq(params.q))).data.aggregations.unique_datasets.value
     const res = []
 
@@ -151,7 +157,7 @@ export const list = async ({ catalogConfig, params }: ListContext<OneGeoSuiteCon
       formats = formats.map(x => extensionTable[x]?.slice(1) ?? x)
 
       res.push({
-        id: `${catalog._id}`,
+        id: `${catalog._source.uuid}`,
         title: catalog._source['metadata-fr'].title,
         description: sources[0].description,
         format: formats.slice(0, 3).join(', ') + (formats.length ? '..' : ''),

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@data-fair/catalog-onegeo-suite",
   "description": "OneGeoSuite plugin for the Data Fair catalogs service.",
-  "version": "0.1.4",
+  "version": "0.1.8",
   "main": "index.ts",
   "type": "module",
   "scripts": {

package/types/importConfig/.type/index.js

@@ -29,8 +29,8 @@ export const schema = {
       "layout": {
         "cols": 6,
         "getItems": {
-          "url": "${context.catalogConfig.url}/fr/indexer/elastic/_search/?q=_id:${context.resourceId}",
-          "itemsResults": "(function() { const links = data.hits.hits[0]._source['metadata-fr'].link; return links.filter(l => l.formats.find(f => ['CSV', 'ODS', 'Excel non structuré', 'Microsoft Excel','ZIP', 'Shapefile (zip)', 'GeoJSON', 'JSON', 'XML', 'GML', 'KML'].includes(f)) || ['WS', 'AFS', 'WFS'].includes(l.service)).map((l, i) =>  [l, i] );  })()",
+          "url": "${context.catalogConfig.url}/fr/indexer/elastic/_search/?q=uuid.keyword:${context.resourceId}%20AND%20is_metadata:true",
+          "itemsResults": "(function() { const links = data.hits.hits[0]._source['metadata-fr'].link; return links.filter(l => l.formats.find(f => ['CSV', 'Excel non structuré', 'Microsoft Excel','ZIP', 'Shapefile (zip)', 'SHAPE-ZIP', 'GeoJSON', 'JSON', 'KML'].includes(f)) && ['WS', 'WFS', undefined].includes(l.service)).map((l, i) =>  [l, i] );  })()",
           "itemTitle": "item[0].service?? 'Local N°' + (item[1] + 1)",
           "itemValue": "item[0].service??  item[0].url"
         }
@@ -39,12 +39,32 @@ export const schema = {
     "format": {
       "type": "string",
       "title": "Format",
+      "description": "Import format, if no service is set, the possible formats are unknown.",
+      "x-i18n-description": {
+        "fr": "Format d’importation, si aucun service n’est défini, les formats possibles sont inconnus."
+      },
       "layout": {
         "if": "parent.data.service != null",
         "cols": 6,
         "getItems": {
-          "url": "${context.catalogConfig.url}/fr/indexer/elastic/_search/?q=${parent.data.service}&q=_id:${context.resourceId}",
-          "itemsResults": "(function(){const service = data.hits.hits[0]._source['metadata-fr'].link.find(x => x.service === parent.data.service || x.url === parent.data.service); let formats = service.formats; if (['WS', 'WFS', 'AFS'].includes(service.service) && !formats.includes('CSV')) {formats.push('CSV');} return formats; })()"
+          "url": "${context.catalogConfig.url}/fr/indexer/elastic/_search/?q=${parent.data.service}&q=uuid.keyword:${context.resourceId}%20AND%20is_metadata:true",
+          "itemsResults": "(function(){ const service = data.hits.hits[0]._source['metadata-fr'].link.find(x => x.service === parent.data.service || x.url === parent.data.service); let formats = service.formats; if (['WS', 'WFS'].includes(service.service) && !formats.includes('CSV')) {formats.push('CSV');} return formats.filter(f => ['CSV', 'Excel non structuré', 'Microsoft Excel','ZIP', 'Shapefile (zip)', 'SHAPE-ZIP', 'GeoJSON', 'JSON', 'KML'].includes(f));})()"
+        }
+      }
+    },
+    "format2": {
+      "type": "string",
+      "title": "Format",
+      "description": "Import format, if no service is set, the possible formats are unknown.",
+      "x-i18n-description": {
+        "fr": "Format d’importation, si aucun service n’est défini, les formats possibles sont inconnus."
+      },
+      "layout": {
+        "if": "parent.data.service == null",
+        "cols": 6,
+        "getItems": {
+          "url": "https://httpbin.org/get",
+          "itemsResults": "(function(){return ['CSV', 'Excel non structuré', 'Microsoft Excel','ZIP', 'Shapefile (zip)', 'SHAPE-ZIP', 'GeoJSON', 'JSON', 'KML']})()"
         }
       }
     }

package/types/index.ts

@@ -1,5 +1,4 @@
 export { schema as configSchema, assertValid as assertConfigValid, type OneGeoSuiteConfig } from './catalogConfig/index.ts'
-export { schema as importConfigSchema } from './importConfig/index.ts'
 
 export type Link = {
   _main: boolean,

package/types/importConfig/index.ts

@@ -1 +0,0 @@
-export * from './.type/index.js'

package/types/importConfig/schema.json

@@ -1,47 +0,0 @@
-{
-  "$id": "https://github.com/data-fair/catalog-onegeo-suite/import-config",
-  "x-exports": [
-    "schema"
-  ],
-  "title": "ImportConfig",
-  "type": "object",
-  "additionalProperties": false,
-  "required": [],
-  "properties": {
-    "service": {
-      "type": "string",
-      "title": "Web Service",
-      "x-i18n-title": {
-        "fr": "Service Web"
-      },
-      "description": "This resource is available from multiple sources (web services, local files). Select the source you want to use for the import.",
-      "x-i18n-description": {
-        "fr": "Cette ressource est disponible via plusieurs sources (services web, fichiers locaux). Sélectionnez la source que vous souhaitez utiliser pour l'import."
-      },
-      "layout": {
-        "cols": 6,
-        "getItems": {
-          "url": "${context.catalogConfig.url}/fr/indexer/elastic/_search/?q=_id:${context.resourceId}",
-          "itemsResults": "(function() { const links = data.hits.hits[0]._source['metadata-fr'].link; return links.filter(l => l.formats.find(f => ['CSV', 'ODS', 'Excel non structuré', 'Microsoft Excel','ZIP', 'Shapefile (zip)', 'GeoJSON', 'JSON', 'XML', 'GML', 'KML'].includes(f)) || ['WS', 'AFS', 'WFS'].includes(l.service)).map((l, i) =>  [l, i] );  })()",
-          "itemTitle": "item[0].service?? 'Local N°' + (item[1] + 1)",
-          "itemValue": "item[0].service??  item[0].url"
-        }
-      }
-    },
-    "format": {
-      "type": "string",
-      "title": "Format",
-      "layout": {
-        "if": "parent.data.service != null",
-        "cols": 6,
-        "getItems": {
-          "url": "${context.catalogConfig.url}/fr/indexer/elastic/_search/?q=${parent.data.service}&q=_id:${context.resourceId}",
-          "itemsResults": "(function(){const service = data.hits.hits[0]._source['metadata-fr'].link.find(x => x.service === parent.data.service || x.url === parent.data.service); let formats = service.formats; if (['WS', 'WFS', 'AFS'].includes(service.service) && !formats.includes('CSV')) {formats.push('CSV');} return formats; })()"
-        }
-      }
-    }
-  },
-  "layout": {
-    "title": null
-  }
-}