@data-fair/catalog-csw 0.1.0

package/README.md

@@ -0,0 +1,3 @@
+# <img alt="Data FAIR logo" src="https://cdn.jsdelivr.net/gh/data-fair/data-fair@master/ui/public/assets/logo.svg" width="40"> @data-fair/catalog-csw
+
+A simple CSW plugin for the Data Fair catalogs service.

package/index.ts

@@ -0,0 +1,34 @@
+import type CatalogPlugin from '@data-fair/types-catalogs'
+import { configSchema, assertConfigValid, type CSWConfig } from '#types'
+import { type CSWCapabilities, capabilities } from './lib/capabilities.ts'
+
+const plugin: CatalogPlugin<CSWConfig, CSWCapabilities> = {
+  async prepare (context) {
+    if (context.catalogConfig.url) {
+      context.catalogConfig.url = context.catalogConfig.url.trim()
+    }
+    return context
+  },
+  async list (context) {
+    const { list } = await import('./lib/list.ts')
+    return list(context)
+  },
+
+  async getResource (context) {
+    const { getResource } = await import('./lib/imports.ts')
+    return getResource(context)
+  },
+
+  metadata: {
+    title: 'CSW',
+    i18n: {
+      en: { description: 'Uses CSW 2.0.2 to import datasets (GeoNetwork, ...)' },
+      fr: { description: 'Utilise du CSW 2.0.2 pour importer des datasets (GeoNetwork, ...)' }
+    },
+    capabilities
+  },
+
+  configSchema,
+  assertConfigValid
+}
+export default plugin

package/lib/capabilities.ts

@@ -0,0 +1,16 @@
+import type { Capability } from '@data-fair/types-catalogs'
+
+/**
+ * The list of capabilities of the plugin.
+ * These capabilities define the actions that can be performed with the plugin.
+ * The capabilities must satisfy the `Capability` type.
+ */
+export const capabilities = [
+  'search',
+  'pagination',
+
+  'import',
+] satisfies Capability[]
+
+export type CSWCapabilities = typeof capabilities
+export default capabilities

package/lib/imports.ts

@@ -0,0 +1,104 @@
+import { XMLParser } from 'fast-xml-parser'
+import path from 'path'
+import axios from '@data-fair/lib-node/axios.js'
+import type { CatalogPlugin, GetResourceContext, Resource } from '@data-fair/types-catalogs'
+import type { CSWConfig } from '#types'
+import { downloadFileWithProgress } from './utils/download.ts'
+import { findBestDownloadUrl } from './utils/link-selection.ts'
+import { getText } from './utils/common.ts'
+
+const parser = new XMLParser({
+  ignoreAttributes: false,
+  attributeNamePrefix: '',
+  removeNSPrefix: true,
+  parseTagValue: true
+})
+
+/**
+ * Fetches the metadata for a given resource ID from the CSW endpoint, determines the best download URL, and downloads the file to a temporary directory.
+ * @param context The context object containing catalog configuration, resource ID, temporary directory path, and logger
+ * @returns An object containing details about the downloaded resource, including title, description, file path, format, and updated date
+ */
+export const getResource = async ({ catalogConfig, resourceId, tmpDir, log }: GetResourceContext<CSWConfig>): ReturnType<CatalogPlugin['getResource']> => {
+  // CSW GetRecordById request body (Requesting full ISO 19139 metadata)
+  const cswBody = `
+    <csw:GetRecordById 
+      xmlns:csw="http://www.opengis.net/cat/csw/2.0.2" 
+      xmlns:gmd="http://www.isotc211.org/2005/gmd" 
+      service="CSW" 
+      version="2.0.2" 
+      outputSchema="http://www.isotc211.org/2005/gmd">
+      <csw:Id>${resourceId}</csw:Id>
+      <csw:ElementSetName>full</csw:ElementSetName>
+    </csw:GetRecordById>`
+
+  const baseUrl = catalogConfig.url
+
+  try {
+    await log.step('Fetching metadata (CSW)')
+    const response = await axios.post(baseUrl, cswBody, {
+      headers: { 'Content-Type': 'application/xml' }
+    })
+
+    const parsed = parser.parse(response.data)
+
+    // Handle responses with or without namespace prefixes
+    const responseRoot = parsed.GetRecordByIdResponse || parsed['csw:GetRecordByIdResponse']
+    if (!responseRoot) {
+      throw new Error('Invalid or empty CSW response')
+    }
+
+    const metadata = responseRoot.MD_Metadata || responseRoot['gmd:MD_Metadata']
+    if (!metadata) {
+      throw new Error('ISO 19139 Metadata not found')
+    }
+
+    // Extract basic info using safe helpers
+    const dataId = metadata.identificationInfo?.MD_DataIdentification || {}
+    const titleObj = dataId.citation?.CI_Citation?.title
+    const titleRecord = getText(titleObj) || resourceId
+    const abstract = getText(dataId.abstract)
+
+    // Determine the best URL for file download
+    const downloadInfo = await findBestDownloadUrl(metadata, resourceId, log)
+
+    if (!downloadInfo || !downloadInfo.url) {
+      throw new Error(`No suitable download link found for ${resourceId}`)
+    }
+
+    await log.step('Downloading file')
+
+    // Construct filename based on format
+    let fileName = `${resourceId}`
+    if (downloadInfo.format === 'shapefile') {
+      fileName += '.zip'
+    } else if (downloadInfo.format === 'geojson') {
+      fileName += '.geojson'
+    } else if (downloadInfo.format === 'csv') {
+      fileName += '.csv'
+    } else if (downloadInfo.format === 'json') {
+      fileName += '.json'
+    } else if (downloadInfo.format === 'kml') {
+      fileName += '.kml'
+    } else {
+      // Fallback to extension from URL
+      fileName += path.extname(downloadInfo.url.split('?')[0]) || ''
+    }
+
+    const destPath = path.join(tmpDir, fileName)
+
+    await downloadFileWithProgress(downloadInfo.url, destPath, resourceId, log)
+
+    return {
+      id: resourceId,
+      title: titleRecord,
+      description: abstract,
+      filePath: destPath,
+      format: downloadInfo.format,
+      updatedAt: getText(metadata.dateStamp?.Date || metadata.dateStamp?.DateTime) || new Date().toISOString(),
+      size: 0
+    } as Resource
+  } catch (error: any) {
+    throw new Error(error.message || 'Error fetching resource from CSW')
+  }
+}

package/lib/list.ts

@@ -0,0 +1,142 @@
+import type { CatalogPlugin, ListContext } from '@data-fair/types-catalogs'
+import type { CSWConfig } from '#types'
+import type { CswRecord } from './utils/types.ts'
+import { XMLParser } from 'fast-xml-parser'
+import axios from '@data-fair/lib-node/axios.js'
+import capabilities from './capabilities.ts'
+import { asArray, getText } from './utils/common.ts'
+
+type ResourceList = Awaited<ReturnType<CatalogPlugin['list']>>['results']
+
+const parser = new XMLParser({
+  ignoreAttributes: false,
+  attributeNamePrefix: '',
+  removeNSPrefix: true
+})
+
+/**
+ * Performs a CSW GetRecords request to list resources based on the provided query and pagination parameters.
+ * @param config The context object containing catalog configuration, query parameters, and logger
+ * @returns An object containing the total count of matched records, an array of resource summaries, and the path for pagination
+ */
+export const list = async (config: ListContext<CSWConfig, typeof capabilities>): ReturnType<CatalogPlugin<CSWConfig>['list']> => {
+  const { catalogConfig, params } = config
+  const query = params?.q ? params.q.trim() : ''
+  const page = Number(params?.page || 1)
+  const size = Number(params?.size || 10)
+  const startPosition = (page - 1) * size + 1
+
+  // Bloc of XML filters to find records with relevant formats in their metadata (WFS, GeoJSON, JSON, CSV, ZIP)
+  const formatFilter = `
+    <ogc:Or>
+      <ogc:PropertyIsLike matchCase="false" wildCard="%" singleChar="_" escapeChar="\\">
+        <ogc:PropertyName>AnyText</ogc:PropertyName>
+        <ogc:Literal>%WFS%</ogc:Literal>
+      </ogc:PropertyIsLike>
+      
+      <ogc:PropertyIsLike matchCase="false" wildCard="%" singleChar="_" escapeChar="\\">
+        <ogc:PropertyName>AnyText</ogc:PropertyName>
+        <ogc:Literal>%JSON%</ogc:Literal>
+      </ogc:PropertyIsLike>
+
+      <ogc:PropertyIsLike matchCase="false" wildCard="%" singleChar="_" escapeChar="\\">
+        <ogc:PropertyName>AnyText</ogc:PropertyName>
+        <ogc:Literal>%CSV%</ogc:Literal>
+      </ogc:PropertyIsLike>
+
+      <ogc:PropertyIsLike matchCase="false" wildCard="%" singleChar="_" escapeChar="\\">
+        <ogc:PropertyName>AnyText</ogc:PropertyName>
+        <ogc:Literal>%ZIP%</ogc:Literal>
+      </ogc:PropertyIsLike>
+    </ogc:Or>`
+
+  const filterBlock = query
+    ? `
+    <ogc:And>
+      <ogc:PropertyIsLike wildCard="%" singleChar="_" escapeChar="\\\\">
+        <ogc:PropertyName>AnyText</ogc:PropertyName>
+        <ogc:Literal>%${query}%</ogc:Literal>
+      </ogc:PropertyIsLike>
+      ${formatFilter}
+    </ogc:And>`
+    : formatFilter
+
+  const constraintBlock = `
+    <csw:Constraint version="1.1.0">
+      <ogc:Filter>
+        ${filterBlock}
+      </ogc:Filter>
+    </csw:Constraint>
+    <ogc:SortBy xmlns:ogc="http://www.opengis.net/ogc">
+      <ogc:SortProperty>
+        <ogc:PropertyName>RevisionDate</ogc:PropertyName>
+        <ogc:SortOrder>DESC</ogc:SortOrder>
+      </ogc:SortProperty>
+    </ogc:SortBy>`
+
+  const cswBody = `
+    <csw:GetRecords 
+      xmlns:csw="http://www.opengis.net/cat/csw/2.0.2" 
+      xmlns:ogc="http://www.opengis.net/ogc" 
+      service="CSW" 
+      version="2.0.2" 
+      resultType="results" 
+      startPosition="${startPosition}" 
+      maxRecords="${size}" 
+      outputSchema="http://www.opengis.net/cat/csw/2.0.2">
+      <csw:Query typeNames="csw:Record">
+        <csw:ElementSetName>summary</csw:ElementSetName>
+        ${constraintBlock}
+      </csw:Query>
+    </csw:GetRecords>`
+
+  try {
+    const baseUrl = catalogConfig.url
+
+    const response = await axios.post(baseUrl, cswBody, {
+      headers: { 'Content-Type': 'application/xml' }
+    })
+
+    const parsed = parser.parse(response.data)
+    const root = parsed.GetRecordsResponse || parsed['csw:GetRecordsResponse']
+    if (!root) {
+      console.error('Réponse XML invalide (pas de GetRecordsResponse)')
+      return { count: 0, results: [], path: [] }
+    }
+
+    const searchResults = root.SearchResults || root['csw:SearchResults']
+    if (!searchResults) {
+      console.error('Pas de SearchResults')
+      return { count: 0, results: [], path: [] }
+    }
+
+    const totalCount = parseInt(searchResults.numberOfRecordsMatched || searchResults['numberOfRecordsMatched'] || '0', 10)
+    const rawRecords = searchResults.SummaryRecord || searchResults.Record || []
+    const records = asArray(rawRecords) as CswRecord[]
+
+    const listResults = records.map((record: any) => {
+      const identifier = getText(record.identifier || record['dc:identifier'])
+      const titleRecord = getText(record.title || record['dc:title']) || 'Sans titre'
+      const rawDateObj = record.modified || record.date || record.dateStamp || record.RevisionDate
+      const dateRaw = getText(rawDateObj)
+      const type = getText(record.type || record['dc:type'])
+      return {
+        id: identifier,
+        title: titleRecord,
+        updatedAt: dateRaw || new Date().toISOString(),
+        type: 'resource',
+        format: type || 'unknown'
+      }
+    }) as ResourceList
+
+    return {
+      count: totalCount,
+      results: listResults,
+      path: []
+    }
+  } catch (error: any) {
+    console.error('ERREUR :', error.message)
+    if (error.response) console.error('Data:', error.response.data)
+    throw new Error('Erreur lors de la recherche CSW')
+  }
+}

package/lib/utils/common.ts

@@ -0,0 +1,12 @@
+export const asArray = (input: any): any[] => {
+  if (!input) return []
+  return Array.isArray(input) ? input : [input]
+}
+
+export const getText = (input: any): string => {
+  if (!input) return ''
+  if (typeof input === 'string') return input
+  if (input.CharacterString) return input.CharacterString
+  if (input['#text']) return input['#text']
+  return ''
+}

package/lib/utils/download.ts

@@ -0,0 +1,122 @@
+import fs from 'fs'
+import axios from '@data-fair/lib-node/axios.js'
+import type { AxiosRequestConfig } from 'axios'
+
+/**
+ * Downloads a file from a URL to a local path.
+ * @param url - The source URL to download from.
+ * @param destPath - The local file system path where the file will be saved.
+ * @param label - A label used for logging (the resource ID).
+ * @param log - The logger object
+ * @param axiosConfig - Optional Axios configuration
+ * @returns A promise that resolves to the destination path upon success.
+ */
+export async function downloadFileWithProgress (
+  url: string,
+  destPath: string,
+  label: string,
+  log: any,
+  axiosConfig: AxiosRequestConfig = {}
+): Promise<string> {
+  // Create a write stream to save the file to the disk
+  const writer = fs.createWriteStream(destPath)
+
+  try {
+    // Make the HTTP GET request
+    const response = await axios.get(url, {
+      ...axiosConfig,
+      responseType: 'stream',
+    })
+
+    // Attempt to get the total file size from headers for progress calculation
+    const totalLength = response.headers['content-length']
+      ? parseInt(response.headers['content-length'], 10)
+      : undefined
+
+    // Initialize the logging task
+    await log.task(`download ${label}`, 'Downloading...', totalLength)
+
+    let downloadedBytes = 0
+    let lastLogged = Date.now()
+    const logInterval = 500 // Update progress log every 500ms to avoid spamming the console
+
+    // Listen to the data chunk event to update progress
+    response.data.on('data', (chunk: Buffer) => {
+      downloadedBytes += chunk.length
+      const now = Date.now()
+      // Only update the log if the interval has passed
+      if (now - lastLogged > logInterval) {
+        lastLogged = now
+        log.progress(`download ${label}`, downloadedBytes, totalLength)
+      }
+    })
+
+    // Pipe the download stream directly into the file writer
+    response.data.pipe(writer)
+
+    // Return a promise that resolves when writing is finished or rejects on error
+    return await new Promise<string>((resolve, reject) => {
+      // Success: The file has been fully written
+      writer.on('finish', async () => {
+        // Ensure the progress bar shows 100% (or the final byte count) at the end
+        await log.progress(`download ${label}`, downloadedBytes, totalLength)
+        resolve(destPath)
+      })
+
+      // Error handling helper
+      const handleError = (err: any) => {
+        // Close the stream explicitly
+        writer.close()
+        // Delete the partial file to avoid corruption
+        fs.unlink(destPath, () => {})
+        reject(err)
+      }
+
+      // Listen for errors on both the file writer and the download stream
+      writer.on('error', handleError)
+      response.data.on('error', handleError)
+    })
+  } catch (err: any) {
+    if (err.response) {
+      const status = err.response.status
+      if (status >= 400 && status < 500) {
+        let msg = `Erreur client (${status})`
+        switch (status) {
+          case 400:
+            msg = 'Requête invalide (400). Les paramètres envoyés sont peut-être incorrects.'
+            break
+          case 401:
+            msg = 'Accès refusé (401). Vérifiez le nom d\'utilisateur et le mot de passe dans la configuration.'
+            break
+          case 403:
+            msg = 'Accès interdit (403). Vous n\'avez pas les droits nécessaires pour accéder à ce fichier.'
+            break
+          case 404:
+            msg = 'Fichier introuvable (404). L\'URL de téléchargement n\'existe plus ou est incorrecte.'
+            break
+          case 408:
+            msg = 'Délai d\'attente dépassé (408). Le serveur a mis trop de temps à répondre.'
+            break
+          case 410:
+            msg = 'Ressource indisponible (410). Le fichier a été définitivement supprimé.'
+            break
+          case 421:
+            msg = 'Requête mal dirigée (421). Le serveur ne peut pas répondre (problème de certificat SSL).'
+            break
+          case 429:
+            msg = 'Trop de requêtes (429). Le serveur limite le nombre de téléchargements (rate limit).'
+            break
+          default:
+            msg = `Erreur client non gérée (${status}).`
+        }
+        await log.error(msg)
+        throw new Error(msg)
+      }
+    }
+    writer.close()
+    if (fs.existsSync(destPath)) {
+      fs.unlinkSync(destPath)
+    }
+    throw err
+  }
+}

package/lib/utils/link-selection.ts

@@ -0,0 +1,232 @@
+import axios from '@data-fair/lib-node/axios.js'
+import { getText, asArray } from './common.ts'
+import type { DownloadCandidate } from './types.ts'
+
+export const isUrlValid = async (url: string, log: any, isWFSTest = false): Promise<boolean> => {
+  try {
+    if (isWFSTest) {
+      const testUrl = new URL(url)
+      testUrl.searchParams.set('COUNT', '1')
+      testUrl.searchParams.set('MAXFEATURES', '1')
+
+      const response = await axios.get(testUrl.toString(), {
+        timeout: 5000,
+        validateStatus: (status) => status < 500
+      })
+
+      if (response.status >= 400) {
+        return false
+      }
+      const content = String(response.data)
+      if (content.includes('ExceptionReport') || content.includes('ServiceException')) {
+        return false
+      }
+      // if the requested format is JSON but the response is XML, it's likely that the WFS doesn't support the requested format
+      const contentType = response.headers['content-type'] || ''
+      const requestedFormat = testUrl.searchParams.get('OUTPUTFORMAT') || ''
+      if (requestedFormat.includes('json') && contentType.includes('xml')) {
+        return false
+      }
+
+      return true
+    } else {
+      await axios.head(url, { timeout: 3000, validateStatus: (s) => s >= 200 && s < 400 })
+      return true
+    }
+  } catch (err) {
+    return false
+  }
+}
+
+const analyzeLink = (linkWrapper: any): DownloadCandidate | null => {
+  const link = linkWrapper.CI_OnlineResource || linkWrapper
+  const url = getText(link.linkage?.URL)
+  const protocol = getText(link.protocol).toLowerCase()
+  const name = getText(link.name)
+
+  if (!url) return null
+
+  const u = url.toLowerCase()
+  const p = protocol
+  const n = name.toLowerCase()
+
+  if (u.includes('service=wfs') && u.includes('outputformat=')) {
+    let format = 'wfs_service'
+    const score = 50
+    if (u.includes('geojson') || u.includes('json')) format = 'geojson'
+    else if (u.includes('csv')) format = 'csv'
+    else if (u.includes('zip') || u.includes('shape')) format = 'shapefile'
+    return { url, format, score }
+  }
+
+  // 2. Shapefile (Zip)
+  if (u.includes('shape-zip') || u.endsWith('.zip') || n.includes('shapefile')) {
+    return { url, format: 'shapefile', score: 10 }
+  }
+
+  // 3. GeoJSON
+  if (u.includes('geojson') || p.includes('geo+json') || n.includes('geojson')) {
+    return { url, format: 'geojson', score: 8 }
+  }
+
+  // 4. CSV
+  if (u.includes('/csv') || u.includes('.csv') || p.includes('text/csv') || name === 'csv') {
+    return { url, format: 'csv', score: 6 }
+  }
+
+  // 5. KML
+  if (u.endsWith('.kml') || p.includes('kml')) {
+    return { url, format: 'kml', score: 4 }
+  }
+
+  // 6. JSON
+  if ((u.includes('/json') || u.includes('.json') || p.includes('application/json') || n === 'json') && !u.includes('geojson')) {
+    return { url, format: 'json', score: 5 }
+  }
+
+  // 7. WFS Brut
+  if (p.includes('ogc:wfs') || p.includes('wfs') || u.includes('service=wfs')) {
+    return { url, format: 'wfs_service', score: 2, layerName: name }
+  }
+
+  return null
+}
+
+/**
+ * Parses the CSW metadata to find the best download URL based on heuristics and validation.
+ * @param metadata The parsed ISO 19139 metadata object
+ * @param resourceId The resource ID (used for logging and WFS typeName fallback)
+ * @param log The logger object for logging progress and warnings
+ * @returns An object containing the best URL and its format, or null if no valid link is found
+ */
+export const findBestDownloadUrl = async (metadata: any, resourceId: string, log: any): Promise<{ url: string, format: string } | null> => {
+  const root = metadata.MD_Metadata || metadata
+  const distributionInfo = root?.distributionInfo?.MD_Distribution
+  if (!distributionInfo) return null
+
+  const declaredFormats: string[] = []
+  if (distributionInfo.distributionFormat) {
+    const rawFormats = asArray(distributionInfo.distributionFormat)
+    for (const f of rawFormats) {
+      const formatName = getText(f.MD_Format?.name).toLowerCase()
+      if (formatName) declaredFormats.push(formatName)
+    }
+  }
+
+  const transferOptions = asArray(distributionInfo.transferOptions)
+
+  const allLinks: any[] = []
+  for (const transfer of transferOptions) {
+    const digitalTransfer = transfer.MD_DigitalTransferOptions || transfer
+    if (digitalTransfer && digitalTransfer.onLine) {
+      allLinks.push(...asArray(digitalTransfer.onLine))
+    }
+  }
+
+  if (allLinks.length === 0) return null
+
+  const candidates: DownloadCandidate[] = []
+  for (const link of allLinks) {
+    const candidate = analyzeLink(link)
+    if (candidate) candidates.push(candidate)
+  }
+
+  candidates.sort((a, b) => b.score - a.score)
+
+  if (candidates.length === 0) return null
+
+  log.info(`${candidates.length} liens candidats trouvés. Vérification...`)
+
+  let bestCandidate: DownloadCandidate | null = null
+
+  for (const candidate of candidates) {
+    // Prioritize direct links with format hints, but allow WFS if no better option is found
+    if (candidate.format !== 'wfs_service' || candidate.url.toLowerCase().includes('outputformat=')) {
+      if (await isUrlValid(candidate.url, log)) {
+        bestCandidate = candidate
+        log.info(`Lien direct validé : ${candidate.url}`)
+        break
+      }
+    } else if (!bestCandidate) {
+      bestCandidate = candidate
+    }
+  }
+
+  if (!bestCandidate) {
+    log.warning('Aucun lien n\'a passé le test de validation.')
+    return null
+  }
+
+  let { url, format, layerName } = bestCandidate
+
+  if (url.includes('/api/data/') && !url.toLowerCase().includes('format=')) {
+    const separator = url.includes('?') ? '&' : '?'
+    url = `${url}${separator}format=csv`
+    format = 'csv'
+  }
+
+  if (format === 'wfs_service' && !url.toLowerCase().includes('outputformat=')) {
+    log.info(`Service WFS détecté sur ${url}, test des formats supportés...`)
+    const [baseUrl, existingQuery] = url.split('?')
+    const params = new URLSearchParams(existingQuery || '')
+    const keysToDelete: string[] = []
+    for (const key of params.keys()) {
+      const lowerKey = key.toLowerCase()
+      if (['service', 'request', 'version', 'typename', 'typenames', 'outputformat', 'srsname'].includes(lowerKey)) {
+        keysToDelete.push(key)
+      }
+    }
+    keysToDelete.forEach(k => params.delete(k))
+
+    params.set('SERVICE', 'WFS')
+    params.set('VERSION', '2.0.0')
+    params.set('REQUEST', 'GetFeature')
+    if (layerName) {
+      params.set('TYPENAMES', layerName)
+    } else {
+      params.set('TYPENAMES', resourceId)
+    }
+
+    const formatsToTry = [
+      { param: 'application/json; subtype=geojson', format: 'geojson' },
+      { param: 'geojson', format: 'geojson' },
+      { param: 'application/json', format: 'geojson' },
+      { param: 'application/vnd.geo+json', format: 'geojson' },
+      { param: 'json', format: 'geojson' },
+      { param: 'SHAPE-ZIP', format: 'shapefile' },
+      { param: 'shapezip', format: 'shapefile' },
+      { param: 'application/zip', format: 'shapefile' },
+      { param: 'application/x-shapefile', format: 'shapefile' },
+      { param: 'csv', format: 'csv' },
+      { param: 'text/csv', format: 'csv' },
+      { param: 'kml', format: 'kml' },
+      { param: 'application/vnd.google-earth.kml+xml', format: 'kml' }
+    ]
+
+    let foundUrl = null
+    let foundFormat = null
+
+    for (const f of formatsToTry) {
+      const testParams = new URLSearchParams(params)
+      testParams.set('OUTPUTFORMAT', f.param)
+      const testUrl = `${baseUrl}?${testParams.toString()}`
+      if (await isUrlValid(testUrl, log, true)) {
+        log.info(`Format WFS supporté trouvé : ${f.param}`)
+        foundUrl = testUrl
+        foundFormat = f.format
+        break // We stop at the first valid format fFound, prioritizing GeoJSON and Shapefile over others
+      }
+    }
+
+    if (foundUrl && foundFormat) {
+      url = foundUrl
+      log.info(`URL finale WFS : ${url}`)
+      format = foundFormat
+    } else {
+      log.error('Ce service WFS ne propose aucun format supporté par DataFair (GeoJSON, Shapefile, KML, CSV)')
+      return null
+    }
+  }
+
+  return { url, format }
+}

package/lib/utils/types.ts

@@ -0,0 +1,18 @@
+type CswValue = string | string[] | undefined
+
+export interface CswRecord {
+  identifier?: CswValue
+  title?: CswValue
+  description?: CswValue
+  modified?: CswValue
+  format?: CswValue
+  protocol?: CswValue
+  references?: { scheme: string, value: string }[]
+}
+
+export interface DownloadCandidate {
+  url: string
+  format: string
+  score: number
+  layerName?: string
+}

package/package.json

@@ -0,0 +1,56 @@
+{
+  "name": "@data-fair/catalog-csw",
+  "description": "A simple CSW plugin for the Data Fair catalogs service.",
+  "version": "0.1.0",
+  "main": "index.ts",
+  "type": "module",
+  "scripts": {
+    "build-types": "df-build-types ./",
+    "check-types": "tsc",
+    "lint": "eslint .",
+    "prepare": "husky || true",
+    "test-base": "NODE_ENV=test EVENTS_LOG_LEVEL=alert node --disable-warning=ExperimentalWarning --test-force-exit --test-concurrency=1 --test",
+    "test-only": "npm run test-base -- --test-only test-it/*.ts",
+    "test": "npm run test-base test-it/*.ts",
+    "quality": "npm run lint && npm run build-types && npm run check-types && npm run test && npm audit --omit=dev --audit-level=critical",
+    "prepublishOnly": "npm run build-types"
+  },
+  "files": [
+    "./lib/**",
+    "./types/**",
+    "index.ts"
+  ],
+  "imports": {
+    "#types": "./types/index.ts",
+    "#type/*": "./types/*"
+  },
+  "keywords": [
+    "data-fair-catalogs-plugin"
+  ],
+  "license": "MIT",
+  "dependencies": {
+    "@data-fair/lib-validation": "^1.0.2",
+    "@data-fair/lib-node": "^2.8.1",
+    "@data-fair/lib-utils": "^1.6.0",
+    "fast-xml-parser": "^5.3.5",
+    "prom-client": "^15.1.3"
+  },
+  "devDependencies": {
+    "@commitlint/cli": "^19.8.0",
+    "@commitlint/config-conventional": "^19.8.0",
+    "@data-fair/lib-types-builder": "^1.8.0",
+    "@data-fair/types-catalogs": "^0.6.0",
+    "@types/fs-extra": "^11.0.4",
+    "chalk": "^5.4.1",
+    "dayjs": "^1.11.13",
+    "draftlog": "^1.0.13",
+    "eslint": "^9.25.1",
+    "fs-extra": "^11.3.0",
+    "husky": "^9.1.7",
+    "neostandard": "^0.12.1",
+    "typescript": "^5.8.3"
+  },
+  "relativeDependencies": {
+    "@data-fair/types-catalogs": "../../catalogs/types-catalogs"
+  }
+}

package/types/catalogConfig/.type/index.d.ts

@@ -0,0 +1,20 @@
+
+export const schemaExports: string[]
+
+// see https://github.com/bcherny/json-schema-to-typescript/issues/439 if some types are not exported
+/**
+ * URL of the CSW service to connect to (GeoNetwork, ...)
+ */
+export type CSWServiceURL = string;
+
+export type CSWConfig = {
+  url: CSWServiceURL;
+}
+
+
+export declare function validate(data: any): data is CswConfig
+export declare function assertValid(data: any, options?: import('@data-fair/lib-validation').AssertValidOptions): asserts data is CswConfig
+export declare function returnValid(data: any, options?: import('@data-fair/lib-validation').AssertValidOptions): CswConfig
+      
+export declare const schema: any
+        

package/types/catalogConfig/.type/index.js

@@ -0,0 +1,50 @@
+/* eslint-disable */
+
+
+import validate from './validate.js'
+import { assertValid as assertValidGeneric } from '@data-fair/lib-validation'
+
+export const schemaExports = [
+  "types",
+  "validate",
+  "schema"
+]
+
+export { validate } from './validate.js'
+export function assertValid(data, options) {
+  assertValidGeneric(validate, data, options)
+}
+export function returnValid(data, options) {
+  assertValid(data, options)
+  return data
+}
+
+export const schema = {
+  "$id": "https://github.com/data-fair/catalog-csw/catalog-config",
+  "x-exports": [
+    "types",
+    "validate",
+    "schema"
+  ],
+  "title": "CSWConfig",
+  "type": "object",
+  "additionalProperties": false,
+  "properties": {
+    "url": {
+      "title": "CSW service URL",
+      "x-i18n-title": {
+        "fr": "URL du service CSW"
+      },
+      "description": "URL of the CSW service to connect to (GeoNetwork, ...)",
+      "x-i18n-description": {
+        "fr": "URL du service CSW auquel se connecter (GeoNetwork, ...)"
+      },
+      "type": "string",
+      "format": "uri",
+      "default": "https://geobretagne.fr/geonetwork/srv/fre/csw"
+    }
+  },
+  "required": [
+    "url"
+  ]
+}

package/types/catalogConfig/.type/validate.js

@@ -0,0 +1,76 @@
+/* eslint-disable */
+// @ts-nocheck
+
+import { fullFormats } from "ajv-formats/dist/formats.js";
+"use strict";
+export const validate = validate14;
+export default validate14;
+const schema16 = {"$id":"https://github.com/data-fair/catalog-csw/catalog-config","x-exports":["types","validate","schema"],"title":"CSWConfig","type":"object","additionalProperties":false,"properties":{"url":{"title":"CSW service URL","x-i18n-title":{"fr":"URL du service CSW"},"description":"URL of the CSW service to connect to (GeoNetwork, ...)","x-i18n-description":{"fr":"URL du service CSW auquel se connecter (GeoNetwork, ...)"},"type":"string","format":"uri","default":"https://geobretagne.fr/geonetwork/srv/fre/csw"}},"required":["url"]};
+const formats0 = fullFormats.uri;
+
+function validate14(data, {instancePath="", parentData, parentDataProperty, rootData=data}={}){
+/*# sourceURL="https://github.com/data-fair/catalog-csw/catalog-config" */;
+let vErrors = null;
+let errors = 0;
+if(data && typeof data == "object" && !Array.isArray(data)){
+if(data.url === undefined){
+const err0 = {instancePath,schemaPath:"#/required",keyword:"required",params:{missingProperty: "url"},message:"must have required property '"+"url"+"'"};
+if(vErrors === null){
+vErrors = [err0];
+}
+else {
+vErrors.push(err0);
+}
+errors++;
+}
+for(const key0 in data){
+if(!(key0 === "url")){
+const err1 = {instancePath,schemaPath:"#/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key0},message:"must NOT have additional properties"};
+if(vErrors === null){
+vErrors = [err1];
+}
+else {
+vErrors.push(err1);
+}
+errors++;
+}
+}
+if(data.url !== undefined){
+let data0 = data.url;
+if(typeof data0 === "string"){
+if(!(formats0(data0))){
+const err2 = {instancePath:instancePath+"/url",schemaPath:"#/properties/url/format",keyword:"format",params:{format: "uri"},message:"must match format \""+"uri"+"\""};
+if(vErrors === null){
+vErrors = [err2];
+}
+else {
+vErrors.push(err2);
+}
+errors++;
+}
+}
+else {
+const err3 = {instancePath:instancePath+"/url",schemaPath:"#/properties/url/type",keyword:"type",params:{type: "string"},message:"must be string"};
+if(vErrors === null){
+vErrors = [err3];
+}
+else {
+vErrors.push(err3);
+}
+errors++;
+}
+}
+}
+else {
+const err4 = {instancePath,schemaPath:"#/type",keyword:"type",params:{type: "object"},message:"must be object"};
+if(vErrors === null){
+vErrors = [err4];
+}
+else {
+vErrors.push(err4);
+}
+errors++;
+}
+validate14.errors = vErrors;
+return errors === 0;
+}

package/types/catalogConfig/index.ts

@@ -0,0 +1 @@
+export * from './.type/index.js'

package/types/catalogConfig/schema.json

@@ -0,0 +1,27 @@
+{
+  "$id": "https://github.com/data-fair/catalog-csw/catalog-config",
+  "x-exports": [
+    "types",
+    "validate",
+    "schema"
+  ],
+  "title": "CSWConfig",
+  "type": "object",
+  "additionalProperties": false,
+  "properties": {
+    "url": {
+      "title": "CSW service URL",
+      "x-i18n-title": {
+        "fr": "URL du service CSW"
+      },
+      "description": "URL of the CSW service to connect to (GeoNetwork, ...)",
+      "x-i18n-description": {
+        "fr": "URL du service CSW auquel se connecter (GeoNetwork, ...)"
+      },
+      "type": "string",
+      "format": "uri",
+      "default": "https://geobretagne.fr/geonetwork/srv/fre/csw"
+    }
+  },
+  "required": ["url"]
+}

package/types/index.ts

@@ -0,0 +1 @@
+export { schema as configSchema, assertValid as assertConfigValid, type CSWConfig } from './catalogConfig/index.ts'