@data-fair/catalog-data-fair 0.1.0 → 0.2.0

package/lib/download.ts

@@ -4,6 +4,7 @@ import axios from '@data-fair/lib-node/axios.js'
 import * as fs from 'fs'
 import { join } from 'path'
 import { Transform } from 'stream'
+import slugify from 'slugify'
 
 /**
  * Retrieves a resource by first fetching its metadata and then downloading the actual resource.
@@ -15,8 +16,8 @@ import { Transform } from 'stream'
 export const getResource = async (context: GetResourceContext<DataFairConfig>): ReturnType<CatalogPlugin['getResource']> => {
   context.log.step('Import de la ressource')
 
-  const resource = await getMetaData(context)
-  resource.filePath = await downloadResource(context, resource)
+  const { resource, file } = await getMetaData(context)
+  resource.filePath = await downloadResource(context, file, resource)
 
   return resource
 }
@@ -27,11 +28,12 @@ export const getResource = async (context: GetResourceContext<DataFairConfig>):
  * @param resourceId the dataset Id to fetch fields from
  * @returns the Resource corresponding to the id by this configuration
  */
-const getMetaData = async ({ catalogConfig, resourceId, log }: GetResourceContext<DataFairConfig>): Promise<Resource> => {
+const getMetaData = async ({ catalogConfig, resourceId, log, secrets }: GetResourceContext<DataFairConfig>): Promise<{ resource: Resource, file: boolean }> => {
   let dataset: DataFairDataset
   try {
     const url = `${catalogConfig.url}/data-fair/api/v1/datasets/${resourceId}`
-    const res = (await axios.get(url))
+    const config = secrets.apiKey ? { headers: { 'x-apiKey': secrets.apiKey } } : undefined
+    const res = (await axios.get(url, config))
     if (res.status !== 200) {
       throw new Error(`HTTP error : ${res.status}, ${res.data}`)
     }
@@ -42,6 +44,17 @@ const getMetaData = async ({ catalogConfig, resourceId, log }: GetResourceContex
     throw new Error(`Erreur lors de la récuperation de la resource DataFair. ${e instanceof Error ? e.message : e}`)
   }
 
+  dataset.schema = (dataset.schema ?? []).map((field) => {
+    if (field['x-extension']) {
+      return {
+        ...field,
+        key: slugify.default(field.key.replace(/^_/, ''), { lower: true, strict: true, replacement: '_' }), // Ensure no leading underscore
+        'x-extension': undefined,   // Remove x-extension property if it exists
+      }
+    }
+    return field
+  })
+
   const resource: Resource = {
     id: resourceId,
     title: dataset.title,
@@ -51,7 +64,7 @@ const getMetaData = async ({ catalogConfig, resourceId, log }: GetResourceContex
     frequency: dataset.frequency,
     image: dataset.image,
     keywords: dataset.keywords,
-    size: dataset.file?.size,
+    size: dataset.file?.size ?? dataset.storage?.size ?? dataset.originalFile?.size,
     schema: dataset.schema,
     filePath: '',
   }
@@ -62,7 +75,8 @@ const getMetaData = async ({ catalogConfig, resourceId, log }: GetResourceContex
       href: dataset.license.href ?? '',
     }
   }
-  return resource
+
+  return { resource, file: !!dataset.file }
 }
 
 /**
@@ -72,12 +86,14 @@ const getMetaData = async ({ catalogConfig, resourceId, log }: GetResourceContex
  * @param res - the metadatas about the resource.
  * @returns A promise resolving to the file path of the downloaded CSV.
  */
-const downloadResource = async (context: GetResourceContext<DataFairConfig>, res: Resource): Promise<string> => {
+const downloadResource = async (context: GetResourceContext<DataFairConfig>, file: boolean, res: Resource): Promise<string> => {
   const filePath = join(context.tmpDir, `${context.resourceId}.csv`)
   try {
-    if (res.size && res.size > 0 && context.importConfig.fields?.length === 0 && context.importConfig.filters?.length === 0) {
+    if (file && !context.importConfig.fields?.length && !context.importConfig.filters?.length) {
+      await context.log.task('downloading', 'Téléchargement en cours...  (taille approximative)', res.size || NaN)
       await downloadResourceFile(filePath, context)
     } else {
+      await context.log.task('downloading', 'Téléchargement en cours...', NaN)
       await downloadResourceLines(filePath, context)
     }
     return filePath
@@ -97,29 +113,47 @@ const downloadResource = async (context: GetResourceContext<DataFairConfig>, res
  * @returns A promise that resolves when the file is successfully downloaded and saved.
  * @throws If there is an error writing the file or fetching the dataset.
  */
-const downloadResourceFile = async (filePath: string, { catalogConfig, resourceId, log }: GetResourceContext<DataFairConfig>): Promise<void> => {
+const downloadResourceFile = async (filePath: string, { catalogConfig, resourceId, log, secrets }: GetResourceContext<DataFairConfig>): Promise<void> => {
   const url = `${catalogConfig.url}/data-fair/api/v1/datasets/${resourceId}/full`
-  log.info('Import des données de la ressource', url)
+  const headers = secrets.apiKey ? { 'x-apiKey': secrets.apiKey } : undefined
 
-  const fileStream = fs.createWriteStream(filePath)
-
-  const response = await axios.get(url, { responseType: 'stream' })
+  const response = await axios.get(url, { responseType: 'stream', headers })
 
   if (response.status !== 200) {
     throw new Error(`Error while fetching data: HTTP ${response.statusText}`)
   }
 
+  let downloaded = 0
+  let logPromise: Promise<void> | null = null
+
   return new Promise<void>((resolve, reject) => {
+    const fileStream = fs.createWriteStream(filePath, { encoding: 'binary' }) // Ensure binary encoding
+
+    response.data.on('data', (chunk: Buffer) => {
+      downloaded += chunk.length
+      if (!logPromise) {
+        logPromise = log.progress('downloading', downloaded)
+          .catch(err => console.warn('Progress logging failed:', err))
+          .finally(() => { logPromise = null })
+      }
+    })
+
     response.data.pipe(fileStream)
+
     fileStream.on('finish', () => {
+      fileStream.close()
       resolve()
     })
+
     response.data.on('error', (err: any) => {
-      fs.unlink(filePath, () => { }) // Delete the file in case of error
+      fileStream.destroy()
+      fs.unlink(filePath, () => { })
       reject(err)
     })
+
     fileStream.on('error', (err) => {
-      fs.unlink(filePath, () => { }) // Delete the file in case of error
+      response.data.destroy()
+      fs.unlink(filePath, () => { })
       reject(err)
     })
   })
@@ -135,13 +169,15 @@ const downloadResourceFile = async (filePath: string, { catalogConfig, resourceI
  * @returns A promise that resolves when the file is successfully downloaded and saved.
  * @throws If there is an error writing the file or fetching the dataset.
  */
-const downloadResourceLines = async (destFile: string, { catalogConfig, resourceId, importConfig, log }: GetResourceContext<DataFairConfig> & { importConfig: ImportConfig }) => {
-  let url: string | null = `${catalogConfig.url}/data-fair/api/v1/datasets/${resourceId}/lines?format=csv&size=3000`
+const downloadResourceLines = async (destFile: string, { catalogConfig, resourceId, importConfig, secrets, log }: GetResourceContext<DataFairConfig> & { importConfig: ImportConfig }) => {
+  let url: string | null = `${catalogConfig.url}/data-fair/api/v1/datasets/${resourceId}/lines?format=csv&size=5000`
 
   if (importConfig.fields) {
     url += '&select=' + importConfig.fields.map(field => field.key).join(',')
   }
 
+  const headers = secrets.apiKey ? { 'x-apiKey': secrets.apiKey } : undefined
+
   if (importConfig.filters) {
     importConfig.filters.forEach((filter) => {
       switch (filter.type) {
@@ -160,13 +196,14 @@ const downloadResourceLines = async (destFile: string, { catalogConfig, resource
     })
   }
 
-  log.info('Import des données de la ressource', url)
+  let downloaded = 0
+  let pendingLogPromise: Promise<void> | null = null
+
   const writer = fs.createWriteStream(destFile)
   let isFirstChunk = true
 
   while (url) {
-    console.log(url)
-    const response = await axios.get(url, { responseType: 'stream' })
+    const response = await axios.get(url, { responseType: 'stream', headers })
     if (response.status !== 200) {
       throw new Error(`Error while fetching data: HTTP ${response.statusText}`)
     }
@@ -189,6 +226,18 @@ const downloadResourceLines = async (destFile: string, { catalogConfig, resource
           }
         }))
       }
+
+      stream.on('data', (chunk: Buffer) => {
+        downloaded += chunk.length
+
+        // Only start a new log promise if there isn't one already running
+        if (!pendingLogPromise) {
+          pendingLogPromise = log.progress('downloading', downloaded)
+            .catch(err => console.warn('Progress logging failed:', err))
+            .finally(() => { pendingLogPromise = null })
+        }
+      })
+
       stream.pipe(writer, { end: false })
       stream.on('end', () => {
         const linkHeader = response.headers.link

package/lib/imports.ts

@@ -20,6 +20,7 @@ const prepareCatalog = (dataFairDatasets: DataFairDataset[]): ResourceList => {
       format: 'csv',
       size: dataFairDataset.file?.size ?? dataFairDataset.storage?.size ?? dataFairDataset.originalFile?.size,
       type: 'resource',
+      origin: dataFairDataset.page
     } as ResourceList[number])
   }
   return catalog
@@ -38,8 +39,9 @@ export const listResources = async (config: ListResourcesContext<DataFairConfig,
 
   let data: DataFairCatalog
   const url = `${config.catalogConfig.url}/data-fair/api/v1/catalog/datasets`
+  const headers = config.secrets.apiKey ? { 'x-apiKey': config.secrets.apiKey } : undefined
   try {
-    const res = (await axios.get(url, { params: dataFairParams }))
+    const res = (await axios.get(url, { params: dataFairParams, headers }))
     if (res.status !== 200) {
       throw new Error(`HTTP error : ${res.status}, ${res.data}`)
     }

package/lib/prepare.ts

@@ -4,15 +4,27 @@ import type { DataFairConfig } from '#types'
 import axios from '@data-fair/lib-node/axios.js'
 
 export default async ({ catalogConfig, capabilities, secrets }: PrepareContext<DataFairConfig, DataFairCapabilities>) => {
-  // To remove when catalog/datasets allows q parameters
-  capabilities = capabilities.filter(c => c !== 'search')
+  // set the apiKey in the secrets field if it exists
+  const apiKey = catalogConfig.apiKey
+  if (apiKey && apiKey !== '*************************') {
+    secrets.apiKey = apiKey
+    catalogConfig.apiKey = '*************************'
+  } else if (secrets?.apiKey && (!apiKey || apiKey === '')) {
+    delete secrets.apiKey
+  } else {
+    // The secret is already set, do nothing
+  }
 
   // test the url
   try {
-    await axios.get(catalogConfig.url + '/data-fair/api/v1/catalog/datasets?size=1&select=id')
+    if (!catalogConfig.url) {
+      throw new Error('URL du catalogue non définie')
+    }
+    const config = secrets.apiKey ? { headers: { 'x-apiKey': secrets.apiKey } } : undefined
+    await axios.get(catalogConfig.url + '/data-fair/api/v1/catalog/datasets?size=1&select=id', config)
   } catch (e) {
     console.error('Erreur URL pendant la configuration : ', e instanceof Error ? e.message : e)
-    throw new Error('Configuration invalide, vérifiez l\'URL')
+    throw new Error(`Configuration invalide, veuillez vérifier l’URL du catalogue et la clé API si nécessaire (${e instanceof Error ? e.message : e})`)
   }
 
   return {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@data-fair/catalog-data-fair",
   "description": "A simple Data Fair plugin for the Data Fair catalogs service.",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "main": "index.ts",
   "type": "module",
   "scripts": {
@@ -31,7 +31,8 @@
   "dependencies": {
     "@data-fair/lib-node": "^2.8.2",
     "@data-fair/lib-utils": "^1.6.0",
-    "prom-client": "^15.1.3"
+    "prom-client": "^15.1.3",
+    "slugify": "^1.6.6"
   },
   "devDependencies": {
     "@commitlint/cli": "^19.8.0",

package/types/catalogConfig/.type/index.d.ts

@@ -6,9 +6,14 @@ export const schemaExports: string[]
  * The url of the catalog
  */
 export type URL = string;
+/**
+ * The Data Fair API key to access the catalog. You can create one from the 'Settings' tab in Data Fair, under the 'API Keys' section.
+ */
+export type APIKeyOptional = string;
 
 export type DataFairConfig = {
   url: URL;
+  apiKey?: APIKeyOptional;
 }
 
 

package/types/catalogConfig/.type/index.js

@@ -38,7 +38,7 @@ export const schema = {
       "title": "URL",
       "description": "The url of the catalog",
       "x-i18n-description": {
-        "fr": "L'URL du site où le catalogue est publié. *L'URL de l'API utilisée sera* *https*[]()*://example.com***/data-fair/api/v1**"
+        "fr": "L'URL du portail où le catalogue est publié. Par exemple: `https://opendata.koumoul.com`."
       },
       "pattern": "^https?://.*[^/]$",
       "errorMessage": "The URL must start with http:// or https:// and must not end with `/`.",
@@ -46,8 +46,25 @@ export const schema = {
         "fr": "L'URL doit commencer par http:// ou https:// et ne pas se terminer par `/`."
       },
       "examples": [
-        "https://example.com"
+        "https://opendata.koumoul.com"
       ]
+    },
+    "apiKey": {
+      "type": "string",
+      "title": "API Key (Optional)",
+      "x-i18n-title": {
+        "fr": "Clé API (Optionnelle)"
+      },
+      "description": "The Data Fair API key to access the catalog. You can create one from the 'Settings' tab in Data Fair, under the 'API Keys' section.",
+      "x-i18n-description": {
+        "fr": "La clé API Data Fair pour accéder au catalogue. Vous pouvez en créer depuis l'onglet 'Paramètres' de Data Fair, dans la section 'Clés d'API'."
+      },
+      "layout": {
+        "props": {
+          "type": "password",
+          "autocomplete": "new-password"
+        }
+      }
     }
   }
 }

package/types/catalogConfig/.type/validate.js

@@ -4,7 +4,7 @@
 "use strict";
 export const validate = validate14;
 export default validate14;
-const schema16 = {"$id":"https://github.com/data-fair/catalog-data-fair/catalog-config","x-exports":["types","validate","schema"],"title":"DataFairConfig","type":"object","additionalProperties":false,"required":["url"],"properties":{"url":{"type":"string","title":"URL","description":"The url of the catalog","x-i18n-description":{"fr":"L'URL du site où le catalogue est publié. *L'URL de l'API utilisée sera* *https*[]()*://example.com***/data-fair/api/v1**"},"pattern":"^https?://.*[^/]$","errorMessage":"The URL must start with http:// or https:// and must not end with `/`.","x-i18n-errorMessage":{"fr":"L'URL doit commencer par http:// ou https:// et ne pas se terminer par `/`."},"examples":["https://example.com"]}}};
+const schema16 = {"$id":"https://github.com/data-fair/catalog-data-fair/catalog-config","x-exports":["types","validate","schema"],"title":"DataFairConfig","type":"object","additionalProperties":false,"required":["url"],"properties":{"url":{"type":"string","title":"URL","description":"The url of the catalog","x-i18n-description":{"fr":"L'URL du portail où le catalogue est publié. Par exemple: `https://opendata.koumoul.com`."},"pattern":"^https?://.*[^/]$","errorMessage":"The URL must start with http:// or https:// and must not end with `/`.","x-i18n-errorMessage":{"fr":"L'URL doit commencer par http:// ou https:// et ne pas se terminer par `/`."},"examples":["https://opendata.koumoul.com"]},"apiKey":{"type":"string","title":"API Key (Optional)","x-i18n-title":{"fr":"Clé API (Optionnelle)"},"description":"The Data Fair API key to access the catalog. You can create one from the 'Settings' tab in Data Fair, under the 'API Keys' section.","x-i18n-description":{"fr":"La clé API Data Fair pour accéder au catalogue. Vous pouvez en créer depuis l'onglet 'Paramètres' de Data Fair, dans la section 'Clés d'API'."},"layout":{"props":{"type":"password","autocomplete":"new-password"}}}}};
 const pattern0 = new RegExp("^https?://.*[^/]$", "u");
 
 function validate14(data, {instancePath="", parentData, parentDataProperty, rootData=data}={}){
@@ -23,7 +23,7 @@ vErrors.push(err0);
 errors++;
 }
 for(const key0 in data){
-if(!(key0 === "url")){
+if(!((key0 === "url") || (key0 === "apiKey"))){
 const err1 = {instancePath,schemaPath:"#/additionalProperties",keyword:"additionalProperties",params:{additionalProperty: key0},message:"must NOT have additional properties"};
 if(vErrors === null){
 vErrors = [err1];
@@ -86,9 +86,9 @@ vErrors = emErrs1;
 errors = emErrs1.length;
 }
 }
-}
-else {
-const err7 = {instancePath,schemaPath:"#/type",keyword:"type",params:{type: "object"},message:"must be object"};
+if(data.apiKey !== undefined){
+if(typeof data.apiKey !== "string"){
+const err7 = {instancePath:instancePath+"/apiKey",schemaPath:"#/properties/apiKey/type",keyword:"type",params:{type: "string"},message:"must be string"};
 if(vErrors === null){
 vErrors = [err7];
 }
@@ -97,6 +97,18 @@ vErrors.push(err7);
 }
 errors++;
 }
+}
+}
+else {
+const err8 = {instancePath,schemaPath:"#/type",keyword:"type",params:{type: "object"},message:"must be object"};
+if(vErrors === null){
+vErrors = [err8];
+}
+else {
+vErrors.push(err8);
+}
+errors++;
+}
 validate14.errors = vErrors;
 return errors === 0;
 }

package/types/catalogConfig/schema.json

@@ -17,14 +17,33 @@
       "title": "URL",
       "description": "The url of the catalog",
       "x-i18n-description": {
-        "fr": "L'URL du site où le catalogue est publié. *L'URL de l'API utilisée sera* *https*[]()*://example.com***/data-fair/api/v1**"
+        "fr": "L'URL du portail où le catalogue est publié. Par exemple: `https://opendata.koumoul.com`."
       },
       "pattern": "^https?://.*[^/]$",
       "errorMessage": "The URL must start with http:// or https:// and must not end with `/`.",
       "x-i18n-errorMessage": {
         "fr": "L'URL doit commencer par http:// ou https:// et ne pas se terminer par `/`."
       },
-      "examples": ["https://example.com"]
+      "examples": [
+        "https://opendata.koumoul.com"
+      ]
+    },
+    "apiKey": {
+      "type": "string",
+      "title": "API Key (Optional)",
+      "x-i18n-title": {
+        "fr": "Clé API (Optionnelle)"
+      },
+      "description": "The Data Fair API key to access the catalog. You can create one from the 'Settings' tab in Data Fair, under the 'API Keys' section.",
+      "x-i18n-description": {
+        "fr": "La clé API Data Fair pour accéder au catalogue. Vous pouvez en créer depuis l'onglet 'Paramètres' de Data Fair, dans la section 'Clés d'API'."
+      },
+      "layout": {
+        "props": {
+          "type": "password",
+          "autocomplete": "new-password"
+        }
+      }
     }
   }
-}
+}

package/types/datafairSchemas/schema.json

@@ -1,5 +1,5 @@
 {
-  "$id": "https://github.com/data-fair/catalog-data-fair/catlog-schemas",
+  "$id": "https://github.com/data-fair/catalog-data-fair/catalog-schemas",
   "x-exports": [
     "types"
   ],