@opentermsarchive/engine 10.3.1 → 10.3.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "10.3.1",
+  "version": "10.3.3",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://opentermsarchive.org",
   "bugs": {

package/scripts/dataset/publish/github/index.js

@@ -30,7 +30,7 @@ export default async function publish({ archivePath, releaseDate, stats }) {
   logger.info('Uploading release asset…');
 
   await octokit.rest.repos.uploadReleaseAsset({
-    data: fsApi.readFileSync(archivePath),
+    data: fsApi.createReadStream(archivePath),
     headers: {
       'content-type': 'application/zip',
       'content-length': fsApi.statSync(archivePath).size,

package/scripts/dataset/publish/index.js

@@ -22,26 +22,29 @@ export default async function publishRelease({ archivePath, releaseDate, stats }
     throw new Error('No publishing platform configured. Please configure at least one of: GitHub (OTA_ENGINE_GITHUB_TOKEN), GitLab (OTA_ENGINE_GITLAB_TOKEN), or data.gouv.fr (OTA_ENGINE_DATAGOUV_API_KEY + datasetId or organizationIdOrSlug in config).');
   }
 
-  const results = await Promise.allSettled(platforms.map(async platform => {
-    const url = await platform.publish();
-
-    return { platform: platform.name, url };
-  }));
-
-  const succeeded = results.filter(result => result.status === 'fulfilled');
-  const failed = results.filter(result => result.status === 'rejected');
+  const succeeded = [];
+  const failed = [];
+
+  // Execute publications sequentially to avoid memory issues with large file uploads
+  for (const platform of platforms) {
+    try {
+      const url = await platform.publish();
+
+      succeeded.push({ platform: platform.name, url });
+    } catch (error) {
+      failed.push({ platform: platform.name, error });
+    }
+  }
 
   if (failed.length) {
     let errorMessage = !succeeded.length ? 'All platforms failed to publish:' : 'Some platforms failed to publish:';
 
-    failed.forEach(rejectedResult => {
-      const index = results.indexOf(rejectedResult);
-
-      errorMessage += `\n  - ${platforms[index].name}: ${rejectedResult.reason.message}`;
+    failed.forEach(({ platform, error }) => {
+      errorMessage += `\n  - ${platform}: ${error.message}`;
     });
 
     logger.error(errorMessage);
   }
 
-  return succeeded.map(result => result.value);
+  return succeeded;
 }

package/src/archivist/extract/index.js

@@ -35,15 +35,16 @@ export async function extractFromHTML(sourceDocument) {
   const filteredDOM = await filter(webPageDOM, sourceDocument);
   const cleanedDOM = filteredDOM.remove(insignificantContentSelectors);
   const selectedDOM = cleanedDOM.select(contentSelectors);
+  const contentSelectorsDisplay = typeof contentSelectors === 'object' ? JSON.stringify(contentSelectors) : contentSelectors;
 
   if (!selectedDOM?.children.length) {
-    throw new Error(`The provided selector "${contentSelectors}" has no match in the web page at '${location}'. This could be due to elements being removed before content selection if "remove" and "select" selectors match the same content.`);
+    throw new Error(`The provided selector "${contentSelectorsDisplay}" has no match in the web page at '${location}'. This could be due to elements being removed before content selection if "remove" and "select" selectors match the same content.`);
   }
 
   const markdownContent = transformFromHTML(selectedDOM);
 
   if (!markdownContent) {
-    throw new Error(`The provided selector "${contentSelectors}" matches an empty content in the web page at '${location}'`);
+    throw new Error(`The provided selector "${contentSelectorsDisplay}" matches an empty content in the web page at '${location}'`);
   }
 
   return markdownContent;