@data-fair/processing-web-scraper 0.2.0 → 0.2.1

package/index.js

@@ -210,8 +210,9 @@ exports.run = async ({ pluginConfig, processingConfig, processingId, dir, tmpDir
     if (page.etag) headers['if-none-match'] = page.etag
     let response
     try {
-      response = await axios.get(page.url, { headers })
+      response = await axios.get(page.url, { headers, maxRedirects: 0 })
     } catch (err) {
+      // content did not change
       if (err.status === 304) {
         await log.debug(`page was not modified since last exploration ${page.url}`)
         sentIds.add(page._id)
@@ -220,6 +221,12 @@ exports.run = async ({ pluginConfig, processingConfig, processingId, dir, tmpDir
         }
         continue
       }
+      // follow a redirect
+      if (err.status === 301) {
+        await log.debug(`page redirected ${page.url} -> ${err.headers.location}`)
+        pages.push({ url: new URL(err.headers.location, page.url).href, source: 'redirect ' + page.url })
+        continue
+      }
       await log.warning(`failed to fetch page ${page.url} - ${err.status || err.message}`)
       if (page.source) await log.warning(`this broken URL comes from ${page.source}`)
       continue

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@data-fair/processing-web-scraper",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "description": "A small Web scraper that publishes its data into data-fair datasets.",
   "main": "index.js",
   "scripts": {
@@ -22,7 +22,7 @@
   },
   "homepage": "https://github.com/data-fair/processing-web-scraper#readme",
   "devDependencies": {
-    "@data-fair/processings-test-utils": "^0.5.0",
+    "@data-fair/processings-test-utils": "^0.5.1",
     "config": "^3.3.6",
     "eslint": "^7.18.0",
     "express": "^4.18.2",