@data-fair/processing-web-scraper 0.6.1 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/scraper.ts +8 -2
- package/package.json +1 -1
package/lib/scraper.ts
CHANGED
|
@@ -279,8 +279,14 @@ export const run = async (context: ProcessingContext<ProcessingConfig>) => {
|
|
|
279
279
|
continue
|
|
280
280
|
}
|
|
281
281
|
if (err.status === 301 || err.status === 302) {
|
|
282
|
-
|
|
283
|
-
|
|
282
|
+
// lack of homogeneity between processings service and dev env
|
|
283
|
+
const location = err.headers?.location ?? err.response?.headers?.location
|
|
284
|
+
if (!location) {
|
|
285
|
+
log.error('redirection without a location header')
|
|
286
|
+
} else {
|
|
287
|
+
await log.debug(`page redirected ${page.url} -> ${err.headers.location}`)
|
|
288
|
+
await pages.push({ url: new URL(err.headers.location, page.url).href, source: 'redirect ' + page.url })
|
|
289
|
+
}
|
|
284
290
|
continue
|
|
285
291
|
}
|
|
286
292
|
await log.warning(`failed to fetch page ${page.url} - ${err.status || err.message}`)
|