@data-fair/processing-web-scraper 0.6.2 → 0.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/scraper.ts +3 -3
- package/package.json +1 -1
package/lib/scraper.ts
CHANGED
|
@@ -282,10 +282,10 @@ export const run = async (context: ProcessingContext<ProcessingConfig>) => {
|
|
|
282
282
|
// lack of homogeneity between processings service and dev env
|
|
283
283
|
const location = err.headers?.location ?? err.response?.headers?.location
|
|
284
284
|
if (!location) {
|
|
285
|
-
log.error('redirection without a location header')
|
|
285
|
+
await log.error('redirection without a location header')
|
|
286
286
|
} else {
|
|
287
|
-
await log.debug(`page redirected ${page.url} -> ${
|
|
288
|
-
await pages.push({ url: new URL(
|
|
287
|
+
await log.debug(`page redirected ${page.url} -> ${location}`)
|
|
288
|
+
await pages.push({ url: new URL(location, page.url).href, source: 'redirect ' + page.url })
|
|
289
289
|
}
|
|
290
290
|
continue
|
|
291
291
|
}
|