@data-fair/processing-web-scraper 0.6.0 → 0.6.2

package/lib/scraper.ts

@@ -1,7 +1,7 @@
 import { createHash } from 'node:crypto'
 import UrlPattern from 'url-pattern'
 import robotsParser from 'robots-parser'
-import cheerio from 'cheerio'
+import * as cheerio from 'cheerio'
 import FormData from 'form-data'
 import type { ProcessingContext } from '@data-fair/lib-common-types/processings.js'
 import type { ProcessingConfig } from '../types/processingConfig/index.ts'
@@ -84,6 +84,8 @@ interface Page {
   parentId?: string
 }
 
+type UrlPatternWithHostname = UrlPattern & { hostname: string }
+
 class PagesIterator {
   pages: Page[] = []
   cursor = -1
@@ -100,7 +102,7 @@ class PagesIterator {
     this.robots = robots
     this.excludeURLPatterns = (processingConfig.excludeURLPatterns || []).map((p: string) => {
       const url = new URL(p)
-      const pattern = new UrlPattern(url.pathname)
+      const pattern = new UrlPattern(url.pathname) as UrlPatternWithHostname
       pattern.hostname = url.hostname
       return pattern
     })
@@ -167,7 +169,7 @@ export const run = async (context: ProcessingContext<ProcessingConfig>) => {
     if (robots[origin]) continue
     try {
       const response = await axios.get(origin + '/robots.txt')
-      robots[origin] = robotsParser(origin + '/robots.txt', response.data)
+      robots[origin] = (robotsParser as any)(origin + '/robots.txt', response.data)
       for (const sitemap of robots[origin].getSitemaps()) {
         if (!sitemaps.includes(sitemap)) {
           await log.info(`add sitemap found in robots.txt ${sitemap}`)
@@ -277,8 +279,14 @@ export const run = async (context: ProcessingContext<ProcessingConfig>) => {
         continue
       }
       if (err.status === 301 || err.status === 302) {
-        await log.debug(`page redirected ${page.url} -> ${err.headers.location}`)
-        await pages.push({ url: new URL(err.headers.location, page.url).href, source: 'redirect ' + page.url })
+        // lack of homogeneity between processings service and dev env
+        const location = err.headers?.location ?? err.response?.headers?.location
+        if (!location) {
+          log.error('redirection without a location header')
+        } else {
+          await log.debug(`page redirected ${page.url} -> ${err.headers.location}`)
+          await pages.push({ url: new URL(err.headers.location, page.url).href, source: 'redirect ' + page.url })
+        }
         continue
       }
       await log.warning(`failed to fetch page ${page.url} - ${err.status || err.message}`)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@data-fair/processing-web-scraper",
-  "version": "0.6.0",
+  "version": "0.6.2",
   "description": "A small Web scraper that publishes its data into data-fair datasets.",
   "main": "index.ts",
   "type": "module",
@@ -10,7 +10,9 @@
     "build-types": "export NODE_OPTIONS='--experimental-strip-types' && df-build-types ./",
     "prepare": "husky || true",
     "test-base": "NODE_ENV=test node --experimental-strip-types --test-force-exit --test-concurrency=1 --test --test-reporter=spec --test-reporter-destination=stdout --test-timeout=300000",
-    "test": "npm run test-base test-it/*.ts"
+    "test": "npm run test-base test-it/*.ts",
+    "check-types": "tsc",
+    "quality": "npm run lint && npm run check-types && npm test"
   },
   "repository": {
     "type": "git",
@@ -42,9 +44,9 @@
   },
   "dependencies": {
     "@data-fair/lib-utils": "^1.9.0",
-    "cheerio": "^1.0.0-rc.12",
-    "form-data": "^4.0.0",
-    "robots-parser": "^3.0.0",
+    "cheerio": "^1.2.0",
+    "form-data": "^4.0.5",
+    "robots-parser": "^3.0.1",
     "url-pattern": "^1.0.3"
   },
   "files": [