@data-fair/processing-web-scraper 0.4.0 → 0.6.0

package/index.ts

@@ -0,0 +1,19 @@
+import type { PrepareFunction, ProcessingContext } from '@data-fair/lib-common-types/processings.js'
+import type { ProcessingConfig } from './types/processingConfig/index.ts'
+
+export const prepare: PrepareFunction<ProcessingConfig> = async ({ processingConfig, secrets }) => {
+  return {
+    processingConfig,
+    secrets
+  }
+}
+
+export const run = async (context: ProcessingContext<ProcessingConfig>) => {
+  const { run } = await import('./lib/scraper.ts')
+  await run(context)
+}
+
+export const stop = async () => {
+  const { stop } = await import('./lib/scraper.ts')
+  await stop()
+}

package/{index.js → lib/scraper.ts}

@@ -1,11 +1,10 @@
-const FormData = require('form-data')
-const crypto = require('crypto')
-const robotsParser = require('robots-parser')
-
-// TODO:
-// handle html but also any file formats
-// add in-links info (at least for files)
-// specifications listed here http://robots-txt.com/
+import { createHash } from 'node:crypto'
+import UrlPattern from 'url-pattern'
+import robotsParser from 'robots-parser'
+import cheerio from 'cheerio'
+import FormData from 'form-data'
+import type { ProcessingContext } from '@data-fair/lib-common-types/processings.js'
+import type { ProcessingConfig } from '../types/processingConfig/index.ts'
 
 const datasetSchema = [
   {
@@ -14,6 +13,12 @@ const datasetSchema = [
     'x-refersTo': 'http://www.w3.org/2000/01/rdf-schema#label',
     'x-capabilities': { textAgg: false }
   },
+  {
+    key: 'description',
+    type: 'string',
+    'x-refersTo': 'https://schema.org/description',
+    'x-capabilities': { values: false, textAgg: false, insensitive: false }
+  },
   {
     key: 'url',
     type: 'string',
@@ -30,7 +35,6 @@ const datasetSchema = [
   {
     key: 'etag',
     type: 'string',
-    separator: ',',
     'x-capabilities': { index: false, values: false, text: false, textStandard: false, textAgg: false, insensitive: false }
   },
   {
@@ -46,10 +50,9 @@ const datasetSchema = [
   }
 ]
 
-// a global variable to manage interruption
-let stopped
+let stopped: boolean | undefined
 
-const normalizeURL = (url, ignoreHash = false, addSlash = false) => {
+const normalizeURL = (url: string, ignoreHash = false, addSlash = false): string => {
   const parsedURL = new URL(url)
   for (const indexSuffix of ['index.html', 'index.php', 'index.jsp', 'index.cgi']) {
     if (parsedURL.pathname.endsWith('/' + indexSuffix)) {
@@ -61,20 +64,41 @@ const normalizeURL = (url, ignoreHash = false, addSlash = false) => {
   return parsedURL.href
 }
 
-const getId = (page) => {
-  return crypto.createHash('sha256').update(normalizeURL(page.url)).digest('base64url').slice(0, 20)
+const getId = async (page: { url: string }): Promise<string> => {
+  return createHash('sha256').update(normalizeURL(page.url)).digest('base64url').slice(0, 20)
+}
+
+interface Page {
+  url: string
+  title?: string
+  description?: string
+  tags?: string[]
+  etag?: string
+  lastModified?: string
+  attachmentPath?: string
+  _id?: string
+  parsedURL?: URL
+  source?: string
+  noindex?: boolean
+  nofollow?: boolean
+  parentId?: string
 }
 
 class PagesIterator {
-  constructor (log, pluginConfig, processingConfig, robots) {
-    this.pages = []
-    this.cursor = -1
+  pages: Page[] = []
+  cursor = -1
+  log: any
+  pluginConfig: any
+  processingConfig: ProcessingConfig
+  robots: Record<string, any>
+  excludeURLPatterns: any[] = []
+
+  constructor (log: any, pluginConfig: any, processingConfig: ProcessingConfig, robots: Record<string, any>) {
     this.log = log
     this.pluginConfig = pluginConfig
     this.processingConfig = processingConfig
     this.robots = robots
-    const UrlPattern = require('url-pattern')
-    this.excludeURLPatterns = (processingConfig.excludeURLPatterns || []).map(p => {
+    this.excludeURLPatterns = (processingConfig.excludeURLPatterns || []).map((p: string) => {
       const url = new URL(p)
       const pattern = new UrlPattern(url.pathname)
       pattern.hostname = url.hostname
@@ -86,19 +110,19 @@ class PagesIterator {
     return this
   }
 
-  push (page) {
+  async push (page: Page | string) {
     if (typeof page === 'string') page = { url: page }
-    if (!this.processingConfig.baseURLs.find(b => page.url.startsWith(b))) return
+    if (!this.processingConfig.baseURLs?.find((b: string) => page.url.startsWith(b))) return
     page.parsedURL = page.parsedURL || new URL(page.url)
     if (page.parsedURL.hash) return
-    if (this.excludeURLPatterns.find(p => p.match(page.parsedURL.pathname) && p.hostname === page.parsedURL.hostname)) {
+    if (this.excludeURLPatterns.find((p: any) => p.match(page.parsedURL!.pathname) && p.hostname === page.parsedURL!.hostname)) {
       return
     }
-    if (this.robots[page.parsedURL.origin] && !this.robots[page.parsedURL.origin].isAllowed(page.url, this.pluginConfig.userAgent || 'data-fair-web-scraper')) {
+    if (this.robots[page.parsedURL!.origin] && !this.robots[page.parsedURL!.origin].isAllowed(page.url, this.pluginConfig.userAgent || 'data-fair-web-scraper')) {
       return
     }
-    page._id = getId(page)
-    if (this.pages.find(p => p._id === page._id)) return
+    page._id = await getId(page)
+    if (this.pages.find((p: Page) => p._id === page._id)) return
     this.pages.push(page)
   }
 
@@ -112,40 +136,45 @@ class PagesIterator {
   }
 }
 
-exports.run = async ({ pluginConfig, processingConfig, processingId, dir, tmpDir, axios, log, patchConfig, ws }) => {
-  let dataset
+export const run = async (context: ProcessingContext<ProcessingConfig>) => {
+  const { pluginConfig, processingConfig, processingId, axios, log, patchConfig, ws } = context
+  let dataset: any
   if (processingConfig.datasetMode === 'create') {
     await log.step('Dataset creation')
     dataset = (await axios.post('api/v1/datasets', {
-      id: processingConfig.dataset.id,
-      title: processingConfig.dataset.title,
+      id: processingConfig.dataset?.id,
+      title: processingConfig.dataset?.title,
       isRest: true,
       schema: datasetSchema,
       extras: { processingId }
     })).data
+    if (dataset.status !== 'finalized') {
+      await ws.waitForJournal(dataset.id, 'finalize-end')
+    }
     await log.info(`dataset created, id="${dataset.id}", title="${dataset.title}"`)
     await patchConfig({ datasetMode: 'update', dataset: { id: dataset.id, title: dataset.title } })
-    await ws.waitForJournal(dataset.id, 'finalize-end')
   } else if (processingConfig.datasetMode === 'update') {
     await log.step('Check dataset')
-    dataset = (await axios.get(`api/v1/datasets/${processingConfig.dataset.id}`)).data
-    if (!dataset) throw new Error(`the dataset does not exist, id="${processingConfig.dataset.id}"`)
+    dataset = (await axios.get(`api/v1/datasets/${processingConfig.dataset?.id}`)).data
+    if (!dataset) throw new Error(`the dataset does not exist, id="${processingConfig.dataset?.id}"`)
     await log.info(`the dataset exists, id="${dataset.id}", title="${dataset.title}"`)
   }
 
-  // parse the robots.txt files if available
-  const robots = {}
+  const robots: Record<string, any> = {}
   const sitemaps = processingConfig.sitemaps || []
-  for (const baseURL of processingConfig.baseURLs) {
+  for (const baseURL of processingConfig.baseURLs || []) {
     const { origin } = new URL(baseURL)
     if (robots[origin]) continue
     try {
       const response = await axios.get(origin + '/robots.txt')
       robots[origin] = robotsParser(origin + '/robots.txt', response.data)
       for (const sitemap of robots[origin].getSitemaps()) {
-        if (!sitemaps.includes(sitemap)) sitemaps.push(sitemap)
+        if (!sitemaps.includes(sitemap)) {
+          await log.info(`add sitemap found in robots.txt ${sitemap}`)
+          sitemaps.push(sitemap)
+        }
       }
-    } catch (err) {
+    } catch (err: any) {
       await log.info(`failed to fetch ${origin + '/robots.txt'} - ${err.status || err.message}`)
     }
   }
@@ -153,32 +182,33 @@ exports.run = async ({ pluginConfig, processingConfig, processingId, dir, tmpDir
   const pages = new PagesIterator(log, pluginConfig, processingConfig, robots)
 
   await log.step('Init pages list')
-  let existingPages
+  let existingPages: Page[] | undefined
   if (processingConfig.datasetMode === 'update') {
     existingPages = (await axios.get(`api/v1/datasets/${dataset.id}/lines`, { params: { select: '_id,url,etag,lastModified', size: 10000 } })).data.results
-    await log.info(`add ${existingPages.length} pages from previous crawls`)
-    for (const page of existingPages) {
-      page.parsedURL = new URL(page.url)
-      if (page.parsedURL.hash) {
-        const parentURL = new URL(page.parsedURL)
-        parentURL.hash = ''
-        page.parentId = getId({ url: parentURL.href })
+    if (existingPages) {
+      await log.info(`add ${existingPages.length} pages from previous crawls`)
+      for (const page of existingPages) {
+        page.parsedURL = new URL(page.url)
+        if (page.parsedURL.hash) {
+          const parentURL = new URL(page.parsedURL)
+          parentURL.hash = ''
+          page.parentId = await getId({ url: parentURL.href })
+        }
+        await pages.push({ ...page, source: 'previous exploration' })
       }
-      await pages.push({ ...page, source: 'previous exploration' })
     }
   }
-  await log.info(`add ${processingConfig.startURLs.length} pages from config`)
-  for (const url of processingConfig.startURLs) {
+  await log.info(`add ${processingConfig.startURLs?.length || 0} pages from config`)
+  for (const url of processingConfig.startURLs || []) {
     await pages.push({ url, source: 'config start URLs' })
   }
 
   for (const sitemapURL of sitemaps) {
     await log.info(`fetch start URLs from sitemap ${sitemapURL}`)
     const sitemap = (await axios.get(sitemapURL)).data
-    const cheerio = require('cheerio')
     const $ = cheerio.load(sitemap)
-    const sitemapURLs = []
-    $('url loc').each(function () {
+    const sitemapURLs: string[] = []
+    $('url loc').each(function (this: any) {
       sitemapURLs.push($(this).text())
     })
     for (const url of sitemapURLs) {
@@ -186,20 +216,16 @@ exports.run = async ({ pluginConfig, processingConfig, processingId, dir, tmpDir
     }
   }
 
-  const sentIds = new Set([])
-  const sendPage = async (page, data, contentType = 'text/html', filename = 'content.html') => {
-    await log.debug('send page', page.url)
+  const sentIds = new Set<string>()
+  const sendPage = async (page: Page, data: any, contentType = 'text/html', filename = 'content.html') => {
     const form = new FormData()
-    // improve page title
+
     if (page.title) {
       page.title = page.title.trim()
       if (processingConfig.titlePrefix && page.title.startsWith(processingConfig.titlePrefix)) {
         page.title = page.title.replace(processingConfig.titlePrefix, '')
       }
     }
-    form.append('title', page.title)
-    form.append('url', page.url)
-    if (page.tags && page.tags.length) form.append('tags', page.tags.join(','))
     data = typeof data === 'string' ? Buffer.from(data) : data
     const dataOpts = {
       contentType,
@@ -207,49 +233,52 @@ exports.run = async ({ pluginConfig, processingConfig, processingId, dir, tmpDir
       knownLength: data.length
     }
     form.append('attachment', data, dataOpts)
-    if (page.lastModified) form.append('lastModified', page.lastModified)
-    if (page.etag) form.append('etag', page.etag)
-    page._id = getId(page)
+    page._id = await getId(page)
     sentIds.add(page._id)
+    const body = { ...page }
+    delete body.source
+    delete body.parsedURL
+    delete body.nofollow
+    delete body.noindex
+    form.append('_body', JSON.stringify(body))
     const headers = {
       ...form.getHeaders(),
       'content-length': form.getLengthSync()
     }
+
     await axios({
       method: 'put',
       url: `api/v1/datasets/${dataset.id}/lines/${page._id}`,
       data: form,
-      headers
+      headers,
+      validateStatus: (status) => status === 200 || status === 304
     })
   }
 
   for await (const page of pages) {
     if (stopped) break
 
-    const crawlDelay = (robots[page.parsedURL.origin] && robots[page.parsedURL.origin].getCrawlDelay()) || pluginConfig.defaultCrawlDelay || 1
+    const crawlDelay = (robots[page.parsedURL!.origin] && robots[page.parsedURL!.origin].getCrawlDelay()) || pluginConfig.defaultCrawlDelay || 1
     await new Promise(resolve => setTimeout(resolve, crawlDelay * 1000))
 
-    // TODO: apply if-none-match and if-modified-since headers if etag or lastModified are available
-    const headers = { 'user-agent': pluginConfig.userAgent || 'data-fair-web-scraper' }
+    const headers: Record<string, string> = { 'user-agent': pluginConfig.userAgent || 'data-fair-web-scraper' }
     if (page.lastModified) headers['if-modified-since'] = page.lastModified
     if (page.etag) headers['if-none-match'] = page.etag
-    let response
+    let response: any
     try {
       response = await axios.get(page.url, { headers, maxRedirects: 0 })
-    } catch (err) {
-      // content did not change
+    } catch (err: any) {
       if (err.status === 304) {
         await log.debug(`page was not modified since last exploration ${page.url}`)
-        sentIds.add(page._id)
-        for (const existingPage of existingPages) {
-          if (existingPage.parentId === page._id) sentIds.add(existingPage._id)
+        sentIds.add(page._id!)
+        for (const existingPage of existingPages || []) {
+          if (existingPage.parentId === page._id) sentIds.add(existingPage._id!)
         }
         continue
       }
-      // follow a redirect
-      if (err.status === 301) {
+      if (err.status === 301 || err.status === 302) {
         await log.debug(`page redirected ${page.url} -> ${err.headers.location}`)
-        pages.push({ url: new URL(err.headers.location, page.url).href, source: 'redirect ' + page.url })
+        await pages.push({ url: new URL(err.headers.location, page.url).href, source: 'redirect ' + page.url })
         continue
       }
       await log.warning(`failed to fetch page ${page.url} - ${err.status || err.message}`)
@@ -258,46 +287,70 @@ exports.run = async ({ pluginConfig, processingConfig, processingId, dir, tmpDir
     }
 
     if (response.headers['x-robots-tag']) {
-      await log.debug('use x-robots-tag header', response.headers['x-robots-tag'])
-      for (const part of response.headers['x-robots-tag'].split(',').map(p => p.trim())) {
+      await log.debug('x-robots-tag header', response.headers['x-robots-tag'])
+      for (const part of response.headers['x-robots-tag'].split(',').map((p: string) => p.trim())) {
         if (part === 'noindex') page.noindex = true
         if (part === 'nofollow') page.nofollow = true
       }
     }
-
     page.lastModified = response.headers['last-modified']
     page.etag = response.headers.etag
 
     const isHTML = (response.headers['content-type'] && response.headers['content-type'].startsWith('text/html;')) || (typeof response.data === 'string' && response.data.trim().startsWith('<html'))
     if (isHTML) {
-      const cheerio = require('cheerio')
       const $ = cheerio.load(response.data)
       const titleSelectors = (processingConfig.titleSelectors || []).concat(['title', 'h1'])
       for (const titleSelector of titleSelectors) {
         page.title = $(titleSelector).text()
         if (page.title) {
-          log.debug(`used title selector "${titleSelector}" -> ${page.title.trim()}`)
+          await log.debug(`used title selector "${titleSelector}" -> ${page.title.trim()}`)
           break
         }
       }
 
-      $('meta').each(function (i, elem) {
+      if (processingConfig.tagsSelectors && processingConfig.tagsSelectors.length) {
+        for (const tagsSelector of processingConfig.tagsSelectors) {
+          $(tagsSelector).each(function (this: any) {
+            const tag = $(this).text().trim()
+            if (tag) {
+              page.tags = page.tags ?? []
+              page.tags!.push(tag)
+            }
+          })
+        }
+      }
+
+      $('meta').each(function (this: any) {
         const name = $(this).attr('name')
+        const property = $(this).attr('property')
+        const content = $(this).attr('content')
         if (name === 'robots') {
-          const content = $(this).attr('content')
-          log.debug('use robots meta', content)
+          log.debug('robots meta', content)
           if (content) {
-            for (const part of content.split(',').map(p => p.trim())) {
+            for (const part of content.split(',').map((p) => p.trim())) {
               if (part === 'noindex') page.noindex = true
               if (part === 'nofollow') page.nofollow = true
             }
           }
         }
+        if (processingConfig.extractKeywords && name === 'keywords' && content) {
+          page.tags = page.tags ?? []
+          for (const tag of content.split(',').map((t) => t.trim()).filter((t) => t)) {
+            page.tags!.push(tag)
+          }
+        }
+        if (processingConfig.extractArticleTags && property === 'article:tag' && content) {
+          page.tags = page.tags ?? []
+          page.tags!.push(content.trim())
+        }
+        if (processingConfig.extractDescription && name === 'description' && content) {
+          page.description = content.trim()
+        }
       })
 
       if (!page.noindex && processingConfig.anchors && processingConfig.anchors.length) {
-        const anchorsPages = []
-        $('a').each(function (i, elem) {
+        const anchorsPages: [Page, string][] = []
+        $('a').each(function (this: any) {
           const href = $(this).attr('href')
           if (!href) return
           const parsedURL = new URL(href, page.url)
@@ -308,7 +361,7 @@ exports.run = async ({ pluginConfig, processingConfig, processingId, dir, tmpDir
               const fragment = anchor.wrapperSelector ? targetElement.closest(anchor.wrapperSelector) : targetElement
               const fragmentHtml = fragment.html()
               if (fragmentHtml) {
-                const anchorPage = { url: parsedURL.href, tags: anchor.tags || [], source: 'anchor ' + page.url }
+                const anchorPage: Page = { url: parsedURL.href, tags: anchor.tags || [], source: 'anchor ' + page.url }
                 if (anchor.titleSelector) anchorPage.title = fragment.find(anchor.titleSelector).text() || page.title
                 else anchorPage.title = targetElement.text() || page.title
                 anchorsPages.push([anchorPage, fragmentHtml])
@@ -324,7 +377,7 @@ exports.run = async ({ pluginConfig, processingConfig, processingId, dir, tmpDir
         }
       }
       if (!page.nofollow) {
-        $('a').each(function (i, elem) {
+        $('a').each(function (this: any) {
           const href = $(this).attr('href')
           if (href) pages.push({ url: new URL(href, page.url).href, source: 'link ' + page.url })
         })
@@ -332,7 +385,7 @@ exports.run = async ({ pluginConfig, processingConfig, processingId, dir, tmpDir
 
       if (!page.noindex) {
         if (processingConfig.prune) {
-          processingConfig.prune.forEach(s => $(s).remove())
+          processingConfig.prune.forEach((s: string) => $(s).remove())
         }
         await sendPage(page, $.html())
       }
@@ -341,7 +394,7 @@ exports.run = async ({ pluginConfig, processingConfig, processingId, dir, tmpDir
 
   if (existingPages) {
     for (const existingPage of existingPages) {
-      if (!sentIds.has(existingPage._id)) {
+      if (!sentIds.has(existingPage._id!)) {
         await log.info('delete previously explored page that was not indexed this time', existingPage.url)
         await axios.delete(`api/v1/datasets/${dataset.id}/lines/${existingPage._id}`)
       }
@@ -349,10 +402,6 @@ exports.run = async ({ pluginConfig, processingConfig, processingId, dir, tmpDir
   }
 }
 
-// used to manage interruption
-// not required but it is a good practice to prevent incoherent state a smuch as possible
-// the run method should finish shortly after calling stop, otherwise the process will be forcibly terminated
-// the grace period before force termination is 20 seconds
-exports.stop = async () => {
+export const stop = async () => {
   stopped = true
 }

package/package.json

@@ -1,12 +1,16 @@
 {
   "name": "@data-fair/processing-web-scraper",
-  "version": "0.4.0",
+  "version": "0.6.0",
   "description": "A small Web scraper that publishes its data into data-fair datasets.",
-  "main": "index.js",
+  "main": "index.ts",
+  "type": "module",
   "scripts": {
-    "test": "mocha --exit",
-    "lint": "eslint --ignore-path .gitignore .",
-    "lint-fix": "eslint --ignore-path .gitignore --fix ."
+    "lint": "eslint .",
+    "lint-fix": "eslint --fix .",
+    "build-types": "export NODE_OPTIONS='--experimental-strip-types' && df-build-types ./",
+    "prepare": "husky || true",
+    "test-base": "NODE_ENV=test node --experimental-strip-types --test-force-exit --test-concurrency=1 --test --test-reporter=spec --test-reporter-destination=stdout --test-timeout=300000",
+    "test": "npm run test-base test-it/*.ts"
   },
   "repository": {
     "type": "git",
@@ -22,17 +26,31 @@
   },
   "homepage": "https://github.com/data-fair/processing-web-scraper#readme",
   "devDependencies": {
-    "@data-fair/processings-test-utils": "^0.5.1",
-    "config": "^3.3.6",
-    "eslint": "^7.18.0",
+    "@commitlint/cli": "^20.4.1",
+    "@commitlint/config-conventional": "^20.4.1",
+    "@data-fair/lib-processing-dev": "^0.2.0",
+    "@data-fair/lib-types-builder": "^1.11.3",
+    "@types/config": "^3.3.5",
+    "@types/express": "^5.0.6",
+    "@types/node": "^25.2.0",
+    "config": "^4.2.0",
+    "eslint": "^9.39.2",
     "express": "^4.18.2",
-    "mocha": "^8.2.1",
-    "standard": "^16.0.3"
+    "husky": "^9.1.7",
+    "neostandard": "^0.12.2",
+    "ws": "^8.19.0"
   },
   "dependencies": {
+    "@data-fair/lib-utils": "^1.9.0",
     "cheerio": "^1.0.0-rc.12",
     "form-data": "^4.0.0",
     "robots-parser": "^3.0.0",
     "url-pattern": "^1.0.3"
-  }
+  },
+  "files": [
+    "plugin-config-schema.json",
+    "processing-config-schema.json",
+    "./index.ts",
+    "./lib/**/*"
+  ]
 }

package/processing-config-schema.json

@@ -67,6 +67,26 @@
         "description": "Par défaut le sélecteur 'title' est utilisé ce qui correspond au titre de page dans les métadonnées HTML.",
         "items": {"type": "string"}
       },
+      "tagsSelectors": {
+        "type": "array",
+        "title": "Sélecteurs d'éléments HTML à utiliser comme étiquettes",
+        "items": {"type": "string"}
+      },
+      "extractKeywords": {
+        "type": "boolean",
+        "title": "Extraire les mots-clés depuis la balise meta name=\"keywords\"",
+        "default": false
+      },
+      "extractArticleTags": {
+        "type": "boolean",
+        "title": "Extraire les étiquettes depuis les balises meta property=\"article:tag\"",
+        "default": false
+      },
+      "extractDescription": {
+        "type": "boolean",
+        "title": "Extraire la description depuis la balise meta name=\"description\"",
+        "default": false
+      },
       "sitemaps": {
         "type": "array",
         "title": "URLs de fichiers sitemap.xml",
@@ -82,7 +102,7 @@
       "excludeURLPatterns": {
         "type": "array",
         "title": "Formats d'URL à exclure de l'exploration",
-        "description": "Exemple: https://data-fair.github.io/master/en(/*)",
+        "description": "Exemple: https://data-fair.github.io/3/en(/*)",
         "items": {"type": "string"}
       },
       "anchors": {

package/.eslintrc.js

@@ -1,13 +0,0 @@
-module.exports = {
-  root: true,
-  parserOptions: {
-    parser: 'babel-eslint',
-    sourceType: 'module'
-  },
-  extends: ['standard'],
-  // add your custom rules here
-  rules: {
-    // allow paren-less arrow functions
-    'arrow-parens': 0
-  }
-}