@mvegter/scrapedin 1.0.32 → 1.1.0

package/src/company/companyScraperTemplate.js

@@ -1,30 +1,29 @@
-const template = {
-    profile: {
-        selector: '.org-top-card',
-        fields: {
-            name: `h1`,
-            headline: `p`,
-            imageurl: {
-                selector: `img.org-top-card-primary-content__logo`,
-                attribute: 'src'
-            }
-        }
-    },
-    about: {
-        selector: '.org-grid__core-rail--no-margin-left',
-        fields: {
-            overview: 'p',
-            types:{
-                selector: 'dl dt',
-                isMultipleFields: true
-            },
-            values:{
-                selector: 'dl dd:not(.org-page-details__employees-on-linkedin-count)',
-                isMultipleFields: true
-            }
-        }
-    }
-}
-
-
-module.exports = template
+const template = {
+  profile: {
+    selector: '.org-top-card, .top-card-layout',
+    fields: {
+      name: 'h1',
+      headline: '.org-top-card-summary__tagline, .top-card-layout__headline, p',
+      imageurl: {
+        selector: 'img.org-top-card-primary-content__logo, img.top-card-layout__entity-image',
+        attribute: 'src'
+      }
+    }
+  },
+  about: {
+    selector: '.org-grid__core-rail--no-margin-left, .org-about-us-organization-description, .core-section-container',
+    fields: {
+      overview: 'p',
+      types: {
+        selector: 'dl dt, .org-page-details__definition-term',
+        isMultipleFields: true
+      },
+      values: {
+        selector: 'dl dd:not(.org-page-details__employees-on-linkedin-count), .org-page-details__definition-text',
+        isMultipleFields: true
+      }
+    }
+  }
+}
+
+module.exports = template

package/src/logger.js

@@ -1,29 +1,29 @@
-const path = require('path')
-const pkg = require('./package')
-const winston = require('winston')
-const logger = winston.createLogger({
-  format: winston.format.combine(
-    winston.format.splat(),
-    winston.format.simple(),
-    winston.format.timestamp(),
-    winston.format.colorize(),
-    winston.format.printf(info => `${pkg.name}: ${info.timestamp} ${info.level}: ${info.message}`)
-  ),
-  transports: [new winston.transports.Console()]
-})
-
-const loggerWrapper = (absoluteFilePath) => {
-  const file = path.relative(__dirname, absoluteFilePath)
-  // Because this file is in the source code root folder, the above will make all paths relative to it: just the info needed for the log.
-
-  return {
-    info: (message) => logger.info(`[${file}] ${message}`),
-    warn: (message) => logger.warn(`[${file}] ${message}`),
-    error: (message, error) => logger.error(`[${file}] ${message}${error && error.stack ? error.stack : (error || '')}`),
-    stopLogging: () => {
-      logger.silent = true
-    }
-  }
-}
-
-module.exports = loggerWrapper
+const path = require('path')
+const pkg = require('./package')
+const winston = require('winston')
+const logger = winston.createLogger({
+  format: winston.format.combine(
+    winston.format.splat(),
+    winston.format.simple(),
+    winston.format.timestamp(),
+    winston.format.colorize(),
+    winston.format.printf(info => `${pkg.name}: ${info.timestamp} ${info.level}: ${info.message}`)
+  ),
+  transports: [new winston.transports.Console()]
+})
+
+const loggerWrapper = (absoluteFilePath) => {
+  const file = path.relative(__dirname, absoluteFilePath)
+  // Because this file is in the source code root folder, the above will make all paths relative to it: just the info needed for the log.
+
+  return {
+    info: (message) => logger.info(`[${file}] ${message}`),
+    warn: (message) => logger.warn(`[${file}] ${message}`),
+    error: (message, error) => logger.error(`[${file}] ${message}${error && error.stack ? error.stack : (error || '')}`),
+    stopLogging: () => {
+      logger.silent = true
+    }
+  }
+}
+
+module.exports = loggerWrapper

package/src/login.js

@@ -1,68 +1,70 @@
-const openPage = require('./openPage')
-const logger = require('./logger')(__filename)
-const pkg = require('./package')
-
-module.exports = async (browser, email, password) => {
-  const url = 'https://www.linkedin.com/login'
-  const page = await openPage({ browser, url })
-  logger.info(`logging at: ${url}`)
-
-  await page.waitForSelector('#username')
-
-  await page.$('#username')
-    .then((emailElement) => emailElement.type(email))
-  await page.$('#password')
-    .then((passwordElement) => passwordElement.type(password))
-
-  await page.$x("//button[contains(text(), 'Sign in')]")
-    .then((button) => button[0].click())
-
-  return page.waitForSelector('input[role=combobox]', {
-    timeout: 15000
-  })
-    .then(async () => {
-      logger.info('logged feed page selector found')
-      await page.close()
-    })
-    .catch(async () => {
-      logger.warn('successful login element was not found')
-      const emailError = await page.evaluate(() => {
-        const e = document.querySelector('div[error-for=username]')
-        if (!e) { return false }
-        const style = window.getComputedStyle(e)
-        return style && style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0'
-      })
-
-      const passwordError = await page.evaluate(() => {
-        const e = document.querySelector('div[error-for=password]')
-        if (!e) { return false }
-        const style = window.getComputedStyle(e)
-        return style && style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0'
-      })
-
-      const manualChallengeRequested = await page.evaluate(() => {
-        const e = document.querySelector('.flow-challenge-content')
-        if (!e) { return false }
-        const style = window.getComputedStyle(e)
-        return style && style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0'
-      })
-
-      if (emailError) {
-        logger.info('wrong username element found')
-        return Promise.reject(new Error(`linkedin: invalid username: ${email}`))
-      }
-
-      if (passwordError) {
-        logger.info('wrong password element found')
-        return Promise.reject(new Error('linkedin: invalid password'))
-      }
-
-      if (page.$(manualChallengeRequested)) {
-        logger.warn('manual check was required')
-        return Promise.reject(new Error(`linkedin: manual check was required, verify if your login is properly working manually or report this issue: ${pkg.name} ${pkg.version} ${pkg.bugs.url}`))
-      }
-
-      logger.error('could not find any element to retrieve a proper error')
-      return Promise.reject(new Error(`${pkg.name} ${pkg.version} login is not working, please report: ${pkg.bugs.url}`))
-    })
-}
+const openPage = require('./openPage')
+const logger = require('./logger')(__filename)
+const pkg = require('./package')
+
+module.exports = async (browser, email, password) => {
+  const url = 'https://www.linkedin.com/login'
+  const page = await openPage({ browser, url })
+  logger.info(`logging at: ${url}`)
+
+  await page.waitForSelector('#username')
+
+  await page.$('#username')
+    .then((emailElement) => emailElement.type(email))
+  await page.$('#password')
+    .then((passwordElement) => passwordElement.type(password))
+
+  await page.locator('button[type="submit"]').click()
+
+  return page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 15000 })
+    .then(async () => {
+      const currentUrl = page.url()
+      if (currentUrl.includes('/feed') || currentUrl.includes('/mynetwork') || currentUrl.includes('/in/')) {
+        logger.info('logged in, redirected to: ' + currentUrl)
+        await page.close()
+        return
+      }
+      throw new Error('unexpected redirect: ' + currentUrl)
+    })
+    .catch(async () => {
+      logger.warn('successful login element was not found')
+      const emailError = await page.evaluate(() => {
+        const e = document.querySelector('div[error-for=username], #error-for-username')
+        if (!e) { return false }
+        const style = window.getComputedStyle(e)
+        return style && style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0'
+      })
+
+      const passwordError = await page.evaluate(() => {
+        const e = document.querySelector('div[error-for=password], #error-for-password')
+        if (!e) { return false }
+        const style = window.getComputedStyle(e)
+        return style && style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0'
+      })
+
+      const manualChallengeRequested = await page.evaluate(() => {
+        const e = document.querySelector('.flow-challenge-content, #challenge, [data-test-id="challenge"]')
+        if (!e) { return false }
+        const style = window.getComputedStyle(e)
+        return style && style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0'
+      })
+
+      if (emailError) {
+        logger.info('wrong username element found')
+        return Promise.reject(new Error(`linkedin: invalid username: ${email}`))
+      }
+
+      if (passwordError) {
+        logger.info('wrong password element found')
+        return Promise.reject(new Error('linkedin: invalid password'))
+      }
+
+      if (manualChallengeRequested) {
+        logger.warn('manual check was required')
+        return Promise.reject(new Error(`linkedin: manual check was required, verify if your login is properly working manually or report this issue: ${pkg.name} ${pkg.version} ${pkg.bugs.url}`))
+      }
+
+      logger.error('could not find any element to retrieve a proper error')
+      return Promise.reject(new Error(`${pkg.name} ${pkg.version} login is not working, please report: ${pkg.bugs.url}`))
+    })
+}

package/src/openPage.js

@@ -1,33 +1,23 @@
-const agents = [
-  'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36'
-  // "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
-  // "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
-  // "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:34.0) Gecko/20100101 Firefox/34.0",
-  // "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
-  // "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0",
-  // "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
-  // "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0",
-  // "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36"
-]
-
-module.exports = async ({ browser, cookies, url, puppeteerAuthenticate }) => {
-  const page = await browser.newPage()
-  await page.setDefaultNavigationTimeout(0)
-
-  if (cookies) {
-    await page.setCookie(...cookies)
-  }
-  await page.setUserAgent(agents[Math.floor(Math.random() * agents.length)])
-  await page.setExtraHTTPHeaders({ 'Accept-Language': 'en-GB,en-US;q=0.9,en;q=0.8' })
-  await page.setViewport({
-    width: 1920,
-    height: 1080
-  })
-
-  if (puppeteerAuthenticate) {
-    await page.authenticate(puppeteerAuthenticate)
-  }
-
-  await page.goto(url, { waitUntil: 'load' })
-  return page
-}
+const AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'
+
+module.exports = async ({ browser, cookies, url, puppeteerAuthenticate }) => {
+  const page = await browser.newPage()
+  await page.setDefaultNavigationTimeout(60000)
+
+  if (cookies) {
+    await page.setCookie(...cookies)
+  }
+  await page.setUserAgent(AGENT)
+  await page.setExtraHTTPHeaders({ 'Accept-Language': 'en-GB,en-US;q=0.9,en;q=0.8' })
+  await page.setViewport({
+    width: 1920,
+    height: 1080
+  })
+
+  if (puppeteerAuthenticate) {
+    await page.authenticate(puppeteerAuthenticate)
+  }
+
+  await page.goto(url, { waitUntil: 'load' })
+  return page
+}

package/src/package.js

@@ -1,11 +1,11 @@
-const pkg = require('../package.json')
-
-// Only specific keys are needed, not the whole file.
-
-module.exports = {
-  bugs: {
-    url: pkg.bugs.url
-  },
-  name: pkg.name,
-  version: pkg.version
-}
+const pkg = require('../package.json')
+
+// Only specific keys are needed, not the whole file.
+
+module.exports = {
+  bugs: {
+    url: pkg.bugs.url
+  },
+  name: pkg.name,
+  version: pkg.version
+}

package/src/profile/cleanProfileData.js

@@ -1,99 +1,98 @@
-const logger = require('../logger')(__filename)
-const pkg = require('../package')
-
-module.exports = (profile) => {
-  if(!profile?.profile?.name) {
-    const messageError = `LinkedIn website changed and ${pkg.name} ${pkg.version} can't read basic data. Please report this issue at ${pkg.bugs.url}`
-    logger.error(messageError, '')
-    throw new Error(messageError)
-  }
-
-  profile.profile.summary = profile?.about?.text
-
-  profile.positions.forEach((position) => {
-    if(position.title){
-        position.title = position.title.replace('Company Name\n', '')
-    }
-    if(position.description) {
-      position.description = position.description.replace('See more', '');
-      position.description = position.description.replace('see more', '');
-	    position.description = position.description.replace('See less', '');
-    }
-    if(position.roles) {
-      position.roles.forEach((role) => {
-        if(role.title) {
-          role.title = role.title.replace('Title\n', '')
-        }
-        if (role.date) {
-          role.date1 = role.date.replace('·', '-').split('-')[0].trim()
-          role.date2 = role.date.replace('·', '-').split('-')[1].trim()
-          delete role.date
-        }
-        if(role.description) {
-          role.description = role.description.replace('See more', '')
-          role.description = role.description.replace('see more', '')
-        }
-      })
-    }
-  })
-
-  if(profile.recommendations.receivedCount) {
-    profile.recommendations.receivedCount = profile.recommendations.receivedCount.replace(/[^\d]/g, '')
-  }
-
-  if(profile.recommendations.givenCount) {
-    profile.recommendations.givenCount = profile.recommendations.givenCount.replace(/[^\d]/g, '')
-  }
-
-  if(profile.recommendations.received) {
-    profile.recommendations.received.forEach((recommendation) => {
-      if(recommendation.summary){
-        recommendation.summary = recommendation.summary.replace('See more', '')
-        recommendation.summary = recommendation.summary.replace('See less', '')
-      }
-    })
-  }
-
-  if(profile.recommendations.given) {
-    profile.recommendations.given.forEach((recommendation) => {
-      if(recommendation.summary){
-        recommendation.summary = recommendation.summary.replace('See more', '')
-        recommendation.summary = recommendation.summary.replace('See less', '')
-      }
-    })
-  }
-
-  if(profile.courses){
-    profile.courses = profile.courses.map(({ name, year }) => {
-      const coursesObj = {}
-      if(name) {
-        coursesObj.name = name.replace('Course name\n', '')
-      }
-      if(year) {
-        coursesObj.year = year.replace('Course number\n', '')
-      }
-      return coursesObj
-    }
-    );
-  }
-
-  if(profile.languages){
-    profile.languages = profile.languages.map(({ name, proficiency }) => ({
-      name: name ? name.replace('Language name\n', '') : undefined,
-      proficiency,
-    }));
-  }
-
-  if(profile.projects){
-    profile.projects = profile.projects.map(
-      ({ name, date, description, link }) => ({
-        name: name ? name.replace('Project name\n', '') : undefined,
-        date,
-        description: description ? description.replace('Project description\n', '') : undefined,
-        link,
-      }),
-    );
-  }
-  
-  return profile
-}
+const logger = require('../logger')(__filename)
+const pkg = require('../package')
+
+module.exports = (profile) => {
+  if (!profile?.profile?.name) {
+    const messageError = `LinkedIn website changed and ${pkg.name} ${pkg.version} can't read basic data. Please report this issue at ${pkg.bugs.url}`
+    logger.error(messageError, '')
+    throw new Error(messageError)
+  }
+
+  profile.profile.summary = profile?.about?.text
+
+  profile.positions.forEach((position) => {
+    if (position.title) {
+      position.title = position.title.replace('Company Name\n', '')
+    }
+    if (position.description) {
+      position.description = position.description.replace('See more', '')
+      position.description = position.description.replace('see more', '')
+      position.description = position.description.replace('See less', '')
+    }
+    if (position.roles) {
+      position.roles.forEach((role) => {
+        if (role.title) {
+          role.title = role.title.replace('Title\n', '')
+        }
+        if (role.date) {
+          role.date1 = role.date.replace('·', '-').split('-')[0].trim()
+          role.date2 = role.date.replace('·', '-').split('-')[1].trim()
+          delete role.date
+        }
+        if (role.description) {
+          role.description = role.description.replace('See more', '')
+          role.description = role.description.replace('see more', '')
+        }
+      })
+    }
+  })
+
+  if (profile.recommendations.receivedCount) {
+    profile.recommendations.receivedCount = profile.recommendations.receivedCount.replace(/[^\d]/g, '')
+  }
+
+  if (profile.recommendations.givenCount) {
+    profile.recommendations.givenCount = profile.recommendations.givenCount.replace(/[^\d]/g, '')
+  }
+
+  if (profile.recommendations.received) {
+    profile.recommendations.received.forEach((recommendation) => {
+      if (recommendation.summary) {
+        recommendation.summary = recommendation.summary.replace('See more', '')
+        recommendation.summary = recommendation.summary.replace('See less', '')
+      }
+    })
+  }
+
+  if (profile.recommendations.given) {
+    profile.recommendations.given.forEach((recommendation) => {
+      if (recommendation.summary) {
+        recommendation.summary = recommendation.summary.replace('See more', '')
+        recommendation.summary = recommendation.summary.replace('See less', '')
+      }
+    })
+  }
+
+  if (profile.courses) {
+    profile.courses = profile.courses.map(({ name, year }) => {
+      const coursesObj = {}
+      if (name) {
+        coursesObj.name = name.replace('Course name\n', '')
+      }
+      if (year) {
+        coursesObj.year = year.replace('Course number\n', '')
+      }
+      return coursesObj
+    })
+  }
+
+  if (profile.languages) {
+    profile.languages = profile.languages.map(({ name, proficiency }) => ({
+      name: name ? name.replace('Language name\n', '') : undefined,
+      proficiency
+    }))
+  }
+
+  if (profile.projects) {
+    profile.projects = profile.projects.map(
+      ({ name, date, description, link }) => ({
+        name: name ? name.replace('Project name\n', '') : undefined,
+        date,
+        description: description ? description.replace('Project description\n', '') : undefined,
+        link
+      })
+    )
+  }
+
+  return profile
+}

package/src/profile/contactInfo.js

@@ -1,48 +1,46 @@
-const logger = require('../logger')(__filename)
-const scrapSection = require('../scrapSection')
-
-const SEE_MORE_SELECTOR = 'a[data-control-name=contact_see_more]'
-const CLOSE_MODAL_SELECTOR = '.artdeco-modal__dismiss';
-
-const template = {
-  selector: '.pv-contact-info__contact-type',
-  fields: {
-    type: 'header',
-    values: {
-      selector: '.pv-contact-info__ci-container',
-      isMultipleFields: true
-    },
-    links: {
-      selector: 'a',
-      attribute: 'href',
-      isMultipleFields: true
-    }
-  }
-} 
-const getContactInfo = async(page) => {
-  await page.waitFor(SEE_MORE_SELECTOR, { timeout: 2000 })
-    .catch(() => {
-      logger.warn('contact-info', 'selector not found')
-      return {}
-    })
-
-  const element = await page.$(SEE_MORE_SELECTOR)
-  if(element){
-    await element.click()
-    const contactInfoIndicatorSelector = '#pv-contact-info'
-    await page.waitFor(contactInfoIndicatorSelector, { timeout: 5000 })
-        .catch(() => {
-          logger.warn('contact info was not found')
-        })
-    
-    const contactInfo = await scrapSection(page, template)
-    const closeButton = await page.$(CLOSE_MODAL_SELECTOR)
-    if(closeButton)
-      await closeButton.click()
-
-    return contactInfo
-  }
-  
-}
-
-module.exports = getContactInfo
+const logger = require('../logger')(__filename)
+const scrapSection = require('../scrapSection')
+
+const SEE_MORE_SELECTOR = '#top-card-text-details-contact-info'
+const CLOSE_MODAL_SELECTOR = '.artdeco-modal__dismiss'
+
+const template = {
+  selector: '.pv-contact-info__contact-type, .ci-vanity-url, .ci-email, .ci-phone, .ci-websites, .ci-birthday, .ci-ims, .ci-address',
+  fields: {
+    type: 'header, h3',
+    values: {
+      selector: '.pv-contact-info__ci-container, .t-14',
+      isMultipleFields: true
+    },
+    links: {
+      selector: 'a',
+      attribute: 'href',
+      isMultipleFields: true
+    }
+  }
+}
+const getContactInfo = async (page) => {
+  await page.waitForSelector(SEE_MORE_SELECTOR, { timeout: 2000 })
+    .catch(() => {
+      logger.warn('contact-info selector not found')
+      return {}
+    })
+
+  const element = await page.$(SEE_MORE_SELECTOR)
+  if (element) {
+    await element.click()
+    const contactInfoIndicatorSelector = '.pv-profile-section__section-info, .artdeco-modal__content'
+    await page.waitForSelector(contactInfoIndicatorSelector, { timeout: 5000 })
+      .catch(() => {
+        logger.warn('contact info was not found')
+      })
+
+    const contactInfo = await scrapSection(page, template)
+    const closeButton = await page.$(CLOSE_MODAL_SELECTOR)
+    if (closeButton) { await closeButton.click() }
+
+    return contactInfo
+  }
+}
+
+module.exports = getContactInfo