@mvegter/scrapedin 1.0.33 → 1.1.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@mvegter/scrapedin",
-  "version": "1.0.33",
-  "description": "linkedin scraper for 2020 website",
+  "version": "1.1.0",
+  "description": "linkedin scraper updated for 2025+ website",
   "keywords": [
     "linkedin",
     "scraper",
@@ -21,14 +21,14 @@
   "author": "Wagner Leonardi <leonardiwagner@gmail.com>",
   "license": "Apache-2.0",
   "dependencies": {
-    "puppeteer": "14.1.1",
+    "puppeteer": "24.36.0",
     "winston": "3.7.2"
   },
   "devDependencies": {
     "standard": "17.0.0"
   },
   "engines": {
-    "node": ">= 7.6.0"
+    "node": ">= 16.0.0"
   },
   "homepage": "https://github.com/linkedtales/scrapedin#readme"
 }

package/src/company/company.js

@@ -5,31 +5,30 @@ const template = require('./companyScraperTemplate')
 const logger = require('../logger')(__filename)
 
 module.exports = async (browser, cookies, url, waitTimeToScrapMs = 500, puppeteerAuthenticate = undefined) => {
-  logger.info(`starting scraping url: ${url}`);
+  logger.info(`starting scraping url: ${url}`)
 
-  let company = {};
+  const company = {}
 
-  let page;
-  if(url.includes('legacySchoolId=')){
-      page = await openPage({ browser, cookies, url, puppeteerAuthenticate });
+  let page
+  if (url.includes('legacySchoolId=')) {
+    page = await openPage({ browser, cookies, url, puppeteerAuthenticate })
 
-      const aboutSelector = 'a[href$="/about/"]';
+    const aboutSelector = 'a[href$="/about/"]'
 
-      company.url = page.url();
-      
-      await page.$eval(aboutSelector, async about => await about.click());
-      await page.waitForNavigation();
-  } else{
-      company.url = url;
-      url = url + '/about';
-      page = await openPage({ browser, cookies, url, puppeteerAuthenticate });
+    company.url = page.url()
+
+    await page.$eval(aboutSelector, async about => await about.click())
+    await page.waitForNavigation()
+  } else {
+    company.url = url
+    url = url + '/about'
+    page = await openPage({ browser, cookies, url, puppeteerAuthenticate })
   }
-  company.about = (await scrapSection(page, template.about))[0];
-  company.profile = (await scrapSection(page, template.profile))[0];
+  company.about = (await scrapSection(page, template.about))[0]
+  company.profile = (await scrapSection(page, template.profile))[0]
+
+  await page.close()
+  logger.info(`finished scraping url: ${url}`)
 
-  await page.close();
-  logger.info(`finished scraping url: ${url}`);
-  
   return company
-    
 }

package/src/company/companyScraperTemplate.js

@@ -1,30 +1,29 @@
 const template = {
-    profile: {
-        selector: '.org-top-card',
-        fields: {
-            name: `h1`,
-            headline: `p`,
-            imageurl: {
-                selector: `img.org-top-card-primary-content__logo`,
-                attribute: 'src'
-            }
-        }
-    },
-    about: {
-        selector: '.org-grid__core-rail--no-margin-left',
-        fields: {
-            overview: 'p',
-            types:{
-                selector: 'dl dt',
-                isMultipleFields: true
-            },
-            values:{
-                selector: 'dl dd:not(.org-page-details__employees-on-linkedin-count)',
-                isMultipleFields: true
-            }
-        }
+  profile: {
+    selector: '.org-top-card, .top-card-layout',
+    fields: {
+      name: 'h1',
+      headline: '.org-top-card-summary__tagline, .top-card-layout__headline, p',
+      imageurl: {
+        selector: 'img.org-top-card-primary-content__logo, img.top-card-layout__entity-image',
+        attribute: 'src'
+      }
     }
+  },
+  about: {
+    selector: '.org-grid__core-rail--no-margin-left, .org-about-us-organization-description, .core-section-container',
+    fields: {
+      overview: 'p',
+      types: {
+        selector: 'dl dt, .org-page-details__definition-term',
+        isMultipleFields: true
+      },
+      values: {
+        selector: 'dl dd:not(.org-page-details__employees-on-linkedin-count), .org-page-details__definition-text',
+        isMultipleFields: true
+      }
+    }
+  }
 }
 
-
 module.exports = template

package/src/login.js

@@ -14,34 +14,36 @@ module.exports = async (browser, email, password) => {
   await page.$('#password')
     .then((passwordElement) => passwordElement.type(password))
 
-  await page.$x("//button[contains(text(), 'Sign in')]")
-    .then((button) => button[0].click())
+  await page.locator('button[type="submit"]').click()
 
-  return page.waitForSelector('input[role=combobox]', {
-    timeout: 15000
-  })
+  return page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 15000 })
     .then(async () => {
-      logger.info('logged feed page selector found')
-      await page.close()
+      const currentUrl = page.url()
+      if (currentUrl.includes('/feed') || currentUrl.includes('/mynetwork') || currentUrl.includes('/in/')) {
+        logger.info('logged in, redirected to: ' + currentUrl)
+        await page.close()
+        return
+      }
+      throw new Error('unexpected redirect: ' + currentUrl)
     })
     .catch(async () => {
       logger.warn('successful login element was not found')
       const emailError = await page.evaluate(() => {
-        const e = document.querySelector('div[error-for=username]')
+        const e = document.querySelector('div[error-for=username], #error-for-username')
         if (!e) { return false }
         const style = window.getComputedStyle(e)
         return style && style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0'
       })
 
       const passwordError = await page.evaluate(() => {
-        const e = document.querySelector('div[error-for=password]')
+        const e = document.querySelector('div[error-for=password], #error-for-password')
         if (!e) { return false }
         const style = window.getComputedStyle(e)
         return style && style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0'
       })
 
       const manualChallengeRequested = await page.evaluate(() => {
-        const e = document.querySelector('.flow-challenge-content')
+        const e = document.querySelector('.flow-challenge-content, #challenge, [data-test-id="challenge"]')
         if (!e) { return false }
         const style = window.getComputedStyle(e)
         return style && style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0'
@@ -57,7 +59,7 @@ module.exports = async (browser, email, password) => {
         return Promise.reject(new Error('linkedin: invalid password'))
       }
 
-      if (page.$(manualChallengeRequested)) {
+      if (manualChallengeRequested) {
         logger.warn('manual check was required')
         return Promise.reject(new Error(`linkedin: manual check was required, verify if your login is properly working manually or report this issue: ${pkg.name} ${pkg.version} ${pkg.bugs.url}`))
       }

package/src/openPage.js

@@ -1,23 +1,13 @@
-const agents = [
-  'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36'
-  // "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
-  // "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
-  // "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:34.0) Gecko/20100101 Firefox/34.0",
-  // "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
-  // "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0",
-  // "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
-  // "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0",
-  // "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36"
-]
+const AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'
 
 module.exports = async ({ browser, cookies, url, puppeteerAuthenticate }) => {
   const page = await browser.newPage()
-  await page.setDefaultNavigationTimeout(0)
+  await page.setDefaultNavigationTimeout(60000)
 
   if (cookies) {
     await page.setCookie(...cookies)
   }
-  await page.setUserAgent(agents[Math.floor(Math.random() * agents.length)])
+  await page.setUserAgent(AGENT)
   await page.setExtraHTTPHeaders({ 'Accept-Language': 'en-GB,en-US;q=0.9,en;q=0.8' })
   await page.setViewport({
     width: 1920,

package/src/profile/cleanProfileData.js

@@ -2,7 +2,7 @@ const logger = require('../logger')(__filename)
 const pkg = require('../package')
 
 module.exports = (profile) => {
-  if(!profile?.profile?.name) {
+  if (!profile?.profile?.name) {
     const messageError = `LinkedIn website changed and ${pkg.name} ${pkg.version} can't read basic data. Please report this issue at ${pkg.bugs.url}`
     logger.error(messageError, '')
     throw new Error(messageError)
@@ -11,17 +11,17 @@ module.exports = (profile) => {
   profile.profile.summary = profile?.about?.text
 
   profile.positions.forEach((position) => {
-    if(position.title){
-        position.title = position.title.replace('Company Name\n', '')
+    if (position.title) {
+      position.title = position.title.replace('Company Name\n', '')
     }
-    if(position.description) {
-      position.description = position.description.replace('See more', '');
-      position.description = position.description.replace('see more', '');
-	    position.description = position.description.replace('See less', '');
+    if (position.description) {
+      position.description = position.description.replace('See more', '')
+      position.description = position.description.replace('see more', '')
+      position.description = position.description.replace('See less', '')
     }
-    if(position.roles) {
+    if (position.roles) {
       position.roles.forEach((role) => {
-        if(role.title) {
+        if (role.title) {
           role.title = role.title.replace('Title\n', '')
         }
         if (role.date) {
@@ -29,7 +29,7 @@ module.exports = (profile) => {
           role.date2 = role.date.replace('·', '-').split('-')[1].trim()
           delete role.date
         }
-        if(role.description) {
+        if (role.description) {
           role.description = role.description.replace('See more', '')
           role.description = role.description.replace('see more', '')
         }
@@ -37,63 +37,62 @@ module.exports = (profile) => {
     }
   })
 
-  if(profile.recommendations.receivedCount) {
+  if (profile.recommendations.receivedCount) {
     profile.recommendations.receivedCount = profile.recommendations.receivedCount.replace(/[^\d]/g, '')
   }
 
-  if(profile.recommendations.givenCount) {
+  if (profile.recommendations.givenCount) {
     profile.recommendations.givenCount = profile.recommendations.givenCount.replace(/[^\d]/g, '')
   }
 
-  if(profile.recommendations.received) {
+  if (profile.recommendations.received) {
     profile.recommendations.received.forEach((recommendation) => {
-      if(recommendation.summary){
+      if (recommendation.summary) {
         recommendation.summary = recommendation.summary.replace('See more', '')
         recommendation.summary = recommendation.summary.replace('See less', '')
       }
     })
   }
 
-  if(profile.recommendations.given) {
+  if (profile.recommendations.given) {
     profile.recommendations.given.forEach((recommendation) => {
-      if(recommendation.summary){
+      if (recommendation.summary) {
         recommendation.summary = recommendation.summary.replace('See more', '')
         recommendation.summary = recommendation.summary.replace('See less', '')
       }
     })
   }
 
-  if(profile.courses){
+  if (profile.courses) {
     profile.courses = profile.courses.map(({ name, year }) => {
       const coursesObj = {}
-      if(name) {
+      if (name) {
         coursesObj.name = name.replace('Course name\n', '')
       }
-      if(year) {
+      if (year) {
         coursesObj.year = year.replace('Course number\n', '')
       }
       return coursesObj
-    }
-    );
+    })
   }
 
-  if(profile.languages){
+  if (profile.languages) {
     profile.languages = profile.languages.map(({ name, proficiency }) => ({
       name: name ? name.replace('Language name\n', '') : undefined,
-      proficiency,
-    }));
+      proficiency
+    }))
   }
 
-  if(profile.projects){
+  if (profile.projects) {
     profile.projects = profile.projects.map(
       ({ name, date, description, link }) => ({
         name: name ? name.replace('Project name\n', '') : undefined,
         date,
         description: description ? description.replace('Project description\n', '') : undefined,
-        link,
-      }),
-    );
+        link
+      })
+    )
   }
-  
+
   return profile
 }

package/src/profile/contactInfo.js

@@ -1,15 +1,15 @@
 const logger = require('../logger')(__filename)
 const scrapSection = require('../scrapSection')
 
-const SEE_MORE_SELECTOR = 'a[data-control-name=contact_see_more]'
-const CLOSE_MODAL_SELECTOR = '.artdeco-modal__dismiss';
+const SEE_MORE_SELECTOR = '#top-card-text-details-contact-info'
+const CLOSE_MODAL_SELECTOR = '.artdeco-modal__dismiss'
 
 const template = {
-  selector: '.pv-contact-info__contact-type',
+  selector: '.pv-contact-info__contact-type, .ci-vanity-url, .ci-email, .ci-phone, .ci-websites, .ci-birthday, .ci-ims, .ci-address',
   fields: {
-    type: 'header',
+    type: 'header, h3',
     values: {
-      selector: '.pv-contact-info__ci-container',
+      selector: '.pv-contact-info__ci-container, .t-14',
       isMultipleFields: true
     },
     links: {
@@ -18,31 +18,29 @@ const template = {
       isMultipleFields: true
     }
   }
-} 
-const getContactInfo = async(page) => {
-  await page.waitFor(SEE_MORE_SELECTOR, { timeout: 2000 })
+}
+const getContactInfo = async (page) => {
+  await page.waitForSelector(SEE_MORE_SELECTOR, { timeout: 2000 })
     .catch(() => {
-      logger.warn('contact-info', 'selector not found')
+      logger.warn('contact-info selector not found')
       return {}
     })
 
   const element = await page.$(SEE_MORE_SELECTOR)
-  if(element){
+  if (element) {
     await element.click()
-    const contactInfoIndicatorSelector = '#pv-contact-info'
-    await page.waitFor(contactInfoIndicatorSelector, { timeout: 5000 })
-        .catch(() => {
-          logger.warn('contact info was not found')
-        })
-    
+    const contactInfoIndicatorSelector = '.pv-profile-section__section-info, .artdeco-modal__content'
+    await page.waitForSelector(contactInfoIndicatorSelector, { timeout: 5000 })
+      .catch(() => {
+        logger.warn('contact info was not found')
+      })
+
     const contactInfo = await scrapSection(page, template)
     const closeButton = await page.$(CLOSE_MODAL_SELECTOR)
-    if(closeButton)
-      await closeButton.click()
+    if (closeButton) { await closeButton.click() }
 
     return contactInfo
   }
-  
 }
 
 module.exports = getContactInfo

package/src/profile/profile.js

@@ -1,81 +1,221 @@
 const openPage = require('../openPage')
-const scrapSection = require('../scrapSection')
-const scrapAccomplishmentPanel = require('./scrapAccomplishmentPanel')
 const scrollToPageBottom = require('./scrollToPageBottom')
 const seeMoreButtons = require('./seeMoreButtons')
 const contactInfo = require('./contactInfo')
-const template = require('./profileScraperTemplate')
 const cleanProfileData = require('./cleanProfileData')
 
 const logger = require('../logger')(__filename)
 
+const extractProfileData = async (page) => {
+  return page.evaluate(() => {
+    const txt = (el) => el ? (el.textContent || '').trim() : ''
+
+    const byViewName = (name) => document.querySelector(`[data-view-name="${name}"]`)
+
+    const getSection = (viewName) => {
+      const el = byViewName(viewName)
+      if (!el) return null
+      return el.querySelector('section') || el.closest('section')
+    }
+
+    // Get items from a section. For sections without ul/li, items are divs
+    // found after the h2 heading: h2.parent.nextSibling > div > div > div
+    const getSectionItems = (viewName) => {
+      const section = getSection(viewName)
+      if (!section) return []
+      const ul = section.querySelector('ul')
+      if (ul) return [...ul.querySelectorAll(':scope > li')]
+      const h2 = section.querySelector('h2')
+      if (!h2) return []
+      const afterH2 = h2.parentElement.nextElementSibling
+      if (!afterH2) return []
+      const itemDivs = afterH2.querySelectorAll(':scope > div > div > div')
+      return [...itemDivs].filter(d => d.querySelectorAll('p').length > 0)
+    }
+
+    // Profile top card: name is in h2, headline/location in p tags
+    const titleName = document.title.replace(' | LinkedIn', '').trim()
+    const mainLevel = byViewName('profile-main-level')
+    const topSection = mainLevel ? (mainLevel.querySelector('section') || mainLevel.closest('section')) : null
+    const nameH2 = topSection ? topSection.querySelector('h2') : null
+    const photoImg = document.querySelector('img[src*="profile-displayphoto"]')
+
+    const profileName = nameH2 ? txt(nameH2) : titleName
+    let headline = ''
+    let location = ''
+
+    if (topSection) {
+      const allP = [...topSection.querySelectorAll('p')]
+      headline = txt(allP[0])
+      for (const p of allP) {
+        const t = txt(p)
+        if (t.includes(',') && !t.includes('at ') && !t.includes('\xB7')) {
+          location = t
+          break
+        }
+      }
+    }
+
+    const profile = {
+      name: profileName,
+      headline,
+      location,
+      connections: '',
+      imageurl: photoImg ? photoImg.getAttribute('src') || '' : ''
+    }
+
+    // Experience: uses LazyColumn with company groups containing ul > li
+    const positions = []
+    const expSection = getSection('profile-card-experience')
+    if (expSection) {
+      const lazyCol = expSection.querySelector('[data-component-type]') || expSection
+      const groups = [...lazyCol.children]
+
+      groups.forEach(group => {
+        const ul = group.querySelector('ul')
+        if (!ul) return
+
+        // Company info is in p tags before the ul
+        const allPs = group.querySelectorAll('p')
+        const beforeUlPs = []
+        for (const p of allPs) {
+          if (ul.contains(p)) break
+          beforeUlPs.push(p)
+        }
+        const companyName = txt(beforeUlPs[0])
+        const companyLink = group.querySelector('a[href*="/company/"]')
+        const companyUrl = companyLink ? companyLink.getAttribute('href') : ''
+        const companyLocation = txt(beforeUlPs[2])
+
+        const lis = [...ul.querySelectorAll(':scope > li')]
+        lis.forEach(li => {
+          const ps = [...li.querySelectorAll('div[role="button"] p')]
+          const descEl = li.querySelector('[data-testid="expandable-text-box"]')
+          const dateStr = txt(ps[2])
+          const dateRange = dateStr.split('\xB7')[0].trim()
+          const dateParts = dateRange.split(' - ')
+          positions.push({
+            title: txt(ps[0]),
+            companyName,
+            link: companyUrl,
+            url: companyUrl,
+            location: companyLocation,
+            description: descEl ? txt(descEl) : '',
+            date: dateRange,
+            date1: dateParts[0] ? dateParts[0].trim() : '',
+            date2: dateParts[1] ? dateParts[1].trim() : ''
+          })
+        })
+      })
+    }
+
+    // Education
+    const eduItems = getSectionItems('profile-card-education')
+    const educations = eduItems.map(item => {
+      const ps = [...item.querySelectorAll('p')]
+      const link = item.querySelector('a[href*="/school/"]') ||
+        item.closest('div')?.parentElement?.querySelector('a[href*="/school/"]')
+      return {
+        title: txt(ps[0]),
+        degree: txt(ps[1]),
+        fieldOfStudy: txt(ps[1]),
+        url: link ? link.getAttribute('href') : '',
+        date1: ps[2] ? txt(ps[2]).split('\u2013')[0].trim() : '',
+        date2: ps[2] ? (txt(ps[2]).split('\u2013')[1] || '').trim() : '',
+        description: ''
+      }
+    })
+
+    // Skills
+    const skillItems = getSectionItems('profile-card-skills')
+    const skills = skillItems.map(item => {
+      const ps = [...item.querySelectorAll('p')]
+      return { title: txt(ps[0]), count: '' }
+    })
+
+    // Languages
+    const langItems = getSectionItems('profile-card-languages')
+    const languages = langItems.map(item => {
+      const ps = [...item.querySelectorAll('p')]
+      return { name: txt(ps[0]), proficiency: txt(ps[1]) }
+    })
+
+    // Projects
+    const projItems = getSectionItems('profile-card-projects')
+    const projects = projItems.map(item => {
+      const ps = [...item.querySelectorAll('p')]
+      const descEl = item.querySelector('[data-testid="expandable-text-box"]')
+      const link = item.querySelector('a[href*="http"]')
+      return {
+        name: txt(ps[0]),
+        date: txt(ps[1]),
+        description: descEl ? txt(descEl) : '',
+        link: link ? link.getAttribute('href') : ''
+      }
+    })
+
+    // Certifications
+    const certItems = getSectionItems('profile-card-licenses-and-certifications')
+    const accomplishments = certItems.map(item => {
+      const ps = [...item.querySelectorAll('p')]
+      return { title: txt(ps[0]), count: '', items: [] }
+    })
+
+    return {
+      profile,
+      about: { text: '' },
+      positions,
+      educations,
+      skills,
+      recommendations: { givenCount: '0', receivedCount: '0', given: [], received: [] },
+      accomplishments,
+      courses: [],
+      languages,
+      projects,
+      peopleAlsoViewed: [],
+      volunteerExperience: [],
+      contact: []
+    }
+  })
+}
+
 module.exports = async (browser, cookies, url, waitTimeToScrapMs = 500, hasToGetContactInfo = false, puppeteerAuthenticate = undefined) => {
   logger.info(`starting scraping url: ${url}`)
 
   const page = await openPage({ browser, cookies, url, puppeteerAuthenticate })
-  const profilePageIndicatorSelector = '.pv-profile-section'
-  await page.waitForSelector(profilePageIndicatorSelector, { timeout: 5000 })
+
+  // Wait for the SDUI profile to fully hydrate
+  await page.waitForFunction(() => {
+    return document.querySelector('[data-view-name="profile-card-experience"]')
+  }, { timeout: 30000 })
     .catch(() => {
-      //why doesn't throw error instead of continuing scraping?
-      //because it can be just a false negative meaning LinkedIn only changed that selector but everything else is fine :)
-      logger.warn('profile selector was not found')
+      logger.warn('profile content did not fully render in time')
     })
 
   logger.info('scrolling page to the bottom')
   await scrollToPageBottom(page)
-  
-  if(waitTimeToScrapMs) {
-    logger.info(`applying 1st delay`)
-    await new Promise((resolve) => { setTimeout(() => { resolve() }, waitTimeToScrapMs / 2)})
-  }
+
+  // Wait for lazy-loaded sections to render after scrolling
+  await new Promise((resolve) => { setTimeout(resolve, 2000) })
+
+  // Scroll again in case new content was loaded
+  await scrollToPageBottom(page)
+  await new Promise((resolve) => { setTimeout(resolve, 1000) })
 
   await seeMoreButtons.clickAll(page)
 
-  if(waitTimeToScrapMs) {
-    logger.info(`applying 2nd (and last) delay`)
-    await new Promise((resolve) => { setTimeout(() => { resolve() }, waitTimeToScrapMs / 2)})
-  }
+  // Final wait for content to settle
+  await new Promise((resolve) => { setTimeout(resolve, 1000) })
 
-  const [profile] = await scrapSection(page, template.profile)
-  const [about] = await scrapSection(page, template.about)
-  const positions = await scrapSection(page, template.positions)
-  const educations = await scrapSection(page, template.educations)
-  const [recommendationsCount] = await scrapSection(page, template.recommendationsCount)
-  const recommendationsReceived = await scrapSection(page, template.recommendationsReceived)
-  const recommendationsGiven = await scrapSection(page, template.recommendationsGiven)
-  const skills = await scrapSection(page, template.skills)
-  const accomplishments = await scrapSection(page, template.accomplishments)
-  const courses = await scrapAccomplishmentPanel(page, 'courses')
-  const languages = await scrapAccomplishmentPanel(page, 'languages')
-  const projects = await scrapAccomplishmentPanel(page, 'projects')
-  const volunteerExperience = await scrapSection(page, template.volunteerExperience)
-  const peopleAlsoViewed = await scrapSection(page, template.peopleAlsoViewed)
-  const contact = hasToGetContactInfo ? await contactInfo(page) : []
+  const rawProfile = await extractProfileData(page)
+
+  if (hasToGetContactInfo) {
+    rawProfile.contact = await contactInfo(page) || []
+  }
 
   await page.close()
   logger.info(`finished scraping url: ${url}`)
 
-  const rawProfile = {
-    profile,
-    about,
-    positions,
-    educations,
-    skills,
-    recommendations: {
-      givenCount: recommendationsCount ? recommendationsCount.given : "0",
-      receivedCount: recommendationsCount ? recommendationsCount.received : "0",
-      given: recommendationsReceived,
-      received: recommendationsGiven
-    },
-    accomplishments,
-    courses,
-    languages,
-    projects,
-    peopleAlsoViewed,
-    volunteerExperience,
-    contact
-  }
-
   const cleanedProfile = cleanProfileData(rawProfile)
   return cleanedProfile
 }

package/src/profile/profileScraperTemplate.js

@@ -1,189 +1,186 @@
-const profileSelector = '.core-rail > *:first-child section >'
-
 const template = {
   profile: {
-    selector: '.pb5',
+    selector: '.scaffold-layout__main',
     fields: {
-			name: `.text-heading-xlarge`,
-      headline: `.text-body-medium`,
-      location: `.pb2 .text-body-small`,
-      connections: `li.text-body-small`,
+      name: '.text-heading-xlarge',
+      headline: '.text-body-medium',
+      location: '.text-body-small.inline.t-black--light.break-words',
+      connections: '.t-bold',
       imageurl: {
-				selector: `img.pv-top-card__photo`,
+        selector: 'img.pv-top-card-profile-picture__image',
         attribute: 'src'
       }
     }
   },
   about: {
-    selector: '.pv-about-section',
+    selector: '#about ~ .display-flex .inline-show-more-text',
     fields: {
-      text: 'div'
+      text: 'span[aria-hidden="true"]'
     }
   },
   positions: {
-    selector: 'div[id="experience"] + div + div li.artdeco-list__item',
+    selector: '#experience ~ .pvs-list__outer-container .pvs-list > li.pvs-list__paged-list-item',
     fields: {
-      title: 'h3',
+      title: '.mr1.hoverable-link-text.t-bold > span',
       link: {
-        selector: 'a',
-        attribute: 'href',
+        selector: 'a.optional-action-target-wrapper',
+        attribute: 'href'
       },
       url: {
-        selector: 'a',
+        selector: 'a.optional-action-target-wrapper',
         attribute: 'href'
       },
-      companyName: 'div.t-bold span:first-child',
-      location: '.pv-entity__location span:last-child',
-      description: '.pv-entity__description',
-      date1: '.pv-entity__date-range span:last-child',
-      date2: '.pv-entity__bullet-item-v2',
+      companyName: '.t-14.t-normal > span',
+      location: '.t-14.t-normal.t-black--light > span',
+      description: '.pvs-list__outer-container .inline-show-more-text span[aria-hidden="true"]',
+      date1: '.pvs-entity__caption-wrapper',
+      date2: '.pvs-entity__caption-wrapper',
       roles: {
-        selector: 'div.full-width > div > ul div[data-view-name]',
+        selector: '.pvs-entity__sub-components li.pvs-list__paged-list-item',
         hasChildrenFields: true,
         fields: {
-          title: 'div.t-bold span:first-child',
-          description: 'div.display-flex div.full-width > span[aria-hidden]',
+          title: '.mr1.hoverable-link-text.t-bold > span',
+          description: '.inline-show-more-text span[aria-hidden="true"]',
           date: '.pvs-entity__caption-wrapper',
-          location: '.pv-entity__location span:last-child'
+          location: '.t-14.t-normal.t-black--light > span'
         }
       }
     }
   },
   educations: {
-    selector: '#education-section li',
+    selector: '#education ~ .pvs-list__outer-container .pvs-list > li.pvs-list__paged-list-item',
     fields: {
-      title: 'h3',
-      degree: 'span[class=pv-entity__comma-item]',
+      title: '.hoverable-link-text.t-bold > span',
+      degree: '.t-14.t-normal > span',
       url: {
         selector: 'a',
         attribute: 'href'
       },
-	    fieldOfStudy: 'p.pv-entity__fos span:nth-child(2)',
-      date1: '.pv-entity__dates time:nth-child(1)',
-      date2: '.pv-entity__dates time:nth-child(2)',
-      description: '.pv-entity__description'
+      fieldOfStudy: '.t-14.t-normal > span',
+      date1: '.pvs-entity__caption-wrapper',
+      date2: '.pvs-entity__caption-wrapper',
+      description: '.inline-show-more-text span[aria-hidden="true"]'
     }
   },
   skills: {
-    selector: '.pv-skill-category-entity__skill-wrapper',
+    selector: '#skills ~ .pvs-list__outer-container .pvs-list > li.pvs-list__paged-list-item',
     fields: {
-      title: '.pv-skill-category-entity__name-text',
-      count: '.pv-skill-category-entity__endorsement-count'
+      title: '.mr1.hoverable-link-text.t-bold > span',
+      count: '.t-14.t-normal.t-black--light > span'
     }
   },
   recommendationsCount: {
-    selector: '.recommendations-inlining',
+    selector: '#recommendations ~ .pvs-list__outer-container',
     fields: {
       received: '.artdeco-tab:nth-child(1)',
       given: '.artdeco-tab:nth-child(2)'
     }
   },
   recommendationsReceived: {
-    selector: '.recommendations-inlining',
+    selector: '#recommendations ~ .pvs-list__outer-container .pvs-list > li.pvs-list__paged-list-item',
     fields: {
       user: {
-        selector: '.pv-recommendation-entity__member',
+        selector: 'a',
         attribute: 'href'
       },
-      text: 'blockquote.pv-recommendation-entity__text',
+      text: '.inline-show-more-text span[aria-hidden="true"]',
       profileImage: {
-        selector: 'a img',
+        selector: 'img',
         attribute: 'src'
       },
       name: {
-        selector: 'a h3'
+        selector: '.t-bold > span'
       },
       userDescription: {
-        selector: '.pv-recommendation-entity__headline'
+        selector: '.t-14.t-normal > span'
       }
     }
   },
   recommendationsGiven: {
-    selector: '.artdeco-tabpanel li.pv-recommendation-entity',
+    selector: '#recommendations ~ .pvs-list__outer-container .pvs-list > li.pvs-list__paged-list-item',
     fields: {
       user: {
-        selector: '.pv-recommendation-entity__member',
+        selector: 'a',
         attribute: 'href'
       },
-      text: 'blockquote.pv-recommendation-entity__text',
+      text: '.inline-show-more-text span[aria-hidden="true"]',
       profileImage: {
-        selector: 'a img',
+        selector: 'img',
         attribute: 'src'
       },
       name: {
-        selector: 'a h3'
+        selector: '.t-bold > span'
       },
       userDescription: {
-        selector: '.pv-recommendation-entity__headline'
+        selector: '.t-14.t-normal > span'
       }
     }
   },
   accomplishments: {
-    selector: '.pv-accomplishments-section > div',
+    selector: '#honors_and_awards ~ .pvs-list__outer-container .pvs-list > li.pvs-list__paged-list-item',
     fields: {
-      count: 'h3 span:last-child',
-      title: '.pv-accomplishments-block__title',
+      count: '.t-14.t-normal.t-black--light > span',
+      title: '.mr1.hoverable-link-text.t-bold > span',
       items: {
-        selector: 'li',
+        selector: '.pvs-list__outer-container li',
         isMultipleFields: true
       }
     }
   },
   peopleAlsoViewed: {
-    selector: 'li.pv-browsemap-section__member-container',
+    selector: '.pv-browsemap-section li',
     fields: {
       user: {
         selector: 'a',
         attribute: 'href'
       },
-      text: 'p',
+      text: '.t-14.t-normal',
       profileImage: {
-        selector: 'a img',
+        selector: 'img',
         attribute: 'src'
       },
       name: {
-        selector: '.name'
+        selector: '.t-bold'
       }
     }
   },
   volunteerExperience: {
-    selector: 'section.volunteering-section li',
+    selector: '#volunteering_experience ~ .pvs-list__outer-container .pvs-list > li.pvs-list__paged-list-item',
     fields: {
-      title: 'h3',
-      experience: 'span[class=pv-entity__secondary-title]',
-      location: '.pv-entity__location span:nth-child(2)',
-      description: '.pv-volunteer-causes',
-      date1: '.pv-entity__date-range span:nth-child(2)',
-      date2: '.pv-entity__bullet-item'
+      title: '.mr1.hoverable-link-text.t-bold > span',
+      experience: '.t-14.t-normal > span',
+      location: '.t-14.t-normal.t-black--light > span',
+      description: '.inline-show-more-text span[aria-hidden="true"]',
+      date1: '.pvs-entity__caption-wrapper',
+      date2: '.pvs-entity__caption-wrapper'
     }
   },
   courses: {
-    selector: '.pv-accomplishments-section',
+    selector: '#courses ~ .pvs-list__outer-container .pvs-list > li.pvs-list__paged-list-item',
     fields: {
-      name: '.pv-accomplishment-entity__title',
-      year: '.pv-accomplishment-entity__course-number'
+      name: '.mr1.hoverable-link-text.t-bold > span',
+      year: '.t-14.t-normal > span'
     }
   },
   languages: {
-    selector: '.pv-accomplishments-block.languages li',
+    selector: '#languages ~ .pvs-list__outer-container .pvs-list > li.pvs-list__paged-list-item',
     fields: {
-      name: '.pv-accomplishment-entity__title',
-      proficiency: '.pv-accomplishment-entity__proficiency',
+      name: '.mr1.t-bold > span',
+      proficiency: '.t-14.t-normal.t-black--light > span'
     }
   },
   projects: {
-    selector: '.pv-accomplishments-block.projects li',
+    selector: '#projects ~ .pvs-list__outer-container .pvs-list > li.pvs-list__paged-list-item',
     fields: {
-      name: '.pv-accomplishment-entity__title',
-      date: '.pv-accomplishment-entity__date',
-      description: '.pv-accomplishment-entity__description',
+      name: '.mr1.hoverable-link-text.t-bold > span',
+      date: '.pvs-entity__caption-wrapper',
+      description: '.inline-show-more-text span[aria-hidden="true"]',
       link: {
-        selector: '.mt4',
+        selector: 'a',
         attribute: 'href'
       }
     }
   }
 }
 
-
 module.exports = template

package/src/profile/scrapAccomplishmentPanel.js

@@ -1,18 +1,11 @@
-const scrapSection = require('../scrapSection');
-const template = require('./profileScraperTemplate');
+const scrapSection = require('../scrapSection')
+const template = require('./profileScraperTemplate')
 
 const scrapAccomplishmentPanel = async (page, section) => {
-  const queryString = `.pv-accomplishments-block.${section} button`
-
-  const openingButton = await page.$(queryString);
-
-  if (openingButton) {
-    await page.evaluate((q) => {
-      document.querySelector(q).click();
-    }, queryString);
-
-    return scrapSection(page, template[section]);
+  if (!template[section]) {
+    return []
   }
-};
+  return scrapSection(page, template[section])
+}
 
-module.exports = scrapAccomplishmentPanel;
+module.exports = scrapAccomplishmentPanel

package/src/profile/scrollToPageBottom.js

@@ -3,21 +3,20 @@ const logger = require('../logger')(__filename)
 module.exports = async (page) => {
   const MAX_TIMES_TO_SCROLL = 25
   const TIMEOUT_BETWEEN_SCROLLS = 500
-  const PAGE_BOTTOM_SELECTOR_STRING = '#expanded-footer'
 
   for (let i = 0; i < MAX_TIMES_TO_SCROLL; i++) {
     await page.evaluate(() => window.scrollBy(0, window.innerHeight))
 
-    const hasReachedEnd = await page.waitForSelector(PAGE_BOTTOM_SELECTOR_STRING, {
-      visible: true,
-      timeout: TIMEOUT_BETWEEN_SCROLLS
-    }).catch(() => {
-      logger.info(`scrolling to page bottom (${i + 1})`)
+    const hasReachedEnd = await page.evaluate(() => {
+      return (window.innerHeight + window.scrollY) >= (document.body.scrollHeight - 200)
     })
 
     if (hasReachedEnd) {
       return
     }
+
+    await new Promise(resolve => setTimeout(resolve, TIMEOUT_BETWEEN_SCROLLS))
+    logger.info(`scrolling to page bottom (${i + 1})`)
   }
 
   logger.warn('page bottom not found')

package/src/profile/seeMoreButtons.js

@@ -1,42 +1,18 @@
 const logger = require('../logger')(__filename)
-const seeMoreButtons = [
-  {
-    id: 'SHOW_MORE_ABOUT',
-    selector: '#line-clamp-show-more-button'
-  },{
-    id: 'SHOW_MORE_EXPERIENCES',
-    selector: '#experience-section .pv-profile-section__see-more-inline'
-  },{
-    id: 'SEE_MORE_EXPERIENCES',
-    selector: '#experience-section .inline-show-more-text__button'
-  },{
-    id: 'SHOW_MORE_CERTIFICATIONS',
-    selector: '#certifications-section .pv-profile-section__see-more-inline'
-  },{
-    id: 'SHOW_MORE_SKILLS',
-    selector: '.pv-skills-section__additional-skills'
-  },{
-    id: 'SEE_MORE_RECOMMENDATIONS',
-    selector: '.recommendations-inlining #line-clamp-show-more-button'
-  }
-]
-
 
-const clickAll = async(page) => {
-  for(let i = 0; i < seeMoreButtons.length; i++){
-    const button = seeMoreButtons[i]
-    const elems = await page.$$(button.selector)
+const clickAll = async (page) => {
+  const clicked = await page.evaluate(() => {
+    let count = 0
+    // Only click expandable text buttons (inline expand, not navigation)
+    const expandButtons = document.querySelectorAll('[data-testid="expandable-text-button"]')
+    expandButtons.forEach(btn => { btn.click(); count++ })
+    return count
+  })
 
-    for(let j = 0; j < elems.length; j++){
-      const elem = elems[j]
-      if (elem) {
-        await elem.click()
-          .catch((e) => logger.warn(`couldn't click on ${button.selector}, it's probably invisible`))
-      }
-    }
+  if (clicked > 0) {
+    logger.info(`clicked ${clicked} show-more buttons`)
+    await new Promise(resolve => setTimeout(resolve, 500))
   }
-
-  return
 }
 
 module.exports = { clickAll }