@mvegter/scrapedin 1.0.33 → 1.2.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@mvegter/scrapedin",
-  "version": "1.0.33",
-  "description": "linkedin scraper for 2020 website",
+  "version": "1.2.0",
+  "description": "linkedin scraper updated for 2025+ website",
   "keywords": [
     "linkedin",
     "scraper",
@@ -21,14 +21,14 @@
   "author": "Wagner Leonardi <leonardiwagner@gmail.com>",
   "license": "Apache-2.0",
   "dependencies": {
-    "puppeteer": "14.1.1",
+    "puppeteer": "24.36.0",
     "winston": "3.7.2"
   },
   "devDependencies": {
     "standard": "17.0.0"
   },
   "engines": {
-    "node": ">= 7.6.0"
+    "node": ">= 16.0.0"
   },
   "homepage": "https://github.com/linkedtales/scrapedin#readme"
 }

package/src/company/company.js

@@ -5,31 +5,30 @@ const template = require('./companyScraperTemplate')
 const logger = require('../logger')(__filename)
 
 module.exports = async (browser, cookies, url, waitTimeToScrapMs = 500, puppeteerAuthenticate = undefined) => {
-  logger.info(`starting scraping url: ${url}`);
+  logger.info(`starting scraping url: ${url}`)
 
-  let company = {};
+  const company = {}
 
-  let page;
-  if(url.includes('legacySchoolId=')){
-      page = await openPage({ browser, cookies, url, puppeteerAuthenticate });
+  let page
+  if (url.includes('legacySchoolId=')) {
+    page = await openPage({ browser, cookies, url, puppeteerAuthenticate })
 
-      const aboutSelector = 'a[href$="/about/"]';
+    const aboutSelector = 'a[href$="/about/"]'
 
-      company.url = page.url();
-      
-      await page.$eval(aboutSelector, async about => await about.click());
-      await page.waitForNavigation();
-  } else{
-      company.url = url;
-      url = url + '/about';
-      page = await openPage({ browser, cookies, url, puppeteerAuthenticate });
+    company.url = page.url()
+
+    await page.$eval(aboutSelector, async about => await about.click())
+    await page.waitForNavigation()
+  } else {
+    company.url = url
+    url = url + '/about'
+    page = await openPage({ browser, cookies, url, puppeteerAuthenticate })
   }
-  company.about = (await scrapSection(page, template.about))[0];
-  company.profile = (await scrapSection(page, template.profile))[0];
+  company.about = (await scrapSection(page, template.about))[0]
+  company.profile = (await scrapSection(page, template.profile))[0]
+
+  await page.close()
+  logger.info(`finished scraping url: ${url}`)
 
-  await page.close();
-  logger.info(`finished scraping url: ${url}`);
-  
   return company
-    
 }

package/src/company/companyScraperTemplate.js

@@ -1,30 +1,29 @@
 const template = {
-    profile: {
-        selector: '.org-top-card',
-        fields: {
-            name: `h1`,
-            headline: `p`,
-            imageurl: {
-                selector: `img.org-top-card-primary-content__logo`,
-                attribute: 'src'
-            }
-        }
-    },
-    about: {
-        selector: '.org-grid__core-rail--no-margin-left',
-        fields: {
-            overview: 'p',
-            types:{
-                selector: 'dl dt',
-                isMultipleFields: true
-            },
-            values:{
-                selector: 'dl dd:not(.org-page-details__employees-on-linkedin-count)',
-                isMultipleFields: true
-            }
-        }
+  profile: {
+    selector: '.org-top-card, .top-card-layout',
+    fields: {
+      name: 'h1',
+      headline: '.org-top-card-summary__tagline, .top-card-layout__headline, p',
+      imageurl: {
+        selector: 'img.org-top-card-primary-content__logo, img.top-card-layout__entity-image',
+        attribute: 'src'
+      }
     }
+  },
+  about: {
+    selector: '.org-grid__core-rail--no-margin-left, .org-about-us-organization-description, .core-section-container',
+    fields: {
+      overview: 'p',
+      types: {
+        selector: 'dl dt, .org-page-details__definition-term',
+        isMultipleFields: true
+      },
+      values: {
+        selector: 'dl dd:not(.org-page-details__employees-on-linkedin-count), .org-page-details__definition-text',
+        isMultipleFields: true
+      }
+    }
+  }
 }
 
-
 module.exports = template

package/src/login.js

@@ -2,67 +2,173 @@ const openPage = require('./openPage')
 const logger = require('./logger')(__filename)
 const pkg = require('./package')
 
+const ACCEPT_COOKIES_SELECTORS = [
+  'button:has-text("Accept")',
+  'button:has-text("Alle accepteren")',
+  'button[action-type="ACCEPT"]',
+  '.cookie-consent-v2__button--accept',
+  '#artdeco-global-alert-container button:has-text("Accept")'
+]
+
+const acceptCookies = async (page) => {
+  for (const selector of ACCEPT_COOKIES_SELECTORS) {
+    try {
+      const btn = await page.$(selector)
+      if (btn) {
+        await btn.click()
+        await new Promise((r) => setTimeout(r, 1000))
+        return
+      }
+    } catch {
+      // selector might not exist, try next
+    }
+  }
+  // Fallback: find any button with Accept text
+  try {
+    await page.evaluate(() => {
+      const btn = Array.from(document.querySelectorAll('button')).find(
+        (b) => b.textContent.trim().toLowerCase() === 'accept'
+      )
+      if (btn) btn.click()
+    })
+    await new Promise((r) => setTimeout(r, 1000))
+  } catch {
+    // ignore
+  }
+}
+
+const fillField = async (page, fieldValue) => {
+  // LinkedIn renders two sets of inputs: hidden (CSS-only, not visible) and visible.
+  // We find all <input> elements matching the autocomplete attribute, then pick
+  // the first one that is actually visible (has non-zero dimensions).
+  const autocomplete = fieldValue === 'username webauthn' ? 'username' : 'current-password'
+
+  const visibleInput = await page.evaluate((auto, val) => {
+    const inputs = Array.from(document.querySelectorAll(`input[autocomplete="${auto}"]`))
+    for (const input of inputs) {
+      const rect = input.getBoundingClientRect()
+      if (rect.width > 0 && rect.height > 0) {
+        const style = window.getComputedStyle(input)
+        if (style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0') {
+          input.focus()
+          input.value = ''
+          return true
+        }
+      }
+    }
+    return false
+  }, autocomplete, fieldValue)
+
+  if (visibleInput) {
+    await page.keyboard.type(fieldValue, { delay: 50 })
+  }
+}
+
+const clickSignIn = async (page) => {
+  // Find the "Sign in" button, excluding "Sign in with Apple" etc.
+  await page.evaluate(() => {
+    const buttons = Array.from(document.querySelectorAll('button'))
+    const signInBtn = buttons.find(
+      (b) => {
+        const text = b.textContent.trim().toLowerCase()
+        return (text === 'sign in' || text === 'inloggen' || text === 'aanmelden') &&
+          !text.includes('apple')
+      }
+    )
+    if (signInBtn) {
+      signInBtn.click()
+      return true
+    }
+    return false
+  })
+}
+
+const LOGGED_IN_PATHS = ['/feed', '/mynetwork', '/in/']
+
 module.exports = async (browser, email, password) => {
   const url = 'https://www.linkedin.com/login'
   const page = await openPage({ browser, url })
   logger.info(`logging at: ${url}`)
 
-  await page.waitForSelector('#username')
+  // Accept cookie consent if present
+  await acceptCookies(page)
+  await new Promise((r) => setTimeout(r, 1000))
 
-  await page.$('#username')
-    .then((emailElement) => emailElement.type(email))
-  await page.$('#password')
-    .then((passwordElement) => passwordElement.type(password))
+  // Fill in email field
+  await fillField(page, 'username webauthn')
+  await new Promise((r) => setTimeout(r, 500))
 
-  await page.$x("//button[contains(text(), 'Sign in')]")
-    .then((button) => button[0].click())
+  // Fill in password field
+  await fillField(page, 'current-password')
+  await new Promise((r) => setTimeout(r, 500))
 
-  return page.waitForSelector('input[role=combobox]', {
-    timeout: 15000
-  })
-    .then(async () => {
-      logger.info('logged feed page selector found')
-      await page.close()
-    })
-    .catch(async () => {
-      logger.warn('successful login element was not found')
-      const emailError = await page.evaluate(() => {
-        const e = document.querySelector('div[error-for=username]')
-        if (!e) { return false }
-        const style = window.getComputedStyle(e)
-        return style && style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0'
-      })
-
-      const passwordError = await page.evaluate(() => {
-        const e = document.querySelector('div[error-for=password]')
-        if (!e) { return false }
-        const style = window.getComputedStyle(e)
-        return style && style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0'
-      })
-
-      const manualChallengeRequested = await page.evaluate(() => {
-        const e = document.querySelector('.flow-challenge-content')
-        if (!e) { return false }
-        const style = window.getComputedStyle(e)
-        return style && style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0'
-      })
-
-      if (emailError) {
-        logger.info('wrong username element found')
-        return Promise.reject(new Error(`linkedin: invalid username: ${email}`))
-      }
+  await clickSignIn(page)
 
-      if (passwordError) {
-        logger.info('wrong password element found')
-        return Promise.reject(new Error('linkedin: invalid password'))
-      }
+  let hadChallenge = false
+
+  try {
+    await page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 15000 })
+  } catch {
+    // Navigation timeout is expected — the page may not navigate if already on login
+  }
 
-      if (page.$(manualChallengeRequested)) {
-        logger.warn('manual check was required')
-        return Promise.reject(new Error(`linkedin: manual check was required, verify if your login is properly working manually or report this issue: ${pkg.name} ${pkg.version} ${pkg.bugs.url}`))
+  // Wait for either the feed/mynetwork or challenge page
+  const maxWaitMs = 120000
+  const start = Date.now()
+  let resolved = false
+  while (Date.now() - start < maxWaitMs && !resolved) {
+    const currentUrl = page.url()
+    if (LOGGED_IN_PATHS.some((p) => currentUrl.includes(p))) {
+      logger.info('logged in, redirected to: ' + currentUrl)
+      resolved = true
+      break
+    }
+    if (currentUrl.includes('/checkpoint')) {
+      if (!hadChallenge) {
+        logger.warn('2FA challenge detected, please complete the verification in the browser window (waiting up to 2 minutes)...')
+        hadChallenge = true
       }
+      await new Promise((r) => setTimeout(r, 2000))
+      continue
+    }
+    await new Promise((r) => setTimeout(r, 500))
+  }
 
-      logger.error('could not find any element to retrieve a proper error')
-      return Promise.reject(new Error(`${pkg.name} ${pkg.version} login is not working, please report: ${pkg.bugs.url}`))
+  if (!resolved) {
+    const finalUrl = page.url()
+    logger.warn('successful login element was not found, url: ' + finalUrl)
+
+    const emailError = await page.evaluate(() => {
+      const e = document.querySelector('div[error-for=username], #error-for-username')
+      if (!e) { return false }
+      const style = window.getComputedStyle(e)
+      return style && style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0'
     })
+
+    const passwordError = await page.evaluate(() => {
+      const e = document.querySelector('div[error-for=password], #error-for-password')
+      if (!e) { return false }
+      const style = window.getComputedStyle(e)
+      return style && style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0'
+    })
+
+    if (emailError) {
+      logger.info('wrong username element found')
+      await page.close()
+      return Promise.reject(new Error(`linkedin: invalid username: ${email}`))
+    }
+
+    if (passwordError) {
+      logger.info('wrong password element found')
+      await page.close()
+      return Promise.reject(new Error('linkedin: invalid password'))
+    }
+
+    logger.error('could not find any element to retrieve a proper error')
+    await page.close()
+    return Promise.reject(new Error(`${pkg.name} ${pkg.version} login is not working, please report: ${pkg.bugs.url}`))
+  }
+
+  await page.close()
+  return { hadChallenge }
 }

package/src/openPage.js

@@ -1,23 +1,13 @@
-const agents = [
-  'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36'
-  // "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
-  // "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
-  // "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:34.0) Gecko/20100101 Firefox/34.0",
-  // "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
-  // "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0",
-  // "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36",
-  // "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0",
-  // "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36"
-]
+const AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'
 
 module.exports = async ({ browser, cookies, url, puppeteerAuthenticate }) => {
   const page = await browser.newPage()
-  await page.setDefaultNavigationTimeout(0)
+  await page.setDefaultNavigationTimeout(60000)
 
   if (cookies) {
     await page.setCookie(...cookies)
   }
-  await page.setUserAgent(agents[Math.floor(Math.random() * agents.length)])
+  await page.setUserAgent(AGENT)
   await page.setExtraHTTPHeaders({ 'Accept-Language': 'en-GB,en-US;q=0.9,en;q=0.8' })
   await page.setViewport({
     width: 1920,
@@ -28,6 +18,6 @@ module.exports = async ({ browser, cookies, url, puppeteerAuthenticate }) => {
     await page.authenticate(puppeteerAuthenticate)
   }
 
-  await page.goto(url, { waitUntil: 'load' })
+  await page.goto(url, { waitUntil: 'networkidle2' })
   return page
 }

package/src/profile/cleanProfileData.js

@@ -2,7 +2,7 @@ const logger = require('../logger')(__filename)
 const pkg = require('../package')
 
 module.exports = (profile) => {
-  if(!profile?.profile?.name) {
+  if (!profile?.profile?.name) {
     const messageError = `LinkedIn website changed and ${pkg.name} ${pkg.version} can't read basic data. Please report this issue at ${pkg.bugs.url}`
     logger.error(messageError, '')
     throw new Error(messageError)
@@ -11,17 +11,17 @@ module.exports = (profile) => {
   profile.profile.summary = profile?.about?.text
 
   profile.positions.forEach((position) => {
-    if(position.title){
-        position.title = position.title.replace('Company Name\n', '')
+    if (position.title) {
+      position.title = position.title.replace('Company Name\n', '')
     }
-    if(position.description) {
-      position.description = position.description.replace('See more', '');
-      position.description = position.description.replace('see more', '');
-	    position.description = position.description.replace('See less', '');
+    if (position.description) {
+      position.description = position.description.replace('See more', '')
+      position.description = position.description.replace('see more', '')
+      position.description = position.description.replace('See less', '')
     }
-    if(position.roles) {
+    if (position.roles) {
       position.roles.forEach((role) => {
-        if(role.title) {
+        if (role.title) {
           role.title = role.title.replace('Title\n', '')
         }
         if (role.date) {
@@ -29,7 +29,7 @@ module.exports = (profile) => {
           role.date2 = role.date.replace('·', '-').split('-')[1].trim()
           delete role.date
         }
-        if(role.description) {
+        if (role.description) {
           role.description = role.description.replace('See more', '')
           role.description = role.description.replace('see more', '')
         }
@@ -37,63 +37,62 @@ module.exports = (profile) => {
     }
   })
 
-  if(profile.recommendations.receivedCount) {
+  if (profile.recommendations.receivedCount) {
     profile.recommendations.receivedCount = profile.recommendations.receivedCount.replace(/[^\d]/g, '')
   }
 
-  if(profile.recommendations.givenCount) {
+  if (profile.recommendations.givenCount) {
     profile.recommendations.givenCount = profile.recommendations.givenCount.replace(/[^\d]/g, '')
   }
 
-  if(profile.recommendations.received) {
+  if (profile.recommendations.received) {
     profile.recommendations.received.forEach((recommendation) => {
-      if(recommendation.summary){
+      if (recommendation.summary) {
         recommendation.summary = recommendation.summary.replace('See more', '')
         recommendation.summary = recommendation.summary.replace('See less', '')
       }
     })
   }
 
-  if(profile.recommendations.given) {
+  if (profile.recommendations.given) {
     profile.recommendations.given.forEach((recommendation) => {
-      if(recommendation.summary){
+      if (recommendation.summary) {
         recommendation.summary = recommendation.summary.replace('See more', '')
         recommendation.summary = recommendation.summary.replace('See less', '')
       }
     })
   }
 
-  if(profile.courses){
+  if (profile.courses) {
     profile.courses = profile.courses.map(({ name, year }) => {
       const coursesObj = {}
-      if(name) {
+      if (name) {
         coursesObj.name = name.replace('Course name\n', '')
       }
-      if(year) {
+      if (year) {
         coursesObj.year = year.replace('Course number\n', '')
       }
       return coursesObj
-    }
-    );
+    })
   }
 
-  if(profile.languages){
+  if (profile.languages) {
     profile.languages = profile.languages.map(({ name, proficiency }) => ({
       name: name ? name.replace('Language name\n', '') : undefined,
-      proficiency,
-    }));
+      proficiency
+    }))
   }
 
-  if(profile.projects){
+  if (profile.projects) {
     profile.projects = profile.projects.map(
       ({ name, date, description, link }) => ({
         name: name ? name.replace('Project name\n', '') : undefined,
         date,
         description: description ? description.replace('Project description\n', '') : undefined,
-        link,
-      }),
-    );
+        link
+      })
+    )
   }
-  
+
   return profile
 }

package/src/profile/contactInfo.js

@@ -1,15 +1,15 @@
 const logger = require('../logger')(__filename)
 const scrapSection = require('../scrapSection')
 
-const SEE_MORE_SELECTOR = 'a[data-control-name=contact_see_more]'
-const CLOSE_MODAL_SELECTOR = '.artdeco-modal__dismiss';
+const SEE_MORE_SELECTOR = '#top-card-text-details-contact-info'
+const CLOSE_MODAL_SELECTOR = '.artdeco-modal__dismiss'
 
 const template = {
-  selector: '.pv-contact-info__contact-type',
+  selector: '.pv-contact-info__contact-type, .ci-vanity-url, .ci-email, .ci-phone, .ci-websites, .ci-birthday, .ci-ims, .ci-address',
   fields: {
-    type: 'header',
+    type: 'header, h3',
     values: {
-      selector: '.pv-contact-info__ci-container',
+      selector: '.pv-contact-info__ci-container, .t-14',
       isMultipleFields: true
     },
     links: {
@@ -18,31 +18,29 @@ const template = {
       isMultipleFields: true
     }
   }
-} 
-const getContactInfo = async(page) => {
-  await page.waitFor(SEE_MORE_SELECTOR, { timeout: 2000 })
+}
+const getContactInfo = async (page) => {
+  await page.waitForSelector(SEE_MORE_SELECTOR, { timeout: 2000 })
     .catch(() => {
-      logger.warn('contact-info', 'selector not found')
+      logger.warn('contact-info selector not found')
       return {}
     })
 
   const element = await page.$(SEE_MORE_SELECTOR)
-  if(element){
+  if (element) {
     await element.click()
-    const contactInfoIndicatorSelector = '#pv-contact-info'
-    await page.waitFor(contactInfoIndicatorSelector, { timeout: 5000 })
-        .catch(() => {
-          logger.warn('contact info was not found')
-        })
-    
+    const contactInfoIndicatorSelector = '.pv-profile-section__section-info, .artdeco-modal__content'
+    await page.waitForSelector(contactInfoIndicatorSelector, { timeout: 5000 })
+      .catch(() => {
+        logger.warn('contact info was not found')
+      })
+
     const contactInfo = await scrapSection(page, template)
     const closeButton = await page.$(CLOSE_MODAL_SELECTOR)
-    if(closeButton)
-      await closeButton.click()
+    if (closeButton) { await closeButton.click() }
 
     return contactInfo
   }
-  
 }
 
 module.exports = getContactInfo

package/src/profile/profile.js

@@ -1,81 +1,276 @@
 const openPage = require('../openPage')
-const scrapSection = require('../scrapSection')
-const scrapAccomplishmentPanel = require('./scrapAccomplishmentPanel')
 const scrollToPageBottom = require('./scrollToPageBottom')
 const seeMoreButtons = require('./seeMoreButtons')
 const contactInfo = require('./contactInfo')
-const template = require('./profileScraperTemplate')
 const cleanProfileData = require('./cleanProfileData')
 
 const logger = require('../logger')(__filename)
 
+const extractProfileData = async (page) => {
+  return page.evaluate(() => {
+    const txt = (el) => el ? (el.textContent || '').trim() : ''
+
+    const findSection = (headingText) => {
+      const sections = document.querySelectorAll('section')
+      for (const s of sections) {
+        const h2 = s.querySelector('h2')
+        if (h2 && h2.textContent.trim().startsWith(headingText)) return s
+      }
+      return null
+    }
+
+    const getContentDivs = (section) => {
+      if (!section) return []
+      const h2 = section.querySelector('h2')
+      if (!h2) return []
+      const content = h2.parentElement?.nextElementSibling
+      if (!content) return []
+      return [...content.querySelectorAll(':scope > div')].filter(d => d.querySelector('p'))
+    }
+
+    // Profile top card
+    const sections = document.querySelectorAll('section')
+    let topSection = null
+    for (const s of sections) {
+      const h2 = s.querySelector('h2')
+      if (h2 && ['0 notifications', 'Suggested for you', 'Analytics', 'Activity'].includes(h2.textContent.trim())) continue
+      if (s.textContent.trim().length > 200) { topSection = s; break }
+    }
+
+    const nameH1 = document.querySelector('h1')
+    const profileName = nameH1 ? txt(nameH1) : document.title.replace(' | LinkedIn', '').trim()
+    let headline = ''
+    let location = ''
+    if (topSection) {
+      const allP = [...topSection.querySelectorAll('p')]
+      headline = txt(allP[0])
+      for (const p of allP) {
+        const t = txt(p)
+        if (t.includes(',') && !t.includes('at ') && !t.includes('\u00B7') && !t.includes('follow')) {
+          location = t
+          break
+        }
+      }
+    }
+    const photoImg = document.querySelector('img[src*="profile-displayphoto"]')
+    const profile = {
+      name: profileName,
+      headline,
+      location,
+      connections: '',
+      imageurl: photoImg ? photoImg.getAttribute('src') || '' : ''
+    }
+
+    // Experience
+    const positions = []
+    const expSection = findSection('Experience')
+    if (expSection) {
+      const companyGroups = expSection.querySelectorAll('[componentkey^="entity-collection"]')
+      companyGroups.forEach(group => {
+        const allP = [...group.querySelectorAll('p')].filter(p => txt(p).length > 0)
+        if (allP.length === 0) return
+        const companyName = txt(allP[0])
+        const companyLink = group.querySelector('a[href*="/company/"]')
+        const companyUrl = companyLink ? companyLink.getAttribute('href') : ''
+
+        const positionLis = group.querySelectorAll('ul > li')
+        if (positionLis.length > 0) {
+          positionLis.forEach(li => {
+            const liPs = [...li.querySelectorAll('p')].filter(p => txt(p).length > 0)
+            if (liPs.length === 0) return
+            const descEl = li.querySelector('[data-testid="expandable-text-box"]')
+            let dateStr = ''
+            for (let i = 1; i < liPs.length; i++) {
+              const t = txt(liPs[i])
+              if (t.includes('\u00B7') || /\d{4}/.test(t)) { dateStr = t; break }
+            }
+            const dateRange = dateStr.split('\u00B7')[0].trim()
+            const dateParts = dateRange.split(' - ')
+            positions.push({
+              title: txt(liPs[0]),
+              companyName,
+              link: companyUrl,
+              url: companyUrl,
+              location: '',
+              description: descEl ? txt(descEl) : '',
+              date: dateRange,
+              date1: dateParts[0] ? dateParts[0].trim() : '',
+              date2: dateParts[1] ? dateParts[1].trim() : ''
+            })
+          })
+        } else {
+          const descEl = group.querySelector('[data-testid="expandable-text-box"]')
+          let dateStr = ''
+          for (let i = 1; i < allP.length; i++) {
+            const t = txt(allP[i])
+            if (t.includes('\u00B7') || /\d{4}/.test(t.split(' ')[0])) { dateStr = t; break }
+          }
+          const dateRange = dateStr.split('\u00B7')[0].trim()
+          const dateParts = dateRange.split(' - ')
+          positions.push({
+            title: txt(allP[3]) || txt(allP[1]),
+            companyName,
+            link: companyUrl,
+            url: companyUrl,
+            location: txt(allP[2]) || '',
+            description: descEl ? txt(descEl) : '',
+            date: dateRange,
+            date1: dateParts[0] ? dateParts[0].trim() : '',
+            date2: dateParts[1] ? dateParts[1].trim() : ''
+          })
+        }
+      })
+    }
+
+    // Education
+    const educations = []
+    const eduSection = findSection('Education')
+    const eduDivs = getContentDivs(eduSection)
+    eduDivs.forEach(div => {
+      const ps = [...div.querySelectorAll('p')].filter(p => txt(p).length > 0)
+      if (ps.length === 0) return
+      const schoolLink = div.querySelector('a[href*="/school/"]')
+      const url = schoolLink ? schoolLink.getAttribute('href') : ''
+      const dateStr = txt(ps[2]) || ''
+      const dateParts = dateStr.split('\u2013')
+      educations.push({
+        title: txt(ps[0]),
+        degree: txt(ps[1]) || '',
+        fieldOfStudy: txt(ps[1]) || '',
+        url,
+        date1: dateParts[0] ? dateParts[0].trim() : '',
+        date2: dateParts[1] ? dateParts[1].trim() : '',
+        description: ''
+      })
+    })
+
+    // Skills
+    const skills = []
+    const skillsSection = findSection('Skills')
+    if (skillsSection) {
+      const allSectionP = [...skillsSection.querySelectorAll('p')].filter(p => txt(p).length > 0)
+      for (let i = 0; i < allSectionP.length; i += 2) {
+        const name = txt(allSectionP[i])
+        if (name && !name.includes('Show all') && !name.includes('Private')) {
+          skills.push({ title: name, count: '' })
+        }
+      }
+    }
+
+    // Languages
+    const languages = []
+    const langSection = findSection('Languages')
+    if (langSection) {
+      const langDivs = getContentDivs(langSection)
+      langDivs.forEach(div => {
+        const ps = [...div.querySelectorAll('p')].filter(p => txt(p).length > 0)
+        for (let i = 0; i < ps.length; i += 2) {
+          const n = txt(ps[i])
+          const pr = ps[i + 1] ? txt(ps[i + 1]) : ''
+          if (n) languages.push({ name: n, proficiency: pr })
+        }
+      })
+    }
+
+    // Projects
+    const projects = []
+    const projSection = findSection('Projects')
+    const projDivs = getContentDivs(projSection)
+    projDivs.forEach(div => {
+      const ps = [...div.querySelectorAll('p')].filter(p => txt(p).length > 0)
+      if (ps.length === 0) return
+      const descEl = div.querySelector('[data-testid="expandable-text-box"]')
+      const link = div.querySelector('a[href*="http"]')
+      projects.push({
+        name: txt(ps[0]),
+        date: txt(ps[1]) || '',
+        description: descEl ? txt(descEl) : '',
+        link: link ? link.getAttribute('href') : ''
+      })
+    })
+
+    // Certifications
+    const accomplishments = []
+    const certSection = findSection('Licenses')
+    const certDivs = getContentDivs(certSection)
+    certDivs.forEach(div => {
+      const ps = [...div.querySelectorAll('p')].filter(p => txt(p).length > 0)
+      if (ps.length > 0) accomplishments.push({ title: txt(ps[0]), count: '', items: [] })
+    })
+
+    return {
+      profile,
+      about: { text: '' },
+      positions,
+      educations,
+      skills,
+      recommendations: { givenCount: '0', receivedCount: '0', given: [], received: [] },
+      accomplishments,
+      courses: [],
+      languages,
+      projects,
+      peopleAlsoViewed: [],
+      volunteerExperience: [],
+      contact: []
+    }
+  })
+}
+
 module.exports = async (browser, cookies, url, waitTimeToScrapMs = 500, hasToGetContactInfo = false, puppeteerAuthenticate = undefined) => {
   logger.info(`starting scraping url: ${url}`)
 
   const page = await openPage({ browser, cookies, url, puppeteerAuthenticate })
-  const profilePageIndicatorSelector = '.pv-profile-section'
-  await page.waitForSelector(profilePageIndicatorSelector, { timeout: 5000 })
+
+  // Check for authwall (expired session)
+  const isAuthwall = await page.evaluate(() =>
+    window.location.href.includes('/authwall') || document.title.toLowerCase().includes('inschrijven')
+  )
+  if (isAuthwall) {
+    await page.close()
+    throw new Error('authwall: LinkedIn session expired, re-authentication required')
+  }
+
+  // Accept cookie consent if present
+  await page.evaluate(() => {
+    const btn = Array.from(document.querySelectorAll('button')).find(
+      (b) => b.textContent.trim().toLowerCase() === 'accept'
+    )
+    if (btn) btn.click()
+  })
+  await new Promise((r) => setTimeout(r, 2000))
+
+  // Wait for sections to appear (profile content is SDUI rendered)
+  await page.waitForFunction(() => {
+    return document.querySelectorAll('section h2').length > 1
+  }, { timeout: 30000 })
     .catch(() => {
-      //why doesn't throw error instead of continuing scraping?
-      //because it can be just a false negative meaning LinkedIn only changed that selector but everything else is fine :)
-      logger.warn('profile selector was not found')
+      logger.warn('profile content did not fully render in time')
     })
 
   logger.info('scrolling page to the bottom')
   await scrollToPageBottom(page)
-  
-  if(waitTimeToScrapMs) {
-    logger.info(`applying 1st delay`)
-    await new Promise((resolve) => { setTimeout(() => { resolve() }, waitTimeToScrapMs / 2)})
-  }
+
+  // Wait for lazy-loaded sections to render after scrolling
+  await new Promise((resolve) => { setTimeout(resolve, 2000) })
+
+  // Scroll again in case new content was loaded
+  await scrollToPageBottom(page)
+  await new Promise((resolve) => { setTimeout(resolve, 1000) })
 
   await seeMoreButtons.clickAll(page)
 
-  if(waitTimeToScrapMs) {
-    logger.info(`applying 2nd (and last) delay`)
-    await new Promise((resolve) => { setTimeout(() => { resolve() }, waitTimeToScrapMs / 2)})
-  }
+  // Final wait for content to settle
+  await new Promise((resolve) => { setTimeout(resolve, 1000) })
 
-  const [profile] = await scrapSection(page, template.profile)
-  const [about] = await scrapSection(page, template.about)
-  const positions = await scrapSection(page, template.positions)
-  const educations = await scrapSection(page, template.educations)
-  const [recommendationsCount] = await scrapSection(page, template.recommendationsCount)
-  const recommendationsReceived = await scrapSection(page, template.recommendationsReceived)
-  const recommendationsGiven = await scrapSection(page, template.recommendationsGiven)
-  const skills = await scrapSection(page, template.skills)
-  const accomplishments = await scrapSection(page, template.accomplishments)
-  const courses = await scrapAccomplishmentPanel(page, 'courses')
-  const languages = await scrapAccomplishmentPanel(page, 'languages')
-  const projects = await scrapAccomplishmentPanel(page, 'projects')
-  const volunteerExperience = await scrapSection(page, template.volunteerExperience)
-  const peopleAlsoViewed = await scrapSection(page, template.peopleAlsoViewed)
-  const contact = hasToGetContactInfo ? await contactInfo(page) : []
+  const rawProfile = await extractProfileData(page)
+
+  if (hasToGetContactInfo) {
+    rawProfile.contact = await contactInfo(page) || []
+  }
 
   await page.close()
   logger.info(`finished scraping url: ${url}`)
 
-  const rawProfile = {
-    profile,
-    about,
-    positions,
-    educations,
-    skills,
-    recommendations: {
-      givenCount: recommendationsCount ? recommendationsCount.given : "0",
-      receivedCount: recommendationsCount ? recommendationsCount.received : "0",
-      given: recommendationsReceived,
-      received: recommendationsGiven
-    },
-    accomplishments,
-    courses,
-    languages,
-    projects,
-    peopleAlsoViewed,
-    volunteerExperience,
-    contact
-  }
-
   const cleanedProfile = cleanProfileData(rawProfile)
   return cleanedProfile
 }

package/src/profile/profileScraperTemplate.js

@@ -1,189 +1,186 @@
-const profileSelector = '.core-rail > *:first-child section >'
-
 const template = {
   profile: {
-    selector: '.pb5',
+    selector: '.scaffold-layout__main',
     fields: {
-			name: `.text-heading-xlarge`,
-      headline: `.text-body-medium`,
-      location: `.pb2 .text-body-small`,
-      connections: `li.text-body-small`,
+      name: '.text-heading-xlarge',
+      headline: '.text-body-medium',
+      location: '.text-body-small.inline.t-black--light.break-words',
+      connections: '.t-bold',
       imageurl: {
-				selector: `img.pv-top-card__photo`,
+        selector: 'img.pv-top-card-profile-picture__image',
         attribute: 'src'
       }
     }
   },
   about: {
-    selector: '.pv-about-section',
+    selector: '#about ~ .display-flex .inline-show-more-text',
     fields: {
-      text: 'div'
+      text: 'span[aria-hidden="true"]'
     }
   },
   positions: {
-    selector: 'div[id="experience"] + div + div li.artdeco-list__item',
+    selector: '#experience ~ .pvs-list__outer-container .pvs-list > li.pvs-list__paged-list-item',
     fields: {
-      title: 'h3',
+      title: '.mr1.hoverable-link-text.t-bold > span',
       link: {
-        selector: 'a',
-        attribute: 'href',
+        selector: 'a.optional-action-target-wrapper',
+        attribute: 'href'
       },
       url: {
-        selector: 'a',
+        selector: 'a.optional-action-target-wrapper',
         attribute: 'href'
       },
-      companyName: 'div.t-bold span:first-child',
-      location: '.pv-entity__location span:last-child',
-      description: '.pv-entity__description',
-      date1: '.pv-entity__date-range span:last-child',
-      date2: '.pv-entity__bullet-item-v2',
+      companyName: '.t-14.t-normal > span',
+      location: '.t-14.t-normal.t-black--light > span',
+      description: '.pvs-list__outer-container .inline-show-more-text span[aria-hidden="true"]',
+      date1: '.pvs-entity__caption-wrapper',
+      date2: '.pvs-entity__caption-wrapper',
       roles: {
-        selector: 'div.full-width > div > ul div[data-view-name]',
+        selector: '.pvs-entity__sub-components li.pvs-list__paged-list-item',
         hasChildrenFields: true,
         fields: {
-          title: 'div.t-bold span:first-child',
-          description: 'div.display-flex div.full-width > span[aria-hidden]',
+          title: '.mr1.hoverable-link-text.t-bold > span',
+          description: '.inline-show-more-text span[aria-hidden="true"]',
           date: '.pvs-entity__caption-wrapper',
-          location: '.pv-entity__location span:last-child'
+          location: '.t-14.t-normal.t-black--light > span'
         }
       }
     }
   },
   educations: {
-    selector: '#education-section li',
+    selector: '#education ~ .pvs-list__outer-container .pvs-list > li.pvs-list__paged-list-item',
     fields: {
-      title: 'h3',
-      degree: 'span[class=pv-entity__comma-item]',
+      title: '.hoverable-link-text.t-bold > span',
+      degree: '.t-14.t-normal > span',
       url: {
         selector: 'a',
         attribute: 'href'
       },
-	    fieldOfStudy: 'p.pv-entity__fos span:nth-child(2)',
-      date1: '.pv-entity__dates time:nth-child(1)',
-      date2: '.pv-entity__dates time:nth-child(2)',
-      description: '.pv-entity__description'
+      fieldOfStudy: '.t-14.t-normal > span',
+      date1: '.pvs-entity__caption-wrapper',
+      date2: '.pvs-entity__caption-wrapper',
+      description: '.inline-show-more-text span[aria-hidden="true"]'
     }
   },
   skills: {
-    selector: '.pv-skill-category-entity__skill-wrapper',
+    selector: '#skills ~ .pvs-list__outer-container .pvs-list > li.pvs-list__paged-list-item',
     fields: {
-      title: '.pv-skill-category-entity__name-text',
-      count: '.pv-skill-category-entity__endorsement-count'
+      title: '.mr1.hoverable-link-text.t-bold > span',
+      count: '.t-14.t-normal.t-black--light > span'
     }
   },
   recommendationsCount: {
-    selector: '.recommendations-inlining',
+    selector: '#recommendations ~ .pvs-list__outer-container',
     fields: {
       received: '.artdeco-tab:nth-child(1)',
       given: '.artdeco-tab:nth-child(2)'
     }
   },
   recommendationsReceived: {
-    selector: '.recommendations-inlining',
+    selector: '#recommendations ~ .pvs-list__outer-container .pvs-list > li.pvs-list__paged-list-item',
     fields: {
       user: {
-        selector: '.pv-recommendation-entity__member',
+        selector: 'a',
         attribute: 'href'
       },
-      text: 'blockquote.pv-recommendation-entity__text',
+      text: '.inline-show-more-text span[aria-hidden="true"]',
       profileImage: {
-        selector: 'a img',
+        selector: 'img',
         attribute: 'src'
       },
       name: {
-        selector: 'a h3'
+        selector: '.t-bold > span'
       },
       userDescription: {
-        selector: '.pv-recommendation-entity__headline'
+        selector: '.t-14.t-normal > span'
       }
     }
   },
   recommendationsGiven: {
-    selector: '.artdeco-tabpanel li.pv-recommendation-entity',
+    selector: '#recommendations ~ .pvs-list__outer-container .pvs-list > li.pvs-list__paged-list-item',
     fields: {
       user: {
-        selector: '.pv-recommendation-entity__member',
+        selector: 'a',
         attribute: 'href'
       },
-      text: 'blockquote.pv-recommendation-entity__text',
+      text: '.inline-show-more-text span[aria-hidden="true"]',
       profileImage: {
-        selector: 'a img',
+        selector: 'img',
         attribute: 'src'
       },
       name: {
-        selector: 'a h3'
+        selector: '.t-bold > span'
       },
       userDescription: {
-        selector: '.pv-recommendation-entity__headline'
+        selector: '.t-14.t-normal > span'
       }
     }
   },
   accomplishments: {
-    selector: '.pv-accomplishments-section > div',
+    selector: '#honors_and_awards ~ .pvs-list__outer-container .pvs-list > li.pvs-list__paged-list-item',
     fields: {
-      count: 'h3 span:last-child',
-      title: '.pv-accomplishments-block__title',
+      count: '.t-14.t-normal.t-black--light > span',
+      title: '.mr1.hoverable-link-text.t-bold > span',
       items: {
-        selector: 'li',
+        selector: '.pvs-list__outer-container li',
         isMultipleFields: true
       }
     }
   },
   peopleAlsoViewed: {
-    selector: 'li.pv-browsemap-section__member-container',
+    selector: '.pv-browsemap-section li',
     fields: {
       user: {
         selector: 'a',
         attribute: 'href'
       },
-      text: 'p',
+      text: '.t-14.t-normal',
       profileImage: {
-        selector: 'a img',
+        selector: 'img',
         attribute: 'src'
       },
       name: {
-        selector: '.name'
+        selector: '.t-bold'
       }
     }
   },
   volunteerExperience: {
-    selector: 'section.volunteering-section li',
+    selector: '#volunteering_experience ~ .pvs-list__outer-container .pvs-list > li.pvs-list__paged-list-item',
     fields: {
-      title: 'h3',
-      experience: 'span[class=pv-entity__secondary-title]',
-      location: '.pv-entity__location span:nth-child(2)',
-      description: '.pv-volunteer-causes',
-      date1: '.pv-entity__date-range span:nth-child(2)',
-      date2: '.pv-entity__bullet-item'
+      title: '.mr1.hoverable-link-text.t-bold > span',
+      experience: '.t-14.t-normal > span',
+      location: '.t-14.t-normal.t-black--light > span',
+      description: '.inline-show-more-text span[aria-hidden="true"]',
+      date1: '.pvs-entity__caption-wrapper',
+      date2: '.pvs-entity__caption-wrapper'
     }
   },
   courses: {
-    selector: '.pv-accomplishments-section',
+    selector: '#courses ~ .pvs-list__outer-container .pvs-list > li.pvs-list__paged-list-item',
     fields: {
-      name: '.pv-accomplishment-entity__title',
-      year: '.pv-accomplishment-entity__course-number'
+      name: '.mr1.hoverable-link-text.t-bold > span',
+      year: '.t-14.t-normal > span'
     }
   },
   languages: {
-    selector: '.pv-accomplishments-block.languages li',
+    selector: '#languages ~ .pvs-list__outer-container .pvs-list > li.pvs-list__paged-list-item',
     fields: {
-      name: '.pv-accomplishment-entity__title',
-      proficiency: '.pv-accomplishment-entity__proficiency',
+      name: '.mr1.t-bold > span',
+      proficiency: '.t-14.t-normal.t-black--light > span'
     }
   },
   projects: {
-    selector: '.pv-accomplishments-block.projects li',
+    selector: '#projects ~ .pvs-list__outer-container .pvs-list > li.pvs-list__paged-list-item',
     fields: {
-      name: '.pv-accomplishment-entity__title',
-      date: '.pv-accomplishment-entity__date',
-      description: '.pv-accomplishment-entity__description',
+      name: '.mr1.hoverable-link-text.t-bold > span',
+      date: '.pvs-entity__caption-wrapper',
+      description: '.inline-show-more-text span[aria-hidden="true"]',
       link: {
-        selector: '.mt4',
+        selector: 'a',
         attribute: 'href'
       }
     }
   }
 }
 
-
 module.exports = template

package/src/profile/scrapAccomplishmentPanel.js

@@ -1,18 +1,11 @@
-const scrapSection = require('../scrapSection');
-const template = require('./profileScraperTemplate');
+const scrapSection = require('../scrapSection')
+const template = require('./profileScraperTemplate')
 
 const scrapAccomplishmentPanel = async (page, section) => {
-  const queryString = `.pv-accomplishments-block.${section} button`
-
-  const openingButton = await page.$(queryString);
-
-  if (openingButton) {
-    await page.evaluate((q) => {
-      document.querySelector(q).click();
-    }, queryString);
-
-    return scrapSection(page, template[section]);
+  if (!template[section]) {
+    return []
   }
-};
+  return scrapSection(page, template[section])
+}
 
-module.exports = scrapAccomplishmentPanel;
+module.exports = scrapAccomplishmentPanel

package/src/profile/scrollToPageBottom.js

@@ -3,21 +3,20 @@ const logger = require('../logger')(__filename)
 module.exports = async (page) => {
   const MAX_TIMES_TO_SCROLL = 25
   const TIMEOUT_BETWEEN_SCROLLS = 500
-  const PAGE_BOTTOM_SELECTOR_STRING = '#expanded-footer'
 
   for (let i = 0; i < MAX_TIMES_TO_SCROLL; i++) {
     await page.evaluate(() => window.scrollBy(0, window.innerHeight))
 
-    const hasReachedEnd = await page.waitForSelector(PAGE_BOTTOM_SELECTOR_STRING, {
-      visible: true,
-      timeout: TIMEOUT_BETWEEN_SCROLLS
-    }).catch(() => {
-      logger.info(`scrolling to page bottom (${i + 1})`)
+    const hasReachedEnd = await page.evaluate(() => {
+      return (window.innerHeight + window.scrollY) >= (document.body.scrollHeight - 200)
     })
 
     if (hasReachedEnd) {
       return
     }
+
+    await new Promise(resolve => setTimeout(resolve, TIMEOUT_BETWEEN_SCROLLS))
+    logger.info(`scrolling to page bottom (${i + 1})`)
   }
 
   logger.warn('page bottom not found')

package/src/profile/seeMoreButtons.js

@@ -1,42 +1,18 @@
 const logger = require('../logger')(__filename)
-const seeMoreButtons = [
-  {
-    id: 'SHOW_MORE_ABOUT',
-    selector: '#line-clamp-show-more-button'
-  },{
-    id: 'SHOW_MORE_EXPERIENCES',
-    selector: '#experience-section .pv-profile-section__see-more-inline'
-  },{
-    id: 'SEE_MORE_EXPERIENCES',
-    selector: '#experience-section .inline-show-more-text__button'
-  },{
-    id: 'SHOW_MORE_CERTIFICATIONS',
-    selector: '#certifications-section .pv-profile-section__see-more-inline'
-  },{
-    id: 'SHOW_MORE_SKILLS',
-    selector: '.pv-skills-section__additional-skills'
-  },{
-    id: 'SEE_MORE_RECOMMENDATIONS',
-    selector: '.recommendations-inlining #line-clamp-show-more-button'
-  }
-]
-
 
-const clickAll = async(page) => {
-  for(let i = 0; i < seeMoreButtons.length; i++){
-    const button = seeMoreButtons[i]
-    const elems = await page.$$(button.selector)
+const clickAll = async (page) => {
+  const clicked = await page.evaluate(() => {
+    let count = 0
+    // Only click expandable text buttons (inline expand, not navigation)
+    const expandButtons = document.querySelectorAll('[data-testid="expandable-text-button"]')
+    expandButtons.forEach(btn => { btn.click(); count++ })
+    return count
+  })
 
-    for(let j = 0; j < elems.length; j++){
-      const elem = elems[j]
-      if (elem) {
-        await elem.click()
-          .catch((e) => logger.warn(`couldn't click on ${button.selector}, it's probably invisible`))
-      }
-    }
+  if (clicked > 0) {
+    logger.info(`clicked ${clicked} show-more buttons`)
+    await new Promise(resolve => setTimeout(resolve, 500))
   }
-
-  return
 }
 
 module.exports = { clickAll }

package/src/scrapedin.js

@@ -4,7 +4,22 @@ const profile = require('./profile/profile')
 const company = require('./company/company')
 const logger = require('./logger')(__filename)
 
-module.exports = async ({ cookies, email, password, isHeadless, hasToLog, hasToGetContactInfo, puppeteerArgs, puppeteerAuthenticate, endpoint } = { isHeadless: true, hasToLog: false }) => {
+const saveBrowserCookies = async (browser, cookiesPath) => {
+  if (!cookiesPath) return
+  try {
+    const pages = await browser.pages()
+    if (pages.length > 0) {
+      const pageCookies = await pages[0].cookies()
+      const fs = require('fs')
+      fs.writeFileSync(cookiesPath, JSON.stringify(pageCookies, null, 2))
+      logger.info('cookies saved to: ' + cookiesPath)
+    }
+  } catch (e) {
+    logger.warn('failed to save cookies: ' + e.message)
+  }
+}
+
+module.exports = async ({ cookies, email, password, isHeadless, hasToLog, hasToGetContactInfo, cookiesPath, puppeteerArgs, puppeteerAuthenticate, endpoint } = { isHeadless: true, hasToLog: false }) => {
   if (!hasToLog) {
     logger.stopLogging()
   }
@@ -26,7 +41,11 @@ module.exports = async ({ cookies, email, password, isHeadless, hasToLog, hasToG
     logger.info('email and password was provided, we\'re going to login...')
 
     try {
-      await login(browser, email, password, logger)
+      const loginResult = await login(browser, email, password, logger)
+      // Only save cookies if login fully completed (no 2FA challenge)
+      if (loginResult && !loginResult.hadChallenge && cookiesPath) {
+        await saveBrowserCookies(browser, cookiesPath)
+      }
     } catch (e) {
       if (!endpoint) {
         await browser.close()