@mvegter/scrapedin 1.1.0 → 1.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mvegter/scrapedin",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "description": "linkedin scraper updated for 2025+ website",
   "keywords": [
     "linkedin",

package/src/login.js

@@ -2,69 +2,173 @@ const openPage = require('./openPage')
 const logger = require('./logger')(__filename)
 const pkg = require('./package')
 
+const ACCEPT_COOKIES_SELECTORS = [
+  'button:has-text("Accept")',
+  'button:has-text("Alle accepteren")',
+  'button[action-type="ACCEPT"]',
+  '.cookie-consent-v2__button--accept',
+  '#artdeco-global-alert-container button:has-text("Accept")'
+]
+
+const acceptCookies = async (page) => {
+  for (const selector of ACCEPT_COOKIES_SELECTORS) {
+    try {
+      const btn = await page.$(selector)
+      if (btn) {
+        await btn.click()
+        await new Promise((r) => setTimeout(r, 1000))
+        return
+      }
+    } catch {
+      // selector might not exist, try next
+    }
+  }
+  // Fallback: find any button with Accept text
+  try {
+    await page.evaluate(() => {
+      const btn = Array.from(document.querySelectorAll('button')).find(
+        (b) => b.textContent.trim().toLowerCase() === 'accept'
+      )
+      if (btn) btn.click()
+    })
+    await new Promise((r) => setTimeout(r, 1000))
+  } catch {
+    // ignore
+  }
+}
+
+const fillField = async (page, fieldValue) => {
+  // LinkedIn renders two sets of inputs: hidden (CSS-only, not visible) and visible.
+  // We find all <input> elements matching the autocomplete attribute, then pick
+  // the first one that is actually visible (has non-zero dimensions).
+  const autocomplete = fieldValue === 'username webauthn' ? 'username' : 'current-password'
+
+  const visibleInput = await page.evaluate((auto, val) => {
+    const inputs = Array.from(document.querySelectorAll(`input[autocomplete="${auto}"]`))
+    for (const input of inputs) {
+      const rect = input.getBoundingClientRect()
+      if (rect.width > 0 && rect.height > 0) {
+        const style = window.getComputedStyle(input)
+        if (style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0') {
+          input.focus()
+          input.value = ''
+          return true
+        }
+      }
+    }
+    return false
+  }, autocomplete, fieldValue)
+
+  if (visibleInput) {
+    await page.keyboard.type(fieldValue, { delay: 50 })
+  }
+}
+
+const clickSignIn = async (page) => {
+  // Find the "Sign in" button, excluding "Sign in with Apple" etc.
+  await page.evaluate(() => {
+    const buttons = Array.from(document.querySelectorAll('button'))
+    const signInBtn = buttons.find(
+      (b) => {
+        const text = b.textContent.trim().toLowerCase()
+        return (text === 'sign in' || text === 'inloggen' || text === 'aanmelden') &&
+          !text.includes('apple')
+      }
+    )
+    if (signInBtn) {
+      signInBtn.click()
+      return true
+    }
+    return false
+  })
+}
+
+const LOGGED_IN_PATHS = ['/feed', '/mynetwork', '/in/']
+
 module.exports = async (browser, email, password) => {
   const url = 'https://www.linkedin.com/login'
   const page = await openPage({ browser, url })
   logger.info(`logging at: ${url}`)
 
-  await page.waitForSelector('#username')
+  // Accept cookie consent if present
+  await acceptCookies(page)
+  await new Promise((r) => setTimeout(r, 1000))
 
-  await page.$('#username')
-    .then((emailElement) => emailElement.type(email))
-  await page.$('#password')
-    .then((passwordElement) => passwordElement.type(password))
+  // Fill in email field
+  await fillField(page, 'username webauthn')
+  await new Promise((r) => setTimeout(r, 500))
 
-  await page.locator('button[type="submit"]').click()
+  // Fill in password field
+  await fillField(page, 'current-password')
+  await new Promise((r) => setTimeout(r, 500))
 
-  return page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 15000 })
-    .then(async () => {
-      const currentUrl = page.url()
-      if (currentUrl.includes('/feed') || currentUrl.includes('/mynetwork') || currentUrl.includes('/in/')) {
-        logger.info('logged in, redirected to: ' + currentUrl)
-        await page.close()
-        return
-      }
-      throw new Error('unexpected redirect: ' + currentUrl)
-    })
-    .catch(async () => {
-      logger.warn('successful login element was not found')
-      const emailError = await page.evaluate(() => {
-        const e = document.querySelector('div[error-for=username], #error-for-username')
-        if (!e) { return false }
-        const style = window.getComputedStyle(e)
-        return style && style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0'
-      })
-
-      const passwordError = await page.evaluate(() => {
-        const e = document.querySelector('div[error-for=password], #error-for-password')
-        if (!e) { return false }
-        const style = window.getComputedStyle(e)
-        return style && style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0'
-      })
-
-      const manualChallengeRequested = await page.evaluate(() => {
-        const e = document.querySelector('.flow-challenge-content, #challenge, [data-test-id="challenge"]')
-        if (!e) { return false }
-        const style = window.getComputedStyle(e)
-        return style && style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0'
-      })
-
-      if (emailError) {
-        logger.info('wrong username element found')
-        return Promise.reject(new Error(`linkedin: invalid username: ${email}`))
-      }
+  await clickSignIn(page)
 
-      if (passwordError) {
-        logger.info('wrong password element found')
-        return Promise.reject(new Error('linkedin: invalid password'))
-      }
+  let hadChallenge = false
+
+  try {
+    await page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 15000 })
+  } catch {
+    // Navigation timeout is expected — the page may not navigate if already on login
+  }
 
-      if (manualChallengeRequested) {
-        logger.warn('manual check was required')
-        return Promise.reject(new Error(`linkedin: manual check was required, verify if your login is properly working manually or report this issue: ${pkg.name} ${pkg.version} ${pkg.bugs.url}`))
+  // Wait for either the feed/mynetwork or challenge page
+  const maxWaitMs = 120000
+  const start = Date.now()
+  let resolved = false
+  while (Date.now() - start < maxWaitMs && !resolved) {
+    const currentUrl = page.url()
+    if (LOGGED_IN_PATHS.some((p) => currentUrl.includes(p))) {
+      logger.info('logged in, redirected to: ' + currentUrl)
+      resolved = true
+      break
+    }
+    if (currentUrl.includes('/checkpoint')) {
+      if (!hadChallenge) {
+        logger.warn('2FA challenge detected, please complete the verification in the browser window (waiting up to 2 minutes)...')
+        hadChallenge = true
       }
+      await new Promise((r) => setTimeout(r, 2000))
+      continue
+    }
+    await new Promise((r) => setTimeout(r, 500))
+  }
 
-      logger.error('could not find any element to retrieve a proper error')
-      return Promise.reject(new Error(`${pkg.name} ${pkg.version} login is not working, please report: ${pkg.bugs.url}`))
+  if (!resolved) {
+    const finalUrl = page.url()
+    logger.warn('successful login element was not found, url: ' + finalUrl)
+
+    const emailError = await page.evaluate(() => {
+      const e = document.querySelector('div[error-for=username], #error-for-username')
+      if (!e) { return false }
+      const style = window.getComputedStyle(e)
+      return style && style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0'
+    })
+
+    const passwordError = await page.evaluate(() => {
+      const e = document.querySelector('div[error-for=password], #error-for-password')
+      if (!e) { return false }
+      const style = window.getComputedStyle(e)
+      return style && style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0'
     })
+
+    if (emailError) {
+      logger.info('wrong username element found')
+      await page.close()
+      return Promise.reject(new Error(`linkedin: invalid username: ${email}`))
+    }
+
+    if (passwordError) {
+      logger.info('wrong password element found')
+      await page.close()
+      return Promise.reject(new Error('linkedin: invalid password'))
+    }
+
+    logger.error('could not find any element to retrieve a proper error')
+    await page.close()
+    return Promise.reject(new Error(`${pkg.name} ${pkg.version} login is not working, please report: ${pkg.bugs.url}`))
+  }
+
+  await page.close()
+  return { hadChallenge }
 }

package/src/openPage.js

@@ -18,6 +18,6 @@ module.exports = async ({ browser, cookies, url, puppeteerAuthenticate }) => {
     await page.authenticate(puppeteerAuthenticate)
   }
 
-  await page.goto(url, { waitUntil: 'load' })
+  await page.goto(url, { waitUntil: 'networkidle2' })
   return page
 }

package/src/profile/profile.js

@@ -10,52 +10,49 @@ const extractProfileData = async (page) => {
   return page.evaluate(() => {
     const txt = (el) => el ? (el.textContent || '').trim() : ''
 
-    const byViewName = (name) => document.querySelector(`[data-view-name="${name}"]`)
-
-    const getSection = (viewName) => {
-      const el = byViewName(viewName)
-      if (!el) return null
-      return el.querySelector('section') || el.closest('section')
+    const findSection = (headingText) => {
+      const sections = document.querySelectorAll('section')
+      for (const s of sections) {
+        const h2 = s.querySelector('h2')
+        if (h2 && h2.textContent.trim().startsWith(headingText)) return s
+      }
+      return null
     }
 
-    // Get items from a section. For sections without ul/li, items are divs
-    // found after the h2 heading: h2.parent.nextSibling > div > div > div
-    const getSectionItems = (viewName) => {
-      const section = getSection(viewName)
+    const getContentDivs = (section) => {
       if (!section) return []
-      const ul = section.querySelector('ul')
-      if (ul) return [...ul.querySelectorAll(':scope > li')]
       const h2 = section.querySelector('h2')
       if (!h2) return []
-      const afterH2 = h2.parentElement.nextElementSibling
-      if (!afterH2) return []
-      const itemDivs = afterH2.querySelectorAll(':scope > div > div > div')
-      return [...itemDivs].filter(d => d.querySelectorAll('p').length > 0)
+      const content = h2.parentElement?.nextElementSibling
+      if (!content) return []
+      return [...content.querySelectorAll(':scope > div')].filter(d => d.querySelector('p'))
     }
 
-    // Profile top card: name is in h2, headline/location in p tags
-    const titleName = document.title.replace(' | LinkedIn', '').trim()
-    const mainLevel = byViewName('profile-main-level')
-    const topSection = mainLevel ? (mainLevel.querySelector('section') || mainLevel.closest('section')) : null
-    const nameH2 = topSection ? topSection.querySelector('h2') : null
-    const photoImg = document.querySelector('img[src*="profile-displayphoto"]')
+    // Profile top card
+    const sections = document.querySelectorAll('section')
+    let topSection = null
+    for (const s of sections) {
+      const h2 = s.querySelector('h2')
+      if (h2 && ['0 notifications', 'Suggested for you', 'Analytics', 'Activity'].includes(h2.textContent.trim())) continue
+      if (s.textContent.trim().length > 200) { topSection = s; break }
+    }
 
-    const profileName = nameH2 ? txt(nameH2) : titleName
+    const nameH1 = document.querySelector('h1')
+    const profileName = nameH1 ? txt(nameH1) : document.title.replace(' | LinkedIn', '').trim()
     let headline = ''
     let location = ''
-
     if (topSection) {
       const allP = [...topSection.querySelectorAll('p')]
       headline = txt(allP[0])
       for (const p of allP) {
         const t = txt(p)
-        if (t.includes(',') && !t.includes('at ') && !t.includes('\xB7')) {
+        if (t.includes(',') && !t.includes('at ') && !t.includes('\u00B7') && !t.includes('follow')) {
           location = t
           break
         }
       }
     }
-
+    const photoImg = document.querySelector('img[src*="profile-displayphoto"]')
     const profile = {
       name: profileName,
       headline,
@@ -64,101 +61,141 @@ const extractProfileData = async (page) => {
       imageurl: photoImg ? photoImg.getAttribute('src') || '' : ''
     }
 
-    // Experience: uses LazyColumn with company groups containing ul > li
+    // Experience
     const positions = []
-    const expSection = getSection('profile-card-experience')
+    const expSection = findSection('Experience')
     if (expSection) {
-      const lazyCol = expSection.querySelector('[data-component-type]') || expSection
-      const groups = [...lazyCol.children]
-
-      groups.forEach(group => {
-        const ul = group.querySelector('ul')
-        if (!ul) return
-
-        // Company info is in p tags before the ul
-        const allPs = group.querySelectorAll('p')
-        const beforeUlPs = []
-        for (const p of allPs) {
-          if (ul.contains(p)) break
-          beforeUlPs.push(p)
-        }
-        const companyName = txt(beforeUlPs[0])
+      const companyGroups = expSection.querySelectorAll('[componentkey^="entity-collection"]')
+      companyGroups.forEach(group => {
+        const allP = [...group.querySelectorAll('p')].filter(p => txt(p).length > 0)
+        if (allP.length === 0) return
+        const companyName = txt(allP[0])
         const companyLink = group.querySelector('a[href*="/company/"]')
         const companyUrl = companyLink ? companyLink.getAttribute('href') : ''
-        const companyLocation = txt(beforeUlPs[2])
-
-        const lis = [...ul.querySelectorAll(':scope > li')]
-        lis.forEach(li => {
-          const ps = [...li.querySelectorAll('div[role="button"] p')]
-          const descEl = li.querySelector('[data-testid="expandable-text-box"]')
-          const dateStr = txt(ps[2])
-          const dateRange = dateStr.split('\xB7')[0].trim()
+
+        const positionLis = group.querySelectorAll('ul > li')
+        if (positionLis.length > 0) {
+          positionLis.forEach(li => {
+            const liPs = [...li.querySelectorAll('p')].filter(p => txt(p).length > 0)
+            if (liPs.length === 0) return
+            const descEl = li.querySelector('[data-testid="expandable-text-box"]')
+            let dateStr = ''
+            for (let i = 1; i < liPs.length; i++) {
+              const t = txt(liPs[i])
+              if (t.includes('\u00B7') || /\d{4}/.test(t)) { dateStr = t; break }
+            }
+            const dateRange = dateStr.split('\u00B7')[0].trim()
+            const dateParts = dateRange.split(' - ')
+            positions.push({
+              title: txt(liPs[0]),
+              companyName,
+              link: companyUrl,
+              url: companyUrl,
+              location: '',
+              description: descEl ? txt(descEl) : '',
+              date: dateRange,
+              date1: dateParts[0] ? dateParts[0].trim() : '',
+              date2: dateParts[1] ? dateParts[1].trim() : ''
+            })
+          })
+        } else {
+          const descEl = group.querySelector('[data-testid="expandable-text-box"]')
+          let dateStr = ''
+          for (let i = 1; i < allP.length; i++) {
+            const t = txt(allP[i])
+            if (t.includes('\u00B7') || /\d{4}/.test(t.split(' ')[0])) { dateStr = t; break }
+          }
+          const dateRange = dateStr.split('\u00B7')[0].trim()
           const dateParts = dateRange.split(' - ')
           positions.push({
-            title: txt(ps[0]),
+            title: txt(allP[3]) || txt(allP[1]),
             companyName,
             link: companyUrl,
             url: companyUrl,
-            location: companyLocation,
+            location: txt(allP[2]) || '',
             description: descEl ? txt(descEl) : '',
             date: dateRange,
             date1: dateParts[0] ? dateParts[0].trim() : '',
             date2: dateParts[1] ? dateParts[1].trim() : ''
           })
-        })
+        }
       })
     }
 
     // Education
-    const eduItems = getSectionItems('profile-card-education')
-    const educations = eduItems.map(item => {
-      const ps = [...item.querySelectorAll('p')]
-      const link = item.querySelector('a[href*="/school/"]') ||
-        item.closest('div')?.parentElement?.querySelector('a[href*="/school/"]')
-      return {
+    const educations = []
+    const eduSection = findSection('Education')
+    const eduDivs = getContentDivs(eduSection)
+    eduDivs.forEach(div => {
+      const ps = [...div.querySelectorAll('p')].filter(p => txt(p).length > 0)
+      if (ps.length === 0) return
+      const schoolLink = div.querySelector('a[href*="/school/"]')
+      const url = schoolLink ? schoolLink.getAttribute('href') : ''
+      const dateStr = txt(ps[2]) || ''
+      const dateParts = dateStr.split('\u2013')
+      educations.push({
         title: txt(ps[0]),
-        degree: txt(ps[1]),
-        fieldOfStudy: txt(ps[1]),
-        url: link ? link.getAttribute('href') : '',
-        date1: ps[2] ? txt(ps[2]).split('\u2013')[0].trim() : '',
-        date2: ps[2] ? (txt(ps[2]).split('\u2013')[1] || '').trim() : '',
+        degree: txt(ps[1]) || '',
+        fieldOfStudy: txt(ps[1]) || '',
+        url,
+        date1: dateParts[0] ? dateParts[0].trim() : '',
+        date2: dateParts[1] ? dateParts[1].trim() : '',
         description: ''
-      }
+      })
     })
 
     // Skills
-    const skillItems = getSectionItems('profile-card-skills')
-    const skills = skillItems.map(item => {
-      const ps = [...item.querySelectorAll('p')]
-      return { title: txt(ps[0]), count: '' }
-    })
+    const skills = []
+    const skillsSection = findSection('Skills')
+    if (skillsSection) {
+      const allSectionP = [...skillsSection.querySelectorAll('p')].filter(p => txt(p).length > 0)
+      for (let i = 0; i < allSectionP.length; i += 2) {
+        const name = txt(allSectionP[i])
+        if (name && !name.includes('Show all') && !name.includes('Private')) {
+          skills.push({ title: name, count: '' })
+        }
+      }
+    }
 
     // Languages
-    const langItems = getSectionItems('profile-card-languages')
-    const languages = langItems.map(item => {
-      const ps = [...item.querySelectorAll('p')]
-      return { name: txt(ps[0]), proficiency: txt(ps[1]) }
-    })
+    const languages = []
+    const langSection = findSection('Languages')
+    if (langSection) {
+      const langDivs = getContentDivs(langSection)
+      langDivs.forEach(div => {
+        const ps = [...div.querySelectorAll('p')].filter(p => txt(p).length > 0)
+        for (let i = 0; i < ps.length; i += 2) {
+          const n = txt(ps[i])
+          const pr = ps[i + 1] ? txt(ps[i + 1]) : ''
+          if (n) languages.push({ name: n, proficiency: pr })
+        }
+      })
+    }
 
     // Projects
-    const projItems = getSectionItems('profile-card-projects')
-    const projects = projItems.map(item => {
-      const ps = [...item.querySelectorAll('p')]
-      const descEl = item.querySelector('[data-testid="expandable-text-box"]')
-      const link = item.querySelector('a[href*="http"]')
-      return {
+    const projects = []
+    const projSection = findSection('Projects')
+    const projDivs = getContentDivs(projSection)
+    projDivs.forEach(div => {
+      const ps = [...div.querySelectorAll('p')].filter(p => txt(p).length > 0)
+      if (ps.length === 0) return
+      const descEl = div.querySelector('[data-testid="expandable-text-box"]')
+      const link = div.querySelector('a[href*="http"]')
+      projects.push({
         name: txt(ps[0]),
-        date: txt(ps[1]),
+        date: txt(ps[1]) || '',
         description: descEl ? txt(descEl) : '',
         link: link ? link.getAttribute('href') : ''
-      }
+      })
     })
 
     // Certifications
-    const certItems = getSectionItems('profile-card-licenses-and-certifications')
-    const accomplishments = certItems.map(item => {
-      const ps = [...item.querySelectorAll('p')]
-      return { title: txt(ps[0]), count: '', items: [] }
+    const accomplishments = []
+    const certSection = findSection('Licenses')
+    const certDivs = getContentDivs(certSection)
+    certDivs.forEach(div => {
+      const ps = [...div.querySelectorAll('p')].filter(p => txt(p).length > 0)
+      if (ps.length > 0) accomplishments.push({ title: txt(ps[0]), count: '', items: [] })
     })
 
     return {
@@ -184,9 +221,27 @@ module.exports = async (browser, cookies, url, waitTimeToScrapMs = 500, hasToGet
 
   const page = await openPage({ browser, cookies, url, puppeteerAuthenticate })
 
-  // Wait for the SDUI profile to fully hydrate
+  // Check for authwall (expired session)
+  const isAuthwall = await page.evaluate(() =>
+    window.location.href.includes('/authwall') || document.title.toLowerCase().includes('inschrijven')
+  )
+  if (isAuthwall) {
+    await page.close()
+    throw new Error('authwall: LinkedIn session expired, re-authentication required')
+  }
+
+  // Accept cookie consent if present
+  await page.evaluate(() => {
+    const btn = Array.from(document.querySelectorAll('button')).find(
+      (b) => b.textContent.trim().toLowerCase() === 'accept'
+    )
+    if (btn) btn.click()
+  })
+  await new Promise((r) => setTimeout(r, 2000))
+
+  // Wait for sections to appear (profile content is SDUI rendered)
   await page.waitForFunction(() => {
-    return document.querySelector('[data-view-name="profile-card-experience"]')
+    return document.querySelectorAll('section h2').length > 1
   }, { timeout: 30000 })
     .catch(() => {
       logger.warn('profile content did not fully render in time')

package/src/scrapedin.js

@@ -4,7 +4,22 @@ const profile = require('./profile/profile')
 const company = require('./company/company')
 const logger = require('./logger')(__filename)
 
-module.exports = async ({ cookies, email, password, isHeadless, hasToLog, hasToGetContactInfo, puppeteerArgs, puppeteerAuthenticate, endpoint } = { isHeadless: true, hasToLog: false }) => {
+const saveBrowserCookies = async (browser, cookiesPath) => {
+  if (!cookiesPath) return
+  try {
+    const pages = await browser.pages()
+    if (pages.length > 0) {
+      const pageCookies = await pages[0].cookies()
+      const fs = require('fs')
+      fs.writeFileSync(cookiesPath, JSON.stringify(pageCookies, null, 2))
+      logger.info('cookies saved to: ' + cookiesPath)
+    }
+  } catch (e) {
+    logger.warn('failed to save cookies: ' + e.message)
+  }
+}
+
+module.exports = async ({ cookies, email, password, isHeadless, hasToLog, hasToGetContactInfo, cookiesPath, puppeteerArgs, puppeteerAuthenticate, endpoint } = { isHeadless: true, hasToLog: false }) => {
   if (!hasToLog) {
     logger.stopLogging()
   }
@@ -26,7 +41,11 @@ module.exports = async ({ cookies, email, password, isHeadless, hasToLog, hasToG
     logger.info('email and password was provided, we\'re going to login...')
 
     try {
-      await login(browser, email, password, logger)
+      const loginResult = await login(browser, email, password, logger)
+      // Only save cookies if login fully completed (no 2FA challenge)
+      if (loginResult && !loginResult.hadChallenge && cookiesPath) {
+        await saveBrowserCookies(browser, cookiesPath)
+      }
     } catch (e) {
       if (!endpoint) {
         await browser.close()