@mvegter/scrapedin 1.0.26 → 1.0.30

package/src/profile/profile.js

@@ -1,81 +1,81 @@
-const openPage = require('../openPage')
-const scrapSection = require('../scrapSection')
-const scrapAccomplishmentPanel = require('./scrapAccomplishmentPanel')
-const scrollToPageBottom = require('./scrollToPageBottom')
-const seeMoreButtons = require('./seeMoreButtons')
-const contactInfo = require('./contactInfo')
-const template = require('./profileScraperTemplate')
-const cleanProfileData = require('./cleanProfileData')
-
-const logger = require('../logger')(__filename)
-
-module.exports = async (browser, cookies, url, waitTimeToScrapMs = 500, hasToGetContactInfo = false, puppeteerAuthenticate = undefined) => {
-  logger.info(`starting scraping url: ${url}`)
-
-  const page = await openPage({ browser, cookies, url, puppeteerAuthenticate })
-  const profilePageIndicatorSelector = '.pv-profile-section'
-  await page.waitForSelector(profilePageIndicatorSelector, { timeout: 5000 })
-    .catch(() => {
-      //why doesn't throw error instead of continuing scraping?
-      //because it can be just a false negative meaning LinkedIn only changed that selector but everything else is fine :)
-      logger.warn('profile selector was not found')
-    })
-
-  logger.info('scrolling page to the bottom')
-  await scrollToPageBottom(page)
-  
-  if(waitTimeToScrapMs) {
-    logger.info(`applying 1st delay`)
-    await new Promise((resolve) => { setTimeout(() => { resolve() }, waitTimeToScrapMs / 2)})
-  }
-
-  await seeMoreButtons.clickAll(page)
-
-  if(waitTimeToScrapMs) {
-    logger.info(`applying 2nd (and last) delay`)
-    await new Promise((resolve) => { setTimeout(() => { resolve() }, waitTimeToScrapMs / 2)})
-  }
-
-  const [profile] = await scrapSection(page, template.profile)
-  const [about] = await scrapSection(page, template.about)
-  const positions = await scrapSection(page, template.positions)
-  const educations = await scrapSection(page, template.educations)
-  const [recommendationsCount] = await scrapSection(page, template.recommendationsCount)
-  const recommendationsReceived = await scrapSection(page, template.recommendationsReceived)
-  const recommendationsGiven = await scrapSection(page, template.recommendationsGiven)
-  const skills = await scrapSection(page, template.skills)
-  const accomplishments = await scrapSection(page, template.accomplishments)
-  const courses = await scrapAccomplishmentPanel(page, 'courses')
-  const languages = await scrapAccomplishmentPanel(page, 'languages')
-  const projects = await scrapAccomplishmentPanel(page, 'projects')
-  const volunteerExperience = await scrapSection(page, template.volunteerExperience)
-  const peopleAlsoViewed = await scrapSection(page, template.peopleAlsoViewed)
-  const contact = hasToGetContactInfo ? await contactInfo(page) : []
-
-  await page.close()
-  logger.info(`finished scraping url: ${url}`)
-
-  const rawProfile = {
-    profile,
-    about,
-    positions,
-    educations,
-    skills,
-    recommendations: {
-      givenCount: recommendationsCount ? recommendationsCount.given : "0",
-      receivedCount: recommendationsCount ? recommendationsCount.received : "0",
-      given: recommendationsReceived,
-      received: recommendationsGiven
-    },
-    accomplishments,
-    courses,
-    languages,
-    projects,
-    peopleAlsoViewed,
-    volunteerExperience,
-    contact
-  }
-
-  const cleanedProfile = cleanProfileData(rawProfile)
-  return cleanedProfile
-}
+const openPage = require('../openPage')
+const scrapSection = require('../scrapSection')
+const scrapAccomplishmentPanel = require('./scrapAccomplishmentPanel')
+const scrollToPageBottom = require('./scrollToPageBottom')
+const seeMoreButtons = require('./seeMoreButtons')
+const contactInfo = require('./contactInfo')
+const template = require('./profileScraperTemplate')
+const cleanProfileData = require('./cleanProfileData')
+
+const logger = require('../logger')(__filename)
+
+module.exports = async (browser, cookies, url, waitTimeToScrapMs = 500, hasToGetContactInfo = false, puppeteerAuthenticate = undefined) => {
+  logger.info(`starting scraping url: ${url}`)
+
+  const page = await openPage({ browser, cookies, url, puppeteerAuthenticate })
+  const profilePageIndicatorSelector = '.pv-profile-section'
+  await page.waitForSelector(profilePageIndicatorSelector, { timeout: 5000 })
+    .catch(() => {
+      //why doesn't throw error instead of continuing scraping?
+      //because it can be just a false negative meaning LinkedIn only changed that selector but everything else is fine :)
+      logger.warn('profile selector was not found')
+    })
+
+  logger.info('scrolling page to the bottom')
+  await scrollToPageBottom(page)
+  
+  if(waitTimeToScrapMs) {
+    logger.info(`applying 1st delay`)
+    await new Promise((resolve) => { setTimeout(() => { resolve() }, waitTimeToScrapMs / 2)})
+  }
+
+  await seeMoreButtons.clickAll(page)
+
+  if(waitTimeToScrapMs) {
+    logger.info(`applying 2nd (and last) delay`)
+    await new Promise((resolve) => { setTimeout(() => { resolve() }, waitTimeToScrapMs / 2)})
+  }
+
+  const [profile] = await scrapSection(page, template.profile)
+  const [about] = await scrapSection(page, template.about)
+  const positions = await scrapSection(page, template.positions)
+  const educations = await scrapSection(page, template.educations)
+  const [recommendationsCount] = await scrapSection(page, template.recommendationsCount)
+  const recommendationsReceived = await scrapSection(page, template.recommendationsReceived)
+  const recommendationsGiven = await scrapSection(page, template.recommendationsGiven)
+  const skills = await scrapSection(page, template.skills)
+  const accomplishments = await scrapSection(page, template.accomplishments)
+  const courses = await scrapAccomplishmentPanel(page, 'courses')
+  const languages = await scrapAccomplishmentPanel(page, 'languages')
+  const projects = await scrapAccomplishmentPanel(page, 'projects')
+  const volunteerExperience = await scrapSection(page, template.volunteerExperience)
+  const peopleAlsoViewed = await scrapSection(page, template.peopleAlsoViewed)
+  const contact = hasToGetContactInfo ? await contactInfo(page) : []
+
+  await page.close()
+  logger.info(`finished scraping url: ${url}`)
+
+  const rawProfile = {
+    profile,
+    about,
+    positions,
+    educations,
+    skills,
+    recommendations: {
+      givenCount: recommendationsCount ? recommendationsCount.given : "0",
+      receivedCount: recommendationsCount ? recommendationsCount.received : "0",
+      given: recommendationsReceived,
+      received: recommendationsGiven
+    },
+    accomplishments,
+    courses,
+    languages,
+    projects,
+    peopleAlsoViewed,
+    volunteerExperience,
+    contact
+  }
+
+  const cleanedProfile = cleanProfileData(rawProfile)
+  return cleanedProfile
+}

package/src/profile/profileScraperTemplate.js

@@ -1,190 +1,189 @@
-const profileSelector = '.core-rail > *:first-child section >'
-
-const template = {
-  profile: {
-    selector: '.pv-top-card',
-    fields: {
-      name: `.pv-text-details__left-panel:first-child h1`,
-      headline: `.pv-text-details__left-panel:first-child .text-body-medium`,
-      location: `.pv-text-details__left-panel:first-child .pb2`,
-      connections: `.pv-top-card--list span`,
-      imageurl: {
-		    selector: `img.pv-top-card__photo`,
-        attribute: 'src'
-      }
-    }
-  },
-  about: {
-    selector: '.pv-about-section',
-    fields: {
-      text: 'div'
-    }
-  },
-  positions: {
-    selector: '#experience-section li:not(.pv-entity__position-group-role-item)',
-    fields: {
-      title: 'h3',
-      link: {
-        selector: 'a',
-        attribute: 'href',
-      },
-      url: {
-        selector: 'a',
-        attribute: 'href'
-      },
-      companyName: '.pv-entity__secondary-title',
-      location: '.pv-entity__location span:last-child',
-      description: '.pv-entity__description',
-      date1: '.pv-entity__date-range span:last-child',
-      date2: '.pv-entity__bullet-item-v2',
-      roles: {
-        selector: 'li',
-        hasChildrenFields: true,
-        fields: {
-          title: 'h3',
-          description: '.pv-entity__description',
-          date1: '.pv-entity__date-range span:last-child',
-          date2: '.pv-entity__bullet-item-v2',
-          location: '.pv-entity__location span:last-child'
-        }
-      }
-    }
-  },
-  educations: {
-    selector: '#education-section li',
-    fields: {
-      title: 'h3',
-      degree: 'span[class=pv-entity__comma-item]',
-      url: {
-        selector: 'a',
-        attribute: 'href'
-      },
-	    fieldOfStudy: 'p.pv-entity__fos span:nth-child(2)',
-      date1: '.pv-entity__dates time:nth-child(1)',
-      date2: '.pv-entity__dates time:nth-child(2)',
-      description: '.pv-entity__description'
-    }
-  },
-  skills: {
-    selector: '.pv-skill-category-entity__skill-wrapper',
-    fields: {
-      title: '.pv-skill-category-entity__name-text',
-      count: '.pv-skill-category-entity__endorsement-count'
-    }
-  },
-  recommendationsCount: {
-    selector: '.recommendations-inlining',
-    fields: {
-      received: '.artdeco-tab:nth-child(1)',
-      given: '.artdeco-tab:nth-child(2)'
-    }
-  },
-  recommendationsReceived: {
-    selector: '.recommendations-inlining',
-    fields: {
-      user: {
-        selector: '.pv-recommendation-entity__member',
-        attribute: 'href'
-      },
-      text: 'blockquote.pv-recommendation-entity__text',
-      profileImage: {
-        selector: 'a img',
-        attribute: 'src'
-      },
-      name: {
-        selector: 'a h3'
-      },
-      userDescription: {
-        selector: '.pv-recommendation-entity__headline'
-      }
-    }
-  },
-  recommendationsGiven: {
-    selector: '.artdeco-tabpanel li.pv-recommendation-entity',
-    fields: {
-      user: {
-        selector: '.pv-recommendation-entity__member',
-        attribute: 'href'
-      },
-      text: 'blockquote.pv-recommendation-entity__text',
-      profileImage: {
-        selector: 'a img',
-        attribute: 'src'
-      },
-      name: {
-        selector: 'a h3'
-      },
-      userDescription: {
-        selector: '.pv-recommendation-entity__headline'
-      }
-    }
-  },
-  accomplishments: {
-    selector: '.pv-accomplishments-section > div',
-    fields: {
-      count: 'h3 span:last-child',
-      title: '.pv-accomplishments-block__title',
-      items: {
-        selector: 'li',
-        isMultipleFields: true
-      }
-    }
-  },
-  peopleAlsoViewed: {
-    selector: 'li.pv-browsemap-section__member-container',
-    fields: {
-      user: {
-        selector: 'a',
-        attribute: 'href'
-      },
-      text: 'p',
-      profileImage: {
-        selector: 'a img',
-        attribute: 'src'
-      },
-      name: {
-        selector: '.name'
-      }
-    }
-  },
-  volunteerExperience: {
-    selector: 'section.volunteering-section li',
-    fields: {
-      title: 'h3',
-      experience: 'span[class=pv-entity__secondary-title]',
-      location: '.pv-entity__location span:nth-child(2)',
-      description: '.pv-volunteer-causes',
-      date1: '.pv-entity__date-range span:nth-child(2)',
-      date2: '.pv-entity__bullet-item'
-    }
-  },
-  courses: {
-    selector: '.pv-accomplishments-section',
-    fields: {
-      name: '.pv-accomplishment-entity__title',
-      year: '.pv-accomplishment-entity__course-number'
-    }
-  },
-  languages: {
-    selector: '.pv-accomplishments-section',
-    fields: {
-      name: '.pv-accomplishment-entity__title',
-      proficiency: '.pv-accomplishment-entity__proficiency',
-    }
-  },
-  projects: {
-    selector: '.pv-accomplishments-block.projects li',
-    fields: {
-      name: '.pv-accomplishment-entity__title',
-      date: '.pv-accomplishment-entity__date',
-      description: '.pv-accomplishment-entity__description',
-      link: {
-        selector: '.pv-accomplishment-entity__external-source',
-        attribute: 'href'
-      }
-    }
-  }
-}
-
-
-module.exports = template
+const profileSelector = '.core-rail > *:first-child section >'
+
+const template = {
+  profile: {
+    selector: '.pv-top-card',
+    fields: {
+      name: `.pv-text-details__left-panel:first-child h1`,
+      headline: `.pv-text-details__left-panel:first-child .text-body-medium`,
+      location: `.pv-text-details__left-panel:first-child .pb2 span`,
+      connections: `.pv-top-card--list span`,
+      imageurl: {
+		    selector: `img.pv-top-card__photo`,
+        attribute: 'src'
+      }
+    }
+  },
+  about: {
+    selector: '.pv-about-section',
+    fields: {
+      text: 'div'
+    }
+  },
+  positions: {
+    selector: 'section[id$="EXPERIENCE-en-US"] li.artdeco-list__item',
+    fields: {
+      title: 'h3',
+      link: {
+        selector: 'a',
+        attribute: 'href',
+      },
+      url: {
+        selector: 'a',
+        attribute: 'href'
+      },
+      companyName: 'span.t-bold span',
+      location: '.pv-entity__location span:last-child',
+      description: '.pv-entity__description',
+      date1: '.pv-entity__date-range span:last-child',
+      date2: '.pv-entity__bullet-item-v2',
+      roles: {
+        selector: 'li',
+        hasChildrenFields: true,
+        fields: {
+          title: 'span.t-bold span:last-child',
+          description: '.pv-entity__description',
+          date: '.t-14.t-normal:last-child span:first-child',
+          location: '.pv-entity__location span:last-child'
+        }
+      }
+    }
+  },
+  educations: {
+    selector: '#education-section li',
+    fields: {
+      title: 'h3',
+      degree: 'span[class=pv-entity__comma-item]',
+      url: {
+        selector: 'a',
+        attribute: 'href'
+      },
+	    fieldOfStudy: 'p.pv-entity__fos span:nth-child(2)',
+      date1: '.pv-entity__dates time:nth-child(1)',
+      date2: '.pv-entity__dates time:nth-child(2)',
+      description: '.pv-entity__description'
+    }
+  },
+  skills: {
+    selector: '.pv-skill-category-entity__skill-wrapper',
+    fields: {
+      title: '.pv-skill-category-entity__name-text',
+      count: '.pv-skill-category-entity__endorsement-count'
+    }
+  },
+  recommendationsCount: {
+    selector: '.recommendations-inlining',
+    fields: {
+      received: '.artdeco-tab:nth-child(1)',
+      given: '.artdeco-tab:nth-child(2)'
+    }
+  },
+  recommendationsReceived: {
+    selector: '.recommendations-inlining',
+    fields: {
+      user: {
+        selector: '.pv-recommendation-entity__member',
+        attribute: 'href'
+      },
+      text: 'blockquote.pv-recommendation-entity__text',
+      profileImage: {
+        selector: 'a img',
+        attribute: 'src'
+      },
+      name: {
+        selector: 'a h3'
+      },
+      userDescription: {
+        selector: '.pv-recommendation-entity__headline'
+      }
+    }
+  },
+  recommendationsGiven: {
+    selector: '.artdeco-tabpanel li.pv-recommendation-entity',
+    fields: {
+      user: {
+        selector: '.pv-recommendation-entity__member',
+        attribute: 'href'
+      },
+      text: 'blockquote.pv-recommendation-entity__text',
+      profileImage: {
+        selector: 'a img',
+        attribute: 'src'
+      },
+      name: {
+        selector: 'a h3'
+      },
+      userDescription: {
+        selector: '.pv-recommendation-entity__headline'
+      }
+    }
+  },
+  accomplishments: {
+    selector: '.pv-accomplishments-section > div',
+    fields: {
+      count: 'h3 span:last-child',
+      title: '.pv-accomplishments-block__title',
+      items: {
+        selector: 'li',
+        isMultipleFields: true
+      }
+    }
+  },
+  peopleAlsoViewed: {
+    selector: 'li.pv-browsemap-section__member-container',
+    fields: {
+      user: {
+        selector: 'a',
+        attribute: 'href'
+      },
+      text: 'p',
+      profileImage: {
+        selector: 'a img',
+        attribute: 'src'
+      },
+      name: {
+        selector: '.name'
+      }
+    }
+  },
+  volunteerExperience: {
+    selector: 'section.volunteering-section li',
+    fields: {
+      title: 'h3',
+      experience: 'span[class=pv-entity__secondary-title]',
+      location: '.pv-entity__location span:nth-child(2)',
+      description: '.pv-volunteer-causes',
+      date1: '.pv-entity__date-range span:nth-child(2)',
+      date2: '.pv-entity__bullet-item'
+    }
+  },
+  courses: {
+    selector: '.pv-accomplishments-section',
+    fields: {
+      name: '.pv-accomplishment-entity__title',
+      year: '.pv-accomplishment-entity__course-number'
+    }
+  },
+  languages: {
+    selector: '.pv-accomplishments-block.languages li',
+    fields: {
+      name: '.pv-accomplishment-entity__title',
+      proficiency: '.pv-accomplishment-entity__proficiency',
+    }
+  },
+  projects: {
+    selector: '.pv-accomplishments-block.projects li',
+    fields: {
+      name: '.pv-accomplishment-entity__title',
+      date: '.pv-accomplishment-entity__date',
+      description: '.pv-accomplishment-entity__description',
+      link: {
+        selector: '.mt4',
+        attribute: 'href'
+      }
+    }
+  }
+}
+
+
+module.exports = template

package/src/profile/scrapAccomplishmentPanel.js

@@ -1,18 +1,18 @@
-const scrapSection = require('../scrapSection');
-const template = require('./profileScraperTemplate');
-
-const scrapAccomplishmentPanel = async (page, section) => {
-  const queryString = `.pv-accomplishments-block.${section} button`
-
-  const openingButton = await page.$(queryString);
-
-  if (openingButton) {
-    await page.evaluate((q) => {
-      document.querySelector(q).click();
-    }, queryString);
-
-    return scrapSection(page, template[section]);
-  }
-};
-
+const scrapSection = require('../scrapSection');
+const template = require('./profileScraperTemplate');
+
+const scrapAccomplishmentPanel = async (page, section) => {
+  const queryString = `.pv-accomplishments-block.${section} button`
+
+  const openingButton = await page.$(queryString);
+
+  if (openingButton) {
+    await page.evaluate((q) => {
+      document.querySelector(q).click();
+    }, queryString);
+
+    return scrapSection(page, template[section]);
+  }
+};
+
 module.exports = scrapAccomplishmentPanel;

package/src/profile/scrollToPageBottom.js

@@ -1,24 +1,24 @@
-const logger = require('../logger')(__filename)
-
-module.exports = async (page) => {
-  const MAX_TIMES_TO_SCROLL = 25
-  const TIMEOUT_BETWEEN_SCROLLS = 500
-  const PAGE_BOTTOM_SELECTOR_STRING = '#expanded-footer'
-
-  for (let i = 0; i < MAX_TIMES_TO_SCROLL; i++) {
-    await page.evaluate(() => window.scrollBy(0, window.innerHeight))
-
-    const hasReachedEnd = await page.waitForSelector(PAGE_BOTTOM_SELECTOR_STRING, {
-      visible: true,
-      timeout: TIMEOUT_BETWEEN_SCROLLS
-    }).catch(() => {
-      logger.info(`scrolling to page bottom (${i + 1})`)
-    })
-
-    if (hasReachedEnd) {
-      return
-    }
-  }
-
-  logger.warn('page bottom not found')
-}
+const logger = require('../logger')(__filename)
+
+module.exports = async (page) => {
+  const MAX_TIMES_TO_SCROLL = 25
+  const TIMEOUT_BETWEEN_SCROLLS = 500
+  const PAGE_BOTTOM_SELECTOR_STRING = '#expanded-footer'
+
+  for (let i = 0; i < MAX_TIMES_TO_SCROLL; i++) {
+    await page.evaluate(() => window.scrollBy(0, window.innerHeight))
+
+    const hasReachedEnd = await page.waitForSelector(PAGE_BOTTOM_SELECTOR_STRING, {
+      visible: true,
+      timeout: TIMEOUT_BETWEEN_SCROLLS
+    }).catch(() => {
+      logger.info(`scrolling to page bottom (${i + 1})`)
+    })
+
+    if (hasReachedEnd) {
+      return
+    }
+  }
+
+  logger.warn('page bottom not found')
+}