@mvegter/scrapedin 1.0.28 → 1.0.31

package/src/profile/seeMoreButtons.js

@@ -1,42 +1,42 @@
-const logger = require('../logger')(__filename)
-const seeMoreButtons = [
-  {
-    id: 'SHOW_MORE_ABOUT',
-    selector: '#line-clamp-show-more-button'
-  },{
-    id: 'SHOW_MORE_EXPERIENCES',
-    selector: '#experience-section .pv-profile-section__see-more-inline'
-  },{
-    id: 'SEE_MORE_EXPERIENCES',
-    selector: '#experience-section .inline-show-more-text__button'
-  },{
-    id: 'SHOW_MORE_CERTIFICATIONS',
-    selector: '#certifications-section .pv-profile-section__see-more-inline'
-  },{
-    id: 'SHOW_MORE_SKILLS',
-    selector: '.pv-skills-section__additional-skills'
-  },{
-    id: 'SEE_MORE_RECOMMENDATIONS',
-    selector: '.recommendations-inlining #line-clamp-show-more-button'
-  }
-]
-
-
-const clickAll = async(page) => {
-  for(let i = 0; i < seeMoreButtons.length; i++){
-    const button = seeMoreButtons[i]
-    const elems = await page.$$(button.selector)
-
-    for(let j = 0; j < elems.length; j++){
-      const elem = elems[j]
-      if (elem) {
-        await elem.click()
-          .catch((e) => logger.warn(`couldn't click on ${button.selector}, it's probably invisible`))
-      }
-    }
-  }
-
-  return
-}
-
-module.exports = { clickAll }
+const logger = require('../logger')(__filename)
+const seeMoreButtons = [
+  {
+    id: 'SHOW_MORE_ABOUT',
+    selector: '#line-clamp-show-more-button'
+  },{
+    id: 'SHOW_MORE_EXPERIENCES',
+    selector: '#experience-section .pv-profile-section__see-more-inline'
+  },{
+    id: 'SEE_MORE_EXPERIENCES',
+    selector: '#experience-section .inline-show-more-text__button'
+  },{
+    id: 'SHOW_MORE_CERTIFICATIONS',
+    selector: '#certifications-section .pv-profile-section__see-more-inline'
+  },{
+    id: 'SHOW_MORE_SKILLS',
+    selector: '.pv-skills-section__additional-skills'
+  },{
+    id: 'SEE_MORE_RECOMMENDATIONS',
+    selector: '.recommendations-inlining #line-clamp-show-more-button'
+  }
+]
+
+
+const clickAll = async(page) => {
+  for(let i = 0; i < seeMoreButtons.length; i++){
+    const button = seeMoreButtons[i]
+    const elems = await page.$$(button.selector)
+
+    for(let j = 0; j < elems.length; j++){
+      const elem = elems[j]
+      if (elem) {
+        await elem.click()
+          .catch((e) => logger.warn(`couldn't click on ${button.selector}, it's probably invisible`))
+      }
+    }
+  }
+
+  return
+}
+
+module.exports = { clickAll }

package/src/scrapSection.js

@@ -1,49 +1,49 @@
-const scrapSelectorFields = (selector, section) => async (scrapedObjectPromise, fieldKey) => {
-  const scrapedObject = await scrapedObjectPromise
-  const field = section.fields[fieldKey]
-
-  // currently field can be a selector string, or an object containing a selector field
-  const fieldSelectorString = await field.selector
-    ? field.selector
-    : field
-
-  const isFieldPresent = await selector.$(fieldSelectorString)
-
-  if (!isFieldPresent) { return scrapedObject }
-
-  if (field.isMultipleFields) {
-    if (field.attribute === 'href') {
-      scrapedObject[fieldKey] = await selector.$$eval(fieldSelectorString, (elems) => elems.map(elem => elem.href ? elem.href.trim() : elem.innerHTML.trim()))
-    } else if (field.attribute === 'src') {
-      scrapedObject[fieldKey] = await selector.$$eval(fieldSelectorString, (elems) => elems.map(elem => elem.src ? elem.src.trim() : elem.innerHTML.trim()))
-    } else {
-      scrapedObject[fieldKey] = await selector.$$eval(fieldSelectorString, (elems) => elems.map(elem => elem.innerText.trim()))
-    }
-  } else if (field.hasChildrenFields) {
-    const fieldChildrenSelectors = await selector.$$(field.selector)
-
-    scrapedObject[fieldKey] = await Promise.all(
-      fieldChildrenSelectors.map((s) => scrapSelector(s, field))
-    )
-  } else if (field.attribute && field.attribute === 'href') {
-    scrapedObject[fieldKey] = await selector.$eval(fieldSelectorString, (elem) => elem && elem.href ? elem.href.trim() : '')
-  } else if (field.attribute && field.attribute === 'src') {
-    scrapedObject[fieldKey] = await selector.$eval(fieldSelectorString, (elem) => elem && elem.src ? elem.src.trim() : '')
-  } else {
-    scrapedObject[fieldKey] = await selector.$eval(fieldSelectorString, (elem) => elem && elem.innerText ? elem.innerText.trim() : '')
-  }
-
-  return scrapedObject
-}
-const scrapSelector = (selector, section) =>
-  Object.keys(section.fields)
-    .reduce(scrapSelectorFields(selector, section), Promise.resolve({}))
-
-module.exports = async (page, section) => {
-  const sectionSelectors = await page.$$(section.selector)
-
-  const scrapedPromises = sectionSelectors
-    .map((selector) => scrapSelector(selector, section))
-
-  return Promise.all(scrapedPromises)
-}
+const scrapSelectorFields = (selector, section) => async (scrapedObjectPromise, fieldKey) => {
+  const scrapedObject = await scrapedObjectPromise
+  const field = section.fields[fieldKey]
+
+  // currently field can be a selector string, or an object containing a selector field
+  const fieldSelectorString = await field.selector
+    ? field.selector
+    : field
+
+  const isFieldPresent = await selector.$(fieldSelectorString)
+
+  if (!isFieldPresent) { return scrapedObject }
+
+  if (field.isMultipleFields) {
+    if (field.attribute === 'href') {
+      scrapedObject[fieldKey] = await selector.$$eval(fieldSelectorString, (elems) => elems.map(elem => elem.href ? elem.href.trim() : elem.innerHTML.trim()))
+    } else if (field.attribute === 'src') {
+      scrapedObject[fieldKey] = await selector.$$eval(fieldSelectorString, (elems) => elems.map(elem => elem.src ? elem.src.trim() : elem.innerHTML.trim()))
+    } else {
+      scrapedObject[fieldKey] = await selector.$$eval(fieldSelectorString, (elems) => elems.map(elem => elem.innerText.trim()))
+    }
+  } else if (field.hasChildrenFields) {
+    const fieldChildrenSelectors = await selector.$$(field.selector)
+
+    scrapedObject[fieldKey] = await Promise.all(
+      fieldChildrenSelectors.map((s) => scrapSelector(s, field))
+    )
+  } else if (field.attribute && field.attribute === 'href') {
+    scrapedObject[fieldKey] = await selector.$eval(fieldSelectorString, (elem) => elem && elem.href ? elem.href.trim() : '')
+  } else if (field.attribute && field.attribute === 'src') {
+    scrapedObject[fieldKey] = await selector.$eval(fieldSelectorString, (elem) => elem && elem.src ? elem.src.trim() : '')
+  } else {
+    scrapedObject[fieldKey] = await selector.$eval(fieldSelectorString, (elem) => elem && elem.innerText ? elem.innerText.trim() : '')
+  }
+
+  return scrapedObject
+}
+const scrapSelector = (selector, section) =>
+  Object.keys(section.fields)
+    .reduce(scrapSelectorFields(selector, section), Promise.resolve({}))
+
+module.exports = async (page, section) => {
+  const sectionSelectors = await page.$$(section.selector)
+
+  const scrapedPromises = sectionSelectors
+    .map((selector) => scrapSelector(selector, section))
+
+  return Promise.all(scrapedPromises)
+}

package/src/scrapedin.js

@@ -1,41 +1,41 @@
-const puppeteer = require('puppeteer')
-const login = require('./login')
-const profile = require('./profile/profile')
-const company = require('./company/company')
-const logger = require('./logger')(__filename)
-
-module.exports = async ({ cookies, email, password, isHeadless, hasToLog, hasToGetContactInfo, puppeteerArgs, puppeteerAuthenticate, endpoint } = { isHeadless: true, hasToLog: false }) => {
-  if (!hasToLog) {
-    logger.stopLogging()
-  }
-  logger.info('initializing')
-
-  let browser
-  if (endpoint) {
-    browser = await puppeteer.connect({
-      browserWSEndpoint: endpoint
-    })
-  } else {
-    const args = Object.assign({ headless: isHeadless, args: ['--no-sandbox'] }, puppeteerArgs)
-    browser = await puppeteer.launch(args)
-  }
-
-  if (cookies) {
-    logger.info('using cookies, login will be bypassed')
-  } else if (email && password) {
-    logger.info('email and password was provided, we\'re going to login...')
-
-    try {
-      await login(browser, email, password, logger)
-    } catch (e) {
-      if (!endpoint) {
-        await browser.close()
-      }
-      throw e
-    }
-  } else {
-    logger.warn('email/password and cookies wasn\'t provided, only public data will be collected')
-  }
-
-  return (url, waitMs) => url.includes('/school/') || url.includes('/company/') ? company(browser, cookies, url, waitMs, hasToGetContactInfo, puppeteerAuthenticate) : profile(browser, cookies, url, waitMs, hasToGetContactInfo, puppeteerAuthenticate)
-}
+const puppeteer = require('puppeteer')
+const login = require('./login')
+const profile = require('./profile/profile')
+const company = require('./company/company')
+const logger = require('./logger')(__filename)
+
+module.exports = async ({ cookies, email, password, isHeadless, hasToLog, hasToGetContactInfo, puppeteerArgs, puppeteerAuthenticate, endpoint } = { isHeadless: true, hasToLog: false }) => {
+  if (!hasToLog) {
+    logger.stopLogging()
+  }
+  logger.info('initializing')
+
+  let browser
+  if (endpoint) {
+    browser = await puppeteer.connect({
+      browserWSEndpoint: endpoint
+    })
+  } else {
+    const args = Object.assign({ headless: isHeadless, args: ['--no-sandbox'] }, puppeteerArgs)
+    browser = await puppeteer.launch(args)
+  }
+
+  if (cookies) {
+    logger.info('using cookies, login will be bypassed')
+  } else if (email && password) {
+    logger.info('email and password was provided, we\'re going to login...')
+
+    try {
+      await login(browser, email, password, logger)
+    } catch (e) {
+      if (!endpoint) {
+        await browser.close()
+      }
+      throw e
+    }
+  } else {
+    logger.warn('email/password and cookies wasn\'t provided, only public data will be collected')
+  }
+
+  return (url, waitMs) => url.includes('/school/') || url.includes('/company/') ? company(browser, cookies, url, waitMs, hasToGetContactInfo, puppeteerAuthenticate) : profile(browser, cookies, url, waitMs, hasToGetContactInfo, puppeteerAuthenticate)
+}