@mvegter/scrapedin 1.0.26 → 1.0.30

package/src/profile/seeMoreButtons.js

@@ -1,42 +1,42 @@
-const logger = require('../logger')(__filename)
-const seeMoreButtons = [
-  {
-    id: 'SHOW_MORE_ABOUT',
-    selector: '#line-clamp-show-more-button'
-  },{
-    id: 'SHOW_MORE_EXPERIENCES',
-    selector: '#experience-section .pv-profile-section__see-more-inline'
-  },{
-    id: 'SEE_MORE_EXPERIENCES',
-    selector: '#experience-section .inline-show-more-text__button'
-  },{
-    id: 'SHOW_MORE_CERTIFICATIONS',
-    selector: '#certifications-section .pv-profile-section__see-more-inline'
-  },{
-    id: 'SHOW_MORE_SKILLS',
-    selector: '.pv-skills-section__additional-skills'
-  },{
-    id: 'SEE_MORE_RECOMMENDATIONS',
-    selector: '.recommendations-inlining #line-clamp-show-more-button'
-  }
-]
-
-
-const clickAll = async(page) => {
-  for(let i = 0; i < seeMoreButtons.length; i++){
-    const button = seeMoreButtons[i]
-    const elems = await page.$$(button.selector)
-
-    for(let j = 0; j < elems.length; j++){
-      const elem = elems[j]
-      if (elem) {
-        await elem.click()
-          .catch((e) => logger.warn(`couldn't click on ${button.selector}, it's probably invisible`))
-      }
-    }
-  }
-
-  return
-}
-
-module.exports = { clickAll }
+const logger = require('../logger')(__filename)
+const seeMoreButtons = [
+  {
+    id: 'SHOW_MORE_ABOUT',
+    selector: '#line-clamp-show-more-button'
+  },{
+    id: 'SHOW_MORE_EXPERIENCES',
+    selector: '#experience-section .pv-profile-section__see-more-inline'
+  },{
+    id: 'SEE_MORE_EXPERIENCES',
+    selector: '#experience-section .inline-show-more-text__button'
+  },{
+    id: 'SHOW_MORE_CERTIFICATIONS',
+    selector: '#certifications-section .pv-profile-section__see-more-inline'
+  },{
+    id: 'SHOW_MORE_SKILLS',
+    selector: '.pv-skills-section__additional-skills'
+  },{
+    id: 'SEE_MORE_RECOMMENDATIONS',
+    selector: '.recommendations-inlining #line-clamp-show-more-button'
+  }
+]
+
+
+const clickAll = async(page) => {
+  for(let i = 0; i < seeMoreButtons.length; i++){
+    const button = seeMoreButtons[i]
+    const elems = await page.$$(button.selector)
+
+    for(let j = 0; j < elems.length; j++){
+      const elem = elems[j]
+      if (elem) {
+        await elem.click()
+          .catch((e) => logger.warn(`couldn't click on ${button.selector}, it's probably invisible`))
+      }
+    }
+  }
+
+  return
+}
+
+module.exports = { clickAll }

package/src/scrapSection.js

@@ -1,49 +1,49 @@
-const scrapSelectorFields = (selector, section) => async (scrapedObjectPromise, fieldKey) => {
-  const scrapedObject = await scrapedObjectPromise
-  const field = section.fields[fieldKey]
-
-  // currently field can be a selector string, or an object containing a selector field
-  const fieldSelectorString = await field.selector
-    ? field.selector
-    : field
-
-  const isFieldPresent = await selector.$(fieldSelectorString)
-
-  if (!isFieldPresent) { return scrapedObject }
-
-  if (field.isMultipleFields) {
-    if (field.attribute === 'href') {
-      scrapedObject[fieldKey] = await selector.$$eval(fieldSelectorString, (elems) => elems.map(elem => elem.href ? elem.href.trim() : elem.innerHTML.trim()))
-    } else if(field.attribute === 'src'){
-      scrapedObject[fieldKey] = await selector.$$eval(fieldSelectorString, (elems) => elems.map(elem => elem.src ? elem.src.trim() : elem.innerHTML.trim()))
-    }else{
-      scrapedObject[fieldKey] = await selector.$$eval(fieldSelectorString, (elems) => elems.map(elem => elem.innerText.trim()))
-    }
-  } else if (field.hasChildrenFields) {
-    const fieldChildrenSelectors = await selector.$$(field.selector)
-
-    scrapedObject[fieldKey] = await Promise.all(
-      fieldChildrenSelectors.map((s) => scrapSelector(s, field))
-    )
-  } else if (field.attribute && field.attribute === 'href') {
-    scrapedObject[fieldKey] = await selector.$eval(fieldSelectorString, (elem) => elem && elem.href ? elem.href.trim() : '')
-  } else if (field.attribute && field.attribute === 'src') {
-    scrapedObject[fieldKey] = await selector.$eval(fieldSelectorString, (elem) => elem && elem.src ? elem.src.trim() : '')
-  } else {
-    scrapedObject[fieldKey] = await selector.$eval(fieldSelectorString, (elem) => elem && elem.innerText ? elem.innerText.trim() : '')
-  }
-
-  return scrapedObject
-}
-const scrapSelector = (selector, section) =>
-  Object.keys(section.fields)
-    .reduce(scrapSelectorFields(selector, section), Promise.resolve({}))
-
-module.exports = async (page, section) => {
-  const sectionSelectors = await page.$$(section.selector)
-
-  const scrapedPromises = sectionSelectors
-    .map((selector) => scrapSelector(selector, section))
-
-  return Promise.all(scrapedPromises)
-}
+const scrapSelectorFields = (selector, section) => async (scrapedObjectPromise, fieldKey) => {
+  const scrapedObject = await scrapedObjectPromise
+  const field = section.fields[fieldKey]
+
+  // currently field can be a selector string, or an object containing a selector field
+  const fieldSelectorString = await field.selector
+    ? field.selector
+    : field
+
+  const isFieldPresent = await selector.$(fieldSelectorString)
+
+  if (!isFieldPresent) { return scrapedObject }
+
+  if (field.isMultipleFields) {
+    if (field.attribute === 'href') {
+      scrapedObject[fieldKey] = await selector.$$eval(fieldSelectorString, (elems) => elems.map(elem => elem.href ? elem.href.trim() : elem.innerHTML.trim()))
+    } else if (field.attribute === 'src') {
+      scrapedObject[fieldKey] = await selector.$$eval(fieldSelectorString, (elems) => elems.map(elem => elem.src ? elem.src.trim() : elem.innerHTML.trim()))
+    } else {
+      scrapedObject[fieldKey] = await selector.$$eval(fieldSelectorString, (elems) => elems.map(elem => elem.innerText.trim()))
+    }
+  } else if (field.hasChildrenFields) {
+    const fieldChildrenSelectors = await selector.$$(field.selector)
+
+    scrapedObject[fieldKey] = await Promise.all(
+      fieldChildrenSelectors.map((s) => scrapSelector(s, field))
+    )
+  } else if (field.attribute && field.attribute === 'href') {
+    scrapedObject[fieldKey] = await selector.$eval(fieldSelectorString, (elem) => elem && elem.href ? elem.href.trim() : '')
+  } else if (field.attribute && field.attribute === 'src') {
+    scrapedObject[fieldKey] = await selector.$eval(fieldSelectorString, (elem) => elem && elem.src ? elem.src.trim() : '')
+  } else {
+    scrapedObject[fieldKey] = await selector.$eval(fieldSelectorString, (elem) => elem && elem.innerText ? elem.innerText.trim() : '')
+  }
+
+  return scrapedObject
+}
+const scrapSelector = (selector, section) =>
+  Object.keys(section.fields)
+    .reduce(scrapSelectorFields(selector, section), Promise.resolve({}))
+
+module.exports = async (page, section) => {
+  const sectionSelectors = await page.$$(section.selector)
+
+  const scrapedPromises = sectionSelectors
+    .map((selector) => scrapSelector(selector, section))
+
+  return Promise.all(scrapedPromises)
+}

package/src/scrapedin.js

@@ -1,41 +1,41 @@
-const puppeteer = require('puppeteer')
-const login = require('./login')
-const profile = require('./profile/profile')
-const company = require('./company/company')
-const logger = require('./logger')(__filename)
-
-module.exports = async ({ cookies, email, password, isHeadless, hasToLog, hasToGetContactInfo, puppeteerArgs, puppeteerAuthenticate, endpoint } = { isHeadless: true, hasToLog: false }) => {
-  if (!hasToLog) {
-    logger.stopLogging()
-  }
-  logger.info('initializing')
-
-  let browser;
-  if(endpoint){
-    browser = await puppeteer.connect({
-      browserWSEndpoint: endpoint,
-    });
-  }else{
-    const args = Object.assign({ headless: isHeadless, args: ['--no-sandbox'] }, puppeteerArgs)
-    browser = await puppeteer.launch(args)
-  }
-
-  if (cookies) {
-    logger.info('using cookies, login will be bypassed')
-  } else if (email && password) {
-    logger.info('email and password was provided, we\'re going to login...')
-
-    try {
-      await login(browser, email, password, logger)
-    } catch (e) {
-      if(!endpoint){
-        await browser.close()
-      }
-      throw e
-    }
-  } else {
-    logger.warn('email/password and cookies wasn\'t provided, only public data will be collected')
-  }
-
-  return (url, waitMs) => url.includes('/school/') || url.includes('/company/') ? company(browser, cookies, url, waitMs, hasToGetContactInfo, puppeteerAuthenticate) :profile(browser, cookies, url, waitMs, hasToGetContactInfo, puppeteerAuthenticate)
-}
+const puppeteer = require('puppeteer')
+const login = require('./login')
+const profile = require('./profile/profile')
+const company = require('./company/company')
+const logger = require('./logger')(__filename)
+
+module.exports = async ({ cookies, email, password, isHeadless, hasToLog, hasToGetContactInfo, puppeteerArgs, puppeteerAuthenticate, endpoint } = { isHeadless: true, hasToLog: false }) => {
+  if (!hasToLog) {
+    logger.stopLogging()
+  }
+  logger.info('initializing')
+
+  let browser
+  if (endpoint) {
+    browser = await puppeteer.connect({
+      browserWSEndpoint: endpoint
+    })
+  } else {
+    const args = Object.assign({ headless: isHeadless, args: ['--no-sandbox'] }, puppeteerArgs)
+    browser = await puppeteer.launch(args)
+  }
+
+  if (cookies) {
+    logger.info('using cookies, login will be bypassed')
+  } else if (email && password) {
+    logger.info('email and password was provided, we\'re going to login...')
+
+    try {
+      await login(browser, email, password, logger)
+    } catch (e) {
+      if (!endpoint) {
+        await browser.close()
+      }
+      throw e
+    }
+  } else {
+    logger.warn('email/password and cookies wasn\'t provided, only public data will be collected')
+  }
+
+  return (url, waitMs) => url.includes('/school/') || url.includes('/company/') ? company(browser, cookies, url, waitMs, hasToGetContactInfo, puppeteerAuthenticate) : profile(browser, cookies, url, waitMs, hasToGetContactInfo, puppeteerAuthenticate)
+}

package/.travis.yml

@@ -1,5 +0,0 @@
-language: node_js
-node_js:
-  - "7.6"
-script:
-  - npm test

package/src/scrapedin.test.js

@@ -1,338 +0,0 @@
-const faker = require('faker')
-const { expect } = require('chai')
-const profile = require('./profile/profile')
-const logger = require('./logger')(__filename)
-const { mock, match } = require('sinon')
-const profileScraperTemplate = require('./profile/profileScraperTemplate')
-const url = faker.internet.url()
-const fakeEvalResult = faker.lorem.words(1)
-
-// Make the linter happy.
-var mocha = require('mocha')
-var it = mocha.it
-
-logger.stopLogging()
-
-it('should get complete profile', async () => {
-  const browserMock = prepareBrowserMock()
-  const result = await profile(browserMock, [], url, 0)
-  const expectedResult = {
-    aboutAlternative: {
-      text: fakeEvalResult
-    },
-    aboutLegacy: {
-      text: fakeEvalResult
-    },
-    accomplishments: [
-      {
-        count: fakeEvalResult,
-        items: [fakeEvalResult],
-        title: fakeEvalResult
-      }
-    ],
-    contact: {},
-    courses: [
-      {
-        name: fakeEvalResult,
-        year: fakeEvalResult
-      }
-    ],
-    educations: [
-      {
-        date1: fakeEvalResult,
-        date2: fakeEvalResult,
-        degree: fakeEvalResult,
-        fieldOfStudy: fakeEvalResult,
-        url: fakeEvalResult,
-        title: fakeEvalResult
-      }
-    ],
-    languages: [
-      {
-        name: fakeEvalResult,
-        proficiency: fakeEvalResult
-      }
-    ],
-    peopleAlsoViewed: [
-      {
-        text: fakeEvalResult,
-        user: fakeEvalResult
-      }
-    ],
-    positions: [
-      {
-        companyName: fakeEvalResult,
-        date1: fakeEvalResult,
-        date2: fakeEvalResult,
-        description: fakeEvalResult,
-        link: fakeEvalResult,
-        location: fakeEvalResult,
-        roles: [
-          {
-            date1: fakeEvalResult,
-            date2: fakeEvalResult,
-            description: fakeEvalResult,
-            location: fakeEvalResult,
-            title: fakeEvalResult
-          }
-        ],
-        title: fakeEvalResult,
-        url: fakeEvalResult
-      }
-    ],
-    profile: {
-      connections: fakeEvalResult,
-      headline: fakeEvalResult,
-      location: fakeEvalResult,
-      name: fakeEvalResult,
-      summary: fakeEvalResult
-    },
-    profileAlternative: {
-      connections: fakeEvalResult,
-      headline: fakeEvalResult,
-      imageurl: fakeEvalResult,
-      location: fakeEvalResult,
-      name: fakeEvalResult
-    },
-    profileLegacy: {
-      connections: fakeEvalResult,
-      headline: fakeEvalResult,
-      location: fakeEvalResult,
-      name: fakeEvalResult,
-      summary: fakeEvalResult
-    },
-    projects: [
-      {
-        date: fakeEvalResult,
-        description: fakeEvalResult,
-        link: fakeEvalResult,
-        name: fakeEvalResult
-      }
-    ],
-    recommendations: {
-      given: [
-        {
-          text: fakeEvalResult,
-          user: fakeEvalResult
-        }
-      ],
-      givenCount: '',
-      received: [
-        {
-          text: fakeEvalResult,
-          user: fakeEvalResult
-        }
-      ],
-      receivedCount: ''
-    },
-    skills: [
-      {
-        count: fakeEvalResult,
-        title: fakeEvalResult
-      }
-    ],
-    volunteerExperience: [
-      {
-        date1: fakeEvalResult,
-        date2: fakeEvalResult,
-        description: fakeEvalResult,
-        experience: fakeEvalResult,
-        location: fakeEvalResult,
-        title: fakeEvalResult
-      }
-    ]
-  }
-
-  expect(result).to.deep.equals(expectedResult)
-})
-
-it('should get an incomplete profile', async () => {
-  const browser = prepareBrowserMock(true)
-
-  const result = await profile(browser, [], url, 0)
-  const expectedResult = {
-    aboutAlternative: {
-      text: ''
-    },
-    aboutLegacy: {
-      text: ''
-    },
-    accomplishments: [
-      {
-        count: '',
-        items: [fakeEvalResult],
-        title: ''
-      }
-    ],
-    contact: {},
-    courses: [{}],
-    educations: [
-      {
-        date1: '',
-        date2: '',
-        degree: '',
-        fieldOfStudy: '',
-        url: ''
-      }
-    ],
-    languages: [
-      {
-        name: undefined,
-        proficiency: ''
-      }
-    ],
-    peopleAlsoViewed: [
-      {
-        text: '',
-        user: ''
-      }
-    ],
-    positions: [
-      {
-        companyName: '',
-        date1: '',
-        date2: '',
-        description: '',
-        link: '',
-        location: '',
-        roles: [
-          {
-            date1: '',
-            date2: '',
-            description: '',
-            location: '',
-            title: ''
-          }
-        ],
-        url: ''
-      }
-    ],
-    profile: {
-      connections: '',
-      headline: '',
-      location: '',
-      name: ''
-    },
-    profileAlternative: {
-      connections: '',
-      headline: '',
-      imageurl: '',
-      location: '',
-      name: ''
-    },
-    profileLegacy: {
-      connections: '',
-      headline: '',
-      location: '',
-      name: ''
-    },
-    projects: [
-      {
-        date: '',
-        description: undefined,
-        link: '',
-        name: undefined
-      }
-    ],
-    recommendations: {
-      given: [
-        {
-          text: '',
-          user: ''
-        }
-      ],
-      givenCount: '',
-      received: [
-        {
-          text: '',
-          user: ''
-        }
-      ],
-      receivedCount: ''
-    },
-    skills: [
-      {
-        count: '',
-        title: ''
-      }
-    ],
-    volunteerExperience: [
-      {
-        date1: '',
-        date2: '',
-        description: '',
-        experience: '',
-        location: ''
-      }
-    ]
-  }
-
-  expect(result).to.deep.equals(expectedResult)
-})
-
-const prepareBrowserMock = (isIncompleteProfile) => {
-  const Page = function () {
-    this.goto = mock().once().withExactArgs(url).resolves()
-    this.setUserAgent = mock().once().resolves()
-    this.setExtraHTTPHeaders = mock().once().resolves()
-    this.setViewport = mock().once().resolves()
-    this.waitFor = mock().once().resolves()
-
-    this.evaluate = mock()
-      .twice()
-      .withExactArgs(match.func)
-      .atLeast(1)
-      .resolves()
-    this.waitForSelector = mock()
-      .withExactArgs(match.string, match.object)
-      .twice()
-      .onCall(0)
-      .rejects()
-      .onCall(1)
-      .resolves(true)
-
-    this.setCookie = mock().once().withExactArgs().resolves()
-
-    this.click = mock().atLeast(1).withExactArgs().resolves()
-    this.$$eval = mock()
-      .withExactArgs(match.string, match.func)
-      .atLeast(1)
-      .callsArgWith(1, [{ innerText: fakeEvalResult }])
-      .resolves([fakeEvalResult])
-
-    this.$eval = mock()
-      .withExactArgs(match.string, match.func)
-      .atLeast(1)
-      .callsArgWith(
-        1,
-        isIncompleteProfile
-          ? undefined
-          : {
-            innerText: fakeEvalResult,
-            src: fakeEvalResult,
-            href: fakeEvalResult
-          }
-      )
-      .resolves(isIncompleteProfile ? '' : fakeEvalResult)
-
-    this.close = mock().once().resolves()
-  }
-
-  Page.prototype.$ = () => new Page()
-
-  if (isIncompleteProfile) {
-    // I couldn't do that with sinon :(
-    Page.prototype.$ = (arg) =>
-      arg === profileScraperTemplate.positions.fields.title
-        ? undefined
-        : Promise.resolve(new Page())
-  }
-
-  Page.prototype.$$ = () => [new Page()]
-
-  const browser = {
-    newPage: mock().once().withExactArgs().resolves(new Page())
-  }
-
-  return browser
-}